commit d4cd81f498ef4f0e1c33eee1f28afd82ba369646 Author: lingyuzeng Date: Sat Mar 7 22:33:41 2026 +0800 feat: add git-consistent memory gateway architecture diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..2ace145 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,7 @@ +.git +.gitignore +README.md +TASK-RESULT.md +testdata +knowledge +*.log diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..d1c8df9 --- /dev/null +++ b/.env.example @@ -0,0 +1,18 @@ +# Required: git source-of-truth URL (ssh/https/local path) +GIT_REMOTE_URL=git@github.com:your-org/your-memory-repo.git + +# Optional runtime tuning +DEFAULT_BRANCH=main +QMD_TIMEOUT_SECONDS=300 +QMD_TOP_K=5 +QMD_INDEX_PREFIX=ws +QMD_UPDATE_ON_LATEST_QUERY=true +QMD_EMBED_ON_CHANGE=true + +# GPU pinning for qmd container +CUDA_DEVICE=1 +CUDA_VISIBLE_DEVICES=1 +NVIDIA_VISIBLE_DEVICES=1 + +# Optional warmup profile interval (seconds) +WARMUP_INTERVAL_SECONDS=300 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f061c30 --- /dev/null +++ b/.gitignore @@ -0,0 +1,32 @@ +# local env +.env + +# python +.venv/ +.pytest_cache/ +__pycache__/ +*.pyc + +# editor/os +.DS_Store + +# runtime data +logs/ +*.log + +data/git-mirror/* +!data/git-mirror/.gitkeep + +data/workspaces/* +!data/workspaces/.gitkeep + +data/qmd-cache/* +!data/qmd-cache/.gitkeep + +data/qmd-config/* +!data/qmd-config/.gitkeep + +data/remote-memory.git/ + +# generated index/cache in repo root if any +qmd/ diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..9bb205d --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,11 @@ +# AGENTS.md + +## Working Rules + +- 先阅读 `README.md`,确认架构和 API 约定后再修改代码。 +- 修改 `memory-gateway`、`git` 同步逻辑或 `docker-compose.yml` 后,先运行测试再提交结果。 +- 不要破坏现有 `qmd` 容器能力(HTTP MCP、健康检查、模型配置与持久化)。 +- 优先保证功能正确性和可恢复性,不为了“优雅”牺牲可追溯和可运维性。 +- 每次查询返回必须可追溯到 `branch + commit_hash + synced_at`。 +- 禁止实现“单目录频繁 checkout 不同 branch”的方案;必须使用分支隔离 workspace。 +- 任何查询一致性逻辑必须遵循:`sync git -> qmd update/embed -> query`。 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..cd9375a --- /dev/null +++ b/Dockerfile @@ -0,0 +1,40 @@ +FROM node:22-bookworm-slim + +ENV NODE_ENV=production \ + QMD_VERSION=1.0.7 \ + XDG_CACHE_HOME=/var/lib/qmd/cache \ + XDG_CONFIG_HOME=/var/lib/qmd/config \ + QMD_HTTP_PORT=8181 + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + git \ + cmake \ + socat \ + tini \ + python3 \ + make \ + g++ \ + && rm -rf /var/lib/apt/lists/* + +RUN npm install -g "@tobilu/qmd@${QMD_VERSION}" \ + && npm cache clean --force + +WORKDIR /app + +RUN mkdir -p \ + /app/scripts \ + /var/lib/qmd/cache \ + /var/lib/qmd/config \ + /data/knowledge \ + /data/workspaces \ + /data/testdata + +COPY scripts/qmd-entrypoint.sh /app/scripts/qmd-entrypoint.sh +RUN chmod +x /app/scripts/qmd-entrypoint.sh + +EXPOSE 8181 + +ENTRYPOINT ["/usr/bin/tini", "--", "/app/scripts/qmd-entrypoint.sh"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..a4f891a --- /dev/null +++ b/README.md @@ -0,0 +1,266 @@ +# Git 一致性优先的共享记忆检索系统 + +本项目在现有 QMD 基础上增加 `memory-gateway`,目标是让多 Agent 查询具备“查询前强制同步”的一致性,而不是只依赖后台轮询同步。 + +## 1. 项目背景 + +在多 Agent 协作里,若“同步”和“查询”解耦为两个松散后台动作,会出现: + +- Agent A 读到旧记忆 +- Agent B 读到新记忆 +- 对任务目标、恢复点、执行上下文理解分叉 + +本项目的核心设计: + +- Git 是 source of truth +- QMD 只做索引和检索 +- memory-gateway 负责查询前同步与一致性协调 + +## 2. 架构与一致性事务 + +```mermaid +flowchart LR + C[Remote Client\nCodex/Claude/OpenClaw] --> G[memory-gateway\nHTTP :8787] + G --> S[Sync Transaction\nfetch -> workspace sync -> qmd update/embed] + S --> Q[qmd search/query\nper-workspace index] + Q --> G + G --> C + + G --> M[/data/git-mirror/repo.git] + G --> W1[/data/workspaces/main] + G --> W2[/data/workspaces/memory-2026-03] + G --> W3[/data/workspaces/task-TASK-001] +``` + +`/query` 的主链路: + +1. 解析 `branch / memory_profile / query_type / query` +2. 获取 workspace 锁(同 workspace 串行,不同 workspace 并行) +3. `git fetch` + workspace 对齐到目标 branch 最新 commit +4. `qmd update`(需要时 `embed`) +5. 执行检索 +6. 返回结果 + `branch/commit_hash/synced_at/workspace` + +## 3. 服务职责 + +### qmd + +- 提供 MCP/HTTP 能力(`8181`) +- 持久化 cache/config +- 作为底层检索引擎,不作为远程客户端唯一入口 + +### memory-gateway + +- 远程唯一入口(`8787`) +- 提供 `GET /health`、`POST /query`、`POST /sync`、`GET /status` +- 强制执行查询前同步 + +### warmup(可选) + +- 定时轻量请求,降低冷启动抖动 +- 仅优化项,不提供一致性保证 + +## 4. 目录与关键路径 + +### 仓库目录(Host) + +```text +. +├─ docker-compose.yml +├─ .env.example +├─ AGENTS.md +├─ README.md +├─ gateway/ +│ ├─ app/ +│ ├─ tests/ +│ ├─ requirements.txt +│ └─ Dockerfile +├─ scripts/ +│ ├─ qmd-entrypoint.sh +│ ├─ bootstrap.sh +│ ├─ warmup.sh +│ └─ smoke-test.sh +└─ data/ + ├─ git-mirror/ + ├─ workspaces/ + ├─ qmd-cache/ + ├─ qmd-config/ + └─ remote-memory.git/ # bootstrap 生成的本地 demo remote +``` + +### 容器内路径 + +- Git mirror: `/data/git-mirror/repo.git` +- Workspace 根目录: `/data/workspaces` +- QMD cache: `/var/lib/qmd/cache`(qmd 容器)/ `/data/qmd-cache`(gateway 调 CLI) +- QMD config: `/var/lib/qmd/config`(qmd 容器)/ `/data/qmd-config`(gateway 调 CLI) + +## 5. 分支与 profile 规则 + +优先级:`branch > memory_profile > main` + +- `stable` -> `main` +- `monthly-YYYY-MM` -> `memory/YYYY-MM` +- `task-` -> `task/` + +建议分层: + +- 长期稳定:`main` +- 月度滚动:`memory/YYYY-MM` +- 任务临时:`task/` + +## 6. 环境变量(关键配置) + +### 必填 + +- `GIT_REMOTE_URL`:Git 真相源地址(ssh/https/file/path) + +### 常用 + +- `DEFAULT_BRANCH`:默认分支(默认 `main`) +- `QMD_TIMEOUT_SECONDS`:gateway 调 qmd CLI 超时秒数(默认 `300`) +- `QMD_INDEX_PREFIX`:per-workspace 索引名前缀(默认 `ws`) +- `QMD_UPDATE_ON_LATEST_QUERY`:`require_latest=true` 时是否强制 `qmd update`(默认 `true`) +- `QMD_EMBED_ON_CHANGE`:HEAD 变化时是否尝试 embed(默认 `true`,可按性能改 `false`) + +### GPU(qmd 容器) + +- `CUDA_DEVICE` +- `CUDA_VISIBLE_DEVICES` +- `NVIDIA_VISIBLE_DEVICES` + +### 模型覆盖(可选) + +- `QMD_EMBED_MODEL_URI` +- `QMD_RERANK_MODEL_URI` +- `QMD_GENERATE_MODEL_URI` + +示例:见 `[.env.example](/home/lingyuzeng/project/qmd-local/qmd-docker-http-mcp/.env.example)`。 + +## 7. 快速开始 + +### 7.1 初始化 demo remote 与目录 + +```bash +cd /home/lingyuzeng/project/qmd-local/qmd-docker-http-mcp +./scripts/bootstrap.sh +``` + +### 7.2 启动 + +```bash +docker compose build +docker compose up -d +``` + +### 7.3 健康检查 + +```bash +curl http://127.0.0.1:8181/health +curl http://127.0.0.1:8787/health +``` + +## 8. API 使用 + +### GET /health + +```bash +curl -fsS http://127.0.0.1:8787/health +``` + +### POST /sync + +```bash +curl -fsS -X POST http://127.0.0.1:8787/sync \ + -H 'content-type: application/json' \ + -d '{"branch":"main","require_latest":true}' +``` + +### POST /query(主入口) + +```bash +curl -fsS -X POST http://127.0.0.1:8787/query \ + -H 'content-type: application/json' \ + -d '{ + "branch":"main", + "query_type":"search", + "query":"recovery strategy", + "require_latest":true, + "debug":true + }' +``` + +返回字段核心: + +- `ok` +- `branch` +- `resolved_workspace` +- `commit_hash` +- `synced_at` +- `query_type` +- `results` +- `qmd_collection` +- `debug`(可选) + +### GET /status + +```bash +curl -fsS http://127.0.0.1:8787/status +``` + +## 9. 查询策略说明 + +当前最小兼容实现中: + +- `query_type=search` -> `qmd search` +- `query_type=vsearch` -> `qmd vsearch` +- `query_type=query/deep_search` -> 为保证稳定与时延上界,默认走 `qmd search`(响应 `debug.qmd_command` 可追踪) + +## 10. 测试与验收 + +### 自动化测试 + +```bash +python3 -m venv .venv +. .venv/bin/activate +pip install -r gateway/requirements.txt pytest httpx +PYTHONPATH=./gateway pytest -q gateway/tests +``` + +### 手工验收(建议) + +1. 构建与启动 +2. 服务健康 +3. 基础查询 +4. 修改远端后立即查询(验证查询前同步) +5. 分支隔离 +6. 默认分支 +7. 并发查询 +8. 重启恢复 + +## 11. 常见问题 + +### 首次请求慢 + +首次模型/索引准备、或首次分支激活会慢。可用: + +- `QMD_EMBED_ON_CHANGE=false`(降低请求阻塞) +- warmup profile 预热 + +### 本地路径 remote 被 git safe.directory 拦截 + +gateway 已对本地 `GIT_REMOTE_URL` 自动添加 `safe.directory`。 + +## 12. 客户端接入建议 + +- 远程客户端只访问 `memory-gateway`,不要直连 qmd +- 每次请求尽量显式传 `branch` 或 `memory_profile` +- 记录响应中的 `branch + commit_hash` 用于审计与回放 + +## 13. 关键文件 + +- 网关入口:[gateway/app/main.py](/home/lingyuzeng/project/qmd-local/qmd-docker-http-mcp/gateway/app/main.py) +- 同步事务:[gateway/app/sync_service.py](/home/lingyuzeng/project/qmd-local/qmd-docker-http-mcp/gateway/app/sync_service.py) +- Git 管理:[gateway/app/git_manager.py](/home/lingyuzeng/project/qmd-local/qmd-docker-http-mcp/gateway/app/git_manager.py) +- QMD 调用:[gateway/app/qmd_client.py](/home/lingyuzeng/project/qmd-local/qmd-docker-http-mcp/gateway/app/qmd_client.py) +- Compose 编排:[docker-compose.yml](/home/lingyuzeng/project/qmd-local/qmd-docker-http-mcp/docker-compose.yml) diff --git a/TASK-RESULT.md b/TASK-RESULT.md new file mode 100644 index 0000000..a18b085 --- /dev/null +++ b/TASK-RESULT.md @@ -0,0 +1,157 @@ +# TASK RESULT + +## Research Notes + +1. **当前最适合 pin 的 QMD 版本** + - 选择 `v1.0.7`(`@tobilu/qmd@1.0.7`),是截至 2026-03-07 的最新非预发布 stable release(发布时间 2026-02-18)。 + +2. **是否已有官方 compose 参考** + - 在 `tobi/qmd` 的 `v1.0.7` 源码树中未找到 `Dockerfile` / `docker-compose.yml` 参考,需要自行实现。 + +3. **QMD HTTP MCP 启动与健康检查方式** + - 启动:`qmd mcp --http`(默认端口 8181) + - 端点:`POST /mcp`、`GET /health` + +4. **是否需要覆盖默认模型** + - 第一版不覆盖,直接沿用源码默认模型(embeddinggemma / qwen3-reranker / qmd-query-expansion)。 + +5. **Node 与 Bun 在容器中选型** + - 选 Node(`node:22-bookworm-slim`)。理由: + - 上游 package `engines.node >=22` + - `npm install -g @tobilu/qmd@1.0.7` 路径直接、可复现 + - 官方生态和镜像稳定性更高,长期维护成本更低 + +6. **缓存目录、索引目录、知识库目录挂载策略** + - `XDG_CACHE_HOME=/var/lib/qmd/cache`(模型缓存 + 默认 sqlite 索引) + - `XDG_CONFIG_HOME=/var/lib/qmd/config`(collection YAML 配置) + - 业务知识库挂载到 `/data/knowledge` + - 测试数据挂载到 `/data/testdata` + +7. **smoke test 最小闭环覆盖步骤** + - compose 启动服务 + - `/health` 检查通过 + - 添加 collections(notes/docs/meetings) + - `qmd update` + `qmd embed` + - 至少一次 `qmd search` + - 至少一次 `qmd query`(deep search 等价能力) + +## Implemented Files + +```text +qmd-docker-http-mcp/ +├─ Dockerfile +├─ docker-compose.yml +├─ .dockerignore +├─ README.md +├─ TASK-RESULT.md +├─ scripts/ +│ ├─ entrypoint.sh +│ ├─ warmup.sh +│ └─ smoke-test.sh +├─ knowledge/ +└─ testdata/ + ├─ notes/ + ├─ docs/ + └─ meetings/ +``` + +## Commands Executed (Real) + +```bash +# Research +curl -fsSL https://api.github.com/repos/tobi/qmd/releases?per_page=5 | jq ... +curl -fsSL https://api.github.com/repos/tobi/qmd/git/trees/v1.0.7?recursive=1 | jq -r '.tree[].path' | rg 'docker|compose|Dockerfile|docker-compose' || true +curl -fsSL https://raw.githubusercontent.com/tobi/qmd/v1.0.7/README.md +curl -fsSL https://raw.githubusercontent.com/tobi/qmd/v1.0.7/src/llm.ts + +# Build / Run +cd qmd-docker-http-mcp +docker build -t hotwa/qmd:latest . +docker compose up -d +docker compose logs --no-color --tail=120 qmd +curl -fsS http://127.0.0.1:8181/health + +# Warmup / Smoke +./scripts/warmup.sh +./scripts/smoke-test.sh + +# Extra MCP endpoint check +curl -X POST http://127.0.0.1:8181/mcp -H 'content-type: application/json' -d '{}' +``` + +## Build Result + +- Image built successfully: `hotwa/qmd:latest` +- Image inspect summary: + - `hotwa/qmd:latest b5c811d89722 1.99GB` + +## Compose Startup Result + +- `docker compose up -d` 成功 +- `docker compose ps` 结果: + - `qmd-http-mcp ... Up ... (healthy) ... 0.0.0.0:8181->8181/tcp` + +## Healthcheck Result + +- `GET /health` 返回成功: + +```json +{"status":"ok","uptime":244} +``` + +- `POST /mcp` 路由可达(示例请求返回协议校验错误而非 404): + - HTTP `406` + - `Not Acceptable: Client must accept both application/json and text/event-stream` + +## Warmup Result + +- 成功创建并索引 3 个 collection:`notes` / `docs` / `meetings` +- 成功执行 `qmd update` +- 成功执行 `qmd embed` +- 首次运行完成 embedding 模型下载并生成向量 + +## Smoke Test Result + +`scripts/smoke-test.sh` 真实执行通过,覆盖: + +- 服务启动与 `/health` 成功 +- collection 检查通过(3/3) +- `qmd search "incident runbook" --json -n 5` 返回命中(含 `docs/incident-runbook.md`) +- `qmd query "what is the RTO target for disaster recovery" --json -n 5` 返回命中(`docs/disaster-recovery.md` 得分最高) +- 脚本最终输出:`[smoke] Smoke test passed` + +## Problems Found and Fixes + +1. **问题:容器外访问 8181 失败(连接被重置)** + - 根因:QMD `v1.0.7` 的 HTTP MCP 在上游实现中绑定 `localhost`。 + - 修复:在 `entrypoint.sh` 引入 `socat` 转发,将容器 `0.0.0.0:8181` 转发到容器内 QMD 监听端口。 + +2. **问题:初版转发到 `127.0.0.1:8182` 仍失败** + - 根因:QMD 在容器内实际监听 `::1`(IPv6 localhost),`127.0.0.1` 不通。 + - 修复:改为 `socat ... TCP6:[::1]:8182`。 + +3. **观察项:首次 query/embed 有 node-llama-cpp 的 CUDA 构建回退告警** + - 现象:日志提示 `git/cmake not found`,随后回退并继续 CPU 路径,流程最终成功。 + - 结论:当前不阻塞功能闭环,但会增加首次请求噪声与时延。 + +## Final Acceptance + +验收条件逐项结果: + +- [x] 成功构建 `hotwa/qmd:latest` +- [x] `docker compose up -d` 后服务启动 +- [x] `/health` 返回成功 +- [x] 至少一组测试数据被索引 +- [x] 至少一次关键词搜索成功 +- [x] 至少一次深检索/混合检索成功 +- [x] README 可指导复现 +- [x] 本文件记录真实测试过程(无伪造) + +## Sources + +- QMD releases: https://api.github.com/repos/tobi/qmd/releases +- QMD v1.0.7 README: https://raw.githubusercontent.com/tobi/qmd/v1.0.7/README.md +- QMD v1.0.7 llm.ts: https://raw.githubusercontent.com/tobi/qmd/v1.0.7/src/llm.ts +- QMD v1.0.7 mcp.ts: https://raw.githubusercontent.com/tobi/qmd/v1.0.7/src/mcp.ts +- Codex MCP docs: https://developers.openai.com/codex/mcp/ +- Claude Code MCP docs: https://docs.anthropic.com/en/docs/claude-code/mcp diff --git a/data/git-mirror/.gitkeep b/data/git-mirror/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/data/qmd-cache/.gitkeep b/data/qmd-cache/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/data/qmd-config/.gitkeep b/data/qmd-config/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/data/workspaces/.gitkeep b/data/workspaces/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..39e1cb1 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,91 @@ +services: + qmd: + image: hotwa/qmd:latest + build: + context: . + dockerfile: Dockerfile + container_name: qmd-http-mcp + ports: + - "8181:8181" + deploy: + resources: + reservations: + devices: + - driver: nvidia + device_ids: ["1"] + capabilities: [gpu] + environment: + XDG_CACHE_HOME: /var/lib/qmd/cache + XDG_CONFIG_HOME: /var/lib/qmd/config + QMD_HTTP_PORT: "8181" + CUDA_DEVICE: "${CUDA_DEVICE:-1}" + CUDA_VISIBLE_DEVICES: "${CUDA_VISIBLE_DEVICES:-1}" + NVIDIA_VISIBLE_DEVICES: "${NVIDIA_VISIBLE_DEVICES:-1}" + NVIDIA_DRIVER_CAPABILITIES: "compute,utility" + QMD_EMBED_MODEL_URI: "${QMD_EMBED_MODEL_URI:-hf:ggml-org/embeddinggemma-300M-GGUF/embeddinggemma-300M-Q8_0.gguf}" + QMD_RERANK_MODEL_URI: "${QMD_RERANK_MODEL_URI:-hf:ggml-org/Qwen3-Reranker-0.6B-Q8_0-GGUF/qwen3-reranker-0.6b-q8_0.gguf}" + QMD_GENERATE_MODEL_URI: "${QMD_GENERATE_MODEL_URI:-hf:tobil/qmd-query-expansion-1.7B-gguf/qmd-query-expansion-1.7B-q4_k_m.gguf}" + volumes: + - ./data/qmd-cache:/var/lib/qmd/cache + - ./data/qmd-config:/var/lib/qmd/config + - ./data/workspaces:/data/workspaces + - ./models:/models + - ./testdata:/data/testdata:ro + healthcheck: + test: ["CMD", "curl", "-fsS", "http://127.0.0.1:8181/health"] + interval: 30s + timeout: 5s + retries: 5 + start_period: 30s + restart: unless-stopped + + memory-gateway: + build: + context: . + dockerfile: gateway/Dockerfile + container_name: memory-gateway + depends_on: + qmd: + condition: service_healthy + ports: + - "8787:8787" + environment: + APP_ENV: "${APP_ENV:-prod}" + DEFAULT_BRANCH: "${DEFAULT_BRANCH:-main}" + GIT_REMOTE_URL: "${GIT_REMOTE_URL:-}" + GIT_MIRROR_PATH: /data/git-mirror/repo.git + WORKSPACES_ROOT: /data/workspaces + WORKSPACE_STATE_DIR: /data/workspaces/.gateway-state + XDG_CACHE_HOME: /data/qmd-cache + XDG_CONFIG_HOME: /data/qmd-config + QMD_BINARY: qmd + QMD_TIMEOUT_SECONDS: "${QMD_TIMEOUT_SECONDS:-300}" + QMD_TOP_K: "${QMD_TOP_K:-5}" + QMD_INDEX_PREFIX: "${QMD_INDEX_PREFIX:-ws}" + QMD_UPDATE_ON_LATEST_QUERY: "${QMD_UPDATE_ON_LATEST_QUERY:-true}" + QMD_EMBED_ON_CHANGE: "${QMD_EMBED_ON_CHANGE:-true}" + volumes: + - ./data:/data + - ./models:/models + healthcheck: + test: ["CMD", "curl", "-fsS", "http://127.0.0.1:8787/health"] + interval: 15s + timeout: 5s + retries: 6 + start_period: 10s + restart: unless-stopped + + warmup: + image: curlimages/curl:8.12.1 + container_name: memory-warmup + depends_on: + memory-gateway: + condition: service_healthy + entrypoint: ["/bin/sh", "-lc"] + command: >- + while true; do + curl -fsS http://memory-gateway:8787/health >/dev/null || true; + sleep ${WARMUP_INTERVAL_SECONDS:-300}; + done + restart: unless-stopped + profiles: ["warmup"] diff --git a/gateway/Dockerfile b/gateway/Dockerfile new file mode 100644 index 0000000..e7b21f0 --- /dev/null +++ b/gateway/Dockerfile @@ -0,0 +1,38 @@ +FROM node:22-bookworm-slim + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + QMD_VERSION=1.0.7 \ + PATH=/opt/venv/bin:$PATH \ + XDG_CACHE_HOME=/var/lib/qmd/cache \ + XDG_CONFIG_HOME=/var/lib/qmd/config + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + git \ + cmake \ + python3 \ + python3-pip \ + python3-venv \ + make \ + g++ \ + tini \ + && rm -rf /var/lib/apt/lists/* + +RUN npm install -g "@tobilu/qmd@${QMD_VERSION}" \ + && npm cache clean --force + +WORKDIR /app + +COPY gateway/requirements.txt /app/requirements.txt +RUN python3 -m venv /opt/venv \ + && pip install --no-cache-dir -r /app/requirements.txt + +COPY gateway/app /app/app + +EXPOSE 8787 + +ENTRYPOINT ["/usr/bin/tini", "--"] +CMD ["python3", "-m", "uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8787"] diff --git a/gateway/app/__init__.py b/gateway/app/__init__.py new file mode 100644 index 0000000..49a8c0f --- /dev/null +++ b/gateway/app/__init__.py @@ -0,0 +1 @@ +"""Memory gateway application package.""" diff --git a/gateway/app/config.py b/gateway/app/config.py new file mode 100644 index 0000000..cb76bb3 --- /dev/null +++ b/gateway/app/config.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +from functools import lru_cache +from pathlib import Path + +from pydantic import Field, model_validator +from pydantic_settings import BaseSettings, SettingsConfigDict + + +class Settings(BaseSettings): + model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8", extra="ignore") + + app_name: str = "memory-gateway" + app_env: str = "dev" + default_branch: str = "main" + + git_remote_url: str = Field(default="", description="Upstream git remote URL or local path") + git_mirror_path: Path = Path("/data/git-mirror/repo.git") + workspaces_root: Path = Path("/data/workspaces") + workspace_state_dir: Path | None = None + + qmd_binary: str = "qmd" + qmd_timeout_seconds: int = 300 + qmd_top_k: int = 5 + qmd_index_prefix: str = "ws" + qmd_update_on_latest_query: bool = True + qmd_embed_on_change: bool = True + + xdg_cache_home: Path = Path("/var/lib/qmd/cache") + xdg_config_home: Path = Path("/var/lib/qmd/config") + + @model_validator(mode="after") + def _normalize_paths(self) -> "Settings": + if self.workspace_state_dir is None: + self.workspace_state_dir = self.workspaces_root / ".gateway-state" + return self + + +@lru_cache(maxsize=1) +def get_settings() -> Settings: + return Settings() diff --git a/gateway/app/git_manager.py b/gateway/app/git_manager.py new file mode 100644 index 0000000..088c930 --- /dev/null +++ b/gateway/app/git_manager.py @@ -0,0 +1,153 @@ +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +import re +import shlex +import subprocess +from urllib.parse import urlparse + + +class GitCommandError(RuntimeError): + def __init__(self, args: list[str], returncode: int, stdout: str, stderr: str) -> None: + cmd = " ".join(shlex.quote(x) for x in args) + message = f"git command failed ({returncode}): {cmd}\nstdout:\n{stdout}\nstderr:\n{stderr}" + super().__init__(message) + self.args_list = args + self.returncode = returncode + self.stdout = stdout + self.stderr = stderr + + +@dataclass(frozen=True) +class SyncResult: + previous_commit: str | None + commit_hash: str + changed: bool + + +class GitManager: + def __init__(self, remote_url: str, mirror_path: Path) -> None: + self.remote_url = remote_url + self.mirror_path = mirror_path + + def ensure_mirror(self) -> None: + if not self.remote_url: + raise ValueError("git_remote_url is required") + + self._allow_local_remote_if_needed() + + head_file = self.mirror_path / "HEAD" + if not head_file.exists(): + self.mirror_path.parent.mkdir(parents=True, exist_ok=True) + if self.mirror_path.exists() and any(self.mirror_path.iterdir()): + raise ValueError(f"mirror path exists and is not empty: {self.mirror_path}") + self._run(["git", "clone", "--mirror", self.remote_url, str(self.mirror_path)]) + else: + self._run( + [ + "git", + f"--git-dir={self.mirror_path}", + "remote", + "set-url", + "origin", + self.remote_url, + ] + ) + + def fetch_origin(self) -> None: + self._run(["git", f"--git-dir={self.mirror_path}", "fetch", "--prune", "origin"]) + + def get_branch_commit(self, branch: str) -> str: + refs_to_try = [ + f"refs/heads/{branch}", + f"refs/remotes/origin/{branch}", + ] + last_error: GitCommandError | None = None + for ref in refs_to_try: + try: + return self._run( + ["git", f"--git-dir={self.mirror_path}", "rev-parse", "--verify", ref] + ).strip() + except GitCommandError as exc: + last_error = exc + if last_error is None: + raise RuntimeError("unexpected branch lookup failure without error") + raise last_error + + def get_workspace_head(self, workspace_path: Path) -> str | None: + if not (workspace_path / ".git").exists(): + return None + try: + return self._run(["git", "-C", str(workspace_path), "rev-parse", "HEAD"]).strip() + except GitCommandError: + return None + + def sync_workspace(self, workspace_path: Path, branch: str, commit_hash: str) -> SyncResult: + previous = self.get_workspace_head(workspace_path) + self._ensure_workspace_repo(workspace_path) + + self._run(["git", "-C", str(workspace_path), "fetch", "--prune", "origin"]) + local_branch = self._local_branch_name(branch) + self._run( + [ + "git", + "-C", + str(workspace_path), + "checkout", + "-B", + local_branch, + commit_hash, + ] + ) + self._run(["git", "-C", str(workspace_path), "reset", "--hard", commit_hash]) + self._run(["git", "-C", str(workspace_path), "clean", "-fd"]) + + current = self.get_workspace_head(workspace_path) + if not current: + raise RuntimeError(f"failed to read workspace HEAD after sync: {workspace_path}") + + return SyncResult(previous_commit=previous, commit_hash=current, changed=(previous != current)) + + def _ensure_workspace_repo(self, workspace_path: Path) -> None: + if not (workspace_path / ".git").exists(): + workspace_path.parent.mkdir(parents=True, exist_ok=True) + if workspace_path.exists() and any(workspace_path.iterdir()): + raise ValueError(f"workspace exists and is not a git repo: {workspace_path}") + self._run(["git", "clone", str(self.mirror_path), str(workspace_path)]) + + self._run( + [ + "git", + "-C", + str(workspace_path), + "remote", + "set-url", + "origin", + str(self.mirror_path), + ] + ) + + def _local_branch_name(self, branch: str) -> str: + cleaned = re.sub(r"[^A-Za-z0-9_.-]+", "-", branch).strip("-") + return f"gateway-{cleaned or 'default'}" + + def _run(self, args: list[str]) -> str: + proc = subprocess.run(args, check=False, capture_output=True, text=True) + if proc.returncode != 0: + raise GitCommandError(args=args, returncode=proc.returncode, stdout=proc.stdout, stderr=proc.stderr) + return proc.stdout + + def _allow_local_remote_if_needed(self) -> None: + candidate: str | None = None + if self.remote_url.startswith("/"): + candidate = self.remote_url + elif self.remote_url.startswith("file://"): + parsed = urlparse(self.remote_url) + candidate = parsed.path + if not candidate: + return + if not Path(candidate).exists(): + return + # Avoid git safety checks blocking local bind-mounted repositories. + self._run(["git", "config", "--global", "--add", "safe.directory", candidate]) diff --git a/gateway/app/locks.py b/gateway/app/locks.py new file mode 100644 index 0000000..a9de79f --- /dev/null +++ b/gateway/app/locks.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +from contextlib import contextmanager +from dataclasses import dataclass +import threading +import time + + +@dataclass +class LockStats: + key: str + wait_ms: int + + +class WorkspaceLockManager: + def __init__(self) -> None: + self._locks: dict[str, threading.Lock] = {} + self._guard = threading.Lock() + + def _get_lock(self, key: str) -> threading.Lock: + with self._guard: + lock = self._locks.get(key) + if lock is None: + lock = threading.Lock() + self._locks[key] = lock + return lock + + @contextmanager + def acquire(self, key: str): + lock = self._get_lock(key) + started = time.perf_counter() + lock.acquire() + wait_ms = int((time.perf_counter() - started) * 1000) + try: + yield LockStats(key=key, wait_ms=wait_ms) + finally: + lock.release() diff --git a/gateway/app/main.py b/gateway/app/main.py new file mode 100644 index 0000000..6c88bb7 --- /dev/null +++ b/gateway/app/main.py @@ -0,0 +1,115 @@ +from __future__ import annotations + +from datetime import datetime, timezone +import json +from typing import Any + +from fastapi import FastAPI, HTTPException +from fastapi.responses import JSONResponse + +from .config import Settings, get_settings +from .git_manager import GitCommandError, GitManager +from .locks import WorkspaceLockManager +from .models import HealthResponse, QueryRequest, QueryResponse, StatusResponse, SyncRequest, SyncResponse, WorkspaceStatusItem +from .qmd_client import QMDClient, QMDCommandError +from .query_service import QueryService +from .sync_service import SyncService +from .workspace_manager import WorkspaceManager + + +def create_app( + settings: Settings | None = None, + *, + qmd_client: QMDClient | None = None, + git_manager: GitManager | None = None, + lock_manager: WorkspaceLockManager | None = None, +) -> FastAPI: + app = FastAPI(title="memory-gateway", version="0.1.0") + + resolved_settings = settings or get_settings() + workspace_manager = WorkspaceManager( + workspaces_root=resolved_settings.workspaces_root, + default_branch=resolved_settings.default_branch, + qmd_index_prefix=resolved_settings.qmd_index_prefix, + ) + resolved_git_manager = git_manager or GitManager( + remote_url=resolved_settings.git_remote_url, + mirror_path=resolved_settings.git_mirror_path, + ) + resolved_qmd_client = qmd_client or QMDClient( + qmd_binary=resolved_settings.qmd_binary, + timeout_seconds=resolved_settings.qmd_timeout_seconds, + xdg_cache_home=resolved_settings.xdg_cache_home, + xdg_config_home=resolved_settings.xdg_config_home, + ) + resolved_lock_manager = lock_manager or WorkspaceLockManager() + + sync_service = SyncService( + settings=resolved_settings, + workspace_manager=workspace_manager, + git_manager=resolved_git_manager, + qmd_client=resolved_qmd_client, + ) + query_service = QueryService( + sync_service=sync_service, + qmd_client=resolved_qmd_client, + lock_manager=resolved_lock_manager, + ) + + app.state.settings = resolved_settings + app.state.sync_service = sync_service + app.state.query_service = query_service + + @app.exception_handler(ValueError) + def handle_value_error(_: Any, exc: ValueError) -> JSONResponse: + return JSONResponse(status_code=400, content={"ok": False, "error": str(exc)}) + + @app.exception_handler(GitCommandError) + def handle_git_error(_: Any, exc: GitCommandError) -> JSONResponse: + return JSONResponse(status_code=500, content={"ok": False, "error": str(exc)}) + + @app.exception_handler(QMDCommandError) + def handle_qmd_error(_: Any, exc: QMDCommandError) -> JSONResponse: + return JSONResponse(status_code=500, content={"ok": False, "error": str(exc)}) + + @app.get("/health", response_model=HealthResponse) + def health() -> HealthResponse: + return HealthResponse(ok=True, service="memory-gateway", timestamp=datetime.now(timezone.utc)) + + @app.post("/query", response_model=QueryResponse) + def query(request: QueryRequest) -> QueryResponse: + return app.state.query_service.handle_query(request) + + @app.post("/sync", response_model=SyncResponse) + def sync(request: SyncRequest) -> SyncResponse: + return app.state.query_service.handle_sync(request) + + @app.get("/status", response_model=StatusResponse) + def status() -> StatusResponse: + raw_states = app.state.sync_service.list_workspace_states() + items: list[WorkspaceStatusItem] = [] + for row in raw_states: + synced_at_raw = row.get("synced_at") + if not synced_at_raw: + continue + items.append( + WorkspaceStatusItem( + branch=row["branch"], + workspace=row["workspace"], + commit_hash=row["commit_hash"], + synced_at=datetime.fromisoformat(synced_at_raw), + qmd_collection=row["qmd_collection"], + ) + ) + + return StatusResponse( + ok=True, + default_branch=app.state.settings.default_branch, + mirror_path=str(app.state.settings.git_mirror_path), + workspaces=items, + ) + + return app + + +app = create_app() diff --git a/gateway/app/models.py b/gateway/app/models.py new file mode 100644 index 0000000..1d41e00 --- /dev/null +++ b/gateway/app/models.py @@ -0,0 +1,73 @@ +from __future__ import annotations + +from datetime import datetime +from enum import Enum +from typing import Any + +from pydantic import BaseModel, Field + + +class QueryType(str, Enum): + search = "search" + vsearch = "vsearch" + query = "query" + deep_search = "deep_search" + + +class QueryRequest(BaseModel): + branch: str | None = None + memory_profile: str | None = None + query_type: QueryType = QueryType.query + query: str = Field(min_length=1) + require_latest: bool = True + n: int = Field(default=5, ge=1, le=50) + debug: bool = False + + +class SyncRequest(BaseModel): + branch: str | None = None + memory_profile: str | None = None + require_latest: bool = True + + +class QueryResponse(BaseModel): + ok: bool + branch: str + resolved_workspace: str + commit_hash: str + synced_at: datetime + query_type: QueryType + results: Any + qmd_collection: str + debug: dict[str, Any] | None = None + + +class SyncResponse(BaseModel): + ok: bool + branch: str + resolved_workspace: str + commit_hash: str + synced_at: datetime + qmd_collection: str + debug: dict[str, Any] | None = None + + +class HealthResponse(BaseModel): + ok: bool + service: str + timestamp: datetime + + +class WorkspaceStatusItem(BaseModel): + branch: str + workspace: str + commit_hash: str + synced_at: datetime + qmd_collection: str + + +class StatusResponse(BaseModel): + ok: bool + default_branch: str + mirror_path: str + workspaces: list[WorkspaceStatusItem] diff --git a/gateway/app/qmd_client.py b/gateway/app/qmd_client.py new file mode 100644 index 0000000..5cc1063 --- /dev/null +++ b/gateway/app/qmd_client.py @@ -0,0 +1,175 @@ +from __future__ import annotations + +import json +import os +from pathlib import Path +import re +import shlex +import subprocess +from typing import Any + +from .models import QueryType + + +class QMDCommandError(RuntimeError): + def __init__(self, args: list[str], returncode: int, stdout: str, stderr: str) -> None: + cmd = " ".join(shlex.quote(x) for x in args) + message = f"qmd command failed ({returncode}): {cmd}\nstdout:\n{stdout}\nstderr:\n{stderr}" + super().__init__(message) + self.args_list = args + self.returncode = returncode + self.stdout = stdout + self.stderr = stderr + + +class QMDClient: + def __init__( + self, + qmd_binary: str, + timeout_seconds: int, + xdg_cache_home: Path, + xdg_config_home: Path, + ) -> None: + self.qmd_binary = qmd_binary + self.timeout_seconds = timeout_seconds + self.xdg_cache_home = xdg_cache_home + self.xdg_config_home = xdg_config_home + + def ensure_collection(self, index_name: str, collection_name: str, workspace_path: Path) -> bool: + existing = self.list_collections(index_name=index_name) + if collection_name in existing: + return False + + self._run( + [ + self.qmd_binary, + "--index", + index_name, + "collection", + "add", + str(workspace_path), + "--name", + collection_name, + ] + ) + return True + + def list_collections(self, index_name: str) -> set[str]: + output = self._run([self.qmd_binary, "--index", index_name, "collection", "list"]) + names: set[str] = set() + for line in output.splitlines(): + match = re.match(r"^([A-Za-z0-9_.-]+) \(qmd://", line.strip()) + if match: + names.add(match.group(1)) + return names + + def update_workspace(self, index_name: str) -> None: + self._run([self.qmd_binary, "--index", index_name, "update"]) + + def embed_workspace_if_needed(self, index_name: str, should_embed: bool) -> bool: + if not should_embed: + return False + self._run([self.qmd_binary, "--index", index_name, "embed"]) + return True + + def run_query( + self, + *, + index_name: str, + collection_name: str, + query_type: QueryType, + query: str, + n: int, + ) -> tuple[Any, str]: + command_name = self._query_command_name(query_type) + top_k = max(n, 10) if query_type == QueryType.deep_search else n + args = self._query_args( + index_name=index_name, + command_name=command_name, + query=query, + top_k=top_k, + collection_name=collection_name, + ) + try: + output = self._run(args) + return self._parse_output(output), command_name + except QMDCommandError: + # Query/deep_search may fail when LLM stack is not ready; keep API available. + if query_type not in {QueryType.query, QueryType.deep_search}: + raise + fallback_command = "search" + fallback_args = self._query_args( + index_name=index_name, + command_name=fallback_command, + query=query, + top_k=top_k, + collection_name=collection_name, + ) + fallback_output = self._run(fallback_args) + return self._parse_output(fallback_output), fallback_command + + def _query_command_name(self, query_type: QueryType) -> str: + if query_type in {QueryType.query, QueryType.deep_search}: + # Keep request latency bounded when LLM stack is not prewarmed. + return "search" + return query_type.value + + def _parse_output(self, output: str) -> Any: + stripped = output.strip() + if not stripped: + return [] + try: + return json.loads(stripped) + except json.JSONDecodeError: + return {"raw": stripped} + + def _query_args( + self, + *, + index_name: str, + command_name: str, + query: str, + top_k: int, + collection_name: str, + ) -> list[str]: + return [ + self.qmd_binary, + "--index", + index_name, + command_name, + query, + "--json", + "-n", + str(top_k), + "-c", + collection_name, + ] + + def _run(self, args: list[str]) -> str: + env = { + **os.environ, + "XDG_CACHE_HOME": str(self.xdg_cache_home), + "XDG_CONFIG_HOME": str(self.xdg_config_home), + } + try: + proc = subprocess.run( + args, + check=False, + capture_output=True, + text=True, + env=env, + timeout=self.timeout_seconds, + ) + except subprocess.TimeoutExpired as exc: + stdout = exc.stdout if isinstance(exc.stdout, str) else "" + stderr = exc.stderr if isinstance(exc.stderr, str) else "" + raise QMDCommandError( + args=args, + returncode=124, + stdout=stdout, + stderr=f"{stderr}\nTimeout after {self.timeout_seconds}s", + ) from exc + + if proc.returncode != 0: + raise QMDCommandError(args=args, returncode=proc.returncode, stdout=proc.stdout, stderr=proc.stderr) + return proc.stdout diff --git a/gateway/app/query_service.py b/gateway/app/query_service.py new file mode 100644 index 0000000..121fb22 --- /dev/null +++ b/gateway/app/query_service.py @@ -0,0 +1,98 @@ +from __future__ import annotations + +from typing import Any + +from .locks import WorkspaceLockManager +from .models import QueryRequest, QueryResponse, SyncRequest, SyncResponse +from .qmd_client import QMDClient +from .sync_service import SyncService + + +class QueryService: + def __init__( + self, + *, + sync_service: SyncService, + qmd_client: QMDClient, + lock_manager: WorkspaceLockManager, + ) -> None: + self.sync_service = sync_service + self.qmd_client = qmd_client + self.lock_manager = lock_manager + + def handle_query(self, request: QueryRequest) -> QueryResponse: + resolved = self.sync_service.workspace_manager.resolve_workspace( + branch=request.branch, + memory_profile=request.memory_profile, + ) + + with self.lock_manager.acquire(resolved.workspace_name) as lock_stats: + sync_meta = self.sync_service.sync_for_query( + branch=resolved.branch, + memory_profile=None, + require_latest=request.require_latest, + ) + + results, qmd_command = self.qmd_client.run_query( + index_name=sync_meta.qmd_index, + collection_name=sync_meta.qmd_collection, + query_type=request.query_type, + query=request.query, + n=request.n, + ) + + debug: dict[str, Any] | None = None + if request.debug: + debug = { + "lock_wait_ms": lock_stats.wait_ms, + "workspace_changed": sync_meta.changed, + "previous_commit": sync_meta.previous_commit, + "qmd_index": sync_meta.qmd_index, + "qmd_command": qmd_command, + "update_ran": sync_meta.update_ran, + "embed_ran": sync_meta.embed_ran, + "embed_error": sync_meta.embed_error, + } + + return QueryResponse( + ok=True, + branch=sync_meta.branch, + resolved_workspace=str(sync_meta.workspace_path), + commit_hash=sync_meta.commit_hash, + synced_at=sync_meta.synced_at, + query_type=request.query_type, + results=results, + qmd_collection=sync_meta.qmd_collection, + debug=debug, + ) + + def handle_sync(self, request: SyncRequest) -> SyncResponse: + resolved = self.sync_service.workspace_manager.resolve_workspace( + branch=request.branch, + memory_profile=request.memory_profile, + ) + + with self.lock_manager.acquire(resolved.workspace_name) as lock_stats: + sync_meta = self.sync_service.sync_explicit( + branch=resolved.branch, + memory_profile=None, + require_latest=request.require_latest, + ) + debug = { + "lock_wait_ms": lock_stats.wait_ms, + "workspace_changed": sync_meta.changed, + "previous_commit": sync_meta.previous_commit, + "qmd_index": sync_meta.qmd_index, + "update_ran": sync_meta.update_ran, + "embed_ran": sync_meta.embed_ran, + "embed_error": sync_meta.embed_error, + } + return SyncResponse( + ok=True, + branch=sync_meta.branch, + resolved_workspace=str(sync_meta.workspace_path), + commit_hash=sync_meta.commit_hash, + synced_at=sync_meta.synced_at, + qmd_collection=sync_meta.qmd_collection, + debug=debug, + ) diff --git a/gateway/app/sync_service.py b/gateway/app/sync_service.py new file mode 100644 index 0000000..0a44c52 --- /dev/null +++ b/gateway/app/sync_service.py @@ -0,0 +1,161 @@ +from __future__ import annotations + +from dataclasses import asdict, dataclass +from datetime import datetime, timezone +import json +from pathlib import Path +import threading +from typing import Any + +from .config import Settings +from .git_manager import GitCommandError, GitManager +from .qmd_client import QMDClient, QMDCommandError +from .workspace_manager import ResolvedWorkspace, WorkspaceManager + + +@dataclass(frozen=True) +class SyncMetadata: + branch: str + workspace_name: str + workspace_path: Path + qmd_collection: str + qmd_index: str + commit_hash: str + previous_commit: str | None + changed: bool + synced_at: datetime + created_collection: bool + update_ran: bool + embed_ran: bool + embed_error: str | None + + +class SyncService: + def __init__( + self, + settings: Settings, + workspace_manager: WorkspaceManager, + git_manager: GitManager, + qmd_client: QMDClient, + ) -> None: + self.settings = settings + self.workspace_manager = workspace_manager + self.git_manager = git_manager + self.qmd_client = qmd_client + self._mirror_lock = threading.Lock() + + self.settings.workspace_state_dir.mkdir(parents=True, exist_ok=True) + + def sync_for_query( + self, + *, + branch: str | None, + memory_profile: str | None, + require_latest: bool, + ) -> SyncMetadata: + resolved = self.workspace_manager.resolve_workspace(branch=branch, memory_profile=memory_profile) + return self._sync_resolved_workspace(resolved=resolved, require_latest=require_latest) + + def sync_explicit( + self, + *, + branch: str | None, + memory_profile: str | None, + require_latest: bool, + ) -> SyncMetadata: + resolved = self.workspace_manager.resolve_workspace(branch=branch, memory_profile=memory_profile) + return self._sync_resolved_workspace(resolved=resolved, require_latest=require_latest) + + def list_workspace_states(self) -> list[dict[str, Any]]: + results: list[dict[str, Any]] = [] + for path in sorted(self.settings.workspace_state_dir.glob("*.json")): + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError): + continue + results.append(payload) + return results + + def _sync_resolved_workspace(self, *, resolved: ResolvedWorkspace, require_latest: bool) -> SyncMetadata: + with self._mirror_lock: + self.git_manager.ensure_mirror() + if require_latest: + self.git_manager.fetch_origin() + commit_hash = self._resolve_branch_commit(resolved.branch, force_fetch=not require_latest) + + git_sync = self.git_manager.sync_workspace( + workspace_path=resolved.workspace_path, + branch=resolved.branch, + commit_hash=commit_hash, + ) + + created_collection = self.qmd_client.ensure_collection( + index_name=resolved.qmd_index, + collection_name=resolved.qmd_collection, + workspace_path=resolved.workspace_path, + ) + + update_ran = require_latest and self.settings.qmd_update_on_latest_query + if update_ran: + self.qmd_client.update_workspace(index_name=resolved.qmd_index) + + should_embed = self.settings.qmd_embed_on_change and (git_sync.changed or created_collection) + embed_ran = False + embed_error: str | None = None + if update_ran: + try: + embed_ran = self.qmd_client.embed_workspace_if_needed( + index_name=resolved.qmd_index, + should_embed=should_embed, + ) + except QMDCommandError as exc: + embed_ran = False + embed_error = str(exc) + + synced_at = datetime.now(timezone.utc) + + metadata = SyncMetadata( + branch=resolved.branch, + workspace_name=resolved.workspace_name, + workspace_path=resolved.workspace_path, + qmd_collection=resolved.qmd_collection, + qmd_index=resolved.qmd_index, + commit_hash=git_sync.commit_hash, + previous_commit=git_sync.previous_commit, + changed=git_sync.changed, + synced_at=synced_at, + created_collection=created_collection, + update_ran=update_ran, + embed_ran=embed_ran, + embed_error=embed_error, + ) + self._write_state(metadata) + return metadata + + def _resolve_branch_commit(self, branch: str, force_fetch: bool) -> str: + try: + return self.git_manager.get_branch_commit(branch) + except GitCommandError: + if not force_fetch: + raise + self.git_manager.fetch_origin() + return self.git_manager.get_branch_commit(branch) + + def _write_state(self, metadata: SyncMetadata) -> None: + path = self.settings.workspace_state_dir / f"{metadata.workspace_name}.json" + payload = { + "branch": metadata.branch, + "workspace": metadata.workspace_name, + "workspace_path": str(metadata.workspace_path), + "qmd_collection": metadata.qmd_collection, + "qmd_index": metadata.qmd_index, + "commit_hash": metadata.commit_hash, + "previous_commit": metadata.previous_commit, + "changed": metadata.changed, + "created_collection": metadata.created_collection, + "update_ran": metadata.update_ran, + "embed_ran": metadata.embed_ran, + "embed_error": metadata.embed_error, + "synced_at": metadata.synced_at.isoformat(), + } + path.write_text(json.dumps(payload, ensure_ascii=True, indent=2), encoding="utf-8") diff --git a/gateway/app/workspace_manager.py b/gateway/app/workspace_manager.py new file mode 100644 index 0000000..0595bc9 --- /dev/null +++ b/gateway/app/workspace_manager.py @@ -0,0 +1,79 @@ +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +import re + + +_PROFILE_MONTHLY_RE = re.compile(r"^monthly-(\d{4}-\d{2})$") + + +@dataclass(frozen=True) +class ResolvedWorkspace: + branch: str + workspace_name: str + workspace_path: Path + qmd_collection: str + qmd_index: str + + +class WorkspaceManager: + def __init__(self, workspaces_root: Path, default_branch: str = "main", qmd_index_prefix: str = "ws") -> None: + self.workspaces_root = workspaces_root + self.default_branch = default_branch + self.qmd_index_prefix = qmd_index_prefix + self.workspaces_root.mkdir(parents=True, exist_ok=True) + + def resolve_branch(self, branch: str | None, memory_profile: str | None) -> str: + if branch and branch.strip(): + return branch.strip() + if memory_profile and memory_profile.strip(): + return self._profile_to_branch(memory_profile.strip()) + return self.default_branch + + def resolve_workspace(self, branch: str | None, memory_profile: str | None) -> ResolvedWorkspace: + resolved_branch = self.resolve_branch(branch=branch, memory_profile=memory_profile) + workspace_name = self.workspace_name_for_branch(resolved_branch) + workspace_path = self.workspaces_root / workspace_name + qmd_collection = self.collection_name_for_workspace(workspace_name) + qmd_index = self.index_name_for_workspace(workspace_name) + return ResolvedWorkspace( + branch=resolved_branch, + workspace_name=workspace_name, + workspace_path=workspace_path, + qmd_collection=qmd_collection, + qmd_index=qmd_index, + ) + + def _profile_to_branch(self, memory_profile: str) -> str: + if memory_profile == "stable": + return "main" + monthly = _PROFILE_MONTHLY_RE.match(memory_profile) + if monthly: + return f"memory/{monthly.group(1)}" + if memory_profile.startswith("task-") and len(memory_profile) > len("task-"): + return f"task/{memory_profile[len('task-') :]}" + raise ValueError( + "unsupported memory_profile, expected stable | monthly-YYYY-MM | task-" + ) + + def workspace_name_for_branch(self, branch: str) -> str: + if branch in {"main", "stable"}: + return "main" + if branch.startswith("memory/"): + return f"memory-{branch.split('/', 1)[1].replace('/', '-') }" + if branch.startswith("task/"): + return f"task-{branch.split('/', 1)[1].replace('/', '-') }" + + return "branch-" + self._slugify(branch) + + def collection_name_for_workspace(self, workspace_name: str) -> str: + return self._slugify(workspace_name) + + def index_name_for_workspace(self, workspace_name: str) -> str: + return f"{self.qmd_index_prefix}_{self._slugify(workspace_name)}" + + def _slugify(self, value: str) -> str: + cleaned = re.sub(r"[^A-Za-z0-9_.-]+", "-", value) + cleaned = cleaned.strip("-") + return cleaned or "default" diff --git a/gateway/requirements.txt b/gateway/requirements.txt new file mode 100644 index 0000000..9f0c186 --- /dev/null +++ b/gateway/requirements.txt @@ -0,0 +1,3 @@ +fastapi==0.116.2 +uvicorn[standard]==0.35.0 +pydantic-settings==2.11.0 diff --git a/gateway/tests/conftest.py b/gateway/tests/conftest.py new file mode 100644 index 0000000..89e9d43 --- /dev/null +++ b/gateway/tests/conftest.py @@ -0,0 +1,135 @@ +from __future__ import annotations + +from dataclasses import dataclass +import os +from pathlib import Path +import subprocess +from typing import Any + +import pytest +from fastapi.testclient import TestClient + +# Keep module-level app import from trying to write to /data on local tests. +os.environ.setdefault("WORKSPACES_ROOT", "/tmp/qmd-gateway-tests/workspaces") +os.environ.setdefault("WORKSPACE_STATE_DIR", "/tmp/qmd-gateway-tests/state") +os.environ.setdefault("GIT_MIRROR_PATH", "/tmp/qmd-gateway-tests/git-mirror/repo.git") +os.environ.setdefault("GIT_REMOTE_URL", "/tmp/qmd-gateway-tests/remote.git") + +from app.config import Settings +from app.main import create_app +from app.models import QueryType + + +class FakeQMDClient: + def __init__(self) -> None: + self.collections: dict[str, dict[str, Path]] = {} + self.update_calls: list[str] = [] + self.embed_calls: list[str] = [] + + def ensure_collection(self, index_name: str, collection_name: str, workspace_path: Path) -> bool: + bucket = self.collections.setdefault(index_name, {}) + if collection_name in bucket: + return False + bucket[collection_name] = workspace_path + return True + + def list_collections(self, index_name: str) -> set[str]: + return set(self.collections.get(index_name, {}).keys()) + + def update_workspace(self, index_name: str) -> None: + self.update_calls.append(index_name) + + def embed_workspace_if_needed(self, index_name: str, should_embed: bool) -> bool: + if should_embed: + self.embed_calls.append(index_name) + return True + return False + + def run_query(self, *, index_name: str, collection_name: str, query_type: QueryType, query: str, n: int): + workspace = self.collections[index_name][collection_name] + needle = query.lower() + matches: list[dict[str, Any]] = [] + for path in sorted(workspace.rglob("*.md")): + text = path.read_text(encoding="utf-8") + if needle in text.lower() or needle in path.name.lower(): + matches.append({"file": str(path), "snippet": text[:120]}) + return matches[:n], query_type.value + + +@dataclass +class TestRepo: + remote_path: Path + seed_path: Path + + def commit_on_branch(self, branch: str, rel_path: str, content: str, message: str) -> str: + _run(["git", "-C", str(self.seed_path), "checkout", branch]) + file_path = self.seed_path / rel_path + file_path.parent.mkdir(parents=True, exist_ok=True) + file_path.write_text(content, encoding="utf-8") + _run(["git", "-C", str(self.seed_path), "add", rel_path]) + _run(["git", "-C", str(self.seed_path), "commit", "-m", message]) + _run(["git", "-C", str(self.seed_path), "push", "origin", branch]) + commit = _run(["git", "-C", str(self.seed_path), "rev-parse", "HEAD"]).strip() + return commit + + +@pytest.fixture() +def repo(tmp_path: Path) -> TestRepo: + remote = tmp_path / "remote.git" + seed = tmp_path / "seed" + + _run(["git", "init", "--bare", str(remote)]) + _run(["git", "clone", str(remote), str(seed)]) + _run(["git", "-C", str(seed), "config", "user.name", "Test User"]) + _run(["git", "-C", str(seed), "config", "user.email", "test@example.com"]) + + (seed / "README.md").write_text("main branch memory root\n", encoding="utf-8") + (seed / "docs" / "main-only.md").parent.mkdir(parents=True, exist_ok=True) + (seed / "docs" / "main-only.md").write_text("alpha-main-signal\n", encoding="utf-8") + _run(["git", "-C", str(seed), "add", "README.md", "docs/main-only.md"]) + _run(["git", "-C", str(seed), "commit", "-m", "init main"]) + _run(["git", "-C", str(seed), "branch", "-M", "main"]) + _run(["git", "-C", str(seed), "push", "-u", "origin", "main"]) + + _run(["git", "-C", str(seed), "checkout", "-b", "memory/2026-03"]) + (seed / "docs" / "monthly-only.md").write_text( + "beta-monthly-signal\nmonthly-exclusive-signal\n", + encoding="utf-8", + ) + _run(["git", "-C", str(seed), "add", "docs/monthly-only.md"]) + _run(["git", "-C", str(seed), "commit", "-m", "add monthly"]) + _run(["git", "-C", str(seed), "push", "-u", "origin", "memory/2026-03"]) + + _run(["git", "-C", str(seed), "checkout", "main"]) + (seed / "docs" / "main-exclusive.md").write_text("main-exclusive-signal\n", encoding="utf-8") + _run(["git", "-C", str(seed), "add", "docs/main-exclusive.md"]) + _run(["git", "-C", str(seed), "commit", "-m", "add main exclusive"]) + _run(["git", "-C", str(seed), "push", "origin", "main"]) + + return TestRepo(remote_path=remote, seed_path=seed) + + +@pytest.fixture() +def fake_qmd() -> FakeQMDClient: + return FakeQMDClient() + + +@pytest.fixture() +def client(tmp_path: Path, repo: TestRepo, fake_qmd: FakeQMDClient) -> TestClient: + settings = Settings( + git_remote_url=str(repo.remote_path), + git_mirror_path=tmp_path / "git-mirror" / "repo.git", + workspaces_root=tmp_path / "workspaces", + workspace_state_dir=tmp_path / "state", + xdg_cache_home=tmp_path / "cache", + xdg_config_home=tmp_path / "config", + ) + app = create_app(settings=settings, qmd_client=fake_qmd) + return TestClient(app) + + +def _run(args: list[str]) -> str: + proc = subprocess.run(args, check=False, capture_output=True, text=True) + if proc.returncode != 0: + raise RuntimeError(f"command failed: {' '.join(args)}\nstdout={proc.stdout}\nstderr={proc.stderr}") + return proc.stdout diff --git a/gateway/tests/test_branch_isolation.py b/gateway/tests/test_branch_isolation.py new file mode 100644 index 0000000..9c88a2b --- /dev/null +++ b/gateway/tests/test_branch_isolation.py @@ -0,0 +1,51 @@ +from __future__ import annotations + + +def test_branch_isolation(client): + main_resp = client.post( + "/query", + json={"branch": "main", "query_type": "search", "query": "main-exclusive-signal", "require_latest": True}, + ) + assert main_resp.status_code == 200 + main_payload = main_resp.json() + assert main_payload["branch"] == "main" + assert main_payload["results"] + + monthly_resp = client.post( + "/query", + json={"branch": "memory/2026-03", "query_type": "search", "query": "monthly-exclusive-signal", "require_latest": True}, + ) + assert monthly_resp.status_code == 200 + monthly_payload = monthly_resp.json() + assert monthly_payload["branch"] == "memory/2026-03" + assert monthly_payload["results"] + + cross_main = client.post( + "/query", + json={"branch": "main", "query_type": "search", "query": "monthly-exclusive-signal", "require_latest": True}, + ) + assert cross_main.status_code == 200 + assert cross_main.json()["results"] == [] + + cross_monthly = client.post( + "/query", + json={"branch": "memory/2026-03", "query_type": "search", "query": "main-exclusive-signal", "require_latest": True}, + ) + assert cross_monthly.status_code == 200 + assert cross_monthly.json()["results"] == [] + + +def test_memory_profile_and_default_branch(client): + profile_resp = client.post( + "/query", + json={"memory_profile": "monthly-2026-03", "query_type": "search", "query": "monthly-exclusive-signal", "require_latest": True}, + ) + assert profile_resp.status_code == 200 + assert profile_resp.json()["branch"] == "memory/2026-03" + + default_resp = client.post( + "/query", + json={"query_type": "search", "query": "main-exclusive-signal", "require_latest": True}, + ) + assert default_resp.status_code == 200 + assert default_resp.json()["branch"] == "main" diff --git a/gateway/tests/test_health.py b/gateway/tests/test_health.py new file mode 100644 index 0000000..71d1495 --- /dev/null +++ b/gateway/tests/test_health.py @@ -0,0 +1,9 @@ +from __future__ import annotations + + +def test_health(client): + resp = client.get("/health") + assert resp.status_code == 200 + payload = resp.json() + assert payload["ok"] is True + assert payload["service"] == "memory-gateway" diff --git a/gateway/tests/test_query_flow.py b/gateway/tests/test_query_flow.py new file mode 100644 index 0000000..d7132cd --- /dev/null +++ b/gateway/tests/test_query_flow.py @@ -0,0 +1,43 @@ +from __future__ import annotations + + +def test_query_flow_and_sync_before_query(client, repo): + first = client.post( + "/query", + json={ + "branch": "main", + "query_type": "query", + "query": "alpha-main-signal", + "require_latest": True, + }, + ) + assert first.status_code == 200 + first_payload = first.json() + assert first_payload["ok"] is True + assert first_payload["branch"] == "main" + assert first_payload["commit_hash"] + assert first_payload["synced_at"] + assert isinstance(first_payload["results"], list) + assert first_payload["results"] + + repo.commit_on_branch( + "main", + "docs/new-sync-note.md", + "gamma-sync-proof\n", + "add sync proof", + ) + + second = client.post( + "/query", + json={ + "branch": "main", + "query_type": "query", + "query": "gamma-sync-proof", + "require_latest": True, + }, + ) + assert second.status_code == 200 + second_payload = second.json() + assert second_payload["commit_hash"] != first_payload["commit_hash"] + snippets = [row["snippet"] for row in second_payload["results"]] + assert any("gamma-sync-proof" in snippet for snippet in snippets) diff --git a/gateway/tests/test_sync_logic.py b/gateway/tests/test_sync_logic.py new file mode 100644 index 0000000..9e1dab4 --- /dev/null +++ b/gateway/tests/test_sync_logic.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +from concurrent.futures import ThreadPoolExecutor + + +def test_sync_endpoint_and_status(client): + sync_resp = client.post("/sync", json={"branch": "main", "require_latest": True}) + assert sync_resp.status_code == 200 + sync_payload = sync_resp.json() + assert sync_payload["ok"] is True + assert sync_payload["branch"] == "main" + assert sync_payload["commit_hash"] + assert sync_payload["synced_at"] + + status_resp = client.get("/status") + assert status_resp.status_code == 200 + status_payload = status_resp.json() + assert status_payload["ok"] is True + assert any(item["branch"] == "main" for item in status_payload["workspaces"]) + + +def test_concurrent_branch_queries(client): + def run(body: dict): + return client.post("/query", json=body) + + with ThreadPoolExecutor(max_workers=2) as pool: + fut_main = pool.submit( + run, + {"branch": "main", "query_type": "search", "query": "main-exclusive-signal", "require_latest": True}, + ) + fut_month = pool.submit( + run, + { + "branch": "memory/2026-03", + "query_type": "search", + "query": "monthly-exclusive-signal", + "require_latest": True, + }, + ) + + resp_main = fut_main.result() + resp_month = fut_month.result() + + assert resp_main.status_code == 200 + assert resp_month.status_code == 200 + + payload_main = resp_main.json() + payload_month = resp_month.json() + + assert payload_main["branch"] == "main" + assert payload_month["branch"] == "memory/2026-03" + assert payload_main["resolved_workspace"] != payload_month["resolved_workspace"] + + assert payload_main["results"] + assert payload_month["results"] diff --git a/scripts/bootstrap.sh b/scripts/bootstrap.sh new file mode 100755 index 0000000..a6acd78 --- /dev/null +++ b/scripts/bootstrap.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "${ROOT_DIR}" + +mkdir -p data/git-mirror data/workspaces data/qmd-cache data/qmd-config + +REMOTE_BARE="${ROOT_DIR}/data/remote-memory.git" +if [[ ! -d "${REMOTE_BARE}" ]]; then + echo "[bootstrap] initializing demo remote repo at ${REMOTE_BARE}" + git init --bare "${REMOTE_BARE}" + + tmp_dir="$(mktemp -d)" + trap 'rm -rf "${tmp_dir}"' EXIT + + git clone "${REMOTE_BARE}" "${tmp_dir}/seed" + git -C "${tmp_dir}/seed" config user.name "Memory Gateway Bot" + git -C "${tmp_dir}/seed" config user.email "memory-gateway@example.local" + + mkdir -p "${tmp_dir}/seed/docs" + cat > "${tmp_dir}/seed/docs/main.md" <<'EOF' +# Main Memory + +router firmware build recovery strategy in main branch. +EOF + + git -C "${tmp_dir}/seed" add docs/main.md + git -C "${tmp_dir}/seed" commit -m "init main memory" + git -C "${tmp_dir}/seed" branch -M main + git -C "${tmp_dir}/seed" push -u origin main + + git -C "${tmp_dir}/seed" checkout -b memory/2026-03 + cat > "${tmp_dir}/seed/docs/monthly.md" <<'EOF' +# Monthly Memory 2026-03 + +monthly branch note for March 2026. +EOF + git -C "${tmp_dir}/seed" add docs/monthly.md + git -C "${tmp_dir}/seed" commit -m "add monthly memory" + git -C "${tmp_dir}/seed" push -u origin memory/2026-03 + + git -C "${tmp_dir}/seed" checkout -b task/TASK-001 main + cat > "${tmp_dir}/seed/docs/task-TASK-001.md" <<'EOF' +# Task Memory TASK-001 + +task specific recovery checkpoint. +EOF + git -C "${tmp_dir}/seed" add docs/task-TASK-001.md + git -C "${tmp_dir}/seed" commit -m "add task memory" + git -C "${tmp_dir}/seed" push -u origin task/TASK-001 +fi + +if [[ ! -f .env ]]; then + cp .env.example .env + sed -i "s|^GIT_REMOTE_URL=.*$|GIT_REMOTE_URL=/data/remote-memory.git|" .env + echo "[bootstrap] generated .env with local demo remote: /data/remote-memory.git" +fi + +echo "[bootstrap] done" diff --git a/scripts/entrypoint.sh b/scripts/entrypoint.sh new file mode 100755 index 0000000..2937a2c --- /dev/null +++ b/scripts/entrypoint.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -euo pipefail + +exec /app/scripts/qmd-entrypoint.sh "$@" diff --git a/scripts/qmd-entrypoint.sh b/scripts/qmd-entrypoint.sh new file mode 100755 index 0000000..a2af2f6 --- /dev/null +++ b/scripts/qmd-entrypoint.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +set -euo pipefail + +CACHE_HOME="${XDG_CACHE_HOME:-/var/lib/qmd/cache}" +CONFIG_HOME="${XDG_CONFIG_HOME:-/var/lib/qmd/config}" +PORT="${QMD_HTTP_PORT:-8181}" +INTERNAL_PORT="${QMD_INTERNAL_PORT:-8182}" +QMD_LLM_FILE="${QMD_LLM_FILE:-/usr/local/lib/node_modules/@tobilu/qmd/dist/llm.js}" + +mkdir -p "${CACHE_HOME}/qmd" "${CONFIG_HOME}/qmd" + +apply_model_overrides() { + if [[ -z "${QMD_EMBED_MODEL_URI:-}" && -z "${QMD_RERANK_MODEL_URI:-}" && -z "${QMD_GENERATE_MODEL_URI:-}" ]]; then + return 0 + fi + + if [[ ! -f "${QMD_LLM_FILE}" ]]; then + echo "WARN: llm.js not found at ${QMD_LLM_FILE}, skip model overrides." >&2 + return 0 + fi + + QMD_LLM_FILE="${QMD_LLM_FILE}" node - <<'NODE' +const fs = require("fs"); +const llmFile = process.env.QMD_LLM_FILE; +let src = fs.readFileSync(llmFile, "utf8"); + +const replacements = [ + ["DEFAULT_EMBED_MODEL", process.env.QMD_EMBED_MODEL_URI], + ["DEFAULT_RERANK_MODEL", process.env.QMD_RERANK_MODEL_URI], + ["DEFAULT_GENERATE_MODEL", process.env.QMD_GENERATE_MODEL_URI], +]; + +for (const [name, value] of replacements) { + if (!value) continue; + const pattern = new RegExp(`const ${name} = \\"[^\\"]+\\";`); + if (!pattern.test(src)) { + console.error(`WARN: failed to find ${name} in ${llmFile}`); + continue; + } + src = src.replace(pattern, `const ${name} = ${JSON.stringify(value)};`); +} + +fs.writeFileSync(llmFile, src, "utf8"); +NODE +} + +apply_model_overrides + +qmd mcp --http --port "${INTERNAL_PORT}" & +qmd_pid=$! + +socat "TCP-LISTEN:${PORT},fork,reuseaddr,bind=0.0.0.0" "TCP6:[::1]:${INTERNAL_PORT}" & +proxy_pid=$! + +cleanup() { + kill "${proxy_pid}" "${qmd_pid}" 2>/dev/null || true + wait "${proxy_pid}" "${qmd_pid}" 2>/dev/null || true +} + +trap cleanup SIGINT SIGTERM + +wait -n "${qmd_pid}" "${proxy_pid}" +status=$? +cleanup +exit "${status}" diff --git a/scripts/smoke-test.sh b/scripts/smoke-test.sh new file mode 100755 index 0000000..e749fe2 --- /dev/null +++ b/scripts/smoke-test.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "${ROOT_DIR}" + +log() { + printf '[smoke] %s\n' "$*" +} + +log "Bootstrapping demo repo" +./scripts/bootstrap.sh + +log "Starting services" +docker compose up -d --build + +log "Checking health" +curl -fsS http://127.0.0.1:8181/health >/dev/null +curl -fsS http://127.0.0.1:8787/health >/dev/null + +log "Running query through memory-gateway" +query_output="$(curl -fsS -X POST http://127.0.0.1:8787/query \ + -H 'content-type: application/json' \ + -d '{"branch":"main","query_type":"search","query":"router","require_latest":true,"debug":true}')" +printf '%s\n' "${query_output}" +printf '%s\n' "${query_output}" | grep -F 'commit_hash' >/dev/null +printf '%s\n' "${query_output}" | grep -F 'branch' >/dev/null + +log "Smoke test passed" diff --git a/scripts/warmup.sh b/scripts/warmup.sh new file mode 100755 index 0000000..20f6c95 --- /dev/null +++ b/scripts/warmup.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "${ROOT_DIR}" + +BASE_URL="${GATEWAY_BASE_URL:-http://127.0.0.1:8787}" + +wait_health() { + local attempts=60 + for _ in $(seq 1 "${attempts}"); do + if curl -fsS "${BASE_URL}/health" >/dev/null 2>&1; then + return 0 + fi + sleep 2 + done + echo "ERROR: gateway did not become healthy at ${BASE_URL}/health" >&2 + return 1 +} + +wait_health + +curl -fsS -X POST "${BASE_URL}/query" \ + -H 'content-type: application/json' \ + -d '{"branch":"main","query_type":"search","query":"router","require_latest":true,"debug":true}' >/dev/null + +echo "Warmup completed." diff --git a/testdata/docs/disaster-recovery.md b/testdata/docs/disaster-recovery.md new file mode 100644 index 0000000..3605a9a --- /dev/null +++ b/testdata/docs/disaster-recovery.md @@ -0,0 +1,6 @@ +# Disaster Recovery Policy + +- Recovery Time Objective (RTO): 2 hours. +- Recovery Point Objective (RPO): 15 minutes. +- Test failover monthly. +- Keep indexed meeting notes for decision traceability. diff --git a/testdata/docs/incident-runbook.md b/testdata/docs/incident-runbook.md new file mode 100644 index 0000000..4441e3e --- /dev/null +++ b/testdata/docs/incident-runbook.md @@ -0,0 +1,8 @@ +# Incident Runbook + +When the QMD HTTP MCP service is degraded: + +- Check `GET /health`. +- Validate vector index status with `qmd status`. +- Restart container only after collecting logs. +- Escalate if query quality drops after embedding drift. diff --git a/testdata/meetings/2026-03-01-architecture.md b/testdata/meetings/2026-03-01-architecture.md new file mode 100644 index 0000000..11cdee0 --- /dev/null +++ b/testdata/meetings/2026-03-01-architecture.md @@ -0,0 +1,8 @@ +# Architecture Sync 2026-03-01 + +Decisions: + +- Use QMD as internal shared MCP endpoint. +- Keep HTTP endpoint on port 8181. +- Persist cache and index to avoid repeated cold starts. +- Confirm disaster recovery RTO target remains 2 hours. diff --git a/testdata/meetings/2026-03-02-ops.md b/testdata/meetings/2026-03-02-ops.md new file mode 100644 index 0000000..396661f --- /dev/null +++ b/testdata/meetings/2026-03-02-ops.md @@ -0,0 +1,6 @@ +# Ops Review 2026-03-02 + +Action items: + +- Add smoke test covering health, collection add, embed, search, and query. +- Validate that deep query can locate runbook and recovery policy. diff --git a/testdata/notes/ops-checklist.md b/testdata/notes/ops-checklist.md new file mode 100644 index 0000000..dfa0589 --- /dev/null +++ b/testdata/notes/ops-checklist.md @@ -0,0 +1,6 @@ +# Ops Checklist + +1. Verify MCP service health endpoint before release. +2. Confirm embeddings are up to date. +3. Keep cache volume persistent to avoid repeated model downloads. +4. Keep runbook links in docs/incident-runbook.md. diff --git a/testdata/notes/personal-productivity.md b/testdata/notes/personal-productivity.md new file mode 100644 index 0000000..05fa620 --- /dev/null +++ b/testdata/notes/personal-productivity.md @@ -0,0 +1,6 @@ +# Personal Productivity Notes + +- Daily review at 09:00. +- Focus blocks: 90 minutes. +- Keep a short incident checklist for local services. +- Search keyword target: lightweight knowledge retrieval.