Merge pull request 'ci: harden the qdrant integration job (5s client timeout + cold storage)' (#60) from ci/harden-qdrant-integration into main

Reviewed-on: #60

.gitea/workflows/docker-build.yaml

@@ -32,15 +32,32 @@ jobs:
           docker rm -f qdrant-${{ github.run_id }} 2>/dev/null || true
           docker run -d --name qdrant-${{ github.run_id }} \
             --network "container:$(hostname)" \
-            docker.io/qdrant/qdrant:latest
+            docker.io/qdrant/qdrant:v1.19
 
       - name: Wait for Qdrant
         run: |
-          for i in $(seq 1 30); do
-            curl -s http://localhost:6333/healthz && echo "QDRANT ready" && break
-            echo "Waiting for Qdrant... ($i/30)"
+          # /healthz answers before the storage engine accepts writes, and a cold
+          # container on a loaded host can take several seconds to create its
+          # first collection. Gate on a real write round-trip, not on /healthz.
+          ready=""
+          for i in $(seq 1 60); do
+            if curl -sf "http://localhost:6333/healthz" > /dev/null 2>&1 \
+              && curl -sf -X PUT "http://localhost:6333/collections/ci_readiness_probe?timeout=60" \
+                   -H 'Content-Type: application/json' \
+                   -d '{"vectors":{"size":4,"distance":"Cosine"}}' > /dev/null 2>&1; then
+              curl -sf -X DELETE "http://localhost:6333/collections/ci_readiness_probe" > /dev/null 2>&1 || true
+              ready="yes"
+              echo "QDRANT ready: write path OK (attempt $i)"
+              break
+            fi
+            echo "Waiting for Qdrant... ($i/60)"
             sleep 1
           done
+          if [ -z "$ready" ]; then
+            echo "Qdrant write path never became ready"
+            docker logs --tail 100 qdrant-${{ github.run_id }} || true
+            exit 1
+          fi
 
       - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7
         with:
@@ -53,6 +70,15 @@ jobs:
         env:
           QDRANT_URL: http://localhost:6333
           COLLECTION_NAME: test_mcp_maildir
+          # The qdrant client defaults to a 5s request timeout, which a cold
+          # collection on a shared runner can exceed; this job has no SLA.
+          QDRANT_TIMEOUT: "60"
+
+      - name: Qdrant diagnostics
+        if: failure()
+        run: |
+          docker ps -a || true
+          docker logs --tail 200 qdrant-${{ github.run_id }} || true
 
       - name: Cleanup Qdrant
         if: always()

.gitea/workflows/main.yaml

@@ -22,7 +22,9 @@ jobs:
         with:
           python-version: "3.14"
       - name: Install dependencies
-        run: pip install -r requirements.txt pytest
+        run: pip install -r requirements.txt pytest ruff==0.17.1
+      - name: Lint (undefined names)
+        run: ruff check src/ tests/ --select F821
       - name: Run unit tests
         run: pytest tests/ -v
         env:
@@ -43,17 +45,32 @@ jobs:
         run: |
           docker run -d --name qdrant-${{ github.run_id }} \
             --network "container:$(hostname)" \
-            docker.io/qdrant/qdrant:latest
+            docker.io/qdrant/qdrant:v1.19
 
       - name: Wait for Qdrant
         run: |
-          for i in $(seq 1 30); do
-            curl -s http://localhost:6333/healthz \
-              && echo "QDRANT ready" && break
-            echo "Waiting for Qdrant... ($i/30)"
+          # /healthz answers before the storage engine accepts writes, and a cold
+          # container on a loaded host can take several seconds to create its
+          # first collection. Gate on a real write round-trip, not on /healthz.
+          ready=""
+          for i in $(seq 1 60); do
+            if curl -sf "http://localhost:6333/healthz" > /dev/null 2>&1 \
+              && curl -sf -X PUT "http://localhost:6333/collections/ci_readiness_probe?timeout=60" \
+                   -H 'Content-Type: application/json' \
+                   -d '{"vectors":{"size":4,"distance":"Cosine"}}' > /dev/null 2>&1; then
+              curl -sf -X DELETE "http://localhost:6333/collections/ci_readiness_probe" > /dev/null 2>&1 || true
+              ready="yes"
+              echo "QDRANT ready: write path OK (attempt $i)"
+              break
+            fi
+            echo "Waiting for Qdrant... ($i/60)"
             sleep 1
           done
-          curl -s http://localhost:6333/healthz > /dev/null
+          if [ -z "$ready" ]; then
+            echo "Qdrant write path never became ready"
+            docker logs --tail 100 qdrant-${{ github.run_id }} || true
+            exit 1
+          fi
 
       - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7
         with:
@@ -65,6 +82,15 @@ jobs:
         env:
           QDRANT_URL: http://localhost:6333
           COLLECTION_NAME: test_mcp_maildir
+          # The qdrant client defaults to a 5s request timeout, which a cold
+          # collection on a shared runner can exceed; this job has no SLA.
+          QDRANT_TIMEOUT: "60"
+
+      - name: Qdrant diagnostics
+        if: failure()
+        run: |
+          docker ps -a || true
+          docker logs --tail 200 qdrant-${{ github.run_id }} || true
 
       - name: Cleanup Qdrant
         if: always()

docker-compose.yml

@@ -1,6 +1,6 @@
 services:
   qdrant:
-    image: docker.io/qdrant/qdrant:latest
+    image: docker.io/qdrant/qdrant:v1.19
     container_name: mcp_maildir_qdrant
     ports:
       - "127.0.0.1:6333:6333" # REST API
@@ -15,7 +15,7 @@ services:
       dockerfile: pkg/Dockerfile
     container_name: mcp_maildir_server
     ports:
-      - "127.0.0.1:8000:8000" # Expose the MCP HTTP (SSE) server
+      - "127.0.0.1:8000:8000" # Expose the MCP Streamable HTTP server (/mcp)
     env_file:
       - .env
     environment:

renovate.json

@@ -3,9 +3,12 @@
   "customManagers": [
     {
       "customType": "regex",
-      "description": "Track qdrant/qdrant Docker image from docker-compose.yml (native manager missed it)",
-      "managerFilePatterns": ["/^docker-compose\\.ya?ml$/"],
-      "matchStrings": ["image:\\s*(?:docker\\.io/)?qdrant/qdrant:(?<currentValue>[^\\s]+)"],
+      "description": "Track the pinned qdrant/qdrant image in docker-compose.yml and the CI workflows (native manager missed it)",
+      "managerFilePatterns": [
+        "/^docker-compose\\.ya?ml$/",
+        "/^\\.gitea/workflows/.*\\.ya?ml$/"
+      ],
+      "matchStrings": ["(?:docker\\.io/)?qdrant/qdrant:(?<currentValue>[^\\s\"']+)"],
       "depNameTemplate": "qdrant/qdrant",
       "datasourceTemplate": "docker"
     }

src/indexer.py

@@ -29,6 +29,13 @@ MAILDIR_FOLDERS = os.environ.get("MAILDIR_FOLDERS", "")
 QDRANT_URL = os.environ.get("QDRANT_URL", "")
 COLLECTION_NAME = os.environ.get("COLLECTION_NAME", "")
 
+# Embedding model and batching. EMBEDDING_MODEL_NAME MUST stay in sync with the
+# one used by server.py: indexer and server have to embed with the same model,
+# otherwise the vectors stored in Qdrant are not comparable with query vectors.
+EMBEDDING_MODEL_NAME = os.environ.get("EMBEDDING_MODEL_NAME", "BAAI/bge-small-en-v1.5")
+BATCH_SIZE = int(os.environ.get("BATCH_SIZE", "100"))
+EMBEDDING_BATCH_SIZE = int(os.environ.get("EMBEDDING_BATCH_SIZE", "64"))
+
 METADATA_COLLECTION = "mcp_indexer_metadata"
 INCREMENTAL_DAYS = int(os.environ.get("INCREMENTAL_DAYS", "7"))
 FORCE_REINDEX = os.environ.get("FORCE_REINDEX", "").lower() in ("1", "true", "yes")

src/server.py

@@ -9,6 +9,7 @@ from typing import Any, Optional
 import qdrant_client as qdrant_pkg
 from dateutil import parser as date_parser
 from email.utils import parseaddr
+import uvicorn
 from dotenv import load_dotenv
 from starlette.requests import Request
 from starlette.responses import JSONResponse
@@ -36,6 +37,10 @@ COLLECTION_NAME = os.environ.get("COLLECTION_NAME", "")
 EMBEDDING_MODEL_NAME = os.environ.get("EMBEDDING_MODEL_NAME", "BAAI/bge-small-en-v1.5")
 DEFAULT_LIMIT = int(os.environ.get("SEARCH_LIMIT", "50"))
 
+# Per-request timeout (seconds) for Qdrant. The client default is 5s, which is
+# tight for a cold collection or a busy host: unset means keep that default.
+QDRANT_TIMEOUT = int(os.environ["QDRANT_TIMEOUT"]) if os.environ.get("QDRANT_TIMEOUT") else None
+
 if not QDRANT_URL:
     raise ValueError("QDRANT_URL environment variable is required.")
 if not COLLECTION_NAME:
@@ -60,8 +65,8 @@ def get_qdrant_client() -> QdrantClient:
     """Returns a singleton Qdrant client."""
     global _qdrant_client
     if _qdrant_client is None:
-        logger.info(f"Connecting to Qdrant at {QDRANT_URL}")
-        _qdrant_client = QdrantClient(url=QDRANT_URL)
+        logger.info(f"Connecting to Qdrant at {QDRANT_URL} (timeout={QDRANT_TIMEOUT})")
+        _qdrant_client = QdrantClient(url=QDRANT_URL, timeout=QDRANT_TIMEOUT)
     return _qdrant_client
 
 
@@ -340,6 +345,23 @@ def read_email(message_id: str):
         return {"error": f"read_email failed: {exc}"}
 
 
+# Streamable HTTP is the transport supported by current MCP clients (OpenCode,
+# Claude, ...). The legacy SSE transport is deprecated and is also a POST
+# target for clients that only speak Streamable HTTP, which answers 405.
+MCP_HOST = "0.0.0.0"
+MCP_PORT = 8000
+MCP_HTTP_PATH = "/mcp"
+
+
+def build_app():
+    """Builds the ASGI app serving the MCP endpoint over Streamable HTTP.
+
+    The MCP endpoint lives at MCP_HTTP_PATH; the custom route declared above
+    keeps serving GET /health for the Kubernetes probes.
+    """
+    return mcp.http_app(path=MCP_HTTP_PATH)
+
+
 if __name__ == "__main__":
     logger.info("Initializing models before starting server...")
     try:
@@ -348,6 +370,5 @@ if __name__ == "__main__":
     except Exception as e:
         logger.error(f"Failed to load models: {e}")
 
-    logger.info("Starting SSE server on 0.0.0.0:8000...")
-    # Start the MCP server using SSE (Server-Sent Events) over HTTP
-    mcp.run(transport="sse", host="0.0.0.0", port=8000)
+    logger.info(f"Starting Streamable HTTP server on {MCP_HOST}:{MCP_PORT}{MCP_HTTP_PATH}...")
+    uvicorn.run(build_app(), host=MCP_HOST, port=MCP_PORT)

tests/test_indexer.py

@@ -21,6 +21,9 @@ from indexer import (
     parse_email_message,
     get_recent_keys,
     METADATA_COLLECTION,
+    EMBEDDING_MODEL_NAME,
+    BATCH_SIZE,
+    EMBEDDING_BATCH_SIZE,
 )
 
 
@@ -227,3 +230,25 @@ class TestGetRecentKeys:
         assert key.split(":")[0] in keys
         for k in keys:
             assert ":" not in k
+
+
+# ---------------------------------------------------------------------------
+# Regression: module-level constants consumed by main()
+# ---------------------------------------------------------------------------
+
+
+def test_indexer_module_constants_are_defined():
+    """Regression (e32806a): the "make indexer importable without env" refactor
+    dropped these three module-level definitions while main() kept using them,
+    so every indexer run died with NameError before touching Qdrant."""
+    assert isinstance(EMBEDDING_MODEL_NAME, str) and EMBEDDING_MODEL_NAME
+    assert isinstance(BATCH_SIZE, int) and BATCH_SIZE > 0
+    assert isinstance(EMBEDDING_BATCH_SIZE, int) and EMBEDDING_BATCH_SIZE > 0
+
+
+def test_embedding_model_is_the_same_as_the_server_one():
+    """Indexer and server must embed with the same model, otherwise the vectors
+    stored by the indexer are not comparable with the query vectors."""
+    from server import EMBEDDING_MODEL_NAME as server_embedding_model
+
+    assert EMBEDDING_MODEL_NAME == server_embedding_model

tests/test_transport.py

@@ -0,0 +1,27 @@
+"""Transport-level tests for the mcp-maildir ASGI app.
+
+Current MCP clients (OpenCode, Claude, ...) only speak Streamable HTTP: they
+POST JSON-RPC to the configured URL. Serving the legacy SSE transport alone
+makes those clients fail with `405 Method Not Allowed`, so the app built by
+`server.build_app()` must expose the MCP endpoint at /mcp.
+
+Env vars are set in tests/conftest.py (shared session-wide).
+"""
+
+from server import build_app, MCP_HTTP_PATH
+
+
+def app_paths() -> set:
+    return {getattr(route, "path", None) for route in build_app().routes}
+
+
+class TestTransport:
+    def test_mcp_endpoint_is_streamable_http(self):
+        assert MCP_HTTP_PATH == "/mcp"
+        assert MCP_HTTP_PATH in app_paths()
+
+    def test_health_endpoint_is_exposed_for_probes(self):
+        assert "/health" in app_paths()
+
+    def test_legacy_sse_endpoint_is_gone(self):
+        assert "/sse" not in app_paths()