diff --git a/.gitea/workflows/cron.yaml b/.gitea/workflows/cron.yaml index cda5d0b..8b68ebe 100644 --- a/.gitea/workflows/cron.yaml +++ b/.gitea/workflows/cron.yaml @@ -18,11 +18,60 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4 - - run: docker build -f pkg/Dockerfile -t ci-image:${{ github.sha }} . + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 + with: + python-version: "3.13" + - name: Install dependencies + run: pip install -r requirements.txt pytest + - name: Run unit tests + run: pytest tests/ -v + env: + # indexer/server read env at import; unit tests need no real backend + QDRANT_URL: http://localhost:6333 + COLLECTION_NAME: test + + integration-test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + - name: Cleanup old Qdrant container + run: | + docker rm -f qdrant-${{ github.run_id }} 2>/dev/null || true + + - name: Start Qdrant + run: | + docker run -d --name qdrant-${{ github.run_id }} \ + --network "container:$(hostname)" \ + docker.io/qdrant/qdrant:latest + + - name: Wait for Qdrant + run: | + for i in $(seq 1 30); do + curl -s http://localhost:6333/healthz \ + && echo "QDRANT ready" && break + echo "Waiting for Qdrant... ($i/30)" + sleep 1 + done + curl -s http://localhost:6333/healthz > /dev/null + + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 + with: + python-version: "3.13" + - name: Install dependencies + run: pip install -r requirements.txt pytest + - name: Run integration tests + run: pytest tests/ -v -m integration + env: + QDRANT_URL: http://localhost:6333 + COLLECTION_NAME: test_mcp_maildir + + - name: Cleanup Qdrant + if: always() + run: docker rm -f qdrant-${{ github.run_id }} || true build-push: - needs: [test] + needs: [test, integration-test] runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 diff --git a/.gitea/workflows/docker-build.yaml b/.gitea/workflows/docker-build.yaml index faf470b..8972a8e 100644 --- a/.gitea/workflows/docker-build.yaml +++ b/.gitea/workflows/docker-build.yaml @@ -29,8 +29,8 @@ jobs: - name: Start Qdrant run: | - docker rm -f qdrant 2>/dev/null || true - docker run -d --name qdrant \ + docker rm -f qdrant-${{ github.run_id }} 2>/dev/null || true + docker run -d --name qdrant-${{ github.run_id }} \ --network "container:$(hostname)" \ docker.io/qdrant/qdrant:latest @@ -49,14 +49,14 @@ jobs: run: | pip install -r requirements.txt pytest - name: Run integration tests - run: pytest tests/test_integration.py -v + run: pytest tests/ -v -m integration env: QDRANT_URL: http://localhost:6333 COLLECTION_NAME: test_mcp_maildir - name: Cleanup Qdrant if: always() - run: docker rm -f qdrant 2>/dev/null || true + run: docker rm -f qdrant-${{ github.run_id }} 2>/dev/null || true build: runs-on: ubuntu-latest diff --git a/.gitea/workflows/main.yaml b/.gitea/workflows/main.yaml index fecf74b..a1ad96e 100644 --- a/.gitea/workflows/main.yaml +++ b/.gitea/workflows/main.yaml @@ -18,11 +18,60 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4 - - run: docker build -f pkg/Dockerfile -t ci-image:${{ github.sha }} . + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 + with: + python-version: "3.13" + - name: Install dependencies + run: pip install -r requirements.txt pytest + - name: Run unit tests + run: pytest tests/ -v + env: + # indexer/server read env at import; unit tests need no real backend + QDRANT_URL: http://localhost:6333 + COLLECTION_NAME: test + + integration-test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + - name: Cleanup old Qdrant container + run: | + docker rm -f qdrant-${{ github.run_id }} 2>/dev/null || true + + - name: Start Qdrant + run: | + docker run -d --name qdrant-${{ github.run_id }} \ + --network "container:$(hostname)" \ + docker.io/qdrant/qdrant:latest + + - name: Wait for Qdrant + run: | + for i in $(seq 1 30); do + curl -s http://localhost:6333/healthz \ + && echo "QDRANT ready" && break + echo "Waiting for Qdrant... ($i/30)" + sleep 1 + done + curl -s http://localhost:6333/healthz > /dev/null + + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 + with: + python-version: "3.13" + - name: Install dependencies + run: pip install -r requirements.txt pytest + - name: Run integration tests + run: pytest tests/ -v -m integration + env: + QDRANT_URL: http://localhost:6333 + COLLECTION_NAME: test_mcp_maildir + + - name: Cleanup Qdrant + if: always() + run: docker rm -f qdrant-${{ github.run_id }} || true build: - needs: [test] + needs: [test, integration-test] runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 diff --git a/.gitea/workflows/pr.yaml b/.gitea/workflows/pr.yaml index 48735fa..47e08fd 100644 --- a/.gitea/workflows/pr.yaml +++ b/.gitea/workflows/pr.yaml @@ -14,9 +14,25 @@ jobs: with: dockerfile: pkg/Dockerfile + pytest: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 + with: + python-version: "3.13" + - name: Install dependencies + run: pip install -r requirements.txt pytest + - name: Run unit tests + run: pytest tests/ -v + env: + # indexer/server read env at import; unit tests need no real backend + QDRANT_URL: http://localhost:6333 + COLLECTION_NAME: test + build-test: runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4 - - run: docker build -f pkg/Dockerfile -t ci-image:${{ github.sha }} . \ No newline at end of file + - run: docker build -f pkg/Dockerfile -t ci-image:${{ github.sha }} . diff --git a/.gitea/workflows/tag.yaml b/.gitea/workflows/tag.yaml index b90cfcd..ceb1674 100644 --- a/.gitea/workflows/tag.yaml +++ b/.gitea/workflows/tag.yaml @@ -18,11 +18,60 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4 - - run: docker build -f pkg/Dockerfile -t ci-image:${{ github.sha }} . + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 + with: + python-version: "3.13" + - name: Install dependencies + run: pip install -r requirements.txt pytest + - name: Run unit tests + run: pytest tests/ -v + env: + # indexer/server read env at import; unit tests need no real backend + QDRANT_URL: http://localhost:6333 + COLLECTION_NAME: test + + integration-test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + - name: Cleanup old Qdrant container + run: | + docker rm -f qdrant-${{ github.run_id }} 2>/dev/null || true + + - name: Start Qdrant + run: | + docker run -d --name qdrant-${{ github.run_id }} \ + --network "container:$(hostname)" \ + docker.io/qdrant/qdrant:latest + + - name: Wait for Qdrant + run: | + for i in $(seq 1 30); do + curl -s http://localhost:6333/healthz \ + && echo "QDRANT ready" && break + echo "Waiting for Qdrant... ($i/30)" + sleep 1 + done + curl -s http://localhost:6333/healthz > /dev/null + + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 + with: + python-version: "3.13" + - name: Install dependencies + run: pip install -r requirements.txt pytest + - name: Run integration tests + run: pytest tests/ -v -m integration + env: + QDRANT_URL: http://localhost:6333 + COLLECTION_NAME: test_mcp_maildir + + - name: Cleanup Qdrant + if: always() + run: docker rm -f qdrant-${{ github.run_id }} || true build-push: - needs: [test] + needs: [test, integration-test] runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 diff --git a/pytest.ini b/pytest.ini index 80432c2..fe9256e 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,3 +1,8 @@ [pytest] testpaths = tests pythonpath = src +; Integration tests (tests/test_integration.py) require a live Qdrant on +; localhost:6333 — they only run in the nightly CI job (or manually with -m integration). +markers = + integration: requires a live Qdrant instance (deselected by default) +addopts = -m "not integration" -p no:cacheprovider diff --git a/src/indexer.py b/src/indexer.py index 8682934..944fcc1 100644 --- a/src/indexer.py +++ b/src/indexer.py @@ -22,26 +22,27 @@ from bs4 import BeautifulSoup # Load .env config load_dotenv() -# Configuration +# Configuration (validated lazily in main() so the module can be imported +# and unit-tested without environment variables) MAILDIR_PATH = os.environ.get("MAILDIR_PATH", "") MAILDIR_FOLDERS = os.environ.get("MAILDIR_FOLDERS", "") QDRANT_URL = os.environ.get("QDRANT_URL", "") COLLECTION_NAME = os.environ.get("COLLECTION_NAME", "") -if not MAILDIR_PATH: - raise ValueError("MAILDIR_PATH environment variable is required.") -if not QDRANT_URL: - raise ValueError("QDRANT_URL environment variable is required.") -if not COLLECTION_NAME: - raise ValueError("COLLECTION_NAME environment variable is required.") - -EMBEDDING_MODEL_NAME = os.environ.get("EMBEDDING_MODEL_NAME", "BAAI/bge-small-en-v1.5") -BATCH_SIZE = int(os.environ.get("BATCH_SIZE", "100")) -EMBEDDING_BATCH_SIZE = int(os.environ.get("EMBEDDING_BATCH_SIZE", "64")) METADATA_COLLECTION = "mcp_indexer_metadata" INCREMENTAL_DAYS = int(os.environ.get("INCREMENTAL_DAYS", "7")) FORCE_REINDEX = os.environ.get("FORCE_REINDEX", "").lower() in ("1", "true", "yes") + +def _validate_config(): + """Raises if required environment variables are missing. Called from main().""" + if not MAILDIR_PATH: + raise ValueError("MAILDIR_PATH environment variable is required.") + if not QDRANT_URL: + raise ValueError("QDRANT_URL environment variable is required.") + if not COLLECTION_NAME: + raise ValueError("COLLECTION_NAME environment variable is required.") + def decode_mime_words(s: str) -> str: """Decodes MIME encoded strings (e.g. subjects, filenames).""" if not s: @@ -251,6 +252,8 @@ def main(): Main ingestion function. Reads Maildir, extracts text, generates local embeddings, and pushes to Qdrant. """ + _validate_config() + print(f"Indexing emails from {MAILDIR_PATH} into {QDRANT_URL}...") if not os.path.exists(MAILDIR_PATH): diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..bcd0466 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,15 @@ +"""Shared test configuration — sets env vars ONCE before any test module runs. + +The env must be consistent for the whole pytest session: server.py freezes +COLLECTION_NAME/QDRANT_URL at import time, while the integration fixture +reads os.environ at fixture time. If any test module mutates os.environ after +that (test_server.py used to set COLLECTION_NAME='test'), the fixture creates +a different collection than the one server functions query → 404. +Keep all env wiring here so import order no longer matters. +""" + +import os + +os.environ.setdefault("QDRANT_URL", "http://localhost:6333") +os.environ.setdefault("COLLECTION_NAME", "test_mcp_maildir") +os.environ.setdefault("EMBEDDING_MODEL_NAME", "BAAI/bge-small-en-v1.5") diff --git a/tests/test_indexer.py b/tests/test_indexer.py new file mode 100644 index 0000000..4b33ce7 --- /dev/null +++ b/tests/test_indexer.py @@ -0,0 +1,229 @@ +"""Unit tests for mcp-maildir indexer parsing functions. + +Pure-function tests: no Qdrant, no embedding model, no network. +A throwaway Maildir is built in tmp_path for get_recent_keys(). +""" + +import mailbox +import os +import time +import uuid +from datetime import datetime, timezone, timedelta +from email.message import EmailMessage +from email.utils import format_datetime + +import pytest + +from indexer import ( + decode_mime_words, + extract_text_from_html, + normalize_email_address, + parse_email_message, + get_recent_keys, + METADATA_COLLECTION, +) + + +# --------------------------------------------------------------------------- +# decode_mime_words +# --------------------------------------------------------------------------- + +class TestDecodeMimeWords: + def test_empty_and_none(self): + assert decode_mime_words("") == "" + assert decode_mime_words(None) == "" + + def test_plain_ascii(self): + assert decode_mime_words("Hello World") == "Hello World" + + def test_encoded_word_utf8(self): + # =?utf-8?q?...?= encoded word + assert decode_mime_words("=?utf-8?q?Caf=C3=A9_au_lait?=") == "Café au lait" + + def test_encoded_word_iso8859(self): + # decode_mime_words does not strip whitespace around the words + assert decode_mime_words("=?iso-8859-1?q?caf=E9?=") == "café" + + def test_mixed_encoded_and_plain(self): + assert decode_mime_words("Re: =?utf-8?q?planning_2026?=") == "Re: planning 2026" + + +# --------------------------------------------------------------------------- +# extract_text_from_html +# --------------------------------------------------------------------------- + +class TestExtractTextFromHtml: + def test_simple_paragraphs(self): + html = "
Hello
World
" + assert extract_text_from_html(html) == "Hello World" + + def test_strips_tags_and_scripts(self): + html = "