test: add indexer unit tests + integration marker; make indexer importable without env

- pytest.ini: 'integration' marker, deselected by default; the integration
  suite requires a live Qdrant and will only run in the nightly CI job.
- src/indexer.py: move env validation from import time into _validate_config()
  called by main() — the module can now be imported without MAILDIR_PATH /
  QDRANT_URL / COLLECTION_NAME set, enabling unit tests.
- tests/test_indexer.py: 27 new unit tests, no network and no model:
  decode_mime_words (encoded-words utf-8/iso-8859-1, mixed), extract_text_from_html
  (tags, scripts), normalize_email_address, parse_email_message with real
  EmailMessage objects (plain bodies, attachments detected and excluded from
  body, HTML alternative, unknown charset fallback via raw bytes message,
  encoded attachment filename), get_recent_keys against a real tmp_path
  Maildir (fresh mtime selected, old mtime excluded, colon-suffix keys).
  mailbox.Maildir(create=True) does not create cur/new/tmp on Python 3.11 —
  the test helper pre-creates them.
This commit is contained in:
OpenCode Cabillot committed 2026-10-09 00:16:10 +00:00
1 parent 6b17ca4ad5
commit e32806aab9
3 files changed
+248 -11

No files matched your search

+14 -11
View File
@@ -22,26 +22,27 @@ from bs4 import BeautifulSoup
# Load .env config
load_dotenv()
# Configuration
# Configuration (validated lazily in main() so the module can be imported
# and unit-tested without environment variables)
MAILDIR_PATH = os.environ.get("MAILDIR_PATH", "")
MAILDIR_FOLDERS = os.environ.get("MAILDIR_FOLDERS", "")
QDRANT_URL = os.environ.get("QDRANT_URL", "")
COLLECTION_NAME = os.environ.get("COLLECTION_NAME", "")
if not MAILDIR_PATH:
raise ValueError("MAILDIR_PATH environment variable is required.")
if not QDRANT_URL:
raise ValueError("QDRANT_URL environment variable is required.")
if not COLLECTION_NAME:
raise ValueError("COLLECTION_NAME environment variable is required.")
EMBEDDING_MODEL_NAME = os.environ.get("EMBEDDING_MODEL_NAME", "BAAI/bge-small-en-v1.5")
BATCH_SIZE = int(os.environ.get("BATCH_SIZE", "100"))
EMBEDDING_BATCH_SIZE = int(os.environ.get("EMBEDDING_BATCH_SIZE", "64"))
METADATA_COLLECTION = "mcp_indexer_metadata"
INCREMENTAL_DAYS = int(os.environ.get("INCREMENTAL_DAYS", "7"))
FORCE_REINDEX = os.environ.get("FORCE_REINDEX", "").lower() in ("1", "true", "yes")
def _validate_config():
"""Raises if required environment variables are missing. Called from main()."""
if not MAILDIR_PATH:
raise ValueError("MAILDIR_PATH environment variable is required.")
if not QDRANT_URL:
raise ValueError("QDRANT_URL environment variable is required.")
if not COLLECTION_NAME:
raise ValueError("COLLECTION_NAME environment variable is required.")
def decode_mime_words(s: str) -> str:
"""Decodes MIME encoded strings (e.g. subjects, filenames)."""
if not s:
@@ -251,6 +252,8 @@ def main():
Main ingestion function.
Reads Maildir, extracts text, generates local embeddings, and pushes to Qdrant.
"""
_validate_config()
print(f"Indexing emails from {MAILDIR_PATH} into {QDRANT_URL}...")
if not os.path.exists(MAILDIR_PATH):