test: add indexer unit tests + integration marker; make indexer importable without env
- pytest.ini: 'integration' marker, deselected by default; the integration suite requires a live Qdrant and will only run in the nightly CI job. - src/indexer.py: move env validation from import time into _validate_config() called by main() — the module can now be imported without MAILDIR_PATH / QDRANT_URL / COLLECTION_NAME set, enabling unit tests. - tests/test_indexer.py: 27 new unit tests, no network and no model: decode_mime_words (encoded-words utf-8/iso-8859-1, mixed), extract_text_from_html (tags, scripts), normalize_email_address, parse_email_message with real EmailMessage objects (plain bodies, attachments detected and excluded from body, HTML alternative, unknown charset fallback via raw bytes message, encoded attachment filename), get_recent_keys against a real tmp_path Maildir (fresh mtime selected, old mtime excluded, colon-suffix keys). mailbox.Maildir(create=True) does not create cur/new/tmp on Python 3.11 — the test helper pre-creates them.
This commit is contained in:
1 parent
6b17ca4ad5
commit
e32806aab9
3 files changed
+248
-11
No files matched your search
+14
-11
@@ -22,26 +22,27 @@ from bs4 import BeautifulSoup
|
||||
# Load .env config
|
||||
load_dotenv()
|
||||
|
||||
# Configuration
|
||||
# Configuration (validated lazily in main() so the module can be imported
|
||||
# and unit-tested without environment variables)
|
||||
MAILDIR_PATH = os.environ.get("MAILDIR_PATH", "")
|
||||
MAILDIR_FOLDERS = os.environ.get("MAILDIR_FOLDERS", "")
|
||||
QDRANT_URL = os.environ.get("QDRANT_URL", "")
|
||||
COLLECTION_NAME = os.environ.get("COLLECTION_NAME", "")
|
||||
|
||||
if not MAILDIR_PATH:
|
||||
raise ValueError("MAILDIR_PATH environment variable is required.")
|
||||
if not QDRANT_URL:
|
||||
raise ValueError("QDRANT_URL environment variable is required.")
|
||||
if not COLLECTION_NAME:
|
||||
raise ValueError("COLLECTION_NAME environment variable is required.")
|
||||
|
||||
EMBEDDING_MODEL_NAME = os.environ.get("EMBEDDING_MODEL_NAME", "BAAI/bge-small-en-v1.5")
|
||||
BATCH_SIZE = int(os.environ.get("BATCH_SIZE", "100"))
|
||||
EMBEDDING_BATCH_SIZE = int(os.environ.get("EMBEDDING_BATCH_SIZE", "64"))
|
||||
METADATA_COLLECTION = "mcp_indexer_metadata"
|
||||
INCREMENTAL_DAYS = int(os.environ.get("INCREMENTAL_DAYS", "7"))
|
||||
FORCE_REINDEX = os.environ.get("FORCE_REINDEX", "").lower() in ("1", "true", "yes")
|
||||
|
||||
|
||||
def _validate_config():
|
||||
"""Raises if required environment variables are missing. Called from main()."""
|
||||
if not MAILDIR_PATH:
|
||||
raise ValueError("MAILDIR_PATH environment variable is required.")
|
||||
if not QDRANT_URL:
|
||||
raise ValueError("QDRANT_URL environment variable is required.")
|
||||
if not COLLECTION_NAME:
|
||||
raise ValueError("COLLECTION_NAME environment variable is required.")
|
||||
|
||||
def decode_mime_words(s: str) -> str:
|
||||
"""Decodes MIME encoded strings (e.g. subjects, filenames)."""
|
||||
if not s:
|
||||
@@ -251,6 +252,8 @@ def main():
|
||||
Main ingestion function.
|
||||
Reads Maildir, extracts text, generates local embeddings, and pushes to Qdrant.
|
||||
"""
|
||||
_validate_config()
|
||||
|
||||
print(f"Indexing emails from {MAILDIR_PATH} into {QDRANT_URL}...")
|
||||
|
||||
if not os.path.exists(MAILDIR_PATH):
|
||||
|
||||
Reference in new issue
Block a user