Implement corpus indexing and MCP search

tests/conftest.py

@@ -0,0 +1,22 @@
+from pathlib import Path
+
+import pytest
+
+from corpus_mcp.database import build_database
+
+
+@pytest.fixture
+def corpus(tmp_path: Path) -> tuple[Path, Path]:
+    source = tmp_path / "source"
+    source.mkdir()
+    (source / "guide.md").write_text(
+        "# Search Guide\n\nThe quick brown fox explains full text search.",
+        encoding="utf-8",
+    )
+    (source / "notes.txt").write_text(
+        "A second document about SQLite indexing and retrieval.",
+        encoding="utf-8",
+    )
+    database = tmp_path / "corpus.sqlite"
+    build_database(source, database, description="Test documentation")
+    return source, database

tests/test_cli.py

@@ -0,0 +1,42 @@
+import subprocess
+import sys
+from pathlib import Path
+
+from corpus_mcp.cli import main
+from corpus_mcp.database import read_corpus_info
+
+
+def test_index_command_accepts_description_file(tmp_path: Path, capsys) -> None:
+    source = tmp_path / "source"
+    source.mkdir()
+    (source / "document.txt").write_text("searchable", encoding="utf-8")
+    description = tmp_path / "description.txt"
+    description.write_text("CLI corpus", encoding="utf-8")
+    database = tmp_path / "corpus.sqlite"
+
+    result = main(
+        [
+            "index",
+            str(source),
+            "--database",
+            str(database),
+            "--description-file",
+            str(description),
+        ]
+    )
+
+    assert result == 0
+    assert "Indexed 1 documents" in capsys.readouterr().out
+    assert read_corpus_info(database)["description"] == "CLI corpus"
+
+
+def test_module_help_smoke_test() -> None:
+    result = subprocess.run(
+        [sys.executable, "-m", "corpus_mcp", "--help"],
+        check=False,
+        capture_output=True,
+        text=True,
+    )
+
+    assert result.returncode == 0
+    assert "corpus-mcp" in result.stdout

tests/test_indexing.py

@@ -0,0 +1,82 @@
+import sqlite3
+from pathlib import Path
+
+import pytest
+
+from corpus_mcp.database import (
+    CorpusError,
+    build_database,
+    iter_chunks,
+    read_corpus_info,
+)
+
+
+def test_indexes_supported_files_recursively(tmp_path: Path) -> None:
+    source = tmp_path / "documents"
+    nested = source / "nested"
+    nested.mkdir(parents=True)
+    (source / "root.TXT").write_text("Root text", encoding="utf-8-sig")
+    (nested / "page.Md").write_text("Nested markdown", encoding="utf-8")
+    (nested / "ignored.rst").write_text("Not indexed", encoding="utf-8")
+    database = tmp_path / "output" / "corpus.sqlite"
+
+    count = build_database(source, database, description="  Example corpus  ")
+
+    assert count == 2
+    info = read_corpus_info(database)
+    assert info["description"] == "Example corpus"
+    assert info["document_count"] == 2
+    with sqlite3.connect(database) as connection:
+        paths = [
+            row[0]
+            for row in connection.execute("SELECT path FROM documents ORDER BY path")
+        ]
+    assert paths == ["nested/page.Md", "root.TXT"]
+
+
+def test_empty_file_is_recorded(tmp_path: Path) -> None:
+    source = tmp_path / "documents"
+    source.mkdir()
+    (source / "empty.txt").write_text("", encoding="utf-8")
+    database = tmp_path / "corpus.sqlite"
+
+    assert build_database(source, database) == 1
+    with sqlite3.connect(database) as connection:
+        assert connection.execute("SELECT count(*) FROM documents").fetchone()[0] == 1
+        assert connection.execute("SELECT count(*) FROM chunks").fetchone()[0] == 0
+
+
+def test_rebuild_is_atomic_when_a_source_cannot_be_decoded(tmp_path: Path) -> None:
+    source = tmp_path / "documents"
+    source.mkdir()
+    (source / "valid.txt").write_text("original", encoding="utf-8")
+    database = tmp_path / "corpus.sqlite"
+    build_database(source, database, description="Original")
+    original_bytes = database.read_bytes()
+    (source / "invalid.txt").write_bytes(b"\xff\xfe\xfa")
+
+    with pytest.raises(CorpusError, match="invalid.txt"):
+        build_database(source, database)
+
+    assert database.read_bytes() == original_bytes
+    assert read_corpus_info(database)["description"] == "Original"
+
+
+def test_chunks_overlap_and_reconstruct_offsets() -> None:
+    text = "paragraph one\n\n" + ("word " * 1_200) + "the end"
+    chunks = list(iter_chunks(text))
+
+    assert len(chunks) >= 2
+    assert chunks[0][0] == 0
+    assert chunks[-1][1] == len(text)
+    for previous, current in zip(chunks, chunks[1:]):
+        assert current[0] < previous[1]
+        assert text[current[0] : current[1]] == current[2]
+
+
+def test_rejects_non_directory_source(tmp_path: Path) -> None:
+    source = tmp_path / "file.txt"
+    source.write_text("text", encoding="utf-8")
+
+    with pytest.raises(CorpusError, match="not a directory"):
+        build_database(source, tmp_path / "corpus.sqlite")

tests/test_search.py

@@ -0,0 +1,106 @@
+from pathlib import Path
+
+import pytest
+
+from corpus_mcp.database import (
+    CorpusError,
+    build_database,
+    read_document_excerpt,
+    search_database,
+)
+
+
+def test_plain_search_returns_highlighted_excerpt(corpus: tuple[Path, Path]) -> None:
+    _, database = corpus
+
+    result = search_database(database, "quick search")
+
+    assert result["syntax"] == "plain"
+    assert [match["path"] for match in result["matches"]] == ["guide.md"]
+    excerpt = result["matches"][0]["excerpts"][0]
+    assert "<match>quick</match>" in excerpt["text"]
+    assert "<match>search</match>" in excerpt["text"]
+
+
+def test_fts5_search_supports_phrases_and_rejects_invalid_syntax(
+    corpus: tuple[Path, Path],
+) -> None:
+    _, database = corpus
+
+    result = search_database(database, '"full text"', syntax="fts5")
+    assert result["matches"][0]["path"] == "guide.md"
+
+    with pytest.raises(CorpusError, match="Invalid FTS5 query"):
+        search_database(database, '"unterminated', syntax="fts5")
+
+
+def test_search_returns_separate_excerpts_from_a_large_document(tmp_path: Path) -> None:
+    source = tmp_path / "source"
+    source.mkdir()
+    content = "needle first\n\n" + ("filler " * 800) + "\n\nneedle second"
+    (source / "large.md").write_text(content, encoding="utf-8")
+    database = tmp_path / "corpus.sqlite"
+    build_database(source, database)
+
+    result = search_database(database, "needle", excerpts_per_document=2)
+
+    excerpts = result["matches"][0]["excerpts"]
+    assert len(excerpts) == 2
+    assert excerpts[0]["chunk_end_char"] <= len(content)
+    assert excerpts[1]["chunk_end_char"] <= len(content)
+
+
+def test_large_document_does_not_hide_other_matching_documents(
+    tmp_path: Path,
+) -> None:
+    source = tmp_path / "source"
+    source.mkdir()
+    (source / "large.txt").write_text(("common term " * 200_000), encoding="utf-8")
+    (source / "small.txt").write_text("common term", encoding="utf-8")
+    database = tmp_path / "corpus.sqlite"
+    build_database(source, database)
+
+    result = search_database(database, "common", limit=2)
+
+    assert {match["path"] for match in result["matches"]} == {
+        "large.txt",
+        "small.txt",
+    }
+
+
+def test_read_document_excerpt_is_bounded_and_navigable(
+    corpus: tuple[Path, Path],
+) -> None:
+    _, database = corpus
+
+    first = read_document_excerpt(database, "guide.md", max_chars=12)
+    assert first["next_offset"] is not None
+    second = read_document_excerpt(
+        database, "guide.md", offset=first["next_offset"], max_chars=12
+    )
+
+    assert len(first["text"]) == 12
+    assert first["previous_offset"] is None
+    assert second["start_char"] == 12
+    assert second["previous_offset"] == 0
+
+
+@pytest.mark.parametrize(
+    ("kwargs", "message"),
+    [
+        ({"offset": -1}, "offset must not be negative"),
+        ({"max_chars": 20_001}, "max_chars must be between"),
+    ],
+)
+def test_read_document_excerpt_validates_bounds(
+    corpus: tuple[Path, Path], kwargs: dict, message: str
+) -> None:
+    _, database = corpus
+    with pytest.raises(CorpusError, match=message):
+        read_document_excerpt(database, "guide.md", **kwargs)
+
+
+def test_search_validates_empty_query(corpus: tuple[Path, Path]) -> None:
+    _, database = corpus
+    with pytest.raises(CorpusError, match="must not be empty"):
+        search_database(database, "  ")

tests/test_server.py

@@ -0,0 +1,29 @@
+from pathlib import Path
+
+import pytest
+from mcp.shared.memory import create_connected_server_and_client_session
+
+from corpus_mcp.server import create_server
+
+
+@pytest.mark.anyio
+async def test_mcp_server_exposes_search_and_description(
+    corpus: tuple[Path, Path],
+) -> None:
+    _, database = corpus
+    server = create_server(database)
+
+    async with create_connected_server_and_client_session(
+        server, raise_exceptions=True
+    ) as session:
+        result = await session.call_tool("search_corpus", {"query": "SQLite"})
+        tools = await session.list_tools()
+
+    assert result.isError is False
+    assert result.structuredContent["matches"][0]["path"] == "notes.txt"
+    assert {tool.name for tool in tools.tools} == {
+        "search_corpus",
+        "read_document_excerpt",
+        "corpus_info",
+    }
+    assert "Test documentation" in server.instructions