optimize plain search

README.md

@@ -71,7 +71,9 @@ returns ranked, highlighted excerpts grouped by document. Plain searches match
 all whitespace-separated terms without interpreting operators. Set `syntax` to
 `fts5` for phrases, `OR`, `NOT`, and prefix expressions. Each result includes
 the containing chunk's character offsets, which can be passed to
-`read_document_excerpt` to retrieve more surrounding context.
+`read_document_excerpt` to retrieve more surrounding context. Search ranks a
+bounded pool of chunk matches so broad queries remain responsive; as a result,
+a document with many highly ranked passages may occupy multiple candidates.
 
 `read_document_excerpt(path, offset=0, max_chars=4000)` reads a bounded range
 and returns previous and next offsets. Responses are capped at 20,000

src/corpus_mcp/database.py

@@ -15,6 +15,7 @@ CHUNK_OVERLAP = 400
 MAX_SEARCH_DOCUMENTS = 20
 MAX_EXCERPTS_PER_DOCUMENT = 5
 MAX_EXCERPT_CHARS = 20_000
+SEARCH_CANDIDATE_MULTIPLIER = 10
 
 
 class CorpusError(Exception):
@@ -352,58 +353,33 @@ def search_database(
     connection = _connect_read_only(database_path)
     try:
         try:
-            documents = connection.execute(
+            rows = connection.execute(
                 """
-                WITH chunk_matches AS (
+                SELECT d.path,
-                    SELECT d.id AS document_id,
+                       c.ordinal,
-                           d.path,
+                       c.start_char,
-                           chunks_fts.rank AS score
+                       c.end_char,
-                      FROM chunks_fts
+                       snippet(
-                      JOIN chunks AS c ON c.id = chunks_fts.rowid
+                           chunks_fts,
-                      JOIN documents AS d ON d.id = c.document_id
+                           0,
-                     WHERE chunks_fts MATCH ?
+                           '<match>',
-                )
+                           '</match>',
-                SELECT document_id, path, min(score) AS document_score
+                           ' ... ',
-                  FROM chunk_matches
+                           48
-                 GROUP BY document_id, path
+                       ) AS excerpt,
-                 ORDER BY document_score ASC, path ASC
+                       chunks_fts.rank AS score
+                  FROM chunks_fts
+                  JOIN chunks AS c ON c.id = chunks_fts.rowid
+                  JOIN documents AS d ON d.id = c.document_id
+                 WHERE chunks_fts MATCH ?
+                 ORDER BY score ASC, d.path ASC, c.ordinal ASC
                  LIMIT ?
                 """,
-                (match_query, limit),
+                (
+                    match_query,
+                    limit * excerpts_per_document * SEARCH_CANDIDATE_MULTIPLIER,
+                ),
             ).fetchall()
-
-            rows: list[sqlite3.Row] = []
-            for document in documents:
-                rows.extend(
-                    connection.execute(
-                        """
-                        SELECT d.path,
-                               c.ordinal,
-                               c.start_char,
-                               c.end_char,
-                               snippet(
-                                   chunks_fts,
-                                   0,
-                                   '<match>',
-                                   '</match>',
-                                   ' ... ',
-                                   48
-                               ) AS excerpt,
-                               chunks_fts.rank AS score
-                          FROM chunks_fts
-                          JOIN chunks AS c ON c.id = chunks_fts.rowid
-                          JOIN documents AS d ON d.id = c.document_id
-                         WHERE chunks_fts MATCH ? AND d.id = ?
-                         ORDER BY score ASC, c.ordinal ASC
-                         LIMIT ?
-                        """,
-                        (
-                            match_query,
-                            document["document_id"],
-                            excerpts_per_document * 3,
-                        ),
-                    ).fetchall()
-                )
         except sqlite3.OperationalError as exc:
             if syntax == "fts5":
                 raise CorpusError(f"Invalid FTS5 query: {exc}") from exc

tests/test_search.py

@@ -1,9 +1,12 @@
 from pathlib import Path
+from typing import Any
 
 import pytest
 
+import corpus_mcp.database as database_module
 from corpus_mcp.database import (
     CorpusError,
+    SEARCH_CANDIDATE_MULTIPLIER,
     build_database,
     read_document_excerpt,
     search_database,
@@ -50,8 +53,8 @@ def test_search_returns_separate_excerpts_from_a_large_document(tmp_path: Path)
     assert excerpts[1]["chunk_end_char"] <= len(content)
 
 
-def test_large_document_does_not_hide_other_matching_documents(
+def test_common_term_search_uses_a_bounded_candidate_pool(
-    tmp_path: Path,
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
 ) -> None:
     source = tmp_path / "source"
     source.mkdir()
@@ -59,13 +62,28 @@ def test_large_document_does_not_hide_other_matching_documents(
     (source / "small.txt").write_text("common term", encoding="utf-8")
     database = tmp_path / "corpus.sqlite"
     build_database(source, database)
+    connection = database_module._connect_read_only(database)
+    executed: list[tuple[str, tuple[Any, ...]]] = []
 
-    result = search_database(database, "common", limit=2)
+    class RecordingConnection:
+        def execute(self, statement: str, parameters: tuple[Any, ...] = ()) -> Any:
+            executed.append((statement, parameters))
+            return connection.execute(statement, parameters)
 
-    assert {match["path"] for match in result["matches"]} == {
+        def close(self) -> None:
-        "large.txt",
+            connection.close()
-        "small.txt",
+
-    }
+    monkeypatch.setattr(
+        database_module,
+        "_connect_read_only",
+        lambda _database: RecordingConnection(),
+    )
+
+    search_database(database, "common", limit=2, excerpts_per_document=3)
+
+    assert len(executed) == 1
+    assert "GROUP BY" not in executed[0][0]
+    assert executed[0][1][-1] == 2 * 3 * SEARCH_CANDIDATE_MULTIPLIER
 
 
 def test_read_document_excerpt_is_bounded_and_navigable(