Merge pull request #16 from mweiden/codex/remove-unused-methods-from-invertedindex

Remove legacy word-search helpers

Author: mweiden

src/search/inverted_index.py

@@ -24,7 +24,6 @@ def __init__(
         model: SentenceTransformer | None = None,
         model_name: str = "sentence-transformers/paraphrase-MiniLM-L3-v2",
     ):
-        self._inverted_index: dict[str, list[tuple[str, int]]] = defaultdict(list)
         self._words_per_doc: dict[str, int] = defaultdict(int)
         self._doc_id_to_url: dict[str, str] = dict()
         self._doc_id_to_title: dict[str, str] = dict()
@@ -38,12 +37,10 @@ def __init__(
 
     @property
     def total_docs(self):
-        return len(self._inverted_index)
+        return len(self._doc_id_to_url)
 
     def insert(self, doc: Node) -> None:
-        counts, total = self._word_count(doc.text)
-        for word, count in counts.items():
-            self._inverted_index[word].append((doc.id, count))
+        _, total = self._word_count(doc.text)
         self._words_per_doc[doc.id] = total
         self._doc_id_to_url[doc.id] = doc.url
         self._doc_id_to_title[doc.id] = doc.title
@@ -65,21 +62,6 @@ def _word_count(self, text: str) -> dict[str, int]:
             total += 1
         return counts, total
 
-    def _search(self, word: str) -> list[tuple[str, int]]:
-        result = self._inverted_index.get(word)
-        return [] if result is None else result
-
-    def search(self, word: str) -> list[SearchResult]:
-        return [
-            SearchResult(
-                id=kv[0],
-                url=self._doc_id_to_url[kv[0]],
-                title=self._doc_id_to_title[kv[0]],
-                score=None,
-            )
-            for kv in self._search(word)
-        ]
-
     def num_words_in_doc(self, doc_id: str) -> int:
         return self._words_per_doc[doc_id]
 

src/search/tests/test_inverted_index.py

@@ -1,10 +1,8 @@
 import numpy as np
 import pytest
-import requests
-from sentence_transformers import SentenceTransformer
 
 from web_crawler.node import Node
-from search.inverted_index import InvertedIndex, SearchResult
+from search.inverted_index import InvertedIndex
 from search.tokenizer import tokenize
 
 
@@ -62,21 +60,10 @@ def get_sentence_embedding_dimension(self) -> int:
     return inverted_index
 
 
-@pytest.fixture
-def search_result0(node0) -> SearchResult:
-    return SearchResult(node0.id, node0.url, node0.title)
-
-
-@pytest.fixture
-def search_result1(node1) -> SearchResult:
-    return SearchResult(node1.id, node1.url, node1.title)
-
-
-def test_insert(inverted_index, search_result0, search_result1):
-    assert list(tokenize("ipsum")) == ["ipsum"]
-    assert list(tokenize("iterators")) == ["iterators"]
-    assert inverted_index.search("ipsum") == [search_result0]
-    assert inverted_index.search("iterators") == [search_result0, search_result1]
+def test_top_k_keywords(inverted_index, node0, node1):
+    assert inverted_index.top_k("ipsum")[0].id == node0.id
+    ids = [r.id for r in inverted_index.top_k("iterators")[:2]]
+    assert ids == [node1.id, node0.id]
 
 
 def test_num_words_in_doc(inverted_index, node0, node1):