lift restrictions on search with frequent terms slightly

[nominatim.git] / src / nominatim_api / search / db_search_builder.py

diff --git a/src/nominatim_api/search/db_search_builder.py b/src/nominatim_api/search/db_search_builder.py
index 29a853a0f387504db8ed651655256e66b54ea363..0269cf1f57edd7699853d6eaf2ba9e5388dda425 100644 (file)
--- a/src/nominatim_api/search/db_search_builder.py
+++ b/src/nominatim_api/search/db_search_builder.py
@@ -167,7 +167,12 @@ class SearchBuilder:
         expected_count = sum(t.count for t in hnrs)
 
         partials = {t.token: t.addr_count for trange in address
         expected_count = sum(t.count for t in hnrs)
 
         partials = {t.token: t.addr_count for trange in address

-                       for t in self.query.get_partials_list(trange)}
+                       for t in self.query.get_partials_list(trange)
+                       if t.is_indexed}
+
+        if not partials:
+            # can happen when none of the partials is indexed
+            return

 
         if expected_count < 8000:
             sdata.lookups.append(dbf.FieldLookup('nameaddress_vector',
 
         if expected_count < 8000:
             sdata.lookups.append(dbf.FieldLookup('nameaddress_vector',


@@ -222,7 +227,7 @@ class SearchBuilder:
             yield penalty, exp_count, dbf.lookup_by_names(list(name_partials.keys()), addr_tokens)
             return
 
             yield penalty, exp_count, dbf.lookup_by_names(list(name_partials.keys()), addr_tokens)
             return
 

-        addr_count = min(t.addr_count for t in addr_partials) if addr_partials else 30000
+        addr_count = min(t.addr_count for t in addr_partials) if addr_partials else 50000

         # Partial term to frequent. Try looking up by rare full names first.
         name_fulls = self.query.get_tokens(name, TokenType.WORD)
         if name_fulls:
         # Partial term to frequent. Try looking up by rare full names first.
         name_fulls = self.query.get_tokens(name, TokenType.WORD)
         if name_fulls:


@@ -230,7 +235,7 @@ class SearchBuilder:
             if partials_indexed:
                 penalty += 1.2 * sum(t.penalty for t in addr_partials if not t.is_indexed)
 
             if partials_indexed:
                 penalty += 1.2 * sum(t.penalty for t in addr_partials if not t.is_indexed)
 

-            if fulls_count < 50000 or addr_count < 30000:
+            if fulls_count < 80000 or addr_count < 50000:

                 yield penalty,fulls_count / (2**len(addr_tokens)), \
                     self.get_full_name_ranking(name_fulls, addr_partials,
                                                fulls_count > 30000 / max(1, len(addr_tokens)))
                 yield penalty,fulls_count / (2**len(addr_tokens)), \
                     self.get_full_name_ranking(name_fulls, addr_partials,
                                                fulls_count > 30000 / max(1, len(addr_tokens)))


@@ -275,20 +280,14 @@ class SearchBuilder:
                               use_lookup: bool) -> List[dbf.FieldLookup]:
         """ Create a ranking expression with full name terms and
             additional address lookup. When 'use_lookup' is true, then
                               use_lookup: bool) -> List[dbf.FieldLookup]:
         """ Create a ranking expression with full name terms and
             additional address lookup. When 'use_lookup' is true, then

-            address lookups will use the index, when the occurences are not
+            address lookups will use the index, when the occurrences are not

             too many.
         """
         # At this point drop unindexed partials from the address.
         # This might yield wrong results, nothing we can do about that.
         if use_lookup:
             addr_restrict_tokens = []
             too many.
         """
         # At this point drop unindexed partials from the address.
         # This might yield wrong results, nothing we can do about that.
         if use_lookup:
             addr_restrict_tokens = []

-            addr_lookup_tokens = []
-            for t in addr_partials:
-                if t.is_indexed:
-                    if t.addr_count > 20000:
-                        addr_restrict_tokens.append(t.token)
-                    else:
-                        addr_lookup_tokens.append(t.token)
+            addr_lookup_tokens = [t.token for t in addr_partials if t.is_indexed]

         else:
             addr_restrict_tokens = [t.token for t in addr_partials if t.is_indexed]
             addr_lookup_tokens = []
         else:
             addr_restrict_tokens = [t.token for t in addr_partials if t.is_indexed]
             addr_lookup_tokens = []