Merge remote-tracking branch 'upstream/master'

[nominatim.git] / src / nominatim_api / search / db_search_builder.py

diff --git a/src/nominatim_api/search/db_search_builder.py b/src/nominatim_api/search/db_search_builder.py
index 7e76de1408234453d2ad4137411b0feff57fb879..4987f156e9b9a6e134e56a48dafade15729165ab 100644 (file)
--- a/src/nominatim_api/search/db_search_builder.py
+++ b/src/nominatim_api/search/db_search_builder.py
@@ -97,7 +97,7 @@ class SearchBuilder:
                 builder = self.build_poi_search(sdata)
             elif assignment.housenumber:
                 hnr_tokens = self.query.get_tokens(assignment.housenumber,
                 builder = self.build_poi_search(sdata)
             elif assignment.housenumber:
                 hnr_tokens = self.query.get_tokens(assignment.housenumber,

-                                                   qmod.TokenType.HOUSENUMBER)
+                                                   qmod.TOKEN_HOUSENUMBER)

                 builder = self.build_housenumber_search(sdata, hnr_tokens, assignment.address)
             else:
                 builder = self.build_special_search(sdata, assignment.address,
                 builder = self.build_housenumber_search(sdata, hnr_tokens, assignment.address)
             else:
                 builder = self.build_special_search(sdata, assignment.address,


@@ -173,7 +173,7 @@ class SearchBuilder:
                                                  list(partials), lookups.LookupAll))
         else:
             addr_fulls = [t.token for t
                                                  list(partials), lookups.LookupAll))
         else:
             addr_fulls = [t.token for t

-                          in self.query.get_tokens(address[0], qmod.TokenType.WORD)]
+                          in self.query.get_tokens(address[0], qmod.TOKEN_WORD)]

             if len(addr_fulls) > 5:
                 return
             sdata.lookups.append(
             if len(addr_fulls) > 5:
                 return
             sdata.lookups.append(


@@ -214,13 +214,13 @@ class SearchBuilder:
             yield penalty, exp_count, dbf.lookup_by_names(list(name_partials.keys()), addr_tokens)
             return
 
             yield penalty, exp_count, dbf.lookup_by_names(list(name_partials.keys()), addr_tokens)
             return
 

-        addr_count = min(t.addr_count for t in addr_partials) if addr_partials else 30000
+        addr_count = min(t.addr_count for t in addr_partials) if addr_partials else 50000

         # Partial term to frequent. Try looking up by rare full names first.
         # Partial term to frequent. Try looking up by rare full names first.

-        name_fulls = self.query.get_tokens(name, qmod.TokenType.WORD)
+        name_fulls = self.query.get_tokens(name, qmod.TOKEN_WORD)

         if name_fulls:
             fulls_count = sum(t.count for t in name_fulls)
 
         if name_fulls:
             fulls_count = sum(t.count for t in name_fulls)
 

-            if fulls_count < 50000 or addr_count < 30000:
+            if fulls_count < 80000 or addr_count < 50000:

                 yield penalty, fulls_count / (2**len(addr_tokens)), \
                     self.get_full_name_ranking(name_fulls, addr_partials,
                                                fulls_count > 30000 / max(1, len(addr_tokens)))
                 yield penalty, fulls_count / (2**len(addr_tokens)), \
                     self.get_full_name_ranking(name_fulls, addr_partials,
                                                fulls_count > 30000 / max(1, len(addr_tokens)))


@@ -268,12 +268,7 @@ class SearchBuilder:
         # This might yield wrong results, nothing we can do about that.
         if use_lookup:
             addr_restrict_tokens = []
         # This might yield wrong results, nothing we can do about that.
         if use_lookup:
             addr_restrict_tokens = []

-            addr_lookup_tokens = []
-            for t in addr_partials:
-                if t.addr_count > 20000:
-                    addr_restrict_tokens.append(t.token)
-                else:
-                    addr_lookup_tokens.append(t.token)
+            addr_lookup_tokens = [t.token for t in addr_partials]

         else:
             addr_restrict_tokens = [t.token for t in addr_partials]
             addr_lookup_tokens = []
         else:
             addr_restrict_tokens = [t.token for t in addr_partials]
             addr_lookup_tokens = []


@@ -285,7 +280,7 @@ class SearchBuilder:
                          db_field: str = 'name_vector') -> dbf.FieldRanking:
         """ Create a ranking expression for a name term in the given range.
         """
                          db_field: str = 'name_vector') -> dbf.FieldRanking:
         """ Create a ranking expression for a name term in the given range.
         """

-        name_fulls = self.query.get_tokens(trange, qmod.TokenType.WORD)
+        name_fulls = self.query.get_tokens(trange, qmod.TOKEN_WORD)

         ranks = [dbf.RankedTokens(t.penalty, [t.token]) for t in name_fulls]
         ranks.sort(key=lambda r: r.penalty)
         # Fallback, sum of penalty for partials
         ranks = [dbf.RankedTokens(t.penalty, [t.token]) for t in name_fulls]
         ranks.sort(key=lambda r: r.penalty)
         # Fallback, sum of penalty for partials


@@ -304,10 +299,10 @@ class SearchBuilder:
         while todo:
             neglen, pos, rank = heapq.heappop(todo)
             for tlist in self.query.nodes[pos].starting:
         while todo:
             neglen, pos, rank = heapq.heappop(todo)
             for tlist in self.query.nodes[pos].starting:

-                if tlist.ttype in (qmod.TokenType.PARTIAL, qmod.TokenType.WORD):
+                if tlist.ttype in (qmod.TOKEN_PARTIAL, qmod.TOKEN_WORD):

                     if tlist.end < trange.end:
                         chgpenalty = PENALTY_WORDCHANGE[self.query.nodes[tlist.end].btype]
                     if tlist.end < trange.end:
                         chgpenalty = PENALTY_WORDCHANGE[self.query.nodes[tlist.end].btype]

-                        if tlist.ttype == qmod.TokenType.PARTIAL:
+                        if tlist.ttype == qmod.TOKEN_PARTIAL:

                             penalty = rank.penalty + chgpenalty \
                                       + max(t.penalty for t in tlist.tokens)
                             heapq.heappush(todo, (neglen - 1, tlist.end,
                             penalty = rank.penalty + chgpenalty \
                                       + max(t.penalty for t in tlist.tokens)
                             heapq.heappush(todo, (neglen - 1, tlist.end,


@@ -317,7 +312,7 @@ class SearchBuilder:
                                 heapq.heappush(todo, (neglen - 1, tlist.end,
                                                       rank.with_token(t, chgpenalty)))
                     elif tlist.end == trange.end:
                                 heapq.heappush(todo, (neglen - 1, tlist.end,
                                                       rank.with_token(t, chgpenalty)))
                     elif tlist.end == trange.end:

-                        if tlist.ttype == qmod.TokenType.PARTIAL:
+                        if tlist.ttype == qmod.TOKEN_PARTIAL:

                             ranks.append(dbf.RankedTokens(rank.penalty
                                                           + max(t.penalty for t in tlist.tokens),
                                                           rank.tokens))
                             ranks.append(dbf.RankedTokens(rank.penalty
                                                           + max(t.penalty for t in tlist.tokens),
                                                           rank.tokens))


@@ -357,11 +352,11 @@ class SearchBuilder:
         if assignment.housenumber:
             sdata.set_strings('housenumbers',
                               self.query.get_tokens(assignment.housenumber,
         if assignment.housenumber:
             sdata.set_strings('housenumbers',
                               self.query.get_tokens(assignment.housenumber,

-                                                    qmod.TokenType.HOUSENUMBER))
+                                                    qmod.TOKEN_HOUSENUMBER))

         if assignment.postcode:
             sdata.set_strings('postcodes',
                               self.query.get_tokens(assignment.postcode,
         if assignment.postcode:
             sdata.set_strings('postcodes',
                               self.query.get_tokens(assignment.postcode,

-                                                    qmod.TokenType.POSTCODE))
+                                                    qmod.TOKEN_POSTCODE))

         if assignment.qualifier:
             tokens = self.get_qualifier_tokens(assignment.qualifier)
             if not tokens:
         if assignment.qualifier:
             tokens = self.get_qualifier_tokens(assignment.qualifier)
             if not tokens:


@@ -391,7 +386,7 @@ class SearchBuilder:
             optionally filtered by the country list from the details
             parameters.
         """
             optionally filtered by the country list from the details
             parameters.
         """

-        tokens = self.query.get_tokens(trange, qmod.TokenType.COUNTRY)
+        tokens = self.query.get_tokens(trange, qmod.TOKEN_COUNTRY)

         if self.details.countries:
             tokens = [t for t in tokens if t.lookup_word in self.details.countries]
 
         if self.details.countries:
             tokens = [t for t in tokens if t.lookup_word in self.details.countries]
 


@@ -402,7 +397,7 @@ class SearchBuilder:
             optionally filtered by the qualifier list from the details
             parameters.
         """
             optionally filtered by the qualifier list from the details
             parameters.
         """

-        tokens = self.query.get_tokens(trange, qmod.TokenType.QUALIFIER)
+        tokens = self.query.get_tokens(trange, qmod.TOKEN_QUALIFIER)

         if self.details.categories:
             tokens = [t for t in tokens if t.get_category() in self.details.categories]
 
         if self.details.categories:
             tokens = [t for t in tokens if t.get_category() in self.details.categories]
 


@@ -415,7 +410,7 @@ class SearchBuilder:
         """
         if assignment.near_item:
             tokens: Dict[Tuple[str, str], float] = {}
         """
         if assignment.near_item:
             tokens: Dict[Tuple[str, str], float] = {}

-            for t in self.query.get_tokens(assignment.near_item, qmod.TokenType.NEAR_ITEM):
+            for t in self.query.get_tokens(assignment.near_item, qmod.TOKEN_NEAR_ITEM):

                 cat = t.get_category()
                 # The category of a near search will be that of near_item.
                 # Thus, if search is restricted to a category parameter,
                 cat = t.get_category()
                 # The category of a near search will be that of near_item.
                 # Thus, if search is restricted to a category parameter,