Merge remote-tracking branch 'upstream/master'

[nominatim.git] / src / nominatim_api / search / db_search_builder.py

diff --git a/src/nominatim_api/search/db_search_builder.py b/src/nominatim_api/search/db_search_builder.py
index 29a853a0f387504db8ed651655256e66b54ea363..1fbb7168bb44a963f31e83bfd99f6f534bcf9be5 100644 (file)
--- a/src/nominatim_api/search/db_search_builder.py
+++ b/src/nominatim_api/search/db_search_builder.py
@@ -42,7 +42,7 @@ def build_poi_search(category: List[Tuple[str, str]],
     class _PoiData(dbf.SearchData):
         penalty = 0.0
         qualifiers = dbf.WeightedCategories(category, [0.0] * len(category))
     class _PoiData(dbf.SearchData):
         penalty = 0.0
         qualifiers = dbf.WeightedCategories(category, [0.0] * len(category))

-        countries=ccs
+        countries = ccs

 
     return dbs.PoiSearch(_PoiData())
 
 
     return dbs.PoiSearch(_PoiData())
 


@@ -55,15 +55,13 @@ class SearchBuilder:
         self.query = query
         self.details = details
 
         self.query = query
         self.details = details
 

-

     @property
     def configured_for_country(self) -> bool:
         """ Return true if the search details are configured to
             allow countries in the result.
         """
         return self.details.min_rank <= 4 and self.details.max_rank >= 4 \
     @property
     def configured_for_country(self) -> bool:
         """ Return true if the search details are configured to
             allow countries in the result.
         """
         return self.details.min_rank <= 4 and self.details.max_rank >= 4 \

-               and self.details.layer_enabled(DataLayer.ADDRESS)
-
+            and self.details.layer_enabled(DataLayer.ADDRESS)

 
     @property
     def configured_for_postcode(self) -> bool:
 
     @property
     def configured_for_postcode(self) -> bool:


@@ -71,8 +69,7 @@ class SearchBuilder:
             allow postcodes in the result.
         """
         return self.details.min_rank <= 5 and self.details.max_rank >= 11\
             allow postcodes in the result.
         """
         return self.details.min_rank <= 5 and self.details.max_rank >= 11\

-               and self.details.layer_enabled(DataLayer.ADDRESS)
-
+            and self.details.layer_enabled(DataLayer.ADDRESS)

 
     @property
     def configured_for_housenumbers(self) -> bool:
 
     @property
     def configured_for_housenumbers(self) -> bool:


@@ -80,8 +77,7 @@ class SearchBuilder:
             allow addresses in the result.
         """
         return self.details.max_rank >= 30 \
             allow addresses in the result.
         """
         return self.details.max_rank >= 30 \

-               and self.details.layer_enabled(DataLayer.ADDRESS)
-
+            and self.details.layer_enabled(DataLayer.ADDRESS)

 
     def build(self, assignment: TokenAssignment) -> Iterator[dbs.AbstractSearch]:
         """ Yield all possible abstract searches for the given token assignment.
 
     def build(self, assignment: TokenAssignment) -> Iterator[dbs.AbstractSearch]:
         """ Yield all possible abstract searches for the given token assignment.


@@ -92,7 +88,7 @@ class SearchBuilder:
 
         near_items = self.get_near_items(assignment)
         if near_items is not None and not near_items:
 
         near_items = self.get_near_items(assignment)
         if near_items is not None and not near_items:

-            return # impossible compbination of near items and category parameter
+            return  # impossible combination of near items and category parameter

 
         if assignment.name is None:
             if near_items and not sdata.postcodes:
 
         if assignment.name is None:
             if near_items and not sdata.postcodes:


@@ -123,7 +119,6 @@ class SearchBuilder:
                 search.penalty += assignment.penalty
                 yield search
 
                 search.penalty += assignment.penalty
                 yield search
 

-

     def build_poi_search(self, sdata: dbf.SearchData) -> Iterator[dbs.AbstractSearch]:
         """ Build abstract search query for a simple category search.
             This kind of search requires an additional geographic constraint.
     def build_poi_search(self, sdata: dbf.SearchData) -> Iterator[dbs.AbstractSearch]:
         """ Build abstract search query for a simple category search.
             This kind of search requires an additional geographic constraint.


@@ -132,7 +127,6 @@ class SearchBuilder:
            and ((self.details.viewbox and self.details.bounded_viewbox) or self.details.near):
             yield dbs.PoiSearch(sdata)
 
            and ((self.details.viewbox and self.details.bounded_viewbox) or self.details.near):
             yield dbs.PoiSearch(sdata)
 

-

     def build_special_search(self, sdata: dbf.SearchData,
                              address: List[TokenRange],
                              is_category: bool) -> Iterator[dbs.AbstractSearch]:
     def build_special_search(self, sdata: dbf.SearchData,
                              address: List[TokenRange],
                              is_category: bool) -> Iterator[dbs.AbstractSearch]:


@@ -157,7 +151,6 @@ class SearchBuilder:
                 penalty += 0.2
             yield dbs.PostcodeSearch(penalty, sdata)
 
                 penalty += 0.2
             yield dbs.PostcodeSearch(penalty, sdata)
 

-

     def build_housenumber_search(self, sdata: dbf.SearchData, hnrs: List[Token],
                                  address: List[TokenRange]) -> Iterator[dbs.AbstractSearch]:
         """ Build a simple address search for special entries where the
     def build_housenumber_search(self, sdata: dbf.SearchData, hnrs: List[Token],
                                  address: List[TokenRange]) -> Iterator[dbs.AbstractSearch]:
         """ Build a simple address search for special entries where the


@@ -167,7 +160,11 @@ class SearchBuilder:
         expected_count = sum(t.count for t in hnrs)
 
         partials = {t.token: t.addr_count for trange in address
         expected_count = sum(t.count for t in hnrs)
 
         partials = {t.token: t.addr_count for trange in address

-                       for t in self.query.get_partials_list(trange)}
+                    for t in self.query.get_partials_list(trange)}
+
+        if not partials:
+            # can happen when none of the partials is indexed
+            return

 
         if expected_count < 8000:
             sdata.lookups.append(dbf.FieldLookup('nameaddress_vector',
 
         if expected_count < 8000:
             sdata.lookups.append(dbf.FieldLookup('nameaddress_vector',


@@ -186,7 +183,6 @@ class SearchBuilder:
         sdata.housenumbers = dbf.WeightedStrings([], [])
         yield dbs.PlaceSearch(0.05, sdata, expected_count)
 
         sdata.housenumbers = dbf.WeightedStrings([], [])
         yield dbs.PlaceSearch(0.05, sdata, expected_count)
 

-

     def build_name_search(self, sdata: dbf.SearchData,
                           name: TokenRange, address: List[TokenRange],
                           is_category: bool) -> Iterator[dbs.AbstractSearch]:
     def build_name_search(self, sdata: dbf.SearchData,
                           name: TokenRange, address: List[TokenRange],
                           is_category: bool) -> Iterator[dbs.AbstractSearch]:


@@ -201,50 +197,43 @@ class SearchBuilder:
                 sdata.lookups = lookup
                 yield dbs.PlaceSearch(penalty + name_penalty, sdata, count)
 
                 sdata.lookups = lookup
                 yield dbs.PlaceSearch(penalty + name_penalty, sdata, count)
 

-
-    def yield_lookups(self, name: TokenRange, address: List[TokenRange])\
-                          -> Iterator[Tuple[float, int, List[dbf.FieldLookup]]]:
+    def yield_lookups(self, name: TokenRange, address: List[TokenRange]
+                      ) -> Iterator[Tuple[float, int, List[dbf.FieldLookup]]]:

         """ Yield all variants how the given name and address should best
             be searched for. This takes into account how frequent the terms
             are and tries to find a lookup that optimizes index use.
         """
         """ Yield all variants how the given name and address should best
             be searched for. This takes into account how frequent the terms
             are and tries to find a lookup that optimizes index use.
         """

-        penalty = 0.0 # extra penalty
+        penalty = 0.0  # extra penalty

         name_partials = {t.token: t for t in self.query.get_partials_list(name)}
 
         addr_partials = [t for r in address for t in self.query.get_partials_list(r)]
         addr_tokens = list({t.token for t in addr_partials})
 
         name_partials = {t.token: t for t in self.query.get_partials_list(name)}
 
         addr_partials = [t for r in address for t in self.query.get_partials_list(r)]
         addr_tokens = list({t.token for t in addr_partials})
 

-        partials_indexed = all(t.is_indexed for t in name_partials.values()) \
-                           and all(t.is_indexed for t in addr_partials)

         exp_count = min(t.count for t in name_partials.values()) / (2**(len(name_partials) - 1))
 
         exp_count = min(t.count for t in name_partials.values()) / (2**(len(name_partials) - 1))
 

-        if (len(name_partials) > 3 or exp_count < 8000) and partials_indexed:
+        if (len(name_partials) > 3 or exp_count < 8000):

             yield penalty, exp_count, dbf.lookup_by_names(list(name_partials.keys()), addr_tokens)
             return
 
             yield penalty, exp_count, dbf.lookup_by_names(list(name_partials.keys()), addr_tokens)
             return
 

-        addr_count = min(t.addr_count for t in addr_partials) if addr_partials else 30000
+        addr_count = min(t.addr_count for t in addr_partials) if addr_partials else 50000

         # Partial term to frequent. Try looking up by rare full names first.
         name_fulls = self.query.get_tokens(name, TokenType.WORD)
         if name_fulls:
             fulls_count = sum(t.count for t in name_fulls)
         # Partial term to frequent. Try looking up by rare full names first.
         name_fulls = self.query.get_tokens(name, TokenType.WORD)
         if name_fulls:
             fulls_count = sum(t.count for t in name_fulls)

-            if partials_indexed:
-                penalty += 1.2 * sum(t.penalty for t in addr_partials if not t.is_indexed)

 
 

-            if fulls_count < 50000 or addr_count < 30000:
-                yield penalty,fulls_count / (2**len(addr_tokens)), \
+            if fulls_count < 80000 or addr_count < 50000:
+                yield penalty, fulls_count / (2**len(addr_tokens)), \

                     self.get_full_name_ranking(name_fulls, addr_partials,
                                                fulls_count > 30000 / max(1, len(addr_tokens)))
 
         # To catch remaining results, lookup by name and address
         # We only do this if there is a reasonable number of results expected.
         exp_count = exp_count / (2**len(addr_tokens)) if addr_tokens else exp_count
                     self.get_full_name_ranking(name_fulls, addr_partials,
                                                fulls_count > 30000 / max(1, len(addr_tokens)))
 
         # To catch remaining results, lookup by name and address
         # We only do this if there is a reasonable number of results expected.
         exp_count = exp_count / (2**len(addr_tokens)) if addr_tokens else exp_count

-        if exp_count < 10000 and addr_count < 20000\
-           and all(t.is_indexed for t in name_partials.values()):
+        if exp_count < 10000 and addr_count < 20000:

             penalty += 0.35 * max(1 if name_fulls else 0.1,
                                   5 - len(name_partials) - len(addr_tokens))
             penalty += 0.35 * max(1 if name_fulls else 0.1,
                                   5 - len(name_partials) - len(addr_tokens))

-            yield penalty, exp_count,\
-                  self.get_name_address_ranking(list(name_partials.keys()), addr_partials)
-
+            yield penalty, exp_count, \
+                self.get_name_address_ranking(list(name_partials.keys()), addr_partials)

 
     def get_name_address_ranking(self, name_tokens: List[int],
                                  addr_partials: List[Token]) -> List[dbf.FieldLookup]:
 
     def get_name_address_ranking(self, name_tokens: List[int],
                                  addr_partials: List[Token]) -> List[dbf.FieldLookup]:


@@ -255,11 +244,10 @@ class SearchBuilder:
         addr_restrict_tokens = []
         addr_lookup_tokens = []
         for t in addr_partials:
         addr_restrict_tokens = []
         addr_lookup_tokens = []
         for t in addr_partials:

-            if t.is_indexed:
-                if t.addr_count > 20000:
-                    addr_restrict_tokens.append(t.token)
-                else:
-                    addr_lookup_tokens.append(t.token)
+            if t.addr_count > 20000:
+                addr_restrict_tokens.append(t.token)
+            else:
+                addr_lookup_tokens.append(t.token)

 
         if addr_restrict_tokens:
             lookup.append(dbf.FieldLookup('nameaddress_vector',
 
         if addr_restrict_tokens:
             lookup.append(dbf.FieldLookup('nameaddress_vector',


@@ -270,33 +258,25 @@ class SearchBuilder:
 
         return lookup
 
 
         return lookup
 

-

     def get_full_name_ranking(self, name_fulls: List[Token], addr_partials: List[Token],
                               use_lookup: bool) -> List[dbf.FieldLookup]:
         """ Create a ranking expression with full name terms and
             additional address lookup. When 'use_lookup' is true, then
     def get_full_name_ranking(self, name_fulls: List[Token], addr_partials: List[Token],
                               use_lookup: bool) -> List[dbf.FieldLookup]:
         """ Create a ranking expression with full name terms and
             additional address lookup. When 'use_lookup' is true, then

-            address lookups will use the index, when the occurences are not
+            address lookups will use the index, when the occurrences are not

             too many.
         """
         # At this point drop unindexed partials from the address.
         # This might yield wrong results, nothing we can do about that.
         if use_lookup:
             addr_restrict_tokens = []
             too many.
         """
         # At this point drop unindexed partials from the address.
         # This might yield wrong results, nothing we can do about that.
         if use_lookup:
             addr_restrict_tokens = []

-            addr_lookup_tokens = []
-            for t in addr_partials:
-                if t.is_indexed:
-                    if t.addr_count > 20000:
-                        addr_restrict_tokens.append(t.token)
-                    else:
-                        addr_lookup_tokens.append(t.token)
+            addr_lookup_tokens = [t.token for t in addr_partials]

         else:
         else:

-            addr_restrict_tokens = [t.token for t in addr_partials if t.is_indexed]
+            addr_restrict_tokens = [t.token for t in addr_partials]

             addr_lookup_tokens = []
 
         return dbf.lookup_by_any_name([t.token for t in name_fulls],
                                       addr_restrict_tokens, addr_lookup_tokens)
 
             addr_lookup_tokens = []
 
         return dbf.lookup_by_any_name([t.token for t in name_fulls],
                                       addr_restrict_tokens, addr_lookup_tokens)
 

-

     def get_name_ranking(self, trange: TokenRange,
                          db_field: str = 'name_vector') -> dbf.FieldRanking:
         """ Create a ranking expression for a name term in the given range.
     def get_name_ranking(self, trange: TokenRange,
                          db_field: str = 'name_vector') -> dbf.FieldRanking:
         """ Create a ranking expression for a name term in the given range.


@@ -309,7 +289,6 @@ class SearchBuilder:
         default = sum(t.penalty for t in name_partials) + 0.2
         return dbf.FieldRanking(db_field, default, ranks)
 
         default = sum(t.penalty for t in name_partials) + 0.2
         return dbf.FieldRanking(db_field, default, ranks)
 

-

     def get_addr_ranking(self, trange: TokenRange) -> dbf.FieldRanking:
         """ Create a list of ranking expressions for an address term
             for the given ranges.
     def get_addr_ranking(self, trange: TokenRange) -> dbf.FieldRanking:
         """ Create a list of ranking expressions for an address term
             for the given ranges.


@@ -318,7 +297,7 @@ class SearchBuilder:
         heapq.heappush(todo, (0, trange.start, dbf.RankedTokens(0.0, [])))
         ranks: List[dbf.RankedTokens] = []
 
         heapq.heappush(todo, (0, trange.start, dbf.RankedTokens(0.0, [])))
         ranks: List[dbf.RankedTokens] = []
 

-        while todo: # pylint: disable=too-many-nested-blocks
+        while todo:

             neglen, pos, rank = heapq.heappop(todo)
             for tlist in self.query.nodes[pos].starting:
                 if tlist.ttype in (TokenType.PARTIAL, TokenType.WORD):
             neglen, pos, rank = heapq.heappop(todo)
             for tlist in self.query.nodes[pos].starting:
                 if tlist.ttype in (TokenType.PARTIAL, TokenType.WORD):


@@ -357,7 +336,6 @@ class SearchBuilder:
 
         return dbf.FieldRanking('nameaddress_vector', default, ranks)
 
 
         return dbf.FieldRanking('nameaddress_vector', default, ranks)
 

-

     def get_search_data(self, assignment: TokenAssignment) -> Optional[dbf.SearchData]:
         """ Collect the tokens for the non-name search fields in the
             assignment.
     def get_search_data(self, assignment: TokenAssignment) -> Optional[dbf.SearchData]:
         """ Collect the tokens for the non-name search fields in the
             assignment.


@@ -404,7 +382,6 @@ class SearchBuilder:
 
         return sdata
 
 
         return sdata
 

-

     def get_country_tokens(self, trange: TokenRange) -> List[Token]:
         """ Return the list of country tokens for the given range,
             optionally filtered by the country list from the details
     def get_country_tokens(self, trange: TokenRange) -> List[Token]:
         """ Return the list of country tokens for the given range,
             optionally filtered by the country list from the details


@@ -416,7 +393,6 @@ class SearchBuilder:
 
         return tokens
 
 
         return tokens
 

-

     def get_qualifier_tokens(self, trange: TokenRange) -> List[Token]:
         """ Return the list of qualifier tokens for the given range,
             optionally filtered by the qualifier list from the details
     def get_qualifier_tokens(self, trange: TokenRange) -> List[Token]:
         """ Return the list of qualifier tokens for the given range,
             optionally filtered by the qualifier list from the details


@@ -428,7 +404,6 @@ class SearchBuilder:
 
         return tokens
 
 
         return tokens
 

-

     def get_near_items(self, assignment: TokenAssignment) -> Optional[dbf.WeightedCategories]:
         """ Collect tokens for near items search or use the categories
             requested per parameter.
     def get_near_items(self, assignment: TokenAssignment) -> Optional[dbf.WeightedCategories]:
         """ Collect tokens for near items search or use the categories
             requested per parameter.