Merge pull request #3383 from lonvia/window-searches

[nominatim.git] / nominatim / api / search / icu_tokenizer.py

diff --git a/nominatim/api/search/icu_tokenizer.py b/nominatim/api/search/icu_tokenizer.py
index eabd329d57e08cac6d8b6cbc8c0a8c3c0fcf8fdd..eb90c122eb43277a18b2d71bf48eb9ab99a375ac 100644 (file)
--- a/nominatim/api/search/icu_tokenizer.py
+++ b/nominatim/api/search/icu_tokenizer.py
@@ -8,7 +8,6 @@
 Implementation of query analysis for the ICU tokenizer.
 """
 from typing import Tuple, Dict, List, Optional, NamedTuple, Iterator, Any, cast
 Implementation of query analysis for the ICU tokenizer.
 """
 from typing import Tuple, Dict, List, Optional, NamedTuple, Iterator, Any, cast

-from copy import copy

 from collections import defaultdict
 import dataclasses
 import difflib
 from collections import defaultdict
 import dataclasses
 import difflib


@@ -98,6 +97,7 @@ class ICUToken(qmod.Token):
         """ Create a ICUToken from the row of the word table.
         """
         count = 1 if row.info is None else row.info.get('count', 1)
         """ Create a ICUToken from the row of the word table.
         """
         count = 1 if row.info is None else row.info.get('count', 1)

+        addr_count = 1 if row.info is None else row.info.get('addr_count', 1)

 
         penalty = 0.0
         if row.type == 'w':
 
         penalty = 0.0
         if row.type == 'w':


@@ -122,9 +122,10 @@ class ICUToken(qmod.Token):
         else:
             lookup_word = row.word_token
 
         else:
             lookup_word = row.word_token
 

-        return ICUToken(penalty=penalty, token=row.word_id, count=count,
+        return ICUToken(penalty=penalty, token=row.word_id, count=max(1, count),

                         lookup_word=lookup_word, is_indexed=True,
                         lookup_word=lookup_word, is_indexed=True,

-                        word_token=row.word_token, info=row.info)
+                        word_token=row.word_token, info=row.info,
+                        addr_count=max(1, addr_count))

 
 
 
 
 
 


@@ -187,11 +188,10 @@ class ICUQueryAnalyzer(AbstractQueryAnalyzer):
                         if trange.start == 0:
                             query.add_token(trange, qmod.TokenType.NEAR_ITEM, token)
                     else:
                         if trange.start == 0:
                             query.add_token(trange, qmod.TokenType.NEAR_ITEM, token)
                     else:

-                        query.add_token(trange, qmod.TokenType.QUALIFIER, token)
-                        if trange.start == 0 or trange.end == query.num_token_slots():
-                            token = copy(token)
-                            token.penalty += 0.1 * (query.num_token_slots())
+                        if trange.start == 0 and trange.end == query.num_token_slots():

                             query.add_token(trange, qmod.TokenType.NEAR_ITEM, token)
                             query.add_token(trange, qmod.TokenType.NEAR_ITEM, token)

+                        else:
+                            query.add_token(trange, qmod.TokenType.QUALIFIER, token)

                 else:
                     query.add_token(trange, DB_TO_TOKEN_TYPE[row.type], token)
 
                 else:
                     query.add_token(trange, DB_TO_TOKEN_TYPE[row.type], token)
 


@@ -259,7 +259,7 @@ class ICUQueryAnalyzer(AbstractQueryAnalyzer):
             if len(part.token) <= 4 and part[0].isdigit()\
                and not node.has_tokens(i+1, qmod.TokenType.HOUSENUMBER):
                 query.add_token(qmod.TokenRange(i, i+1), qmod.TokenType.HOUSENUMBER,
             if len(part.token) <= 4 and part[0].isdigit()\
                and not node.has_tokens(i+1, qmod.TokenType.HOUSENUMBER):
                 query.add_token(qmod.TokenRange(i, i+1), qmod.TokenType.HOUSENUMBER,

-                                ICUToken(0.5, 0, 1, part.token, True, part.token, None))
+                                ICUToken(0.5, 0, 1, 1, part.token, True, part.token, None))

 
 
     def rerank_tokens(self, query: qmod.QueryStruct, parts: QueryParts) -> None:
 
 
     def rerank_tokens(self, query: qmod.QueryStruct, parts: QueryParts) -> None: