test/python/api/search/test_token_assignment.py

   1 # SPDX-License-Identifier: GPL-3.0-or-later
   2 #
   3 # This file is part of Nominatim. (https://nominatim.org)
   4 #
   5 # Copyright (C) 2023 by the Nominatim developer community.
   6 # For a full list of authors see the git log.
   7 """
   8 Test for creation of token assignments from tokenized queries.
   9 """
  10 import pytest
  11
  12 from nominatim.api.search.query import QueryStruct, Phrase, PhraseType, BreakType, TokenType, TokenRange, Token
  13 from nominatim.api.search.token_assignment import yield_token_assignments, TokenAssignment, PENALTY_TOKENCHANGE
  14
  15 class MyToken(Token):
  16     def get_category(self):
  17         return 'this', 'that'
  18
  19
  20 def make_query(*args):
  21     q = None
  22     dummy = MyToken(3.0, 45, 1, 'foo', True)
  23
  24     for btype, ptype, tlist in args:
  25         if q is None:
  26             q = QueryStruct([Phrase(ptype, '')])
  27         else:
  28             q.add_node(btype, ptype)
  29
  30         start = len(q.nodes) - 1
  31         for end, ttype in tlist:
  32             q.add_token(TokenRange(start, end), ttype, [dummy])
  33
  34     q.add_node(BreakType.END, PhraseType.NONE)
  35
  36     return q
  37
  38
  39 def check_assignments(actual, *expected):
  40     todo = list(expected)
  41     for assignment in actual:
  42         assert assignment in todo, f"Unexpected assignment: {assignment}"
  43         todo.remove(assignment)
  44
  45     assert not todo, f"Missing assignments: {expected}"
  46
  47
  48 def test_query_with_missing_tokens():
  49     q = QueryStruct([Phrase(PhraseType.NONE, '')])
  50     q.add_node(BreakType.END, PhraseType.NONE)
  51
  52     assert list(yield_token_assignments(q)) == []
  53
  54
  55 def test_one_word_query():
  56     q = make_query((BreakType.START, PhraseType.NONE,
  57                     [(1, TokenType.PARTIAL),
  58                      (1, TokenType.WORD),
  59                      (1, TokenType.HOUSENUMBER)]))
  60
  61     res = list(yield_token_assignments(q))
  62     assert res == [TokenAssignment(name=TokenRange(0, 1))]
  63
  64
  65 def test_single_postcode():
  66     q = make_query((BreakType.START, PhraseType.NONE,
  67                     [(1, TokenType.POSTCODE)]))
  68
  69     res = list(yield_token_assignments(q))
  70     assert res == [TokenAssignment(postcode=TokenRange(0, 1))]
  71
  72
  73 def test_single_country_name():
  74     q = make_query((BreakType.START, PhraseType.NONE,
  75                     [(1, TokenType.COUNTRY)]))
  76
  77     res = list(yield_token_assignments(q))
  78     assert res == [TokenAssignment(country=TokenRange(0, 1))]
  79
  80
  81 def test_single_word_poi_search():
  82     q = make_query((BreakType.START, PhraseType.NONE,
  83                     [(1, TokenType.CATEGORY),
  84                      (1, TokenType.QUALIFIER)]))
  85
  86     res = list(yield_token_assignments(q))
  87     assert res == [TokenAssignment(category=TokenRange(0, 1))]
  88
  89
  90 @pytest.mark.parametrize('btype', [BreakType.WORD, BreakType.PART, BreakType.TOKEN])
  91 def test_multiple_simple_words(btype):
  92     q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.PARTIAL)]),
  93                    (btype, PhraseType.NONE, [(2, TokenType.PARTIAL)]),
  94                    (btype, PhraseType.NONE, [(3, TokenType.PARTIAL)]))
  95
  96     penalty = PENALTY_TOKENCHANGE[btype]
  97
  98     check_assignments(yield_token_assignments(q),
  99                       TokenAssignment(name=TokenRange(0, 3)),
 100                       TokenAssignment(penalty=penalty, name=TokenRange(0, 2),
 101                                       address=[TokenRange(2, 3)]),
 102                       TokenAssignment(penalty=penalty, name=TokenRange(0, 1),
 103                                       address=[TokenRange(1, 3)]),
 104                       TokenAssignment(penalty=penalty, name=TokenRange(1, 3),
 105                                       address=[TokenRange(0, 1)]),
 106                       TokenAssignment(penalty=penalty, name=TokenRange(2, 3),
 107                                       address=[TokenRange(0, 2)])
 108                      )
 109
 110
 111 def test_multiple_words_respect_phrase_break():
 112     q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.PARTIAL)]),
 113                    (BreakType.PHRASE, PhraseType.NONE, [(2, TokenType.PARTIAL)]))
 114
 115     check_assignments(yield_token_assignments(q),
 116                       TokenAssignment(name=TokenRange(0, 1),
 117                                       address=[TokenRange(1, 2)]),
 118                       TokenAssignment(name=TokenRange(1, 2),
 119                                       address=[TokenRange(0, 1)]))
 120
 121
 122 def test_housenumber_and_street():
 123     q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.HOUSENUMBER)]),
 124                    (BreakType.PHRASE, PhraseType.NONE, [(2, TokenType.PARTIAL)]))
 125
 126     check_assignments(yield_token_assignments(q),
 127                       TokenAssignment(name=TokenRange(1, 2),
 128                                       housenumber=TokenRange(0, 1)))
 129
 130
 131 def test_housenumber_and_street_backwards():
 132     q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.PARTIAL)]),
 133                    (BreakType.PHRASE, PhraseType.NONE, [(2, TokenType.HOUSENUMBER)]))
 134
 135     check_assignments(yield_token_assignments(q),
 136                       TokenAssignment(name=TokenRange(0, 1),
 137                                       housenumber=TokenRange(1, 2)))
 138
 139
 140 def test_housenumber_and_postcode():
 141     q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.PARTIAL)]),
 142                    (BreakType.WORD, PhraseType.NONE, [(2, TokenType.HOUSENUMBER)]),
 143                    (BreakType.WORD, PhraseType.NONE, [(3, TokenType.PARTIAL)]),
 144                    (BreakType.WORD, PhraseType.NONE, [(4, TokenType.POSTCODE)]))
 145
 146     check_assignments(yield_token_assignments(q),
 147                       TokenAssignment(penalty=pytest.approx(0.3),
 148                                       name=TokenRange(0, 1),
 149                                       housenumber=TokenRange(1, 2),
 150                                       address=[TokenRange(2, 3)],
 151                                       postcode=TokenRange(3, 4)))
 152
 153 def test_postcode_and_housenumber():
 154     q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.PARTIAL)]),
 155                    (BreakType.WORD, PhraseType.NONE, [(2, TokenType.POSTCODE)]),
 156                    (BreakType.WORD, PhraseType.NONE, [(3, TokenType.PARTIAL)]),
 157                    (BreakType.WORD, PhraseType.NONE, [(4, TokenType.HOUSENUMBER)]))
 158
 159     check_assignments(yield_token_assignments(q),
 160                       TokenAssignment(penalty=pytest.approx(0.3),
 161                                       name=TokenRange(2, 3),
 162                                       housenumber=TokenRange(3, 4),
 163                                       address=[TokenRange(0, 1)],
 164                                       postcode=TokenRange(1, 2)))
 165
 166
 167 def test_country_housenumber_postcode():
 168     q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.COUNTRY)]),
 169                    (BreakType.WORD, PhraseType.NONE, [(2, TokenType.PARTIAL)]),
 170                    (BreakType.WORD, PhraseType.NONE, [(3, TokenType.HOUSENUMBER)]),
 171                    (BreakType.WORD, PhraseType.NONE, [(4, TokenType.POSTCODE)]))
 172
 173     check_assignments(yield_token_assignments(q))
 174
 175
 176 @pytest.mark.parametrize('ttype', [TokenType.POSTCODE, TokenType.COUNTRY,
 177                                    TokenType.CATEGORY, TokenType.QUALIFIER])
 178 def test_housenumber_with_only_special_terms(ttype):
 179     q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.HOUSENUMBER)]),
 180                    (BreakType.WORD, PhraseType.NONE, [(2, ttype)]))
 181
 182     check_assignments(yield_token_assignments(q))
 183
 184
 185 @pytest.mark.parametrize('ttype', [TokenType.POSTCODE, TokenType.HOUSENUMBER, TokenType.COUNTRY])
 186 def test_multiple_special_tokens(ttype):
 187     q = make_query((BreakType.START, PhraseType.NONE, [(1, ttype)]),
 188                    (BreakType.PHRASE, PhraseType.NONE, [(2, TokenType.PARTIAL)]),
 189                    (BreakType.PHRASE, PhraseType.NONE, [(3, ttype)]))
 190
 191     check_assignments(yield_token_assignments(q))
 192
 193
 194 def test_housenumber_many_phrases():
 195     q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.PARTIAL)]),
 196                    (BreakType.PHRASE, PhraseType.NONE, [(2, TokenType.PARTIAL)]),
 197                    (BreakType.PHRASE, PhraseType.NONE, [(3, TokenType.PARTIAL)]),
 198                    (BreakType.PHRASE, PhraseType.NONE, [(4, TokenType.HOUSENUMBER)]),
 199                    (BreakType.WORD, PhraseType.NONE, [(5, TokenType.PARTIAL)]))
 200
 201     check_assignments(yield_token_assignments(q),
 202                       TokenAssignment(penalty=0.1,
 203                                       name=TokenRange(4, 5),
 204                                       housenumber=TokenRange(3, 4),\
 205                                       address=[TokenRange(0, 1), TokenRange(1, 2),
 206                                                TokenRange(2, 3)]))
 207
 208
 209 def test_country_at_beginning():
 210     q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.COUNTRY)]),
 211                    (BreakType.WORD, PhraseType.NONE, [(2, TokenType.PARTIAL)]))
 212
 213     check_assignments(yield_token_assignments(q),
 214                       TokenAssignment(penalty=0.1, name=TokenRange(1, 2),
 215                                       country=TokenRange(0, 1)))
 216
 217
 218 def test_country_at_end():
 219     q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.PARTIAL)]),
 220                    (BreakType.WORD, PhraseType.NONE, [(2, TokenType.COUNTRY)]))
 221
 222     check_assignments(yield_token_assignments(q),
 223                       TokenAssignment(penalty=0.1, name=TokenRange(0, 1),
 224                                       country=TokenRange(1, 2)))
 225
 226
 227 def test_country_in_middle():
 228     q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.PARTIAL)]),
 229                    (BreakType.WORD, PhraseType.NONE, [(2, TokenType.COUNTRY)]),
 230                    (BreakType.WORD, PhraseType.NONE, [(3, TokenType.PARTIAL)]))
 231
 232     check_assignments(yield_token_assignments(q))
 233
 234
 235 def test_postcode_with_designation():
 236     q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.POSTCODE)]),
 237                    (BreakType.PHRASE, PhraseType.NONE, [(2, TokenType.PARTIAL)]))
 238
 239     check_assignments(yield_token_assignments(q),
 240                       TokenAssignment(name=TokenRange(1, 2),
 241                                       postcode=TokenRange(0, 1)),
 242                       TokenAssignment(postcode=TokenRange(0, 1),
 243                                       address=[TokenRange(1, 2)]))
 244
 245
 246 def test_postcode_with_designation_backwards():
 247     q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.PARTIAL)]),
 248                    (BreakType.PHRASE, PhraseType.NONE, [(2, TokenType.POSTCODE)]))
 249
 250     check_assignments(yield_token_assignments(q),
 251                       TokenAssignment(name=TokenRange(0, 1),
 252                                       postcode=TokenRange(1, 2)),
 253                       TokenAssignment(postcode=TokenRange(1, 2),
 254                                       address=[TokenRange(0, 1)]))
 255
 256
 257 def test_category_at_beginning():
 258     q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.CATEGORY)]),
 259                    (BreakType.WORD, PhraseType.NONE, [(2, TokenType.PARTIAL)]))
 260
 261     check_assignments(yield_token_assignments(q),
 262                       TokenAssignment(penalty=0.1, name=TokenRange(1, 2),
 263                                       category=TokenRange(0, 1)))
 264
 265
 266 def test_category_at_end():
 267     q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.PARTIAL)]),
 268                    (BreakType.WORD, PhraseType.NONE, [(2, TokenType.CATEGORY)]))
 269
 270     check_assignments(yield_token_assignments(q),
 271                       TokenAssignment(penalty=0.1, name=TokenRange(0, 1),
 272                                       category=TokenRange(1, 2)))
 273
 274
 275 def test_category_in_middle():
 276     q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.PARTIAL)]),
 277                    (BreakType.WORD, PhraseType.NONE, [(2, TokenType.CATEGORY)]),
 278                    (BreakType.WORD, PhraseType.NONE, [(3, TokenType.PARTIAL)]))
 279
 280     check_assignments(yield_token_assignments(q))
 281
 282
 283 def test_qualifier_at_beginning():
 284     q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.QUALIFIER)]),
 285                    (BreakType.WORD, PhraseType.NONE, [(2, TokenType.PARTIAL)]),
 286                    (BreakType.WORD, PhraseType.NONE, [(3, TokenType.PARTIAL)]))
 287
 288
 289     check_assignments(yield_token_assignments(q),
 290                       TokenAssignment(penalty=0.1, name=TokenRange(1, 3),
 291                                       qualifier=TokenRange(0, 1)),
 292                       TokenAssignment(penalty=0.2, name=TokenRange(1, 2),
 293                                       qualifier=TokenRange(0, 1),
 294                                       address=[TokenRange(2, 3)]))
 295
 296
 297 def test_qualifier_after_name():
 298     q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.PARTIAL)]),
 299                    (BreakType.WORD, PhraseType.NONE, [(2, TokenType.PARTIAL)]),
 300                    (BreakType.WORD, PhraseType.NONE, [(3, TokenType.QUALIFIER)]),
 301                    (BreakType.WORD, PhraseType.NONE, [(4, TokenType.PARTIAL)]),
 302                    (BreakType.WORD, PhraseType.NONE, [(5, TokenType.PARTIAL)]))
 303
 304
 305     check_assignments(yield_token_assignments(q),
 306                       TokenAssignment(penalty=0.2, name=TokenRange(0, 2),
 307                                       qualifier=TokenRange(2, 3),
 308                                       address=[TokenRange(3, 5)]),
 309                       TokenAssignment(penalty=0.2, name=TokenRange(3, 5),
 310                                       qualifier=TokenRange(2, 3),
 311                                       address=[TokenRange(0, 2)]))
 312
 313
 314 def test_qualifier_before_housenumber():
 315     q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.QUALIFIER)]),
 316                    (BreakType.WORD, PhraseType.NONE, [(2, TokenType.HOUSENUMBER)]),
 317                    (BreakType.WORD, PhraseType.NONE, [(3, TokenType.PARTIAL)]))
 318
 319     check_assignments(yield_token_assignments(q))
 320
 321
 322 def test_qualifier_after_housenumber():
 323     q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.HOUSENUMBER)]),
 324                    (BreakType.WORD, PhraseType.NONE, [(2, TokenType.QUALIFIER)]),
 325                    (BreakType.WORD, PhraseType.NONE, [(3, TokenType.PARTIAL)]))
 326
 327     check_assignments(yield_token_assignments(q))