test/python/api/search/test_token_assignment.py

   1 # SPDX-License-Identifier: GPL-3.0-or-later
   2 #
   3 # This file is part of Nominatim. (https://nominatim.org)
   4 #
   5 # Copyright (C) 2023 by the Nominatim developer community.
   6 # For a full list of authors see the git log.
   7 """
   8 Test for creation of token assignments from tokenized queries.
   9 """
  10 import pytest
  11
  12 from nominatim.api.search.query import QueryStruct, Phrase, PhraseType, BreakType, TokenType, TokenRange, Token
  13 from nominatim.api.search.token_assignment import yield_token_assignments, TokenAssignment, PENALTY_TOKENCHANGE
  14
  15 class MyToken(Token):
  16     def get_category(self):
  17         return 'this', 'that'
  18
  19
  20 def make_query(*args):
  21     q = None
  22     dummy = MyToken(3.0, 45, 1, 'foo', True)
  23
  24     for btype, ptype, tlist in args:
  25         if q is None:
  26             q = QueryStruct([Phrase(ptype, '')])
  27         else:
  28             q.add_node(btype, ptype)
  29
  30         start = len(q.nodes) - 1
  31         for end, ttype in tlist:
  32             q.add_token(TokenRange(start, end), ttype, dummy)
  33
  34     q.add_node(BreakType.END, PhraseType.NONE)
  35
  36     return q
  37
  38
  39 def check_assignments(actual, *expected):
  40     todo = list(expected)
  41     for assignment in actual:
  42         assert assignment in todo, f"Unexpected assignment: {assignment}"
  43         todo.remove(assignment)
  44
  45     assert not todo, f"Missing assignments: {expected}"
  46
  47
  48 def test_query_with_missing_tokens():
  49     q = QueryStruct([Phrase(PhraseType.NONE, '')])
  50     q.add_node(BreakType.END, PhraseType.NONE)
  51
  52     assert list(yield_token_assignments(q)) == []
  53
  54
  55 def test_one_word_query():
  56     q = make_query((BreakType.START, PhraseType.NONE,
  57                     [(1, TokenType.PARTIAL),
  58                      (1, TokenType.WORD),
  59                      (1, TokenType.HOUSENUMBER)]))
  60
  61     res = list(yield_token_assignments(q))
  62     assert res == [TokenAssignment(name=TokenRange(0, 1))]
  63
  64
  65 def test_single_postcode():
  66     q = make_query((BreakType.START, PhraseType.NONE,
  67                     [(1, TokenType.POSTCODE)]))
  68
  69     res = list(yield_token_assignments(q))
  70     assert res == [TokenAssignment(postcode=TokenRange(0, 1))]
  71
  72
  73 def test_single_country_name():
  74     q = make_query((BreakType.START, PhraseType.NONE,
  75                     [(1, TokenType.COUNTRY)]))
  76
  77     res = list(yield_token_assignments(q))
  78     assert res == [TokenAssignment(country=TokenRange(0, 1))]
  79
  80
  81 def test_single_word_poi_search():
  82     q = make_query((BreakType.START, PhraseType.NONE,
  83                     [(1, TokenType.CATEGORY),
  84                      (1, TokenType.QUALIFIER)]))
  85
  86     res = list(yield_token_assignments(q))
  87     assert res == [TokenAssignment(category=TokenRange(0, 1))]
  88
  89
  90 @pytest.mark.parametrize('btype', [BreakType.WORD, BreakType.PART, BreakType.TOKEN])
  91 def test_multiple_simple_words(btype):
  92     q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.PARTIAL)]),
  93                    (btype, PhraseType.NONE, [(2, TokenType.PARTIAL)]),
  94                    (btype, PhraseType.NONE, [(3, TokenType.PARTIAL)]))
  95
  96     penalty = PENALTY_TOKENCHANGE[btype]
  97
  98     check_assignments(yield_token_assignments(q),
  99                       TokenAssignment(name=TokenRange(0, 3)),
 100                       TokenAssignment(penalty=penalty, name=TokenRange(0, 2),
 101                                       address=[TokenRange(2, 3)]),
 102                       TokenAssignment(penalty=penalty, name=TokenRange(0, 1),
 103                                       address=[TokenRange(1, 3)]),
 104                       TokenAssignment(penalty=penalty, name=TokenRange(1, 3),
 105                                       address=[TokenRange(0, 1)]),
 106                       TokenAssignment(penalty=penalty, name=TokenRange(2, 3),
 107                                       address=[TokenRange(0, 2)])
 108                      )
 109
 110
 111 def test_multiple_words_respect_phrase_break():
 112     q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.PARTIAL)]),
 113                    (BreakType.PHRASE, PhraseType.NONE, [(2, TokenType.PARTIAL)]))
 114
 115     check_assignments(yield_token_assignments(q),
 116                       TokenAssignment(name=TokenRange(0, 1),
 117                                       address=[TokenRange(1, 2)]),
 118                       TokenAssignment(name=TokenRange(1, 2),
 119                                       address=[TokenRange(0, 1)]))
 120
 121
 122 def test_housenumber_and_street():
 123     q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.HOUSENUMBER)]),
 124                    (BreakType.PHRASE, PhraseType.NONE, [(2, TokenType.PARTIAL)]))
 125
 126     check_assignments(yield_token_assignments(q),
 127                       TokenAssignment(name=TokenRange(1, 2),
 128                                       housenumber=TokenRange(0, 1)),
 129                       TokenAssignment(address=[TokenRange(1, 2)],
 130                                       housenumber=TokenRange(0, 1)))
 131
 132
 133 def test_housenumber_and_street_backwards():
 134     q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.PARTIAL)]),
 135                    (BreakType.PHRASE, PhraseType.NONE, [(2, TokenType.HOUSENUMBER)]))
 136
 137     check_assignments(yield_token_assignments(q),
 138                       TokenAssignment(name=TokenRange(0, 1),
 139                                       housenumber=TokenRange(1, 2)),
 140                       TokenAssignment(address=[TokenRange(0, 1)],
 141                                       housenumber=TokenRange(1, 2)))
 142
 143
 144 def test_housenumber_and_postcode():
 145     q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.PARTIAL)]),
 146                    (BreakType.WORD, PhraseType.NONE, [(2, TokenType.HOUSENUMBER)]),
 147                    (BreakType.WORD, PhraseType.NONE, [(3, TokenType.PARTIAL)]),
 148                    (BreakType.WORD, PhraseType.NONE, [(4, TokenType.POSTCODE)]))
 149
 150     check_assignments(yield_token_assignments(q),
 151                       TokenAssignment(penalty=pytest.approx(0.3),
 152                                       name=TokenRange(0, 1),
 153                                       housenumber=TokenRange(1, 2),
 154                                       address=[TokenRange(2, 3)],
 155                                       postcode=TokenRange(3, 4)),
 156                       TokenAssignment(penalty=pytest.approx(0.3),
 157                                       housenumber=TokenRange(1, 2),
 158                                       address=[TokenRange(0, 1), TokenRange(2, 3)],
 159                                       postcode=TokenRange(3, 4)))
 160
 161 def test_postcode_and_housenumber():
 162     q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.PARTIAL)]),
 163                    (BreakType.WORD, PhraseType.NONE, [(2, TokenType.POSTCODE)]),
 164                    (BreakType.WORD, PhraseType.NONE, [(3, TokenType.PARTIAL)]),
 165                    (BreakType.WORD, PhraseType.NONE, [(4, TokenType.HOUSENUMBER)]))
 166
 167     check_assignments(yield_token_assignments(q),
 168                       TokenAssignment(penalty=pytest.approx(0.3),
 169                                       name=TokenRange(2, 3),
 170                                       housenumber=TokenRange(3, 4),
 171                                       address=[TokenRange(0, 1)],
 172                                       postcode=TokenRange(1, 2)),
 173                       TokenAssignment(penalty=pytest.approx(0.3),
 174                                       housenumber=TokenRange(3, 4),
 175                                       address=[TokenRange(0, 1), TokenRange(2, 3)],
 176                                       postcode=TokenRange(1, 2)))
 177
 178
 179 def test_country_housenumber_postcode():
 180     q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.COUNTRY)]),
 181                    (BreakType.WORD, PhraseType.NONE, [(2, TokenType.PARTIAL)]),
 182                    (BreakType.WORD, PhraseType.NONE, [(3, TokenType.HOUSENUMBER)]),
 183                    (BreakType.WORD, PhraseType.NONE, [(4, TokenType.POSTCODE)]))
 184
 185     check_assignments(yield_token_assignments(q))
 186
 187
 188 @pytest.mark.parametrize('ttype', [TokenType.POSTCODE, TokenType.COUNTRY,
 189                                    TokenType.CATEGORY, TokenType.QUALIFIER])
 190 def test_housenumber_with_only_special_terms(ttype):
 191     q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.HOUSENUMBER)]),
 192                    (BreakType.WORD, PhraseType.NONE, [(2, ttype)]))
 193
 194     check_assignments(yield_token_assignments(q))
 195
 196
 197 @pytest.mark.parametrize('ttype', [TokenType.POSTCODE, TokenType.HOUSENUMBER, TokenType.COUNTRY])
 198 def test_multiple_special_tokens(ttype):
 199     q = make_query((BreakType.START, PhraseType.NONE, [(1, ttype)]),
 200                    (BreakType.PHRASE, PhraseType.NONE, [(2, TokenType.PARTIAL)]),
 201                    (BreakType.PHRASE, PhraseType.NONE, [(3, ttype)]))
 202
 203     check_assignments(yield_token_assignments(q))
 204
 205
 206 def test_housenumber_many_phrases():
 207     q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.PARTIAL)]),
 208                    (BreakType.PHRASE, PhraseType.NONE, [(2, TokenType.PARTIAL)]),
 209                    (BreakType.PHRASE, PhraseType.NONE, [(3, TokenType.PARTIAL)]),
 210                    (BreakType.PHRASE, PhraseType.NONE, [(4, TokenType.HOUSENUMBER)]),
 211                    (BreakType.WORD, PhraseType.NONE, [(5, TokenType.PARTIAL)]))
 212
 213     check_assignments(yield_token_assignments(q),
 214                       TokenAssignment(penalty=0.1,
 215                                       name=TokenRange(4, 5),
 216                                       housenumber=TokenRange(3, 4),\
 217                                       address=[TokenRange(0, 1), TokenRange(1, 2),
 218                                                TokenRange(2, 3)]),
 219                       TokenAssignment(penalty=0.1,
 220                                       housenumber=TokenRange(3, 4),\
 221                                       address=[TokenRange(0, 1), TokenRange(1, 2),
 222                                                TokenRange(2, 3), TokenRange(4, 5)]))
 223
 224
 225 def test_country_at_beginning():
 226     q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.COUNTRY)]),
 227                    (BreakType.WORD, PhraseType.NONE, [(2, TokenType.PARTIAL)]))
 228
 229     check_assignments(yield_token_assignments(q),
 230                       TokenAssignment(penalty=0.1, name=TokenRange(1, 2),
 231                                       country=TokenRange(0, 1)))
 232
 233
 234 def test_country_at_end():
 235     q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.PARTIAL)]),
 236                    (BreakType.WORD, PhraseType.NONE, [(2, TokenType.COUNTRY)]))
 237
 238     check_assignments(yield_token_assignments(q),
 239                       TokenAssignment(penalty=0.1, name=TokenRange(0, 1),
 240                                       country=TokenRange(1, 2)))
 241
 242
 243 def test_country_in_middle():
 244     q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.PARTIAL)]),
 245                    (BreakType.WORD, PhraseType.NONE, [(2, TokenType.COUNTRY)]),
 246                    (BreakType.WORD, PhraseType.NONE, [(3, TokenType.PARTIAL)]))
 247
 248     check_assignments(yield_token_assignments(q))
 249
 250
 251 def test_postcode_with_designation():
 252     q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.POSTCODE)]),
 253                    (BreakType.PHRASE, PhraseType.NONE, [(2, TokenType.PARTIAL)]))
 254
 255     check_assignments(yield_token_assignments(q),
 256                       TokenAssignment(name=TokenRange(1, 2),
 257                                       postcode=TokenRange(0, 1)),
 258                       TokenAssignment(postcode=TokenRange(0, 1),
 259                                       address=[TokenRange(1, 2)]))
 260
 261
 262 def test_postcode_with_designation_backwards():
 263     q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.PARTIAL)]),
 264                    (BreakType.PHRASE, PhraseType.NONE, [(2, TokenType.POSTCODE)]))
 265
 266     check_assignments(yield_token_assignments(q),
 267                       TokenAssignment(name=TokenRange(0, 1),
 268                                       postcode=TokenRange(1, 2)),
 269                       TokenAssignment(postcode=TokenRange(1, 2),
 270                                       address=[TokenRange(0, 1)]))
 271
 272
 273 def test_category_at_beginning():
 274     q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.CATEGORY)]),
 275                    (BreakType.WORD, PhraseType.NONE, [(2, TokenType.PARTIAL)]))
 276
 277     check_assignments(yield_token_assignments(q),
 278                       TokenAssignment(penalty=0.1, name=TokenRange(1, 2),
 279                                       category=TokenRange(0, 1)))
 280
 281
 282 def test_category_at_end():
 283     q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.PARTIAL)]),
 284                    (BreakType.WORD, PhraseType.NONE, [(2, TokenType.CATEGORY)]))
 285
 286     check_assignments(yield_token_assignments(q),
 287                       TokenAssignment(penalty=0.1, name=TokenRange(0, 1),
 288                                       category=TokenRange(1, 2)))
 289
 290
 291 def test_category_in_middle():
 292     q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.PARTIAL)]),
 293                    (BreakType.WORD, PhraseType.NONE, [(2, TokenType.CATEGORY)]),
 294                    (BreakType.WORD, PhraseType.NONE, [(3, TokenType.PARTIAL)]))
 295
 296     check_assignments(yield_token_assignments(q))
 297
 298
 299 def test_qualifier_at_beginning():
 300     q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.QUALIFIER)]),
 301                    (BreakType.WORD, PhraseType.NONE, [(2, TokenType.PARTIAL)]),
 302                    (BreakType.WORD, PhraseType.NONE, [(3, TokenType.PARTIAL)]))
 303
 304
 305     check_assignments(yield_token_assignments(q),
 306                       TokenAssignment(penalty=0.1, name=TokenRange(1, 3),
 307                                       qualifier=TokenRange(0, 1)),
 308                       TokenAssignment(penalty=0.2, name=TokenRange(1, 2),
 309                                       qualifier=TokenRange(0, 1),
 310                                       address=[TokenRange(2, 3)]))
 311
 312
 313 def test_qualifier_after_name():
 314     q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.PARTIAL)]),
 315                    (BreakType.WORD, PhraseType.NONE, [(2, TokenType.PARTIAL)]),
 316                    (BreakType.WORD, PhraseType.NONE, [(3, TokenType.QUALIFIER)]),
 317                    (BreakType.WORD, PhraseType.NONE, [(4, TokenType.PARTIAL)]),
 318                    (BreakType.WORD, PhraseType.NONE, [(5, TokenType.PARTIAL)]))
 319
 320
 321     check_assignments(yield_token_assignments(q),
 322                       TokenAssignment(penalty=0.2, name=TokenRange(0, 2),
 323                                       qualifier=TokenRange(2, 3),
 324                                       address=[TokenRange(3, 5)]),
 325                       TokenAssignment(penalty=0.2, name=TokenRange(3, 5),
 326                                       qualifier=TokenRange(2, 3),
 327                                       address=[TokenRange(0, 2)]))
 328
 329
 330 def test_qualifier_before_housenumber():
 331     q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.QUALIFIER)]),
 332                    (BreakType.WORD, PhraseType.NONE, [(2, TokenType.HOUSENUMBER)]),
 333                    (BreakType.WORD, PhraseType.NONE, [(3, TokenType.PARTIAL)]))
 334
 335     check_assignments(yield_token_assignments(q))
 336
 337
 338 def test_qualifier_after_housenumber():
 339     q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.HOUSENUMBER)]),
 340                    (BreakType.WORD, PhraseType.NONE, [(2, TokenType.QUALIFIER)]),
 341                    (BreakType.WORD, PhraseType.NONE, [(3, TokenType.PARTIAL)]))
 342
 343     check_assignments(yield_token_assignments(q))