#
# This file is part of Nominatim. (https://nominatim.org)
#
-# Copyright (C) 2023 by the Nominatim developer community.
+# Copyright (C) 2024 by the Nominatim developer community.
# For a full list of authors see the git log.
"""
Test for creation of token assignments from tokenized queries.
"""
import pytest
-from nominatim.api.search.query import QueryStruct, Phrase, PhraseType, BreakType, TokenType, TokenRange, Token
-from nominatim.api.search.token_assignment import yield_token_assignments, TokenAssignment, PENALTY_TOKENCHANGE
+from nominatim_api.search.query import QueryStruct, Phrase, TokenRange, Token
+import nominatim_api.search.query as qmod
+from nominatim_api.search.token_assignment import yield_token_assignments, TokenAssignment, PENALTY_TOKENCHANGE
class MyToken(Token):
def get_category(self):
def make_query(*args):
- q = None
- dummy = MyToken(3.0, 45, 1, 'foo', True)
+ q = QueryStruct([Phrase(args[0][1], '')])
+ dummy = MyToken(penalty=3.0, token=45, count=1, addr_count=1,
+ lookup_word='foo')
- for btype, ptype, tlist in args:
- if q is None:
- q = QueryStruct([Phrase(ptype, '')])
- else:
- q.add_node(btype, ptype)
+ for btype, ptype, _ in args[1:]:
+ q.add_node(btype, ptype)
+ q.add_node(qmod.BREAK_END, qmod.PHRASE_ANY)
- start = len(q.nodes) - 1
- for end, ttype in tlist:
+ for start, t in enumerate(args):
+ for end, ttype in t[2]:
q.add_token(TokenRange(start, end), ttype, dummy)
- q.add_node(BreakType.END, PhraseType.NONE)
-
return q
def test_query_with_missing_tokens():
- q = QueryStruct([Phrase(PhraseType.NONE, '')])
- q.add_node(BreakType.END, PhraseType.NONE)
+ q = QueryStruct([Phrase(qmod.PHRASE_ANY, '')])
+ q.add_node(qmod.BREAK_END, qmod.PHRASE_ANY)
assert list(yield_token_assignments(q)) == []
def test_one_word_query():
- q = make_query((BreakType.START, PhraseType.NONE,
- [(1, TokenType.PARTIAL),
- (1, TokenType.WORD),
- (1, TokenType.HOUSENUMBER)]))
+ q = make_query((qmod.BREAK_START, qmod.PHRASE_ANY,
+ [(1, qmod.TOKEN_PARTIAL),
+ (1, qmod.TOKEN_WORD),
+ (1, qmod.TOKEN_HOUSENUMBER)]))
res = list(yield_token_assignments(q))
assert res == [TokenAssignment(name=TokenRange(0, 1))]
def test_single_postcode():
- q = make_query((BreakType.START, PhraseType.NONE,
- [(1, TokenType.POSTCODE)]))
+ q = make_query((qmod.BREAK_START, qmod.PHRASE_ANY,
+ [(1, qmod.TOKEN_POSTCODE)]))
res = list(yield_token_assignments(q))
assert res == [TokenAssignment(postcode=TokenRange(0, 1))]
def test_single_country_name():
- q = make_query((BreakType.START, PhraseType.NONE,
- [(1, TokenType.COUNTRY)]))
+ q = make_query((qmod.BREAK_START, qmod.PHRASE_ANY,
+ [(1, qmod.TOKEN_COUNTRY)]))
res = list(yield_token_assignments(q))
assert res == [TokenAssignment(country=TokenRange(0, 1))]
def test_single_word_poi_search():
- q = make_query((BreakType.START, PhraseType.NONE,
- [(1, TokenType.CATEGORY),
- (1, TokenType.QUALIFIER)]))
+ q = make_query((qmod.BREAK_START, qmod.PHRASE_ANY,
+ [(1, qmod.TOKEN_NEAR_ITEM),
+ (1, qmod.TOKEN_QUALIFIER)]))
res = list(yield_token_assignments(q))
- assert res == [TokenAssignment(category=TokenRange(0, 1))]
+ assert res == [TokenAssignment(near_item=TokenRange(0, 1))]
-@pytest.mark.parametrize('btype', [BreakType.WORD, BreakType.PART, BreakType.TOKEN])
+@pytest.mark.parametrize('btype', [qmod.BREAK_WORD, qmod.BREAK_PART, qmod.BREAK_TOKEN])
def test_multiple_simple_words(btype):
- q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.PARTIAL)]),
- (btype, PhraseType.NONE, [(2, TokenType.PARTIAL)]),
- (btype, PhraseType.NONE, [(3, TokenType.PARTIAL)]))
+ q = make_query((qmod.BREAK_START, qmod.PHRASE_ANY, [(1, qmod.TOKEN_PARTIAL)]),
+ (btype, qmod.PHRASE_ANY, [(2, qmod.TOKEN_PARTIAL)]),
+ (btype, qmod.PHRASE_ANY, [(3, qmod.TOKEN_PARTIAL)]))
penalty = PENALTY_TOKENCHANGE[btype]
def test_multiple_words_respect_phrase_break():
- q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.PARTIAL)]),
- (BreakType.PHRASE, PhraseType.NONE, [(2, TokenType.PARTIAL)]))
+ q = make_query((qmod.BREAK_START, qmod.PHRASE_ANY, [(1, qmod.TOKEN_PARTIAL)]),
+ (qmod.BREAK_PHRASE, qmod.PHRASE_ANY, [(2, qmod.TOKEN_PARTIAL)]))
check_assignments(yield_token_assignments(q),
TokenAssignment(name=TokenRange(0, 1),
def test_housenumber_and_street():
- q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.HOUSENUMBER)]),
- (BreakType.PHRASE, PhraseType.NONE, [(2, TokenType.PARTIAL)]))
+ q = make_query((qmod.BREAK_START, qmod.PHRASE_ANY, [(1, qmod.TOKEN_HOUSENUMBER)]),
+ (qmod.BREAK_PHRASE, qmod.PHRASE_ANY, [(2, qmod.TOKEN_PARTIAL)]))
check_assignments(yield_token_assignments(q),
TokenAssignment(name=TokenRange(1, 2),
def test_housenumber_and_street_backwards():
- q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.PARTIAL)]),
- (BreakType.PHRASE, PhraseType.NONE, [(2, TokenType.HOUSENUMBER)]))
+ q = make_query((qmod.BREAK_START, qmod.PHRASE_ANY, [(1, qmod.TOKEN_PARTIAL)]),
+ (qmod.BREAK_PHRASE, qmod.PHRASE_ANY, [(2, qmod.TOKEN_HOUSENUMBER)]))
check_assignments(yield_token_assignments(q),
TokenAssignment(name=TokenRange(0, 1),
def test_housenumber_and_postcode():
- q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.PARTIAL)]),
- (BreakType.WORD, PhraseType.NONE, [(2, TokenType.HOUSENUMBER)]),
- (BreakType.WORD, PhraseType.NONE, [(3, TokenType.PARTIAL)]),
- (BreakType.WORD, PhraseType.NONE, [(4, TokenType.POSTCODE)]))
+ q = make_query((qmod.BREAK_START, qmod.PHRASE_ANY, [(1, qmod.TOKEN_PARTIAL)]),
+ (qmod.BREAK_WORD, qmod.PHRASE_ANY, [(2, qmod.TOKEN_HOUSENUMBER)]),
+ (qmod.BREAK_WORD, qmod.PHRASE_ANY, [(3, qmod.TOKEN_PARTIAL)]),
+ (qmod.BREAK_WORD, qmod.PHRASE_ANY, [(4, qmod.TOKEN_POSTCODE)]))
check_assignments(yield_token_assignments(q),
TokenAssignment(penalty=pytest.approx(0.3),
postcode=TokenRange(3, 4)))
def test_postcode_and_housenumber():
- q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.PARTIAL)]),
- (BreakType.WORD, PhraseType.NONE, [(2, TokenType.POSTCODE)]),
- (BreakType.WORD, PhraseType.NONE, [(3, TokenType.PARTIAL)]),
- (BreakType.WORD, PhraseType.NONE, [(4, TokenType.HOUSENUMBER)]))
+ q = make_query((qmod.BREAK_START, qmod.PHRASE_ANY, [(1, qmod.TOKEN_PARTIAL)]),
+ (qmod.BREAK_WORD, qmod.PHRASE_ANY, [(2, qmod.TOKEN_POSTCODE)]),
+ (qmod.BREAK_WORD, qmod.PHRASE_ANY, [(3, qmod.TOKEN_PARTIAL)]),
+ (qmod.BREAK_WORD, qmod.PHRASE_ANY, [(4, qmod.TOKEN_HOUSENUMBER)]))
check_assignments(yield_token_assignments(q),
TokenAssignment(penalty=pytest.approx(0.3),
def test_country_housenumber_postcode():
- q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.COUNTRY)]),
- (BreakType.WORD, PhraseType.NONE, [(2, TokenType.PARTIAL)]),
- (BreakType.WORD, PhraseType.NONE, [(3, TokenType.HOUSENUMBER)]),
- (BreakType.WORD, PhraseType.NONE, [(4, TokenType.POSTCODE)]))
+ q = make_query((qmod.BREAK_START, qmod.PHRASE_ANY, [(1, qmod.TOKEN_COUNTRY)]),
+ (qmod.BREAK_WORD, qmod.PHRASE_ANY, [(2, qmod.TOKEN_PARTIAL)]),
+ (qmod.BREAK_WORD, qmod.PHRASE_ANY, [(3, qmod.TOKEN_HOUSENUMBER)]),
+ (qmod.BREAK_WORD, qmod.PHRASE_ANY, [(4, qmod.TOKEN_POSTCODE)]))
check_assignments(yield_token_assignments(q))
-@pytest.mark.parametrize('ttype', [TokenType.POSTCODE, TokenType.COUNTRY,
- TokenType.CATEGORY, TokenType.QUALIFIER])
+@pytest.mark.parametrize('ttype', [qmod.TOKEN_POSTCODE, qmod.TOKEN_COUNTRY,
+ qmod.TOKEN_NEAR_ITEM, qmod.TOKEN_QUALIFIER])
def test_housenumber_with_only_special_terms(ttype):
- q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.HOUSENUMBER)]),
- (BreakType.WORD, PhraseType.NONE, [(2, ttype)]))
+ q = make_query((qmod.BREAK_START, qmod.PHRASE_ANY, [(1, qmod.TOKEN_HOUSENUMBER)]),
+ (qmod.BREAK_WORD, qmod.PHRASE_ANY, [(2, ttype)]))
check_assignments(yield_token_assignments(q))
-@pytest.mark.parametrize('ttype', [TokenType.POSTCODE, TokenType.HOUSENUMBER, TokenType.COUNTRY])
+@pytest.mark.parametrize('ttype', [qmod.TOKEN_POSTCODE, qmod.TOKEN_HOUSENUMBER, qmod.TOKEN_COUNTRY])
def test_multiple_special_tokens(ttype):
- q = make_query((BreakType.START, PhraseType.NONE, [(1, ttype)]),
- (BreakType.PHRASE, PhraseType.NONE, [(2, TokenType.PARTIAL)]),
- (BreakType.PHRASE, PhraseType.NONE, [(3, ttype)]))
+ q = make_query((qmod.BREAK_START, qmod.PHRASE_ANY, [(1, ttype)]),
+ (qmod.BREAK_PHRASE, qmod.PHRASE_ANY, [(2, qmod.TOKEN_PARTIAL)]),
+ (qmod.BREAK_PHRASE, qmod.PHRASE_ANY, [(3, ttype)]))
check_assignments(yield_token_assignments(q))
def test_housenumber_many_phrases():
- q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.PARTIAL)]),
- (BreakType.PHRASE, PhraseType.NONE, [(2, TokenType.PARTIAL)]),
- (BreakType.PHRASE, PhraseType.NONE, [(3, TokenType.PARTIAL)]),
- (BreakType.PHRASE, PhraseType.NONE, [(4, TokenType.HOUSENUMBER)]),
- (BreakType.WORD, PhraseType.NONE, [(5, TokenType.PARTIAL)]))
+ q = make_query((qmod.BREAK_START, qmod.PHRASE_ANY, [(1, qmod.TOKEN_PARTIAL)]),
+ (qmod.BREAK_PHRASE, qmod.PHRASE_ANY, [(2, qmod.TOKEN_PARTIAL)]),
+ (qmod.BREAK_PHRASE, qmod.PHRASE_ANY, [(3, qmod.TOKEN_PARTIAL)]),
+ (qmod.BREAK_PHRASE, qmod.PHRASE_ANY, [(4, qmod.TOKEN_HOUSENUMBER)]),
+ (qmod.BREAK_WORD, qmod.PHRASE_ANY, [(5, qmod.TOKEN_PARTIAL)]))
check_assignments(yield_token_assignments(q),
TokenAssignment(penalty=0.1,
def test_country_at_beginning():
- q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.COUNTRY)]),
- (BreakType.WORD, PhraseType.NONE, [(2, TokenType.PARTIAL)]))
+ q = make_query((qmod.BREAK_START, qmod.PHRASE_ANY, [(1, qmod.TOKEN_COUNTRY)]),
+ (qmod.BREAK_WORD, qmod.PHRASE_ANY, [(2, qmod.TOKEN_PARTIAL)]))
check_assignments(yield_token_assignments(q),
TokenAssignment(penalty=0.1, name=TokenRange(1, 2),
def test_country_at_end():
- q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.PARTIAL)]),
- (BreakType.WORD, PhraseType.NONE, [(2, TokenType.COUNTRY)]))
+ q = make_query((qmod.BREAK_START, qmod.PHRASE_ANY, [(1, qmod.TOKEN_PARTIAL)]),
+ (qmod.BREAK_WORD, qmod.PHRASE_ANY, [(2, qmod.TOKEN_COUNTRY)]))
check_assignments(yield_token_assignments(q),
TokenAssignment(penalty=0.1, name=TokenRange(0, 1),
def test_country_in_middle():
- q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.PARTIAL)]),
- (BreakType.WORD, PhraseType.NONE, [(2, TokenType.COUNTRY)]),
- (BreakType.WORD, PhraseType.NONE, [(3, TokenType.PARTIAL)]))
+ q = make_query((qmod.BREAK_START, qmod.PHRASE_ANY, [(1, qmod.TOKEN_PARTIAL)]),
+ (qmod.BREAK_WORD, qmod.PHRASE_ANY, [(2, qmod.TOKEN_COUNTRY)]),
+ (qmod.BREAK_WORD, qmod.PHRASE_ANY, [(3, qmod.TOKEN_PARTIAL)]))
check_assignments(yield_token_assignments(q))
def test_postcode_with_designation():
- q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.POSTCODE)]),
- (BreakType.PHRASE, PhraseType.NONE, [(2, TokenType.PARTIAL)]))
+ q = make_query((qmod.BREAK_START, qmod.PHRASE_ANY, [(1, qmod.TOKEN_POSTCODE)]),
+ (qmod.BREAK_PHRASE, qmod.PHRASE_ANY, [(2, qmod.TOKEN_PARTIAL)]))
check_assignments(yield_token_assignments(q),
TokenAssignment(penalty=0.1, name=TokenRange(1, 2),
def test_postcode_with_designation_backwards():
- q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.PARTIAL)]),
- (BreakType.PHRASE, PhraseType.NONE, [(2, TokenType.POSTCODE)]))
+ q = make_query((qmod.BREAK_START, qmod.PHRASE_ANY, [(1, qmod.TOKEN_PARTIAL)]),
+ (qmod.BREAK_PHRASE, qmod.PHRASE_ANY, [(2, qmod.TOKEN_POSTCODE)]))
check_assignments(yield_token_assignments(q),
TokenAssignment(name=TokenRange(0, 1),
address=[TokenRange(0, 1)]))
-def test_category_at_beginning():
- q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.CATEGORY)]),
- (BreakType.WORD, PhraseType.NONE, [(2, TokenType.PARTIAL)]))
+def test_near_item_at_beginning():
+ q = make_query((qmod.BREAK_START, qmod.PHRASE_ANY, [(1, qmod.TOKEN_NEAR_ITEM)]),
+ (qmod.BREAK_WORD, qmod.PHRASE_ANY, [(2, qmod.TOKEN_PARTIAL)]))
check_assignments(yield_token_assignments(q),
TokenAssignment(penalty=0.1, name=TokenRange(1, 2),
- category=TokenRange(0, 1)))
+ near_item=TokenRange(0, 1)))
-def test_category_at_end():
- q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.PARTIAL)]),
- (BreakType.WORD, PhraseType.NONE, [(2, TokenType.CATEGORY)]))
+def test_near_item_at_end():
+ q = make_query((qmod.BREAK_START, qmod.PHRASE_ANY, [(1, qmod.TOKEN_PARTIAL)]),
+ (qmod.BREAK_WORD, qmod.PHRASE_ANY, [(2, qmod.TOKEN_NEAR_ITEM)]))
check_assignments(yield_token_assignments(q),
TokenAssignment(penalty=0.1, name=TokenRange(0, 1),
- category=TokenRange(1, 2)))
+ near_item=TokenRange(1, 2)))
-def test_category_in_middle():
- q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.PARTIAL)]),
- (BreakType.WORD, PhraseType.NONE, [(2, TokenType.CATEGORY)]),
- (BreakType.WORD, PhraseType.NONE, [(3, TokenType.PARTIAL)]))
+def test_near_item_in_middle():
+ q = make_query((qmod.BREAK_START, qmod.PHRASE_ANY, [(1, qmod.TOKEN_PARTIAL)]),
+ (qmod.BREAK_WORD, qmod.PHRASE_ANY, [(2, qmod.TOKEN_NEAR_ITEM)]),
+ (qmod.BREAK_WORD, qmod.PHRASE_ANY, [(3, qmod.TOKEN_PARTIAL)]))
check_assignments(yield_token_assignments(q))
def test_qualifier_at_beginning():
- q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.QUALIFIER)]),
- (BreakType.WORD, PhraseType.NONE, [(2, TokenType.PARTIAL)]),
- (BreakType.WORD, PhraseType.NONE, [(3, TokenType.PARTIAL)]))
+ q = make_query((qmod.BREAK_START, qmod.PHRASE_ANY, [(1, qmod.TOKEN_QUALIFIER)]),
+ (qmod.BREAK_WORD, qmod.PHRASE_ANY, [(2, qmod.TOKEN_PARTIAL)]),
+ (qmod.BREAK_WORD, qmod.PHRASE_ANY, [(3, qmod.TOKEN_PARTIAL)]))
check_assignments(yield_token_assignments(q),
def test_qualifier_after_name():
- q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.PARTIAL)]),
- (BreakType.WORD, PhraseType.NONE, [(2, TokenType.PARTIAL)]),
- (BreakType.WORD, PhraseType.NONE, [(3, TokenType.QUALIFIER)]),
- (BreakType.WORD, PhraseType.NONE, [(4, TokenType.PARTIAL)]),
- (BreakType.WORD, PhraseType.NONE, [(5, TokenType.PARTIAL)]))
+ q = make_query((qmod.BREAK_START, qmod.PHRASE_ANY, [(1, qmod.TOKEN_PARTIAL)]),
+ (qmod.BREAK_WORD, qmod.PHRASE_ANY, [(2, qmod.TOKEN_PARTIAL)]),
+ (qmod.BREAK_WORD, qmod.PHRASE_ANY, [(3, qmod.TOKEN_QUALIFIER)]),
+ (qmod.BREAK_WORD, qmod.PHRASE_ANY, [(4, qmod.TOKEN_PARTIAL)]),
+ (qmod.BREAK_WORD, qmod.PHRASE_ANY, [(5, qmod.TOKEN_PARTIAL)]))
check_assignments(yield_token_assignments(q),
def test_qualifier_before_housenumber():
- q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.QUALIFIER)]),
- (BreakType.WORD, PhraseType.NONE, [(2, TokenType.HOUSENUMBER)]),
- (BreakType.WORD, PhraseType.NONE, [(3, TokenType.PARTIAL)]))
+ q = make_query((qmod.BREAK_START, qmod.PHRASE_ANY, [(1, qmod.TOKEN_QUALIFIER)]),
+ (qmod.BREAK_WORD, qmod.PHRASE_ANY, [(2, qmod.TOKEN_HOUSENUMBER)]),
+ (qmod.BREAK_WORD, qmod.PHRASE_ANY, [(3, qmod.TOKEN_PARTIAL)]))
check_assignments(yield_token_assignments(q))
def test_qualifier_after_housenumber():
- q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.HOUSENUMBER)]),
- (BreakType.WORD, PhraseType.NONE, [(2, TokenType.QUALIFIER)]),
- (BreakType.WORD, PhraseType.NONE, [(3, TokenType.PARTIAL)]))
+ q = make_query((qmod.BREAK_START, qmod.PHRASE_ANY, [(1, qmod.TOKEN_HOUSENUMBER)]),
+ (qmod.BREAK_WORD, qmod.PHRASE_ANY, [(2, qmod.TOKEN_QUALIFIER)]),
+ (qmod.BREAK_WORD, qmod.PHRASE_ANY, [(3, qmod.TOKEN_PARTIAL)]))
+
+ check_assignments(yield_token_assignments(q))
+
+
+def test_qualifier_in_middle_of_phrase():
+ q = make_query((qmod.BREAK_START, qmod.PHRASE_ANY, [(1, qmod.TOKEN_PARTIAL)]),
+ (qmod.BREAK_PHRASE, qmod.PHRASE_ANY, [(2, qmod.TOKEN_PARTIAL)]),
+ (qmod.BREAK_WORD, qmod.PHRASE_ANY, [(3, qmod.TOKEN_QUALIFIER)]),
+ (qmod.BREAK_WORD, qmod.PHRASE_ANY, [(4, qmod.TOKEN_PARTIAL)]),
+ (qmod.BREAK_PHRASE, qmod.PHRASE_ANY, [(5, qmod.TOKEN_PARTIAL)]))
check_assignments(yield_token_assignments(q))
+
projects / nominatim.git / blobdiff