X-Git-Url: https://git.openstreetmap.org./nominatim.git/blobdiff_plain/a413aae8a3962be4d623844d867604df68a9a211..4577669213ea392fa7e25a2fce444f387763f4c8:/test/python/api/search/test_token_assignment.py?ds=sidebyside diff --git a/test/python/api/search/test_token_assignment.py b/test/python/api/search/test_token_assignment.py index f78d5430..8af23d89 100644 --- a/test/python/api/search/test_token_assignment.py +++ b/test/python/api/search/test_token_assignment.py @@ -2,15 +2,16 @@ # # This file is part of Nominatim. (https://nominatim.org) # -# Copyright (C) 2023 by the Nominatim developer community. +# Copyright (C) 2024 by the Nominatim developer community. # For a full list of authors see the git log. """ Test for creation of token assignments from tokenized queries. """ import pytest -from nominatim.api.search.query import QueryStruct, Phrase, PhraseType, BreakType, TokenType, TokenRange, Token -from nominatim.api.search.token_assignment import yield_token_assignments, TokenAssignment, PENALTY_TOKENCHANGE +from nominatim_api.search.query import QueryStruct, Phrase, PhraseType, TokenType, TokenRange, Token +import nominatim_api.search.query as qmod +from nominatim_api.search.token_assignment import yield_token_assignments, TokenAssignment, PENALTY_TOKENCHANGE class MyToken(Token): def get_category(self): @@ -18,21 +19,18 @@ class MyToken(Token): def make_query(*args): - q = None - dummy = MyToken(3.0, 45, 1, 'foo', True) + q = QueryStruct([Phrase(args[0][1], '')]) + dummy = MyToken(penalty=3.0, token=45, count=1, addr_count=1, + lookup_word='foo') - for btype, ptype, tlist in args: - if q is None: - q = QueryStruct([Phrase(ptype, '')]) - else: - q.add_node(btype, ptype) + for btype, ptype, _ in args[1:]: + q.add_node(btype, ptype) + q.add_node(qmod.BREAK_END, PhraseType.NONE) - start = len(q.nodes) - 1 - for end, ttype in tlist: + for start, t in enumerate(args): + for end, ttype in t[2]: q.add_token(TokenRange(start, end), ttype, dummy) - q.add_node(BreakType.END, PhraseType.NONE) - return q @@ -47,13 +45,13 @@ def check_assignments(actual, *expected): def test_query_with_missing_tokens(): q = QueryStruct([Phrase(PhraseType.NONE, '')]) - q.add_node(BreakType.END, PhraseType.NONE) + q.add_node(qmod.BREAK_END, PhraseType.NONE) assert list(yield_token_assignments(q)) == [] def test_one_word_query(): - q = make_query((BreakType.START, PhraseType.NONE, + q = make_query((qmod.BREAK_START, PhraseType.NONE, [(1, TokenType.PARTIAL), (1, TokenType.WORD), (1, TokenType.HOUSENUMBER)])) @@ -63,7 +61,7 @@ def test_one_word_query(): def test_single_postcode(): - q = make_query((BreakType.START, PhraseType.NONE, + q = make_query((qmod.BREAK_START, PhraseType.NONE, [(1, TokenType.POSTCODE)])) res = list(yield_token_assignments(q)) @@ -71,7 +69,7 @@ def test_single_postcode(): def test_single_country_name(): - q = make_query((BreakType.START, PhraseType.NONE, + q = make_query((qmod.BREAK_START, PhraseType.NONE, [(1, TokenType.COUNTRY)])) res = list(yield_token_assignments(q)) @@ -79,17 +77,17 @@ def test_single_country_name(): def test_single_word_poi_search(): - q = make_query((BreakType.START, PhraseType.NONE, - [(1, TokenType.CATEGORY), + q = make_query((qmod.BREAK_START, PhraseType.NONE, + [(1, TokenType.NEAR_ITEM), (1, TokenType.QUALIFIER)])) res = list(yield_token_assignments(q)) - assert res == [TokenAssignment(category=TokenRange(0, 1))] + assert res == [TokenAssignment(near_item=TokenRange(0, 1))] -@pytest.mark.parametrize('btype', [BreakType.WORD, BreakType.PART, BreakType.TOKEN]) +@pytest.mark.parametrize('btype', [qmod.BREAK_WORD, qmod.BREAK_PART, qmod.BREAK_TOKEN]) def test_multiple_simple_words(btype): - q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.PARTIAL)]), + q = make_query((qmod.BREAK_START, PhraseType.NONE, [(1, TokenType.PARTIAL)]), (btype, PhraseType.NONE, [(2, TokenType.PARTIAL)]), (btype, PhraseType.NONE, [(3, TokenType.PARTIAL)])) @@ -109,8 +107,8 @@ def test_multiple_simple_words(btype): def test_multiple_words_respect_phrase_break(): - q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.PARTIAL)]), - (BreakType.PHRASE, PhraseType.NONE, [(2, TokenType.PARTIAL)])) + q = make_query((qmod.BREAK_START, PhraseType.NONE, [(1, TokenType.PARTIAL)]), + (qmod.BREAK_PHRASE, PhraseType.NONE, [(2, TokenType.PARTIAL)])) check_assignments(yield_token_assignments(q), TokenAssignment(name=TokenRange(0, 1), @@ -120,8 +118,8 @@ def test_multiple_words_respect_phrase_break(): def test_housenumber_and_street(): - q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.HOUSENUMBER)]), - (BreakType.PHRASE, PhraseType.NONE, [(2, TokenType.PARTIAL)])) + q = make_query((qmod.BREAK_START, PhraseType.NONE, [(1, TokenType.HOUSENUMBER)]), + (qmod.BREAK_PHRASE, PhraseType.NONE, [(2, TokenType.PARTIAL)])) check_assignments(yield_token_assignments(q), TokenAssignment(name=TokenRange(1, 2), @@ -131,8 +129,8 @@ def test_housenumber_and_street(): def test_housenumber_and_street_backwards(): - q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.PARTIAL)]), - (BreakType.PHRASE, PhraseType.NONE, [(2, TokenType.HOUSENUMBER)])) + q = make_query((qmod.BREAK_START, PhraseType.NONE, [(1, TokenType.PARTIAL)]), + (qmod.BREAK_PHRASE, PhraseType.NONE, [(2, TokenType.HOUSENUMBER)])) check_assignments(yield_token_assignments(q), TokenAssignment(name=TokenRange(0, 1), @@ -142,10 +140,10 @@ def test_housenumber_and_street_backwards(): def test_housenumber_and_postcode(): - q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.PARTIAL)]), - (BreakType.WORD, PhraseType.NONE, [(2, TokenType.HOUSENUMBER)]), - (BreakType.WORD, PhraseType.NONE, [(3, TokenType.PARTIAL)]), - (BreakType.WORD, PhraseType.NONE, [(4, TokenType.POSTCODE)])) + q = make_query((qmod.BREAK_START, PhraseType.NONE, [(1, TokenType.PARTIAL)]), + (qmod.BREAK_WORD, PhraseType.NONE, [(2, TokenType.HOUSENUMBER)]), + (qmod.BREAK_WORD, PhraseType.NONE, [(3, TokenType.PARTIAL)]), + (qmod.BREAK_WORD, PhraseType.NONE, [(4, TokenType.POSTCODE)])) check_assignments(yield_token_assignments(q), TokenAssignment(penalty=pytest.approx(0.3), @@ -159,10 +157,10 @@ def test_housenumber_and_postcode(): postcode=TokenRange(3, 4))) def test_postcode_and_housenumber(): - q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.PARTIAL)]), - (BreakType.WORD, PhraseType.NONE, [(2, TokenType.POSTCODE)]), - (BreakType.WORD, PhraseType.NONE, [(3, TokenType.PARTIAL)]), - (BreakType.WORD, PhraseType.NONE, [(4, TokenType.HOUSENUMBER)])) + q = make_query((qmod.BREAK_START, PhraseType.NONE, [(1, TokenType.PARTIAL)]), + (qmod.BREAK_WORD, PhraseType.NONE, [(2, TokenType.POSTCODE)]), + (qmod.BREAK_WORD, PhraseType.NONE, [(3, TokenType.PARTIAL)]), + (qmod.BREAK_WORD, PhraseType.NONE, [(4, TokenType.HOUSENUMBER)])) check_assignments(yield_token_assignments(q), TokenAssignment(penalty=pytest.approx(0.3), @@ -177,38 +175,38 @@ def test_postcode_and_housenumber(): def test_country_housenumber_postcode(): - q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.COUNTRY)]), - (BreakType.WORD, PhraseType.NONE, [(2, TokenType.PARTIAL)]), - (BreakType.WORD, PhraseType.NONE, [(3, TokenType.HOUSENUMBER)]), - (BreakType.WORD, PhraseType.NONE, [(4, TokenType.POSTCODE)])) + q = make_query((qmod.BREAK_START, PhraseType.NONE, [(1, TokenType.COUNTRY)]), + (qmod.BREAK_WORD, PhraseType.NONE, [(2, TokenType.PARTIAL)]), + (qmod.BREAK_WORD, PhraseType.NONE, [(3, TokenType.HOUSENUMBER)]), + (qmod.BREAK_WORD, PhraseType.NONE, [(4, TokenType.POSTCODE)])) check_assignments(yield_token_assignments(q)) @pytest.mark.parametrize('ttype', [TokenType.POSTCODE, TokenType.COUNTRY, - TokenType.CATEGORY, TokenType.QUALIFIER]) + TokenType.NEAR_ITEM, TokenType.QUALIFIER]) def test_housenumber_with_only_special_terms(ttype): - q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.HOUSENUMBER)]), - (BreakType.WORD, PhraseType.NONE, [(2, ttype)])) + q = make_query((qmod.BREAK_START, PhraseType.NONE, [(1, TokenType.HOUSENUMBER)]), + (qmod.BREAK_WORD, PhraseType.NONE, [(2, ttype)])) check_assignments(yield_token_assignments(q)) @pytest.mark.parametrize('ttype', [TokenType.POSTCODE, TokenType.HOUSENUMBER, TokenType.COUNTRY]) def test_multiple_special_tokens(ttype): - q = make_query((BreakType.START, PhraseType.NONE, [(1, ttype)]), - (BreakType.PHRASE, PhraseType.NONE, [(2, TokenType.PARTIAL)]), - (BreakType.PHRASE, PhraseType.NONE, [(3, ttype)])) + q = make_query((qmod.BREAK_START, PhraseType.NONE, [(1, ttype)]), + (qmod.BREAK_PHRASE, PhraseType.NONE, [(2, TokenType.PARTIAL)]), + (qmod.BREAK_PHRASE, PhraseType.NONE, [(3, ttype)])) check_assignments(yield_token_assignments(q)) def test_housenumber_many_phrases(): - q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.PARTIAL)]), - (BreakType.PHRASE, PhraseType.NONE, [(2, TokenType.PARTIAL)]), - (BreakType.PHRASE, PhraseType.NONE, [(3, TokenType.PARTIAL)]), - (BreakType.PHRASE, PhraseType.NONE, [(4, TokenType.HOUSENUMBER)]), - (BreakType.WORD, PhraseType.NONE, [(5, TokenType.PARTIAL)])) + q = make_query((qmod.BREAK_START, PhraseType.NONE, [(1, TokenType.PARTIAL)]), + (qmod.BREAK_PHRASE, PhraseType.NONE, [(2, TokenType.PARTIAL)]), + (qmod.BREAK_PHRASE, PhraseType.NONE, [(3, TokenType.PARTIAL)]), + (qmod.BREAK_PHRASE, PhraseType.NONE, [(4, TokenType.HOUSENUMBER)]), + (qmod.BREAK_WORD, PhraseType.NONE, [(5, TokenType.PARTIAL)])) check_assignments(yield_token_assignments(q), TokenAssignment(penalty=0.1, @@ -223,8 +221,8 @@ def test_housenumber_many_phrases(): def test_country_at_beginning(): - q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.COUNTRY)]), - (BreakType.WORD, PhraseType.NONE, [(2, TokenType.PARTIAL)])) + q = make_query((qmod.BREAK_START, PhraseType.NONE, [(1, TokenType.COUNTRY)]), + (qmod.BREAK_WORD, PhraseType.NONE, [(2, TokenType.PARTIAL)])) check_assignments(yield_token_assignments(q), TokenAssignment(penalty=0.1, name=TokenRange(1, 2), @@ -232,8 +230,8 @@ def test_country_at_beginning(): def test_country_at_end(): - q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.PARTIAL)]), - (BreakType.WORD, PhraseType.NONE, [(2, TokenType.COUNTRY)])) + q = make_query((qmod.BREAK_START, PhraseType.NONE, [(1, TokenType.PARTIAL)]), + (qmod.BREAK_WORD, PhraseType.NONE, [(2, TokenType.COUNTRY)])) check_assignments(yield_token_assignments(q), TokenAssignment(penalty=0.1, name=TokenRange(0, 1), @@ -241,65 +239,65 @@ def test_country_at_end(): def test_country_in_middle(): - q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.PARTIAL)]), - (BreakType.WORD, PhraseType.NONE, [(2, TokenType.COUNTRY)]), - (BreakType.WORD, PhraseType.NONE, [(3, TokenType.PARTIAL)])) + q = make_query((qmod.BREAK_START, PhraseType.NONE, [(1, TokenType.PARTIAL)]), + (qmod.BREAK_WORD, PhraseType.NONE, [(2, TokenType.COUNTRY)]), + (qmod.BREAK_WORD, PhraseType.NONE, [(3, TokenType.PARTIAL)])) check_assignments(yield_token_assignments(q)) def test_postcode_with_designation(): - q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.POSTCODE)]), - (BreakType.PHRASE, PhraseType.NONE, [(2, TokenType.PARTIAL)])) + q = make_query((qmod.BREAK_START, PhraseType.NONE, [(1, TokenType.POSTCODE)]), + (qmod.BREAK_PHRASE, PhraseType.NONE, [(2, TokenType.PARTIAL)])) check_assignments(yield_token_assignments(q), - TokenAssignment(name=TokenRange(1, 2), + TokenAssignment(penalty=0.1, name=TokenRange(1, 2), postcode=TokenRange(0, 1)), TokenAssignment(postcode=TokenRange(0, 1), address=[TokenRange(1, 2)])) def test_postcode_with_designation_backwards(): - q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.PARTIAL)]), - (BreakType.PHRASE, PhraseType.NONE, [(2, TokenType.POSTCODE)])) + q = make_query((qmod.BREAK_START, PhraseType.NONE, [(1, TokenType.PARTIAL)]), + (qmod.BREAK_PHRASE, PhraseType.NONE, [(2, TokenType.POSTCODE)])) check_assignments(yield_token_assignments(q), TokenAssignment(name=TokenRange(0, 1), postcode=TokenRange(1, 2)), - TokenAssignment(postcode=TokenRange(1, 2), + TokenAssignment(penalty=0.1, postcode=TokenRange(1, 2), address=[TokenRange(0, 1)])) -def test_category_at_beginning(): - q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.CATEGORY)]), - (BreakType.WORD, PhraseType.NONE, [(2, TokenType.PARTIAL)])) +def test_near_item_at_beginning(): + q = make_query((qmod.BREAK_START, PhraseType.NONE, [(1, TokenType.NEAR_ITEM)]), + (qmod.BREAK_WORD, PhraseType.NONE, [(2, TokenType.PARTIAL)])) check_assignments(yield_token_assignments(q), TokenAssignment(penalty=0.1, name=TokenRange(1, 2), - category=TokenRange(0, 1))) + near_item=TokenRange(0, 1))) -def test_category_at_end(): - q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.PARTIAL)]), - (BreakType.WORD, PhraseType.NONE, [(2, TokenType.CATEGORY)])) +def test_near_item_at_end(): + q = make_query((qmod.BREAK_START, PhraseType.NONE, [(1, TokenType.PARTIAL)]), + (qmod.BREAK_WORD, PhraseType.NONE, [(2, TokenType.NEAR_ITEM)])) check_assignments(yield_token_assignments(q), TokenAssignment(penalty=0.1, name=TokenRange(0, 1), - category=TokenRange(1, 2))) + near_item=TokenRange(1, 2))) -def test_category_in_middle(): - q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.PARTIAL)]), - (BreakType.WORD, PhraseType.NONE, [(2, TokenType.CATEGORY)]), - (BreakType.WORD, PhraseType.NONE, [(3, TokenType.PARTIAL)])) +def test_near_item_in_middle(): + q = make_query((qmod.BREAK_START, PhraseType.NONE, [(1, TokenType.PARTIAL)]), + (qmod.BREAK_WORD, PhraseType.NONE, [(2, TokenType.NEAR_ITEM)]), + (qmod.BREAK_WORD, PhraseType.NONE, [(3, TokenType.PARTIAL)])) check_assignments(yield_token_assignments(q)) def test_qualifier_at_beginning(): - q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.QUALIFIER)]), - (BreakType.WORD, PhraseType.NONE, [(2, TokenType.PARTIAL)]), - (BreakType.WORD, PhraseType.NONE, [(3, TokenType.PARTIAL)])) + q = make_query((qmod.BREAK_START, PhraseType.NONE, [(1, TokenType.QUALIFIER)]), + (qmod.BREAK_WORD, PhraseType.NONE, [(2, TokenType.PARTIAL)]), + (qmod.BREAK_WORD, PhraseType.NONE, [(3, TokenType.PARTIAL)])) check_assignments(yield_token_assignments(q), @@ -311,11 +309,11 @@ def test_qualifier_at_beginning(): def test_qualifier_after_name(): - q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.PARTIAL)]), - (BreakType.WORD, PhraseType.NONE, [(2, TokenType.PARTIAL)]), - (BreakType.WORD, PhraseType.NONE, [(3, TokenType.QUALIFIER)]), - (BreakType.WORD, PhraseType.NONE, [(4, TokenType.PARTIAL)]), - (BreakType.WORD, PhraseType.NONE, [(5, TokenType.PARTIAL)])) + q = make_query((qmod.BREAK_START, PhraseType.NONE, [(1, TokenType.PARTIAL)]), + (qmod.BREAK_WORD, PhraseType.NONE, [(2, TokenType.PARTIAL)]), + (qmod.BREAK_WORD, PhraseType.NONE, [(3, TokenType.QUALIFIER)]), + (qmod.BREAK_WORD, PhraseType.NONE, [(4, TokenType.PARTIAL)]), + (qmod.BREAK_WORD, PhraseType.NONE, [(5, TokenType.PARTIAL)])) check_assignments(yield_token_assignments(q), @@ -328,16 +326,27 @@ def test_qualifier_after_name(): def test_qualifier_before_housenumber(): - q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.QUALIFIER)]), - (BreakType.WORD, PhraseType.NONE, [(2, TokenType.HOUSENUMBER)]), - (BreakType.WORD, PhraseType.NONE, [(3, TokenType.PARTIAL)])) + q = make_query((qmod.BREAK_START, PhraseType.NONE, [(1, TokenType.QUALIFIER)]), + (qmod.BREAK_WORD, PhraseType.NONE, [(2, TokenType.HOUSENUMBER)]), + (qmod.BREAK_WORD, PhraseType.NONE, [(3, TokenType.PARTIAL)])) check_assignments(yield_token_assignments(q)) def test_qualifier_after_housenumber(): - q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.HOUSENUMBER)]), - (BreakType.WORD, PhraseType.NONE, [(2, TokenType.QUALIFIER)]), - (BreakType.WORD, PhraseType.NONE, [(3, TokenType.PARTIAL)])) + q = make_query((qmod.BREAK_START, PhraseType.NONE, [(1, TokenType.HOUSENUMBER)]), + (qmod.BREAK_WORD, PhraseType.NONE, [(2, TokenType.QUALIFIER)]), + (qmod.BREAK_WORD, PhraseType.NONE, [(3, TokenType.PARTIAL)])) + + check_assignments(yield_token_assignments(q)) + + +def test_qualifier_in_middle_of_phrase(): + q = make_query((qmod.BREAK_START, PhraseType.NONE, [(1, TokenType.PARTIAL)]), + (qmod.BREAK_PHRASE, PhraseType.NONE, [(2, TokenType.PARTIAL)]), + (qmod.BREAK_WORD, PhraseType.NONE, [(3, TokenType.QUALIFIER)]), + (qmod.BREAK_WORD, PhraseType.NONE, [(4, TokenType.PARTIAL)]), + (qmod.BREAK_PHRASE, PhraseType.NONE, [(5, TokenType.PARTIAL)])) check_assignments(yield_token_assignments(q)) +