replace BreakType enum with simple char constants

[nominatim.git] / test / python / api / search / test_token_assignment.py

diff --git a/test/python/api/search/test_token_assignment.py b/test/python/api/search/test_token_assignment.py
index 8cbcccb90d675f512f50fe9b2e8699623e025be3..8af23d8962b4bd0a88f40439a302f1cc2c88a098 100644 (file)
--- a/test/python/api/search/test_token_assignment.py
+++ b/test/python/api/search/test_token_assignment.py
@@ -2,15 +2,16 @@
 #
 # This file is part of Nominatim. (https://nominatim.org)
 #
 #
 # This file is part of Nominatim. (https://nominatim.org)
 #

-# Copyright (C) 2023 by the Nominatim developer community.
+# Copyright (C) 2024 by the Nominatim developer community.

 # For a full list of authors see the git log.
 """
 Test for creation of token assignments from tokenized queries.
 """
 import pytest
 
 # For a full list of authors see the git log.
 """
 Test for creation of token assignments from tokenized queries.
 """
 import pytest
 

-from nominatim.api.search.query import QueryStruct, Phrase, PhraseType, BreakType, TokenType, TokenRange, Token
-from nominatim.api.search.token_assignment import yield_token_assignments, TokenAssignment, PENALTY_TOKENCHANGE
+from nominatim_api.search.query import QueryStruct, Phrase, PhraseType, TokenType, TokenRange, Token
+import nominatim_api.search.query as qmod
+from nominatim_api.search.token_assignment import yield_token_assignments, TokenAssignment, PENALTY_TOKENCHANGE

 
 class MyToken(Token):
     def get_category(self):
 
 class MyToken(Token):
     def get_category(self):


@@ -18,20 +19,17 @@ class MyToken(Token):
 
 
 def make_query(*args):
 
 
 def make_query(*args):

-    q = None
-    dummy = MyToken(3.0, 45, 1, 'foo', True)
+    q = QueryStruct([Phrase(args[0][1], '')])
+    dummy = MyToken(penalty=3.0, token=45, count=1, addr_count=1,
+                    lookup_word='foo')

 
 

-    for btype, ptype, tlist in args:
-        if q is None:
-            q = QueryStruct([Phrase(ptype, '')])
-        else:
-            q.add_node(btype, ptype)
+    for btype, ptype, _ in args[1:]:
+        q.add_node(btype, ptype)
+    q.add_node(qmod.BREAK_END, PhraseType.NONE)

 
 

-        start = len(q.nodes) - 1
-        for end, ttype in tlist:
-            q.add_token(TokenRange(start, end), ttype, [dummy])
-
-    q.add_node(BreakType.END, PhraseType.NONE)
+    for start, t in enumerate(args):
+        for end, ttype in t[2]:
+            q.add_token(TokenRange(start, end), ttype, dummy)

 
     return q
 
 
     return q
 


@@ -47,13 +45,13 @@ def check_assignments(actual, *expected):
 
 def test_query_with_missing_tokens():
     q = QueryStruct([Phrase(PhraseType.NONE, '')])
 
 def test_query_with_missing_tokens():
     q = QueryStruct([Phrase(PhraseType.NONE, '')])

-    q.add_node(BreakType.END, PhraseType.NONE)
+    q.add_node(qmod.BREAK_END, PhraseType.NONE)

 
     assert list(yield_token_assignments(q)) == []
 
 
 def test_one_word_query():
 
     assert list(yield_token_assignments(q)) == []
 
 
 def test_one_word_query():

-    q = make_query((BreakType.START, PhraseType.NONE,
+    q = make_query((qmod.BREAK_START, PhraseType.NONE,

                     [(1, TokenType.PARTIAL),
                      (1, TokenType.WORD),
                      (1, TokenType.HOUSENUMBER)]))
                     [(1, TokenType.PARTIAL),
                      (1, TokenType.WORD),
                      (1, TokenType.HOUSENUMBER)]))


@@ -63,7 +61,7 @@ def test_one_word_query():
 
 
 def test_single_postcode():
 
 
 def test_single_postcode():

-    q = make_query((BreakType.START, PhraseType.NONE,
+    q = make_query((qmod.BREAK_START, PhraseType.NONE,

                     [(1, TokenType.POSTCODE)]))
 
     res = list(yield_token_assignments(q))
                     [(1, TokenType.POSTCODE)]))
 
     res = list(yield_token_assignments(q))


@@ -71,7 +69,7 @@ def test_single_postcode():
 
 
 def test_single_country_name():
 
 
 def test_single_country_name():

-    q = make_query((BreakType.START, PhraseType.NONE,
+    q = make_query((qmod.BREAK_START, PhraseType.NONE,

                     [(1, TokenType.COUNTRY)]))
 
     res = list(yield_token_assignments(q))
                     [(1, TokenType.COUNTRY)]))
 
     res = list(yield_token_assignments(q))


@@ -79,17 +77,17 @@ def test_single_country_name():
 
 
 def test_single_word_poi_search():
 
 
 def test_single_word_poi_search():

-    q = make_query((BreakType.START, PhraseType.NONE,
-                    [(1, TokenType.CATEGORY),
+    q = make_query((qmod.BREAK_START, PhraseType.NONE,
+                    [(1, TokenType.NEAR_ITEM),

                      (1, TokenType.QUALIFIER)]))
 
     res = list(yield_token_assignments(q))
                      (1, TokenType.QUALIFIER)]))
 
     res = list(yield_token_assignments(q))

-    assert res == [TokenAssignment(category=TokenRange(0, 1))]
+    assert res == [TokenAssignment(near_item=TokenRange(0, 1))]

 
 
 
 

-@pytest.mark.parametrize('btype', [BreakType.WORD, BreakType.PART, BreakType.TOKEN])
+@pytest.mark.parametrize('btype', [qmod.BREAK_WORD, qmod.BREAK_PART, qmod.BREAK_TOKEN])

 def test_multiple_simple_words(btype):
 def test_multiple_simple_words(btype):

-    q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.PARTIAL)]),
+    q = make_query((qmod.BREAK_START, PhraseType.NONE, [(1, TokenType.PARTIAL)]),

                    (btype, PhraseType.NONE, [(2, TokenType.PARTIAL)]),
                    (btype, PhraseType.NONE, [(3, TokenType.PARTIAL)]))
 
                    (btype, PhraseType.NONE, [(2, TokenType.PARTIAL)]),
                    (btype, PhraseType.NONE, [(3, TokenType.PARTIAL)]))
 


@@ -109,8 +107,8 @@ def test_multiple_simple_words(btype):
 
 
 def test_multiple_words_respect_phrase_break():
 
 
 def test_multiple_words_respect_phrase_break():

-    q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.PARTIAL)]),
-                   (BreakType.PHRASE, PhraseType.NONE, [(2, TokenType.PARTIAL)]))
+    q = make_query((qmod.BREAK_START, PhraseType.NONE, [(1, TokenType.PARTIAL)]),
+                   (qmod.BREAK_PHRASE, PhraseType.NONE, [(2, TokenType.PARTIAL)]))

 
     check_assignments(yield_token_assignments(q),
                       TokenAssignment(name=TokenRange(0, 1),
 
     check_assignments(yield_token_assignments(q),
                       TokenAssignment(name=TokenRange(0, 1),


@@ -120,95 +118,111 @@ def test_multiple_words_respect_phrase_break():
 
 
 def test_housenumber_and_street():
 
 
 def test_housenumber_and_street():

-    q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.HOUSENUMBER)]),
-                   (BreakType.PHRASE, PhraseType.NONE, [(2, TokenType.PARTIAL)]))
+    q = make_query((qmod.BREAK_START, PhraseType.NONE, [(1, TokenType.HOUSENUMBER)]),
+                   (qmod.BREAK_PHRASE, PhraseType.NONE, [(2, TokenType.PARTIAL)]))

 
     check_assignments(yield_token_assignments(q),
                       TokenAssignment(name=TokenRange(1, 2),
 
     check_assignments(yield_token_assignments(q),
                       TokenAssignment(name=TokenRange(1, 2),

+                                      housenumber=TokenRange(0, 1)),
+                      TokenAssignment(address=[TokenRange(1, 2)],

                                       housenumber=TokenRange(0, 1)))
 
 
 def test_housenumber_and_street_backwards():
                                       housenumber=TokenRange(0, 1)))
 
 
 def test_housenumber_and_street_backwards():

-    q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.PARTIAL)]),
-                   (BreakType.PHRASE, PhraseType.NONE, [(2, TokenType.HOUSENUMBER)]))
+    q = make_query((qmod.BREAK_START, PhraseType.NONE, [(1, TokenType.PARTIAL)]),
+                   (qmod.BREAK_PHRASE, PhraseType.NONE, [(2, TokenType.HOUSENUMBER)]))

 
     check_assignments(yield_token_assignments(q),
                       TokenAssignment(name=TokenRange(0, 1),
 
     check_assignments(yield_token_assignments(q),
                       TokenAssignment(name=TokenRange(0, 1),

+                                      housenumber=TokenRange(1, 2)),
+                      TokenAssignment(address=[TokenRange(0, 1)],

                                       housenumber=TokenRange(1, 2)))
 
 
 def test_housenumber_and_postcode():
                                       housenumber=TokenRange(1, 2)))
 
 
 def test_housenumber_and_postcode():

-    q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.PARTIAL)]),
-                   (BreakType.WORD, PhraseType.NONE, [(2, TokenType.HOUSENUMBER)]),
-                   (BreakType.WORD, PhraseType.NONE, [(3, TokenType.PARTIAL)]),
-                   (BreakType.WORD, PhraseType.NONE, [(4, TokenType.POSTCODE)]))
+    q = make_query((qmod.BREAK_START, PhraseType.NONE, [(1, TokenType.PARTIAL)]),
+                   (qmod.BREAK_WORD, PhraseType.NONE, [(2, TokenType.HOUSENUMBER)]),
+                   (qmod.BREAK_WORD, PhraseType.NONE, [(3, TokenType.PARTIAL)]),
+                   (qmod.BREAK_WORD, PhraseType.NONE, [(4, TokenType.POSTCODE)]))

 
     check_assignments(yield_token_assignments(q),
                       TokenAssignment(penalty=pytest.approx(0.3),
                                       name=TokenRange(0, 1),
                                       housenumber=TokenRange(1, 2),
                                       address=[TokenRange(2, 3)],
 
     check_assignments(yield_token_assignments(q),
                       TokenAssignment(penalty=pytest.approx(0.3),
                                       name=TokenRange(0, 1),
                                       housenumber=TokenRange(1, 2),
                                       address=[TokenRange(2, 3)],

+                                      postcode=TokenRange(3, 4)),
+                      TokenAssignment(penalty=pytest.approx(0.3),
+                                      housenumber=TokenRange(1, 2),
+                                      address=[TokenRange(0, 1), TokenRange(2, 3)],

                                       postcode=TokenRange(3, 4)))
 
 def test_postcode_and_housenumber():
                                       postcode=TokenRange(3, 4)))
 
 def test_postcode_and_housenumber():

-    q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.PARTIAL)]),
-                   (BreakType.WORD, PhraseType.NONE, [(2, TokenType.POSTCODE)]),
-                   (BreakType.WORD, PhraseType.NONE, [(3, TokenType.PARTIAL)]),
-                   (BreakType.WORD, PhraseType.NONE, [(4, TokenType.HOUSENUMBER)]))
+    q = make_query((qmod.BREAK_START, PhraseType.NONE, [(1, TokenType.PARTIAL)]),
+                   (qmod.BREAK_WORD, PhraseType.NONE, [(2, TokenType.POSTCODE)]),
+                   (qmod.BREAK_WORD, PhraseType.NONE, [(3, TokenType.PARTIAL)]),
+                   (qmod.BREAK_WORD, PhraseType.NONE, [(4, TokenType.HOUSENUMBER)]))

 
     check_assignments(yield_token_assignments(q),
                       TokenAssignment(penalty=pytest.approx(0.3),
                                       name=TokenRange(2, 3),
                                       housenumber=TokenRange(3, 4),
                                       address=[TokenRange(0, 1)],
 
     check_assignments(yield_token_assignments(q),
                       TokenAssignment(penalty=pytest.approx(0.3),
                                       name=TokenRange(2, 3),
                                       housenumber=TokenRange(3, 4),
                                       address=[TokenRange(0, 1)],

+                                      postcode=TokenRange(1, 2)),
+                      TokenAssignment(penalty=pytest.approx(0.3),
+                                      housenumber=TokenRange(3, 4),
+                                      address=[TokenRange(0, 1), TokenRange(2, 3)],

                                       postcode=TokenRange(1, 2)))
 
 
 def test_country_housenumber_postcode():
                                       postcode=TokenRange(1, 2)))
 
 
 def test_country_housenumber_postcode():

-    q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.COUNTRY)]),
-                   (BreakType.WORD, PhraseType.NONE, [(2, TokenType.PARTIAL)]),
-                   (BreakType.WORD, PhraseType.NONE, [(3, TokenType.HOUSENUMBER)]),
-                   (BreakType.WORD, PhraseType.NONE, [(4, TokenType.POSTCODE)]))
+    q = make_query((qmod.BREAK_START, PhraseType.NONE, [(1, TokenType.COUNTRY)]),
+                   (qmod.BREAK_WORD, PhraseType.NONE, [(2, TokenType.PARTIAL)]),
+                   (qmod.BREAK_WORD, PhraseType.NONE, [(3, TokenType.HOUSENUMBER)]),
+                   (qmod.BREAK_WORD, PhraseType.NONE, [(4, TokenType.POSTCODE)]))

 
     check_assignments(yield_token_assignments(q))
 
 
 @pytest.mark.parametrize('ttype', [TokenType.POSTCODE, TokenType.COUNTRY,
 
     check_assignments(yield_token_assignments(q))
 
 
 @pytest.mark.parametrize('ttype', [TokenType.POSTCODE, TokenType.COUNTRY,

-                                   TokenType.CATEGORY, TokenType.QUALIFIER])
+                                   TokenType.NEAR_ITEM, TokenType.QUALIFIER])

 def test_housenumber_with_only_special_terms(ttype):
 def test_housenumber_with_only_special_terms(ttype):

-    q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.HOUSENUMBER)]),
-                   (BreakType.WORD, PhraseType.NONE, [(2, ttype)]))
+    q = make_query((qmod.BREAK_START, PhraseType.NONE, [(1, TokenType.HOUSENUMBER)]),
+                   (qmod.BREAK_WORD, PhraseType.NONE, [(2, ttype)]))

 
     check_assignments(yield_token_assignments(q))
 
 
 @pytest.mark.parametrize('ttype', [TokenType.POSTCODE, TokenType.HOUSENUMBER, TokenType.COUNTRY])
 def test_multiple_special_tokens(ttype):
 
     check_assignments(yield_token_assignments(q))
 
 
 @pytest.mark.parametrize('ttype', [TokenType.POSTCODE, TokenType.HOUSENUMBER, TokenType.COUNTRY])
 def test_multiple_special_tokens(ttype):

-    q = make_query((BreakType.START, PhraseType.NONE, [(1, ttype)]),
-                   (BreakType.PHRASE, PhraseType.NONE, [(2, TokenType.PARTIAL)]),
-                   (BreakType.PHRASE, PhraseType.NONE, [(3, ttype)]))
+    q = make_query((qmod.BREAK_START, PhraseType.NONE, [(1, ttype)]),
+                   (qmod.BREAK_PHRASE, PhraseType.NONE, [(2, TokenType.PARTIAL)]),
+                   (qmod.BREAK_PHRASE, PhraseType.NONE, [(3, ttype)]))

 
     check_assignments(yield_token_assignments(q))
 
 
 def test_housenumber_many_phrases():
 
     check_assignments(yield_token_assignments(q))
 
 
 def test_housenumber_many_phrases():

-    q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.PARTIAL)]),
-                   (BreakType.PHRASE, PhraseType.NONE, [(2, TokenType.PARTIAL)]),
-                   (BreakType.PHRASE, PhraseType.NONE, [(3, TokenType.PARTIAL)]),
-                   (BreakType.PHRASE, PhraseType.NONE, [(4, TokenType.HOUSENUMBER)]),
-                   (BreakType.WORD, PhraseType.NONE, [(5, TokenType.PARTIAL)]))
+    q = make_query((qmod.BREAK_START, PhraseType.NONE, [(1, TokenType.PARTIAL)]),
+                   (qmod.BREAK_PHRASE, PhraseType.NONE, [(2, TokenType.PARTIAL)]),
+                   (qmod.BREAK_PHRASE, PhraseType.NONE, [(3, TokenType.PARTIAL)]),
+                   (qmod.BREAK_PHRASE, PhraseType.NONE, [(4, TokenType.HOUSENUMBER)]),
+                   (qmod.BREAK_WORD, PhraseType.NONE, [(5, TokenType.PARTIAL)]))

 
     check_assignments(yield_token_assignments(q),
                       TokenAssignment(penalty=0.1,
                                       name=TokenRange(4, 5),
                                       housenumber=TokenRange(3, 4),\
                                       address=[TokenRange(0, 1), TokenRange(1, 2),
 
     check_assignments(yield_token_assignments(q),
                       TokenAssignment(penalty=0.1,
                                       name=TokenRange(4, 5),
                                       housenumber=TokenRange(3, 4),\
                                       address=[TokenRange(0, 1), TokenRange(1, 2),

-                                               TokenRange(2, 3)]))
+                                               TokenRange(2, 3)]),
+                      TokenAssignment(penalty=0.1,
+                                      housenumber=TokenRange(3, 4),\
+                                      address=[TokenRange(0, 1), TokenRange(1, 2),
+                                               TokenRange(2, 3), TokenRange(4, 5)]))

 
 
 def test_country_at_beginning():
 
 
 def test_country_at_beginning():

-    q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.COUNTRY)]),
-                   (BreakType.WORD, PhraseType.NONE, [(2, TokenType.PARTIAL)]))
+    q = make_query((qmod.BREAK_START, PhraseType.NONE, [(1, TokenType.COUNTRY)]),
+                   (qmod.BREAK_WORD, PhraseType.NONE, [(2, TokenType.PARTIAL)]))

 
     check_assignments(yield_token_assignments(q),
                       TokenAssignment(penalty=0.1, name=TokenRange(1, 2),
 
     check_assignments(yield_token_assignments(q),
                       TokenAssignment(penalty=0.1, name=TokenRange(1, 2),


@@ -216,8 +230,8 @@ def test_country_at_beginning():
 
 
 def test_country_at_end():
 
 
 def test_country_at_end():

-    q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.PARTIAL)]),
-                   (BreakType.WORD, PhraseType.NONE, [(2, TokenType.COUNTRY)]))
+    q = make_query((qmod.BREAK_START, PhraseType.NONE, [(1, TokenType.PARTIAL)]),
+                   (qmod.BREAK_WORD, PhraseType.NONE, [(2, TokenType.COUNTRY)]))

 
     check_assignments(yield_token_assignments(q),
                       TokenAssignment(penalty=0.1, name=TokenRange(0, 1),
 
     check_assignments(yield_token_assignments(q),
                       TokenAssignment(penalty=0.1, name=TokenRange(0, 1),


@@ -225,65 +239,65 @@ def test_country_at_end():
 
 
 def test_country_in_middle():
 
 
 def test_country_in_middle():

-    q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.PARTIAL)]),
-                   (BreakType.WORD, PhraseType.NONE, [(2, TokenType.COUNTRY)]),
-                   (BreakType.WORD, PhraseType.NONE, [(3, TokenType.PARTIAL)]))
+    q = make_query((qmod.BREAK_START, PhraseType.NONE, [(1, TokenType.PARTIAL)]),
+                   (qmod.BREAK_WORD, PhraseType.NONE, [(2, TokenType.COUNTRY)]),
+                   (qmod.BREAK_WORD, PhraseType.NONE, [(3, TokenType.PARTIAL)]))

 
     check_assignments(yield_token_assignments(q))
 
 
 def test_postcode_with_designation():
 
     check_assignments(yield_token_assignments(q))
 
 
 def test_postcode_with_designation():

-    q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.POSTCODE)]),
-                   (BreakType.PHRASE, PhraseType.NONE, [(2, TokenType.PARTIAL)]))
+    q = make_query((qmod.BREAK_START, PhraseType.NONE, [(1, TokenType.POSTCODE)]),
+                   (qmod.BREAK_PHRASE, PhraseType.NONE, [(2, TokenType.PARTIAL)]))

 
     check_assignments(yield_token_assignments(q),
 
     check_assignments(yield_token_assignments(q),

-                      TokenAssignment(name=TokenRange(1, 2),
+                      TokenAssignment(penalty=0.1, name=TokenRange(1, 2),

                                       postcode=TokenRange(0, 1)),
                       TokenAssignment(postcode=TokenRange(0, 1),
                                       address=[TokenRange(1, 2)]))
 
 
 def test_postcode_with_designation_backwards():
                                       postcode=TokenRange(0, 1)),
                       TokenAssignment(postcode=TokenRange(0, 1),
                                       address=[TokenRange(1, 2)]))
 
 
 def test_postcode_with_designation_backwards():

-    q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.PARTIAL)]),
-                   (BreakType.PHRASE, PhraseType.NONE, [(2, TokenType.POSTCODE)]))
+    q = make_query((qmod.BREAK_START, PhraseType.NONE, [(1, TokenType.PARTIAL)]),
+                   (qmod.BREAK_PHRASE, PhraseType.NONE, [(2, TokenType.POSTCODE)]))

 
     check_assignments(yield_token_assignments(q),
                       TokenAssignment(name=TokenRange(0, 1),
                                       postcode=TokenRange(1, 2)),
 
     check_assignments(yield_token_assignments(q),
                       TokenAssignment(name=TokenRange(0, 1),
                                       postcode=TokenRange(1, 2)),

-                      TokenAssignment(postcode=TokenRange(1, 2),
+                      TokenAssignment(penalty=0.1, postcode=TokenRange(1, 2),

                                       address=[TokenRange(0, 1)]))
 
 
                                       address=[TokenRange(0, 1)]))
 
 

-def test_category_at_beginning():
-    q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.CATEGORY)]),
-                   (BreakType.WORD, PhraseType.NONE, [(2, TokenType.PARTIAL)]))
+def test_near_item_at_beginning():
+    q = make_query((qmod.BREAK_START, PhraseType.NONE, [(1, TokenType.NEAR_ITEM)]),
+                   (qmod.BREAK_WORD, PhraseType.NONE, [(2, TokenType.PARTIAL)]))

 
     check_assignments(yield_token_assignments(q),
                       TokenAssignment(penalty=0.1, name=TokenRange(1, 2),
 
     check_assignments(yield_token_assignments(q),
                       TokenAssignment(penalty=0.1, name=TokenRange(1, 2),

-                                      category=TokenRange(0, 1)))
+                                      near_item=TokenRange(0, 1)))

 
 
 
 

-def test_category_at_end():
-    q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.PARTIAL)]),
-                   (BreakType.WORD, PhraseType.NONE, [(2, TokenType.CATEGORY)]))
+def test_near_item_at_end():
+    q = make_query((qmod.BREAK_START, PhraseType.NONE, [(1, TokenType.PARTIAL)]),
+                   (qmod.BREAK_WORD, PhraseType.NONE, [(2, TokenType.NEAR_ITEM)]))

 
     check_assignments(yield_token_assignments(q),
                       TokenAssignment(penalty=0.1, name=TokenRange(0, 1),
 
     check_assignments(yield_token_assignments(q),
                       TokenAssignment(penalty=0.1, name=TokenRange(0, 1),

-                                      category=TokenRange(1, 2)))
+                                      near_item=TokenRange(1, 2)))

 
 
 
 

-def test_category_in_middle():
-    q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.PARTIAL)]),
-                   (BreakType.WORD, PhraseType.NONE, [(2, TokenType.CATEGORY)]),
-                   (BreakType.WORD, PhraseType.NONE, [(3, TokenType.PARTIAL)]))
+def test_near_item_in_middle():
+    q = make_query((qmod.BREAK_START, PhraseType.NONE, [(1, TokenType.PARTIAL)]),
+                   (qmod.BREAK_WORD, PhraseType.NONE, [(2, TokenType.NEAR_ITEM)]),
+                   (qmod.BREAK_WORD, PhraseType.NONE, [(3, TokenType.PARTIAL)]))

 
     check_assignments(yield_token_assignments(q))
 
 
 def test_qualifier_at_beginning():
 
     check_assignments(yield_token_assignments(q))
 
 
 def test_qualifier_at_beginning():

-    q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.QUALIFIER)]),
-                   (BreakType.WORD, PhraseType.NONE, [(2, TokenType.PARTIAL)]),
-                   (BreakType.WORD, PhraseType.NONE, [(3, TokenType.PARTIAL)]))
+    q = make_query((qmod.BREAK_START, PhraseType.NONE, [(1, TokenType.QUALIFIER)]),
+                   (qmod.BREAK_WORD, PhraseType.NONE, [(2, TokenType.PARTIAL)]),
+                   (qmod.BREAK_WORD, PhraseType.NONE, [(3, TokenType.PARTIAL)]))

 
 
     check_assignments(yield_token_assignments(q),
 
 
     check_assignments(yield_token_assignments(q),


@@ -295,11 +309,11 @@ def test_qualifier_at_beginning():
 
 
 def test_qualifier_after_name():
 
 
 def test_qualifier_after_name():

-    q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.PARTIAL)]),
-                   (BreakType.WORD, PhraseType.NONE, [(2, TokenType.PARTIAL)]),
-                   (BreakType.WORD, PhraseType.NONE, [(3, TokenType.QUALIFIER)]),
-                   (BreakType.WORD, PhraseType.NONE, [(4, TokenType.PARTIAL)]),
-                   (BreakType.WORD, PhraseType.NONE, [(5, TokenType.PARTIAL)]))
+    q = make_query((qmod.BREAK_START, PhraseType.NONE, [(1, TokenType.PARTIAL)]),
+                   (qmod.BREAK_WORD, PhraseType.NONE, [(2, TokenType.PARTIAL)]),
+                   (qmod.BREAK_WORD, PhraseType.NONE, [(3, TokenType.QUALIFIER)]),
+                   (qmod.BREAK_WORD, PhraseType.NONE, [(4, TokenType.PARTIAL)]),
+                   (qmod.BREAK_WORD, PhraseType.NONE, [(5, TokenType.PARTIAL)]))

 
 
     check_assignments(yield_token_assignments(q),
 
 
     check_assignments(yield_token_assignments(q),


@@ -312,16 +326,27 @@ def test_qualifier_after_name():
 
 
 def test_qualifier_before_housenumber():
 
 
 def test_qualifier_before_housenumber():

-    q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.QUALIFIER)]),
-                   (BreakType.WORD, PhraseType.NONE, [(2, TokenType.HOUSENUMBER)]),
-                   (BreakType.WORD, PhraseType.NONE, [(3, TokenType.PARTIAL)]))
+    q = make_query((qmod.BREAK_START, PhraseType.NONE, [(1, TokenType.QUALIFIER)]),
+                   (qmod.BREAK_WORD, PhraseType.NONE, [(2, TokenType.HOUSENUMBER)]),
+                   (qmod.BREAK_WORD, PhraseType.NONE, [(3, TokenType.PARTIAL)]))

 
     check_assignments(yield_token_assignments(q))
 
 
 def test_qualifier_after_housenumber():
 
     check_assignments(yield_token_assignments(q))
 
 
 def test_qualifier_after_housenumber():

-    q = make_query((BreakType.START, PhraseType.NONE, [(1, TokenType.HOUSENUMBER)]),
-                   (BreakType.WORD, PhraseType.NONE, [(2, TokenType.QUALIFIER)]),
-                   (BreakType.WORD, PhraseType.NONE, [(3, TokenType.PARTIAL)]))
+    q = make_query((qmod.BREAK_START, PhraseType.NONE, [(1, TokenType.HOUSENUMBER)]),
+                   (qmod.BREAK_WORD, PhraseType.NONE, [(2, TokenType.QUALIFIER)]),
+                   (qmod.BREAK_WORD, PhraseType.NONE, [(3, TokenType.PARTIAL)]))
+
+    check_assignments(yield_token_assignments(q))
+
+
+def test_qualifier_in_middle_of_phrase():
+    q = make_query((qmod.BREAK_START, PhraseType.NONE, [(1, TokenType.PARTIAL)]),
+                   (qmod.BREAK_PHRASE, PhraseType.NONE, [(2, TokenType.PARTIAL)]),
+                   (qmod.BREAK_WORD, PhraseType.NONE, [(3, TokenType.QUALIFIER)]),
+                   (qmod.BREAK_WORD, PhraseType.NONE, [(4, TokenType.PARTIAL)]),
+                   (qmod.BREAK_PHRASE, PhraseType.NONE, [(5, TokenType.PARTIAL)]))

 
     check_assignments(yield_token_assignments(q))
 
     check_assignments(yield_token_assignments(q))

+