Merge pull request #3099 from lonvia/determine-place-address-from-tokenizer

Use information from tokenizer to determine street vs. place address

diff --git a/docs/develop/Tokenizers.md b/docs/develop/Tokenizers.md
index 273e65e2126381a65b2d27361d71269c2ec439c5..eb0d4ea2e513b030a9dd3e09262196e5875d83ed 100644 (file)
--- a/docs/develop/Tokenizers.md
+++ b/docs/develop/Tokenizers.md
@@ -189,6 +189,28 @@ a house number token text. If a place has multiple house numbers they must
 be listed with a semicolon as delimiter. Must be NULL when the place has no
 house numbers.
 
+```sql
+FUNCTION token_is_street_address(info JSONB) RETURNS BOOLEAN
+```
+
+Return true if this is an object that should be parented against a street.
+Only relevant for objects with address rank 30.
+
+```sql
+FUNCTION token_has_addr_street(info JSONB) RETURNS BOOLEAN
+```
+
+Return true if there are street names to match against for finding the
+parent of the object.
+
+
+```sql
+FUNCTION token_has_addr_place(info JSONB) RETURNS BOOLEAN
+```
+
+Return true if there are place names to match against for finding the
+parent of the object.
+
 ```sql
 FUNCTION token_matches_street(info JSONB, street_tokens INTEGER[]) RETURNS BOOLEAN
 ```

diff --git a/lib-sql/functions/placex_triggers.sql b/lib-sql/functions/placex_triggers.sql
index 99d2872f69a83d67a4e4958520478738a1e5c501..a252e9aacd1d4d9de035b4884b935ce92b850fe7 100644 (file)
--- a/lib-sql/functions/placex_triggers.sql
+++ b/lib-sql/functions/placex_triggers.sql
@@ -996,7 +996,7 @@ BEGIN
 
     {% if debug %}RAISE WARNING 'finding street for % %', NEW.osm_type, NEW.osm_id;{% endif %}
     NEW.parent_place_id := null;
-    is_place_address := coalesce(not NEW.address ? 'street' and NEW.address ? 'place', FALSE);
+    is_place_address := not token_is_street_address(NEW.token_info);
 
     -- We have to find our parent road.
     NEW.parent_place_id := find_parent_for_poi(NEW.osm_type, NEW.osm_id,
@@ -1013,7 +1013,7 @@ BEGIN
       SELECT p.country_code, p.postcode, p.name FROM placex p
        WHERE p.place_id = NEW.parent_place_id INTO location;
 
-      IF is_place_address THEN
+      IF is_place_address and NEW.address ? 'place' THEN
         -- Check if the addr:place tag is part of the parent name
         SELECT count(*) INTO i
           FROM svals(location.name) AS pname WHERE pname = NEW.address->'place';

diff --git a/lib-sql/tokenizer/icu_tokenizer.sql b/lib-sql/tokenizer/icu_tokenizer.sql
index 599d0eb089eaeff6b5be795734e3a9b139b97117..04fcedcb4370c4805e2bbb9a198cb6b626d509b4 100644 (file)
--- a/lib-sql/tokenizer/icu_tokenizer.sql
+++ b/lib-sql/tokenizer/icu_tokenizer.sql
@@ -41,10 +41,17 @@ AS $$
 $$ LANGUAGE SQL IMMUTABLE STRICT;
 
 
+CREATE OR REPLACE FUNCTION token_is_street_address(info JSONB)
+  RETURNS BOOLEAN
+AS $$
+  SELECT info->>'street' is not null or info->>'place' is null;
+$$ LANGUAGE SQL IMMUTABLE;
+
+
 CREATE OR REPLACE FUNCTION token_has_addr_street(info JSONB)
   RETURNS BOOLEAN
 AS $$
-  SELECT info->>'street' is not null;
+  SELECT info->>'street' is not null and info->>'street' != '{}';
 $$ LANGUAGE SQL IMMUTABLE;
 
 

diff --git a/lib-sql/tokenizer/legacy_tokenizer.sql b/lib-sql/tokenizer/legacy_tokenizer.sql
index 5826f74ac25392b3bff857c1c71f054028839ba2..8c8f56e1c264e937bf8617ab56b5e6e9d83325e2 100644 (file)
--- a/lib-sql/tokenizer/legacy_tokenizer.sql
+++ b/lib-sql/tokenizer/legacy_tokenizer.sql
@@ -41,10 +41,17 @@ AS $$
 $$ LANGUAGE SQL IMMUTABLE STRICT;
 
 
+CREATE OR REPLACE FUNCTION token_is_street_address(info JSONB)
+  RETURNS BOOLEAN
+AS $$
+  SELECT info->>'street' is not null or info->>'place_search' is null;
+$$ LANGUAGE SQL IMMUTABLE;
+
+
 CREATE OR REPLACE FUNCTION token_has_addr_street(info JSONB)
   RETURNS BOOLEAN
 AS $$
-  SELECT info->>'street' is not null;
+  SELECT info->>'street' is not null and info->>'street' != '{}';
 $$ LANGUAGE SQL IMMUTABLE;
 
 

diff --git a/nominatim/data/place_name.py b/nominatim/data/place_name.py
index 47464e2b323b4c753f7e58e2c433e77dec57ea59..abba3544e5025b09051a7ae220ec2e77d3fa2930 100644 (file)
--- a/nominatim/data/place_name.py
+++ b/nominatim/data/place_name.py
@@ -34,7 +34,7 @@ class PlaceName:
 
 
     def __repr__(self) -> str:
-        return f"PlaceName(name='{self.name}',kind='{self.kind}',suffix='{self.suffix}')"
+        return f"PlaceName(name={self.name!r},kind={self.kind!r},suffix={self.suffix!r})"
 
 
     def clone(self, name: Optional[str] = None,

diff --git a/nominatim/tokenizer/icu_tokenizer.py b/nominatim/tokenizer/icu_tokenizer.py
index 79f383f6f1c11b2e8aac46b09fe1afb75e26a981..b6e646377b0e7f9d6dea691cf1ffcc6e1295632d 100644 (file)
--- a/nominatim/tokenizer/icu_tokenizer.py
+++ b/nominatim/tokenizer/icu_tokenizer.py
@@ -720,7 +720,7 @@ class _TokenInfo:
         self.names: Optional[str] = None
         self.housenumbers: Set[str] = set()
         self.housenumber_tokens: Set[int] = set()
-        self.street_tokens: Set[int] = set()
+        self.street_tokens: Optional[Set[int]] = None
         self.place_tokens: Set[int] = set()
         self.address_tokens: Dict[str, str] = {}
         self.postcode: Optional[str] = None
@@ -742,7 +742,7 @@ class _TokenInfo:
             out['hnr'] = ';'.join(self.housenumbers)
             out['hnr_tokens'] = self._mk_array(self.housenumber_tokens)
 
-        if self.street_tokens:
+        if self.street_tokens is not None:
             out['street'] = self._mk_array(self.street_tokens)
 
         if self.place_tokens:
@@ -776,6 +776,8 @@ class _TokenInfo:
     def add_street(self, tokens: Iterable[int]) -> None:
         """ Add addr:street match terms.
         """
+        if self.street_tokens is None:
+            self.street_tokens = set()
         self.street_tokens.update(tokens)
 
 

diff --git a/nominatim/tokenizer/legacy_tokenizer.py b/nominatim/tokenizer/legacy_tokenizer.py
index a50dedb2da7f2ffb51ad836445d2282bd0408280..e09700d9ddb8856a8d52fc6f7de1e9b748b9bbd7 100644 (file)
--- a/nominatim/tokenizer/legacy_tokenizer.py
+++ b/nominatim/tokenizer/legacy_tokenizer.py
@@ -564,14 +564,13 @@ class _TokenInfo:
     def add_street(self, conn: Connection, street: str) -> None:
         """ Add addr:street match terms.
         """
-        def _get_street(name: str) -> List[int]:
+        def _get_street(name: str) -> Optional[str]:
             with conn.cursor() as cur:
-                return cast(List[int],
+                return cast(Optional[str],
                             cur.scalar("SELECT word_ids_from_name(%s)::text", (name, )))
 
         tokens = self.cache.streets.get(street, _get_street)
-        if tokens:
-            self.data['street'] = tokens
+        self.data['street'] = tokens or '{}'
 
 
     def add_place(self, conn: Connection, place: str) -> None:

diff --git a/test/python/tokenizer/test_icu.py b/test/python/tokenizer/test_icu.py
index 7f0ffce16b5a3aa8c90e228aa9abd9f57cd6097a..2d9da69a632d938ed733bbb44cc06435eec5f955 100644 (file)
--- a/test/python/tokenizer/test_icu.py
+++ b/test/python/tokenizer/test_icu.py
@@ -523,7 +523,7 @@ class TestPlaceAddress:
     def test_process_place_nonexisting_street(self):
         info = self.process_address(street='Grand Road')
 
-        assert 'street' not in info
+        assert info['street'] == '{}'
 
 
     def test_process_place_multiple_street_tags(self):
@@ -538,7 +538,7 @@ class TestPlaceAddress:
     def test_process_place_street_empty(self):
         info = self.process_address(street='🜵')
 
-        assert 'street' not in info
+        assert info['street'] == '{}'
 
 
     def test_process_place_street_from_cache(self):

diff --git a/test/python/tokenizer/test_legacy.py b/test/python/tokenizer/test_legacy.py
index 57a82b8a0f4aa5af2f1dc6c2377db0fca8d83959..d63ee8e14624d5db3e652cfd677511f755b6a487 100644 (file)
--- a/test/python/tokenizer/test_legacy.py
+++ b/test/python/tokenizer/test_legacy.py
@@ -549,7 +549,7 @@ class TestPlaceAddress:
     def test_process_place_street_empty(self):
         info = self.process_address(street='🜵')
 
-        assert 'street' not in info
+        assert info['street'] == '{}'
 
 
     def test_process_place_place(self):