osm2pgsql style: add modification for name and address, with tests

diff --git a/lib-lua/themes/nominatim/init.lua b/lib-lua/themes/nominatim/init.lua
index c2dfb130c7744de582660f8321805e6bda8dc3ae..f8706bb92b6c285809091902df85fe8174c50eb6 100644 (file)
--- a/lib-lua/themes/nominatim/init.lua
+++ b/lib-lua/themes/nominatim/init.lua
@@ -636,10 +636,10 @@ function module.process_tags(o)
     end
 
     -- name keys
-    local fallback = o:grab_name_parts{groups=NAMES}
+    local fallback = o:grab_name_parts{groups=NAME_FILTER}
 
     -- address keys
-    if o:grab_address_parts{groups=ADDRESS_TAGS} > 0 and fallback == nil then
+    if o:grab_address_parts{groups=ADDRESS_FILTER} > 0 and fallback == nil then
         fallback = {'place', 'house', PlaceTransform.always}
     end
     if o.address.country ~= nil and #o.address.country ~= 2 then
@@ -686,7 +686,7 @@ function module.set_prefilters(data)
 end
 
 
-function module.ignore_tags(data)
+function module.ignore_keys(data)
     merge_filters_into_main('delete', data)
     add_pre_filter{delete = data}
 end
@@ -735,47 +735,71 @@ function module.add_main_tags(data)
 end
 
 
-function module.set_name_tags(data)
-    NAMES = module.tag_group(data)
-
-    for _, lst in pairs(data) do
-        for _, k in ipairs(lst) do
-            local key = process_key(k)
-            if key ~= nil then
-                module.TAGINFO_NAME_KEYS[key] = true
-            end
+function module.modify_name_tags(data)
+    for k,v in pairs(data) do
+        if next(v) then
+            NAMES[k] = v
+        else
+            NAMES[k] = nil
         end
     end
+    NAME_FILTER = module.tag_group(NAMES)
     remove_group_from_main('fallback:name')
-    merge_filters_into_main('fallback:name', data.house)
+    if data.house ~= nil then
+        merge_filters_into_main('fallback:name', data.house)
+    end
+end
+
+
+function module.set_name_tags(data)
+    NAMES = {}
+    module.modify_name_tags(data)
 end
 
 
 function module.set_address_tags(data)
-    if data.postcode_fallback ~= nil then
-        POSTCODE_FALLBACK = data.postcode_fallback
-        data.postcode_fallback = nil
-    end
-    ADDRESS_TAGS = module.tag_group(data)
-
-    for _, lst in pairs(data) do
-        if lst ~= nil then
-            for _, k in ipairs(lst) do
-                local key = process_key(k)
-                if key ~= nil then
-                    module.TAGINFO_ADDRESS_KEYS[key] = true
-                end
-            end
+    ADDRESS_TAGS = {}
+    module.modify_address_tags(data)
+end
+
+
+function module.modify_address_tags(data)
+    for k, v in pairs(data) do
+        if k == 'postcode_fallback' then
+            POSTCODE_FALLBACK = v
+        elseif next(v) == nil then
+            ADDRESS_TAGS[k] = nil
+        else
+            ADDRESS_TAGS[k] = v
         end
     end
 
+    ADDRESS_FILTER = module.tag_group(ADDRESS_TAGS)
+
     remove_group_from_main('fallback:address')
-    remove_group_from_main('fallback:postcode')
     merge_filters_into_main('fallback:address', data.main)
+    merge_filters_into_main('fallback:address', data.interpolation)
+    remove_group_from_main('fallback:postcode')
     if POSTCODE_FALLBACK then
         merge_filters_into_main('fallback:postcode', data.postcode)
     end
-    merge_filters_into_main('fallback:address', data.interpolation)
+end
+
+
+function module.set_address_tags(data)
+    ADDRESS_TAGS_SOURCE = {}
+    module.modify_address_tags(data)
+end
+
+
+function module.set_postcode_fallback(enable)
+    if POSTCODE_FALLBACK ~= enable then
+        remove_group_from_main('fallback:postcode')
+        if enable then
+            merge_filters_into_main('fallback:postcode', ADDRESS_TAGS.postcode)
+        end
+    end
+    POSTCODE_FALLBACK = enable
 end
 
 

diff --git a/test/bdd/osm2pgsql/import/custom_style.feature b/test/bdd/osm2pgsql/import/custom_style.feature
index 2ca95c91a563d0ab0a46d7a4c3cfbdff0d7856da..405afdb5c4dc8c2d0c363e43439d64338c7ff0fe 100644 (file)
--- a/test/bdd/osm2pgsql/import/custom_style.feature
+++ b/test/bdd/osm2pgsql/import/custom_style.feature
@@ -2,7 +2,7 @@
 Feature: Import with custom styles by osm2pgsql
     Tests for the example customizations given in the documentation.
 
-    Scenario: Custom main tags
+    Scenario: Custom main tags (set new ones)
         Given the lua style file
             """
             local flex = require('import-full')
@@ -28,6 +28,35 @@ Feature: Import with custom styles by osm2pgsql
             | N13    | highway  | primary        |
             | N15    | highway  | primary        |
 
+    Scenario: Custom main tags (modify existing)
+        Given the lua style file
+            """
+            local flex = require('import-full')
+
+            flex.add_main_tags{
+                amenity = {prison = 'delete'},
+                highway = {stop = 'named'},
+                aeroway = 'named'
+            }
+            """
+        When loading osm data
+            """
+            n10 Tamenity=hotel x0 y0
+            n11 Tamenity=prison x0 y0
+            n12 Thighway=stop x0 y0
+            n13 Thighway=stop,name=BigStop x0 y0
+            n14 Thighway=give_way x0 y0
+            n15 Thighway=bus_stop x0 y0
+            n16 Taeroway=no,name=foo x0 y0
+            n17 Taeroway=taxiway,name=D15 x0 y0
+            """
+        Then place contains exactly
+            | object | class   | type     |
+            | N10    | amenity | hotel    |
+            | N13    | highway | stop     |
+            | N15    | highway | bus_stop |
+            | N17    | aeroway | taxiway  |
+
     Scenario: Prefiltering tags
         Given the lua style file
             """
@@ -56,6 +85,38 @@ Feature: Import with custom styles by osm2pgsql
             | N4:tourism | - |
             | N4:amenity | - |
 
+    Scenario: Ignore some tags
+        Given the lua style file
+            """
+            local flex = require('import-extratags')
+
+            flex.ignore_keys{'ref:*', 'surface'}
+            """
+        When loading osm data
+            """
+            n100 Thighway=residential,ref=34,ref:bodo=34,surface=gray,extra=1 x0 y0
+            """
+        Then place contains exactly
+            | object | name         | extratags    |
+            | N100   | 'ref' : '34' | 'extra': '1' |
+
+
+    Scenario: Add for extratags
+        Given the lua style file
+            """
+            local flex = require('import-full')
+
+            flex.add_for_extratags{'ref:*', 'surface'}
+            """
+        When loading osm data
+            """
+            n100 Thighway=residential,ref=34,ref:bodo=34,surface=gray,extra=1 x0 y0
+            """
+        Then place contains exactly
+            | object | name         | extratags    |
+            | N100   | 'ref' : '34' | 'ref:bodo': '34', 'surface': 'gray' |
+
+
     Scenario: Name tags
         Given the lua style file
             """
@@ -78,6 +139,22 @@ Feature: Import with custom styles by osm2pgsql
             | N3:highway | 'name': 'Greens'           |
             | N4:highway | 'name': 'Red', 'ref': '45' |
 
+    Scenario: Modify name tags
+        Given the lua style file
+            """
+            local flex = require('import-full')
+
+            flex.modify_name_tags{house = {}, extra = {'o'}}
+            """
+        When loading osm data
+            """
+            n1 Ttourism=hotel,ref=45,o=good
+            n2 Taddr:housename=Old,addr:street=Away
+            """
+        Then place contains exactly
+            | object     | name        |
+            | N1:tourism | 'o': 'good' |
+
     Scenario: Address tags
         Given the lua style file
             """
@@ -101,7 +178,24 @@ Feature: Import with custom styles by osm2pgsql
             | N1:tourism | hotel | 'street': 'Foo' |
             | N2:place   | house | 'housenumber': '23', 'street': 'Budd', 'postcode': '5567' |
 
-    Scenario: Unused handling
+    Scenario: Modify address tags
+        Given the lua style file
+            """
+            local flex = require('import-full')
+
+            flex.set_address_tags{
+                extra = {'addr:*'},
+            }
+            """
+        When loading osm data
+            """
+            n2 Taddr:housenumber=23,addr:street=Budd,is_in:city=Faraway,postal_code=5567 x0 y0
+            """
+        Then place contains exactly
+            | object     | type  | address |
+            | N2:place   | house | 'housenumber': '23', 'street': 'Budd', 'postcode': '5567' |
+
+    Scenario: Unused handling (delete)
         Given the lua style file
             """
             local flex = require('import-full')
@@ -122,6 +216,31 @@ Feature: Import with custom styles by osm2pgsql
             | N1:tourism | hotel | 'tiger:county': 'Fargo' | -                |
             | N2:tourism | hotel | -                       | 'else': 'other'  |
 
+    Scenario: Unused handling (extra)
+        Given the lua style file
+            """
+            local flex = require('flex-base')
+            flex.set_main_tags{highway = 'always',
+                               wikipedia = 'extra'}
+            flex.add_for_extratags{'wikipedia:*', 'wikidata'}
+            flex.set_unused_handling{extra_keys = {'surface'}}
+            """
+        When loading osm data
+            """
+            n100 Thighway=path,foo=bar,wikipedia=en:Path x0 y0
+            n234 Thighway=path,surface=rough x0 y0
+            n445 Thighway=path,name=something x0 y0
+            n446 Thighway=path,wikipedia:en=Path,wikidata=Q23 x0 y0
+            n567 Thighway=path,surface=dirt,wikipedia:en=Path x0 y0
+            """
+        Then place contains exactly
+            | object       | type  | extratags              |
+            | N100:highway | path  | 'wikipedia': 'en:Path' |
+            | N234:highway | path  | 'surface': 'rough' |
+            | N445:highway | path  | - |
+            | N446:highway | path  | 'wikipedia:en': 'Path', 'wikidata': 'Q23' |
+            | N567:highway | path  | 'surface': 'dirt', 'wikipedia:en': 'Path' |
+
     Scenario: Additional relation types
         Given the lua style file
             """