From d1b7c14f7987a20c218c96fe8071427536f8b36e Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Fri, 6 Dec 2024 11:00:25 +0100 Subject: [PATCH] osm2pgsql style: add modification for name and address, with tests --- lib-lua/themes/nominatim/init.lua | 82 +++++++----- .../bdd/osm2pgsql/import/custom_style.feature | 123 +++++++++++++++++- 2 files changed, 174 insertions(+), 31 deletions(-) diff --git a/lib-lua/themes/nominatim/init.lua b/lib-lua/themes/nominatim/init.lua index c2dfb130..f8706bb9 100644 --- a/lib-lua/themes/nominatim/init.lua +++ b/lib-lua/themes/nominatim/init.lua @@ -636,10 +636,10 @@ function module.process_tags(o) end -- name keys - local fallback = o:grab_name_parts{groups=NAMES} + local fallback = o:grab_name_parts{groups=NAME_FILTER} -- address keys - if o:grab_address_parts{groups=ADDRESS_TAGS} > 0 and fallback == nil then + if o:grab_address_parts{groups=ADDRESS_FILTER} > 0 and fallback == nil then fallback = {'place', 'house', PlaceTransform.always} end if o.address.country ~= nil and #o.address.country ~= 2 then @@ -686,7 +686,7 @@ function module.set_prefilters(data) end -function module.ignore_tags(data) +function module.ignore_keys(data) merge_filters_into_main('delete', data) add_pre_filter{delete = data} end @@ -735,47 +735,71 @@ function module.add_main_tags(data) end -function module.set_name_tags(data) - NAMES = module.tag_group(data) - - for _, lst in pairs(data) do - for _, k in ipairs(lst) do - local key = process_key(k) - if key ~= nil then - module.TAGINFO_NAME_KEYS[key] = true - end +function module.modify_name_tags(data) + for k,v in pairs(data) do + if next(v) then + NAMES[k] = v + else + NAMES[k] = nil end end + NAME_FILTER = module.tag_group(NAMES) remove_group_from_main('fallback:name') - merge_filters_into_main('fallback:name', data.house) + if data.house ~= nil then + merge_filters_into_main('fallback:name', data.house) + end +end + + +function module.set_name_tags(data) + NAMES = {} + module.modify_name_tags(data) end function module.set_address_tags(data) - if data.postcode_fallback ~= nil then - POSTCODE_FALLBACK = data.postcode_fallback - data.postcode_fallback = nil - end - ADDRESS_TAGS = module.tag_group(data) - - for _, lst in pairs(data) do - if lst ~= nil then - for _, k in ipairs(lst) do - local key = process_key(k) - if key ~= nil then - module.TAGINFO_ADDRESS_KEYS[key] = true - end - end + ADDRESS_TAGS = {} + module.modify_address_tags(data) +end + + +function module.modify_address_tags(data) + for k, v in pairs(data) do + if k == 'postcode_fallback' then + POSTCODE_FALLBACK = v + elseif next(v) == nil then + ADDRESS_TAGS[k] = nil + else + ADDRESS_TAGS[k] = v end end + ADDRESS_FILTER = module.tag_group(ADDRESS_TAGS) + remove_group_from_main('fallback:address') - remove_group_from_main('fallback:postcode') merge_filters_into_main('fallback:address', data.main) + merge_filters_into_main('fallback:address', data.interpolation) + remove_group_from_main('fallback:postcode') if POSTCODE_FALLBACK then merge_filters_into_main('fallback:postcode', data.postcode) end - merge_filters_into_main('fallback:address', data.interpolation) +end + + +function module.set_address_tags(data) + ADDRESS_TAGS_SOURCE = {} + module.modify_address_tags(data) +end + + +function module.set_postcode_fallback(enable) + if POSTCODE_FALLBACK ~= enable then + remove_group_from_main('fallback:postcode') + if enable then + merge_filters_into_main('fallback:postcode', ADDRESS_TAGS.postcode) + end + end + POSTCODE_FALLBACK = enable end diff --git a/test/bdd/osm2pgsql/import/custom_style.feature b/test/bdd/osm2pgsql/import/custom_style.feature index 2ca95c91..405afdb5 100644 --- a/test/bdd/osm2pgsql/import/custom_style.feature +++ b/test/bdd/osm2pgsql/import/custom_style.feature @@ -2,7 +2,7 @@ Feature: Import with custom styles by osm2pgsql Tests for the example customizations given in the documentation. - Scenario: Custom main tags + Scenario: Custom main tags (set new ones) Given the lua style file """ local flex = require('import-full') @@ -28,6 +28,35 @@ Feature: Import with custom styles by osm2pgsql | N13 | highway | primary | | N15 | highway | primary | + Scenario: Custom main tags (modify existing) + Given the lua style file + """ + local flex = require('import-full') + + flex.add_main_tags{ + amenity = {prison = 'delete'}, + highway = {stop = 'named'}, + aeroway = 'named' + } + """ + When loading osm data + """ + n10 Tamenity=hotel x0 y0 + n11 Tamenity=prison x0 y0 + n12 Thighway=stop x0 y0 + n13 Thighway=stop,name=BigStop x0 y0 + n14 Thighway=give_way x0 y0 + n15 Thighway=bus_stop x0 y0 + n16 Taeroway=no,name=foo x0 y0 + n17 Taeroway=taxiway,name=D15 x0 y0 + """ + Then place contains exactly + | object | class | type | + | N10 | amenity | hotel | + | N13 | highway | stop | + | N15 | highway | bus_stop | + | N17 | aeroway | taxiway | + Scenario: Prefiltering tags Given the lua style file """ @@ -56,6 +85,38 @@ Feature: Import with custom styles by osm2pgsql | N4:tourism | - | | N4:amenity | - | + Scenario: Ignore some tags + Given the lua style file + """ + local flex = require('import-extratags') + + flex.ignore_keys{'ref:*', 'surface'} + """ + When loading osm data + """ + n100 Thighway=residential,ref=34,ref:bodo=34,surface=gray,extra=1 x0 y0 + """ + Then place contains exactly + | object | name | extratags | + | N100 | 'ref' : '34' | 'extra': '1' | + + + Scenario: Add for extratags + Given the lua style file + """ + local flex = require('import-full') + + flex.add_for_extratags{'ref:*', 'surface'} + """ + When loading osm data + """ + n100 Thighway=residential,ref=34,ref:bodo=34,surface=gray,extra=1 x0 y0 + """ + Then place contains exactly + | object | name | extratags | + | N100 | 'ref' : '34' | 'ref:bodo': '34', 'surface': 'gray' | + + Scenario: Name tags Given the lua style file """ @@ -78,6 +139,22 @@ Feature: Import with custom styles by osm2pgsql | N3:highway | 'name': 'Greens' | | N4:highway | 'name': 'Red', 'ref': '45' | + Scenario: Modify name tags + Given the lua style file + """ + local flex = require('import-full') + + flex.modify_name_tags{house = {}, extra = {'o'}} + """ + When loading osm data + """ + n1 Ttourism=hotel,ref=45,o=good + n2 Taddr:housename=Old,addr:street=Away + """ + Then place contains exactly + | object | name | + | N1:tourism | 'o': 'good' | + Scenario: Address tags Given the lua style file """ @@ -101,7 +178,24 @@ Feature: Import with custom styles by osm2pgsql | N1:tourism | hotel | 'street': 'Foo' | | N2:place | house | 'housenumber': '23', 'street': 'Budd', 'postcode': '5567' | - Scenario: Unused handling + Scenario: Modify address tags + Given the lua style file + """ + local flex = require('import-full') + + flex.set_address_tags{ + extra = {'addr:*'}, + } + """ + When loading osm data + """ + n2 Taddr:housenumber=23,addr:street=Budd,is_in:city=Faraway,postal_code=5567 x0 y0 + """ + Then place contains exactly + | object | type | address | + | N2:place | house | 'housenumber': '23', 'street': 'Budd', 'postcode': '5567' | + + Scenario: Unused handling (delete) Given the lua style file """ local flex = require('import-full') @@ -122,6 +216,31 @@ Feature: Import with custom styles by osm2pgsql | N1:tourism | hotel | 'tiger:county': 'Fargo' | - | | N2:tourism | hotel | - | 'else': 'other' | + Scenario: Unused handling (extra) + Given the lua style file + """ + local flex = require('flex-base') + flex.set_main_tags{highway = 'always', + wikipedia = 'extra'} + flex.add_for_extratags{'wikipedia:*', 'wikidata'} + flex.set_unused_handling{extra_keys = {'surface'}} + """ + When loading osm data + """ + n100 Thighway=path,foo=bar,wikipedia=en:Path x0 y0 + n234 Thighway=path,surface=rough x0 y0 + n445 Thighway=path,name=something x0 y0 + n446 Thighway=path,wikipedia:en=Path,wikidata=Q23 x0 y0 + n567 Thighway=path,surface=dirt,wikipedia:en=Path x0 y0 + """ + Then place contains exactly + | object | type | extratags | + | N100:highway | path | 'wikipedia': 'en:Path' | + | N234:highway | path | 'surface': 'rough' | + | N445:highway | path | - | + | N446:highway | path | 'wikipedia:en': 'Path', 'wikidata': 'Q23' | + | N567:highway | path | 'surface': 'dirt', 'wikipedia:en': 'Path' | + Scenario: Additional relation types Given the lua style file """ -- 2.39.5