From: Sarah Hoffmann Date: Fri, 23 Dec 2022 16:35:28 +0000 (+0100) Subject: add tests for examples in lua style documentation X-Git-Tag: v4.3.0~121^2~2 X-Git-Url: https://git.openstreetmap.org./nominatim.git/commitdiff_plain/200eae3bc0b559e7df7c1d1fb50141830c2f1849?hp=9321e425a4b6a42f8c6579a27f575d40a07efa12 add tests for examples in lua style documentation And fix all the errors the tests have found. --- diff --git a/docs/customize/Import-Styles.md b/docs/customize/Import-Styles.md index 1f171c16..6085d4e4 100644 --- a/docs/customize/Import-Styles.md +++ b/docs/customize/Import-Styles.md @@ -49,7 +49,9 @@ during the processing. There are no built-in defaults for the tag lists, so all the functions need to be called from your style script to fully process the data. Make sure you start from one of the default style and only modify - the data you are interested in. + the data you are interested in. You can also derive your style from an + existing style by importing the appropriate module, e.g. + `local flex = require('import-street')`. Many of the following functions take _key match lists_. These lists can contain three kinds of strings to match against tag keys: @@ -91,10 +93,12 @@ key, then this is used as default for values that are not listed. !!! example ``` lua + local flex = require('import-full') + flex.set_main_tags{ - boundary = {'administrative' = named}, - highway = {'always', street_lamp = 'named'} - landuse = 'fallback', + boundary = {administrative = 'named'}, + highway = {'always', street_lamp = 'named'}, + landuse = 'fallback' } ``` @@ -134,9 +138,11 @@ Any other string is matched exactly against tag keys. !!! example ``` lua + local flex = require('import-full') + flex.set_prefilters{ delete_keys = {'source', 'source:*'}, - extra_tags = {'amenity' = {'yes', 'no'} + extra_tags = {amenity = {'yes', 'no'}} } flex.set_main_tags{ amenity = 'always' @@ -168,7 +174,9 @@ They take _key match lists_ for main and extra names respectively. !!! example ``` lua - flex.set_main_tags{'highway' = {'traffic_light' = 'named'}} + local flex = require('flex-base') + + flex.set_main_tags{highway = {traffic_light = 'named'}} flex.set_name_tags{main = {'name', 'name:*'}, extra = {'ref'} } @@ -204,6 +212,8 @@ are accepted, all other values are discarded. !!! example ``` lua + local flex = require('import-full') + flex.set_address_tags{ main = {'addr:housenumber'}, extra = {'addr:*'}, @@ -239,6 +249,8 @@ above. !!! example ``` lua + local flex = require('import-full') + flex.set_address_tags{ main = {'addr:housenumber'}, extra = {'addr:*', 'tiger:county'} @@ -274,6 +286,8 @@ kinds of geometries can be used: !!! Example ``` lua + local flex = require('import-full') + flex.RELATION_TYPES['site'] = flex.relation_as_multipolygon ``` @@ -292,6 +306,8 @@ logic. !!! Example ``` lua + local flex = require('import-full') + function osm2pgsql.process_relation(object) if object.tags.boundary ~= 'administrative' or object.tags.admin_level ~= '2' then flex.process_relation(object) @@ -312,6 +328,8 @@ responsible for filtering the tags and writing out the rows into Postgresql. !!! Example ``` lua + local flex = require('import-full') + local original_process_tags = flex.process_tags function flex.process_tags(o) diff --git a/nominatim/tools/exec_utils.py b/nominatim/tools/exec_utils.py index 2e5c4801..ab2ccc7c 100644 --- a/nominatim/tools/exec_utils.py +++ b/nominatim/tools/exec_utils.py @@ -125,7 +125,7 @@ def run_osm2pgsql(options: Mapping[str, Any]) -> None: ] if str(options['osm2pgsql_style']).endswith('.lua'): - env['LUA_PATH'] = ';'.join((str(options['osm2pgsql_style_path'] / 'flex-base.lua'), + env['LUA_PATH'] = ';'.join((str(options['osm2pgsql_style_path'] / '?.lua'), os.environ.get('LUAPATH', ';'))) cmd.extend(('--output', 'flex')) else: diff --git a/settings/flex-base.lua b/settings/flex-base.lua index a7a3c58c..0e112736 100644 --- a/settings/flex-base.lua +++ b/settings/flex-base.lua @@ -461,8 +461,8 @@ end function module.set_prefilters(data) PRE_DELETE = module.tag_match{keys = data.delete_keys, tags = data.delete_tags} - PRE_EXTRAS = module.tag_match{keys = data.extratag_keys, - tags = data.extratag_tags} + PRE_EXTRAS = module.tag_match{keys = data.extra_keys, + tags = data.extra_tags} end function module.set_main_tags(data) @@ -484,12 +484,12 @@ end function module.set_unused_handling(data) if data.extra_keys == nil and data.extra_tags == nil then - POST_DELETE = module.tag_match{data.delete_keys, tags = data.delete_tags} + POST_DELETE = module.tag_match{keys = data.delete_keys, tags = data.delete_tags} POST_EXTRAS = nil SAVE_EXTRA_MAINS = true elseif data.delete_keys == nil and data.delete_tags == nil then POST_DELETE = nil - POST_EXTRAS = module.tag_match{data.extra_keys, tags = data.extra_tags} + POST_EXTRAS = module.tag_match{keys = data.extra_keys, tags = data.extra_tags} SAVE_EXTRA_MAINS = false else error("unused handler can have only 'extra_keys' or 'delete_keys' set.") diff --git a/settings/import-address.lua b/settings/import-address.lua index 2bd92535..00d089cc 100644 --- a/settings/import-address.lua +++ b/settings/import-address.lua @@ -1,4 +1,4 @@ -flex = require('flex-base') +local flex = require('flex-base') flex.set_main_tags{ highway = {'always', @@ -32,7 +32,7 @@ flex.set_prefilters{delete_keys = {'building', 'source', 'noexit', 'crossing', 'give_way', 'stop'}, landuse = {'cemetry', 'no'}, boundary = {'place'}}, - extratag_keys = {'wikipedia', 'wikipedia:*', 'wikidata', 'capital', 'area'} + extra_keys = {'wikipedia', 'wikipedia:*', 'wikidata', 'capital', 'area'} } flex.set_name_tags{main = {'name', 'name:*', @@ -65,3 +65,5 @@ flex.set_address_tags{main = {'addr:housenumber', flex.set_unused_handling{extra_keys = {'place'}} + +return flex diff --git a/settings/import-admin.lua b/settings/import-admin.lua index c593c34a..a1164acc 100644 --- a/settings/import-admin.lua +++ b/settings/import-admin.lua @@ -15,7 +15,7 @@ flex.set_prefilters{delete_keys = {'building', 'source', 'highway', 'addr:street:name', 'addr:street:type'}, delete_tags = {landuse = {'cemetry', 'no'}, boundary = {'place'}}, - extratag_keys = {'wikipedia', 'wikipedia:*', 'wikidata', 'capital'} + extra_keys = {'wikipedia', 'wikipedia:*', 'wikidata', 'capital'} } flex.set_name_tags{main = {'name', 'name:*', @@ -42,3 +42,5 @@ flex.set_address_tags{extra = {'addr:*', 'is_in:*'}, } flex.set_unused_handling{extra_keys = {'place'}} + +return flex diff --git a/settings/import-extratags.lua b/settings/import-extratags.lua index 22c3d9c8..d634d4a1 100644 --- a/settings/import-extratags.lua +++ b/settings/import-extratags.lua @@ -1,4 +1,4 @@ -flex = require('flex-base') +local flex = require('flex-base') flex.set_main_tags{ building = 'fallback', @@ -74,7 +74,7 @@ flex.set_prefilters{delete_keys = {'note', 'note:*', 'source', '*source', 'attri waterway = {'riverbank'}, building = {'no'}, boundary = {'place'}}, - extratag_keys = {'*:prefix', '*:suffix', 'name:prefix:*', 'name:suffix:*', + extra_keys = {'*:prefix', '*:suffix', 'name:prefix:*', 'name:suffix:*', 'name:etymology', 'name:signed', 'name:botanical', 'wikidata', '*:wikidata', 'addr:street:name', 'addr:street:type'} @@ -110,3 +110,5 @@ flex.set_address_tags{main = {'addr:housenumber', flex.set_unused_handling{delete_keys = {'tiger:*'}} + +return flex diff --git a/settings/import-full.lua b/settings/import-full.lua index e00313ce..a932fa50 100644 --- a/settings/import-full.lua +++ b/settings/import-full.lua @@ -1,4 +1,4 @@ -flex = require('flex-base') +local flex = require('flex-base') flex.set_main_tags{ building = 'fallback', @@ -74,7 +74,7 @@ flex.set_prefilters{delete_keys = {'note', 'note:*', 'source', '*source', 'attri waterway = {'riverbank'}, building = {'no'}, boundary = {'place'}}, - extratag_keys = {'*:prefix', '*:suffix', 'name:prefix:*', 'name:suffix:*', + extra_keys = {'*:prefix', '*:suffix', 'name:prefix:*', 'name:suffix:*', 'name:etymology', 'name:signed', 'name:botanical', 'wikidata', '*:wikidata', 'addr:street:name', 'addr:street:type'} @@ -110,3 +110,5 @@ flex.set_address_tags{main = {'addr:housenumber', flex.set_unused_handling{extra_keys = {'place'}} + +return flex diff --git a/settings/import-street.lua b/settings/import-street.lua index 34bff572..acadf01e 100644 --- a/settings/import-street.lua +++ b/settings/import-street.lua @@ -1,4 +1,4 @@ -flex = require('flex-base') +local flex = require('flex-base') flex.set_main_tags{ highway = {'always', @@ -32,7 +32,7 @@ flex.set_prefilters{delete_keys = {'building', 'source', 'noexit', 'crossing', 'give_way', 'stop'}, landuse = {'cemetry', 'no'}, boundary = {'place'}}, - extratag_keys = {'wikipedia', 'wikipedia:*', 'wikidata', 'capital', 'area'} + extra_keys = {'wikipedia', 'wikipedia:*', 'wikidata', 'capital', 'area'} } flex.set_name_tags{main = {'name', 'name:*', @@ -64,3 +64,5 @@ flex.set_address_tags{main = {'addr:housenumber', } flex.set_unused_handling{extra_keys = {'place'}} + +return flex diff --git a/test/bdd/osm2pgsql/import/custom_style.feature b/test/bdd/osm2pgsql/import/custom_style.feature new file mode 100644 index 00000000..2ca95c91 --- /dev/null +++ b/test/bdd/osm2pgsql/import/custom_style.feature @@ -0,0 +1,200 @@ +@DB +Feature: Import with custom styles by osm2pgsql + Tests for the example customizations given in the documentation. + + Scenario: Custom main tags + Given the lua style file + """ + local flex = require('import-full') + + flex.set_main_tags{ + boundary = {administrative = 'named'}, + highway = {'always', street_lamp = 'named'}, + landuse = 'fallback' + } + """ + When loading osm data + """ + n10 Tboundary=administrative x0 y0 + n11 Tboundary=administrative,name=Foo x0 y0 + n12 Tboundary=electoral x0 y0 + n13 Thighway=primary x0 y0 + n14 Thighway=street_lamp x0 y0 + n15 Thighway=primary,landuse=street x0 y0 + """ + Then place contains exactly + | object | class | type | + | N11 | boundary | administrative | + | N13 | highway | primary | + | N15 | highway | primary | + + Scenario: Prefiltering tags + Given the lua style file + """ + local flex = require('import-full') + + flex.set_prefilters{ + delete_keys = {'source', 'source:*'}, + extra_tags = {amenity = {'yes', 'no'}} + } + flex.set_main_tags{ + amenity = 'always', + tourism = 'always' + } + """ + When loading osm data + """ + n1 Tamenity=yes x0 y6 + n2 Tamenity=hospital,source=survey x3 y6 + n3 Ttourism=hotel,amenity=yes x0 y0 + n4 Ttourism=hotel,amenity=telephone x0 y0 + """ + Then place contains exactly + | object | extratags | + | N2:amenity | - | + | N3:tourism | 'amenity': 'yes' | + | N4:tourism | - | + | N4:amenity | - | + + Scenario: Name tags + Given the lua style file + """ + local flex = require('flex-base') + + flex.set_main_tags{highway = {traffic_light = 'named'}} + flex.set_name_tags{main = {'name', 'name:*'}, + extra = {'ref'} + } + """ + When loading osm data + """ + n1 Thighway=stop,name=Something x0 y0 + n2 Thighway=traffic_light,ref=453-4 x0 y0 + n3 Thighway=traffic_light,name=Greens x0 y0 + n4 Thighway=traffic_light,name=Red,ref=45 x0 y0 + """ + Then place contains exactly + | object | name | + | N3:highway | 'name': 'Greens' | + | N4:highway | 'name': 'Red', 'ref': '45' | + + Scenario: Address tags + Given the lua style file + """ + local flex = require('import-full') + + flex.set_address_tags{ + main = {'addr:housenumber'}, + extra = {'addr:*'}, + postcode = {'postal_code', 'postcode', 'addr:postcode'}, + country = {'country-code', 'ISO3166-1'} + } + """ + When loading osm data + """ + n1 Ttourism=hotel,addr:street=Foo x0 y0 + n2 Taddr:housenumber=23,addr:street=Budd,postal_code=5567 x0 y0 + n3 Taddr:street=None,addr:city=Where x0 y0 + """ + Then place contains exactly + | object | type | address | + | N1:tourism | hotel | 'street': 'Foo' | + | N2:place | house | 'housenumber': '23', 'street': 'Budd', 'postcode': '5567' | + + Scenario: Unused handling + Given the lua style file + """ + local flex = require('import-full') + + flex.set_address_tags{ + main = {'addr:housenumber'}, + extra = {'addr:*', 'tiger:county'} + } + flex.set_unused_handling{delete_keys = {'tiger:*'}} + """ + When loading osm data + """ + n1 Ttourism=hotel,tiger:county=Fargo x0 y0 + n2 Ttourism=hotel,tiger:xxd=56,else=other x0 y0 + """ + Then place contains exactly + | object | type | address | extratags | + | N1:tourism | hotel | 'tiger:county': 'Fargo' | - | + | N2:tourism | hotel | - | 'else': 'other' | + + Scenario: Additional relation types + Given the lua style file + """ + local flex = require('import-full') + + flex.RELATION_TYPES['site'] = flex.relation_as_multipolygon + """ + And the grid + | 1 | 2 | + | 4 | 3 | + When loading osm data + """ + n1 + n2 + n3 + n4 + w1 Nn1,n2,n3,n4,n1 + r1 Ttype=multipolygon,amenity=school Mw1@ + r2 Ttype=site,amenity=school Mw1@ + """ + Then place contains exactly + | object | type | + | R1:amenity | school | + | R2:amenity | school | + + Scenario: Exclude country relations + Given the lua style file + """ + local flex = require('import-full') + + function osm2pgsql.process_relation(object) + if object.tags.boundary ~= 'administrative' or object.tags.admin_level ~= '2' then + flex.process_relation(object) + end + end + """ + And the grid + | 1 | 2 | + | 4 | 3 | + When loading osm data + """ + n1 + n2 + n3 + n4 + w1 Nn1,n2,n3,n4,n1 + r1 Ttype=multipolygon,boundary=administrative,admin_level=4,name=Small Mw1@ + r2 Ttype=multipolygon,boundary=administrative,admin_level=2,name=Big Mw1@ + """ + Then place contains exactly + | object | type | + | R1:boundary | administrative | + + Scenario: Customize processing functions + Given the lua style file + """ + local flex = require('import-full') + + local original_process_tags = flex.process_tags + + function flex.process_tags(o) + if o.object.tags.highway ~= nil and o.object.tags.access == 'no' then + return + end + + original_process_tags(o) + end + """ + When loading osm data + """ + n1 Thighway=residential x0 y0 + n2 Thighway=residential,access=no x0 y0 + """ + Then place contains exactly + | object | type | + | N1:highway | residential | diff --git a/test/bdd/steps/steps_osm_data.py b/test/bdd/steps/steps_osm_data.py index 7590b17c..336fb707 100644 --- a/test/bdd/steps/steps_osm_data.py +++ b/test/bdd/steps/steps_osm_data.py @@ -49,6 +49,15 @@ def write_opl_file(opl, grid): return fd.name +@given('the lua style file') +def lua_style_file(context): + """ Define a custom style file to use for the import. + """ + style = Path(context.nominatim.website_dir.name) / 'custom.lua' + style.write_text(context.text) + context.nominatim.test_env['NOMINATIM_IMPORT_STYLE'] = str(style) + + @given(u'the ([0-9.]+ )?grid(?: with origin (?P.*))?') def define_node_grid(context, grid_step, origin): """