]> git.openstreetmap.org Git - nominatim.git/commitdiff
add tests for examples in lua style documentation
authorSarah Hoffmann <lonvia@denofr.de>
Fri, 23 Dec 2022 16:35:28 +0000 (17:35 +0100)
committerSarah Hoffmann <lonvia@denofr.de>
Fri, 23 Dec 2022 16:35:28 +0000 (17:35 +0100)
And fix all the errors the tests have found.

docs/customize/Import-Styles.md
nominatim/tools/exec_utils.py
settings/flex-base.lua
settings/import-address.lua
settings/import-admin.lua
settings/import-extratags.lua
settings/import-full.lua
settings/import-street.lua
test/bdd/osm2pgsql/import/custom_style.feature [new file with mode: 0644]
test/bdd/steps/steps_osm_data.py

index 1f171c16fb732879ceab2fb920fa0b2ef935631a..6085d4e473f0efb4944d2a5eac9c0316f79da733 100644 (file)
@@ -49,7 +49,9 @@ during the processing.
     There are no built-in defaults for the tag lists, so all the functions
     need to be called from your style script to fully process the data.
     Make sure you start from one of the default style and only modify
     There are no built-in defaults for the tag lists, so all the functions
     need to be called from your style script to fully process the data.
     Make sure you start from one of the default style and only modify
-    the data you are interested in.
+    the data you are interested in. You can also derive your style from an
+    existing style by importing the appropriate module, e.g.
+    `local flex = require('import-street')`.
 
 Many of the following functions take _key match lists_. These lists can
 contain three kinds of strings to match against tag keys:
 
 Many of the following functions take _key match lists_. These lists can
 contain three kinds of strings to match against tag keys:
@@ -91,10 +93,12 @@ key, then this is used as default for values that are not listed.
 
 !!! example
     ``` lua
 
 !!! example
     ``` lua
+    local flex = require('import-full')
+
     flex.set_main_tags{
     flex.set_main_tags{
-        boundary = {'administrative' = named},
-        highway = {'always', street_lamp = 'named'}
-        landuse = 'fallback',
+        boundary = {administrative = 'named'},
+        highway = {'always', street_lamp = 'named'},
+        landuse = 'fallback'
     }
     ```
 
     }
     ```
 
@@ -134,9 +138,11 @@ Any other string is matched exactly against tag keys.
 
 !!! example
     ``` lua
 
 !!! example
     ``` lua
+    local flex = require('import-full')
+
     flex.set_prefilters{
         delete_keys = {'source', 'source:*'},
     flex.set_prefilters{
         delete_keys = {'source', 'source:*'},
-        extra_tags = {'amenity' = {'yes', 'no'}
+        extra_tags = {amenity = {'yes', 'no'}}
     }
     flex.set_main_tags{
         amenity = 'always'
     }
     flex.set_main_tags{
         amenity = 'always'
@@ -168,7 +174,9 @@ They take _key match lists_ for main and extra names respectively.
 
 !!! example
     ``` lua
 
 !!! example
     ``` lua
-    flex.set_main_tags{'highway' = {'traffic_light' = 'named'}}
+    local flex = require('flex-base')
+
+    flex.set_main_tags{highway = {traffic_light = 'named'}}
     flex.set_name_tags{main = {'name', 'name:*'},
                        extra = {'ref'}
                       }
     flex.set_name_tags{main = {'name', 'name:*'},
                        extra = {'ref'}
                       }
@@ -204,6 +212,8 @@ are accepted, all other values are discarded.
 
 !!! example
     ``` lua
 
 !!! example
     ``` lua
+    local flex = require('import-full')
+
     flex.set_address_tags{
         main = {'addr:housenumber'},
         extra = {'addr:*'},
     flex.set_address_tags{
         main = {'addr:housenumber'},
         extra = {'addr:*'},
@@ -239,6 +249,8 @@ above.
 
 !!! example
     ``` lua
 
 !!! example
     ``` lua
+    local flex = require('import-full')
+
     flex.set_address_tags{
         main = {'addr:housenumber'},
         extra = {'addr:*', 'tiger:county'}
     flex.set_address_tags{
         main = {'addr:housenumber'},
         extra = {'addr:*', 'tiger:county'}
@@ -274,6 +286,8 @@ kinds of geometries can be used:
 
 !!! Example
     ``` lua
 
 !!! Example
     ``` lua
+    local flex = require('import-full')
+
     flex.RELATION_TYPES['site'] = flex.relation_as_multipolygon
     ```
 
     flex.RELATION_TYPES['site'] = flex.relation_as_multipolygon
     ```
 
@@ -292,6 +306,8 @@ logic.
 
 !!! Example
     ``` lua
 
 !!! Example
     ``` lua
+    local flex = require('import-full')
+
     function osm2pgsql.process_relation(object)
         if object.tags.boundary ~= 'administrative' or object.tags.admin_level ~= '2' then
           flex.process_relation(object)
     function osm2pgsql.process_relation(object)
         if object.tags.boundary ~= 'administrative' or object.tags.admin_level ~= '2' then
           flex.process_relation(object)
@@ -312,6 +328,8 @@ responsible for filtering the tags and writing out the rows into Postgresql.
 
 !!! Example
     ``` lua
 
 !!! Example
     ``` lua
+    local flex = require('import-full')
+
     local original_process_tags = flex.process_tags
 
     function flex.process_tags(o)
     local original_process_tags = flex.process_tags
 
     function flex.process_tags(o)
index 2e5c4801606115f1fe3adfa6616bb39d241ab5f3..ab2ccc7cf9d83d4047cd5a7f5b7c14ba634bb29a 100644 (file)
@@ -125,7 +125,7 @@ def run_osm2pgsql(options: Mapping[str, Any]) -> None:
           ]
 
     if str(options['osm2pgsql_style']).endswith('.lua'):
           ]
 
     if str(options['osm2pgsql_style']).endswith('.lua'):
-        env['LUA_PATH'] = ';'.join((str(options['osm2pgsql_style_path'] / 'flex-base.lua'),
+        env['LUA_PATH'] = ';'.join((str(options['osm2pgsql_style_path'] / '?.lua'),
                                     os.environ.get('LUAPATH', ';')))
         cmd.extend(('--output', 'flex'))
     else:
                                     os.environ.get('LUAPATH', ';')))
         cmd.extend(('--output', 'flex'))
     else:
index a7a3c58c39f748aece06b5587024ec24c3661891..0e11273633cfc6dabcda67b1f3f1f50954527fe8 100644 (file)
@@ -461,8 +461,8 @@ end
 
 function module.set_prefilters(data)
     PRE_DELETE = module.tag_match{keys = data.delete_keys, tags = data.delete_tags}
 
 function module.set_prefilters(data)
     PRE_DELETE = module.tag_match{keys = data.delete_keys, tags = data.delete_tags}
-    PRE_EXTRAS = module.tag_match{keys = data.extratag_keys,
-                                  tags = data.extratag_tags}
+    PRE_EXTRAS = module.tag_match{keys = data.extra_keys,
+                                  tags = data.extra_tags}
 end
 
 function module.set_main_tags(data)
 end
 
 function module.set_main_tags(data)
@@ -484,12 +484,12 @@ end
 
 function module.set_unused_handling(data)
     if data.extra_keys == nil and data.extra_tags == nil then
 
 function module.set_unused_handling(data)
     if data.extra_keys == nil and data.extra_tags == nil then
-        POST_DELETE = module.tag_match{data.delete_keys, tags = data.delete_tags}
+        POST_DELETE = module.tag_match{keys = data.delete_keys, tags = data.delete_tags}
         POST_EXTRAS = nil
         SAVE_EXTRA_MAINS = true
     elseif data.delete_keys == nil and data.delete_tags == nil then
         POST_DELETE = nil
         POST_EXTRAS = nil
         SAVE_EXTRA_MAINS = true
     elseif data.delete_keys == nil and data.delete_tags == nil then
         POST_DELETE = nil
-        POST_EXTRAS = module.tag_match{data.extra_keys, tags = data.extra_tags}
+        POST_EXTRAS = module.tag_match{keys = data.extra_keys, tags = data.extra_tags}
         SAVE_EXTRA_MAINS = false
     else
         error("unused handler can have only 'extra_keys' or 'delete_keys' set.")
         SAVE_EXTRA_MAINS = false
     else
         error("unused handler can have only 'extra_keys' or 'delete_keys' set.")
index 2bd925356572e332838344179577af6eaa0e4365..00d089ccd95af0219f83764cffe399bab28de23a 100644 (file)
@@ -1,4 +1,4 @@
-flex = require('flex-base')
+local flex = require('flex-base')
 
 flex.set_main_tags{
     highway = {'always',
 
 flex.set_main_tags{
     highway = {'always',
@@ -32,7 +32,7 @@ flex.set_prefilters{delete_keys = {'building', 'source',
                                               'noexit', 'crossing', 'give_way', 'stop'},
                                    landuse = {'cemetry', 'no'},
                                    boundary = {'place'}},
                                               'noexit', 'crossing', 'give_way', 'stop'},
                                    landuse = {'cemetry', 'no'},
                                    boundary = {'place'}},
-                    extratag_keys = {'wikipedia', 'wikipedia:*', 'wikidata', 'capital', 'area'}
+                    extra_keys = {'wikipedia', 'wikipedia:*', 'wikidata', 'capital', 'area'}
                    }
 
 flex.set_name_tags{main = {'name', 'name:*',
                    }
 
 flex.set_name_tags{main = {'name', 'name:*',
@@ -65,3 +65,5 @@ flex.set_address_tags{main = {'addr:housenumber',
 
 
 flex.set_unused_handling{extra_keys = {'place'}}
 
 
 flex.set_unused_handling{extra_keys = {'place'}}
+
+return flex
index c593c34a2a0ea06fe49564841156141c5a8ac7ee..a1164acc5060f550c04dc64a1d678d71ed953e5a 100644 (file)
@@ -15,7 +15,7 @@ flex.set_prefilters{delete_keys = {'building', 'source', 'highway',
                                    'addr:street:name', 'addr:street:type'},
                     delete_tags = {landuse = {'cemetry', 'no'},
                                    boundary = {'place'}},
                                    'addr:street:name', 'addr:street:type'},
                     delete_tags = {landuse = {'cemetry', 'no'},
                                    boundary = {'place'}},
-                    extratag_keys = {'wikipedia', 'wikipedia:*', 'wikidata', 'capital'}
+                    extra_keys = {'wikipedia', 'wikipedia:*', 'wikidata', 'capital'}
                    }
 
 flex.set_name_tags{main = {'name', 'name:*',
                    }
 
 flex.set_name_tags{main = {'name', 'name:*',
@@ -42,3 +42,5 @@ flex.set_address_tags{extra = {'addr:*', 'is_in:*'},
                      }
 
 flex.set_unused_handling{extra_keys = {'place'}}
                      }
 
 flex.set_unused_handling{extra_keys = {'place'}}
+
+return flex
index 22c3d9c8392803192950fe5291387991efd7b747..d634d4a1ba9d0625aef597a39ef746427e9621f0 100644 (file)
@@ -1,4 +1,4 @@
-flex = require('flex-base')
+local flex = require('flex-base')
 
 flex.set_main_tags{
     building = 'fallback',
 
 flex.set_main_tags{
     building = 'fallback',
@@ -74,7 +74,7 @@ flex.set_prefilters{delete_keys = {'note', 'note:*', 'source', '*source', 'attri
                                    waterway = {'riverbank'},
                                    building = {'no'},
                                    boundary = {'place'}},
                                    waterway = {'riverbank'},
                                    building = {'no'},
                                    boundary = {'place'}},
-                    extratag_keys = {'*:prefix', '*:suffix', 'name:prefix:*', 'name:suffix:*',
+                    extra_keys = {'*:prefix', '*:suffix', 'name:prefix:*', 'name:suffix:*',
                                'name:etymology', 'name:signed', 'name:botanical',
                                'wikidata', '*:wikidata',
                                'addr:street:name', 'addr:street:type'}
                                'name:etymology', 'name:signed', 'name:botanical',
                                'wikidata', '*:wikidata',
                                'addr:street:name', 'addr:street:type'}
@@ -110,3 +110,5 @@ flex.set_address_tags{main = {'addr:housenumber',
 
 
 flex.set_unused_handling{delete_keys = {'tiger:*'}}
 
 
 flex.set_unused_handling{delete_keys = {'tiger:*'}}
+
+return flex
index e00313cebccf9284ef0dd977fb3821e5c18bbfff..a932fa50e38dc263790e3378a543ed6ffdb11726 100644 (file)
@@ -1,4 +1,4 @@
-flex = require('flex-base')
+local flex = require('flex-base')
 
 flex.set_main_tags{
     building = 'fallback',
 
 flex.set_main_tags{
     building = 'fallback',
@@ -74,7 +74,7 @@ flex.set_prefilters{delete_keys = {'note', 'note:*', 'source', '*source', 'attri
                                    waterway = {'riverbank'},
                                    building = {'no'},
                                    boundary = {'place'}},
                                    waterway = {'riverbank'},
                                    building = {'no'},
                                    boundary = {'place'}},
-                    extratag_keys = {'*:prefix', '*:suffix', 'name:prefix:*', 'name:suffix:*',
+                    extra_keys = {'*:prefix', '*:suffix', 'name:prefix:*', 'name:suffix:*',
                                'name:etymology', 'name:signed', 'name:botanical',
                                'wikidata', '*:wikidata',
                                'addr:street:name', 'addr:street:type'}
                                'name:etymology', 'name:signed', 'name:botanical',
                                'wikidata', '*:wikidata',
                                'addr:street:name', 'addr:street:type'}
@@ -110,3 +110,5 @@ flex.set_address_tags{main = {'addr:housenumber',
 
 
 flex.set_unused_handling{extra_keys = {'place'}}
 
 
 flex.set_unused_handling{extra_keys = {'place'}}
+
+return flex
index 34bff5725fa91d31bfb658b23c18007ba1428f0d..acadf01e7769c8229cba9dfff09c0f95c2755e56 100644 (file)
@@ -1,4 +1,4 @@
-flex = require('flex-base')
+local flex = require('flex-base')
 
 flex.set_main_tags{
     highway = {'always',
 
 flex.set_main_tags{
     highway = {'always',
@@ -32,7 +32,7 @@ flex.set_prefilters{delete_keys = {'building', 'source',
                                               'noexit', 'crossing', 'give_way', 'stop'},
                                    landuse = {'cemetry', 'no'},
                                    boundary = {'place'}},
                                               'noexit', 'crossing', 'give_way', 'stop'},
                                    landuse = {'cemetry', 'no'},
                                    boundary = {'place'}},
-                    extratag_keys = {'wikipedia', 'wikipedia:*', 'wikidata', 'capital', 'area'}
+                    extra_keys = {'wikipedia', 'wikipedia:*', 'wikidata', 'capital', 'area'}
                    }
 
 flex.set_name_tags{main = {'name', 'name:*',
                    }
 
 flex.set_name_tags{main = {'name', 'name:*',
@@ -64,3 +64,5 @@ flex.set_address_tags{main = {'addr:housenumber',
                      }
 
 flex.set_unused_handling{extra_keys = {'place'}}
                      }
 
 flex.set_unused_handling{extra_keys = {'place'}}
+
+return flex
diff --git a/test/bdd/osm2pgsql/import/custom_style.feature b/test/bdd/osm2pgsql/import/custom_style.feature
new file mode 100644 (file)
index 0000000..2ca95c9
--- /dev/null
@@ -0,0 +1,200 @@
+@DB
+Feature: Import with custom styles by osm2pgsql
+    Tests for the example customizations given in the documentation.
+
+    Scenario: Custom main tags
+        Given the lua style file
+            """
+            local flex = require('import-full')
+
+            flex.set_main_tags{
+                boundary = {administrative = 'named'},
+                highway = {'always', street_lamp = 'named'},
+                landuse = 'fallback'
+            }
+            """
+        When loading osm data
+            """
+            n10 Tboundary=administrative x0 y0
+            n11 Tboundary=administrative,name=Foo x0 y0
+            n12 Tboundary=electoral x0 y0
+            n13 Thighway=primary x0 y0
+            n14 Thighway=street_lamp x0 y0
+            n15 Thighway=primary,landuse=street x0 y0
+            """
+        Then place contains exactly
+            | object | class    | type           |
+            | N11    | boundary | administrative |
+            | N13    | highway  | primary        |
+            | N15    | highway  | primary        |
+
+    Scenario: Prefiltering tags
+        Given the lua style file
+            """
+            local flex = require('import-full')
+
+            flex.set_prefilters{
+                delete_keys = {'source', 'source:*'},
+                extra_tags = {amenity = {'yes', 'no'}}
+            }
+            flex.set_main_tags{
+                amenity = 'always',
+                tourism = 'always'
+            }
+            """
+        When loading osm data
+            """
+            n1 Tamenity=yes x0 y6
+            n2 Tamenity=hospital,source=survey x3 y6
+            n3 Ttourism=hotel,amenity=yes x0 y0
+            n4 Ttourism=hotel,amenity=telephone x0 y0
+            """
+        Then place contains exactly
+            | object     | extratags              |
+            | N2:amenity | -                      |
+            | N3:tourism | 'amenity': 'yes'       |
+            | N4:tourism | - |
+            | N4:amenity | - |
+
+    Scenario: Name tags
+        Given the lua style file
+            """
+            local flex = require('flex-base')
+
+            flex.set_main_tags{highway = {traffic_light = 'named'}}
+            flex.set_name_tags{main = {'name', 'name:*'},
+                               extra = {'ref'}
+                              }
+            """
+        When loading osm data
+            """
+            n1 Thighway=stop,name=Something x0 y0
+            n2 Thighway=traffic_light,ref=453-4 x0 y0
+            n3 Thighway=traffic_light,name=Greens x0 y0
+            n4 Thighway=traffic_light,name=Red,ref=45 x0 y0
+            """
+        Then place contains exactly
+            | object     | name                       |
+            | N3:highway | 'name': 'Greens'           |
+            | N4:highway | 'name': 'Red', 'ref': '45' |
+
+    Scenario: Address tags
+        Given the lua style file
+            """
+            local flex = require('import-full')
+
+            flex.set_address_tags{
+                main = {'addr:housenumber'},
+                extra = {'addr:*'},
+                postcode = {'postal_code', 'postcode', 'addr:postcode'},
+                country = {'country-code', 'ISO3166-1'}
+            }
+            """
+        When loading osm data
+            """
+            n1 Ttourism=hotel,addr:street=Foo x0 y0
+            n2 Taddr:housenumber=23,addr:street=Budd,postal_code=5567 x0 y0
+            n3 Taddr:street=None,addr:city=Where x0 y0
+            """
+        Then place contains exactly
+            | object     | type  | address |
+            | N1:tourism | hotel | 'street': 'Foo' |
+            | N2:place   | house | 'housenumber': '23', 'street': 'Budd', 'postcode': '5567' |
+
+    Scenario: Unused handling
+        Given the lua style file
+            """
+            local flex = require('import-full')
+
+            flex.set_address_tags{
+                main = {'addr:housenumber'},
+                extra = {'addr:*', 'tiger:county'}
+            }
+            flex.set_unused_handling{delete_keys = {'tiger:*'}}
+            """
+        When loading osm data
+            """
+            n1 Ttourism=hotel,tiger:county=Fargo x0 y0
+            n2 Ttourism=hotel,tiger:xxd=56,else=other x0 y0
+            """
+        Then place contains exactly
+            | object     | type  | address                 | extratags        |
+            | N1:tourism | hotel | 'tiger:county': 'Fargo' | -                |
+            | N2:tourism | hotel | -                       | 'else': 'other'  |
+
+    Scenario: Additional relation types
+        Given the lua style file
+            """
+            local flex = require('import-full')
+
+            flex.RELATION_TYPES['site'] = flex.relation_as_multipolygon
+            """
+        And the grid
+            | 1 | 2 |
+            | 4 | 3 |
+        When loading osm data
+            """
+            n1
+            n2
+            n3
+            n4
+            w1 Nn1,n2,n3,n4,n1
+            r1 Ttype=multipolygon,amenity=school Mw1@
+            r2 Ttype=site,amenity=school Mw1@
+            """
+        Then place contains exactly
+            | object     | type   |
+            | R1:amenity | school |
+            | R2:amenity | school |
+
+    Scenario: Exclude country relations
+        Given the lua style file
+            """
+            local flex = require('import-full')
+
+            function osm2pgsql.process_relation(object)
+                if object.tags.boundary ~= 'administrative' or object.tags.admin_level ~= '2' then
+                  flex.process_relation(object)
+                end
+            end
+            """
+       And the grid
+            | 1 | 2 |
+            | 4 | 3 |
+       When loading osm data
+            """
+            n1
+            n2
+            n3
+            n4
+            w1 Nn1,n2,n3,n4,n1
+            r1 Ttype=multipolygon,boundary=administrative,admin_level=4,name=Small Mw1@
+            r2 Ttype=multipolygon,boundary=administrative,admin_level=2,name=Big Mw1@
+            """
+        Then place contains exactly
+            | object      | type           |
+            | R1:boundary | administrative |
+
+    Scenario: Customize processing functions
+        Given the lua style file
+            """
+            local flex = require('import-full')
+
+            local original_process_tags = flex.process_tags
+
+            function flex.process_tags(o)
+                if o.object.tags.highway ~= nil and o.object.tags.access == 'no' then
+                    return
+                end
+
+                original_process_tags(o)
+            end
+            """
+        When loading osm data
+            """
+            n1 Thighway=residential x0 y0
+            n2 Thighway=residential,access=no x0 y0
+            """
+        Then place contains exactly
+            | object     | type        |
+            | N1:highway | residential |
index 7590b17c9c81c7a97c4f9cfc0bdf339db2cc4405..336fb707038bc02d775f84c85ba3165f3b583221 100644 (file)
@@ -49,6 +49,15 @@ def write_opl_file(opl, grid):
 
         return fd.name
 
 
         return fd.name
 
+@given('the lua style file')
+def lua_style_file(context):
+    """ Define a custom style file to use for the import.
+    """
+    style = Path(context.nominatim.website_dir.name) / 'custom.lua'
+    style.write_text(context.text)
+    context.nominatim.test_env['NOMINATIM_IMPORT_STYLE'] = str(style)
+
+
 @given(u'the ([0-9.]+ )?grid(?: with origin (?P<origin>.*))?')
 def define_node_grid(context, grid_step, origin):
     """
 @given(u'the ([0-9.]+ )?grid(?: with origin (?P<origin>.*))?')
 def define_node_grid(context, grid_step, origin):
     """