From: Sarah Hoffmann Date: Sat, 25 Mar 2017 22:12:51 +0000 (+0100) Subject: adapt osm2pgsql tests for new address column X-Git-Tag: v3.0.0~42^2~6 X-Git-Url: https://git.openstreetmap.org./nominatim.git/commitdiff_plain/49b5e4dbad99de544b796ffddd487963b04bf7c7 adapt osm2pgsql tests for new address column --- diff --git a/test/bdd/environment.py b/test/bdd/environment.py index 58494deb..aca7929d 100644 --- a/test/bdd/environment.py +++ b/test/bdd/environment.py @@ -4,6 +4,7 @@ import os import psycopg2 import psycopg2.extras import subprocess +import tempfile from nose.tools import * # for assert functions from sys import version_info as python_version @@ -88,17 +89,20 @@ class NominatimEnvironment(object): conn.commit() conn.close() - # execute osm2pgsql on an empty file to get the right tables - osm2pgsql = os.path.join(self.build_dir, 'osm2pgsql', 'osm2pgsql') - proc = subprocess.Popen([osm2pgsql, '-lsc', '-r', 'xml', - '-O', 'gazetteer', '-d', self.template_db, '-'], - cwd=self.build_dir, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - [outstr, errstr] = proc.communicate(input=b'') - logger.debug("running osm2pgsql for template: %s\n%s\n%s" % (osm2pgsql, outstr, errstr)) - self.run_setup_script('create-functions', 'create-tables', - 'create-partition-tables', 'create-partition-functions', - 'load-data', 'create-search-indices') + # execute osm2pgsql import on an empty file to get the right tables + with tempfile.NamedTemporaryFile(dir='/tmp', suffix='.xml') as fd: + fd.write(b'') + fd.flush() + self.run_setup_script('import-data', + 'ignore-errors', + 'create-functions', + 'create-tables', + 'create-partition-tables', + 'create-partition-functions', + 'load-data', + 'create-search-indices', + osm_file=fd.name, + osm2pgsql_cache='200') def setup_api_db(self, context): self.write_nominatim_config(self.api_test_db) diff --git a/test/bdd/osm2pgsql/import/tags.feature b/test/bdd/osm2pgsql/import/tags.feature index d81b6c72..7db8d629 100644 --- a/test/bdd/osm2pgsql/import/tags.feature +++ b/test/bdd/osm2pgsql/import/tags.feature @@ -102,8 +102,8 @@ Feature: Tag evaluation n1 T=,name=real """ Then place contains - | object | name | - | N1 | 'name' : 'real' | + | object | class | type | name | + | N1 | | | 'name' : 'real' | Examples: | key | value | @@ -300,7 +300,7 @@ Feature: Tag evaluation n11 T= """ Then place contains - | object | class | type | postcode | + | object | class | type | addr+postcode | | N10 | highway | secondary | | | N11 | place | postcode | | And place has no entry for N10:place @@ -320,9 +320,9 @@ Feature: Tag evaluation n20 Tamenity=hospital,addr:place=Foo%20%Town """ Then place contains - | object | class | type | street | addr_place | - | N10 | amenity | hospital | Foo St | None | - | N20 | amenity | hospital | - | Foo Town | + | object | class | type | addr+street | addr+place | + | N10 | amenity | hospital | Foo St | - | + | N20 | amenity | hospital | - | Foo Town | Scenario Outline: Import of country @@ -331,16 +331,16 @@ Feature: Tag evaluation n10 Tplace=village,= """ Then place contains - | object | class | type | country_code | + | object | class | type | addr+country | | N10 | place | village | | Examples: - | key | value | - | country_code | us | - | ISO3166-1 | XX | - | is_in:country_code | __ | - | addr:country | .. | - | addr:country_code | cv | + | key | value | + | country_code | us | + | ISO3166-1 | XX | + | is_in:country_code | __ | + | addr:country | .. | + | addr:country_code | cv | Scenario Outline: Ignore country codes with wrong length When loading osm data @@ -348,7 +348,7 @@ Feature: Tag evaluation n10 Tplace=village,country_code= """ Then place contains - | object | class | type | country_code | + | object | class | type | addr+country | | N10 | place | village | - | Examples: @@ -368,24 +368,11 @@ Feature: Tag evaluation n13 Tbuilding=yes,addr:conscriptionnumber=3,addr:streetnumber=111 """ Then place contains - | object | class | type | housenumber | - | N10 | building | yes | 4b | - | N11 | building | yes | 003 | - | N12 | building | yes | 2345 | - | N13 | building | yes | 3/111 | - - Scenario: Import of address interpolations - When loading osm data - """ - n10 Taddr:interpolation=odd - n11 Taddr:housenumber=10,addr:interpolation=odd - n12 Taddr:interpolation=odd,addr:housenumber=23 - """ - Then place contains - | object | class | type | housenumber | - | N10 | place | houses | odd | - | N11 | place | houses | odd | - | N12 | place | houses | odd | + | object | class | type | address | + | N10 | building | yes | 'housenumber' : '4b' | + | N11 | building | yes | 'conscriptionnumber' : '003' | + | N12 | building | yes | 'streetnumber' : '2345' | + | N13 | building | yes | 'conscriptionnumber' : '3', 'streetnumber' : '111' | Scenario: Shorten tiger:county tags When loading osm data @@ -395,26 +382,26 @@ Feature: Tag evaluation n12 Tplace=village,tiger:county=Feebourgh """ Then place contains - | object | class | type | isin | + | object | class | type | addr+tiger:county | | N10 | place | village | Feebourgh county | - | N11 | place | village | Alabama,Feebourgh county | + | N11 | place | village | Feebourgh county | | N12 | place | village | Feebourgh county | Scenario Outline: Import of address tags When loading osm data """ - n10 Tplace=village,= + n10 Tplace=village,addr:= + n11 Tplace=village,is_in:= """ Then place contains - | object | class | type | isin | - | N10 | place | village | | + | object | class | type | address | + | N10 | place | village | '' : '' | Examples: - | key | value | - | is_in:country | Xanadu | - | addr:suburb | hinein | - | addr:city | Sydney | - | addr:state | Jura | + | key | value | + | suburb | hinein | + | city | Sydney | + | state | Jura | Scenario: Import of isin tags with space When loading osm data @@ -423,9 +410,9 @@ Feature: Tag evaluation n11 Tplace=village,addr:county=le%20%havre """ Then place contains - | object | class | type | isin | - | N10 | place | village | Stockholm, Sweden | - | N11 | place | village | le havre | + | object | class | type | address | + | N10 | place | village | 'is_in' : 'Stockholm, Sweden' | + | N11 | place | village | 'county' : 'le havre' | Scenario: Import of admin level When loading osm data @@ -438,8 +425,8 @@ Feature: Tag evaluation Then place contains | object | class | type | admin_level | | N10 | amenity | hospital | 3 | - | N11 | amenity | hospital | 100 | - | N12 | amenity | hospital | 100 | + | N11 | amenity | hospital | 15 | + | N12 | amenity | hospital | 15 | | N13 | amenity | hospital | 3 | Scenario Outline: Import of extra tags @@ -550,5 +537,5 @@ Feature: Tag evaluation n290393920 Taddr:city=Perpignan,addr:country=FR,addr:housenumber=43\,addr:postcode=66000,addr:street=Rue%20%Pierre%20%Constant%20%d`Ivry,source=cadastre-dgi-fr%20%source%20%:%20%Direction%20%Générale%20%des%20%Impôts%20%-%20%Cadastre%20%;%20%mise%20%à%20%jour%20%:2008 """ Then place contains - | object | class | type | housenumber | - | N290393920 | place | house| 43\ | + | object | class | type | address | + | N290393920 | place | house| 'city' : 'Perpignan', 'country' : 'FR', 'housenumber' : '43\\', 'postcode' : '66000', 'street' : 'Rue Pierre Constant d`Ivry' | diff --git a/test/bdd/osm2pgsql/update/relation.feature b/test/bdd/osm2pgsql/update/relation.feature index 0eccd3e6..83ce9129 100644 --- a/test/bdd/osm2pgsql/update/relation.feature +++ b/test/bdd/osm2pgsql/update/relation.feature @@ -137,7 +137,7 @@ Feature: Update of relations by osm2pgsql r1 Ttype=boundary,boundary=administrative,name=Foo,country_code=XX,admin_level=2 Mw1@ """ Then place contains - | object | country_code | name | + | object | addr+country | name | | R1 | XX | 'name' : 'Foo' | Scenario: Country boundary names are extended when country_code known @@ -154,6 +154,6 @@ Feature: Update of relations by osm2pgsql r1 Ttype=boundary,boundary=administrative,name=Foo,country_code=ch,admin_level=2 Mw1@ """ Then place contains - | object | country_code | name+name:de | name+name | + | object | addr+country | name+name:de | name+name | | R1 | ch | Schweiz | Foo | diff --git a/test/bdd/steps/db_ops.py b/test/bdd/steps/db_ops.py index df1d1688..c4384f07 100644 --- a/test/bdd/steps/db_ops.py +++ b/test/bdd/steps/db_ops.py @@ -20,9 +20,10 @@ class PlaceColumn: self.add_hstore('name', key[5:], value) elif key.startswith('extra+'): self.add_hstore('extratags', key[6:], value) + elif key.startswith('addr+'): + self.add_hstore('address', key[6:], value) else: - assert_in(key, ('class', 'type', 'street', 'addr_place', - 'isin', 'postcode')) + assert_in(key, ('class', 'type')) self.columns[key] = None if value == '' else value def set_key_name(self, value): @@ -39,10 +40,16 @@ class PlaceColumn: self.columns['admin_level'] = int(value) def set_key_housenr(self, value): - self.columns['housenumber'] = None if value == '' else value + self.add_hstore('address', 'housenumber', None if value == '' else value) + + def set_key_street(self, value): + self.add_hstore('address', 'street', None if value == '' else value) + + def set_key_addr_place(self, value): + self.add_hstore('address', 'place', None if value == '' else value) def set_key_country(self, value): - self.columns['country_code'] = None if value == '' else value + self.add_hstore('address', 'country', None if value == '' else value) def set_key_geometry(self, value): self.geometry = self.context.osm.parse_geometry(value, self.context.scene) @@ -223,15 +230,11 @@ def import_and_index_data_from_place_table(context): cur = context.db.cursor() cur.execute( """insert into placex (osm_type, osm_id, class, type, name, admin_level, - housenumber, street, addr_place, isin, postcode, country_code, extratags, - geometry) + address, extratags, geometry) select * from place where not (class='place' and type='houses' and osm_type='W')""") cur.execute( - """insert into location_property_osmline - (osm_id, interpolationtype, street, addr_place, - postcode, calculated_country_code, linegeo) - SELECT osm_id, housenumber, street, addr_place, - postcode, country_code, geometry from place + """insert into location_property_osmline (osm_id, address, linegeo) + SELECT osm_id, address, geometry from place WHERE class='place' and type='houses' and osm_type='W' and ST_GeometryType(geometry) = 'ST_LineString'""") context.db.commit() @@ -338,7 +341,7 @@ def check_placex_contents(context, exact): expected_content.add((res['osm_type'], res['osm_id'], res['class'])) for h in row.headings: msg = "%s: %s" % (row['object'], h) - if h in ('name', 'extratags'): + if h in ('name', 'extratags', 'address'): if row[h] == '-': assert_is_none(res[h], msg) else: @@ -348,6 +351,12 @@ def check_placex_contents(context, exact): assert_equals(res['name'][h[5:]], row[h], msg) elif h.startswith('extratags+'): assert_equals(res['extratags'][h[10:]], row[h], msg) + elif h.startswith('addr+'): + if row[h] == '-': + if res['address'] is not None: + assert_not_in(h[5:], res['address']) + else: + assert_equals(res['address'][h[5:]], row[h], msg) elif h in ('linked_place_id', 'parent_place_id'): if row[h] == '0': assert_equals(0, res[h], msg) diff --git a/test/bdd/steps/osm_data.py b/test/bdd/steps/osm_data.py index d0d8d892..f56a106a 100644 --- a/test/bdd/steps/osm_data.py +++ b/test/bdd/steps/osm_data.py @@ -79,14 +79,10 @@ def update_from_osm_file(context): cur = context.db.cursor() cur.execute("""insert into placex (osm_type, osm_id, class, type, name, - admin_level, housenumber, street, addr_place, isin, postcode, - country_code, extratags, geometry) select * from place""") + admin_level, address, extratags, geometry) select * from place""") cur.execute( - """insert into location_property_osmline - (osm_id, interpolationtype, street, addr_place, - postcode, calculated_country_code, linegeo) - SELECT osm_id, housenumber, street, addr_place, - postcode, country_code, geometry from place + """insert into location_property_osmline (osm_id, address, linegeo) + SELECT osm_id, address, geometry from place WHERE class='place' and type='houses' and osm_type='W' and ST_GeometryType(geometry) = 'ST_LineString'""") context.db.commit()