From caa8210112d608609f6825dff6893b6e9c83b7c6 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Tue, 27 Nov 2018 00:17:00 +0100 Subject: [PATCH] Switch to configurable style for osm2pgsql Includes the full style, which is the same as now (minus sidwalk exclusion) and a minimal style for boundaries only. --- lib/setup/SetupClass.php | 22 ++- settings/defaults.php | 1 + settings/import-admin.style | 64 +++++++ settings/import-full.style | 233 +++++++++++++++++++++++++ test/bdd/osm2pgsql/import/tags.feature | 12 +- utils/update.php | 2 +- 6 files changed, 315 insertions(+), 19 deletions(-) create mode 100644 settings/import-admin.style create mode 100644 settings/import-full.style diff --git a/lib/setup/SetupClass.php b/lib/setup/SetupClass.php index d7de0145..e6b07998 100755 --- a/lib/setup/SetupClass.php +++ b/lib/setup/SetupClass.php @@ -10,7 +10,7 @@ class SetupFunctions protected $iInstances; protected $sModulePath; protected $aDSNInfo; - protected $sVerbose; + protected $bVerbose; protected $sIgnoreErrors; protected $bEnableDiffUpdates; protected $bEnableDebugStatements; @@ -46,7 +46,7 @@ class SetupFunctions } // setting member variables based on command line options stored in $aCMDResult - $this->sVerbose = $aCMDResult['verbose']; + $this->bVerbose = $aCMDResult['verbose']; //setting default values which are not set by the update.php array if (isset($aCMDResult['ignore-errors'])) { @@ -189,6 +189,8 @@ class SetupFunctions fail("osm2pgsql not found in '$osm2pgsql'"); } + $osm2pgsql .= ' -S '.CONST_Import_Style; + if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) { $osm2pgsql .= ' --flat-nodes '.CONST_Osm2pgsql_Flatnode_File; } @@ -405,7 +407,7 @@ class SetupFunctions $sSQL .= " and not (class='place' and type='houses' and osm_type='W'"; $sSQL .= " and ST_GeometryType(geometry) = 'ST_LineString')"; $sSQL .= ' and ST_IsValid(geometry)'; - if ($this->sVerbose) echo "$sSQL\n"; + if ($this->bVerbose) echo "$sSQL\n"; if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) { fail(pg_last_error($aDBInstances[$i]->connection)); } @@ -417,7 +419,7 @@ class SetupFunctions $sSQL .= ' (osm_id, address, linegeo)'; $sSQL .= ' SELECT osm_id, address, geometry from place where '; $sSQL .= "class='place' and type='houses' and osm_type='W' and ST_GeometryType(geometry) = 'ST_LineString'"; - if ($this->sVerbose) echo "$sSQL\n"; + if ($this->bVerbose) echo "$sSQL\n"; if (!pg_send_query($aDBInstances[$iLoadThreads]->connection, $sSQL)) { fail(pg_last_error($aDBInstances[$iLoadThreads]->connection)); } @@ -708,7 +710,7 @@ class SetupFunctions if (!$bFound) array_push($aDropTables, $sTable); } foreach ($aDropTables as $sDrop) { - if ($this->sVerbose) echo "Dropping table $sDrop\n"; + if ($this->bVerbose) echo "Dropping table $sDrop\n"; @pg_query($this->oDB->connection, "DROP TABLE $sDrop CASCADE"); // ignore warnings/errors as they might be caused by a table having // been deleted already by CASCADE @@ -716,7 +718,7 @@ class SetupFunctions if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) { if (file_exists(CONST_Osm2pgsql_Flatnode_File)) { - if ($this->sVerbose) echo 'Deleting '.CONST_Osm2pgsql_Flatnode_File."\n"; + if ($this->bVerbose) echo 'Deleting '.CONST_Osm2pgsql_Flatnode_File."\n"; unlink(CONST_Osm2pgsql_Flatnode_File); } } @@ -740,7 +742,7 @@ class SetupFunctions runSQLScript( $sScript, $bfatal, - $this->sVerbose, + $this->bVerbose, $this->sIgnoreErrors ); } @@ -794,7 +796,7 @@ class SetupFunctions if (!file_exists($sFilename)) fail('unable to find '.$sFilename); $sCMD = 'psql -p '.$this->aDSNInfo['port'].' -d '.$this->aDSNInfo['database']; - if (!$this->sVerbose) { + if (!$this->bVerbose) { $sCMD .= ' -q'; } if (isset($this->aDSNInfo['hostspec'])) { @@ -857,6 +859,10 @@ class SetupFunctions private function runWithPgEnv($sCmd) { + if ($this->bVerbose) { + echo "Execute: $sCmd\n"; + } + $aProcEnv = null; if (isset($this->aDSNInfo['password'])) { diff --git a/settings/defaults.php b/settings/defaults.php index 2b3ddeb2..2a2aea36 100644 --- a/settings/defaults.php +++ b/settings/defaults.php @@ -51,6 +51,7 @@ if (isset($_GET['debug']) && $_GET['debug']) @define('CONST_Debug', true); @define('CONST_Wikipedia_Data_Path', CONST_ExtraDataPath); @define('CONST_Phrase_Config', CONST_BasePath.'/settings/phrase_settings.php'); @define('CONST_Address_Level_Config', CONST_BasePath.'/settings/address-levels.json'); +@define('CONST_Import_Style', CONST_BasePath.'/settings/import-full.style'); // osm2pgsql settings @define('CONST_Osm2pgsql_Flatnode_File', null); diff --git a/settings/import-admin.style b/settings/import-admin.style new file mode 100644 index 00000000..89a1eb16 --- /dev/null +++ b/settings/import-admin.style @@ -0,0 +1,64 @@ +[ +{ + "keys" : ["name:prefix", "name:suffix", "name:botanical", "*wikidata"], + "values" : { + "" : "skip" + } +}, +{ + "keys" : ["ref", "int_ref", "nat_ref", "reg_ref", "loc_ref", "old_ref", + "iata", "icao", "pcode"], + "values" : { + "" : "ref" + } +}, +{ + "keys" : ["name", "name:*", "int_name", "int_name:*", "nat_name", "nat_name:*", + "reg_name", "reg_name:*", "loc_name", "loc_name:*", + "old_name", "old_name:*", "alt_name", "alt_name:*", "alt_name_*", + "official_name", "official_name:*", "place_name", "place_name:*", + "short_name", "short_name:*", "brand"], + "values" : { + "" : "name" + } +}, +{ + "keys" : ["landuse"], + "values" : { + "cemetry" : "skip", + "" : "fallback" + } +}, +{ + "keys" : ["boundary"], + "values" : { + "administrative" : "main" + } +}, +{ + "keys" : ["place"], + "values" : { + "" : "main" + } +}, +{ + "keys" : ["postal_code", "postcode", "addr:postcode", + "tiger:zip_left", "tiger:zip_right"], + "values" : { + "" : "postcode" + } +}, +{ + "keys" : ["country_code", "ISO3166-1", "is_in:country_code", "is_in_country", + "addr:country", "addr:country", "addr:country_code"], + "values" : { + "" : "country" + } +}, +{ + "keys" : ["addr:*", "is_in:*", "tiger:county"], + "values" : { + "" : "address" + } +} +] diff --git a/settings/import-full.style b/settings/import-full.style new file mode 100644 index 00000000..f84b9305 --- /dev/null +++ b/settings/import-full.style @@ -0,0 +1,233 @@ +[ +{ + "keys" : ["*source"], + "values" : { + "" : "skip" + } +}, +{ + "keys" : ["name:prefix", "name:suffix", "name:botanical", "wikidata", + "*:wikidata"], + "values" : { + "" : "extra" + } +}, +{ + "keys" : ["ref", "int_ref", "nat_ref", "reg_ref", "loc_ref", "old_ref", + "iata", "icao", "pcode", "pcode:*"], + "values" : { + "" : "ref" + } +}, +{ + "keys" : ["name", "name:*", "int_name", "int_name:*", "nat_name", "nat_name:*", + "reg_name", "reg_name:*", "loc_name", "loc_name:*", + "old_name", "old_name:*", "alt_name", "alt_name:*", "alt_name_*", + "official_name", "official_name:*", "place_name", "place_name:*", + "short_name", "short_name:*", "brand"], + "values" : { + "" : "name" + } +}, +{ + "keys" : ["addr:housename"], + "values" : { + "" : "name,house" + } +}, +{ + "keys" : ["emergency"], + "values" : { + "fire_hydrant" : "skip", + "yes" : "skip", + "no" : "skip", + "" : "main" + } +}, +{ + "keys" : ["historic", "military"], + "values" : { + "no" : "skip", + "yes" : "skip", + "" : "main" + } +}, +{ + "keys" : ["natural"], + "values" : { + "yes" : "skip", + "no" : "skip", + "coastline" : "skip", + "" : "main,with_name" + } +}, +{ + "keys" : ["landuse"], + "values" : { + "cemetry" : "main,with_name", + "" : "main,fallback,with_name" + } +}, +{ + "keys" : ["highway"], + "values" : { + "no" : "skip", + "turning_circle" : "skip", + "mini_roundabout" : "skip", + "noexit" : "skip", + "crossing" : "skip", + "traffic_signals" : "main,with_name", + "service" : "main,with_name", + "cycleway" : "main,with_name", + "path" : "main,with_name", + "footway" : "main,with_name", + "steps" : "main,with_name", + "bridleway" : "main,with_name", + "track" : "main,with_name", + "byway": "main,with_name", + "motorway_link" : "main,with_name", + "trunk_link" : "main,with_name", + "primary_link" : "main,with_name", + "secondary_link" : "main,with_name", + "tertiary_link" : "main,with_name", + "" : "main" + } +}, +{ + "keys" : ["railway"], + "values" : { + "level_crossing" : "skip", + "no" : "skip", + "" : "main,with_name" + } +}, +{ + "keys" : ["man_made"], + "values" : { + "survey_point" : "skip", + "cutline" : "skip", + "" : "main" + } +}, +{ + "keys" : ["aerialway"], + "values" : { + "pylon" : "skip", + "no" : "skip", + "" : "main" + } +}, +{ + "keys" : ["boundary"], + "values" : { + "" : "main,with_name" + } +}, +{ + "keys" : ["amenity"], + "values" : { + "restaurant" : "main,operator", + "fuel" : "main,operator" + } +}, +{ + "keys" : ["aeroway", "amenity", "club", "craft", "leisure", + "office", "mountain_pass"], + "values" : { + "no" : "skip", + "" : "main" + } +}, +{ + "keys" : ["shop"], + "values" : { + "no" : "skip", + "" : "main,operator" + } +}, +{ + "keys" : ["tourism"], + "values" : { + "yes" : "skip", + "no" : "skip", + "" : "main,operator" + } +}, +{ + "keys" : ["bridge", "tunnel"], + "values" : { + "" : "main,with_name_key" + } +}, +{ + "keys" : ["waterway"], + "values" : { + "riverbank" : "skip", + "" : "main,with_name" + } +}, +{ + "keys" : ["place"], + "values" : { + "" : "main" + } +}, +{ + "keys" : ["junction"], + "values" : { + "" : "main,fallback,with_name" + } +}, +{ + "keys" : ["postal_code", "postcode", "addr:postcode", + "tiger:zip_left", "tiger:zip_right"], + "values" : { + "" : "postcode" + } +}, +{ + "keys" : ["country_code", "ISO3166-1", "is_in:country_code", "is_in_country", + "addr:country", "addr:country", "addr:country_code"], + "values" : { + "" : "country" + } +}, +{ + "keys" : ["addr:housenumber", "addr:conscriptionnumber", "addr:streetnumber"], + "values" : { + "" : "address,house" + } +}, +{ + "keys" : ["addr:*", "is_in:*", "tiger:county", "is_in"], + "values" : { + "" : "address" + } +}, +{ + "keys" : ["building"], + "values" : { + "no" : "skip", + "" : "main,fallback,with_name" + } +}, +{ + "keys" : ["tracktype", "traffic_calming", "service", "cuisine", "capital", + "dispensing", "religion", "denomination", "sport", + "internet_access", "lanes", "surface", "smoothness", "width", + "est_width", "incline", "opening_hours", "collection_times", + "service_times", "disused", "wheelchair", "sac_scale", + "trail_visibility", "mtb:scale", "mtb:description", "wood", + "drive_through", "drive_in", "access", "vehicle", "bicyle", + "foot", "goods", "hgv", "motor_vehicle", "motor_car", "oneway", + "date_on", "date_off", "day_on", "day_off", "hour_on", "hour_off", + "maxweight", "maxheight", "maxspeed", "fee", "toll", "charge", + "population", "description", "image", "attribution", "fax", + "email", "url", "website", "phone", "real_ale", "smoking", + "food", "camera", "brewery", "locality", "wikipedia", + "wikipedia:*", "access:*", "contact:*", "drink:*", "toll:*"], + "values" : { + "" : "extra" + } +} +] diff --git a/test/bdd/osm2pgsql/import/tags.feature b/test/bdd/osm2pgsql/import/tags.feature index 2e44e145..94506fec 100644 --- a/test/bdd/osm2pgsql/import/tags.feature +++ b/test/bdd/osm2pgsql/import/tags.feature @@ -231,14 +231,6 @@ Feature: Tag evaluation | boundary | administrative | | waterway | stream | - Scenario: Footways are not included if they are sidewalks - When loading osm data - """ - n2 Thighway=footway,name=To%20%Hell,footway=sidewalk - n23 Thighway=footway,name=x - """ - Then place has no entry for N2 - Scenario: named junctions are included if there is no other tag When loading osm data """ @@ -538,9 +530,9 @@ Feature: Tag evaluation Then place contains | object | class | type | | N10 | tourism | hotel | - | N12 | building| yes | + | N12 | building| shed | | N13 | building| yes | - | N14 | building| yes | + | N14 | place | postcode | And place has no entry for N10:building And place has no entry for N11 diff --git a/utils/update.php b/utils/update.php index c933d26e..3086a70b 100644 --- a/utils/update.php +++ b/utils/update.php @@ -63,7 +63,7 @@ if ($iCacheMemory + 500 > getTotalMemoryMB()) { $iCacheMemory = getCacheMemoryMB(); echo "WARNING: resetting cache memory to $iCacheMemory\n"; } -$sOsm2pgsqlCmd = CONST_Osm2pgsql_Binary.' -klas --number-processes 1 -C '.$iCacheMemory.' -O gazetteer -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port']; +$sOsm2pgsqlCmd = CONST_Osm2pgsql_Binary.' -klas --number-processes 1 -C '.$iCacheMemory.' -O gazetteer -S '.CONST_Import_Style.' -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port']; if (isset($aDSNInfo['username']) && $aDSNInfo['username']) { $sOsm2pgsqlCmd .= ' -U ' . $aDSNInfo['username']; } -- 2.39.5