From: Brian Quinion Date: Thu, 23 Jan 2014 14:40:42 +0000 (+0000) Subject: Merge branch 'master' of github.com:twain47/Nominatim X-Git-Tag: v2.2.0~17 X-Git-Url: https://git.openstreetmap.org./nominatim.git/commitdiff_plain/e591269fd8c357b922f2809d21f726a2469788b3?hp=47fdb33adf065b46df7104675e0251dd2ffb9bf4 Merge branch 'master' of github.com:twain47/Nominatim --- diff --git a/AUTHORS b/AUTHORS index 704d6bed..1769f616 100644 --- a/AUTHORS +++ b/AUTHORS @@ -1 +1,14 @@ -Nominatim was written by Brian Quinion. +Nominatim was written by: + + Brian Quinion + Sarah Hoffmann + Frederik Ramm + Michael Spreng + Daniele Forsi + mfn + Grant Slater + Andree Klattenhoff + IrlJidel + appelflap + b3nn0 + Spin0us diff --git a/ChangeLog b/ChangeLog index e69de29b..19a6a760 100644 --- a/ChangeLog +++ b/ChangeLog @@ -0,0 +1,28 @@ +2.0.1 + + * delete outdated entries from location_area_country + * remove remaining uses of INTEGER, to allow node ids larger than 2^31 + +2.1 + + * update to openlayers 2.12 (now custom built) + * update fallback OSM boundaries + * add support for postgresql 9.2/9.3 and postgis 2.x + * add structured queries + * add options for polygon output in various formats + (geojson, svg, kml, postgis text) + * maintenance functions for deleting objects and updating regions + (plcae_force_update/place_force_delete) + * web view for polygons that need deleting + * rate limiting using memcache + * improve layout of details page + * add support for boundary:postal_code + * full CORS support + * improve parenting of POIs + * support for extract daily diffs from Geofabrik + * support for addresses without a street + (addr:place and conscription number house numbers) + * improve layout of word and search_name_* tables + * support for US ZIP+4 codes + * refactoring of front-end PHP code + * lots of smaller bug fixes diff --git a/configure.ac b/configure.ac index e89b2217..a92bbeeb 100644 --- a/configure.ac +++ b/configure.ac @@ -1,4 +1,4 @@ -AC_INIT(Nominatim,2.1) +AC_INIT(Nominatim,2.2) if git rev-parse HEAD 2>/dev/null >/dev/null; then AC_SUBST([PACKAGE_VERSION], [$PACKAGE_VERSION-git-`git rev-parse --short HEAD`]) fi diff --git a/lib/Geocode.php b/lib/Geocode.php index d5612240..e3fa25b6 100644 --- a/lib/Geocode.php +++ b/lib/Geocode.php @@ -33,6 +33,7 @@ protected $iMinAddressRank = 0; protected $iMaxAddressRank = 30; protected $aAddressRankList = array(); + protected $exactMatchCache = array(); protected $sAllowedTypesSQLList = false; @@ -277,13 +278,13 @@ // Get the details for display (is this a redundant extra step?) $sPlaceIDs = join(',',$aPlaceIDs); - $sSQL = "select osm_type,osm_id,class,type,admin_level,rank_search,rank_address,min(place_id) as place_id,calculated_country_code as country_code,"; + $sSQL = "select osm_type,osm_id,class,type,admin_level,rank_search,rank_address,min(place_id) as place_id, min(parent_place_id) as parent_place_id, calculated_country_code as country_code,"; $sSQL .= "get_address_by_language(place_id, $sLanguagePrefArraySQL) as langaddress,"; $sSQL .= "get_name_by_language(name, $sLanguagePrefArraySQL) as placename,"; $sSQL .= "get_name_by_language(name, ARRAY['ref']) as ref,"; $sSQL .= "avg(ST_X(centroid)) as lon,avg(ST_Y(centroid)) as lat, "; $sSQL .= "coalesce(importance,0.75-(rank_search::float/40)) as importance, "; - $sSQL .= "(select max(p.importance*(p.rank_address+2)) from place_addressline s, placex p where s.place_id = min(placex.place_id) and p.place_id = s.address_place_id and s.isaddress and p.importance is not null) as addressimportance, "; + $sSQL .= "(select max(p.importance*(p.rank_address+2)) from place_addressline s, placex p where s.place_id = min(CASE WHEN placex.rank_search < 28 THEN placex.place_id ELSE placex.parent_place_id END) and p.place_id = s.address_place_id and s.isaddress and p.importance is not null) as addressimportance, "; $sSQL .= "(extratags->'place') as extra_place "; $sSQL .= "from placex where place_id in ($sPlaceIDs) "; $sSQL .= "and (placex.rank_address between $this->iMinAddressRank and $this->iMaxAddressRank "; @@ -302,26 +303,26 @@ if (30 >= $this->iMinAddressRank && 30 <= $this->iMaxAddressRank) { $sSQL .= " union "; - $sSQL .= "select 'T' as osm_type,place_id as osm_id,'place' as class,'house' as type,null as admin_level,30 as rank_search,30 as rank_address,min(place_id) as place_id,'us' as country_code,"; + $sSQL .= "select 'T' as osm_type,place_id as osm_id,'place' as class,'house' as type,null as admin_level,30 as rank_search,30 as rank_address,min(place_id) as place_id, min(parent_place_id) as parent_place_id,'us' as country_code,"; $sSQL .= "get_address_by_language(place_id, $sLanguagePrefArraySQL) as langaddress,"; $sSQL .= "null as placename,"; $sSQL .= "null as ref,"; $sSQL .= "avg(ST_X(centroid)) as lon,avg(ST_Y(centroid)) as lat, "; $sSQL .= "-0.15 as importance, "; - $sSQL .= "(select max(p.importance*(p.rank_address+2)) from place_addressline s, placex p where s.place_id = min(location_property_tiger.place_id) and p.place_id = s.address_place_id and s.isaddress and p.importance is not null) as addressimportance, "; + $sSQL .= "(select max(p.importance*(p.rank_address+2)) from place_addressline s, placex p where s.place_id = min(location_property_tiger.parent_place_id) and p.place_id = s.address_place_id and s.isaddress and p.importance is not null) as addressimportance, "; $sSQL .= "null as extra_place "; $sSQL .= "from location_property_tiger where place_id in ($sPlaceIDs) "; $sSQL .= "and 30 between $this->iMinAddressRank and $this->iMaxAddressRank "; $sSQL .= "group by place_id"; if (!$this->bDeDupe) $sSQL .= ",place_id"; $sSQL .= " union "; - $sSQL .= "select 'L' as osm_type,place_id as osm_id,'place' as class,'house' as type,null as admin_level,30 as rank_search,30 as rank_address,min(place_id) as place_id,'us' as country_code,"; + $sSQL .= "select 'L' as osm_type,place_id as osm_id,'place' as class,'house' as type,null as admin_level,30 as rank_search,30 as rank_address,min(place_id) as place_id, min(parent_place_id) as parent_place_id,'us' as country_code,"; $sSQL .= "get_address_by_language(place_id, $sLanguagePrefArraySQL) as langaddress,"; $sSQL .= "null as placename,"; $sSQL .= "null as ref,"; $sSQL .= "avg(ST_X(centroid)) as lon,avg(ST_Y(centroid)) as lat, "; $sSQL .= "-0.10 as importance, "; - $sSQL .= "(select max(p.importance*(p.rank_address+2)) from place_addressline s, placex p where s.place_id = min(location_property_aux.place_id) and p.place_id = s.address_place_id and s.isaddress and p.importance is not null) as addressimportance, "; + $sSQL .= "(select max(p.importance*(p.rank_address+2)) from place_addressline s, placex p where s.place_id = min(location_property_aux.parent_place_id) and p.place_id = s.address_place_id and s.isaddress and p.importance is not null) as addressimportance, "; $sSQL .= "null as extra_place "; $sSQL .= "from location_property_aux where place_id in ($sPlaceIDs) "; $sSQL .= "and 30 between $this->iMinAddressRank and $this->iMaxAddressRank "; @@ -342,6 +343,36 @@ return $aSearchResults; } + /* Perform the actual query lookup. + + Returns an ordered list of results, each with the following fields: + osm_type: type of corresponding OSM object + N - node + W - way + R - relation + P - postcode (internally computed) + osm_id: id of corresponding OSM object + class: general object class (corresponds to tag key of primary OSM tag) + type: subclass of object (corresponds to tag value of primary OSM tag) + admin_level: see http://wiki.openstreetmap.org/wiki/Admin_level + rank_search: rank in search hierarchy + (see also http://wiki.openstreetmap.org/wiki/Nominatim/Development_overview#Country_to_street_level) + rank_address: rank in address hierarchy (determines orer in address) + place_id: internal key (may differ between different instances) + country_code: ISO country code + langaddress: localized full address + placename: localized name of object + ref: content of ref tag (if available) + lon: longitude + lat: latitude + importance: importance of place based on Wikipedia link count + addressimportance: cumulated importance of address elements + extra_place: type of place (for admin boundaries, if there is a place tag) + aBoundingBox: bounding Box + label: short description of the object class/type (English only) + name: full name (currently the same as langaddress) + foundorder: secondary ordering for places with same importance + */ function lookup() { if (!$this->sQuery && !$this->aStructuredQuery) return false; @@ -1148,12 +1179,16 @@ $aOrder[] = "$sImportanceSQL DESC"; if (sizeof($aSearch['aFullNameAddress'])) { - $aOrder[] = '(select count(*) from (select unnest(ARRAY['.join($aSearch['aFullNameAddress'],",").']) INTERSECT select unnest(nameaddress_vector))s) DESC'; + $sExactMatchSQL = '(select count(*) from (select unnest(ARRAY['.join($aSearch['aFullNameAddress'],",").']) INTERSECT select unnest(nameaddress_vector))s) as exactmatch'; + $aOrder[] = 'exactmatch DESC'; + } else { + $sExactMatchSQL = '0::int as exactmatch'; } if (sizeof($aTerms)) { - $sSQL = "select place_id"; + $sSQL = "select place_id, "; + $sSQL .= $sExactMatchSQL; $sSQL .= " from search_name"; $sSQL .= " where ".join(' and ',$aTerms); $sSQL .= " order by ".join(', ',$aOrder); @@ -1181,6 +1216,7 @@ //if ($aViewBoxRow['in_small'] == 't') $bViewBoxMatch = 1; //else if ($aViewBoxRow['in_large'] == 't') $bViewBoxMatch = 2; $aPlaceIDs[] = $aViewBoxRow['place_id']; + $this->exactMatchCache[$aViewBoxRow['place_id']] = $aViewBoxRow['exactmatch']; } } //var_Dump($aPlaceIDs); @@ -1586,7 +1622,24 @@ $aResult['importance'] = $aResult['importance'] + ($iCountWords*0.1); // 0.1 is a completely arbitrary number but something in the range 0.1 to 0.5 would seem right $aResult['name'] = $aResult['langaddress']; - $aResult['foundorder'] = -$aResult['addressimportance']; + // secondary ordering (for results with same importance (the smaller the better): + // - approximate importance of address parts + $aResult['foundorder'] = -$aResult['addressimportance']/10; + // - number of exact matches from the query + if (isset($this->exactMatchCache[$aResult['place_id']])) + $aResult['foundorder'] -= $this->exactMatchCache[$aResult['place_id']]; + else if (isset($this->exactMatchCache[$aResult['parent_place_id']])) + $aResult['foundorder'] -= $this->exactMatchCache[$aResult['parent_place_id']]; + // - importance of the class/type + if (isset($aClassType[$aResult['class'].':'.$aResult['type']]['importance']) + && $aClassType[$aResult['class'].':'.$aResult['type']]['importance']) + { + $aResult['foundorder'] = $aResult['foundorder'] + 0.000001 * $aClassType[$aResult['class'].':'.$aResult['type']]['importance']; + } + else + { + $aResult['foundorder'] = $aResult['foundorder'] + 0.001; + } $aSearchResults[$iResNum] = $aResult; } uasort($aSearchResults, 'byImportance'); diff --git a/lib/ReverseGeocode.php b/lib/ReverseGeocode.php index 8503e1eb..cb8c0c6b 100644 --- a/lib/ReverseGeocode.php +++ b/lib/ReverseGeocode.php @@ -95,8 +95,8 @@ $sSQL .= ' and rank_search != 28 and rank_search >= '.$iMaxRank; $sSQL .= ' and (name is not null or housenumber is not null)'; $sSQL .= ' and class not in (\'waterway\',\'railway\',\'tunnel\',\'bridge\')'; - $sSQL .= ' and (ST_GeometryType(geometry) not in (\'ST_Polygon\',\'ST_MultiPolygon\') '; $sSQL .= ' and indexed_status = 0 '; + $sSQL .= ' and (ST_GeometryType(geometry) not in (\'ST_Polygon\',\'ST_MultiPolygon\') '; $sSQL .= ' OR ST_DWithin('.$sPointSQL.', centroid, '.$fSearchDiam.'))'; $sSQL .= ' ORDER BY ST_distance('.$sPointSQL.', geometry) ASC limit 1'; if (CONST_Debug) var_dump($sSQL); diff --git a/lib/init-website.php b/lib/init-website.php index bcf3ebf7..e47b73fb 100644 --- a/lib/init-website.php +++ b/lib/init-website.php @@ -55,8 +55,7 @@ if (strpos(CONST_BlockedIPs, ','.$_SERVER["REMOTE_ADDR"].',') !== false || $fBucketVal >= CONST_ConnectionBucket_BlockLimit) { echo "Your IP has been blocked. \n"; - echo "Please create a nominatim trac ticket (http://trac.openstreetmap.org/newticket?component=nominatim) to request this to be removed. \n"; - echo "Information on the Nominatim usage policy can be found here: http://wiki.openstreetmap.org/wiki/Nominatim#Usage_Policy \n"; + echo CONST_BlockMessage; exit; } diff --git a/lib/template/details-html.php b/lib/template/details-html.php index ba41b931..83df4814 100644 --- a/lib/template/details-html.php +++ b/lib/template/details-html.php @@ -80,7 +80,7 @@ echo '

'; if ($aPointDetails['icon']) { - echo ''; + echo ''.$aPointDetails['icon'].''; } echo $aPointDetails['localname'].'

'; echo '
'; diff --git a/lib/template/search-xml.php b/lib/template/search-xml.php index dfcf1375..693330bb 100644 --- a/lib/template/search-xml.php +++ b/lib/template/search-xml.php @@ -80,7 +80,7 @@ echo " display_name='".htmlspecialchars($aResult['name'], ENT_QUOTES)."'"; echo " class='".htmlspecialchars($aResult['class'])."'"; - echo " type='".htmlspecialchars($aResult['type'])."'"; + echo " type='".htmlspecialchars($aResult['type'], ENT_QUOTES)."'"; echo " importance='".htmlspecialchars($aResult['importance'])."'"; if (isset($aResult['icon']) && $aResult['icon']) { diff --git a/munin/nominatim_importlag b/munin/nominatim_importlag index 9f401311..0960ff5a 100755 --- a/munin/nominatim_importlag +++ b/munin/nominatim_importlag @@ -5,26 +5,42 @@ # Can be configured through libpq environment variables, for example # PGUSER, PGDATABASE, etc. See man page of psql for more information # -# Parameters: +# To configure munin for a default installation, add these lines to +# the file /etc/munin/plugin-conf.d/munin-node or in any file in the +# directory /etc/munin/plugin-conf.d/ +# +# [nominatim_*] +# user www-data +# env.PGUSER www-data +# env.PGPORT 5432 +# env.PGDATABASE nominatim +# env.age_warning 21600 +# env.age_critical 86400 + +# Parameters: # # config (required) # autoconf (optional - used by munin-config) # - + +. $MUNIN_LIBDIR/plugins/plugin.sh + if [ "$1" = "config" ]; then - - echo 'graph_title Data import lag' - echo 'graph_args --base 1000 -l 0' - echo 'graph_vlabel minutes' - echo 'graph_category nominatim' - echo 'age.label DB import age' - echo 'age.type GAUGE' + + echo 'graph_title Data import lag' + echo 'graph_args --base 1000 -l 0' + echo 'graph_vlabel minutes' + echo 'graph_category nominatim' + echo 'age.label DB import age' + echo 'age.type GAUGE' echo 'age.cdef age,60,/' + print_warning age + print_critical age exit 0 fi - + delay=`psql -c "copy (select extract(epoch from timezone('utc', now())-lastimportdate)::int from import_status) to stdout"` - - + + echo "age.value $delay" diff --git a/nominatim/nominatim.c b/nominatim/nominatim.c index d0eeff22..7a9e6e9e 100644 --- a/nominatim/nominatim.c +++ b/nominatim/nominatim.c @@ -115,7 +115,7 @@ int main(int argc, char *argv[]) PGconn *conn; - fprintf(stderr, "nominatim SVN version %s\n\n", VERSION); + fprintf(stderr, "nominatim version %s\n\n", VERSION); while (1) { diff --git a/osm2pgsql b/osm2pgsql index 0005c331..86d1e0ed 160000 --- a/osm2pgsql +++ b/osm2pgsql @@ -1 +1 @@ -Subproject commit 0005c3312cdbddb51fdb749315d4c7df59daa82a +Subproject commit 86d1e0ede1e01593a3d1b4f2284e9c0bab873ce4 diff --git a/settings/settings.php b/settings/settings.php index 0ce34ad2..27b3446b 100644 --- a/settings/settings.php +++ b/settings/settings.php @@ -55,6 +55,7 @@ @define('CONST_ClosedForIndexingExceptionIPs', ''); @define('CONST_BlockedIPs', ''); @define('CONST_BulkUserIPs', ''); + @define('CONST_BlockMessage', ''); // additional info to show for blocked IPs @define('CONST_Website_BaseURL', 'http://'.php_uname('n').'/'); @define('CONST_Tile_Default', 'Mapnik'); diff --git a/sql/functions.sql b/sql/functions.sql index 6d2ae5f8..1883d998 100644 --- a/sql/functions.sql +++ b/sql/functions.sql @@ -347,7 +347,7 @@ BEGIN w := getorcreate_word_id(s); - IF NOT (ARRAY[w] <@ result) THEN + IF w IS NOT NULL AND NOT (ARRAY[w] <@ result) THEN result := result || w; END IF; @@ -415,7 +415,7 @@ BEGIN w := getorcreate_word_id(s); - IF NOT (ARRAY[w] <@ result) THEN + IF w IS NOT NULL AND NOT (ARRAY[w] <@ result) THEN result := result || w; END IF; @@ -938,10 +938,6 @@ BEGIN --DEBUG: RAISE WARNING '% %',NEW.osm_type,NEW.osm_id; -- just block these - IF NEW.class = 'highway' and NEW.type in ('turning_circle','traffic_signals','mini_roundabout','noexit','crossing') THEN --- RAISE WARNING 'bad highway %',NEW.osm_id; - RETURN null; - END IF; IF NEW.class in ('landuse','natural') and NEW.name is null THEN -- RAISE WARNING 'empty landuse %',NEW.osm_id; RETURN null; @@ -1127,7 +1123,11 @@ BEGIN return NULL; END IF; NEW.rank_search := NEW.admin_level * 2; - NEW.rank_address := NEW.rank_search; + IF NEW.type = 'administrative' THEN + NEW.rank_address := NEW.rank_search; + ELSE + NEW.rank_address := 0; + END IF; ELSEIF NEW.class = 'landuse' AND ST_GeometryType(NEW.geometry) in ('ST_Polygon','ST_MultiPolygon') THEN NEW.rank_search := 22; NEW.rank_address := NEW.rank_search; @@ -1245,7 +1245,7 @@ BEGIN -- Note: won't work on initial import because the classtype tables -- do not yet exist. It won't hurt either. classtable := 'place_classtype_' || NEW.class || '_' || NEW.type; - SELECT count(*)>0 FROM pg_tables WHERE tablename = classtable INTO result; + SELECT count(*)>0 FROM pg_tables WHERE tablename = classtable and schemaname = current_schema() INTO result; IF result THEN EXECUTE 'INSERT INTO ' || classtable::regclass || ' (place_id, centroid) VALUES ($1,$2)' USING NEW.place_id, ST_Centroid(NEW.geometry); @@ -1987,7 +1987,7 @@ BEGIN -- remove from tables for special search classtable := 'place_classtype_' || OLD.class || '_' || OLD.type; - SELECT count(*)>0 FROM pg_tables WHERE tablename = classtable INTO b; + SELECT count(*)>0 FROM pg_tables WHERE tablename = classtable and schemaname = current_schema() INTO b; IF b THEN EXECUTE 'DELETE FROM ' || classtable::regclass || ' WHERE place_id = $1' USING OLD.place_id; END IF; @@ -2044,9 +2044,6 @@ BEGIN END IF; -- Just block these - lots and pointless - IF NEW.class = 'highway' and NEW.type in ('turning_circle','traffic_signals','mini_roundabout','noexit','crossing') THEN - RETURN null; - END IF; IF NEW.class in ('landuse','natural') and NEW.name is null THEN RETURN null; END IF; @@ -2097,6 +2094,9 @@ BEGIN IF existingplacex.osm_type IS NULL THEN IF existing.osm_type IS NOT NULL THEN + -- pathological case caused by the triggerless copy into place during initial import + -- force delete even for large areas, it will be reinserted later + UPDATE place set geometry = ST_SetSRID(ST_Point(0,0), 4326) where osm_type = NEW.osm_type and osm_id = NEW.osm_id and class = NEW.class and type = NEW.type; DELETE from place where osm_type = NEW.osm_type and osm_id = NEW.osm_id and class = NEW.class and type = NEW.type; END IF; @@ -2255,7 +2255,7 @@ BEGIN country_code = NEW.country_code, parent_place_id = null, extratags = NEW.extratags, - admin_level = NEW.admin_level, + admin_level = CASE WHEN NEW.admin_level > 15 THEN 15 ELSE NEW.admin_level END, indexed_status = 2, geometry = NEW.geometry where place_id = existingplacex.place_id; @@ -2481,7 +2481,7 @@ BEGIN CASE WHEN class = 'place' and type = 'postcode' THEN hstore('name', postcode) ELSE name END as name, class, type, admin_level, fromarea, isaddress, CASE WHEN address_place_id = for_place_id AND rank_address = 0 THEN 100 WHEN rank_address = 11 THEN 5 ELSE rank_address END as rank_address, - distance,calculated_country_code + distance,calculated_country_code,postcode from place_addressline join placex on (address_place_id = placex.place_id) where place_addressline.place_id = for_place_id and (cached_rank_address > 0 AND cached_rank_address < searchrankaddress) @@ -2496,6 +2496,9 @@ BEGIN IF searchpostcode IS NOT NULL and location.type = 'postcode' THEN location.isaddress := FALSE; END IF; + IF searchpostcode IS NULL and location.isaddress and location.type != 'postcode' and location.postcode IS NOT NULL THEN + searchpostcode := location.postcode; + END IF; IF location.rank_address = 4 AND location.isaddress THEN hadcountry := true; END IF; diff --git a/sql/tables.sql b/sql/tables.sql index 3bb41e28..038a373e 100644 --- a/sql/tables.sql +++ b/sql/tables.sql @@ -212,16 +212,9 @@ CREATE INDEX idx_placex_adminname on placex USING BTREE (make_standard_name(name DROP SEQUENCE seq_place; CREATE SEQUENCE seq_place start 1; GRANT SELECT on placex to "www-data" ; -GRANT UPDATE ON placex to "www-data" ; GRANT SELECT ON search_name to "www-data" ; -GRANT DELETE on search_name to "www-data" ; -GRANT INSERT on search_name to "www-data" ; GRANT SELECT on place_addressline to "www-data" ; -GRANT INSERT ON place_addressline to "www-data" ; -GRANT DELETE on place_addressline to "www-data" ; GRANT SELECT ON seq_word to "www-data" ; -GRANT UPDATE ON seq_word to "www-data" ; -GRANT INSERT ON word to "www-data" ; GRANT SELECT ON planet_osm_ways to "www-data" ; GRANT SELECT ON planet_osm_rels to "www-data" ; GRANT SELECT on location_area to "www-data" ; diff --git a/utils/setup.php b/utils/setup.php index 0e1a1f4d..fb16254d 100755 --- a/utils/setup.php +++ b/utils/setup.php @@ -125,7 +125,13 @@ pgsqlRunScript('CREATE EXTENSION hstore'); } - pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/postgis.sql'); + $fPostgisVersion = (float) CONST_Postgis_Version; + if ($fPostgisVersion < 2.0) { + pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/postgis.sql'); + pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/spatial_ref_sys.sql'); + } else { + pgsqlRunScript('CREATE EXTENSION postgis'); + } $sVersionString = $oDB->getOne('select postgis_full_version()'); preg_match('#POSTGIS="([0-9]+)[.]([0-9]+)[.]([0-9]+)( r([0-9]+))?"#', $sVersionString, $aMatches); if (CONST_Postgis_Version != $aMatches[1].'.'.$aMatches[2]) @@ -134,7 +140,6 @@ exit; } - pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/spatial_ref_sys.sql'); pgsqlRunScriptFile(CONST_BasePath.'/data/country_name.sql'); pgsqlRunScriptFile(CONST_BasePath.'/data/country_naturalearthdata.sql'); pgsqlRunScriptFile(CONST_BasePath.'/data/country_osm_grid.sql'); diff --git a/utils/tigerAddressImport.py b/utils/tigerAddressImport.py index a784b93b..c4bca715 100755 --- a/utils/tigerAddressImport.py +++ b/utils/tigerAddressImport.py @@ -3325,6 +3325,102 @@ county_fips = { '56041' : 'Uinta, WY' , '56043' : 'Washakie, WY' , '56045' : 'Weston, WY' , +'60000' : 'AMERICAN SAMOA', +'60010' : 'Eastern district, AS' , +'60020' : 'Manua district, AS' , +'60030' : 'Rose island, AS' , +'60040' : 'Swains island, AS' , +'60050' : 'Western district, AS' , +'66000' : 'GUAM' , +'66010' : 'Guam, GU' , +'69000' : 'COMMONWEALTH OF THE NORTHERN MARIANA ISLANDS' , +'69085' : 'Commonwealth of the Northern Mariana Islands, MP' , +'69100' : 'Commonwealth of the Northern Mariana Islands, MP' , +'69110' : 'Commonwealth of the Northern Mariana Islands, MP' , +'69120' : 'Commonwealth of the Northern Mariana Islands, MP' , +'72000' : 'PUERTO RICO', +'72001' : 'Adjuntas, PR', +'72003' : 'Aguada, PR', +'72005' : 'Aguadilla, PR', +'72007' : 'Aguas Buenas, PR', +'72009' : 'Aibonito, PR', +'72011' : 'Anasco, PR', +'72013' : 'Arecibo, PR', +'72015' : 'Arroyo, PR', +'72017' : 'Barceloneta, PR', +'72019' : 'Barranquitas, PR', +'72021' : 'Bayamon, PR', +'72023' : 'Cabo Rojo, PR', +'72025' : 'Caguas, PR', +'72027' : 'Camuy, PR', +'72029' : 'Canovanas, PR', +'72031' : 'Carolina, PR', +'72033' : 'Catano, PR', +'72035' : 'Cayey, PR', +'72037' : 'Ceiba, PR', +'72039' : 'Ciales, PR', +'72041' : 'Cidra, PR', +'72043' : 'Coamo, PR', +'72045' : 'Comerio, PR', +'72047' : 'Corozal, PR', +'72049' : 'Culebra, PR', +'72051' : 'Dorado, PR', +'72053' : 'Fajardo, PR', +'72054' : 'Florida, PR', +'72055' : 'Guanica, PR', +'72057' : 'Guayama, PR', +'72059' : 'Guayanilla, PR', +'72061' : 'Guaynabo, PR', +'72063' : 'Gurabo, PR', +'72065' : 'Hatillo, PR', +'72067' : 'Hormigueros, PR', +'72069' : 'Humacao, PR', +'72071' : 'Isabela, PR', +'72073' : 'Jayuya, PR', +'72075' : 'Juana Diaz, PR', +'72077' : 'Juncos, PR', +'72079' : 'Lajas, PR', +'72081' : 'Lares, PR', +'72083' : 'Las Marias, PR', +'72085' : 'Las Piedras, PR', +'72087' : 'Loiza, PR', +'72089' : 'Luquillo, PR', +'72091' : 'Manati, PR', +'72093' : 'Maricao, PR', +'72095' : 'Maunabo, PR', +'72097' : 'Mayaguez, PR', +'72099' : 'Moca, PR', +'72101' : 'Morovis, PR', +'72103' : 'Naguabo, PR', +'72105' : 'Naranjito, PR', +'72107' : 'Orocovis, PR', +'72109' : 'Patillas, PR', +'72111' : 'Penuelas, PR', +'72113' : 'Ponce, PR', +'72115' : 'Quebradillas, PR', +'72117' : 'Rincon, PR', +'72119' : 'Rio Grande, PR', +'72121' : 'Sabana Grande, PR', +'72123' : 'Salinas, PR', +'72125' : 'San German, PR', +'72127' : 'San Juan, PR', +'72129' : 'San Lorenzo, PR', +'72131' : 'San Sebastian, PR', +'72133' : 'Santa Isabel, PR', +'72135' : 'Toa Alta, PR', +'72137' : 'Toa Baja, PR', +'72139' : 'Trujillo Alto, PR', +'72141' : 'Utuado, PR', +'72143' : 'Vega Alta, PR', +'72145' : 'Vega Baja, PR', +'72147' : 'Vieques, PR', +'72149' : 'Villalba, PR', +'72151' : 'Yabucoa, PR', +'72153' : 'Yauco, PR', +'78000' : 'VIRGIN ISLANDS' , +'78010' : 'St. Croix, VI' , +'78020' : 'St. John, VI' , +'78030' : 'St. Thomas, VI' , } def fipsstate(fips,countyfp): diff --git a/utils/update.php b/utils/update.php index d8509b37..8dd1dfc3 100755 --- a/utils/update.php +++ b/utils/update.php @@ -74,6 +74,7 @@ $oDB =& getDB(); $aDSNInfo = DB::parseDSN(CONST_Database_DSN); + if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432; // cache memory to be used by osm2pgsql, should not be more than the available memory $iCacheMemory = (isset($aResult['osm2pgsql-cache'])?$aResult['osm2pgsql-cache']:2000); @@ -82,7 +83,7 @@ $iCacheMemory = getCacheMemoryMB(); echo "WARNING: resetting cache memory to $iCacheMemory\n"; } - $sOsm2pgsqlCmd = CONST_Osm2pgsql_Binary.' -klas -C '.$iCacheMemory.' -O gazetteer -d '.$aDSNInfo['database']; + $sOsm2pgsqlCmd = CONST_Osm2pgsql_Binary.' -klas -C '.$iCacheMemory.' -O gazetteer -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port']; if (!is_null(CONST_Osm2pgsql_Flatnode_File)) { $sOsm2pgsqlCmd .= ' --flat-nodes '.CONST_Osm2pgsql_Flatnode_File; @@ -360,7 +361,7 @@ if ($aResult['index']) { - passthru(CONST_BasePath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -t '.$aResult['index-instances'].' -r '.$aResult['index-rank']); + passthru(CONST_BasePath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$aResult['index-instances'].' -r '.$aResult['index-rank']); } if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) @@ -377,7 +378,7 @@ $sCMDDownload = $sOsmosisCMD.' --read-replication-interval workingDirectory='.$sOsmosisConfigDirectory.' --simplify-change --write-xml-change '.$sImportFile; $sCMDCheckReplicationLag = $sOsmosisCMD.' -q --read-replication-lag workingDirectory='.$sOsmosisConfigDirectory; $sCMDImport = $sOsm2pgsqlCmd.' '.$sImportFile; - $sCMDIndex = $sBasePath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -t '.$aResult['index-instances']; + $sCMDIndex = $sBasePath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$aResult['index-instances']; if (!$aResult['no-npi']) { $sCMDIndex .= '-F '; } @@ -417,8 +418,9 @@ exec($sCMDCheckReplicationLag, $aReplicationLag, $iErrorLevel); } // There are new replication files - use osmosis to download the file - echo "\nReplication Delay is ".$aReplicationLag[0]."\n"; + echo "\n".date('Y-m-d H:i:s')." Replication Delay is ".$aReplicationLag[0]."\n"; } + $fStartTime = time(); $fCMDStartTime = time(); echo $sCMDDownload."\n"; exec($sCMDDownload, $sJunk, $iErrorLevel); @@ -431,9 +433,10 @@ } $iFileSize = filesize($sImportFile); $sBatchEnd = getosmosistimestamp($sOsmosisConfigDirectory); - echo "Completed for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n"; $sSQL = "INSERT INTO import_osmosis_log values ('$sBatchEnd',$iFileSize,'".date('Y-m-d H:i:s',$fCMDStartTime)."','".date('Y-m-d H:i:s')."','osmosis')"; + var_Dump($sSQL); $oDB->query($sSQL); + echo date('Y-m-d H:i:s')." Completed osmosis step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n"; } $iFileSize = filesize($sImportFile); @@ -446,12 +449,12 @@ if ($iErrorLevel) { echo "Error: $iErrorLevel\n"; - exit; + exit($iErrorLevel); } - echo "Completed for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n"; $sSQL = "INSERT INTO import_osmosis_log values ('$sBatchEnd',$iFileSize,'".date('Y-m-d H:i:s',$fCMDStartTime)."','".date('Y-m-d H:i:s')."','osm2pgsql')"; var_Dump($sSQL); $oDB->query($sSQL); + echo date('Y-m-d H:i:s')." Completed osm2pgsql step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n"; // Archive for debug? unlink($sImportFile); @@ -461,15 +464,15 @@ // Index file $sThisIndexCmd = $sCMDIndex; + $fCMDStartTime = time(); if (!$aResult['no-npi']) { - $fCMDStartTime = time(); $iFileID = $oDB->getOne('select nextval(\'file\')'); if (PEAR::isError($iFileID)) { echo $iFileID->getMessage()."\n"; - exit; + exit(-1); } $sFileDir = CONST_BasePath.'/export/diff/'; $sFileDir .= str_pad(floor($iFileID/1000000), 3, '0', STR_PAD_LEFT); @@ -495,7 +498,7 @@ if ($iErrorLevel) { echo "Error: $iErrorLevel\n"; - exit; + exit($iErrorLevel); } if (!$aResult['no-npi']) @@ -510,7 +513,7 @@ if ($iErrorLevel) { echo "Error: $iErrorLevel\n"; - exit; + exit($iErrorLevel); } rename($sFileDir.'/'.str_pad($iFileID % 1000, 3, '0', STR_PAD_LEFT).".npi.out.bz2", @@ -518,26 +521,27 @@ } } - echo "Completed for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n"; $sSQL = "INSERT INTO import_osmosis_log values ('$sBatchEnd',$iFileSize,'".date('Y-m-d H:i:s',$fCMDStartTime)."','".date('Y-m-d H:i:s')."','index')"; + var_Dump($sSQL); $oDB->query($sSQL); + echo date('Y-m-d H:i:s')." Completed index step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n"; $sSQL = "update import_status set lastimportdate = '$sBatchEnd'"; $oDB->query($sSQL); $fDuration = time() - $fStartTime; - echo "Completed for $sBatchEnd in ".round($fDuration/60,2)."\n"; + echo date('Y-m-d H:i:s')." Completed all for $sBatchEnd in ".round($fDuration/60,2)." minutes\n"; if (!$aResult['import-osmosis-all']) exit; if ( CONST_Replication_Update_Interval > 60 ) { - $iSleep = round(CONST_Replication_Update_Interval*0.8); + $iSleep = max(0,(strtotime($sBatchEnd)+CONST_Replication_Update_Interval-time())); } else { $iSleep = max(0,CONST_Replication_Update_Interval-$fDuration); } - echo "Sleeping $iSleep seconds\n"; + echo date('Y-m-d H:i:s')." Sleeping $iSleep seconds\n"; sleep($iSleep); }