From: Sarah Hoffmann Date: Mon, 27 Feb 2017 21:30:54 +0000 (+0100) Subject: Merge remote-tracking branch 'upstream/master' X-Git-Tag: deploy~402 X-Git-Url: https://git.openstreetmap.org./nominatim.git/commitdiff_plain/8bc3bb16a116f0152b49b00dff53fb1f37f0fb31?hp=-c Merge remote-tracking branch 'upstream/master' --- 8bc3bb16a116f0152b49b00dff53fb1f37f0fb31 diff --combined CMakeLists.txt index 71075c82,28600018..89624069 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@@ -42,6 -42,17 +42,17 @@@ unset(PostgreSQL_TYPE_INCLUDE_DIR CACHE set(PostgreSQL_TYPE_INCLUDE_DIR "/usr/include/") find_package(PostgreSQL REQUIRED) include_directories(${PostgreSQL_INCLUDE_DIRS}) + link_directories(${PostgreSQL_LIBRARY_DIRS}) + + find_program(OSMOSIS osmosis) + if (NOT EXISTS "${OSMOSIS}") + set(OSMOSIS_PATH "/nonexistent") + message(WARNING "Osmosis not found (required for updates)") + else() + set(OSMOSIS_PATH "${OSMOSIS}") + message(STATUS "Using osmosis at ${OSMOSIS_PATH}") + endif() + find_program(PG_CONFIG pg_config) execute_process(COMMAND ${PG_CONFIG} --pgxs @@@ -75,14 -86,6 +86,14 @@@ set(CUSTOMFILE website/reverse.php website/search.php website/status.php + website/403.html + website/509.html + website/crossdomain.xml + website/favicon.ico + website/last_update.php + website/nominatim.xml + website/robots.txt + website/taginfo.json utils/blocks.php utils/country_languages.php utils/imports.php diff --combined lib/Geocode.php index 5fc3dc6e,c6341829..1f4eebef --- a/lib/Geocode.php +++ b/lib/Geocode.php @@@ -24,7 -24,7 +24,7 @@@ class Geocod protected $aExcludePlaceIDs = array(); protected $bDeDupe = true; - protected $bReverseInPlan = false; + protected $bReverseInPlan = true; protected $iLimit = 20; protected $iFinalLimit = 10; @@@ -1326,6 -1326,7 +1326,7 @@@ $aOrder[0] .= " SELECT place_id "; $aOrder[0] .= " FROM location_property_osmline "; $aOrder[0] .= " WHERE parent_place_id = search_name.place_id"; + $aOrder[0] .= " AND startnumber is not NULL"; $aOrder[0] .= " AND ".intval($aSearch['sHouseNumber']).">=startnumber "; $aOrder[0] .= " AND ".intval($aSearch['sHouseNumber'])."<=endnumber "; $aOrder[0] .= " LIMIT 1"; @@@ -1337,20 -1338,19 +1338,20 @@@ // TODO: filter out the pointless search terms (2 letter name tokens and less) // they might be right - but they are just too darned expensive to run if (sizeof($aSearch['aName'])) $aTerms[] = "name_vector @> ARRAY[".join($aSearch['aName'], ",")."]"; - if (sizeof($aSearch['aNameNonSearch'])) $aTerms[] = "array_cat(name_vector,ARRAY[]::integer[]) @> ARRAY[".join($aSearch['aNameNonSearch'], ",")."]"; + //if (sizeof($aSearch['aNameNonSearch'])) $aTerms[] = "array_cat(name_vector,ARRAY[]::integer[]) @> ARRAY[".join($aSearch['aNameNonSearch'], ",")."]"; if (sizeof($aSearch['aAddress']) && $aSearch['aName'] != $aSearch['aAddress']) { // For infrequent name terms disable index usage for address if (CONST_Search_NameOnlySearchFrequencyThreshold && sizeof($aSearch['aName']) == 1 && $aWordFrequencyScores[$aSearch['aName'][reset($aSearch['aName'])]] < CONST_Search_NameOnlySearchFrequencyThreshold ) { - $aTerms[] = "array_cat(nameaddress_vector,ARRAY[]::integer[]) @> ARRAY[".join(array_merge($aSearch['aAddress'], $aSearch['aAddressNonSearch']), ",")."]"; + //$aTerms[] = "array_cat(nameaddress_vector,ARRAY[]::integer[]) @> ARRAY[".join(array_merge($aSearch['aAddress'], $aSearch['aAddressNonSearch']), ",")."]"; + $aTerms[] = "array_cat(nameaddress_vector,ARRAY[]::integer[]) @> ARRAY[".join($aSearch['aAddress'],",")."]"; } else { $aTerms[] = "nameaddress_vector @> ARRAY[".join($aSearch['aAddress'], ",")."]"; - if (sizeof($aSearch['aAddressNonSearch'])) { + /*if (sizeof($aSearch['aAddressNonSearch'])) { $aTerms[] = "array_cat(nameaddress_vector,ARRAY[]::integer[]) @> ARRAY[".join($aSearch['aAddressNonSearch'], ",")."]"; - } + }*/ } } if ($aSearch['sCountryCode']) $aTerms[] = "country_code = '".pg_escape_string($aSearch['sCountryCode'])."'"; @@@ -1463,7 -1463,7 +1464,7 @@@ // do we need to use transliteration and the regex for housenumbers??? //new query for lines, not housenumbers anymore $sSQL = "SELECT distinct place_id FROM location_property_osmline"; - $sSQL .= " WHERE parent_place_id in (".$sPlaceIDs.") and ("; + $sSQL .= " WHERE startnumber is not NULL and parent_place_id in (".$sPlaceIDs.") and ("; if ($searchedHousenumber%2 == 0) { //if housenumber is even, look for housenumber in streets with interpolationtype even or all $sSQL .= "interpolationtype='even'"; @@@ -1682,7 -1682,7 +1683,7 @@@ $sSQL .= " SELECT place_id "; $sSQL .= " FROM location_property_osmline "; $sSQL .= " WHERE place_id in (".join(',', array_keys($aResultPlaceIDs)).")"; - $sSQL .= " AND (30 between $this->iMinAddressRank and $this->iMaxAddressRank)"; + $sSQL .= " AND startnumber is not NULL AND (30 between $this->iMinAddressRank and $this->iMaxAddressRank)"; if (CONST_Debug) var_dump($sSQL); $aFilteredPlaceIDs = chksql($this->oDB->getCol($sSQL)); $tempIDs = array(); diff --combined sql/functions.sql index 4df13e67,61228c42..dc259710 --- a/sql/functions.sql +++ b/sql/functions.sql @@@ -598,7 -598,7 +598,7 @@@ BEGI IF addr_street is null and addr_place is null THEN select nodes from planet_osm_ways where id = wayid INTO waynodes; - FOR location IN SELECT placex.street, placex.addr_place from placex + FOR location IN SELECT placex.street, placex.addr_place from placex where osm_type = 'N' and osm_id = ANY(waynodes) and (placex.street is not null or placex.addr_place is not null) and indexed_status < 100 @@@ -647,101 -647,27 +647,27 @@@ $ LANGUAGE plpgsql; - CREATE OR REPLACE FUNCTION insert_osmline(wayid BIGINT, interpolationtype TEXT, - street TEXT, addr_place TEXT, - defpostalcode TEXT, country_code TEXT, - geom GEOMETRY) - RETURNS INTEGER AS $$ - DECLARE - - newpoints INTEGER; - waynodes BIGINT[]; - nodeid BIGINT; - prevnode RECORD; - nextnode RECORD; - startnumber INTEGER; - endnumber INTEGER; - housenum INTEGER; - linegeo GEOMETRY; - splitline GEOMETRY; - sectiongeo GEOMETRY; - pointgeo GEOMETRY; - place_centroid GEOMETRY; - calculated_country_code VARCHAR(2); - partition INTEGER; - geometry_sector INTEGER; - + CREATE OR REPLACE FUNCTION osmline_insert() RETURNS TRIGGER + AS $$ BEGIN - place_centroid := ST_PointOnSurface(geom); - calculated_country_code := lower(get_country_code(place_centroid)); - partition := get_partition(calculated_country_code); - geometry_sector := geometry_sector(partition, place_centroid); - - IF interpolationtype != 'odd' AND interpolationtype != 'even' AND interpolationtype!='all' THEN - -- other interpolation types than odd/even/all (e.g. numeric ones) are not supported - RETURN 0; - END IF; - - select nodes from planet_osm_ways where id = wayid INTO waynodes; - - IF array_upper(waynodes, 1) IS NULL THEN - RETURN 0; - END IF; - - linegeo := geom; - startnumber := NULL; - - FOR nodeidpos in 1..array_upper(waynodes, 1) LOOP + NEW.place_id := nextval('seq_place'); + NEW.indexed_date := now(); - select * from place where osm_type = 'N' and osm_id = waynodes[nodeidpos]::BIGINT - and housenumber is not NULL limit 1 INTO nextnode; - --RAISE NOTICE 'Nextnode.place_id: %s', nextnode.place_id; - IF nextnode.osm_id IS NOT NULL THEN - --RAISE NOTICE 'place_id is not null'; - IF nodeidpos > 1 and nodeidpos < array_upper(waynodes, 1) THEN - -- Make sure that the point is actually on the line. That might - -- be a bit paranoid but ensures that the algorithm still works - -- should osm2pgsql attempt to repair geometries. - splitline := split_line_on_node(linegeo, nextnode.geometry); - sectiongeo := ST_GeometryN(splitline, 1); - linegeo := ST_GeometryN(splitline, 2); - ELSE - sectiongeo = linegeo; + IF NEW.indexed_status IS NULL THEN + IF NEW.interpolationtype NOT IN ('odd', 'even', 'all') THEN + -- other interpolation types than odd/even/all (e.g. numeric ones) are not supported + RETURN NULL; END IF; - endnumber := substring(nextnode.housenumber,'[0-9]+')::integer; - IF startnumber IS NOT NULL AND endnumber IS NOT NULL - AND startnumber != endnumber - AND ST_GeometryType(sectiongeo) = 'ST_LineString' THEN - - IF (startnumber > endnumber) THEN - housenum := endnumber; - endnumber := startnumber; - startnumber := housenum; - sectiongeo := ST_Reverse(sectiongeo); - END IF; - - insert into location_property_osmline - values (sectiongeo, nextval('seq_place'), partition, wayid, NULL, - startnumber, endnumber, interpolationtype, - coalesce(street, prevnode.street, nextnode.street), - coalesce(addr_place, prevnode.addr_place, nextnode.addr_place), - coalesce(defpostalcode, prevnode.postcode, nextnode.postcode), - calculated_country_code, geometry_sector, 2, now()); - END IF; + NEW.indexed_status := 1; --STATUS_NEW - -- early break if we are out of line string, - -- might happen when a line string loops back on itself - IF ST_GeometryType(linegeo) != 'ST_LineString' THEN - RETURN 0; - END IF; + NEW.calculated_country_code := lower(get_country_code(NEW.linegeo)); - startnumber := substring(nextnode.housenumber,'[0-9]+')::integer; - prevnode := nextnode; - END IF; - END LOOP; + NEW.partition := get_partition(NEW.calculated_country_code); + NEW.geometry_sector := geometry_sector(NEW.partition, NEW.linegeo); + END IF; - RETURN 1; + RETURN NEW; END; $$ LANGUAGE plpgsql; @@@ -759,19 -685,6 +685,6 @@@ DECLAR classtable TEXT; line RECORD; BEGIN - --DEBUG: RAISE WARNING '% %',NEW.osm_type,NEW.osm_id; - - -- ignore interpolated addresses, not necessary anymore, cause interpolated addresses are now in location_property_osmline - IF NEW.class = 'place' and NEW.type = 'address' THEN - RETURN NEW; - END IF; - - IF ST_IsEmpty(NEW.geometry) OR NOT ST_IsValid(NEW.geometry) OR ST_X(ST_Centroid(NEW.geometry))::text in ('NaN','Infinity','-Infinity') OR ST_Y(ST_Centroid(NEW.geometry))::text in ('NaN','Infinity','-Infinity') THEN - -- block all invalid geometary - just not worth the risk. seg faults are causing serious problems. - RAISE WARNING 'invalid geometry %',NEW.osm_id; - RETURN NULL; - END IF; - --DEBUG: RAISE WARNING '% % % %',NEW.osm_type,NEW.osm_id,NEW.class,NEW.type; NEW.place_id := nextval('seq_place'); @@@ -1069,6 -982,18 +982,18 @@@ TRIGGE AS $$ DECLARE place_centroid GEOMETRY; + waynodes BIGINT[]; + prevnode RECORD; + nextnode RECORD; + startnumber INTEGER; + endnumber INTEGER; + housenum INTEGER; + linegeo GEOMETRY; + splitline GEOMETRY; + sectiongeo GEOMETRY; + street TEXT; + addr_place TEXT; + postcode TEXT; BEGIN -- deferred delete IF OLD.indexed_status = 100 THEN @@@ -1080,19 -1005,106 +1005,106 @@@ RETURN NEW; END IF; - -- do the reparenting: (finally here, because ALL places in placex, that are needed for reparenting, need to be up to date) - -- (the osm interpolationline in location_property_osmline was marked for reparenting in placex_insert/placex_delete with index_status = 1 or 2 (1 inset, 2 delete) + -- do the reparenting: (finally here, because ALL places in placex, + -- that are needed for reparenting, need to be up to date) + -- (the osm interpolationline in location_property_osmline was marked for + -- reparenting in placex_insert/placex_delete with index_status = 1 or 2 (1 inset, 2 delete) -- => index.c: sets index_status back to 0 -- => triggers this function) place_centroid := ST_PointOnSurface(NEW.linegeo); - -- marking descendants for reparenting is not needed, because there are actually no descendants for interpolation lines + -- marking descendants for reparenting is not needed, because there are + -- actually no descendants for interpolation lines NEW.parent_place_id = get_interpolation_parent(NEW.osm_id, NEW.street, NEW.addr_place, NEW.partition, place_centroid, NEW.linegeo); - return NEW; + + -- if we are just updating then our work is done + IF OLD.indexed_status != 1 THEN + return NEW; + END IF; + + -- otherwise split the line as necessary + select nodes from planet_osm_ways where id = NEW.osm_id INTO waynodes; + + IF array_upper(waynodes, 1) IS NULL THEN + RETURN 0; + END IF; + + linegeo := NEW.linegeo; + startnumber := NULL; + street := NEW.street; + addr_place := NEW.addr_place; + postcode := NEW.postcode; + + FOR nodeidpos in 1..array_upper(waynodes, 1) LOOP + + select * from place where osm_type = 'N' and osm_id = waynodes[nodeidpos]::BIGINT + and housenumber is not NULL limit 1 INTO nextnode; + --RAISE NOTICE 'Nextnode.place_id: %s', nextnode.place_id; + IF nextnode.osm_id IS NOT NULL THEN + --RAISE NOTICE 'place_id is not null'; + IF nodeidpos > 1 and nodeidpos < array_upper(waynodes, 1) THEN + -- Make sure that the point is actually on the line. That might + -- be a bit paranoid but ensures that the algorithm still works + -- should osm2pgsql attempt to repair geometries. + splitline := split_line_on_node(linegeo, nextnode.geometry); + sectiongeo := ST_GeometryN(splitline, 1); + linegeo := ST_GeometryN(splitline, 2); + ELSE + sectiongeo = linegeo; + END IF; + endnumber := substring(nextnode.housenumber,'[0-9]+')::integer; + + IF startnumber IS NOT NULL AND endnumber IS NOT NULL + AND startnumber != endnumber + AND ST_GeometryType(sectiongeo) = 'ST_LineString' THEN + + IF (startnumber > endnumber) THEN + housenum := endnumber; + endnumber := startnumber; + startnumber := housenum; + sectiongeo := ST_Reverse(sectiongeo); + END IF; + + IF NEW.startnumber IS NULL THEN + NEW.startnumber := startnumber; + NEW.endnumber := endnumber; + NEW.linegeo := sectiongeo; + NEW.street := coalesce(street, prevnode.street, nextnode.street); + NEW.addr_place := coalesce(addr_place, prevnode.addr_place, nextnode.addr_place); + NEW.postcode := coalesce(postcode, prevnode.postcode, nextnode.postcode); + ELSE + insert into location_property_osmline + (linegeo, partition, osm_id, parent_place_id, + startnumber, endnumber, interpolationtype, + street, addr_place, postcode, calculated_country_code, + geometry_sector, indexed_status) + values (sectiongeo, NEW.partition, NEW.osm_id, NEW.parent_place_id, + startnumber, endnumber, NEW.interpolationtype, + coalesce(street, prevnode.street, nextnode.street), + coalesce(addr_place, prevnode.addr_place, nextnode.addr_place), + coalesce(postcode, prevnode.postcode, nextnode.postcode), + NEW.calculated_country_code, NEW.geometry_sector, 0); + END IF; + END IF; + + -- early break if we are out of line string, + -- might happen when a line string loops back on itself + IF ST_GeometryType(linegeo) != 'ST_LineString' THEN + RETURN NEW; + END IF; + + startnumber := substring(nextnode.housenumber,'[0-9]+')::integer; + prevnode := nextnode; + END IF; + END LOOP; + + RETURN NEW; END; $$ LANGUAGE plpgsql; + + CREATE OR REPLACE FUNCTION placex_update() RETURNS TRIGGER AS $$ @@@ -1924,7 -1936,7 +1936,7 @@@ BEGI RETURN null; END IF; - -- decide, whether it is an osm interpolation line => insert_osmline, or else just insert into placex + -- decide, whether it is an osm interpolation line => insert intoosmline, or else just placex IF NEW.class='place' and NEW.type='houses' and NEW.osm_type='W' and ST_GeometryType(NEW.geometry) = 'ST_LineString' THEN -- Have we already done this place? select * from place where osm_type = NEW.osm_type and osm_id = NEW.osm_id and class = NEW.class and type = NEW.type INTO existing; @@@ -1949,11 -1961,16 +1961,16 @@@ update placex p set indexed_status = 2 from planet_osm_ways w where w.id = NEW.osm_id and p.osm_type = 'N' and p.osm_id = any(w.nodes); - -- insert new line into location_property_osmline, use function insert_osmline + + + INSERT INTO location_property_osmline + (osm_id, interpolationtype, street, + addr_place, postcode, calculated_country_code, linegeo) + VALUES (NEW.osm_id, NEW.housenumber, NEW.street, + NEW.addr_place, NEW.postcode, NEW.country_code, NEW.geometry); IF existing.osm_type IS NULL THEN - i = insert_osmline(NEW.osm_id, NEW.housenumber, NEW.street, NEW.addr_place, NEW.postcode, NEW.country_code, NEW.geometry); return NEW; END IF; @@@ -1978,8 -1995,6 +1995,6 @@@ admin_level = NEW.admin_level, geometry = NEW.geometry where osm_type = NEW.osm_type and osm_id = NEW.osm_id and class = NEW.class and type = NEW.type; - - i = insert_osmline(NEW.osm_id, NEW.housenumber, NEW.street, NEW.addr_place, NEW.postcode, NEW.country_code, NEW.geometry); END IF; RETURN NULL; @@@ -2402,7 -2417,7 +2417,7 @@@ BEGI CASE WHEN class = 'place' and type = 'postcode' THEN hstore('name', postcode) ELSE name END as name, CASE WHEN extratags ? 'place' THEN 'place' ELSE class END as class, CASE WHEN extratags ? 'place' THEN extratags->'place' ELSE type END as type, - admin_level, fromarea, isaddress, + admin_level, fromarea, isaddress and linked_place_id is NULL as isaddress, CASE WHEN address_place_id = for_place_id AND rank_address = 0 THEN 100 WHEN rank_address = 11 THEN 5 ELSE rank_address END as rank_address, distance,calculated_country_code,postcode from place_addressline join placex on (address_place_id = placex.place_id)