result := '{}'::INTEGER[];
s := make_standard_name(src);
- w := getorcreate_name_id(s);
+ w := getorcreate_name_id(s, src);
IF NOT (ARRAY[w] <@ result) THEN
result := result || w;
END LOOP;
END IF;
+ words := regexp_split_to_array(src, E'[,;()]');
+ IF array_upper(words, 1) != 1 THEN
+ FOR j IN 1..array_upper(words, 1) LOOP
+ s := make_standard_name(words[j]);
+ IF s != '' THEN
+ w := getorcreate_word_id(s);
+ IF NOT (ARRAY[w] <@ result) THEN
+ result := result || w;
+ END IF;
+ END IF;
+ END LOOP;
+ END IF;
+
+ s := regexp_replace(src, '市$', '');
+ IF s != src THEN
+ s := make_standard_name(s);
+ IF s != '' THEN
+ w := getorcreate_name_id(s, src);
+ IF NOT (ARRAY[w] <@ result) THEN
+ result := result || w;
+ END IF;
+ END IF;
+ END IF;
+
RETURN result;
END;
$$
place_centre GEOMETRY;
nearcountry RECORD;
BEGIN
- place_centre := ST_Centroid(place);
+ place_centre := ST_PointOnSurface(place);
--RAISE WARNING 'start: %', ST_AsText(place_centre);
NEW.place_id := nextval('seq_place');
NEW.indexed_status := 1; --STATUS_NEW
- IF NEW.rank_search >= 4 THEN
- NEW.country_code := lower(get_country_code(NEW.geometry, NEW.country_code));
- ELSE
- NEW.country_code := NULL;
- END IF;
+ NEW.country_code := lower(get_country_code(NEW.geometry, NEW.country_code));
NEW.partition := get_partition(NEW.geometry, NEW.country_code);
NEW.geometry_sector := geometry_sector(NEW.partition, NEW.geometry);
IF NEW.type in ('continent') THEN
NEW.rank_search := 2;
NEW.rank_address := NEW.rank_search;
+ NEW.country_code := NULL;
ELSEIF NEW.type in ('sea') THEN
NEW.rank_search := 2;
NEW.rank_address := 0;
+ NEW.country_code := NULL;
ELSEIF NEW.type in ('country') THEN
NEW.rank_search := 4;
NEW.rank_address := NEW.rank_search;
END IF;
IF (NEW.extratags -> 'capital') = 'yes' THEN
- NEW.rank_search := NEW.rank_search -1;
+ NEW.rank_search := NEW.rank_search - 1;
+ END IF;
+
+ -- a country code make no sense below rank 4 (country)
+ IF NEW.rank_address < 4 THEN
+ NEW.country_code := NULL;
END IF;
-- Block import below rank 22
-- RETURN NULL;
-- END IF;
- RETURN NEW; -- The following is not needed until doing diff updates, and slows the main index process down
+ RETURN NEW; -- @DIFFUPDATES@ The following is not needed until doing diff updates, and slows the main index process down
IF (ST_GeometryType(NEW.geometry) in ('ST_Polygon','ST_MultiPolygon') AND ST_IsValid(NEW.geometry)) THEN
-- Performance: We just can't handle re-indexing for country level changes
END LOOP;
NEW.importance := null;
- select language||':'||title,importance from get_wikipedia_match(NEW.extratags) INTO NEW.wikipedia,NEW.importance;
+ select language||':'||title,importance from get_wikipedia_match(NEW.extratags, NEW.country_code) INTO NEW.wikipedia,NEW.importance;
IF NEW.importance IS NULL THEN
select language||':'||title,importance from wikipedia_article where osm_type = NEW.osm_type and osm_id = NEW.osm_id order by importance desc limit 1 INTO NEW.wikipedia,NEW.importance;
END IF;
---RAISE WARNING '% %', NEW.place_id, NEW.rank_search;
+--RAISE WARNING 'before low level% %', NEW.place_id, NEW.rank_search;
-- For low level elements we inherit from our parent road
IF (NEW.rank_search > 27 OR (NEW.type = 'postcode' AND NEW.rank_search = 25)) THEN
-- see if we have any special relation members
select members from planet_osm_rels where id = NEW.osm_id INTO relation_members;
+-- RAISE WARNING 'get_osm_rel_members, label';
FOR relMember IN select get_osm_rel_members(relation_members,ARRAY['label']) as member LOOP
- select * from placex where osm_type = upper(substring(relMember.member,1,1))
- and osm_id = substring(relMember.member,2,10000)::integer order by rank_search desc limit 1 into linkedPlacex;
+ select * from placex where osm_type = upper(substring(relMember.member,1,1))::char(1)
+ and osm_id = substring(relMember.member,2,10000)::bigint order by rank_search desc limit 1 into linkedPlacex;
-- If we don't already have one use this as the centre point of the geometry
IF NEW.centroid IS NULL THEN
FOR relMember IN select get_osm_rel_members(relation_members,ARRAY['admin_center','admin_centre']) as member LOOP
- select * from placex where osm_type = upper(substring(relMember.member,1,1))
- and osm_id = substring(relMember.member,2,10000)::integer order by rank_search desc limit 1 into linkedPlacex;
+ select * from placex where osm_type = upper(substring(relMember.member,1,1))::char(1)
+ and osm_id = substring(relMember.member,2,10000)::bigint order by rank_search desc limit 1 into linkedPlacex;
-- For an admin centre we also want a name match - still not perfect, for example 'new york, new york'
-- But that can be fixed by explicitly setting the label in the data
IF make_standard_name(NEW.name->'name') = make_standard_name(linkedPlacex.name->'name')
AND NEW.rank_search = linkedPlacex.rank_search THEN
+
-- If we don't already have one use this as the centre point of the geometry
IF NEW.centroid IS NULL THEN
NEW.centroid := coalesce(linkedPlacex.centroid,st_centroid(linkedPlacex.geometry));
make_standard_name(name->'name') = make_standard_name(NEW.name->'name')
AND placex.rank_search = NEW.rank_search
AND placex.place_id != NEW.place_id
- AND placex.osm_type = 'N' AND placex.rank_search < 26
+ AND placex.osm_type = 'N'::char(1) AND placex.rank_search < 26
AND st_contains(NEW.geometry, placex.geometry)
LOOP
-- Did we gain a wikipedia tag in the process? then we need to recalculate our importance
IF NEW.importance is null THEN
- select language||':'||title,importance from get_wikipedia_match(NEW.extratags) INTO NEW.wikipedia,NEW.importance;
+ select language||':'||title,importance from get_wikipedia_match(NEW.extratags, NEW.country_code) INTO NEW.wikipedia,NEW.importance;
END IF;
-- Still null? how about looking it up by the node id
IF NEW.importance IS NULL THEN
- select language||':'||title,importance from wikipedia_article where osm_type = 'N' and osm_id = linked_node_id order by importance desc limit 1 INTO NEW.wikipedia,NEW.importance;
+ select language||':'||title,importance from wikipedia_article where osm_type = 'N'::char(1) and osm_id = linked_node_id order by importance desc limit 1 INTO NEW.wikipedia,NEW.importance;
END IF;
END IF;
importance FLOAT
);
-CREATE OR REPLACE FUNCTION get_wikipedia_match(extratags HSTORE) RETURNS wikipedia_article_match
+CREATE OR REPLACE FUNCTION get_wikipedia_match(extratags HSTORE, country_code varchar(2)) RETURNS wikipedia_article_match
AS $$
DECLARE
langs TEXT[];
wiki_article_language TEXT;
result wikipedia_article_match;
BEGIN
- langs := ARRAY['','ar','bg','ca','cs','da','de','en','es','eo','eu','fa','fr','ko','hi','hr','id','it','he','lt','hu','ms','nl','ja','no','pl','pt','kk','ro','ru','sk','sl','sr','fi','sv','tr','uk','vi','vo','war','zh'];
+ langs := ARRAY['english','country','ar','bg','ca','cs','da','de','en','es','eo','eu','fa','fr','ko','hi','hr','id','it','he','lt','hu','ms','nl','ja','no','pl','pt','kk','ro','ru','sk','sl','sr','fi','sv','tr','uk','vi','vo','war','zh'];
i := 1;
WHILE langs[i] IS NOT NULL LOOP
- wiki_article := extratags->(case when langs[i] = '' THEN 'wikipedia' ELSE 'wikipedia:'||langs[i] END);
+ wiki_article := extratags->(case when langs[i] in ('english','country') THEN 'wikipedia' ELSE 'wikipedia:'||langs[i] END);
IF wiki_article is not null THEN
- wiki_article := replace(regexp_replace(wiki_article,E'(.*?)([a-z]+).wikipedia.org/wiki/',E'\\2:'),' ','_');
- wiki_article_title := split_part(wiki_article, ':', 2);
+ wiki_article := regexp_replace(wiki_article,E'^(.*?)([a-z]{2,3}).wikipedia.org/wiki/',E'\\2:');
+ wiki_article := regexp_replace(wiki_article,E'^(.*?)([a-z]{2,3}).wikipedia.org/w/index.php\\?title=',E'\\2:');
+ wiki_article := regexp_replace(wiki_article,E'^(.*?)/([a-z]{2,3})/wiki/',E'\\2:');
+ wiki_article := regexp_replace(wiki_article,E'^(.*?)([a-z]{2,3})[=:]',E'\\2:');
+ wiki_article := replace(wiki_article,' ','_');
+ wiki_article_title := trim(split_part(wiki_article, ':', 2));
IF wiki_article_title IS NULL OR wiki_article_title = '' THEN
- wiki_article_title := wiki_article;
- wiki_article_language := CASE WHEN langs[i] = '' THEN 'en' ELSE langs[i] END;
+ wiki_article_title := trim(wiki_article);
+ wiki_article_language := CASE WHEN langs[i] = 'english' THEN 'en' WHEN langs[i] = 'country' THEN get_country_language_code(country_code) ELSE langs[i] END;
ELSE
- wiki_article_language := lower(split_part(wiki_article, ':', 1));
+ wiki_article_language := lower(trim(split_part(wiki_article, ':', 1)));
END IF;
select wikipedia_article.language,wikipedia_article.title,wikipedia_article.importance
if (file_exists($sWikiArticlesFile))
{
echo "Importing wikipedia articles...";
- pgsqlRunRestoreData($sWikiArticlesFile);
+ pgsqlRunDropAndRestore($sWikiArticlesFile);
echo "...done\n";
}
else
if (file_exists($sWikiRedirectsFile))
{
echo "Importing wikipedia redirects...";
- pgsqlRunRestoreData($sWikiRedirectsFile);
+ pgsqlRunDropAndRestore($sWikiRedirectsFile);
echo "...done\n";
}
else
if (!pg_query($oDB->connection, 'CREATE SEQUENCE seq_place start 100000')) fail(pg_last_error($oDB->connection));
echo '.';
+ // pre-create the word list
+ if (!pg_query($oDB->connection, 'select count(make_keywords(v)) from (select distinct svals(name) as v from place) as w where v is not null;')) fail(pg_last_error($oDB->connection));
+ echo '.';
+ if (!pg_query($oDB->connection, 'select count(make_keywords(v)) from (select distinct postcode as v from place) as w where v is not null;')) fail(pg_last_error($oDB->connection));
+ echo '.';
+ if (!pg_query($oDB->connection, 'select count(getorcreate_housenumber_id(v)) from (select distinct housenumber as v from place where housenumber is not null) as w;')) fail(pg_last_error($oDB->connection));
+ echo '.';
+
$aDBInstances = array();
for($i = 0; $i < $iInstances; $i++)
{
$sOutputFile = '';
if (isset($aCMDResult['index-output'])) $sOutputFile = ' -F '.$aCMDResult['index-output'];
$sBaseCmd = CONST_BasePath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -t '.$iInstances.$sOutputFile;
- passthru($sBaseCmd.' -R 4');
- pgsqlRunScript('ANALYSE');
- passthru($sBaseCmd.' -r 5 -R 25');
- pgsqlRunScript('ANALYSE');
- passthru($sBaseCmd.' -r 26');
+ passthru($sBaseCmd);
}
if ($aCMDResult['create-search-indices'] || $aCMDResult['all'])
proc_close($hProcess);
}
+
+ function pgsqlRunDropAndRestore($sDumpFile)
+ {
+ // Convert database DSN to psql paramaters
+ $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
+ if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
+ $sCMD = 'pg_restore -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -Fc --clean '.$sDumpFile;
+
+ $aDescriptors = array(
+ 0 => array('pipe', 'r'),
+ 1 => array('pipe', 'w'),
+ 2 => array('file', '/dev/null', 'a')
+ );
+ $ahPipes = null;
+ $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
+ if (!is_resource($hProcess)) fail('unable to start pg_restore');
+
+ fclose($ahPipes[0]);
+
+ // TODO: error checking
+ while(!feof($ahPipes[1]))
+ {
+ echo fread($ahPipes[1], 4096);
+ }
+ fclose($ahPipes[1]);
+
+ proc_close($hProcess);
+ }
projects / nominatim.git / commitdiff