add normalized version of special search terms on import

[nominatim.git] / utils / specialphrases.php

diff --git a/utils/specialphrases.php b/utils/specialphrases.php
index ab6f910c49a543d2c07999cee6afe1520c80368e..156169761fef71102ff21f47a802375f5796064b 100755 (executable)
--- a/utils/specialphrases.php
+++ b/utils/specialphrases.php
@@ -12,34 +12,27 @@ $aCMDOptions
    array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
    array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
    array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
-   array('countries', '', 0, 1, 0, 0, 'bool', 'Create import script for country codes and names'),
    array('wiki-import', '', 0, 1, 0, 0, 'bool', 'Create import script for search phrases '),
   );
 getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true);
 
 include(CONST_InstallPath.'/settings/phrase_settings.php');
 
-
-if ($aCMDResult['countries']) {
-    echo "select getorcreate_country(make_standard_name('uk'), 'gb');\n";
-    echo "select getorcreate_country(make_standard_name('united states'), 'us');\n";
-    echo "select count(*) from (select getorcreate_country(make_standard_name(country_code), country_code) from country_name where country_code is not null) as x;\n";
-
-    echo "select count(*) from (select getorcreate_country(make_standard_name(get_name_by_language(country_name.name,ARRAY['name'])), country_code) from country_name where get_name_by_language(country_name.name, ARRAY['name']) is not null) as x;\n";
-    foreach ($aLanguageIn as $sLanguage) {
-        echo "select count(*) from (select getorcreate_country(make_standard_name(get_name_by_language(country_name.name,ARRAY['name:".$sLanguage."'])), country_code) from country_name where get_name_by_language(country_name.name, ARRAY['name:".$sLanguage."']) is not null) as x;\n";
-    }
-}
-
 if ($aCMDResult['wiki-import']) {
+    $oNormalizer = Transliterator::createFromRules(":: NFD (); [:Nonspacing Mark:] >;  :: lower (); [[:Punctuation:][:Space:]]+ > ' '; :: NFC ();");
     $aPairs = array();
 
-    foreach ($aLanguageIn as $sLanguage) {
+    $sLanguageIn = CONST_Languages ? CONST_Languages :
+        ('af,ar,br,ca,cs,de,en,es,et,eu,fa,fi,fr,gl,hr,hu,'.
+         'ia,is,it,ja,mk,nl,no,pl,ps,pt,ru,sk,sl,sv,uk,vi');
+
+    foreach (explode(',', $sLanguageIn) as $sLanguage) {
         $sURL = 'http://wiki.openstreetmap.org/wiki/Special:Export/Nominatim/Special_Phrases/'.strtoupper($sLanguage);
         $sWikiPageXML = file_get_contents($sURL);
         if (preg_match_all('#\\| ([^|]+) \\|\\| ([^|]+) \\|\\| ([^|]+) \\|\\| ([^|]+) \\|\\| ([\\-YN])#', $sWikiPageXML, $aMatches, PREG_SET_ORDER)) {
             foreach ($aMatches as $aMatch) {
                 $sLabel = trim($aMatch[1]);
+                $sTrans = pg_escape_string($oNormalizer->transliterate($sLabel));
                 $sClass = trim($aMatch[2]);
                 $sType = trim($aMatch[3]);
                 // hack around a bug where building=yes was imported with
@@ -66,13 +59,13 @@ if ($aCMDResult['wiki-import']) {
 
                 switch (trim($aMatch[4])) {
                     case 'near':
-                        echo "select getorcreate_amenityoperator(make_standard_name('".pg_escape_string($sLabel)."'), '$sClass', '$sType', 'near');\n";
+                        echo "select getorcreate_amenityoperator(make_standard_name('".pg_escape_string($sLabel)."'), '$sTrans', '$sClass', '$sType', 'near');\n";
                         break;
                     case 'in':
-                        echo "select getorcreate_amenityoperator(make_standard_name('".pg_escape_string($sLabel)."'), '$sClass', '$sType', 'in');\n";
+                        echo "select getorcreate_amenityoperator(make_standard_name('".pg_escape_string($sLabel)."'), '$sTrans', '$sClass', '$sType', 'in');\n";
                         break;
                     default:
-                        echo "select getorcreate_amenity(make_standard_name('".pg_escape_string($sLabel)."'), '$sClass', '$sType');\n";
+                        echo "select getorcreate_amenity(make_standard_name('".pg_escape_string($sLabel)."'), '$sTrans', '$sClass', '$sType');\n";
                         break;
                 }
             }