bdd: convert API tests to smaller test db

[nominatim.git] / utils / specialphrases.php

diff --git a/utils/specialphrases.php b/utils/specialphrases.php
old mode 100755 (executable)
new mode 100644 (file)
index 1348069..b1df643
--- a/utils/specialphrases.php
+++ b/utils/specialphrases.php
@@ -1,113 +1,162 @@
-#!/usr/bin/php -Cq

 <?php
 
 <?php
 

-        require_once(dirname(dirname(__FILE__)).'/lib/init-cmd.php');
-        ini_set('memory_limit', '800M');
-        ini_set('display_errors', 'stderr');
-
-        $aCMDOptions = array(
-                "Import and export special phrases",
-                array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
-                array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
-                array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
-                array('countries', '', 0, 1, 0, 0, 'bool', 'Create import script for country codes and names'),
-                array('wiki-import', '', 0, 1, 0, 0, 'bool', 'Create import script for search phrases '),
-        );
-        getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true);
-
-               include(CONST_BasePath.'/settings/phrase_settings.php');
+require_once(CONST_LibDir.'/init-cmd.php');
+ini_set('memory_limit', '800M');
+ini_set('display_errors', 'stderr');
+
+$aCMDOptions
+= array(
+   'Import and export special phrases',
+   array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
+   array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
+   array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
+   array('wiki-import', '', 0, 1, 0, 0, 'bool', 'Create import script for search phrases '),
+   array('project-dir', '', 0, 1, 1, 1, 'realpath', 'Base directory of the Nominatim installation (default: .)'),
+  );
+getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true);
+
+loadSettings($aCMDResult['project-dir'] ?? getcwd());
+setupHTTPProxy();
+
+include(getSettingConfig('PHRASE_CONFIG', 'phrase_settings.php'));
+
+if ($aCMDResult['wiki-import']) {
+    $oNormalizer = Transliterator::createFromRules(getSetting('TERM_NORMALIZATION'));
+    $aPairs = array();
+
+    $sLanguageIn = getSetting(
+        'LANGUAGES',
+        'af,ar,br,ca,cs,de,en,es,et,eu,fa,fi,fr,gl,hr,hu,'.
+        'ia,is,it,ja,mk,nl,no,pl,ps,pt,ru,sk,sl,sv,uk,vi'
+    );
+
+    foreach (explode(',', $sLanguageIn) as $sLanguage) {
+        $sURL = 'https://wiki.openstreetmap.org/wiki/Special:Export/Nominatim/Special_Phrases/'.strtoupper($sLanguage);
+        $sWikiPageXML = file_get_contents($sURL);
+
+        if (!preg_match_all(
+            '#\\| ([^|]+) \\|\\| ([^|]+) \\|\\| ([^|]+) \\|\\| ([^|]+) \\|\\| ([\\-YN])#',
+            $sWikiPageXML,
+            $aMatches,
+            PREG_SET_ORDER
+        )) {
+            continue;
+        }

 
 

+        foreach ($aMatches as $aMatch) {
+            $sLabel = trim($aMatch[1]);
+            if ($oNormalizer !== null) {
+                $sTrans = pg_escape_string($oNormalizer->transliterate($sLabel));
+            } else {
+                $sTrans = null;
+            }
+            $sClass = trim($aMatch[2]);
+            $sType = trim($aMatch[3]);
+            // hack around a bug where building=yes was imported with
+            // quotes into the wiki
+            $sType = preg_replace('/(&quot;|")/', '', $sType);
+            // sanity check, in case somebody added garbage in the wiki
+            if (preg_match('/^\\w+$/', $sClass) < 1
+                || preg_match('/^\\w+$/', $sType) < 1
+            ) {
+                trigger_error("Bad class/type for language $sLanguage: $sClass=$sType");
+                exit;
+            }
+            // blacklisting: disallow certain class/type combinations
+            if (isset($aTagsBlacklist[$sClass]) && in_array($sType, $aTagsBlacklist[$sClass])) {
+                // fwrite(STDERR, "Blacklisted: ".$sClass."/".$sType."\n");
+                continue;
+            }
+            // whitelisting: if class is in whitelist, allow only tags in the list
+            if (isset($aTagsWhitelist[$sClass]) && !in_array($sType, $aTagsWhitelist[$sClass])) {
+                // fwrite(STDERR, "Non-Whitelisted: ".$sClass."/".$sType."\n");
+                continue;
+            }
+            $aPairs[$sClass.'|'.$sType] = array($sClass, $sType);
+
+            switch (trim($aMatch[4])) {
+                case 'near':
+                    printf(
+                        "SELECT getorcreate_amenityoperator(make_standard_name('%s'), '%s', '%s', '%s', 'near');\n",
+                        pg_escape_string($sLabel),
+                        $sTrans,
+                        $sClass,
+                        $sType
+                    );
+                    break;
+                case 'in':
+                    printf(
+                        "SELECT getorcreate_amenityoperator(make_standard_name('%s'), '%s', '%s', '%s', 'in');\n",
+                        pg_escape_string($sLabel),
+                        $sTrans,
+                        $sClass,
+                        $sType
+                    );
+                    break;
+                default:
+                    printf(
+                        "SELECT getorcreate_amenity(make_standard_name('%s'), '%s', '%s', '%s');\n",
+                        pg_escape_string($sLabel),
+                        $sTrans,
+                        $sClass,
+                        $sType
+                    );
+                    break;
+            }
+        }
+    }

 
 

-    if ($aCMDResult['countries']) {
-        echo "select getorcreate_country(make_standard_name('uk'), 'gb');\n";
-        echo "select getorcreate_country(make_standard_name('united states'), 'us');\n";
-        echo "select count(*) from (select getorcreate_country(make_standard_name(country_code), country_code) from country_name where country_code is not null) as x;\n";
+    echo 'CREATE INDEX idx_placex_classtype ON placex (class, type);';

 
 

-        echo "select count(*) from (select getorcreate_country(make_standard_name(get_name_by_language(country_name.name,ARRAY['name'])), country_code) from country_name where get_name_by_language(country_name.name, ARRAY['name']) is not null) as x;\n";
-        foreach($aLanguageIn as $sLanguage)
-               {
-            echo "select count(*) from (select getorcreate_country(make_standard_name(get_name_by_language(country_name.name,ARRAY['name:".$sLanguage."'])), country_code) from country_name where get_name_by_language(country_name.name, ARRAY['name:".$sLanguage."']) is not null) as x;\n";
+    foreach ($aPairs as $aPair) {
+        $sql_tablespace = getSetting('TABLESPACE_AUX_DATA');
+        if ($sql_tablespace) {
+            $sql_tablespace = ' TABLESPACE '.$sql_tablespace;

         }
         }

+
+        printf(
+            'CREATE TABLE place_classtype_%s_%s'
+            . $sql_tablespace
+            . ' AS'
+            . ' SELECT place_id AS place_id,st_centroid(geometry) AS centroid FROM placex'
+            . " WHERE class = '%s' AND type = '%s'"
+            . ";\n",
+            pg_escape_string($aPair[0]),
+            pg_escape_string($aPair[1]),
+            pg_escape_string($aPair[0]),
+            pg_escape_string($aPair[1])
+        );
+
+        printf(
+            'CREATE INDEX idx_place_classtype_%s_%s_centroid'
+            . ' ON place_classtype_%s_%s USING GIST (centroid)'
+            . $sql_tablespace
+            . ";\n",
+            pg_escape_string($aPair[0]),
+            pg_escape_string($aPair[1]),
+            pg_escape_string($aPair[0]),
+            pg_escape_string($aPair[1])
+        );
+
+        printf(
+            'CREATE INDEX idx_place_classtype_%s_%s_place_id'
+            . ' ON place_classtype_%s_%s USING btree(place_id)'
+            . $sql_tablespace
+            . ";\n",
+            pg_escape_string($aPair[0]),
+            pg_escape_string($aPair[1]),
+            pg_escape_string($aPair[0]),
+            pg_escape_string($aPair[1])
+        );
+
+        printf(
+            'GRANT SELECT ON place_classtype_%s_%s TO "%s"'
+            . ";\n",
+            pg_escape_string($aPair[0]),
+            pg_escape_string($aPair[1]),
+            getSetting('DATABASE_WEBUSER')
+        );

     }
 
     }
 

-       if ($aCMDResult['wiki-import'])
-       {
-               $aPairs = array();
-
-               foreach($aLanguageIn as $sLanguage)
-               {
-                       $sURL = 'http://wiki.openstreetmap.org/wiki/Special:Export/Nominatim/Special_Phrases/'.strtoupper($sLanguage);
-                       $sWikiPageXML = file_get_contents($sURL);
-                       if (preg_match_all('#\\| ([^|]+) \\|\\| ([^|]+) \\|\\| ([^|]+) \\|\\| ([^|]+) \\|\\| ([\\-YN])#', $sWikiPageXML, $aMatches, PREG_SET_ORDER))
-                       {
-                               foreach($aMatches as $aMatch)
-                               {
-                                       $sLabel = trim($aMatch[1]);
-                                       $sClass = trim($aMatch[2]);
-                                       $sType = trim($aMatch[3]);
-                                       # hack around a bug where building=yes was imported with
-                                       # quotes into the wiki
-                                       $sType = preg_replace('/&quot;/', '', $sType);
-                                       # sanity check, in case somebody added garbage in the wiki
-                                       if (preg_match('/^\\w+$/', $sClass) < 1 ||
-                                               preg_match('/^\\w+$/', $sType) < 1) {
-                                               trigger_error("Bad class/type for language $sLanguage: $sClass=$sType");
-                                               exit;
-                                       }
-                                       # blacklisting: disallow certain class/type combinations
-                                       if (isset($aTagsBlacklist[$sClass]) && in_array($sType, $aTagsBlacklist[$sClass])) {
-                                               # fwrite(STDERR, "Blacklisted: ".$sClass."/".$sType."\n");
-                                               continue;
-                                       }
-                                       # whitelisting: if class is in whitelist, allow only tags in the list
-                                       if (isset($aTagsWhitelist[$sClass])     && !in_array($sType, $aTagsWhitelist[$sClass])) {
-                                               # fwrite(STDERR, "Non-Whitelisted: ".$sClass."/".$sType."\n");
-                                               continue;
-                                       }
-                                       $aPairs[$sClass.'|'.$sType] = array($sClass, $sType);
-
-                                       switch(trim($aMatch[4]))
-                                       {
-                                       case 'near':
-                                               echo "select getorcreate_amenityoperator(make_standard_name('".pg_escape_string($sLabel)."'), '$sClass', '$sType', 'near');\n";
-                                               break;
-                                       case 'in':
-                                               echo "select getorcreate_amenityoperator(make_standard_name('".pg_escape_string($sLabel)."'), '$sClass', '$sType', 'in');\n";
-                                               break;
-                                       default:
-                                               echo "select getorcreate_amenity(make_standard_name('".pg_escape_string($sLabel)."'), '$sClass', '$sType');\n";
-                                               break;
-                                       }
-                               }
-                       }
-               }
-
-        echo "create index idx_placex_classtype on placex (class, type);";
-
-               foreach($aPairs as $aPair)
-               {
-                       echo "create table place_classtype_".pg_escape_string($aPair[0])."_".pg_escape_string($aPair[1]);
-                       if (CONST_Tablespace_Aux_Data)
-                               echo " tablespace ".CONST_Tablespace_Aux_Data;
-                       echo " as select place_id as place_id,st_centroid(geometry) as centroid from placex where ";
-                       echo "class = '".pg_escape_string($aPair[0])."' and type = '".pg_escape_string($aPair[1])."'";
-                       echo ";\n";
-
-                       echo "CREATE INDEX idx_place_classtype_".pg_escape_string($aPair[0])."_".pg_escape_string($aPair[1])."_centroid ";
-                       echo "ON place_classtype_".pg_escape_string($aPair[0])."_".pg_escape_string($aPair[1])." USING GIST (centroid)";
-                       if (CONST_Tablespace_Aux_Index)
-                               echo " tablespace ".CONST_Tablespace_Aux_Index;
-                       echo ";\n";
-
-                       echo "CREATE INDEX idx_place_classtype_".pg_escape_string($aPair[0])."_".pg_escape_string($aPair[1])."_place_id ";
-                       echo "ON place_classtype_".pg_escape_string($aPair[0])."_".pg_escape_string($aPair[1])." USING btree(place_id)";
-                       if (CONST_Tablespace_Aux_Index)
-                               echo " tablespace ".CONST_Tablespace_Aux_Index;
-                       echo ";\n";
-
-            echo "GRANT SELECT ON place_classtype_".pg_escape_string($aPair[0])."_".pg_escape_string($aPair[1])." TO \"www-data\";\n";
-
-               }
-
-        echo "drop index idx_placex_classtype;";
-       }
+    echo 'DROP INDEX idx_placex_classtype;';
+}