fix warming for ICU tokenizer

Running the warm-up search requests requires querying
the most frequent words. This must be done via the tokenizer
to honor the different formats of the word table.

diff --git a/lib-php/admin/warm.php b/lib-php/admin/warm.php
index 39a375062fe914ce3a178a467554d2dbc718e092..338ec2da64f6668d03caee6e450637186561e804 100644 (file)
--- a/lib-php/admin/warm.php
+++ b/lib-php/admin/warm.php
@@ -86,8 +86,13 @@ if (!$aResult['reverse-only']) {
     if ($bVerbose) {
         echo "\n";
     }
     if ($bVerbose) {
         echo "\n";
     }

+
+    $oTokenizer = new \Nominatim\Tokenizer($oDB);
+
+    $aWords = $oTokenizer->mostFrequentWords(1000);
+

     $sSQL = 'SELECT word FROM word WHERE word is not null ORDER BY search_name_count DESC LIMIT 1000';
     $sSQL = 'SELECT word FROM word WHERE word is not null ORDER BY search_name_count DESC LIMIT 1000';

-    foreach ($oDB->getCol($sSQL) as $sWord) {
+    foreach ($aWords as $sWord) {

         if ($bVerbose) {
             echo "$sWord = ";
         }
         if ($bVerbose) {
             echo "$sWord = ";
         }

diff --git a/lib-php/tokenizer/icu_tokenizer.php b/lib-php/tokenizer/icu_tokenizer.php
index ca224a224861f5bd2e0f126cf10be5ea0edf5488..f4dd3aebd4db2a4e853df13037e75ecb64b3f47a 100644 (file)
--- a/lib-php/tokenizer/icu_tokenizer.php
+++ b/lib-php/tokenizer/icu_tokenizer.php
@@ -40,6 +40,15 @@ class Tokenizer
         return $this->oNormalizer->transliterate($sTerm);
     }
 
         return $this->oNormalizer->transliterate($sTerm);
     }
 

+
+    public function mostFrequentWords($iNum)
+    {
+        $sSQL = "SELECT word FROM word WHERE type = 'W'";
+        $sSQL .= "ORDER BY info->'count' DESC LIMIT ".$iNum;
+        return $this->oDB->getCol($sSQL);
+    }
+
+

     private function makeStandardWord($sTerm)
     {
         return trim($this->oTransliterator->transliterate(' '.$sTerm.' '));
     private function makeStandardWord($sTerm)
     {
         return trim($this->oTransliterator->transliterate(' '.$sTerm.' '));

diff --git a/lib-php/tokenizer/legacy_tokenizer.php b/lib-php/tokenizer/legacy_tokenizer.php
index b508d220aca3f07e861fd0bbb6efc39a13cf5528..d5686f6464eaddf0c4a8aa8cd2f6bc0024d4b887 100644 (file)
--- a/lib-php/tokenizer/legacy_tokenizer.php
+++ b/lib-php/tokenizer/legacy_tokenizer.php
@@ -48,6 +48,14 @@ class Tokenizer
     }
 
 
     }
 
 

+    public function mostFrequentWords($iNum)
+    {
+        $sSQL = 'SELECT word FROM word WHERE word is not null ';
+        $sSQL .= 'ORDER BY search_name_count DESC LIMIT '.$iNum;
+        return $this->oDB->getCol($sSQL);
+    }
+
+

     public function tokensForSpecialTerm($sTerm)
     {
         $aResults = array();
     public function tokensForSpecialTerm($sTerm)
     {
         $aResults = array();