move warm script to python code

diff --git a/lib-php/admin/warm.php b/lib-php/admin/warm.php
deleted file mode 100644 (file)
index 32f78f4..0000000
--- a/lib-php/admin/warm.php
+++ /dev/null
@@ -1,115 +0,0 @@
-<?php
-/**
- * SPDX-License-Identifier: GPL-2.0-only
- *
- * This file is part of Nominatim. (https://nominatim.org)
- *
- * Copyright (C) 2022 by the Nominatim developer community.
- * For a full list of authors see the git log.
- */
-@define('CONST_LibDir', dirname(dirname(__FILE__)));
-
-require_once(CONST_LibDir.'/init-cmd.php');
-require_once(CONST_LibDir.'/log.php');
-require_once(CONST_LibDir.'/PlaceLookup.php');
-require_once(CONST_LibDir.'/ReverseGeocode.php');
-
-ini_set('memory_limit', '800M');
-
-$aCMDOptions = array(
-                'Tools to warm nominatim db',
-                array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
-                array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
-                array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
-                array('reverse-only', '', 0, 1, 0, 0, 'bool', 'Warm reverse only'),
-                array('search-only', '', 0, 1, 0, 0, 'bool', 'Warm search only'),
-                array('project-dir', '', 0, 1, 1, 1, 'realpath', 'Base directory of the Nominatim installation (default: .)'),
-               );
-getCmdOpt($_SERVER['argv'], $aCMDOptions, $aResult, true, true);
-
-loadSettings($aCMDResult['project-dir'] ?? getcwd());
-
-@define('CONST_Database_DSN', getSetting('DATABASE_DSN'));
-@define('CONST_Default_Language', getSetting('DEFAULT_LANGUAGE', false));
-@define('CONST_Log_DB', getSettingBool('LOG_DB'));
-@define('CONST_Log_File', getSetting('LOG_FILE', false));
-@define('CONST_NoAccessControl', getSettingBool('CORS_NOACCESSCONTROL'));
-@define('CONST_Places_Max_ID_count', getSetting('LOOKUP_MAX_COUNT'));
-@define('CONST_PolygonOutput_MaximumTypes', getSetting('POLYGON_OUTPUT_MAX_TYPES'));
-@define('CONST_Search_BatchMode', getSettingBool('SEARCH_BATCH_MODE'));
-@define('CONST_Search_NameOnlySearchFrequencyThreshold', getSetting('SEARCH_NAME_ONLY_THRESHOLD'));
-@define('CONST_Use_US_Tiger_Data', getSettingBool('USE_US_TIGER_DATA'));
-@define('CONST_MapIcon_URL', getSetting('MAPICON_URL', false));
-@define('CONST_TokenizerDir', CONST_InstallDir.'/tokenizer');
-@define('CONST_Search_WithinCountries', getSetting('SEARCH_WITHIN_COUNTRIES', false));
-
-require_once(CONST_LibDir.'/Geocode.php');
-
-$oDB = new Nominatim\DB();
-$oDB->connect();
-
-$bVerbose = $aResult['verbose'];
-
-function print_results($aResults, $bVerbose)
-{
-    if ($bVerbose) {
-        if ($aResults && count($aResults)) {
-            echo $aResults[0]['langaddress']."\n";
-        } else {
-            echo "<not found>\n";
-        }
-    } else {
-        echo '.';
-    }
-}
-
-if (!$aResult['search-only']) {
-    $oReverseGeocode = new Nominatim\ReverseGeocode($oDB);
-    $oReverseGeocode->setZoom(20);
-    $oPlaceLookup = new Nominatim\PlaceLookup($oDB);
-    $oPlaceLookup->setIncludeAddressDetails(true);
-    $oPlaceLookup->setLanguagePreference(array('en'));
-
-    echo 'Warm reverse: ';
-    if ($bVerbose) {
-        echo "\n";
-    }
-    for ($i = 0; $i < 1000; $i++) {
-        $fLat = rand(-9000, 9000) / 100;
-        $fLon = rand(-18000, 18000) / 100;
-        if ($bVerbose) {
-            echo "$fLat, $fLon = ";
-        }
-
-        $oLookup = $oReverseGeocode->lookup($fLat, $fLon);
-        $aSearchResults = $oLookup ? $oPlaceLookup->lookup(array($oLookup->iId => $oLookup)) : null;
-        print_results($aSearchResults, $bVerbose);
-    }
-    echo "\n";
-}
-
-if (!$aResult['reverse-only']) {
-    $oGeocode = new Nominatim\Geocode($oDB);
-
-    echo 'Warm search: ';
-    if ($bVerbose) {
-        echo "\n";
-    }
-
-    $oTokenizer = new \Nominatim\Tokenizer($oDB);
-
-    $aWords = $oTokenizer->mostFrequentWords(1000);
-
-    $sSQL = 'SELECT word FROM word WHERE word is not null ORDER BY search_name_count DESC LIMIT 1000';
-    foreach ($aWords as $sWord) {
-        if ($bVerbose) {
-            echo "$sWord = ";
-        }
-
-        $oGeocode->setLanguagePreference(array('en'));
-        $oGeocode->setQuery($sWord);
-        $aSearchResults = $oGeocode->lookup();
-        print_results($aSearchResults, $bVerbose);
-    }
-    echo "\n";
-}

diff --git a/nominatim/clicmd/admin.py b/nominatim/clicmd/admin.py
index 0c7739603e94c7a73bef52c5e556506f34f6bc50..a84b0db2157a3daf3062da9430b0fbbbff9ea08d 100644 (file)
--- a/nominatim/clicmd/admin.py
+++ b/nominatim/clicmd/admin.py
@@ -9,9 +9,11 @@ Implementation of the 'admin' subcommand.
 """
 import logging
 import argparse
+import random
 
-from nominatim.tools.exec_utils import run_legacy_script
+from nominatim.db.connection import connect
 from nominatim.clicmd.args import NominatimArgs
+import nominatim.api as napi
 
 # Do not repeat documentation of subcommand classes.
 # pylint: disable=C0111
@@ -81,11 +83,25 @@ class AdminFuncs:
 
         return 1
 
+
     def _warm(self, args: NominatimArgs) -> int:
         LOG.warning('Warming database caches')
-        params = ['warm.php']
-        if args.target == 'reverse':
-            params.append('--reverse-only')
-        if args.target == 'search':
-            params.append('--search-only')
-        return run_legacy_script(*params, config=args.config)
+
+        api = napi.NominatimAPI(args.project_dir)
+
+        if args.target != 'reverse':
+            for _ in range(1000):
+                api.reverse((random.uniform(-90, 90), random.uniform(-180, 180)),
+                            address_details=True)
+
+        if args.target != 'search':
+            from ..tokenizer import factory as tokenizer_factory
+
+            tokenizer = tokenizer_factory.get_tokenizer_for_db(args.config)
+            with connect(args.config.get_libpq_dsn()) as conn:
+                words = tokenizer.most_frequent_words(conn, 1000)
+
+            for word in words:
+                api.search(word)
+
+        return 0

diff --git a/nominatim/tokenizer/base.py b/nominatim/tokenizer/base.py
index afbd1914b35d84219812afdd64f3061d306944f5..f0fd9dd0e178c5a1319ce6e79a560cb438567f12 100644 (file)
--- a/nominatim/tokenizer/base.py
+++ b/nominatim/tokenizer/base.py
@@ -13,6 +13,7 @@ from typing import List, Tuple, Dict, Any, Optional, Iterable
 from pathlib import Path
 
 from nominatim.config import Configuration
+from nominatim.db.connection import Connection
 from nominatim.data.place_info import PlaceInfo
 from nominatim.typing import Protocol
 
@@ -233,6 +234,13 @@ class AbstractTokenizer(ABC):
         """
 
 
+    @abstractmethod
+    def most_frequent_words(self, conn: Connection, num: int) -> List[str]:
+        """ Return a list of the `num` most frequent full words
+            in the database.
+        """
+
+
 class TokenizerModule(Protocol):
     """ Interface that must be exported by modules that implement their
         own tokenizer.

diff --git a/nominatim/tokenizer/icu_tokenizer.py b/nominatim/tokenizer/icu_tokenizer.py
index b6e646377b0e7f9d6dea691cf1ffcc6e1295632d..39c1cbc648ca33fe4ecce4ec77421db1de96b1c2 100644 (file)
--- a/nominatim/tokenizer/icu_tokenizer.py
+++ b/nominatim/tokenizer/icu_tokenizer.py
@@ -183,6 +183,18 @@ class ICUTokenizer(AbstractTokenizer):
                                self.loader.make_token_analysis())
 
 
+    def most_frequent_words(self, conn: Connection, num: int) -> List[str]:
+        """ Return a list of the `num` most frequent full words
+            in the database.
+        """
+        with conn.cursor() as cur:
+            cur.execute("""SELECT word, sum((info->'count')::int) as count
+                             FROM word WHERE type = 'W'
+                             GROUP BY word
+                             ORDER BY count DESC LIMIT %s""", (num,))
+            return list(s[0].split('@')[0] for s in cur)
+
+
     def _install_php(self, phpdir: Path, overwrite: bool = True) -> None:
         """ Install the php script for the tokenizer.
         """

diff --git a/nominatim/tokenizer/legacy_tokenizer.py b/nominatim/tokenizer/legacy_tokenizer.py
index e09700d9ddb8856a8d52fc6f7de1e9b748b9bbd7..1b68a494383bb72804d662b8bcdff8456b87c6db 100644 (file)
--- a/nominatim/tokenizer/legacy_tokenizer.py
+++ b/nominatim/tokenizer/legacy_tokenizer.py
@@ -256,6 +256,16 @@ class LegacyTokenizer(AbstractTokenizer):
         return LegacyNameAnalyzer(self.dsn, normalizer)
 
 
+    def most_frequent_words(self, conn: Connection, num: int) -> List[str]:
+        """ Return a list of the `num` most frequent full words
+            in the database.
+        """
+        with conn.cursor() as cur:
+            cur.execute(""" SELECT word FROM word WHERE word is not null
+                              ORDER BY search_name_count DESC LIMIT %s""", (num,))
+            return list(s[0] for s in cur)
+
+
     def _install_php(self, config: Configuration, overwrite: bool = True) -> None:
         """ Install the php script for the tokenizer.
         """

diff --git a/test/python/cli/test_cmd_admin.py b/test/python/cli/test_cmd_admin.py
index 696e2dd2a616ea64b2a5369842db6dfb40b32d8f..75ae3cd2a3bc9921b56b3ee0f950382874c6ec61 100644 (file)
--- a/test/python/cli/test_cmd_admin.py
+++ b/test/python/cli/test_cmd_admin.py
@@ -19,17 +19,6 @@ import nominatim.tools.migration
 import nominatim.clicmd.admin
 
 
-@pytest.mark.parametrize("params", [('--warm', ),
-                                    ('--warm', '--reverse-only'),
-                                    ('--warm', '--search-only')])
-def test_admin_command_legacy(cli_call, mock_func_factory, params):
-    mock_run_legacy = mock_func_factory(nominatim.clicmd.admin, 'run_legacy_script')
-
-    assert cli_call('admin', *params) == 0
-
-    assert mock_run_legacy.called == 1
-
-
 def test_admin_command_check_database(cli_call, mock_func_factory):
     mock = mock_func_factory(nominatim.tools.check_database, 'check_database')