remove old nominatim.py in favour of 'nominatim index'

The PHP scripts need to know the position of the nominatim
tool in order to call it. This is handed in as environment
variable, so it can be set by the Python script.

diff --git a/cmake/script.tmpl b/cmake/script.tmpl
index 30b8717bac8af7d67918d157f7d7f4ff59b2bf47..aa25a1248418d064916454880eff55f94a71adf5 100755 (executable)
--- a/cmake/script.tmpl
+++ b/cmake/script.tmpl
@@ -8,5 +8,6 @@ require('@CMAKE_SOURCE_DIR@/lib/dotenv_loader.php');
 @define('CONST_DataDir', '@CMAKE_SOURCE_DIR@');
 
 loadDotEnv();
 @define('CONST_DataDir', '@CMAKE_SOURCE_DIR@');
 
 loadDotEnv();

+$_SERVER['NOMINATIM_NOMINATIM_TOOL'] = '@CMAKE_BINARY_DIR@/nominatim';

 
 require_once('@CMAKE_SOURCE_DIR@/lib/admin/@script_source@');
 
 require_once('@CMAKE_SOURCE_DIR@/lib/admin/@script_source@');

diff --git a/cmake/tool.tmpl b/cmake/tool.tmpl
index 40f2b8ea7920e910641577002a2aac1d051fadfd..43646792d19d8824753a331f39b3111fa1d6f357 100755 (executable)
--- a/cmake/tool.tmpl
+++ b/cmake/tool.tmpl
@@ -1,8 +1,11 @@
 #!/usr/bin/env python3
 import sys
 #!/usr/bin/env python3
 import sys

+import os

 
 sys.path.insert(1, '@CMAKE_SOURCE_DIR@')
 
 
 sys.path.insert(1, '@CMAKE_SOURCE_DIR@')
 

+os.environ['NOMINATIM_NOMINATIM_TOOL'] = __file__
+

 from nominatim import cli
 
 exit(cli.nominatim(module_dir='@CMAKE_BINARY_DIR@/module',
 from nominatim import cli
 
 exit(cli.nominatim(module_dir='@CMAKE_BINARY_DIR@/module',

diff --git a/lib/Shell.php b/lib/Shell.php
index 59c4473bde94d43b2569331fac3814211d0b10ed..72f90735e9763e798cb354155e8b077b37666f7e 100644 (file)
--- a/lib/Shell.php
+++ b/lib/Shell.php
@@ -7,7 +7,7 @@ class Shell
     public function __construct($sBaseCmd, ...$aParams)
     {
         if (!$sBaseCmd) {
     public function __construct($sBaseCmd, ...$aParams)
     {
         if (!$sBaseCmd) {

-            throw new Exception('Command missing in new() call');
+            throw new \Exception('Command missing in new() call');

         }
         $this->baseCmd = $sBaseCmd;
         $this->aParams = array();
         }
         $this->baseCmd = $sBaseCmd;
         $this->aParams = array();

diff --git a/lib/admin/update.php b/lib/admin/update.php
index 50f611d71aa9ab7c9294cc649f7394eb030703ff..fe9658b54f42ee458d25fcfb75be6ae38a1782fd 100644 (file)
--- a/lib/admin/update.php
+++ b/lib/admin/update.php
@@ -105,25 +105,14 @@ if ($fPostgresVersion >= 11.0) {
 }
 
 
 }
 
 

-$oIndexCmd = (new \Nominatim\Shell(CONST_DataDir.'/nominatim/nominatim.py'))
-             ->addParams('--database', $aDSNInfo['database'])
-             ->addParams('--port', $aDSNInfo['port'])
-             ->addParams('--threads', $aResult['index-instances']);
-if (!$aResult['quiet']) {
-    $oIndexCmd->addParams('--verbose');
+$oIndexCmd = (new \Nominatim\Shell(getSetting('NOMINATIM_TOOL')))
+             ->addParams('index');
+if ($aResult['quiet']) {
+    $oIndexCmd->addParams('--quiet');

 }
 if ($aResult['verbose']) {
     $oIndexCmd->addParams('--verbose');
 }
 }
 if ($aResult['verbose']) {
     $oIndexCmd->addParams('--verbose');
 }

-if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) {
-    $oIndexCmd->addParams('--host', $aDSNInfo['hostspec']);
-}
-if (isset($aDSNInfo['username']) && $aDSNInfo['username']) {
-    $oIndexCmd->addParams('--username', $aDSNInfo['username']);
-}
-if (isset($aDSNInfo['password']) && $aDSNInfo['password']) {
-    $oIndexCmd->addEnvPair('PGPASSWORD', $aDSNInfo['password']);
-}

 
 $sPyosmiumBin = getSetting('PYOSMIUM_BINARY');
 $sBaseURL = getSetting('REPLICATION_URL');
 
 $sPyosmiumBin = getSetting('PYOSMIUM_BINARY');
 $sBaseURL = getSetting('REPLICATION_URL');


@@ -288,15 +277,9 @@ if ($aResult['recompute-word-counts']) {
 }
 
 if ($aResult['index']) {
 }
 
 if ($aResult['index']) {

-    $oCmd = (clone $oIndexCmd)
-            ->addParams('--minrank', $aResult['index-rank'], '-b');
-    $oCmd->run();
-

     $oCmd = (clone $oIndexCmd)
             ->addParams('--minrank', $aResult['index-rank']);
     $oCmd->run();
     $oCmd = (clone $oIndexCmd)
             ->addParams('--minrank', $aResult['index-rank']);
     $oCmd->run();

-
-    $oDB->exec('update import_status set indexed = true');

 }
 
 if ($aResult['update-address-levels']) {
 }
 
 if ($aResult['update-address-levels']) {


@@ -438,15 +421,6 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
         if (!$aResult['no-index']) {
             $fCMDStartTime = time();
 
         if (!$aResult['no-index']) {
             $fCMDStartTime = time();
 

-            $oThisIndexCmd = clone($oIndexCmd);
-            $oThisIndexCmd->addParams('-b');
-            echo $oThisIndexCmd->escapedCmd()."\n";
-            $iErrorLevel = $oThisIndexCmd->run();
-            if ($iErrorLevel) {
-                echo "Error: $iErrorLevel\n";
-                exit($iErrorLevel);
-            }
-

             $oThisIndexCmd = clone($oIndexCmd);
             echo $oThisIndexCmd->escapedCmd()."\n";
             $iErrorLevel = $oThisIndexCmd->run();
             $oThisIndexCmd = clone($oIndexCmd);
             echo $oThisIndexCmd->escapedCmd()."\n";
             $iErrorLevel = $oThisIndexCmd->run();


@@ -463,9 +437,6 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
             var_Dump($sSQL);
             $oDB->exec($sSQL);
             echo date('Y-m-d H:i:s')." Completed index step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60, 2)." minutes\n";
             var_Dump($sSQL);
             $oDB->exec($sSQL);
             echo date('Y-m-d H:i:s')." Completed index step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60, 2)." minutes\n";

-
-            $sSQL = 'update import_status set indexed = true';
-            $oDB->exec($sSQL);

         } else {
             if ($aResult['import-osmosis-all']) {
                 echo "Error: --no-index cannot be used with continuous imports (--import-osmosis-all).\n";
         } else {
             if ($aResult['import-osmosis-all']) {
                 echo "Error: --no-index cannot be used with continuous imports (--import-osmosis-all).\n";

diff --git a/lib/setup/SetupClass.php b/lib/setup/SetupClass.php
index 77b14a8a4c27260a3a5bde6c303a43f86b16af83..d17fdca7c8b0763324c60fd2e985775cf7db524e 100755 (executable)
--- a/lib/setup/SetupClass.php
+++ b/lib/setup/SetupClass.php
@@ -549,26 +549,15 @@ class SetupFunctions
     {
         $this->checkModulePresence(); // raises exception on failure
 
     {
         $this->checkModulePresence(); // raises exception on failure
 

-        $oBaseCmd = (new \Nominatim\Shell(CONST_DataDir.'/nominatim/nominatim.py'))
-                    ->addParams('--database', $this->aDSNInfo['database'])
-                    ->addParams('--port', $this->aDSNInfo['port'])
-                    ->addParams('--threads', $this->iInstances);
+        $oBaseCmd = (new \Nominatim\Shell(getSetting('NOMINATIM_TOOL')))
+                    ->addParams('index');

 
 

-        if (!$this->bQuiet) {
-            $oBaseCmd->addParams('-v');
+        if ($this->bQuiet) {
+            $oBaseCmd->addParams('-q');

         }
         if ($this->bVerbose) {
             $oBaseCmd->addParams('-v');
         }
         }
         if ($this->bVerbose) {
             $oBaseCmd->addParams('-v');
         }

-        if (isset($this->aDSNInfo['hostspec'])) {
-            $oBaseCmd->addParams('--host', $this->aDSNInfo['hostspec']);
-        }
-        if (isset($this->aDSNInfo['username'])) {
-            $oBaseCmd->addParams('--user', $this->aDSNInfo['username']);
-        }
-        if (isset($this->aDSNInfo['password'])) {
-            $oBaseCmd->addEnvPair('PGPASSWORD', $this->aDSNInfo['password']);
-        }

 
         info('Index ranks 0 - 4');
         $oCmd = (clone $oBaseCmd)->addParams('--maxrank', 4);
 
         info('Index ranks 0 - 4');
         $oCmd = (clone $oBaseCmd)->addParams('--maxrank', 4);


@@ -581,14 +570,14 @@ class SetupFunctions
         if (!$bIndexNoanalyse) $this->pgsqlRunScript('ANALYSE');
 
         info('Index administrative boundaries');
         if (!$bIndexNoanalyse) $this->pgsqlRunScript('ANALYSE');
 
         info('Index administrative boundaries');

-        $oCmd = (clone $oBaseCmd)->addParams('-b');
+        $oCmd = (clone $oBaseCmd)->addParams('--boundaries-only');

         $iStatus = $oCmd->run();
         if ($iStatus != 0) {
             fail('error status ' . $iStatus . ' running nominatim!');
         }
 
         info('Index ranks 5 - 25');
         $iStatus = $oCmd->run();
         if ($iStatus != 0) {
             fail('error status ' . $iStatus . ' running nominatim!');
         }
 
         info('Index ranks 5 - 25');

-        $oCmd = (clone $oBaseCmd)->addParams('--minrank', 5, '--maxrank', 25);
+        $oCmd = (clone $oBaseCmd)->addParams('--no-boundaries', '--minrank', 5, '--maxrank', 25);

         $iStatus = $oCmd->run();
         if ($iStatus != 0) {
             fail('error status ' . $iStatus . ' running nominatim!');
         $iStatus = $oCmd->run();
         if ($iStatus != 0) {
             fail('error status ' . $iStatus . ' running nominatim!');


@@ -597,7 +586,7 @@ class SetupFunctions
         if (!$bIndexNoanalyse) $this->pgsqlRunScript('ANALYSE');
 
         info('Index ranks 26 - 30');
         if (!$bIndexNoanalyse) $this->pgsqlRunScript('ANALYSE');
 
         info('Index ranks 26 - 30');

-        $oCmd = (clone $oBaseCmd)->addParams('--minrank', 26);
+        $oCmd = (clone $oBaseCmd)->addParams('--no-boundaries', '--minrank', 26);

         $iStatus = $oCmd->run();
         if ($iStatus != 0) {
             fail('error status ' . $iStatus . ' running nominatim!');
         $iStatus = $oCmd->run();
         if ($iStatus != 0) {
             fail('error status ' . $iStatus . ' running nominatim!');

diff --git a/nominatim/cli.py b/nominatim/cli.py
index acb6839fa303937847bc54616f226713149ad836..65ea90bbc3534b4421558e883ecaf31796e5b0b5 100644 (file)
--- a/nominatim/cli.py
+++ b/nominatim/cli.py
@@ -78,7 +78,7 @@ class CommandlineParser:
         args.project_dir = Path(args.project_dir)
 
         logging.basicConfig(stream=sys.stderr,
         args.project_dir = Path(args.project_dir)
 
         logging.basicConfig(stream=sys.stderr,

-                            format='%(asctime)s %(levelname)s: %(message)s',
+                            format='%(asctime)s: %(message)s',

                             datefmt='%Y-%m-%d %H:%M:%S',
                             level=max(4 - args.verbose, 1) * 10)
 
                             datefmt='%Y-%m-%d %H:%M:%S',
                             level=max(4 - args.verbose, 1) * 10)
 


@@ -328,6 +328,9 @@ class UpdateIndex:
         if not args.boundaries_only:
             indexer.index_by_rank(args.minrank, args.maxrank)
 
         if not args.boundaries_only:
             indexer.index_by_rank(args.minrank, args.maxrank)
 

+        if not args.no_boundaries and not args.boundaries_only:
+            indexer.update_status_table()
+

         return 0
 
 
         return 0
 
 

diff --git a/nominatim/indexer/indexer.py b/nominatim/indexer/indexer.py
index d86303c4d018a3a91667414e0c36c1b7db40a93f..094d127987bb5d7c6b88f007d83c2e1481aba9e2 100644 (file)
--- a/nominatim/indexer/indexer.py
+++ b/nominatim/indexer/indexer.py
@@ -124,6 +124,13 @@ class Indexer:
         else:
             self.index(RankRunner(maxrank))
 
         else:
             self.index(RankRunner(maxrank))
 

+    def update_status_table(self):
+        """ Update the status in the status table to 'indexed'.
+        """
+        with self.conn.cursor() as cur:
+            cur.execute('UPDATE import_status SET indexed = true')
+        self.conn.commit()
+

     def index(self, obj, batch=1):
         """ Index a single rank or table. `obj` describes the SQL to use
             for indexing. `batch` describes the number of objects that
     def index(self, obj, batch=1):
         """ Index a single rank or table. `obj` describes the SQL to use
             for indexing. `batch` describes the number of objects that

diff --git a/nominatim/indexer/progress.py b/nominatim/indexer/progress.py
index 99120673faa67680216ac5fc48d6c8f93da62d03..c9d8816be989fb99675341a512c6806efcf06465 100644 (file)
--- a/nominatim/indexer/progress.py
+++ b/nominatim/indexer/progress.py
@@ -26,7 +26,7 @@ class ProgressLogger:
         self.done_places = 0
         self.rank_start_time = datetime.now()
         self.log_interval = log_interval
         self.done_places = 0
         self.rank_start_time = datetime.now()
         self.log_interval = log_interval

-        self.next_info = INITIAL_PROGRESS if LOG.isEnabledFor(logging.INFO) else total + 1
+        self.next_info = INITIAL_PROGRESS if LOG.isEnabledFor(logging.WARNING) else total + 1

 
     def add(self, num=1):
         """ Mark `num` places as processed. Print a log message if the
 
     def add(self, num=1):
         """ Mark `num` places as processed. Print a log message if the


@@ -47,9 +47,9 @@ class ProgressLogger:
         places_per_sec = self.done_places / done_time
         eta = (self.total_places - self.done_places) / places_per_sec
 
         places_per_sec = self.done_places / done_time
         eta = (self.total_places - self.done_places) / places_per_sec
 

-        LOG.info("Done %d in %d @ %.3f per second - %s ETA (seconds): %.2f",
-                 self.done_places, int(done_time),
-                 places_per_sec, self.name, eta)
+        LOG.warning("Done %d in %d @ %.3f per second - %s ETA (seconds): %.2f",
+                    self.done_places, int(done_time),
+                    places_per_sec, self.name, eta)

 
         self.next_info += int(places_per_sec) * self.log_interval
 
 
         self.next_info += int(places_per_sec) * self.log_interval
 

diff --git a/nominatim/nominatim.py b/nominatim/nominatim.py
deleted file mode 100755 (executable)
index fdc2bcb..0000000
--- a/nominatim/nominatim.py
+++ /dev/null
@@ -1,84 +0,0 @@
-#! /usr/bin/env python3
-#-----------------------------------------------------------------------------
-# nominatim - [description]
-#-----------------------------------------------------------------------------
-#
-# Indexing tool for the Nominatim database.
-#
-# Based on C version by Brian Quinion
-#
-# This program is free software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 2
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
-#-----------------------------------------------------------------------------
-from argparse import ArgumentParser, RawDescriptionHelpFormatter
-import logging
-import sys
-import getpass
-
-from indexer.indexer import Indexer
-
-def nominatim_arg_parser():
-    """ Setup the command-line parser for the tool.
-    """
-    parser = ArgumentParser(description="Indexing tool for Nominatim.",
-                            formatter_class=RawDescriptionHelpFormatter)
-
-    parser.add_argument('-d', '--database',
-                        dest='dbname', action='store', default='nominatim',
-                        help='Name of the PostgreSQL database to connect to.')
-    parser.add_argument('-U', '--username',
-                        dest='user', action='store',
-                        help='PostgreSQL user name.')
-    parser.add_argument('-W', '--password',
-                        dest='password_prompt', action='store_true',
-                        help='Force password prompt.')
-    parser.add_argument('-H', '--host',
-                        dest='host', action='store',
-                        help='PostgreSQL server hostname or socket location.')
-    parser.add_argument('-P', '--port',
-                        dest='port', action='store',
-                        help='PostgreSQL server port')
-    parser.add_argument('-b', '--boundary-only',
-                        dest='boundary_only', action='store_true',
-                        help='Only index administrative boundaries (ignores min/maxrank).')
-    parser.add_argument('-r', '--minrank',
-                        dest='minrank', type=int, metavar='RANK', default=0,
-                        help='Minimum/starting rank.')
-    parser.add_argument('-R', '--maxrank',
-                        dest='maxrank', type=int, metavar='RANK', default=30,
-                        help='Maximum/finishing rank.')
-    parser.add_argument('-t', '--threads',
-                        dest='threads', type=int, metavar='NUM', default=1,
-                        help='Number of threads to create for indexing.')
-    parser.add_argument('-v', '--verbose',
-                        dest='loglevel', action='count', default=0,
-                        help='Increase verbosity')
-
-    return parser
-
-if __name__ == '__main__':
-    OPTIONS = nominatim_arg_parser().parse_args(sys.argv[1:])
-
-    logging.basicConfig(stream=sys.stderr, format='%(levelname)s: %(message)s',
-                        level=max(3 - OPTIONS.loglevel, 0) * 10)
-
-    OPTIONS.password = None
-    if OPTIONS.password_prompt:
-        PASSWORD = getpass.getpass("Database password: ")
-        OPTIONS.password = PASSWORD
-
-    if OPTIONS.boundary_only:
-        Indexer(OPTIONS).index_boundaries()
-    else:
-        Indexer(OPTIONS).index_by_rank()

diff --git a/test/bdd/steps/nominatim_environment.py b/test/bdd/steps/nominatim_environment.py
index 68d7b2f43544ef1626d29e7ca66962b1bea682ff..0ee921375e8544594e65ddb2b213ceffe8cb1a59 100644 (file)
--- a/test/bdd/steps/nominatim_environment.py
+++ b/test/bdd/steps/nominatim_environment.py
@@ -91,6 +91,7 @@ class NominatimEnvironment:
         self.test_env['NOMINATIM_BINDIR'] = self.src_dir / 'utils'
         self.test_env['NOMINATIM_DATABASE_MODULE_PATH'] = self.build_dir / 'module'
         self.test_env['NOMINATIM_OSM2PGSQL_BINARY'] = self.build_dir / 'osm2pgsql' / 'osm2pgsql'
         self.test_env['NOMINATIM_BINDIR'] = self.src_dir / 'utils'
         self.test_env['NOMINATIM_DATABASE_MODULE_PATH'] = self.build_dir / 'module'
         self.test_env['NOMINATIM_OSM2PGSQL_BINARY'] = self.build_dir / 'osm2pgsql' / 'osm2pgsql'

+        self.test_env['NOMINATIM_NOMINATIM_TOOL'] = self.build_dir / 'nominatim'

 
         if self.server_module_path:
             self.test_env['NOMINATIM_DATABASE_MODULE_PATH'] = self.server_module_path
 
         if self.server_module_path:
             self.test_env['NOMINATIM_DATABASE_MODULE_PATH'] = self.server_module_path