bdd: use python library where possible

Replace calls to PHP scripts with direct calls into the
nominatim Python library where possible. This speed up
tests quite a bit.

diff --git a/lib-php/admin/setup.php b/lib-php/admin/setup.php
index 6493460d305695c3a5a49246c5e7d67e25c6d5fc..3aa6b828d15c336e102018a4c6cb6f4774a6171d 100644 (file)
--- a/lib-php/admin/setup.php
+++ b/lib-php/admin/setup.php
@@ -69,7 +69,11 @@ $iInstances = max(1, $aCMDResult['threads'] ?? (min(16, getProcessorCount()) - 1
 
 function run($oCmd) {
     global $iInstances;
 
 function run($oCmd) {
     global $iInstances;

+    global $aCMDResult;

     $oCmd->addParams('--threads', $iInstances);
     $oCmd->addParams('--threads', $iInstances);

+    if ($aCMDResult['ignore-errors'] ?? false) {
+        $oCmd->addParams('--ignore-errors');
+    }

     $oCmd->run(true);
 }
 
     $oCmd->run(true);
 }
 

diff --git a/nominatim/cli.py b/nominatim/cli.py
index 35c6c1f09da01757efa572cd623dc26827e7967a..7459711f9b43dd1c8e370c209d14e6c38989fa6d 100644 (file)
--- a/nominatim/cli.py
+++ b/nominatim/cli.py
@@ -75,12 +75,14 @@ class CommandlineParser:
             setattr(args, arg, Path(kwargs[arg]))
         args.project_dir = Path(args.project_dir).resolve()
 
             setattr(args, arg, Path(kwargs[arg]))
         args.project_dir = Path(args.project_dir).resolve()
 

-        logging.basicConfig(stream=sys.stderr,
-                            format='%(asctime)s: %(message)s',
-                            datefmt='%Y-%m-%d %H:%M:%S',
-                            level=max(4 - args.verbose, 1) * 10)
-
-        args.config = Configuration(args.project_dir, args.config_dir)
+        if 'cli_args' not in kwargs:
+            logging.basicConfig(stream=sys.stderr,
+                                format='%(asctime)s: %(message)s',
+                                datefmt='%Y-%m-%d %H:%M:%S',
+                                level=max(4 - args.verbose, 1) * 10)
+
+        args.config = Configuration(args.project_dir, args.config_dir,
+                                    environ=kwargs.get('environ', os.environ))

 
         log = logging.getLogger()
         log.warning('Using project directory: %s', str(args.project_dir))
 
         log = logging.getLogger()
         log.warning('Using project directory: %s', str(args.project_dir))

diff --git a/nominatim/clicmd/setup.py b/nominatim/clicmd/setup.py
index 8f717cbb9d16ad52b3076c0aef207e16bf00b401..f8229b391d78ff602afbaaa03864e4b3d9600ed8 100644 (file)
--- a/nominatim/clicmd/setup.py
+++ b/nominatim/clicmd/setup.py
@@ -75,7 +75,8 @@ class SetupAll:
             LOG.warning('Importing OSM data file')
             database_import.import_osm_data(Path(args.osm_file),
                                             args.osm2pgsql_options(0, 1),
             LOG.warning('Importing OSM data file')
             database_import.import_osm_data(Path(args.osm_file),
                                             args.osm2pgsql_options(0, 1),

-                                            drop=args.no_updates)
+                                            drop=args.no_updates,
+                                            ignore_errors=args.ignore_errors)

 
             LOG.warning('Create functions (1st pass)')
             with connect(args.config.get_libpq_dsn()) as conn:
 
             LOG.warning('Create functions (1st pass)')
             with connect(args.config.get_libpq_dsn()) as conn:

diff --git a/nominatim/clicmd/transition.py b/nominatim/clicmd/transition.py
index 1d78062ed9995a22a82377f81401622d7e4516b6..210eec70a2ce656995f3258f20b3e68837ffd609 100644 (file)
--- a/nominatim/clicmd/transition.py
+++ b/nominatim/clicmd/transition.py
@@ -48,6 +48,8 @@ class AdminTransition:
                            help='Size of cache to be used by osm2pgsql (in MB)')
         group.add_argument('--no-analyse', action='store_true',
                            help='Do not perform analyse operations during index')
                            help='Size of cache to be used by osm2pgsql (in MB)')
         group.add_argument('--no-analyse', action='store_true',
                            help='Do not perform analyse operations during index')

+        group.add_argument('--ignore-errors', action='store_true',
+                           help="Ignore certain erros on import.")

 
     @staticmethod
     def run(args):
 
     @staticmethod
     def run(args):


@@ -75,7 +77,8 @@ class AdminTransition:
                 raise UsageError('Missing required --osm-file argument')
             database_import.import_osm_data(Path(args.osm_file),
                                             args.osm2pgsql_options(0, 1),
                 raise UsageError('Missing required --osm-file argument')
             database_import.import_osm_data(Path(args.osm_file),
                                             args.osm2pgsql_options(0, 1),

-                                            drop=args.drop)
+                                            drop=args.drop,
+                                            ignore_errors=args.ignore_errors)

 
         if args.load_data:
             LOG.warning('Load data')
 
         if args.load_data:
             LOG.warning('Load data')

diff --git a/nominatim/tools/database_import.py b/nominatim/tools/database_import.py
index 6e65e73ad101148c150e971710e5bdc5e47d70c2..7d71386fc7921b752c10cfd57f24644aded895d2 100644 (file)
--- a/nominatim/tools/database_import.py
+++ b/nominatim/tools/database_import.py
@@ -145,7 +145,7 @@ def import_base_data(dsn, sql_dir, ignore_partitions=False):
             conn.commit()
 
 
             conn.commit()
 
 

-def import_osm_data(osm_file, options, drop=False):
+def import_osm_data(osm_file, options, drop=False, ignore_errors=False):

     """ Import the given OSM file. 'options' contains the list of
         default settings for osm2pgsql.
     """
     """ Import the given OSM file. 'options' contains the list of
         default settings for osm2pgsql.
     """


@@ -164,10 +164,11 @@ def import_osm_data(osm_file, options, drop=False):
     run_osm2pgsql(options)
 
     with connect(options['dsn']) as conn:
     run_osm2pgsql(options)
 
     with connect(options['dsn']) as conn:

-        with conn.cursor() as cur:
-            cur.execute('SELECT * FROM place LIMIT 1')
-            if cur.rowcount == 0:
-                raise UsageError('No data imported by osm2pgsql.')
+        if not ignore_errors:
+            with conn.cursor() as cur:
+                cur.execute('SELECT * FROM place LIMIT 1')
+                if cur.rowcount == 0:
+                    raise UsageError('No data imported by osm2pgsql.')

 
         if drop:
             conn.drop_table('planet_osm_nodes')
 
         if drop:
             conn.drop_table('planet_osm_nodes')

diff --git a/test/bdd/steps/nominatim_environment.py b/test/bdd/steps/nominatim_environment.py
index 811faf5cda6a9837a597feeb8a4afd4ebc548912..168334b1659d381b1ea98875d0ce90a14f8e412e 100644 (file)
--- a/test/bdd/steps/nominatim_environment.py
+++ b/test/bdd/steps/nominatim_environment.py
@@ -7,6 +7,7 @@ import psycopg2.extras
 
 sys.path.insert(1, str((Path(__file__) / '..' / '..' / '..' / '..').resolve()))
 
 
 sys.path.insert(1, str((Path(__file__) / '..' / '..' / '..' / '..').resolve()))
 

+from nominatim import cli

 from nominatim.config import Configuration
 from nominatim.tools import refresh
 from steps.utils import run_script
 from nominatim.config import Configuration
 from nominatim.tools import refresh
 from steps.utils import run_script


@@ -88,18 +89,18 @@ class NominatimEnvironment:
         self.test_env['NOMINATIM_FLATNODE_FILE'] = ''
         self.test_env['NOMINATIM_IMPORT_STYLE'] = 'full'
         self.test_env['NOMINATIM_USE_US_TIGER_DATA'] = 'yes'
         self.test_env['NOMINATIM_FLATNODE_FILE'] = ''
         self.test_env['NOMINATIM_IMPORT_STYLE'] = 'full'
         self.test_env['NOMINATIM_USE_US_TIGER_DATA'] = 'yes'

-        self.test_env['NOMINATIM_DATADIR'] = self.src_dir / 'data'
-        self.test_env['NOMINATIM_SQLDIR'] = self.src_dir / 'lib-sql'
-        self.test_env['NOMINATIM_CONFIGDIR'] = self.src_dir / 'settings'
-        self.test_env['NOMINATIM_DATABASE_MODULE_SRC_PATH'] = self.build_dir / 'module'
-        self.test_env['NOMINATIM_OSM2PGSQL_BINARY'] = self.build_dir / 'osm2pgsql' / 'osm2pgsql'
-        self.test_env['NOMINATIM_NOMINATIM_TOOL'] = self.build_dir / 'nominatim'
+        self.test_env['NOMINATIM_DATADIR'] = str((self.src_dir / 'data').resolve())
+        self.test_env['NOMINATIM_SQLDIR'] = str((self.src_dir / 'lib-sql').resolve())
+        self.test_env['NOMINATIM_CONFIGDIR'] = str((self.src_dir / 'settings').resolve())
+        self.test_env['NOMINATIM_DATABASE_MODULE_SRC_PATH'] = str((self.build_dir / 'module').resolve())
+        self.test_env['NOMINATIM_OSM2PGSQL_BINARY'] = str((self.build_dir / 'osm2pgsql' / 'osm2pgsql').resolve())
+        self.test_env['NOMINATIM_NOMINATIM_TOOL'] = str((self.build_dir / 'nominatim').resolve())

 
         if self.server_module_path:
             self.test_env['NOMINATIM_DATABASE_MODULE_PATH'] = self.server_module_path
         else:
             # avoid module being copied into the temporary environment
 
         if self.server_module_path:
             self.test_env['NOMINATIM_DATABASE_MODULE_PATH'] = self.server_module_path
         else:
             # avoid module being copied into the temporary environment

-            self.test_env['NOMINATIM_DATABASE_MODULE_PATH'] = self.build_dir / 'module'
+            self.test_env['NOMINATIM_DATABASE_MODULE_PATH'] = str((self.build_dir / 'module').resolve())

 
         if self.website_dir is not None:
             self.website_dir.cleanup()
 
         if self.website_dir is not None:
             self.website_dir.cleanup()


@@ -182,9 +183,9 @@ class NominatimEnvironment:
         self.test_env['NOMINATIM_WIKIPEDIA_DATA_PATH'] = str(testdata.resolve())
 
         try:
         self.test_env['NOMINATIM_WIKIPEDIA_DATA_PATH'] = str(testdata.resolve())
 
         try:

-            self.run_setup_script('all', osm_file=self.api_test_file)
+            self.run_nominatim('import', '--osm-file', str(self.api_test_file))

             self.run_setup_script('import-tiger-data')
             self.run_setup_script('import-tiger-data')

-            self.run_setup_script('drop')
+            self.run_nominatim('freeze')

 
             phrase_file = str((testdata / 'specialphrases_testdb.sql').resolve())
             run_script(['psql', '-d', self.api_test_db, '-f', phrase_file])
 
             phrase_file = str((testdata / 'specialphrases_testdb.sql').resolve())
             run_script(['psql', '-d', self.api_test_db, '-f', phrase_file])


@@ -249,12 +250,25 @@ class NominatimEnvironment:
         """
         with db.cursor() as cur:
             while True:
         """
         with db.cursor() as cur:
             while True:

-                self.run_update_script('index')
+                self.run_nominatim('index')

 
                 cur.execute("SELECT 'a' FROM placex WHERE indexed_status != 0 LIMIT 1")
                 if cur.rowcount == 0:
                     return
 
 
                 cur.execute("SELECT 'a' FROM placex WHERE indexed_status != 0 LIMIT 1")
                 if cur.rowcount == 0:
                     return
 

+    def run_nominatim(self, *cmdline):
+        """ Run the nominatim command-line tool via the library.
+        """
+        cli.nominatim(module_dir='',
+                      osm2pgsql_path=str(self.build_dir / 'osm2pgsql' / 'osm2pgsql'),
+                      phplib_dir=str(self.src_dir / 'lib-php'),
+                      sqllib_dir=str(self.src_dir / 'lib-sql'),
+                      data_dir=str(self.src_dir / 'data'),
+                      config_dir=str(self.src_dir / 'settings'),
+                      cli_args=cmdline,
+                      phpcgi_path='',
+                      environ=self.test_env)
+

     def run_setup_script(self, *args, **kwargs):
         """ Run the Nominatim setup script with the given arguments.
         """
     def run_setup_script(self, *args, **kwargs):
         """ Run the Nominatim setup script with the given arguments.
         """


@@ -285,7 +299,7 @@ class NominatimEnvironment:
         """ Copy data from place to the placex and location_property_osmline
             tables invoking the appropriate triggers.
         """
         """ Copy data from place to the placex and location_property_osmline
             tables invoking the appropriate triggers.
         """

-        self.run_setup_script('create-functions', 'create-partition-functions')
+        self.run_nominatim('refresh', '--functions', '--no-diff-updates')

 
         with db.cursor() as cur:
             cur.execute("""INSERT INTO placex (osm_type, osm_id, class, type,
 
         with db.cursor() as cur:
             cur.execute("""INSERT INTO placex (osm_type, osm_id, class, type,

diff --git a/test/bdd/steps/steps_db_ops.py b/test/bdd/steps/steps_db_ops.py
index c549f3eb5476e144f3921b3e1d92ea445755f528..9d443b434da263654ed0f40c4f5e843bf2a0d433 100644 (file)
--- a/test/bdd/steps/steps_db_ops.py
+++ b/test/bdd/steps/steps_db_ops.py
@@ -5,6 +5,7 @@ import psycopg2.extras
 from place_inserter import PlaceColumn
 from table_compare import NominatimID, DBRow
 
 from place_inserter import PlaceColumn
 from table_compare import NominatimID, DBRow
 

+from nominatim.indexer.indexer import Indexer

 
 def check_database_integrity(context):
     """ Check some generic constraints on the tables.
 
 def check_database_integrity(context):
     """ Check some generic constraints on the tables.


@@ -85,7 +86,12 @@ def import_and_index_data_from_place_table(context):
     """ Import data previously set up in the place table.
     """
     context.nominatim.copy_from_place(context.db)
     """ Import data previously set up in the place table.
     """
     context.nominatim.copy_from_place(context.db)

-    context.nominatim.run_setup_script('calculate-postcodes', 'index', 'index-noanalyse')
+    context.nominatim.run_setup_script('calculate-postcodes')
+
+    # Call directly as the refresh function does not include postcodes.
+    indexer = Indexer(context.nominatim.test_env['NOMINATIM_DATABASE_DSN'][6:], 1)
+    indexer.index_full(analyse=False)
+

     check_database_integrity(context)
 
 @when("updating places")
     check_database_integrity(context)
 
 @when("updating places")


@@ -93,8 +99,7 @@ def update_place_table(context):
     """ Update the place table with the given data. Also runs all triggers
         related to updates and reindexes the new data.
     """
     """ Update the place table with the given data. Also runs all triggers
         related to updates and reindexes the new data.
     """

-    context.nominatim.run_setup_script(
-        'create-functions', 'create-partition-functions', 'enable-diff-updates')
+    context.nominatim.run_nominatim('refresh', '--functions')

     with context.db.cursor() as cur:
         for row in context.table:
             PlaceColumn(context).add_row(row, False).db_insert(cur)
     with context.db.cursor() as cur:
         for row in context.table:
             PlaceColumn(context).add_row(row, False).db_insert(cur)


@@ -106,7 +111,7 @@ def update_place_table(context):
 def update_postcodes(context):
     """ Rerun the calculation of postcodes.
     """
 def update_postcodes(context):
     """ Rerun the calculation of postcodes.
     """

-    context.nominatim.run_update_script('calculate-postcodes')
+    context.nominatim.run_nominatim('refresh', '--postcodes')

 
 @when("marking for delete (?P<oids>.*)")
 def delete_places(context, oids):
 
 @when("marking for delete (?P<oids>.*)")
 def delete_places(context, oids):


@@ -114,8 +119,7 @@ def delete_places(context, oids):
         separated by commas. Also runs all triggers
         related to updates and reindexes the new data.
     """
         separated by commas. Also runs all triggers
         related to updates and reindexes the new data.
     """

-    context.nominatim.run_setup_script(
-        'create-functions', 'create-partition-functions', 'enable-diff-updates')
+    context.nominatim.run_nominatim('refresh', '--functions')

     with context.db.cursor() as cur:
         for oid in oids.split(','):
             NominatimID(oid).query_osm_id(cur, 'DELETE FROM place WHERE {}')
     with context.db.cursor() as cur:
         for oid in oids.split(','):
             NominatimID(oid).query_osm_id(cur, 'DELETE FROM place WHERE {}')

diff --git a/test/bdd/steps/steps_osm_data.py b/test/bdd/steps/steps_osm_data.py
index 3858198b680112017d785c522a62ca4f1b6c2243..844fb27484f984a51873e3268315f9d94756174b 100644 (file)
--- a/test/bdd/steps/steps_osm_data.py
+++ b/test/bdd/steps/steps_osm_data.py
@@ -75,9 +75,8 @@ def update_from_osm_file(context):
     The data is expected as attached text in OPL format.
     """
     context.nominatim.copy_from_place(context.db)
     The data is expected as attached text in OPL format.
     """
     context.nominatim.copy_from_place(context.db)

-    context.nominatim.run_setup_script('index', 'index-noanalyse')
-    context.nominatim.run_setup_script('create-functions', 'create-partition-functions',
-                                       'enable-diff-updates')
+    context.nominatim.run_nominatim('index')
+    context.nominatim.run_nominatim('refresh', '--functions')

 
     # create an OSM file and import it
     fname = write_opl_file(context.text, context.osm)
 
     # create an OSM file and import it
     fname = write_opl_file(context.text, context.osm)