Correct some typos

[nominatim.git] / test / bdd / steps / nominatim_environment.py

diff --git a/test/bdd/steps/nominatim_environment.py b/test/bdd/steps/nominatim_environment.py
index 7eb6f3dd474232d127defa82aa1993fa7c4f6a81..1fc6f887ee0c11d66a6b2b9fa06ee29ada36608b 100644 (file)
--- a/test/bdd/steps/nominatim_environment.py
+++ b/test/bdd/steps/nominatim_environment.py
@@ -1,4 +1,11 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# This file is part of Nominatim. (https://nominatim.org)
+#
+# Copyright (C) 2022 by the Nominatim developer community.
+# For a full list of authors see the git log.

 from pathlib import Path
 from pathlib import Path

+import importlib

 import sys
 import tempfile
 
 import sys
 import tempfile
 


@@ -9,7 +16,7 @@ sys.path.insert(1, str((Path(__file__) / '..' / '..' / '..' / '..').resolve()))
 
 from nominatim import cli
 from nominatim.config import Configuration
 
 from nominatim import cli
 from nominatim.config import Configuration

-from nominatim.db.connection import _Connection
+from nominatim.db.connection import Connection

 from nominatim.tools import refresh
 from nominatim.tokenizer import factory as tokenizer_factory
 from steps.utils import run_script
 from nominatim.tools import refresh
 from nominatim.tokenizer import factory as tokenizer_factory
 from steps.utils import run_script


@@ -30,18 +37,25 @@ class NominatimEnvironment:
         self.api_test_db = config['API_TEST_DB']
         self.api_test_file = config['API_TEST_FILE']
         self.tokenizer = config['TOKENIZER']
         self.api_test_db = config['API_TEST_DB']
         self.api_test_file = config['API_TEST_FILE']
         self.tokenizer = config['TOKENIZER']

+        self.import_style = config['STYLE']

         self.server_module_path = config['SERVER_MODULE_PATH']
         self.reuse_template = not config['REMOVE_TEMPLATE']
         self.keep_scenario_db = config['KEEP_TEST_DB']
         self.code_coverage_path = config['PHPCOV']
         self.code_coverage_id = 1
 
         self.server_module_path = config['SERVER_MODULE_PATH']
         self.reuse_template = not config['REMOVE_TEMPLATE']
         self.keep_scenario_db = config['KEEP_TEST_DB']
         self.code_coverage_path = config['PHPCOV']
         self.code_coverage_id = 1
 

-        self.default_config = Configuration(None, self.src_dir / 'settings').get_os_env()
+        self.default_config = Configuration(None).get_os_env()

         self.test_env = None
         self.template_db_done = False
         self.api_db_done = False
         self.website_dir = None
 
         self.test_env = None
         self.template_db_done = False
         self.api_db_done = False
         self.website_dir = None
 

+        self.api_engine = None
+        if config['API_ENGINE'] != 'php':
+            if not hasattr(self, f"create_api_request_func_{config['API_ENGINE']}"):
+                raise RuntimeError(f"Unknown API engine '{config['API_ENGINE']}'")
+            self.api_engine = getattr(self, f"create_api_request_func_{config['API_ENGINE']}")()
+

     def connect_database(self, dbname):
         """ Return a connection to the database with the given name.
             Uses configured host, user and port.
     def connect_database(self, dbname):
         """ Return a connection to the database with the given name.
             Uses configured host, user and port.


@@ -55,7 +69,7 @@ class NominatimEnvironment:
             dbargs['user'] = self.db_user
         if self.db_pass:
             dbargs['password'] = self.db_pass
             dbargs['user'] = self.db_user
         if self.db_pass:
             dbargs['password'] = self.db_pass

-        conn = psycopg2.connect(connection_factory=_Connection, **dbargs)
+        conn = psycopg2.connect(connection_factory=Connection, **dbargs)

         return conn
 
     def next_code_coverage_file(self):
         return conn
 
     def next_code_coverage_file(self):


@@ -72,7 +86,10 @@ class NominatimEnvironment:
             be picked up by dotenv and creates a project directory with the
             appropriate website scripts.
         """
             be picked up by dotenv and creates a project directory with the
             appropriate website scripts.
         """

-        dsn = 'pgsql:dbname={}'.format(dbname)
+        if dbname.startswith('sqlite:'):
+            dsn = 'sqlite:dbname={}'.format(dbname[7:])
+        else:
+            dsn = 'pgsql:dbname={}'.format(dbname)

         if self.db_host:
             dsn += ';host=' + self.db_host
         if self.db_port:
         if self.db_host:
             dsn += ';host=' + self.db_host
         if self.db_port:


@@ -82,13 +99,9 @@ class NominatimEnvironment:
         if self.db_pass:
             dsn += ';password=' + self.db_pass
 
         if self.db_pass:
             dsn += ';password=' + self.db_pass
 

-        if self.website_dir is not None \
-           and self.test_env is not None \
-           and dsn == self.test_env['NOMINATIM_DATABASE_DSN']:
-            return # environment already set uo
-

         self.test_env = dict(self.default_config)
         self.test_env['NOMINATIM_DATABASE_DSN'] = dsn
         self.test_env = dict(self.default_config)
         self.test_env['NOMINATIM_DATABASE_DSN'] = dsn

+        self.test_env['NOMINATIM_LANGUAGES'] = 'en,de,fr,ja'

         self.test_env['NOMINATIM_FLATNODE_FILE'] = ''
         self.test_env['NOMINATIM_IMPORT_STYLE'] = 'full'
         self.test_env['NOMINATIM_USE_US_TIGER_DATA'] = 'yes'
         self.test_env['NOMINATIM_FLATNODE_FILE'] = ''
         self.test_env['NOMINATIM_IMPORT_STYLE'] = 'full'
         self.test_env['NOMINATIM_USE_US_TIGER_DATA'] = 'yes'


@@ -97,9 +110,10 @@ class NominatimEnvironment:
         self.test_env['NOMINATIM_CONFIGDIR'] = str((self.src_dir / 'settings').resolve())
         self.test_env['NOMINATIM_DATABASE_MODULE_SRC_PATH'] = str((self.build_dir / 'module').resolve())
         self.test_env['NOMINATIM_OSM2PGSQL_BINARY'] = str((self.build_dir / 'osm2pgsql' / 'osm2pgsql').resolve())
         self.test_env['NOMINATIM_CONFIGDIR'] = str((self.src_dir / 'settings').resolve())
         self.test_env['NOMINATIM_DATABASE_MODULE_SRC_PATH'] = str((self.build_dir / 'module').resolve())
         self.test_env['NOMINATIM_OSM2PGSQL_BINARY'] = str((self.build_dir / 'osm2pgsql' / 'osm2pgsql').resolve())

-        self.test_env['NOMINATIM_NOMINATIM_TOOL'] = str((self.build_dir / 'nominatim').resolve())

         if self.tokenizer is not None:
             self.test_env['NOMINATIM_TOKENIZER'] = self.tokenizer
         if self.tokenizer is not None:
             self.test_env['NOMINATIM_TOKENIZER'] = self.tokenizer

+        if self.import_style is not None:
+            self.test_env['NOMINATIM_IMPORT_STYLE'] = self.import_style

 
         if self.server_module_path:
             self.test_env['NOMINATIM_DATABASE_MODULE_PATH'] = self.server_module_path
 
         if self.server_module_path:
             self.test_env['NOMINATIM_DATABASE_MODULE_PATH'] = self.server_module_path


@@ -121,13 +135,9 @@ class NominatimEnvironment:
 
 
     def get_test_config(self):
 
 
     def get_test_config(self):

-        cfg = Configuration(Path(self.website_dir.name), self.src_dir / 'settings',
-                            environ=self.test_env)
+        cfg = Configuration(Path(self.website_dir.name), environ=self.test_env)

         cfg.set_libdirs(module=self.build_dir / 'module',
         cfg.set_libdirs(module=self.build_dir / 'module',

-                        osm2pgsql=self.build_dir / 'osm2pgsql' / 'osm2pgsql',
-                        php=self.src_dir / 'lib-php',
-                        sql=self.src_dir / 'lib-sql',
-                        data=self.src_dir / 'data')
+                        osm2pgsql=self.build_dir / 'osm2pgsql' / 'osm2pgsql')

         return cfg
 
     def get_libpq_dsn(self):
         return cfg
 
     def get_libpq_dsn(self):


@@ -166,22 +176,23 @@ class NominatimEnvironment:
 
         self.template_db_done = True
 
 
         self.template_db_done = True
 

-        if self._reuse_or_drop_db(self.template_db):
-            return
-

         self.write_nominatim_config(self.template_db)
 
         self.write_nominatim_config(self.template_db)
 

-        try:
-            # execute nominatim import on an empty file to get the right tables
-            with tempfile.NamedTemporaryFile(dir='/tmp', suffix='.xml') as fd:
-                fd.write(b'<osm version="0.6"></osm>')
-                fd.flush()
-                self.run_nominatim('import', '--osm-file', fd.name,
-                                             '--osm2pgsql-cache', '1',
-                                             '--ignore-errors')
-        except:
-            self.db_drop_database(self.template_db)
-            raise
+        if not self._reuse_or_drop_db(self.template_db):
+            try:
+                # execute nominatim import on an empty file to get the right tables
+                with tempfile.NamedTemporaryFile(dir='/tmp', suffix='.xml') as fd:
+                    fd.write(b'<osm version="0.6"></osm>')
+                    fd.flush()
+                    self.run_nominatim('import', '--osm-file', fd.name,
+                                                 '--osm2pgsql-cache', '1',
+                                                 '--ignore-errors',
+                                                 '--offline', '--index-noanalyse')
+            except:
+                self.db_drop_database(self.template_db)
+                raise
+
+        self.run_nominatim('refresh', '--functions')

 
 
     def setup_api_db(self):
 
 
     def setup_api_db(self):


@@ -189,33 +200,34 @@ class NominatimEnvironment:
         """
         self.write_nominatim_config(self.api_test_db)
 
         """
         self.write_nominatim_config(self.api_test_db)
 

+        if self.api_test_db.startswith('sqlite:'):
+            return
+

         if not self.api_db_done:
             self.api_db_done = True
 
             if not self._reuse_or_drop_db(self.api_test_db):
         if not self.api_db_done:
             self.api_db_done = True
 
             if not self._reuse_or_drop_db(self.api_test_db):

-                testdata = Path('__file__') / '..' / '..' / 'testdb'
-                self.test_env['NOMINATIM_WIKIPEDIA_DATA_PATH'] = str(testdata.resolve())
+                testdata = (Path(__file__) / '..' / '..' / '..' / 'testdb').resolve()
+                self.test_env['NOMINATIM_WIKIPEDIA_DATA_PATH'] = str(testdata)
+                simp_file = Path(self.website_dir.name) / 'secondary_importance.sql.gz'
+                simp_file.symlink_to(testdata / 'secondary_importance.sql.gz')

 
                 try:
                     self.run_nominatim('import', '--osm-file', str(self.api_test_file))
 
                 try:
                     self.run_nominatim('import', '--osm-file', str(self.api_test_file))

-                    if self.tokenizer != 'legacy_icu':
-                        self.run_nominatim('add-data', '--tiger-data', str((testdata / 'tiger').resolve()))
+                    self.run_nominatim('add-data', '--tiger-data', str(testdata / 'tiger'))

                     self.run_nominatim('freeze')
 
                     self.run_nominatim('freeze')
 

-                    if self.tokenizer != 'legacy_icu':
-                        phrase_file = str((testdata / 'specialphrases_testdb.sql').resolve())
+                    if self.tokenizer == 'legacy':
+                        phrase_file = str(testdata / 'specialphrases_testdb.sql')

                         run_script(['psql', '-d', self.api_test_db, '-f', phrase_file])
                     else:
                         run_script(['psql', '-d', self.api_test_db, '-f', phrase_file])
                     else:

-                        # XXX Temporary use the wiki while there is no CSV import
-                        # available.
-                        self.test_env['NOMINATIM_LANGUAGES'] = 'en'
-                        self.run_nominatim('special-phrases', '--import-from-wiki')
-                        del self.test_env['NOMINATIM_LANGUAGES']
+                        csv_path = str(testdata / 'full_en_phrases_test.csv')
+                        self.run_nominatim('special-phrases', '--import-from-csv', csv_path)

                 except:
                     self.db_drop_database(self.api_test_db)
                     raise
 
                 except:
                     self.db_drop_database(self.api_test_db)
                     raise
 

-        tokenizer_factory.create_tokenizer(self.get_test_config(), init_db=False)
+        tokenizer_factory.get_tokenizer_for_db(self.get_test_config())

 
 
     def setup_unknown_db(self):
 
 
     def setup_unknown_db(self):


@@ -258,8 +270,8 @@ class NominatimEnvironment:
             self.db_drop_database(self.test_db)
 
     def _reuse_or_drop_db(self, name):
             self.db_drop_database(self.test_db)
 
     def _reuse_or_drop_db(self, name):

-        """ Check for the existance of the given DB. If reuse is enabled,
-            then the function checks for existance and returns True if the
+        """ Check for the existence of the given DB. If reuse is enabled,
+            then the function checks for existnce and returns True if the

             database is already there. Otherwise an existing database is
             dropped and always false returned.
         """
             database is already there. Otherwise an existing database is
             dropped and always false returned.
         """


@@ -298,12 +310,7 @@ class NominatimEnvironment:
 
         cli.nominatim(module_dir='',
                       osm2pgsql_path=str(self.build_dir / 'osm2pgsql' / 'osm2pgsql'),
 
         cli.nominatim(module_dir='',
                       osm2pgsql_path=str(self.build_dir / 'osm2pgsql' / 'osm2pgsql'),

-                      phplib_dir=str(self.src_dir / 'lib-php'),
-                      sqllib_dir=str(self.src_dir / 'lib-sql'),
-                      data_dir=str(self.src_dir / 'data'),
-                      config_dir=str(self.src_dir / 'settings'),

                       cli_args=cmdline,
                       cli_args=cmdline,

-                      phpcgi_path='',

                       environ=self.test_env)
 
 
                       environ=self.test_env)
 
 


@@ -328,3 +335,40 @@ class NominatimEnvironment:
                               WHERE class='place' and type='houses'
                                     and osm_type='W'
                                     and ST_GeometryType(geometry) = 'ST_LineString'""")
                               WHERE class='place' and type='houses'
                                     and osm_type='W'
                                     and ST_GeometryType(geometry) = 'ST_LineString'""")

+
+
+    def create_api_request_func_starlette(self):
+        import nominatim.server.starlette.server
+        from asgi_lifespan import LifespanManager
+        import httpx
+
+        async def _request(endpoint, params, project_dir, environ, http_headers):
+            app = nominatim.server.starlette.server.get_application(project_dir, environ)
+
+            async with LifespanManager(app):
+                async with httpx.AsyncClient(app=app, base_url="http://nominatim.test") as client:
+                    response = await client.get(f"/{endpoint}", params=params,
+                                                headers=http_headers)
+
+            return response.text, response.status_code
+
+        return _request
+
+
+    def create_api_request_func_falcon(self):
+        import nominatim.server.falcon.server
+        import falcon.testing
+
+        async def _request(endpoint, params, project_dir, environ, http_headers):
+            app = nominatim.server.falcon.server.get_application(project_dir, environ)
+
+            async with falcon.testing.ASGIConductor(app) as conductor:
+                response = await conductor.get(f"/{endpoint}", params=params,
+                                               headers=http_headers)
+
+            return response.text, response.status_code
+
+        return _request
+
+
+