actions: run tests against psycopg

[nominatim.git] / nominatim / tools / exec_utils.py

diff --git a/nominatim/tools/exec_utils.py b/nominatim/tools/exec_utils.py
index f373f347dd23936fd155edda465373fcb09e42d4..566ac06edc57d4374b4979095866c64c1e959d60 100644 (file)
--- a/nominatim/tools/exec_utils.py
+++ b/nominatim/tools/exec_utils.py
@@ -1,44 +1,57 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# This file is part of Nominatim. (https://nominatim.org)
+#
+# Copyright (C) 2022 by the Nominatim developer community.
+# For a full list of authors see the git log.

 """
 Helper functions for executing external programs.
 """
 """
 Helper functions for executing external programs.
 """

+from typing import Any, Union, Optional, Mapping, IO
+from pathlib import Path

 import logging
 import os
 import subprocess
 import urllib.request as urlrequest
 from urllib.parse import urlencode
 
 import logging
 import os
 import subprocess
 import urllib.request as urlrequest
 from urllib.parse import urlencode
 

-from psycopg2.extensions import parse_dsn
-
-from ..version import NOMINATIM_VERSION
+from nominatim.config import Configuration
+from nominatim.typing import StrPath
+from nominatim.version import NOMINATIM_VERSION
+from nominatim.db.connection import get_pg_env

 
 LOG = logging.getLogger()
 
 
 LOG = logging.getLogger()
 

-def run_legacy_script(script, *args, nominatim_env=None, throw_on_fail=False):
+def run_legacy_script(script: StrPath, *args: Union[int, str],
+                      config: Configuration,
+                      throw_on_fail: bool = False) -> int:

     """ Run a Nominatim PHP script with the given arguments.
 
         Returns the exit code of the script. If `throw_on_fail` is True
         then throw a `CalledProcessError` on a non-zero exit.
     """
     cmd = ['/usr/bin/env', 'php', '-Cq',
     """ Run a Nominatim PHP script with the given arguments.
 
         Returns the exit code of the script. If `throw_on_fail` is True
         then throw a `CalledProcessError` on a non-zero exit.
     """
     cmd = ['/usr/bin/env', 'php', '-Cq',

-           nominatim_env.phplib_dir / 'admin' / script]
+           str(config.lib_dir.php / 'admin' / script)]

     cmd.extend([str(a) for a in args])
 
     cmd.extend([str(a) for a in args])
 

-    env = nominatim_env.config.get_os_env()
-    env['NOMINATIM_DATADIR'] = str(nominatim_env.data_dir)
-    env['NOMINATIM_SQLDIR'] = str(nominatim_env.sqllib_dir)
-    env['NOMINATIM_CONFIGDIR'] = str(nominatim_env.config_dir)
-    env['NOMINATIM_DATABASE_MODULE_SRC_PATH'] = nominatim_env.module_dir
+    env = config.get_os_env()
+    env['NOMINATIM_DATADIR'] = str(config.lib_dir.data)
+    env['NOMINATIM_SQLDIR'] = str(config.lib_dir.sql)
+    env['NOMINATIM_CONFIGDIR'] = str(config.config_dir)
+    env['NOMINATIM_DATABASE_MODULE_SRC_PATH'] = str(config.lib_dir.module)

     if not env['NOMINATIM_OSM2PGSQL_BINARY']:
     if not env['NOMINATIM_OSM2PGSQL_BINARY']:

-        env['NOMINATIM_OSM2PGSQL_BINARY'] = nominatim_env.osm2pgsql_path
+        env['NOMINATIM_OSM2PGSQL_BINARY'] = str(config.lib_dir.osm2pgsql)

 
 

-    proc = subprocess.run(cmd, cwd=str(nominatim_env.project_dir), env=env,
+    proc = subprocess.run(cmd, cwd=str(config.project_dir), env=env,

                           check=throw_on_fail)
 
     return proc.returncode
 
                           check=throw_on_fail)
 
     return proc.returncode
 

-def run_api_script(endpoint, project_dir, extra_env=None, phpcgi_bin=None,
-                   params=None):
-    """ Execute a Nominiatim API function.
+def run_api_script(endpoint: str, project_dir: Path,
+                   extra_env: Optional[Mapping[str, str]] = None,
+                   phpcgi_bin: Optional[Path] = None,
+                   params: Optional[Mapping[str, Any]] = None) -> int:
+    """ Execute a Nominatim API function.

 
         The function needs a project directory that contains the website
         directory with the scripts to be executed. The scripts will be run
 
         The function needs a project directory that contains the website
         directory with the scripts to be executed. The scripts will be run


@@ -51,10 +64,10 @@ def run_api_script(endpoint, project_dir, extra_env=None, phpcgi_bin=None,
     query_string = urlencode(params or {})
 
     env = dict(QUERY_STRING=query_string,
     query_string = urlencode(params or {})
 
     env = dict(QUERY_STRING=query_string,

-               SCRIPT_NAME='/{}.php'.format(endpoint),
-               REQUEST_URI='/{}.php?{}'.format(endpoint, query_string),
+               SCRIPT_NAME=f'/{endpoint}.php',
+               REQUEST_URI=f'/{endpoint}.php?{query_string}',

                CONTEXT_DOCUMENT_ROOT=webdir,
                CONTEXT_DOCUMENT_ROOT=webdir,

-               SCRIPT_FILENAME='{}/{}.php'.format(webdir, endpoint),
+               SCRIPT_FILENAME=f'{webdir}/{endpoint}.php',

                HTTP_HOST='localhost',
                HTTP_USER_AGENT='nominatim-tool',
                REMOTE_ADDR='0.0.0.0',
                HTTP_HOST='localhost',
                HTTP_USER_AGENT='nominatim-tool',
                REMOTE_ADDR='0.0.0.0',


@@ -72,7 +85,9 @@ def run_api_script(endpoint, project_dir, extra_env=None, phpcgi_bin=None,
     else:
         cmd = [str(phpcgi_bin)]
 
     else:
         cmd = [str(phpcgi_bin)]
 

-    proc = subprocess.run(cmd, cwd=str(project_dir), env=env, capture_output=True,
+    proc = subprocess.run(cmd, cwd=str(project_dir), env=env,
+                          stdout=subprocess.PIPE,
+                          stderr=subprocess.PIPE,

                           check=False)
 
     if proc.returncode != 0 or proc.stderr:
                           check=False)
 
     if proc.returncode != 0 or proc.stderr:


@@ -90,59 +105,69 @@ def run_api_script(endpoint, project_dir, extra_env=None, phpcgi_bin=None,
     return 0
 
 
     return 0
 
 

-def run_php_server(server_address, base_dir):
+def run_php_server(server_address: str, base_dir: StrPath) -> None:

     """ Run the built-in server from the given directory.
     """
     subprocess.run(['/usr/bin/env', 'php', '-S', server_address],
                    cwd=str(base_dir), check=True)
 
 
     """ Run the built-in server from the given directory.
     """
     subprocess.run(['/usr/bin/env', 'php', '-S', server_address],
                    cwd=str(base_dir), check=True)
 
 

-def run_osm2pgsql(options):
+def run_osm2pgsql(options: Mapping[str, Any]) -> None:

     """ Run osm2pgsql with the given options.
     """
     """ Run osm2pgsql with the given options.
     """

-    env = os.environ
-    cmd = [options['osm2pgsql'],
+    env = get_pg_env(options['dsn'])
+    cmd = [str(options['osm2pgsql']),

            '--hstore', '--latlon', '--slim',
            '--hstore', '--latlon', '--slim',

-           '--with-forward-dependencies', 'false',

            '--log-progress', 'true',
            '--log-progress', 'true',

-           '--number-processes', str(options['threads']),
+           '--number-processes', '1' if options['append'] else str(options['threads']),

            '--cache', str(options['osm2pgsql_cache']),
            '--cache', str(options['osm2pgsql_cache']),

-           '--output', 'gazetteer',

            '--style', str(options['osm2pgsql_style'])
           ]
            '--style', str(options['osm2pgsql_style'])
           ]

-    if options['append']:
-        cmd.append('--append')
+
+    if str(options['osm2pgsql_style']).endswith('.lua'):
+        env['LUA_PATH'] = ';'.join((str(options['osm2pgsql_style_path'] / '?.lua'),
+                                    os.environ.get('LUAPATH', ';')))
+        cmd.extend(('--output', 'flex'))
+    else:
+        cmd.extend(('--output', 'gazetteer'))
+
+    cmd.append('--append' if options['append'] else '--create')

 
     if options['flatnode_file']:
         cmd.extend(('--flat-nodes', options['flatnode_file']))
 
 
     if options['flatnode_file']:
         cmd.extend(('--flat-nodes', options['flatnode_file']))
 

-    dsn = parse_dsn(options['dsn'])
-    if 'password' in dsn:
-        env['PGPASSWORD'] = dsn['password']
-    if 'dbname' in dsn:
-        cmd.extend(('-d', dsn['dbname']))
-    if 'user' in dsn:
-        cmd.extend(('--username', dsn['user']))
-    for param in ('host', 'port'):
-        if param in dsn:
-            cmd.extend(('--' + param, dsn[param]))
+    for key, param in (('slim_data', '--tablespace-slim-data'),
+                       ('slim_index', '--tablespace-slim-index'),
+                       ('main_data', '--tablespace-main-data'),
+                       ('main_index', '--tablespace-main-index')):
+        if options['tablespaces'][key]:
+            cmd.extend((param, options['tablespaces'][key]))

 
     if options.get('disable_jit', False):
         env['PGOPTIONS'] = '-c jit=off -c max_parallel_workers_per_gather=0'
 
 
     if options.get('disable_jit', False):
         env['PGOPTIONS'] = '-c jit=off -c max_parallel_workers_per_gather=0'
 

-    cmd.append(str(options['import_file']))
+    if 'import_data' in options:
+        cmd.extend(('-r', 'xml', '-'))
+    elif isinstance(options['import_file'], list):
+        for fname in options['import_file']:
+            cmd.append(str(fname))
+    else:
+        cmd.append(str(options['import_file']))

 
 

-    subprocess.run(cmd, cwd=options.get('cwd', '.'), env=env, check=True)
+    subprocess.run(cmd, cwd=options.get('cwd', '.'),
+                   input=options.get('import_data'),
+                   env=env, check=True)

 
 
 
 

-def get_url(url):
+def get_url(url: str) -> str:

     """ Get the contents from the given URL and return it as a UTF-8 string.
     """
     """ Get the contents from the given URL and return it as a UTF-8 string.
     """

-    headers = {"User-Agent" : "Nominatim/" + NOMINATIM_VERSION}
+    headers = {"User-Agent": f"Nominatim/{NOMINATIM_VERSION!s}"}

 
     try:
 
     try:

-        with urlrequest.urlopen(urlrequest.Request(url, headers=headers)) as response:
+        request = urlrequest.Request(url, headers=headers)
+        with urlrequest.urlopen(request) as response: # type: IO[bytes]

             return response.read().decode('utf-8')
             return response.read().decode('utf-8')

-    except:
+    except Exception:

         LOG.fatal('Failed to load URL: %s', url)
         raise
         LOG.fatal('Failed to load URL: %s', url)
         raise