Revert "work round typing bug in pyosmium 4.0"

[nominatim.git] / src / nominatim_db / tools / exec_utils.py

diff --git a/src/nominatim_db/tools/exec_utils.py b/src/nominatim_db/tools/exec_utils.py
index 5450c43bb36f66eda9fccb41d1dff8d272195e08..4cbbf95d9b650c2e8d11f6c2b36b4d5c32e78c60 100644 (file)
--- a/src/nominatim_db/tools/exec_utils.py
+++ b/src/nominatim_db/tools/exec_utils.py
@@ -7,36 +7,29 @@
 """
 Helper functions for executing external programs.
 """
 """
 Helper functions for executing external programs.
 """

-from typing import Any, Mapping
+from typing import Any, Mapping, List, Optional

 import logging
 import os
 import logging
 import os

+import re

 import subprocess
 import shutil
 
 import subprocess
 import shutil
 

-from nominatim_core.typing import StrPath
-from nominatim_core.db.connection import get_pg_env
+from ..db.connection import get_pg_env
+from ..errors import UsageError
+from ..version import OSM2PGSQL_REQUIRED_VERSION

 
 LOG = logging.getLogger()
 
 
 LOG = logging.getLogger()
 

-def run_php_server(server_address: str, base_dir: StrPath) -> None:
-    """ Run the built-in server from the given directory.
-    """
-    subprocess.run(['/usr/bin/env', 'php', '-S', server_address],
-                   cwd=str(base_dir), check=True)
-

 
 def run_osm2pgsql(options: Mapping[str, Any]) -> None:
     """ Run osm2pgsql with the given options.
     """
 
 def run_osm2pgsql(options: Mapping[str, Any]) -> None:
     """ Run osm2pgsql with the given options.
     """

-    env = get_pg_env(options['dsn'])
+    _check_osm2pgsql_version(options['osm2pgsql'])

 
 

-    osm2pgsql_cmd = options['osm2pgsql']
-    if osm2pgsql_cmd is None:
-        osm2pgsql_cmd = shutil.which('osm2pgsql')
-        if osm2pgsql_cmd is None:
-            raise RuntimeError('osm2pgsql executable not found. Please install osm2pgsql first.')
+    env = get_pg_env(options['dsn'])

 
 

-    cmd = [str(osm2pgsql_cmd),
+    cmd = [_find_osm2pgsql_cmd(options['osm2pgsql']),
+           '--append' if options['append'] else '--create',

            '--slim',
            '--log-progress', 'true',
            '--number-processes', '1' if options['append'] else str(options['threads']),
            '--slim',
            '--log-progress', 'true',
            '--number-processes', '1' if options['append'] else str(options['threads']),


@@ -48,25 +41,20 @@ def run_osm2pgsql(options: Mapping[str, Any]) -> None:
         env['LUA_PATH'] = ';'.join((str(options['osm2pgsql_style_path'] / '?.lua'),
                                     os.environ.get('LUAPATH', ';')))
         cmd.extend(('--output', 'flex'))
         env['LUA_PATH'] = ';'.join((str(options['osm2pgsql_style_path'] / '?.lua'),
                                     os.environ.get('LUAPATH', ';')))
         cmd.extend(('--output', 'flex'))

+
+        for flavour in ('data', 'index'):
+            if options['tablespaces'][f"main_{flavour}"]:
+                env[f"NOMINATIM_TABLESPACE_PLACE_{flavour.upper()}"] = \
+                    options['tablespaces'][f"main_{flavour}"]

     else:
         cmd.extend(('--output', 'gazetteer', '--hstore', '--latlon'))
     else:
         cmd.extend(('--output', 'gazetteer', '--hstore', '--latlon'))

+        cmd.extend(_mk_tablespace_options('main', options))

 
 

-    cmd.append('--append' if options['append'] else '--create')

 
     if options['flatnode_file']:
         cmd.extend(('--flat-nodes', options['flatnode_file']))
 
 
     if options['flatnode_file']:
         cmd.extend(('--flat-nodes', options['flatnode_file']))
 

-    for key, param in (('slim_data', '--tablespace-slim-data'),
-                       ('slim_index', '--tablespace-slim-index'),
-                       ('main_data', '--tablespace-main-data'),
-                       ('main_index', '--tablespace-main-index')):
-        if options['tablespaces'][key]:
-            cmd.extend((param, options['tablespaces'][key]))
-
-    if options['tablespaces']['main_data']:
-        env['NOMINATIM_TABLESPACE_PLACE_DATA'] = options['tablespaces']['main_data']
-    if options['tablespaces']['main_index']:
-        env['NOMINATIM_TABLESPACE_PLACE_INDEX'] = options['tablespaces']['main_index']
+    cmd.extend(_mk_tablespace_options('slim', options))

 
     if options.get('disable_jit', False):
         env['PGOPTIONS'] = '-c jit=off -c max_parallel_workers_per_gather=0'
 
     if options.get('disable_jit', False):
         env['PGOPTIONS'] = '-c jit=off -c max_parallel_workers_per_gather=0'


@@ -82,3 +70,43 @@ def run_osm2pgsql(options: Mapping[str, Any]) -> None:
     subprocess.run(cmd, cwd=options.get('cwd', '.'),
                    input=options.get('import_data'),
                    env=env, check=True)
     subprocess.run(cmd, cwd=options.get('cwd', '.'),
                    input=options.get('import_data'),
                    env=env, check=True)

+
+
+def _mk_tablespace_options(ttype: str, options: Mapping[str, Any]) -> List[str]:
+    cmds: List[str] = []
+    for flavour in ('data', 'index'):
+        if options['tablespaces'][f"{ttype}_{flavour}"]:
+            cmds.extend((f"--tablespace-{ttype}-{flavour}",
+                         options['tablespaces'][f"{ttype}_{flavour}"]))
+
+    return cmds
+
+
+def _find_osm2pgsql_cmd(cmdline: Optional[str]) -> str:
+    if cmdline is not None:
+        return cmdline
+
+    in_path = shutil.which('osm2pgsql')
+    if in_path is None:
+        raise UsageError('osm2pgsql executable not found. Please install osm2pgsql first.')
+
+    return str(in_path)
+
+
+def _check_osm2pgsql_version(cmdline: Optional[str]) -> None:
+    cmd = [_find_osm2pgsql_cmd(cmdline), '--version']
+
+    result = subprocess.run(cmd, capture_output=True, check=True)
+
+    if not result.stderr:
+        raise UsageError("osm2pgsql does not print version information.")
+
+    verinfo = result.stderr.decode('UTF-8')
+
+    match = re.search(r'osm2pgsql version (\d+)\.(\d+)', verinfo)
+    if match is None:
+        raise UsageError(f"No version information found in output: {verinfo}")
+
+    if (int(match[1]), int(match[2])) < OSM2PGSQL_REQUIRED_VERSION:
+        raise UsageError(f"osm2pgsql is too old. Found version {match[1]}.{match[2]}. "
+                         f"Need at least version {'.'.join(map(str, OSM2PGSQL_REQUIRED_VERSION))}.")