add skeleton code for convert function

diff --git a/nominatim/cli.py b/nominatim/cli.py
index 1029ee7a91247f693c75d990c2de46df7e35e3ef..88a6078284424b4dc3beacf7d45757eddc1af3bb 100644 (file)
--- a/nominatim/cli.py
+++ b/nominatim/cli.py
@@ -206,6 +206,7 @@ def get_set_parser() -> CommandlineParser:
     parser.add_subcommand('admin', clicmd.AdminFuncs())
 
     parser.add_subcommand('export', clicmd.QueryExport())
     parser.add_subcommand('admin', clicmd.AdminFuncs())
 
     parser.add_subcommand('export', clicmd.QueryExport())

+    parser.add_subcommand('convert', clicmd.ConvertDB())

     parser.add_subcommand('serve', AdminServe())
 
     parser.add_subcommand('search', clicmd.APISearch())
     parser.add_subcommand('serve', AdminServe())
 
     parser.add_subcommand('search', clicmd.APISearch())

diff --git a/nominatim/clicmd/__init__.py b/nominatim/clicmd/__init__.py
index 235dff0cec3a44938ea2bb144cc2d5fc8f04ef66..c8de68c144eb53704d0a96e05bd82f44f209d2df 100644 (file)
--- a/nominatim/clicmd/__init__.py
+++ b/nominatim/clicmd/__init__.py
@@ -25,3 +25,4 @@ from nominatim.clicmd.admin import AdminFuncs as AdminFuncs
 from nominatim.clicmd.freeze import SetupFreeze as SetupFreeze
 from nominatim.clicmd.special_phrases import ImportSpecialPhrases as ImportSpecialPhrases
 from nominatim.clicmd.export import QueryExport as QueryExport
 from nominatim.clicmd.freeze import SetupFreeze as SetupFreeze
 from nominatim.clicmd.special_phrases import ImportSpecialPhrases as ImportSpecialPhrases
 from nominatim.clicmd.export import QueryExport as QueryExport

+from nominatim.clicmd.convert import ConvertDB as ConvertDB

diff --git a/nominatim/clicmd/args.py b/nominatim/clicmd/args.py
index e632e4c709c0b3c14535d285189584b89a2315b3..eb3a3b6145f3c76dc569ebbcdbd6cbbc4fa48b37 100644 (file)
--- a/nominatim/clicmd/args.py
+++ b/nominatim/clicmd/args.py
@@ -101,6 +101,9 @@ class NominatimArgs:
     language: Optional[str]
     restrict_to_country: Optional[str]
 
     language: Optional[str]
     restrict_to_country: Optional[str]
 

+    # Arguments to 'convert'
+    output: Path
+

     # Arguments to 'refresh'
     postcodes: bool
     word_tokens: bool
     # Arguments to 'refresh'
     postcodes: bool
     word_tokens: bool

diff --git a/nominatim/clicmd/convert.py b/nominatim/clicmd/convert.py
new file mode 100644 (file)
index 0000000..26b3fb1
--- /dev/null
+++ b/nominatim/clicmd/convert.py
@@ -0,0 +1,95 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+#
+# This file is part of Nominatim. (https://nominatim.org)
+#
+# Copyright (C) 2023 by the Nominatim developer community.
+# For a full list of authors see the git log.
+"""
+Implementation of the 'convert' subcommand.
+"""
+from typing import Set, Any, Union, Optional, Sequence
+import argparse
+import asyncio
+from pathlib import Path
+
+from nominatim.clicmd.args import NominatimArgs
+from nominatim.errors import UsageError
+
+# Do not repeat documentation of subcommand classes.
+# pylint: disable=C0111
+# Using non-top-level imports to avoid eventually unused imports.
+# pylint: disable=E0012,C0415
+
+class WithAction(argparse.Action):
+    """ Special action that saves a list of flags, given on the command-line
+        as `--with-foo` or `--without-foo`.
+    """
+    def __init__(self, option_strings: Sequence[str], dest: Any,
+                 default: bool = True, **kwargs: Any) -> None:
+        if 'nargs' in kwargs:
+            raise ValueError("nargs not allowed.")
+        if option_strings is None:
+            raise ValueError("Positional parameter not allowed.")
+
+        self.dest_set = kwargs.pop('dest_set')
+        full_option_strings = []
+        for opt in option_strings:
+            if not opt.startswith('--'):
+                raise ValueError("short-form options not allowed")
+            if default:
+                self.dest_set.add(opt[2:])
+            full_option_strings.append(f"--with-{opt[2:]}")
+            full_option_strings.append(f"--without-{opt[2:]}")
+
+        super().__init__(full_option_strings, argparse.SUPPRESS, nargs=0, **kwargs)
+
+
+    def __call__(self, parser: argparse.ArgumentParser, namespace: argparse.Namespace,
+                 values: Union[str, Sequence[Any], None],
+                 option_string: Optional[str] = None) -> None:
+        assert option_string
+        if option_string.startswith('--with-'):
+            self.dest_set.add(option_string[7:])
+        if option_string.startswith('--without-'):
+            self.dest_set.discard(option_string[10:])
+
+
+class ConvertDB:
+    """ Convert an existing database into a different format. (EXPERIMENTAL)
+
+        Dump a read-only version of the database in a different format.
+        At the moment only a SQLite database suitable for reverse lookup
+        can be created.
+    """
+
+    def __init__(self) -> None:
+        self.options: Set[str] = set()
+
+    def add_args(self, parser: argparse.ArgumentParser) -> None:
+        parser.add_argument('--format', default='sqlite',
+                            choices=('sqlite', ),
+                            help='Format of the output database (must be sqlite currently)')
+        parser.add_argument('--output', '-o', required=True, type=Path,
+                            help='File to write the database to.')
+        group = parser.add_argument_group('Switches to define database layout'
+                                          '(currently no effect)')
+        group.add_argument('--reverse', action=WithAction, dest_set=self.options, default=True,
+                           help='Enable/disable support for reverse and lookup API'
+                                ' (default: enabled)')
+        group.add_argument('--search', action=WithAction, dest_set=self.options, default=False,
+                           help='Enable/disable support for search API (default: disabled)')
+        group.add_argument('--details', action=WithAction, dest_set=self.options, default=True,
+                           help='Enable/disable support for details API (default: enabled)')
+
+
+    def run(self, args: NominatimArgs) -> int:
+        if args.output.exists():
+            raise UsageError(f"File '{args.output}' already exists. Refusing to overwrite.")
+
+        if args.format == 'sqlite':
+            from ..tools import convert_sqlite
+
+            asyncio.run(convert_sqlite.convert(args.project_dir, args.output, self.options))
+            return 0
+
+        return 1

diff --git a/nominatim/tools/convert_sqlite.py b/nominatim/tools/convert_sqlite.py
new file mode 100644 (file)
index 0000000..42977ce
--- /dev/null
+++ b/nominatim/tools/convert_sqlite.py
@@ -0,0 +1,30 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+#
+# This file is part of Nominatim. (https://nominatim.org)
+#
+# Copyright (C) 2023 by the Nominatim developer community.
+# For a full list of authors see the git log.
+"""
+Exporting a Nominatim database to SQlite.
+"""
+from typing import Set
+from pathlib import Path
+
+import sqlalchemy as sa
+
+import nominatim.api as napi
+
+async def convert(project_dir: Path, outfile: Path, options: Set[str]) -> None:
+    """ Export an existing database to sqlite. The resulting database
+        will be usable against the Python frontend of Nominatim.
+    """
+    api = napi.NominatimAPIAsync(project_dir)
+
+    try:
+        outapi = napi.NominatimAPIAsync(project_dir,
+                                        {'NOMINATIM_DATABASE_DSN': f"sqlite:dbname={outfile}"})
+
+        async with api.begin() as inconn, outapi.begin() as outconn:
+            pass
+    finally:
+        await api.close()