X-Git-Url: https://git.openstreetmap.org./nominatim.git/blobdiff_plain/04690ad8c4734c15efc920bc0dab59cf78e3e7cb..3fb739de9275680d4f482d363f22c535442f0017:/nominatim/cli.py diff --git a/nominatim/cli.py b/nominatim/cli.py index a40dc07e..88a60782 100644 --- a/nominatim/cli.py +++ b/nominatim/cli.py @@ -1,20 +1,35 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# This file is part of Nominatim. (https://nominatim.org) +# +# Copyright (C) 2023 by the Nominatim developer community. +# For a full list of authors see the git log. """ Command-line interface to the Nominatim functions for import, update, database administration and querying. """ +from typing import Optional, Any +import importlib +import logging +import os import sys import argparse -import logging from pathlib import Path -from .config import Configuration -from .admin.exec_utils import run_legacy_script +from nominatim.config import Configuration +from nominatim.tools.exec_utils import run_php_server +from nominatim.errors import UsageError +from nominatim import clicmd +from nominatim import version +from nominatim.clicmd.args import NominatimArgs, Subcommand + +LOG = logging.getLogger() class CommandlineParser: """ Wraps some of the common functions for parsing the command line and setting up subcommands. """ - def __init__(self, prog, description): + def __init__(self, prog: str, description: Optional[str]): self.parser = argparse.ArgumentParser( prog=prog, description=description, @@ -23,6 +38,10 @@ class CommandlineParser: self.subs = self.parser.add_subparsers(title='available commands', dest='subcommand') + # Global arguments that only work if no sub-command given + self.parser.add_argument('--version', action='store_true', + help='Print Nominatim version and exit') + # Arguments added to every sub-command self.default_args = argparse.ArgumentParser(add_help=False) group = self.default_args.add_argument_group('Default arguments') @@ -39,11 +58,22 @@ class CommandlineParser: help='Number of parallel threads to use') - def add_subcommand(self, name, cmd): + def nominatim_version_text(self) -> str: + """ Program name and version number as string + """ + text = f'Nominatim version {version.NOMINATIM_VERSION!s}' + if version.GIT_COMMIT_HASH is not None: + text += f' ({version.GIT_COMMIT_HASH})' + return text + + + def add_subcommand(self, name: str, cmd: Subcommand) -> None: """ Add a subcommand to the parser. The subcommand must be a class with a function add_args() that adds the parameters for the subcommand and a run() function that executes the command. """ + assert cmd.__doc__ is not None + parser = self.subs.add_parser(name, parents=[self.default_args], help=cmd.__doc__.split('\n', 1)[0], description=cmd.__doc__, @@ -52,340 +82,145 @@ class CommandlineParser: parser.set_defaults(command=cmd) cmd.add_args(parser) - def run(self, **kwargs): + + def run(self, **kwargs: Any) -> int: """ Parse the command line arguments of the program and execute the appropriate subcommand. """ - args = self.parser.parse_args() - - if args.subcommand is None: - return self.parser.print_help() - - for arg in ('module_dir', 'osm2pgsql_path', 'phplib_dir', 'data_dir'): - setattr(args, arg, Path(kwargs[arg])) - args.project_dir = Path(args.project_dir) - - logging.basicConfig(stream=sys.stderr, - format='%(asctime)s %(levelname)s: %(message)s', - datefmt='%Y-%m-%d %H:%M:%S', - level=max(4 - args.verbose, 1) * 10) - - args.config = Configuration(args.project_dir, args.data_dir / 'settings') + args = NominatimArgs() + try: + self.parser.parse_args(args=kwargs.get('cli_args'), namespace=args) + except SystemExit: + return 1 - args.command.run(args) + if args.version: + print(self.nominatim_version_text()) + return 0 - -class SetupAll: + if args.subcommand is None: + self.parser.print_help() + return 1 + + args.project_dir = Path(args.project_dir).resolve() + + if 'cli_args' not in kwargs: + logging.basicConfig(stream=sys.stderr, + format='%(asctime)s: %(message)s', + datefmt='%Y-%m-%d %H:%M:%S', + level=max(4 - args.verbose, 1) * 10) + + args.config = Configuration(args.project_dir, + environ=kwargs.get('environ', os.environ)) + args.config.set_libdirs(module=kwargs['module_dir'], + osm2pgsql=kwargs['osm2pgsql_path']) + + log = logging.getLogger() + log.warning('Using project directory: %s', str(args.project_dir)) + + try: + return args.command.run(args) + except UsageError as exception: + if log.isEnabledFor(logging.DEBUG): + raise # use Python's exception printing + log.fatal('FATAL: %s', exception) + + # If we get here, then execution has failed in some way. + return 1 + + +# Subcommand classes +# +# Each class needs to implement two functions: add_args() adds the CLI parameters +# for the subfunction, run() executes the subcommand. +# +# The class documentation doubles as the help text for the command. The +# first line is also used in the summary when calling the program without +# a subcommand. +# +# No need to document the functions each time. +# pylint: disable=C0111 +class AdminServe: """\ - Create a new Nominatim database from an OSM file. - """ + Start a simple web server for serving the API. - @staticmethod - def add_args(parser): - group_name = parser.add_argument_group('Required arguments') - group = group_name.add_mutually_exclusive_group(required=True) - group.add_argument('--osm-file', - help='OSM file to be imported.') - group.add_argument('--continue', nargs=1, - choices=['load-data', 'indexing', 'db-postprocess'], - help='Continue an import that was interrupted') - group = parser.add_argument_group('Optional arguments') - group.add_argument('--osm2pgsql-cache', metavar='SIZE', type=int, - help='Size of cache to be used by osm2pgsql (in MB)') - group.add_argument('--reverse-only', action='store_true', - help='Do not create tables and indexes for searching') - group.add_argument('--enable-debug-statements', action='store_true', - help='Include debug warning statements in SQL code') - group.add_argument('--no-partitions', action='store_true', - help="""Do not partition search indices - (speeds up import of single country extracts)""") - group.add_argument('--no-updates', action='store_true', - help="""Do not keep tables that are only needed for - updating the database later""") - group = parser.add_argument_group('Expert options') - group.add_argument('--ignore-errors', action='store_true', - help='Continue import even when errors in SQL are present') - group.add_argument('--disable-token-precalc', action='store_true', - help='Disable name precalculation') - group.add_argument('--index-noanalyse', action='store_true', - help='Do not perform analyse operations during index') - - - @staticmethod - def run(args): - print("TODO: ./utils/setup.php", args) - - -class SetupFreeze: - """\ - Make database read-only. + This command starts a built-in webserver to serve the website + from the current project directory. This webserver is only suitable + for testing and development. Do not use it in production setups! - About half of data in the Nominatim database is kept only to be able to - keep the data up-to-date with new changes made in OpenStreetMap. This - command drops all this data and only keeps the part needed for geocoding - itself. + There are different webservers available. The default 'php' engine + runs the classic PHP frontend. The other engines are Python servers + which run the new Python frontend code. This is highly experimental + at the moment and may not include the full API. - This command has the same effect as the `--no-updates` option for imports. + By the default, the webserver can be accessed at: http://127.0.0.1:8088 """ - @staticmethod - def add_args(parser): - pass # No options + def add_args(self, parser: argparse.ArgumentParser) -> None: + group = parser.add_argument_group('Server arguments') + group.add_argument('--server', default='127.0.0.1:8088', + help='The address the server will listen to.') + group.add_argument('--engine', default='php', + choices=('php', 'falcon', 'starlette'), + help='Webserver framework to run. (default: php)') - @staticmethod - def run(args): - print("TODO: setup drop", args) - -class SetupSpecialPhrases: - """\ - Maintain special phrases. - """ - - @staticmethod - def add_args(parser): - group = parser.add_argument_group('Input arguments') - group.add_argument('--from-wiki', action='store_true', - help='Pull special phrases from the OSM wiki.') - group = parser.add_argument_group('Output arguments') - group.add_argument('-o', '--output', default='-', - type=argparse.FileType('w', encoding='UTF-8'), - help="""File to write the preprocessed phrases to. - If omitted, it will be written to stdout.""") - - @staticmethod - def run(args): - print("./utils/specialphrases.php --from-wiki", args) - - -class UpdateReplication: - """\ - Update the database using an online replication service. - """ - - @staticmethod - def add_args(parser): - group = parser.add_argument_group('Arguments for initialisation') - group.add_argument('--init', action='store_true', - help='Initialise the update process') - group.add_argument('--no-update-functions', dest='update_functions', - action='store_false', - help="""Do not update the trigger function to - support differential updates.""") - group = parser.add_argument_group('Arguments for updates') - group.add_argument('--check-for-updates', action='store_true', - help='Check if new updates are available and exit') - group.add_argument('--once', action='store_true', - help="""Download and apply updates only once. When - not set, updates are continuously applied""") - group.add_argument('--no-index', action='store_false', dest='do_index', - help="""Do not index the new data. Only applicable - together with --once""") - - @staticmethod - def run(args): - if args.init: - print('./utils/update.php --init-updates', args) + def run(self, args: NominatimArgs) -> int: + if args.engine == 'php': + run_php_server(args.server, args.project_dir / 'website') else: - print('./utils/update.php --import-osmosis(-all)', args) - + import uvicorn # pylint: disable=import-outside-toplevel + server_info = args.server.split(':', 1) + host = server_info[0] + if len(server_info) > 1: + if not server_info[1].isdigit(): + raise UsageError('Invalid format for --server parameter. Use :') + port = int(server_info[1]) + else: + port = 8088 -class UpdateAddData: - """\ - Add additional data from a file or an online source. + server_module = importlib.import_module(f'nominatim.server.{args.engine}.server') - Data is only imported, not indexed. You need to call `nominatim-update index` - to complete the process. - """ - - @staticmethod - def add_args(parser): - group_name = parser.add_argument_group('Source') - group = group_name.add_mutually_exclusive_group(required=True) - group.add_argument('--file', metavar='FILE', - help='Import data from an OSM file') - group.add_argument('--diff', metavar='FILE', - help='Import data from an OSM diff file') - group.add_argument('--node', metavar='ID', type=int, - help='Import a single node from the API') - group.add_argument('--way', metavar='ID', type=int, - help='Import a single way from the API') - group.add_argument('--relation', metavar='ID', type=int, - help='Import a single relation from the API') - group.add_argument('--tiger-data', metavar='DIR', - help='Add housenumbers from the US TIGER census database.') - group = parser.add_argument_group('Extra arguments') - group.add_argument('--use-main-api', action='store_true', - help='Use OSM API instead of Overpass to download objects') - - @staticmethod - def run(args): - print('./utils/update.php --import-*', args) - - -class UpdateIndex: - """\ - Reindex all new and modified data. - """ + app = server_module.get_application(args.project_dir) + uvicorn.run(app, host=host, port=port) - @staticmethod - def add_args(parser): - pass + return 0 - @staticmethod - def run(args): - print('./utils/update.php --index', args) - -class UpdateRefresh: - """\ - Recompute auxillary data used by the indexing process. - - These functions must not be run in parallel with other update commands. - """ - - @staticmethod - def add_args(parser): - group = parser.add_argument_group('Data arguments') - group.add_argument('--postcodes', action='store_true', - help='Update postcode centroid table') - group.add_argument('--word-counts', action='store_true', - help='Compute frequency of full-word search terms') - group.add_argument('--address-levels', action='store_true', - help='Reimport address level configuration') - group.add_argument('--importance', action='store_true', - help='Recompute place importances') - group.add_argument('--functions', action='store_true', - help='Update the PL/pgSQL functions in the database') - group.add_argument('--wiki-data', - help='Update Wikipedia/data importance numbers.') - group.add_argument('--website', action='store_true', - help='Refresh the directory that serves the scripts for the web API') - group = parser.add_argument_group('Arguments for function refresh') - group.add_argument('--no-diff-updates', action='store_false', dest='diffs', - help='Do not enable code for propagating updates') - - @staticmethod - def run(args): - print('./utils/update.php', args) - - -class AdminCheckDatabase: +def get_set_parser() -> CommandlineParser: """\ - Check that the database is complete and operational. + Initializes the parser and adds various subcommands for + nominatim cli. """ + parser = CommandlineParser('nominatim', nominatim.__doc__) - @staticmethod - def add_args(parser): - pass # No options + parser.add_subcommand('import', clicmd.SetupAll()) + parser.add_subcommand('freeze', clicmd.SetupFreeze()) + parser.add_subcommand('replication', clicmd.UpdateReplication()) - @staticmethod - def run(args): - print("TODO: ./utils/check_import_finished.php", args) + parser.add_subcommand('special-phrases', clicmd.ImportSpecialPhrases()) + parser.add_subcommand('add-data', clicmd.UpdateAddData()) + parser.add_subcommand('index', clicmd.UpdateIndex()) + parser.add_subcommand('refresh', clicmd.UpdateRefresh()) -class AdminWarm: - """\ - Warm database caches for search and reverse queries. - """ + parser.add_subcommand('admin', clicmd.AdminFuncs()) - @staticmethod - def add_args(parser): - group = parser.add_argument_group('Target arguments') - group.add_argument('--search-only', action='store_const', dest='target', - const='search', - help="Only pre-warm tables for search queries") - group.add_argument('--reverse-only', action='store_const', dest='target', - const='reverse', - help="Only pre-warm tables for reverse queries") - - @staticmethod - def run(args): - params = ['warm.php'] - if args.target == 'reverse': - params.append('--reverse-only') - if args.target == 'search': - params.append('--search-only') - return run_legacy_script(*params, nominatim_env=args) - - -class QueryExport: - """\ - Export addresses as CSV file from a Nominatim database. - """ + parser.add_subcommand('export', clicmd.QueryExport()) + parser.add_subcommand('convert', clicmd.ConvertDB()) + parser.add_subcommand('serve', AdminServe()) - @staticmethod - def add_args(parser): - group = parser.add_argument_group('Output arguments') - group.add_argument('--output-type', default='street', - choices=('continent', 'country', 'state', 'county', - 'city', 'suburb', 'street', 'path'), - help='Type of places to output (default: street)') - group.add_argument('--output-format', - default='street;suburb;city;county;state;country', - help="""Semicolon-separated list of address types - (see --output-type). Multiple ranks can be - merged into one column by simply using a - comma-separated list.""") - group.add_argument('--output-all-postcodes', action='store_true', - help="""List all postcodes for address instead of - just the most likely one""") - group.add_argument('--language', - help="""Preferred language for output - (use local name, if omitted)""") - group = parser.add_argument_group('Filter arguments') - group.add_argument('--restrict-to-country', metavar='COUNTRY_CODE', - help='Export only objects within country') - group.add_argument('--restrict-to-osm-node', metavar='ID', type=int, - help='Export only children of this OSM node') - group.add_argument('--restrict-to-osm-way', metavar='ID', type=int, - help='Export only children of this OSM way') - group.add_argument('--restrict-to-osm-relation', metavar='ID', type=int, - help='Export only children of this OSM relation') - - - @staticmethod - def run(args): - print("TODO: ./utils/export.php", args) - - -class QueryTodo: - """\ - Todo - """ - @staticmethod - def add_args(parser): - pass + parser.add_subcommand('search', clicmd.APISearch()) + parser.add_subcommand('reverse', clicmd.APIReverse()) + parser.add_subcommand('lookup', clicmd.APILookup()) + parser.add_subcommand('details', clicmd.APIDetails()) + parser.add_subcommand('status', clicmd.APIStatus()) - def run(args): - print("TODO: searching") + return parser -def nominatim(**kwargs): +def nominatim(**kwargs: Any) -> int: """\ Command-line tools for importing, updating, administrating and querying the Nominatim database. """ - parser = CommandlineParser('nominatim', nominatim.__doc__) - - parser.add_subcommand('import', SetupAll) - parser.add_subcommand('freeze', SetupFreeze) - parser.add_subcommand('replication', UpdateReplication) - - parser.add_subcommand('check-database', AdminCheckDatabase) - parser.add_subcommand('warm', AdminWarm) - - parser.add_subcommand('special-phrases', SetupSpecialPhrases) - - parser.add_subcommand('add-data', UpdateAddData) - parser.add_subcommand('index', UpdateIndex) - parser.add_subcommand('refresh', UpdateRefresh) - - parser.add_subcommand('export', QueryExport) - parser.add_subcommand('search', QueryTodo) - parser.add_subcommand('reverse', QueryTodo) - parser.add_subcommand('lookup', QueryTodo) - parser.add_subcommand('details', QueryTodo) - parser.add_subcommand('status', QueryTodo) - - parser.run(**kwargs) + return get_set_parser().run(**kwargs)