X-Git-Url: https://git.openstreetmap.org./nominatim.git/blobdiff_plain/98dbc84836dd834dd41cb4ada675456746e872f1..3475e1dfd6c1ef44b7e2cc046d62a6f50108b6a0:/nominatim/cli.py diff --git a/nominatim/cli.py b/nominatim/cli.py index 9d1e36d1..65ea90bb 100644 --- a/nominatim/cli.py +++ b/nominatim/cli.py @@ -3,6 +3,7 @@ Command-line interface to the Nominatim functions for import, update, database administration and querying. """ import sys +import os import argparse import logging from pathlib import Path @@ -10,6 +11,17 @@ from pathlib import Path from .config import Configuration from .admin.exec_utils import run_legacy_script +from .indexer.indexer import Indexer + +def _num_system_cpus(): + try: + cpus = len(os.sched_getaffinity(0)) + except NotImplementedError: + cpus = None + + return cpus or os.cpu_count() + + class CommandlineParser: """ Wraps some of the common functions for parsing the command line and setting up subcommands. @@ -66,13 +78,25 @@ class CommandlineParser: args.project_dir = Path(args.project_dir) logging.basicConfig(stream=sys.stderr, - format='%(asctime)s %(levelname)s: %(message)s', + format='%(asctime)s: %(message)s', datefmt='%Y-%m-%d %H:%M:%S', level=max(4 - args.verbose, 1) * 10) args.config = Configuration(args.project_dir, args.data_dir / 'settings') - args.command.run(args) + return args.command.run(args) + +##### Subcommand classes +# +# Each class needs to implement two functions: add_args() adds the CLI parameters +# for the subfunction, run() executes the subcommand. +# +# The class documentation doubles as the help text for the command. The +# first line is also used in the summary when calling the program without +# a subcommand. +# +# No need to document the functions each time. +# pylint: disable=C0111 class SetupAll: @@ -86,7 +110,7 @@ class SetupAll: group = group_name.add_mutually_exclusive_group(required=True) group.add_argument('--osm-file', help='OSM file to be imported.') - group.add_argument('--continue', nargs=1, dest='continue_at', + group.add_argument('--continue', dest='continue_at', choices=['load-data', 'indexing', 'db-postprocess'], help='Continue an import that was interrupted') group = parser.add_argument_group('Optional arguments') @@ -180,7 +204,7 @@ class SetupSpecialPhrases: def run(args): if args.output.name != '': raise NotImplementedError('Only output to stdout is currently implemented.') - return run_legacy_script('specialphrases.php', '--wiki-import' , nominatim_env=args) + return run_legacy_script('specialphrases.php', '--wiki-import', nominatim_env=args) class UpdateReplication: @@ -213,7 +237,7 @@ class UpdateReplication: if args.init: params.append('--init-updates') if not args.update_functions: - params.apend('--no-update-functions') + params.append('--no-update-functions') elif args.check_for_updates: params.append('--check-for-updates') else: @@ -258,9 +282,10 @@ class UpdateAddData: @staticmethod def run(args): if args.tiger_data: + os.environ['NOMINATIM_TIGER_DATA_PATH'] = args.tiger_data return run_legacy_script('setup.php', '--import-tiger-data', nominatim_env=args) - params = [ 'update.php'] + params = ['update.php'] if args.file: params.extend(('--import-file', args.file)) elif args.diff: @@ -270,7 +295,7 @@ class UpdateAddData: elif args.way: params.extend(('--import-way', args.way)) elif args.relation: - params.extend(('--import-relation' , args.relation)) + params.extend(('--import-relation', args.relation)) if args.use_main_api: params.append('--use-main-api') return run_legacy_script(*params, nominatim_env=args) @@ -283,16 +308,35 @@ class UpdateIndex: @staticmethod def add_args(parser): - pass + group = parser.add_argument_group('Filter arguments') + group.add_argument('--boundaries-only', action='store_true', + help="""Index only administrative boundaries.""") + group.add_argument('--no-boundaries', action='store_true', + help="""Index everything except administrative boundaries.""") + group.add_argument('--minrank', '-r', type=int, metavar='RANK', default=0, + help='Minimum/starting rank') + group.add_argument('--maxrank', '-R', type=int, metavar='RANK', default=30, + help='Maximum/finishing rank') @staticmethod def run(args): - return run_legacy_script('update.php', '--index', nominatim_env=args) + indexer = Indexer(args.config.get_libpq_dsn(), + args.threads or _num_system_cpus() or 1) + + if not args.no_boundaries: + indexer.index_boundaries(args.minrank, args.maxrank) + if not args.boundaries_only: + indexer.index_by_rank(args.minrank, args.maxrank) + + if not args.no_boundaries and not args.boundaries_only: + indexer.update_status_table() + + return 0 class UpdateRefresh: """\ - Recompute auxillary data used by the indexing process. + Recompute auxiliary data used by the indexing process. These functions must not be run in parallel with other update commands. """ @@ -306,12 +350,12 @@ class UpdateRefresh: help='Compute frequency of full-word search terms') group.add_argument('--address-levels', action='store_true', help='Reimport address level configuration') - group.add_argument('--importance', action='store_true', - help='Recompute place importances (expensive!)') group.add_argument('--functions', action='store_true', help='Update the PL/pgSQL functions in the database') group.add_argument('--wiki-data', action='store_true', help='Update Wikipedia/data importance numbers.') + group.add_argument('--importance', action='store_true', + help='Recompute place importances (expensive!)') group.add_argument('--website', action='store_true', help='Refresh the directory that serves the scripts for the web API') group = parser.add_argument_group('Arguments for function refresh') @@ -331,9 +375,6 @@ class UpdateRefresh: if args.address_levels: run_legacy_script('update.php', '--update-address-levels', nominatim_env=args, throw_on_fail=True) - if args.importance: - run_legacy_script('update.php', '--recompute-importance', - nominatim_env=args, throw_on_fail=True) if args.functions: params = ['setup.php', '--create-functions', '--create-partition-functions'] if args.diffs: @@ -344,6 +385,10 @@ class UpdateRefresh: if args.wiki_data: run_legacy_script('setup.php', '--import-wikipedia-articles', nominatim_env=args, throw_on_fail=True) + # Attention: importance MUST come after wiki data import. + if args.importance: + run_legacy_script('update.php', '--recompute-importance', + nominatim_env=args, throw_on_fail=True) if args.website: run_legacy_script('setup.php', '--setup-website', nominatim_env=args, throw_on_fail=True) @@ -435,11 +480,11 @@ class QueryExport: if args.restrict_to_country: params.extend(('--restrict-to-country', args.restrict_to_country)) if args.restrict_to_osm_node: - params.exted(('--restrict-to-osm-node', args.restrict_to_osm_node)) + params.extend(('--restrict-to-osm-node', args.restrict_to_osm_node)) if args.restrict_to_osm_way: - params.exted(('--restrict-to-osm-way', args.restrict_to_osm_way)) + params.extend(('--restrict-to-osm-way', args.restrict_to_osm_way)) if args.restrict_to_osm_relation: - params.exted(('--restrict-to-osm-relation', args.restrict_to_osm_relation)) + params.extend(('--restrict-to-osm-relation', args.restrict_to_osm_relation)) return run_legacy_script(*params, nominatim_env=args) @@ -451,7 +496,8 @@ class QueryTodo: def add_args(parser): pass - def run(args): + @staticmethod + def run(args): # pylint: disable=W0613 print("TODO: searching") @@ -482,4 +528,4 @@ def nominatim(**kwargs): parser.add_subcommand('details', QueryTodo) parser.add_subcommand('status', QueryTodo) - parser.run(**kwargs) + return parser.run(**kwargs)