nominatim/cli.py

   1 """
   2 Command-line interface to the Nominatim functions for import, update,
   3 database administration and querying.
   4 """
   5 import datetime as dt
   6 import os
   7 import sys
   8 import time
   9 import argparse
  10 import logging
  11 from pathlib import Path
  12
  13 from .config import Configuration
  14 from .tools.exec_utils import run_legacy_script, run_api_script
  15 from .db.connection import connect
  16 from .db import status
  17 from .errors import UsageError
  18
  19 LOG = logging.getLogger()
  20
  21 def _num_system_cpus():
  22     try:
  23         cpus = len(os.sched_getaffinity(0))
  24     except NotImplementedError:
  25         cpus = None
  26
  27     return cpus or os.cpu_count()
  28
  29
  30 class CommandlineParser:
  31     """ Wraps some of the common functions for parsing the command line
  32         and setting up subcommands.
  33     """
  34     def __init__(self, prog, description):
  35         self.parser = argparse.ArgumentParser(
  36             prog=prog,
  37             description=description,
  38             formatter_class=argparse.RawDescriptionHelpFormatter)
  39
  40         self.subs = self.parser.add_subparsers(title='available commands',
  41                                                dest='subcommand')
  42
  43         # Arguments added to every sub-command
  44         self.default_args = argparse.ArgumentParser(add_help=False)
  45         group = self.default_args.add_argument_group('Default arguments')
  46         group.add_argument('-h', '--help', action='help',
  47                            help='Show this help message and exit')
  48         group.add_argument('-q', '--quiet', action='store_const', const=0,
  49                            dest='verbose', default=1,
  50                            help='Print only error messages')
  51         group.add_argument('-v', '--verbose', action='count', default=1,
  52                            help='Increase verboseness of output')
  53         group.add_argument('--project-dir', metavar='DIR', default='.',
  54                            help='Base directory of the Nominatim installation (default:.)')
  55         group.add_argument('-j', '--threads', metavar='NUM', type=int,
  56                            help='Number of parallel threads to use')
  57
  58
  59     def add_subcommand(self, name, cmd):
  60         """ Add a subcommand to the parser. The subcommand must be a class
  61             with a function add_args() that adds the parameters for the
  62             subcommand and a run() function that executes the command.
  63         """
  64         parser = self.subs.add_parser(name, parents=[self.default_args],
  65                                       help=cmd.__doc__.split('\n', 1)[0],
  66                                       description=cmd.__doc__,
  67                                       formatter_class=argparse.RawDescriptionHelpFormatter,
  68                                       add_help=False)
  69         parser.set_defaults(command=cmd)
  70         cmd.add_args(parser)
  71
  72     def run(self, **kwargs):
  73         """ Parse the command line arguments of the program and execute the
  74             appropriate subcommand.
  75         """
  76         args = self.parser.parse_args(args=kwargs.get('cli_args'))
  77
  78         if args.subcommand is None:
  79             self.parser.print_help()
  80             return 1
  81
  82         for arg in ('module_dir', 'osm2pgsql_path', 'phplib_dir', 'data_dir', 'phpcgi_path'):
  83             setattr(args, arg, Path(kwargs[arg]))
  84         args.project_dir = Path(args.project_dir)
  85
  86         logging.basicConfig(stream=sys.stderr,
  87                             format='%(asctime)s: %(message)s',
  88                             datefmt='%Y-%m-%d %H:%M:%S',
  89                             level=max(4 - args.verbose, 1) * 10)
  90
  91         args.config = Configuration(args.project_dir, args.data_dir / 'settings')
  92
  93         try:
  94             return args.command.run(args)
  95         except UsageError as e:
  96             log = logging.getLogger()
  97             if log.isEnabledFor(logging.DEBUG):
  98                 raise # use Python's exception printing
  99             log.fatal('FATAL: ' + str(e))
 100
 101         # If we get here, then execution has failed in some way.
 102         return 1
 103
 104
 105 def _osm2pgsql_options_from_args(args, default_cache, default_threads):
 106     """ Set up the stanadrd osm2pgsql from the command line arguments.
 107     """
 108     return dict(osm2pgsql=args.osm2pgsql_path,
 109                 osm2pgsql_cache=args.osm2pgsql_cache or default_cache,
 110                 osm2pgsql_style=args.config.get_import_style_file(),
 111                 threads=args.threads or default_threads,
 112                 dsn=args.config.get_libpq_dsn(),
 113                 flatnode_file=args.config.FLATNODE_FILE)
 114
 115 ##### Subcommand classes
 116 #
 117 # Each class needs to implement two functions: add_args() adds the CLI parameters
 118 # for the subfunction, run() executes the subcommand.
 119 #
 120 # The class documentation doubles as the help text for the command. The
 121 # first line is also used in the summary when calling the program without
 122 # a subcommand.
 123 #
 124 # No need to document the functions each time.
 125 # pylint: disable=C0111
 126
 127
 128 class SetupAll:
 129     """\
 130     Create a new Nominatim database from an OSM file.
 131     """
 132
 133     @staticmethod
 134     def add_args(parser):
 135         group_name = parser.add_argument_group('Required arguments')
 136         group = group_name.add_mutually_exclusive_group(required=True)
 137         group.add_argument('--osm-file',
 138                            help='OSM file to be imported.')
 139         group.add_argument('--continue', dest='continue_at',
 140                            choices=['load-data', 'indexing', 'db-postprocess'],
 141                            help='Continue an import that was interrupted')
 142         group = parser.add_argument_group('Optional arguments')
 143         group.add_argument('--osm2pgsql-cache', metavar='SIZE', type=int,
 144                            help='Size of cache to be used by osm2pgsql (in MB)')
 145         group.add_argument('--reverse-only', action='store_true',
 146                            help='Do not create tables and indexes for searching')
 147         group.add_argument('--enable-debug-statements', action='store_true',
 148                            help='Include debug warning statements in SQL code')
 149         group.add_argument('--no-partitions', action='store_true',
 150                            help="""Do not partition search indices
 151                                    (speeds up import of single country extracts)""")
 152         group.add_argument('--no-updates', action='store_true',
 153                            help="""Do not keep tables that are only needed for
 154                                    updating the database later""")
 155         group = parser.add_argument_group('Expert options')
 156         group.add_argument('--ignore-errors', action='store_true',
 157                            help='Continue import even when errors in SQL are present')
 158         group.add_argument('--index-noanalyse', action='store_true',
 159                            help='Do not perform analyse operations during index')
 160
 161
 162     @staticmethod
 163     def run(args):
 164         params = ['setup.php']
 165         if args.osm_file:
 166             params.extend(('--all', '--osm-file', args.osm_file))
 167         else:
 168             if args.continue_at == 'load-data':
 169                 params.append('--load-data')
 170             if args.continue_at in ('load-data', 'indexing'):
 171                 params.append('--index')
 172             params.extend(('--create-search-indices', '--create-country-names',
 173                            '--setup-website'))
 174         if args.osm2pgsql_cache:
 175             params.extend(('--osm2pgsql-cache', args.osm2pgsql_cache))
 176         if args.reverse_only:
 177             params.append('--reverse-only')
 178         if args.enable_debug_statements:
 179             params.append('--enable-debug-statements')
 180         if args.no_partitions:
 181             params.append('--no-partitions')
 182         if args.no_updates:
 183             params.append('--drop')
 184         if args.ignore_errors:
 185             params.append('--ignore-errors')
 186         if args.index_noanalyse:
 187             params.append('--index-noanalyse')
 188
 189         return run_legacy_script(*params, nominatim_env=args)
 190
 191
 192 class SetupFreeze:
 193     """\
 194     Make database read-only.
 195
 196     About half of data in the Nominatim database is kept only to be able to
 197     keep the data up-to-date with new changes made in OpenStreetMap. This
 198     command drops all this data and only keeps the part needed for geocoding
 199     itself.
 200
 201     This command has the same effect as the `--no-updates` option for imports.
 202     """
 203
 204     @staticmethod
 205     def add_args(parser):
 206         pass # No options
 207
 208     @staticmethod
 209     def run(args):
 210         return run_legacy_script('setup.php', '--drop', nominatim_env=args)
 211
 212
 213 class SetupSpecialPhrases:
 214     """\
 215     Maintain special phrases.
 216     """
 217
 218     @staticmethod
 219     def add_args(parser):
 220         group = parser.add_argument_group('Input arguments')
 221         group.add_argument('--from-wiki', action='store_true',
 222                            help='Pull special phrases from the OSM wiki.')
 223         group = parser.add_argument_group('Output arguments')
 224         group.add_argument('-o', '--output', default='-',
 225                            help="""File to write the preprocessed phrases to.
 226                                    If omitted, it will be written to stdout.""")
 227
 228     @staticmethod
 229     def run(args):
 230         if args.output != '-':
 231             raise NotImplementedError('Only output to stdout is currently implemented.')
 232         return run_legacy_script('specialphrases.php', '--wiki-import', nominatim_env=args)
 233
 234
 235 class UpdateReplication:
 236     """\
 237     Update the database using an online replication service.
 238     """
 239
 240     @staticmethod
 241     def add_args(parser):
 242         group = parser.add_argument_group('Arguments for initialisation')
 243         group.add_argument('--init', action='store_true',
 244                            help='Initialise the update process')
 245         group.add_argument('--no-update-functions', dest='update_functions',
 246                            action='store_false',
 247                            help="""Do not update the trigger function to
 248                                    support differential updates.""")
 249         group = parser.add_argument_group('Arguments for updates')
 250         group.add_argument('--check-for-updates', action='store_true',
 251                            help='Check if new updates are available and exit')
 252         group.add_argument('--once', action='store_true',
 253                            help="""Download and apply updates only once. When
 254                                    not set, updates are continuously applied""")
 255         group.add_argument('--no-index', action='store_false', dest='do_index',
 256                            help="""Do not index the new data. Only applicable
 257                                    together with --once""")
 258         group.add_argument('--osm2pgsql-cache', metavar='SIZE', type=int,
 259                            help='Size of cache to be used by osm2pgsql (in MB)')
 260
 261     @staticmethod
 262     def _init_replication(args):
 263         from .tools import replication, refresh
 264
 265         LOG.warning("Initialising replication updates")
 266         conn = connect(args.config.get_libpq_dsn())
 267         replication.init_replication(conn, base_url=args.config.REPLICATION_URL)
 268         if args.update_functions:
 269             LOG.warning("Create functions")
 270             refresh.create_functions(conn, args.config, args.data_dir,
 271                                      True, False)
 272         conn.close()
 273         return 0
 274
 275
 276     @staticmethod
 277     def _check_for_updates(args):
 278         from .tools import replication
 279
 280         conn = connect(args.config.get_libpq_dsn())
 281         ret = replication.check_for_updates(conn, base_url=args.config.REPLICATION_URL)
 282         conn.close()
 283         return ret
 284
 285
 286     @staticmethod
 287     def _update(args):
 288         from .tools import replication
 289         from .indexer.indexer import Indexer
 290
 291         params = _osm2pgsql_options_from_args(args, 2000, 1)
 292         params.update(base_url=args.config.REPLICATION_URL,
 293                       update_interval=args.config.get_int('REPLICATION_UPDATE_INTERVAL'),
 294                       import_file=args.project_dir / 'osmosischange.osc',
 295                       max_diff_size=args.config.get_int('REPLICATION_MAX_DIFF'),
 296                       indexed_only=not args.once)
 297
 298         # Sanity check to not overwhelm the Geofabrik servers.
 299         if 'download.geofabrik.de'in params['base_url']\
 300            and params['update_interval'] < 86400:
 301             LOG.fatal("Update interval too low for download.geofabrik.de.\n"
 302                       "Please check install documentation "
 303                       "(https://nominatim.org/release-docs/latest/admin/Import-and-Update#"
 304                       "setting-up-the-update-process).")
 305             raise UsageError("Invalid replication update interval setting.")
 306
 307         if not args.once:
 308             if not args.do_index:
 309                 LOG.fatal("Indexing cannot be disabled when running updates continuously.")
 310                 raise UsageError("Bad argument '--no-index'.")
 311             recheck_interval = args.config.get_int('REPLICATION_RECHECK_INTERVAL')
 312
 313         while True:
 314             conn = connect(args.config.get_libpq_dsn())
 315             start = dt.datetime.now(dt.timezone.utc)
 316             state = replication.update(conn, params)
 317             status.log_status(conn, start, 'import')
 318             conn.close()
 319
 320             if state is not replication.UpdateState.NO_CHANGES and args.do_index:
 321                 start = dt.datetime.now(dt.timezone.utc)
 322                 indexer = Indexer(args.config.get_libpq_dsn(),
 323                                   args.threads or 1)
 324                 indexer.index_boundaries(0, 30)
 325                 indexer.index_by_rank(0, 30)
 326
 327                 conn = connect(args.config.get_libpq_dsn())
 328                 status.set_indexed(conn, True)
 329                 status.log_status(conn, start, 'index')
 330                 conn.close()
 331
 332             if args.once:
 333                 break
 334
 335             if state is replication.UpdateState.NO_CHANGES:
 336                 LOG.warning("No new changes. Sleeping for %d sec.", recheck_interval)
 337                 time.sleep(recheck_interval)
 338
 339         return state.value
 340
 341     @staticmethod
 342     def run(args):
 343         try:
 344             import osmium # pylint: disable=W0611
 345         except ModuleNotFoundError:
 346             LOG.fatal("pyosmium not installed. Replication functions not available.\n"
 347                       "To install pyosmium via pip: pip3 install osmium")
 348             return 1
 349
 350         if args.init:
 351             return UpdateReplication._init_replication(args)
 352
 353         if args.check_for_updates:
 354             return UpdateReplication._check_for_updates(args)
 355
 356         return UpdateReplication._update(args)
 357
 358 class UpdateAddData:
 359     """\
 360     Add additional data from a file or an online source.
 361
 362     Data is only imported, not indexed. You need to call `nominatim-update index`
 363     to complete the process.
 364     """
 365
 366     @staticmethod
 367     def add_args(parser):
 368         group_name = parser.add_argument_group('Source')
 369         group = group_name.add_mutually_exclusive_group(required=True)
 370         group.add_argument('--file', metavar='FILE',
 371                            help='Import data from an OSM file')
 372         group.add_argument('--diff', metavar='FILE',
 373                            help='Import data from an OSM diff file')
 374         group.add_argument('--node', metavar='ID', type=int,
 375                            help='Import a single node from the API')
 376         group.add_argument('--way', metavar='ID', type=int,
 377                            help='Import a single way from the API')
 378         group.add_argument('--relation', metavar='ID', type=int,
 379                            help='Import a single relation from the API')
 380         group.add_argument('--tiger-data', metavar='DIR',
 381                            help='Add housenumbers from the US TIGER census database.')
 382         group = parser.add_argument_group('Extra arguments')
 383         group.add_argument('--use-main-api', action='store_true',
 384                            help='Use OSM API instead of Overpass to download objects')
 385
 386     @staticmethod
 387     def run(args):
 388         if args.tiger_data:
 389             os.environ['NOMINATIM_TIGER_DATA_PATH'] = args.tiger_data
 390             return run_legacy_script('setup.php', '--import-tiger-data', nominatim_env=args)
 391
 392         params = ['update.php']
 393         if args.file:
 394             params.extend(('--import-file', args.file))
 395         elif args.diff:
 396             params.extend(('--import-diff', args.diff))
 397         elif args.node:
 398             params.extend(('--import-node', args.node))
 399         elif args.way:
 400             params.extend(('--import-way', args.way))
 401         elif args.relation:
 402             params.extend(('--import-relation', args.relation))
 403         if args.use_main_api:
 404             params.append('--use-main-api')
 405         return run_legacy_script(*params, nominatim_env=args)
 406
 407
 408 class UpdateIndex:
 409     """\
 410     Reindex all new and modified data.
 411     """
 412
 413     @staticmethod
 414     def add_args(parser):
 415         group = parser.add_argument_group('Filter arguments')
 416         group.add_argument('--boundaries-only', action='store_true',
 417                            help="""Index only administrative boundaries.""")
 418         group.add_argument('--no-boundaries', action='store_true',
 419                            help="""Index everything except administrative boundaries.""")
 420         group.add_argument('--minrank', '-r', type=int, metavar='RANK', default=0,
 421                            help='Minimum/starting rank')
 422         group.add_argument('--maxrank', '-R', type=int, metavar='RANK', default=30,
 423                            help='Maximum/finishing rank')
 424
 425     @staticmethod
 426     def run(args):
 427         from .indexer.indexer import Indexer
 428
 429         indexer = Indexer(args.config.get_libpq_dsn(),
 430                           args.threads or _num_system_cpus() or 1)
 431
 432         if not args.no_boundaries:
 433             indexer.index_boundaries(args.minrank, args.maxrank)
 434         if not args.boundaries_only:
 435             indexer.index_by_rank(args.minrank, args.maxrank)
 436
 437         if not args.no_boundaries and not args.boundaries_only \
 438            and args.minrank == 0 and args.maxrank == 30:
 439             conn = connect(args.config.get_libpq_dsn())
 440             status.set_indexed(conn, True)
 441             conn.close()
 442
 443         return 0
 444
 445
 446 class UpdateRefresh:
 447     """\
 448     Recompute auxiliary data used by the indexing process.
 449
 450     These functions must not be run in parallel with other update commands.
 451     """
 452
 453     @staticmethod
 454     def add_args(parser):
 455         group = parser.add_argument_group('Data arguments')
 456         group.add_argument('--postcodes', action='store_true',
 457                            help='Update postcode centroid table')
 458         group.add_argument('--word-counts', action='store_true',
 459                            help='Compute frequency of full-word search terms')
 460         group.add_argument('--address-levels', action='store_true',
 461                            help='Reimport address level configuration')
 462         group.add_argument('--functions', action='store_true',
 463                            help='Update the PL/pgSQL functions in the database')
 464         group.add_argument('--wiki-data', action='store_true',
 465                            help='Update Wikipedia/data importance numbers.')
 466         group.add_argument('--importance', action='store_true',
 467                            help='Recompute place importances (expensive!)')
 468         group.add_argument('--website', action='store_true',
 469                            help='Refresh the directory that serves the scripts for the web API')
 470         group = parser.add_argument_group('Arguments for function refresh')
 471         group.add_argument('--no-diff-updates', action='store_false', dest='diffs',
 472                            help='Do not enable code for propagating updates')
 473         group.add_argument('--enable-debug-statements', action='store_true',
 474                            help='Enable debug warning statements in functions')
 475
 476     @staticmethod
 477     def run(args):
 478         from .tools import refresh
 479
 480         if args.postcodes:
 481             LOG.warning("Update postcodes centroid")
 482             conn = connect(args.config.get_libpq_dsn())
 483             refresh.update_postcodes(conn, args.data_dir)
 484             conn.close()
 485
 486         if args.word_counts:
 487             LOG.warning('Recompute frequency of full-word search terms')
 488             conn = connect(args.config.get_libpq_dsn())
 489             refresh.recompute_word_counts(conn, args.data_dir)
 490             conn.close()
 491
 492         if args.address_levels:
 493             cfg = Path(args.config.ADDRESS_LEVEL_CONFIG)
 494             LOG.warning('Updating address levels from %s', cfg)
 495             conn = connect(args.config.get_libpq_dsn())
 496             refresh.load_address_levels_from_file(conn, cfg)
 497             conn.close()
 498
 499         if args.functions:
 500             LOG.warning('Create functions')
 501             conn = connect(args.config.get_libpq_dsn())
 502             refresh.create_functions(conn, args.config, args.data_dir,
 503                                      args.diffs, args.enable_debug_statements)
 504             conn.close()
 505
 506         if args.wiki_data:
 507             run_legacy_script('setup.php', '--import-wikipedia-articles',
 508                               nominatim_env=args, throw_on_fail=True)
 509         # Attention: importance MUST come after wiki data import.
 510         if args.importance:
 511             run_legacy_script('update.php', '--recompute-importance',
 512                               nominatim_env=args, throw_on_fail=True)
 513         if args.website:
 514             run_legacy_script('setup.php', '--setup-website',
 515                               nominatim_env=args, throw_on_fail=True)
 516
 517         return 0
 518
 519
 520 class AdminCheckDatabase:
 521     """\
 522     Check that the database is complete and operational.
 523     """
 524
 525     @staticmethod
 526     def add_args(parser):
 527         pass # No options
 528
 529     @staticmethod
 530     def run(args):
 531         return run_legacy_script('check_import_finished.php', nominatim_env=args)
 532
 533
 534 class AdminWarm:
 535     """\
 536     Warm database caches for search and reverse queries.
 537     """
 538
 539     @staticmethod
 540     def add_args(parser):
 541         group = parser.add_argument_group('Target arguments')
 542         group.add_argument('--search-only', action='store_const', dest='target',
 543                            const='search',
 544                            help="Only pre-warm tables for search queries")
 545         group.add_argument('--reverse-only', action='store_const', dest='target',
 546                            const='reverse',
 547                            help="Only pre-warm tables for reverse queries")
 548
 549     @staticmethod
 550     def run(args):
 551         params = ['warm.php']
 552         if args.target == 'reverse':
 553             params.append('--reverse-only')
 554         if args.target == 'search':
 555             params.append('--search-only')
 556         return run_legacy_script(*params, nominatim_env=args)
 557
 558
 559 class QueryExport:
 560     """\
 561     Export addresses as CSV file from the database.
 562     """
 563
 564     @staticmethod
 565     def add_args(parser):
 566         group = parser.add_argument_group('Output arguments')
 567         group.add_argument('--output-type', default='street',
 568                            choices=('continent', 'country', 'state', 'county',
 569                                     'city', 'suburb', 'street', 'path'),
 570                            help='Type of places to output (default: street)')
 571         group.add_argument('--output-format',
 572                            default='street;suburb;city;county;state;country',
 573                            help="""Semicolon-separated list of address types
 574                                    (see --output-type). Multiple ranks can be
 575                                    merged into one column by simply using a
 576                                    comma-separated list.""")
 577         group.add_argument('--output-all-postcodes', action='store_true',
 578                            help="""List all postcodes for address instead of
 579                                    just the most likely one""")
 580         group.add_argument('--language',
 581                            help="""Preferred language for output
 582                                    (use local name, if omitted)""")
 583         group = parser.add_argument_group('Filter arguments')
 584         group.add_argument('--restrict-to-country', metavar='COUNTRY_CODE',
 585                            help='Export only objects within country')
 586         group.add_argument('--restrict-to-osm-node', metavar='ID', type=int,
 587                            help='Export only children of this OSM node')
 588         group.add_argument('--restrict-to-osm-way', metavar='ID', type=int,
 589                            help='Export only children of this OSM way')
 590         group.add_argument('--restrict-to-osm-relation', metavar='ID', type=int,
 591                            help='Export only children of this OSM relation')
 592
 593
 594     @staticmethod
 595     def run(args):
 596         params = ['export.php',
 597                   '--output-type', args.output_type,
 598                   '--output-format', args.output_format]
 599         if args.output_all_postcodes:
 600             params.append('--output-all-postcodes')
 601         if args.language:
 602             params.extend(('--language', args.language))
 603         if args.restrict_to_country:
 604             params.extend(('--restrict-to-country', args.restrict_to_country))
 605         if args.restrict_to_osm_node:
 606             params.extend(('--restrict-to-osm-node', args.restrict_to_osm_node))
 607         if args.restrict_to_osm_way:
 608             params.extend(('--restrict-to-osm-way', args.restrict_to_osm_way))
 609         if args.restrict_to_osm_relation:
 610             params.extend(('--restrict-to-osm-relation', args.restrict_to_osm_relation))
 611
 612         return run_legacy_script(*params, nominatim_env=args)
 613
 614 STRUCTURED_QUERY = (
 615     ('street', 'housenumber and street'),
 616     ('city', 'city, town or village'),
 617     ('county', 'county'),
 618     ('state', 'state'),
 619     ('country', 'country'),
 620     ('postalcode', 'postcode')
 621 )
 622
 623 EXTRADATA_PARAMS = (
 624     ('addressdetails', 'Include a breakdown of the address into elements.'),
 625     ('extratags', """Include additional information if available
 626                      (e.g. wikipedia link, opening hours)."""),
 627     ('namedetails', 'Include a list of alternative names.')
 628 )
 629
 630 DETAILS_SWITCHES = (
 631     ('addressdetails', 'Include a breakdown of the address into elements.'),
 632     ('keywords', 'Include a list of name keywords and address keywords.'),
 633     ('linkedplaces', 'Include a details of places that are linked with this one.'),
 634     ('hierarchy', 'Include details of places lower in the address hierarchy.'),
 635     ('group_hierarchy', 'Group the places by type.'),
 636     ('polygon_geojson', 'Include geometry of result.')
 637 )
 638
 639 def _add_api_output_arguments(parser):
 640     group = parser.add_argument_group('Output arguments')
 641     group.add_argument('--format', default='jsonv2',
 642                        choices=['xml', 'json', 'jsonv2', 'geojson', 'geocodejson'],
 643                        help='Format of result')
 644     for name, desc in EXTRADATA_PARAMS:
 645         group.add_argument('--' + name, action='store_true', help=desc)
 646
 647     group.add_argument('--lang', '--accept-language', metavar='LANGS',
 648                        help='Preferred language order for presenting search results')
 649     group.add_argument('--polygon-output',
 650                        choices=['geojson', 'kml', 'svg', 'text'],
 651                        help='Output geometry of results as a GeoJSON, KML, SVG or WKT.')
 652     group.add_argument('--polygon-threshold', type=float, metavar='TOLERANCE',
 653                        help="""Simplify output geometry.
 654                                Parameter is difference tolerance in degrees.""")
 655
 656
 657 class APISearch:
 658     """\
 659     Execute API search query.
 660     """
 661
 662     @staticmethod
 663     def add_args(parser):
 664         group = parser.add_argument_group('Query arguments')
 665         group.add_argument('--query',
 666                            help='Free-form query string')
 667         for name, desc in STRUCTURED_QUERY:
 668             group.add_argument('--' + name, help='Structured query: ' + desc)
 669
 670         _add_api_output_arguments(parser)
 671
 672         group = parser.add_argument_group('Result limitation')
 673         group.add_argument('--countrycodes', metavar='CC,..',
 674                            help='Limit search results to one or more countries.')
 675         group.add_argument('--exclude_place_ids', metavar='ID,..',
 676                            help='List of search object to be excluded')
 677         group.add_argument('--limit', type=int,
 678                            help='Limit the number of returned results')
 679         group.add_argument('--viewbox', metavar='X1,Y1,X2,Y2',
 680                            help='Preferred area to find search results')
 681         group.add_argument('--bounded', action='store_true',
 682                            help='Strictly restrict results to viewbox area')
 683
 684         group = parser.add_argument_group('Other arguments')
 685         group.add_argument('--no-dedupe', action='store_false', dest='dedupe',
 686                            help='Do not remove duplicates from the result list')
 687
 688
 689     @staticmethod
 690     def run(args):
 691         if args.query:
 692             params = dict(q=args.query)
 693         else:
 694             params = {k : getattr(args, k) for k, _ in STRUCTURED_QUERY if getattr(args, k)}
 695
 696         for param, _ in EXTRADATA_PARAMS:
 697             if getattr(args, param):
 698                 params[param] = '1'
 699         for param in ('format', 'countrycodes', 'exclude_place_ids', 'limit', 'viewbox'):
 700             if getattr(args, param):
 701                 params[param] = getattr(args, param)
 702         if args.lang:
 703             params['accept-language'] = args.lang
 704         if args.polygon_output:
 705             params['polygon_' + args.polygon_output] = '1'
 706         if args.polygon_threshold:
 707             params['polygon_threshold'] = args.polygon_threshold
 708         if args.bounded:
 709             params['bounded'] = '1'
 710         if not args.dedupe:
 711             params['dedupe'] = '0'
 712
 713         return run_api_script('search', args.project_dir,
 714                               phpcgi_bin=args.phpcgi_path, params=params)
 715
 716 class APIReverse:
 717     """\
 718     Execute API reverse query.
 719     """
 720
 721     @staticmethod
 722     def add_args(parser):
 723         group = parser.add_argument_group('Query arguments')
 724         group.add_argument('--lat', type=float, required=True,
 725                            help='Latitude of coordinate to look up (in WGS84)')
 726         group.add_argument('--lon', type=float, required=True,
 727                            help='Longitude of coordinate to look up (in WGS84)')
 728         group.add_argument('--zoom', type=int,
 729                            help='Level of detail required for the address')
 730
 731         _add_api_output_arguments(parser)
 732
 733
 734     @staticmethod
 735     def run(args):
 736         params = dict(lat=args.lat, lon=args.lon)
 737         if args.zoom is not None:
 738             params['zoom'] = args.zoom
 739
 740         for param, _ in EXTRADATA_PARAMS:
 741             if getattr(args, param):
 742                 params[param] = '1'
 743         if args.format:
 744             params['format'] = args.format
 745         if args.lang:
 746             params['accept-language'] = args.lang
 747         if args.polygon_output:
 748             params['polygon_' + args.polygon_output] = '1'
 749         if args.polygon_threshold:
 750             params['polygon_threshold'] = args.polygon_threshold
 751
 752         return run_api_script('reverse', args.project_dir,
 753                               phpcgi_bin=args.phpcgi_path, params=params)
 754
 755
 756 class APILookup:
 757     """\
 758     Execute API reverse query.
 759     """
 760
 761     @staticmethod
 762     def add_args(parser):
 763         group = parser.add_argument_group('Query arguments')
 764         group.add_argument('--id', metavar='OSMID',
 765                            action='append', required=True, dest='ids',
 766                            help='OSM id to lookup in format <NRW><id> (may be repeated)')
 767
 768         _add_api_output_arguments(parser)
 769
 770
 771     @staticmethod
 772     def run(args):
 773         params = dict(osm_ids=','.join(args.ids))
 774
 775         for param, _ in EXTRADATA_PARAMS:
 776             if getattr(args, param):
 777                 params[param] = '1'
 778         if args.format:
 779             params['format'] = args.format
 780         if args.lang:
 781             params['accept-language'] = args.lang
 782         if args.polygon_output:
 783             params['polygon_' + args.polygon_output] = '1'
 784         if args.polygon_threshold:
 785             params['polygon_threshold'] = args.polygon_threshold
 786
 787         return run_api_script('lookup', args.project_dir,
 788                               phpcgi_bin=args.phpcgi_path, params=params)
 789
 790
 791 class APIDetails:
 792     """\
 793     Execute API lookup query.
 794     """
 795
 796     @staticmethod
 797     def add_args(parser):
 798         group = parser.add_argument_group('Query arguments')
 799         objs = group.add_mutually_exclusive_group(required=True)
 800         objs.add_argument('--node', '-n', type=int,
 801                           help="Look up the OSM node with the given ID.")
 802         objs.add_argument('--way', '-w', type=int,
 803                           help="Look up the OSM way with the given ID.")
 804         objs.add_argument('--relation', '-r', type=int,
 805                           help="Look up the OSM relation with the given ID.")
 806         objs.add_argument('--place_id', '-p', type=int,
 807                           help='Database internal identifier of the OSM object to look up.')
 808         group.add_argument('--class', dest='object_class',
 809                            help="""Class type to disambiguated multiple entries
 810                                    of the same object.""")
 811
 812         group = parser.add_argument_group('Output arguments')
 813         for name, desc in DETAILS_SWITCHES:
 814             group.add_argument('--' + name, action='store_true', help=desc)
 815         group.add_argument('--lang', '--accept-language', metavar='LANGS',
 816                            help='Preferred language order for presenting search results')
 817
 818     @staticmethod
 819     def run(args):
 820         if args.node:
 821             params = dict(osmtype='N', osmid=args.node)
 822         elif args.way:
 823             params = dict(osmtype='W', osmid=args.node)
 824         elif args.relation:
 825             params = dict(osmtype='R', osmid=args.node)
 826         else:
 827             params = dict(place_id=args.place_id)
 828         if args.object_class:
 829             params['class'] = args.object_class
 830         for name, _ in DETAILS_SWITCHES:
 831             params[name] = '1' if getattr(args, name) else '0'
 832
 833         return run_api_script('details', args.project_dir,
 834                               phpcgi_bin=args.phpcgi_path, params=params)
 835
 836
 837 class APIStatus:
 838     """\
 839     Execute API status query.
 840     """
 841
 842     @staticmethod
 843     def add_args(parser):
 844         group = parser.add_argument_group('API parameters')
 845         group.add_argument('--format', default='text', choices=['text', 'json'],
 846                            help='Format of result')
 847
 848     @staticmethod
 849     def run(args):
 850         return run_api_script('status', args.project_dir,
 851                               phpcgi_bin=args.phpcgi_path,
 852                               params=dict(format=args.format))
 853
 854
 855 def nominatim(**kwargs):
 856     """\
 857     Command-line tools for importing, updating, administrating and
 858     querying the Nominatim database.
 859     """
 860     parser = CommandlineParser('nominatim', nominatim.__doc__)
 861
 862     parser.add_subcommand('import', SetupAll)
 863     parser.add_subcommand('freeze', SetupFreeze)
 864     parser.add_subcommand('replication', UpdateReplication)
 865
 866     parser.add_subcommand('check-database', AdminCheckDatabase)
 867     parser.add_subcommand('warm', AdminWarm)
 868
 869     parser.add_subcommand('special-phrases', SetupSpecialPhrases)
 870
 871     parser.add_subcommand('add-data', UpdateAddData)
 872     parser.add_subcommand('index', UpdateIndex)
 873     parser.add_subcommand('refresh', UpdateRefresh)
 874
 875     parser.add_subcommand('export', QueryExport)
 876
 877     if kwargs.get('phpcgi_path'):
 878         parser.add_subcommand('search', APISearch)
 879         parser.add_subcommand('reverse', APIReverse)
 880         parser.add_subcommand('lookup', APILookup)
 881         parser.add_subcommand('details', APIDetails)
 882         parser.add_subcommand('status', APIStatus)
 883     else:
 884         parser.parser.epilog = 'php-cgi not found. Query commands not available.'
 885
 886     return parser.run(**kwargs)