nominatim/cli.py

   1 """
   2 Command-line interface to the Nominatim functions for import, update,
   3 database administration and querying.
   4 """
   5 import datetime as dt
   6 import os
   7 import sys
   8 import time
   9 import argparse
  10 import logging
  11 from pathlib import Path
  12
  13 from .config import Configuration
  14 from .tools.exec_utils import run_legacy_script, run_api_script, run_php_server
  15 from .db.connection import connect
  16 from .db import status
  17 from .errors import UsageError
  18
  19 LOG = logging.getLogger()
  20
  21 def _num_system_cpus():
  22     try:
  23         cpus = len(os.sched_getaffinity(0))
  24     except NotImplementedError:
  25         cpus = None
  26
  27     return cpus or os.cpu_count()
  28
  29
  30 class CommandlineParser:
  31     """ Wraps some of the common functions for parsing the command line
  32         and setting up subcommands.
  33     """
  34     def __init__(self, prog, description):
  35         self.parser = argparse.ArgumentParser(
  36             prog=prog,
  37             description=description,
  38             formatter_class=argparse.RawDescriptionHelpFormatter)
  39
  40         self.subs = self.parser.add_subparsers(title='available commands',
  41                                                dest='subcommand')
  42
  43         # Arguments added to every sub-command
  44         self.default_args = argparse.ArgumentParser(add_help=False)
  45         group = self.default_args.add_argument_group('Default arguments')
  46         group.add_argument('-h', '--help', action='help',
  47                            help='Show this help message and exit')
  48         group.add_argument('-q', '--quiet', action='store_const', const=0,
  49                            dest='verbose', default=1,
  50                            help='Print only error messages')
  51         group.add_argument('-v', '--verbose', action='count', default=1,
  52                            help='Increase verboseness of output')
  53         group.add_argument('--project-dir', metavar='DIR', default='.',
  54                            help='Base directory of the Nominatim installation (default:.)')
  55         group.add_argument('-j', '--threads', metavar='NUM', type=int,
  56                            help='Number of parallel threads to use')
  57
  58
  59     def add_subcommand(self, name, cmd):
  60         """ Add a subcommand to the parser. The subcommand must be a class
  61             with a function add_args() that adds the parameters for the
  62             subcommand and a run() function that executes the command.
  63         """
  64         parser = self.subs.add_parser(name, parents=[self.default_args],
  65                                       help=cmd.__doc__.split('\n', 1)[0],
  66                                       description=cmd.__doc__,
  67                                       formatter_class=argparse.RawDescriptionHelpFormatter,
  68                                       add_help=False)
  69         parser.set_defaults(command=cmd)
  70         cmd.add_args(parser)
  71
  72     def run(self, **kwargs):
  73         """ Parse the command line arguments of the program and execute the
  74             appropriate subcommand.
  75         """
  76         args = self.parser.parse_args(args=kwargs.get('cli_args'))
  77
  78         if args.subcommand is None:
  79             self.parser.print_help()
  80             return 1
  81
  82         for arg in ('module_dir', 'osm2pgsql_path', 'phplib_dir', 'data_dir', 'phpcgi_path'):
  83             setattr(args, arg, Path(kwargs[arg]))
  84         args.project_dir = Path(args.project_dir).resolve()
  85
  86         logging.basicConfig(stream=sys.stderr,
  87                             format='%(asctime)s: %(message)s',
  88                             datefmt='%Y-%m-%d %H:%M:%S',
  89                             level=max(4 - args.verbose, 1) * 10)
  90
  91         args.config = Configuration(args.project_dir, args.data_dir / 'settings')
  92
  93         log = logging.getLogger()
  94         log.warning('Using project directory: %s', str(args.project_dir))
  95
  96         try:
  97             return args.command.run(args)
  98         except UsageError as exception:
  99             if log.isEnabledFor(logging.DEBUG):
 100                 raise # use Python's exception printing
 101             log.fatal('FATAL: %s', exception)
 102
 103         # If we get here, then execution has failed in some way.
 104         return 1
 105
 106
 107 def _osm2pgsql_options_from_args(args, default_cache, default_threads):
 108     """ Set up the stanadrd osm2pgsql from the command line arguments.
 109     """
 110     return dict(osm2pgsql=args.osm2pgsql_path,
 111                 osm2pgsql_cache=args.osm2pgsql_cache or default_cache,
 112                 osm2pgsql_style=args.config.get_import_style_file(),
 113                 threads=args.threads or default_threads,
 114                 dsn=args.config.get_libpq_dsn(),
 115                 flatnode_file=args.config.FLATNODE_FILE)
 116
 117 ##### Subcommand classes
 118 #
 119 # Each class needs to implement two functions: add_args() adds the CLI parameters
 120 # for the subfunction, run() executes the subcommand.
 121 #
 122 # The class documentation doubles as the help text for the command. The
 123 # first line is also used in the summary when calling the program without
 124 # a subcommand.
 125 #
 126 # No need to document the functions each time.
 127 # pylint: disable=C0111
 128 # Using non-top-level imports to make pyosmium optional for replication only.
 129 # pylint: disable=E0012,C0415
 130
 131
 132 class SetupAll:
 133     """\
 134     Create a new Nominatim database from an OSM file.
 135     """
 136
 137     @staticmethod
 138     def add_args(parser):
 139         group_name = parser.add_argument_group('Required arguments')
 140         group = group_name.add_mutually_exclusive_group(required=True)
 141         group.add_argument('--osm-file',
 142                            help='OSM file to be imported.')
 143         group.add_argument('--continue', dest='continue_at',
 144                            choices=['load-data', 'indexing', 'db-postprocess'],
 145                            help='Continue an import that was interrupted')
 146         group = parser.add_argument_group('Optional arguments')
 147         group.add_argument('--osm2pgsql-cache', metavar='SIZE', type=int,
 148                            help='Size of cache to be used by osm2pgsql (in MB)')
 149         group.add_argument('--reverse-only', action='store_true',
 150                            help='Do not create tables and indexes for searching')
 151         group.add_argument('--enable-debug-statements', action='store_true',
 152                            help='Include debug warning statements in SQL code')
 153         group.add_argument('--no-partitions', action='store_true',
 154                            help="""Do not partition search indices
 155                                    (speeds up import of single country extracts)""")
 156         group.add_argument('--no-updates', action='store_true',
 157                            help="""Do not keep tables that are only needed for
 158                                    updating the database later""")
 159         group = parser.add_argument_group('Expert options')
 160         group.add_argument('--ignore-errors', action='store_true',
 161                            help='Continue import even when errors in SQL are present')
 162         group.add_argument('--index-noanalyse', action='store_true',
 163                            help='Do not perform analyse operations during index')
 164
 165
 166     @staticmethod
 167     def run(args):
 168         params = ['setup.php']
 169         if args.osm_file:
 170             params.extend(('--all', '--osm-file', args.osm_file))
 171         else:
 172             if args.continue_at == 'load-data':
 173                 params.append('--load-data')
 174             if args.continue_at in ('load-data', 'indexing'):
 175                 params.append('--index')
 176             params.extend(('--create-search-indices', '--create-country-names',
 177                            '--setup-website'))
 178         if args.osm2pgsql_cache:
 179             params.extend(('--osm2pgsql-cache', args.osm2pgsql_cache))
 180         if args.reverse_only:
 181             params.append('--reverse-only')
 182         if args.enable_debug_statements:
 183             params.append('--enable-debug-statements')
 184         if args.no_partitions:
 185             params.append('--no-partitions')
 186         if args.no_updates:
 187             params.append('--drop')
 188         if args.ignore_errors:
 189             params.append('--ignore-errors')
 190         if args.index_noanalyse:
 191             params.append('--index-noanalyse')
 192
 193         return run_legacy_script(*params, nominatim_env=args)
 194
 195
 196 class SetupFreeze:
 197     """\
 198     Make database read-only.
 199
 200     About half of data in the Nominatim database is kept only to be able to
 201     keep the data up-to-date with new changes made in OpenStreetMap. This
 202     command drops all this data and only keeps the part needed for geocoding
 203     itself.
 204
 205     This command has the same effect as the `--no-updates` option for imports.
 206     """
 207
 208     @staticmethod
 209     def add_args(parser):
 210         pass # No options
 211
 212     @staticmethod
 213     def run(args):
 214         return run_legacy_script('setup.php', '--drop', nominatim_env=args)
 215
 216
 217 class SetupSpecialPhrases:
 218     """\
 219     Maintain special phrases.
 220     """
 221
 222     @staticmethod
 223     def add_args(parser):
 224         group = parser.add_argument_group('Input arguments')
 225         group.add_argument('--from-wiki', action='store_true',
 226                            help='Pull special phrases from the OSM wiki.')
 227         group = parser.add_argument_group('Output arguments')
 228         group.add_argument('-o', '--output', default='-',
 229                            help="""File to write the preprocessed phrases to.
 230                                    If omitted, it will be written to stdout.""")
 231
 232     @staticmethod
 233     def run(args):
 234         if args.output != '-':
 235             raise NotImplementedError('Only output to stdout is currently implemented.')
 236         return run_legacy_script('specialphrases.php', '--wiki-import', nominatim_env=args)
 237
 238
 239 class UpdateReplication:
 240     """\
 241     Update the database using an online replication service.
 242     """
 243
 244     @staticmethod
 245     def add_args(parser):
 246         group = parser.add_argument_group('Arguments for initialisation')
 247         group.add_argument('--init', action='store_true',
 248                            help='Initialise the update process')
 249         group.add_argument('--no-update-functions', dest='update_functions',
 250                            action='store_false',
 251                            help="""Do not update the trigger function to
 252                                    support differential updates.""")
 253         group = parser.add_argument_group('Arguments for updates')
 254         group.add_argument('--check-for-updates', action='store_true',
 255                            help='Check if new updates are available and exit')
 256         group.add_argument('--once', action='store_true',
 257                            help="""Download and apply updates only once. When
 258                                    not set, updates are continuously applied""")
 259         group.add_argument('--no-index', action='store_false', dest='do_index',
 260                            help="""Do not index the new data. Only applicable
 261                                    together with --once""")
 262         group.add_argument('--osm2pgsql-cache', metavar='SIZE', type=int,
 263                            help='Size of cache to be used by osm2pgsql (in MB)')
 264
 265     @staticmethod
 266     def _init_replication(args):
 267         from .tools import replication, refresh
 268
 269         LOG.warning("Initialising replication updates")
 270         conn = connect(args.config.get_libpq_dsn())
 271         replication.init_replication(conn, base_url=args.config.REPLICATION_URL)
 272         if args.update_functions:
 273             LOG.warning("Create functions")
 274             refresh.create_functions(conn, args.config, args.data_dir,
 275                                      True, False)
 276         conn.close()
 277         return 0
 278
 279
 280     @staticmethod
 281     def _check_for_updates(args):
 282         from .tools import replication
 283
 284         conn = connect(args.config.get_libpq_dsn())
 285         ret = replication.check_for_updates(conn, base_url=args.config.REPLICATION_URL)
 286         conn.close()
 287         return ret
 288
 289     @staticmethod
 290     def _report_update(batchdate, start_import, start_index):
 291         def round_time(delta):
 292             return dt.timedelta(seconds=int(delta.total_seconds()))
 293
 294         end = dt.datetime.now(dt.timezone.utc)
 295         LOG.warning("Update completed. Import: %s. %sTotal: %s. Remaining backlog: %s.",
 296                     round_time((start_index or end) - start_import),
 297                     "Indexing: {} ".format(round_time(end - start_index))
 298                     if start_index else '',
 299                     round_time(end - start_import),
 300                     round_time(end - batchdate))
 301
 302     @staticmethod
 303     def _update(args):
 304         from .tools import replication
 305         from .indexer.indexer import Indexer
 306
 307         params = _osm2pgsql_options_from_args(args, 2000, 1)
 308         params.update(base_url=args.config.REPLICATION_URL,
 309                       update_interval=args.config.get_int('REPLICATION_UPDATE_INTERVAL'),
 310                       import_file=args.project_dir / 'osmosischange.osc',
 311                       max_diff_size=args.config.get_int('REPLICATION_MAX_DIFF'),
 312                       indexed_only=not args.once)
 313
 314         # Sanity check to not overwhelm the Geofabrik servers.
 315         if 'download.geofabrik.de'in params['base_url']\
 316            and params['update_interval'] < 86400:
 317             LOG.fatal("Update interval too low for download.geofabrik.de.\n"
 318                       "Please check install documentation "
 319                       "(https://nominatim.org/release-docs/latest/admin/Import-and-Update#"
 320                       "setting-up-the-update-process).")
 321             raise UsageError("Invalid replication update interval setting.")
 322
 323         if not args.once:
 324             if not args.do_index:
 325                 LOG.fatal("Indexing cannot be disabled when running updates continuously.")
 326                 raise UsageError("Bad argument '--no-index'.")
 327             recheck_interval = args.config.get_int('REPLICATION_RECHECK_INTERVAL')
 328
 329         while True:
 330             conn = connect(args.config.get_libpq_dsn())
 331             start = dt.datetime.now(dt.timezone.utc)
 332             state = replication.update(conn, params)
 333             status.log_status(conn, start, 'import')
 334             batchdate, _, _ = status.get_status(conn)
 335             conn.close()
 336
 337             if state is not replication.UpdateState.NO_CHANGES and args.do_index:
 338                 index_start = dt.datetime.now(dt.timezone.utc)
 339                 indexer = Indexer(args.config.get_libpq_dsn(),
 340                                   args.threads or 1)
 341                 indexer.index_boundaries(0, 30)
 342                 indexer.index_by_rank(0, 30)
 343
 344                 conn = connect(args.config.get_libpq_dsn())
 345                 status.set_indexed(conn, True)
 346                 status.log_status(conn, index_start, 'index')
 347                 conn.close()
 348             else:
 349                 index_start = None
 350
 351             if LOG.isEnabledFor(logging.WARNING):
 352                 UpdateReplication._report_update(batchdate, start, index_start)
 353
 354             if args.once:
 355                 break
 356
 357             if state is replication.UpdateState.NO_CHANGES:
 358                 LOG.warning("No new changes. Sleeping for %d sec.", recheck_interval)
 359                 time.sleep(recheck_interval)
 360
 361         return state.value
 362
 363     @staticmethod
 364     def run(args):
 365         try:
 366             import osmium # pylint: disable=W0611
 367         except ModuleNotFoundError:
 368             LOG.fatal("pyosmium not installed. Replication functions not available.\n"
 369                       "To install pyosmium via pip: pip3 install osmium")
 370             return 1
 371
 372         if args.init:
 373             return UpdateReplication._init_replication(args)
 374
 375         if args.check_for_updates:
 376             return UpdateReplication._check_for_updates(args)
 377
 378         return UpdateReplication._update(args)
 379
 380 class UpdateAddData:
 381     """\
 382     Add additional data from a file or an online source.
 383
 384     Data is only imported, not indexed. You need to call `nominatim-update index`
 385     to complete the process.
 386     """
 387
 388     @staticmethod
 389     def add_args(parser):
 390         group_name = parser.add_argument_group('Source')
 391         group = group_name.add_mutually_exclusive_group(required=True)
 392         group.add_argument('--file', metavar='FILE',
 393                            help='Import data from an OSM file')
 394         group.add_argument('--diff', metavar='FILE',
 395                            help='Import data from an OSM diff file')
 396         group.add_argument('--node', metavar='ID', type=int,
 397                            help='Import a single node from the API')
 398         group.add_argument('--way', metavar='ID', type=int,
 399                            help='Import a single way from the API')
 400         group.add_argument('--relation', metavar='ID', type=int,
 401                            help='Import a single relation from the API')
 402         group.add_argument('--tiger-data', metavar='DIR',
 403                            help='Add housenumbers from the US TIGER census database.')
 404         group = parser.add_argument_group('Extra arguments')
 405         group.add_argument('--use-main-api', action='store_true',
 406                            help='Use OSM API instead of Overpass to download objects')
 407
 408     @staticmethod
 409     def run(args):
 410         if args.tiger_data:
 411             os.environ['NOMINATIM_TIGER_DATA_PATH'] = args.tiger_data
 412             return run_legacy_script('setup.php', '--import-tiger-data', nominatim_env=args)
 413
 414         params = ['update.php']
 415         if args.file:
 416             params.extend(('--import-file', args.file))
 417         elif args.diff:
 418             params.extend(('--import-diff', args.diff))
 419         elif args.node:
 420             params.extend(('--import-node', args.node))
 421         elif args.way:
 422             params.extend(('--import-way', args.way))
 423         elif args.relation:
 424             params.extend(('--import-relation', args.relation))
 425         if args.use_main_api:
 426             params.append('--use-main-api')
 427         return run_legacy_script(*params, nominatim_env=args)
 428
 429
 430 class UpdateIndex:
 431     """\
 432     Reindex all new and modified data.
 433     """
 434
 435     @staticmethod
 436     def add_args(parser):
 437         group = parser.add_argument_group('Filter arguments')
 438         group.add_argument('--boundaries-only', action='store_true',
 439                            help="""Index only administrative boundaries.""")
 440         group.add_argument('--no-boundaries', action='store_true',
 441                            help="""Index everything except administrative boundaries.""")
 442         group.add_argument('--minrank', '-r', type=int, metavar='RANK', default=0,
 443                            help='Minimum/starting rank')
 444         group.add_argument('--maxrank', '-R', type=int, metavar='RANK', default=30,
 445                            help='Maximum/finishing rank')
 446
 447     @staticmethod
 448     def run(args):
 449         from .indexer.indexer import Indexer
 450
 451         indexer = Indexer(args.config.get_libpq_dsn(),
 452                           args.threads or _num_system_cpus() or 1)
 453
 454         if not args.no_boundaries:
 455             indexer.index_boundaries(args.minrank, args.maxrank)
 456         if not args.boundaries_only:
 457             indexer.index_by_rank(args.minrank, args.maxrank)
 458
 459         if not args.no_boundaries and not args.boundaries_only \
 460            and args.minrank == 0 and args.maxrank == 30:
 461             conn = connect(args.config.get_libpq_dsn())
 462             status.set_indexed(conn, True)
 463             conn.close()
 464
 465         return 0
 466
 467
 468 class UpdateRefresh:
 469     """\
 470     Recompute auxiliary data used by the indexing process.
 471
 472     These functions must not be run in parallel with other update commands.
 473     """
 474
 475     @staticmethod
 476     def add_args(parser):
 477         group = parser.add_argument_group('Data arguments')
 478         group.add_argument('--postcodes', action='store_true',
 479                            help='Update postcode centroid table')
 480         group.add_argument('--word-counts', action='store_true',
 481                            help='Compute frequency of full-word search terms')
 482         group.add_argument('--address-levels', action='store_true',
 483                            help='Reimport address level configuration')
 484         group.add_argument('--functions', action='store_true',
 485                            help='Update the PL/pgSQL functions in the database')
 486         group.add_argument('--wiki-data', action='store_true',
 487                            help='Update Wikipedia/data importance numbers.')
 488         group.add_argument('--importance', action='store_true',
 489                            help='Recompute place importances (expensive!)')
 490         group.add_argument('--website', action='store_true',
 491                            help='Refresh the directory that serves the scripts for the web API')
 492         group = parser.add_argument_group('Arguments for function refresh')
 493         group.add_argument('--no-diff-updates', action='store_false', dest='diffs',
 494                            help='Do not enable code for propagating updates')
 495         group.add_argument('--enable-debug-statements', action='store_true',
 496                            help='Enable debug warning statements in functions')
 497
 498     @staticmethod
 499     def run(args):
 500         from .tools import refresh
 501
 502         if args.postcodes:
 503             LOG.warning("Update postcodes centroid")
 504             conn = connect(args.config.get_libpq_dsn())
 505             refresh.update_postcodes(conn, args.data_dir)
 506             conn.close()
 507
 508         if args.word_counts:
 509             LOG.warning('Recompute frequency of full-word search terms')
 510             conn = connect(args.config.get_libpq_dsn())
 511             refresh.recompute_word_counts(conn, args.data_dir)
 512             conn.close()
 513
 514         if args.address_levels:
 515             cfg = Path(args.config.ADDRESS_LEVEL_CONFIG)
 516             LOG.warning('Updating address levels from %s', cfg)
 517             conn = connect(args.config.get_libpq_dsn())
 518             refresh.load_address_levels_from_file(conn, cfg)
 519             conn.close()
 520
 521         if args.functions:
 522             LOG.warning('Create functions')
 523             conn = connect(args.config.get_libpq_dsn())
 524             refresh.create_functions(conn, args.config, args.data_dir,
 525                                      args.diffs, args.enable_debug_statements)
 526             conn.close()
 527
 528         if args.wiki_data:
 529             run_legacy_script('setup.php', '--import-wikipedia-articles',
 530                               nominatim_env=args, throw_on_fail=True)
 531         # Attention: importance MUST come after wiki data import.
 532         if args.importance:
 533             run_legacy_script('update.php', '--recompute-importance',
 534                               nominatim_env=args, throw_on_fail=True)
 535         if args.website:
 536             run_legacy_script('setup.php', '--setup-website',
 537                               nominatim_env=args, throw_on_fail=True)
 538
 539         return 0
 540
 541
 542 class AdminCheckDatabase:
 543     """\
 544     Check that the database is complete and operational.
 545     """
 546
 547     @staticmethod
 548     def add_args(parser):
 549         pass # No options
 550
 551     @staticmethod
 552     def run(args):
 553         return run_legacy_script('check_import_finished.php', nominatim_env=args)
 554
 555
 556 class AdminWarm:
 557     """\
 558     Warm database caches for search and reverse queries.
 559     """
 560
 561     @staticmethod
 562     def add_args(parser):
 563         group = parser.add_argument_group('Target arguments')
 564         group.add_argument('--search-only', action='store_const', dest='target',
 565                            const='search',
 566                            help="Only pre-warm tables for search queries")
 567         group.add_argument('--reverse-only', action='store_const', dest='target',
 568                            const='reverse',
 569                            help="Only pre-warm tables for reverse queries")
 570
 571     @staticmethod
 572     def run(args):
 573         params = ['warm.php']
 574         if args.target == 'reverse':
 575             params.append('--reverse-only')
 576         if args.target == 'search':
 577             params.append('--search-only')
 578         return run_legacy_script(*params, nominatim_env=args)
 579
 580
 581 class QueryExport:
 582     """\
 583     Export addresses as CSV file from the database.
 584     """
 585
 586     @staticmethod
 587     def add_args(parser):
 588         group = parser.add_argument_group('Output arguments')
 589         group.add_argument('--output-type', default='street',
 590                            choices=('continent', 'country', 'state', 'county',
 591                                     'city', 'suburb', 'street', 'path'),
 592                            help='Type of places to output (default: street)')
 593         group.add_argument('--output-format',
 594                            default='street;suburb;city;county;state;country',
 595                            help="""Semicolon-separated list of address types
 596                                    (see --output-type). Multiple ranks can be
 597                                    merged into one column by simply using a
 598                                    comma-separated list.""")
 599         group.add_argument('--output-all-postcodes', action='store_true',
 600                            help="""List all postcodes for address instead of
 601                                    just the most likely one""")
 602         group.add_argument('--language',
 603                            help="""Preferred language for output
 604                                    (use local name, if omitted)""")
 605         group = parser.add_argument_group('Filter arguments')
 606         group.add_argument('--restrict-to-country', metavar='COUNTRY_CODE',
 607                            help='Export only objects within country')
 608         group.add_argument('--restrict-to-osm-node', metavar='ID', type=int,
 609                            help='Export only children of this OSM node')
 610         group.add_argument('--restrict-to-osm-way', metavar='ID', type=int,
 611                            help='Export only children of this OSM way')
 612         group.add_argument('--restrict-to-osm-relation', metavar='ID', type=int,
 613                            help='Export only children of this OSM relation')
 614
 615
 616     @staticmethod
 617     def run(args):
 618         params = ['export.php',
 619                   '--output-type', args.output_type,
 620                   '--output-format', args.output_format]
 621         if args.output_all_postcodes:
 622             params.append('--output-all-postcodes')
 623         if args.language:
 624             params.extend(('--language', args.language))
 625         if args.restrict_to_country:
 626             params.extend(('--restrict-to-country', args.restrict_to_country))
 627         if args.restrict_to_osm_node:
 628             params.extend(('--restrict-to-osm-node', args.restrict_to_osm_node))
 629         if args.restrict_to_osm_way:
 630             params.extend(('--restrict-to-osm-way', args.restrict_to_osm_way))
 631         if args.restrict_to_osm_relation:
 632             params.extend(('--restrict-to-osm-relation', args.restrict_to_osm_relation))
 633
 634         return run_legacy_script(*params, nominatim_env=args)
 635
 636
 637 class AdminServe:
 638     """\
 639     Start a simple web server for serving the API.
 640
 641     This command starts the built-in PHP webserver to serve the website
 642     from the current project directory. This webserver is only suitable
 643     for testing and develop. Do not use it in production setups!
 644
 645     By the default, the webserver can be accessed at: http://127.0.0.1:8088
 646     """
 647
 648     @staticmethod
 649     def add_args(parser):
 650         group = parser.add_argument_group('Server arguments')
 651         group.add_argument('--server', default='127.0.0.1:8088',
 652                            help='The address the server will listen to.')
 653
 654     @staticmethod
 655     def run(args):
 656         run_php_server(args.server, args.project_dir / 'website')
 657
 658 STRUCTURED_QUERY = (
 659     ('street', 'housenumber and street'),
 660     ('city', 'city, town or village'),
 661     ('county', 'county'),
 662     ('state', 'state'),
 663     ('country', 'country'),
 664     ('postalcode', 'postcode')
 665 )
 666
 667 EXTRADATA_PARAMS = (
 668     ('addressdetails', 'Include a breakdown of the address into elements.'),
 669     ('extratags', """Include additional information if available
 670                      (e.g. wikipedia link, opening hours)."""),
 671     ('namedetails', 'Include a list of alternative names.')
 672 )
 673
 674 DETAILS_SWITCHES = (
 675     ('addressdetails', 'Include a breakdown of the address into elements.'),
 676     ('keywords', 'Include a list of name keywords and address keywords.'),
 677     ('linkedplaces', 'Include a details of places that are linked with this one.'),
 678     ('hierarchy', 'Include details of places lower in the address hierarchy.'),
 679     ('group_hierarchy', 'Group the places by type.'),
 680     ('polygon_geojson', 'Include geometry of result.')
 681 )
 682
 683 def _add_api_output_arguments(parser):
 684     group = parser.add_argument_group('Output arguments')
 685     group.add_argument('--format', default='jsonv2',
 686                        choices=['xml', 'json', 'jsonv2', 'geojson', 'geocodejson'],
 687                        help='Format of result')
 688     for name, desc in EXTRADATA_PARAMS:
 689         group.add_argument('--' + name, action='store_true', help=desc)
 690
 691     group.add_argument('--lang', '--accept-language', metavar='LANGS',
 692                        help='Preferred language order for presenting search results')
 693     group.add_argument('--polygon-output',
 694                        choices=['geojson', 'kml', 'svg', 'text'],
 695                        help='Output geometry of results as a GeoJSON, KML, SVG or WKT.')
 696     group.add_argument('--polygon-threshold', type=float, metavar='TOLERANCE',
 697                        help="""Simplify output geometry.
 698                                Parameter is difference tolerance in degrees.""")
 699
 700
 701 class APISearch:
 702     """\
 703     Execute API search query.
 704     """
 705
 706     @staticmethod
 707     def add_args(parser):
 708         group = parser.add_argument_group('Query arguments')
 709         group.add_argument('--query',
 710                            help='Free-form query string')
 711         for name, desc in STRUCTURED_QUERY:
 712             group.add_argument('--' + name, help='Structured query: ' + desc)
 713
 714         _add_api_output_arguments(parser)
 715
 716         group = parser.add_argument_group('Result limitation')
 717         group.add_argument('--countrycodes', metavar='CC,..',
 718                            help='Limit search results to one or more countries.')
 719         group.add_argument('--exclude_place_ids', metavar='ID,..',
 720                            help='List of search object to be excluded')
 721         group.add_argument('--limit', type=int,
 722                            help='Limit the number of returned results')
 723         group.add_argument('--viewbox', metavar='X1,Y1,X2,Y2',
 724                            help='Preferred area to find search results')
 725         group.add_argument('--bounded', action='store_true',
 726                            help='Strictly restrict results to viewbox area')
 727
 728         group = parser.add_argument_group('Other arguments')
 729         group.add_argument('--no-dedupe', action='store_false', dest='dedupe',
 730                            help='Do not remove duplicates from the result list')
 731
 732
 733     @staticmethod
 734     def run(args):
 735         if args.query:
 736             params = dict(q=args.query)
 737         else:
 738             params = {k : getattr(args, k) for k, _ in STRUCTURED_QUERY if getattr(args, k)}
 739
 740         for param, _ in EXTRADATA_PARAMS:
 741             if getattr(args, param):
 742                 params[param] = '1'
 743         for param in ('format', 'countrycodes', 'exclude_place_ids', 'limit', 'viewbox'):
 744             if getattr(args, param):
 745                 params[param] = getattr(args, param)
 746         if args.lang:
 747             params['accept-language'] = args.lang
 748         if args.polygon_output:
 749             params['polygon_' + args.polygon_output] = '1'
 750         if args.polygon_threshold:
 751             params['polygon_threshold'] = args.polygon_threshold
 752         if args.bounded:
 753             params['bounded'] = '1'
 754         if not args.dedupe:
 755             params['dedupe'] = '0'
 756
 757         return run_api_script('search', args.project_dir,
 758                               phpcgi_bin=args.phpcgi_path, params=params)
 759
 760 class APIReverse:
 761     """\
 762     Execute API reverse query.
 763     """
 764
 765     @staticmethod
 766     def add_args(parser):
 767         group = parser.add_argument_group('Query arguments')
 768         group.add_argument('--lat', type=float, required=True,
 769                            help='Latitude of coordinate to look up (in WGS84)')
 770         group.add_argument('--lon', type=float, required=True,
 771                            help='Longitude of coordinate to look up (in WGS84)')
 772         group.add_argument('--zoom', type=int,
 773                            help='Level of detail required for the address')
 774
 775         _add_api_output_arguments(parser)
 776
 777
 778     @staticmethod
 779     def run(args):
 780         params = dict(lat=args.lat, lon=args.lon)
 781         if args.zoom is not None:
 782             params['zoom'] = args.zoom
 783
 784         for param, _ in EXTRADATA_PARAMS:
 785             if getattr(args, param):
 786                 params[param] = '1'
 787         if args.format:
 788             params['format'] = args.format
 789         if args.lang:
 790             params['accept-language'] = args.lang
 791         if args.polygon_output:
 792             params['polygon_' + args.polygon_output] = '1'
 793         if args.polygon_threshold:
 794             params['polygon_threshold'] = args.polygon_threshold
 795
 796         return run_api_script('reverse', args.project_dir,
 797                               phpcgi_bin=args.phpcgi_path, params=params)
 798
 799
 800 class APILookup:
 801     """\
 802     Execute API reverse query.
 803     """
 804
 805     @staticmethod
 806     def add_args(parser):
 807         group = parser.add_argument_group('Query arguments')
 808         group.add_argument('--id', metavar='OSMID',
 809                            action='append', required=True, dest='ids',
 810                            help='OSM id to lookup in format <NRW><id> (may be repeated)')
 811
 812         _add_api_output_arguments(parser)
 813
 814
 815     @staticmethod
 816     def run(args):
 817         params = dict(osm_ids=','.join(args.ids))
 818
 819         for param, _ in EXTRADATA_PARAMS:
 820             if getattr(args, param):
 821                 params[param] = '1'
 822         if args.format:
 823             params['format'] = args.format
 824         if args.lang:
 825             params['accept-language'] = args.lang
 826         if args.polygon_output:
 827             params['polygon_' + args.polygon_output] = '1'
 828         if args.polygon_threshold:
 829             params['polygon_threshold'] = args.polygon_threshold
 830
 831         return run_api_script('lookup', args.project_dir,
 832                               phpcgi_bin=args.phpcgi_path, params=params)
 833
 834
 835 class APIDetails:
 836     """\
 837     Execute API lookup query.
 838     """
 839
 840     @staticmethod
 841     def add_args(parser):
 842         group = parser.add_argument_group('Query arguments')
 843         objs = group.add_mutually_exclusive_group(required=True)
 844         objs.add_argument('--node', '-n', type=int,
 845                           help="Look up the OSM node with the given ID.")
 846         objs.add_argument('--way', '-w', type=int,
 847                           help="Look up the OSM way with the given ID.")
 848         objs.add_argument('--relation', '-r', type=int,
 849                           help="Look up the OSM relation with the given ID.")
 850         objs.add_argument('--place_id', '-p', type=int,
 851                           help='Database internal identifier of the OSM object to look up.')
 852         group.add_argument('--class', dest='object_class',
 853                            help="""Class type to disambiguated multiple entries
 854                                    of the same object.""")
 855
 856         group = parser.add_argument_group('Output arguments')
 857         for name, desc in DETAILS_SWITCHES:
 858             group.add_argument('--' + name, action='store_true', help=desc)
 859         group.add_argument('--lang', '--accept-language', metavar='LANGS',
 860                            help='Preferred language order for presenting search results')
 861
 862     @staticmethod
 863     def run(args):
 864         if args.node:
 865             params = dict(osmtype='N', osmid=args.node)
 866         elif args.way:
 867             params = dict(osmtype='W', osmid=args.node)
 868         elif args.relation:
 869             params = dict(osmtype='R', osmid=args.node)
 870         else:
 871             params = dict(place_id=args.place_id)
 872         if args.object_class:
 873             params['class'] = args.object_class
 874         for name, _ in DETAILS_SWITCHES:
 875             params[name] = '1' if getattr(args, name) else '0'
 876
 877         return run_api_script('details', args.project_dir,
 878                               phpcgi_bin=args.phpcgi_path, params=params)
 879
 880
 881 class APIStatus:
 882     """\
 883     Execute API status query.
 884     """
 885
 886     @staticmethod
 887     def add_args(parser):
 888         group = parser.add_argument_group('API parameters')
 889         group.add_argument('--format', default='text', choices=['text', 'json'],
 890                            help='Format of result')
 891
 892     @staticmethod
 893     def run(args):
 894         return run_api_script('status', args.project_dir,
 895                               phpcgi_bin=args.phpcgi_path,
 896                               params=dict(format=args.format))
 897
 898
 899 def nominatim(**kwargs):
 900     """\
 901     Command-line tools for importing, updating, administrating and
 902     querying the Nominatim database.
 903     """
 904     parser = CommandlineParser('nominatim', nominatim.__doc__)
 905
 906     parser.add_subcommand('import', SetupAll)
 907     parser.add_subcommand('freeze', SetupFreeze)
 908     parser.add_subcommand('replication', UpdateReplication)
 909
 910     parser.add_subcommand('check-database', AdminCheckDatabase)
 911     parser.add_subcommand('warm', AdminWarm)
 912
 913     parser.add_subcommand('special-phrases', SetupSpecialPhrases)
 914
 915     parser.add_subcommand('add-data', UpdateAddData)
 916     parser.add_subcommand('index', UpdateIndex)
 917     parser.add_subcommand('refresh', UpdateRefresh)
 918
 919     parser.add_subcommand('export', QueryExport)
 920     parser.add_subcommand('serve', AdminServe)
 921
 922     if kwargs.get('phpcgi_path'):
 923         parser.add_subcommand('search', APISearch)
 924         parser.add_subcommand('reverse', APIReverse)
 925         parser.add_subcommand('lookup', APILookup)
 926         parser.add_subcommand('details', APIDetails)
 927         parser.add_subcommand('status', APIStatus)
 928     else:
 929         parser.parser.epilog = 'php-cgi not found. Query commands not available.'
 930
 931     return parser.run(**kwargs)