nominatim/cli.py

   1 """
   2 Command-line interface to the Nominatim functions for import, update,
   3 database administration and querying.
   4 """
   5 import datetime as dt
   6 import os
   7 import socket
   8 import sys
   9 import time
  10 import argparse
  11 import logging
  12 from pathlib import Path
  13
  14 from .config import Configuration
  15 from .tools.exec_utils import run_legacy_script, run_api_script, run_php_server
  16 from .db.connection import connect
  17 from .db import status
  18 from .errors import UsageError
  19
  20 LOG = logging.getLogger()
  21
  22 def _num_system_cpus():
  23     try:
  24         cpus = len(os.sched_getaffinity(0))
  25     except NotImplementedError:
  26         cpus = None
  27
  28     return cpus or os.cpu_count()
  29
  30
  31 class CommandlineParser:
  32     """ Wraps some of the common functions for parsing the command line
  33         and setting up subcommands.
  34     """
  35     def __init__(self, prog, description):
  36         self.parser = argparse.ArgumentParser(
  37             prog=prog,
  38             description=description,
  39             formatter_class=argparse.RawDescriptionHelpFormatter)
  40
  41         self.subs = self.parser.add_subparsers(title='available commands',
  42                                                dest='subcommand')
  43
  44         # Arguments added to every sub-command
  45         self.default_args = argparse.ArgumentParser(add_help=False)
  46         group = self.default_args.add_argument_group('Default arguments')
  47         group.add_argument('-h', '--help', action='help',
  48                            help='Show this help message and exit')
  49         group.add_argument('-q', '--quiet', action='store_const', const=0,
  50                            dest='verbose', default=1,
  51                            help='Print only error messages')
  52         group.add_argument('-v', '--verbose', action='count', default=1,
  53                            help='Increase verboseness of output')
  54         group.add_argument('--project-dir', metavar='DIR', default='.',
  55                            help='Base directory of the Nominatim installation (default:.)')
  56         group.add_argument('-j', '--threads', metavar='NUM', type=int,
  57                            help='Number of parallel threads to use')
  58
  59
  60     def add_subcommand(self, name, cmd):
  61         """ Add a subcommand to the parser. The subcommand must be a class
  62             with a function add_args() that adds the parameters for the
  63             subcommand and a run() function that executes the command.
  64         """
  65         parser = self.subs.add_parser(name, parents=[self.default_args],
  66                                       help=cmd.__doc__.split('\n', 1)[0],
  67                                       description=cmd.__doc__,
  68                                       formatter_class=argparse.RawDescriptionHelpFormatter,
  69                                       add_help=False)
  70         parser.set_defaults(command=cmd)
  71         cmd.add_args(parser)
  72
  73     def run(self, **kwargs):
  74         """ Parse the command line arguments of the program and execute the
  75             appropriate subcommand.
  76         """
  77         args = self.parser.parse_args(args=kwargs.get('cli_args'))
  78
  79         if args.subcommand is None:
  80             self.parser.print_help()
  81             return 1
  82
  83         for arg in ('module_dir', 'osm2pgsql_path', 'phplib_dir', 'data_dir', 'phpcgi_path'):
  84             setattr(args, arg, Path(kwargs[arg]))
  85         args.project_dir = Path(args.project_dir).resolve()
  86
  87         logging.basicConfig(stream=sys.stderr,
  88                             format='%(asctime)s: %(message)s',
  89                             datefmt='%Y-%m-%d %H:%M:%S',
  90                             level=max(4 - args.verbose, 1) * 10)
  91
  92         args.config = Configuration(args.project_dir, args.data_dir / 'settings')
  93
  94         log = logging.getLogger()
  95         log.warning('Using project directory: %s', str(args.project_dir))
  96
  97         try:
  98             return args.command.run(args)
  99         except UsageError as exception:
 100             if log.isEnabledFor(logging.DEBUG):
 101                 raise # use Python's exception printing
 102             log.fatal('FATAL: %s', exception)
 103
 104         # If we get here, then execution has failed in some way.
 105         return 1
 106
 107
 108 def _osm2pgsql_options_from_args(args, default_cache, default_threads):
 109     """ Set up the stanadrd osm2pgsql from the command line arguments.
 110     """
 111     return dict(osm2pgsql=args.osm2pgsql_path,
 112                 osm2pgsql_cache=args.osm2pgsql_cache or default_cache,
 113                 osm2pgsql_style=args.config.get_import_style_file(),
 114                 threads=args.threads or default_threads,
 115                 dsn=args.config.get_libpq_dsn(),
 116                 flatnode_file=args.config.FLATNODE_FILE)
 117
 118 ##### Subcommand classes
 119 #
 120 # Each class needs to implement two functions: add_args() adds the CLI parameters
 121 # for the subfunction, run() executes the subcommand.
 122 #
 123 # The class documentation doubles as the help text for the command. The
 124 # first line is also used in the summary when calling the program without
 125 # a subcommand.
 126 #
 127 # No need to document the functions each time.
 128 # pylint: disable=C0111
 129 # Using non-top-level imports to make pyosmium optional for replication only.
 130 # pylint: disable=E0012,C0415
 131
 132
 133 class SetupAll:
 134     """\
 135     Create a new Nominatim database from an OSM file.
 136     """
 137
 138     @staticmethod
 139     def add_args(parser):
 140         group_name = parser.add_argument_group('Required arguments')
 141         group = group_name.add_mutually_exclusive_group(required=True)
 142         group.add_argument('--osm-file',
 143                            help='OSM file to be imported.')
 144         group.add_argument('--continue', dest='continue_at',
 145                            choices=['load-data', 'indexing', 'db-postprocess'],
 146                            help='Continue an import that was interrupted')
 147         group = parser.add_argument_group('Optional arguments')
 148         group.add_argument('--osm2pgsql-cache', metavar='SIZE', type=int,
 149                            help='Size of cache to be used by osm2pgsql (in MB)')
 150         group.add_argument('--reverse-only', action='store_true',
 151                            help='Do not create tables and indexes for searching')
 152         group.add_argument('--enable-debug-statements', action='store_true',
 153                            help='Include debug warning statements in SQL code')
 154         group.add_argument('--no-partitions', action='store_true',
 155                            help="""Do not partition search indices
 156                                    (speeds up import of single country extracts)""")
 157         group.add_argument('--no-updates', action='store_true',
 158                            help="""Do not keep tables that are only needed for
 159                                    updating the database later""")
 160         group = parser.add_argument_group('Expert options')
 161         group.add_argument('--ignore-errors', action='store_true',
 162                            help='Continue import even when errors in SQL are present')
 163         group.add_argument('--index-noanalyse', action='store_true',
 164                            help='Do not perform analyse operations during index')
 165
 166
 167     @staticmethod
 168     def run(args):
 169         params = ['setup.php']
 170         if args.osm_file:
 171             params.extend(('--all', '--osm-file', args.osm_file))
 172         else:
 173             if args.continue_at == 'load-data':
 174                 params.append('--load-data')
 175             if args.continue_at in ('load-data', 'indexing'):
 176                 params.append('--index')
 177             params.extend(('--create-search-indices', '--create-country-names',
 178                            '--setup-website'))
 179         if args.osm2pgsql_cache:
 180             params.extend(('--osm2pgsql-cache', args.osm2pgsql_cache))
 181         if args.reverse_only:
 182             params.append('--reverse-only')
 183         if args.enable_debug_statements:
 184             params.append('--enable-debug-statements')
 185         if args.no_partitions:
 186             params.append('--no-partitions')
 187         if args.no_updates:
 188             params.append('--drop')
 189         if args.ignore_errors:
 190             params.append('--ignore-errors')
 191         if args.index_noanalyse:
 192             params.append('--index-noanalyse')
 193
 194         return run_legacy_script(*params, nominatim_env=args)
 195
 196
 197 class SetupFreeze:
 198     """\
 199     Make database read-only.
 200
 201     About half of data in the Nominatim database is kept only to be able to
 202     keep the data up-to-date with new changes made in OpenStreetMap. This
 203     command drops all this data and only keeps the part needed for geocoding
 204     itself.
 205
 206     This command has the same effect as the `--no-updates` option for imports.
 207     """
 208
 209     @staticmethod
 210     def add_args(parser):
 211         pass # No options
 212
 213     @staticmethod
 214     def run(args):
 215         return run_legacy_script('setup.php', '--drop', nominatim_env=args)
 216
 217
 218 class SetupSpecialPhrases:
 219     """\
 220     Maintain special phrases.
 221     """
 222
 223     @staticmethod
 224     def add_args(parser):
 225         group = parser.add_argument_group('Input arguments')
 226         group.add_argument('--from-wiki', action='store_true',
 227                            help='Pull special phrases from the OSM wiki.')
 228         group = parser.add_argument_group('Output arguments')
 229         group.add_argument('-o', '--output', default='-',
 230                            help="""File to write the preprocessed phrases to.
 231                                    If omitted, it will be written to stdout.""")
 232
 233     @staticmethod
 234     def run(args):
 235         if args.output != '-':
 236             raise NotImplementedError('Only output to stdout is currently implemented.')
 237         return run_legacy_script('specialphrases.php', '--wiki-import', nominatim_env=args)
 238
 239
 240 class UpdateReplication:
 241     """\
 242     Update the database using an online replication service.
 243     """
 244
 245     @staticmethod
 246     def add_args(parser):
 247         group = parser.add_argument_group('Arguments for initialisation')
 248         group.add_argument('--init', action='store_true',
 249                            help='Initialise the update process')
 250         group.add_argument('--no-update-functions', dest='update_functions',
 251                            action='store_false',
 252                            help="""Do not update the trigger function to
 253                                    support differential updates.""")
 254         group = parser.add_argument_group('Arguments for updates')
 255         group.add_argument('--check-for-updates', action='store_true',
 256                            help='Check if new updates are available and exit')
 257         group.add_argument('--once', action='store_true',
 258                            help="""Download and apply updates only once. When
 259                                    not set, updates are continuously applied""")
 260         group.add_argument('--no-index', action='store_false', dest='do_index',
 261                            help="""Do not index the new data. Only applicable
 262                                    together with --once""")
 263         group.add_argument('--osm2pgsql-cache', metavar='SIZE', type=int,
 264                            help='Size of cache to be used by osm2pgsql (in MB)')
 265         group = parser.add_argument_group('Download parameters')
 266         group.add_argument('--socket-timeout', dest='socket_timeout', type=int, default=60,
 267                            help='Set timeout for file downloads.')
 268
 269     @staticmethod
 270     def _init_replication(args):
 271         from .tools import replication, refresh
 272
 273         socket.setdefaulttimeout(args.socket_timeout)
 274
 275         LOG.warning("Initialising replication updates")
 276         conn = connect(args.config.get_libpq_dsn())
 277         replication.init_replication(conn, base_url=args.config.REPLICATION_URL)
 278         if args.update_functions:
 279             LOG.warning("Create functions")
 280             refresh.create_functions(conn, args.config, args.data_dir,
 281                                      True, False)
 282         conn.close()
 283         return 0
 284
 285
 286     @staticmethod
 287     def _check_for_updates(args):
 288         from .tools import replication
 289
 290         conn = connect(args.config.get_libpq_dsn())
 291         ret = replication.check_for_updates(conn, base_url=args.config.REPLICATION_URL)
 292         conn.close()
 293         return ret
 294
 295     @staticmethod
 296     def _report_update(batchdate, start_import, start_index):
 297         def round_time(delta):
 298             return dt.timedelta(seconds=int(delta.total_seconds()))
 299
 300         end = dt.datetime.now(dt.timezone.utc)
 301         LOG.warning("Update completed. Import: %s. %sTotal: %s. Remaining backlog: %s.",
 302                     round_time((start_index or end) - start_import),
 303                     "Indexing: {} ".format(round_time(end - start_index))
 304                     if start_index else '',
 305                     round_time(end - start_import),
 306                     round_time(end - batchdate))
 307
 308     @staticmethod
 309     def _update(args):
 310         from .tools import replication
 311         from .indexer.indexer import Indexer
 312
 313         params = _osm2pgsql_options_from_args(args, 2000, 1)
 314         params.update(base_url=args.config.REPLICATION_URL,
 315                       update_interval=args.config.get_int('REPLICATION_UPDATE_INTERVAL'),
 316                       import_file=args.project_dir / 'osmosischange.osc',
 317                       max_diff_size=args.config.get_int('REPLICATION_MAX_DIFF'),
 318                       indexed_only=not args.once)
 319
 320         # Sanity check to not overwhelm the Geofabrik servers.
 321         if 'download.geofabrik.de'in params['base_url']\
 322            and params['update_interval'] < 86400:
 323             LOG.fatal("Update interval too low for download.geofabrik.de.\n"
 324                       "Please check install documentation "
 325                       "(https://nominatim.org/release-docs/latest/admin/Import-and-Update#"
 326                       "setting-up-the-update-process).")
 327             raise UsageError("Invalid replication update interval setting.")
 328
 329         if not args.once:
 330             if not args.do_index:
 331                 LOG.fatal("Indexing cannot be disabled when running updates continuously.")
 332                 raise UsageError("Bad argument '--no-index'.")
 333             recheck_interval = args.config.get_int('REPLICATION_RECHECK_INTERVAL')
 334
 335         while True:
 336             conn = connect(args.config.get_libpq_dsn())
 337             start = dt.datetime.now(dt.timezone.utc)
 338             state = replication.update(conn, params)
 339             status.log_status(conn, start, 'import')
 340             batchdate, _, _ = status.get_status(conn)
 341             conn.close()
 342
 343             if state is not replication.UpdateState.NO_CHANGES and args.do_index:
 344                 index_start = dt.datetime.now(dt.timezone.utc)
 345                 indexer = Indexer(args.config.get_libpq_dsn(),
 346                                   args.threads or 1)
 347                 indexer.index_boundaries(0, 30)
 348                 indexer.index_by_rank(0, 30)
 349
 350                 conn = connect(args.config.get_libpq_dsn())
 351                 status.set_indexed(conn, True)
 352                 status.log_status(conn, index_start, 'index')
 353                 conn.close()
 354             else:
 355                 index_start = None
 356
 357             if LOG.isEnabledFor(logging.WARNING):
 358                 UpdateReplication._report_update(batchdate, start, index_start)
 359
 360             if args.once:
 361                 break
 362
 363             if state is replication.UpdateState.NO_CHANGES:
 364                 LOG.warning("No new changes. Sleeping for %d sec.", recheck_interval)
 365                 time.sleep(recheck_interval)
 366
 367         return state.value
 368
 369     @staticmethod
 370     def run(args):
 371         try:
 372             import osmium # pylint: disable=W0611
 373         except ModuleNotFoundError:
 374             LOG.fatal("pyosmium not installed. Replication functions not available.\n"
 375                       "To install pyosmium via pip: pip3 install osmium")
 376             return 1
 377
 378         if args.init:
 379             return UpdateReplication._init_replication(args)
 380
 381         if args.check_for_updates:
 382             return UpdateReplication._check_for_updates(args)
 383
 384         return UpdateReplication._update(args)
 385
 386 class UpdateAddData:
 387     """\
 388     Add additional data from a file or an online source.
 389
 390     Data is only imported, not indexed. You need to call `nominatim-update index`
 391     to complete the process.
 392     """
 393
 394     @staticmethod
 395     def add_args(parser):
 396         group_name = parser.add_argument_group('Source')
 397         group = group_name.add_mutually_exclusive_group(required=True)
 398         group.add_argument('--file', metavar='FILE',
 399                            help='Import data from an OSM file')
 400         group.add_argument('--diff', metavar='FILE',
 401                            help='Import data from an OSM diff file')
 402         group.add_argument('--node', metavar='ID', type=int,
 403                            help='Import a single node from the API')
 404         group.add_argument('--way', metavar='ID', type=int,
 405                            help='Import a single way from the API')
 406         group.add_argument('--relation', metavar='ID', type=int,
 407                            help='Import a single relation from the API')
 408         group.add_argument('--tiger-data', metavar='DIR',
 409                            help='Add housenumbers from the US TIGER census database.')
 410         group = parser.add_argument_group('Extra arguments')
 411         group.add_argument('--use-main-api', action='store_true',
 412                            help='Use OSM API instead of Overpass to download objects')
 413
 414     @staticmethod
 415     def run(args):
 416         if args.tiger_data:
 417             os.environ['NOMINATIM_TIGER_DATA_PATH'] = args.tiger_data
 418             return run_legacy_script('setup.php', '--import-tiger-data', nominatim_env=args)
 419
 420         params = ['update.php']
 421         if args.file:
 422             params.extend(('--import-file', args.file))
 423         elif args.diff:
 424             params.extend(('--import-diff', args.diff))
 425         elif args.node:
 426             params.extend(('--import-node', args.node))
 427         elif args.way:
 428             params.extend(('--import-way', args.way))
 429         elif args.relation:
 430             params.extend(('--import-relation', args.relation))
 431         if args.use_main_api:
 432             params.append('--use-main-api')
 433         return run_legacy_script(*params, nominatim_env=args)
 434
 435
 436 class UpdateIndex:
 437     """\
 438     Reindex all new and modified data.
 439     """
 440
 441     @staticmethod
 442     def add_args(parser):
 443         group = parser.add_argument_group('Filter arguments')
 444         group.add_argument('--boundaries-only', action='store_true',
 445                            help="""Index only administrative boundaries.""")
 446         group.add_argument('--no-boundaries', action='store_true',
 447                            help="""Index everything except administrative boundaries.""")
 448         group.add_argument('--minrank', '-r', type=int, metavar='RANK', default=0,
 449                            help='Minimum/starting rank')
 450         group.add_argument('--maxrank', '-R', type=int, metavar='RANK', default=30,
 451                            help='Maximum/finishing rank')
 452
 453     @staticmethod
 454     def run(args):
 455         from .indexer.indexer import Indexer
 456
 457         indexer = Indexer(args.config.get_libpq_dsn(),
 458                           args.threads or _num_system_cpus() or 1)
 459
 460         if not args.no_boundaries:
 461             indexer.index_boundaries(args.minrank, args.maxrank)
 462         if not args.boundaries_only:
 463             indexer.index_by_rank(args.minrank, args.maxrank)
 464
 465         if not args.no_boundaries and not args.boundaries_only \
 466            and args.minrank == 0 and args.maxrank == 30:
 467             conn = connect(args.config.get_libpq_dsn())
 468             status.set_indexed(conn, True)
 469             conn.close()
 470
 471         return 0
 472
 473
 474 class UpdateRefresh:
 475     """\
 476     Recompute auxiliary data used by the indexing process.
 477
 478     These functions must not be run in parallel with other update commands.
 479     """
 480
 481     @staticmethod
 482     def add_args(parser):
 483         group = parser.add_argument_group('Data arguments')
 484         group.add_argument('--postcodes', action='store_true',
 485                            help='Update postcode centroid table')
 486         group.add_argument('--word-counts', action='store_true',
 487                            help='Compute frequency of full-word search terms')
 488         group.add_argument('--address-levels', action='store_true',
 489                            help='Reimport address level configuration')
 490         group.add_argument('--functions', action='store_true',
 491                            help='Update the PL/pgSQL functions in the database')
 492         group.add_argument('--wiki-data', action='store_true',
 493                            help='Update Wikipedia/data importance numbers.')
 494         group.add_argument('--importance', action='store_true',
 495                            help='Recompute place importances (expensive!)')
 496         group.add_argument('--website', action='store_true',
 497                            help='Refresh the directory that serves the scripts for the web API')
 498         group = parser.add_argument_group('Arguments for function refresh')
 499         group.add_argument('--no-diff-updates', action='store_false', dest='diffs',
 500                            help='Do not enable code for propagating updates')
 501         group.add_argument('--enable-debug-statements', action='store_true',
 502                            help='Enable debug warning statements in functions')
 503
 504     @staticmethod
 505     def run(args):
 506         from .tools import refresh
 507
 508         if args.postcodes:
 509             LOG.warning("Update postcodes centroid")
 510             conn = connect(args.config.get_libpq_dsn())
 511             refresh.update_postcodes(conn, args.data_dir)
 512             conn.close()
 513
 514         if args.word_counts:
 515             LOG.warning('Recompute frequency of full-word search terms')
 516             conn = connect(args.config.get_libpq_dsn())
 517             refresh.recompute_word_counts(conn, args.data_dir)
 518             conn.close()
 519
 520         if args.address_levels:
 521             cfg = Path(args.config.ADDRESS_LEVEL_CONFIG)
 522             LOG.warning('Updating address levels from %s', cfg)
 523             conn = connect(args.config.get_libpq_dsn())
 524             refresh.load_address_levels_from_file(conn, cfg)
 525             conn.close()
 526
 527         if args.functions:
 528             LOG.warning('Create functions')
 529             conn = connect(args.config.get_libpq_dsn())
 530             refresh.create_functions(conn, args.config, args.data_dir,
 531                                      args.diffs, args.enable_debug_statements)
 532             conn.close()
 533
 534         if args.wiki_data:
 535             run_legacy_script('setup.php', '--import-wikipedia-articles',
 536                               nominatim_env=args, throw_on_fail=True)
 537         # Attention: importance MUST come after wiki data import.
 538         if args.importance:
 539             run_legacy_script('update.php', '--recompute-importance',
 540                               nominatim_env=args, throw_on_fail=True)
 541         if args.website:
 542             run_legacy_script('setup.php', '--setup-website',
 543                               nominatim_env=args, throw_on_fail=True)
 544
 545         return 0
 546
 547
 548 class AdminCheckDatabase:
 549     """\
 550     Check that the database is complete and operational.
 551     """
 552
 553     @staticmethod
 554     def add_args(parser):
 555         pass # No options
 556
 557     @staticmethod
 558     def run(args):
 559         return run_legacy_script('check_import_finished.php', nominatim_env=args)
 560
 561
 562 class AdminWarm:
 563     """\
 564     Warm database caches for search and reverse queries.
 565     """
 566
 567     @staticmethod
 568     def add_args(parser):
 569         group = parser.add_argument_group('Target arguments')
 570         group.add_argument('--search-only', action='store_const', dest='target',
 571                            const='search',
 572                            help="Only pre-warm tables for search queries")
 573         group.add_argument('--reverse-only', action='store_const', dest='target',
 574                            const='reverse',
 575                            help="Only pre-warm tables for reverse queries")
 576
 577     @staticmethod
 578     def run(args):
 579         params = ['warm.php']
 580         if args.target == 'reverse':
 581             params.append('--reverse-only')
 582         if args.target == 'search':
 583             params.append('--search-only')
 584         return run_legacy_script(*params, nominatim_env=args)
 585
 586
 587 class QueryExport:
 588     """\
 589     Export addresses as CSV file from the database.
 590     """
 591
 592     @staticmethod
 593     def add_args(parser):
 594         group = parser.add_argument_group('Output arguments')
 595         group.add_argument('--output-type', default='street',
 596                            choices=('continent', 'country', 'state', 'county',
 597                                     'city', 'suburb', 'street', 'path'),
 598                            help='Type of places to output (default: street)')
 599         group.add_argument('--output-format',
 600                            default='street;suburb;city;county;state;country',
 601                            help="""Semicolon-separated list of address types
 602                                    (see --output-type). Multiple ranks can be
 603                                    merged into one column by simply using a
 604                                    comma-separated list.""")
 605         group.add_argument('--output-all-postcodes', action='store_true',
 606                            help="""List all postcodes for address instead of
 607                                    just the most likely one""")
 608         group.add_argument('--language',
 609                            help="""Preferred language for output
 610                                    (use local name, if omitted)""")
 611         group = parser.add_argument_group('Filter arguments')
 612         group.add_argument('--restrict-to-country', metavar='COUNTRY_CODE',
 613                            help='Export only objects within country')
 614         group.add_argument('--restrict-to-osm-node', metavar='ID', type=int,
 615                            help='Export only children of this OSM node')
 616         group.add_argument('--restrict-to-osm-way', metavar='ID', type=int,
 617                            help='Export only children of this OSM way')
 618         group.add_argument('--restrict-to-osm-relation', metavar='ID', type=int,
 619                            help='Export only children of this OSM relation')
 620
 621
 622     @staticmethod
 623     def run(args):
 624         params = ['export.php',
 625                   '--output-type', args.output_type,
 626                   '--output-format', args.output_format]
 627         if args.output_all_postcodes:
 628             params.append('--output-all-postcodes')
 629         if args.language:
 630             params.extend(('--language', args.language))
 631         if args.restrict_to_country:
 632             params.extend(('--restrict-to-country', args.restrict_to_country))
 633         if args.restrict_to_osm_node:
 634             params.extend(('--restrict-to-osm-node', args.restrict_to_osm_node))
 635         if args.restrict_to_osm_way:
 636             params.extend(('--restrict-to-osm-way', args.restrict_to_osm_way))
 637         if args.restrict_to_osm_relation:
 638             params.extend(('--restrict-to-osm-relation', args.restrict_to_osm_relation))
 639
 640         return run_legacy_script(*params, nominatim_env=args)
 641
 642
 643 class AdminServe:
 644     """\
 645     Start a simple web server for serving the API.
 646
 647     This command starts the built-in PHP webserver to serve the website
 648     from the current project directory. This webserver is only suitable
 649     for testing and develop. Do not use it in production setups!
 650
 651     By the default, the webserver can be accessed at: http://127.0.0.1:8088
 652     """
 653
 654     @staticmethod
 655     def add_args(parser):
 656         group = parser.add_argument_group('Server arguments')
 657         group.add_argument('--server', default='127.0.0.1:8088',
 658                            help='The address the server will listen to.')
 659
 660     @staticmethod
 661     def run(args):
 662         run_php_server(args.server, args.project_dir / 'website')
 663
 664 STRUCTURED_QUERY = (
 665     ('street', 'housenumber and street'),
 666     ('city', 'city, town or village'),
 667     ('county', 'county'),
 668     ('state', 'state'),
 669     ('country', 'country'),
 670     ('postalcode', 'postcode')
 671 )
 672
 673 EXTRADATA_PARAMS = (
 674     ('addressdetails', 'Include a breakdown of the address into elements.'),
 675     ('extratags', """Include additional information if available
 676                      (e.g. wikipedia link, opening hours)."""),
 677     ('namedetails', 'Include a list of alternative names.')
 678 )
 679
 680 DETAILS_SWITCHES = (
 681     ('addressdetails', 'Include a breakdown of the address into elements.'),
 682     ('keywords', 'Include a list of name keywords and address keywords.'),
 683     ('linkedplaces', 'Include a details of places that are linked with this one.'),
 684     ('hierarchy', 'Include details of places lower in the address hierarchy.'),
 685     ('group_hierarchy', 'Group the places by type.'),
 686     ('polygon_geojson', 'Include geometry of result.')
 687 )
 688
 689 def _add_api_output_arguments(parser):
 690     group = parser.add_argument_group('Output arguments')
 691     group.add_argument('--format', default='jsonv2',
 692                        choices=['xml', 'json', 'jsonv2', 'geojson', 'geocodejson'],
 693                        help='Format of result')
 694     for name, desc in EXTRADATA_PARAMS:
 695         group.add_argument('--' + name, action='store_true', help=desc)
 696
 697     group.add_argument('--lang', '--accept-language', metavar='LANGS',
 698                        help='Preferred language order for presenting search results')
 699     group.add_argument('--polygon-output',
 700                        choices=['geojson', 'kml', 'svg', 'text'],
 701                        help='Output geometry of results as a GeoJSON, KML, SVG or WKT.')
 702     group.add_argument('--polygon-threshold', type=float, metavar='TOLERANCE',
 703                        help="""Simplify output geometry.
 704                                Parameter is difference tolerance in degrees.""")
 705
 706
 707 class APISearch:
 708     """\
 709     Execute API search query.
 710     """
 711
 712     @staticmethod
 713     def add_args(parser):
 714         group = parser.add_argument_group('Query arguments')
 715         group.add_argument('--query',
 716                            help='Free-form query string')
 717         for name, desc in STRUCTURED_QUERY:
 718             group.add_argument('--' + name, help='Structured query: ' + desc)
 719
 720         _add_api_output_arguments(parser)
 721
 722         group = parser.add_argument_group('Result limitation')
 723         group.add_argument('--countrycodes', metavar='CC,..',
 724                            help='Limit search results to one or more countries.')
 725         group.add_argument('--exclude_place_ids', metavar='ID,..',
 726                            help='List of search object to be excluded')
 727         group.add_argument('--limit', type=int,
 728                            help='Limit the number of returned results')
 729         group.add_argument('--viewbox', metavar='X1,Y1,X2,Y2',
 730                            help='Preferred area to find search results')
 731         group.add_argument('--bounded', action='store_true',
 732                            help='Strictly restrict results to viewbox area')
 733
 734         group = parser.add_argument_group('Other arguments')
 735         group.add_argument('--no-dedupe', action='store_false', dest='dedupe',
 736                            help='Do not remove duplicates from the result list')
 737
 738
 739     @staticmethod
 740     def run(args):
 741         if args.query:
 742             params = dict(q=args.query)
 743         else:
 744             params = {k : getattr(args, k) for k, _ in STRUCTURED_QUERY if getattr(args, k)}
 745
 746         for param, _ in EXTRADATA_PARAMS:
 747             if getattr(args, param):
 748                 params[param] = '1'
 749         for param in ('format', 'countrycodes', 'exclude_place_ids', 'limit', 'viewbox'):
 750             if getattr(args, param):
 751                 params[param] = getattr(args, param)
 752         if args.lang:
 753             params['accept-language'] = args.lang
 754         if args.polygon_output:
 755             params['polygon_' + args.polygon_output] = '1'
 756         if args.polygon_threshold:
 757             params['polygon_threshold'] = args.polygon_threshold
 758         if args.bounded:
 759             params['bounded'] = '1'
 760         if not args.dedupe:
 761             params['dedupe'] = '0'
 762
 763         return run_api_script('search', args.project_dir,
 764                               phpcgi_bin=args.phpcgi_path, params=params)
 765
 766 class APIReverse:
 767     """\
 768     Execute API reverse query.
 769     """
 770
 771     @staticmethod
 772     def add_args(parser):
 773         group = parser.add_argument_group('Query arguments')
 774         group.add_argument('--lat', type=float, required=True,
 775                            help='Latitude of coordinate to look up (in WGS84)')
 776         group.add_argument('--lon', type=float, required=True,
 777                            help='Longitude of coordinate to look up (in WGS84)')
 778         group.add_argument('--zoom', type=int,
 779                            help='Level of detail required for the address')
 780
 781         _add_api_output_arguments(parser)
 782
 783
 784     @staticmethod
 785     def run(args):
 786         params = dict(lat=args.lat, lon=args.lon)
 787         if args.zoom is not None:
 788             params['zoom'] = args.zoom
 789
 790         for param, _ in EXTRADATA_PARAMS:
 791             if getattr(args, param):
 792                 params[param] = '1'
 793         if args.format:
 794             params['format'] = args.format
 795         if args.lang:
 796             params['accept-language'] = args.lang
 797         if args.polygon_output:
 798             params['polygon_' + args.polygon_output] = '1'
 799         if args.polygon_threshold:
 800             params['polygon_threshold'] = args.polygon_threshold
 801
 802         return run_api_script('reverse', args.project_dir,
 803                               phpcgi_bin=args.phpcgi_path, params=params)
 804
 805
 806 class APILookup:
 807     """\
 808     Execute API reverse query.
 809     """
 810
 811     @staticmethod
 812     def add_args(parser):
 813         group = parser.add_argument_group('Query arguments')
 814         group.add_argument('--id', metavar='OSMID',
 815                            action='append', required=True, dest='ids',
 816                            help='OSM id to lookup in format <NRW><id> (may be repeated)')
 817
 818         _add_api_output_arguments(parser)
 819
 820
 821     @staticmethod
 822     def run(args):
 823         params = dict(osm_ids=','.join(args.ids))
 824
 825         for param, _ in EXTRADATA_PARAMS:
 826             if getattr(args, param):
 827                 params[param] = '1'
 828         if args.format:
 829             params['format'] = args.format
 830         if args.lang:
 831             params['accept-language'] = args.lang
 832         if args.polygon_output:
 833             params['polygon_' + args.polygon_output] = '1'
 834         if args.polygon_threshold:
 835             params['polygon_threshold'] = args.polygon_threshold
 836
 837         return run_api_script('lookup', args.project_dir,
 838                               phpcgi_bin=args.phpcgi_path, params=params)
 839
 840
 841 class APIDetails:
 842     """\
 843     Execute API lookup query.
 844     """
 845
 846     @staticmethod
 847     def add_args(parser):
 848         group = parser.add_argument_group('Query arguments')
 849         objs = group.add_mutually_exclusive_group(required=True)
 850         objs.add_argument('--node', '-n', type=int,
 851                           help="Look up the OSM node with the given ID.")
 852         objs.add_argument('--way', '-w', type=int,
 853                           help="Look up the OSM way with the given ID.")
 854         objs.add_argument('--relation', '-r', type=int,
 855                           help="Look up the OSM relation with the given ID.")
 856         objs.add_argument('--place_id', '-p', type=int,
 857                           help='Database internal identifier of the OSM object to look up.')
 858         group.add_argument('--class', dest='object_class',
 859                            help="""Class type to disambiguated multiple entries
 860                                    of the same object.""")
 861
 862         group = parser.add_argument_group('Output arguments')
 863         for name, desc in DETAILS_SWITCHES:
 864             group.add_argument('--' + name, action='store_true', help=desc)
 865         group.add_argument('--lang', '--accept-language', metavar='LANGS',
 866                            help='Preferred language order for presenting search results')
 867
 868     @staticmethod
 869     def run(args):
 870         if args.node:
 871             params = dict(osmtype='N', osmid=args.node)
 872         elif args.way:
 873             params = dict(osmtype='W', osmid=args.node)
 874         elif args.relation:
 875             params = dict(osmtype='R', osmid=args.node)
 876         else:
 877             params = dict(place_id=args.place_id)
 878         if args.object_class:
 879             params['class'] = args.object_class
 880         for name, _ in DETAILS_SWITCHES:
 881             params[name] = '1' if getattr(args, name) else '0'
 882
 883         return run_api_script('details', args.project_dir,
 884                               phpcgi_bin=args.phpcgi_path, params=params)
 885
 886
 887 class APIStatus:
 888     """\
 889     Execute API status query.
 890     """
 891
 892     @staticmethod
 893     def add_args(parser):
 894         group = parser.add_argument_group('API parameters')
 895         group.add_argument('--format', default='text', choices=['text', 'json'],
 896                            help='Format of result')
 897
 898     @staticmethod
 899     def run(args):
 900         return run_api_script('status', args.project_dir,
 901                               phpcgi_bin=args.phpcgi_path,
 902                               params=dict(format=args.format))
 903
 904
 905 def nominatim(**kwargs):
 906     """\
 907     Command-line tools for importing, updating, administrating and
 908     querying the Nominatim database.
 909     """
 910     parser = CommandlineParser('nominatim', nominatim.__doc__)
 911
 912     parser.add_subcommand('import', SetupAll)
 913     parser.add_subcommand('freeze', SetupFreeze)
 914     parser.add_subcommand('replication', UpdateReplication)
 915
 916     parser.add_subcommand('check-database', AdminCheckDatabase)
 917     parser.add_subcommand('warm', AdminWarm)
 918
 919     parser.add_subcommand('special-phrases', SetupSpecialPhrases)
 920
 921     parser.add_subcommand('add-data', UpdateAddData)
 922     parser.add_subcommand('index', UpdateIndex)
 923     parser.add_subcommand('refresh', UpdateRefresh)
 924
 925     parser.add_subcommand('export', QueryExport)
 926     parser.add_subcommand('serve', AdminServe)
 927
 928     if kwargs.get('phpcgi_path'):
 929         parser.add_subcommand('search', APISearch)
 930         parser.add_subcommand('reverse', APIReverse)
 931         parser.add_subcommand('lookup', APILookup)
 932         parser.add_subcommand('details', APIDetails)
 933         parser.add_subcommand('status', APIStatus)
 934     else:
 935         parser.parser.epilog = 'php-cgi not found. Query commands not available.'
 936
 937     return parser.run(**kwargs)