nominatim/cli.py

   1 """
   2 Command-line interface to the Nominatim functions for import, update,
   3 database administration and querying.
   4 """
   5 import sys
   6 import os
   7 import argparse
   8 import logging
   9 from pathlib import Path
  10
  11 from .config import Configuration
  12 from .admin.exec_utils import run_legacy_script
  13
  14 class CommandlineParser:
  15     """ Wraps some of the common functions for parsing the command line
  16         and setting up subcommands.
  17     """
  18     def __init__(self, prog, description):
  19         self.parser = argparse.ArgumentParser(
  20             prog=prog,
  21             description=description,
  22             formatter_class=argparse.RawDescriptionHelpFormatter)
  23
  24         self.subs = self.parser.add_subparsers(title='available commands',
  25                                                dest='subcommand')
  26
  27         # Arguments added to every sub-command
  28         self.default_args = argparse.ArgumentParser(add_help=False)
  29         group = self.default_args.add_argument_group('Default arguments')
  30         group.add_argument('-h', '--help', action='help',
  31                            help='Show this help message and exit')
  32         group.add_argument('-q', '--quiet', action='store_const', const=0,
  33                            dest='verbose', default=1,
  34                            help='Print only error messages')
  35         group.add_argument('-v', '--verbose', action='count', default=1,
  36                            help='Increase verboseness of output')
  37         group.add_argument('--project-dir', metavar='DIR', default='.',
  38                            help='Base directory of the Nominatim installation (default:.)')
  39         group.add_argument('-j', '--threads', metavar='NUM', type=int,
  40                            help='Number of parallel threads to use')
  41
  42
  43     def add_subcommand(self, name, cmd):
  44         """ Add a subcommand to the parser. The subcommand must be a class
  45             with a function add_args() that adds the parameters for the
  46             subcommand and a run() function that executes the command.
  47         """
  48         parser = self.subs.add_parser(name, parents=[self.default_args],
  49                                       help=cmd.__doc__.split('\n', 1)[0],
  50                                       description=cmd.__doc__,
  51                                       formatter_class=argparse.RawDescriptionHelpFormatter,
  52                                       add_help=False)
  53         parser.set_defaults(command=cmd)
  54         cmd.add_args(parser)
  55
  56     def run(self, **kwargs):
  57         """ Parse the command line arguments of the program and execute the
  58             appropriate subcommand.
  59         """
  60         args = self.parser.parse_args()
  61
  62         if args.subcommand is None:
  63             return self.parser.print_help()
  64
  65         for arg in ('module_dir', 'osm2pgsql_path', 'phplib_dir', 'data_dir'):
  66             setattr(args, arg, Path(kwargs[arg]))
  67         args.project_dir = Path(args.project_dir)
  68
  69         logging.basicConfig(stream=sys.stderr,
  70                             format='%(asctime)s %(levelname)s: %(message)s',
  71                             datefmt='%Y-%m-%d %H:%M:%S',
  72                             level=max(4 - args.verbose, 1) * 10)
  73
  74         args.config = Configuration(args.project_dir, args.data_dir / 'settings')
  75
  76         return args.command.run(args)
  77
  78 ##### Subcommand classes
  79 #
  80 # Each class needs to implement two functions: add_args() adds the CLI parameters
  81 # for the subfunction, run() executes the subcommand.
  82 #
  83 # The class documentation doubles as the help text for the command. The
  84 # first line is also used in the summary when calling the program without
  85 # a subcommand.
  86 #
  87 # No need to document the functions each time.
  88 # pylint: disable=C0111
  89
  90
  91 class SetupAll:
  92     """\
  93     Create a new Nominatim database from an OSM file.
  94     """
  95
  96     @staticmethod
  97     def add_args(parser):
  98         group_name = parser.add_argument_group('Required arguments')
  99         group = group_name.add_mutually_exclusive_group(required=True)
 100         group.add_argument('--osm-file',
 101                            help='OSM file to be imported.')
 102         group.add_argument('--continue', dest='continue_at',
 103                            choices=['load-data', 'indexing', 'db-postprocess'],
 104                            help='Continue an import that was interrupted')
 105         group = parser.add_argument_group('Optional arguments')
 106         group.add_argument('--osm2pgsql-cache', metavar='SIZE', type=int,
 107                            help='Size of cache to be used by osm2pgsql (in MB)')
 108         group.add_argument('--reverse-only', action='store_true',
 109                            help='Do not create tables and indexes for searching')
 110         group.add_argument('--enable-debug-statements', action='store_true',
 111                            help='Include debug warning statements in SQL code')
 112         group.add_argument('--no-partitions', action='store_true',
 113                            help="""Do not partition search indices
 114                                    (speeds up import of single country extracts)""")
 115         group.add_argument('--no-updates', action='store_true',
 116                            help="""Do not keep tables that are only needed for
 117                                    updating the database later""")
 118         group = parser.add_argument_group('Expert options')
 119         group.add_argument('--ignore-errors', action='store_true',
 120                            help='Continue import even when errors in SQL are present')
 121         group.add_argument('--index-noanalyse', action='store_true',
 122                            help='Do not perform analyse operations during index')
 123
 124
 125     @staticmethod
 126     def run(args):
 127         params = ['setup.php']
 128         if args.osm_file:
 129             params.extend(('--all', '--osm-file', args.osm_file))
 130         else:
 131             if args.continue_at == 'load-data':
 132                 params.append('--load-data')
 133             if args.continue_at in ('load-data', 'indexing'):
 134                 params.append('--index')
 135             params.extend(('--create-search-indices', '--create-country-names',
 136                            '--setup-website'))
 137         if args.osm2pgsql_cache:
 138             params.extend(('--osm2pgsql-cache', args.osm2pgsql_cache))
 139         if args.reverse_only:
 140             params.append('--reverse-only')
 141         if args.enable_debug_statements:
 142             params.append('--enable-debug-statements')
 143         if args.no_partitions:
 144             params.append('--no-partitions')
 145         if args.no_updates:
 146             params.append('--drop')
 147         if args.ignore_errors:
 148             params.append('--ignore-errors')
 149         if args.index_noanalyse:
 150             params.append('--index-noanalyse')
 151
 152         return run_legacy_script(*params, nominatim_env=args)
 153
 154
 155 class SetupFreeze:
 156     """\
 157     Make database read-only.
 158
 159     About half of data in the Nominatim database is kept only to be able to
 160     keep the data up-to-date with new changes made in OpenStreetMap. This
 161     command drops all this data and only keeps the part needed for geocoding
 162     itself.
 163
 164     This command has the same effect as the `--no-updates` option for imports.
 165     """
 166
 167     @staticmethod
 168     def add_args(parser):
 169         pass # No options
 170
 171     @staticmethod
 172     def run(args):
 173         return run_legacy_script('setup.php', '--drop', nominatim_env=args)
 174
 175
 176 class SetupSpecialPhrases:
 177     """\
 178     Maintain special phrases.
 179     """
 180
 181     @staticmethod
 182     def add_args(parser):
 183         group = parser.add_argument_group('Input arguments')
 184         group.add_argument('--from-wiki', action='store_true',
 185                            help='Pull special phrases from the OSM wiki.')
 186         group = parser.add_argument_group('Output arguments')
 187         group.add_argument('-o', '--output', default='-',
 188                            type=argparse.FileType('w', encoding='UTF-8'),
 189                            help="""File to write the preprocessed phrases to.
 190                                    If omitted, it will be written to stdout.""")
 191
 192     @staticmethod
 193     def run(args):
 194         if args.output.name != '<stdout>':
 195             raise NotImplementedError('Only output to stdout is currently implemented.')
 196         return run_legacy_script('specialphrases.php', '--wiki-import', nominatim_env=args)
 197
 198
 199 class UpdateReplication:
 200     """\
 201     Update the database using an online replication service.
 202     """
 203
 204     @staticmethod
 205     def add_args(parser):
 206         group = parser.add_argument_group('Arguments for initialisation')
 207         group.add_argument('--init', action='store_true',
 208                            help='Initialise the update process')
 209         group.add_argument('--no-update-functions', dest='update_functions',
 210                            action='store_false',
 211                            help="""Do not update the trigger function to
 212                                    support differential updates.""")
 213         group = parser.add_argument_group('Arguments for updates')
 214         group.add_argument('--check-for-updates', action='store_true',
 215                            help='Check if new updates are available and exit')
 216         group.add_argument('--once', action='store_true',
 217                            help="""Download and apply updates only once. When
 218                                    not set, updates are continuously applied""")
 219         group.add_argument('--no-index', action='store_false', dest='do_index',
 220                            help="""Do not index the new data. Only applicable
 221                                    together with --once""")
 222
 223     @staticmethod
 224     def run(args):
 225         params = ['update.php']
 226         if args.init:
 227             params.append('--init-updates')
 228             if not args.update_functions:
 229                 params.append('--no-update-functions')
 230         elif args.check_for_updates:
 231             params.append('--check-for-updates')
 232         else:
 233             if args.once:
 234                 params.append('--import-osmosis')
 235             else:
 236                 params.append('--import-osmosis-all')
 237             if not args.do_index:
 238                 params.append('--no-index')
 239
 240         return run_legacy_script(*params, nominatim_env=args)
 241
 242
 243 class UpdateAddData:
 244     """\
 245     Add additional data from a file or an online source.
 246
 247     Data is only imported, not indexed. You need to call `nominatim-update index`
 248     to complete the process.
 249     """
 250
 251     @staticmethod
 252     def add_args(parser):
 253         group_name = parser.add_argument_group('Source')
 254         group = group_name.add_mutually_exclusive_group(required=True)
 255         group.add_argument('--file', metavar='FILE',
 256                            help='Import data from an OSM file')
 257         group.add_argument('--diff', metavar='FILE',
 258                            help='Import data from an OSM diff file')
 259         group.add_argument('--node', metavar='ID', type=int,
 260                            help='Import a single node from the API')
 261         group.add_argument('--way', metavar='ID', type=int,
 262                            help='Import a single way from the API')
 263         group.add_argument('--relation', metavar='ID', type=int,
 264                            help='Import a single relation from the API')
 265         group.add_argument('--tiger-data', metavar='DIR',
 266                            help='Add housenumbers from the US TIGER census database.')
 267         group = parser.add_argument_group('Extra arguments')
 268         group.add_argument('--use-main-api', action='store_true',
 269                            help='Use OSM API instead of Overpass to download objects')
 270
 271     @staticmethod
 272     def run(args):
 273         if args.tiger_data:
 274             os.environ['NOMINATIM_TIGER_DATA_PATH'] = args.tiger_data
 275             return run_legacy_script('setup.php', '--import-tiger-data', nominatim_env=args)
 276
 277         params = ['update.php']
 278         if args.file:
 279             params.extend(('--import-file', args.file))
 280         elif args.diff:
 281             params.extend(('--import-diff', args.diff))
 282         elif args.node:
 283             params.extend(('--import-node', args.node))
 284         elif args.way:
 285             params.extend(('--import-way', args.way))
 286         elif args.relation:
 287             params.extend(('--import-relation', args.relation))
 288         if args.use_main_api:
 289             params.append('--use-main-api')
 290         return run_legacy_script(*params, nominatim_env=args)
 291
 292
 293 class UpdateIndex:
 294     """\
 295     Reindex all new and modified data.
 296     """
 297
 298     @staticmethod
 299     def add_args(parser):
 300         pass
 301
 302     @staticmethod
 303     def run(args):
 304         return run_legacy_script('update.php', '--index', nominatim_env=args)
 305
 306
 307 class UpdateRefresh:
 308     """\
 309     Recompute auxillary data used by the indexing process.
 310
 311     These functions must not be run in parallel with other update commands.
 312     """
 313
 314     @staticmethod
 315     def add_args(parser):
 316         group = parser.add_argument_group('Data arguments')
 317         group.add_argument('--postcodes', action='store_true',
 318                            help='Update postcode centroid table')
 319         group.add_argument('--word-counts', action='store_true',
 320                            help='Compute frequency of full-word search terms')
 321         group.add_argument('--address-levels', action='store_true',
 322                            help='Reimport address level configuration')
 323         group.add_argument('--functions', action='store_true',
 324                            help='Update the PL/pgSQL functions in the database')
 325         group.add_argument('--wiki-data', action='store_true',
 326                            help='Update Wikipedia/data importance numbers.')
 327         group.add_argument('--importance', action='store_true',
 328                            help='Recompute place importances (expensive!)')
 329         group.add_argument('--website', action='store_true',
 330                            help='Refresh the directory that serves the scripts for the web API')
 331         group = parser.add_argument_group('Arguments for function refresh')
 332         group.add_argument('--no-diff-updates', action='store_false', dest='diffs',
 333                            help='Do not enable code for propagating updates')
 334         group.add_argument('--enable-debug-statements', action='store_true',
 335                            help='Enable debug warning statements in functions')
 336
 337     @staticmethod
 338     def run(args):
 339         if args.postcodes:
 340             run_legacy_script('update.php', '--calculate-postcodes',
 341                               nominatim_env=args, throw_on_fail=True)
 342         if args.word_counts:
 343             run_legacy_script('update.php', '--recompute-word-counts',
 344                               nominatim_env=args, throw_on_fail=True)
 345         if args.address_levels:
 346             run_legacy_script('update.php', '--update-address-levels',
 347                               nominatim_env=args, throw_on_fail=True)
 348         if args.functions:
 349             params = ['setup.php', '--create-functions', '--create-partition-functions']
 350             if args.diffs:
 351                 params.append('--enable-diff-updates')
 352             if args.enable_debug_statements:
 353                 params.append('--enable-debug-statements')
 354             run_legacy_script(*params, nominatim_env=args, throw_on_fail=True)
 355         if args.wiki_data:
 356             run_legacy_script('setup.php', '--import-wikipedia-articles',
 357                               nominatim_env=args, throw_on_fail=True)
 358         # Attention: importance MUST come after wiki data import.
 359         if args.importance:
 360             run_legacy_script('update.php', '--recompute-importance',
 361                               nominatim_env=args, throw_on_fail=True)
 362         if args.website:
 363             run_legacy_script('setup.php', '--setup-website',
 364                               nominatim_env=args, throw_on_fail=True)
 365
 366
 367 class AdminCheckDatabase:
 368     """\
 369     Check that the database is complete and operational.
 370     """
 371
 372     @staticmethod
 373     def add_args(parser):
 374         pass # No options
 375
 376     @staticmethod
 377     def run(args):
 378         return run_legacy_script('check_import_finished.php', nominatim_env=args)
 379
 380
 381 class AdminWarm:
 382     """\
 383     Warm database caches for search and reverse queries.
 384     """
 385
 386     @staticmethod
 387     def add_args(parser):
 388         group = parser.add_argument_group('Target arguments')
 389         group.add_argument('--search-only', action='store_const', dest='target',
 390                            const='search',
 391                            help="Only pre-warm tables for search queries")
 392         group.add_argument('--reverse-only', action='store_const', dest='target',
 393                            const='reverse',
 394                            help="Only pre-warm tables for reverse queries")
 395
 396     @staticmethod
 397     def run(args):
 398         params = ['warm.php']
 399         if args.target == 'reverse':
 400             params.append('--reverse-only')
 401         if args.target == 'search':
 402             params.append('--search-only')
 403         return run_legacy_script(*params, nominatim_env=args)
 404
 405
 406 class QueryExport:
 407     """\
 408     Export addresses as CSV file from a Nominatim database.
 409     """
 410
 411     @staticmethod
 412     def add_args(parser):
 413         group = parser.add_argument_group('Output arguments')
 414         group.add_argument('--output-type', default='street',
 415                            choices=('continent', 'country', 'state', 'county',
 416                                     'city', 'suburb', 'street', 'path'),
 417                            help='Type of places to output (default: street)')
 418         group.add_argument('--output-format',
 419                            default='street;suburb;city;county;state;country',
 420                            help="""Semicolon-separated list of address types
 421                                    (see --output-type). Multiple ranks can be
 422                                    merged into one column by simply using a
 423                                    comma-separated list.""")
 424         group.add_argument('--output-all-postcodes', action='store_true',
 425                            help="""List all postcodes for address instead of
 426                                    just the most likely one""")
 427         group.add_argument('--language',
 428                            help="""Preferred language for output
 429                                    (use local name, if omitted)""")
 430         group = parser.add_argument_group('Filter arguments')
 431         group.add_argument('--restrict-to-country', metavar='COUNTRY_CODE',
 432                            help='Export only objects within country')
 433         group.add_argument('--restrict-to-osm-node', metavar='ID', type=int,
 434                            help='Export only children of this OSM node')
 435         group.add_argument('--restrict-to-osm-way', metavar='ID', type=int,
 436                            help='Export only children of this OSM way')
 437         group.add_argument('--restrict-to-osm-relation', metavar='ID', type=int,
 438                            help='Export only children of this OSM relation')
 439
 440
 441     @staticmethod
 442     def run(args):
 443         params = ['export.php',
 444                   '--output-type', args.output_type,
 445                   '--output-format', args.output_format]
 446         if args.output_all_postcodes:
 447             params.append('--output-all-postcodes')
 448         if args.language:
 449             params.extend(('--language', args.language))
 450         if args.restrict_to_country:
 451             params.extend(('--restrict-to-country', args.restrict_to_country))
 452         if args.restrict_to_osm_node:
 453             params.extend(('--restrict-to-osm-node', args.restrict_to_osm_node))
 454         if args.restrict_to_osm_way:
 455             params.extend(('--restrict-to-osm-way', args.restrict_to_osm_way))
 456         if args.restrict_to_osm_relation:
 457             params.extend(('--restrict-to-osm-relation', args.restrict_to_osm_relation))
 458
 459         return run_legacy_script(*params, nominatim_env=args)
 460
 461 class QueryTodo:
 462     """\
 463     Todo
 464     """
 465     @staticmethod
 466     def add_args(parser):
 467         pass
 468
 469     @staticmethod
 470     def run(args): # pylint: disable=W0613
 471         print("TODO: searching")
 472
 473
 474 def nominatim(**kwargs):
 475     """\
 476     Command-line tools for importing, updating, administrating and
 477     querying the Nominatim database.
 478     """
 479     parser = CommandlineParser('nominatim', nominatim.__doc__)
 480
 481     parser.add_subcommand('import', SetupAll)
 482     parser.add_subcommand('freeze', SetupFreeze)
 483     parser.add_subcommand('replication', UpdateReplication)
 484
 485     parser.add_subcommand('check-database', AdminCheckDatabase)
 486     parser.add_subcommand('warm', AdminWarm)
 487
 488     parser.add_subcommand('special-phrases', SetupSpecialPhrases)
 489
 490     parser.add_subcommand('add-data', UpdateAddData)
 491     parser.add_subcommand('index', UpdateIndex)
 492     parser.add_subcommand('refresh', UpdateRefresh)
 493
 494     parser.add_subcommand('export', QueryExport)
 495     parser.add_subcommand('search', QueryTodo)
 496     parser.add_subcommand('reverse', QueryTodo)
 497     parser.add_subcommand('lookup', QueryTodo)
 498     parser.add_subcommand('details', QueryTodo)
 499     parser.add_subcommand('status', QueryTodo)
 500
 501     return parser.run(**kwargs)