nominatim/cli.py

   1 """
   2 Command-line interface to the Nominatim functions for import, update,
   3 database administration and querying.
   4 """
   5 import sys
   6 import argparse
   7 import logging
   8 from pathlib import Path
   9
  10 from .config import Configuration
  11 from .admin.exec_utils import run_legacy_script
  12
  13 class CommandlineParser:
  14     """ Wraps some of the common functions for parsing the command line
  15         and setting up subcommands.
  16     """
  17     def __init__(self, prog, description):
  18         self.parser = argparse.ArgumentParser(
  19             prog=prog,
  20             description=description,
  21             formatter_class=argparse.RawDescriptionHelpFormatter)
  22
  23         self.subs = self.parser.add_subparsers(title='available commands',
  24                                                dest='subcommand')
  25
  26         # Arguments added to every sub-command
  27         self.default_args = argparse.ArgumentParser(add_help=False)
  28         group = self.default_args.add_argument_group('Default arguments')
  29         group.add_argument('-h', '--help', action='help',
  30                            help='Show this help message and exit')
  31         group.add_argument('-q', '--quiet', action='store_const', const=0,
  32                            dest='verbose', default=1,
  33                            help='Print only error messages')
  34         group.add_argument('-v', '--verbose', action='count', default=1,
  35                            help='Increase verboseness of output')
  36         group.add_argument('--project-dir', metavar='DIR', default='.',
  37                            help='Base directory of the Nominatim installation (default:.)')
  38         group.add_argument('-j', '--threads', metavar='NUM', type=int,
  39                            help='Number of parallel threads to use')
  40
  41
  42     def add_subcommand(self, name, cmd):
  43         """ Add a subcommand to the parser. The subcommand must be a class
  44             with a function add_args() that adds the parameters for the
  45             subcommand and a run() function that executes the command.
  46         """
  47         parser = self.subs.add_parser(name, parents=[self.default_args],
  48                                       help=cmd.__doc__.split('\n', 1)[0],
  49                                       description=cmd.__doc__,
  50                                       formatter_class=argparse.RawDescriptionHelpFormatter,
  51                                       add_help=False)
  52         parser.set_defaults(command=cmd)
  53         cmd.add_args(parser)
  54
  55     def run(self, **kwargs):
  56         """ Parse the command line arguments of the program and execute the
  57             appropriate subcommand.
  58         """
  59         args = self.parser.parse_args()
  60
  61         if args.subcommand is None:
  62             return self.parser.print_help()
  63
  64         for arg in ('module_dir', 'osm2pgsql_path', 'phplib_dir', 'data_dir'):
  65             setattr(args, arg, Path(kwargs[arg]))
  66         args.project_dir = Path(args.project_dir)
  67
  68         logging.basicConfig(stream=sys.stderr,
  69                             format='%(asctime)s %(levelname)s: %(message)s',
  70                             datefmt='%Y-%m-%d %H:%M:%S',
  71                             level=max(4 - args.verbose, 1) * 10)
  72
  73         args.config = Configuration(args.project_dir, args.data_dir / 'settings')
  74
  75         return args.command.run(args)
  76
  77 ##### Subcommand classes
  78 #
  79 # Each class needs to implement two functions: add_args() adds the CLI parameters
  80 # for the subfunction, run() executes the subcommand.
  81 #
  82 # The class documentation doubles as the help text for the command. The
  83 # first line is also used in the summary when calling the program without
  84 # a subcommand.
  85 #
  86 # No need to document the functions each time.
  87 # pylint: disable=C0111
  88
  89
  90 class SetupAll:
  91     """\
  92     Create a new Nominatim database from an OSM file.
  93     """
  94
  95     @staticmethod
  96     def add_args(parser):
  97         group_name = parser.add_argument_group('Required arguments')
  98         group = group_name.add_mutually_exclusive_group(required=True)
  99         group.add_argument('--osm-file',
 100                            help='OSM file to be imported.')
 101         group.add_argument('--continue', nargs=1, dest='continue_at',
 102                            choices=['load-data', 'indexing', 'db-postprocess'],
 103                            help='Continue an import that was interrupted')
 104         group = parser.add_argument_group('Optional arguments')
 105         group.add_argument('--osm2pgsql-cache', metavar='SIZE', type=int,
 106                            help='Size of cache to be used by osm2pgsql (in MB)')
 107         group.add_argument('--reverse-only', action='store_true',
 108                            help='Do not create tables and indexes for searching')
 109         group.add_argument('--enable-debug-statements', action='store_true',
 110                            help='Include debug warning statements in SQL code')
 111         group.add_argument('--no-partitions', action='store_true',
 112                            help="""Do not partition search indices
 113                                    (speeds up import of single country extracts)""")
 114         group.add_argument('--no-updates', action='store_true',
 115                            help="""Do not keep tables that are only needed for
 116                                    updating the database later""")
 117         group = parser.add_argument_group('Expert options')
 118         group.add_argument('--ignore-errors', action='store_true',
 119                            help='Continue import even when errors in SQL are present')
 120         group.add_argument('--index-noanalyse', action='store_true',
 121                            help='Do not perform analyse operations during index')
 122
 123
 124     @staticmethod
 125     def run(args):
 126         params = ['setup.php']
 127         if args.osm_file:
 128             params.extend(('--all', '--osm-file', args.osm_file))
 129         else:
 130             if args.continue_at == 'load-data':
 131                 params.append('--load-data')
 132             if args.continue_at in ('load-data', 'indexing'):
 133                 params.append('--index')
 134             params.extend(('--create-search-indices', '--create-country-names',
 135                            '--setup-website'))
 136         if args.osm2pgsql_cache:
 137             params.extend(('--osm2pgsql-cache', args.osm2pgsql_cache))
 138         if args.reverse_only:
 139             params.append('--reverse-only')
 140         if args.enable_debug_statements:
 141             params.append('--enable-debug-statements')
 142         if args.no_partitions:
 143             params.append('--no-partitions')
 144         if args.no_updates:
 145             params.append('--drop')
 146         if args.ignore_errors:
 147             params.append('--ignore-errors')
 148         if args.index_noanalyse:
 149             params.append('--index-noanalyse')
 150
 151         return run_legacy_script(*params, nominatim_env=args)
 152
 153
 154 class SetupFreeze:
 155     """\
 156     Make database read-only.
 157
 158     About half of data in the Nominatim database is kept only to be able to
 159     keep the data up-to-date with new changes made in OpenStreetMap. This
 160     command drops all this data and only keeps the part needed for geocoding
 161     itself.
 162
 163     This command has the same effect as the `--no-updates` option for imports.
 164     """
 165
 166     @staticmethod
 167     def add_args(parser):
 168         pass # No options
 169
 170     @staticmethod
 171     def run(args):
 172         return run_legacy_script('setup.php', '--drop', nominatim_env=args)
 173
 174
 175 class SetupSpecialPhrases:
 176     """\
 177     Maintain special phrases.
 178     """
 179
 180     @staticmethod
 181     def add_args(parser):
 182         group = parser.add_argument_group('Input arguments')
 183         group.add_argument('--from-wiki', action='store_true',
 184                            help='Pull special phrases from the OSM wiki.')
 185         group = parser.add_argument_group('Output arguments')
 186         group.add_argument('-o', '--output', default='-',
 187                            type=argparse.FileType('w', encoding='UTF-8'),
 188                            help="""File to write the preprocessed phrases to.
 189                                    If omitted, it will be written to stdout.""")
 190
 191     @staticmethod
 192     def run(args):
 193         if args.output.name != '<stdout>':
 194             raise NotImplementedError('Only output to stdout is currently implemented.')
 195         return run_legacy_script('specialphrases.php', '--wiki-import', nominatim_env=args)
 196
 197
 198 class UpdateReplication:
 199     """\
 200     Update the database using an online replication service.
 201     """
 202
 203     @staticmethod
 204     def add_args(parser):
 205         group = parser.add_argument_group('Arguments for initialisation')
 206         group.add_argument('--init', action='store_true',
 207                            help='Initialise the update process')
 208         group.add_argument('--no-update-functions', dest='update_functions',
 209                            action='store_false',
 210                            help="""Do not update the trigger function to
 211                                    support differential updates.""")
 212         group = parser.add_argument_group('Arguments for updates')
 213         group.add_argument('--check-for-updates', action='store_true',
 214                            help='Check if new updates are available and exit')
 215         group.add_argument('--once', action='store_true',
 216                            help="""Download and apply updates only once. When
 217                                    not set, updates are continuously applied""")
 218         group.add_argument('--no-index', action='store_false', dest='do_index',
 219                            help="""Do not index the new data. Only applicable
 220                                    together with --once""")
 221
 222     @staticmethod
 223     def run(args):
 224         params = ['update.php']
 225         if args.init:
 226             params.append('--init-updates')
 227             if not args.update_functions:
 228                 params.append('--no-update-functions')
 229         elif args.check_for_updates:
 230             params.append('--check-for-updates')
 231         else:
 232             if args.once:
 233                 params.append('--import-osmosis')
 234             else:
 235                 params.append('--import-osmosis-all')
 236             if not args.do_index:
 237                 params.append('--no-index')
 238
 239         return run_legacy_script(*params, nominatim_env=args)
 240
 241
 242 class UpdateAddData:
 243     """\
 244     Add additional data from a file or an online source.
 245
 246     Data is only imported, not indexed. You need to call `nominatim-update index`
 247     to complete the process.
 248     """
 249
 250     @staticmethod
 251     def add_args(parser):
 252         group_name = parser.add_argument_group('Source')
 253         group = group_name.add_mutually_exclusive_group(required=True)
 254         group.add_argument('--file', metavar='FILE',
 255                            help='Import data from an OSM file')
 256         group.add_argument('--diff', metavar='FILE',
 257                            help='Import data from an OSM diff file')
 258         group.add_argument('--node', metavar='ID', type=int,
 259                            help='Import a single node from the API')
 260         group.add_argument('--way', metavar='ID', type=int,
 261                            help='Import a single way from the API')
 262         group.add_argument('--relation', metavar='ID', type=int,
 263                            help='Import a single relation from the API')
 264         group.add_argument('--tiger-data', metavar='DIR',
 265                            help='Add housenumbers from the US TIGER census database.')
 266         group = parser.add_argument_group('Extra arguments')
 267         group.add_argument('--use-main-api', action='store_true',
 268                            help='Use OSM API instead of Overpass to download objects')
 269
 270     @staticmethod
 271     def run(args):
 272         if args.tiger_data:
 273             return run_legacy_script('setup.php', '--import-tiger-data', nominatim_env=args)
 274
 275         params = ['update.php']
 276         if args.file:
 277             params.extend(('--import-file', args.file))
 278         elif args.diff:
 279             params.extend(('--import-diff', args.diff))
 280         elif args.node:
 281             params.extend(('--import-node', args.node))
 282         elif args.way:
 283             params.extend(('--import-way', args.way))
 284         elif args.relation:
 285             params.extend(('--import-relation', args.relation))
 286         if args.use_main_api:
 287             params.append('--use-main-api')
 288         return run_legacy_script(*params, nominatim_env=args)
 289
 290
 291 class UpdateIndex:
 292     """\
 293     Reindex all new and modified data.
 294     """
 295
 296     @staticmethod
 297     def add_args(parser):
 298         pass
 299
 300     @staticmethod
 301     def run(args):
 302         return run_legacy_script('update.php', '--index', nominatim_env=args)
 303
 304
 305 class UpdateRefresh:
 306     """\
 307     Recompute auxillary data used by the indexing process.
 308
 309     These functions must not be run in parallel with other update commands.
 310     """
 311
 312     @staticmethod
 313     def add_args(parser):
 314         group = parser.add_argument_group('Data arguments')
 315         group.add_argument('--postcodes', action='store_true',
 316                            help='Update postcode centroid table')
 317         group.add_argument('--word-counts', action='store_true',
 318                            help='Compute frequency of full-word search terms')
 319         group.add_argument('--address-levels', action='store_true',
 320                            help='Reimport address level configuration')
 321         group.add_argument('--importance', action='store_true',
 322                            help='Recompute place importances (expensive!)')
 323         group.add_argument('--functions', action='store_true',
 324                            help='Update the PL/pgSQL functions in the database')
 325         group.add_argument('--wiki-data', action='store_true',
 326                            help='Update Wikipedia/data importance numbers.')
 327         group.add_argument('--website', action='store_true',
 328                            help='Refresh the directory that serves the scripts for the web API')
 329         group = parser.add_argument_group('Arguments for function refresh')
 330         group.add_argument('--no-diff-updates', action='store_false', dest='diffs',
 331                            help='Do not enable code for propagating updates')
 332         group.add_argument('--enable-debug-statements', action='store_true',
 333                            help='Enable debug warning statements in functions')
 334
 335     @staticmethod
 336     def run(args):
 337         if args.postcodes:
 338             run_legacy_script('update.php', '--calculate-postcodes',
 339                               nominatim_env=args, throw_on_fail=True)
 340         if args.word_counts:
 341             run_legacy_script('update.php', '--recompute-word-counts',
 342                               nominatim_env=args, throw_on_fail=True)
 343         if args.address_levels:
 344             run_legacy_script('update.php', '--update-address-levels',
 345                               nominatim_env=args, throw_on_fail=True)
 346         if args.importance:
 347             run_legacy_script('update.php', '--recompute-importance',
 348                               nominatim_env=args, throw_on_fail=True)
 349         if args.functions:
 350             params = ['setup.php', '--create-functions', '--create-partition-functions']
 351             if args.diffs:
 352                 params.append('--enable-diff-updates')
 353             if args.enable_debug_statements:
 354                 params.append('--enable-debug-statements')
 355             run_legacy_script(*params, nominatim_env=args, throw_on_fail=True)
 356         if args.wiki_data:
 357             run_legacy_script('setup.php', '--import-wikipedia-articles',
 358                               nominatim_env=args, throw_on_fail=True)
 359         if args.website:
 360             run_legacy_script('setup.php', '--setup-website',
 361                               nominatim_env=args, throw_on_fail=True)
 362
 363
 364 class AdminCheckDatabase:
 365     """\
 366     Check that the database is complete and operational.
 367     """
 368
 369     @staticmethod
 370     def add_args(parser):
 371         pass # No options
 372
 373     @staticmethod
 374     def run(args):
 375         return run_legacy_script('check_import_finished.php', nominatim_env=args)
 376
 377
 378 class AdminWarm:
 379     """\
 380     Warm database caches for search and reverse queries.
 381     """
 382
 383     @staticmethod
 384     def add_args(parser):
 385         group = parser.add_argument_group('Target arguments')
 386         group.add_argument('--search-only', action='store_const', dest='target',
 387                            const='search',
 388                            help="Only pre-warm tables for search queries")
 389         group.add_argument('--reverse-only', action='store_const', dest='target',
 390                            const='reverse',
 391                            help="Only pre-warm tables for reverse queries")
 392
 393     @staticmethod
 394     def run(args):
 395         params = ['warm.php']
 396         if args.target == 'reverse':
 397             params.append('--reverse-only')
 398         if args.target == 'search':
 399             params.append('--search-only')
 400         return run_legacy_script(*params, nominatim_env=args)
 401
 402
 403 class QueryExport:
 404     """\
 405     Export addresses as CSV file from a Nominatim database.
 406     """
 407
 408     @staticmethod
 409     def add_args(parser):
 410         group = parser.add_argument_group('Output arguments')
 411         group.add_argument('--output-type', default='street',
 412                            choices=('continent', 'country', 'state', 'county',
 413                                     'city', 'suburb', 'street', 'path'),
 414                            help='Type of places to output (default: street)')
 415         group.add_argument('--output-format',
 416                            default='street;suburb;city;county;state;country',
 417                            help="""Semicolon-separated list of address types
 418                                    (see --output-type). Multiple ranks can be
 419                                    merged into one column by simply using a
 420                                    comma-separated list.""")
 421         group.add_argument('--output-all-postcodes', action='store_true',
 422                            help="""List all postcodes for address instead of
 423                                    just the most likely one""")
 424         group.add_argument('--language',
 425                            help="""Preferred language for output
 426                                    (use local name, if omitted)""")
 427         group = parser.add_argument_group('Filter arguments')
 428         group.add_argument('--restrict-to-country', metavar='COUNTRY_CODE',
 429                            help='Export only objects within country')
 430         group.add_argument('--restrict-to-osm-node', metavar='ID', type=int,
 431                            help='Export only children of this OSM node')
 432         group.add_argument('--restrict-to-osm-way', metavar='ID', type=int,
 433                            help='Export only children of this OSM way')
 434         group.add_argument('--restrict-to-osm-relation', metavar='ID', type=int,
 435                            help='Export only children of this OSM relation')
 436
 437
 438     @staticmethod
 439     def run(args):
 440         params = ['export.php',
 441                   '--output-type', args.output_type,
 442                   '--output-format', args.output_format]
 443         if args.output_all_postcodes:
 444             params.append('--output-all-postcodes')
 445         if args.language:
 446             params.extend(('--language', args.language))
 447         if args.restrict_to_country:
 448             params.extend(('--restrict-to-country', args.restrict_to_country))
 449         if args.restrict_to_osm_node:
 450             params.extend(('--restrict-to-osm-node', args.restrict_to_osm_node))
 451         if args.restrict_to_osm_way:
 452             params.extend(('--restrict-to-osm-way', args.restrict_to_osm_way))
 453         if args.restrict_to_osm_relation:
 454             params.extend(('--restrict-to-osm-relation', args.restrict_to_osm_relation))
 455
 456         return run_legacy_script(*params, nominatim_env=args)
 457
 458 class QueryTodo:
 459     """\
 460     Todo
 461     """
 462     @staticmethod
 463     def add_args(parser):
 464         pass
 465
 466     @staticmethod
 467     def run(args): # pylint: disable=W0613
 468         print("TODO: searching")
 469
 470
 471 def nominatim(**kwargs):
 472     """\
 473     Command-line tools for importing, updating, administrating and
 474     querying the Nominatim database.
 475     """
 476     parser = CommandlineParser('nominatim', nominatim.__doc__)
 477
 478     parser.add_subcommand('import', SetupAll)
 479     parser.add_subcommand('freeze', SetupFreeze)
 480     parser.add_subcommand('replication', UpdateReplication)
 481
 482     parser.add_subcommand('check-database', AdminCheckDatabase)
 483     parser.add_subcommand('warm', AdminWarm)
 484
 485     parser.add_subcommand('special-phrases', SetupSpecialPhrases)
 486
 487     parser.add_subcommand('add-data', UpdateAddData)
 488     parser.add_subcommand('index', UpdateIndex)
 489     parser.add_subcommand('refresh', UpdateRefresh)
 490
 491     parser.add_subcommand('export', QueryExport)
 492     parser.add_subcommand('search', QueryTodo)
 493     parser.add_subcommand('reverse', QueryTodo)
 494     parser.add_subcommand('lookup', QueryTodo)
 495     parser.add_subcommand('details', QueryTodo)
 496     parser.add_subcommand('status', QueryTodo)
 497
 498     return parser.run(**kwargs)