nominatim/cli.py

   1 """
   2 Command-line interface to the Nominatim functions for import, update,
   3 database administration and querying.
   4 """
   5 import logging
   6 import os
   7 import sys
   8 import argparse
   9 from pathlib import Path
  10
  11 from .config import Configuration
  12 from .tools.exec_utils import run_legacy_script, run_php_server
  13 from .errors import UsageError
  14 from . import clicmd
  15
  16 LOG = logging.getLogger()
  17
  18
  19 class CommandlineParser:
  20     """ Wraps some of the common functions for parsing the command line
  21         and setting up subcommands.
  22     """
  23     def __init__(self, prog, description):
  24         self.parser = argparse.ArgumentParser(
  25             prog=prog,
  26             description=description,
  27             formatter_class=argparse.RawDescriptionHelpFormatter)
  28
  29         self.subs = self.parser.add_subparsers(title='available commands',
  30                                                dest='subcommand')
  31
  32         # Arguments added to every sub-command
  33         self.default_args = argparse.ArgumentParser(add_help=False)
  34         group = self.default_args.add_argument_group('Default arguments')
  35         group.add_argument('-h', '--help', action='help',
  36                            help='Show this help message and exit')
  37         group.add_argument('-q', '--quiet', action='store_const', const=0,
  38                            dest='verbose', default=1,
  39                            help='Print only error messages')
  40         group.add_argument('-v', '--verbose', action='count', default=1,
  41                            help='Increase verboseness of output')
  42         group.add_argument('--project-dir', metavar='DIR', default='.',
  43                            help='Base directory of the Nominatim installation (default:.)')
  44         group.add_argument('-j', '--threads', metavar='NUM', type=int,
  45                            help='Number of parallel threads to use')
  46
  47
  48     def add_subcommand(self, name, cmd):
  49         """ Add a subcommand to the parser. The subcommand must be a class
  50             with a function add_args() that adds the parameters for the
  51             subcommand and a run() function that executes the command.
  52         """
  53         parser = self.subs.add_parser(name, parents=[self.default_args],
  54                                       help=cmd.__doc__.split('\n', 1)[0],
  55                                       description=cmd.__doc__,
  56                                       formatter_class=argparse.RawDescriptionHelpFormatter,
  57                                       add_help=False)
  58         parser.set_defaults(command=cmd)
  59         cmd.add_args(parser)
  60
  61     def run(self, **kwargs):
  62         """ Parse the command line arguments of the program and execute the
  63             appropriate subcommand.
  64         """
  65         args = self.parser.parse_args(args=kwargs.get('cli_args'))
  66
  67         if args.subcommand is None:
  68             self.parser.print_help()
  69             return 1
  70
  71         for arg in ('module_dir', 'osm2pgsql_path', 'phplib_dir', 'sqllib_dir',
  72                     'data_dir', 'config_dir', 'phpcgi_path'):
  73             setattr(args, arg, Path(kwargs[arg]))
  74         args.project_dir = Path(args.project_dir).resolve()
  75
  76         logging.basicConfig(stream=sys.stderr,
  77                             format='%(asctime)s: %(message)s',
  78                             datefmt='%Y-%m-%d %H:%M:%S',
  79                             level=max(4 - args.verbose, 1) * 10)
  80
  81         args.config = Configuration(args.project_dir, args.config_dir)
  82
  83         log = logging.getLogger()
  84         log.warning('Using project directory: %s', str(args.project_dir))
  85
  86         try:
  87             return args.command.run(args)
  88         except UsageError as exception:
  89             if log.isEnabledFor(logging.DEBUG):
  90                 raise # use Python's exception printing
  91             log.fatal('FATAL: %s', exception)
  92
  93         # If we get here, then execution has failed in some way.
  94         return 1
  95
  96
  97 ##### Subcommand classes
  98 #
  99 # Each class needs to implement two functions: add_args() adds the CLI parameters
 100 # for the subfunction, run() executes the subcommand.
 101 #
 102 # The class documentation doubles as the help text for the command. The
 103 # first line is also used in the summary when calling the program without
 104 # a subcommand.
 105 #
 106 # No need to document the functions each time.
 107 # pylint: disable=C0111
 108 # Using non-top-level imports to make pyosmium optional for replication only.
 109 # pylint: disable=E0012,C0415
 110
 111
 112 class SetupAll:
 113     """\
 114     Create a new Nominatim database from an OSM file.
 115     """
 116
 117     @staticmethod
 118     def add_args(parser):
 119         group_name = parser.add_argument_group('Required arguments')
 120         group = group_name.add_mutually_exclusive_group(required=True)
 121         group.add_argument('--osm-file',
 122                            help='OSM file to be imported.')
 123         group.add_argument('--continue', dest='continue_at',
 124                            choices=['load-data', 'indexing', 'db-postprocess'],
 125                            help='Continue an import that was interrupted')
 126         group = parser.add_argument_group('Optional arguments')
 127         group.add_argument('--osm2pgsql-cache', metavar='SIZE', type=int,
 128                            help='Size of cache to be used by osm2pgsql (in MB)')
 129         group.add_argument('--reverse-only', action='store_true',
 130                            help='Do not create tables and indexes for searching')
 131         group.add_argument('--enable-debug-statements', action='store_true',
 132                            help='Include debug warning statements in SQL code')
 133         group.add_argument('--no-partitions', action='store_true',
 134                            help="""Do not partition search indices
 135                                    (speeds up import of single country extracts)""")
 136         group.add_argument('--no-updates', action='store_true',
 137                            help="""Do not keep tables that are only needed for
 138                                    updating the database later""")
 139         group = parser.add_argument_group('Expert options')
 140         group.add_argument('--ignore-errors', action='store_true',
 141                            help='Continue import even when errors in SQL are present')
 142         group.add_argument('--index-noanalyse', action='store_true',
 143                            help='Do not perform analyse operations during index')
 144
 145
 146     @staticmethod
 147     def run(args):
 148         params = ['setup.php']
 149         if args.osm_file:
 150             params.extend(('--all', '--osm-file', args.osm_file))
 151         else:
 152             if args.continue_at == 'load-data':
 153                 params.append('--load-data')
 154             if args.continue_at in ('load-data', 'indexing'):
 155                 params.append('--index')
 156             params.extend(('--create-search-indices', '--create-country-names',
 157                            '--setup-website'))
 158         if args.osm2pgsql_cache:
 159             params.extend(('--osm2pgsql-cache', args.osm2pgsql_cache))
 160         if args.reverse_only:
 161             params.append('--reverse-only')
 162         if args.enable_debug_statements:
 163             params.append('--enable-debug-statements')
 164         if args.no_partitions:
 165             params.append('--no-partitions')
 166         if args.no_updates:
 167             params.append('--drop')
 168         if args.ignore_errors:
 169             params.append('--ignore-errors')
 170         if args.index_noanalyse:
 171             params.append('--index-noanalyse')
 172
 173         return run_legacy_script(*params, nominatim_env=args)
 174
 175
 176 class SetupSpecialPhrases:
 177     """\
 178     Maintain special phrases.
 179     """
 180
 181     @staticmethod
 182     def add_args(parser):
 183         group = parser.add_argument_group('Input arguments')
 184         group.add_argument('--from-wiki', action='store_true',
 185                            help='Pull special phrases from the OSM wiki.')
 186         group = parser.add_argument_group('Output arguments')
 187         group.add_argument('-o', '--output', default='-',
 188                            help="""File to write the preprocessed phrases to.
 189                                    If omitted, it will be written to stdout.""")
 190
 191     @staticmethod
 192     def run(args):
 193         if args.output != '-':
 194             raise NotImplementedError('Only output to stdout is currently implemented.')
 195         return run_legacy_script('specialphrases.php', '--wiki-import', nominatim_env=args)
 196
 197
 198 class UpdateAddData:
 199     """\
 200     Add additional data from a file or an online source.
 201
 202     Data is only imported, not indexed. You need to call `nominatim-update index`
 203     to complete the process.
 204     """
 205
 206     @staticmethod
 207     def add_args(parser):
 208         group_name = parser.add_argument_group('Source')
 209         group = group_name.add_mutually_exclusive_group(required=True)
 210         group.add_argument('--file', metavar='FILE',
 211                            help='Import data from an OSM file')
 212         group.add_argument('--diff', metavar='FILE',
 213                            help='Import data from an OSM diff file')
 214         group.add_argument('--node', metavar='ID', type=int,
 215                            help='Import a single node from the API')
 216         group.add_argument('--way', metavar='ID', type=int,
 217                            help='Import a single way from the API')
 218         group.add_argument('--relation', metavar='ID', type=int,
 219                            help='Import a single relation from the API')
 220         group.add_argument('--tiger-data', metavar='DIR',
 221                            help='Add housenumbers from the US TIGER census database.')
 222         group = parser.add_argument_group('Extra arguments')
 223         group.add_argument('--use-main-api', action='store_true',
 224                            help='Use OSM API instead of Overpass to download objects')
 225
 226     @staticmethod
 227     def run(args):
 228         if args.tiger_data:
 229             os.environ['NOMINATIM_TIGER_DATA_PATH'] = args.tiger_data
 230             return run_legacy_script('setup.php', '--import-tiger-data', nominatim_env=args)
 231
 232         params = ['update.php']
 233         if args.file:
 234             params.extend(('--import-file', args.file))
 235         elif args.diff:
 236             params.extend(('--import-diff', args.diff))
 237         elif args.node:
 238             params.extend(('--import-node', args.node))
 239         elif args.way:
 240             params.extend(('--import-way', args.way))
 241         elif args.relation:
 242             params.extend(('--import-relation', args.relation))
 243         if args.use_main_api:
 244             params.append('--use-main-api')
 245         return run_legacy_script(*params, nominatim_env=args)
 246
 247
 248 class QueryExport:
 249     """\
 250     Export addresses as CSV file from the database.
 251     """
 252
 253     @staticmethod
 254     def add_args(parser):
 255         group = parser.add_argument_group('Output arguments')
 256         group.add_argument('--output-type', default='street',
 257                            choices=('continent', 'country', 'state', 'county',
 258                                     'city', 'suburb', 'street', 'path'),
 259                            help='Type of places to output (default: street)')
 260         group.add_argument('--output-format',
 261                            default='street;suburb;city;county;state;country',
 262                            help="""Semicolon-separated list of address types
 263                                    (see --output-type). Multiple ranks can be
 264                                    merged into one column by simply using a
 265                                    comma-separated list.""")
 266         group.add_argument('--output-all-postcodes', action='store_true',
 267                            help="""List all postcodes for address instead of
 268                                    just the most likely one""")
 269         group.add_argument('--language',
 270                            help="""Preferred language for output
 271                                    (use local name, if omitted)""")
 272         group = parser.add_argument_group('Filter arguments')
 273         group.add_argument('--restrict-to-country', metavar='COUNTRY_CODE',
 274                            help='Export only objects within country')
 275         group.add_argument('--restrict-to-osm-node', metavar='ID', type=int,
 276                            help='Export only children of this OSM node')
 277         group.add_argument('--restrict-to-osm-way', metavar='ID', type=int,
 278                            help='Export only children of this OSM way')
 279         group.add_argument('--restrict-to-osm-relation', metavar='ID', type=int,
 280                            help='Export only children of this OSM relation')
 281
 282
 283     @staticmethod
 284     def run(args):
 285         params = ['export.php',
 286                   '--output-type', args.output_type,
 287                   '--output-format', args.output_format]
 288         if args.output_all_postcodes:
 289             params.append('--output-all-postcodes')
 290         if args.language:
 291             params.extend(('--language', args.language))
 292         if args.restrict_to_country:
 293             params.extend(('--restrict-to-country', args.restrict_to_country))
 294         if args.restrict_to_osm_node:
 295             params.extend(('--restrict-to-osm-node', args.restrict_to_osm_node))
 296         if args.restrict_to_osm_way:
 297             params.extend(('--restrict-to-osm-way', args.restrict_to_osm_way))
 298         if args.restrict_to_osm_relation:
 299             params.extend(('--restrict-to-osm-relation', args.restrict_to_osm_relation))
 300
 301         return run_legacy_script(*params, nominatim_env=args)
 302
 303
 304 class AdminServe:
 305     """\
 306     Start a simple web server for serving the API.
 307
 308     This command starts the built-in PHP webserver to serve the website
 309     from the current project directory. This webserver is only suitable
 310     for testing and develop. Do not use it in production setups!
 311
 312     By the default, the webserver can be accessed at: http://127.0.0.1:8088
 313     """
 314
 315     @staticmethod
 316     def add_args(parser):
 317         group = parser.add_argument_group('Server arguments')
 318         group.add_argument('--server', default='127.0.0.1:8088',
 319                            help='The address the server will listen to.')
 320
 321     @staticmethod
 322     def run(args):
 323         run_php_server(args.server, args.project_dir / 'website')
 324
 325
 326 def nominatim(**kwargs):
 327     """\
 328     Command-line tools for importing, updating, administrating and
 329     querying the Nominatim database.
 330     """
 331     parser = CommandlineParser('nominatim', nominatim.__doc__)
 332
 333     parser.add_subcommand('import', SetupAll)
 334     parser.add_subcommand('freeze', clicmd.SetupFreeze)
 335     parser.add_subcommand('replication', clicmd.UpdateReplication)
 336
 337     parser.add_subcommand('special-phrases', SetupSpecialPhrases)
 338
 339     parser.add_subcommand('add-data', UpdateAddData)
 340     parser.add_subcommand('index', clicmd.UpdateIndex)
 341     parser.add_subcommand('refresh', clicmd.UpdateRefresh)
 342
 343     parser.add_subcommand('admin', clicmd.AdminFuncs)
 344
 345     parser.add_subcommand('export', QueryExport)
 346     parser.add_subcommand('serve', AdminServe)
 347
 348     if kwargs.get('phpcgi_path'):
 349         parser.add_subcommand('search', clicmd.APISearch)
 350         parser.add_subcommand('reverse', clicmd.APIReverse)
 351         parser.add_subcommand('lookup', clicmd.APILookup)
 352         parser.add_subcommand('details', clicmd.APIDetails)
 353         parser.add_subcommand('status', clicmd.APIStatus)
 354     else:
 355         parser.parser.epilog = 'php-cgi not found. Query commands not available.'
 356
 357     parser.add_subcommand('transition', clicmd.AdminTransition)
 358
 359     return parser.run(**kwargs)