nominatim/cli.py

   1 """
   2 Command-line interface to the Nominatim functions for import, update,
   3 database administration and querying.
   4 """
   5 import logging
   6 import os
   7 import sys
   8 import argparse
   9 from pathlib import Path
  10
  11 from .config import Configuration
  12 from .tools.exec_utils import run_legacy_script, run_php_server
  13 from .errors import UsageError
  14 from . import clicmd
  15 from .clicmd.args import NominatimArgs
  16
  17 LOG = logging.getLogger()
  18
  19
  20 class CommandlineParser:
  21     """ Wraps some of the common functions for parsing the command line
  22         and setting up subcommands.
  23     """
  24     def __init__(self, prog, description):
  25         self.parser = argparse.ArgumentParser(
  26             prog=prog,
  27             description=description,
  28             formatter_class=argparse.RawDescriptionHelpFormatter)
  29
  30         self.subs = self.parser.add_subparsers(title='available commands',
  31                                                dest='subcommand')
  32
  33         # Arguments added to every sub-command
  34         self.default_args = argparse.ArgumentParser(add_help=False)
  35         group = self.default_args.add_argument_group('Default arguments')
  36         group.add_argument('-h', '--help', action='help',
  37                            help='Show this help message and exit')
  38         group.add_argument('-q', '--quiet', action='store_const', const=0,
  39                            dest='verbose', default=1,
  40                            help='Print only error messages')
  41         group.add_argument('-v', '--verbose', action='count', default=1,
  42                            help='Increase verboseness of output')
  43         group.add_argument('--project-dir', metavar='DIR', default='.',
  44                            help='Base directory of the Nominatim installation (default:.)')
  45         group.add_argument('-j', '--threads', metavar='NUM', type=int,
  46                            help='Number of parallel threads to use')
  47
  48
  49     def add_subcommand(self, name, cmd):
  50         """ Add a subcommand to the parser. The subcommand must be a class
  51             with a function add_args() that adds the parameters for the
  52             subcommand and a run() function that executes the command.
  53         """
  54         parser = self.subs.add_parser(name, parents=[self.default_args],
  55                                       help=cmd.__doc__.split('\n', 1)[0],
  56                                       description=cmd.__doc__,
  57                                       formatter_class=argparse.RawDescriptionHelpFormatter,
  58                                       add_help=False)
  59         parser.set_defaults(command=cmd)
  60         cmd.add_args(parser)
  61
  62     def run(self, **kwargs):
  63         """ Parse the command line arguments of the program and execute the
  64             appropriate subcommand.
  65         """
  66         args = NominatimArgs()
  67         self.parser.parse_args(args=kwargs.get('cli_args'), namespace=args)
  68
  69         if args.subcommand is None:
  70             self.parser.print_help()
  71             return 1
  72
  73         for arg in ('module_dir', 'osm2pgsql_path', 'phplib_dir', 'sqllib_dir',
  74                     'data_dir', 'config_dir', 'phpcgi_path'):
  75             setattr(args, arg, Path(kwargs[arg]))
  76         args.project_dir = Path(args.project_dir).resolve()
  77
  78         logging.basicConfig(stream=sys.stderr,
  79                             format='%(asctime)s: %(message)s',
  80                             datefmt='%Y-%m-%d %H:%M:%S',
  81                             level=max(4 - args.verbose, 1) * 10)
  82
  83         args.config = Configuration(args.project_dir, args.config_dir)
  84
  85         log = logging.getLogger()
  86         log.warning('Using project directory: %s', str(args.project_dir))
  87
  88         try:
  89             return args.command.run(args)
  90         except UsageError as exception:
  91             if log.isEnabledFor(logging.DEBUG):
  92                 raise # use Python's exception printing
  93             log.fatal('FATAL: %s', exception)
  94
  95         # If we get here, then execution has failed in some way.
  96         return 1
  97
  98
  99 ##### Subcommand classes
 100 #
 101 # Each class needs to implement two functions: add_args() adds the CLI parameters
 102 # for the subfunction, run() executes the subcommand.
 103 #
 104 # The class documentation doubles as the help text for the command. The
 105 # first line is also used in the summary when calling the program without
 106 # a subcommand.
 107 #
 108 # No need to document the functions each time.
 109 # pylint: disable=C0111
 110 # Using non-top-level imports to make pyosmium optional for replication only.
 111 # pylint: disable=E0012,C0415
 112
 113
 114 class SetupAll:
 115     """\
 116     Create a new Nominatim database from an OSM file.
 117     """
 118
 119     @staticmethod
 120     def add_args(parser):
 121         group_name = parser.add_argument_group('Required arguments')
 122         group = group_name.add_mutually_exclusive_group(required=True)
 123         group.add_argument('--osm-file',
 124                            help='OSM file to be imported.')
 125         group.add_argument('--continue', dest='continue_at',
 126                            choices=['load-data', 'indexing', 'db-postprocess'],
 127                            help='Continue an import that was interrupted')
 128         group = parser.add_argument_group('Optional arguments')
 129         group.add_argument('--osm2pgsql-cache', metavar='SIZE', type=int,
 130                            help='Size of cache to be used by osm2pgsql (in MB)')
 131         group.add_argument('--reverse-only', action='store_true',
 132                            help='Do not create tables and indexes for searching')
 133         group.add_argument('--enable-debug-statements', action='store_true',
 134                            help='Include debug warning statements in SQL code')
 135         group.add_argument('--no-partitions', action='store_true',
 136                            help="""Do not partition search indices
 137                                    (speeds up import of single country extracts)""")
 138         group.add_argument('--no-updates', action='store_true',
 139                            help="""Do not keep tables that are only needed for
 140                                    updating the database later""")
 141         group = parser.add_argument_group('Expert options')
 142         group.add_argument('--ignore-errors', action='store_true',
 143                            help='Continue import even when errors in SQL are present')
 144         group.add_argument('--index-noanalyse', action='store_true',
 145                            help='Do not perform analyse operations during index')
 146
 147
 148     @staticmethod
 149     def run(args):
 150         params = ['setup.php']
 151         if args.osm_file:
 152             params.extend(('--all', '--osm-file', args.osm_file))
 153         else:
 154             if args.continue_at == 'load-data':
 155                 params.append('--load-data')
 156             if args.continue_at in ('load-data', 'indexing'):
 157                 params.append('--index')
 158             params.extend(('--create-search-indices', '--create-country-names',
 159                            '--setup-website'))
 160         if args.osm2pgsql_cache:
 161             params.extend(('--osm2pgsql-cache', args.osm2pgsql_cache))
 162         if args.reverse_only:
 163             params.append('--reverse-only')
 164         if args.enable_debug_statements:
 165             params.append('--enable-debug-statements')
 166         if args.no_partitions:
 167             params.append('--no-partitions')
 168         if args.no_updates:
 169             params.append('--drop')
 170         if args.ignore_errors:
 171             params.append('--ignore-errors')
 172         if args.index_noanalyse:
 173             params.append('--index-noanalyse')
 174         if args.threads:
 175             params.extend(('--threads', args.threads))
 176
 177         return run_legacy_script(*params, nominatim_env=args)
 178
 179
 180 class SetupSpecialPhrases:
 181     """\
 182     Maintain special phrases.
 183     """
 184
 185     @staticmethod
 186     def add_args(parser):
 187         group = parser.add_argument_group('Input arguments')
 188         group.add_argument('--from-wiki', action='store_true',
 189                            help='Pull special phrases from the OSM wiki.')
 190         group = parser.add_argument_group('Output arguments')
 191         group.add_argument('-o', '--output', default='-',
 192                            help="""File to write the preprocessed phrases to.
 193                                    If omitted, it will be written to stdout.""")
 194
 195     @staticmethod
 196     def run(args):
 197         if args.output != '-':
 198             raise NotImplementedError('Only output to stdout is currently implemented.')
 199         return run_legacy_script('specialphrases.php', '--wiki-import', nominatim_env=args)
 200
 201
 202 class UpdateAddData:
 203     """\
 204     Add additional data from a file or an online source.
 205
 206     Data is only imported, not indexed. You need to call `nominatim-update index`
 207     to complete the process.
 208     """
 209
 210     @staticmethod
 211     def add_args(parser):
 212         group_name = parser.add_argument_group('Source')
 213         group = group_name.add_mutually_exclusive_group(required=True)
 214         group.add_argument('--file', metavar='FILE',
 215                            help='Import data from an OSM file')
 216         group.add_argument('--diff', metavar='FILE',
 217                            help='Import data from an OSM diff file')
 218         group.add_argument('--node', metavar='ID', type=int,
 219                            help='Import a single node from the API')
 220         group.add_argument('--way', metavar='ID', type=int,
 221                            help='Import a single way from the API')
 222         group.add_argument('--relation', metavar='ID', type=int,
 223                            help='Import a single relation from the API')
 224         group.add_argument('--tiger-data', metavar='DIR',
 225                            help='Add housenumbers from the US TIGER census database.')
 226         group = parser.add_argument_group('Extra arguments')
 227         group.add_argument('--use-main-api', action='store_true',
 228                            help='Use OSM API instead of Overpass to download objects')
 229
 230     @staticmethod
 231     def run(args):
 232         if args.tiger_data:
 233             os.environ['NOMINATIM_TIGER_DATA_PATH'] = args.tiger_data
 234             return run_legacy_script('setup.php', '--import-tiger-data', nominatim_env=args)
 235
 236         params = ['update.php']
 237         if args.file:
 238             params.extend(('--import-file', args.file))
 239         elif args.diff:
 240             params.extend(('--import-diff', args.diff))
 241         elif args.node:
 242             params.extend(('--import-node', args.node))
 243         elif args.way:
 244             params.extend(('--import-way', args.way))
 245         elif args.relation:
 246             params.extend(('--import-relation', args.relation))
 247         if args.use_main_api:
 248             params.append('--use-main-api')
 249         return run_legacy_script(*params, nominatim_env=args)
 250
 251
 252 class QueryExport:
 253     """\
 254     Export addresses as CSV file from the database.
 255     """
 256
 257     @staticmethod
 258     def add_args(parser):
 259         group = parser.add_argument_group('Output arguments')
 260         group.add_argument('--output-type', default='street',
 261                            choices=('continent', 'country', 'state', 'county',
 262                                     'city', 'suburb', 'street', 'path'),
 263                            help='Type of places to output (default: street)')
 264         group.add_argument('--output-format',
 265                            default='street;suburb;city;county;state;country',
 266                            help="""Semicolon-separated list of address types
 267                                    (see --output-type). Multiple ranks can be
 268                                    merged into one column by simply using a
 269                                    comma-separated list.""")
 270         group.add_argument('--output-all-postcodes', action='store_true',
 271                            help="""List all postcodes for address instead of
 272                                    just the most likely one""")
 273         group.add_argument('--language',
 274                            help="""Preferred language for output
 275                                    (use local name, if omitted)""")
 276         group = parser.add_argument_group('Filter arguments')
 277         group.add_argument('--restrict-to-country', metavar='COUNTRY_CODE',
 278                            help='Export only objects within country')
 279         group.add_argument('--restrict-to-osm-node', metavar='ID', type=int,
 280                            help='Export only children of this OSM node')
 281         group.add_argument('--restrict-to-osm-way', metavar='ID', type=int,
 282                            help='Export only children of this OSM way')
 283         group.add_argument('--restrict-to-osm-relation', metavar='ID', type=int,
 284                            help='Export only children of this OSM relation')
 285
 286
 287     @staticmethod
 288     def run(args):
 289         params = ['export.php',
 290                   '--output-type', args.output_type,
 291                   '--output-format', args.output_format]
 292         if args.output_all_postcodes:
 293             params.append('--output-all-postcodes')
 294         if args.language:
 295             params.extend(('--language', args.language))
 296         if args.restrict_to_country:
 297             params.extend(('--restrict-to-country', args.restrict_to_country))
 298         if args.restrict_to_osm_node:
 299             params.extend(('--restrict-to-osm-node', args.restrict_to_osm_node))
 300         if args.restrict_to_osm_way:
 301             params.extend(('--restrict-to-osm-way', args.restrict_to_osm_way))
 302         if args.restrict_to_osm_relation:
 303             params.extend(('--restrict-to-osm-relation', args.restrict_to_osm_relation))
 304
 305         return run_legacy_script(*params, nominatim_env=args)
 306
 307
 308 class AdminServe:
 309     """\
 310     Start a simple web server for serving the API.
 311
 312     This command starts the built-in PHP webserver to serve the website
 313     from the current project directory. This webserver is only suitable
 314     for testing and develop. Do not use it in production setups!
 315
 316     By the default, the webserver can be accessed at: http://127.0.0.1:8088
 317     """
 318
 319     @staticmethod
 320     def add_args(parser):
 321         group = parser.add_argument_group('Server arguments')
 322         group.add_argument('--server', default='127.0.0.1:8088',
 323                            help='The address the server will listen to.')
 324
 325     @staticmethod
 326     def run(args):
 327         run_php_server(args.server, args.project_dir / 'website')
 328
 329
 330 def nominatim(**kwargs):
 331     """\
 332     Command-line tools for importing, updating, administrating and
 333     querying the Nominatim database.
 334     """
 335     parser = CommandlineParser('nominatim', nominatim.__doc__)
 336
 337     parser.add_subcommand('import', SetupAll)
 338     parser.add_subcommand('freeze', clicmd.SetupFreeze)
 339     parser.add_subcommand('replication', clicmd.UpdateReplication)
 340
 341     parser.add_subcommand('special-phrases', SetupSpecialPhrases)
 342
 343     parser.add_subcommand('add-data', UpdateAddData)
 344     parser.add_subcommand('index', clicmd.UpdateIndex)
 345     parser.add_subcommand('refresh', clicmd.UpdateRefresh)
 346
 347     parser.add_subcommand('admin', clicmd.AdminFuncs)
 348
 349     parser.add_subcommand('export', QueryExport)
 350     parser.add_subcommand('serve', AdminServe)
 351
 352     if kwargs.get('phpcgi_path'):
 353         parser.add_subcommand('search', clicmd.APISearch)
 354         parser.add_subcommand('reverse', clicmd.APIReverse)
 355         parser.add_subcommand('lookup', clicmd.APILookup)
 356         parser.add_subcommand('details', clicmd.APIDetails)
 357         parser.add_subcommand('status', clicmd.APIStatus)
 358     else:
 359         parser.parser.epilog = 'php-cgi not found. Query commands not available.'
 360
 361     parser.add_subcommand('transition', clicmd.AdminTransition)
 362
 363     return parser.run(**kwargs)