2 Command-line interface to the Nominatim functions for import, update,
3 database administration and querying.
9 from pathlib import Path
11 from .config import Configuration
12 from .tools.exec_utils import run_legacy_script, run_api_script
14 from .indexer.indexer import Indexer
16 def _num_system_cpus():
18 cpus = len(os.sched_getaffinity(0))
19 except NotImplementedError:
22 return cpus or os.cpu_count()
25 class CommandlineParser:
26 """ Wraps some of the common functions for parsing the command line
27 and setting up subcommands.
29 def __init__(self, prog, description):
30 self.parser = argparse.ArgumentParser(
32 description=description,
33 formatter_class=argparse.RawDescriptionHelpFormatter)
35 self.subs = self.parser.add_subparsers(title='available commands',
38 # Arguments added to every sub-command
39 self.default_args = argparse.ArgumentParser(add_help=False)
40 group = self.default_args.add_argument_group('Default arguments')
41 group.add_argument('-h', '--help', action='help',
42 help='Show this help message and exit')
43 group.add_argument('-q', '--quiet', action='store_const', const=0,
44 dest='verbose', default=1,
45 help='Print only error messages')
46 group.add_argument('-v', '--verbose', action='count', default=1,
47 help='Increase verboseness of output')
48 group.add_argument('--project-dir', metavar='DIR', default='.',
49 help='Base directory of the Nominatim installation (default:.)')
50 group.add_argument('-j', '--threads', metavar='NUM', type=int,
51 help='Number of parallel threads to use')
54 def add_subcommand(self, name, cmd):
55 """ Add a subcommand to the parser. The subcommand must be a class
56 with a function add_args() that adds the parameters for the
57 subcommand and a run() function that executes the command.
59 parser = self.subs.add_parser(name, parents=[self.default_args],
60 help=cmd.__doc__.split('\n', 1)[0],
61 description=cmd.__doc__,
62 formatter_class=argparse.RawDescriptionHelpFormatter,
64 parser.set_defaults(command=cmd)
67 def run(self, **kwargs):
68 """ Parse the command line arguments of the program and execute the
69 appropriate subcommand.
71 args = self.parser.parse_args(args=kwargs.get('cli_args'))
73 if args.subcommand is None:
74 self.parser.print_help()
77 for arg in ('module_dir', 'osm2pgsql_path', 'phplib_dir', 'data_dir', 'phpcgi_path'):
78 setattr(args, arg, Path(kwargs[arg]))
79 args.project_dir = Path(args.project_dir)
81 logging.basicConfig(stream=sys.stderr,
82 format='%(asctime)s: %(message)s',
83 datefmt='%Y-%m-%d %H:%M:%S',
84 level=max(4 - args.verbose, 1) * 10)
86 args.config = Configuration(args.project_dir, args.data_dir / 'settings')
88 return args.command.run(args)
90 ##### Subcommand classes
92 # Each class needs to implement two functions: add_args() adds the CLI parameters
93 # for the subfunction, run() executes the subcommand.
95 # The class documentation doubles as the help text for the command. The
96 # first line is also used in the summary when calling the program without
99 # No need to document the functions each time.
100 # pylint: disable=C0111
105 Create a new Nominatim database from an OSM file.
109 def add_args(parser):
110 group_name = parser.add_argument_group('Required arguments')
111 group = group_name.add_mutually_exclusive_group(required=True)
112 group.add_argument('--osm-file',
113 help='OSM file to be imported.')
114 group.add_argument('--continue', dest='continue_at',
115 choices=['load-data', 'indexing', 'db-postprocess'],
116 help='Continue an import that was interrupted')
117 group = parser.add_argument_group('Optional arguments')
118 group.add_argument('--osm2pgsql-cache', metavar='SIZE', type=int,
119 help='Size of cache to be used by osm2pgsql (in MB)')
120 group.add_argument('--reverse-only', action='store_true',
121 help='Do not create tables and indexes for searching')
122 group.add_argument('--enable-debug-statements', action='store_true',
123 help='Include debug warning statements in SQL code')
124 group.add_argument('--no-partitions', action='store_true',
125 help="""Do not partition search indices
126 (speeds up import of single country extracts)""")
127 group.add_argument('--no-updates', action='store_true',
128 help="""Do not keep tables that are only needed for
129 updating the database later""")
130 group = parser.add_argument_group('Expert options')
131 group.add_argument('--ignore-errors', action='store_true',
132 help='Continue import even when errors in SQL are present')
133 group.add_argument('--index-noanalyse', action='store_true',
134 help='Do not perform analyse operations during index')
139 params = ['setup.php']
141 params.extend(('--all', '--osm-file', args.osm_file))
143 if args.continue_at == 'load-data':
144 params.append('--load-data')
145 if args.continue_at in ('load-data', 'indexing'):
146 params.append('--index')
147 params.extend(('--create-search-indices', '--create-country-names',
149 if args.osm2pgsql_cache:
150 params.extend(('--osm2pgsql-cache', args.osm2pgsql_cache))
151 if args.reverse_only:
152 params.append('--reverse-only')
153 if args.enable_debug_statements:
154 params.append('--enable-debug-statements')
155 if args.no_partitions:
156 params.append('--no-partitions')
158 params.append('--drop')
159 if args.ignore_errors:
160 params.append('--ignore-errors')
161 if args.index_noanalyse:
162 params.append('--index-noanalyse')
164 return run_legacy_script(*params, nominatim_env=args)
169 Make database read-only.
171 About half of data in the Nominatim database is kept only to be able to
172 keep the data up-to-date with new changes made in OpenStreetMap. This
173 command drops all this data and only keeps the part needed for geocoding
176 This command has the same effect as the `--no-updates` option for imports.
180 def add_args(parser):
185 return run_legacy_script('setup.php', '--drop', nominatim_env=args)
188 class SetupSpecialPhrases:
190 Maintain special phrases.
194 def add_args(parser):
195 group = parser.add_argument_group('Input arguments')
196 group.add_argument('--from-wiki', action='store_true',
197 help='Pull special phrases from the OSM wiki.')
198 group = parser.add_argument_group('Output arguments')
199 group.add_argument('-o', '--output', default='-',
200 help="""File to write the preprocessed phrases to.
201 If omitted, it will be written to stdout.""")
205 if args.output != '-':
206 raise NotImplementedError('Only output to stdout is currently implemented.')
207 return run_legacy_script('specialphrases.php', '--wiki-import', nominatim_env=args)
210 class UpdateReplication:
212 Update the database using an online replication service.
216 def add_args(parser):
217 group = parser.add_argument_group('Arguments for initialisation')
218 group.add_argument('--init', action='store_true',
219 help='Initialise the update process')
220 group.add_argument('--no-update-functions', dest='update_functions',
221 action='store_false',
222 help="""Do not update the trigger function to
223 support differential updates.""")
224 group = parser.add_argument_group('Arguments for updates')
225 group.add_argument('--check-for-updates', action='store_true',
226 help='Check if new updates are available and exit')
227 group.add_argument('--once', action='store_true',
228 help="""Download and apply updates only once. When
229 not set, updates are continuously applied""")
230 group.add_argument('--no-index', action='store_false', dest='do_index',
231 help="""Do not index the new data. Only applicable
232 together with --once""")
236 params = ['update.php']
238 params.append('--init-updates')
239 if not args.update_functions:
240 params.append('--no-update-functions')
241 elif args.check_for_updates:
242 params.append('--check-for-updates')
245 params.append('--import-osmosis')
247 params.append('--import-osmosis-all')
248 if not args.do_index:
249 params.append('--no-index')
251 return run_legacy_script(*params, nominatim_env=args)
256 Add additional data from a file or an online source.
258 Data is only imported, not indexed. You need to call `nominatim-update index`
259 to complete the process.
263 def add_args(parser):
264 group_name = parser.add_argument_group('Source')
265 group = group_name.add_mutually_exclusive_group(required=True)
266 group.add_argument('--file', metavar='FILE',
267 help='Import data from an OSM file')
268 group.add_argument('--diff', metavar='FILE',
269 help='Import data from an OSM diff file')
270 group.add_argument('--node', metavar='ID', type=int,
271 help='Import a single node from the API')
272 group.add_argument('--way', metavar='ID', type=int,
273 help='Import a single way from the API')
274 group.add_argument('--relation', metavar='ID', type=int,
275 help='Import a single relation from the API')
276 group.add_argument('--tiger-data', metavar='DIR',
277 help='Add housenumbers from the US TIGER census database.')
278 group = parser.add_argument_group('Extra arguments')
279 group.add_argument('--use-main-api', action='store_true',
280 help='Use OSM API instead of Overpass to download objects')
285 os.environ['NOMINATIM_TIGER_DATA_PATH'] = args.tiger_data
286 return run_legacy_script('setup.php', '--import-tiger-data', nominatim_env=args)
288 params = ['update.php']
290 params.extend(('--import-file', args.file))
292 params.extend(('--import-diff', args.diff))
294 params.extend(('--import-node', args.node))
296 params.extend(('--import-way', args.way))
298 params.extend(('--import-relation', args.relation))
299 if args.use_main_api:
300 params.append('--use-main-api')
301 return run_legacy_script(*params, nominatim_env=args)
306 Reindex all new and modified data.
310 def add_args(parser):
311 group = parser.add_argument_group('Filter arguments')
312 group.add_argument('--boundaries-only', action='store_true',
313 help="""Index only administrative boundaries.""")
314 group.add_argument('--no-boundaries', action='store_true',
315 help="""Index everything except administrative boundaries.""")
316 group.add_argument('--minrank', '-r', type=int, metavar='RANK', default=0,
317 help='Minimum/starting rank')
318 group.add_argument('--maxrank', '-R', type=int, metavar='RANK', default=30,
319 help='Maximum/finishing rank')
323 indexer = Indexer(args.config.get_libpq_dsn(),
324 args.threads or _num_system_cpus() or 1)
326 if not args.no_boundaries:
327 indexer.index_boundaries(args.minrank, args.maxrank)
328 if not args.boundaries_only:
329 indexer.index_by_rank(args.minrank, args.maxrank)
331 if not args.no_boundaries and not args.boundaries_only:
332 indexer.update_status_table()
339 Recompute auxiliary data used by the indexing process.
341 These functions must not be run in parallel with other update commands.
345 def add_args(parser):
346 group = parser.add_argument_group('Data arguments')
347 group.add_argument('--postcodes', action='store_true',
348 help='Update postcode centroid table')
349 group.add_argument('--word-counts', action='store_true',
350 help='Compute frequency of full-word search terms')
351 group.add_argument('--address-levels', action='store_true',
352 help='Reimport address level configuration')
353 group.add_argument('--functions', action='store_true',
354 help='Update the PL/pgSQL functions in the database')
355 group.add_argument('--wiki-data', action='store_true',
356 help='Update Wikipedia/data importance numbers.')
357 group.add_argument('--importance', action='store_true',
358 help='Recompute place importances (expensive!)')
359 group.add_argument('--website', action='store_true',
360 help='Refresh the directory that serves the scripts for the web API')
361 group = parser.add_argument_group('Arguments for function refresh')
362 group.add_argument('--no-diff-updates', action='store_false', dest='diffs',
363 help='Do not enable code for propagating updates')
364 group.add_argument('--enable-debug-statements', action='store_true',
365 help='Enable debug warning statements in functions')
370 run_legacy_script('update.php', '--calculate-postcodes',
371 nominatim_env=args, throw_on_fail=True)
373 run_legacy_script('update.php', '--recompute-word-counts',
374 nominatim_env=args, throw_on_fail=True)
375 if args.address_levels:
376 run_legacy_script('update.php', '--update-address-levels',
377 nominatim_env=args, throw_on_fail=True)
379 params = ['setup.php', '--create-functions', '--create-partition-functions']
381 params.append('--enable-diff-updates')
382 if args.enable_debug_statements:
383 params.append('--enable-debug-statements')
384 run_legacy_script(*params, nominatim_env=args, throw_on_fail=True)
386 run_legacy_script('setup.php', '--import-wikipedia-articles',
387 nominatim_env=args, throw_on_fail=True)
388 # Attention: importance MUST come after wiki data import.
390 run_legacy_script('update.php', '--recompute-importance',
391 nominatim_env=args, throw_on_fail=True)
393 run_legacy_script('setup.php', '--setup-website',
394 nominatim_env=args, throw_on_fail=True)
398 class AdminCheckDatabase:
400 Check that the database is complete and operational.
404 def add_args(parser):
409 return run_legacy_script('check_import_finished.php', nominatim_env=args)
414 Warm database caches for search and reverse queries.
418 def add_args(parser):
419 group = parser.add_argument_group('Target arguments')
420 group.add_argument('--search-only', action='store_const', dest='target',
422 help="Only pre-warm tables for search queries")
423 group.add_argument('--reverse-only', action='store_const', dest='target',
425 help="Only pre-warm tables for reverse queries")
429 params = ['warm.php']
430 if args.target == 'reverse':
431 params.append('--reverse-only')
432 if args.target == 'search':
433 params.append('--search-only')
434 return run_legacy_script(*params, nominatim_env=args)
439 Export addresses as CSV file from the database.
443 def add_args(parser):
444 group = parser.add_argument_group('Output arguments')
445 group.add_argument('--output-type', default='street',
446 choices=('continent', 'country', 'state', 'county',
447 'city', 'suburb', 'street', 'path'),
448 help='Type of places to output (default: street)')
449 group.add_argument('--output-format',
450 default='street;suburb;city;county;state;country',
451 help="""Semicolon-separated list of address types
452 (see --output-type). Multiple ranks can be
453 merged into one column by simply using a
454 comma-separated list.""")
455 group.add_argument('--output-all-postcodes', action='store_true',
456 help="""List all postcodes for address instead of
457 just the most likely one""")
458 group.add_argument('--language',
459 help="""Preferred language for output
460 (use local name, if omitted)""")
461 group = parser.add_argument_group('Filter arguments')
462 group.add_argument('--restrict-to-country', metavar='COUNTRY_CODE',
463 help='Export only objects within country')
464 group.add_argument('--restrict-to-osm-node', metavar='ID', type=int,
465 help='Export only children of this OSM node')
466 group.add_argument('--restrict-to-osm-way', metavar='ID', type=int,
467 help='Export only children of this OSM way')
468 group.add_argument('--restrict-to-osm-relation', metavar='ID', type=int,
469 help='Export only children of this OSM relation')
474 params = ['export.php',
475 '--output-type', args.output_type,
476 '--output-format', args.output_format]
477 if args.output_all_postcodes:
478 params.append('--output-all-postcodes')
480 params.extend(('--language', args.language))
481 if args.restrict_to_country:
482 params.extend(('--restrict-to-country', args.restrict_to_country))
483 if args.restrict_to_osm_node:
484 params.extend(('--restrict-to-osm-node', args.restrict_to_osm_node))
485 if args.restrict_to_osm_way:
486 params.extend(('--restrict-to-osm-way', args.restrict_to_osm_way))
487 if args.restrict_to_osm_relation:
488 params.extend(('--restrict-to-osm-relation', args.restrict_to_osm_relation))
490 return run_legacy_script(*params, nominatim_env=args)
493 ('street', 'housenumber and street'),
494 ('city', 'city, town or village'),
495 ('county', 'county'),
497 ('country', 'country'),
498 ('postalcode', 'postcode')
502 ('addressdetails', 'Include a breakdown of the address into elements.'),
503 ('extratags', """Include additional information if available
504 (e.g. wikipedia link, opening hours)."""),
505 ('namedetails', 'Include a list of alternative names.')
509 ('addressdetails', 'Include a breakdown of the address into elements.'),
510 ('keywords', 'Include a list of name keywords and address keywords.'),
511 ('linkedplaces', 'Include a details of places that are linked with this one.'),
512 ('hierarchy', 'Include details of places lower in the address hierarchy.'),
513 ('group_hierarchy', 'Group the places by type.'),
514 ('polygon_geojson', 'Include geometry of result.')
517 def _add_api_output_arguments(parser):
518 group = parser.add_argument_group('Output arguments')
519 group.add_argument('--format', default='jsonv2',
520 choices=['xml', 'json', 'jsonv2', 'geojson', 'geocodejson'],
521 help='Format of result')
522 for name, desc in EXTRADATA_PARAMS:
523 group.add_argument('--' + name, action='store_true', help=desc)
525 group.add_argument('--lang', '--accept-language', metavar='LANGS',
526 help='Preferred language order for presenting search results')
527 group.add_argument('--polygon-output',
528 choices=['geojson', 'kml', 'svg', 'text'],
529 help='Output geometry of results as a GeoJSON, KML, SVG or WKT.')
530 group.add_argument('--polygon-threshold', type=float, metavar='TOLERANCE',
531 help="""Simplify output geometry.
532 Parameter is difference tolerance in degrees.""")
537 Execute API search query.
541 def add_args(parser):
542 group = parser.add_argument_group('Query arguments')
543 group.add_argument('--query',
544 help='Free-form query string')
545 for name, desc in STRUCTURED_QUERY:
546 group.add_argument('--' + name, help='Structured query: ' + desc)
548 _add_api_output_arguments(parser)
550 group = parser.add_argument_group('Result limitation')
551 group.add_argument('--countrycodes', metavar='CC,..',
552 help='Limit search results to one or more countries.')
553 group.add_argument('--exclude_place_ids', metavar='ID,..',
554 help='List of search object to be excluded')
555 group.add_argument('--limit', type=int,
556 help='Limit the number of returned results')
557 group.add_argument('--viewbox', metavar='X1,Y1,X2,Y2',
558 help='Preferred area to find search results')
559 group.add_argument('--bounded', action='store_true',
560 help='Strictly restrict results to viewbox area')
562 group = parser.add_argument_group('Other arguments')
563 group.add_argument('--no-dedupe', action='store_false', dest='dedupe',
564 help='Do not remove duplicates from the result list')
570 params = dict(q=args.query)
572 params = {k : getattr(args, k) for k, _ in STRUCTURED_QUERY if getattr(args, k)}
574 for param, _ in EXTRADATA_PARAMS:
575 if getattr(args, param):
577 for param in ('format', 'countrycodes', 'exclude_place_ids', 'limit', 'viewbox'):
578 if getattr(args, param):
579 params[param] = getattr(args, param)
581 params['accept-language'] = args.lang
582 if args.polygon_output:
583 params['polygon_' + args.polygon_output] = '1'
584 if args.polygon_threshold:
585 params['polygon_threshold'] = args.polygon_threshold
587 params['bounded'] = '1'
589 params['dedupe'] = '0'
591 return run_api_script('search', args.project_dir,
592 phpcgi_bin=args.phpcgi_path, params=params)
596 Execute API reverse query.
600 def add_args(parser):
601 group = parser.add_argument_group('Query arguments')
602 group.add_argument('--lat', type=float, required=True,
603 help='Latitude of coordinate to look up (in WGS84)')
604 group.add_argument('--lon', type=float, required=True,
605 help='Longitude of coordinate to look up (in WGS84)')
606 group.add_argument('--zoom', type=int,
607 help='Level of detail required for the address')
609 _add_api_output_arguments(parser)
614 params = dict(lat=args.lat, lon=args.lon)
615 if args.zoom is not None:
616 params['zoom'] = args.zoom
618 for param, _ in EXTRADATA_PARAMS:
619 if getattr(args, param):
622 params['format'] = args.format
624 params['accept-language'] = args.lang
625 if args.polygon_output:
626 params['polygon_' + args.polygon_output] = '1'
627 if args.polygon_threshold:
628 params['polygon_threshold'] = args.polygon_threshold
630 return run_api_script('reverse', args.project_dir,
631 phpcgi_bin=args.phpcgi_path, params=params)
636 Execute API reverse query.
640 def add_args(parser):
641 group = parser.add_argument_group('Query arguments')
642 group.add_argument('--id', metavar='OSMID',
643 action='append', required=True, dest='ids',
644 help='OSM id to lookup in format <NRW><id> (may be repeated)')
646 _add_api_output_arguments(parser)
651 params = dict(osm_ids=','.join(args.ids))
653 for param, _ in EXTRADATA_PARAMS:
654 if getattr(args, param):
657 params['format'] = args.format
659 params['accept-language'] = args.lang
660 if args.polygon_output:
661 params['polygon_' + args.polygon_output] = '1'
662 if args.polygon_threshold:
663 params['polygon_threshold'] = args.polygon_threshold
665 return run_api_script('lookup', args.project_dir,
666 phpcgi_bin=args.phpcgi_path, params=params)
671 Execute API lookup query.
675 def add_args(parser):
676 group = parser.add_argument_group('Query arguments')
677 objs = group.add_mutually_exclusive_group(required=True)
678 objs.add_argument('--node', '-n', type=int,
679 help="Look up the OSM node with the given ID.")
680 objs.add_argument('--way', '-w', type=int,
681 help="Look up the OSM way with the given ID.")
682 objs.add_argument('--relation', '-r', type=int,
683 help="Look up the OSM relation with the given ID.")
684 objs.add_argument('--place_id', '-p', type=int,
685 help='Database internal identifier of the OSM object to look up.')
686 group.add_argument('--class', dest='object_class',
687 help="""Class type to disambiguated multiple entries
688 of the same object.""")
690 group = parser.add_argument_group('Output arguments')
691 for name, desc in DETAILS_SWITCHES:
692 group.add_argument('--' + name, action='store_true', help=desc)
693 group.add_argument('--lang', '--accept-language', metavar='LANGS',
694 help='Preferred language order for presenting search results')
699 params = dict(osmtype='N', osmid=args.node)
701 params = dict(osmtype='W', osmid=args.node)
703 params = dict(osmtype='R', osmid=args.node)
705 params = dict(place_id=args.place_id)
706 if args.object_class:
707 params['class'] = args.object_class
708 for name, _ in DETAILS_SWITCHES:
709 params[name] = '1' if getattr(args, name) else '0'
711 return run_api_script('details', args.project_dir,
712 phpcgi_bin=args.phpcgi_path, params=params)
717 Execute API status query.
721 def add_args(parser):
722 group = parser.add_argument_group('API parameters')
723 group.add_argument('--format', default='text', choices=['text', 'json'],
724 help='Format of result')
728 return run_api_script('status', args.project_dir,
729 phpcgi_bin=args.phpcgi_path,
730 params=dict(format=args.format))
733 def nominatim(**kwargs):
735 Command-line tools for importing, updating, administrating and
736 querying the Nominatim database.
738 parser = CommandlineParser('nominatim', nominatim.__doc__)
740 parser.add_subcommand('import', SetupAll)
741 parser.add_subcommand('freeze', SetupFreeze)
742 parser.add_subcommand('replication', UpdateReplication)
744 parser.add_subcommand('check-database', AdminCheckDatabase)
745 parser.add_subcommand('warm', AdminWarm)
747 parser.add_subcommand('special-phrases', SetupSpecialPhrases)
749 parser.add_subcommand('add-data', UpdateAddData)
750 parser.add_subcommand('index', UpdateIndex)
751 parser.add_subcommand('refresh', UpdateRefresh)
753 parser.add_subcommand('export', QueryExport)
755 if kwargs.get('phpcgi_path'):
756 parser.add_subcommand('search', APISearch)
757 parser.add_subcommand('reverse', APIReverse)
758 parser.add_subcommand('lookup', APILookup)
759 parser.add_subcommand('details', APIDetails)
760 parser.add_subcommand('status', APIStatus)
762 parser.parser.epilog = 'php-cgi not found. Query commands not available.'
764 return parser.run(**kwargs)