2 Command-line interface to the Nominatim functions for import, update,
3 database administration and querying.
9 from pathlib import Path
11 from .config import Configuration
12 from .tools.exec_utils import run_legacy_script, run_api_script
14 def _num_system_cpus():
16 cpus = len(os.sched_getaffinity(0))
17 except NotImplementedError:
20 return cpus or os.cpu_count()
23 class CommandlineParser:
24 """ Wraps some of the common functions for parsing the command line
25 and setting up subcommands.
27 def __init__(self, prog, description):
28 self.parser = argparse.ArgumentParser(
30 description=description,
31 formatter_class=argparse.RawDescriptionHelpFormatter)
33 self.subs = self.parser.add_subparsers(title='available commands',
36 # Arguments added to every sub-command
37 self.default_args = argparse.ArgumentParser(add_help=False)
38 group = self.default_args.add_argument_group('Default arguments')
39 group.add_argument('-h', '--help', action='help',
40 help='Show this help message and exit')
41 group.add_argument('-q', '--quiet', action='store_const', const=0,
42 dest='verbose', default=1,
43 help='Print only error messages')
44 group.add_argument('-v', '--verbose', action='count', default=1,
45 help='Increase verboseness of output')
46 group.add_argument('--project-dir', metavar='DIR', default='.',
47 help='Base directory of the Nominatim installation (default:.)')
48 group.add_argument('-j', '--threads', metavar='NUM', type=int,
49 help='Number of parallel threads to use')
52 def add_subcommand(self, name, cmd):
53 """ Add a subcommand to the parser. The subcommand must be a class
54 with a function add_args() that adds the parameters for the
55 subcommand and a run() function that executes the command.
57 parser = self.subs.add_parser(name, parents=[self.default_args],
58 help=cmd.__doc__.split('\n', 1)[0],
59 description=cmd.__doc__,
60 formatter_class=argparse.RawDescriptionHelpFormatter,
62 parser.set_defaults(command=cmd)
65 def run(self, **kwargs):
66 """ Parse the command line arguments of the program and execute the
67 appropriate subcommand.
69 args = self.parser.parse_args(args=kwargs.get('cli_args'))
71 if args.subcommand is None:
72 self.parser.print_help()
75 for arg in ('module_dir', 'osm2pgsql_path', 'phplib_dir', 'data_dir', 'phpcgi_path'):
76 setattr(args, arg, Path(kwargs[arg]))
77 args.project_dir = Path(args.project_dir)
79 logging.basicConfig(stream=sys.stderr,
80 format='%(asctime)s: %(message)s',
81 datefmt='%Y-%m-%d %H:%M:%S',
82 level=max(4 - args.verbose, 1) * 10)
84 args.config = Configuration(args.project_dir, args.data_dir / 'settings')
86 return args.command.run(args)
88 ##### Subcommand classes
90 # Each class needs to implement two functions: add_args() adds the CLI parameters
91 # for the subfunction, run() executes the subcommand.
93 # The class documentation doubles as the help text for the command. The
94 # first line is also used in the summary when calling the program without
97 # No need to document the functions each time.
98 # pylint: disable=C0111
103 Create a new Nominatim database from an OSM file.
107 def add_args(parser):
108 group_name = parser.add_argument_group('Required arguments')
109 group = group_name.add_mutually_exclusive_group(required=True)
110 group.add_argument('--osm-file',
111 help='OSM file to be imported.')
112 group.add_argument('--continue', dest='continue_at',
113 choices=['load-data', 'indexing', 'db-postprocess'],
114 help='Continue an import that was interrupted')
115 group = parser.add_argument_group('Optional arguments')
116 group.add_argument('--osm2pgsql-cache', metavar='SIZE', type=int,
117 help='Size of cache to be used by osm2pgsql (in MB)')
118 group.add_argument('--reverse-only', action='store_true',
119 help='Do not create tables and indexes for searching')
120 group.add_argument('--enable-debug-statements', action='store_true',
121 help='Include debug warning statements in SQL code')
122 group.add_argument('--no-partitions', action='store_true',
123 help="""Do not partition search indices
124 (speeds up import of single country extracts)""")
125 group.add_argument('--no-updates', action='store_true',
126 help="""Do not keep tables that are only needed for
127 updating the database later""")
128 group = parser.add_argument_group('Expert options')
129 group.add_argument('--ignore-errors', action='store_true',
130 help='Continue import even when errors in SQL are present')
131 group.add_argument('--index-noanalyse', action='store_true',
132 help='Do not perform analyse operations during index')
137 params = ['setup.php']
139 params.extend(('--all', '--osm-file', args.osm_file))
141 if args.continue_at == 'load-data':
142 params.append('--load-data')
143 if args.continue_at in ('load-data', 'indexing'):
144 params.append('--index')
145 params.extend(('--create-search-indices', '--create-country-names',
147 if args.osm2pgsql_cache:
148 params.extend(('--osm2pgsql-cache', args.osm2pgsql_cache))
149 if args.reverse_only:
150 params.append('--reverse-only')
151 if args.enable_debug_statements:
152 params.append('--enable-debug-statements')
153 if args.no_partitions:
154 params.append('--no-partitions')
156 params.append('--drop')
157 if args.ignore_errors:
158 params.append('--ignore-errors')
159 if args.index_noanalyse:
160 params.append('--index-noanalyse')
162 return run_legacy_script(*params, nominatim_env=args)
167 Make database read-only.
169 About half of data in the Nominatim database is kept only to be able to
170 keep the data up-to-date with new changes made in OpenStreetMap. This
171 command drops all this data and only keeps the part needed for geocoding
174 This command has the same effect as the `--no-updates` option for imports.
178 def add_args(parser):
183 return run_legacy_script('setup.php', '--drop', nominatim_env=args)
186 class SetupSpecialPhrases:
188 Maintain special phrases.
192 def add_args(parser):
193 group = parser.add_argument_group('Input arguments')
194 group.add_argument('--from-wiki', action='store_true',
195 help='Pull special phrases from the OSM wiki.')
196 group = parser.add_argument_group('Output arguments')
197 group.add_argument('-o', '--output', default='-',
198 help="""File to write the preprocessed phrases to.
199 If omitted, it will be written to stdout.""")
203 if args.output != '-':
204 raise NotImplementedError('Only output to stdout is currently implemented.')
205 return run_legacy_script('specialphrases.php', '--wiki-import', nominatim_env=args)
208 class UpdateReplication:
210 Update the database using an online replication service.
214 def add_args(parser):
215 group = parser.add_argument_group('Arguments for initialisation')
216 group.add_argument('--init', action='store_true',
217 help='Initialise the update process')
218 group.add_argument('--no-update-functions', dest='update_functions',
219 action='store_false',
220 help="""Do not update the trigger function to
221 support differential updates.""")
222 group = parser.add_argument_group('Arguments for updates')
223 group.add_argument('--check-for-updates', action='store_true',
224 help='Check if new updates are available and exit')
225 group.add_argument('--once', action='store_true',
226 help="""Download and apply updates only once. When
227 not set, updates are continuously applied""")
228 group.add_argument('--no-index', action='store_false', dest='do_index',
229 help="""Do not index the new data. Only applicable
230 together with --once""")
234 params = ['update.php']
236 params.append('--init-updates')
237 if not args.update_functions:
238 params.append('--no-update-functions')
239 elif args.check_for_updates:
240 params.append('--check-for-updates')
243 params.append('--import-osmosis')
245 params.append('--import-osmosis-all')
246 if not args.do_index:
247 params.append('--no-index')
249 return run_legacy_script(*params, nominatim_env=args)
254 Add additional data from a file or an online source.
256 Data is only imported, not indexed. You need to call `nominatim-update index`
257 to complete the process.
261 def add_args(parser):
262 group_name = parser.add_argument_group('Source')
263 group = group_name.add_mutually_exclusive_group(required=True)
264 group.add_argument('--file', metavar='FILE',
265 help='Import data from an OSM file')
266 group.add_argument('--diff', metavar='FILE',
267 help='Import data from an OSM diff file')
268 group.add_argument('--node', metavar='ID', type=int,
269 help='Import a single node from the API')
270 group.add_argument('--way', metavar='ID', type=int,
271 help='Import a single way from the API')
272 group.add_argument('--relation', metavar='ID', type=int,
273 help='Import a single relation from the API')
274 group.add_argument('--tiger-data', metavar='DIR',
275 help='Add housenumbers from the US TIGER census database.')
276 group = parser.add_argument_group('Extra arguments')
277 group.add_argument('--use-main-api', action='store_true',
278 help='Use OSM API instead of Overpass to download objects')
283 os.environ['NOMINATIM_TIGER_DATA_PATH'] = args.tiger_data
284 return run_legacy_script('setup.php', '--import-tiger-data', nominatim_env=args)
286 params = ['update.php']
288 params.extend(('--import-file', args.file))
290 params.extend(('--import-diff', args.diff))
292 params.extend(('--import-node', args.node))
294 params.extend(('--import-way', args.way))
296 params.extend(('--import-relation', args.relation))
297 if args.use_main_api:
298 params.append('--use-main-api')
299 return run_legacy_script(*params, nominatim_env=args)
304 Reindex all new and modified data.
308 def add_args(parser):
309 group = parser.add_argument_group('Filter arguments')
310 group.add_argument('--boundaries-only', action='store_true',
311 help="""Index only administrative boundaries.""")
312 group.add_argument('--no-boundaries', action='store_true',
313 help="""Index everything except administrative boundaries.""")
314 group.add_argument('--minrank', '-r', type=int, metavar='RANK', default=0,
315 help='Minimum/starting rank')
316 group.add_argument('--maxrank', '-R', type=int, metavar='RANK', default=30,
317 help='Maximum/finishing rank')
321 from .indexer.indexer import Indexer
323 indexer = Indexer(args.config.get_libpq_dsn(),
324 args.threads or _num_system_cpus() or 1)
326 if not args.no_boundaries:
327 indexer.index_boundaries(args.minrank, args.maxrank)
328 if not args.boundaries_only:
329 indexer.index_by_rank(args.minrank, args.maxrank)
331 if not args.no_boundaries and not args.boundaries_only:
332 indexer.update_status_table()
339 Recompute auxiliary data used by the indexing process.
341 These functions must not be run in parallel with other update commands.
345 def add_args(parser):
346 group = parser.add_argument_group('Data arguments')
347 group.add_argument('--postcodes', action='store_true',
348 help='Update postcode centroid table')
349 group.add_argument('--word-counts', action='store_true',
350 help='Compute frequency of full-word search terms')
351 group.add_argument('--address-levels', action='store_true',
352 help='Reimport address level configuration')
353 group.add_argument('--functions', action='store_true',
354 help='Update the PL/pgSQL functions in the database')
355 group.add_argument('--wiki-data', action='store_true',
356 help='Update Wikipedia/data importance numbers.')
357 group.add_argument('--importance', action='store_true',
358 help='Recompute place importances (expensive!)')
359 group.add_argument('--website', action='store_true',
360 help='Refresh the directory that serves the scripts for the web API')
361 group = parser.add_argument_group('Arguments for function refresh')
362 group.add_argument('--no-diff-updates', action='store_false', dest='diffs',
363 help='Do not enable code for propagating updates')
364 group.add_argument('--enable-debug-statements', action='store_true',
365 help='Enable debug warning statements in functions')
370 run_legacy_script('update.php', '--calculate-postcodes',
371 nominatim_env=args, throw_on_fail=True)
373 run_legacy_script('update.php', '--recompute-word-counts',
374 nominatim_env=args, throw_on_fail=True)
375 if args.address_levels:
376 run_legacy_script('update.php', '--update-address-levels',
377 nominatim_env=args, throw_on_fail=True)
379 params = ['setup.php', '--create-functions', '--create-partition-functions']
381 params.append('--enable-diff-updates')
382 if args.enable_debug_statements:
383 params.append('--enable-debug-statements')
384 run_legacy_script(*params, nominatim_env=args, throw_on_fail=True)
386 run_legacy_script('setup.php', '--import-wikipedia-articles',
387 nominatim_env=args, throw_on_fail=True)
388 # Attention: importance MUST come after wiki data import.
390 run_legacy_script('update.php', '--recompute-importance',
391 nominatim_env=args, throw_on_fail=True)
393 run_legacy_script('setup.php', '--setup-website',
394 nominatim_env=args, throw_on_fail=True)
398 class AdminCheckDatabase:
400 Check that the database is complete and operational.
404 def add_args(parser):
409 return run_legacy_script('check_import_finished.php', nominatim_env=args)
414 Warm database caches for search and reverse queries.
418 def add_args(parser):
419 group = parser.add_argument_group('Target arguments')
420 group.add_argument('--search-only', action='store_const', dest='target',
422 help="Only pre-warm tables for search queries")
423 group.add_argument('--reverse-only', action='store_const', dest='target',
425 help="Only pre-warm tables for reverse queries")
429 params = ['warm.php']
430 if args.target == 'reverse':
431 params.append('--reverse-only')
432 if args.target == 'search':
433 params.append('--search-only')
434 return run_legacy_script(*params, nominatim_env=args)
439 Export addresses as CSV file from the database.
443 def add_args(parser):
444 group = parser.add_argument_group('Output arguments')
445 group.add_argument('--output-type', default='street',
446 choices=('continent', 'country', 'state', 'county',
447 'city', 'suburb', 'street', 'path'),
448 help='Type of places to output (default: street)')
449 group.add_argument('--output-format',
450 default='street;suburb;city;county;state;country',
451 help="""Semicolon-separated list of address types
452 (see --output-type). Multiple ranks can be
453 merged into one column by simply using a
454 comma-separated list.""")
455 group.add_argument('--output-all-postcodes', action='store_true',
456 help="""List all postcodes for address instead of
457 just the most likely one""")
458 group.add_argument('--language',
459 help="""Preferred language for output
460 (use local name, if omitted)""")
461 group = parser.add_argument_group('Filter arguments')
462 group.add_argument('--restrict-to-country', metavar='COUNTRY_CODE',
463 help='Export only objects within country')
464 group.add_argument('--restrict-to-osm-node', metavar='ID', type=int,
465 help='Export only children of this OSM node')
466 group.add_argument('--restrict-to-osm-way', metavar='ID', type=int,
467 help='Export only children of this OSM way')
468 group.add_argument('--restrict-to-osm-relation', metavar='ID', type=int,
469 help='Export only children of this OSM relation')
474 params = ['export.php',
475 '--output-type', args.output_type,
476 '--output-format', args.output_format]
477 if args.output_all_postcodes:
478 params.append('--output-all-postcodes')
480 params.extend(('--language', args.language))
481 if args.restrict_to_country:
482 params.extend(('--restrict-to-country', args.restrict_to_country))
483 if args.restrict_to_osm_node:
484 params.extend(('--restrict-to-osm-node', args.restrict_to_osm_node))
485 if args.restrict_to_osm_way:
486 params.extend(('--restrict-to-osm-way', args.restrict_to_osm_way))
487 if args.restrict_to_osm_relation:
488 params.extend(('--restrict-to-osm-relation', args.restrict_to_osm_relation))
490 return run_legacy_script(*params, nominatim_env=args)
493 ('street', 'housenumber and street'),
494 ('city', 'city, town or village'),
495 ('county', 'county'),
497 ('country', 'country'),
498 ('postalcode', 'postcode')
502 ('addressdetails', 'Include a breakdown of the address into elements.'),
503 ('extratags', """Include additional information if available
504 (e.g. wikipedia link, opening hours)."""),
505 ('namedetails', 'Include a list of alternative names.')
509 ('addressdetails', 'Include a breakdown of the address into elements.'),
510 ('keywords', 'Include a list of name keywords and address keywords.'),
511 ('linkedplaces', 'Include a details of places that are linked with this one.'),
512 ('hierarchy', 'Include details of places lower in the address hierarchy.'),
513 ('group_hierarchy', 'Group the places by type.'),
514 ('polygon_geojson', 'Include geometry of result.')
517 def _add_api_output_arguments(parser):
518 group = parser.add_argument_group('Output arguments')
519 group.add_argument('--format', default='jsonv2',
520 choices=['xml', 'json', 'jsonv2', 'geojson', 'geocodejson'],
521 help='Format of result')
522 for name, desc in EXTRADATA_PARAMS:
523 group.add_argument('--' + name, action='store_true', help=desc)
525 group.add_argument('--lang', '--accept-language', metavar='LANGS',
526 help='Preferred language order for presenting search results')
527 group.add_argument('--polygon-output',
528 choices=['geojson', 'kml', 'svg', 'text'],
529 help='Output geometry of results as a GeoJSON, KML, SVG or WKT.')
530 group.add_argument('--polygon-threshold', type=float, metavar='TOLERANCE',
531 help="""Simplify output geometry.
532 Parameter is difference tolerance in degrees.""")
537 Execute API search query.
541 def add_args(parser):
542 group = parser.add_argument_group('Query arguments')
543 group.add_argument('--query',
544 help='Free-form query string')
545 for name, desc in STRUCTURED_QUERY:
546 group.add_argument('--' + name, help='Structured query: ' + desc)
548 _add_api_output_arguments(parser)
550 group = parser.add_argument_group('Result limitation')
551 group.add_argument('--countrycodes', metavar='CC,..',
552 help='Limit search results to one or more countries.')
553 group.add_argument('--exclude_place_ids', metavar='ID,..',
554 help='List of search object to be excluded')
555 group.add_argument('--limit', type=int,
556 help='Limit the number of returned results')
557 group.add_argument('--viewbox', metavar='X1,Y1,X2,Y2',
558 help='Preferred area to find search results')
559 group.add_argument('--bounded', action='store_true',
560 help='Strictly restrict results to viewbox area')
562 group = parser.add_argument_group('Other arguments')
563 group.add_argument('--no-dedupe', action='store_false', dest='dedupe',
564 help='Do not remove duplicates from the result list')
570 params = dict(q=args.query)
572 params = {k : getattr(args, k) for k, _ in STRUCTURED_QUERY if getattr(args, k)}
574 for param, _ in EXTRADATA_PARAMS:
575 if getattr(args, param):
577 for param in ('format', 'countrycodes', 'exclude_place_ids', 'limit', 'viewbox'):
578 if getattr(args, param):
579 params[param] = getattr(args, param)
581 params['accept-language'] = args.lang
582 if args.polygon_output:
583 params['polygon_' + args.polygon_output] = '1'
584 if args.polygon_threshold:
585 params['polygon_threshold'] = args.polygon_threshold
587 params['bounded'] = '1'
589 params['dedupe'] = '0'
591 return run_api_script('search', args.project_dir,
592 phpcgi_bin=args.phpcgi_path, params=params)
596 Execute API reverse query.
600 def add_args(parser):
601 group = parser.add_argument_group('Query arguments')
602 group.add_argument('--lat', type=float, required=True,
603 help='Latitude of coordinate to look up (in WGS84)')
604 group.add_argument('--lon', type=float, required=True,
605 help='Longitude of coordinate to look up (in WGS84)')
606 group.add_argument('--zoom', type=int,
607 help='Level of detail required for the address')
609 _add_api_output_arguments(parser)
614 params = dict(lat=args.lat, lon=args.lon)
615 if args.zoom is not None:
616 params['zoom'] = args.zoom
618 for param, _ in EXTRADATA_PARAMS:
619 if getattr(args, param):
622 params['format'] = args.format
624 params['accept-language'] = args.lang
625 if args.polygon_output:
626 params['polygon_' + args.polygon_output] = '1'
627 if args.polygon_threshold:
628 params['polygon_threshold'] = args.polygon_threshold
630 return run_api_script('reverse', args.project_dir,
631 phpcgi_bin=args.phpcgi_path, params=params)
636 Execute API reverse query.
640 def add_args(parser):
641 group = parser.add_argument_group('Query arguments')
642 group.add_argument('--id', metavar='OSMID',
643 action='append', required=True, dest='ids',
644 help='OSM id to lookup in format <NRW><id> (may be repeated)')
646 _add_api_output_arguments(parser)
651 params = dict(osm_ids=','.join(args.ids))
653 for param, _ in EXTRADATA_PARAMS:
654 if getattr(args, param):
657 params['format'] = args.format
659 params['accept-language'] = args.lang
660 if args.polygon_output:
661 params['polygon_' + args.polygon_output] = '1'
662 if args.polygon_threshold:
663 params['polygon_threshold'] = args.polygon_threshold
665 return run_api_script('lookup', args.project_dir,
666 phpcgi_bin=args.phpcgi_path, params=params)
671 Execute API lookup query.
675 def add_args(parser):
676 group = parser.add_argument_group('Query arguments')
677 objs = group.add_mutually_exclusive_group(required=True)
678 objs.add_argument('--node', '-n', type=int,
679 help="Look up the OSM node with the given ID.")
680 objs.add_argument('--way', '-w', type=int,
681 help="Look up the OSM way with the given ID.")
682 objs.add_argument('--relation', '-r', type=int,
683 help="Look up the OSM relation with the given ID.")
684 objs.add_argument('--place_id', '-p', type=int,
685 help='Database internal identifier of the OSM object to look up.')
686 group.add_argument('--class', dest='object_class',
687 help="""Class type to disambiguated multiple entries
688 of the same object.""")
690 group = parser.add_argument_group('Output arguments')
691 for name, desc in DETAILS_SWITCHES:
692 group.add_argument('--' + name, action='store_true', help=desc)
693 group.add_argument('--lang', '--accept-language', metavar='LANGS',
694 help='Preferred language order for presenting search results')
699 params = dict(osmtype='N', osmid=args.node)
701 params = dict(osmtype='W', osmid=args.node)
703 params = dict(osmtype='R', osmid=args.node)
705 params = dict(place_id=args.place_id)
706 if args.object_class:
707 params['class'] = args.object_class
708 for name, _ in DETAILS_SWITCHES:
709 params[name] = '1' if getattr(args, name) else '0'
711 return run_api_script('details', args.project_dir,
712 phpcgi_bin=args.phpcgi_path, params=params)
717 Execute API status query.
721 def add_args(parser):
722 group = parser.add_argument_group('API parameters')
723 group.add_argument('--format', default='text', choices=['text', 'json'],
724 help='Format of result')
728 return run_api_script('status', args.project_dir,
729 phpcgi_bin=args.phpcgi_path,
730 params=dict(format=args.format))
733 def nominatim(**kwargs):
735 Command-line tools for importing, updating, administrating and
736 querying the Nominatim database.
738 parser = CommandlineParser('nominatim', nominatim.__doc__)
740 parser.add_subcommand('import', SetupAll)
741 parser.add_subcommand('freeze', SetupFreeze)
742 parser.add_subcommand('replication', UpdateReplication)
744 parser.add_subcommand('check-database', AdminCheckDatabase)
745 parser.add_subcommand('warm', AdminWarm)
747 parser.add_subcommand('special-phrases', SetupSpecialPhrases)
749 parser.add_subcommand('add-data', UpdateAddData)
750 parser.add_subcommand('index', UpdateIndex)
751 parser.add_subcommand('refresh', UpdateRefresh)
753 parser.add_subcommand('export', QueryExport)
755 if kwargs.get('phpcgi_path'):
756 parser.add_subcommand('search', APISearch)
757 parser.add_subcommand('reverse', APIReverse)
758 parser.add_subcommand('lookup', APILookup)
759 parser.add_subcommand('details', APIDetails)
760 parser.add_subcommand('status', APIStatus)
762 parser.parser.epilog = 'php-cgi not found. Query commands not available.'
764 return parser.run(**kwargs)