remove API comparison util

[nominatim.git] / nominatim / cli.py

diff --git a/nominatim/cli.py b/nominatim/cli.py
index 62299e9f985eb58518187e8faed26ff1cd4f767c..6c110ce72203feea60beb83571761def0ee2d036 100644 (file)
--- a/nominatim/cli.py
+++ b/nominatim/cli.py
@@ -3,12 +3,24 @@ Command-line interface to the Nominatim functions for import, update,
 database administration and querying.
 """
 import sys
 database administration and querying.
 """
 import sys

+import os

 import argparse
 import logging
 from pathlib import Path
 
 from .config import Configuration
 import argparse
 import logging
 from pathlib import Path
 
 from .config import Configuration

-from .admin.exec_utils import run_legacy_script
+from .tools.exec_utils import run_legacy_script, run_api_script
+
+from .indexer.indexer import Indexer
+
+def _num_system_cpus():
+    try:
+        cpus = len(os.sched_getaffinity(0))
+    except NotImplementedError:
+        cpus = None
+
+    return cpus or os.cpu_count()
+

 
 class CommandlineParser:
     """ Wraps some of the common functions for parsing the command line
 
 class CommandlineParser:
     """ Wraps some of the common functions for parsing the command line


@@ -56,17 +68,18 @@ class CommandlineParser:
         """ Parse the command line arguments of the program and execute the
             appropriate subcommand.
         """
         """ Parse the command line arguments of the program and execute the
             appropriate subcommand.
         """

-        args = self.parser.parse_args()
+        args = self.parser.parse_args(args=kwargs.get('cli_args'))

 
         if args.subcommand is None:
 
         if args.subcommand is None:

-            return self.parser.print_help()
+            self.parser.print_help()
+            return 1

 
 

-        for arg in ('module_dir', 'osm2pgsql_path', 'phplib_dir', 'data_dir'):
+        for arg in ('module_dir', 'osm2pgsql_path', 'phplib_dir', 'data_dir', 'phpcgi_path'):

             setattr(args, arg, Path(kwargs[arg]))
         args.project_dir = Path(args.project_dir)
 
         logging.basicConfig(stream=sys.stderr,
             setattr(args, arg, Path(kwargs[arg]))
         args.project_dir = Path(args.project_dir)
 
         logging.basicConfig(stream=sys.stderr,

-                            format='%(asctime)s %(levelname)s: %(message)s',
+                            format='%(asctime)s: %(message)s',

                             datefmt='%Y-%m-%d %H:%M:%S',
                             level=max(4 - args.verbose, 1) * 10)
 
                             datefmt='%Y-%m-%d %H:%M:%S',
                             level=max(4 - args.verbose, 1) * 10)
 


@@ -98,7 +111,7 @@ class SetupAll:
         group = group_name.add_mutually_exclusive_group(required=True)
         group.add_argument('--osm-file',
                            help='OSM file to be imported.')
         group = group_name.add_mutually_exclusive_group(required=True)
         group.add_argument('--osm-file',
                            help='OSM file to be imported.')

-        group.add_argument('--continue', nargs=1, dest='continue_at',
+        group.add_argument('--continue', dest='continue_at',

                            choices=['load-data', 'indexing', 'db-postprocess'],
                            help='Continue an import that was interrupted')
         group = parser.add_argument_group('Optional arguments')
                            choices=['load-data', 'indexing', 'db-postprocess'],
                            help='Continue an import that was interrupted')
         group = parser.add_argument_group('Optional arguments')


@@ -184,13 +197,12 @@ class SetupSpecialPhrases:
                            help='Pull special phrases from the OSM wiki.')
         group = parser.add_argument_group('Output arguments')
         group.add_argument('-o', '--output', default='-',
                            help='Pull special phrases from the OSM wiki.')
         group = parser.add_argument_group('Output arguments')
         group.add_argument('-o', '--output', default='-',

-                           type=argparse.FileType('w', encoding='UTF-8'),

                            help="""File to write the preprocessed phrases to.
                                    If omitted, it will be written to stdout.""")
 
     @staticmethod
     def run(args):
                            help="""File to write the preprocessed phrases to.
                                    If omitted, it will be written to stdout.""")
 
     @staticmethod
     def run(args):

-        if args.output.name != '<stdout>':
+        if args.output != '-':

             raise NotImplementedError('Only output to stdout is currently implemented.')
         return run_legacy_script('specialphrases.php', '--wiki-import', nominatim_env=args)
 
             raise NotImplementedError('Only output to stdout is currently implemented.')
         return run_legacy_script('specialphrases.php', '--wiki-import', nominatim_env=args)
 


@@ -270,6 +282,7 @@ class UpdateAddData:
     @staticmethod
     def run(args):
         if args.tiger_data:
     @staticmethod
     def run(args):
         if args.tiger_data:

+            os.environ['NOMINATIM_TIGER_DATA_PATH'] = args.tiger_data

             return run_legacy_script('setup.php', '--import-tiger-data', nominatim_env=args)
 
         params = ['update.php']
             return run_legacy_script('setup.php', '--import-tiger-data', nominatim_env=args)
 
         params = ['update.php']


@@ -295,16 +308,35 @@ class UpdateIndex:
 
     @staticmethod
     def add_args(parser):
 
     @staticmethod
     def add_args(parser):

-        pass
+        group = parser.add_argument_group('Filter arguments')
+        group.add_argument('--boundaries-only', action='store_true',
+                           help="""Index only administrative boundaries.""")
+        group.add_argument('--no-boundaries', action='store_true',
+                           help="""Index everything except administrative boundaries.""")
+        group.add_argument('--minrank', '-r', type=int, metavar='RANK', default=0,
+                           help='Minimum/starting rank')
+        group.add_argument('--maxrank', '-R', type=int, metavar='RANK', default=30,
+                           help='Maximum/finishing rank')

 
     @staticmethod
     def run(args):
 
     @staticmethod
     def run(args):

-        return run_legacy_script('update.php', '--index', nominatim_env=args)
+        indexer = Indexer(args.config.get_libpq_dsn(),
+                          args.threads or _num_system_cpus() or 1)
+
+        if not args.no_boundaries:
+            indexer.index_boundaries(args.minrank, args.maxrank)
+        if not args.boundaries_only:
+            indexer.index_by_rank(args.minrank, args.maxrank)
+
+        if not args.no_boundaries and not args.boundaries_only:
+            indexer.update_status_table()
+
+        return 0

 
 
 class UpdateRefresh:
     """\
 
 
 class UpdateRefresh:
     """\

-    Recompute auxillary data used by the indexing process.
+    Recompute auxiliary data used by the indexing process.

 
     These functions must not be run in parallel with other update commands.
     """
 
     These functions must not be run in parallel with other update commands.
     """


@@ -318,12 +350,12 @@ class UpdateRefresh:
                            help='Compute frequency of full-word search terms')
         group.add_argument('--address-levels', action='store_true',
                            help='Reimport address level configuration')
                            help='Compute frequency of full-word search terms')
         group.add_argument('--address-levels', action='store_true',
                            help='Reimport address level configuration')

-        group.add_argument('--importance', action='store_true',
-                           help='Recompute place importances (expensive!)')

         group.add_argument('--functions', action='store_true',
                            help='Update the PL/pgSQL functions in the database')
         group.add_argument('--wiki-data', action='store_true',
                            help='Update Wikipedia/data importance numbers.')
         group.add_argument('--functions', action='store_true',
                            help='Update the PL/pgSQL functions in the database')
         group.add_argument('--wiki-data', action='store_true',
                            help='Update Wikipedia/data importance numbers.')

+        group.add_argument('--importance', action='store_true',
+                           help='Recompute place importances (expensive!)')

         group.add_argument('--website', action='store_true',
                            help='Refresh the directory that serves the scripts for the web API')
         group = parser.add_argument_group('Arguments for function refresh')
         group.add_argument('--website', action='store_true',
                            help='Refresh the directory that serves the scripts for the web API')
         group = parser.add_argument_group('Arguments for function refresh')


@@ -343,9 +375,6 @@ class UpdateRefresh:
         if args.address_levels:
             run_legacy_script('update.php', '--update-address-levels',
                               nominatim_env=args, throw_on_fail=True)
         if args.address_levels:
             run_legacy_script('update.php', '--update-address-levels',
                               nominatim_env=args, throw_on_fail=True)

-        if args.importance:
-            run_legacy_script('update.php', '--recompute-importance',
-                              nominatim_env=args, throw_on_fail=True)

         if args.functions:
             params = ['setup.php', '--create-functions', '--create-partition-functions']
             if args.diffs:
         if args.functions:
             params = ['setup.php', '--create-functions', '--create-partition-functions']
             if args.diffs:


@@ -356,9 +385,14 @@ class UpdateRefresh:
         if args.wiki_data:
             run_legacy_script('setup.php', '--import-wikipedia-articles',
                               nominatim_env=args, throw_on_fail=True)
         if args.wiki_data:
             run_legacy_script('setup.php', '--import-wikipedia-articles',
                               nominatim_env=args, throw_on_fail=True)

+        # Attention: importance MUST come after wiki data import.
+        if args.importance:
+            run_legacy_script('update.php', '--recompute-importance',
+                              nominatim_env=args, throw_on_fail=True)

         if args.website:
             run_legacy_script('setup.php', '--setup-website',
                               nominatim_env=args, throw_on_fail=True)
         if args.website:
             run_legacy_script('setup.php', '--setup-website',
                               nominatim_env=args, throw_on_fail=True)

+        return 0

 
 
 class AdminCheckDatabase:
 
 
 class AdminCheckDatabase:


@@ -402,7 +436,7 @@ class AdminWarm:
 
 class QueryExport:
     """\
 
 class QueryExport:
     """\

-    Export addresses as CSV file from a Nominatim database.
+    Export addresses as CSV file from the database.

     """
 
     @staticmethod
     """
 
     @staticmethod


@@ -455,17 +489,245 @@ class QueryExport:
 
         return run_legacy_script(*params, nominatim_env=args)
 
 
         return run_legacy_script(*params, nominatim_env=args)
 

-class QueryTodo:
+STRUCTURED_QUERY = (
+    ('street', 'housenumber and street'),
+    ('city', 'city, town or village'),
+    ('county', 'county'),
+    ('state', 'state'),
+    ('country', 'country'),
+    ('postalcode', 'postcode')
+)
+
+EXTRADATA_PARAMS = (
+    ('addressdetails', 'Include a breakdown of the address into elements.'),
+    ('extratags', """Include additional information if available
+                     (e.g. wikipedia link, opening hours)."""),
+    ('namedetails', 'Include a list of alternative names.')
+)
+
+DETAILS_SWITCHES = (
+    ('addressdetails', 'Include a breakdown of the address into elements.'),
+    ('keywords', 'Include a list of name keywords and address keywords.'),
+    ('linkedplaces', 'Include a details of places that are linked with this one.'),
+    ('hierarchy', 'Include details of places lower in the address hierarchy.'),
+    ('group_hierarchy', 'Group the places by type.'),
+    ('polygon_geojson', 'Include geometry of result.')
+)
+
+def _add_api_output_arguments(parser):
+    group = parser.add_argument_group('Output arguments')
+    group.add_argument('--format', default='jsonv2',
+                       choices=['xml', 'json', 'jsonv2', 'geojson', 'geocodejson'],
+                       help='Format of result')
+    for name, desc in EXTRADATA_PARAMS:
+        group.add_argument('--' + name, action='store_true', help=desc)
+
+    group.add_argument('--lang', '--accept-language', metavar='LANGS',
+                       help='Preferred language order for presenting search results')
+    group.add_argument('--polygon-output',
+                       choices=['geojson', 'kml', 'svg', 'text'],
+                       help='Output geometry of results as a GeoJSON, KML, SVG or WKT.')
+    group.add_argument('--polygon-threshold', type=float, metavar='TOLERANCE',
+                       help="""Simplify output geometry.
+                               Parameter is difference tolerance in degrees.""")
+
+
+class APISearch:

     """\
     """\

-    Todo
+    Execute API search query.

     """
     """

+
+    @staticmethod
+    def add_args(parser):
+        group = parser.add_argument_group('Query arguments')
+        group.add_argument('--query',
+                           help='Free-form query string')
+        for name, desc in STRUCTURED_QUERY:
+            group.add_argument('--' + name, help='Structured query: ' + desc)
+
+        _add_api_output_arguments(parser)
+
+        group = parser.add_argument_group('Result limitation')
+        group.add_argument('--countrycodes', metavar='CC,..',
+                           help='Limit search results to one or more countries.')
+        group.add_argument('--exclude_place_ids', metavar='ID,..',
+                           help='List of search object to be excluded')
+        group.add_argument('--limit', type=int,
+                           help='Limit the number of returned results')
+        group.add_argument('--viewbox', metavar='X1,Y1,X2,Y2',
+                           help='Preferred area to find search results')
+        group.add_argument('--bounded', action='store_true',
+                           help='Strictly restrict results to viewbox area')
+
+        group = parser.add_argument_group('Other arguments')
+        group.add_argument('--no-dedupe', action='store_false', dest='dedupe',
+                           help='Do not remove duplicates from the result list')
+
+
+    @staticmethod
+    def run(args):
+        if args.query:
+            params = dict(q=args.query)
+        else:
+            params = {k : getattr(args, k) for k, _ in STRUCTURED_QUERY if getattr(args, k)}
+
+        for param, _ in EXTRADATA_PARAMS:
+            if getattr(args, param):
+                params[param] = '1'
+        for param in ('format', 'countrycodes', 'exclude_place_ids', 'limit', 'viewbox'):
+            if getattr(args, param):
+                params[param] = getattr(args, param)
+        if args.lang:
+            params['accept-language'] = args.lang
+        if args.polygon_output:
+            params['polygon_' + args.polygon_output] = '1'
+        if args.polygon_threshold:
+            params['polygon_threshold'] = args.polygon_threshold
+        if args.bounded:
+            params['bounded'] = '1'
+        if not args.dedupe:
+            params['dedupe'] = '0'
+
+        return run_api_script('search', args.project_dir,
+                              phpcgi_bin=args.phpcgi_path, params=params)
+
+class APIReverse:
+    """\
+    Execute API reverse query.
+    """
+

     @staticmethod
     def add_args(parser):
     @staticmethod
     def add_args(parser):

-        pass
+        group = parser.add_argument_group('Query arguments')
+        group.add_argument('--lat', type=float, required=True,
+                           help='Latitude of coordinate to look up (in WGS84)')
+        group.add_argument('--lon', type=float, required=True,
+                           help='Longitude of coordinate to look up (in WGS84)')
+        group.add_argument('--zoom', type=int,
+                           help='Level of detail required for the address')
+
+        _add_api_output_arguments(parser)
+

 
     @staticmethod
 
     @staticmethod

-    def run(args): # pylint: disable=W0613
-        print("TODO: searching")
+    def run(args):
+        params = dict(lat=args.lat, lon=args.lon)
+        if args.zoom is not None:
+            params['zoom'] = args.zoom
+
+        for param, _ in EXTRADATA_PARAMS:
+            if getattr(args, param):
+                params[param] = '1'
+        if args.format:
+            params['format'] = args.format
+        if args.lang:
+            params['accept-language'] = args.lang
+        if args.polygon_output:
+            params['polygon_' + args.polygon_output] = '1'
+        if args.polygon_threshold:
+            params['polygon_threshold'] = args.polygon_threshold
+
+        return run_api_script('reverse', args.project_dir,
+                              phpcgi_bin=args.phpcgi_path, params=params)
+
+
+class APILookup:
+    """\
+    Execute API reverse query.
+    """
+
+    @staticmethod
+    def add_args(parser):
+        group = parser.add_argument_group('Query arguments')
+        group.add_argument('--id', metavar='OSMID',
+                           action='append', required=True, dest='ids',
+                           help='OSM id to lookup in format <NRW><id> (may be repeated)')
+
+        _add_api_output_arguments(parser)
+
+
+    @staticmethod
+    def run(args):
+        params = dict(osm_ids=','.join(args.ids))
+
+        for param, _ in EXTRADATA_PARAMS:
+            if getattr(args, param):
+                params[param] = '1'
+        if args.format:
+            params['format'] = args.format
+        if args.lang:
+            params['accept-language'] = args.lang
+        if args.polygon_output:
+            params['polygon_' + args.polygon_output] = '1'
+        if args.polygon_threshold:
+            params['polygon_threshold'] = args.polygon_threshold
+
+        return run_api_script('lookup', args.project_dir,
+                              phpcgi_bin=args.phpcgi_path, params=params)
+
+
+class APIDetails:
+    """\
+    Execute API lookup query.
+    """
+
+    @staticmethod
+    def add_args(parser):
+        group = parser.add_argument_group('Query arguments')
+        objs = group.add_mutually_exclusive_group(required=True)
+        objs.add_argument('--node', '-n', type=int,
+                          help="Look up the OSM node with the given ID.")
+        objs.add_argument('--way', '-w', type=int,
+                          help="Look up the OSM way with the given ID.")
+        objs.add_argument('--relation', '-r', type=int,
+                          help="Look up the OSM relation with the given ID.")
+        objs.add_argument('--place_id', '-p', type=int,
+                          help='Database internal identifier of the OSM object to look up.')
+        group.add_argument('--class', dest='object_class',
+                           help="""Class type to disambiguated multiple entries
+                                   of the same object.""")
+
+        group = parser.add_argument_group('Output arguments')
+        for name, desc in DETAILS_SWITCHES:
+            group.add_argument('--' + name, action='store_true', help=desc)
+        group.add_argument('--lang', '--accept-language', metavar='LANGS',
+                           help='Preferred language order for presenting search results')
+
+    @staticmethod
+    def run(args):
+        if args.node:
+            params = dict(osmtype='N', osmid=args.node)
+        elif args.way:
+            params = dict(osmtype='W', osmid=args.node)
+        elif args.relation:
+            params = dict(osmtype='R', osmid=args.node)
+        else:
+            params = dict(place_id=args.place_id)
+        if args.object_class:
+            params['class'] = args.object_class
+        for name, _ in DETAILS_SWITCHES:
+            params[name] = '1' if getattr(args, name) else '0'
+
+        return run_api_script('details', args.project_dir,
+                              phpcgi_bin=args.phpcgi_path, params=params)
+
+
+class APIStatus:
+    """\
+    Execute API status query.
+    """
+
+    @staticmethod
+    def add_args(parser):
+        group = parser.add_argument_group('API parameters')
+        group.add_argument('--format', default='text', choices=['text', 'json'],
+                           help='Format of result')
+
+    @staticmethod
+    def run(args):
+        return run_api_script('status', args.project_dir,
+                              phpcgi_bin=args.phpcgi_path,
+                              params=dict(format=args.format))

 
 
 def nominatim(**kwargs):
 
 
 def nominatim(**kwargs):


@@ -489,10 +751,14 @@ def nominatim(**kwargs):
     parser.add_subcommand('refresh', UpdateRefresh)
 
     parser.add_subcommand('export', QueryExport)
     parser.add_subcommand('refresh', UpdateRefresh)
 
     parser.add_subcommand('export', QueryExport)

-    parser.add_subcommand('search', QueryTodo)
-    parser.add_subcommand('reverse', QueryTodo)
-    parser.add_subcommand('lookup', QueryTodo)
-    parser.add_subcommand('details', QueryTodo)
-    parser.add_subcommand('status', QueryTodo)
+
+    if kwargs.get('phpcgi_path'):
+        parser.add_subcommand('search', APISearch)
+        parser.add_subcommand('reverse', APIReverse)
+        parser.add_subcommand('lookup', APILookup)
+        parser.add_subcommand('details', APIDetails)
+        parser.add_subcommand('status', APIStatus)
+    else:
+        parser.parser.epilog = 'php-cgi not found. Query commands not available.'

 
     return parser.run(**kwargs)
 
     return parser.run(**kwargs)