probe for php_cgi in cmake to be used for querying

[nominatim.git] / nominatim / cli.py

diff --git a/nominatim/cli.py b/nominatim/cli.py
index a40dc07ea07f0375b956f5ca2267448fd84216f1..4fadaa9f76c64cc3f4123912191006eab2d2b9c0 100644 (file)
--- a/nominatim/cli.py
+++ b/nominatim/cli.py
@@ -3,6 +3,7 @@ Command-line interface to the Nominatim functions for import, update,
 database administration and querying.
 """
 import sys
 database administration and querying.
 """
 import sys

+import os

 import argparse
 import logging
 from pathlib import Path
 import argparse
 import logging
 from pathlib import Path


@@ -10,6 +11,17 @@ from pathlib import Path
 from .config import Configuration
 from .admin.exec_utils import run_legacy_script
 
 from .config import Configuration
 from .admin.exec_utils import run_legacy_script
 

+from .indexer.indexer import Indexer
+
+def _num_system_cpus():
+    try:
+        cpus = len(os.sched_getaffinity(0))
+    except NotImplementedError:
+        cpus = None
+
+    return cpus or os.cpu_count()
+
+

 class CommandlineParser:
     """ Wraps some of the common functions for parsing the command line
         and setting up subcommands.
 class CommandlineParser:
     """ Wraps some of the common functions for parsing the command line
         and setting up subcommands.


@@ -66,13 +78,25 @@ class CommandlineParser:
         args.project_dir = Path(args.project_dir)
 
         logging.basicConfig(stream=sys.stderr,
         args.project_dir = Path(args.project_dir)
 
         logging.basicConfig(stream=sys.stderr,

-                            format='%(asctime)s %(levelname)s: %(message)s',
+                            format='%(asctime)s: %(message)s',

                             datefmt='%Y-%m-%d %H:%M:%S',
                             level=max(4 - args.verbose, 1) * 10)
 
         args.config = Configuration(args.project_dir, args.data_dir / 'settings')
 
                             datefmt='%Y-%m-%d %H:%M:%S',
                             level=max(4 - args.verbose, 1) * 10)
 
         args.config = Configuration(args.project_dir, args.data_dir / 'settings')
 

-        args.command.run(args)
+        return args.command.run(args)
+
+##### Subcommand classes
+#
+# Each class needs to implement two functions: add_args() adds the CLI parameters
+# for the subfunction, run() executes the subcommand.
+#
+# The class documentation doubles as the help text for the command. The
+# first line is also used in the summary when calling the program without
+# a subcommand.
+#
+# No need to document the functions each time.
+# pylint: disable=C0111

 
 
 class SetupAll:
 
 
 class SetupAll:


@@ -86,7 +110,7 @@ class SetupAll:
         group = group_name.add_mutually_exclusive_group(required=True)
         group.add_argument('--osm-file',
                            help='OSM file to be imported.')
         group = group_name.add_mutually_exclusive_group(required=True)
         group.add_argument('--osm-file',
                            help='OSM file to be imported.')

-        group.add_argument('--continue', nargs=1,
+        group.add_argument('--continue', dest='continue_at',

                            choices=['load-data', 'indexing', 'db-postprocess'],
                            help='Continue an import that was interrupted')
         group = parser.add_argument_group('Optional arguments')
                            choices=['load-data', 'indexing', 'db-postprocess'],
                            help='Continue an import that was interrupted')
         group = parser.add_argument_group('Optional arguments')


@@ -105,15 +129,38 @@ class SetupAll:
         group = parser.add_argument_group('Expert options')
         group.add_argument('--ignore-errors', action='store_true',
                            help='Continue import even when errors in SQL are present')
         group = parser.add_argument_group('Expert options')
         group.add_argument('--ignore-errors', action='store_true',
                            help='Continue import even when errors in SQL are present')

-        group.add_argument('--disable-token-precalc', action='store_true',
-                           help='Disable name precalculation')

         group.add_argument('--index-noanalyse', action='store_true',
                            help='Do not perform analyse operations during index')
 
 
     @staticmethod
     def run(args):
         group.add_argument('--index-noanalyse', action='store_true',
                            help='Do not perform analyse operations during index')
 
 
     @staticmethod
     def run(args):

-        print("TODO: ./utils/setup.php", args)
+        params = ['setup.php']
+        if args.osm_file:
+            params.extend(('--all', '--osm-file', args.osm_file))
+        else:
+            if args.continue_at == 'load-data':
+                params.append('--load-data')
+            if args.continue_at in ('load-data', 'indexing'):
+                params.append('--index')
+            params.extend(('--create-search-indices', '--create-country-names',
+                           '--setup-website'))
+        if args.osm2pgsql_cache:
+            params.extend(('--osm2pgsql-cache', args.osm2pgsql_cache))
+        if args.reverse_only:
+            params.append('--reverse-only')
+        if args.enable_debug_statements:
+            params.append('--enable-debug-statements')
+        if args.no_partitions:
+            params.append('--no-partitions')
+        if args.no_updates:
+            params.append('--drop')
+        if args.ignore_errors:
+            params.append('--ignore-errors')
+        if args.index_noanalyse:
+            params.append('--index-noanalyse')
+
+        return run_legacy_script(*params, nominatim_env=args)

 
 
 class SetupFreeze:
 
 
 class SetupFreeze:


@@ -134,7 +181,7 @@ class SetupFreeze:
 
     @staticmethod
     def run(args):
 
     @staticmethod
     def run(args):

-        print("TODO: setup drop", args)
+        return run_legacy_script('setup.php', '--drop', nominatim_env=args)

 
 
 class SetupSpecialPhrases:
 
 
 class SetupSpecialPhrases:


@@ -155,7 +202,9 @@ class SetupSpecialPhrases:
 
     @staticmethod
     def run(args):
 
     @staticmethod
     def run(args):

-        print("./utils/specialphrases.php --from-wiki", args)
+        if args.output.name != '<stdout>':
+            raise NotImplementedError('Only output to stdout is currently implemented.')
+        return run_legacy_script('specialphrases.php', '--wiki-import', nominatim_env=args)

 
 
 class UpdateReplication:
 
 
 class UpdateReplication:


@@ -184,10 +233,22 @@ class UpdateReplication:
 
     @staticmethod
     def run(args):
 
     @staticmethod
     def run(args):

+        params = ['update.php']

         if args.init:
         if args.init:

-            print('./utils/update.php --init-updates', args)
+            params.append('--init-updates')
+            if not args.update_functions:
+                params.append('--no-update-functions')
+        elif args.check_for_updates:
+            params.append('--check-for-updates')

         else:
         else:

-            print('./utils/update.php --import-osmosis(-all)', args)
+            if args.once:
+                params.append('--import-osmosis')
+            else:
+                params.append('--import-osmosis-all')
+            if not args.do_index:
+                params.append('--no-index')
+
+        return run_legacy_script(*params, nominatim_env=args)

 
 
 class UpdateAddData:
 
 
 class UpdateAddData:


@@ -220,7 +281,24 @@ class UpdateAddData:
 
     @staticmethod
     def run(args):
 
     @staticmethod
     def run(args):

-        print('./utils/update.php --import-*', args)
+        if args.tiger_data:
+            os.environ['NOMINATIM_TIGER_DATA_PATH'] = args.tiger_data
+            return run_legacy_script('setup.php', '--import-tiger-data', nominatim_env=args)
+
+        params = ['update.php']
+        if args.file:
+            params.extend(('--import-file', args.file))
+        elif args.diff:
+            params.extend(('--import-diff', args.diff))
+        elif args.node:
+            params.extend(('--import-node', args.node))
+        elif args.way:
+            params.extend(('--import-way', args.way))
+        elif args.relation:
+            params.extend(('--import-relation', args.relation))
+        if args.use_main_api:
+            params.append('--use-main-api')
+        return run_legacy_script(*params, nominatim_env=args)

 
 
 class UpdateIndex:
 
 
 class UpdateIndex:


@@ -230,16 +308,35 @@ class UpdateIndex:
 
     @staticmethod
     def add_args(parser):
 
     @staticmethod
     def add_args(parser):

-        pass
+        group = parser.add_argument_group('Filter arguments')
+        group.add_argument('--boundaries-only', action='store_true',
+                           help="""Index only administrative boundaries.""")
+        group.add_argument('--no-boundaries', action='store_true',
+                           help="""Index everything except administrative boundaries.""")
+        group.add_argument('--minrank', '-r', type=int, metavar='RANK', default=0,
+                           help='Minimum/starting rank')
+        group.add_argument('--maxrank', '-R', type=int, metavar='RANK', default=30,
+                           help='Maximum/finishing rank')

 
     @staticmethod
     def run(args):
 
     @staticmethod
     def run(args):

-        print('./utils/update.php --index', args)
+        indexer = Indexer(args.config.get_libpq_dsn(),
+                          args.threads or _num_system_cpus() or 1)
+
+        if not args.no_boundaries:
+            indexer.index_boundaries(args.minrank, args.maxrank)
+        if not args.boundaries_only:
+            indexer.index_by_rank(args.minrank, args.maxrank)
+
+        if not args.no_boundaries and not args.boundaries_only:
+            indexer.update_status_table()
+
+        return 0

 
 
 class UpdateRefresh:
     """\
 
 
 class UpdateRefresh:
     """\

-    Recompute auxillary data used by the indexing process.
+    Recompute auxiliary data used by the indexing process.

 
     These functions must not be run in parallel with other update commands.
     """
 
     These functions must not be run in parallel with other update commands.
     """


@@ -253,21 +350,48 @@ class UpdateRefresh:
                            help='Compute frequency of full-word search terms')
         group.add_argument('--address-levels', action='store_true',
                            help='Reimport address level configuration')
                            help='Compute frequency of full-word search terms')
         group.add_argument('--address-levels', action='store_true',
                            help='Reimport address level configuration')

-        group.add_argument('--importance', action='store_true',
-                           help='Recompute place importances')

         group.add_argument('--functions', action='store_true',
                            help='Update the PL/pgSQL functions in the database')
         group.add_argument('--functions', action='store_true',
                            help='Update the PL/pgSQL functions in the database')

-        group.add_argument('--wiki-data',
+        group.add_argument('--wiki-data', action='store_true',

                            help='Update Wikipedia/data importance numbers.')
                            help='Update Wikipedia/data importance numbers.')

+        group.add_argument('--importance', action='store_true',
+                           help='Recompute place importances (expensive!)')

         group.add_argument('--website', action='store_true',
                            help='Refresh the directory that serves the scripts for the web API')
         group = parser.add_argument_group('Arguments for function refresh')
         group.add_argument('--no-diff-updates', action='store_false', dest='diffs',
                            help='Do not enable code for propagating updates')
         group.add_argument('--website', action='store_true',
                            help='Refresh the directory that serves the scripts for the web API')
         group = parser.add_argument_group('Arguments for function refresh')
         group.add_argument('--no-diff-updates', action='store_false', dest='diffs',
                            help='Do not enable code for propagating updates')

+        group.add_argument('--enable-debug-statements', action='store_true',
+                           help='Enable debug warning statements in functions')

 
     @staticmethod
     def run(args):
 
     @staticmethod
     def run(args):

-        print('./utils/update.php', args)
+        if args.postcodes:
+            run_legacy_script('update.php', '--calculate-postcodes',
+                              nominatim_env=args, throw_on_fail=True)
+        if args.word_counts:
+            run_legacy_script('update.php', '--recompute-word-counts',
+                              nominatim_env=args, throw_on_fail=True)
+        if args.address_levels:
+            run_legacy_script('update.php', '--update-address-levels',
+                              nominatim_env=args, throw_on_fail=True)
+        if args.functions:
+            params = ['setup.php', '--create-functions', '--create-partition-functions']
+            if args.diffs:
+                params.append('--enable-diff-updates')
+            if args.enable_debug_statements:
+                params.append('--enable-debug-statements')
+            run_legacy_script(*params, nominatim_env=args, throw_on_fail=True)
+        if args.wiki_data:
+            run_legacy_script('setup.php', '--import-wikipedia-articles',
+                              nominatim_env=args, throw_on_fail=True)
+        # Attention: importance MUST come after wiki data import.
+        if args.importance:
+            run_legacy_script('update.php', '--recompute-importance',
+                              nominatim_env=args, throw_on_fail=True)
+        if args.website:
+            run_legacy_script('setup.php', '--setup-website',
+                              nominatim_env=args, throw_on_fail=True)

 
 
 class AdminCheckDatabase:
 
 
 class AdminCheckDatabase:


@@ -281,7 +405,7 @@ class AdminCheckDatabase:
 
     @staticmethod
     def run(args):
 
     @staticmethod
     def run(args):

-        print("TODO: ./utils/check_import_finished.php", args)
+        return run_legacy_script('check_import_finished.php', nominatim_env=args)

 
 
 class AdminWarm:
 
 
 class AdminWarm:


@@ -311,7 +435,7 @@ class AdminWarm:
 
 class QueryExport:
     """\
 
 class QueryExport:
     """\

-    Export addresses as CSV file from a Nominatim database.
+    Export addresses as CSV file from the database.

     """
 
     @staticmethod
     """
 
     @staticmethod


@@ -346,8 +470,23 @@ class QueryExport:
 
     @staticmethod
     def run(args):
 
     @staticmethod
     def run(args):

-        print("TODO: ./utils/export.php", args)
+        params = ['export.php',
+                  '--output-type', args.output_type,
+                  '--output-format', args.output_format]
+        if args.output_all_postcodes:
+            params.append('--output-all-postcodes')
+        if args.language:
+            params.extend(('--language', args.language))
+        if args.restrict_to_country:
+            params.extend(('--restrict-to-country', args.restrict_to_country))
+        if args.restrict_to_osm_node:
+            params.extend(('--restrict-to-osm-node', args.restrict_to_osm_node))
+        if args.restrict_to_osm_way:
+            params.extend(('--restrict-to-osm-way', args.restrict_to_osm_way))
+        if args.restrict_to_osm_relation:
+            params.extend(('--restrict-to-osm-relation', args.restrict_to_osm_relation))

 
 

+        return run_legacy_script(*params, nominatim_env=args)

 
 class QueryTodo:
     """\
 
 class QueryTodo:
     """\


@@ -357,7 +496,8 @@ class QueryTodo:
     def add_args(parser):
         pass
 
     def add_args(parser):
         pass
 

-    def run(args):
+    @staticmethod
+    def run(args): # pylint: disable=W0613

         print("TODO: searching")
 
 
         print("TODO: searching")
 
 


@@ -382,10 +522,14 @@ def nominatim(**kwargs):
     parser.add_subcommand('refresh', UpdateRefresh)
 
     parser.add_subcommand('export', QueryExport)
     parser.add_subcommand('refresh', UpdateRefresh)
 
     parser.add_subcommand('export', QueryExport)

-    parser.add_subcommand('search', QueryTodo)
-    parser.add_subcommand('reverse', QueryTodo)
-    parser.add_subcommand('lookup', QueryTodo)
-    parser.add_subcommand('details', QueryTodo)
-    parser.add_subcommand('status', QueryTodo)

 
 

-    parser.run(**kwargs)
+    if kwargs.get('phpcgi_path'):
+        parser.add_subcommand('search', QueryTodo)
+        parser.add_subcommand('reverse', QueryTodo)
+        parser.add_subcommand('lookup', QueryTodo)
+        parser.add_subcommand('details', QueryTodo)
+        parser.add_subcommand('status', QueryTodo)
+    else:
+        parser.parser.epilog = 'php-cgi not found. Query commands not available.'
+
+    return parser.run(**kwargs)