implementaion of 'nominatim index'

[nominatim.git] / nominatim / cli.py

diff --git a/nominatim/cli.py b/nominatim/cli.py
index 62299e9f985eb58518187e8faed26ff1cd4f767c..acb6839fa303937847bc54616f226713149ad836 100644 (file)
--- a/nominatim/cli.py
+++ b/nominatim/cli.py
@@ -3,6 +3,7 @@ Command-line interface to the Nominatim functions for import, update,
 database administration and querying.
 """
 import sys
 database administration and querying.
 """
 import sys

+import os

 import argparse
 import logging
 from pathlib import Path
 import argparse
 import logging
 from pathlib import Path


@@ -10,6 +11,17 @@ from pathlib import Path
 from .config import Configuration
 from .admin.exec_utils import run_legacy_script
 
 from .config import Configuration
 from .admin.exec_utils import run_legacy_script
 

+from .indexer.indexer import Indexer
+
+def _num_system_cpus():
+    try:
+        cpus = len(os.sched_getaffinity(0))
+    except NotImplementedError:
+        cpus = None
+
+    return cpus or os.cpu_count()
+
+

 class CommandlineParser:
     """ Wraps some of the common functions for parsing the command line
         and setting up subcommands.
 class CommandlineParser:
     """ Wraps some of the common functions for parsing the command line
         and setting up subcommands.


@@ -98,7 +110,7 @@ class SetupAll:
         group = group_name.add_mutually_exclusive_group(required=True)
         group.add_argument('--osm-file',
                            help='OSM file to be imported.')
         group = group_name.add_mutually_exclusive_group(required=True)
         group.add_argument('--osm-file',
                            help='OSM file to be imported.')

-        group.add_argument('--continue', nargs=1, dest='continue_at',
+        group.add_argument('--continue', dest='continue_at',

                            choices=['load-data', 'indexing', 'db-postprocess'],
                            help='Continue an import that was interrupted')
         group = parser.add_argument_group('Optional arguments')
                            choices=['load-data', 'indexing', 'db-postprocess'],
                            help='Continue an import that was interrupted')
         group = parser.add_argument_group('Optional arguments')


@@ -270,6 +282,7 @@ class UpdateAddData:
     @staticmethod
     def run(args):
         if args.tiger_data:
     @staticmethod
     def run(args):
         if args.tiger_data:

+            os.environ['NOMINATIM_TIGER_DATA_PATH'] = args.tiger_data

             return run_legacy_script('setup.php', '--import-tiger-data', nominatim_env=args)
 
         params = ['update.php']
             return run_legacy_script('setup.php', '--import-tiger-data', nominatim_env=args)
 
         params = ['update.php']


@@ -295,16 +308,32 @@ class UpdateIndex:
 
     @staticmethod
     def add_args(parser):
 
     @staticmethod
     def add_args(parser):

-        pass
+        group = parser.add_argument_group('Filter arguments')
+        group.add_argument('--boundaries-only', action='store_true',
+                           help="""Index only administrative boundaries.""")
+        group.add_argument('--no-boundaries', action='store_true',
+                           help="""Index everything except administrative boundaries.""")
+        group.add_argument('--minrank', '-r', type=int, metavar='RANK', default=0,
+                           help='Minimum/starting rank')
+        group.add_argument('--maxrank', '-R', type=int, metavar='RANK', default=30,
+                           help='Maximum/finishing rank')

 
     @staticmethod
     def run(args):
 
     @staticmethod
     def run(args):

-        return run_legacy_script('update.php', '--index', nominatim_env=args)
+        indexer = Indexer(args.config.get_libpq_dsn(),
+                          args.threads or _num_system_cpus() or 1)
+
+        if not args.no_boundaries:
+            indexer.index_boundaries(args.minrank, args.maxrank)
+        if not args.boundaries_only:
+            indexer.index_by_rank(args.minrank, args.maxrank)
+
+        return 0

 
 
 class UpdateRefresh:
     """\
 
 
 class UpdateRefresh:
     """\

-    Recompute auxillary data used by the indexing process.
+    Recompute auxiliary data used by the indexing process.

 
     These functions must not be run in parallel with other update commands.
     """
 
     These functions must not be run in parallel with other update commands.
     """


@@ -318,12 +347,12 @@ class UpdateRefresh:
                            help='Compute frequency of full-word search terms')
         group.add_argument('--address-levels', action='store_true',
                            help='Reimport address level configuration')
                            help='Compute frequency of full-word search terms')
         group.add_argument('--address-levels', action='store_true',
                            help='Reimport address level configuration')

-        group.add_argument('--importance', action='store_true',
-                           help='Recompute place importances (expensive!)')

         group.add_argument('--functions', action='store_true',
                            help='Update the PL/pgSQL functions in the database')
         group.add_argument('--wiki-data', action='store_true',
                            help='Update Wikipedia/data importance numbers.')
         group.add_argument('--functions', action='store_true',
                            help='Update the PL/pgSQL functions in the database')
         group.add_argument('--wiki-data', action='store_true',
                            help='Update Wikipedia/data importance numbers.')

+        group.add_argument('--importance', action='store_true',
+                           help='Recompute place importances (expensive!)')

         group.add_argument('--website', action='store_true',
                            help='Refresh the directory that serves the scripts for the web API')
         group = parser.add_argument_group('Arguments for function refresh')
         group.add_argument('--website', action='store_true',
                            help='Refresh the directory that serves the scripts for the web API')
         group = parser.add_argument_group('Arguments for function refresh')


@@ -343,9 +372,6 @@ class UpdateRefresh:
         if args.address_levels:
             run_legacy_script('update.php', '--update-address-levels',
                               nominatim_env=args, throw_on_fail=True)
         if args.address_levels:
             run_legacy_script('update.php', '--update-address-levels',
                               nominatim_env=args, throw_on_fail=True)

-        if args.importance:
-            run_legacy_script('update.php', '--recompute-importance',
-                              nominatim_env=args, throw_on_fail=True)

         if args.functions:
             params = ['setup.php', '--create-functions', '--create-partition-functions']
             if args.diffs:
         if args.functions:
             params = ['setup.php', '--create-functions', '--create-partition-functions']
             if args.diffs:


@@ -356,6 +382,10 @@ class UpdateRefresh:
         if args.wiki_data:
             run_legacy_script('setup.php', '--import-wikipedia-articles',
                               nominatim_env=args, throw_on_fail=True)
         if args.wiki_data:
             run_legacy_script('setup.php', '--import-wikipedia-articles',
                               nominatim_env=args, throw_on_fail=True)

+        # Attention: importance MUST come after wiki data import.
+        if args.importance:
+            run_legacy_script('update.php', '--recompute-importance',
+                              nominatim_env=args, throw_on_fail=True)

         if args.website:
             run_legacy_script('setup.php', '--setup-website',
                               nominatim_env=args, throw_on_fail=True)
         if args.website:
             run_legacy_script('setup.php', '--setup-website',
                               nominatim_env=args, throw_on_fail=True)