Merge remote-tracking branch 'upstream/master'

[nominatim.git] / src / nominatim_db / clicmd / index.py

diff --git a/src/nominatim_db/clicmd/index.py b/src/nominatim_db/clicmd/index.py
index 7766a1d10b4f3a7e9cb968194495bba1092aedac..f1890a657a1fa6f23f3e3bc4209fc1c4a37082c5 100644 (file)
--- a/src/nominatim_db/clicmd/index.py
+++ b/src/nominatim_db/clicmd/index.py
@@ -8,18 +8,14 @@
 Implementation of the 'index' subcommand.
 """
 import argparse
+import asyncio
 
 import psutil
 
-from nominatim_core.db import status
-from nominatim_core.db.connection import connect
+from ..db import status
+from ..db.connection import connect
 from .args import NominatimArgs
 
-# Do not repeat documentation of subcommand classes.
-# pylint: disable=C0111
-# Using non-top-level imports to avoid eventually unused imports.
-# pylint: disable=E0012,C0415
-
 
 class UpdateIndex:
     """\
@@ -42,21 +38,8 @@ class UpdateIndex:
         group.add_argument('--maxrank', '-R', type=int, metavar='RANK', default=30,
                            help='Maximum/finishing rank')
 
-
     def run(self, args: NominatimArgs) -> int:
-        from ..indexer.indexer import Indexer
-        from ..tokenizer import factory as tokenizer_factory
-
-        tokenizer = tokenizer_factory.get_tokenizer_for_db(args.config)
-
-        indexer = Indexer(args.config.get_libpq_dsn(), tokenizer,
-                          args.threads or psutil.cpu_count() or 1)
-
-        if not args.no_boundaries:
-            indexer.index_boundaries(args.minrank, args.maxrank)
-        if not args.boundaries_only:
-            indexer.index_by_rank(args.minrank, args.maxrank)
-            indexer.index_postcodes()
+        asyncio.run(self._do_index(args))
 
         if not args.no_boundaries and not args.boundaries_only \
            and args.minrank == 0 and args.maxrank == 30:
@@ -64,3 +47,21 @@ class UpdateIndex:
                 status.set_indexed(conn, True)
 
         return 0
+
+    async def _do_index(self, args: NominatimArgs) -> None:
+        from ..tokenizer import factory as tokenizer_factory
+
+        tokenizer = tokenizer_factory.get_tokenizer_for_db(args.config)
+        from ..indexer.indexer import Indexer
+
+        indexer = Indexer(args.config.get_libpq_dsn(), tokenizer,
+                          args.threads or psutil.cpu_count() or 1)
+
+        has_pending = True  # run at least once
+        while has_pending:
+            if not args.no_boundaries:
+                await indexer.index_boundaries(args.minrank, args.maxrank)
+            if not args.boundaries_only:
+                await indexer.index_by_rank(args.minrank, args.maxrank)
+                await indexer.index_postcodes()
+            has_pending = indexer.has_pending()