Merge pull request #3110 from lonvia/sql-lambda-queries

[nominatim.git] / nominatim / tools / migration.py

diff --git a/nominatim/tools/migration.py b/nominatim/tools/migration.py
index 147a9f9c3c5faa28d1a275ac85437e39cc154f26..ffeb4958f409ce285e7fa8f91ba2d342ffe2cfe9 100644 (file)
--- a/nominatim/tools/migration.py
+++ b/nominatim/tools/migration.py
@@ -15,16 +15,14 @@ from psycopg2 import sql as pysql
 from nominatim.config import Configuration
 from nominatim.db import properties
 from nominatim.db.connection import connect, Connection
-from nominatim.version import NOMINATIM_VERSION, version_str
+from nominatim.version import NominatimVersion, NOMINATIM_VERSION, parse_version
 from nominatim.tools import refresh
 from nominatim.tokenizer import factory as tokenizer_factory
 from nominatim.errors import UsageError
 
 LOG = logging.getLogger()
 
-VersionTuple = Tuple[int, int, int, int]
-
-_MIGRATION_FUNCTIONS : List[Tuple[VersionTuple, Callable[..., None]]] = []
+_MIGRATION_FUNCTIONS : List[Tuple[NominatimVersion, Callable[..., None]]] = []
 
 def migrate(config: Configuration, paths: Any) -> int:
     """ Check for the current database version and execute migrations,
@@ -37,8 +35,7 @@ def migrate(config: Configuration, paths: Any) -> int:
             db_version_str = None
 
         if db_version_str is not None:
-            parts = db_version_str.split('.')
-            db_version = tuple(int(x) for x in parts[:2] + parts[2].split('-'))
+            db_version = parse_version(db_version_str)
 
             if db_version == NOMINATIM_VERSION:
                 LOG.warning("Database already at latest version (%s)", db_version_str)
@@ -49,31 +46,28 @@ def migrate(config: Configuration, paths: Any) -> int:
             db_version = _guess_version(conn)
 
 
-        has_run_migration = False
         for version, func in _MIGRATION_FUNCTIONS:
-            if db_version <= version:
+            if db_version < version or \
+               (db_version == (3, 5, 0, 99) and version == (3, 5, 0, 99)):
                 title = func.__doc__ or ''
-                LOG.warning("Running: %s (%s)", title.split('\n', 1)[0],
-                            version_str(version))
+                LOG.warning("Running: %s (%s)", title.split('\n', 1)[0], version)
                 kwargs = dict(conn=conn, config=config, paths=paths)
                 func(**kwargs)
                 conn.commit()
-                has_run_migration = True
 
-        if has_run_migration:
-            LOG.warning('Updating SQL functions.')
-            refresh.create_functions(conn, config)
-            tokenizer = tokenizer_factory.get_tokenizer_for_db(config)
-            tokenizer.update_sql_functions(config)
+        LOG.warning('Updating SQL functions.')
+        refresh.create_functions(conn, config)
+        tokenizer = tokenizer_factory.get_tokenizer_for_db(config)
+        tokenizer.update_sql_functions(config)
 
-        properties.set_property(conn, 'database_version', version_str())
+        properties.set_property(conn, 'database_version', str(NOMINATIM_VERSION))
 
         conn.commit()
 
     return 0
 
 
-def _guess_version(conn: Connection) -> VersionTuple:
+def _guess_version(conn: Connection) -> NominatimVersion:
     """ Guess a database version when there is no property table yet.
         Only migrations for 3.6 and later are supported, so bail out
         when the version seems older.
@@ -89,7 +83,7 @@ def _guess_version(conn: Connection) -> VersionTuple:
                       'prior to 3.6.0. Automatic migration not possible.')
             raise UsageError('Migration not possible.')
 
-    return (3, 5, 0, 99)
+    return NominatimVersion(3, 5, 0, 99)
 
 
 
@@ -108,7 +102,8 @@ def _migration(major: int, minor: int, patch: int = 0,
         there.
     """
     def decorator(func: Callable[..., None]) -> Callable[..., None]:
-        _MIGRATION_FUNCTIONS.append(((major, minor, patch, dbpatch), func))
+        version = NominatimVersion(major, minor, patch, dbpatch)
+        _MIGRATION_FUNCTIONS.append((version, func))
         return func
 
     return decorator
@@ -348,3 +343,42 @@ def split_pending_index(conn: Connection, **_: Any) -> None:
                            WHERE class = 'boundary' and type = 'administrative'
                                  and indexed_status > 0""")
             cur.execute("DROP INDEX IF EXISTS idx_placex_pendingsector")
+
+
+@_migration(4, 2, 99, 0)
+def enable_forward_dependencies(conn: Connection, **_: Any) -> None:
+    """ Create indexes for updates with forward dependency tracking (long-running).
+    """
+    if conn.table_exists('planet_osm_ways'):
+        with conn.cursor() as cur:
+            cur.execute("""SELECT * FROM pg_indexes
+                           WHERE tablename = 'planet_osm_ways'
+                                 and indexdef LIKE '%nodes%'""")
+            if cur.rowcount == 0:
+                cur.execute("""CREATE OR REPLACE FUNCTION public.planet_osm_index_bucket(bigint[])
+                               RETURNS bigint[]
+                               LANGUAGE sql IMMUTABLE
+                                AS $function$
+                                  SELECT ARRAY(SELECT DISTINCT unnest($1) >> 5)
+                                $function$""")
+                cur.execute("""CREATE INDEX planet_osm_ways_nodes_bucket_idx
+                                 ON planet_osm_ways
+                                 USING gin (planet_osm_index_bucket(nodes))
+                                 WITH (fastupdate=off)""")
+                cur.execute("""CREATE INDEX planet_osm_rels_parts_idx
+                                 ON planet_osm_rels USING gin (parts)
+                                 WITH (fastupdate=off)""")
+                cur.execute("ANALYZE planet_osm_ways")
+
+
+@_migration(4, 2, 99, 1)
+def add_improved_geometry_reverse_placenode_index(conn: Connection, **_: Any) -> None:
+    """ Create improved index for reverse lookup of place nodes.
+    """
+    with conn.cursor() as cur:
+        cur.execute("""CREATE INDEX IF NOT EXISTS idx_placex_geometry_reverse_lookupPlaceNode
+                       ON placex
+                       USING gist (ST_Buffer(geometry, reverse_place_diameter(rank_search)))
+                       WHERE rank_address between 4 and 25 AND type != 'postcode'
+                         AND name is not null AND linked_place_id is null AND osm_type = 'N'
+                    """)