added subcommand to clean deleted relations for issue # 2444

[nominatim.git] / nominatim / tools / migration.py

diff --git a/nominatim/tools/migration.py b/nominatim/tools/migration.py
index 147a9f9c3c5faa28d1a275ac85437e39cc154f26..ffeb4958f409ce285e7fa8f91ba2d342ffe2cfe9 100644 (file)
--- a/nominatim/tools/migration.py
+++ b/nominatim/tools/migration.py
@@ -15,16 +15,14 @@ from psycopg2 import sql as pysql
 from nominatim.config import Configuration
 from nominatim.db import properties
 from nominatim.db.connection import connect, Connection
 from nominatim.config import Configuration
 from nominatim.db import properties
 from nominatim.db.connection import connect, Connection

-from nominatim.version import NOMINATIM_VERSION, version_str
+from nominatim.version import NominatimVersion, NOMINATIM_VERSION, parse_version

 from nominatim.tools import refresh
 from nominatim.tokenizer import factory as tokenizer_factory
 from nominatim.errors import UsageError
 
 LOG = logging.getLogger()
 
 from nominatim.tools import refresh
 from nominatim.tokenizer import factory as tokenizer_factory
 from nominatim.errors import UsageError
 
 LOG = logging.getLogger()
 

-VersionTuple = Tuple[int, int, int, int]
-
-_MIGRATION_FUNCTIONS : List[Tuple[VersionTuple, Callable[..., None]]] = []
+_MIGRATION_FUNCTIONS : List[Tuple[NominatimVersion, Callable[..., None]]] = []

 
 def migrate(config: Configuration, paths: Any) -> int:
     """ Check for the current database version and execute migrations,
 
 def migrate(config: Configuration, paths: Any) -> int:
     """ Check for the current database version and execute migrations,


@@ -37,8 +35,7 @@ def migrate(config: Configuration, paths: Any) -> int:
             db_version_str = None
 
         if db_version_str is not None:
             db_version_str = None
 
         if db_version_str is not None:

-            parts = db_version_str.split('.')
-            db_version = tuple(int(x) for x in parts[:2] + parts[2].split('-'))
+            db_version = parse_version(db_version_str)

 
             if db_version == NOMINATIM_VERSION:
                 LOG.warning("Database already at latest version (%s)", db_version_str)
 
             if db_version == NOMINATIM_VERSION:
                 LOG.warning("Database already at latest version (%s)", db_version_str)


@@ -49,31 +46,28 @@ def migrate(config: Configuration, paths: Any) -> int:
             db_version = _guess_version(conn)
 
 
             db_version = _guess_version(conn)
 
 

-        has_run_migration = False

         for version, func in _MIGRATION_FUNCTIONS:
         for version, func in _MIGRATION_FUNCTIONS:

-            if db_version <= version:
+            if db_version < version or \
+               (db_version == (3, 5, 0, 99) and version == (3, 5, 0, 99)):

                 title = func.__doc__ or ''
                 title = func.__doc__ or ''

-                LOG.warning("Running: %s (%s)", title.split('\n', 1)[0],
-                            version_str(version))
+                LOG.warning("Running: %s (%s)", title.split('\n', 1)[0], version)

                 kwargs = dict(conn=conn, config=config, paths=paths)
                 func(**kwargs)
                 conn.commit()
                 kwargs = dict(conn=conn, config=config, paths=paths)
                 func(**kwargs)
                 conn.commit()

-                has_run_migration = True

 
 

-        if has_run_migration:
-            LOG.warning('Updating SQL functions.')
-            refresh.create_functions(conn, config)
-            tokenizer = tokenizer_factory.get_tokenizer_for_db(config)
-            tokenizer.update_sql_functions(config)
+        LOG.warning('Updating SQL functions.')
+        refresh.create_functions(conn, config)
+        tokenizer = tokenizer_factory.get_tokenizer_for_db(config)
+        tokenizer.update_sql_functions(config)

 
 

-        properties.set_property(conn, 'database_version', version_str())
+        properties.set_property(conn, 'database_version', str(NOMINATIM_VERSION))

 
         conn.commit()
 
     return 0
 
 
 
         conn.commit()
 
     return 0
 
 

-def _guess_version(conn: Connection) -> VersionTuple:
+def _guess_version(conn: Connection) -> NominatimVersion:

     """ Guess a database version when there is no property table yet.
         Only migrations for 3.6 and later are supported, so bail out
         when the version seems older.
     """ Guess a database version when there is no property table yet.
         Only migrations for 3.6 and later are supported, so bail out
         when the version seems older.


@@ -89,7 +83,7 @@ def _guess_version(conn: Connection) -> VersionTuple:
                       'prior to 3.6.0. Automatic migration not possible.')
             raise UsageError('Migration not possible.')
 
                       'prior to 3.6.0. Automatic migration not possible.')
             raise UsageError('Migration not possible.')
 

-    return (3, 5, 0, 99)
+    return NominatimVersion(3, 5, 0, 99)

 
 
 
 
 
 


@@ -108,7 +102,8 @@ def _migration(major: int, minor: int, patch: int = 0,
         there.
     """
     def decorator(func: Callable[..., None]) -> Callable[..., None]:
         there.
     """
     def decorator(func: Callable[..., None]) -> Callable[..., None]:

-        _MIGRATION_FUNCTIONS.append(((major, minor, patch, dbpatch), func))
+        version = NominatimVersion(major, minor, patch, dbpatch)
+        _MIGRATION_FUNCTIONS.append((version, func))

         return func
 
     return decorator
         return func
 
     return decorator


@@ -348,3 +343,42 @@ def split_pending_index(conn: Connection, **_: Any) -> None:
                            WHERE class = 'boundary' and type = 'administrative'
                                  and indexed_status > 0""")
             cur.execute("DROP INDEX IF EXISTS idx_placex_pendingsector")
                            WHERE class = 'boundary' and type = 'administrative'
                                  and indexed_status > 0""")
             cur.execute("DROP INDEX IF EXISTS idx_placex_pendingsector")

+
+
+@_migration(4, 2, 99, 0)
+def enable_forward_dependencies(conn: Connection, **_: Any) -> None:
+    """ Create indexes for updates with forward dependency tracking (long-running).
+    """
+    if conn.table_exists('planet_osm_ways'):
+        with conn.cursor() as cur:
+            cur.execute("""SELECT * FROM pg_indexes
+                           WHERE tablename = 'planet_osm_ways'
+                                 and indexdef LIKE '%nodes%'""")
+            if cur.rowcount == 0:
+                cur.execute("""CREATE OR REPLACE FUNCTION public.planet_osm_index_bucket(bigint[])
+                               RETURNS bigint[]
+                               LANGUAGE sql IMMUTABLE
+                                AS $function$
+                                  SELECT ARRAY(SELECT DISTINCT unnest($1) >> 5)
+                                $function$""")
+                cur.execute("""CREATE INDEX planet_osm_ways_nodes_bucket_idx
+                                 ON planet_osm_ways
+                                 USING gin (planet_osm_index_bucket(nodes))
+                                 WITH (fastupdate=off)""")
+                cur.execute("""CREATE INDEX planet_osm_rels_parts_idx
+                                 ON planet_osm_rels USING gin (parts)
+                                 WITH (fastupdate=off)""")
+                cur.execute("ANALYZE planet_osm_ways")
+
+
+@_migration(4, 2, 99, 1)
+def add_improved_geometry_reverse_placenode_index(conn: Connection, **_: Any) -> None:
+    """ Create improved index for reverse lookup of place nodes.
+    """
+    with conn.cursor() as cur:
+        cur.execute("""CREATE INDEX IF NOT EXISTS idx_placex_geometry_reverse_lookupPlaceNode
+                       ON placex
+                       USING gist (ST_Buffer(geometry, reverse_place_diameter(rank_search)))
+                       WHERE rank_address between 4 and 25 AND type != 'postcode'
+                         AND name is not null AND linked_place_id is null AND osm_type = 'N'
+                    """)