flez: add other default styles

[nominatim.git] / nominatim / tools / migration.py

diff --git a/nominatim/tools/migration.py b/nominatim/tools/migration.py
index 8eaad2f70a20e914ff5dfa503d65ecb886745159..10bca15c117c78e0210f82aaf68aab0ad96b6920 100644 (file)
--- a/nominatim/tools/migration.py
+++ b/nominatim/tools/migration.py
@@ -7,12 +7,14 @@
 """
 Functions for database migration to newer software versions.
 """
 """
 Functions for database migration to newer software versions.
 """

+from typing import List, Tuple, Callable, Any

 import logging
 
 from psycopg2 import sql as pysql
 
 import logging
 
 from psycopg2 import sql as pysql
 

+from nominatim.config import Configuration

 from nominatim.db import properties
 from nominatim.db import properties

-from nominatim.db.connection import connect
+from nominatim.db.connection import connect, Connection

 from nominatim.version import NOMINATIM_VERSION, version_str
 from nominatim.tools import refresh
 from nominatim.tokenizer import factory as tokenizer_factory
 from nominatim.version import NOMINATIM_VERSION, version_str
 from nominatim.tools import refresh
 from nominatim.tokenizer import factory as tokenizer_factory


@@ -20,9 +22,11 @@ from nominatim.errors import UsageError
 
 LOG = logging.getLogger()
 
 
 LOG = logging.getLogger()
 

-_MIGRATION_FUNCTIONS = []
+VersionTuple = Tuple[int, int, int, int]

 
 

-def migrate(config, paths):
+_MIGRATION_FUNCTIONS : List[Tuple[VersionTuple, Callable[..., None]]] = []
+
+def migrate(config: Configuration, paths: Any) -> int:

     """ Check for the current database version and execute migrations,
         if necesssary.
     """
     """ Check for the current database version and execute migrations,
         if necesssary.
     """


@@ -48,7 +52,8 @@ def migrate(config, paths):
         has_run_migration = False
         for version, func in _MIGRATION_FUNCTIONS:
             if db_version <= version:
         has_run_migration = False
         for version, func in _MIGRATION_FUNCTIONS:
             if db_version <= version:

-                LOG.warning("Runnning: %s (%s)", func.__doc__.split('\n', 1)[0],
+                title = func.__doc__ or ''
+                LOG.warning("Running: %s (%s)", title.split('\n', 1)[0],

                             version_str(version))
                 kwargs = dict(conn=conn, config=config, paths=paths)
                 func(**kwargs)
                             version_str(version))
                 kwargs = dict(conn=conn, config=config, paths=paths)
                 func(**kwargs)


@@ -68,7 +73,7 @@ def migrate(config, paths):
     return 0
 
 
     return 0
 
 

-def _guess_version(conn):
+def _guess_version(conn: Connection) -> VersionTuple:

     """ Guess a database version when there is no property table yet.
         Only migrations for 3.6 and later are supported, so bail out
         when the version seems older.
     """ Guess a database version when there is no property table yet.
         Only migrations for 3.6 and later are supported, so bail out
         when the version seems older.


@@ -88,7 +93,8 @@ def _guess_version(conn):
 
 
 
 
 
 

-def _migration(major, minor, patch=0, dbpatch=0):
+def _migration(major: int, minor: int, patch: int = 0,
+               dbpatch: int = 0) -> Callable[[Callable[..., None]], Callable[..., None]]:

     """ Decorator for a single migration step. The parameters describe the
         version after which the migration is applicable, i.e before changing
         from the given version to the next, the migration is required.
     """ Decorator for a single migration step. The parameters describe the
         version after which the migration is applicable, i.e before changing
         from the given version to the next, the migration is required.


@@ -101,7 +107,7 @@ def _migration(major, minor, patch=0, dbpatch=0):
         process, so the migration functions may leave a temporary state behind
         there.
     """
         process, so the migration functions may leave a temporary state behind
         there.
     """

-    def decorator(func):
+    def decorator(func: Callable[..., None]) -> Callable[..., None]:

         _MIGRATION_FUNCTIONS.append(((major, minor, patch, dbpatch), func))
         return func
 
         _MIGRATION_FUNCTIONS.append(((major, minor, patch, dbpatch), func))
         return func
 


@@ -109,7 +115,7 @@ def _migration(major, minor, patch=0, dbpatch=0):
 
 
 @_migration(3, 5, 0, 99)
 
 
 @_migration(3, 5, 0, 99)

-def import_status_timestamp_change(conn, **_):
+def import_status_timestamp_change(conn: Connection, **_: Any) -> None:

     """ Add timezone to timestamp in status table.
 
         The import_status table has been changed to include timezone information
     """ Add timezone to timestamp in status table.
 
         The import_status table has been changed to include timezone information


@@ -121,7 +127,7 @@ def import_status_timestamp_change(conn, **_):
 
 
 @_migration(3, 5, 0, 99)
 
 
 @_migration(3, 5, 0, 99)

-def add_nominatim_property_table(conn, config, **_):
+def add_nominatim_property_table(conn: Connection, config: Configuration, **_: Any) -> None:

     """ Add nominatim_property table.
     """
     if not conn.table_exists('nominatim_properties'):
     """ Add nominatim_property table.
     """
     if not conn.table_exists('nominatim_properties'):


@@ -130,11 +136,10 @@ def add_nominatim_property_table(conn, config, **_):
                                         property TEXT,
                                         value TEXT);
                                      GRANT SELECT ON TABLE nominatim_properties TO {};
                                         property TEXT,
                                         value TEXT);
                                      GRANT SELECT ON TABLE nominatim_properties TO {};

-                                  """)
-                             .format(pysql.Identifier(config.DATABASE_WEBUSER)))
+                                  """).format(pysql.Identifier(config.DATABASE_WEBUSER)))

 
 @_migration(3, 6, 0, 0)
 
 @_migration(3, 6, 0, 0)

-def change_housenumber_transliteration(conn, **_):
+def change_housenumber_transliteration(conn: Connection, **_: Any) -> None:

     """ Transliterate housenumbers.
 
         The database schema switched from saving raw housenumbers in
     """ Transliterate housenumbers.
 
         The database schema switched from saving raw housenumbers in


@@ -165,7 +170,7 @@ def change_housenumber_transliteration(conn, **_):
 
 
 @_migration(3, 7, 0, 0)
 
 
 @_migration(3, 7, 0, 0)

-def switch_placenode_geometry_index(conn, **_):
+def switch_placenode_geometry_index(conn: Connection, **_: Any) -> None:

     """ Replace idx_placex_geometry_reverse_placeNode index.
 
         Make the index slightly more permissive, so that it can also be used
     """ Replace idx_placex_geometry_reverse_placeNode index.
 
         Make the index slightly more permissive, so that it can also be used


@@ -182,7 +187,7 @@ def switch_placenode_geometry_index(conn, **_):
 
 
 @_migration(3, 7, 0, 1)
 
 
 @_migration(3, 7, 0, 1)

-def install_legacy_tokenizer(conn, config, **_):
+def install_legacy_tokenizer(conn: Connection, config: Configuration, **_: Any) -> None:

     """ Setup legacy tokenizer.
 
         If no other tokenizer has been configured yet, then create the
     """ Setup legacy tokenizer.
 
         If no other tokenizer has been configured yet, then create the


@@ -197,15 +202,15 @@ def install_legacy_tokenizer(conn, config, **_):
                                         (table, ))
                 if has_column == 0:
                     cur.execute(pysql.SQL('ALTER TABLE {} ADD COLUMN token_info JSONB')
                                         (table, ))
                 if has_column == 0:
                     cur.execute(pysql.SQL('ALTER TABLE {} ADD COLUMN token_info JSONB')

-                                     .format(pysql.Identifier(table)))
+                                .format(pysql.Identifier(table)))

         tokenizer = tokenizer_factory.create_tokenizer(config, init_db=False,
                                                        module_name='legacy')
 
         tokenizer = tokenizer_factory.create_tokenizer(config, init_db=False,
                                                        module_name='legacy')
 

-        tokenizer.migrate_database(config)
+        tokenizer.migrate_database(config) # type: ignore[attr-defined]

 
 
 @_migration(4, 0, 99, 0)
 
 
 @_migration(4, 0, 99, 0)

-def create_tiger_housenumber_index(conn, **_):
+def create_tiger_housenumber_index(conn: Connection, **_: Any) -> None:

     """ Create idx_location_property_tiger_parent_place_id with included
         house number.
 
     """ Create idx_location_property_tiger_parent_place_id with included
         house number.
 


@@ -222,7 +227,7 @@ def create_tiger_housenumber_index(conn, **_):
 
 
 @_migration(4, 0, 99, 1)
 
 
 @_migration(4, 0, 99, 1)

-def create_interpolation_index_on_place(conn, **_):
+def create_interpolation_index_on_place(conn: Connection, **_: Any) -> None:

     """ Create idx_place_interpolations for lookup of interpolation lines
         on updates.
     """
     """ Create idx_place_interpolations for lookup of interpolation lines
         on updates.
     """


@@ -233,10 +238,10 @@ def create_interpolation_index_on_place(conn, **_):
 
 
 @_migration(4, 0, 99, 2)
 
 
 @_migration(4, 0, 99, 2)

-def add_step_column_for_interpolation(conn, **_):
+def add_step_column_for_interpolation(conn: Connection, **_: Any) -> None:

     """ Add a new column 'step' to the interpolations table.
 
     """ Add a new column 'step' to the interpolations table.
 

-        Also convers the data into the stricter format which requires that
+        Also converts the data into the stricter format which requires that

         startnumbers comply with the odd/even requirements.
     """
     if conn.table_has_column('location_property_osmline', 'step'):
         startnumbers comply with the odd/even requirements.
     """
     if conn.table_has_column('location_property_osmline', 'step'):


@@ -268,7 +273,7 @@ def add_step_column_for_interpolation(conn, **_):
 
 
 @_migration(4, 0, 99, 3)
 
 
 @_migration(4, 0, 99, 3)

-def add_step_column_for_tiger(conn, **_):
+def add_step_column_for_tiger(conn: Connection, **_: Any) -> None:

     """ Add a new column 'step' to the tiger data table.
     """
     if conn.table_has_column('location_property_tiger', 'step'):
     """ Add a new column 'step' to the tiger data table.
     """
     if conn.table_has_column('location_property_tiger', 'step'):


@@ -283,7 +288,7 @@ def add_step_column_for_tiger(conn, **_):
 
 
 @_migration(4, 0, 99, 4)
 
 
 @_migration(4, 0, 99, 4)

-def add_derived_name_column_for_country_names(conn, **_):
+def add_derived_name_column_for_country_names(conn: Connection, **_: Any) -> None:

     """ Add a new column 'derived_name' which in the future takes the
         country names as imported from OSM data.
     """
     """ Add a new column 'derived_name' which in the future takes the
         country names as imported from OSM data.
     """


@@ -293,7 +298,7 @@ def add_derived_name_column_for_country_names(conn, **_):
 
 
 @_migration(4, 0, 99, 5)
 
 
 @_migration(4, 0, 99, 5)

-def mark_internal_country_names(conn, config, **_):
+def mark_internal_country_names(conn: Connection, config: Configuration, **_: Any) -> None:

     """ Names from the country table should be marked as internal to prevent
         them from being deleted. Only necessary for ICU tokenizer.
     """
     """ Names from the country table should be marked as internal to prevent
         them from being deleted. Only necessary for ICU tokenizer.
     """


@@ -310,3 +315,62 @@ def mark_internal_country_names(conn, config, **_):
                     names = {}
                 names['countrycode'] = country_code
                 analyzer.add_country_names(country_code, names)
                     names = {}
                 names['countrycode'] = country_code
                 analyzer.add_country_names(country_code, names)

+
+
+@_migration(4, 1, 99, 0)
+def add_place_deletion_todo_table(conn: Connection, **_: Any) -> None:
+    """ Add helper table for deleting data on updates.
+
+        The table is only necessary when updates are possible, i.e.
+        the database is not in freeze mode.
+    """
+    if conn.table_exists('place'):
+        with conn.cursor() as cur:
+            cur.execute("""CREATE TABLE IF NOT EXISTS place_to_be_deleted (
+                             osm_type CHAR(1),
+                             osm_id BIGINT,
+                             class TEXT,
+                             type TEXT,
+                             deferred BOOLEAN)""")
+
+
+@_migration(4, 1, 99, 1)
+def split_pending_index(conn: Connection, **_: Any) -> None:
+    """ Reorganise indexes for pending updates.
+    """
+    if conn.table_exists('place'):
+        with conn.cursor() as cur:
+            cur.execute("""CREATE INDEX IF NOT EXISTS idx_placex_rank_address_sector
+                           ON placex USING BTREE (rank_address, geometry_sector)
+                           WHERE indexed_status > 0""")
+            cur.execute("""CREATE INDEX IF NOT EXISTS idx_placex_rank_boundaries_sector
+                           ON placex USING BTREE (rank_search, geometry_sector)
+                           WHERE class = 'boundary' and type = 'administrative'
+                                 and indexed_status > 0""")
+            cur.execute("DROP INDEX IF EXISTS idx_placex_pendingsector")
+
+
+@_migration(4, 2, 99, 0)
+def enable_forward_dependencies(conn: Connection, **_: Any) -> None:
+    """ Create indexes for updates with forward dependency tracking (long-running).
+    """
+    if conn.table_exists('planet_osm_ways'):
+        with conn.cursor() as cur:
+            cur.execute("""SELECT * FROM pg_indexes
+                           WHERE tablename = 'planet_osm_ways'
+                                 and indexdef LIKE '%nodes%'""")
+            if cur.rowcount == 0:
+                cur.execute("""CREATE OR REPLACE FUNCTION public.planet_osm_index_bucket(bigint[])
+                               RETURNS bigint[]
+                               LANGUAGE sql IMMUTABLE
+                                AS $function$
+                                  SELECT ARRAY(SELECT DISTINCT unnest($1) >> 5)
+                                $function$""")
+                cur.execute("""CREATE INDEX planet_osm_ways_nodes_bucket_idx
+                                 ON planet_osm_ways
+                                 USING gin (planet_osm_index_bucket(nodes))
+                                 WITH (fastupdate=off)""")
+                cur.execute("""CREATE INDEX planet_osm_rels_parts_idx
+                                 ON planet_osm_rels USING gin (parts)
+                                 WITH (fastupdate=off)""")
+                cur.execute("ANALYZE planet_osm_ways")