From 764a41b9739dd3221167976adc853f2453dc9119 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Tue, 2 Mar 2021 21:26:13 +0100 Subject: [PATCH] automatic migration from 3.6 release Adds a 'admin --migrate' command that checks for the current database version and runs any necessary migrations. Also has migrations going back to 3.6. --- nominatim/clicmd/admin.py | 27 ++++--- nominatim/tools/migration.py | 132 +++++++++++++++++++++++++++++++++++ 2 files changed, 150 insertions(+), 9 deletions(-) create mode 100644 nominatim/tools/migration.py diff --git a/nominatim/clicmd/admin.py b/nominatim/clicmd/admin.py index fd9382eb..03d7ca8a 100644 --- a/nominatim/clicmd/admin.py +++ b/nominatim/clicmd/admin.py @@ -20,13 +20,16 @@ class AdminFuncs: @staticmethod def add_args(parser): - group = parser.add_argument_group('Admin task arguments') - group.add_argument('--warm', action='store_true', - help='Warm database caches for search and reverse queries.') - group.add_argument('--check-database', action='store_true', - help='Check that the database is complete and operational.') - group.add_argument('--analyse-indexing', action='store_true', - help='Print performance analysis of the indexing process.') + group = parser.add_argument_group('Admin tasks') + objs = group.add_mutually_exclusive_group(required=True) + objs.add_argument('--warm', action='store_true', + help='Warm database caches for search and reverse queries.') + objs.add_argument('--check-database', action='store_true', + help='Check that the database is complete and operational.') + objs.add_argument('--migrate', action='store_true', + help='Migrate the database to a new software version.') + objs.add_argument('--analyse-indexing', action='store_true', + help='Print performance analysis of the indexing process.') group = parser.add_argument_group('Arguments for cache warming') group.add_argument('--search-only', action='store_const', dest='target', const='search', @@ -44,7 +47,7 @@ class AdminFuncs: @staticmethod def run(args): if args.warm: - AdminFuncs._warm(args) + return AdminFuncs._warm(args) if args.check_database: LOG.warning('Checking database') @@ -56,8 +59,14 @@ class AdminFuncs: from ..tools import admin with connect(args.config.get_libpq_dsn()) as conn: admin.analyse_indexing(conn, osm_id=args.osm_id, place_id=args.place_id) + return 0 - return 0 + if args.migrate: + LOG.warning('Checking for necessary database migrations') + from ..tools import migration + return migration.migrate(args.config, args) + + return 1 @staticmethod diff --git a/nominatim/tools/migration.py b/nominatim/tools/migration.py new file mode 100644 index 00000000..756a1d3a --- /dev/null +++ b/nominatim/tools/migration.py @@ -0,0 +1,132 @@ +""" +Functions for database migration to newer software versions. +""" +import logging + +from ..db import properties +from ..db.connection import connect +from ..version import NOMINATIM_VERSION +from . import refresh, database_import +from ..errors import UsageError + +LOG = logging.getLogger() + +_MIGRATION_FUNCTIONS = [] + +def migrate(config, paths): + """ Check for the current database version and execute migrations, + if necesssary. + """ + with connect(config.get_libpq_dsn()) as conn: + if conn.table_exists('nominatim_properties'): + db_version_str = properties.get_property(conn, 'database_version') + else: + db_version_str = None + + if db_version_str is not None: + parts = db_version_str.split('.') + db_version = tuple([int(x) for x in parts[:2] + parts[2].split('-')]) + + if db_version == NOMINATIM_VERSION: + LOG.warning("Database already at latest version (%s)", db_version_str) + return 0 + + LOG.info("Detected database version: %s", db_version_str) + else: + db_version = _guess_version(conn) + + + has_run_migration = False + for version, func in _MIGRATION_FUNCTIONS: + if db_version <= version: + LOG.warning("Runnning: %s (%s)", func.__doc__.split('\n', 1)[0], + '{0[0]}.{0[1]}.{0[2]}-{0[3]}'.format(version)) + kwargs = dict(conn=conn, config=config, paths=paths) + func(**kwargs) + has_run_migration = True + + if has_run_migration: + LOG.warning('Updating SQL functions.') + refresh.create_functions(conn, config, paths.sqllib_dir) + + properties.set_property(conn, 'database_version', + '{0[0]}.{0[1]}.{0[2]}-{0[3]}'.format(NOMINATIM_VERSION)) + + conn.commit() + + return 0 + + +def _guess_version(conn): + """ Guess a database version when there is no property table yet. + Only migrations for 3.6 and later are supported, so bail out + when the version seems older. + """ + with conn.cursor() as cur: + # In version 3.6, the country_name table was updated. Check for that. + cnt = cur.scalar("""SELECT count(*) FROM + (SELECT svals(name) FROM country_name + WHERE country_code = 'gb')x; + """) + if cnt < 100: + LOG.fatal('It looks like your database was imported with a version ' + 'prior to 3.6.0. Automatic migration not possible.') + raise UsageError('Migration not possible.') + + return (3, 5, 0, 99) + + + +def _migration(major, minor, patch=0, dbpatch=0): + """ Decorator for a single migration step. The parameters describe the + version after which the migration is applicable, i.e before changing + from the given version to the next, the migration is required. + + All migrations are run in the order in which they are defined in this + file. Do not run global SQL scripts for migrations as you cannot be sure + that these scripts do the same in later versions. + + Functions will always be reimported in full at the end of the migration + process, so the migration functions may leave a temporary state behind + there. + """ + def decorator(func): + _MIGRATION_FUNCTIONS.append(((major, minor, patch, dbpatch), func)) + + return decorator + + +@_migration(3, 5, 0, 99) +def import_status_timestamp_change(conn, **_): + """ Add timezone to timestamp in status table. + + The import_status table has been changed to include timezone information + with the time stamp. + """ + with conn.cursor() as cur: + cur.execute("""ALTER TABLE import_status ALTER COLUMN lastimportdate + TYPE timestamp with time zone;""") + + +@_migration(3, 5, 0, 99) +def install_database_module_in_project_directory(conn, config, paths, **_): + """ Install database module in project directory. + + The database module needs to be present in the project directory + since those were introduced. + """ + database_import.install_module(paths.module_dir, paths.project_dir, + config.DATABASE_MODULE_PATH, conn=conn) + + +@_migration(3, 5, 0, 99) +def add_nominatim_property_table(conn, config, **_): + """ Add nominatim_property table. + """ + if not conn.table_exists('nominatim_properties'): + with conn.cursor() as cur: + cur.execute("""CREATE TABLE nominatim_properties ( + property TEXT, + value TEXT); + GRANT SELECT ON TABLE nominatim_properties TO "{}"; + """.format(config.DATABASE_WEBUSER)) -- 2.39.5