# Database Migrations
-This page describes database migrations necessary to update existing databases
-to newer versions of Nominatim.
+Since version 3.7.0 Nominatim offers automatic migrations. Please follow
+the following steps:
-SQL statements should be executed from the PostgreSQL commandline. Execute
-`psql nominatim` to enter command line mode.
+* stop any updates that are potentially running
+* update Nominatim to the nwer version
+* goto your project directory and run `nominatim admin --migrate`
+* (optionally) restart updates
-## 3.6.0 -> master
-
-### Status table contains now time zone information
+Below you find additional migrations and hints about other structural and
+breaking changes.
-The `import_status` table has been changed to include timezone information
-with the time stamp. You need to alter an existing table before running
-any replication functions with:
+!!! note
+ If you are migrating from a version <3.6, then you still have to follow
+ the manual migration steps up to 3.6.
-```sql
-ALTER TABLE import_status ALTER COLUMN lastimportdate TYPE timestamp with time zone;
-```
+## 3.6.0 -> master
### New location for data files
@staticmethod
def add_args(parser):
- group = parser.add_argument_group('Admin task arguments')
- group.add_argument('--warm', action='store_true',
- help='Warm database caches for search and reverse queries.')
- group.add_argument('--check-database', action='store_true',
- help='Check that the database is complete and operational.')
- group.add_argument('--analyse-indexing', action='store_true',
- help='Print performance analysis of the indexing process.')
+ group = parser.add_argument_group('Admin tasks')
+ objs = group.add_mutually_exclusive_group(required=True)
+ objs.add_argument('--warm', action='store_true',
+ help='Warm database caches for search and reverse queries.')
+ objs.add_argument('--check-database', action='store_true',
+ help='Check that the database is complete and operational.')
+ objs.add_argument('--migrate', action='store_true',
+ help='Migrate the database to a new software version.')
+ objs.add_argument('--analyse-indexing', action='store_true',
+ help='Print performance analysis of the indexing process.')
group = parser.add_argument_group('Arguments for cache warming')
group.add_argument('--search-only', action='store_const', dest='target',
const='search',
@staticmethod
def run(args):
if args.warm:
- AdminFuncs._warm(args)
+ return AdminFuncs._warm(args)
if args.check_database:
LOG.warning('Checking database')
from ..tools import admin
with connect(args.config.get_libpq_dsn()) as conn:
admin.analyse_indexing(conn, osm_id=args.osm_id, place_id=args.place_id)
+ return 0
- return 0
+ if args.migrate:
+ LOG.warning('Checking for necessary database migrations')
+ from ..tools import migration
+ return migration.migrate(args.config, args)
+
+ return 1
@staticmethod
--- /dev/null
+"""
+Functions for database migration to newer software versions.
+"""
+import logging
+
+from ..db import properties
+from ..db.connection import connect
+from ..version import NOMINATIM_VERSION
+from . import refresh, database_import
+from ..errors import UsageError
+
+LOG = logging.getLogger()
+
+_MIGRATION_FUNCTIONS = []
+
+def migrate(config, paths):
+ """ Check for the current database version and execute migrations,
+ if necesssary.
+ """
+ with connect(config.get_libpq_dsn()) as conn:
+ if conn.table_exists('nominatim_properties'):
+ db_version_str = properties.get_property(conn, 'database_version')
+ else:
+ db_version_str = None
+
+ if db_version_str is not None:
+ parts = db_version_str.split('.')
+ db_version = tuple([int(x) for x in parts[:2] + parts[2].split('-')])
+
+ if db_version == NOMINATIM_VERSION:
+ LOG.warning("Database already at latest version (%s)", db_version_str)
+ return 0
+
+ LOG.info("Detected database version: %s", db_version_str)
+ else:
+ db_version = _guess_version(conn)
+
+
+ has_run_migration = False
+ for version, func in _MIGRATION_FUNCTIONS:
+ if db_version <= version:
+ LOG.warning("Runnning: %s (%s)", func.__doc__.split('\n', 1)[0],
+ '{0[0]}.{0[1]}.{0[2]}-{0[3]}'.format(version))
+ kwargs = dict(conn=conn, config=config, paths=paths)
+ func(**kwargs)
+ has_run_migration = True
+
+ if has_run_migration:
+ LOG.warning('Updating SQL functions.')
+ refresh.create_functions(conn, config, paths.sqllib_dir)
+
+ properties.set_property(conn, 'database_version',
+ '{0[0]}.{0[1]}.{0[2]}-{0[3]}'.format(NOMINATIM_VERSION))
+
+ conn.commit()
+
+ return 0
+
+
+def _guess_version(conn):
+ """ Guess a database version when there is no property table yet.
+ Only migrations for 3.6 and later are supported, so bail out
+ when the version seems older.
+ """
+ with conn.cursor() as cur:
+ # In version 3.6, the country_name table was updated. Check for that.
+ cnt = cur.scalar("""SELECT count(*) FROM
+ (SELECT svals(name) FROM country_name
+ WHERE country_code = 'gb')x;
+ """)
+ if cnt < 100:
+ LOG.fatal('It looks like your database was imported with a version '
+ 'prior to 3.6.0. Automatic migration not possible.')
+ raise UsageError('Migration not possible.')
+
+ return (3, 5, 0, 99)
+
+
+
+def _migration(major, minor, patch=0, dbpatch=0):
+ """ Decorator for a single migration step. The parameters describe the
+ version after which the migration is applicable, i.e before changing
+ from the given version to the next, the migration is required.
+
+ All migrations are run in the order in which they are defined in this
+ file. Do not run global SQL scripts for migrations as you cannot be sure
+ that these scripts do the same in later versions.
+
+ Functions will always be reimported in full at the end of the migration
+ process, so the migration functions may leave a temporary state behind
+ there.
+ """
+ def decorator(func):
+ _MIGRATION_FUNCTIONS.append(((major, minor, patch, dbpatch), func))
+
+ return decorator
+
+
+@_migration(3, 5, 0, 99)
+def import_status_timestamp_change(conn, **_):
+ """ Add timezone to timestamp in status table.
+
+ The import_status table has been changed to include timezone information
+ with the time stamp.
+ """
+ with conn.cursor() as cur:
+ cur.execute("""ALTER TABLE import_status ALTER COLUMN lastimportdate
+ TYPE timestamp with time zone;""")
+
+
+@_migration(3, 5, 0, 99)
+def install_database_module_in_project_directory(conn, config, paths, **_):
+ """ Install database module in project directory.
+
+ The database module needs to be present in the project directory
+ since those were introduced.
+ """
+ database_import.install_module(paths.module_dir, paths.project_dir,
+ config.DATABASE_MODULE_PATH, conn=conn)
+
+
+@_migration(3, 5, 0, 99)
+def add_nominatim_property_table(conn, config, **_):
+ """ Add nominatim_property table.
+ """
+ if not conn.table_exists('nominatim_properties'):
+ with conn.cursor() as cur:
+ cur.execute("""CREATE TABLE nominatim_properties (
+ property TEXT,
+ value TEXT);
+ GRANT SELECT ON TABLE nominatim_properties TO "{}";
+ """.format(config.DATABASE_WEBUSER))
projects / nominatim.git / commitdiff