X-Git-Url: https://git.openstreetmap.org./nominatim.git/blobdiff_plain/b169e4c88cd99075c7e932aeb6587c97f51e7ed1..6dbec931f58c659075a903db1e8eab15b1c93b03:/nominatim/tools/check_database.py diff --git a/nominatim/tools/check_database.py b/nominatim/tools/check_database.py index e99a3572..7ac31271 100644 --- a/nominatim/tools/check_database.py +++ b/nominatim/tools/check_database.py @@ -1,13 +1,18 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# This file is part of Nominatim. (https://nominatim.org) +# +# Copyright (C) 2022 by the Nominatim developer community. +# For a full list of authors see the git log. """ Collection of functions that check if the database is complete and functional. """ from enum import Enum from textwrap import dedent -import psycopg2 - -from ..db.connection import connect -from ..errors import UsageError +from nominatim.db.connection import connect +from nominatim.errors import UsageError +from nominatim.tokenizer import factory as tokenizer_factory CHECKLIST = [] @@ -18,6 +23,7 @@ class CheckState(Enum): FAIL = 1 FATAL = 2 NOT_APPLICABLE = 3 + WARN = 4 def _check(hint=None): """ Decorator for checks. It adds the function to the list of @@ -25,6 +31,7 @@ def _check(hint=None): """ def decorator(func): title = func.__doc__.split('\n', 1)[0].strip() + def run_check(conn, config): print(title, end=' ... ') ret = func(conn, config) @@ -34,6 +41,11 @@ def _check(hint=None): params = {} if ret == CheckState.OK: print('\033[92mOK\033[0m') + elif ret == CheckState.WARN: + print('\033[93mWARNING\033[0m') + if hint: + print('') + print(dedent(hint.format(**params))) elif ret == CheckState.NOT_APPLICABLE: print('not applicable') else: @@ -47,7 +59,7 @@ def _check(hint=None): return decorator -class _BadConnection: # pylint: disable=R0903 +class _BadConnection: def __init__(self, msg): self.msg = msg @@ -60,7 +72,7 @@ def check_database(config): """ Run a number of checks on the database and return the status. """ try: - conn = connect(config.get_libpq_dsn()) + conn = connect(config.get_libpq_dsn()).connection except UsageError as err: conn = _BadConnection(str(err)) @@ -70,7 +82,7 @@ def check_database(config): if ret == CheckState.FATAL: conn.close() return 1 - if ret != CheckState.OK: + if ret in (CheckState.FATAL, CheckState.FAIL): overall_result = 1 conn.close() @@ -78,13 +90,12 @@ def check_database(config): def _get_indexes(conn): - indexes = ['idx_word_word_id', - 'idx_place_addressline_address_place_id', + indexes = ['idx_place_addressline_address_place_id', 'idx_placex_rank_search', 'idx_placex_rank_address', 'idx_placex_parent_place_id', 'idx_placex_geometry_reverse_lookuppolygon', - 'idx_placex_geometry_reverse_placenode', + 'idx_placex_geometry_placenode', 'idx_osmline_parent_place_id', 'idx_osmline_parent_osm_id', 'idx_postcode_id', @@ -94,16 +105,18 @@ def _get_indexes(conn): indexes.extend(('idx_search_name_nameaddress_vector', 'idx_search_name_name_vector', 'idx_search_name_centroid')) + if conn.server_version_tuple() >= (11, 0, 0): + indexes.extend(('idx_placex_housenumber', + 'idx_osmline_parent_osm_id_with_hnr')) if conn.table_exists('place'): indexes.extend(('idx_placex_pendingsector', 'idx_location_area_country_place_id', - 'idx_place_osm_unique' - )) + 'idx_place_osm_unique')) return indexes -### CHECK FUNCTIONS +# CHECK FUNCTIONS # # Functions are exectured in the order they appear here. @@ -146,7 +159,7 @@ def check_placex_table(conn, config): @_check(hint="""placex table has no data. Did the import finish sucessfully?""") -def check_placex_size(conn, config): # pylint: disable=W0613 +def check_placex_size(conn, _): """ Checking for placex content """ with conn.cursor() as cur: @@ -154,30 +167,40 @@ def check_placex_size(conn, config): # pylint: disable=W0613 return CheckState.OK if cnt > 0 else CheckState.FATAL -@_check(hint="""\ - The Postgresql extension nominatim.so was not correctly loaded. - Error: {error} +@_check(hint="""{msg}""") +def check_tokenizer(_, config): + """ Checking that tokenizer works + """ + try: + tokenizer = tokenizer_factory.get_tokenizer_for_db(config) + except UsageError: + return CheckState.FAIL, dict(msg="""\ + Cannot load tokenizer. Did the import finish sucessfully?""") - Hints: - * Check the output of the CMmake/make installation step - * Does nominatim.so exist? - * Does nominatim.so exist on the database server? - * Can nominatim.so be accessed by the database user? + result = tokenizer.check_database(config) + + if result is None: + return CheckState.OK + + return CheckState.FAIL, dict(msg=result) + + +@_check(hint="""\ + Wikipedia/Wikidata importance tables missing. + Quality of search results may be degraded. Reverse geocoding is unaffected. + See https://nominatim.org/release-docs/latest/admin/Import/#wikipediawikidata-rankings """) -def check_module(conn, config): # pylint: disable=W0613 - """ Checking that nominatim.so module is installed +def check_existance_wikipedia(conn, _): + """ Checking for wikipedia/wikidata data """ + if not conn.table_exists('search_name'): + return CheckState.NOT_APPLICABLE + with conn.cursor() as cur: - try: - out = cur.scalar("SELECT make_standard_name('a')") - except psycopg2.ProgrammingError as err: - return CheckState.FAIL, dict(error=str(err)) + cnt = cur.scalar('SELECT count(*) FROM wikipedia_article') - if out != 'a': - return CheckState.FAIL, dict(error='Unexpected result for make_standard_name()') - - return CheckState.OK + return CheckState.WARN if cnt == 0 else CheckState.OK @_check(hint="""\ @@ -185,7 +208,7 @@ def check_module(conn, config): # pylint: disable=W0613 To index the remaining entries, run: {index_cmd} """) -def check_indexing(conn, config): # pylint: disable=W0613 +def check_indexing(conn, _): """ Checking indexing status """ with conn.cursor() as cur: @@ -194,24 +217,23 @@ def check_indexing(conn, config): # pylint: disable=W0613 if cnt == 0: return CheckState.OK - if conn.index_exists('idx_word_word_id'): + if conn.index_exists('idx_placex_rank_search'): # Likely just an interrupted update. index_cmd = 'nominatim index' else: - # Looks like the import process got interupted. + # Looks like the import process got interrupted. index_cmd = 'nominatim import --continue indexing' return CheckState.FAIL, dict(count=cnt, index_cmd=index_cmd) - @_check(hint="""\ The following indexes are missing: {indexes} Rerun the index creation with: nominatim import --continue db-postprocess """) -def check_database_indexes(conn, config): # pylint: disable=W0613 +def check_database_indexes(conn, _): """ Checking that database indexes are complete """ missing = [] @@ -233,7 +255,7 @@ def check_database_indexes(conn, config): # pylint: disable=W0613 Invalid indexes: {indexes} """) -def check_database_index_valid(conn, config): # pylint: disable=W0613 +def check_database_index_valid(conn, _): """ Checking that all database indexes are valid """ with conn.cursor() as cur: