nominatim/tools/check_database.py

   1 # SPDX-License-Identifier: GPL-2.0-only
   2 #
   3 # This file is part of Nominatim. (https://nominatim.org)
   4 #
   5 # Copyright (C) 2022 by the Nominatim developer community.
   6 # For a full list of authors see the git log.
   7 """
   8 Collection of functions that check if the database is complete and functional.
   9 """
  10 from enum import Enum
  11 from textwrap import dedent
  12
  13 from nominatim.db.connection import connect
  14 from nominatim.errors import UsageError
  15 from nominatim.tokenizer import factory as tokenizer_factory
  16
  17 CHECKLIST = []
  18
  19 class CheckState(Enum):
  20     """ Possible states of a check. FATAL stops check execution entirely.
  21     """
  22     OK = 0
  23     FAIL = 1
  24     FATAL = 2
  25     NOT_APPLICABLE = 3
  26     WARN = 4
  27
  28 def _check(hint=None):
  29     """ Decorator for checks. It adds the function to the list of
  30         checks to execute and adds the code for printing progress messages.
  31     """
  32     def decorator(func):
  33         title = func.__doc__.split('\n', 1)[0].strip()
  34
  35         def run_check(conn, config):
  36             print(title, end=' ... ')
  37             ret = func(conn, config)
  38             if isinstance(ret, tuple):
  39                 ret, params = ret
  40             else:
  41                 params = {}
  42             if ret == CheckState.OK:
  43                 print('\033[92mOK\033[0m')
  44             elif ret == CheckState.WARN:
  45                 print('\033[93mWARNING\033[0m')
  46                 if hint:
  47                     print('')
  48                     print(dedent(hint.format(**params)))
  49             elif ret == CheckState.NOT_APPLICABLE:
  50                 print('not applicable')
  51             else:
  52                 print('\x1B[31mFailed\033[0m')
  53                 if hint:
  54                     print(dedent(hint.format(**params)))
  55             return ret
  56
  57         CHECKLIST.append(run_check)
  58         return run_check
  59
  60     return decorator
  61
  62 class _BadConnection:
  63
  64     def __init__(self, msg):
  65         self.msg = msg
  66
  67     def close(self):
  68         """ Dummy function to provide the implementation.
  69         """
  70
  71 def check_database(config):
  72     """ Run a number of checks on the database and return the status.
  73     """
  74     try:
  75         conn = connect(config.get_libpq_dsn()).connection
  76     except UsageError as err:
  77         conn = _BadConnection(str(err))
  78
  79     overall_result = 0
  80     for check in CHECKLIST:
  81         ret = check(conn, config)
  82         if ret == CheckState.FATAL:
  83             conn.close()
  84             return 1
  85         if ret in (CheckState.FATAL, CheckState.FAIL):
  86             overall_result = 1
  87
  88     conn.close()
  89     return overall_result
  90
  91
  92 def _get_indexes(conn):
  93     indexes = ['idx_place_addressline_address_place_id',
  94                'idx_placex_rank_search',
  95                'idx_placex_rank_address',
  96                'idx_placex_parent_place_id',
  97                'idx_placex_geometry_reverse_lookuppolygon',
  98                'idx_placex_geometry_placenode',
  99                'idx_osmline_parent_place_id',
 100                'idx_osmline_parent_osm_id',
 101                'idx_postcode_id',
 102                'idx_postcode_postcode'
 103               ]
 104     if conn.table_exists('search_name'):
 105         indexes.extend(('idx_search_name_nameaddress_vector',
 106                         'idx_search_name_name_vector',
 107                         'idx_search_name_centroid'))
 108         if conn.server_version_tuple() >= (11, 0, 0):
 109             indexes.extend(('idx_placex_housenumber',
 110                             'idx_osmline_parent_osm_id_with_hnr'))
 111     if conn.table_exists('place'):
 112         indexes.extend(('idx_placex_pendingsector',
 113                         'idx_location_area_country_place_id',
 114                         'idx_place_osm_unique'))
 115
 116     return indexes
 117
 118
 119 # CHECK FUNCTIONS
 120 #
 121 # Functions are exectured in the order they appear here.
 122
 123 @_check(hint="""\
 124              {error}
 125
 126              Hints:
 127              * Is the database server started?
 128              * Check the NOMINATIM_DATABASE_DSN variable in your local .env
 129              * Try connecting to the database with the same settings
 130
 131              Project directory: {config.project_dir}
 132              Current setting of NOMINATIM_DATABASE_DSN: {config.DATABASE_DSN}
 133              """)
 134 def check_connection(conn, config):
 135     """ Checking database connection
 136     """
 137     if isinstance(conn, _BadConnection):
 138         return CheckState.FATAL, dict(error=conn.msg, config=config)
 139
 140     return CheckState.OK
 141
 142 @_check(hint="""\
 143              placex table not found
 144
 145              Hints:
 146              * Are you connecting to the right database?
 147              * Did the import process finish without errors?
 148
 149              Project directory: {config.project_dir}
 150              Current setting of NOMINATIM_DATABASE_DSN: {config.DATABASE_DSN}
 151              """)
 152 def check_placex_table(conn, config):
 153     """ Checking for placex table
 154     """
 155     if conn.table_exists('placex'):
 156         return CheckState.OK
 157
 158     return CheckState.FATAL, dict(config=config)
 159
 160
 161 @_check(hint="""placex table has no data. Did the import finish sucessfully?""")
 162 def check_placex_size(conn, _):
 163     """ Checking for placex content
 164     """
 165     with conn.cursor() as cur:
 166         cnt = cur.scalar('SELECT count(*) FROM (SELECT * FROM placex LIMIT 100) x')
 167
 168     return CheckState.OK if cnt > 0 else CheckState.FATAL
 169
 170
 171 @_check(hint="""{msg}""")
 172 def check_tokenizer(_, config):
 173     """ Checking that tokenizer works
 174     """
 175     try:
 176         tokenizer = tokenizer_factory.get_tokenizer_for_db(config)
 177     except UsageError:
 178         return CheckState.FAIL, dict(msg="""\
 179             Cannot load tokenizer. Did the import finish sucessfully?""")
 180
 181     result = tokenizer.check_database(config)
 182
 183     if result is None:
 184         return CheckState.OK
 185
 186     return CheckState.FAIL, dict(msg=result)
 187
 188
 189 @_check(hint="""\
 190              Wikipedia/Wikidata importance tables missing.
 191              Quality of search results may be degraded. Reverse geocoding is unaffected.
 192              See https://nominatim.org/release-docs/latest/admin/Import/#wikipediawikidata-rankings
 193              """)
 194 def check_existance_wikipedia(conn, _):
 195     """ Checking for wikipedia/wikidata data
 196     """
 197     with conn.cursor() as cur:
 198         cnt = cur.scalar('SELECT count(*) FROM wikipedia_article')
 199
 200         return CheckState.WARN if cnt == 0 else CheckState.OK
 201
 202
 203 @_check(hint="""\
 204              The indexing didn't finish. {count} entries are not yet indexed.
 205
 206              To index the remaining entries, run:   {index_cmd}
 207              """)
 208 def check_indexing(conn, _):
 209     """ Checking indexing status
 210     """
 211     with conn.cursor() as cur:
 212         cnt = cur.scalar('SELECT count(*) FROM placex WHERE indexed_status > 0')
 213
 214     if cnt == 0:
 215         return CheckState.OK
 216
 217     if conn.index_exists('idx_placex_rank_search'):
 218         # Likely just an interrupted update.
 219         index_cmd = 'nominatim index'
 220     else:
 221         # Looks like the import process got interrupted.
 222         index_cmd = 'nominatim import --continue indexing'
 223
 224     return CheckState.FAIL, dict(count=cnt, index_cmd=index_cmd)
 225
 226
 227 @_check(hint="""\
 228              The following indexes are missing:
 229                {indexes}
 230
 231              Rerun the index creation with:   nominatim import --continue db-postprocess
 232              """)
 233 def check_database_indexes(conn, _):
 234     """ Checking that database indexes are complete
 235     """
 236     missing = []
 237     for index in _get_indexes(conn):
 238         if not conn.index_exists(index):
 239             missing.append(index)
 240
 241     if missing:
 242         return CheckState.FAIL, dict(indexes='\n  '.join(missing))
 243
 244     return CheckState.OK
 245
 246
 247 @_check(hint="""\
 248              At least one index is invalid. That can happen, e.g. when index creation was
 249              disrupted and later restarted. You should delete the affected indices
 250              and recreate them.
 251
 252              Invalid indexes:
 253                {indexes}
 254              """)
 255 def check_database_index_valid(conn, _):
 256     """ Checking that all database indexes are valid
 257     """
 258     with conn.cursor() as cur:
 259         cur.execute(""" SELECT relname FROM pg_class, pg_index
 260                         WHERE pg_index.indisvalid = false
 261                         AND pg_index.indexrelid = pg_class.oid""")
 262
 263         broken = list(cur)
 264
 265     if broken:
 266         return CheckState.FAIL, dict(indexes='\n  '.join(broken))
 267
 268     return CheckState.OK
 269
 270
 271 @_check(hint="""\
 272              {error}
 273              Run TIGER import again:   nominatim add-data --tiger-data <DIR>
 274              """)
 275 def check_tiger_table(conn, config):
 276     """ Checking TIGER external data table.
 277     """
 278     if not config.get_bool('USE_US_TIGER_DATA'):
 279         return CheckState.NOT_APPLICABLE
 280
 281     if not conn.table_exists('location_property_tiger'):
 282         return CheckState.FAIL, dict(error='TIGER data table not found.')
 283
 284     with conn.cursor() as cur:
 285         if cur.scalar('SELECT count(*) FROM location_property_tiger') == 0:
 286             return CheckState.FAIL, dict(error='TIGER data table is empty.')
 287
 288     return CheckState.OK