nominatim/tools/check_database.py

   1 """
   2 Collection of functions that check if the database is complete and functional.
   3 """
   4 from enum import Enum
   5 from textwrap import dedent
   6
   7 import psycopg2
   8
   9 from nominatim.db.connection import connect
  10 from nominatim.errors import UsageError
  11
  12 CHECKLIST = []
  13
  14 class CheckState(Enum):
  15     """ Possible states of a check. FATAL stops check execution entirely.
  16     """
  17     OK = 0
  18     FAIL = 1
  19     FATAL = 2
  20     NOT_APPLICABLE = 3
  21
  22 def _check(hint=None):
  23     """ Decorator for checks. It adds the function to the list of
  24         checks to execute and adds the code for printing progress messages.
  25     """
  26     def decorator(func):
  27         title = func.__doc__.split('\n', 1)[0].strip()
  28         def run_check(conn, config):
  29             print(title, end=' ... ')
  30             ret = func(conn, config)
  31             if isinstance(ret, tuple):
  32                 ret, params = ret
  33             else:
  34                 params = {}
  35             if ret == CheckState.OK:
  36                 print('\033[92mOK\033[0m')
  37             elif ret == CheckState.NOT_APPLICABLE:
  38                 print('not applicable')
  39             else:
  40                 print('\x1B[31mFailed\033[0m')
  41                 if hint:
  42                     print(dedent(hint.format(**params)))
  43             return ret
  44
  45         CHECKLIST.append(run_check)
  46         return run_check
  47
  48     return decorator
  49
  50 class _BadConnection: # pylint: disable=R0903
  51
  52     def __init__(self, msg):
  53         self.msg = msg
  54
  55     def close(self):
  56         """ Dummy function to provide the implementation.
  57         """
  58
  59 def check_database(config):
  60     """ Run a number of checks on the database and return the status.
  61     """
  62     try:
  63         conn = connect(config.get_libpq_dsn()).connection
  64     except UsageError as err:
  65         conn = _BadConnection(str(err))
  66
  67     overall_result = 0
  68     for check in CHECKLIST:
  69         ret = check(conn, config)
  70         if ret == CheckState.FATAL:
  71             conn.close()
  72             return 1
  73         if ret in (CheckState.FATAL, CheckState.FAIL):
  74             overall_result = 1
  75
  76     conn.close()
  77     return overall_result
  78
  79
  80 def _get_indexes(conn):
  81     indexes = ['idx_word_word_id',
  82                'idx_place_addressline_address_place_id',
  83                'idx_placex_rank_search',
  84                'idx_placex_rank_address',
  85                'idx_placex_parent_place_id',
  86                'idx_placex_geometry_reverse_lookuppolygon',
  87                'idx_placex_geometry_placenode',
  88                'idx_placex_housenumber',
  89                'idx_osmline_parent_place_id',
  90                'idx_osmline_parent_osm_id',
  91                'idx_postcode_id',
  92                'idx_postcode_postcode'
  93               ]
  94     if conn.table_exists('search_name'):
  95         indexes.extend(('idx_search_name_nameaddress_vector',
  96                         'idx_search_name_name_vector',
  97                         'idx_search_name_centroid'))
  98     if conn.table_exists('place'):
  99         indexes.extend(('idx_placex_pendingsector',
 100                         'idx_location_area_country_place_id',
 101                         'idx_place_osm_unique'
 102                        ))
 103
 104     return indexes
 105
 106
 107 ### CHECK FUNCTIONS
 108 #
 109 # Functions are exectured in the order they appear here.
 110
 111 @_check(hint="""\
 112              {error}
 113
 114              Hints:
 115              * Is the database server started?
 116              * Check the NOMINATIM_DATABASE_DSN variable in your local .env
 117              * Try connecting to the database with the same settings
 118
 119              Project directory: {config.project_dir}
 120              Current setting of NOMINATIM_DATABASE_DSN: {config.DATABASE_DSN}
 121              """)
 122 def check_connection(conn, config):
 123     """ Checking database connection
 124     """
 125     if isinstance(conn, _BadConnection):
 126         return CheckState.FATAL, dict(error=conn.msg, config=config)
 127
 128     return CheckState.OK
 129
 130 @_check(hint="""\
 131              placex table not found
 132
 133              Hints:
 134              * Are you connecting to the right database?
 135              * Did the import process finish without errors?
 136
 137              Project directory: {config.project_dir}
 138              Current setting of NOMINATIM_DATABASE_DSN: {config.DATABASE_DSN}
 139              """)
 140 def check_placex_table(conn, config):
 141     """ Checking for placex table
 142     """
 143     if conn.table_exists('placex'):
 144         return CheckState.OK
 145
 146     return CheckState.FATAL, dict(config=config)
 147
 148
 149 @_check(hint="""placex table has no data. Did the import finish sucessfully?""")
 150 def check_placex_size(conn, config): # pylint: disable=W0613
 151     """ Checking for placex content
 152     """
 153     with conn.cursor() as cur:
 154         cnt = cur.scalar('SELECT count(*) FROM (SELECT * FROM placex LIMIT 100) x')
 155
 156     return CheckState.OK if cnt > 0 else CheckState.FATAL
 157
 158
 159 @_check(hint="""\
 160              The Postgresql extension nominatim.so was not correctly loaded.
 161
 162              Error: {error}
 163
 164              Hints:
 165              * Check the output of the CMmake/make installation step
 166              * Does nominatim.so exist?
 167              * Does nominatim.so exist on the database server?
 168              * Can nominatim.so be accessed by the database user?
 169              """)
 170 def check_module(conn, config): # pylint: disable=W0613
 171     """ Checking that nominatim.so module is installed
 172     """
 173     with conn.cursor() as cur:
 174         try:
 175             out = cur.scalar("SELECT make_standard_name('a')")
 176         except psycopg2.ProgrammingError as err:
 177             return CheckState.FAIL, dict(error=str(err))
 178
 179         if out != 'a':
 180             return CheckState.FAIL, dict(error='Unexpected result for make_standard_name()')
 181
 182         return CheckState.OK
 183
 184
 185 @_check(hint="""\
 186              The indexing didn't finish. {count} entries are not yet indexed.
 187
 188              To index the remaining entries, run:   {index_cmd}
 189              """)
 190 def check_indexing(conn, config): # pylint: disable=W0613
 191     """ Checking indexing status
 192     """
 193     with conn.cursor() as cur:
 194         cnt = cur.scalar('SELECT count(*) FROM placex WHERE indexed_status > 0')
 195
 196     if cnt == 0:
 197         return CheckState.OK
 198
 199     if conn.index_exists('idx_word_word_id'):
 200         # Likely just an interrupted update.
 201         index_cmd = 'nominatim index'
 202     else:
 203         # Looks like the import process got interrupted.
 204         index_cmd = 'nominatim import --continue indexing'
 205
 206     return CheckState.FAIL, dict(count=cnt, index_cmd=index_cmd)
 207
 208
 209 @_check(hint="""\
 210              The following indexes are missing:
 211                {indexes}
 212
 213              Rerun the index creation with:   nominatim import --continue db-postprocess
 214              """)
 215 def check_database_indexes(conn, config): # pylint: disable=W0613
 216     """ Checking that database indexes are complete
 217     """
 218     missing = []
 219     for index in _get_indexes(conn):
 220         if not conn.index_exists(index):
 221             missing.append(index)
 222
 223     if missing:
 224         return CheckState.FAIL, dict(indexes='\n  '.join(missing))
 225
 226     return CheckState.OK
 227
 228
 229 @_check(hint="""\
 230              At least one index is invalid. That can happen, e.g. when index creation was
 231              disrupted and later restarted. You should delete the affected indices
 232              and recreate them.
 233
 234              Invalid indexes:
 235                {indexes}
 236              """)
 237 def check_database_index_valid(conn, config): # pylint: disable=W0613
 238     """ Checking that all database indexes are valid
 239     """
 240     with conn.cursor() as cur:
 241         cur.execute(""" SELECT relname FROM pg_class, pg_index
 242                         WHERE pg_index.indisvalid = false
 243                         AND pg_index.indexrelid = pg_class.oid""")
 244
 245         broken = list(cur)
 246
 247     if broken:
 248         return CheckState.FAIL, dict(indexes='\n  '.join(broken))
 249
 250     return CheckState.OK
 251
 252
 253 @_check(hint="""\
 254              {error}
 255              Run TIGER import again:   nominatim add-data --tiger-data <DIR>
 256              """)
 257 def check_tiger_table(conn, config):
 258     """ Checking TIGER external data table.
 259     """
 260     if not config.get_bool('USE_US_TIGER_DATA'):
 261         return CheckState.NOT_APPLICABLE
 262
 263     if not conn.table_exists('location_property_tiger'):
 264         return CheckState.FAIL, dict(error='TIGER data table not found.')
 265
 266     with conn.cursor() as cur:
 267         if cur.scalar('SELECT count(*) FROM location_property_tiger') == 0:
 268             return CheckState.FAIL, dict(error='TIGER data table is empty.')
 269
 270     return CheckState.OK