nominatim/tools/check_database.py

   1 """
   2 Collection of functions that check if the database is complete and functional.
   3 """
   4 from enum import Enum
   5 from textwrap import dedent
   6
   7 import psycopg2
   8
   9 from ..db.connection import connect
  10 from ..errors import UsageError
  11
  12 CHECKLIST = []
  13
  14 class CheckState(Enum):
  15     """ Possible states of a check. FATAL stops check execution entirely.
  16     """
  17     OK = 0
  18     FAIL = 1
  19     FATAL = 2
  20     NOT_APPLICABLE = 3
  21
  22 def _check(hint=None):
  23     """ Decorator for checks. It adds the function to the list of
  24         checks to execute and adds the code for printing progress messages.
  25     """
  26     def decorator(func):
  27         title = func.__doc__.split('\n', 1)[0].strip()
  28         def run_check(conn, config):
  29             print(title, end=' ... ')
  30             ret = func(conn, config)
  31             if isinstance(ret, tuple):
  32                 ret, params = ret
  33             else:
  34                 params = {}
  35             if ret == CheckState.OK:
  36                 print('\033[92mOK\033[0m')
  37             elif ret == CheckState.NOT_APPLICABLE:
  38                 print('not applicable')
  39             else:
  40                 print('\x1B[31mFailed\033[0m')
  41                 if hint:
  42                     print(dedent(hint.format(**params)))
  43             return ret
  44
  45         CHECKLIST.append(run_check)
  46         return run_check
  47
  48     return decorator
  49
  50 class _BadConnection: # pylint: disable=R0903
  51
  52     def __init__(self, msg):
  53         self.msg = msg
  54
  55     def close(self):
  56         """ Dummy function to provide the implementation.
  57         """
  58
  59 def check_database(config):
  60     """ Run a number of checks on the database and return the status.
  61     """
  62     try:
  63         conn = connect(config.get_libpq_dsn()).connection
  64     except UsageError as err:
  65         conn = _BadConnection(str(err))
  66
  67     overall_result = 0
  68     for check in CHECKLIST:
  69         ret = check(conn, config)
  70         if ret == CheckState.FATAL:
  71             conn.close()
  72             return 1
  73         if ret in (CheckState.FATAL, CheckState.FAIL):
  74             overall_result = 1
  75
  76     conn.close()
  77     return overall_result
  78
  79
  80 def _get_indexes(conn):
  81     indexes = ['idx_word_word_id',
  82                'idx_place_addressline_address_place_id',
  83                'idx_placex_rank_search',
  84                'idx_placex_rank_address',
  85                'idx_placex_parent_place_id',
  86                'idx_placex_geometry_reverse_lookuppolygon',
  87                'idx_placex_geometry_reverse_placenode',
  88                'idx_osmline_parent_place_id',
  89                'idx_osmline_parent_osm_id',
  90                'idx_postcode_id',
  91                'idx_postcode_postcode'
  92               ]
  93     if conn.table_exists('search_name'):
  94         indexes.extend(('idx_search_name_nameaddress_vector',
  95                         'idx_search_name_name_vector',
  96                         'idx_search_name_centroid'))
  97     if conn.table_exists('place'):
  98         indexes.extend(('idx_placex_pendingsector',
  99                         'idx_location_area_country_place_id',
 100                         'idx_place_osm_unique'
 101                        ))
 102
 103     return indexes
 104
 105
 106 ### CHECK FUNCTIONS
 107 #
 108 # Functions are exectured in the order they appear here.
 109
 110 @_check(hint="""\
 111              {error}
 112
 113              Hints:
 114              * Is the database server started?
 115              * Check the NOMINATIM_DATABASE_DSN variable in your local .env
 116              * Try connecting to the database with the same settings
 117
 118              Project directory: {config.project_dir}
 119              Current setting of NOMINATIM_DATABASE_DSN: {config.DATABASE_DSN}
 120              """)
 121 def check_connection(conn, config):
 122     """ Checking database connection
 123     """
 124     if isinstance(conn, _BadConnection):
 125         return CheckState.FATAL, dict(error=conn.msg, config=config)
 126
 127     return CheckState.OK
 128
 129 @_check(hint="""\
 130              placex table not found
 131
 132              Hints:
 133              * Are you connecting to the right database?
 134              * Did the import process finish without errors?
 135
 136              Project directory: {config.project_dir}
 137              Current setting of NOMINATIM_DATABASE_DSN: {config.DATABASE_DSN}
 138              """)
 139 def check_placex_table(conn, config):
 140     """ Checking for placex table
 141     """
 142     if conn.table_exists('placex'):
 143         return CheckState.OK
 144
 145     return CheckState.FATAL, dict(config=config)
 146
 147
 148 @_check(hint="""placex table has no data. Did the import finish sucessfully?""")
 149 def check_placex_size(conn, config): # pylint: disable=W0613
 150     """ Checking for placex content
 151     """
 152     with conn.cursor() as cur:
 153         cnt = cur.scalar('SELECT count(*) FROM (SELECT * FROM placex LIMIT 100) x')
 154
 155     return CheckState.OK if cnt > 0 else CheckState.FATAL
 156
 157
 158 @_check(hint="""\
 159              The Postgresql extension nominatim.so was not correctly loaded.
 160
 161              Error: {error}
 162
 163              Hints:
 164              * Check the output of the CMmake/make installation step
 165              * Does nominatim.so exist?
 166              * Does nominatim.so exist on the database server?
 167              * Can nominatim.so be accessed by the database user?
 168              """)
 169 def check_module(conn, config): # pylint: disable=W0613
 170     """ Checking that nominatim.so module is installed
 171     """
 172     with conn.cursor() as cur:
 173         try:
 174             out = cur.scalar("SELECT make_standard_name('a')")
 175         except psycopg2.ProgrammingError as err:
 176             return CheckState.FAIL, dict(error=str(err))
 177
 178         if out != 'a':
 179             return CheckState.FAIL, dict(error='Unexpected result for make_standard_name()')
 180
 181         return CheckState.OK
 182
 183
 184 @_check(hint="""\
 185              The indexing didn't finish. {count} entries are not yet indexed.
 186
 187              To index the remaining entries, run:   {index_cmd}
 188              """)
 189 def check_indexing(conn, config): # pylint: disable=W0613
 190     """ Checking indexing status
 191     """
 192     with conn.cursor() as cur:
 193         cnt = cur.scalar('SELECT count(*) FROM placex WHERE indexed_status > 0')
 194
 195     if cnt == 0:
 196         return CheckState.OK
 197
 198     if conn.index_exists('idx_word_word_id'):
 199         # Likely just an interrupted update.
 200         index_cmd = 'nominatim index'
 201     else:
 202         # Looks like the import process got interrupted.
 203         index_cmd = 'nominatim import --continue indexing'
 204
 205     return CheckState.FAIL, dict(count=cnt, index_cmd=index_cmd)
 206
 207
 208 @_check(hint="""\
 209              The following indexes are missing:
 210                {indexes}
 211
 212              Rerun the index creation with:   nominatim import --continue db-postprocess
 213              """)
 214 def check_database_indexes(conn, config): # pylint: disable=W0613
 215     """ Checking that database indexes are complete
 216     """
 217     missing = []
 218     for index in _get_indexes(conn):
 219         if not conn.index_exists(index):
 220             missing.append(index)
 221
 222     if missing:
 223         return CheckState.FAIL, dict(indexes='\n  '.join(missing))
 224
 225     return CheckState.OK
 226
 227
 228 @_check(hint="""\
 229              At least one index is invalid. That can happen, e.g. when index creation was
 230              disrupted and later restarted. You should delete the affected indices
 231              and recreate them.
 232
 233              Invalid indexes:
 234                {indexes}
 235              """)
 236 def check_database_index_valid(conn, config): # pylint: disable=W0613
 237     """ Checking that all database indexes are valid
 238     """
 239     with conn.cursor() as cur:
 240         cur.execute(""" SELECT relname FROM pg_class, pg_index
 241                         WHERE pg_index.indisvalid = false
 242                         AND pg_index.indexrelid = pg_class.oid""")
 243
 244         broken = list(cur)
 245
 246     if broken:
 247         return CheckState.FAIL, dict(indexes='\n  '.join(broken))
 248
 249     return CheckState.OK
 250
 251
 252 @_check(hint="""\
 253              {error}
 254              Run TIGER import again:   nominatim add-data --tiger-data <DIR>
 255              """)
 256 def check_tiger_table(conn, config):
 257     """ Checking TIGER external data table.
 258     """
 259     if not config.get_bool('USE_US_TIGER_DATA'):
 260         return CheckState.NOT_APPLICABLE
 261
 262     if not conn.table_exists('location_property_tiger'):
 263         return CheckState.FAIL, dict(error='TIGER data table not found.')
 264
 265     with conn.cursor() as cur:
 266         if cur.scalar('SELECT count(*) FROM location_property_tiger') == 0:
 267             return CheckState.FAIL, dict(error='TIGER data table is empty.')
 268
 269     return CheckState.OK