only add well-formatted postcodes to location table

[nominatim.git] / nominatim / tools / check_database.py

diff --git a/nominatim/tools/check_database.py b/nominatim/tools/check_database.py
index 2d57c5ee323ee8e1f620a26932b2e51be6530384..7ac31271568382aedc011e6dc46d8c3b3e8bb57c 100644 (file)
--- a/nominatim/tools/check_database.py
+++ b/nominatim/tools/check_database.py
@@ -1,13 +1,18 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# This file is part of Nominatim. (https://nominatim.org)
+#
+# Copyright (C) 2022 by the Nominatim developer community.
+# For a full list of authors see the git log.
 """
 Collection of functions that check if the database is complete and functional.
 """
 from enum import Enum
 from textwrap import dedent
 
-import psycopg2
-
 from nominatim.db.connection import connect
 from nominatim.errors import UsageError
+from nominatim.tokenizer import factory as tokenizer_factory
 
 CHECKLIST = []
 
@@ -18,6 +23,7 @@ class CheckState(Enum):
     FAIL = 1
     FATAL = 2
     NOT_APPLICABLE = 3
+    WARN = 4
 
 def _check(hint=None):
     """ Decorator for checks. It adds the function to the list of
@@ -25,6 +31,7 @@ def _check(hint=None):
     """
     def decorator(func):
         title = func.__doc__.split('\n', 1)[0].strip()
+
         def run_check(conn, config):
             print(title, end=' ... ')
             ret = func(conn, config)
@@ -34,6 +41,11 @@ def _check(hint=None):
                 params = {}
             if ret == CheckState.OK:
                 print('\033[92mOK\033[0m')
+            elif ret == CheckState.WARN:
+                print('\033[93mWARNING\033[0m')
+                if hint:
+                    print('')
+                    print(dedent(hint.format(**params)))
             elif ret == CheckState.NOT_APPLICABLE:
                 print('not applicable')
             else:
@@ -78,8 +90,7 @@ def check_database(config):
 
 
 def _get_indexes(conn):
-    indexes = ['idx_word_word_id',
-               'idx_place_addressline_address_place_id',
+    indexes = ['idx_place_addressline_address_place_id',
                'idx_placex_rank_search',
                'idx_placex_rank_address',
                'idx_placex_parent_place_id',
@@ -100,13 +111,12 @@ def _get_indexes(conn):
     if conn.table_exists('place'):
         indexes.extend(('idx_placex_pendingsector',
                         'idx_location_area_country_place_id',
-                        'idx_place_osm_unique'
-                       ))
+                        'idx_place_osm_unique'))
 
     return indexes
 
 
-### CHECK FUNCTIONS
+# CHECK FUNCTIONS
 #
 # Functions are exectured in the order they appear here.
 
@@ -149,7 +159,7 @@ def check_placex_table(conn, config):
 
 
 @_check(hint="""placex table has no data. Did the import finish sucessfully?""")
-def check_placex_size(conn, config): # pylint: disable=W0613
+def check_placex_size(conn, _):
     """ Checking for placex content
     """
     with conn.cursor() as cur:
@@ -158,30 +168,39 @@ def check_placex_size(conn, config): # pylint: disable=W0613
     return CheckState.OK if cnt > 0 else CheckState.FATAL
 
 
-@_check(hint="""\
-             The Postgresql extension nominatim.so was not correctly loaded.
+@_check(hint="""{msg}""")
+def check_tokenizer(_, config):
+    """ Checking that tokenizer works
+    """
+    try:
+        tokenizer = tokenizer_factory.get_tokenizer_for_db(config)
+    except UsageError:
+        return CheckState.FAIL, dict(msg="""\
+            Cannot load tokenizer. Did the import finish sucessfully?""")
 
-             Error: {error}
+    result = tokenizer.check_database(config)
 
-             Hints:
-             * Check the output of the CMmake/make installation step
-             * Does nominatim.so exist?
-             * Does nominatim.so exist on the database server?
-             * Can nominatim.so be accessed by the database user?
+    if result is None:
+        return CheckState.OK
+
+    return CheckState.FAIL, dict(msg=result)
+
+
+@_check(hint="""\
+             Wikipedia/Wikidata importance tables missing.
+             Quality of search results may be degraded. Reverse geocoding is unaffected.
+             See https://nominatim.org/release-docs/latest/admin/Import/#wikipediawikidata-rankings
              """)
-def check_module(conn, config): # pylint: disable=W0613
-    """ Checking that nominatim.so module is installed
+def check_existance_wikipedia(conn, _):
+    """ Checking for wikipedia/wikidata data
     """
-    with conn.cursor() as cur:
-        try:
-            out = cur.scalar("SELECT make_standard_name('a')")
-        except psycopg2.ProgrammingError as err:
-            return CheckState.FAIL, dict(error=str(err))
+    if not conn.table_exists('search_name'):
+        return CheckState.NOT_APPLICABLE
 
-        if out != 'a':
-            return CheckState.FAIL, dict(error='Unexpected result for make_standard_name()')
+    with conn.cursor() as cur:
+        cnt = cur.scalar('SELECT count(*) FROM wikipedia_article')
 
-        return CheckState.OK
+        return CheckState.WARN if cnt == 0 else CheckState.OK
 
 
 @_check(hint="""\
@@ -189,7 +208,7 @@ def check_module(conn, config): # pylint: disable=W0613
 
              To index the remaining entries, run:   {index_cmd}
              """)
-def check_indexing(conn, config): # pylint: disable=W0613
+def check_indexing(conn, _):
     """ Checking indexing status
     """
     with conn.cursor() as cur:
@@ -198,7 +217,7 @@ def check_indexing(conn, config): # pylint: disable=W0613
     if cnt == 0:
         return CheckState.OK
 
-    if conn.index_exists('idx_word_word_id'):
+    if conn.index_exists('idx_placex_rank_search'):
         # Likely just an interrupted update.
         index_cmd = 'nominatim index'
     else:
@@ -214,7 +233,7 @@ def check_indexing(conn, config): # pylint: disable=W0613
 
              Rerun the index creation with:   nominatim import --continue db-postprocess
              """)
-def check_database_indexes(conn, config): # pylint: disable=W0613
+def check_database_indexes(conn, _):
     """ Checking that database indexes are complete
     """
     missing = []
@@ -236,7 +255,7 @@ def check_database_indexes(conn, config): # pylint: disable=W0613
              Invalid indexes:
                {indexes}
              """)
-def check_database_index_valid(conn, config): # pylint: disable=W0613
+def check_database_index_valid(conn, _):
     """ Checking that all database indexes are valid
     """
     with conn.cursor() as cur: