Merge remote-tracking branch 'upstream/master'

[nominatim.git] / nominatim / tools / check_database.py

diff --git a/nominatim/tools/check_database.py b/nominatim/tools/check_database.py
index 7b8da200b5a8598416bb9ef78f76dc3eda8347a5..7ac31271568382aedc011e6dc46d8c3b3e8bb57c 100644 (file)
--- a/nominatim/tools/check_database.py
+++ b/nominatim/tools/check_database.py
@@ -1,13 +1,18 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# This file is part of Nominatim. (https://nominatim.org)
+#
+# Copyright (C) 2022 by the Nominatim developer community.
+# For a full list of authors see the git log.

 """
 Collection of functions that check if the database is complete and functional.
 """
 from enum import Enum
 from textwrap import dedent
 
 """
 Collection of functions that check if the database is complete and functional.
 """
 from enum import Enum
 from textwrap import dedent
 

-import psycopg2
-
-from ..db.connection import connect
-from ..errors import UsageError
+from nominatim.db.connection import connect
+from nominatim.errors import UsageError
+from nominatim.tokenizer import factory as tokenizer_factory

 
 CHECKLIST = []
 
 
 CHECKLIST = []
 


@@ -18,6 +23,7 @@ class CheckState(Enum):
     FAIL = 1
     FATAL = 2
     NOT_APPLICABLE = 3
     FAIL = 1
     FATAL = 2
     NOT_APPLICABLE = 3

+    WARN = 4

 
 def _check(hint=None):
     """ Decorator for checks. It adds the function to the list of
 
 def _check(hint=None):
     """ Decorator for checks. It adds the function to the list of


@@ -25,6 +31,7 @@ def _check(hint=None):
     """
     def decorator(func):
         title = func.__doc__.split('\n', 1)[0].strip()
     """
     def decorator(func):
         title = func.__doc__.split('\n', 1)[0].strip()

+

         def run_check(conn, config):
             print(title, end=' ... ')
             ret = func(conn, config)
         def run_check(conn, config):
             print(title, end=' ... ')
             ret = func(conn, config)


@@ -34,6 +41,11 @@ def _check(hint=None):
                 params = {}
             if ret == CheckState.OK:
                 print('\033[92mOK\033[0m')
                 params = {}
             if ret == CheckState.OK:
                 print('\033[92mOK\033[0m')

+            elif ret == CheckState.WARN:
+                print('\033[93mWARNING\033[0m')
+                if hint:
+                    print('')
+                    print(dedent(hint.format(**params)))

             elif ret == CheckState.NOT_APPLICABLE:
                 print('not applicable')
             else:
             elif ret == CheckState.NOT_APPLICABLE:
                 print('not applicable')
             else:


@@ -47,7 +59,7 @@ def _check(hint=None):
 
     return decorator
 
 
     return decorator
 

-class _BadConnection: # pylint: disable=R0903
+class _BadConnection:

 
     def __init__(self, msg):
         self.msg = msg
 
     def __init__(self, msg):
         self.msg = msg


@@ -60,7 +72,7 @@ def check_database(config):
     """ Run a number of checks on the database and return the status.
     """
     try:
     """ Run a number of checks on the database and return the status.
     """
     try:

-        conn = connect(config.get_libpq_dsn())
+        conn = connect(config.get_libpq_dsn()).connection

     except UsageError as err:
         conn = _BadConnection(str(err))
 
     except UsageError as err:
         conn = _BadConnection(str(err))
 


@@ -78,13 +90,12 @@ def check_database(config):
 
 
 def _get_indexes(conn):
 
 
 def _get_indexes(conn):

-    indexes = ['idx_word_word_id',
-               'idx_place_addressline_address_place_id',
+    indexes = ['idx_place_addressline_address_place_id',

                'idx_placex_rank_search',
                'idx_placex_rank_address',
                'idx_placex_parent_place_id',
                'idx_placex_geometry_reverse_lookuppolygon',
                'idx_placex_rank_search',
                'idx_placex_rank_address',
                'idx_placex_parent_place_id',
                'idx_placex_geometry_reverse_lookuppolygon',

-               'idx_placex_geometry_reverse_placenode',
+               'idx_placex_geometry_placenode',

                'idx_osmline_parent_place_id',
                'idx_osmline_parent_osm_id',
                'idx_postcode_id',
                'idx_osmline_parent_place_id',
                'idx_osmline_parent_osm_id',
                'idx_postcode_id',


@@ -94,16 +105,18 @@ def _get_indexes(conn):
         indexes.extend(('idx_search_name_nameaddress_vector',
                         'idx_search_name_name_vector',
                         'idx_search_name_centroid'))
         indexes.extend(('idx_search_name_nameaddress_vector',
                         'idx_search_name_name_vector',
                         'idx_search_name_centroid'))

+        if conn.server_version_tuple() >= (11, 0, 0):
+            indexes.extend(('idx_placex_housenumber',
+                            'idx_osmline_parent_osm_id_with_hnr'))

     if conn.table_exists('place'):
         indexes.extend(('idx_placex_pendingsector',
                         'idx_location_area_country_place_id',
     if conn.table_exists('place'):
         indexes.extend(('idx_placex_pendingsector',
                         'idx_location_area_country_place_id',

-                        'idx_place_osm_unique'
-                       ))
+                        'idx_place_osm_unique'))

 
     return indexes
 
 
 
     return indexes
 
 

-### CHECK FUNCTIONS
+# CHECK FUNCTIONS

 #
 # Functions are exectured in the order they appear here.
 
 #
 # Functions are exectured in the order they appear here.
 


@@ -146,7 +159,7 @@ def check_placex_table(conn, config):
 
 
 @_check(hint="""placex table has no data. Did the import finish sucessfully?""")
 
 
 @_check(hint="""placex table has no data. Did the import finish sucessfully?""")

-def check_placex_size(conn, config): # pylint: disable=W0613
+def check_placex_size(conn, _):

     """ Checking for placex content
     """
     with conn.cursor() as cur:
     """ Checking for placex content
     """
     with conn.cursor() as cur:


@@ -155,30 +168,39 @@ def check_placex_size(conn, config): # pylint: disable=W0613
     return CheckState.OK if cnt > 0 else CheckState.FATAL
 
 
     return CheckState.OK if cnt > 0 else CheckState.FATAL
 
 

-@_check(hint="""\
-             The Postgresql extension nominatim.so was not correctly loaded.
+@_check(hint="""{msg}""")
+def check_tokenizer(_, config):
+    """ Checking that tokenizer works
+    """
+    try:
+        tokenizer = tokenizer_factory.get_tokenizer_for_db(config)
+    except UsageError:
+        return CheckState.FAIL, dict(msg="""\
+            Cannot load tokenizer. Did the import finish sucessfully?""")

 
 

-             Error: {error}
+    result = tokenizer.check_database(config)

 
 

-             Hints:
-             * Check the output of the CMmake/make installation step
-             * Does nominatim.so exist?
-             * Does nominatim.so exist on the database server?
-             * Can nominatim.so be accessed by the database user?
+    if result is None:
+        return CheckState.OK
+
+    return CheckState.FAIL, dict(msg=result)
+
+
+@_check(hint="""\
+             Wikipedia/Wikidata importance tables missing.
+             Quality of search results may be degraded. Reverse geocoding is unaffected.
+             See https://nominatim.org/release-docs/latest/admin/Import/#wikipediawikidata-rankings

              """)
              """)

-def check_module(conn, config): # pylint: disable=W0613
-    """ Checking that nominatim.so module is installed
+def check_existance_wikipedia(conn, _):
+    """ Checking for wikipedia/wikidata data

     """
     """

-    with conn.cursor() as cur:
-        try:
-            out = cur.scalar("SELECT make_standard_name('a')")
-        except psycopg2.ProgrammingError as err:
-            return CheckState.FAIL, dict(error=str(err))
+    if not conn.table_exists('search_name'):
+        return CheckState.NOT_APPLICABLE

 
 

-        if out != 'a':
-            return CheckState.FAIL, dict(error='Unexpected result for make_standard_name()')
+    with conn.cursor() as cur:
+        cnt = cur.scalar('SELECT count(*) FROM wikipedia_article')

 
 

-        return CheckState.OK
+        return CheckState.WARN if cnt == 0 else CheckState.OK

 
 
 @_check(hint="""\
 
 
 @_check(hint="""\


@@ -186,7 +208,7 @@ def check_module(conn, config): # pylint: disable=W0613
 
              To index the remaining entries, run:   {index_cmd}
              """)
 
              To index the remaining entries, run:   {index_cmd}
              """)

-def check_indexing(conn, config): # pylint: disable=W0613
+def check_indexing(conn, _):

     """ Checking indexing status
     """
     with conn.cursor() as cur:
     """ Checking indexing status
     """
     with conn.cursor() as cur:


@@ -195,7 +217,7 @@ def check_indexing(conn, config): # pylint: disable=W0613
     if cnt == 0:
         return CheckState.OK
 
     if cnt == 0:
         return CheckState.OK
 

-    if conn.index_exists('idx_word_word_id'):
+    if conn.index_exists('idx_placex_rank_search'):

         # Likely just an interrupted update.
         index_cmd = 'nominatim index'
     else:
         # Likely just an interrupted update.
         index_cmd = 'nominatim index'
     else:


@@ -211,7 +233,7 @@ def check_indexing(conn, config): # pylint: disable=W0613
 
              Rerun the index creation with:   nominatim import --continue db-postprocess
              """)
 
              Rerun the index creation with:   nominatim import --continue db-postprocess
              """)

-def check_database_indexes(conn, config): # pylint: disable=W0613
+def check_database_indexes(conn, _):

     """ Checking that database indexes are complete
     """
     missing = []
     """ Checking that database indexes are complete
     """
     missing = []


@@ -233,7 +255,7 @@ def check_database_indexes(conn, config): # pylint: disable=W0613
              Invalid indexes:
                {indexes}
              """)
              Invalid indexes:
                {indexes}
              """)

-def check_database_index_valid(conn, config): # pylint: disable=W0613
+def check_database_index_valid(conn, _):

     """ Checking that all database indexes are valid
     """
     with conn.cursor() as cur:
     """ Checking that all database indexes are valid
     """
     with conn.cursor() as cur: