Merge pull request #3030 from lonvia/interpolation-corner-cases

[nominatim.git] / nominatim / tools / check_database.py

diff --git a/nominatim/tools/check_database.py b/nominatim/tools/check_database.py
index d4f793b46334c27f304fbfe898268a4f23f564a9..437775db6a874561d5451d3a8683fd085a3ba17c 100644 (file)
--- a/nominatim/tools/check_database.py
+++ b/nominatim/tools/check_database.py
@@ -1,10 +1,18 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# This file is part of Nominatim. (https://nominatim.org)
+#
+# Copyright (C) 2022 by the Nominatim developer community.
+# For a full list of authors see the git log.

 """
 Collection of functions that check if the database is complete and functional.
 """
 """
 Collection of functions that check if the database is complete and functional.
 """

+from typing import Callable, Optional, Any, Union, Tuple, Mapping, List

 from enum import Enum
 from textwrap import dedent
 
 from enum import Enum
 from textwrap import dedent
 

-from nominatim.db.connection import connect
+from nominatim.config import Configuration
+from nominatim.db.connection import connect, Connection

 from nominatim.errors import UsageError
 from nominatim.tokenizer import factory as tokenizer_factory
 
 from nominatim.errors import UsageError
 from nominatim.tokenizer import factory as tokenizer_factory
 


@@ -17,14 +25,19 @@ class CheckState(Enum):
     FAIL = 1
     FATAL = 2
     NOT_APPLICABLE = 3
     FAIL = 1
     FATAL = 2
     NOT_APPLICABLE = 3

+    WARN = 4

 
 

-def _check(hint=None):
+CheckResult = Union[CheckState, Tuple[CheckState, Mapping[str, Any]]]
+CheckFunc = Callable[[Connection, Configuration], CheckResult]
+
+def _check(hint: Optional[str] = None) -> Callable[[CheckFunc], CheckFunc]:

     """ Decorator for checks. It adds the function to the list of
         checks to execute and adds the code for printing progress messages.
     """
     """ Decorator for checks. It adds the function to the list of
         checks to execute and adds the code for printing progress messages.
     """

-    def decorator(func):
-        title = func.__doc__.split('\n', 1)[0].strip()
-        def run_check(conn, config):
+    def decorator(func: CheckFunc) -> CheckFunc:
+        title = (func.__doc__ or '').split('\n', 1)[0].strip()
+
+        def run_check(conn: Connection, config: Configuration) -> CheckState:

             print(title, end=' ... ')
             ret = func(conn, config)
             if isinstance(ret, tuple):
             print(title, end=' ... ')
             ret = func(conn, config)
             if isinstance(ret, tuple):


@@ -33,6 +46,11 @@ def _check(hint=None):
                 params = {}
             if ret == CheckState.OK:
                 print('\033[92mOK\033[0m')
                 params = {}
             if ret == CheckState.OK:
                 print('\033[92mOK\033[0m')

+            elif ret == CheckState.WARN:
+                print('\033[93mWARNING\033[0m')
+                if hint:
+                    print('')
+                    print(dedent(hint.format(**params)))

             elif ret == CheckState.NOT_APPLICABLE:
                 print('not applicable')
             else:
             elif ret == CheckState.NOT_APPLICABLE:
                 print('not applicable')
             else:


@@ -48,20 +66,20 @@ def _check(hint=None):
 
 class _BadConnection:
 
 
 class _BadConnection:
 

-    def __init__(self, msg):
+    def __init__(self, msg: str) -> None:

         self.msg = msg
 
         self.msg = msg
 

-    def close(self):
+    def close(self) -> None:

         """ Dummy function to provide the implementation.
         """
 
         """ Dummy function to provide the implementation.
         """
 

-def check_database(config):
+def check_database(config: Configuration) -> int:

     """ Run a number of checks on the database and return the status.
     """
     try:
         conn = connect(config.get_libpq_dsn()).connection
     except UsageError as err:
     """ Run a number of checks on the database and return the status.
     """
     try:
         conn = connect(config.get_libpq_dsn()).connection
     except UsageError as err:

-        conn = _BadConnection(str(err))
+        conn = _BadConnection(str(err)) # type: ignore[assignment]

 
     overall_result = 0
     for check in CHECKLIST:
 
     overall_result = 0
     for check in CHECKLIST:


@@ -76,7 +94,7 @@ def check_database(config):
     return overall_result
 
 
     return overall_result
 
 

-def _get_indexes(conn):
+def _get_indexes(conn: Connection) -> List[str]:

     indexes = ['idx_place_addressline_address_place_id',
                'idx_placex_rank_search',
                'idx_placex_rank_address',
     indexes = ['idx_place_addressline_address_place_id',
                'idx_placex_rank_search',
                'idx_placex_rank_address',


@@ -96,15 +114,15 @@ def _get_indexes(conn):
             indexes.extend(('idx_placex_housenumber',
                             'idx_osmline_parent_osm_id_with_hnr'))
     if conn.table_exists('place'):
             indexes.extend(('idx_placex_housenumber',
                             'idx_osmline_parent_osm_id_with_hnr'))
     if conn.table_exists('place'):

-        indexes.extend(('idx_placex_pendingsector',
-                        'idx_location_area_country_place_id',
-                        'idx_place_osm_unique'
-                       ))
+        indexes.extend(('idx_location_area_country_place_id',
+                        'idx_place_osm_unique',
+                        'idx_placex_rank_address_sector',
+                        'idx_placex_rank_boundaries_sector'))

 
     return indexes
 
 
 
     return indexes
 
 

-### CHECK FUNCTIONS
+# CHECK FUNCTIONS

 #
 # Functions are exectured in the order they appear here.
 
 #
 # Functions are exectured in the order they appear here.
 


@@ -119,7 +137,7 @@ def _get_indexes(conn):
              Project directory: {config.project_dir}
              Current setting of NOMINATIM_DATABASE_DSN: {config.DATABASE_DSN}
              """)
              Project directory: {config.project_dir}
              Current setting of NOMINATIM_DATABASE_DSN: {config.DATABASE_DSN}
              """)

-def check_connection(conn, config):
+def check_connection(conn: Any, config: Configuration) -> CheckResult:

     """ Checking database connection
     """
     if isinstance(conn, _BadConnection):
     """ Checking database connection
     """
     if isinstance(conn, _BadConnection):


@@ -137,7 +155,7 @@ def check_connection(conn, config):
              Project directory: {config.project_dir}
              Current setting of NOMINATIM_DATABASE_DSN: {config.DATABASE_DSN}
              """)
              Project directory: {config.project_dir}
              Current setting of NOMINATIM_DATABASE_DSN: {config.DATABASE_DSN}
              """)

-def check_placex_table(conn, config):
+def check_placex_table(conn: Connection, config: Configuration) -> CheckResult:

     """ Checking for placex table
     """
     if conn.table_exists('placex'):
     """ Checking for placex table
     """
     if conn.table_exists('placex'):


@@ -146,8 +164,8 @@ def check_placex_table(conn, config):
     return CheckState.FATAL, dict(config=config)
 
 
     return CheckState.FATAL, dict(config=config)
 
 

-@_check(hint="""placex table has no data. Did the import finish sucessfully?""")
-def check_placex_size(conn, _):
+@_check(hint="""placex table has no data. Did the import finish successfully?""")
+def check_placex_size(conn: Connection, _: Configuration) -> CheckResult:

     """ Checking for placex content
     """
     with conn.cursor() as cur:
     """ Checking for placex content
     """
     with conn.cursor() as cur:


@@ -157,16 +175,16 @@ def check_placex_size(conn, _):
 
 
 @_check(hint="""{msg}""")
 
 
 @_check(hint="""{msg}""")

-def check_tokenizer(_, config):
+def check_tokenizer(_: Connection, config: Configuration) -> CheckResult:

     """ Checking that tokenizer works
     """
     try:
         tokenizer = tokenizer_factory.get_tokenizer_for_db(config)
     except UsageError:
         return CheckState.FAIL, dict(msg="""\
     """ Checking that tokenizer works
     """
     try:
         tokenizer = tokenizer_factory.get_tokenizer_for_db(config)
     except UsageError:
         return CheckState.FAIL, dict(msg="""\

-            Cannot load tokenizer. Did the import finish sucessfully?""")
+            Cannot load tokenizer. Did the import finish successfully?""")

 
 

-    result = tokenizer.check_database()
+    result = tokenizer.check_database(config)

 
     if result is None:
         return CheckState.OK
 
     if result is None:
         return CheckState.OK


@@ -174,12 +192,29 @@ def check_tokenizer(_, config):
     return CheckState.FAIL, dict(msg=result)
 
 
     return CheckState.FAIL, dict(msg=result)
 
 

+@_check(hint="""\
+             Wikipedia/Wikidata importance tables missing.
+             Quality of search results may be degraded. Reverse geocoding is unaffected.
+             See https://nominatim.org/release-docs/latest/admin/Import/#wikipediawikidata-rankings
+             """)
+def check_existance_wikipedia(conn: Connection, _: Configuration) -> CheckResult:
+    """ Checking for wikipedia/wikidata data
+    """
+    if not conn.table_exists('search_name') or not conn.table_exists('place'):
+        return CheckState.NOT_APPLICABLE
+
+    with conn.cursor() as cur:
+        cnt = cur.scalar('SELECT count(*) FROM wikipedia_article')
+
+        return CheckState.WARN if cnt == 0 else CheckState.OK
+
+

 @_check(hint="""\
              The indexing didn't finish. {count} entries are not yet indexed.
 
              To index the remaining entries, run:   {index_cmd}
              """)
 @_check(hint="""\
              The indexing didn't finish. {count} entries are not yet indexed.
 
              To index the remaining entries, run:   {index_cmd}
              """)

-def check_indexing(conn, _):
+def check_indexing(conn: Connection, _: Configuration) -> CheckResult:

     """ Checking indexing status
     """
     with conn.cursor() as cur:
     """ Checking indexing status
     """
     with conn.cursor() as cur:


@@ -204,7 +239,7 @@ def check_indexing(conn, _):
 
              Rerun the index creation with:   nominatim import --continue db-postprocess
              """)
 
              Rerun the index creation with:   nominatim import --continue db-postprocess
              """)

-def check_database_indexes(conn, _):
+def check_database_indexes(conn: Connection, _: Configuration) -> CheckResult:

     """ Checking that database indexes are complete
     """
     missing = []
     """ Checking that database indexes are complete
     """
     missing = []


@@ -226,7 +261,7 @@ def check_database_indexes(conn, _):
              Invalid indexes:
                {indexes}
              """)
              Invalid indexes:
                {indexes}
              """)

-def check_database_index_valid(conn, _):
+def check_database_index_valid(conn: Connection, _: Configuration) -> CheckResult:

     """ Checking that all database indexes are valid
     """
     with conn.cursor() as cur:
     """ Checking that all database indexes are valid
     """
     with conn.cursor() as cur:


@@ -234,7 +269,7 @@ def check_database_index_valid(conn, _):
                         WHERE pg_index.indisvalid = false
                         AND pg_index.indexrelid = pg_class.oid""")
 
                         WHERE pg_index.indisvalid = false
                         AND pg_index.indexrelid = pg_class.oid""")
 

-        broken = list(cur)
+        broken = [c[0] for c in cur]

 
     if broken:
         return CheckState.FAIL, dict(indexes='\n  '.join(broken))
 
     if broken:
         return CheckState.FAIL, dict(indexes='\n  '.join(broken))


@@ -246,7 +281,7 @@ def check_database_index_valid(conn, _):
              {error}
              Run TIGER import again:   nominatim add-data --tiger-data <DIR>
              """)
              {error}
              Run TIGER import again:   nominatim add-data --tiger-data <DIR>
              """)

-def check_tiger_table(conn, config):
+def check_tiger_table(conn: Connection, config: Configuration) -> CheckResult:

     """ Checking TIGER external data table.
     """
     if not config.get_bool('USE_US_TIGER_DATA'):
     """ Checking TIGER external data table.
     """
     if not config.get_bool('USE_US_TIGER_DATA'):