read partition and languages from config file

[nominatim.git] / nominatim / tools / check_database.py

diff --git a/nominatim/tools/check_database.py b/nominatim/tools/check_database.py
index e99a3572d49553b28eee6ca3b14f7baf93c55fa7..d116554fea20f6e9b5e261adc2a48b0434fa5531 100644 (file)
--- a/nominatim/tools/check_database.py
+++ b/nominatim/tools/check_database.py
@@ -4,10 +4,9 @@ Collection of functions that check if the database is complete and functional.
 from enum import Enum
 from textwrap import dedent
 
 from enum import Enum
 from textwrap import dedent
 

-import psycopg2
-
-from ..db.connection import connect
-from ..errors import UsageError
+from nominatim.db.connection import connect
+from nominatim.errors import UsageError
+from nominatim.tokenizer import factory as tokenizer_factory

 
 CHECKLIST = []
 
 
 CHECKLIST = []
 


@@ -25,6 +24,7 @@ def _check(hint=None):
     """
     def decorator(func):
         title = func.__doc__.split('\n', 1)[0].strip()
     """
     def decorator(func):
         title = func.__doc__.split('\n', 1)[0].strip()

+

         def run_check(conn, config):
             print(title, end=' ... ')
             ret = func(conn, config)
         def run_check(conn, config):
             print(title, end=' ... ')
             ret = func(conn, config)


@@ -47,7 +47,7 @@ def _check(hint=None):
 
     return decorator
 
 
     return decorator
 

-class _BadConnection: # pylint: disable=R0903
+class _BadConnection:

 
     def __init__(self, msg):
         self.msg = msg
 
     def __init__(self, msg):
         self.msg = msg


@@ -60,7 +60,7 @@ def check_database(config):
     """ Run a number of checks on the database and return the status.
     """
     try:
     """ Run a number of checks on the database and return the status.
     """
     try:

-        conn = connect(config.get_libpq_dsn())
+        conn = connect(config.get_libpq_dsn()).connection

     except UsageError as err:
         conn = _BadConnection(str(err))
 
     except UsageError as err:
         conn = _BadConnection(str(err))
 


@@ -70,7 +70,7 @@ def check_database(config):
         if ret == CheckState.FATAL:
             conn.close()
             return 1
         if ret == CheckState.FATAL:
             conn.close()
             return 1

-        if ret != CheckState.OK:
+        if ret in (CheckState.FATAL, CheckState.FAIL):

             overall_result = 1
 
     conn.close()
             overall_result = 1
 
     conn.close()


@@ -78,13 +78,12 @@ def check_database(config):
 
 
 def _get_indexes(conn):
 
 
 def _get_indexes(conn):

-    indexes = ['idx_word_word_id',
-               'idx_place_addressline_address_place_id',
+    indexes = ['idx_place_addressline_address_place_id',

                'idx_placex_rank_search',
                'idx_placex_rank_address',
                'idx_placex_parent_place_id',
                'idx_placex_geometry_reverse_lookuppolygon',
                'idx_placex_rank_search',
                'idx_placex_rank_address',
                'idx_placex_parent_place_id',
                'idx_placex_geometry_reverse_lookuppolygon',

-               'idx_placex_geometry_reverse_placenode',
+               'idx_placex_geometry_placenode',

                'idx_osmline_parent_place_id',
                'idx_osmline_parent_osm_id',
                'idx_postcode_id',
                'idx_osmline_parent_place_id',
                'idx_osmline_parent_osm_id',
                'idx_postcode_id',


@@ -94,16 +93,18 @@ def _get_indexes(conn):
         indexes.extend(('idx_search_name_nameaddress_vector',
                         'idx_search_name_name_vector',
                         'idx_search_name_centroid'))
         indexes.extend(('idx_search_name_nameaddress_vector',
                         'idx_search_name_name_vector',
                         'idx_search_name_centroid'))

+        if conn.server_version_tuple() >= (11, 0, 0):
+            indexes.extend(('idx_placex_housenumber',
+                            'idx_osmline_parent_osm_id_with_hnr'))

     if conn.table_exists('place'):
         indexes.extend(('idx_placex_pendingsector',
                         'idx_location_area_country_place_id',
     if conn.table_exists('place'):
         indexes.extend(('idx_placex_pendingsector',
                         'idx_location_area_country_place_id',

-                        'idx_place_osm_unique'
-                       ))
+                        'idx_place_osm_unique'))

 
     return indexes
 
 
 
     return indexes
 
 

-### CHECK FUNCTIONS
+# CHECK FUNCTIONS

 #
 # Functions are exectured in the order they appear here.
 
 #
 # Functions are exectured in the order they appear here.
 


@@ -146,7 +147,7 @@ def check_placex_table(conn, config):
 
 
 @_check(hint="""placex table has no data. Did the import finish sucessfully?""")
 
 
 @_check(hint="""placex table has no data. Did the import finish sucessfully?""")

-def check_placex_size(conn, config): # pylint: disable=W0613
+def check_placex_size(conn, _):

     """ Checking for placex content
     """
     with conn.cursor() as cur:
     """ Checking for placex content
     """
     with conn.cursor() as cur:


@@ -154,38 +155,31 @@ def check_placex_size(conn, config): # pylint: disable=W0613
 
     return CheckState.OK if cnt > 0 else CheckState.FATAL
 
 
     return CheckState.OK if cnt > 0 else CheckState.FATAL
 

-@_check(hint="""\
-             The Postgresql extension nominatim.so was not correctly loaded.

 
 

-             Error: {error}
-
-             Hints:
-             * Check the output of the CMmake/make installation step
-             * Does nominatim.so exist?
-             * Does nominatim.so exist on the database server?
-             * Can nominatim.so be accessed by the database user?
-             """)
-def check_module(conn, config): # pylint: disable=W0613
-    """ Checking that nominatim.so module is installed
+@_check(hint="""{msg}""")
+def check_tokenizer(_, config):
+    """ Checking that tokenizer works

     """
     """

-    with conn.cursor() as cur:
-        try:
-            out = cur.scalar("SELECT make_standard_name('a')")
-        except psycopg2.ProgrammingError as err:
-            return CheckState.FAIL, dict(error=str(err))
+    try:
+        tokenizer = tokenizer_factory.get_tokenizer_for_db(config)
+    except UsageError:
+        return CheckState.FAIL, dict(msg="""\
+            Cannot load tokenizer. Did the import finish sucessfully?""")

 
 

-        if out != 'a':
-            return CheckState.FAIL, dict(error='Unexpected result for make_standard_name()')
+    result = tokenizer.check_database()

 
 

+    if result is None:

         return CheckState.OK
 
         return CheckState.OK
 

+    return CheckState.FAIL, dict(msg=result)
+

 
 @_check(hint="""\
              The indexing didn't finish. {count} entries are not yet indexed.
 
              To index the remaining entries, run:   {index_cmd}
              """)
 
 @_check(hint="""\
              The indexing didn't finish. {count} entries are not yet indexed.
 
              To index the remaining entries, run:   {index_cmd}
              """)

-def check_indexing(conn, config): # pylint: disable=W0613
+def check_indexing(conn, _):

     """ Checking indexing status
     """
     with conn.cursor() as cur:
     """ Checking indexing status
     """
     with conn.cursor() as cur:


@@ -194,24 +188,23 @@ def check_indexing(conn, config): # pylint: disable=W0613
     if cnt == 0:
         return CheckState.OK
 
     if cnt == 0:
         return CheckState.OK
 

-    if conn.index_exists('idx_word_word_id'):
+    if conn.index_exists('idx_placex_rank_search'):

         # Likely just an interrupted update.
         index_cmd = 'nominatim index'
     else:
         # Likely just an interrupted update.
         index_cmd = 'nominatim index'
     else:

-        # Looks like the import process got interupted.
+        # Looks like the import process got interrupted.

         index_cmd = 'nominatim import --continue indexing'
 
     return CheckState.FAIL, dict(count=cnt, index_cmd=index_cmd)
 
 
         index_cmd = 'nominatim import --continue indexing'
 
     return CheckState.FAIL, dict(count=cnt, index_cmd=index_cmd)
 
 

-

 @_check(hint="""\
              The following indexes are missing:
                {indexes}
 
              Rerun the index creation with:   nominatim import --continue db-postprocess
              """)
 @_check(hint="""\
              The following indexes are missing:
                {indexes}
 
              Rerun the index creation with:   nominatim import --continue db-postprocess
              """)

-def check_database_indexes(conn, config): # pylint: disable=W0613
+def check_database_indexes(conn, _):

     """ Checking that database indexes are complete
     """
     missing = []
     """ Checking that database indexes are complete
     """
     missing = []


@@ -233,7 +226,7 @@ def check_database_indexes(conn, config): # pylint: disable=W0613
              Invalid indexes:
                {indexes}
              """)
              Invalid indexes:
                {indexes}
              """)

-def check_database_index_valid(conn, config): # pylint: disable=W0613
+def check_database_index_valid(conn, _):

     """ Checking that all database indexes are valid
     """
     with conn.cursor() as cur:
     """ Checking that all database indexes are valid
     """
     with conn.cursor() as cur: