Fix: Removed error if endstatement is wrong and improved tests

[nominatim.git] / nominatim / tools / tiger_data.py

diff --git a/nominatim/tools/tiger_data.py b/nominatim/tools/tiger_data.py
index e3adfd33a760c995ca25c53c057e5b5a00c8ee7d..c655f91d73bf096f0af4fcf7cbc7ce75c4415bd0 100644 (file)
--- a/nominatim/tools/tiger_data.py
+++ b/nominatim/tools/tiger_data.py
@@ -44,9 +44,7 @@ def handle_threaded_sql_statements(sel, file):
     lines = 0
     end_of_file = False
     # Using pool of database connections to execute sql statements
     lines = 0
     end_of_file = False
     # Using pool of database connections to execute sql statements

-    while True:
-        if end_of_file:
-            break
+    while not end_of_file:

         for key, _ in sel.select(1):
             conn = key.data
             try:
         for key, _ in sel.select(1):
             conn = key.data
             try:


@@ -61,8 +59,22 @@ def handle_threaded_sql_statements(sel, file):
                         print('. ', end='', flush=True)
                         lines = 0
             except Exception as exc: # pylint: disable=broad-except
                         print('. ', end='', flush=True)
                         lines = 0
             except Exception as exc: # pylint: disable=broad-except

-                LOG.error('Wrong SQL statement: %s', exc)
+                LOG.info('Wrong SQL statement: %s', exc)

 
 

+def handle_unregister_connection_pool(sel, place_threads):
+    """ Handles unregistering pool of connections
+    """
+
+    while place_threads > 0:
+        for key, _ in sel.select(1):
+            conn = key.data
+            sel.unregister(conn)
+            try:
+                conn.wait()
+            except Exception as exc: # pylint: disable=broad-except
+                LOG.info('Wrong SQL statement: %s', exc)
+            conn.close()
+            place_threads -= 1

 
 def add_tiger_data(dsn, data_dir, threads, config, sqllib_dir):
     """ Import tiger data from directory or tar file
 
 def add_tiger_data(dsn, data_dir, threads, config, sqllib_dir):
     """ Import tiger data from directory or tar file


@@ -97,13 +109,7 @@ def add_tiger_data(dsn, data_dir, threads, config, sqllib_dir):
         handle_threaded_sql_statements(sel, file)
 
     # Unregistering pool of database connections
         handle_threaded_sql_statements(sel, file)
 
     # Unregistering pool of database connections

-    while place_threads > 0:
-        for key, _ in sel.select(1):
-            conn = key.data
-            sel.unregister(conn)
-            conn.wait()
-            conn.close()
-            place_threads -= 1
+    handle_unregister_connection_pool(sel, place_threads)

 
     if tar:
         tar.close()
 
     if tar:
         tar.close()