when adding Tiger data, check first if database is in frozen state

diff --git a/.github/actions/build-nominatim/action.yml b/.github/actions/build-nominatim/action.yml
index d5f04efe9c48882594cf625064c4e9f8a76ae420..6508f03ec57d4b7b52fa5aaee2f23d9d9b82e490 100644 (file)
--- a/.github/actions/build-nominatim/action.yml
+++ b/.github/actions/build-nominatim/action.yml
@@ -29,8 +29,8 @@ runs:
             if [ "$FLAVOUR" == "oldstuff" ]; then
                 pip3 install MarkupSafe==2.0.1 python-dotenv psycopg2==2.7.7 jinja2==2.8 psutil==5.4.2 pyicu==2.9 osmium PyYAML==5.1 sqlalchemy==1.4 GeoAlchemy2==0.10.0 datrie asyncpg
             else
             if [ "$FLAVOUR" == "oldstuff" ]; then
                 pip3 install MarkupSafe==2.0.1 python-dotenv psycopg2==2.7.7 jinja2==2.8 psutil==5.4.2 pyicu==2.9 osmium PyYAML==5.1 sqlalchemy==1.4 GeoAlchemy2==0.10.0 datrie asyncpg
             else

-                sudo apt-get install -y -qq python3-icu python3-datrie python3-pyosmium python3-jinja2 python3-psutil python3-psycopg2 python3-dotenv python3-yaml python3-asyncpg
-                pip3 install sqlalchemy GeoAlchemy2
+                sudo apt-get install -y -qq python3-icu python3-datrie python3-pyosmium python3-jinja2 python3-psutil python3-psycopg2 python3-dotenv python3-yaml
+                pip3 install sqlalchemy GeoAlchemy2 asyncpg

             fi
           shell: bash
           env:
             fi
           shell: bash
           env:

diff --git a/nominatim/tools/freeze.py b/nominatim/tools/freeze.py
index 39c3279d3ce55718cc086cd7bf1278e5e82e9082..602def55b55ac3bb8b52212d43cab04d2361f37b 100644 (file)
--- a/nominatim/tools/freeze.py
+++ b/nominatim/tools/freeze.py
@@ -50,3 +50,9 @@ def drop_flatnode_file(fpath: Optional[Path]) -> None:
     """
     if fpath and fpath.exists():
         fpath.unlink()
     """
     if fpath and fpath.exists():
         fpath.unlink()

+
+def is_frozen(conn: Connection) -> bool:
+    """ Returns true if database is in a frozen state
+    """
+
+    return conn.table_exists('place') is False

diff --git a/nominatim/tools/tiger_data.py b/nominatim/tools/tiger_data.py
index 4a32bb1ebf93848b0126d0dfd31b5b0bd472c858..70cecae58648997f0bc6a81a863fee7b420432f5 100644 (file)
--- a/nominatim/tools/tiger_data.py
+++ b/nominatim/tools/tiger_data.py
@@ -23,6 +23,7 @@ from nominatim.db.sql_preprocessor import SQLPreprocessor
 from nominatim.errors import UsageError
 from nominatim.data.place_info import PlaceInfo
 from nominatim.tokenizer.base import AbstractAnalyzer, AbstractTokenizer
 from nominatim.errors import UsageError
 from nominatim.data.place_info import PlaceInfo
 from nominatim.tokenizer.base import AbstractAnalyzer, AbstractTokenizer

+from nominatim.tools import freeze

 
 LOG = logging.getLogger()
 
 
 LOG = logging.getLogger()
 


@@ -113,6 +114,13 @@ def add_tiger_data(data_dir: str, config: Configuration, threads: int,
     """
     dsn = config.get_libpq_dsn()
 
     """
     dsn = config.get_libpq_dsn()
 

+    with connect(dsn) as conn:
+        is_frozen = freeze.is_frozen(conn)
+        conn.close()
+
+        if is_frozen:
+            raise UsageError("Tiger cannot be imported when database frozen (Github issue #3048)")
+

     with TigerInput(data_dir) as tar:
         if not tar:
             return 1
     with TigerInput(data_dir) as tar:
         if not tar:
             return 1

diff --git a/test/python/tools/test_freeze.py b/test/python/tools/test_freeze.py
index 3ebb1730e46bd788e4c4cf24ada301bd94c0d6af..0d44501a6360357ee9b22c19fa3b20bb317af2c7 100644 (file)
--- a/test/python/tools/test_freeze.py
+++ b/test/python/tools/test_freeze.py
@@ -30,6 +30,8 @@ def test_drop_tables(temp_db_conn, temp_db_cursor, table_factory):
     for table in NOMINATIM_RUNTIME_TABLES + NOMINATIM_DROP_TABLES:
         table_factory(table)
 
     for table in NOMINATIM_RUNTIME_TABLES + NOMINATIM_DROP_TABLES:
         table_factory(table)
 

+    assert not freeze.is_frozen(temp_db_conn)
+

     freeze.drop_update_tables(temp_db_conn)
 
     for table in NOMINATIM_RUNTIME_TABLES:
     freeze.drop_update_tables(temp_db_conn)
 
     for table in NOMINATIM_RUNTIME_TABLES:


@@ -38,6 +40,8 @@ def test_drop_tables(temp_db_conn, temp_db_cursor, table_factory):
     for table in NOMINATIM_DROP_TABLES:
         assert not temp_db_cursor.table_exists(table)
 
     for table in NOMINATIM_DROP_TABLES:
         assert not temp_db_cursor.table_exists(table)
 

+    assert freeze.is_frozen(temp_db_conn)
+

 def test_drop_flatnode_file_no_file():
     freeze.drop_flatnode_file(None)
 
 def test_drop_flatnode_file_no_file():
     freeze.drop_flatnode_file(None)
 


@@ -46,7 +50,7 @@ def test_drop_flatnode_file_file_already_gone(tmp_path):
     freeze.drop_flatnode_file(tmp_path / 'something.store')
 
 
     freeze.drop_flatnode_file(tmp_path / 'something.store')
 
 

-def test_drop_flatnode_file_delte(tmp_path):
+def test_drop_flatnode_file_delete(tmp_path):

     flatfile = tmp_path / 'flatnode.store'
     flatfile.write_text('Some content')
 
     flatfile = tmp_path / 'flatnode.store'
     flatfile.write_text('Some content')
 

diff --git a/test/python/tools/test_tiger_data.py b/test/python/tools/test_tiger_data.py
index f027c4ffef19d1eb877038a57153a982fe4db61a..4b1a1b8959b43eca48d8c72f9578cc3f7b01f811 100644 (file)
--- a/test/python/tools/test_tiger_data.py
+++ b/test/python/tools/test_tiger_data.py
@@ -12,7 +12,7 @@ from textwrap import dedent
 
 import pytest
 
 
 import pytest
 

-from nominatim.tools import tiger_data
+from nominatim.tools import tiger_data, freeze

 from nominatim.errors import UsageError
 
 class MockTigerTable:
 from nominatim.errors import UsageError
 
 class MockTigerTable:


@@ -27,6 +27,9 @@ class MockTigerTable:
                                                token_info JSONB,
                                                postcode TEXT)""")
 
                                                token_info JSONB,
                                                postcode TEXT)""")
 

+            # We need this table to determine if the database is frozen or not
+            cur.execute("CREATE TABLE place (number INTEGER)")
+

     def count(self):
         with self.conn.cursor() as cur:
             return cur.scalar("SELECT count(*) FROM tiger")
     def count(self):
         with self.conn.cursor() as cur:
             return cur.scalar("SELECT count(*) FROM tiger")


@@ -80,6 +83,17 @@ def test_add_tiger_data(def_config, src_dir, tiger_table, tokenizer_mock, thread
     assert tiger_table.count() == 6213
 
 
     assert tiger_table.count() == 6213
 
 

+def test_add_tiger_data_database_frozen(def_config, temp_db_conn, tiger_table, tokenizer_mock,
+                                 tmp_path):
+    freeze.drop_update_tables(temp_db_conn)
+
+    with pytest.raises(UsageError) as excinfo:
+        tiger_data.add_tiger_data(str(tmp_path), def_config, 1, tokenizer_mock())
+
+        assert "database frozen" in str(excinfo.value)
+
+    assert tiger_table.count() == 0
+

 def test_add_tiger_data_no_files(def_config, tiger_table, tokenizer_mock,
                                  tmp_path):
     tiger_data.add_tiger_data(str(tmp_path), def_config, 1, tokenizer_mock())
 def test_add_tiger_data_no_files(def_config, tiger_table, tokenizer_mock,
                                  tmp_path):
     tiger_data.add_tiger_data(str(tmp_path), def_config, 1, tokenizer_mock())