Initial implementation of GeoTIFF import functionality

diff --git a/nominatim/clicmd/refresh.py b/nominatim/clicmd/refresh.py
index dce28d983a1365656c43b3e57bc79eb20c8befc8..6f307a65ae159cc2ac841588c2cf81b89435ac83 100644 (file)
--- a/nominatim/clicmd/refresh.py
+++ b/nominatim/clicmd/refresh.py
@@ -63,6 +63,8 @@ class UpdateRefresh:
                            help='Update the PL/pgSQL functions in the database')
         group.add_argument('--wiki-data', action='store_true',
                            help='Update Wikipedia/data importance numbers')
+        group.add_argument('--osm-views', action='store_true',
+                           help='Update OSM views/data importance numbers')
         group.add_argument('--importance', action='store_true',
                            help='Recompute place importances (expensive!)')
         group.add_argument('--website', action='store_true',
@@ -130,6 +132,15 @@ class UpdateRefresh:
                                                  data_path) > 0:
                 LOG.fatal('FATAL: Wikipedia importance dump file not found')
                 return 1
+        
+        if args.osm_views:
+            data_path = Path(args.config.OSM_VIEWS_DATA_PATH
+                             or args.project_dir)
+            LOG.warning('Import OSM views GeoTIFF data from %s', data_path)
+            if refresh.import_osm_views_geotiff(args.config.get_libpq_dsn(),
+                                                 data_path) > 0:
+                LOG.fatal('FATAL: OSM views GeoTIFF file not found')
+                return 1
 
         # Attention: importance MUST come after wiki data import.
         if args.importance:

diff --git a/nominatim/clicmd/setup.py b/nominatim/clicmd/setup.py
index 297244331b5f6e5065030516007a35f38a80965c..c7366c3a9e7edd7b7e4d8f493017e429f7bfe245 100644 (file)
--- a/nominatim/clicmd/setup.py
+++ b/nominatim/clicmd/setup.py
@@ -58,8 +58,10 @@ class SetupAll:
         group2.add_argument('--no-updates', action='store_true',
                            help="Do not keep tables that are only needed for "
                                 "updating the database later")
+        group2.add_argument('--osm-views', action='store_true',
+                            help='Import OSM views GeoTIFF')
         group2.add_argument('--offline', action='store_true',
-                           help="Do not attempt to load any additional data from the internet")
+                            help="Do not attempt to load any additional data from the internet")
         group3 = parser.add_argument_group('Expert options')
         group3.add_argument('--ignore-errors', action='store_true',
                            help='Continue import even when errors in SQL are present')
@@ -103,7 +105,15 @@ class SetupAll:
             if refresh.import_wikipedia_articles(args.config.get_libpq_dsn(),
                                                  data_path) > 0:
                 LOG.error('Wikipedia importance dump file not found. '
-                          'Will be using default importances.')
+                          'Calculating importance values of locations will not use Wikipedia importance data.')
+            
+            LOG.warning('Importing OSM views GeoTIFF data')
+            database_import.import_osm_views_geotiff()
+            data_path = Path(args.config.OSM_VIEWS_DATA_PATH or args.project_dir)
+            if refresh.import_osm_views_geotiff(args.config.get_libpq_dsn(),
+                                                 data_path) > 0:
+                LOG.error('OSM views GeoTIFF file not found. '
+                          'Calculating importance values of locations will not use OSM views data.')
 
         if args.continue_at is None or args.continue_at == 'load-data':
             LOG.warning('Initialise tables')

diff --git a/nominatim/tools/database_import.py b/nominatim/tools/database_import.py
index f6ebe90d0983588ccc101988f5085d6692f405a0..20883b96ce0289bb2c2c28b6210695ee51759057 100644 (file)
--- a/nominatim/tools/database_import.py
+++ b/nominatim/tools/database_import.py
@@ -75,6 +75,7 @@ def setup_database_skeleton(dsn: str, rouser: Optional[str] = None) -> None:
         with conn.cursor() as cur:
             cur.execute('CREATE EXTENSION IF NOT EXISTS hstore')
             cur.execute('CREATE EXTENSION IF NOT EXISTS postgis')
+            cur.execute('CREATE EXTENSION IF NOT EXISTS postgis_raster')
         conn.commit()
 
         _require_version('PostGIS',
@@ -246,3 +247,8 @@ def create_search_indices(conn: Connection, config: Configuration,
 
     sql.run_parallel_sql_file(config.get_libpq_dsn(),
                               'indices.sql', min(8, threads), drop=drop)
+
+
+def import_osm_views_geotiff():
+    """Import OSM views GeoTIFF file"""
+    subprocess.run("raster2pgsql -s 4326 -I -C -t 100x100 -e osmviews.tiff public.osmviews | psql nominatim", shell=True, check=True)

diff --git a/nominatim/tools/refresh.py b/nominatim/tools/refresh.py
index 8c1e9d9bbb24882601e4d0d8bed6c3c54a41f498..a3b6c4f086397cb4467e45c6406f346f7e11def8 100644 (file)
--- a/nominatim/tools/refresh.py
+++ b/nominatim/tools/refresh.py
@@ -146,6 +146,25 @@ def import_wikipedia_articles(dsn: str, data_path: Path, ignore_errors: bool = F
 
     return 0
 
+def import_osm_views_geotiff(dsn, data_path, ignore_errors=False):
+    """ Replaces the OSM views table with new data.
+        
+        Returns 0 if all was well and 1 if the GeoTIFF file could not
+        be found. Throws an exception if there was an error reading the file.
+    """
+    datafile = data_path / 'osmviews.tiff'
+
+    if not datafile.exists():
+        return 1
+
+    pre_code = """BEGIN;
+                  DROP TABLE IF EXISTS "osmviews";
+               """
+    post_code = "COMMIT"
+    execute_file(dsn, datafile, ignore_errors=ignore_errors,
+                 pre_code=pre_code, post_code=post_code)
+
+    return 0
 
 def recompute_importance(conn: Connection) -> None:
     """ Recompute wikipedia links and importance for all entries in placex.

diff --git a/settings/env.defaults b/settings/env.defaults
index 3115f4382aacf582c5a1054e78c03130bde9f00f..c975926254fcfc9919425d740abd6303c7876abc 100644 (file)
--- a/settings/env.defaults
+++ b/settings/env.defaults
@@ -86,6 +86,10 @@ NOMINATIM_TIGER_DATA_PATH=
 # When unset, the data is expected to be located in the project directory.
 NOMINATIM_WIKIPEDIA_DATA_PATH=
 
+# Directory where to find OSM views GeoTIFF file.
+# When unset, the data is expected to be located in the project directory.
+NOMINATIM_OSM_VIEWS_DATA_PATH=
+
 # Configuration file for special phrase import.
 # OBSOLETE: use `nominatim special-phrases --config <file>` or simply put
 #           a custom phrase-settings.json into your project directory.

diff --git a/test/python/cli/test_cmd_import.py b/test/python/cli/test_cmd_import.py
index 737c4e5c369b0eaea017ae19b468033fa639b03b..b6a8721fb61188de9c5a8c0a65913bfdb90db379 100644 (file)
--- a/test/python/cli/test_cmd_import.py
+++ b/test/python/cli/test_cmd_import.py
@@ -40,6 +40,7 @@ class TestCliImportWithDb:
             mock_func_factory(nominatim.data.country_info, 'setup_country_tables'),
             mock_func_factory(nominatim.tools.database_import, 'import_osm_data'),
             mock_func_factory(nominatim.tools.refresh, 'import_wikipedia_articles'),
+            mock_func_factory(nominatim.tools.refresh, 'import_osm_views_geotiff'),
             mock_func_factory(nominatim.tools.database_import, 'truncate_data_tables'),
             mock_func_factory(nominatim.tools.database_import, 'load_data'),
             mock_func_factory(nominatim.tools.database_import, 'create_tables'),

diff --git a/test/python/cli/test_cmd_refresh.py b/test/python/cli/test_cmd_refresh.py
index 7f44765bcade9ce08cb4c0f41da6f95f19e822c5..ed3a68babce0a21256c860f94345cedcf5928ec2 100644 (file)
--- a/test/python/cli/test_cmd_refresh.py
+++ b/test/python/cli/test_cmd_refresh.py
@@ -24,6 +24,7 @@ class TestRefresh:
     @pytest.mark.parametrize("command,func", [
                              ('address-levels', 'load_address_levels_from_config'),
                              ('wiki-data', 'import_wikipedia_articles'),
+                             ('osm-views', 'import_osm_views_geotiff')
                              ('importance', 'recompute_importance'),
                              ('website', 'setup_website'),
                              ])
@@ -71,15 +72,21 @@ class TestRefresh:
 
         assert self.call_nominatim('refresh', '--wiki-data') == 1
 
+    def test_refresh_osm_views_geotiff_file_not_found(self, monkeypatch):
+        monkeypatch.setenv('NOMINATIM_OSM_VIEWS_DATA_PATH', 'gjoiergjeroi345Q')
 
-    def test_refresh_importance_computed_after_wiki_import(self, monkeypatch):
+        assert self.call_nominatim('refresh', '--osm-views') == 1
+
+    def test_refresh_importance_computed_after_wiki_and_osm_views_import(self, monkeypatch):
         calls = []
         monkeypatch.setattr(nominatim.tools.refresh, 'import_wikipedia_articles',
                             lambda *args, **kwargs: calls.append('import') or 0)
+        monkeypatch.setattr(nominatim.tools.refresh, 'import_osm_views_geotiff',
+                            lambda *args, **kwargs: calls.append('import') or 0)
         monkeypatch.setattr(nominatim.tools.refresh, 'recompute_importance',
                             lambda *args, **kwargs: calls.append('update'))
 
-        assert self.call_nominatim('refresh', '--importance', '--wiki-data') == 0
+        assert self.call_nominatim('refresh', '--importance', '--wiki-data', '--osm-views') == 0
 
         assert calls == ['import', 'update']
 

diff --git a/test/python/tools/test_freeze.py b/test/python/tools/test_freeze.py
index 3ebb1730e46bd788e4c4cf24ada301bd94c0d6af..6e8525500bf56934355bc29ed2ef43d7d93a0d96 100644 (file)
--- a/test/python/tools/test_freeze.py
+++ b/test/python/tools/test_freeze.py
@@ -21,6 +21,7 @@ NOMINATIM_DROP_TABLES = [
     'address_levels',
     'location_area', 'location_area_country', 'location_area_large_100',
     'location_road_1',
+    'osmviews'
     'place', 'planet_osm_nodes', 'planet_osm_rels', 'planet_osm_ways',
     'search_name_111',
     'wikipedia_article', 'wikipedia_redirect'

diff --git a/test/python/tools/test_refresh.py b/test/python/tools/test_refresh.py
index ac52aa36091643923d1c6791038b78ddc8f8c8d9..c8ebdab8734eb94d0900811bd20d52ee28d1b3cb 100644 (file)
--- a/test/python/tools/test_refresh.py
+++ b/test/python/tools/test_refresh.py
@@ -17,6 +17,10 @@ def test_refresh_import_wikipedia_not_existing(dsn):
     assert refresh.import_wikipedia_articles(dsn, Path('.')) == 1
 
 
+def test_refresh_import_osm_views_geotiff_not_existing(dsn):
+    assert refresh.import_osm_views_geotiff(dsn, Path('.')) == 1
+
+
 @pytest.mark.parametrize("replace", (True, False))
 def test_refresh_import_wikipedia(dsn, src_dir, table_factory, temp_db_cursor, replace):
     if replace:
@@ -30,6 +34,17 @@ def test_refresh_import_wikipedia(dsn, src_dir, table_factory, temp_db_cursor, r
     assert temp_db_cursor.table_rows('wikipedia_redirect') > 0
 
 
+@pytest.mark.parametrize("replace", (True, False))
+def test_refresh_import_osm_views_geotiff(dsn, src_dir, table_factory, temp_db_cursor, replace):
+    if replace:
+        table_factory('osmviews')
+
+    # use the small osm views GeoTIFF file for the API testdb
+    assert refresh.import_osm_views_geotiff(dsn, src_dir / 'test' / 'testdb') == 0
+
+    assert temp_db_cursor.table_rows('osmviews') > 0
+
+
 def test_recompute_importance(placex_table, table_factory, temp_db_conn, temp_db_cursor):
     temp_db_cursor.execute("""CREATE OR REPLACE FUNCTION compute_importance(extratags HSTORE,
                                               country_code varchar(2),