Merge pull request #3108 from mtmail/remove-legacy-wikipedia-tag-syntax

[nominatim.git] / nominatim / tools / refresh.py

diff --git a/nominatim/tools/refresh.py b/nominatim/tools/refresh.py
index 531de14db8d4911270cf603a4bc6fc0c27f29337..43e5b1eb387a4641ef49274bc63eaa485c171eda 100644 (file)
--- a/nominatim/tools/refresh.py
+++ b/nominatim/tools/refresh.py
@@ -9,7 +9,6 @@ Functions for bringing auxiliary data in the database up-to-date.
 """
 from typing import MutableSequence, Tuple, Any, Type, Mapping, Sequence, List, cast
 import logging
-import subprocess
 from textwrap import dedent
 from pathlib import Path
 
@@ -19,7 +18,7 @@ from nominatim.config import Configuration
 from nominatim.db.connection import Connection, connect
 from nominatim.db.utils import execute_file
 from nominatim.db.sql_preprocessor import SQLPreprocessor
-from nominatim.version import version_str
+from nominatim.version import NOMINATIM_VERSION
 
 LOG = logging.getLogger()
 
@@ -121,6 +120,7 @@ PHP_CONST_DEFS = (
     ('Search_NameOnlySearchFrequencyThreshold', 'SEARCH_NAME_ONLY_THRESHOLD', str),
     ('Use_US_Tiger_Data', 'USE_US_TIGER_DATA', bool),
     ('MapIcon_URL', 'MAPICON_URL', str),
+    ('Search_WithinCountries', 'SEARCH_WITHIN_COUNTRIES', bool),
 )
 
 
@@ -147,48 +147,23 @@ def import_wikipedia_articles(dsn: str, data_path: Path, ignore_errors: bool = F
 
     return 0
 
-def import_osm_views_geotiff(dsn: str, data_path: Path) -> int:
-    """ Replaces the OSM views table with new data.
+def import_secondary_importance(dsn: str, data_path: Path, ignore_errors: bool = False) -> int:
+    """ Replaces the secondary importance raster data table with new data.
 
-        Returns 0 if all was well and 1 if the OSM views GeoTIFF file could not
+        Returns 0 if all was well and 1 if the raster SQL file could not
         be found. Throws an exception if there was an error reading the file.
     """
-    datafile = data_path / 'osmviews.tiff'
+    datafile = data_path / 'secondary_importance.sql.gz'
     if not datafile.exists():
         return 1
-    with connect(dsn) as conn:
 
+    with connect(dsn) as conn:
         postgis_version = conn.postgis_version_tuple()
         if postgis_version[0] < 3:
+            LOG.error('PostGIS version is too old for using OSM raster data.')
             return 2
 
-        with conn.cursor() as cur:
-            cur.drop_table("osm_views")
-            cur.drop_table("osm_views_stat")
-
-            # -ovr: 6 -> zoom 12, 5 -> zoom 13, 4 -> zoom 14, 3 -> zoom 15
-            reproject_geotiff = f"gdalwarp -q -multi -ovr 3 -overwrite \
-                -co COMPRESS=LZW -tr 0.01 0.01 -t_srs EPSG:4326 {datafile} raster2import.tiff"
-            subprocess.run(["/bin/bash", "-c" , reproject_geotiff], check=True)
-
-            tile_size = 256
-            import_geotiff = f"raster2pgsql -I -C -Y -t {tile_size}x{tile_size} raster2import.tiff \
-                public.osm_views | psql {dsn} > /dev/null"
-            subprocess.run(["/bin/bash", "-c" , import_geotiff], check=True)
-
-            cleanup = "rm raster2import.tiff"
-            subprocess.run(["/bin/bash", "-c" , cleanup], check=True)
-
-            # To normalize osm views data, the max view value is needed
-            cur.execute(f"""
-            CREATE TABLE osm_views_stat AS (
-                SELECT MAX(ST_Value(osm_views.rast, 1, x, y)) AS max_views_count
-                FROM osm_views CROSS JOIN
-                generate_series(1, {tile_size}) As x
-                CROSS JOIN generate_series(1, {tile_size}) As y
-                WHERE x <= ST_Width(rast) AND y <= ST_Height(rast));
-            """)
-            conn.commit()
+    execute_file(dsn, datafile, ignore_errors=ignore_errors)
 
     return 0
 
@@ -202,7 +177,7 @@ def recompute_importance(conn: Connection) -> None:
         cur.execute("""
             UPDATE placex SET (wikipedia, importance) =
                (SELECT wikipedia, importance
-                FROM compute_importance(extratags, country_code, osm_type, osm_id, centroid))
+                FROM compute_importance(extratags, country_code, rank_search, centroid))
             """)
         cur.execute("""
             UPDATE placex s SET wikipedia = d.wikipedia, importance = d.importance
@@ -242,30 +217,33 @@ def setup_website(basedir: Path, config: Configuration, conn: Connection) -> Non
         LOG.info('Creating website directory.')
         basedir.mkdir()
 
-    template = dedent(f"""\
+    assert config.project_dir is not None
+    basedata = dedent(f"""\
                       <?php
 
                       @define('CONST_Debug', $_GET['debug'] ?? false);
                       @define('CONST_LibDir', '{config.lib_dir.php}');
                       @define('CONST_TokenizerDir', '{config.project_dir / 'tokenizer'}');
-                      @define('CONST_NominatimVersion', '{version_str()}');
+                      @define('CONST_NominatimVersion', '{NOMINATIM_VERSION!s}');
 
                       """)
 
     for php_name, conf_name, var_type in PHP_CONST_DEFS:
         varout = _quote_php_variable(var_type, config, conf_name)
 
-        template += f"@define('CONST_{php_name}', {varout});\n"
+        basedata += f"@define('CONST_{php_name}', {varout});\n"
 
-    template += f"\nrequire_once('{config.lib_dir.php}/website/{{}}');\n"
+    template = "\nrequire_once(CONST_LibDir.'/website/{}');\n"
 
     search_name_table_exists = bool(conn and conn.table_exists('search_name'))
 
     for script in WEBSITE_SCRIPTS:
         if not search_name_table_exists and script == 'search.php':
-            (basedir / script).write_text(template.format('reverse-only-search.php'), 'utf-8')
+            out = template.format('reverse-only-search.php')
         else:
-            (basedir / script).write_text(template.format(script), 'utf-8')
+            out = template.format(script)
+
+        (basedir / script).write_text(basedata + out, 'utf-8')
 
 
 def invalidate_osm_object(osm_type: str, osm_id: int, conn: Connection,