add full tokens to addresses

[nominatim.git] / nominatim / tools / refresh.py

diff --git a/nominatim/tools/refresh.py b/nominatim/tools/refresh.py
index a3b6c4f086397cb4467e45c6406f346f7e11def8..008fc7143313469a7962934df9ded6c8415b3e7b 100644 (file)
--- a/nominatim/tools/refresh.py
+++ b/nominatim/tools/refresh.py
@@ -15,10 +15,10 @@ from pathlib import Path
 from psycopg2 import sql as pysql
 
 from nominatim.config import Configuration
 from psycopg2 import sql as pysql
 
 from nominatim.config import Configuration

-from nominatim.db.connection import Connection
+from nominatim.db.connection import Connection, connect

 from nominatim.db.utils import execute_file
 from nominatim.db.sql_preprocessor import SQLPreprocessor
 from nominatim.db.utils import execute_file
 from nominatim.db.sql_preprocessor import SQLPreprocessor

-from nominatim.version import version_str
+from nominatim.version import NOMINATIM_VERSION

 
 LOG = logging.getLogger()
 
 
 LOG = logging.getLogger()
 


@@ -120,6 +120,7 @@ PHP_CONST_DEFS = (
     ('Search_NameOnlySearchFrequencyThreshold', 'SEARCH_NAME_ONLY_THRESHOLD', str),
     ('Use_US_Tiger_Data', 'USE_US_TIGER_DATA', bool),
     ('MapIcon_URL', 'MAPICON_URL', str),
     ('Search_NameOnlySearchFrequencyThreshold', 'SEARCH_NAME_ONLY_THRESHOLD', str),
     ('Use_US_Tiger_Data', 'USE_US_TIGER_DATA', bool),
     ('MapIcon_URL', 'MAPICON_URL', str),

+    ('Search_WithinCountries', 'SEARCH_WITHIN_COUNTRIES', bool),

 )
 
 
 )
 
 


@@ -146,23 +147,23 @@ def import_wikipedia_articles(dsn: str, data_path: Path, ignore_errors: bool = F
 
     return 0
 
 
     return 0
 

-def import_osm_views_geotiff(dsn, data_path, ignore_errors=False):
-    """ Replaces the OSM views table with new data.
-        
-        Returns 0 if all was well and 1 if the GeoTIFF file could not
+def import_secondary_importance(dsn: str, data_path: Path, ignore_errors: bool = False) -> int:
+    """ Replaces the secondary importance raster data table with new data.
+
+        Returns 0 if all was well and 1 if the raster SQL file could not

         be found. Throws an exception if there was an error reading the file.
     """
         be found. Throws an exception if there was an error reading the file.
     """

-    datafile = data_path / 'osmviews.tiff'
-
+    datafile = data_path / 'secondary_importance.sql.gz'

     if not datafile.exists():
         return 1
 
     if not datafile.exists():
         return 1
 

-    pre_code = """BEGIN;
-                  DROP TABLE IF EXISTS "osmviews";
-               """
-    post_code = "COMMIT"
-    execute_file(dsn, datafile, ignore_errors=ignore_errors,
-                 pre_code=pre_code, post_code=post_code)
+    with connect(dsn) as conn:
+        postgis_version = conn.postgis_version_tuple()
+        if postgis_version[0] < 3:
+            LOG.error('PostGIS version is too old for using OSM raster data.')
+            return 2
+
+    execute_file(dsn, datafile, ignore_errors=ignore_errors)

 
     return 0
 
 
     return 0
 


@@ -176,7 +177,7 @@ def recompute_importance(conn: Connection) -> None:
         cur.execute("""
             UPDATE placex SET (wikipedia, importance) =
                (SELECT wikipedia, importance
         cur.execute("""
             UPDATE placex SET (wikipedia, importance) =
                (SELECT wikipedia, importance

-                FROM compute_importance(extratags, country_code, osm_type, osm_id))
+                FROM compute_importance(extratags, country_code, rank_search, centroid))

             """)
         cur.execute("""
             UPDATE placex s SET wikipedia = d.wikipedia, importance = d.importance
             """)
         cur.execute("""
             UPDATE placex s SET wikipedia = d.wikipedia, importance = d.importance


@@ -212,34 +213,41 @@ def _quote_php_variable(var_type: Type[Any], config: Configuration,
 def setup_website(basedir: Path, config: Configuration, conn: Connection) -> None:
     """ Create the website script stubs.
     """
 def setup_website(basedir: Path, config: Configuration, conn: Connection) -> None:
     """ Create the website script stubs.
     """

+    if config.lib_dir.php is None:
+        LOG.info("Python frontend does not require website setup. Skipping.")
+        return
+

     if not basedir.exists():
         LOG.info('Creating website directory.')
         basedir.mkdir()
 
     if not basedir.exists():
         LOG.info('Creating website directory.')
         basedir.mkdir()
 

-    template = dedent(f"""\
+    assert config.project_dir is not None
+    basedata = dedent(f"""\

                       <?php
 
                       @define('CONST_Debug', $_GET['debug'] ?? false);
                       @define('CONST_LibDir', '{config.lib_dir.php}');
                       @define('CONST_TokenizerDir', '{config.project_dir / 'tokenizer'}');
                       <?php
 
                       @define('CONST_Debug', $_GET['debug'] ?? false);
                       @define('CONST_LibDir', '{config.lib_dir.php}');
                       @define('CONST_TokenizerDir', '{config.project_dir / 'tokenizer'}');

-                      @define('CONST_NominatimVersion', '{version_str()}');
+                      @define('CONST_NominatimVersion', '{NOMINATIM_VERSION!s}');

 
                       """)
 
     for php_name, conf_name, var_type in PHP_CONST_DEFS:
         varout = _quote_php_variable(var_type, config, conf_name)
 
 
                       """)
 
     for php_name, conf_name, var_type in PHP_CONST_DEFS:
         varout = _quote_php_variable(var_type, config, conf_name)
 

-        template += f"@define('CONST_{php_name}', {varout});\n"
+        basedata += f"@define('CONST_{php_name}', {varout});\n"

 
 

-    template += f"\nrequire_once('{config.lib_dir.php}/website/{{}}');\n"
+    template = "\nrequire_once(CONST_LibDir.'/website/{}');\n"

 
     search_name_table_exists = bool(conn and conn.table_exists('search_name'))
 
     for script in WEBSITE_SCRIPTS:
         if not search_name_table_exists and script == 'search.php':
 
     search_name_table_exists = bool(conn and conn.table_exists('search_name'))
 
     for script in WEBSITE_SCRIPTS:
         if not search_name_table_exists and script == 'search.php':

-            (basedir / script).write_text(template.format('reverse-only-search.php'), 'utf-8')
+            out = template.format('reverse-only-search.php')

         else:
         else:

-            (basedir / script).write_text(template.format(script), 'utf-8')
+            out = template.format(script)
+
+        (basedir / script).write_text(basedata + out, 'utf-8')

 
 
 def invalidate_osm_object(osm_type: str, osm_id: int, conn: Connection,
 
 
 def invalidate_osm_object(osm_type: str, osm_id: int, conn: Connection,