Merge pull request #3108 from mtmail/remove-legacy-wikipedia-tag-syntax

[nominatim.git] / nominatim / tools / refresh.py

diff --git a/nominatim/tools/refresh.py b/nominatim/tools/refresh.py
index 95be4c0f6a16c9be3c8f437aa3f65e989c0880cb..43e5b1eb387a4641ef49274bc63eaa485c171eda 100644 (file)
--- a/nominatim/tools/refresh.py
+++ b/nominatim/tools/refresh.py
@@ -7,20 +7,25 @@
 """
 Functions for bringing auxiliary data in the database up-to-date.
 """
 """
 Functions for bringing auxiliary data in the database up-to-date.
 """

+from typing import MutableSequence, Tuple, Any, Type, Mapping, Sequence, List, cast

 import logging
 from textwrap import dedent
 from pathlib import Path
 
 from psycopg2 import sql as pysql
 
 import logging
 from textwrap import dedent
 from pathlib import Path
 
 from psycopg2 import sql as pysql
 

+from nominatim.config import Configuration
+from nominatim.db.connection import Connection, connect

 from nominatim.db.utils import execute_file
 from nominatim.db.sql_preprocessor import SQLPreprocessor
 from nominatim.version import NOMINATIM_VERSION
 
 LOG = logging.getLogger()
 
 from nominatim.db.utils import execute_file
 from nominatim.db.sql_preprocessor import SQLPreprocessor
 from nominatim.version import NOMINATIM_VERSION
 
 LOG = logging.getLogger()
 

+OSM_TYPE = {'N': 'node', 'W': 'way', 'R': 'relation'}

 
 

-def _add_address_level_rows_from_entry(rows, entry):
+def _add_address_level_rows_from_entry(rows: MutableSequence[Tuple[Any, ...]],
+                                       entry: Mapping[str, Any]) -> None:

     """ Converts a single entry from the JSON format for address rank
         descriptions into a flat format suitable for inserting into a
         PostgreSQL table and adds these lines to `rows`.
     """ Converts a single entry from the JSON format for address rank
         descriptions into a flat format suitable for inserting into a
         PostgreSQL table and adds these lines to `rows`.


@@ -37,35 +42,39 @@ def _add_address_level_rows_from_entry(rows, entry):
             for country in countries:
                 rows.append((country, key, value, rank_search, rank_address))
 
             for country in countries:
                 rows.append((country, key, value, rank_search, rank_address))
 

-def load_address_levels(conn, table, levels):
+
+def load_address_levels(conn: Connection, table: str, levels: Sequence[Mapping[str, Any]]) -> None:

     """ Replace the `address_levels` table with the contents of `levels'.
 
         A new table is created any previously existing table is dropped.
         The table has the following columns:
             country, class, type, rank_search, rank_address
     """
     """ Replace the `address_levels` table with the contents of `levels'.
 
         A new table is created any previously existing table is dropped.
         The table has the following columns:
             country, class, type, rank_search, rank_address
     """

-    rows = []
+    rows: List[Tuple[Any, ...]]  = []

     for entry in levels:
         _add_address_level_rows_from_entry(rows, entry)
 
     with conn.cursor() as cur:
         cur.drop_table(table)
 
     for entry in levels:
         _add_address_level_rows_from_entry(rows, entry)
 
     with conn.cursor() as cur:
         cur.drop_table(table)
 

-        cur.execute("""CREATE TABLE {} (country_code varchar(2),
+        cur.execute(pysql.SQL("""CREATE TABLE {} (
+                                        country_code varchar(2),

                                         class TEXT,
                                         type TEXT,
                                         rank_search SMALLINT,
                                         class TEXT,
                                         type TEXT,
                                         rank_search SMALLINT,

-                                        rank_address SMALLINT)""".format(table))
+                                        rank_address SMALLINT)
+                              """).format(pysql.Identifier(table)))

 
         cur.execute_values(pysql.SQL("INSERT INTO {} VALUES %s")
                            .format(pysql.Identifier(table)), rows)
 
 
         cur.execute_values(pysql.SQL("INSERT INTO {} VALUES %s")
                            .format(pysql.Identifier(table)), rows)
 

-        cur.execute('CREATE UNIQUE INDEX ON {} (country_code, class, type)'.format(table))
+        cur.execute(pysql.SQL('CREATE UNIQUE INDEX ON {} (country_code, class, type)')
+                    .format(pysql.Identifier(table)))

 
     conn.commit()
 
 
 
     conn.commit()
 
 

-def load_address_levels_from_config(conn, config):
+def load_address_levels_from_config(conn: Connection, config: Configuration) -> None:

     """ Replace the `address_levels` table with the content as
         defined in the given configuration. Uses the parameter
         NOMINATIM_ADDRESS_LEVEL_CONFIG to determine the location of the
     """ Replace the `address_levels` table with the content as
         defined in the given configuration. Uses the parameter
         NOMINATIM_ADDRESS_LEVEL_CONFIG to determine the location of the


@@ -75,7 +84,9 @@ def load_address_levels_from_config(conn, config):
     load_address_levels(conn, 'address_levels', cfg)
 
 
     load_address_levels(conn, 'address_levels', cfg)
 
 

-def create_functions(conn, config, enable_diff_updates=True, enable_debug=False):
+def create_functions(conn: Connection, config: Configuration,
+                     enable_diff_updates: bool = True,
+                     enable_debug: bool = False) -> None:

     """ (Re)create the PL/pgSQL functions.
     """
     sql = SQLPreprocessor(conn, config)
     """ (Re)create the PL/pgSQL functions.
     """
     sql = SQLPreprocessor(conn, config)


@@ -109,13 +120,14 @@ PHP_CONST_DEFS = (
     ('Search_NameOnlySearchFrequencyThreshold', 'SEARCH_NAME_ONLY_THRESHOLD', str),
     ('Use_US_Tiger_Data', 'USE_US_TIGER_DATA', bool),
     ('MapIcon_URL', 'MAPICON_URL', str),
     ('Search_NameOnlySearchFrequencyThreshold', 'SEARCH_NAME_ONLY_THRESHOLD', str),
     ('Use_US_Tiger_Data', 'USE_US_TIGER_DATA', bool),
     ('MapIcon_URL', 'MAPICON_URL', str),

+    ('Search_WithinCountries', 'SEARCH_WITHIN_COUNTRIES', bool),

 )
 
 
 )
 
 

-def import_wikipedia_articles(dsn, data_path, ignore_errors=False):
+def import_wikipedia_articles(dsn: str, data_path: Path, ignore_errors: bool = False) -> int:

     """ Replaces the wikipedia importance tables with new data.
         The import is run in a single transaction so that the new data
     """ Replaces the wikipedia importance tables with new data.
         The import is run in a single transaction so that the new data

-        is replace seemlessly.
+        is replace seamlessly.

 
         Returns 0 if all was well and 1 if the importance file could not
         be found. Throws an exception if there was an error reading the file.
 
         Returns 0 if all was well and 1 if the importance file could not
         be found. Throws an exception if there was an error reading the file.


@@ -135,8 +147,27 @@ def import_wikipedia_articles(dsn, data_path, ignore_errors=False):
 
     return 0
 
 
     return 0
 

+def import_secondary_importance(dsn: str, data_path: Path, ignore_errors: bool = False) -> int:
+    """ Replaces the secondary importance raster data table with new data.
+
+        Returns 0 if all was well and 1 if the raster SQL file could not
+        be found. Throws an exception if there was an error reading the file.
+    """
+    datafile = data_path / 'secondary_importance.sql.gz'
+    if not datafile.exists():
+        return 1
+
+    with connect(dsn) as conn:
+        postgis_version = conn.postgis_version_tuple()
+        if postgis_version[0] < 3:
+            LOG.error('PostGIS version is too old for using OSM raster data.')
+            return 2
+
+    execute_file(dsn, datafile, ignore_errors=ignore_errors)
+
+    return 0

 
 

-def recompute_importance(conn):
+def recompute_importance(conn: Connection) -> None:

     """ Recompute wikipedia links and importance for all entries in placex.
         This is a long-running operations that must not be executed in
         parallel with updates.
     """ Recompute wikipedia links and importance for all entries in placex.
         This is a long-running operations that must not be executed in
         parallel with updates.


@@ -146,7 +177,7 @@ def recompute_importance(conn):
         cur.execute("""
             UPDATE placex SET (wikipedia, importance) =
                (SELECT wikipedia, importance
         cur.execute("""
             UPDATE placex SET (wikipedia, importance) =
                (SELECT wikipedia, importance

-                FROM compute_importance(extratags, country_code, osm_type, osm_id))
+                FROM compute_importance(extratags, country_code, rank_search, centroid))

             """)
         cur.execute("""
             UPDATE placex s SET wikipedia = d.wikipedia, importance = d.importance
             """)
         cur.execute("""
             UPDATE placex s SET wikipedia = d.wikipedia, importance = d.importance


@@ -159,18 +190,19 @@ def recompute_importance(conn):
     conn.commit()
 
 
     conn.commit()
 
 

-def _quote_php_variable(var_type, config, conf_name):
+def _quote_php_variable(var_type: Type[Any], config: Configuration,
+                        conf_name: str) -> str:

     if var_type == bool:
         return 'true' if config.get_bool(conf_name) else 'false'
 
     if var_type == int:
     if var_type == bool:
         return 'true' if config.get_bool(conf_name) else 'false'
 
     if var_type == int:

-        return getattr(config, conf_name)
+        return cast(str, getattr(config, conf_name))

 
     if not getattr(config, conf_name):
         return 'false'
 
     if var_type == Path:
 
     if not getattr(config, conf_name):
         return 'false'
 
     if var_type == Path:

-        value = str(config.get_path(conf_name))
+        value = str(config.get_path(conf_name) or '')

     else:
         value = getattr(config, conf_name)
 
     else:
         value = getattr(config, conf_name)
 


@@ -178,35 +210,63 @@ def _quote_php_variable(var_type, config, conf_name):
     return f"'{quoted}'"
 
 
     return f"'{quoted}'"
 
 

-def setup_website(basedir, config, conn):
+def setup_website(basedir: Path, config: Configuration, conn: Connection) -> None:

     """ Create the website script stubs.
     """
     if not basedir.exists():
         LOG.info('Creating website directory.')
         basedir.mkdir()
 
     """ Create the website script stubs.
     """
     if not basedir.exists():
         LOG.info('Creating website directory.')
         basedir.mkdir()
 

-    template = dedent("""\
+    assert config.project_dir is not None
+    basedata = dedent(f"""\

                       <?php
 
                       @define('CONST_Debug', $_GET['debug'] ?? false);
                       <?php
 
                       @define('CONST_Debug', $_GET['debug'] ?? false);

-                      @define('CONST_LibDir', '{0}');
-                      @define('CONST_TokenizerDir', '{2}');
-                      @define('CONST_NominatimVersion', '{1[0]}.{1[1]}.{1[2]}-{1[3]}');
+                      @define('CONST_LibDir', '{config.lib_dir.php}');
+                      @define('CONST_TokenizerDir', '{config.project_dir / 'tokenizer'}');
+                      @define('CONST_NominatimVersion', '{NOMINATIM_VERSION!s}');

 
 

-                      """.format(config.lib_dir.php, NOMINATIM_VERSION,
-                                 config.project_dir / 'tokenizer'))
+                      """)

 
     for php_name, conf_name, var_type in PHP_CONST_DEFS:
         varout = _quote_php_variable(var_type, config, conf_name)
 
 
     for php_name, conf_name, var_type in PHP_CONST_DEFS:
         varout = _quote_php_variable(var_type, config, conf_name)
 

-        template += f"@define('CONST_{php_name}', {varout});\n"
+        basedata += f"@define('CONST_{php_name}', {varout});\n"

 
 

-    template += f"\nrequire_once('{config.lib_dir.php}/website/{{}}');\n"
+    template = "\nrequire_once(CONST_LibDir.'/website/{}');\n"

 
     search_name_table_exists = bool(conn and conn.table_exists('search_name'))
 
     for script in WEBSITE_SCRIPTS:
         if not search_name_table_exists and script == 'search.php':
 
     search_name_table_exists = bool(conn and conn.table_exists('search_name'))
 
     for script in WEBSITE_SCRIPTS:
         if not search_name_table_exists and script == 'search.php':

-            (basedir / script).write_text(template.format('reverse-only-search.php'), 'utf-8')
+            out = template.format('reverse-only-search.php')

         else:
         else:

-            (basedir / script).write_text(template.format(script), 'utf-8')
+            out = template.format(script)
+
+        (basedir / script).write_text(basedata + out, 'utf-8')
+
+
+def invalidate_osm_object(osm_type: str, osm_id: int, conn: Connection,
+                          recursive: bool = True) -> None:
+    """ Mark the given OSM object for reindexing. When 'recursive' is set
+        to True (the default), then all dependent objects are marked for
+        reindexing as well.
+
+        'osm_type' must be on of 'N' (node), 'W' (way) or 'R' (relation).
+        If the given object does not exist, then nothing happens.
+    """
+    assert osm_type in ('N', 'R', 'W')
+
+    LOG.warning("Invalidating OSM %s %s%s.",
+                OSM_TYPE[osm_type], osm_id,
+                ' and its dependent places' if recursive else '')
+
+    with conn.cursor() as cur:
+        if recursive:
+            sql = """SELECT place_force_update(place_id)
+                     FROM placex WHERE osm_type = %s and osm_id = %s"""
+        else:
+            sql = """UPDATE placex SET indexed_status = 2
+                     WHERE osm_type = %s and osm_id = %s"""
+
+        cur.execute(sql, (osm_type, osm_id))