"""
Functions for bringing auxiliary data in the database up-to-date.
"""
+from typing import MutableSequence, Tuple, Any, Type, Mapping, Sequence, List, cast
import logging
from textwrap import dedent
from pathlib import Path
from psycopg2 import sql as pysql
+from nominatim.config import Configuration
+from nominatim.db.connection import Connection, connect
from nominatim.db.utils import execute_file
from nominatim.db.sql_preprocessor import SQLPreprocessor
-from nominatim.version import version_str
+from nominatim.version import NOMINATIM_VERSION
LOG = logging.getLogger()
OSM_TYPE = {'N': 'node', 'W': 'way', 'R': 'relation'}
-def _add_address_level_rows_from_entry(rows, entry):
+def _add_address_level_rows_from_entry(rows: MutableSequence[Tuple[Any, ...]],
+ entry: Mapping[str, Any]) -> None:
""" Converts a single entry from the JSON format for address rank
descriptions into a flat format suitable for inserting into a
PostgreSQL table and adds these lines to `rows`.
for country in countries:
rows.append((country, key, value, rank_search, rank_address))
-def load_address_levels(conn, table, levels):
+
+def load_address_levels(conn: Connection, table: str, levels: Sequence[Mapping[str, Any]]) -> None:
""" Replace the `address_levels` table with the contents of `levels'.
A new table is created any previously existing table is dropped.
The table has the following columns:
country, class, type, rank_search, rank_address
"""
- rows = []
+ rows: List[Tuple[Any, ...]] = []
for entry in levels:
_add_address_level_rows_from_entry(rows, entry)
with conn.cursor() as cur:
cur.drop_table(table)
- cur.execute("""CREATE TABLE {} (country_code varchar(2),
+ cur.execute(pysql.SQL("""CREATE TABLE {} (
+ country_code varchar(2),
class TEXT,
type TEXT,
rank_search SMALLINT,
- rank_address SMALLINT)""".format(table))
+ rank_address SMALLINT)
+ """).format(pysql.Identifier(table)))
cur.execute_values(pysql.SQL("INSERT INTO {} VALUES %s")
.format(pysql.Identifier(table)), rows)
- cur.execute('CREATE UNIQUE INDEX ON {} (country_code, class, type)'.format(table))
+ cur.execute(pysql.SQL('CREATE UNIQUE INDEX ON {} (country_code, class, type)')
+ .format(pysql.Identifier(table)))
conn.commit()
-def load_address_levels_from_config(conn, config):
+def load_address_levels_from_config(conn: Connection, config: Configuration) -> None:
""" Replace the `address_levels` table with the content as
defined in the given configuration. Uses the parameter
NOMINATIM_ADDRESS_LEVEL_CONFIG to determine the location of the
load_address_levels(conn, 'address_levels', cfg)
-def create_functions(conn, config, enable_diff_updates=True, enable_debug=False):
+def create_functions(conn: Connection, config: Configuration,
+ enable_diff_updates: bool = True,
+ enable_debug: bool = False) -> None:
""" (Re)create the PL/pgSQL functions.
"""
sql = SQLPreprocessor(conn, config)
)
-def import_wikipedia_articles(dsn, data_path, ignore_errors=False):
+def import_wikipedia_articles(dsn: str, data_path: Path, ignore_errors: bool = False) -> int:
""" Replaces the wikipedia importance tables with new data.
The import is run in a single transaction so that the new data
- is replace seemlessly.
+ is replace seamlessly.
Returns 0 if all was well and 1 if the importance file could not
be found. Throws an exception if there was an error reading the file.
return 0
+def import_secondary_importance(dsn: str, data_path: Path, ignore_errors: bool = False) -> int:
+ """ Replaces the secondary importance raster data table with new data.
+
+ Returns 0 if all was well and 1 if the raster SQL file could not
+ be found. Throws an exception if there was an error reading the file.
+ """
+ datafile = data_path / 'secondary_importance.sql.gz'
+ if not datafile.exists():
+ return 1
-def recompute_importance(conn):
+ with connect(dsn) as conn:
+ postgis_version = conn.postgis_version_tuple()
+ if postgis_version[0] < 3:
+ LOG.error('PostGIS version is too old for using OSM raster data.')
+ return 2
+
+ execute_file(dsn, datafile, ignore_errors=ignore_errors)
+
+ return 0
+
+def recompute_importance(conn: Connection) -> None:
""" Recompute wikipedia links and importance for all entries in placex.
This is a long-running operations that must not be executed in
parallel with updates.
cur.execute("""
UPDATE placex SET (wikipedia, importance) =
(SELECT wikipedia, importance
- FROM compute_importance(extratags, country_code, osm_type, osm_id))
+ FROM compute_importance(extratags, country_code, rank_search, centroid))
""")
cur.execute("""
UPDATE placex s SET wikipedia = d.wikipedia, importance = d.importance
conn.commit()
-def _quote_php_variable(var_type, config, conf_name):
+def _quote_php_variable(var_type: Type[Any], config: Configuration,
+ conf_name: str) -> str:
if var_type == bool:
return 'true' if config.get_bool(conf_name) else 'false'
if var_type == int:
- return getattr(config, conf_name)
+ return cast(str, getattr(config, conf_name))
if not getattr(config, conf_name):
return 'false'
if var_type == Path:
- value = str(config.get_path(conf_name))
+ value = str(config.get_path(conf_name) or '')
else:
value = getattr(config, conf_name)
return f"'{quoted}'"
-def setup_website(basedir, config, conn):
+def setup_website(basedir: Path, config: Configuration, conn: Connection) -> None:
""" Create the website script stubs.
"""
if not basedir.exists():
LOG.info('Creating website directory.')
basedir.mkdir()
- template = dedent(f"""\
+ assert config.project_dir is not None
+ basedata = dedent(f"""\
<?php
@define('CONST_Debug', $_GET['debug'] ?? false);
@define('CONST_LibDir', '{config.lib_dir.php}');
@define('CONST_TokenizerDir', '{config.project_dir / 'tokenizer'}');
- @define('CONST_NominatimVersion', '{version_str()}');
+ @define('CONST_NominatimVersion', '{NOMINATIM_VERSION!s}');
""")
for php_name, conf_name, var_type in PHP_CONST_DEFS:
varout = _quote_php_variable(var_type, config, conf_name)
- template += f"@define('CONST_{php_name}', {varout});\n"
+ basedata += f"@define('CONST_{php_name}', {varout});\n"
- template += f"\nrequire_once('{config.lib_dir.php}/website/{{}}');\n"
+ template = "\nrequire_once(CONST_LibDir.'/website/{}');\n"
search_name_table_exists = bool(conn and conn.table_exists('search_name'))
for script in WEBSITE_SCRIPTS:
if not search_name_table_exists and script == 'search.php':
- (basedir / script).write_text(template.format('reverse-only-search.php'), 'utf-8')
+ out = template.format('reverse-only-search.php')
else:
- (basedir / script).write_text(template.format(script), 'utf-8')
+ out = template.format(script)
+
+ (basedir / script).write_text(basedata + out, 'utf-8')
-def invalidate_osm_object(osm_type, osm_id, conn, recursive=True):
+def invalidate_osm_object(osm_type: str, osm_id: int, conn: Connection,
+ recursive: bool = True) -> None:
""" Mark the given OSM object for reindexing. When 'recursive' is set
to True (the default), then all dependent objects are marked for
reindexing as well.
projects / nominatim.git / blobdiff