1 # SPDX-License-Identifier: GPL-2.0-only
3 # This file is part of Nominatim. (https://nominatim.org)
5 # Copyright (C) 2022 by the Nominatim developer community.
6 # For a full list of authors see the git log.
8 Functions for bringing auxiliary data in the database up-to-date.
11 from textwrap import dedent
12 from pathlib import Path
14 from psycopg2 import sql as pysql
16 from nominatim.db.utils import execute_file
17 from nominatim.db.sql_preprocessor import SQLPreprocessor
18 from nominatim.version import version_str
20 LOG = logging.getLogger()
22 OSM_TYPE = {'N': 'node', 'W': 'way', 'R': 'relation'}
24 def _add_address_level_rows_from_entry(rows, entry):
25 """ Converts a single entry from the JSON format for address rank
26 descriptions into a flat format suitable for inserting into a
27 PostgreSQL table and adds these lines to `rows`.
29 countries = entry.get('countries') or (None, )
30 for key, values in entry['tags'].items():
31 for value, ranks in values.items():
32 if isinstance(ranks, list):
33 rank_search, rank_address = ranks
35 rank_search = rank_address = ranks
38 for country in countries:
39 rows.append((country, key, value, rank_search, rank_address))
41 def load_address_levels(conn, table, levels):
42 """ Replace the `address_levels` table with the contents of `levels'.
44 A new table is created any previously existing table is dropped.
45 The table has the following columns:
46 country, class, type, rank_search, rank_address
50 _add_address_level_rows_from_entry(rows, entry)
52 with conn.cursor() as cur:
55 cur.execute(pysql.SQL("""CREATE TABLE {} (
56 country_code varchar(2),
60 rank_address SMALLINT)""")
61 .format(pysql.Identifier(table)))
63 cur.execute_values(pysql.SQL("INSERT INTO {} VALUES %s")
64 .format(pysql.Identifier(table)), rows)
66 cur.execute(pysql.SQL('CREATE UNIQUE INDEX ON {} (country_code, class, type)')
67 .format(pysql.Identifier(table)))
72 def load_address_levels_from_config(conn, config):
73 """ Replace the `address_levels` table with the content as
74 defined in the given configuration. Uses the parameter
75 NOMINATIM_ADDRESS_LEVEL_CONFIG to determine the location of the
78 cfg = config.load_sub_configuration('', config='ADDRESS_LEVEL_CONFIG')
79 load_address_levels(conn, 'address_levels', cfg)
82 def create_functions(conn, config, enable_diff_updates=True, enable_debug=False):
83 """ (Re)create the PL/pgSQL functions.
85 sql = SQLPreprocessor(conn, config)
87 sql.run_sql_file(conn, 'functions.sql',
88 disable_diff_updates=not enable_diff_updates,
103 # constants needed by PHP scripts: PHP name, config name, type
105 ('Database_DSN', 'DATABASE_DSN', str),
106 ('Default_Language', 'DEFAULT_LANGUAGE', str),
107 ('Log_DB', 'LOG_DB', bool),
108 ('Log_File', 'LOG_FILE', Path),
109 ('NoAccessControl', 'CORS_NOACCESSCONTROL', bool),
110 ('Places_Max_ID_count', 'LOOKUP_MAX_COUNT', int),
111 ('PolygonOutput_MaximumTypes', 'POLYGON_OUTPUT_MAX_TYPES', int),
112 ('Search_BatchMode', 'SEARCH_BATCH_MODE', bool),
113 ('Search_NameOnlySearchFrequencyThreshold', 'SEARCH_NAME_ONLY_THRESHOLD', str),
114 ('Use_US_Tiger_Data', 'USE_US_TIGER_DATA', bool),
115 ('MapIcon_URL', 'MAPICON_URL', str),
119 def import_wikipedia_articles(dsn, data_path, ignore_errors=False):
120 """ Replaces the wikipedia importance tables with new data.
121 The import is run in a single transaction so that the new data
122 is replace seemlessly.
124 Returns 0 if all was well and 1 if the importance file could not
125 be found. Throws an exception if there was an error reading the file.
127 datafile = data_path / 'wikimedia-importance.sql.gz'
129 if not datafile.exists():
133 DROP TABLE IF EXISTS "wikipedia_article";
134 DROP TABLE IF EXISTS "wikipedia_redirect"
137 execute_file(dsn, datafile, ignore_errors=ignore_errors,
138 pre_code=pre_code, post_code=post_code)
143 def recompute_importance(conn):
144 """ Recompute wikipedia links and importance for all entries in placex.
145 This is a long-running operations that must not be executed in
146 parallel with updates.
148 with conn.cursor() as cur:
149 cur.execute('ALTER TABLE placex DISABLE TRIGGER ALL')
151 UPDATE placex SET (wikipedia, importance) =
152 (SELECT wikipedia, importance
153 FROM compute_importance(extratags, country_code, osm_type, osm_id))
156 UPDATE placex s SET wikipedia = d.wikipedia, importance = d.importance
158 WHERE s.place_id = d.linked_place_id and d.wikipedia is not null
159 and (s.wikipedia is null or s.importance < d.importance);
162 cur.execute('ALTER TABLE placex ENABLE TRIGGER ALL')
166 def _quote_php_variable(var_type, config, conf_name):
168 return 'true' if config.get_bool(conf_name) else 'false'
171 return getattr(config, conf_name)
173 if not getattr(config, conf_name):
177 value = str(config.get_path(conf_name))
179 value = getattr(config, conf_name)
181 quoted = value.replace("'", "\\'")
185 def setup_website(basedir, config, conn):
186 """ Create the website script stubs.
188 if not basedir.exists():
189 LOG.info('Creating website directory.')
192 template = dedent(f"""\
195 @define('CONST_Debug', $_GET['debug'] ?? false);
196 @define('CONST_LibDir', '{config.lib_dir.php}');
197 @define('CONST_TokenizerDir', '{config.project_dir / 'tokenizer'}');
198 @define('CONST_NominatimVersion', '{version_str()}');
202 for php_name, conf_name, var_type in PHP_CONST_DEFS:
203 varout = _quote_php_variable(var_type, config, conf_name)
205 template += f"@define('CONST_{php_name}', {varout});\n"
207 template += f"\nrequire_once('{config.lib_dir.php}/website/{{}}');\n"
209 search_name_table_exists = bool(conn and conn.table_exists('search_name'))
211 for script in WEBSITE_SCRIPTS:
212 if not search_name_table_exists and script == 'search.php':
213 (basedir / script).write_text(template.format('reverse-only-search.php'), 'utf-8')
215 (basedir / script).write_text(template.format(script), 'utf-8')
218 def invalidate_osm_object(osm_type, osm_id, conn, recursive=True):
219 """ Mark the given OSM object for reindexing. When 'recursive' is set
220 to True (the default), then all dependent objects are marked for
223 'osm_type' must be on of 'N' (node), 'W' (way) or 'R' (relation).
224 If the given object does not exist, then nothing happens.
226 assert osm_type in ('N', 'R', 'W')
228 LOG.warning("Invalidating OSM %s %s%s.",
229 OSM_TYPE[osm_type], osm_id,
230 ' and its dependent places' if recursive else '')
232 with conn.cursor() as cur:
234 sql = """SELECT place_force_update(place_id)
235 FROM placex WHERE osm_type = %s and osm_id = %s"""
237 sql = """UPDATE placex SET indexed_status = 2
238 WHERE osm_type = %s and osm_id = %s"""
240 cur.execute(sql, (osm_type, osm_id))