X-Git-Url: https://git.openstreetmap.org./nominatim.git/blobdiff_plain/1ce223a83bb38585d48137633380db37aa92af2e..8b6489c60e4def399d4205d2a3a07f364fcab9ac:/nominatim/tools/database_import.py diff --git a/nominatim/tools/database_import.py b/nominatim/tools/database_import.py index 28a10ebe..a4d7220f 100644 --- a/nominatim/tools/database_import.py +++ b/nominatim/tools/database_import.py @@ -9,6 +9,7 @@ from pathlib import Path import psutil import psycopg2.extras +from psycopg2 import sql as pysql from nominatim.db.connection import connect, get_pg_env from nominatim.db import utils as db_utils @@ -130,9 +131,8 @@ def import_osm_data(osm_file, options, drop=False, ignore_errors=False): if drop: conn.drop_table('planet_osm_nodes') - if drop: - if options['flatnode_file']: - Path(options['flatnode_file']).unlink() + if drop and options['flatnode_file']: + Path(options['flatnode_file']).unlink() def create_tables(conn, config, reverse_only=False): @@ -185,7 +185,12 @@ def truncate_data_tables(conn): conn.commit() -_COPY_COLUMNS = 'osm_type, osm_id, class, type, name, admin_level, address, extratags, geometry' + +_COPY_COLUMNS = pysql.SQL(',').join(map(pysql.Identifier, + ('osm_type', 'osm_id', 'class', 'type', + 'name', 'admin_level', 'address', + 'extratags', 'geometry'))) + def load_data(dsn, threads): """ Copy data into the word and placex table. @@ -196,12 +201,15 @@ def load_data(dsn, threads): for imod in range(place_threads): conn = DBConnection(dsn) conn.connect() - conn.perform("""INSERT INTO placex ({0}) - SELECT {0} FROM place - WHERE osm_id % {1} = {2} - AND NOT (class='place' and (type='houses' or type='postcode')) - AND ST_IsValid(geometry) - """.format(_COPY_COLUMNS, place_threads, imod)) + conn.perform( + pysql.SQL("""INSERT INTO placex ({columns}) + SELECT {columns} FROM place + WHERE osm_id % {total} = {mod} + AND NOT (class='place' and (type='houses' or type='postcode')) + AND ST_IsValid(geometry) + """).format(columns=_COPY_COLUMNS, + total=pysql.Literal(place_threads), + mod=pysql.Literal(imod))) sel.register(conn, selectors.EVENT_READ, conn) # Address interpolations go into another table. @@ -251,6 +259,7 @@ def create_search_indices(conn, config, drop=False): sql.run_sql_file(conn, 'indices.sql', drop=drop) + def create_country_names(conn, tokenizer, languages=None): """ Add default country names to search index. `languages` is a comma- separated list of language codes as used in OSM. If `languages` is not @@ -262,8 +271,7 @@ def create_country_names(conn, tokenizer, languages=None): def _include_key(key): return key == 'name' or \ - (key.startswith('name:') \ - and (not languages or key[5:] in languages)) + (key.startswith('name:') and (not languages or key[5:] in languages)) with conn.cursor() as cur: psycopg2.extras.register_hstore(cur) @@ -272,15 +280,15 @@ def create_country_names(conn, tokenizer, languages=None): with tokenizer.name_analyzer() as analyzer: for code, name in cur: - names = [code] + names = {'countrycode': code} if code == 'gb': - names.append('UK') + names['short_name'] = 'UK' if code == 'us': - names.append('United States') + names['short_name'] = 'United States' # country names (only in languages as provided) if name: - names.extend((v for k, v in name.items() if _include_key(k))) + names.update(((k, v) for k, v in name.items() if _include_key(k))) analyzer.add_country_names(code, names)