From bb2bd76f91ed6e4b1530a7fb66553c7fd92afd9d Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Wed, 11 May 2022 08:59:28 +0200 Subject: [PATCH] pylint: avoid explicit use of format() function Use psycopg2 SQL formatters for SQL and formatted string literals everywhere else. --- nominatim/clicmd/replication.py | 5 ++- nominatim/clicmd/setup.py | 10 +++--- nominatim/config.py | 2 +- nominatim/db/connection.py | 2 +- nominatim/db/sql_preprocessor.py | 6 ++-- nominatim/db/status.py | 2 +- nominatim/indexer/runners.py | 4 +-- nominatim/tokenizer/legacy_tokenizer.py | 14 ++++----- .../token_analysis/config_variants.py | 4 +-- nominatim/tools/database_import.py | 2 +- nominatim/tools/exec_utils.py | 6 ++-- nominatim/tools/migration.py | 18 ++++++----- nominatim/tools/postcodes.py | 4 +-- nominatim/tools/refresh.py | 9 ++++-- .../tools/special_phrases/sp_csv_loader.py | 2 +- .../tools/special_phrases/sp_importer.py | 31 ++++++++++--------- nominatim/version.py | 4 +-- 17 files changed, 66 insertions(+), 59 deletions(-) diff --git a/nominatim/clicmd/replication.py b/nominatim/clicmd/replication.py index 849a0e49..9d946304 100644 --- a/nominatim/clicmd/replication.py +++ b/nominatim/clicmd/replication.py @@ -21,7 +21,7 @@ LOG = logging.getLogger() # Do not repeat documentation of subcommand classes. # pylint: disable=C0111 # Using non-top-level imports to make pyosmium optional for replication only. -# pylint: disable=E0012,C0415 +# pylint: disable=C0415 class UpdateReplication: """\ @@ -96,8 +96,7 @@ class UpdateReplication: end = dt.datetime.now(dt.timezone.utc) LOG.warning("Update completed. Import: %s. %sTotal: %s. Remaining backlog: %s.", round_time((start_index or end) - start_import), - "Indexing: {} ".format(round_time(end - start_index)) - if start_index else '', + f"Indexing: {round_time(end - start_index)} " if start_index else '', round_time(end - start_import), round_time(end - batchdate)) diff --git a/nominatim/clicmd/setup.py b/nominatim/clicmd/setup.py index 7968fbce..b643c5ba 100644 --- a/nominatim/clicmd/setup.py +++ b/nominatim/clicmd/setup.py @@ -19,7 +19,7 @@ from nominatim.version import version_str # Do not repeat documentation of subcommand classes. # pylint: disable=C0111 # Using non-top-level imports to avoid eventually unused imports. -# pylint: disable=E0012,C0415 +# pylint: disable=C0415 LOG = logging.getLogger() @@ -194,10 +194,10 @@ class SetupAll: LOG.warning('Creating support index') if tablespace: tablespace = 'TABLESPACE ' + tablespace - cur.execute("""CREATE INDEX idx_placex_pendingsector - ON placex USING BTREE (rank_address,geometry_sector) - {} WHERE indexed_status > 0 - """.format(tablespace)) + cur.execute(f"""CREATE INDEX idx_placex_pendingsector + ON placex USING BTREE (rank_address,geometry_sector) + {tablespace} WHERE indexed_status > 0 + """) conn.commit() diff --git a/nominatim/config.py b/nominatim/config.py index a3f91055..ef261079 100644 --- a/nominatim/config.py +++ b/nominatim/config.py @@ -144,7 +144,7 @@ class Configuration: style = self.__getattr__('IMPORT_STYLE') if style in ('admin', 'street', 'address', 'full', 'extratags'): - return self.config_dir / 'import-{}.style'.format(style) + return self.config_dir / f'import-{style}.style' return self.find_config_file('', 'IMPORT_STYLE') diff --git a/nominatim/db/connection.py b/nominatim/db/connection.py index 45bc173d..90194e63 100644 --- a/nominatim/db/connection.py +++ b/nominatim/db/connection.py @@ -163,7 +163,7 @@ def connect(dsn): ctxmgr.connection = conn return ctxmgr except psycopg2.OperationalError as err: - raise UsageError("Cannot connect to database: {}".format(err)) from err + raise UsageError(f"Cannot connect to database: {err}") from err # Translation from PG connection string parameters to PG environment variables. diff --git a/nominatim/db/sql_preprocessor.py b/nominatim/db/sql_preprocessor.py index 10d93666..4de53886 100644 --- a/nominatim/db/sql_preprocessor.py +++ b/nominatim/db/sql_preprocessor.py @@ -39,10 +39,10 @@ def _setup_tablespace_sql(config): out = {} for subset in ('ADDRESS', 'SEARCH', 'AUX'): for kind in ('DATA', 'INDEX'): - tspace = getattr(config, 'TABLESPACE_{}_{}'.format(subset, kind)) + tspace = getattr(config, f'TABLESPACE_{subset}_{kind}') if tspace: - tspace = 'TABLESPACE "{}"'.format(tspace) - out['{}_{}'.format(subset.lower(), kind.lower())] = tspace + tspace = f'TABLESPACE "{tspace}"' + out[f'{subset.lower()}_{kind.lower()}'] = tspace return out diff --git a/nominatim/db/status.py b/nominatim/db/status.py index 12b24a83..d31196b3 100644 --- a/nominatim/db/status.py +++ b/nominatim/db/status.py @@ -34,7 +34,7 @@ def compute_database_date(conn): LOG.info("Using node id %d for timestamp lookup", osmid) # Get the node from the API to find the timestamp when it was created. - node_url = 'https://www.openstreetmap.org/api/0.6/node/{}/1'.format(osmid) + node_url = f'https://www.openstreetmap.org/api/0.6/node/{osmid}/1' data = get_url(node_url) match = re.search(r'timestamp="((\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2}))Z"', data) diff --git a/nominatim/indexer/runners.py b/nominatim/indexer/runners.py index ac7a0015..9a30ffe6 100644 --- a/nominatim/indexer/runners.py +++ b/nominatim/indexer/runners.py @@ -66,7 +66,7 @@ class RankRunner(AbstractPlacexRunner): """ def name(self): - return "rank {}".format(self.rank) + return f"rank {self.rank}" def sql_count_objects(self): return pysql.SQL("""SELECT count(*) FROM placex @@ -86,7 +86,7 @@ class BoundaryRunner(AbstractPlacexRunner): """ def name(self): - return "boundaries rank {}".format(self.rank) + return f"boundaries rank {self.rank}" def sql_count_objects(self): return pysql.SQL("""SELECT count(*) FROM placex diff --git a/nominatim/tokenizer/legacy_tokenizer.py b/nominatim/tokenizer/legacy_tokenizer.py index 97ce6d16..299e2fde 100644 --- a/nominatim/tokenizer/legacy_tokenizer.py +++ b/nominatim/tokenizer/legacy_tokenizer.py @@ -74,10 +74,10 @@ def _check_module(module_dir, conn): with conn.cursor() as cur: try: cur.execute("""CREATE FUNCTION nominatim_test_import_func(text) - RETURNS text AS '{}/nominatim.so', 'transliteration' + RETURNS text AS %s, 'transliteration' LANGUAGE c IMMUTABLE STRICT; DROP FUNCTION nominatim_test_import_func(text) - """.format(module_dir)) + """, (f'{module_dir}/nominatim.so', )) except psycopg2.DatabaseError as err: LOG.fatal("Error accessing database module: %s", err) raise UsageError("Database module cannot be accessed.") from err @@ -250,12 +250,12 @@ class LegacyTokenizer(AbstractTokenizer): php_file = self.data_dir / "tokenizer.php" if not php_file.exists() or overwrite: - php_file.write_text(dedent("""\ + php_file.write_text(dedent(f"""\ ', rule) if len(parts) != 4: - raise UsageError("Syntax error in variant rule: " + rule) + raise UsageError(f"Syntax error in variant rule: {rule}") decompose = parts[1] is None src_terms = [self._parse_variant_word(t) for t in parts[0].split(',')] @@ -89,7 +89,7 @@ class _VariantMaker: name = name.strip() match = re.fullmatch(r'([~^]?)([^~$^]*)([~$]?)', name) if match is None or (match.group(1) == '~' and match.group(3) == '~'): - raise UsageError("Invalid variant word descriptor '{}'".format(name)) + raise UsageError(f"Invalid variant word descriptor '{name}'") norm_name = self.norm.transliterate(match.group(2)).strip() if not norm_name: return None diff --git a/nominatim/tools/database_import.py b/nominatim/tools/database_import.py index caec9035..50938c19 100644 --- a/nominatim/tools/database_import.py +++ b/nominatim/tools/database_import.py @@ -234,7 +234,7 @@ def create_search_indices(conn, config, drop=False): bad_indices = [row[0] for row in list(cur)] for idx in bad_indices: LOG.info("Drop invalid index %s.", idx) - cur.execute('DROP INDEX "{}"'.format(idx)) + cur.execute(pysql.SQL('DROP INDEX {}').format(pysql.Identifier(idx))) conn.commit() sql = SQLPreprocessor(conn, config) diff --git a/nominatim/tools/exec_utils.py b/nominatim/tools/exec_utils.py index db2a4c2e..a81a8d6b 100644 --- a/nominatim/tools/exec_utils.py +++ b/nominatim/tools/exec_utils.py @@ -55,10 +55,10 @@ def run_api_script(endpoint, project_dir, extra_env=None, phpcgi_bin=None, query_string = urlencode(params or {}) env = dict(QUERY_STRING=query_string, - SCRIPT_NAME='/{}.php'.format(endpoint), - REQUEST_URI='/{}.php?{}'.format(endpoint, query_string), + SCRIPT_NAME=f'/{endpoint}.php', + REQUEST_URI=f'/{endpoint}.php?{query_string}', CONTEXT_DOCUMENT_ROOT=webdir, - SCRIPT_FILENAME='{}/{}.php'.format(webdir, endpoint), + SCRIPT_FILENAME=f'{webdir}/{endpoint}.php', HTTP_HOST='localhost', HTTP_USER_AGENT='nominatim-tool', REMOTE_ADDR='0.0.0.0', diff --git a/nominatim/tools/migration.py b/nominatim/tools/migration.py index 290016ad..8eaad2f7 100644 --- a/nominatim/tools/migration.py +++ b/nominatim/tools/migration.py @@ -9,6 +9,8 @@ Functions for database migration to newer software versions. """ import logging +from psycopg2 import sql as pysql + from nominatim.db import properties from nominatim.db.connection import connect from nominatim.version import NOMINATIM_VERSION, version_str @@ -47,7 +49,7 @@ def migrate(config, paths): for version, func in _MIGRATION_FUNCTIONS: if db_version <= version: LOG.warning("Runnning: %s (%s)", func.__doc__.split('\n', 1)[0], - '{}.{}.{}-{}'.format(*version)) + version_str(version)) kwargs = dict(conn=conn, config=config, paths=paths) func(**kwargs) conn.commit() @@ -124,11 +126,12 @@ def add_nominatim_property_table(conn, config, **_): """ if not conn.table_exists('nominatim_properties'): with conn.cursor() as cur: - cur.execute("""CREATE TABLE nominatim_properties ( - property TEXT, - value TEXT); - GRANT SELECT ON TABLE nominatim_properties TO "{}"; - """.format(config.DATABASE_WEBUSER)) + cur.execute(pysql.SQL("""CREATE TABLE nominatim_properties ( + property TEXT, + value TEXT); + GRANT SELECT ON TABLE nominatim_properties TO {}; + """) + .format(pysql.Identifier(config.DATABASE_WEBUSER))) @_migration(3, 6, 0, 0) def change_housenumber_transliteration(conn, **_): @@ -193,7 +196,8 @@ def install_legacy_tokenizer(conn, config, **_): and column_name = 'token_info'""", (table, )) if has_column == 0: - cur.execute('ALTER TABLE {} ADD COLUMN token_info JSONB'.format(table)) + cur.execute(pysql.SQL('ALTER TABLE {} ADD COLUMN token_info JSONB') + .format(pysql.Identifier(table))) tokenizer = tokenizer_factory.create_tokenizer(config, init_db=False, module_name='legacy') diff --git a/nominatim/tools/postcodes.py b/nominatim/tools/postcodes.py index adc58ec5..cd778e5f 100644 --- a/nominatim/tools/postcodes.py +++ b/nominatim/tools/postcodes.py @@ -136,13 +136,13 @@ class _CountryPostcodesCollector: def _open_external(self, project_dir): - fname = project_dir / '{}_postcodes.csv'.format(self.country) + fname = project_dir / f'{self.country}_postcodes.csv' if fname.is_file(): LOG.info("Using external postcode file '%s'.", fname) return open(fname, 'r') - fname = project_dir / '{}_postcodes.csv.gz'.format(self.country) + fname = project_dir / f'{self.country}_postcodes.csv.gz' if fname.is_file(): LOG.info("Using external postcode file '%s'.", fname) diff --git a/nominatim/tools/refresh.py b/nominatim/tools/refresh.py index b9a926f2..ca6f5c62 100644 --- a/nominatim/tools/refresh.py +++ b/nominatim/tools/refresh.py @@ -52,16 +52,19 @@ def load_address_levels(conn, table, levels): with conn.cursor() as cur: cur.drop_table(table) - cur.execute("""CREATE TABLE {} (country_code varchar(2), + cur.execute(pysql.SQL("""CREATE TABLE {} ( + country_code varchar(2), class TEXT, type TEXT, rank_search SMALLINT, - rank_address SMALLINT)""".format(table)) + rank_address SMALLINT)""") + .format(pysql.Identifier(table))) cur.execute_values(pysql.SQL("INSERT INTO {} VALUES %s") .format(pysql.Identifier(table)), rows) - cur.execute('CREATE UNIQUE INDEX ON {} (country_code, class, type)'.format(table)) + cur.execute(pysql.SQL('CREATE UNIQUE INDEX ON {} (country_code, class, type)') + .format(pysql.Identifier(table))) conn.commit() diff --git a/nominatim/tools/special_phrases/sp_csv_loader.py b/nominatim/tools/special_phrases/sp_csv_loader.py index a32a4b39..2a67687f 100644 --- a/nominatim/tools/special_phrases/sp_csv_loader.py +++ b/nominatim/tools/special_phrases/sp_csv_loader.py @@ -54,4 +54,4 @@ class SPCsvLoader(Iterator): _, extension = os.path.splitext(self.csv_path) if extension != '.csv': - raise UsageError('The file {} is not a csv file.'.format(self.csv_path)) + raise UsageError(f'The file {self.csv_path} is not a csv file.') diff --git a/nominatim/tools/special_phrases/sp_importer.py b/nominatim/tools/special_phrases/sp_importer.py index 195f8387..9eefaa19 100644 --- a/nominatim/tools/special_phrases/sp_importer.py +++ b/nominatim/tools/special_phrases/sp_importer.py @@ -16,7 +16,7 @@ import logging import re -from psycopg2.sql import Identifier, Literal, SQL +from psycopg2.sql import Identifier, SQL from nominatim.tools.special_phrases.importer_statistics import SpecialPhrasesImporterStatistics LOG = logging.getLogger() @@ -195,35 +195,36 @@ class SPImporter(): """ table_name = _classtype_table(phrase_class, phrase_type) with self.db_connection.cursor() as db_cursor: - db_cursor.execute(SQL(""" - CREATE TABLE IF NOT EXISTS {{}} {} - AS SELECT place_id AS place_id,st_centroid(geometry) AS centroid FROM placex - WHERE class = {{}} AND type = {{}}""".format(sql_tablespace)) - .format(Identifier(table_name), Literal(phrase_class), - Literal(phrase_type))) + db_cursor.execute(SQL("""CREATE TABLE IF NOT EXISTS {} {} AS + SELECT place_id AS place_id, + st_centroid(geometry) AS centroid + FROM placex + WHERE class = %s AND type = %s""") + .format(Identifier(table_name), SQL(sql_tablespace)), + (phrase_class, phrase_type)) def _create_place_classtype_indexes(self, sql_tablespace, phrase_class, phrase_type): """ Create indexes on centroid and place_id for the place_classtype table. """ - index_prefix = 'idx_place_classtype_{}_{}_'.format(phrase_class, phrase_type) + index_prefix = f'idx_place_classtype_{phrase_class}_{phrase_type}_' base_table = _classtype_table(phrase_class, phrase_type) # Index on centroid if not self.db_connection.index_exists(index_prefix + 'centroid'): with self.db_connection.cursor() as db_cursor: - db_cursor.execute(SQL(""" - CREATE INDEX {{}} ON {{}} USING GIST (centroid) {}""".format(sql_tablespace)) - .format(Identifier(index_prefix + 'centroid'), - Identifier(base_table)), sql_tablespace) + db_cursor.execute(SQL("CREATE INDEX {} ON {} USING GIST (centroid) {}") + .format(Identifier(index_prefix + 'centroid'), + Identifier(base_table), + SQL(sql_tablespace))) # Index on place_id if not self.db_connection.index_exists(index_prefix + 'place_id'): with self.db_connection.cursor() as db_cursor: - db_cursor.execute(SQL( - """CREATE INDEX {{}} ON {{}} USING btree(place_id) {}""".format(sql_tablespace)) + db_cursor.execute(SQL("CREATE INDEX {} ON {} USING btree(place_id) {}") .format(Identifier(index_prefix + 'place_id'), - Identifier(base_table))) + Identifier(base_table), + SQL(sql_tablespace))) def _grant_access_to_webuser(self, phrase_class, phrase_type): diff --git a/nominatim/version.py b/nominatim/version.py index d4f69da1..88d42af9 100644 --- a/nominatim/version.py +++ b/nominatim/version.py @@ -37,8 +37,8 @@ GIT_COMMIT_HASH = None # pylint: disable=consider-using-f-string -def version_str(): +def version_str(version=NOMINATIM_VERSION): """ Return a human-readable string of the version. """ - return '{}.{}.{}-{}'.format(*NOMINATIM_VERSION) + return '{}.{}.{}-{}'.format(*version) -- 2.39.5