X-Git-Url: https://git.openstreetmap.org./nominatim.git/blobdiff_plain/c3788d765ed4e7ddf00794085af757714bc102cf..4160c0a1942657f58f44d73e7637b0aaced93e9f:/nominatim/tools/tiger_data.py diff --git a/nominatim/tools/tiger_data.py b/nominatim/tools/tiger_data.py index 8610880f..4a32bb1e 100644 --- a/nominatim/tools/tiger_data.py +++ b/nominatim/tools/tiger_data.py @@ -7,50 +7,82 @@ """ Functions for importing tiger data and handling tarbar and directory files """ +from typing import Any, TextIO, List, Union, cast import csv import io import logging import os import tarfile +from psycopg2.extras import Json + +from nominatim.config import Configuration from nominatim.db.connection import connect from nominatim.db.async_connection import WorkerPool from nominatim.db.sql_preprocessor import SQLPreprocessor from nominatim.errors import UsageError -from nominatim.indexer.place_info import PlaceInfo +from nominatim.data.place_info import PlaceInfo +from nominatim.tokenizer.base import AbstractAnalyzer, AbstractTokenizer LOG = logging.getLogger() - -def handle_tarfile_or_directory(data_dir): - """ Handles tarfile or directory for importing tiger data +class TigerInput: + """ Context manager that goes through Tiger input files which may + either be in a directory or gzipped together in a tar file. """ - tar = None - if data_dir.endswith('.tar.gz'): - try: - tar = tarfile.open(data_dir) - except tarfile.ReadError as err: - LOG.fatal("Cannot open '%s'. Is this a tar file?", data_dir) - raise UsageError("Cannot open Tiger data file.") from err - - csv_files = [i for i in tar.getmembers() if i.name.endswith('.csv')] - LOG.warning("Found %d CSV files in tarfile with path %s", len(csv_files), data_dir) - if not csv_files: - LOG.warning("Tiger data import selected but no files in tarfile's path %s", data_dir) - return None, None - else: - files = os.listdir(data_dir) - csv_files = [os.path.join(data_dir, i) for i in files if i.endswith('.csv')] - LOG.warning("Found %d CSV files in path %s", len(csv_files), data_dir) - if not csv_files: - LOG.warning("Tiger data import selected but no files found in path %s", data_dir) - return None, None - - return csv_files, tar - - -def handle_threaded_sql_statements(pool, fd, analyzer): + def __init__(self, data_dir: str) -> None: + self.tar_handle = None + self.files: List[Union[str, tarfile.TarInfo]] = [] + + if data_dir.endswith('.tar.gz'): + try: + self.tar_handle = tarfile.open(data_dir) # pylint: disable=consider-using-with + except tarfile.ReadError as err: + LOG.fatal("Cannot open '%s'. Is this a tar file?", data_dir) + raise UsageError("Cannot open Tiger data file.") from err + + self.files = [i for i in self.tar_handle.getmembers() if i.name.endswith('.csv')] + LOG.warning("Found %d CSV files in tarfile with path %s", len(self.files), data_dir) + else: + files = os.listdir(data_dir) + self.files = [os.path.join(data_dir, i) for i in files if i.endswith('.csv')] + LOG.warning("Found %d CSV files in path %s", len(self.files), data_dir) + + if not self.files: + LOG.warning("Tiger data import selected but no files found at %s", data_dir) + + + def __enter__(self) -> 'TigerInput': + return self + + + def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: + if self.tar_handle: + self.tar_handle.close() + self.tar_handle = None + + + def next_file(self) -> TextIO: + """ Return a file handle to the next file to be processed. + Raises an IndexError if there is no file left. + """ + fname = self.files.pop(0) + + if self.tar_handle is not None: + extracted = self.tar_handle.extractfile(fname) + assert extracted is not None + return io.TextIOWrapper(extracted) + + return open(cast(str, fname), encoding='utf-8') + + + def __len__(self) -> int: + return len(self.files) + + +def handle_threaded_sql_statements(pool: WorkerPool, fd: TextIO, + analyzer: AbstractAnalyzer) -> None: """ Handles sql statement with multiplexing """ lines = 0 @@ -63,7 +95,7 @@ def handle_threaded_sql_statements(pool, fd, analyzer): address = dict(street=row['street'], postcode=row['postcode']) args = ('SRID=4326;' + row['geometry'], int(row['from']), int(row['to']), row['interpolation'], - PlaceInfo({'address': address}).analyze(analyzer), + Json(analyzer.process_place(PlaceInfo({'address': address}))), analyzer.normalize_postcode(row['postcode'])) except ValueError: continue @@ -75,39 +107,35 @@ def handle_threaded_sql_statements(pool, fd, analyzer): lines = 0 -def add_tiger_data(data_dir, config, threads, tokenizer): +def add_tiger_data(data_dir: str, config: Configuration, threads: int, + tokenizer: AbstractTokenizer) -> int: """ Import tiger data from directory or tar file `data dir`. """ dsn = config.get_libpq_dsn() - files, tar = handle_tarfile_or_directory(data_dir) - if not files: - return + with TigerInput(data_dir) as tar: + if not tar: + return 1 - with connect(dsn) as conn: - sql = SQLPreprocessor(conn, config) - sql.run_sql_file(conn, 'tiger_import_start.sql') + with connect(dsn) as conn: + sql = SQLPreprocessor(conn, config) + sql.run_sql_file(conn, 'tiger_import_start.sql') - # Reading files and then for each file line handling - # sql_query in chunks. - place_threads = max(1, threads - 1) + # Reading files and then for each file line handling + # sql_query in chunks. + place_threads = max(1, threads - 1) - with WorkerPool(dsn, place_threads, ignore_sql_errors=True) as pool: - with tokenizer.name_analyzer() as analyzer: - for fname in files: - if not tar: - fd = open(fname) - else: - fd = io.TextIOWrapper(tar.extractfile(fname)) + with WorkerPool(dsn, place_threads, ignore_sql_errors=True) as pool: + with tokenizer.name_analyzer() as analyzer: + while tar: + with tar.next_file() as fd: + handle_threaded_sql_statements(pool, fd, analyzer) - handle_threaded_sql_statements(pool, fd, analyzer) + print('\n') - fd.close() - - if tar: - tar.close() - print('\n') LOG.warning("Creating indexes on Tiger data") with connect(dsn) as conn: sql = SQLPreprocessor(conn, config) sql.run_sql_file(conn, 'tiger_import_finish.sql') + + return 0