- file = tar.extractfile(sql_file)
- lines = 0
- end_of_file = False
- total_used_threads = place_threads
- while True:
- if end_of_file:
- break
- for imod in range(place_threads):
- conn = DBConnection(dsn)
- conn.connect()
-
- sql_query = file.readline()
- lines += 1
-
- if not sql_query:
- end_of_file = True
- total_used_threads = imod
- break
-
- conn.perform(sql_query)
- sel.register(conn, selectors.EVENT_READ, conn)
-
- if lines == 1000:
- print('. ', end='', flush=True)
- lines = 0
-
- todo = min(place_threads, total_used_threads)
- while todo > 0:
- for key, _ in sel.select(1):
- try:
- conn = key.data
- sel.unregister(conn)
- conn.wait()
- conn.close()
- todo -= 1
- except:
- todo -= 1
-
- if is_tarfile:
- tar.close()
- print('\n')
+ files = os.listdir(data_dir)
+ self.files = [os.path.join(data_dir, i) for i in files if i.endswith('.csv')]
+ LOG.warning("Found %d CSV files in path %s", len(self.files), data_dir)
+
+ if not self.files:
+ LOG.warning("Tiger data import selected but no files found at %s", data_dir)
+
+
+ def __enter__(self) -> 'TigerInput':
+ return self
+
+
+ def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
+ if self.tar_handle:
+ self.tar_handle.close()
+ self.tar_handle = None
+
+
+ def next_file(self) -> TextIO:
+ """ Return a file handle to the next file to be processed.
+ Raises an IndexError if there is no file left.
+ """
+ fname = self.files.pop(0)
+
+ if self.tar_handle is not None:
+ extracted = self.tar_handle.extractfile(fname)
+ assert extracted is not None
+ return io.TextIOWrapper(extracted)
+
+ return open(cast(str, fname), encoding='utf-8')
+
+
+ def __len__(self) -> int:
+ return len(self.files)
+
+
+def handle_threaded_sql_statements(pool: WorkerPool, fd: TextIO,
+ analyzer: AbstractAnalyzer) -> None:
+ """ Handles sql statement with multiplexing
+ """
+ lines = 0
+ # Using pool of database connections to execute sql statements
+
+ sql = "SELECT tiger_line_import(%s, %s, %s, %s, %s, %s)"
+
+ for row in csv.DictReader(fd, delimiter=';'):
+ try:
+ address = dict(street=row['street'], postcode=row['postcode'])
+ args = ('SRID=4326;' + row['geometry'],
+ int(row['from']), int(row['to']), row['interpolation'],
+ Json(analyzer.process_place(PlaceInfo({'address': address}))),
+ analyzer.normalize_postcode(row['postcode']))
+ except ValueError:
+ continue
+ pool.next_free_worker().perform(sql, args=args)
+
+ lines += 1
+ if lines == 1000:
+ print('.', end='', flush=True)
+ lines = 0
+
+
+def add_tiger_data(data_dir: str, config: Configuration, threads: int,
+ tokenizer: AbstractTokenizer) -> int:
+ """ Import tiger data from directory or tar file `data dir`.
+ """
+ dsn = config.get_libpq_dsn()
+
+ with TigerInput(data_dir) as tar:
+ if not tar:
+ return 1
+
+ with connect(dsn) as conn:
+ sql = SQLPreprocessor(conn, config)
+ sql.run_sql_file(conn, 'tiger_import_start.sql')
+
+ # Reading files and then for each file line handling
+ # sql_query in <threads - 1> chunks.
+ place_threads = max(1, threads - 1)
+
+ with WorkerPool(dsn, place_threads, ignore_sql_errors=True) as pool:
+ with tokenizer.name_analyzer() as analyzer:
+ while tar:
+ with tar.next_file() as fd:
+ handle_threaded_sql_statements(pool, fd, analyzer)
+
+ print('\n')
+
projects / nominatim.git / blobdiff