add migration to mark internal country names

[nominatim.git] / nominatim / tools / tiger_data.py

diff --git a/nominatim/tools/tiger_data.py b/nominatim/tools/tiger_data.py
index ffb558861daab592bbdc2ad3b7a900cde54791c8..8610880ff9f8f8104c1bf96f0dc144d4a7f957ef 100644 (file)
--- a/nominatim/tools/tiger_data.py
+++ b/nominatim/tools/tiger_data.py
@@ -1,3 +1,9 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# This file is part of Nominatim. (https://nominatim.org)
+#
+# Copyright (C) 2022 by the Nominatim developer community.
+# For a full list of authors see the git log.

 """
 Functions for importing tiger data and handling tarbar and directory files
 """
 """
 Functions for importing tiger data and handling tarbar and directory files
 """


@@ -7,12 +13,11 @@ import logging
 import os
 import tarfile
 
 import os
 import tarfile
 

-import psycopg2.extras
-

 from nominatim.db.connection import connect
 from nominatim.db.async_connection import WorkerPool
 from nominatim.db.sql_preprocessor import SQLPreprocessor
 from nominatim.db.connection import connect
 from nominatim.db.async_connection import WorkerPool
 from nominatim.db.sql_preprocessor import SQLPreprocessor

-
+from nominatim.errors import UsageError
+from nominatim.indexer.place_info import PlaceInfo

 
 LOG = logging.getLogger()
 
 
 LOG = logging.getLogger()
 


@@ -23,7 +28,12 @@ def handle_tarfile_or_directory(data_dir):
 
     tar = None
     if data_dir.endswith('.tar.gz'):
 
     tar = None
     if data_dir.endswith('.tar.gz'):

-        tar = tarfile.open(data_dir)
+        try:
+            tar = tarfile.open(data_dir)
+        except tarfile.ReadError as err:
+            LOG.fatal("Cannot open '%s'. Is this a tar file?", data_dir)
+            raise UsageError("Cannot open Tiger data file.") from err
+

         csv_files = [i for i in tar.getmembers() if i.name.endswith('.csv')]
         LOG.warning("Found %d CSV files in tarfile with path %s", len(csv_files), data_dir)
         if not csv_files:
         csv_files = [i for i in tar.getmembers() if i.name.endswith('.csv')]
         LOG.warning("Found %d CSV files in tarfile with path %s", len(csv_files), data_dir)
         if not csv_files:


@@ -53,7 +63,7 @@ def handle_threaded_sql_statements(pool, fd, analyzer):
             address = dict(street=row['street'], postcode=row['postcode'])
             args = ('SRID=4326;' + row['geometry'],
                     int(row['from']), int(row['to']), row['interpolation'],
             address = dict(street=row['street'], postcode=row['postcode'])
             args = ('SRID=4326;' + row['geometry'],
                     int(row['from']), int(row['to']), row['interpolation'],

-                    psycopg2.extras.Json(analyzer.process_place(dict(address=address))),
+                    PlaceInfo({'address': address}).analyze(analyzer),

                     analyzer.normalize_postcode(row['postcode']))
         except ValueError:
             continue
                     analyzer.normalize_postcode(row['postcode']))
         except ValueError:
             continue