add tests for new data invalidation functions

[nominatim.git] / nominatim / tools / country_info.py

diff --git a/nominatim/tools/country_info.py b/nominatim/tools/country_info.py
index 5b37c0b645dfffe301bfc649b38c5a77b758367b..f6a6293f46e46573c88d5e288c653b47f66a1cbc 100644 (file)
--- a/nominatim/tools/country_info.py
+++ b/nominatim/tools/country_info.py
@@ -12,13 +12,14 @@ import psycopg2.extras
 from nominatim.db import utils as db_utils
 from nominatim.db.connection import connect
 
 from nominatim.db import utils as db_utils
 from nominatim.db.connection import connect
 

+

 class _CountryInfo:
     """ Caches country-specific properties from the configuration file.
     """
 
     def __init__(self):
         self._info = {}
 class _CountryInfo:
     """ Caches country-specific properties from the configuration file.
     """
 
     def __init__(self):
         self._info = {}

-
+        self._key_prefix = 'name'

 
     def load(self, config):
         """ Load the country properties from the configuration files,
 
     def load(self, config):
         """ Load the country properties from the configuration files,


@@ -33,16 +34,24 @@ class _CountryInfo:
                 elif not isinstance(prop['languages'], list):
                     prop['languages'] = [x.strip()
                                          for x in prop['languages'].split(',')]
                 elif not isinstance(prop['languages'], list):
                     prop['languages'] = [x.strip()
                                          for x in prop['languages'].split(',')]

-
+                if 'names' not in prop or prop['names'] is None:
+                    prop['names'] = {self._key_prefix: {}}

 
     def items(self):
         """ Return tuples of (country_code, property dict) as iterable.
         """
         return self._info.items()
 
 
     def items(self):
         """ Return tuples of (country_code, property dict) as iterable.
         """
         return self._info.items()
 

+    def key_prefix(self):
+        """ Return the prefix that will be attached to the keys of the country
+            names values when storing them in the database
+        """
+        return self._key_prefix
+

 
 _COUNTRY_INFO = _CountryInfo()
 
 
 _COUNTRY_INFO = _CountryInfo()
 

+

 def setup_country_config(config):
     """ Load country properties from the configuration file.
         Needs to be called before using any other functions in this
 def setup_country_config(config):
     """ Load country properties from the configuration file.
         Needs to be called before using any other functions in this


@@ -61,9 +70,11 @@ def setup_country_tables(dsn, sql_dir, ignore_partitions=False):
     """ Create and populate the tables with basic static data that provides
         the background for geocoding. Data is assumed to not yet exist.
     """
     """ Create and populate the tables with basic static data that provides
         the background for geocoding. Data is assumed to not yet exist.
     """

-    db_utils.execute_file(dsn, sql_dir / 'country_name.sql')

     db_utils.execute_file(dsn, sql_dir / 'country_osm_grid.sql.gz')
 
     db_utils.execute_file(dsn, sql_dir / 'country_osm_grid.sql.gz')
 

+    def add_prefix_to_keys(names, prefix):
+        return {prefix+':'+k: v for k, v in names.items()}
+

     params = []
     for ccode, props in _COUNTRY_INFO.items():
         if ccode is not None and props is not None:
     params = []
     for ccode, props in _COUNTRY_INFO.items():
         if ccode is not None and props is not None:


@@ -71,16 +82,26 @@ def setup_country_tables(dsn, sql_dir, ignore_partitions=False):
                 partition = 0
             else:
                 partition = props.get('partition')
                 partition = 0
             else:
                 partition = props.get('partition')

-            lang = props['languages'][0] if len(props['languages']) == 1 else None
-            params.append((ccode, partition, lang))
-
+            lang = props['languages'][0] if len(
+                props['languages']) == 1 else None
+            name = add_prefix_to_keys(props.get('names').get(
+                _COUNTRY_INFO.key_prefix()), _COUNTRY_INFO.key_prefix())
+            params.append((ccode, name, lang, partition))

     with connect(dsn) as conn:
         with conn.cursor() as cur:
     with connect(dsn) as conn:
         with conn.cursor() as cur:

+            psycopg2.extras.register_hstore(cur)
+            cur.execute(
+                """ CREATE TABLE public.country_name (
+                        country_code character varying(2),
+                        name public.hstore,
+                        derived_name public.hstore,
+                        country_default_language_code text,
+                        partition integer
+                    ); """)

             cur.execute_values(
             cur.execute_values(

-                """ UPDATE country_name
-                    SET partition = part, country_default_language_code = lang
-                    FROM (VALUES %s) AS v (cc, part, lang)
-                    WHERE country_code = v.cc""", params)
+                """ INSERT INTO public.country_name
+                    (country_code, name, country_default_language_code, partition) VALUES %s
+                """, params)

         conn.commit()
 
 
         conn.commit()
 
 


@@ -94,8 +115,9 @@ def create_country_names(conn, tokenizer, languages=None):
         languages = languages.split(',')
 
     def _include_key(key):
         languages = languages.split(',')
 
     def _include_key(key):

-        return key == 'name' or \
-               (key.startswith('name:') and (not languages or key[5:] in languages))
+        return key == _COUNTRY_INFO.key_prefix() or \
+            (key.startswith(_COUNTRY_INFO.key_prefix()+':') and
+             (not languages or key[len(_COUNTRY_INFO.key_prefix())+1:] in languages))

 
     with conn.cursor() as cur:
         psycopg2.extras.register_hstore(cur)
 
     with conn.cursor() as cur:
         psycopg2.extras.register_hstore(cur)


@@ -112,7 +134,8 @@ def create_country_names(conn, tokenizer, languages=None):
 
                 # country names (only in languages as provided)
                 if name:
 
                 # country names (only in languages as provided)
                 if name:

-                    names.update(((k, v) for k, v in name.items() if _include_key(k)))
+                    names.update(((k, v)
+                                  for k, v in name.items() if _include_key(k)))

 
                 analyzer.add_country_names(code, names)
 
 
                 analyzer.add_country_names(code, names)