Ported createCountryNames() to python and added tests

[nominatim.git] / nominatim / tools / database_import.py

diff --git a/nominatim/tools/database_import.py b/nominatim/tools/database_import.py
index ab8ccdd2a129365a03930d1568619b38dc14d128..9d4cad1693bbc6c089ac930c57add5f3493dc5ce 100644 (file)
--- a/nominatim/tools/database_import.py
+++ b/nominatim/tools/database_import.py
@@ -14,6 +14,7 @@ import psycopg2
 from ..db.connection import connect, get_pg_env
 from ..db import utils as db_utils
 from ..db.async_connection import DBConnection
 from ..db.connection import connect, get_pg_env
 from ..db import utils as db_utils
 from ..db.async_connection import DBConnection

+from ..db.sql_preprocessor import SQLPreprocessor

 from .exec_utils import run_osm2pgsql
 from ..errors import UsageError
 from ..version import POSTGRESQL_REQUIRED_VERSION, POSTGIS_REQUIRED_VERSION
 from .exec_utils import run_osm2pgsql
 from ..errors import UsageError
 from ..version import POSTGRESQL_REQUIRED_VERSION, POSTGIS_REQUIRED_VERSION


@@ -178,6 +179,32 @@ def import_osm_data(osm_file, options, drop=False, ignore_errors=False):
             Path(options['flatnode_file']).unlink()
 
 
             Path(options['flatnode_file']).unlink()
 
 

+def create_tables(conn, config, sqllib_dir, reverse_only=False):
+    """ Create the set of basic tables.
+        When `reverse_only` is True, then the main table for searching will
+        be skipped and only reverse search is possible.
+    """
+    sql = SQLPreprocessor(conn, config, sqllib_dir)
+    sql.env.globals['db']['reverse_only'] = reverse_only
+
+    sql.run_sql_file(conn, 'tables.sql')
+
+
+def create_table_triggers(conn, config, sqllib_dir):
+    """ Create the triggers for the tables. The trigger functions must already
+        have been imported with refresh.create_functions().
+    """
+    sql = SQLPreprocessor(conn, config, sqllib_dir)
+    sql.run_sql_file(conn, 'table-triggers.sql')
+
+
+def create_partition_tables(conn, config, sqllib_dir):
+    """ Create tables that have explicit partitioning.
+    """
+    sql = SQLPreprocessor(conn, config, sqllib_dir)
+    sql.run_sql_file(conn, 'partition-tables.src.sql')
+
+

 def truncate_data_tables(conn, max_word_frequency=None):
     """ Truncate all data tables to prepare for a fresh load.
     """
 def truncate_data_tables(conn, max_word_frequency=None):
     """ Truncate all data tables to prepare for a fresh load.
     """


@@ -258,3 +285,59 @@ def load_data(dsn, data_dir, threads):
     with connect(dsn) as conn:
         with conn.cursor() as cur:
             cur.execute('ANALYSE')
     with connect(dsn) as conn:
         with conn.cursor() as cur:
             cur.execute('ANALYSE')

+
+
+def create_search_indices(conn, config, sqllib_dir, drop=False):
+    """ Create tables that have explicit partitioning.
+    """
+
+    # If index creation failed and left an index invalid, they need to be
+    # cleaned out first, so that the script recreates them.
+    with conn.cursor() as cur:
+        cur.execute("""SELECT relname FROM pg_class, pg_index
+                       WHERE pg_index.indisvalid = false
+                             AND pg_index.indexrelid = pg_class.oid""")
+        bad_indices = [row[0] for row in list(cur)]
+        for idx in bad_indices:
+            LOG.info("Drop invalid index %s.", idx)
+            cur.execute('DROP INDEX "{}"'.format(idx))
+    conn.commit()
+
+    sql = SQLPreprocessor(conn, config, sqllib_dir)
+
+    sql.run_sql_file(conn, 'indices.sql', drop=drop)
+
+def create_country_names(conn, config):
+    """ Create search index for default country names.
+    """
+
+    with conn.cursor() as cur:
+        cur.execute("""select
+                       getorcreate_country(make_standard_name('uk')
+                       , 'gb')""")
+        cur.execute("""select getorcreate_country(make_standard_name('united states'), 'us')""")
+        cur.execute("""select count(*) from
+                       (select getorcreate_country(make_standard_name(country_code),
+                       country_code) from country_name where country_code is not null) as x""")
+        cur.execute("""select count(*) from
+                        (select getorcreate_country(make_standard_name(name->'name'),
+                        country_code) from country_name where name ? 'name') as x""")
+        sql_statement = """select count(*) from (select getorcreate_country(make_standard_name(v)
+                           , country_code) from (select country_code, skeys(name)
+                           as k, svals(name) as v from country_name) x where k """
+
+        languages = config.LANGUAGES
+
+        if languages:
+            sql_statement += 'in '
+            delim = '('
+            for language in languages.split(','):
+                sql_statement += delim + "'name:" + language + "'"
+                delim = ','
+
+            sql_statement += ')'
+        else:
+            sql_statement += "like 'name:%'"
+        sql_statement += ') v'
+        cur.execute(sql_statement)
+    conn.commit()