-import logging
import os
from pathlib import Path
-import subprocess
import tempfile
import psycopg2
import psycopg2.extras
-LOG = logging.getLogger(__name__)
+from steps.utils import run_script
class NominatimEnvironment:
""" Collects all functions for the execution of Nominatim functions.
self.template_db = config['TEMPLATE_DB']
self.test_db = config['TEST_DB']
self.api_test_db = config['API_TEST_DB']
+ self.api_test_file = config['API_TEST_FILE']
self.server_module_path = config['SERVER_MODULE_PATH']
self.reuse_template = not config['REMOVE_TEMPLATE']
self.keep_scenario_db = config['KEEP_TEST_DB']
self.code_coverage_path = config['PHPCOV']
self.code_coverage_id = 1
- self.test_env = None
+ self.test_env = None
self.template_db_done = False
+ self.api_db_done = False
self.website_dir = None
def connect_database(self, dbname):
self.template_db_done = True
- if self.reuse_template:
- # check that the template is there
- conn = self.connect_database('postgres')
- cur = conn.cursor()
- cur.execute('select count(*) from pg_database where datname = %s',
- (self.template_db,))
- if cur.fetchone()[0] == 1:
- return
- conn.close()
- else:
- # just in case... make sure a previous table has been dropped
- self.db_drop_database(self.template_db)
+ if self._reuse_or_drop_db(self.template_db):
+ return
try:
# call the first part of database setup
"""
self.write_nominatim_config(self.api_test_db)
+ if self.api_db_done:
+ return
+
+ self.api_db_done = True
+
+ if self._reuse_or_drop_db(self.api_test_db):
+ return
+
+ testdata = Path('__file__') / '..' / '..' / 'testdb'
+ self.test_env['NOMINATIM_TIGER_DATA_PATH'] = str((testdata / 'tiger').resolve())
+ self.test_env['NOMINATIM_WIKIPEDIA_DATA_PATH'] = str(testdata.resolve())
+
+ try:
+ self.run_setup_script('all', osm_file=self.api_test_file)
+ self.run_setup_script('import-tiger-data')
+
+ phrase_file = str((testdata / 'specialphrases_testdb.sql').resolve())
+ run_script(['psql', '-d', self.api_test_db, '-f', phrase_file])
+ except:
+ self.db_drop_database(self.api_test_db)
+ raise
+
+
def setup_unknown_db(self):
""" Setup a test against a non-existing database.
"""
cur.execute('CREATE DATABASE {} TEMPLATE = {}'.format(self.test_db, self.template_db))
conn.close()
context.db = self.connect_database(self.test_db)
+ context.db.autocommit = True
psycopg2.extras.register_hstore(context.db, globally=False)
def teardown_db(self, context):
if not self.keep_scenario_db:
self.db_drop_database(self.test_db)
+ def _reuse_or_drop_db(self, name):
+ """ Check for the existance of the given DB. If reuse is enabled,
+ then the function checks for existance and returns True if the
+ database is already there. Otherwise an existing database is
+ dropped and always false returned.
+ """
+ if self.reuse_template:
+ conn = self.connect_database('postgres')
+ with conn.cursor() as cur:
+ cur.execute('select count(*) from pg_database where datname = %s',
+ (name,))
+ if cur.fetchone()[0] == 1:
+ return True
+ conn.close()
+ else:
+ self.db_drop_database(name)
+
+ return False
+
+ def reindex_placex(self, db):
+ """ Run the indexing step until all data in the placex has
+ been processed. Indexing during updates can produce more data
+ to index under some circumstances. That is why indexing may have
+ to be run multiple times.
+ """
+ with db.cursor() as cur:
+ while True:
+ self.run_update_script('index')
+
+ cur.execute("SELECT 'a' FROM placex WHERE indexed_status != 0 LIMIT 1")
+ if cur.rowcount == 0:
+ return
+
def run_setup_script(self, *args, **kwargs):
""" Run the Nominatim setup script with the given arguments.
"""
else:
cwd = self.build_dir
- proc = subprocess.Popen(cmd, cwd=cwd, env=self.test_env,
- stdout=subprocess.PIPE, stderr=subprocess.PIPE)
- (outp, outerr) = proc.communicate()
- outerr = outerr.decode('utf-8').replace('\\n', '\n')
- LOG.debug("run_nominatim_script: %s\n%s\n%s", cmd, outp, outerr)
- assert (proc.returncode == 0), "Script '%s' failed:\n%s\n%s\n" % (script, outp, outerr)
+ run_script(cmd, cwd=cwd, env=self.test_env)
+
+ def copy_from_place(self, db):
+ """ Copy data from place to the placex and location_property_osmline
+ tables invoking the appropriate triggers.
+ """
+ self.run_setup_script('create-functions', 'create-partition-functions')
+
+ with db.cursor() as cur:
+ cur.execute("""INSERT INTO placex (osm_type, osm_id, class, type,
+ name, admin_level, address,
+ extratags, geometry)
+ SELECT osm_type, osm_id, class, type,
+ name, admin_level, address,
+ extratags, geometry
+ FROM place
+ WHERE not (class='place' and type='houses' and osm_type='W')""")
+ cur.execute("""INSERT INTO location_property_osmline (osm_id, address, linegeo)
+ SELECT osm_id, address, geometry
+ FROM place
+ WHERE class='place' and type='houses'
+ and osm_type='W'
+ and ST_GeometryType(geometry) = 'ST_LineString'""")
projects / nominatim.git / blobdiff