test/bdd/steps/nominatim_environment.py

   1 import logging
   2 import os
   3 from pathlib import Path
   4 import subprocess
   5 import tempfile
   6
   7 import psycopg2
   8 import psycopg2.extras
   9
  10 LOG = logging.getLogger(__name__)
  11
  12 class NominatimEnvironment:
  13     """ Collects all functions for the execution of Nominatim functions.
  14     """
  15
  16     def __init__(self, config):
  17         self.build_dir = Path(config['BUILDDIR']).resolve()
  18         self.src_dir = (Path(__file__) / '..' / '..' / '..' / '..').resolve()
  19         self.db_host = config['DB_HOST']
  20         self.db_port = config['DB_PORT']
  21         self.db_user = config['DB_USER']
  22         self.db_pass = config['DB_PASS']
  23         self.template_db = config['TEMPLATE_DB']
  24         self.test_db = config['TEST_DB']
  25         self.api_test_db = config['API_TEST_DB']
  26         self.server_module_path = config['SERVER_MODULE_PATH']
  27         self.reuse_template = not config['REMOVE_TEMPLATE']
  28         self.keep_scenario_db = config['KEEP_TEST_DB']
  29         self.code_coverage_path = config['PHPCOV']
  30         self.code_coverage_id = 1
  31         self.test_env = None
  32
  33         self.template_db_done = False
  34         self.website_dir = None
  35
  36     def connect_database(self, dbname):
  37         """ Return a connection to the database with the given name.
  38             Uses configured host, user and port.
  39         """
  40         dbargs = {'database': dbname}
  41         if self.db_host:
  42             dbargs['host'] = self.db_host
  43         if self.db_port:
  44             dbargs['port'] = self.db_port
  45         if self.db_user:
  46             dbargs['user'] = self.db_user
  47         if self.db_pass:
  48             dbargs['password'] = self.db_pass
  49         conn = psycopg2.connect(**dbargs)
  50         return conn
  51
  52     def next_code_coverage_file(self):
  53         """ Generate the next name for a coverage file.
  54         """
  55         fn = Path(self.code_coverage_path) / "{:06d}.cov".format(self.code_coverage_id)
  56         self.code_coverage_id += 1
  57
  58         return fn.resolve()
  59
  60     def write_nominatim_config(self, dbname):
  61         """ Set up a custom test configuration that connects to the given
  62             database. This sets up the environment variables so that they can
  63             be picked up by dotenv and creates a project directory with the
  64             appropriate website scripts.
  65         """
  66         dsn = 'pgsql:dbname={}'.format(dbname)
  67         if self.db_host:
  68             dsn += ';host=' + self.db_host
  69         if self.db_port:
  70             dsn += ';port=' + self.db_port
  71         if self.db_user:
  72             dsn += ';user=' + self.db_user
  73         if self.db_pass:
  74             dsn += ';password=' + self.db_pass
  75
  76         if self.website_dir is not None \
  77            and self.test_env is not None \
  78            and dsn == self.test_env['NOMINATIM_DATABASE_DSN']:
  79             return # environment already set uo
  80
  81         self.test_env = os.environ
  82         self.test_env['NOMINATIM_DATABASE_DSN'] = dsn
  83         self.test_env['NOMINATIM_FLATNODE_FILE'] = ''
  84         self.test_env['NOMINATIM_IMPORT_STYLE'] = 'full'
  85         self.test_env['NOMINATIM_USE_US_TIGER_DATA'] = 'yes'
  86
  87         if self.server_module_path:
  88             self.test_env['NOMINATIM_DATABASE_MODULE_PATH'] = self.server_module_path
  89
  90         if self.website_dir is not None:
  91             self.website_dir.cleanup()
  92
  93         self.website_dir = tempfile.TemporaryDirectory()
  94         self.run_setup_script('setup-website')
  95
  96
  97     def db_drop_database(self, name):
  98         """ Drop the database with the given name.
  99         """
 100         conn = self.connect_database('postgres')
 101         conn.set_isolation_level(0)
 102         cur = conn.cursor()
 103         cur.execute('DROP DATABASE IF EXISTS {}'.format(name))
 104         conn.close()
 105
 106     def setup_template_db(self):
 107         """ Setup a template database that already contains common test data.
 108             Having a template database speeds up tests considerably but at
 109             the price that the tests sometimes run with stale data.
 110         """
 111         if self.template_db_done:
 112             return
 113
 114         self.template_db_done = True
 115
 116         if self.reuse_template:
 117             # check that the template is there
 118             conn = self.connect_database('postgres')
 119             cur = conn.cursor()
 120             cur.execute('select count(*) from pg_database where datname = %s',
 121                         (self.template_db,))
 122             if cur.fetchone()[0] == 1:
 123                 return
 124             conn.close()
 125         else:
 126             # just in case... make sure a previous table has been dropped
 127             self.db_drop_database(self.template_db)
 128
 129         try:
 130             # call the first part of database setup
 131             self.write_nominatim_config(self.template_db)
 132             self.run_setup_script('create-db', 'setup-db')
 133             # remove external data to speed up indexing for tests
 134             conn = self.connect_database(self.template_db)
 135             cur = conn.cursor()
 136             cur.execute("""select tablename from pg_tables
 137                            where tablename in ('gb_postcode', 'us_postcode')""")
 138             for t in cur:
 139                 conn.cursor().execute('TRUNCATE TABLE {}'.format(t[0]))
 140             conn.commit()
 141             conn.close()
 142
 143             # execute osm2pgsql import on an empty file to get the right tables
 144             with tempfile.NamedTemporaryFile(dir='/tmp', suffix='.xml') as fd:
 145                 fd.write(b'<osm version="0.6"></osm>')
 146                 fd.flush()
 147                 self.run_setup_script('import-data',
 148                                       'ignore-errors',
 149                                       'create-functions',
 150                                       'create-tables',
 151                                       'create-partition-tables',
 152                                       'create-partition-functions',
 153                                       'load-data',
 154                                       'create-search-indices',
 155                                       osm_file=fd.name,
 156                                       osm2pgsql_cache='200')
 157         except:
 158             self.db_drop_database(self.template_db)
 159             raise
 160
 161
 162     def setup_api_db(self):
 163         """ Setup a test against the API test database.
 164         """
 165         self.write_nominatim_config(self.api_test_db)
 166
 167     def setup_unknown_db(self):
 168         """ Setup a test against a non-existing database.
 169         """
 170         self.write_nominatim_config('UNKNOWN_DATABASE_NAME')
 171
 172     def setup_db(self, context):
 173         """ Setup a test against a fresh, empty test database.
 174         """
 175         self.setup_template_db()
 176         self.write_nominatim_config(self.test_db)
 177         conn = self.connect_database(self.template_db)
 178         conn.set_isolation_level(0)
 179         cur = conn.cursor()
 180         cur.execute('DROP DATABASE IF EXISTS {}'.format(self.test_db))
 181         cur.execute('CREATE DATABASE {} TEMPLATE = {}'.format(self.test_db, self.template_db))
 182         conn.close()
 183         context.db = self.connect_database(self.test_db)
 184         psycopg2.extras.register_hstore(context.db, globally=False)
 185
 186     def teardown_db(self, context):
 187         """ Remove the test database, if it exists.
 188         """
 189         if 'db' in context:
 190             context.db.close()
 191
 192         if not self.keep_scenario_db:
 193             self.db_drop_database(self.test_db)
 194
 195     def run_setup_script(self, *args, **kwargs):
 196         """ Run the Nominatim setup script with the given arguments.
 197         """
 198         self.run_nominatim_script('setup', *args, **kwargs)
 199
 200     def run_update_script(self, *args, **kwargs):
 201         """ Run the Nominatim update script with the given arguments.
 202         """
 203         self.run_nominatim_script('update', *args, **kwargs)
 204
 205     def run_nominatim_script(self, script, *args, **kwargs):
 206         """ Run one of the Nominatim utility scripts with the given arguments.
 207         """
 208         cmd = ['/usr/bin/env', 'php', '-Cq']
 209         cmd.append((Path(self.build_dir) / 'utils' / '{}.php'.format(script)).resolve())
 210         cmd.extend(['--' + x for x in args])
 211         for k, v in kwargs.items():
 212             cmd.extend(('--' + k.replace('_', '-'), str(v)))
 213
 214         if self.website_dir is not None:
 215             cwd = self.website_dir.name
 216         else:
 217             cwd = self.build_dir
 218
 219         proc = subprocess.Popen(cmd, cwd=cwd, env=self.test_env,
 220                                 stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 221         (outp, outerr) = proc.communicate()
 222         outerr = outerr.decode('utf-8').replace('\\n', '\n')
 223         LOG.debug("run_nominatim_script: %s\n%s\n%s", cmd, outp, outerr)
 224         assert (proc.returncode == 0), "Script '%s' failed:\n%s\n%s\n" % (script, outp, outerr)