test/bdd/steps/nominatim_environment.py

   1 import os
   2 from pathlib import Path
   3 import tempfile
   4
   5 import psycopg2
   6 import psycopg2.extras
   7
   8 from steps.utils import run_script
   9
  10 class NominatimEnvironment:
  11     """ Collects all functions for the execution of Nominatim functions.
  12     """
  13
  14     def __init__(self, config):
  15         self.build_dir = Path(config['BUILDDIR']).resolve()
  16         self.src_dir = (Path(__file__) / '..' / '..' / '..' / '..').resolve()
  17         self.db_host = config['DB_HOST']
  18         self.db_port = config['DB_PORT']
  19         self.db_user = config['DB_USER']
  20         self.db_pass = config['DB_PASS']
  21         self.template_db = config['TEMPLATE_DB']
  22         self.test_db = config['TEST_DB']
  23         self.api_test_db = config['API_TEST_DB']
  24         self.server_module_path = config['SERVER_MODULE_PATH']
  25         self.reuse_template = not config['REMOVE_TEMPLATE']
  26         self.keep_scenario_db = config['KEEP_TEST_DB']
  27         self.code_coverage_path = config['PHPCOV']
  28         self.code_coverage_id = 1
  29         self.test_env = None
  30
  31         self.template_db_done = False
  32         self.website_dir = None
  33
  34     def connect_database(self, dbname):
  35         """ Return a connection to the database with the given name.
  36             Uses configured host, user and port.
  37         """
  38         dbargs = {'database': dbname}
  39         if self.db_host:
  40             dbargs['host'] = self.db_host
  41         if self.db_port:
  42             dbargs['port'] = self.db_port
  43         if self.db_user:
  44             dbargs['user'] = self.db_user
  45         if self.db_pass:
  46             dbargs['password'] = self.db_pass
  47         conn = psycopg2.connect(**dbargs)
  48         return conn
  49
  50     def next_code_coverage_file(self):
  51         """ Generate the next name for a coverage file.
  52         """
  53         fn = Path(self.code_coverage_path) / "{:06d}.cov".format(self.code_coverage_id)
  54         self.code_coverage_id += 1
  55
  56         return fn.resolve()
  57
  58     def write_nominatim_config(self, dbname):
  59         """ Set up a custom test configuration that connects to the given
  60             database. This sets up the environment variables so that they can
  61             be picked up by dotenv and creates a project directory with the
  62             appropriate website scripts.
  63         """
  64         dsn = 'pgsql:dbname={}'.format(dbname)
  65         if self.db_host:
  66             dsn += ';host=' + self.db_host
  67         if self.db_port:
  68             dsn += ';port=' + self.db_port
  69         if self.db_user:
  70             dsn += ';user=' + self.db_user
  71         if self.db_pass:
  72             dsn += ';password=' + self.db_pass
  73
  74         if self.website_dir is not None \
  75            and self.test_env is not None \
  76            and dsn == self.test_env['NOMINATIM_DATABASE_DSN']:
  77             return # environment already set uo
  78
  79         self.test_env = os.environ
  80         self.test_env['NOMINATIM_DATABASE_DSN'] = dsn
  81         self.test_env['NOMINATIM_FLATNODE_FILE'] = ''
  82         self.test_env['NOMINATIM_IMPORT_STYLE'] = 'full'
  83         self.test_env['NOMINATIM_USE_US_TIGER_DATA'] = 'yes'
  84
  85         if self.server_module_path:
  86             self.test_env['NOMINATIM_DATABASE_MODULE_PATH'] = self.server_module_path
  87
  88         if self.website_dir is not None:
  89             self.website_dir.cleanup()
  90
  91         self.website_dir = tempfile.TemporaryDirectory()
  92         self.run_setup_script('setup-website')
  93
  94
  95     def db_drop_database(self, name):
  96         """ Drop the database with the given name.
  97         """
  98         conn = self.connect_database('postgres')
  99         conn.set_isolation_level(0)
 100         cur = conn.cursor()
 101         cur.execute('DROP DATABASE IF EXISTS {}'.format(name))
 102         conn.close()
 103
 104     def setup_template_db(self):
 105         """ Setup a template database that already contains common test data.
 106             Having a template database speeds up tests considerably but at
 107             the price that the tests sometimes run with stale data.
 108         """
 109         if self.template_db_done:
 110             return
 111
 112         self.template_db_done = True
 113
 114         if self.reuse_template:
 115             # check that the template is there
 116             conn = self.connect_database('postgres')
 117             cur = conn.cursor()
 118             cur.execute('select count(*) from pg_database where datname = %s',
 119                         (self.template_db,))
 120             if cur.fetchone()[0] == 1:
 121                 return
 122             conn.close()
 123         else:
 124             # just in case... make sure a previous table has been dropped
 125             self.db_drop_database(self.template_db)
 126
 127         try:
 128             # call the first part of database setup
 129             self.write_nominatim_config(self.template_db)
 130             self.run_setup_script('create-db', 'setup-db')
 131             # remove external data to speed up indexing for tests
 132             conn = self.connect_database(self.template_db)
 133             cur = conn.cursor()
 134             cur.execute("""select tablename from pg_tables
 135                            where tablename in ('gb_postcode', 'us_postcode')""")
 136             for t in cur:
 137                 conn.cursor().execute('TRUNCATE TABLE {}'.format(t[0]))
 138             conn.commit()
 139             conn.close()
 140
 141             # execute osm2pgsql import on an empty file to get the right tables
 142             with tempfile.NamedTemporaryFile(dir='/tmp', suffix='.xml') as fd:
 143                 fd.write(b'<osm version="0.6"></osm>')
 144                 fd.flush()
 145                 self.run_setup_script('import-data',
 146                                       'ignore-errors',
 147                                       'create-functions',
 148                                       'create-tables',
 149                                       'create-partition-tables',
 150                                       'create-partition-functions',
 151                                       'load-data',
 152                                       'create-search-indices',
 153                                       osm_file=fd.name,
 154                                       osm2pgsql_cache='200')
 155         except:
 156             self.db_drop_database(self.template_db)
 157             raise
 158
 159
 160     def setup_api_db(self):
 161         """ Setup a test against the API test database.
 162         """
 163         self.write_nominatim_config(self.api_test_db)
 164
 165     def setup_unknown_db(self):
 166         """ Setup a test against a non-existing database.
 167         """
 168         self.write_nominatim_config('UNKNOWN_DATABASE_NAME')
 169
 170     def setup_db(self, context):
 171         """ Setup a test against a fresh, empty test database.
 172         """
 173         self.setup_template_db()
 174         self.write_nominatim_config(self.test_db)
 175         conn = self.connect_database(self.template_db)
 176         conn.set_isolation_level(0)
 177         cur = conn.cursor()
 178         cur.execute('DROP DATABASE IF EXISTS {}'.format(self.test_db))
 179         cur.execute('CREATE DATABASE {} TEMPLATE = {}'.format(self.test_db, self.template_db))
 180         conn.close()
 181         context.db = self.connect_database(self.test_db)
 182         context.db.autocommit = True
 183         psycopg2.extras.register_hstore(context.db, globally=False)
 184
 185     def teardown_db(self, context):
 186         """ Remove the test database, if it exists.
 187         """
 188         if 'db' in context:
 189             context.db.close()
 190
 191         if not self.keep_scenario_db:
 192             self.db_drop_database(self.test_db)
 193
 194     def run_setup_script(self, *args, **kwargs):
 195         """ Run the Nominatim setup script with the given arguments.
 196         """
 197         self.run_nominatim_script('setup', *args, **kwargs)
 198
 199     def run_update_script(self, *args, **kwargs):
 200         """ Run the Nominatim update script with the given arguments.
 201         """
 202         self.run_nominatim_script('update', *args, **kwargs)
 203
 204     def run_nominatim_script(self, script, *args, **kwargs):
 205         """ Run one of the Nominatim utility scripts with the given arguments.
 206         """
 207         cmd = ['/usr/bin/env', 'php', '-Cq']
 208         cmd.append((Path(self.build_dir) / 'utils' / '{}.php'.format(script)).resolve())
 209         cmd.extend(['--' + x for x in args])
 210         for k, v in kwargs.items():
 211             cmd.extend(('--' + k.replace('_', '-'), str(v)))
 212
 213         if self.website_dir is not None:
 214             cwd = self.website_dir.name
 215         else:
 216             cwd = self.build_dir
 217
 218         run_script(cmd, cwd=cwd, env=self.test_env)
 219
 220     def copy_from_place(self, db):
 221         """ Copy data from place to the placex and location_property_osmline
 222             tables invoking the appropriate triggers.
 223         """
 224         self.run_setup_script('create-functions', 'create-partition-functions')
 225
 226         with db.cursor() as cur:
 227             cur.execute("""INSERT INTO placex (osm_type, osm_id, class, type,
 228                                                name, admin_level, address,
 229                                                extratags, geometry)
 230                              SELECT osm_type, osm_id, class, type,
 231                                     name, admin_level, address,
 232                                     extratags, geometry
 233                                FROM place
 234                                WHERE not (class='place' and type='houses' and osm_type='W')""")
 235             cur.execute("""INSERT INTO location_property_osmline (osm_id, address, linegeo)
 236                              SELECT osm_id, address, geometry
 237                                FROM place
 238                               WHERE class='place' and type='houses'
 239                                     and osm_type='W'
 240                                     and ST_GeometryType(geometry) = 'ST_LineString'""")