test/bdd/steps/nominatim_environment.py

   1 import os
   2 from pathlib import Path
   3 import tempfile
   4
   5 import psycopg2
   6 import psycopg2.extras
   7
   8 from steps.utils import run_script
   9
  10 class NominatimEnvironment:
  11     """ Collects all functions for the execution of Nominatim functions.
  12     """
  13
  14     def __init__(self, config):
  15         self.build_dir = Path(config['BUILDDIR']).resolve()
  16         self.src_dir = (Path(__file__) / '..' / '..' / '..' / '..').resolve()
  17         self.db_host = config['DB_HOST']
  18         self.db_port = config['DB_PORT']
  19         self.db_user = config['DB_USER']
  20         self.db_pass = config['DB_PASS']
  21         self.template_db = config['TEMPLATE_DB']
  22         self.test_db = config['TEST_DB']
  23         self.api_test_db = config['API_TEST_DB']
  24         self.server_module_path = config['SERVER_MODULE_PATH']
  25         self.reuse_template = not config['REMOVE_TEMPLATE']
  26         self.keep_scenario_db = config['KEEP_TEST_DB']
  27         self.code_coverage_path = config['PHPCOV']
  28         self.code_coverage_id = 1
  29         self.test_env = None
  30
  31         self.template_db_done = False
  32         self.website_dir = None
  33
  34     def connect_database(self, dbname):
  35         """ Return a connection to the database with the given name.
  36             Uses configured host, user and port.
  37         """
  38         dbargs = {'database': dbname}
  39         if self.db_host:
  40             dbargs['host'] = self.db_host
  41         if self.db_port:
  42             dbargs['port'] = self.db_port
  43         if self.db_user:
  44             dbargs['user'] = self.db_user
  45         if self.db_pass:
  46             dbargs['password'] = self.db_pass
  47         conn = psycopg2.connect(**dbargs)
  48         return conn
  49
  50     def next_code_coverage_file(self):
  51         """ Generate the next name for a coverage file.
  52         """
  53         fn = Path(self.code_coverage_path) / "{:06d}.cov".format(self.code_coverage_id)
  54         self.code_coverage_id += 1
  55
  56         return fn.resolve()
  57
  58     def write_nominatim_config(self, dbname):
  59         """ Set up a custom test configuration that connects to the given
  60             database. This sets up the environment variables so that they can
  61             be picked up by dotenv and creates a project directory with the
  62             appropriate website scripts.
  63         """
  64         dsn = 'pgsql:dbname={}'.format(dbname)
  65         if self.db_host:
  66             dsn += ';host=' + self.db_host
  67         if self.db_port:
  68             dsn += ';port=' + self.db_port
  69         if self.db_user:
  70             dsn += ';user=' + self.db_user
  71         if self.db_pass:
  72             dsn += ';password=' + self.db_pass
  73
  74         if self.website_dir is not None \
  75            and self.test_env is not None \
  76            and dsn == self.test_env['NOMINATIM_DATABASE_DSN']:
  77             return # environment already set uo
  78
  79         self.test_env = os.environ
  80         self.test_env['NOMINATIM_DATABASE_DSN'] = dsn
  81         self.test_env['NOMINATIM_FLATNODE_FILE'] = ''
  82         self.test_env['NOMINATIM_IMPORT_STYLE'] = 'full'
  83         self.test_env['NOMINATIM_USE_US_TIGER_DATA'] = 'yes'
  84
  85         if self.server_module_path:
  86             self.test_env['NOMINATIM_DATABASE_MODULE_PATH'] = self.server_module_path
  87
  88         if self.website_dir is not None:
  89             self.website_dir.cleanup()
  90
  91         self.website_dir = tempfile.TemporaryDirectory()
  92         self.run_setup_script('setup-website')
  93
  94
  95     def db_drop_database(self, name):
  96         """ Drop the database with the given name.
  97         """
  98         conn = self.connect_database('postgres')
  99         conn.set_isolation_level(0)
 100         cur = conn.cursor()
 101         cur.execute('DROP DATABASE IF EXISTS {}'.format(name))
 102         conn.close()
 103
 104     def setup_template_db(self):
 105         """ Setup a template database that already contains common test data.
 106             Having a template database speeds up tests considerably but at
 107             the price that the tests sometimes run with stale data.
 108         """
 109         if self.template_db_done:
 110             return
 111
 112         self.template_db_done = True
 113
 114         if self.reuse_template:
 115             # check that the template is there
 116             conn = self.connect_database('postgres')
 117             cur = conn.cursor()
 118             cur.execute('select count(*) from pg_database where datname = %s',
 119                         (self.template_db,))
 120             if cur.fetchone()[0] == 1:
 121                 return
 122             conn.close()
 123         else:
 124             # just in case... make sure a previous table has been dropped
 125             self.db_drop_database(self.template_db)
 126
 127         try:
 128             # call the first part of database setup
 129             self.write_nominatim_config(self.template_db)
 130             self.run_setup_script('create-db', 'setup-db')
 131             # remove external data to speed up indexing for tests
 132             conn = self.connect_database(self.template_db)
 133             cur = conn.cursor()
 134             cur.execute("""select tablename from pg_tables
 135                            where tablename in ('gb_postcode', 'us_postcode')""")
 136             for t in cur:
 137                 conn.cursor().execute('TRUNCATE TABLE {}'.format(t[0]))
 138             conn.commit()
 139             conn.close()
 140
 141             # execute osm2pgsql import on an empty file to get the right tables
 142             with tempfile.NamedTemporaryFile(dir='/tmp', suffix='.xml') as fd:
 143                 fd.write(b'<osm version="0.6"></osm>')
 144                 fd.flush()
 145                 self.run_setup_script('import-data',
 146                                       'ignore-errors',
 147                                       'create-functions',
 148                                       'create-tables',
 149                                       'create-partition-tables',
 150                                       'create-partition-functions',
 151                                       'load-data',
 152                                       'create-search-indices',
 153                                       osm_file=fd.name,
 154                                       osm2pgsql_cache='200')
 155         except:
 156             self.db_drop_database(self.template_db)
 157             raise
 158
 159
 160     def setup_api_db(self):
 161         """ Setup a test against the API test database.
 162         """
 163         self.write_nominatim_config(self.api_test_db)
 164
 165     def setup_unknown_db(self):
 166         """ Setup a test against a non-existing database.
 167         """
 168         self.write_nominatim_config('UNKNOWN_DATABASE_NAME')
 169
 170     def setup_db(self, context):
 171         """ Setup a test against a fresh, empty test database.
 172         """
 173         self.setup_template_db()
 174         self.write_nominatim_config(self.test_db)
 175         conn = self.connect_database(self.template_db)
 176         conn.set_isolation_level(0)
 177         cur = conn.cursor()
 178         cur.execute('DROP DATABASE IF EXISTS {}'.format(self.test_db))
 179         cur.execute('CREATE DATABASE {} TEMPLATE = {}'.format(self.test_db, self.template_db))
 180         conn.close()
 181         context.db = self.connect_database(self.test_db)
 182         context.db.autocommit = True
 183         psycopg2.extras.register_hstore(context.db, globally=False)
 184
 185     def teardown_db(self, context):
 186         """ Remove the test database, if it exists.
 187         """
 188         if 'db' in context:
 189             context.db.close()
 190
 191         if not self.keep_scenario_db:
 192             self.db_drop_database(self.test_db)
 193
 194     def reindex_placex(self, db):
 195         """ Run the indexing step until all data in the placex has
 196             been processed. Indexing during updates can produce more data
 197             to index under some circumstances. That is why indexing may have
 198             to be run multiple times.
 199         """
 200         with db.cursor() as cur:
 201             while True:
 202                 self.run_update_script('index')
 203
 204                 cur.execute("SELECT 'a' FROM placex WHERE indexed_status != 0 LIMIT 1")
 205                 if cur.rowcount == 0:
 206                     return
 207
 208     def run_setup_script(self, *args, **kwargs):
 209         """ Run the Nominatim setup script with the given arguments.
 210         """
 211         self.run_nominatim_script('setup', *args, **kwargs)
 212
 213     def run_update_script(self, *args, **kwargs):
 214         """ Run the Nominatim update script with the given arguments.
 215         """
 216         self.run_nominatim_script('update', *args, **kwargs)
 217
 218     def run_nominatim_script(self, script, *args, **kwargs):
 219         """ Run one of the Nominatim utility scripts with the given arguments.
 220         """
 221         cmd = ['/usr/bin/env', 'php', '-Cq']
 222         cmd.append((Path(self.build_dir) / 'utils' / '{}.php'.format(script)).resolve())
 223         cmd.extend(['--' + x for x in args])
 224         for k, v in kwargs.items():
 225             cmd.extend(('--' + k.replace('_', '-'), str(v)))
 226
 227         if self.website_dir is not None:
 228             cwd = self.website_dir.name
 229         else:
 230             cwd = self.build_dir
 231
 232         run_script(cmd, cwd=cwd, env=self.test_env)
 233
 234     def copy_from_place(self, db):
 235         """ Copy data from place to the placex and location_property_osmline
 236             tables invoking the appropriate triggers.
 237         """
 238         self.run_setup_script('create-functions', 'create-partition-functions')
 239
 240         with db.cursor() as cur:
 241             cur.execute("""INSERT INTO placex (osm_type, osm_id, class, type,
 242                                                name, admin_level, address,
 243                                                extratags, geometry)
 244                              SELECT osm_type, osm_id, class, type,
 245                                     name, admin_level, address,
 246                                     extratags, geometry
 247                                FROM place
 248                                WHERE not (class='place' and type='houses' and osm_type='W')""")
 249             cur.execute("""INSERT INTO location_property_osmline (osm_id, address, linegeo)
 250                              SELECT osm_id, address, geometry
 251                                FROM place
 252                               WHERE class='place' and type='houses'
 253                                     and osm_type='W'
 254                                     and ST_GeometryType(geometry) = 'ST_LineString'""")