test/bdd/steps/nominatim_environment.py

   1 from pathlib import Path
   2 import sys
   3 import tempfile
   4
   5 import psycopg2
   6 import psycopg2.extras
   7
   8 sys.path.insert(1, str((Path(__file__) / '..' / '..' / '..' / '..').resolve()))
   9
  10 from nominatim.config import Configuration
  11 from steps.utils import run_script
  12
  13 class NominatimEnvironment:
  14     """ Collects all functions for the execution of Nominatim functions.
  15     """
  16
  17     def __init__(self, config):
  18         self.build_dir = Path(config['BUILDDIR']).resolve()
  19         self.src_dir = (Path(__file__) / '..' / '..' / '..' / '..').resolve()
  20         self.db_host = config['DB_HOST']
  21         self.db_port = config['DB_PORT']
  22         self.db_user = config['DB_USER']
  23         self.db_pass = config['DB_PASS']
  24         self.template_db = config['TEMPLATE_DB']
  25         self.test_db = config['TEST_DB']
  26         self.api_test_db = config['API_TEST_DB']
  27         self.api_test_file = config['API_TEST_FILE']
  28         self.server_module_path = config['SERVER_MODULE_PATH']
  29         self.reuse_template = not config['REMOVE_TEMPLATE']
  30         self.keep_scenario_db = config['KEEP_TEST_DB']
  31         self.code_coverage_path = config['PHPCOV']
  32         self.code_coverage_id = 1
  33
  34         self.default_config = Configuration(None, self.src_dir / 'settings').get_os_env()
  35         self.test_env = None
  36         self.template_db_done = False
  37         self.api_db_done = False
  38         self.website_dir = None
  39
  40     def connect_database(self, dbname):
  41         """ Return a connection to the database with the given name.
  42             Uses configured host, user and port.
  43         """
  44         dbargs = {'database': dbname}
  45         if self.db_host:
  46             dbargs['host'] = self.db_host
  47         if self.db_port:
  48             dbargs['port'] = self.db_port
  49         if self.db_user:
  50             dbargs['user'] = self.db_user
  51         if self.db_pass:
  52             dbargs['password'] = self.db_pass
  53         conn = psycopg2.connect(**dbargs)
  54         return conn
  55
  56     def next_code_coverage_file(self):
  57         """ Generate the next name for a coverage file.
  58         """
  59         fn = Path(self.code_coverage_path) / "{:06d}.cov".format(self.code_coverage_id)
  60         self.code_coverage_id += 1
  61
  62         return fn.resolve()
  63
  64     def write_nominatim_config(self, dbname):
  65         """ Set up a custom test configuration that connects to the given
  66             database. This sets up the environment variables so that they can
  67             be picked up by dotenv and creates a project directory with the
  68             appropriate website scripts.
  69         """
  70         dsn = 'pgsql:dbname={}'.format(dbname)
  71         if self.db_host:
  72             dsn += ';host=' + self.db_host
  73         if self.db_port:
  74             dsn += ';port=' + self.db_port
  75         if self.db_user:
  76             dsn += ';user=' + self.db_user
  77         if self.db_pass:
  78             dsn += ';password=' + self.db_pass
  79
  80         if self.website_dir is not None \
  81            and self.test_env is not None \
  82            and dsn == self.test_env['NOMINATIM_DATABASE_DSN']:
  83             return # environment already set uo
  84
  85         self.test_env = dict(self.default_config)
  86         self.test_env['NOMINATIM_DATABASE_DSN'] = dsn
  87         self.test_env['NOMINATIM_FLATNODE_FILE'] = ''
  88         self.test_env['NOMINATIM_IMPORT_STYLE'] = 'full'
  89         self.test_env['NOMINATIM_USE_US_TIGER_DATA'] = 'yes'
  90         self.test_env['NOMINATIM_DATADIR'] = self.src_dir
  91         self.test_env['NOMINATIM_BINDIR'] = self.src_dir / 'utils'
  92         self.test_env['NOMINATIM_DATABASE_MODULE_PATH'] = self.build_dir / 'module'
  93         self.test_env['NOMINATIM_OSM2PGSQL_BINARY'] = self.build_dir / 'osm2pgsql' / 'osm2pgsql'
  94
  95         if self.server_module_path:
  96             self.test_env['NOMINATIM_DATABASE_MODULE_PATH'] = self.server_module_path
  97
  98         if self.website_dir is not None:
  99             self.website_dir.cleanup()
 100
 101         self.website_dir = tempfile.TemporaryDirectory()
 102         self.run_setup_script('setup-website')
 103
 104
 105     def db_drop_database(self, name):
 106         """ Drop the database with the given name.
 107         """
 108         conn = self.connect_database('postgres')
 109         conn.set_isolation_level(0)
 110         cur = conn.cursor()
 111         cur.execute('DROP DATABASE IF EXISTS {}'.format(name))
 112         conn.close()
 113
 114     def setup_template_db(self):
 115         """ Setup a template database that already contains common test data.
 116             Having a template database speeds up tests considerably but at
 117             the price that the tests sometimes run with stale data.
 118         """
 119         if self.template_db_done:
 120             return
 121
 122         self.template_db_done = True
 123
 124         if self._reuse_or_drop_db(self.template_db):
 125             return
 126
 127         try:
 128             # call the first part of database setup
 129             self.write_nominatim_config(self.template_db)
 130             self.run_setup_script('create-db', 'setup-db')
 131             # remove external data to speed up indexing for tests
 132             conn = self.connect_database(self.template_db)
 133             cur = conn.cursor()
 134             cur.execute("""select tablename from pg_tables
 135                            where tablename in ('gb_postcode', 'us_postcode')""")
 136             for t in cur:
 137                 conn.cursor().execute('TRUNCATE TABLE {}'.format(t[0]))
 138             conn.commit()
 139             conn.close()
 140
 141             # execute osm2pgsql import on an empty file to get the right tables
 142             with tempfile.NamedTemporaryFile(dir='/tmp', suffix='.xml') as fd:
 143                 fd.write(b'<osm version="0.6"></osm>')
 144                 fd.flush()
 145                 self.run_setup_script('import-data',
 146                                       'ignore-errors',
 147                                       'create-functions',
 148                                       'create-tables',
 149                                       'create-partition-tables',
 150                                       'create-partition-functions',
 151                                       'load-data',
 152                                       'create-search-indices',
 153                                       osm_file=fd.name,
 154                                       osm2pgsql_cache='200')
 155         except:
 156             self.db_drop_database(self.template_db)
 157             raise
 158
 159
 160     def setup_api_db(self):
 161         """ Setup a test against the API test database.
 162         """
 163         self.write_nominatim_config(self.api_test_db)
 164
 165         if self.api_db_done:
 166             return
 167
 168         self.api_db_done = True
 169
 170         if self._reuse_or_drop_db(self.api_test_db):
 171             return
 172
 173         testdata = Path('__file__') / '..' / '..' / 'testdb'
 174         self.test_env['NOMINATIM_TIGER_DATA_PATH'] = str((testdata / 'tiger').resolve())
 175         self.test_env['NOMINATIM_WIKIPEDIA_DATA_PATH'] = str(testdata.resolve())
 176
 177         try:
 178             self.run_setup_script('all', osm_file=self.api_test_file)
 179             self.run_setup_script('import-tiger-data')
 180
 181             phrase_file = str((testdata / 'specialphrases_testdb.sql').resolve())
 182             run_script(['psql', '-d', self.api_test_db, '-f', phrase_file])
 183         except:
 184             self.db_drop_database(self.api_test_db)
 185             raise
 186
 187
 188     def setup_unknown_db(self):
 189         """ Setup a test against a non-existing database.
 190         """
 191         self.write_nominatim_config('UNKNOWN_DATABASE_NAME')
 192
 193     def setup_db(self, context):
 194         """ Setup a test against a fresh, empty test database.
 195         """
 196         self.setup_template_db()
 197         self.write_nominatim_config(self.test_db)
 198         conn = self.connect_database(self.template_db)
 199         conn.set_isolation_level(0)
 200         cur = conn.cursor()
 201         cur.execute('DROP DATABASE IF EXISTS {}'.format(self.test_db))
 202         cur.execute('CREATE DATABASE {} TEMPLATE = {}'.format(self.test_db, self.template_db))
 203         conn.close()
 204         context.db = self.connect_database(self.test_db)
 205         context.db.autocommit = True
 206         psycopg2.extras.register_hstore(context.db, globally=False)
 207
 208     def teardown_db(self, context):
 209         """ Remove the test database, if it exists.
 210         """
 211         if 'db' in context:
 212             context.db.close()
 213
 214         if not self.keep_scenario_db:
 215             self.db_drop_database(self.test_db)
 216
 217     def _reuse_or_drop_db(self, name):
 218         """ Check for the existance of the given DB. If reuse is enabled,
 219             then the function checks for existance and returns True if the
 220             database is already there. Otherwise an existing database is
 221             dropped and always false returned.
 222         """
 223         if self.reuse_template:
 224             conn = self.connect_database('postgres')
 225             with conn.cursor() as cur:
 226                 cur.execute('select count(*) from pg_database where datname = %s',
 227                             (name,))
 228                 if cur.fetchone()[0] == 1:
 229                     return True
 230             conn.close()
 231         else:
 232             self.db_drop_database(name)
 233
 234         return False
 235
 236     def reindex_placex(self, db):
 237         """ Run the indexing step until all data in the placex has
 238             been processed. Indexing during updates can produce more data
 239             to index under some circumstances. That is why indexing may have
 240             to be run multiple times.
 241         """
 242         with db.cursor() as cur:
 243             while True:
 244                 self.run_update_script('index')
 245
 246                 cur.execute("SELECT 'a' FROM placex WHERE indexed_status != 0 LIMIT 1")
 247                 if cur.rowcount == 0:
 248                     return
 249
 250     def run_setup_script(self, *args, **kwargs):
 251         """ Run the Nominatim setup script with the given arguments.
 252         """
 253         self.run_nominatim_script('setup', *args, **kwargs)
 254
 255     def run_update_script(self, *args, **kwargs):
 256         """ Run the Nominatim update script with the given arguments.
 257         """
 258         self.run_nominatim_script('update', *args, **kwargs)
 259
 260     def run_nominatim_script(self, script, *args, **kwargs):
 261         """ Run one of the Nominatim utility scripts with the given arguments.
 262         """
 263         cmd = ['/usr/bin/env', 'php', '-Cq']
 264         cmd.append((Path(self.src_dir) / 'lib' / 'admin' / '{}.php'.format(script)).resolve())
 265         cmd.extend(['--' + x for x in args])
 266         for k, v in kwargs.items():
 267             cmd.extend(('--' + k.replace('_', '-'), str(v)))
 268
 269         if self.website_dir is not None:
 270             cwd = self.website_dir.name
 271         else:
 272             cwd = None
 273
 274         run_script(cmd, cwd=cwd, env=self.test_env)
 275
 276     def copy_from_place(self, db):
 277         """ Copy data from place to the placex and location_property_osmline
 278             tables invoking the appropriate triggers.
 279         """
 280         self.run_setup_script('create-functions', 'create-partition-functions')
 281
 282         with db.cursor() as cur:
 283             cur.execute("""INSERT INTO placex (osm_type, osm_id, class, type,
 284                                                name, admin_level, address,
 285                                                extratags, geometry)
 286                              SELECT osm_type, osm_id, class, type,
 287                                     name, admin_level, address,
 288                                     extratags, geometry
 289                                FROM place
 290                                WHERE not (class='place' and type='houses' and osm_type='W')""")
 291             cur.execute("""INSERT INTO location_property_osmline (osm_id, address, linegeo)
 292                              SELECT osm_id, address, geometry
 293                                FROM place
 294                               WHERE class='place' and type='houses'
 295                                     and osm_type='W'
 296                                     and ST_GeometryType(geometry) = 'ST_LineString'""")