]> git.openstreetmap.org Git - nominatim.git/blob - test/bdd/environment.py
UI: update the switch-to-reverse link after each map click
[nominatim.git] / test / bdd / environment.py
1 from behave import *
2 import logging
3 import os
4 import psycopg2
5 import psycopg2.extras
6 import subprocess
7 import tempfile
8 from nose.tools import * # for assert functions
9 from sys import version_info as python_version
10
11 logger = logging.getLogger(__name__)
12
13 userconfig = {
14     'BUILDDIR' : os.path.join(os.path.split(__file__)[0], "../../build"),
15     'REMOVE_TEMPLATE' : False,
16     'KEEP_TEST_DB' : False,
17     'DB_HOST' : None,
18     'DB_PORT' : None,
19     'DB_USER' : None,
20     'DB_PASS' : None,
21     'TEMPLATE_DB' : 'test_template_nominatim',
22     'TEST_DB' : 'test_nominatim',
23     'API_TEST_DB' : 'test_api_nominatim',
24     'TEST_SETTINGS_FILE' : '/tmp/nominatim_settings.php',
25     'SERVER_MODULE_PATH' : None,
26     'PHPCOV' : False, # set to output directory to enable code coverage
27 }
28
29 use_step_matcher("re")
30
31 class NominatimEnvironment(object):
32     """ Collects all functions for the execution of Nominatim functions.
33     """
34
35     def __init__(self, config):
36         self.build_dir = os.path.abspath(config['BUILDDIR'])
37         self.src_dir = os.path.abspath(os.path.join(os.path.split(__file__)[0], "../.."))
38         self.db_host = config['DB_HOST']
39         self.db_port = config['DB_PORT']
40         self.db_user = config['DB_USER']
41         self.db_pass = config['DB_PASS']
42         self.template_db = config['TEMPLATE_DB']
43         self.test_db = config['TEST_DB']
44         self.api_test_db = config['API_TEST_DB']
45         self.server_module_path = config['SERVER_MODULE_PATH']
46         self.local_settings_file = config['TEST_SETTINGS_FILE']
47         self.reuse_template = not config['REMOVE_TEMPLATE']
48         self.keep_scenario_db = config['KEEP_TEST_DB']
49         self.code_coverage_path = config['PHPCOV']
50         self.code_coverage_id = 1
51         os.environ['NOMINATIM_SETTINGS'] = self.local_settings_file
52
53         self.template_db_done = False
54
55     def connect_database(self, dbname):
56         dbargs = {'database': dbname}
57         if self.db_host:
58             dbargs['host'] = self.db_host
59         if self.db_port:
60             dbargs['port'] = self.db_port
61         if self.db_user:
62             dbargs['user'] = self.db_user
63         if self.db_pass:
64             dbargs['password'] = self.db_pass
65         conn = psycopg2.connect(**dbargs)
66         return conn
67
68     def next_code_coverage_file(self):
69         fn = os.path.join(self.code_coverage_path, "%06d.cov" % self.code_coverage_id)
70         self.code_coverage_id += 1
71
72         return fn
73
74     def write_nominatim_config(self, dbname):
75         f = open(self.local_settings_file, 'w')
76         f.write("<?php\n  @define('CONST_Database_DSN', 'pgsql://%s:%s@%s%s/%s');\n" %
77                 (self.db_user if self.db_user else '',
78                  self.db_pass if self.db_pass else '',
79                  self.db_host if self.db_host else '',
80                  (':' + self.db_port) if self.db_port else '',
81                  dbname))
82         f.write("@define('CONST_Osm2pgsql_Flatnode_File', null);\n")
83         f.close()
84
85     def cleanup(self):
86         try:
87             os.remove(self.local_settings_file)
88         except OSError:
89             pass # ignore missing file
90
91     def db_drop_database(self, name):
92         conn = self.connect_database('postgres')
93         conn.set_isolation_level(0)
94         cur = conn.cursor()
95         cur.execute('DROP DATABASE IF EXISTS %s' % (name, ))
96         conn.close()
97
98     def setup_template_db(self):
99         if self.template_db_done:
100             return
101
102         self.template_db_done = True
103
104         if self.reuse_template:
105             # check that the template is there
106             conn = self.connect_database('postgres')
107             cur = conn.cursor()
108             cur.execute('select count(*) from pg_database where datname = %s',
109                         (self.template_db,))
110             if cur.fetchone()[0] == 1:
111                 return
112             conn.close()
113         else:
114             # just in case... make sure a previous table has been dropped
115             self.db_drop_database(self.template_db)
116
117         try:
118             # call the first part of database setup
119             self.write_nominatim_config(self.template_db)
120             self.run_setup_script('create-db', 'setup-db')
121             # remove external data to speed up indexing for tests
122             conn = self.connect_database(self.template_db)
123             cur = conn.cursor()
124             cur.execute("""select tablename from pg_tables
125                            where tablename in ('gb_postcode', 'us_postcode')""")
126             for t in cur:
127                 conn.cursor().execute('TRUNCATE TABLE %s' % (t[0],))
128             conn.commit()
129             conn.close()
130
131             # execute osm2pgsql import on an empty file to get the right tables
132             with tempfile.NamedTemporaryFile(dir='/tmp', suffix='.xml') as fd:
133                 fd.write(b'<osm version="0.6"></osm>')
134                 fd.flush()
135                 self.run_setup_script('import-data',
136                                       'ignore-errors',
137                                       'create-functions',
138                                       'create-tables',
139                                       'create-partition-tables',
140                                       'create-partition-functions',
141                                       'load-data',
142                                       'create-search-indices',
143                                       osm_file=fd.name,
144                                       osm2pgsql_cache='200')
145         except:
146             self.db_drop_database(self.template_db)
147             raise
148
149
150     def setup_api_db(self, context):
151         self.write_nominatim_config(self.api_test_db)
152
153     def setup_unknown_db(self, context):
154         self.write_nominatim_config('UNKNOWN_DATABASE_NAME')
155
156     def setup_db(self, context):
157         self.setup_template_db()
158         self.write_nominatim_config(self.test_db)
159         conn = self.connect_database(self.template_db)
160         conn.set_isolation_level(0)
161         cur = conn.cursor()
162         cur.execute('DROP DATABASE IF EXISTS %s' % (self.test_db, ))
163         cur.execute('CREATE DATABASE %s TEMPLATE = %s' % (self.test_db, self.template_db))
164         conn.close()
165         context.db = self.connect_database(self.test_db)
166         if python_version[0] < 3:
167             psycopg2.extras.register_hstore(context.db, globally=False, unicode=True)
168         else:
169             psycopg2.extras.register_hstore(context.db, globally=False)
170
171     def teardown_db(self, context):
172         if 'db' in context:
173             context.db.close()
174
175         if not self.keep_scenario_db:
176             self.db_drop_database(self.test_db)
177
178     def run_setup_script(self, *args, **kwargs):
179         if self.server_module_path:
180             kwargs = dict(kwargs)
181             kwargs['module_path'] = self.server_module_path
182         self.run_nominatim_script('setup', *args, **kwargs)
183
184     def run_update_script(self, *args, **kwargs):
185         self.run_nominatim_script('update', *args, **kwargs)
186
187     def run_nominatim_script(self, script, *args, **kwargs):
188         cmd = ['/usr/bin/env', 'php', '-Cq']
189         cmd.append(os.path.join(self.build_dir, 'utils', '%s.php' % script))
190         cmd.extend(['--%s' % x for x in args])
191         for k, v in kwargs.items():
192             cmd.extend(('--' + k.replace('_', '-'), str(v)))
193         proc = subprocess.Popen(cmd, cwd=self.build_dir,
194                                 stdout=subprocess.PIPE, stderr=subprocess.PIPE)
195         (outp, outerr) = proc.communicate()
196         logger.debug("run_nominatim_script: %s\n%s\n%s" % (cmd, outp, outerr))
197         assert (proc.returncode == 0), "Script '%s' failed:\n%s\n%s\n" % (script, outp, outerr)
198
199
200 class OSMDataFactory(object):
201
202     def __init__(self):
203         scriptpath = os.path.dirname(os.path.abspath(__file__))
204         self.scene_path = os.environ.get('SCENE_PATH',
205                            os.path.join(scriptpath, '..', 'scenes', 'data'))
206         self.scene_cache = {}
207         self.clear_grid()
208
209     def parse_geometry(self, geom, scene):
210         if geom.find(':') >= 0:
211             return "ST_SetSRID(%s, 4326)" % self.get_scene_geometry(scene, geom)
212
213         if geom.find(',') < 0:
214             out = "POINT(%s)" % self.mk_wkt_point(geom)
215         elif geom.find('(') < 0:
216             line = ','.join([self.mk_wkt_point(x) for x in geom.split(',')])
217             out = "LINESTRING(%s)" % line
218         else:
219             inner = geom.strip('() ')
220             line = ','.join([self.mk_wkt_point(x) for x in inner.split(',')])
221             out = "POLYGON((%s))" % line
222
223         return "ST_SetSRID('%s'::geometry, 4326)" % out
224
225     def mk_wkt_point(self, point):
226         geom = point.strip()
227         if geom.find(' ') >= 0:
228             return geom
229         else:
230             pt = self.grid_node(int(geom))
231             assert_is_not_none(pt, "Point not found in grid")
232             return "%f %f" % pt
233
234     def get_scene_geometry(self, default_scene, name):
235         geoms = []
236         for obj in name.split('+'):
237             oname = obj.strip()
238             if oname.startswith(':'):
239                 assert_is_not_none(default_scene, "You need to set a scene")
240                 defscene = self.load_scene(default_scene)
241                 wkt = defscene[oname[1:]]
242             else:
243                 scene, obj = oname.split(':', 2)
244                 scene_geoms = self.load_scene(scene)
245                 wkt = scene_geoms[obj]
246
247             geoms.append("'%s'::geometry" % wkt)
248
249         if len(geoms) == 1:
250             return geoms[0]
251         else:
252             return 'ST_LineMerge(ST_Collect(ARRAY[%s]))' % ','.join(geoms)
253
254     def load_scene(self, name):
255         if name in self.scene_cache:
256             return self.scene_cache[name]
257
258         scene = {}
259         with open(os.path.join(self.scene_path, "%s.wkt" % name), 'r') as fd:
260             for line in fd:
261                 if line.strip():
262                     obj, wkt = line.split('|', 2)
263                     scene[obj.strip()] = wkt.strip()
264             self.scene_cache[name] = scene
265
266         return scene
267
268     def clear_grid(self):
269         self.grid = {}
270
271     def add_grid_node(self, nodeid, x, y):
272         self.grid[nodeid] = (x, y)
273
274     def grid_node(self, nodeid):
275         return self.grid.get(nodeid)
276
277
278 def before_all(context):
279     # logging setup
280     context.config.setup_logging()
281     # set up -D options
282     for k,v in userconfig.items():
283         context.config.userdata.setdefault(k, v)
284     logging.debug('User config: %s' %(str(context.config.userdata)))
285     # Nominatim test setup
286     context.nominatim = NominatimEnvironment(context.config.userdata)
287     context.osm = OSMDataFactory()
288
289 def after_all(context):
290     context.nominatim.cleanup()
291
292
293 def before_scenario(context, scenario):
294     if 'DB' in context.tags:
295         context.nominatim.setup_db(context)
296     elif 'APIDB' in context.tags:
297         context.nominatim.setup_api_db(context)
298     elif 'UNKNOWNDB' in context.tags:
299         context.nominatim.setup_unknown_db(context)
300     context.scene = None
301
302 def after_scenario(context, scenario):
303     if 'DB' in context.tags:
304         context.nominatim.teardown_db(context)