]> git.openstreetmap.org Git - nominatim.git/blob - test/python/api/search/test_search_places.py
remove support for unindexed tokens
[nominatim.git] / test / python / api / search / test_search_places.py
1 # SPDX-License-Identifier: GPL-3.0-or-later
2 #
3 # This file is part of Nominatim. (https://nominatim.org)
4 #
5 # Copyright (C) 2024 by the Nominatim developer community.
6 # For a full list of authors see the git log.
7 """
8 Tests for running the generic place searcher.
9 """
10 import json
11
12 import pytest
13
14 import nominatim_api as napi
15 from nominatim_api.types import SearchDetails
16 from nominatim_api.search.db_searches import PlaceSearch
17 from nominatim_api.search.db_search_fields import WeightedStrings, WeightedCategories,\
18                                                   FieldLookup, FieldRanking, RankedTokens
19 from nominatim_api.search.db_search_lookups import LookupAll, LookupAny, Restrict
20
21 APIOPTIONS = ['search']
22
23 def run_search(apiobj, frontend, global_penalty, lookup, ranking, count=2,
24                hnrs=[], pcs=[], ccodes=[], quals=[],
25                details=SearchDetails()):
26     class MySearchData:
27         penalty = global_penalty
28         postcodes = WeightedStrings(pcs, [0.0] * len(pcs))
29         countries = WeightedStrings(ccodes, [0.0] * len(ccodes))
30         housenumbers = WeightedStrings(hnrs, [0.0] * len(hnrs))
31         qualifiers = WeightedCategories(quals, [0.0] * len(quals))
32         lookups = lookup
33         rankings = ranking
34
35     search = PlaceSearch(0.0, MySearchData(), count)
36
37     if frontend is None:
38         api = apiobj
39     else:
40         api = frontend(apiobj, options=APIOPTIONS)
41
42     async def run():
43         async with api._async_api.begin() as conn:
44             return await search.lookup(conn, details)
45
46     results = api._loop.run_until_complete(run())
47     results.sort(key=lambda r: r.accuracy)
48
49     return results
50
51
52 class TestNameOnlySearches:
53
54     @pytest.fixture(autouse=True)
55     def fill_database(self, apiobj):
56         apiobj.add_placex(place_id=100, country_code='us',
57                           centroid=(5.6, 4.3))
58         apiobj.add_search_name(100, names=[1,2,10,11], country_code='us',
59                                centroid=(5.6, 4.3))
60         apiobj.add_placex(place_id=101, country_code='mx',
61                           centroid=(-10.3, 56.9))
62         apiobj.add_search_name(101, names=[1,2,20,21], country_code='mx',
63                                centroid=(-10.3, 56.9))
64
65
66     @pytest.mark.parametrize('lookup_type', [LookupAll, Restrict])
67     @pytest.mark.parametrize('rank,res', [([10], [100, 101]),
68                                           ([20], [101, 100])])
69     def test_lookup_all_match(self, apiobj, frontend, lookup_type, rank, res):
70         lookup = FieldLookup('name_vector', [1,2], lookup_type)
71         ranking = FieldRanking('name_vector', 0.4, [RankedTokens(0.0, rank)])
72
73         results = run_search(apiobj, frontend, 0.1, [lookup], [ranking])
74
75         assert [r.place_id for r in results] == res
76
77
78     @pytest.mark.parametrize('lookup_type', [LookupAll, Restrict])
79     def test_lookup_all_partial_match(self, apiobj, frontend, lookup_type):
80         lookup = FieldLookup('name_vector', [1,20], lookup_type)
81         ranking = FieldRanking('name_vector', 0.4, [RankedTokens(0.0, [21])])
82
83         results = run_search(apiobj, frontend, 0.1, [lookup], [ranking])
84
85         assert len(results) == 1
86         assert results[0].place_id == 101
87
88     @pytest.mark.parametrize('rank,res', [([10], [100, 101]),
89                                           ([20], [101, 100])])
90     def test_lookup_any_match(self, apiobj, frontend, rank, res):
91         lookup = FieldLookup('name_vector', [11,21], LookupAny)
92         ranking = FieldRanking('name_vector', 0.4, [RankedTokens(0.0, rank)])
93
94         results = run_search(apiobj, frontend, 0.1, [lookup], [ranking])
95
96         assert [r.place_id for r in results] == res
97
98
99     def test_lookup_any_partial_match(self, apiobj, frontend):
100         lookup = FieldLookup('name_vector', [20], LookupAll)
101         ranking = FieldRanking('name_vector', 0.4, [RankedTokens(0.0, [21])])
102
103         results = run_search(apiobj, frontend, 0.1, [lookup], [ranking])
104
105         assert len(results) == 1
106         assert results[0].place_id == 101
107
108
109     @pytest.mark.parametrize('cc,res', [('us', 100), ('mx', 101)])
110     def test_lookup_restrict_country(self, apiobj, frontend, cc, res):
111         lookup = FieldLookup('name_vector', [1,2], LookupAll)
112         ranking = FieldRanking('name_vector', 0.4, [RankedTokens(0.0, [10])])
113
114         results = run_search(apiobj, frontend, 0.1, [lookup], [ranking], ccodes=[cc])
115
116         assert [r.place_id for r in results] == [res]
117
118
119     def test_lookup_restrict_placeid(self, apiobj, frontend):
120         lookup = FieldLookup('name_vector', [1,2], LookupAll)
121         ranking = FieldRanking('name_vector', 0.4, [RankedTokens(0.0, [10])])
122
123         results = run_search(apiobj, frontend, 0.1, [lookup], [ranking],
124                              details=SearchDetails(excluded=[101]))
125
126         assert [r.place_id for r in results] == [100]
127
128
129     @pytest.mark.parametrize('geom', [napi.GeometryFormat.GEOJSON,
130                                       napi.GeometryFormat.KML,
131                                       napi.GeometryFormat.SVG,
132                                       napi.GeometryFormat.TEXT])
133     def test_return_geometries(self, apiobj, frontend, geom):
134         lookup = FieldLookup('name_vector', [20], LookupAll)
135         ranking = FieldRanking('name_vector', 0.4, [RankedTokens(0.0, [21])])
136
137         results = run_search(apiobj, frontend, 0.1, [lookup], [ranking],
138                              details=SearchDetails(geometry_output=geom))
139
140         assert geom.name.lower() in results[0].geometry
141
142
143     @pytest.mark.parametrize('factor,npoints', [(0.0, 3), (1.0, 2)])
144     def test_return_simplified_geometry(self, apiobj, frontend, factor, npoints):
145         apiobj.add_placex(place_id=333, country_code='us',
146                           centroid=(9.0, 9.0),
147                           geometry='LINESTRING(8.9 9.0, 9.0 9.0, 9.1 9.0)')
148         apiobj.add_search_name(333, names=[55], country_code='us',
149                                centroid=(5.6, 4.3))
150
151         lookup = FieldLookup('name_vector', [55], LookupAll)
152         ranking = FieldRanking('name_vector', 0.4, [RankedTokens(0.0, [21])])
153
154         results = run_search(apiobj, frontend, 0.1, [lookup], [ranking],
155                              details=SearchDetails(geometry_output=napi.GeometryFormat.GEOJSON,
156                                                    geometry_simplification=factor))
157
158         assert len(results) == 1
159         result = results[0]
160         geom = json.loads(result.geometry['geojson'])
161
162         assert result.place_id == 333
163         assert len(geom['coordinates']) == npoints
164
165
166     @pytest.mark.parametrize('viewbox', ['5.0,4.0,6.0,5.0', '5.7,4.0,6.0,5.0'])
167     @pytest.mark.parametrize('wcount,rids', [(2, [100, 101]), (20000, [100])])
168     def test_prefer_viewbox(self, apiobj, frontend, viewbox, wcount, rids):
169         lookup = FieldLookup('name_vector', [1, 2], LookupAll)
170         ranking = FieldRanking('name_vector', 0.2, [RankedTokens(0.0, [21])])
171
172         api = frontend(apiobj, options=APIOPTIONS)
173         results = run_search(api, None, 0.1, [lookup], [ranking])
174         assert [r.place_id for r in results] == [101, 100]
175
176         results = run_search(api, None, 0.1, [lookup], [ranking], count=wcount,
177                              details=SearchDetails.from_kwargs({'viewbox': viewbox}))
178         assert [r.place_id for r in results] == rids
179
180
181     @pytest.mark.parametrize('viewbox', ['5.0,4.0,6.0,5.0', '5.55,4.27,5.62,4.31'])
182     def test_force_viewbox(self, apiobj, frontend, viewbox):
183         lookup = FieldLookup('name_vector', [1, 2], LookupAll)
184
185         details=SearchDetails.from_kwargs({'viewbox': viewbox,
186                                            'bounded_viewbox': True})
187
188         results = run_search(apiobj, frontend, 0.1, [lookup], [], details=details)
189         assert [r.place_id for r in results] == [100]
190
191
192     def test_prefer_near(self, apiobj, frontend):
193         lookup = FieldLookup('name_vector', [1, 2], LookupAll)
194         ranking = FieldRanking('name_vector', 0.4, [RankedTokens(0.0, [21])])
195
196         api = frontend(apiobj, options=APIOPTIONS)
197         results = run_search(api, None, 0.1, [lookup], [ranking])
198         assert [r.place_id for r in results] == [101, 100]
199
200         results = run_search(api, None, 0.1, [lookup], [ranking],
201                              details=SearchDetails.from_kwargs({'near': '5.6,4.3'}))
202         results.sort(key=lambda r: -r.importance)
203         assert [r.place_id for r in results] == [100, 101]
204
205
206     @pytest.mark.parametrize('radius', [0.09, 0.11])
207     def test_force_near(self, apiobj, frontend, radius):
208         lookup = FieldLookup('name_vector', [1, 2], LookupAll)
209
210         details=SearchDetails.from_kwargs({'near': '5.6,4.3',
211                                            'near_radius': radius})
212
213         results = run_search(apiobj, frontend, 0.1, [lookup], [], details=details)
214
215         assert [r.place_id for r in results] == [100]
216
217
218 class TestStreetWithHousenumber:
219
220     @pytest.fixture(autouse=True)
221     def fill_database(self, apiobj):
222         apiobj.add_placex(place_id=1, class_='place', type='house',
223                           parent_place_id=1000,
224                           housenumber='20 a', country_code='es')
225         apiobj.add_placex(place_id=2, class_='place', type='house',
226                           parent_place_id=1000,
227                           housenumber='21;22', country_code='es')
228         apiobj.add_placex(place_id=1000, class_='highway', type='residential',
229                           rank_search=26, rank_address=26,
230                           country_code='es')
231         apiobj.add_search_name(1000, names=[1,2,10,11],
232                                search_rank=26, address_rank=26,
233                                country_code='es')
234         apiobj.add_placex(place_id=91, class_='place', type='house',
235                           parent_place_id=2000,
236                           housenumber='20', country_code='pt')
237         apiobj.add_placex(place_id=92, class_='place', type='house',
238                           parent_place_id=2000,
239                           housenumber='22', country_code='pt')
240         apiobj.add_placex(place_id=93, class_='place', type='house',
241                           parent_place_id=2000,
242                           housenumber='24', country_code='pt')
243         apiobj.add_placex(place_id=2000, class_='highway', type='residential',
244                           rank_search=26, rank_address=26,
245                           country_code='pt')
246         apiobj.add_search_name(2000, names=[1,2,20,21],
247                                search_rank=26, address_rank=26,
248                                country_code='pt')
249
250
251     @pytest.mark.parametrize('hnr,res', [('20', [91, 1]), ('20 a', [1]),
252                                          ('21', [2]), ('22', [2, 92]),
253                                          ('24', [93]), ('25', [])])
254     def test_lookup_by_single_housenumber(self, apiobj, frontend, hnr, res):
255         lookup = FieldLookup('name_vector', [1,2], LookupAll)
256         ranking = FieldRanking('name_vector', 0.3, [RankedTokens(0.0, [10])])
257
258         results = run_search(apiobj, frontend, 0.1, [lookup], [ranking], hnrs=[hnr])
259
260         assert [r.place_id for r in results] == res + [1000, 2000]
261
262
263     @pytest.mark.parametrize('cc,res', [('es', [2, 1000]), ('pt', [92, 2000])])
264     def test_lookup_with_country_restriction(self, apiobj, frontend, cc, res):
265         lookup = FieldLookup('name_vector', [1,2], LookupAll)
266         ranking = FieldRanking('name_vector', 0.3, [RankedTokens(0.0, [10])])
267
268         results = run_search(apiobj, frontend, 0.1, [lookup], [ranking], hnrs=['22'],
269                              ccodes=[cc])
270
271         assert [r.place_id for r in results] == res
272
273
274     def test_lookup_exclude_housenumber_placeid(self, apiobj, frontend):
275         lookup = FieldLookup('name_vector', [1,2], LookupAll)
276         ranking = FieldRanking('name_vector', 0.3, [RankedTokens(0.0, [10])])
277
278         results = run_search(apiobj, frontend, 0.1, [lookup], [ranking], hnrs=['22'],
279                              details=SearchDetails(excluded=[92]))
280
281         assert [r.place_id for r in results] == [2, 1000, 2000]
282
283
284     def test_lookup_exclude_street_placeid(self, apiobj, frontend):
285         lookup = FieldLookup('name_vector', [1,2], LookupAll)
286         ranking = FieldRanking('name_vector', 0.3, [RankedTokens(0.0, [10])])
287
288         results = run_search(apiobj, frontend, 0.1, [lookup], [ranking], hnrs=['22'],
289                              details=SearchDetails(excluded=[1000]))
290
291         assert [r.place_id for r in results] == [2, 92, 2000]
292
293
294     def test_lookup_only_house_qualifier(self, apiobj, frontend):
295         lookup = FieldLookup('name_vector', [1,2], LookupAll)
296         ranking = FieldRanking('name_vector', 0.3, [RankedTokens(0.0, [10])])
297
298         results = run_search(apiobj, frontend, 0.1, [lookup], [ranking], hnrs=['22'],
299                              quals=[('place', 'house')])
300
301         assert [r.place_id for r in results] == [2, 92]
302
303
304     def test_lookup_only_street_qualifier(self, apiobj, frontend):
305         lookup = FieldLookup('name_vector', [1,2], LookupAll)
306         ranking = FieldRanking('name_vector', 0.3, [RankedTokens(0.0, [10])])
307
308         results = run_search(apiobj, frontend, 0.1, [lookup], [ranking], hnrs=['22'],
309                              quals=[('highway', 'residential')])
310
311         assert [r.place_id for r in results] == [1000, 2000]
312
313
314     @pytest.mark.parametrize('rank,found', [(26, True), (27, False), (30, False)])
315     def test_lookup_min_rank(self, apiobj, frontend, rank, found):
316         lookup = FieldLookup('name_vector', [1,2], LookupAll)
317         ranking = FieldRanking('name_vector', 0.3, [RankedTokens(0.0, [10])])
318
319         results = run_search(apiobj, frontend, 0.1, [lookup], [ranking], hnrs=['22'],
320                              details=SearchDetails(min_rank=rank))
321
322         assert [r.place_id for r in results] == ([2, 92, 1000, 2000] if found else [2, 92])
323
324
325     @pytest.mark.parametrize('geom', [napi.GeometryFormat.GEOJSON,
326                                       napi.GeometryFormat.KML,
327                                       napi.GeometryFormat.SVG,
328                                       napi.GeometryFormat.TEXT])
329     def test_return_geometries(self, apiobj, frontend, geom):
330         lookup = FieldLookup('name_vector', [1, 2], LookupAll)
331
332         results = run_search(apiobj, frontend, 0.1, [lookup], [], hnrs=['20', '21', '22'],
333                              details=SearchDetails(geometry_output=geom))
334
335         assert results
336         assert all(geom.name.lower() in r.geometry for r in results)
337
338
339 def test_very_large_housenumber(apiobj, frontend):
340     apiobj.add_placex(place_id=93, class_='place', type='house',
341                       parent_place_id=2000,
342                       housenumber='2467463524544', country_code='pt')
343     apiobj.add_placex(place_id=2000, class_='highway', type='residential',
344                       rank_search=26, rank_address=26,
345                       country_code='pt')
346     apiobj.add_search_name(2000, names=[1,2],
347                            search_rank=26, address_rank=26,
348                            country_code='pt')
349
350     lookup = FieldLookup('name_vector', [1, 2], LookupAll)
351
352     results = run_search(apiobj, frontend, 0.1, [lookup], [], hnrs=['2467463524544'],
353                          details=SearchDetails())
354
355     assert results
356     assert [r.place_id for r in results] == [93, 2000]
357
358
359 @pytest.mark.parametrize('wcount,rids', [(2, [990, 991]), (30000, [990])])
360 def test_name_and_postcode(apiobj, frontend, wcount, rids):
361     apiobj.add_placex(place_id=990, class_='highway', type='service',
362                       rank_search=27, rank_address=27,
363                       postcode='11225',
364                       centroid=(10.0, 10.0),
365                       geometry='LINESTRING(9.995 10, 10.005 10)')
366     apiobj.add_search_name(990, names=[111], centroid=(10.0, 10.0),
367                            search_rank=27, address_rank=27)
368     apiobj.add_placex(place_id=991, class_='highway', type='service',
369                       rank_search=27, rank_address=27,
370                       postcode='11221',
371                       centroid=(10.3, 10.3),
372                       geometry='LINESTRING(9.995 10.3, 10.005 10.3)')
373     apiobj.add_search_name(991, names=[111], centroid=(10.3, 10.3),
374                            search_rank=27, address_rank=27)
375     apiobj.add_postcode(place_id=100, country_code='ch', postcode='11225',
376                         geometry='POINT(10 10)')
377
378     lookup = FieldLookup('name_vector', [111], LookupAll)
379
380     results = run_search(apiobj, frontend, 0.1, [lookup], [], pcs=['11225'], count=wcount,
381                          details=SearchDetails())
382
383     assert results
384     assert [r.place_id for r in results] == rids
385
386
387 class TestInterpolations:
388
389     @pytest.fixture(autouse=True)
390     def fill_database(self, apiobj):
391         apiobj.add_placex(place_id=990, class_='highway', type='service',
392                           rank_search=27, rank_address=27,
393                           centroid=(10.0, 10.0),
394                           geometry='LINESTRING(9.995 10, 10.005 10)')
395         apiobj.add_search_name(990, names=[111],
396                                search_rank=27, address_rank=27)
397         apiobj.add_placex(place_id=991, class_='place', type='house',
398                           parent_place_id=990,
399                           rank_search=30, rank_address=30,
400                           housenumber='23',
401                           centroid=(10.0, 10.00002))
402         apiobj.add_osmline(place_id=992,
403                            parent_place_id=990,
404                            startnumber=21, endnumber=29, step=2,
405                            centroid=(10.0, 10.00001),
406                            geometry='LINESTRING(9.995 10.00001, 10.005 10.00001)')
407
408
409     @pytest.mark.parametrize('hnr,res', [('21', [992]), ('22', []), ('23', [991])])
410     def test_lookup_housenumber(self, apiobj, frontend, hnr, res):
411         lookup = FieldLookup('name_vector', [111], LookupAll)
412
413         results = run_search(apiobj, frontend, 0.1, [lookup], [], hnrs=[hnr])
414
415         assert [r.place_id for r in results] == res + [990]
416
417
418     @pytest.mark.parametrize('geom', [napi.GeometryFormat.GEOJSON,
419                                       napi.GeometryFormat.KML,
420                                       napi.GeometryFormat.SVG,
421                                       napi.GeometryFormat.TEXT])
422     def test_osmline_with_geometries(self, apiobj, frontend, geom):
423         lookup = FieldLookup('name_vector', [111], LookupAll)
424
425         results = run_search(apiobj, frontend, 0.1, [lookup], [], hnrs=['21'],
426                              details=SearchDetails(geometry_output=geom))
427
428         assert results[0].place_id == 992
429         assert geom.name.lower() in results[0].geometry
430
431
432
433 class TestTiger:
434
435     @pytest.fixture(autouse=True)
436     def fill_database(self, apiobj):
437         apiobj.add_placex(place_id=990, class_='highway', type='service',
438                           rank_search=27, rank_address=27,
439                           country_code='us',
440                           centroid=(10.0, 10.0),
441                           geometry='LINESTRING(9.995 10, 10.005 10)')
442         apiobj.add_search_name(990, names=[111], country_code='us',
443                                search_rank=27, address_rank=27)
444         apiobj.add_placex(place_id=991, class_='place', type='house',
445                           parent_place_id=990,
446                           rank_search=30, rank_address=30,
447                           housenumber='23',
448                           country_code='us',
449                           centroid=(10.0, 10.00002))
450         apiobj.add_tiger(place_id=992,
451                          parent_place_id=990,
452                          startnumber=21, endnumber=29, step=2,
453                          centroid=(10.0, 10.00001),
454                          geometry='LINESTRING(9.995 10.00001, 10.005 10.00001)')
455
456
457     @pytest.mark.parametrize('hnr,res', [('21', [992]), ('22', []), ('23', [991])])
458     def test_lookup_housenumber(self, apiobj, frontend, hnr, res):
459         lookup = FieldLookup('name_vector', [111], LookupAll)
460
461         results = run_search(apiobj, frontend, 0.1, [lookup], [], hnrs=[hnr])
462
463         assert [r.place_id for r in results] == res + [990]
464
465
466     @pytest.mark.parametrize('geom', [napi.GeometryFormat.GEOJSON,
467                                       napi.GeometryFormat.KML,
468                                       napi.GeometryFormat.SVG,
469                                       napi.GeometryFormat.TEXT])
470     def test_tiger_with_geometries(self, apiobj, frontend, geom):
471         lookup = FieldLookup('name_vector', [111], LookupAll)
472
473         results = run_search(apiobj, frontend, 0.1, [lookup], [], hnrs=['21'],
474                              details=SearchDetails(geometry_output=geom))
475
476         assert results[0].place_id == 992
477         assert geom.name.lower() in results[0].geometry
478
479
480 class TestLayersRank30:
481
482     @pytest.fixture(autouse=True)
483     def fill_database(self, apiobj):
484         apiobj.add_placex(place_id=223, class_='place', type='house',
485                           housenumber='1',
486                           rank_address=30,
487                           rank_search=30)
488         apiobj.add_search_name(223, names=[34],
489                                importance=0.0009,
490                                address_rank=30, search_rank=30)
491         apiobj.add_placex(place_id=224, class_='amenity', type='toilet',
492                           rank_address=30,
493                           rank_search=30)
494         apiobj.add_search_name(224, names=[34],
495                                importance=0.0008,
496                                address_rank=30, search_rank=30)
497         apiobj.add_placex(place_id=225, class_='man_made', type='tower',
498                           rank_address=0,
499                           rank_search=30)
500         apiobj.add_search_name(225, names=[34],
501                                importance=0.0007,
502                                address_rank=0, search_rank=30)
503         apiobj.add_placex(place_id=226, class_='railway', type='station',
504                           rank_address=0,
505                           rank_search=30)
506         apiobj.add_search_name(226, names=[34],
507                                importance=0.0006,
508                                address_rank=0, search_rank=30)
509         apiobj.add_placex(place_id=227, class_='natural', type='cave',
510                           rank_address=0,
511                           rank_search=30)
512         apiobj.add_search_name(227, names=[34],
513                                importance=0.0005,
514                                address_rank=0, search_rank=30)
515
516
517     @pytest.mark.parametrize('layer,res', [(napi.DataLayer.ADDRESS, [223]),
518                                            (napi.DataLayer.POI, [224]),
519                                            (napi.DataLayer.ADDRESS | napi.DataLayer.POI, [223, 224]),
520                                            (napi.DataLayer.MANMADE, [225]),
521                                            (napi.DataLayer.RAILWAY, [226]),
522                                            (napi.DataLayer.NATURAL, [227]),
523                                            (napi.DataLayer.MANMADE | napi.DataLayer.NATURAL, [225, 227]),
524                                            (napi.DataLayer.MANMADE | napi.DataLayer.RAILWAY, [225, 226])])
525     def test_layers_rank30(self, apiobj, frontend, layer, res):
526         lookup = FieldLookup('name_vector', [34], LookupAny)
527
528         results = run_search(apiobj, frontend, 0.1, [lookup], [],
529                              details=SearchDetails(layers=layer))
530
531         assert [r.place_id for r in results] == res