From cc45930ef90c82bb332e9ce9bf418bd763f618b2 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Wed, 5 Jul 2023 14:07:11 +0200 Subject: [PATCH] avoid lookup via partials on frequent words Drops expensive searches via partials on terms like 'rue de'. See #2979. --- nominatim/api/search/db_search_builder.py | 28 ++++++++++--------- .../api/search/test_db_search_builder.py | 14 +++++++++- 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/nominatim/api/search/db_search_builder.py b/nominatim/api/search/db_search_builder.py index 67db3247..2a3153be 100644 --- a/nominatim/api/search/db_search_builder.py +++ b/nominatim/api/search/db_search_builder.py @@ -235,19 +235,21 @@ class SearchBuilder: yield penalty, sum(t.count for t in rare_names), lookup # To catch remaining results, lookup by name and address - if all(t.is_indexed for t in name_partials): - lookup = [dbf.FieldLookup('name_vector', - [t.token for t in name_partials], 'lookup_all')] - else: - # we don't have the partials, try with the non-rare names - non_rare_names = [t.token for t in name_fulls if t.count >= 1000] - if not non_rare_names: - return - lookup = [dbf.FieldLookup('name_vector', non_rare_names, 'lookup_any')] - if addr_tokens: - lookup.append(dbf.FieldLookup('nameaddress_vector', addr_tokens, 'lookup_all')) - yield penalty + 0.1 * max(0, 5 - len(name_partials) - len(addr_tokens)),\ - min(exp_name_count, exp_addr_count), lookup + # We only do this if there is a reasonable number of results expected. + if min(exp_name_count, exp_addr_count) < 10000: + if all(t.is_indexed for t in name_partials): + lookup = [dbf.FieldLookup('name_vector', + [t.token for t in name_partials], 'lookup_all')] + else: + # we don't have the partials, try with the non-rare names + non_rare_names = [t.token for t in name_fulls if t.count >= 1000] + if not non_rare_names: + return + lookup = [dbf.FieldLookup('name_vector', non_rare_names, 'lookup_any')] + if addr_tokens: + lookup.append(dbf.FieldLookup('nameaddress_vector', addr_tokens, 'lookup_all')) + yield penalty + 0.1 * max(0, 5 - len(name_partials) - len(addr_tokens)),\ + min(exp_name_count, exp_addr_count), lookup def get_name_ranking(self, trange: TokenRange) -> dbf.FieldRanking: diff --git a/test/python/api/search/test_db_search_builder.py b/test/python/api/search/test_db_search_builder.py index 9631850e..63589ffc 100644 --- a/test/python/api/search/test_db_search_builder.py +++ b/test/python/api/search/test_db_search_builder.py @@ -382,7 +382,7 @@ def test_frequent_partials_in_name_but_not_in_address(): def test_frequent_partials_in_name_and_address(): - searches = make_counted_searches(10000, 1, 10000, 1) + searches = make_counted_searches(9999, 1, 9999, 1) assert len(searches) == 2 @@ -393,3 +393,15 @@ def test_frequent_partials_in_name_and_address(): {('name_vector', 'lookup_any'), ('nameaddress_vector', 'restrict')} assert set((l.column, l.lookup_type) for l in searches[1].lookups) == \ {('nameaddress_vector', 'lookup_all'), ('name_vector', 'lookup_all')} + + +def test_too_frequent_partials_in_name_and_address(): + searches = make_counted_searches(10000, 1, 10000, 1) + + assert len(searches) == 1 + + assert all(isinstance(s, dbs.PlaceSearch) for s in searches) + searches.sort(key=lambda s: s.penalty) + + assert set((l.column, l.lookup_type) for l in searches[0].lookups) == \ + {('name_vector', 'lookup_any'), ('nameaddress_vector', 'restrict')} -- 2.39.5