X-Git-Url: https://git.openstreetmap.org./nominatim.git/blobdiff_plain/d0a1e8e3110bfe3dea221933ab4e8964e8d74f66..df15f13c628a3e15dee4bcad44e97b2cacb40b9f:/nominatim/api/search/token_assignment.py diff --git a/nominatim/api/search/token_assignment.py b/nominatim/api/search/token_assignment.py index 11da2359..ca907b79 100644 --- a/nominatim/api/search/token_assignment.py +++ b/nominatim/api/search/token_assignment.py @@ -46,7 +46,7 @@ class TokenAssignment: # pylint: disable=too-many-instance-attributes housenumber: Optional[qmod.TokenRange] = None postcode: Optional[qmod.TokenRange] = None country: Optional[qmod.TokenRange] = None - category: Optional[qmod.TokenRange] = None + near_item: Optional[qmod.TokenRange] = None qualifier: Optional[qmod.TokenRange] = None @@ -64,15 +64,15 @@ class TokenAssignment: # pylint: disable=too-many-instance-attributes out.postcode = token.trange elif token.ttype == qmod.TokenType.COUNTRY: out.country = token.trange - elif token.ttype == qmod.TokenType.CATEGORY: - out.category = token.trange + elif token.ttype == qmod.TokenType.NEAR_ITEM: + out.near_item = token.trange elif token.ttype == qmod.TokenType.QUALIFIER: out.qualifier = token.trange return out class _TokenSequence: - """ Working state used to put together the token assignements. + """ Working state used to put together the token assignments. Represents an intermediate state while traversing the tokenized query. @@ -109,7 +109,7 @@ class _TokenSequence: """ # Country and category must be the final term for left-to-right return len(self.seq) > 1 and \ - self.seq[-1].ttype in (qmod.TokenType.COUNTRY, qmod.TokenType.CATEGORY) + self.seq[-1].ttype in (qmod.TokenType.COUNTRY, qmod.TokenType.NEAR_ITEM) def appendable(self, ttype: qmod.TokenType) -> Optional[int]: @@ -132,6 +132,11 @@ class _TokenSequence: # Name tokens are always acceptable and don't change direction if ttype == qmod.TokenType.PARTIAL: + # qualifiers cannot appear in the middle of the query. They need + # to be near the next phrase. + if self.direction == -1 \ + and any(t.ttype == qmod.TokenType.QUALIFIER for t in self.seq[:-1]): + return None return self.direction # Other tokens may only appear once @@ -165,22 +170,22 @@ class _TokenSequence: if ttype == qmod.TokenType.COUNTRY: return None if self.direction == -1 else 1 - if ttype == qmod.TokenType.CATEGORY: + if ttype == qmod.TokenType.NEAR_ITEM: return self.direction if ttype == qmod.TokenType.QUALIFIER: if self.direction == 1: if (len(self.seq) == 1 - and self.seq[0].ttype in (qmod.TokenType.PARTIAL, qmod.TokenType.CATEGORY)) \ + and self.seq[0].ttype in (qmod.TokenType.PARTIAL, qmod.TokenType.NEAR_ITEM)) \ or (len(self.seq) == 2 - and self.seq[0].ttype == qmod.TokenType.CATEGORY + and self.seq[0].ttype == qmod.TokenType.NEAR_ITEM and self.seq[1].ttype == qmod.TokenType.PARTIAL): return 1 return None if self.direction == -1: return -1 - tempseq = self.seq[1:] if self.seq[0].ttype == qmod.TokenType.CATEGORY else self.seq + tempseq = self.seq[1:] if self.seq[0].ttype == qmod.TokenType.NEAR_ITEM else self.seq if len(tempseq) == 0: return 1 if len(tempseq) == 1 and self.seq[0].ttype == qmod.TokenType.HOUSENUMBER: @@ -233,10 +238,10 @@ class _TokenSequence: def recheck_sequence(self) -> bool: """ Check that the sequence is a fully valid token assignment - and addapt direction and penalties further if necessary. + and adapt direction and penalties further if necessary. This function catches some impossible assignments that need - forward context and can therefore not be exluded when building + forward context and can therefore not be excluded when building the assignment. """ # housenumbers may not be further than 2 words from the beginning. @@ -253,10 +258,103 @@ class _TokenSequence: priors = sum(1 for t in self.seq[hnrpos+1:] if t.ttype == qmod.TokenType.PARTIAL) if not self._adapt_penalty_from_priors(priors, 1): return False + if any(t.ttype == qmod.TokenType.NEAR_ITEM for t in self.seq): + self.penalty += 1.0 return True + def _get_assignments_postcode(self, base: TokenAssignment, + query_len: int) -> Iterator[TokenAssignment]: + """ Yield possible assignments of Postcode searches with an + address component. + """ + assert base.postcode is not None + + if (base.postcode.start == 0 and self.direction != -1)\ + or (base.postcode.end == query_len and self.direction != 1): + log().comment('postcode search') + #
, should give preference to address search + if base.postcode.start == 0: + penalty = self.penalty + self.direction = -1 # name searches are only possible backwards + else: + penalty = self.penalty + 0.1 + self.direction = 1 # name searches are only possible forwards + yield dataclasses.replace(base, penalty=penalty) + + + def _get_assignments_address_forward(self, base: TokenAssignment, + query: qmod.QueryStruct) -> Iterator[TokenAssignment]: + """ Yield possible assignments of address searches with + left-to-right reading. + """ + first = base.address[0] + + log().comment('first word = name') + yield dataclasses.replace(base, penalty=self.penalty, + name=first, address=base.address[1:]) + + # To paraphrase: + # * if another name term comes after the first one and before the + # housenumber + # * a qualifier comes after the name + # * the containing phrase is strictly typed + if (base.housenumber and first.end < base.housenumber.start)\ + or (base.qualifier and base.qualifier > first)\ + or (query.nodes[first.start].ptype != qmod.PhraseType.NONE): + return + + penalty = self.penalty + + # Penalty for: + # * , , , ... + # * queries that are comma-separated + if (base.housenumber and base.housenumber > first) or len(query.source) > 1: + penalty += 0.25 + + for i in range(first.start + 1, first.end): + name, addr = first.split(i) + log().comment(f'split first word = name ({i - first.start})') + yield dataclasses.replace(base, name=name, address=[addr] + base.address[1:], + penalty=penalty + PENALTY_TOKENCHANGE[query.nodes[i].btype]) + + + def _get_assignments_address_backward(self, base: TokenAssignment, + query: qmod.QueryStruct) -> Iterator[TokenAssignment]: + """ Yield possible assignments of address searches with + right-to-left reading. + """ + last = base.address[-1] + + if self.direction == -1 or len(base.address) > 1: + log().comment('last word = name') + yield dataclasses.replace(base, penalty=self.penalty, + name=last, address=base.address[:-1]) + + # To paraphrase: + # * if another name term comes before the last one and after the + # housenumber + # * a qualifier comes before the name + # * the containing phrase is strictly typed + if (base.housenumber and last.start > base.housenumber.end)\ + or (base.qualifier and base.qualifier < last)\ + or (query.nodes[last.start].ptype != qmod.PhraseType.NONE): + return + + penalty = self.penalty + if base.housenumber and base.housenumber < last: + penalty += 0.4 + if len(query.source) > 1: + penalty += 0.25 + + for i in range(last.start + 1, last.end): + addr, name = last.split(i) + log().comment(f'split last word = name ({i - last.start})') + yield dataclasses.replace(base, name=name, address=base.address[:-1] + [addr], + penalty=penalty + PENALTY_TOKENCHANGE[query.nodes[i].btype]) + + def get_assignments(self, query: qmod.QueryStruct) -> Iterator[TokenAssignment]: """ Yield possible assignments for the current sequence. @@ -265,69 +363,35 @@ class _TokenSequence: """ base = TokenAssignment.from_ranges(self.seq) + num_addr_tokens = sum(t.end - t.start for t in base.address) + if num_addr_tokens > 50: + return + # Postcode search (postcode-only search is covered in next case) if base.postcode is not None and base.address: - if (base.postcode.start == 0 and self.direction != -1)\ - or (base.postcode.end == query.num_token_slots() and self.direction != 1): - log().comment('postcode search') - #
, should give preference to address search - if base.postcode.start == 0: - penalty = self.penalty - else: - penalty = self.penalty + 0.1 - yield dataclasses.replace(base, penalty=penalty) + yield from self._get_assignments_postcode(base, query.num_token_slots()) # Postcode or country-only search if not base.address: - if not base.housenumber and (base.postcode or base.country or base.category): + if not base.housenumber and (base.postcode or base.country or base.near_item): log().comment('postcode/country search') yield dataclasses.replace(base, penalty=self.penalty) else: # ,
should give preference to postcode search if base.postcode and base.postcode.start == 0: self.penalty += 0.1 - # Use entire first word as name - if self.direction != -1: - log().comment('first word = name') - yield dataclasses.replace(base, name=base.address[0], - penalty=self.penalty, - address=base.address[1:]) - - # Use entire last word as name - if self.direction == -1 or (self.direction == 0 and len(base.address) > 1): - log().comment('last word = name') - yield dataclasses.replace(base, name=base.address[-1], - penalty=self.penalty, - address=base.address[:-1]) - - # variant for special housenumber searches - if base.housenumber: - yield dataclasses.replace(base, penalty=self.penalty) - # Use beginning of first word as name + # Right-to-left reading of the address if self.direction != -1: - first = base.address[0] - if (not base.housenumber or first.end >= base.housenumber.start)\ - and (not base.qualifier or first.start >= base.qualifier.end): - for i in range(first.start + 1, first.end): - name, addr = first.split(i) - penalty = self.penalty + PENALTY_TOKENCHANGE[query.nodes[i].btype] - log().comment(f'split first word = name ({i - first.start})') - yield dataclasses.replace(base, name=name, penalty=penalty, - address=[addr] + base.address[1:]) - - # Use end of last word as name + yield from self._get_assignments_address_forward(base, query) + + # Left-to-right reading of the address if self.direction != 1: - last = base.address[-1] - if (not base.housenumber or last.start <= base.housenumber.end)\ - and (not base.qualifier or last.end <= base.qualifier.start): - for i in range(last.start + 1, last.end): - addr, name = last.split(i) - penalty = self.penalty + PENALTY_TOKENCHANGE[query.nodes[i].btype] - log().comment(f'split last word = name ({i - last.start})') - yield dataclasses.replace(base, name=name, penalty=penalty, - address=base.address[:-1] + [addr]) + yield from self._get_assignments_address_backward(base, query) + # variant for special housenumber searches + if base.housenumber and not base.qualifier: + yield dataclasses.replace(base, penalty=self.penalty) def yield_token_assignments(query: qmod.QueryStruct) -> Iterator[TokenAssignment]: