X-Git-Url: https://git.openstreetmap.org./nominatim.git/blobdiff_plain/b453b0ea95e7b1244912b7bc9fc26f58acb8ec80..06796745ff3aa67a2a658f28f3d3016894359847:/settings/icu_tokenizer.yaml diff --git a/settings/icu_tokenizer.yaml b/settings/icu_tokenizer.yaml index c6601faf..f30578a2 100644 --- a/settings/icu_tokenizer.yaml +++ b/settings/icu_tokenizer.yaml @@ -8,8 +8,8 @@ normalization: - "ª > a" - "º > o" - "[[:Punctuation:][:Symbol:]\u02bc] > ' '" - - "ß > 'ss'" # German szet is unimbigiously equal to double ss - - "[^[:Letter:] [:Number:] [:Space:]] >" + - "ß > 'ss'" # German szet is unambiguously equal to double ss + - "[^[:alnum:] [:Canonical_Combining_Class=Virama:] [:Space:]] >" - "[:Lm:] >" - ":: [[:Number:]] Latin ()" - ":: [[:Number:]] Ascii ();" @@ -24,7 +24,19 @@ transliteration: - ":: lower ()" - "[^a-z0-9[:Space:]] >" - ":: NFC ()" + - "[:Space:]+ > ' '" sanitizers: + - step: clean-housenumbers + filter-kind: + - housenumber + - conscriptionnumber + - streetnumber + convert-to-name: + - (\A|.*,)[^\d,]{3,}(,.*|\Z) + - step: clean-postcodes + convert-to-address: yes + default-pattern: "[A-Z0-9- ]{3,12}" + - step: clean-tiger-tags - step: split-name-list - step: strip-brace-terms - step: tag-analyzer-by-language @@ -34,6 +46,10 @@ sanitizers: mode: append token-analysis: - analyzer: generic + - id: "@housenumber" + analyzer: housenumbers + - id: "@postcode" + analyzer: postcodes - id: bg analyzer: generic mode: variant-only