X-Git-Url: https://git.openstreetmap.org./nominatim.git/blobdiff_plain/97a10ec2186faa9aa53a215662714c760e980a00..7282d816c8d70fa70e199ec8e4b5b679fbaa1caa:/settings/icu_tokenizer.yaml diff --git a/settings/icu_tokenizer.yaml b/settings/icu_tokenizer.yaml index 41760c49..c5a809c6 100644 --- a/settings/icu_tokenizer.yaml +++ b/settings/icu_tokenizer.yaml @@ -7,9 +7,9 @@ normalization: - "'nº' > 'no'" - "ª > a" - "º > o" - - "[[:Punctuation:][:Symbol:]] > ' '" - - "ß > 'ss'" # German szet is unimbigiously equal to double ss - - "[^[:Letter:] [:Number:] [:Space:]] >" + - "[[:Punctuation:][:Symbol:]\u02bc] > ' '" + - "ß > 'ss'" # German szet is unambiguously equal to double ss + - "[^[:alnum:] [:Canonical_Combining_Class=Virama:] [:Space:]] >" - "[:Lm:] >" - ":: [[:Number:]] Latin ()" - ":: [[:Number:]] Ascii ();" @@ -21,19 +21,37 @@ transliteration: - !include icu-rules/extended-unicode-to-asccii.yaml - ":: Ascii ()" - ":: NFD ()" - - "[^[:Ascii:]] >" - ":: lower ()" + - "[^a-z0-9[:Space:]] >" - ":: NFC ()" + - "[:Space:]+ > ' '" sanitizers: + - step: clean-housenumbers + filter-kind: + - housenumber + - conscriptionnumber + - streetnumber + convert-to-name: + - (\A|.*,)[^\d,]{3,}(,.*|\Z) + - step: clean-postcodes + convert-to-address: yes + default-pattern: "[A-Z0-9- ]{3,12}" + - step: clean-tiger-tags - step: split-name-list + delimiters: ; - step: strip-brace-terms - step: tag-analyzer-by-language filter-kind: [".*name.*"] whitelist: [bg,ca,cs,da,de,el,en,es,et,eu,fi,fr,gl,hu,it,ja,mg,ms,nl,no,pl,pt,ro,ru,sk,sl,sv,tr,uk,vi] use-defaults: all mode: append + - step: tag-japanese token-analysis: - analyzer: generic + - id: "@housenumber" + analyzer: housenumbers + - id: "@postcode" + analyzer: postcodes - id: bg analyzer: generic mode: variant-only @@ -59,6 +77,13 @@ token-analysis: mode: variant-only variants: - !include icu-rules/variants-de.yaml + mutations: + - pattern: ä + replacements: ["ä", "ae"] + - pattern: ö + replacements: ["ö", "oe"] + - pattern: ü + replacements: ["ü", "ue"] - id: el analyzer: generic mode: variant-only