X-Git-Url: https://git.openstreetmap.org./nominatim.git/blobdiff_plain/b7d4ff3201376126d9b4979fffe6b4521469d861..14cc047cf4e8e2028be9c34cb6b79ef20816b532:/settings/icu_tokenizer.yaml?ds=inline diff --git a/settings/icu_tokenizer.yaml b/settings/icu_tokenizer.yaml index c0c8c043..c5a809c6 100644 --- a/settings/icu_tokenizer.yaml +++ b/settings/icu_tokenizer.yaml @@ -7,9 +7,9 @@ normalization: - "'nº' > 'no'" - "ª > a" - "º > o" - - "[[:Punctuation:][:Symbol:]] > ' '" - - "ß > 'ss'" # German szet is unimbigiously equal to double ss - - "[^[:Letter:] [:Number:] [:Space:]] >" + - "[[:Punctuation:][:Symbol:]\u02bc] > ' '" + - "ß > 'ss'" # German szet is unambiguously equal to double ss + - "[^[:alnum:] [:Canonical_Combining_Class=Virama:] [:Space:]] >" - "[:Lm:] >" - ":: [[:Number:]] Latin ()" - ":: [[:Number:]] Ascii ();" @@ -21,37 +21,191 @@ transliteration: - !include icu-rules/extended-unicode-to-asccii.yaml - ":: Ascii ()" - ":: NFD ()" - - "[^[:Ascii:]] >" - ":: lower ()" + - "[^a-z0-9[:Space:]] >" - ":: NFC ()" -variants: - - !include icu-rules/variants-bg.yaml - - !include icu-rules/variants-ca.yaml - - !include icu-rules/variants-cs.yaml - - !include icu-rules/variants-da.yaml - - !include icu-rules/variants-de.yaml - - !include icu-rules/variants-el.yaml - - !include icu-rules/variants-en.yaml - - !include icu-rules/variants-es.yaml - - !include icu-rules/variants-et.yaml - - !include icu-rules/variants-eu.yaml - - !include icu-rules/variants-fi.yaml - - !include icu-rules/variants-fr.yaml - - !include icu-rules/variants-gl.yaml - - !include icu-rules/variants-hu.yaml - - !include icu-rules/variants-it.yaml - - !include icu-rules/variants-ja.yaml - - !include icu-rules/variants-mg.yaml - - !include icu-rules/variants-ms.yaml - - !include icu-rules/variants-nl.yaml - - !include icu-rules/variants-no.yaml - - !include icu-rules/variants-pl.yaml - - !include icu-rules/variants-pt.yaml - - !include icu-rules/variants-ro.yaml - - !include icu-rules/variants-ru.yaml - - !include icu-rules/variants-sk.yaml - - !include icu-rules/variants-sl.yaml - - !include icu-rules/variants-sv.yaml - - !include icu-rules/variants-tr.yaml - - !include icu-rules/variants-uk.yaml - - !include icu-rules/variants-vi.yaml + - "[:Space:]+ > ' '" +sanitizers: + - step: clean-housenumbers + filter-kind: + - housenumber + - conscriptionnumber + - streetnumber + convert-to-name: + - (\A|.*,)[^\d,]{3,}(,.*|\Z) + - step: clean-postcodes + convert-to-address: yes + default-pattern: "[A-Z0-9- ]{3,12}" + - step: clean-tiger-tags + - step: split-name-list + delimiters: ; + - step: strip-brace-terms + - step: tag-analyzer-by-language + filter-kind: [".*name.*"] + whitelist: [bg,ca,cs,da,de,el,en,es,et,eu,fi,fr,gl,hu,it,ja,mg,ms,nl,no,pl,pt,ro,ru,sk,sl,sv,tr,uk,vi] + use-defaults: all + mode: append + - step: tag-japanese +token-analysis: + - analyzer: generic + - id: "@housenumber" + analyzer: housenumbers + - id: "@postcode" + analyzer: postcodes + - id: bg + analyzer: generic + mode: variant-only + variants: + - !include icu-rules/variants-bg.yaml + - id: ca + analyzer: generic + mode: variant-only + variants: + - !include icu-rules/variants-ca.yaml + - id: cs + analyzer: generic + mode: variant-only + variants: + - !include icu-rules/variants-cs.yaml + - id: da + analyzer: generic + mode: variant-only + variants: + - !include icu-rules/variants-da.yaml + - id: de + analyzer: generic + mode: variant-only + variants: + - !include icu-rules/variants-de.yaml + mutations: + - pattern: ä + replacements: ["ä", "ae"] + - pattern: ö + replacements: ["ö", "oe"] + - pattern: ü + replacements: ["ü", "ue"] + - id: el + analyzer: generic + mode: variant-only + variants: + - !include icu-rules/variants-el.yaml + - id: en + analyzer: generic + mode: variant-only + variants: + - !include icu-rules/variants-en.yaml + - id: es + analyzer: generic + mode: variant-only + variants: + - !include icu-rules/variants-es.yaml + - id: et + analyzer: generic + mode: variant-only + variants: + - !include icu-rules/variants-et.yaml + - id: eu + analyzer: generic + mode: variant-only + variants: + - !include icu-rules/variants-eu.yaml + - id: fi + analyzer: generic + mode: variant-only + variants: + - !include icu-rules/variants-fi.yaml + - id: fr + analyzer: generic + mode: variant-only + variants: + - !include icu-rules/variants-fr.yaml + - id: gl + analyzer: generic + mode: variant-only + variants: + - !include icu-rules/variants-gl.yaml + - id: hu + analyzer: generic + mode: variant-only + variants: + - !include icu-rules/variants-hu.yaml + - id: it + analyzer: generic + mode: variant-only + variants: + - !include icu-rules/variants-it.yaml + - id: ja + analyzer: generic + mode: variant-only + variants: + - !include icu-rules/variants-ja.yaml + - id: mg + analyzer: generic + mode: variant-only + variants: + - !include icu-rules/variants-mg.yaml + - id: ms + analyzer: generic + mode: variant-only + variants: + - !include icu-rules/variants-ms.yaml + - id: nl + analyzer: generic + mode: variant-only + variants: + - !include icu-rules/variants-nl.yaml + - id: no + analyzer: generic + mode: variant-only + variants: + - !include icu-rules/variants-no.yaml + - id: pl + analyzer: generic + mode: variant-only + variants: + - !include icu-rules/variants-pl.yaml + - id: pt + analyzer: generic + mode: variant-only + variants: + - !include icu-rules/variants-pt.yaml + - id: ro + analyzer: generic + mode: variant-only + variants: + - !include icu-rules/variants-ro.yaml + - id: ru + analyzer: generic + mode: variant-only + variants: + - !include icu-rules/variants-ru.yaml + - id: sk + analyzer: generic + mode: variant-only + variants: + - !include icu-rules/variants-sk.yaml + - id: sl + analyzer: generic + mode: variant-only + variants: + - !include icu-rules/variants-sl.yaml + - id: sv + analyzer: generic + mode: variant-only + variants: + - !include icu-rules/variants-sv.yaml + - id: tr + analyzer: generic + mode: variant-only + variants: + - !include icu-rules/variants-tr.yaml + - id: uk + analyzer: generic + mode: variant-only + variants: + - !include icu-rules/variants-uk.yaml + - id: vi + analyzer: generic + mode: variant-only + variants: + - !include icu-rules/variants-vi.yaml