settings/icu_tokenizer.yaml

   1 normalization:
   2     - ":: lower ()"
   3     - ":: Hans-Hant"
   4     - !include icu-rules/unicode-digits-to-decimal.yaml
   5     - "'№' > 'no'"
   6     - "'n°' > 'no'"
   7     - "'nº' > 'no'"
   8     - "ª > a"
   9     - "º > o"
  10     - "[[:Punctuation:][:Symbol:]]  > ' '"
  11     - "ß > 'ss'" # German szet is unimbigiously equal to double ss
  12     - "[^[:Letter:] [:Number:] [:Space:]] >"
  13     - "[:Lm:] >"
  14     - ":: [[:Number:]] Latin ()"
  15     - ":: [[:Number:]] Ascii ();"
  16     - ":: [[:Number:]] NFD ();"
  17     - "[[:Nonspacing Mark:] [:Cf:]] >;"
  18     - "[:Space:]+ > ' '"
  19 transliteration:
  20     - ":: Latin ()"
  21     - !include icu-rules/extended-unicode-to-asccii.yaml
  22     - ":: Ascii ()"
  23     - ":: NFD ()"
  24     - "[^[:Ascii:]] >"
  25     - ":: lower ()"
  26     - ":: NFC ()"
  27 sanitizers:
  28     - step: split-name-list
  29     - step: strip-brace-terms
  30     - step: tag-analyzer-by-language
  31       filter-kind: [".*name.*"]
  32       whitelist: [bg,ca,cs,da,de,el,en,es,et,eu,fi,fr,gl,hu,it,ja,mg,ms,nl,no,pl,pt,ro,ru,sk,sl,sv,tr,uk,vi]
  33       use-defaults: all
  34       mode: append
  35 token-analysis:
  36     - analyzer: generic
  37     - id: bg
  38       analyzer: generic
  39       mode: variant-only
  40       variants:
  41           - !include icu-rules/variants-bg.yaml
  42     - id: ca
  43       analyzer: generic
  44       mode: variant-only
  45       variants:
  46           - !include icu-rules/variants-ca.yaml
  47     - id: cs
  48       analyzer: generic
  49       mode: variant-only
  50       variants:
  51           - !include icu-rules/variants-cs.yaml
  52     - id: da
  53       analyzer: generic
  54       mode: variant-only
  55       variants:
  56           - !include icu-rules/variants-da.yaml
  57     - id: de
  58       analyzer: generic
  59       mode: variant-only
  60       variants:
  61           - !include icu-rules/variants-de.yaml
  62     - id: el
  63       analyzer: generic
  64       mode: variant-only
  65       variants:
  66           - !include icu-rules/variants-el.yaml
  67     - id: en
  68       analyzer: generic
  69       mode: variant-only
  70       variants:
  71           - !include icu-rules/variants-en.yaml
  72     - id: es
  73       analyzer: generic
  74       mode: variant-only
  75       variants:
  76           - !include icu-rules/variants-es.yaml
  77     - id: et
  78       analyzer: generic
  79       mode: variant-only
  80       variants:
  81           - !include icu-rules/variants-et.yaml
  82     - id: eu
  83       analyzer: generic
  84       mode: variant-only
  85       variants:
  86           - !include icu-rules/variants-eu.yaml
  87     - id: fi
  88       analyzer: generic
  89       mode: variant-only
  90       variants:
  91           - !include icu-rules/variants-fi.yaml
  92     - id: fr
  93       analyzer: generic
  94       mode: variant-only
  95       variants:
  96           - !include icu-rules/variants-fr.yaml
  97     - id: gl
  98       analyzer: generic
  99       mode: variant-only
 100       variants:
 101           - !include icu-rules/variants-gl.yaml
 102     - id: hu
 103       analyzer: generic
 104       mode: variant-only
 105       variants:
 106           - !include icu-rules/variants-hu.yaml
 107     - id: it
 108       analyzer: generic
 109       mode: variant-only
 110       variants:
 111           - !include icu-rules/variants-it.yaml
 112     - id: ja
 113       analyzer: generic
 114       mode: variant-only
 115       variants:
 116           - !include icu-rules/variants-ja.yaml
 117     - id: mg
 118       analyzer: generic
 119       mode: variant-only
 120       variants:
 121           - !include icu-rules/variants-mg.yaml
 122     - id: ms
 123       analyzer: generic
 124       mode: variant-only
 125       variants:
 126           - !include icu-rules/variants-ms.yaml
 127     - id: nl
 128       analyzer: generic
 129       mode: variant-only
 130       variants:
 131           - !include icu-rules/variants-nl.yaml
 132     - id: no
 133       analyzer: generic
 134       mode: variant-only
 135       variants:
 136           - !include icu-rules/variants-no.yaml
 137     - id: pl
 138       analyzer: generic
 139       mode: variant-only
 140       variants:
 141           - !include icu-rules/variants-pl.yaml
 142     - id: pt
 143       analyzer: generic
 144       mode: variant-only
 145       variants:
 146           - !include icu-rules/variants-pt.yaml
 147     - id: ro
 148       analyzer: generic
 149       mode: variant-only
 150       variants:
 151           - !include icu-rules/variants-ro.yaml
 152     - id: ru
 153       analyzer: generic
 154       mode: variant-only
 155       variants:
 156           - !include icu-rules/variants-ru.yaml
 157     - id: sk
 158       analyzer: generic
 159       mode: variant-only
 160       variants:
 161           - !include icu-rules/variants-sk.yaml
 162     - id: sl
 163       analyzer: generic
 164       mode: variant-only
 165       variants:
 166           - !include icu-rules/variants-sl.yaml
 167     - id: sv
 168       analyzer: generic
 169       mode: variant-only
 170       variants:
 171           - !include icu-rules/variants-sv.yaml
 172     - id: tr
 173       analyzer: generic
 174       mode: variant-only
 175       variants:
 176           - !include icu-rules/variants-tr.yaml
 177     - id: uk
 178       analyzer: generic
 179       mode: variant-only
 180       variants:
 181           - !include icu-rules/variants-uk.yaml
 182     - id: vi
 183       analyzer: generic
 184       mode: variant-only
 185       variants:
 186           - !include icu-rules/variants-vi.yaml