avoid special characters in word tokens

Transliteration should only consist of ASCII letters
and numbers. Avoid any other characters.

diff --git a/settings/icu_tokenizer.yaml b/settings/icu_tokenizer.yaml
index 41760c49e0fbd2122d2f1e7fd1966fc4278d1975..e5cbeb6f54ffcef3c3c0340e078c76f6084123e0 100644 (file)
--- a/settings/icu_tokenizer.yaml
+++ b/settings/icu_tokenizer.yaml
@@ -21,8 +21,8 @@ transliteration:
     - !include icu-rules/extended-unicode-to-asccii.yaml
     - ":: Ascii ()"
     - ":: NFD ()"
-    - "[^[:Ascii:]] >"
     - ":: lower ()"
+    - "[^a-z0-9[:Space:]] >"
     - ":: NFC ()"
 sanitizers:
     - step: split-name-list