improve normalization

[nominatim.git] / settings / legacy_icu_tokenizer.yaml

diff --git a/settings/legacy_icu_tokenizer.yaml b/settings/legacy_icu_tokenizer.yaml
index a3f1c02735238b21d96af8d9b5f3bbeaa7629749..7972b156d455d8841daa020328252edef9da9056 100644 (file)
--- a/settings/legacy_icu_tokenizer.yaml
+++ b/settings/legacy_icu_tokenizer.yaml
@@ -1,20 +1,29 @@
 normalization:
 normalization:

-    - ":: NFD ()"
-    - "[[:Nonspacing Mark:] [:Cf:]] >"

     - ":: lower ()"
     - ":: lower ()"

+    - !include icu-rules/unicode-digits-to-decimal.yaml
+    - "'№' > 'no'"
+    - "'n°' > 'no'"
+    - "'nº' > 'no'"
+    - "ª > a"
+    - "º > o"
+    - "[[:Punctuation:][:Symbol:]]  > ' '"

     - "ß > 'ss'" # German szet is unimbigiously equal to double ss
     - "ß > 'ss'" # German szet is unimbigiously equal to double ss

-    - "[[:Punctuation:][:Space:]]+ > ' '"
-    - ":: NFC ()"
+    - "[^[:Letter:] [:Number:] [:Space:]] >"
+    - "[:Lm:] >"
+    - ":: [[:Number:]] Latin ()"
+    - ":: [[:Number:]] Ascii ();"
+    - ":: [[:Number:]] NFD ();"
+    - "[[:Nonspacing Mark:] [:Cf:]] >;"
+    - "[:Space:]+ > ' '"

 transliteration:
 transliteration:

+    - ":: Latin ()"

     - !include icu-rules/extended-unicode-to-asccii.yaml
     - ":: Ascii ()"
     - ":: NFD ()"
     - !include icu-rules/extended-unicode-to-asccii.yaml
     - ":: Ascii ()"
     - ":: NFD ()"

-    - "'' >"
-    - "[[:Nonspacing Mark:] [:Cf:]] >"

     - "[^[:Ascii:]] >"
     - ":: lower ()"
     - "[^[:Ascii:]] >"
     - ":: lower ()"

-    - "[[:Punctuation:][:Space:]]+ > ' '"

     - ":: NFC ()"
     - ":: NFC ()"

+    - "[:Space:]+ > ' '"

 variants:
   - words:
     - ~hal => hal
 variants:
   - words:
     - ~hal => hal