Merge pull request #3463 from lonvia/sqlalchemy14-with-psycopg

[nominatim.git] / docs / customize / Tokenizers.md

diff --git a/docs/customize/Tokenizers.md b/docs/customize/Tokenizers.md
index 58606c29d0176822ce3364324520b34634a5e242..6b811a9adb13004560064df2c8796e700e9fdea0 100644 (file)
--- a/docs/customize/Tokenizers.md
+++ b/docs/customize/Tokenizers.md
@@ -102,7 +102,7 @@ Here is an example configuration file:
 ``` yaml
 normalization:
     - ":: lower ()"
 ``` yaml
 normalization:
     - ":: lower ()"

-    - "ß > 'ss'" # German szet is unimbigiously equal to double ss
+    - "ß > 'ss'" # German szet is unambiguously equal to double ss

 transliteration:
     - !include /etc/nominatim/icu-rules/extended-unicode-to-asccii.yaml
     - ":: Ascii ()"
 transliteration:
     - !include /etc/nominatim/icu-rules/extended-unicode-to-asccii.yaml
     - ":: Ascii ()"


@@ -128,7 +128,7 @@ The configuration file contains four sections:
 The normalization and transliteration sections each define a set of
 ICU rules that are applied to the names.
 
 The normalization and transliteration sections each define a set of
 ICU rules that are applied to the names.
 

-The **normalisation** rules are applied after sanitation. They should remove
+The **normalization** rules are applied after sanitation. They should remove

 any information that is not relevant for search at all. Usual rules to be
 applied here are: lower-casing, removing of special characters, cleanup of
 spaces.
 any information that is not relevant for search at all. Usual rules to be
 applied here are: lower-casing, removing of special characters, cleanup of
 spaces.


@@ -175,53 +175,67 @@ The following is a list of sanitizers that are shipped with Nominatim.
 
 ##### split-name-list
 
 
 ##### split-name-list
 

-::: nominatim.tokenizer.sanitizers.split_name_list
-    selection:
+::: nominatim_db.tokenizer.sanitizers.split_name_list
+    options:

         members: False
         members: False

-    rendering:

         heading_level: 6
         heading_level: 6

+        docstring_section_style: spacy

 
 ##### strip-brace-terms
 
 
 ##### strip-brace-terms
 

-::: nominatim.tokenizer.sanitizers.strip_brace_terms
-    selection:
+::: nominatim_db.tokenizer.sanitizers.strip_brace_terms
+    options:

         members: False
         members: False

-    rendering:

         heading_level: 6
         heading_level: 6

+        docstring_section_style: spacy

 
 ##### tag-analyzer-by-language
 
 
 ##### tag-analyzer-by-language
 

-::: nominatim.tokenizer.sanitizers.tag_analyzer_by_language
-    selection:
+::: nominatim_db.tokenizer.sanitizers.tag_analyzer_by_language
+    options:

         members: False
         members: False

-    rendering:

         heading_level: 6
         heading_level: 6

+        docstring_section_style: spacy

 
 ##### clean-housenumbers
 
 
 ##### clean-housenumbers
 

-::: nominatim.tokenizer.sanitizers.clean_housenumbers
-    selection:
+::: nominatim_db.tokenizer.sanitizers.clean_housenumbers
+    options:

         members: False
         members: False

-    rendering:

         heading_level: 6
         heading_level: 6

+        docstring_section_style: spacy

 
 ##### clean-postcodes
 
 
 ##### clean-postcodes
 

-::: nominatim.tokenizer.sanitizers.clean_postcodes
-    selection:
+::: nominatim_db.tokenizer.sanitizers.clean_postcodes
+    options:

         members: False
         members: False

-    rendering:

         heading_level: 6
         heading_level: 6

+        docstring_section_style: spacy

 
 ##### clean-tiger-tags
 
 
 ##### clean-tiger-tags
 

-::: nominatim.tokenizer.sanitizers.clean_tiger_tags
-    selection:
+::: nominatim_db.tokenizer.sanitizers.clean_tiger_tags
+    options:

         members: False
         members: False

-    rendering:

         heading_level: 6
         heading_level: 6

+        docstring_section_style: spacy

 
 

+#### delete-tags

 
 

+::: nominatim_db.tokenizer.sanitizers.delete_tags
+    options:
+        members: False
+        heading_level: 6
+        docstring_section_style: spacy
+
+#### tag-japanese
+
+::: nominatim_db.tokenizer.sanitizers.tag_japanese
+    options:
+        members: False
+        heading_level: 6
+        docstring_section_style: spacy

 
 #### Token Analysis
 
 
 #### Token Analysis
 


@@ -380,7 +394,7 @@ The analyzer cannot be customized.
 ##### Postcode token analyzer
 
 The analyzer `postcodes` is pupose-made to analyze postcodes. It supports
 ##### Postcode token analyzer
 
 The analyzer `postcodes` is pupose-made to analyze postcodes. It supports

-a 'lookup' varaint of the token, which produces variants with optional
+a 'lookup' variant of the token, which produces variants with optional

 spaces. Use together with the clean-postcodes sanitizer.
 
 The analyzer cannot be customized.
 spaces. Use together with the clean-postcodes sanitizer.
 
 The analyzer cannot be customized.