All punctuation will be converted to '-'. Soft breaks : may be
added by preprocessors. The break signs are only used during
query analysis and are ignored during import token analysis.
- "'nº' > 'no'"
- "ª > a"
- "º > o"
- "'nº' > 'no'"
- "ª > a"
- "º > o"
- - "[[:Punctuation:][:Symbol:]\u02bc] > ' '"
+ - "[[:Punctuation:][:Symbol:][\u02bc] - [-:]]+ > '-'"
- "ß > 'ss'" # German szet is unambiguously equal to double ss
- "ß > 'ss'" # German szet is unambiguously equal to double ss
- - "[^[:alnum:] [:Canonical_Combining_Class=Virama:] [:Space:]] >"
+ - "[^[:alnum:] [:Canonical_Combining_Class=Virama:] [:Space:] [-:]] >"
- "[:Lm:] >"
- ":: [[:Number:]] Latin ()"
- ":: [[:Number:]] Ascii ();"
- ":: [[:Number:]] NFD ();"
- "[[:Nonspacing Mark:] [:Cf:]] >;"
- "[:Lm:] >"
- ":: [[:Number:]] Latin ()"
- ":: [[:Number:]] Ascii ();"
- ":: [[:Number:]] NFD ();"
- "[[:Nonspacing Mark:] [:Cf:]] >;"
+ - "[-:]?[:Space:]+[-:]? > ' '"
- ":: Latin ()"
- !include icu-rules/extended-unicode-to-asccii.yaml
- ":: Ascii ()"
- ":: Latin ()"
- !include icu-rules/extended-unicode-to-asccii.yaml
- ":: Ascii ()"
"""
assert self.query_analyzer is not None
qwords = [word for phrase in query.source
"""
assert self.query_analyzer is not None
qwords = [word for phrase in query.source
- for word in re.split('[, ]+', phrase.text) if word]
+ for word in re.split('[-,: ]+', phrase.text) if word]
distance = 0.0
norm = self.query_analyzer.normalize_text(' '.join((result.display_name,
result.country_code or '')))
distance = 0.0
norm = self.query_analyzer.normalize_text(' '.join((result.display_name,
result.country_code or '')))
- words = set((w for w in norm.split(' ') if w))
+ words = set((w for w in re.split('[-,: ]+', norm) if w))
if not words:
continue
for qword in qwords:
if not words:
continue
for qword in qwords:
def __init__(self, norm_rules: str, trans_rules: str,
analysis_rules: Mapping[Optional[str], 'TokenAnalyzerRule']):
def __init__(self, norm_rules: str, trans_rules: str,
analysis_rules: Mapping[Optional[str], 'TokenAnalyzerRule']):
+ # additional break signs are not relevant during name analysis
+ norm_rules += ";[[:Space:][-:]]+ > ' ';"
self.normalizer = Transliterator.createFromRules("icu_normalization",
norm_rules)
trans_rules += ";[:Space:]+ > ' '"
self.normalizer = Transliterator.createFromRules("icu_normalization",
norm_rules)
trans_rules += ";[:Space:]+ > ' '"