tokenized query.
"""
log().section('Analyze query (using ICU tokenizer)')
- normalized = list(filter(lambda p: p.text,
- (qmod.Phrase(p.ptype, self.normalize_text(p.text))
- for p in phrases)))
- if len(normalized) == 1 \
- and normalized[0].text.count(' ') > 3 \
- and max(len(s) for s in normalized[0].text.split()) < 3:
+ for func in self.preprocessors:
+ phrases = func(phrases)
++
++ if len(phrases) == 1 \
++ and phrases[0].text.count(' ') > 3 \
++ and max(len(s) for s in phrases[0].text.split()) < 3:
+ normalized = []
- query = qmod.QueryStruct(normalized)
++
+ query = qmod.QueryStruct(phrases)
+
log().var_dump('Normalized query', query.source)
if not query.source:
return query