From: Sarah Hoffmann Date: Fri, 28 Feb 2025 16:19:37 +0000 (+0100) Subject: show token begin and end in debug output X-Git-Url: https://git.openstreetmap.org./nominatim.git/commitdiff_plain/e67ae701ac76129d24578cc70b8e25987a162a81 show token begin and end in debug output --- diff --git a/src/nominatim_api/search/icu_tokenizer.py b/src/nominatim_api/search/icu_tokenizer.py index 01513103..09827826 100644 --- a/src/nominatim_api/search/icu_tokenizer.py +++ b/src/nominatim_api/search/icu_tokenizer.py @@ -298,12 +298,12 @@ class ICUQueryAnalyzer(AbstractQueryAnalyzer): def _dump_word_tokens(query: qmod.QueryStruct) -> Iterator[List[Any]]: - yield ['type', 'token', 'word_token', 'lookup_word', 'penalty', 'count', 'info'] - for node in query.nodes: + yield ['type', 'from', 'to', 'token', 'word_token', 'lookup_word', 'penalty', 'count', 'info'] + for i, node in enumerate(query.nodes): for tlist in node.starting: for token in tlist.tokens: t = cast(ICUToken, token) - yield [tlist.ttype, t.token, t.word_token or '', + yield [tlist.ttype, str(i), str(tlist.end), t.token, t.word_token or '', t.lookup_word or '', t.penalty, t.count, t.info]