icu: move token deduplication into TokenInfo

[nominatim.git] / nominatim / clicmd / refresh.py

diff --git a/nominatim/clicmd/refresh.py b/nominatim/clicmd/refresh.py
index 4df283f8c781c8a7ecc41ef58d6ced7601bb2557..b8a88b6d615b5b5c04445f393a71a81c1b6cc112 100644 (file)
--- a/nominatim/clicmd/refresh.py
+++ b/nominatim/clicmd/refresh.py
@@ -39,6 +39,8 @@ class UpdateRefresh:
         group = parser.add_argument_group('Data arguments')
         group.add_argument('--postcodes', action='store_true',
                            help='Update postcode centroid table')
+        group.add_argument('--word-tokens', action='store_true',
+                           help='Clean up search terms')
         group.add_argument('--word-counts', action='store_true',
                            help='Compute frequency of full-word search terms')
         group.add_argument('--address-levels', action='store_true',
@@ -76,6 +78,11 @@ class UpdateRefresh:
                 LOG.error("The place table doesn't exist. "
                           "Postcode updates on a frozen database is not possible.")
 
+        if args.word_tokens:
+            LOG.warning('Updating word tokens')
+            tokenizer = self._get_tokenizer(args.config)
+            tokenizer.update_word_tokens()
+
         if args.word_counts:
             LOG.warning('Recompute word statistics')
             self._get_tokenizer(args.config).update_statistics()