remove Token from explicit input for SearchDescription extension

[nominatim.git] / lib-php / tokenizer / legacy_tokenizer.php

diff --git a/lib-php/tokenizer/legacy_tokenizer.php b/lib-php/tokenizer/legacy_tokenizer.php
index 238fbcf45e48a2120d4b0f2b566032999bc963be..064b41667a9322bb6cb164dd6f7bb041490d1257 100644 (file)
--- a/lib-php/tokenizer/legacy_tokenizer.php
+++ b/lib-php/tokenizer/legacy_tokenizer.php
@@ -218,9 +218,12 @@ class Tokenizer
                     (int) $aWord['count'],
                     substr_count($aWord['word_token'], ' ')
                 );
-            } else {
+            // For backward compatibility: ignore all partial tokens with more
+            // than one word.
+            } elseif (strpos($aWord['word_token'], ' ') === false) {
                 $oToken = new Token\Partial(
                     $iId,
+                    $aWord['word_token'],
                     (int) $aWord['count']
                 );
             }