remove country restriction from tokenizer

[nominatim.git] / test / php / Nominatim / PhraseTest.php

diff --git a/test/php/Nominatim/PhraseTest.php b/test/php/Nominatim/PhraseTest.php
index 35a8a80480d2c503999432edfe230152b3be20c6..e4c2bbd1be0f80130b66a018e0ee0790f217ab4f 100644 (file)
--- a/test/php/Nominatim/PhraseTest.php
+++ b/test/php/Nominatim/PhraseTest.php
@@ -2,8 +2,31 @@
 
 namespace Nominatim;
 
-require_once '../../lib/Phrase.php';
+require_once(CONST_LibDir.'/Phrase.php');
 
+class TokensFullSet
+{
+    public function containsAny($sTerm)
+    {
+        return true;
+    }
+}
+
+// phpcs:ignore PSR1.Classes.ClassDeclaration.MultipleClasses
+class TokensPartialSet
+{
+    public function __construct($aTokens)
+    {
+        $this->aTokens = array_flip($aTokens);
+    }
+
+    public function containsAny($sTerm)
+    {
+        return isset($this->aTokens[$sTerm]);
+    }
+}
+
+// phpcs:ignore PSR1.Classes.ClassDeclaration.MultipleClasses
 class PhraseTest extends \PHPUnit\Framework\TestCase
 {
 
@@ -21,17 +44,16 @@ class PhraseTest extends \PHPUnit\Framework\TestCase
     public function testEmptyPhrase()
     {
         $oPhrase = new Phrase('', '');
+        $oPhrase->computeWordSets(array(), new TokensFullSet());
 
-        $this->assertEquals(
-            array(array('')),
-            $oPhrase->getWordSets()
-        );
+        $this->assertNull($oPhrase->getWordSets());
     }
 
 
     public function testSingleWordPhrase()
     {
         $oPhrase = new Phrase('a', '');
+        $oPhrase->computeWordSets(array('a'), new TokensFullSet());
 
         $this->assertEquals(
             '(a)',
@@ -43,20 +65,23 @@ class PhraseTest extends \PHPUnit\Framework\TestCase
     public function testMultiWordPhrase()
     {
         $oPhrase = new Phrase('a b', '');
+        $oPhrase->computeWordSets(array('a', 'b'), new TokensFullSet());
         $this->assertEquals(
             '(a b),(a|b)',
             $this->serializeSets($oPhrase->getWordSets())
         );
 
         $oPhrase = new Phrase('a b c', '');
+        $oPhrase->computeWordSets(array('a', 'b', 'c'), new TokensFullSet());
         $this->assertEquals(
-            '(a b c),(a|b c),(a|b|c),(a b|c)',
+            '(a b c),(a|b c),(a b|c),(a|b|c)',
             $this->serializeSets($oPhrase->getWordSets())
         );
 
         $oPhrase = new Phrase('a b c d', '');
+        $oPhrase->computeWordSets(array('a', 'b', 'c', 'd'), new TokensFullSet());
         $this->assertEquals(
-            '(a b c d),(a|b c d),(a|b|c d),(a|b|c|d),(a|b c|d),(a b|c d),(a b|c|d),(a b c|d)',
+            '(a b c d),(a b c|d),(a b|c d),(a|b c d),(a b|c|d),(a|b c|d),(a|b|c d),(a|b|c|d)',
             $this->serializeSets($oPhrase->getWordSets())
         );
     }
@@ -65,25 +90,50 @@ class PhraseTest extends \PHPUnit\Framework\TestCase
     public function testInverseWordSets()
     {
         $oPhrase = new Phrase('a b c', '');
+        $oPhrase->computeWordSets(array('a', 'b', 'c'), new TokensFullSet());
         $oPhrase->invertWordSets();
 
         $this->assertEquals(
-            '(a b c),(c|a b),(c|b|a),(b c|a)',
+            '(a b c),(b c|a),(c|a b),(c|b|a)',
             $this->serializeSets($oPhrase->getWordSets())
         );
     }
 
 
-    public function testMaxDepth()
+    public function testMaxWordSets()
     {
-        $oPhrase = new Phrase(join(' ', array_fill(0, 4, 'a')), '');
+        $aWords = array_fill(0, 4, 'a');
+        $oPhrase = new Phrase(join(' ', $aWords), '');
+        $oPhrase->computeWordSets($aWords, new TokensFullSet());
         $this->assertEquals(8, count($oPhrase->getWordSets()));
         $oPhrase->invertWordSets();
         $this->assertEquals(8, count($oPhrase->getWordSets()));
 
-        $oPhrase = new Phrase(join(' ', array_fill(0, 18, 'a')), '');
-        $this->assertEquals(41226, count($oPhrase->getWordSets()));
+        $aWords = array_fill(0, 18, 'a');
+        $oPhrase = new Phrase(join(' ', $aWords), '');
+        $oPhrase->computeWordSets($aWords, new TokensFullSet());
+        $this->assertEquals(100, count($oPhrase->getWordSets()));
         $oPhrase->invertWordSets();
-        $this->assertEquals(41226, count($oPhrase->getWordSets()));
+        $this->assertEquals(100, count($oPhrase->getWordSets()));
+    }
+
+
+    public function testPartialTokensShortTerm()
+    {
+        $oPhrase = new Phrase('a b c d', '');
+        $oPhrase->computeWordSets(array('a', 'b', 'c', 'd'), new TokensPartialSet(array('a', 'b', 'd', 'b c', 'b c d')));
+        $this->assertEquals(
+            '(a|b c d),(a|b c|d)',
+            $this->serializeSets($oPhrase->getWordSets())
+        );
+    }
+
+
+    public function testPartialTokensLongTerm()
+    {
+        $aWords = array_fill(0, 18, 'a');
+        $oPhrase = new Phrase(join(' ', $aWords), '');
+        $oPhrase->computeWordSets($aWords, new TokensPartialSet(array('a', 'a a a a a')));
+        $this->assertEquals(80, count($oPhrase->getWordSets()));
     }
 }