Revert "ICU: additional ranking by matching of normalised term"

This reverts commit b6a831443c8f46bfe1be59ecd1cb7a6847fdf948.

diff --git a/lib-php/TokenWord.php b/lib-php/TokenWord.php
index b9e28d91db7dde65cc235affd505b7b54081ce95..59456e35aaef3d309b8d2da23aca21df2d546b9f 100644 (file)
--- a/lib-php/TokenWord.php
+++ b/lib-php/TokenWord.php
@@ -13,15 +13,12 @@ class Word
     private $iSearchNameCount;
     /// Number of terms in the word.
     private $iTermCount;
     private $iSearchNameCount;
     /// Number of terms in the word.
     private $iTermCount;

-    /// Match score.
-    private $iMatchScore;

 
 

-    public function __construct($iId, $iSearchNameCount, $iTermCount, $iMatchScore = 1)
+    public function __construct($iId, $iSearchNameCount, $iTermCount)

     {
         $this->iId = $iId;
         $this->iSearchNameCount = $iSearchNameCount;
         $this->iTermCount = $iTermCount;
     {
         $this->iId = $iId;
         $this->iSearchNameCount = $iSearchNameCount;
         $this->iTermCount = $iTermCount;

-        $this->iMatchScore = $iMatchScore;

     }
 
     public function getId()
     }
 
     public function getId()


@@ -66,13 +63,13 @@ class Word
             if ($this->iTermCount > 1
                 && ($oPosition->isPhrase('') || !$oPosition->isFirstPhrase())
             ) {
             if ($this->iTermCount > 1
                 && ($oPosition->isPhrase('') || !$oPosition->isFirstPhrase())
             ) {

-                $oNewSearch = $oSearch->clone($this->iMatchScore);
+                $oNewSearch = $oSearch->clone(1);

                 $oNewSearch->addAddressToken($this->iId);
 
                 return array($oNewSearch);
             }
         } elseif (!$oSearch->hasName(true)) {
                 $oNewSearch->addAddressToken($this->iId);
 
                 return array($oNewSearch);
             }
         } elseif (!$oSearch->hasName(true)) {

-            $oNewSearch = $oSearch->clone($this->iMatchScore);
+            $oNewSearch = $oSearch->clone(1);

             $oNewSearch->addNameToken(
                 $this->iId,
                 CONST_Search_NameOnlySearchFrequencyThreshold
             $oNewSearch->addNameToken(
                 $this->iId,
                 CONST_Search_NameOnlySearchFrequencyThreshold


@@ -93,8 +90,7 @@ class Word
                 'Type' => 'word',
                 'Info' => array(
                            'count' => $this->iSearchNameCount,
                 'Type' => 'word',
                 'Info' => array(
                            'count' => $this->iSearchNameCount,

-                           'terms' => $this->iTermCount,
-                           'score' => $this->iMatchScore
+                           'terms' => $this->iTermCount

                           )
                );
     }
                           )
                );
     }

diff --git a/lib-php/tokenizer/icu_tokenizer.php b/lib-php/tokenizer/icu_tokenizer.php
index cebdac476829f240cf1183ca2371f91fbc604ba0..f4dd3aebd4db2a4e853df13037e75ecb64b3f47a 100644 (file)
--- a/lib-php/tokenizer/icu_tokenizer.php
+++ b/lib-php/tokenizer/icu_tokenizer.php
@@ -89,26 +89,19 @@ class Tokenizer
         $aWordLists = array();
         $aTokens = array();
         foreach ($aPhrases as $iPhrase => $oPhrase) {
         $aWordLists = array();
         $aTokens = array();
         foreach ($aPhrases as $iPhrase => $oPhrase) {

-            $sNormPhrase = $this->normalizeString($oPhrase->getPhrase());
-            Debug::printVar('Phrase', $sNormPhrase);
-
-            $oWordList = new SimpleWordList($sNormPhrase);
-
-            foreach ($oWordList->getTokens() as $sToken) {
-                $sTransToken = $this->makeStandardWord($sToken);
-                if (!isset($aTokens[$sTransToken])) {
-                    $aTokens[$sTransToken] = array();
-                }
-                $aTokens[$sTransToken][$sToken] = $sToken;
-            }
+            $sNormQuery .= ','.$this->normalizeString($oPhrase->getPhrase());
+            $sPhrase = $this->makeStandardWord($oPhrase->getPhrase());
+            Debug::printVar('Phrase', $sPhrase);

 
 

+            $oWordList = new SimpleWordList($sPhrase);
+            $aTokens = array_merge($aTokens, $oWordList->getTokens());

             $aWordLists[] = $oWordList;
         }
 
         Debug::printVar('Tokens', $aTokens);
         Debug::printVar('WordLists', $aWordLists);
 
             $aWordLists[] = $oWordList;
         }
 
         Debug::printVar('Tokens', $aTokens);
         Debug::printVar('WordLists', $aWordLists);
 

-        $oValidTokens = $this->computeValidTokens($aTokens);
+        $oValidTokens = $this->computeValidTokens($aTokens, $sNormQuery);

 
         foreach ($aPhrases as $iPhrase => $oPhrase) {
             $oPhrase->setWordSets($aWordLists[$iPhrase]->getWordSets($oValidTokens));
 
         foreach ($aPhrases as $iPhrase => $oPhrase) {
             $oPhrase->setWordSets($aWordLists[$iPhrase]->getWordSets($oValidTokens));


@@ -118,16 +111,16 @@ class Tokenizer
     }
 
 
     }
 
 

-    private function computeValidTokens($aTokens)
+    private function computeValidTokens($aTokens, $sNormQuery)

     {
         $oValidTokens = new TokenList();
 
         if (!empty($aTokens)) {
     {
         $oValidTokens = new TokenList();
 
         if (!empty($aTokens)) {

-            $this->addTokensFromDB($oValidTokens, $aTokens);
+            $this->addTokensFromDB($oValidTokens, $aTokens, $sNormQuery);

 
             // Try more interpretations for Tokens that could not be matched.
 
             // Try more interpretations for Tokens that could not be matched.

-            foreach ($aTokens as $sToken => $aNormalized) {
-                if (!$oValidTokens->contains($sToken)) {
+            foreach ($aTokens as $sToken) {
+                if ($sToken[0] != ' ' && !$oValidTokens->contains($sToken)) {

                     if (preg_match('/^([0-9]{5}) [0-9]{4}$/', $sToken, $aData)) {
                         // US ZIP+4 codes - merge in the 5-digit ZIP code
                         $oValidTokens->addToken(
                     if (preg_match('/^([0-9]{5}) [0-9]{4}$/', $sToken, $aData)) {
                         // US ZIP+4 codes - merge in the 5-digit ZIP code
                         $oValidTokens->addToken(


@@ -150,7 +143,7 @@ class Tokenizer
     }
 
 
     }
 
 

-    private function addTokensFromDB(&$oValidTokens, $aTokens)
+    private function addTokensFromDB(&$oValidTokens, $aTokens, $sNormQuery)

     {
         // Check which tokens we have, get the ID numbers
         $sSQL = 'SELECT word_id, word_token, type, word,';
     {
         // Check which tokens we have, get the ID numbers
         $sSQL = 'SELECT word_id, word_token, type, word,';


@@ -158,7 +151,7 @@ class Tokenizer
         $sSQL .= "      info->>'class' as class, info->>'type' as ctype,";
         $sSQL .= "      info->>'count' as count";
         $sSQL .= ' FROM word WHERE word_token in (';
         $sSQL .= "      info->>'class' as class, info->>'type' as ctype,";
         $sSQL .= "      info->>'count' as count";
         $sSQL .= ' FROM word WHERE word_token in (';

-        $sSQL .= join(',', $this->oDB->getDBQuotedList(array_keys($aTokens))).')';
+        $sSQL .= join(',', $this->oDB->getDBQuotedList($aTokens)).')';

 
         Debug::printSQL($sSQL);
 
 
         Debug::printSQL($sSQL);
 


@@ -167,23 +160,18 @@ class Tokenizer
         foreach ($aDBWords as $aWord) {
             $iId = (int) $aWord['word_id'];
             $sTok = $aWord['word_token'];
         foreach ($aDBWords as $aWord) {
             $iId = (int) $aWord['word_id'];
             $sTok = $aWord['word_token'];

-            $aNorms = $aTokens[$sTok];

 
             switch ($aWord['type']) {
                 case 'C':  // country name tokens
                     if ($aWord['word'] !== null) {
 
             switch ($aWord['type']) {
                 case 'C':  // country name tokens
                     if ($aWord['word'] !== null) {

-                        foreach ($aNorms as $sNorm) {
-                            $oValidTokens->addToken(
-                                $sNorm,
-                                new Token\Country($iId, $aWord['word'])
-                            );
-                        }
+                        $oValidTokens->addToken(
+                            $sTok,
+                            new Token\Country($iId, $aWord['word'])
+                        );

                     }
                     break;
                 case 'H':  // house number tokens
                     }
                     break;
                 case 'H':  // house number tokens

-                    foreach ($aNorms as $sNorm) {
-                        $oValidTokens->addToken($sNorm, new Token\HouseNumber($iId, $aWord['word_token']));
-                    }
+                    $oValidTokens->addToken($sTok, new Token\HouseNumber($iId, $aWord['word_token']));

                     break;
                 case 'P':  // postcode tokens
                     // Postcodes are not normalized, so they may have content
                     break;
                 case 'P':  // postcode tokens
                     // Postcodes are not normalized, so they may have content


@@ -193,48 +181,37 @@ class Tokenizer
                         && pg_escape_string($aWord['word']) == $aWord['word']
                     ) {
                         $sNormPostcode = $this->normalizeString($aWord['word']);
                         && pg_escape_string($aWord['word']) == $aWord['word']
                     ) {
                         $sNormPostcode = $this->normalizeString($aWord['word']);

-                        foreach ($aNorms as $sNorm) {
-                            if ($sNormPostcode == $sNorm) {
-                                $oValidTokens->addToken(
-                                    $sNorm,
-                                    new Token\Postcode($iId, $aWord['word'], null)
-                                );
-                            }
+                        if (strpos($sNormQuery, $sNormPostcode) !== false) {
+                            $oValidTokens->addToken(
+                                $sTok,
+                                new Token\Postcode($iId, $aWord['word'], null)
+                            );

                         }
                     }
                     break;
                 case 'S':  // tokens for classification terms (special phrases)
                     if ($aWord['class'] !== null && $aWord['ctype'] !== null) {
                         }
                     }
                     break;
                 case 'S':  // tokens for classification terms (special phrases)
                     if ($aWord['class'] !== null && $aWord['ctype'] !== null) {

-                        foreach ($aNorms as $sNorm) {
-                            if ($aWord['word'] == $sNorm) {
-                                $oValidTokens->addToken($sTok, new Token\SpecialTerm(
-                                    $iId,
-                                    $aWord['class'],
-                                    $aWord['ctype'],
-                                    (isset($aWord['operator'])) ? Operator::NEAR : Operator::NONE
-                                ));
-                            }
-                        }
-                    }
-                    break;
-                case 'W': // full-word tokens
-                    foreach ($aNorms as $sNorm) {
-                        $oValidTokens->addToken($sNorm, new Token\Word(
+                        $oValidTokens->addToken($sTok, new Token\SpecialTerm(

                             $iId,
                             $iId,

-                            (int) $aWord['count'],
-                            substr_count($aWord['word_token'], ' ') + 1,
-                            levenshtein($aWord['word'], $sNorm) + 1
+                            $aWord['class'],
+                            $aWord['ctype'],
+                            (isset($aWord['operator'])) ? Operator::NEAR : Operator::NONE

                         ));
                     }
                     break;
                         ));
                     }
                     break;

+                case 'W': // full-word tokens
+                    $oValidTokens->addToken($sTok, new Token\Word(
+                        $iId,
+                        (int) $aWord['count'],
+                        substr_count($aWord['word_token'], ' ')
+                    ));
+                    break;

                 case 'w':  // partial word terms
                 case 'w':  // partial word terms

-                    foreach ($aNorms as $sNorm) {
-                        $oValidTokens->addToken($sNorm, new Token\Partial(
-                            $iId,
-                            $aWord['word_token'],
-                            (int) $aWord['count']
-                        ));
-                    }
+                    $oValidTokens->addToken($sTok, new Token\Partial(
+                        $iId,
+                        $aWord['word_token'],
+                        (int) $aWord['count']
+                    ));

                     break;
                 default:
                     break;
                     break;
                 default:
                     break;