lib-php/tokenizer/icu_tokenizer.php

   1 <?php
   2 /**
   3  * SPDX-License-Identifier: GPL-2.0-only
   4  *
   5  * This file is part of Nominatim. (https://nominatim.org)
   6  *
   7  * Copyright (C) 2022 by the Nominatim developer community.
   8  * For a full list of authors see the git log.
   9  */
  10
  11 namespace Nominatim;
  12
  13 require_once(CONST_LibDir.'/SimpleWordList.php');
  14
  15 class Tokenizer
  16 {
  17     private $oDB;
  18
  19     private $oNormalizer;
  20     private $oTransliterator;
  21
  22     public function __construct(&$oDB)
  23     {
  24         $this->oDB =& $oDB;
  25         $this->oNormalizer = \Transliterator::createFromRules(CONST_Term_Normalization_Rules);
  26         $this->oTransliterator = \Transliterator::createFromRules(CONST_Transliteration);
  27     }
  28
  29     public function checkStatus()
  30     {
  31         $sSQL = 'SELECT word_id FROM word WHERE word_id is not null limit 1';
  32         $iWordID = $this->oDB->getOne($sSQL);
  33         if ($iWordID === false) {
  34             throw new \Exception('Query failed', 703);
  35         }
  36         if (!$iWordID) {
  37             throw new \Exception('No value', 704);
  38         }
  39     }
  40
  41
  42     public function normalizeString($sTerm)
  43     {
  44         if ($this->oNormalizer === null) {
  45             return $sTerm;
  46         }
  47
  48         return $this->oNormalizer->transliterate($sTerm);
  49     }
  50
  51
  52     public function mostFrequentWords($iNum)
  53     {
  54         $sSQL = "SELECT word FROM word WHERE type = 'W'";
  55         $sSQL .= "ORDER BY info->'count' DESC LIMIT ".$iNum;
  56         return $this->oDB->getCol($sSQL);
  57     }
  58
  59
  60     private function makeStandardWord($sTerm)
  61     {
  62         return trim($this->oTransliterator->transliterate(' '.$sTerm.' '));
  63     }
  64
  65
  66     public function tokensForSpecialTerm($sTerm)
  67     {
  68         $aResults = array();
  69
  70         $sSQL = "SELECT word_id, info->>'class' as class, info->>'type' as type ";
  71         $sSQL .= '   FROM word WHERE word_token = :term and type = \'S\'';
  72
  73         Debug::printVar('Term', $sTerm);
  74         Debug::printSQL($sSQL);
  75         $aSearchWords = $this->oDB->getAll($sSQL, array(':term' => $this->makeStandardWord($sTerm)));
  76
  77         Debug::printVar('Results', $aSearchWords);
  78
  79         foreach ($aSearchWords as $aSearchTerm) {
  80             $aResults[] = new \Nominatim\Token\SpecialTerm(
  81                 $aSearchTerm['word_id'],
  82                 $aSearchTerm['class'],
  83                 $aSearchTerm['type'],
  84                 \Nominatim\Operator::TYPE
  85             );
  86         }
  87
  88         Debug::printVar('Special term tokens', $aResults);
  89
  90         return $aResults;
  91     }
  92
  93
  94     public function extractTokensFromPhrases(&$aPhrases)
  95     {
  96         $sNormQuery = '';
  97         $aWordLists = array();
  98         $aTokens = array();
  99         foreach ($aPhrases as $iPhrase => $oPhrase) {
 100             $sNormQuery .= ','.$this->normalizeString($oPhrase->getPhrase());
 101             $sPhrase = $this->makeStandardWord($oPhrase->getPhrase());
 102             Debug::printVar('Phrase', $sPhrase);
 103
 104             $oWordList = new SimpleWordList($sPhrase);
 105             $aTokens = array_merge($aTokens, $oWordList->getTokens());
 106             $aWordLists[] = $oWordList;
 107         }
 108
 109         Debug::printVar('Tokens', $aTokens);
 110         Debug::printVar('WordLists', $aWordLists);
 111
 112         $oValidTokens = $this->computeValidTokens($aTokens, $sNormQuery);
 113
 114         foreach ($aPhrases as $iPhrase => $oPhrase) {
 115             $oPhrase->setWordSets($aWordLists[$iPhrase]->getWordSets($oValidTokens));
 116         }
 117
 118         return $oValidTokens;
 119     }
 120
 121
 122     private function computeValidTokens($aTokens, $sNormQuery)
 123     {
 124         $oValidTokens = new TokenList();
 125
 126         if (!empty($aTokens)) {
 127             $this->addTokensFromDB($oValidTokens, $aTokens, $sNormQuery);
 128
 129             // Try more interpretations for Tokens that could not be matched.
 130             foreach ($aTokens as $sToken) {
 131                 if ($sToken[0] != ' ' && !$oValidTokens->contains($sToken)) {
 132                     if (preg_match('/^([0-9]{5}) [0-9]{4}$/', $sToken, $aData)) {
 133                         // US ZIP+4 codes - merge in the 5-digit ZIP code
 134                         $oValidTokens->addToken(
 135                             $sToken,
 136                             new Token\Postcode(null, $aData[1], 'us')
 137                         );
 138                     } elseif (preg_match('/^[0-9]+$/', $sToken)) {
 139                         // Unknown single word token with a number.
 140                         // Assume it is a house number.
 141                         $oValidTokens->addToken(
 142                             $sToken,
 143                             new Token\HouseNumber(null, trim($sToken))
 144                         );
 145                     }
 146                 }
 147             }
 148         }
 149
 150         return $oValidTokens;
 151     }
 152
 153
 154     private function addTokensFromDB(&$oValidTokens, $aTokens, $sNormQuery)
 155     {
 156         // Check which tokens we have, get the ID numbers
 157         $sSQL = 'SELECT word_id, word_token, type, word,';
 158         $sSQL .= "      info->>'op' as operator,";
 159         $sSQL .= "      info->>'class' as class, info->>'type' as ctype,";
 160         $sSQL .= "      info->>'count' as count,";
 161         $sSQL .= "      info->>'lookup' as lookup";
 162         $sSQL .= ' FROM word WHERE word_token in (';
 163         $sSQL .= join(',', $this->oDB->getDBQuotedList($aTokens)).')';
 164
 165         Debug::printSQL($sSQL);
 166
 167         $aDBWords = $this->oDB->getAll($sSQL, null, 'Could not get word tokens.');
 168
 169         foreach ($aDBWords as $aWord) {
 170             $iId = (int) $aWord['word_id'];
 171             $sTok = $aWord['word_token'];
 172
 173             switch ($aWord['type']) {
 174                 case 'C':  // country name tokens
 175                     if ($aWord['word'] !== null) {
 176                         $oValidTokens->addToken(
 177                             $sTok,
 178                             new Token\Country($iId, $aWord['word'])
 179                         );
 180                     }
 181                     break;
 182                 case 'H':  // house number tokens
 183                     $sLookup = $aWord['lookup'] ?? $aWord['word_token'];
 184                     $oValidTokens->addToken($sTok, new Token\HouseNumber($iId, $sLookup));
 185                     break;
 186                 case 'P':  // postcode tokens
 187                     // Postcodes are not normalized, so they may have content
 188                     // that makes SQL injection possible. Reject postcodes
 189                     // that would need special escaping.
 190                     if ($aWord['word'] !== null
 191                         && pg_escape_string($aWord['word']) == $aWord['word']
 192                     ) {
 193                         $sNormPostcode = $this->normalizeString($aWord['word']);
 194                         if (strpos($sNormQuery, $sNormPostcode) !== false) {
 195                             $oValidTokens->addToken(
 196                                 $sTok,
 197                                 new Token\Postcode($iId, $aWord['word'], null)
 198                             );
 199                         }
 200                     }
 201                     break;
 202                 case 'S':  // tokens for classification terms (special phrases)
 203                     if ($aWord['class'] !== null && $aWord['ctype'] !== null) {
 204                         $oValidTokens->addToken($sTok, new Token\SpecialTerm(
 205                             $iId,
 206                             $aWord['class'],
 207                             $aWord['ctype'],
 208                             (isset($aWord['operator'])) ? Operator::NEAR : Operator::NONE
 209                         ));
 210                     }
 211                     break;
 212                 case 'W': // full-word tokens
 213                     $oValidTokens->addToken($sTok, new Token\Word(
 214                         $iId,
 215                         (int) $aWord['count'],
 216                         substr_count($aWord['word_token'], ' ')
 217                     ));
 218                     break;
 219                 case 'w':  // partial word terms
 220                     $oValidTokens->addToken($sTok, new Token\Partial(
 221                         $iId,
 222                         $aWord['word_token'],
 223                         (int) $aWord['count']
 224                     ));
 225                     break;
 226                 default:
 227                     break;
 228             }
 229         }
 230     }
 231 }