X-Git-Url: https://git.openstreetmap.org./nominatim.git/blobdiff_plain/2ae293aeb629050f03b3f9b88b80c3c1e724df5e..0fb8eade136ea03e7853aca0795ca69833c33661:/lib-php/tokenizer/legacy_icu_tokenizer.php?ds=sidebyside diff --git a/lib-php/tokenizer/legacy_icu_tokenizer.php b/lib-php/tokenizer/legacy_icu_tokenizer.php index 09cfe70f..690ef136 100644 --- a/lib-php/tokenizer/legacy_icu_tokenizer.php +++ b/lib-php/tokenizer/legacy_icu_tokenizer.php @@ -8,7 +8,6 @@ class Tokenizer private $oNormalizer; private $oTransliterator; - private $aCountryRestriction; public function __construct(&$oDB) { @@ -19,23 +18,17 @@ class Tokenizer public function checkStatus() { - $sSQL = "SELECT word_id FROM word WHERE word_token IN (' a')"; + $sSQL = 'SELECT word_id FROM word WHERE word_id is not null limit 1'; $iWordID = $this->oDB->getOne($sSQL); if ($iWordID === false) { - throw new Exception('Query failed', 703); + throw new \Exception('Query failed', 703); } if (!$iWordID) { - throw new Exception('No value', 704); + throw new \Exception('No value', 704); } } - public function setCountryRestriction($aCountries) - { - $this->aCountryRestriction = $aCountries; - } - - public function normalizeString($sTerm) { if ($this->oNormalizer === null) { @@ -47,9 +40,7 @@ class Tokenizer private function makeStandardWord($sTerm) { - $sNorm = ' '.$this->oTransliterator->transliterate($sTerm).' '; - - return trim(str_replace(CONST_Abbreviations[0], CONST_Abbreviations[1], $sNorm)); + return trim($this->oTransliterator->transliterate(' '.$sTerm.' ')); } @@ -57,9 +48,8 @@ class Tokenizer { $aResults = array(); - $sSQL = 'SELECT word_id, class, type FROM word '; - $sSQL .= ' WHERE word_token = \' \' || :term'; - $sSQL .= ' AND class is not null AND class not in (\'place\')'; + $sSQL = "SELECT word_id, info->>'class' as class, info->>'type' as type "; + $sSQL .= ' FROM word WHERE word_token = :term and type = \'S\''; Debug::printVar('Term', $sTerm); Debug::printSQL($sSQL); @@ -90,6 +80,7 @@ class Tokenizer foreach ($aPhrases as $iPhrase => $oPhrase) { $sNormQuery .= ','.$this->normalizeString($oPhrase->getPhrase()); $sPhrase = $this->makeStandardWord($oPhrase->getPhrase()); + Debug::printVar('Phrase', $sPhrase); if (strlen($sPhrase) > 0) { $aWords = explode(' ', $sPhrase); Tokenizer::addTokens($aTokens, $aWords); @@ -121,14 +112,14 @@ class Tokenizer // Try more interpretations for Tokens that could not be matched. foreach ($aTokens as $sToken) { - if ($sToken[0] == ' ' && !$oValidTokens->contains($sToken)) { - if (preg_match('/^ ([0-9]{5}) [0-9]{4}$/', $sToken, $aData)) { + if ($sToken[0] != ' ' && !$oValidTokens->contains($sToken)) { + if (preg_match('/^([0-9]{5}) [0-9]{4}$/', $sToken, $aData)) { // US ZIP+4 codes - merge in the 5-digit ZIP code $oValidTokens->addToken( $sToken, new Token\Postcode(null, $aData[1], 'us') ); - } elseif (preg_match('/^ [0-9]+$/', $sToken)) { + } elseif (preg_match('/^[0-9]+$/', $sToken)) { // Unknown single word token with a number. // Assume it is a house number. $oValidTokens->addToken( @@ -147,8 +138,10 @@ class Tokenizer private function addTokensFromDB(&$oValidTokens, $aTokens, $sNormQuery) { // Check which tokens we have, get the ID numbers - $sSQL = 'SELECT word_id, word_token, word, class, type, country_code,'; - $sSQL .= ' operator, coalesce(search_name_count, 0) as count'; + $sSQL = 'SELECT word_id, word_token, type, word,'; + $sSQL .= " info->>'op' as operator,"; + $sSQL .= " info->>'class' as class, info->>'type' as ctype,"; + $sSQL .= " info->>'count' as count"; $sSQL .= ' FROM word WHERE word_token in ('; $sSQL .= join(',', $this->oDB->getDBQuotedList($aTokens)).')'; @@ -157,56 +150,63 @@ class Tokenizer $aDBWords = $this->oDB->getAll($sSQL, null, 'Could not get word tokens.'); foreach ($aDBWords as $aWord) { - $oToken = null; $iId = (int) $aWord['word_id']; + $sTok = $aWord['word_token']; - if ($aWord['class']) { - // Special terms need to appear in their normalized form. - // (postcodes are not normalized in the word table) - $sNormWord = $this->normalizeString($aWord['word']); - if ($aWord['word'] && strpos($sNormQuery, $sNormWord) === false) { - continue; - } - - if ($aWord['class'] == 'place' && $aWord['type'] == 'house') { - $oToken = new Token\HouseNumber($iId, trim($aWord['word_token'])); - } elseif ($aWord['class'] == 'place' && $aWord['type'] == 'postcode') { - if ($aWord['word'] + switch ($aWord['type']) { + case 'C': // country name tokens + if ($aWord['word'] !== null) { + $oValidTokens->addToken( + $sTok, + new Token\Country($iId, $aWord['word']) + ); + } + break; + case 'H': // house number tokens + $oValidTokens->addToken($sTok, new Token\HouseNumber($iId, $aWord['word_token'])); + break; + case 'P': // postcode tokens + // Postcodes are not normalized, so they may have content + // that makes SQL injection possible. Reject postcodes + // that would need special escaping. + if ($aWord['word'] !== null && pg_escape_string($aWord['word']) == $aWord['word'] ) { - $oToken = new Token\Postcode( + $sNormPostcode = $this->normalizeString($aWord['word']); + if (strpos($sNormQuery, $sNormPostcode) !== false) { + $oValidTokens->addToken( + $sTok, + new Token\Postcode($iId, $aWord['word'], null) + ); + } + } + break; + case 'S': // tokens for classification terms (special phrases) + if ($aWord['class'] !== null && $aWord['ctype'] !== null) { + $oValidTokens->addToken($sTok, new Token\SpecialTerm( $iId, - $aWord['word'], - $aWord['country_code'] - ); + $aWord['class'], + $aWord['ctype'], + (isset($aWord['operator'])) ? Operator::NEAR : Operator::NONE + )); } - } else { - // near and in operator the same at the moment - $oToken = new Token\SpecialTerm( + break; + case 'W': // full-word tokens + $oValidTokens->addToken($sTok, new Token\Word( $iId, - $aWord['class'], - $aWord['type'], - $aWord['operator'] ? Operator::NEAR : Operator::NONE - ); - } - } elseif ($aWord['country_code']) { - // Filter country tokens that do not match restricted countries. - if (!$this->aCountryRestriction - || in_array($aWord['country_code'], $this->aCountryRestriction) - ) { - $oToken = new Token\Country($iId, $aWord['country_code']); - } - } else { - $oToken = new Token\Word( - $iId, - $aWord['word_token'][0] != ' ', - (int) $aWord['count'], - substr_count($aWord['word_token'], ' ') - ); - } - - if ($oToken) { - $oValidTokens->addToken($aWord['word_token'], $oToken); + (int) $aWord['count'], + substr_count($aWord['word_token'], ' ') + )); + break; + case 'w': // partial word terms + $oValidTokens->addToken($sTok, new Token\Partial( + $iId, + $aWord['word_token'], + (int) $aWord['count'] + )); + break; + default: + break; } } } @@ -225,12 +225,10 @@ class Tokenizer for ($i = 0; $i < $iNumWords; $i++) { $sPhrase = $aWords[$i]; - $aTokens[' '.$sPhrase] = ' '.$sPhrase; $aTokens[$sPhrase] = $sPhrase; for ($j = $i + 1; $j < $iNumWords; $j++) { $sPhrase .= ' '.$aWords[$j]; - $aTokens[' '.$sPhrase] = ' '.$sPhrase; $aTokens[$sPhrase] = $sPhrase; } }