Merge remote-tracking branch 'lonvia/address-importance'

[nominatim.git] / website / search.php

diff --git a/website/search.php b/website/search.php
index 77a3d135cba4cc6fafa05f807855397520e4372a..dbb364048b3c1ec51a56bee2d2c442cc9adcb610 100755 (executable)
--- a/website/search.php
+++ b/website/search.php
@@ -286,7 +286,8 @@
                {
                        // Start with a blank search
                        $aSearches = array(
                {
                        // Start with a blank search
                        $aSearches = array(

-                               array('iSearchRank' => 0, 'iNamePhrase' => -1, 'sCountryCode' => false, 'aName'=>array(), 'aAddress'=>array(), 
+                               array('iSearchRank' => 0, 'iNamePhrase' => -1, 'sCountryCode' => false, 'aName'=>array(), 'aAddress'=>array(), 'aFullNameAddress'=>array(),
+                                       'aNameNonSearch'=>array(), 'aAddressNonSearch'=>array(),

                                        'sOperator'=>'', 'aFeatureName' => array(), 'sClass'=>'', 'sType'=>'', 'sHouseNumber'=>'', 'fLat'=>'', 'fLon'=>'', 'fRadius'=>'')
                        );
 
                                        'sOperator'=>'', 'aFeatureName' => array(), 'sClass'=>'', 'sType'=>'', 'sHouseNumber'=>'', 'fLat'=>'', 'fLon'=>'', 'fRadius'=>'')
                        );
 


@@ -398,7 +399,7 @@
                        // Check which tokens we have, get the ID numbers                       
                        $sSQL = 'select word_id,word_token, word, class, type, location, country_code, operator, search_name_count';
                        $sSQL .= ' from word where word_token in ('.join(',',array_map("getDBQuoted",$aTokens)).')';
                        // Check which tokens we have, get the ID numbers                       
                        $sSQL = 'select word_id,word_token, word, class, type, location, country_code, operator, search_name_count';
                        $sSQL .= ' from word where word_token in ('.join(',',array_map("getDBQuoted",$aTokens)).')';

-                       $sSQL .= ' and search_name_count < '.CONST_Max_Word_Frequency;
+//                     $sSQL .= ' and search_name_count < '.CONST_Max_Word_Frequency;

 //                     $sSQL .= ' group by word_token, word, class, type, location, country_code';
 
                        if (CONST_Debug) var_Dump($sSQL);
 //                     $sSQL .= ' group by word_token, word, class, type, location, country_code';
 
                        if (CONST_Debug) var_Dump($sSQL);


@@ -413,8 +414,16 @@
                                failInternalError("Could not get word tokens.", $sSQL, $aDatabaseWords);
                        }
                        $aPossibleMainWordIDs = array();
                                failInternalError("Could not get word tokens.", $sSQL, $aDatabaseWords);
                        }
                        $aPossibleMainWordIDs = array();

+                       $aWordFrequencyScores = array();

                        foreach($aDatabaseWords as $aToken)
                        {
                        foreach($aDatabaseWords as $aToken)
                        {

+                               // Very special case - require 2 letter country param to match the country code found
+                               if ($bStructuredPhrases && $aToken['country_code'] && !empty($aStructuredQuery['country']) 
+                                  && strlen($aStructuredQuery['country']) == 2 && strtolower($aStructuredQuery['country']) != $aToken['country_code'])
+                               {
+                                       continue;
+                               }
+

                                if (isset($aValidTokens[$aToken['word_token']]))
                                {
                                        $aValidTokens[$aToken['word_token']][] = $aToken;
                                if (isset($aValidTokens[$aToken['word_token']]))
                                {
                                        $aValidTokens[$aToken['word_token']][] = $aToken;


@@ -423,7 +432,8 @@
                                {
                                        $aValidTokens[$aToken['word_token']] = array($aToken);
                                }
                                {
                                        $aValidTokens[$aToken['word_token']] = array($aToken);
                                }

-                               if ($aToken['word_token'][0]==' ' && !$aToken['class'] && !$aToken['country_code']) $aPossibleMainWordIDs[$aToken['word_id']] = 1 + $aToken['search_name_count'];
+                               if ($aToken['word_token'][0]==' ' && !$aToken['class'] && !$aToken['country_code']) $aPossibleMainWordIDs[$aToken['word_id']] = 1;
+                               $aWordFrequencyScores[$aToken['word_id']] = $aToken['search_name_count'] + 1;

                        }
                        if (CONST_Debug) var_Dump($aPhrases, $aValidTokens);
 
                        }
                        if (CONST_Debug) var_Dump($aPhrases, $aValidTokens);
 


@@ -599,12 +609,13 @@
                                                                                {
                                                                                        if (sizeof($aSearch['aName']))
                                                                                        {
                                                                                {
                                                                                        if (sizeof($aSearch['aName']))
                                                                                        {

-                                                                                               if (($sPhraseType != 'street' && $sPhraseType != 'country') && (!isset($aValidTokens[$sToken]) || strlen($sToken) < 4 || strpos($sToken, ' ') !== false))
+                                                                                               if ((!$bStructuredPhrases || $iPhrase > 0) && $sPhraseType != 'country' && (!isset($aValidTokens[$sToken]) || strlen($sToken) < 4 || strpos($sToken, ' ') !== false))

                                                                                                {
                                                                                                        $aSearch['aAddress'][$aSearchTerm['word_id']] = $aSearchTerm['word_id'];
                                                                                                }
                                                                                                else
                                                                                                {
                                                                                                {
                                                                                                        $aSearch['aAddress'][$aSearchTerm['word_id']] = $aSearchTerm['word_id'];
                                                                                                }
                                                                                                else
                                                                                                {

+                                                                                                       $aCurrentSearch['aFullNameAddress'][$aSearchTerm['word_id']] = $aSearchTerm['word_id'];

                                                                                                        $aSearch['iSearchRank'] += 1000; // skip;
                                                                                                }
                                                                                        }
                                                                                                        $aSearch['iSearchRank'] += 1000; // skip;
                                                                                                }
                                                                                        }


@@ -624,12 +635,35 @@
                                                                        {
                                                                                if (isset($aSearchTerm['word_id']) && $aSearchTerm['word_id'])
                                                                                {
                                                                        {
                                                                                if (isset($aSearchTerm['word_id']) && $aSearchTerm['word_id'])
                                                                                {

-                                                                                       if (($sPhraseType != 'street') && sizeof($aCurrentSearch['aName']) && strlen($sToken) >= 4)
+                                                                                       if ((!$bStructuredPhrases || $iPhrase > 0) && sizeof($aCurrentSearch['aName']) && strlen($sToken) >= 4)

                                                                                        {
                                                                                                $aSearch = $aCurrentSearch;
                                                                                                $aSearch['iSearchRank'] += 1;
                                                                                        {
                                                                                                $aSearch = $aCurrentSearch;
                                                                                                $aSearch['iSearchRank'] += 1;

-                                                                                               $aSearch['aAddress'][$aSearchTerm['word_id']] = $aSearchTerm['word_id'];
-                                                                                               if ($aSearch['iSearchRank'] < $iMaxRank) $aNewWordsetSearches[] = $aSearch;
+                                                                                               if ($aWordFrequencyScores[$aSearchTerm['word_id']] < CONST_Max_Word_Frequency)
+                                                                                               {
+                                                                                                       $aSearch['aAddress'][$aSearchTerm['word_id']] = $aSearchTerm['word_id'];
+                                                                                                       if ($aSearch['iSearchRank'] < $iMaxRank) $aNewWordsetSearches[] = $aSearch;
+                                                                                               }
+                                                                                               elseif (isset($aValidTokens[' '.$sToken])) // revert to the token version?
+                                                                                               {
+                                                                                                       foreach($aValidTokens[' '.$sToken] as $aSearchTermToken)
+                                                                                                       {
+                                                                                                               if (empty($aSearchTermToken['country_code']) 
+                                                                                                                       && empty($aSearchTermToken['lat'])
+                                                                                                                       && empty($aSearchTermToken['class']))
+                                                                                                               {
+                                                                                                                       $aSearch = $aCurrentSearch;
+                                                                                                                       $aSearch['iSearchRank'] += 1;
+                                                                                                                       $aSearch['aAddress'][$aSearchTermToken['word_id']] = $aSearchTermToken['word_id'];
+                                                                                                                       if ($aSearch['iSearchRank'] < $iMaxRank) $aNewWordsetSearches[] = $aSearch;
+                                                                                                               }
+                                                                                                       }
+                                                                                               }
+                                                                                               else
+                                                                                               {
+                                                                                                       $aSearch['aAddressNonSearch'][$aSearchTerm['word_id']] = $aSearchTerm['word_id'];
+                                                                                                       if ($aSearch['iSearchRank'] < $iMaxRank) $aNewWordsetSearches[] = $aSearch;
+                                                                                               }

                                                                                        }
 
                                                                                        if (!sizeof($aCurrentSearch['aName']) || $aCurrentSearch['iNamePhrase'] == $iPhrase)
                                                                                        }
 
                                                                                        if (!sizeof($aCurrentSearch['aName']) || $aCurrentSearch['iNamePhrase'] == $iPhrase)


@@ -637,7 +671,10 @@
                                                                                                $aSearch = $aCurrentSearch;
                                                                                                $aSearch['iSearchRank'] += 2;
                                                                                                if (preg_match('#^[0-9]+$#', $sToken)) $aSearch['iSearchRank'] += 2;
                                                                                                $aSearch = $aCurrentSearch;
                                                                                                $aSearch['iSearchRank'] += 2;
                                                                                                if (preg_match('#^[0-9]+$#', $sToken)) $aSearch['iSearchRank'] += 2;

-                                                                                               $aSearch['aName'][$aSearchTerm['word_id']] = $aSearchTerm['word_id'];
+                                                                                               if ($aWordFrequencyScores[$aSearchTerm['word_id']] < CONST_Max_Word_Frequency)
+                                                                                                       $aSearch['aName'][$aSearchTerm['word_id']] = $aSearchTerm['word_id'];
+                                                                                               else
+                                                                                                       $aSearch['aNameNonSearch'][$aSearchTerm['word_id']] = $aSearchTerm['word_id'];

                                                                                                $aSearch['iNamePhrase'] = $iPhrase;
                                                                                                if ($aSearch['iSearchRank'] < $iMaxRank) $aNewWordsetSearches[] = $aSearch;
                                                                                        }
                                                                                                $aSearch['iNamePhrase'] = $iPhrase;
                                                                                                if ($aSearch['iSearchRank'] < $iMaxRank) $aNewWordsetSearches[] = $aSearch;
                                                                                        }


@@ -885,18 +922,20 @@
                                                        // TODO: filter out the pointless search terms (2 letter name tokens and less)
                                                        // they might be right - but they are just too darned expensive to run
                                                        if (sizeof($aSearch['aName'])) $aTerms[] = "name_vector @> ARRAY[".join($aSearch['aName'],",")."]";
                                                        // TODO: filter out the pointless search terms (2 letter name tokens and less)
                                                        // they might be right - but they are just too darned expensive to run
                                                        if (sizeof($aSearch['aName'])) $aTerms[] = "name_vector @> ARRAY[".join($aSearch['aName'],",")."]";

+                                                       if (sizeof($aSearch['aNameNonSearch'])) $aTerms[] = "array_cat(name_vector,ARRAY[]::integer[]) @> ARRAY[".join($aSearch['aNameNonSearch'],",")."]";

                                                        if (sizeof($aSearch['aAddress']) && $aSearch['aName'] != $aSearch['aAddress']) 
                                                        {
                                                                // For infrequent name terms disable index usage for address
                                                                if (CONST_Search_NameOnlySearchFrequencyThreshold && 
                                                                        sizeof($aSearch['aName']) == 1 && 
                                                        if (sizeof($aSearch['aAddress']) && $aSearch['aName'] != $aSearch['aAddress']) 
                                                        {
                                                                // For infrequent name terms disable index usage for address
                                                                if (CONST_Search_NameOnlySearchFrequencyThreshold && 
                                                                        sizeof($aSearch['aName']) == 1 && 

-                                                                       $aPossibleMainWordIDs[$aSearch['aName'][reset($aSearch['aName'])]] < CONST_Search_NameOnlySearchFrequencyThreshold)
+                                                                       $aWordFrequencyScores[$aSearch['aName'][reset($aSearch['aName'])]] < CONST_Search_NameOnlySearchFrequencyThreshold)

                                                                {
                                                                {

-                                                                       $aTerms[] = "array_cat(nameaddress_vector,ARRAY[]::integer[]) @> ARRAY[".join($aSearch['aAddress'],",")."]";
+                                                                       $aTerms[] = "array_cat(nameaddress_vector,ARRAY[]::integer[]) @> ARRAY[".join(array_merge($aSearch['aAddress'],$aSearch['aAddressNonSearch']),",")."]";

                                                                }
                                                                else
                                                                {
                                                                        $aTerms[] = "nameaddress_vector @> ARRAY[".join($aSearch['aAddress'],",")."]";
                                                                }
                                                                else
                                                                {
                                                                        $aTerms[] = "nameaddress_vector @> ARRAY[".join($aSearch['aAddress'],",")."]";

+                                                                       if (sizeof($aSearch['aAddressNonSearch'])) $aTerms[] = "array_cat(nameaddress_vector,ARRAY[]::integer[]) @> ARRAY[".join($aSearch['aAddressNonSearch'],",")."]";

                                                                }
                                                        }
                                                        if ($aSearch['sCountryCode']) $aTerms[] = "country_code = '".pg_escape_string($aSearch['sCountryCode'])."'";
                                                                }
                                                        }
                                                        if ($aSearch['sCountryCode']) $aTerms[] = "country_code = '".pg_escape_string($aSearch['sCountryCode'])."'";


@@ -918,11 +957,13 @@
                                                        if ($bBoundingBoxSearch) $aTerms[] = "centroid && $sViewboxSmallSQL";
                                                        if ($sNearPointSQL) $aOrder[] = "ST_Distance($sNearPointSQL, centroid) asc";
 
                                                        if ($bBoundingBoxSearch) $aTerms[] = "centroid && $sViewboxSmallSQL";
                                                        if ($sNearPointSQL) $aOrder[] = "ST_Distance($sNearPointSQL, centroid) asc";
 

-                                                       $sImportanceSQL = 'case when importance = 0 OR importance IS NULL then 0.75-(search_rank::float/40) else importance end';
-
+                                                       $sImportanceSQL = '(case when importance = 0 OR importance IS NULL then 0.75-(search_rank::float/40) else importance end)';

                                                        if ($sViewboxSmallSQL) $sImportanceSQL .= " * case when ST_Contains($sViewboxSmallSQL, centroid) THEN 1 ELSE 0.5 END";
                                                        if ($sViewboxLargeSQL) $sImportanceSQL .= " * case when ST_Contains($sViewboxLargeSQL, centroid) THEN 1 ELSE 0.5 END";
                                                        $aOrder[] = "$sImportanceSQL DESC";
                                                        if ($sViewboxSmallSQL) $sImportanceSQL .= " * case when ST_Contains($sViewboxSmallSQL, centroid) THEN 1 ELSE 0.5 END";
                                                        if ($sViewboxLargeSQL) $sImportanceSQL .= " * case when ST_Contains($sViewboxLargeSQL, centroid) THEN 1 ELSE 0.5 END";
                                                        $aOrder[] = "$sImportanceSQL DESC";

+                                                       if (sizeof($aSearch['aFullNameAddress']))
+                                                               $aOrder[] = '(select count(*) from (select unnest(ARRAY['.join($aSearch['aFullNameAddress'],",").']) INTERSECT select unnest(nameaddress_vector))s) DESC';
+

                                                
                                                        if (sizeof($aTerms))
                                                        {
                                                
                                                        if (sizeof($aTerms))
                                                        {


@@ -1169,16 +1210,17 @@
                                        $sSQL .= "get_name_by_language(name, ARRAY['ref']) as ref,";
                                        $sSQL .= "avg(ST_X(centroid)) as lon,avg(ST_Y(centroid)) as lat, ";
 //                                     $sSQL .= $sOrderSQL." as porder, ";
                                        $sSQL .= "get_name_by_language(name, ARRAY['ref']) as ref,";
                                        $sSQL .= "avg(ST_X(centroid)) as lon,avg(ST_Y(centroid)) as lat, ";
 //                                     $sSQL .= $sOrderSQL." as porder, ";

-                                       $sSQL .= "coalesce(importance,0.75-(rank_search::float/40)) as importance ";
+                                       $sSQL .= "coalesce(importance,0.75-(rank_search::float/40)) as importance, ";
+                                       $sSQL .= "(select max(p.importance*(p.rank_address+2)) from place_addressline s, placex p where s.place_id = min(placex.place_id) and p.place_id = s.address_place_id and s.isaddress and p.importance is not null) as addressimportance ";

                                        $sSQL .= "from placex where place_id in ($sPlaceIDs) ";
                                        $sSQL .= "and placex.rank_address between $iMinAddressRank and $iMaxAddressRank ";
                                        if ($sAllowedTypesSQLList) $sSQL .= "and placex.class in $sAllowedTypesSQLList ";
                                        $sSQL .= "and linked_place_id is null ";
                                        $sSQL .= "group by osm_type,osm_id,class,type,admin_level,rank_search,rank_address,calculated_country_code,importance";
                                        if (!$bDeDupe) $sSQL .= ",place_id";
                                        $sSQL .= "from placex where place_id in ($sPlaceIDs) ";
                                        $sSQL .= "and placex.rank_address between $iMinAddressRank and $iMaxAddressRank ";
                                        if ($sAllowedTypesSQLList) $sSQL .= "and placex.class in $sAllowedTypesSQLList ";
                                        $sSQL .= "and linked_place_id is null ";
                                        $sSQL .= "group by osm_type,osm_id,class,type,admin_level,rank_search,rank_address,calculated_country_code,importance";
                                        if (!$bDeDupe) $sSQL .= ",place_id";

-                                       $sSQL .= ",get_address_by_language(place_id, $sLanguagePrefArraySQL) ";
-                                       $sSQL .= ",get_name_by_language(name, $sLanguagePrefArraySQL) ";
-                                       $sSQL .= ",get_name_by_language(name, ARRAY['ref']) ";
+                                       $sSQL .= ",langaddress ";
+                                       $sSQL .= ",placename ";
+                                       $sSQL .= ",ref ";

                                        $sSQL .= " union ";
                                        $sSQL .= "select 'T' as osm_type,place_id as osm_id,'place' as class,'house' as type,null as admin_level,30 as rank_search,30 as rank_address,min(place_id) as place_id,'us' as country_code,";
                                        $sSQL .= "get_address_by_language(place_id, $sLanguagePrefArraySQL) as langaddress,";
                                        $sSQL .= " union ";
                                        $sSQL .= "select 'T' as osm_type,place_id as osm_id,'place' as class,'house' as type,null as admin_level,30 as rank_search,30 as rank_address,min(place_id) as place_id,'us' as country_code,";
                                        $sSQL .= "get_address_by_language(place_id, $sLanguagePrefArraySQL) as langaddress,";


@@ -1186,7 +1228,8 @@
                                        $sSQL .= "null as ref,";
                                        $sSQL .= "avg(ST_X(centroid)) as lon,avg(ST_Y(centroid)) as lat, ";
 //                                     $sSQL .= $sOrderSQL." as porder, ";
                                        $sSQL .= "null as ref,";
                                        $sSQL .= "avg(ST_X(centroid)) as lon,avg(ST_Y(centroid)) as lat, ";
 //                                     $sSQL .= $sOrderSQL." as porder, ";

-                                       $sSQL .= "-0.15 as importance ";
+                                       $sSQL .= "-0.15 as importance, ";
+                                       $sSQL .= "(select max(p.importance*(p.rank_address+2)) from place_addressline s, placex p where s.place_id = min(location_property_tiger.place_id) and p.place_id = s.address_place_id and s.isaddress and p.importance is not null) as addressimportance ";

                                        $sSQL .= "from location_property_tiger where place_id in ($sPlaceIDs) ";
                                        $sSQL .= "and 30 between $iMinAddressRank and $iMaxAddressRank ";
                                        $sSQL .= "group by place_id";
                                        $sSQL .= "from location_property_tiger where place_id in ($sPlaceIDs) ";
                                        $sSQL .= "and 30 between $iMinAddressRank and $iMaxAddressRank ";
                                        $sSQL .= "group by place_id";


@@ -1198,7 +1241,8 @@
                                        $sSQL .= "null as ref,";
                                        $sSQL .= "avg(ST_X(centroid)) as lon,avg(ST_Y(centroid)) as lat, ";
 //                                     $sSQL .= $sOrderSQL." as porder, ";
                                        $sSQL .= "null as ref,";
                                        $sSQL .= "avg(ST_X(centroid)) as lon,avg(ST_Y(centroid)) as lat, ";
 //                                     $sSQL .= $sOrderSQL." as porder, ";

-                                       $sSQL .= "-0.10 as importance ";
+                                       $sSQL .= "-0.10 as importance, ";
+                                       $sSQL .= "(select max(p.importance*(p.rank_address+2)) from place_addressline s, placex p where s.place_id = min(location_property_aux.place_id) and p.place_id = s.address_place_id and s.isaddress and p.importance is not null) as addressimportance ";

                                        $sSQL .= "from location_property_aux where place_id in ($sPlaceIDs) ";
                                        $sSQL .= "and 30 between $iMinAddressRank and $iMaxAddressRank ";
                                        $sSQL .= "group by place_id";
                                        $sSQL .= "from location_property_aux where place_id in ($sPlaceIDs) ";
                                        $sSQL .= "and 30 between $iMinAddressRank and $iMaxAddressRank ";
                                        $sSQL .= "group by place_id";


@@ -1294,7 +1338,7 @@
 //var_Dump($aSearchResults);
 //exit;
        $aClassType = getClassTypesWithImportance();
 //var_Dump($aSearchResults);
 //exit;
        $aClassType = getClassTypesWithImportance();

-       $aRecheckWords = preg_split('/\b/',$sQuery);
+       $aRecheckWords = preg_split('/\b/u',$sQuery);

        foreach($aRecheckWords as $i => $sWord)
        {
                if (!$sWord) unset($aRecheckWords[$i]);
        foreach($aRecheckWords as $i => $sWord)
        {
                if (!$sWord) unset($aRecheckWords[$i]);


@@ -1444,6 +1488,7 @@
                {
                        if (stripos($sAddress, $sWord)!==false) $iCountWords++;
                }
                {
                        if (stripos($sAddress, $sWord)!==false) $iCountWords++;
                }

+

                $aResult['importance'] = $aResult['importance'] + ($iCountWords*0.1); // 0.1 is a completely arbitrary number but something in the range 0.1 to 0.5 would seem right
 
 //if (CONST_Debug) var_dump($aResult['class'].':'.$aResult['type'].':'.$aResult['admin_level']);
                $aResult['importance'] = $aResult['importance'] + ($iCountWords*0.1); // 0.1 is a completely arbitrary number but something in the range 0.1 to 0.5 would seem right
 
 //if (CONST_Debug) var_dump($aResult['class'].':'.$aResult['type'].':'.$aResult['admin_level']);


@@ -1464,7 +1509,7 @@
                }
 */
                $aResult['name'] = $aResult['langaddress'];
                }
 */
                $aResult['name'] = $aResult['langaddress'];

-               $aResult['foundorder'] = $iResNum;
+               $aResult['foundorder'] = -$aResult['addressimportance'];

                $aSearchResults[$iResNum] = $aResult;
        }
        uasort($aSearchResults, 'byImportance');
                $aSearchResults[$iResNum] = $aResult;
        }
        uasort($aSearchResults, 'byImportance');


@@ -1498,7 +1543,7 @@
                if (sizeof($aSearchResults) >= $iFinalLimit) break;
        }
 
                if (sizeof($aSearchResults) >= $iFinalLimit) break;
        }
 

-       $sDataDate = $oDB->getOne("select TO_CHAR(lastimportdate - '1 day'::interval,'YYYY/MM/DD') from import_status limit 1");
+       $sDataDate = $oDB->getOne("select TO_CHAR(lastimportdate - '2 minutes'::interval,'YYYY/MM/DD HH24:MI')||' GMT' from import_status limit 1");

 
        if (isset($_GET['nearlat']) && isset($_GET['nearlon']))
        {
 
        if (isset($_GET['nearlat']) && isset($_GET['nearlon']))
        {