From 73566a9f1584b348bc5357eb155f030e28070735 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Thu, 6 Aug 2020 16:15:17 +0200 Subject: [PATCH] remove unused functions from setup and update Removes the defunct --osmosis-init and --no-api switches and the unsupported (and unnecessary) deduplicate. Also removes 'experimental' from --setup-website as this is a required function now. --- utils/setup.php | 3 +- utils/update.php | 76 +----------------------------------------------- 2 files changed, 2 insertions(+), 77 deletions(-) diff --git a/utils/setup.php b/utils/setup.php index 23b7b3a6..5d323cee 100644 --- a/utils/setup.php +++ b/utils/setup.php @@ -38,13 +38,12 @@ $aCMDOptions array('disable-token-precalc', '', 0, 1, 0, 0, 'bool', 'Disable name precalculation (EXPERT)'), array('import-tiger-data', '', 0, 1, 0, 0, 'bool', 'Import tiger data (not included in \'all\')'), array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'), - array('osmosis-init', '', 0, 1, 0, 0, 'bool', 'Generate default osmosis configuration'), array('index', '', 0, 1, 0, 0, 'bool', 'Index the data'), array('index-noanalyse', '', 0, 1, 0, 0, 'bool', 'Do not perform analyse operations during index (EXPERT)'), array('create-search-indices', '', 0, 1, 0, 0, 'bool', 'Create additional indices required for search and update'), array('create-country-names', '', 0, 1, 0, 0, 'bool', 'Create default list of searchable country names'), array('drop', '', 0, 1, 0, 0, 'bool', 'Drop tables needed for updates, making the database readonly (EXPERIMENTAL)'), - array('setup-website', '', 0, 1, 0, 0, 'bool', 'Used to compile environment variables for the website (EXPERIMENTAL)'), + array('setup-website', '', 0, 1, 0, 0, 'bool', 'Used to compile environment variables for the website'), ); // $aCMDOptions passed to getCmdOpt by reference diff --git a/utils/update.php b/utils/update.php index cba58d31..9e74b4ba 100644 --- a/utils/update.php +++ b/utils/update.php @@ -39,11 +39,9 @@ $aCMDOptions array('index-rank', '', 0, 1, 1, 1, 'int', 'Rank to start indexing from'), array('index-instances', '', 0, 1, 1, 1, 'int', 'Number of indexing instances (threads)'), - array('deduplicate', '', 0, 1, 0, 0, 'bool', 'Deduplicate tokens'), array('recompute-word-counts', '', 0, 1, 0, 0, 'bool', 'Compute frequency of full-word search terms'), array('update-address-levels', '', 0, 1, 0, 0, 'bool', 'Reimport address level configuration (EXPERT)'), - array('recompute-importance', '', 0, 1, 0, 0, 'bool', 'Recompute place importances'), - array('no-npi', '', 0, 1, 0, 0, 'bool', '(obsolete)'), + array('recompute-importance', '', 0, 1, 0, 0, 'bool', 'Recompute place importances') ); getCmdOpt($_SERVER['argv'], $aCMDOptions, $aResult, true, true); @@ -272,78 +270,6 @@ if ($bHaveDiff) { } } -if ($aResult['deduplicate']) { - $oDB = new Nominatim\DB(); - $oDB->connect(); - - if ($oDB->getPostgresVersion() < 9.3) { - fail('ERROR: deduplicate is only currently supported in postgresql 9.3'); - } - - $sSQL = 'select partition from country_name order by country_code'; - $aPartitions = $oDB->getCol($sSQL); - $aPartitions[] = 0; - - // we don't care about empty search_name_* partitions, they can't contain mentions of duplicates - foreach ($aPartitions as $i => $sPartition) { - $sSQL = 'select count(*) from search_name_'.$sPartition; - $nEntries = $oDB->getOne($sSQL); - if ($nEntries == 0) { - unset($aPartitions[$i]); - } - } - - $sSQL = "select word_token,count(*) from word where substr(word_token, 1, 1) = ' '"; - $sSQL .= ' and class is null and type is null and country_code is null'; - $sSQL .= ' group by word_token having count(*) > 1 order by word_token'; - $aDuplicateTokens = $oDB->getAll($sSQL); - foreach ($aDuplicateTokens as $aToken) { - if (trim($aToken['word_token']) == '' || trim($aToken['word_token']) == '-') continue; - echo 'Deduping '.$aToken['word_token']."\n"; - $sSQL = 'select word_id,'; - $sSQL .= ' (select count(*) from search_name where nameaddress_vector @> ARRAY[word_id]) as num'; - $sSQL .= " from word where word_token = '".$aToken['word_token']; - $sSQL .= "' and class is null and type is null and country_code is null order by num desc"; - $aTokenSet = $oDB->getAll($sSQL); - - $aKeep = array_shift($aTokenSet); - $iKeepID = $aKeep['word_id']; - - foreach ($aTokenSet as $aRemove) { - $sSQL = 'update search_name set'; - $sSQL .= ' name_vector = array_replace(name_vector,'.$aRemove['word_id'].','.$iKeepID.'),'; - $sSQL .= ' nameaddress_vector = array_replace(nameaddress_vector,'.$aRemove['word_id'].','.$iKeepID.')'; - $sSQL .= ' where name_vector @> ARRAY['.$aRemove['word_id'].']'; - $oDB->exec($sSQL); - - $sSQL = 'update search_name set'; - $sSQL .= ' nameaddress_vector = array_replace(nameaddress_vector,'.$aRemove['word_id'].','.$iKeepID.')'; - $sSQL .= ' where nameaddress_vector @> ARRAY['.$aRemove['word_id'].']'; - $oDB->exec($sSQL); - - $sSQL = 'update location_area_country set'; - $sSQL .= ' keywords = array_replace(keywords,'.$aRemove['word_id'].','.$iKeepID.')'; - $sSQL .= ' where keywords @> ARRAY['.$aRemove['word_id'].']'; - $oDB->exec($sSQL); - - foreach ($aPartitions as $sPartition) { - $sSQL = 'update search_name_'.$sPartition.' set'; - $sSQL .= ' name_vector = array_replace(name_vector,'.$aRemove['word_id'].','.$iKeepID.')'; - $sSQL .= ' where name_vector @> ARRAY['.$aRemove['word_id'].']'; - $oDB->exec($sSQL); - - $sSQL = 'update location_area_country set'; - $sSQL .= ' keywords = array_replace(keywords,'.$aRemove['word_id'].','.$iKeepID.')'; - $sSQL .= ' where keywords @> ARRAY['.$aRemove['word_id'].']'; - $oDB->exec($sSQL); - } - - $sSQL = 'delete from word where word_id = '.$aRemove['word_id']; - $oDB->exec($sSQL); - } - } -} - if ($aResult['recompute-word-counts']) { info('Recompute frequency of full-word search terms'); $sTemplate = file_get_contents(CONST_BasePath.'/sql/words_from_search_name.sql'); -- 2.39.5