From 7db0da40ad02694b82fe9a387f277a58ca7c5fce Mon Sep 17 00:00:00 2001 From: marc tobias Date: Mon, 2 Dec 2019 14:02:05 +0100 Subject: [PATCH] new script utils/check_import_finished.php --- CMakeLists.txt | 1 + lib/DB.php | 70 +++++++++++++++ utils/check_import_finished.php | 151 ++++++++++++++++++++++++++++++++ 3 files changed, 222 insertions(+) create mode 100755 utils/check_import_finished.php diff --git a/CMakeLists.txt b/CMakeLists.txt index 6bd85880..d0b72a48 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -108,6 +108,7 @@ set(WEBSITESCRIPTS ) set(CUSTOMSCRIPTS + utils/check_import_finished.php utils/country_languages.php utils/export.php utils/query.php diff --git a/lib/DB.php b/lib/DB.php index e4aa4349..6307d6ca 100644 --- a/lib/DB.php +++ b/lib/DB.php @@ -240,6 +240,76 @@ class DB return ($this->getOne($sSQL, array(':tablename' => $sTableName)) == 1); } + /** + * Check if an index exists in the database. Optional filtered by tablename + * + * @param string $sTableName + * + * @return boolean + */ + public function indexExists($sIndexName, $sTableName = null) + { + return in_array($sIndexName, $this->getListOfIndices($sTableName)); + } + + /** + * Returns a list of index names in the database, optional filtered by tablename + * + * @param string $sTableName + * + * @return array + */ + public function getListOfIndices($sTableName = null) + { + // table_name | index_name | column_name + // -----------------------+---------------------------------+-------------- + // country_name | idx_country_name_country_code | country_code + // country_osm_grid | idx_country_osm_grid_geometry | geometry + // import_polygon_delete | idx_import_polygon_delete_osmid | osm_id + // import_polygon_delete | idx_import_polygon_delete_osmid | osm_type + // import_polygon_error | idx_import_polygon_error_osmid | osm_id + // import_polygon_error | idx_import_polygon_error_osmid | osm_type + $sSql = <<< END +SELECT + t.relname as table_name, + i.relname as index_name, + a.attname as column_name +FROM + pg_class t, + pg_class i, + pg_index ix, + pg_attribute a +WHERE + t.oid = ix.indrelid + and i.oid = ix.indexrelid + and a.attrelid = t.oid + and a.attnum = ANY(ix.indkey) + and t.relkind = 'r' + and i.relname NOT LIKE 'pg_%' + FILTERS + ORDER BY + t.relname, + i.relname, + a.attname +END; + + $aRows = null; + if ($sTableName) { + $sSql = str_replace('FILTERS', 'and t.relname = :tablename', $sSql); + $aRows = $this->getAll($sSql, array(':tablename' => $sTableName)); + } else { + $sSql = str_replace('FILTERS', '', $sSql); + $aRows = $this->getAll($sSql); + } + + $aIndexNames = array_unique(array_map(function ($aRow) { + return $aRow['index_name']; + }, $aRows)); + sort($aIndexNames); + + return $aIndexNames; + } + /** * Since the DSN includes the database name, checks if the connection works. * diff --git a/utils/check_import_finished.php b/utils/check_import_finished.php new file mode 100755 index 00000000..7828f5cf --- /dev/null +++ b/utils/check_import_finished.php @@ -0,0 +1,151 @@ + "\033[92m", + 'red' => "\x1B[31m", + 'normal' => "\033[0m" +); + +$print_success = function ($message = 'OK') use ($term_colors) { + echo $term_colors['green'].$message.$term_colors['normal']."\n"; +}; + +$print_fail = function ($message = 'Failed') use ($term_colors) { + echo $term_colors['red'].$message.$term_colors['normal']."\n"; +}; + + +$oDB = new Nominatim\DB; + + +echo 'Checking database got created ... '; +if ($oDB->databaseExists()) { + $print_success(); +} else { + $print_fail(); + echo <<< END + Hints: + * Is the database server started? + * Check the CONST_Database_DSN variable in build/settings/local.php + * Try connecting to the database with the same settings + +END; + exit(1); +} + + +echo 'Checking nominatim.so module installed ... '; +$sStandardWord = $oDB->getOne("SELECT make_standard_name('a')"); +if ($sStandardWord === 'a') { + $print_success(); +} else { + $print_fail(); + echo <<< END + The Postgresql extension nominatim.so was not found in the database. + Hints: + * Check the output of the CMmake/make installation step + * Does nominatim.so exist? + * Does nominatim.so exist on the database server? + * Can nominatim.so be accessed by the database user? + +END; + exit(1); +} + +echo 'Checking place table ... '; +if ($oDB->tableExists('place')) { + $print_success(); +} else { + $print_fail(); + echo <<< END + * The import didn't finish. + Hints: + * Check the output of the utils/setup.php you ran. + Usually the osm2pgsql step failed. Check for errors related to + * the file you imported not containing any places + * harddrive full + * out of memory (RAM) + * osm2pgsql killed by other scripts, for consuming to much memory + +END; + exit(1); +} + + + +echo 'Checking indexing status ... '; +$iUnindexed = $oDB->getOne('SELECT count(*) FROM placex WHERE indexed_status > 0'); +if ($iUnindexed == 0) { + $print_success(); +} else { + $print_fail(); + echo <<< END + The indexing didn't finish. There is still $iUnindexed places. See the + question 'Can a stopped/killed import process be resumed?' in the + troubleshooting guide. + +END; + exit(1); +} + +echo "Search index creation\n"; +$aExpectedIndices = array( + // sql/indices.src.sql + 'idx_word_word_id', + 'idx_place_addressline_address_place_id', + 'idx_placex_rank_search', + 'idx_placex_rank_address', + 'idx_placex_pendingsector', + 'idx_placex_parent_place_id', + 'idx_placex_geometry_reverse_lookuppoint', + 'idx_placex_geometry_reverse_lookuppolygon', + 'idx_placex_geometry_reverse_placenode', + 'idx_location_area_country_place_id', + 'idx_osmline_parent_place_id', + 'idx_osmline_parent_osm_id', + 'idx_place_osm_unique', + 'idx_postcode_id', + 'idx_postcode_postcode', + + // sql/indices_search.src.sql + 'idx_search_name_nameaddress_vector', + 'idx_search_name_name_vector', + 'idx_search_name_centroid' +); + +foreach ($aExpectedIndices as $sExpectedIndex) { + echo "Checking index $sExpectedIndex ... "; + if ($oDB->indexExists($sExpectedIndex)) { + $print_success(); + } else { + $print_fail(); + echo <<< END + Hints: + * Rerun the setup.php --create-search-indices step + +END; + exit(1); + } +} + +if (CONST_Use_US_Tiger_Data) { + echo 'Checking TIGER table exists ... '; + if ($oDB->tableExists('location_property_tiger')) { + $print_success(); + } else { + $print_fail(); + echo <<< END + Table 'location_property_tiger' does not exist. Run the TIGER data + import again. + +END; + exit(1); + } +} + + + + +exit(0); -- 2.39.5