return ($this->getOne($sSQL, array(':tablename' => $sTableName)) == 1);
}
- /**
- * Returns a list of table names in the database
- *
- * @return array[]
- */
- public function getListOfTables()
- {
- return $this->getCol("SELECT tablename FROM pg_tables WHERE schemaname='public'");
- }
-
/**
* Deletes a table. Returns true if deleted or didn't exist.
*
return $this->exec('DROP TABLE IF EXISTS '.$sTableName.' CASCADE') == 0;
}
- /**
- * Check if an index exists in the database. Optional filtered by tablename
- *
- * @param string $sTableName
- *
- * @return boolean
- */
- public function indexExists($sIndexName, $sTableName = null)
- {
- return in_array($sIndexName, $this->getListOfIndices($sTableName));
- }
-
- /**
- * Returns a list of index names in the database, optional filtered by tablename
- *
- * @param string $sTableName
- *
- * @return array
- */
- public function getListOfIndices($sTableName = null)
- {
- // table_name | index_name | column_name
- // -----------------------+---------------------------------+--------------
- // country_name | idx_country_name_country_code | country_code
- // country_osm_grid | idx_country_osm_grid_geometry | geometry
- // import_polygon_delete | idx_import_polygon_delete_osmid | osm_id
- // import_polygon_delete | idx_import_polygon_delete_osmid | osm_type
- // import_polygon_error | idx_import_polygon_error_osmid | osm_id
- // import_polygon_error | idx_import_polygon_error_osmid | osm_type
- $sSql = <<< END
-SELECT
- t.relname as table_name,
- i.relname as index_name,
- a.attname as column_name
-FROM
- pg_class t,
- pg_class i,
- pg_index ix,
- pg_attribute a
-WHERE
- t.oid = ix.indrelid
- and i.oid = ix.indexrelid
- and a.attrelid = t.oid
- and a.attnum = ANY(ix.indkey)
- and t.relkind = 'r'
- and i.relname NOT LIKE 'pg_%'
- FILTERS
- ORDER BY
- t.relname,
- i.relname,
- a.attname
-END;
-
- $aRows = null;
- if ($sTableName) {
- $sSql = str_replace('FILTERS', 'and t.relname = :tablename', $sSql);
- $aRows = $this->getAll($sSql, array(':tablename' => $sTableName));
- } else {
- $sSql = str_replace('FILTERS', '', $sSql);
- $aRows = $this->getAll($sSql);
- }
-
- $aIndexNames = array_unique(array_map(function ($aRow) {
- return $aRow['index_name'];
- }, $aRows));
- sort($aIndexNames);
-
- return $aIndexNames;
- }
-
/**
* Tries to connect to the database but on failure doesn't throw an exception.
*
loadSettings(getcwd());
-$term_colors = array(
- 'green' => "\033[92m",
- 'red' => "\x1B[31m",
- 'normal' => "\033[0m"
-);
-
-$print_success = function ($message = 'OK') use ($term_colors) {
- echo $term_colors['green'].$message.$term_colors['normal']."\n";
-};
-
-$print_fail = function ($message = 'Failed') use ($term_colors) {
- echo $term_colors['red'].$message.$term_colors['normal']."\n";
-};
-
-$oDB = new Nominatim\DB;
-
-
-function isReverseOnlyInstallation()
-{
- global $oDB;
- return !$oDB->tableExists('search_name');
-}
-
-// Check (guess) if the setup.php included --drop
-function isNoUpdateInstallation()
-{
- global $oDB;
- return $oDB->tableExists('placex') && !$oDB->tableExists('planet_osm_rels') ;
-}
-
-
-echo 'Checking database got created ... ';
-if ($oDB->checkConnection()) {
- $print_success();
-} else {
- $print_fail();
- echo <<< END
- Hints:
- * Is the database server started?
- * Check the NOMINATIM_DATABASE_DSN variable in your local .env
- * Try connecting to the database with the same settings
-
-END;
- exit(1);
-}
-
-
-echo 'Checking nominatim.so module installed ... ';
-$sStandardWord = $oDB->getOne("SELECT make_standard_name('a')");
-if ($sStandardWord === 'a') {
- $print_success();
-} else {
- $print_fail();
- echo <<< END
- The Postgresql extension nominatim.so was not found in the database.
- Hints:
- * Check the output of the CMmake/make installation step
- * Does nominatim.so exist?
- * Does nominatim.so exist on the database server?
- * Can nominatim.so be accessed by the database user?
-
-END;
- exit(1);
-}
-
-if (!isNoUpdateInstallation()) {
- echo 'Checking place table ... ';
- if ($oDB->tableExists('place')) {
- $print_success();
- } else {
- $print_fail();
- echo <<< END
- * The import didn't finish.
- Hints:
- * Check the output of the utils/setup.php you ran.
- Usually the osm2pgsql step failed. Check for errors related to
- * the file you imported not containing any places
- * harddrive full
- * out of memory (RAM)
- * osm2pgsql killed by other scripts, for consuming to much memory
-
- END;
- exit(1);
- }
-}
-
-
-echo 'Checking indexing status ... ';
-$iUnindexed = $oDB->getOne('SELECT count(*) FROM placex WHERE indexed_status > 0');
-if ($iUnindexed == 0) {
- $print_success();
-} else {
- $print_fail();
- echo <<< END
- The indexing didn't finish. There is still $iUnindexed places. See the
- question 'Can a stopped/killed import process be resumed?' in the
- troubleshooting guide.
-
-END;
- exit(1);
-}
-
-echo "Search index creation\n";
-$aExpectedIndices = array(
- // sql/indices.src.sql
- 'idx_word_word_id',
- 'idx_place_addressline_address_place_id',
- 'idx_placex_rank_search',
- 'idx_placex_rank_address',
- 'idx_placex_parent_place_id',
- 'idx_placex_geometry_reverse_lookuppolygon',
- 'idx_placex_geometry_reverse_placenode',
- 'idx_osmline_parent_place_id',
- 'idx_osmline_parent_osm_id',
- 'idx_postcode_id',
- 'idx_postcode_postcode'
-);
-if (!isReverseOnlyInstallation()) {
- $aExpectedIndices = array_merge($aExpectedIndices, array(
- // sql/indices_search.src.sql
- 'idx_search_name_nameaddress_vector',
- 'idx_search_name_name_vector',
- 'idx_search_name_centroid'
- ));
-}
-if (!isNoUpdateInstallation()) {
- $aExpectedIndices = array_merge($aExpectedIndices, array(
- 'idx_placex_pendingsector',
- 'idx_location_area_country_place_id',
- 'idx_place_osm_unique',
- ));
-}
-
-foreach ($aExpectedIndices as $sExpectedIndex) {
- echo "Checking index $sExpectedIndex ... ";
- if ($oDB->indexExists($sExpectedIndex)) {
- $print_success();
- } else {
- $print_fail();
- echo <<< END
- Hints:
- * Run './utils/setup.php --create-search-indices --ignore-errors' to
- create missing indices.
-
-END;
- exit(1);
- }
-}
-
-echo 'Checking search indices are valid ... ';
-$sSQL = <<< END
- SELECT relname
- FROM pg_class, pg_index
- WHERE pg_index.indisvalid = false
- AND pg_index.indexrelid = pg_class.oid;
-END;
-$aInvalid = $oDB->getCol($sSQL);
-if (empty($aInvalid)) {
- $print_success();
-} else {
- $print_fail();
- echo <<< END
- At least one index is invalid. That can happen, e.g. when index creation was
- disrupted and later restarted. You should delete the affected indices and
- run the index stage of setup again.
- See the question 'Can a stopped/killed import process be resumed?' in the
- troubleshooting guide.
- Affected indices:
-END;
- echo join(', ', $aInvalid) . "\n";
- exit(1);
-}
-
-
-
-if (getSettingBool('USE_US_TIGER_DATA')) {
- echo 'Checking TIGER table exists ... ';
- if ($oDB->tableExists('location_property_tiger')) {
- $print_success();
- } else {
- $print_fail();
- echo <<< END
- Table 'location_property_tiger' does not exist. Run the TIGER data
- import again.
-
-END;
- exit(1);
- }
-}
-
-
-
-
-exit(0);
+(new \Nominatim\Shell(getSetting('NOMINATIM_TOOL')))
+ ->addParams('admin', '--check-database')
+ ->run();
return "'".$s."'";
}
-function fwriteConstDef($rFile, $sConstName, $value)
-{
- $sEscapedValue;
-
- if (is_bool($value)) {
- $sEscapedValue = $value ? 'true' : 'false';
- } elseif (is_numeric($value)) {
- $sEscapedValue = strval($value);
- } elseif (!$value) {
- $sEscapedValue = 'false';
- } else {
- $sEscapedValue = addQuotes(str_replace("'", "\\'", (string)$value));
- }
-
- fwrite($rFile, "@define('CONST_$sConstName', $sEscapedValue);\n");
-}
-
-
function parseLatLon($sQuery)
{
$sFound = null;
return array($sFound, $fQueryLat, $fQueryLon);
}
-function createPointsAroundCenter($fLon, $fLat, $fRadius)
-{
- $iSteps = max(8, min(100, ($fRadius * 40000)^2));
- $fStepSize = (2*pi())/$iSteps;
- $aPolyPoints = array();
- for ($f = 0; $f < 2*pi(); $f += $fStepSize) {
- $aPolyPoints[] = array('', $fLon+($fRadius*sin($f)), $fLat+($fRadius*cos($f)) );
- }
- return $aPolyPoints;
-}
-
function closestHouseNumber($aRow)
{
$fHouse = $aRow['startnumber']
return max(min($aRow['endnumber'], $iHn), $aRow['startnumber']);
}
-
-function getSearchRankLabel($iRank)
-{
- if (!isset($iRank)) return 'unknown';
- if ($iRank < 2) return 'continent';
- if ($iRank < 4) return 'sea';
- if ($iRank < 8) return 'country';
- if ($iRank < 12) return 'state';
- if ($iRank < 16) return 'county';
- if ($iRank == 16) return 'city';
- if ($iRank == 17) return 'town / island';
- if ($iRank == 18) return 'village / hamlet';
- if ($iRank == 20) return 'suburb';
- if ($iRank == 21) return 'postcode area';
- if ($iRank == 22) return 'croft / farm / locality / islet';
- if ($iRank == 23) return 'postcode area';
- if ($iRank == 25) return 'postcode point';
- if ($iRank == 26) return 'street / major landmark';
- if ($iRank == 27) return 'minory street / path';
- if ($iRank == 28) return 'house / building';
- return 'other: ' . $iRank;
-}
public function drop()
{
- info('Drop tables only required for updates');
-
- // The implementation is potentially a bit dangerous because it uses
- // a positive selection of tables to keep, and deletes everything else.
- // Including any tables that the unsuspecting user might have manually
- // created. USE AT YOUR OWN PERIL.
- // tables we want to keep. everything else goes.
- $aKeepTables = array(
- '*columns',
- 'import_polygon_*',
- 'import_status',
- 'place_addressline',
- 'location_postcode',
- 'location_property*',
- 'placex',
- 'search_name',
- 'seq_*',
- 'word',
- 'query_log',
- 'new_query_log',
- 'spatial_ref_sys',
- 'country_name',
- 'place_classtype_*',
- 'country_osm_grid'
- );
-
- $aDropTables = array();
- $aHaveTables = $this->db()->getListOfTables();
-
- foreach ($aHaveTables as $sTable) {
- $bFound = false;
- foreach ($aKeepTables as $sKeep) {
- if (fnmatch($sKeep, $sTable)) {
- $bFound = true;
- break;
- }
- }
- if (!$bFound) array_push($aDropTables, $sTable);
- }
- foreach ($aDropTables as $sDrop) {
- $this->dropTable($sDrop);
- }
-
- $this->removeFlatnodeFile();
+ (clone($this->oNominatimCmd))->addParams('freeze')->run();
}
/**
*/
public function setupWebsite()
{
- if (!is_dir(CONST_InstallDir.'/website')) {
- info('Creating directory for website scripts at: '.CONST_InstallDir.'/website');
- mkdir(CONST_InstallDir.'/website');
- }
-
- $aScripts = array(
- 'deletable.php',
- 'details.php',
- 'lookup.php',
- 'polygons.php',
- 'reverse.php',
- 'search.php',
- 'status.php'
- );
-
- foreach ($aScripts as $sScript) {
- $rFile = fopen(CONST_InstallDir.'/website/'.$sScript, 'w');
-
- fwrite($rFile, "<?php\n\n");
- fwrite($rFile, '@define(\'CONST_Debug\', $_GET[\'debug\'] ?? false);'."\n\n");
-
- fwriteConstDef($rFile, 'LibDir', CONST_LibDir);
- fwriteConstDef($rFile, 'Database_DSN', getSetting('DATABASE_DSN'));
- fwriteConstDef($rFile, 'Default_Language', getSetting('DEFAULT_LANGUAGE'));
- fwriteConstDef($rFile, 'Log_DB', getSettingBool('LOG_DB'));
- fwriteConstDef($rFile, 'Log_File', getSetting('LOG_FILE'));
- fwriteConstDef($rFile, 'Max_Word_Frequency', (int)getSetting('MAX_WORD_FREQUENCY'));
- fwriteConstDef($rFile, 'NoAccessControl', getSettingBool('CORS_NOACCESSCONTROL'));
- fwriteConstDef($rFile, 'Places_Max_ID_count', (int)getSetting('LOOKUP_MAX_COUNT'));
- fwriteConstDef($rFile, 'PolygonOutput_MaximumTypes', getSetting('POLYGON_OUTPUT_MAX_TYPES'));
- fwriteConstDef($rFile, 'Search_BatchMode', getSettingBool('SEARCH_BATCH_MODE'));
- fwriteConstDef($rFile, 'Search_NameOnlySearchFrequencyThreshold', getSetting('SEARCH_NAME_ONLY_THRESHOLD'));
- fwriteConstDef($rFile, 'Term_Normalization_Rules', getSetting('TERM_NORMALIZATION'));
- fwriteConstDef($rFile, 'Use_Aux_Location_data', getSettingBool('USE_AUX_LOCATION_DATA'));
- fwriteConstDef($rFile, 'Use_US_Tiger_Data', getSettingBool('USE_US_TIGER_DATA'));
- fwriteConstDef($rFile, 'MapIcon_URL', getSetting('MAPICON_URL'));
-
- fwrite($rFile, 'require_once(\''.CONST_LibDir.'/website/'.$sScript."');\n");
- fclose($rFile);
-
- chmod(CONST_InstallDir.'/website/'.$sScript, 0755);
- }
+ (clone($this->oNominatimCmd))->addParams('refresh', '--website')->run();
}
/**
// Be nice about our error messages for broken geometry
- if (!$sPlaceId) {
+ if (!$sPlaceId && $oDB->tableExists('import_polygon_error')) {
$sSQL = 'SELECT ';
$sSQL .= ' osm_type, ';
$sSQL .= ' osm_id, ';
}
$aPointDetails['localname'] = $aPointDetails['localname']?$aPointDetails['localname']:$aPointDetails['housenumber'];
-$aPointDetails['rank_search_label'] = getSearchRankLabel($aPointDetails['rank_search']); // only used in HTML format
// Get all alternative names (languages, etc)
$sSQL = 'SELECT (each(name)).key,(each(name)).value FROM placex ';
return run_legacy_script(*params, nominatim_env=args)
-class SetupFreeze:
- """\
- Make database read-only.
-
- About half of data in the Nominatim database is kept only to be able to
- keep the data up-to-date with new changes made in OpenStreetMap. This
- command drops all this data and only keeps the part needed for geocoding
- itself.
-
- This command has the same effect as the `--no-updates` option for imports.
- """
-
- @staticmethod
- def add_args(parser):
- pass # No options
-
- @staticmethod
- def run(args):
- return run_legacy_script('setup.php', '--drop', nominatim_env=args)
-
-
class SetupSpecialPhrases:
"""\
Maintain special phrases.
parser = CommandlineParser('nominatim', nominatim.__doc__)
parser.add_subcommand('import', SetupAll)
- parser.add_subcommand('freeze', SetupFreeze)
+ parser.add_subcommand('freeze', clicmd.SetupFreeze)
parser.add_subcommand('replication', clicmd.UpdateReplication)
parser.add_subcommand('special-phrases', SetupSpecialPhrases)
from .index import UpdateIndex
from .refresh import UpdateRefresh
from .admin import AdminFuncs
+from .freeze import SetupFreeze
"""
Implementation of the 'admin' subcommand.
"""
+import logging
+
from ..tools.exec_utils import run_legacy_script
from ..db.connection import connect
# Using non-top-level imports to avoid eventually unused imports.
# pylint: disable=E0012,C0415
+LOG = logging.getLogger()
+
class AdminFuncs:
"""\
Analyse and maintain the database.
@staticmethod
def run(args):
- from ..tools import admin
if args.warm:
AdminFuncs._warm(args)
if args.check_database:
- run_legacy_script('check_import_finished.php', nominatim_env=args)
+ LOG.warning('Checking database')
+ from ..tools import check_database
+ return check_database.check_database(args.config)
if args.analyse_indexing:
+ LOG.warning('Analysing performance of indexing function')
+ from ..tools import admin
conn = connect(args.config.get_libpq_dsn())
admin.analyse_indexing(conn, osm_id=args.osm_id, place_id=args.place_id)
conn.close()
@staticmethod
def _warm(args):
+ LOG.warning('Warming database caches')
params = ['warm.php']
if args.target == 'reverse':
params.append('--reverse-only')
--- /dev/null
+"""
+Implementation of the 'freeze' subcommand.
+"""
+
+from ..db.connection import connect
+
+# Do not repeat documentation of subcommand classes.
+# pylint: disable=C0111
+# Using non-top-level imports to avoid eventually unused imports.
+# pylint: disable=E0012,C0415
+
+class SetupFreeze:
+ """\
+ Make database read-only.
+
+ About half of data in the Nominatim database is kept only to be able to
+ keep the data up-to-date with new changes made in OpenStreetMap. This
+ command drops all this data and only keeps the part needed for geocoding
+ itself.
+
+ This command has the same effect as the `--no-updates` option for imports.
+ """
+
+ @staticmethod
+ def add_args(parser):
+ pass # No options
+
+ @staticmethod
+ def run(args):
+ from ..tools import freeze
+
+ conn = connect(args.config.get_libpq_dsn())
+ freeze.drop_update_tables(conn)
+ freeze.drop_flatnode_file(args.config.FLATNODE_FILE)
+ conn.close()
+
+ return 0
run_legacy_script('update.php', '--recompute-importance',
nominatim_env=args, throw_on_fail=True)
if args.website:
- run_legacy_script('setup.php', '--setup-website',
- nominatim_env=args, throw_on_fail=True)
+ webdir = args.project_dir / 'website'
+ LOG.warning('Setting up website directory at %s', webdir)
+ refresh.setup_website(webdir, args.phplib_dir, args.config)
return 0
Nominatim uses dotenv to configure the software. Configuration options
are resolved in the following order:
- * from the OS environment
+ * from the OS environment (or the dirctionary given in `environ`
* from the .env file in the project directory of the installation
* from the default installation in the configuration directory
avoid conflicts with other environment variables.
"""
- def __init__(self, project_dir, config_dir):
+ def __init__(self, project_dir, config_dir, environ=None):
+ self.environ = environ or os.environ
self.project_dir = project_dir
self.config_dir = config_dir
self._config = dotenv_values(str((config_dir / 'env.defaults').resolve()))
def __getattr__(self, name):
name = 'NOMINATIM_' + name
- return os.environ.get(name) or self._config[name]
+ return self.environ.get(name) or self._config[name]
def get_bool(self, name):
""" Return the given configuration parameter as a boolean.
merged in.
"""
env = dict(self._config)
- env.update(os.environ)
+ env.update(self.environ)
return env
import psycopg2.extensions
import psycopg2.extras
+from ..errors import UsageError
+
class _Cursor(psycopg2.extras.DictCursor):
""" A cursor returning dict-like objects and providing specialised
execution functions.
"""
return super().cursor(cursor_factory=cursor_factory, **kwargs)
+
def table_exists(self, table):
""" Check that a table with the given name exists in the database.
"""
with self.cursor() as cur:
num = cur.scalar("""SELECT count(*) FROM pg_tables
- WHERE tablename = %s""", (table, ))
+ WHERE tablename = %s and schemaname = 'public'""", (table, ))
return num == 1
+
+ def index_exists(self, index, table=None):
+ """ Check that an index with the given name exists in the database.
+ If table is not None then the index must relate to the given
+ table.
+ """
+ with self.cursor() as cur:
+ cur.execute("""SELECT tablename FROM pg_indexes
+ WHERE indexname = %s and schemaname = 'public'""", (index, ))
+ if cur.rowcount == 0:
+ return False
+
+ if table is not None:
+ row = cur.fetchone()
+ return row[0] == table
+
+ return True
+
+
def server_version_tuple(self):
""" Return the server version as a tuple of (major, minor).
Converts correctly for pre-10 and post-10 PostgreSQL versions.
""" Open a connection to the database using the specialised connection
factory.
"""
- return psycopg2.connect(dsn, connection_factory=_Connection)
+ try:
+ return psycopg2.connect(dsn, connection_factory=_Connection)
+ except psycopg2.OperationalError as err:
+ raise UsageError("Cannot connect to database: {}".format(err)) from err
--- /dev/null
+"""
+Collection of functions that check if the database is complete and functional.
+"""
+from enum import Enum
+from textwrap import dedent
+
+import psycopg2
+
+from ..db.connection import connect
+from ..errors import UsageError
+
+CHECKLIST = []
+
+class CheckState(Enum):
+ """ Possible states of a check. FATAL stops check execution entirely.
+ """
+ OK = 0
+ FAIL = 1
+ FATAL = 2
+ NOT_APPLICABLE = 3
+
+def _check(hint=None):
+ """ Decorator for checks. It adds the function to the list of
+ checks to execute and adds the code for printing progress messages.
+ """
+ def decorator(func):
+ title = func.__doc__.split('\n', 1)[0].strip()
+ def run_check(conn, config):
+ print(title, end=' ... ')
+ ret = func(conn, config)
+ if isinstance(ret, tuple):
+ ret, params = ret
+ else:
+ params = {}
+ if ret == CheckState.OK:
+ print('\033[92mOK\033[0m')
+ elif ret == CheckState.NOT_APPLICABLE:
+ print('not applicable')
+ else:
+ print('\x1B[31mFailed\033[0m')
+ if hint:
+ print(dedent(hint.format(**params)))
+ return ret
+
+ CHECKLIST.append(run_check)
+ return run_check
+
+ return decorator
+
+class _BadConnection: # pylint: disable=R0903
+
+ def __init__(self, msg):
+ self.msg = msg
+
+ def close(self):
+ """ Dummy function to provide the implementation.
+ """
+
+def check_database(config):
+ """ Run a number of checks on the database and return the status.
+ """
+ try:
+ conn = connect(config.get_libpq_dsn())
+ except UsageError as err:
+ conn = _BadConnection(str(err))
+
+ overall_result = 0
+ for check in CHECKLIST:
+ ret = check(conn, config)
+ if ret == CheckState.FATAL:
+ conn.close()
+ return 1
+ if ret in (CheckState.FATAL, CheckState.FAIL):
+ overall_result = 1
+
+ conn.close()
+ return overall_result
+
+
+def _get_indexes(conn):
+ indexes = ['idx_word_word_id',
+ 'idx_place_addressline_address_place_id',
+ 'idx_placex_rank_search',
+ 'idx_placex_rank_address',
+ 'idx_placex_parent_place_id',
+ 'idx_placex_geometry_reverse_lookuppolygon',
+ 'idx_placex_geometry_reverse_placenode',
+ 'idx_osmline_parent_place_id',
+ 'idx_osmline_parent_osm_id',
+ 'idx_postcode_id',
+ 'idx_postcode_postcode'
+ ]
+ if conn.table_exists('search_name'):
+ indexes.extend(('idx_search_name_nameaddress_vector',
+ 'idx_search_name_name_vector',
+ 'idx_search_name_centroid'))
+ if conn.table_exists('place'):
+ indexes.extend(('idx_placex_pendingsector',
+ 'idx_location_area_country_place_id',
+ 'idx_place_osm_unique'
+ ))
+
+ return indexes
+
+
+### CHECK FUNCTIONS
+#
+# Functions are exectured in the order they appear here.
+
+@_check(hint="""\
+ {error}
+
+ Hints:
+ * Is the database server started?
+ * Check the NOMINATIM_DATABASE_DSN variable in your local .env
+ * Try connecting to the database with the same settings
+
+ Project directory: {config.project_dir}
+ Current setting of NOMINATIM_DATABASE_DSN: {config.DATABASE_DSN}
+ """)
+def check_connection(conn, config):
+ """ Checking database connection
+ """
+ if isinstance(conn, _BadConnection):
+ return CheckState.FATAL, dict(error=conn.msg, config=config)
+
+ return CheckState.OK
+
+@_check(hint="""\
+ placex table not found
+
+ Hints:
+ * Are you connecting to the right database?
+ * Did the import process finish without errors?
+
+ Project directory: {config.project_dir}
+ Current setting of NOMINATIM_DATABASE_DSN: {config.DATABASE_DSN}
+ """)
+def check_placex_table(conn, config):
+ """ Checking for placex table
+ """
+ if conn.table_exists('placex'):
+ return CheckState.OK
+
+ return CheckState.FATAL, dict(config=config)
+
+
+@_check(hint="""placex table has no data. Did the import finish sucessfully?""")
+def check_placex_size(conn, config): # pylint: disable=W0613
+ """ Checking for placex content
+ """
+ with conn.cursor() as cur:
+ cnt = cur.scalar('SELECT count(*) FROM (SELECT * FROM placex LIMIT 100) x')
+
+ return CheckState.OK if cnt > 0 else CheckState.FATAL
+
+
+@_check(hint="""\
+ The Postgresql extension nominatim.so was not correctly loaded.
+
+ Error: {error}
+
+ Hints:
+ * Check the output of the CMmake/make installation step
+ * Does nominatim.so exist?
+ * Does nominatim.so exist on the database server?
+ * Can nominatim.so be accessed by the database user?
+ """)
+def check_module(conn, config): # pylint: disable=W0613
+ """ Checking that nominatim.so module is installed
+ """
+ with conn.cursor() as cur:
+ try:
+ out = cur.scalar("SELECT make_standard_name('a')")
+ except psycopg2.ProgrammingError as err:
+ return CheckState.FAIL, dict(error=str(err))
+
+ if out != 'a':
+ return CheckState.FAIL, dict(error='Unexpected result for make_standard_name()')
+
+ return CheckState.OK
+
+
+@_check(hint="""\
+ The indexing didn't finish. {count} entries are not yet indexed.
+
+ To index the remaining entries, run: {index_cmd}
+ """)
+def check_indexing(conn, config): # pylint: disable=W0613
+ """ Checking indexing status
+ """
+ with conn.cursor() as cur:
+ cnt = cur.scalar('SELECT count(*) FROM placex WHERE indexed_status > 0')
+
+ if cnt == 0:
+ return CheckState.OK
+
+ if conn.index_exists('idx_word_word_id'):
+ # Likely just an interrupted update.
+ index_cmd = 'nominatim index'
+ else:
+ # Looks like the import process got interrupted.
+ index_cmd = 'nominatim import --continue indexing'
+
+ return CheckState.FAIL, dict(count=cnt, index_cmd=index_cmd)
+
+
+@_check(hint="""\
+ The following indexes are missing:
+ {indexes}
+
+ Rerun the index creation with: nominatim import --continue db-postprocess
+ """)
+def check_database_indexes(conn, config): # pylint: disable=W0613
+ """ Checking that database indexes are complete
+ """
+ missing = []
+ for index in _get_indexes(conn):
+ if not conn.index_exists(index):
+ missing.append(index)
+
+ if missing:
+ return CheckState.FAIL, dict(indexes='\n '.join(missing))
+
+ return CheckState.OK
+
+
+@_check(hint="""\
+ At least one index is invalid. That can happen, e.g. when index creation was
+ disrupted and later restarted. You should delete the affected indices
+ and recreate them.
+
+ Invalid indexes:
+ {indexes}
+ """)
+def check_database_index_valid(conn, config): # pylint: disable=W0613
+ """ Checking that all database indexes are valid
+ """
+ with conn.cursor() as cur:
+ cur.execute(""" SELECT relname FROM pg_class, pg_index
+ WHERE pg_index.indisvalid = false
+ AND pg_index.indexrelid = pg_class.oid""")
+
+ broken = list(cur)
+
+ if broken:
+ return CheckState.FAIL, dict(indexes='\n '.join(broken))
+
+ return CheckState.OK
+
+
+@_check(hint="""\
+ {error}
+ Run TIGER import again: nominatim add-data --tiger-data <DIR>
+ """)
+def check_tiger_table(conn, config):
+ """ Checking TIGER external data table.
+ """
+ if not config.get_bool('USE_US_TIGER_DATA'):
+ return CheckState.NOT_APPLICABLE
+
+ if not conn.table_exists('location_property_tiger'):
+ return CheckState.FAIL, dict(error='TIGER data table not found.')
+
+ with conn.cursor() as cur:
+ if cur.scalar('SELECT count(*) FROM location_property_tiger') == 0:
+ return CheckState.FAIL, dict(error='TIGER data table is empty.')
+
+ return CheckState.OK
--- /dev/null
+"""
+Functions for removing unnecessary data from the database.
+"""
+from pathlib import Path
+
+UPDATE_TABLES = [
+ 'address_levels',
+ 'gb_postcode',
+ 'import_osmosis_log',
+ 'import_polygon_%',
+ 'location_area%',
+ 'location_road%',
+ 'place',
+ 'planet_osm_%',
+ 'search_name_%',
+ 'us_postcode',
+ 'wikipedia_%'
+]
+
+def drop_update_tables(conn):
+ """ Drop all tables only necessary for updating the database from
+ OSM replication data.
+ """
+
+ where = ' or '.join(["(tablename LIKE '{}')".format(t) for t in UPDATE_TABLES])
+
+ with conn.cursor() as cur:
+ cur.execute("SELECT tablename FROM pg_tables WHERE " + where)
+ tables = [r[0] for r in cur]
+
+ for table in tables:
+ cur.execute('DROP TABLE IF EXISTS "{}" CASCADE'.format(table))
+
+ conn.commit()
+
+
+def drop_flatnode_file(fname):
+ """ Remove the flatnode file if it exists.
+ """
+ if fname:
+ fpath = Path(fname)
+ if fpath.exists():
+ fpath.unlink()
Functions for bringing auxiliary data in the database up-to-date.
"""
import json
+import logging
import re
+from textwrap import dedent
from psycopg2.extras import execute_values
from ..db.utils import execute_file
+LOG = logging.getLogger()
+
def update_postcodes(conn, sql_dir):
""" Recalculate postcode centroids and add, remove and update entries in the
location_postcode table. `conn` is an opne connection to the database.
cur.execute(sql)
conn.commit()
+
+
+WEBSITE_SCRIPTS = (
+ 'deletable.php',
+ 'details.php',
+ 'lookup.php',
+ 'polygons.php',
+ 'reverse.php',
+ 'search.php',
+ 'status.php'
+)
+
+# constants needed by PHP scripts: PHP name, config name, type
+PHP_CONST_DEFS = (
+ ('Database_DSN', 'DATABASE_DSN', str),
+ ('Default_Language', 'DEFAULT_LANGUAGE', str),
+ ('Log_DB', 'LOG_DB', bool),
+ ('Log_File', 'LOG_FILE', str),
+ ('Max_Word_Frequency', 'MAX_WORD_FREQUENCY', int),
+ ('NoAccessControl', 'CORS_NOACCESSCONTROL', bool),
+ ('Places_Max_ID_count', 'LOOKUP_MAX_COUNT', int),
+ ('PolygonOutput_MaximumTypes', 'POLYGON_OUTPUT_MAX_TYPES', int),
+ ('Search_BatchMode', 'SEARCH_BATCH_MODE', bool),
+ ('Search_NameOnlySearchFrequencyThreshold', 'SEARCH_NAME_ONLY_THRESHOLD', str),
+ ('Term_Normalization_Rules', 'TERM_NORMALIZATION', str),
+ ('Use_Aux_Location_data', 'USE_AUX_LOCATION_DATA', bool),
+ ('Use_US_Tiger_Data', 'USE_US_TIGER_DATA', bool),
+ ('MapIcon_URL', 'MAPICON_URL', str),
+)
+
+
+def setup_website(basedir, phplib_dir, config):
+ """ Create the website script stubs.
+ """
+ if not basedir.exists():
+ LOG.info('Creating website directory.')
+ basedir.mkdir()
+
+ template = dedent("""\
+ <?php
+
+ @define('CONST_Debug', $_GET['debug'] ?? false);
+ @define('CONST_LibDir', '{}');
+
+ """.format(phplib_dir))
+
+ for php_name, conf_name, var_type in PHP_CONST_DEFS:
+ if var_type == bool:
+ varout = 'true' if config.get_bool(conf_name) else 'false'
+ elif var_type == int:
+ varout = getattr(config, conf_name)
+ elif not getattr(config, conf_name):
+ varout = 'false'
+ else:
+ varout = "'{}'".format(getattr(config, conf_name).replace("'", "\\'"))
+
+ template += "@define('CONST_{}', {});\n".format(php_name, varout)
+
+ template += "\nrequire_once('{}/website/{{}}');\n".format(phplib_dir)
+
+ for script in WEBSITE_SCRIPTS:
+ (basedir / script).write_text(template.format(script), 'utf-8')
sys.path.insert(1, str((Path(__file__) / '..' / '..' / '..' / '..').resolve()))
from nominatim.config import Configuration
+from nominatim.tools import refresh
from steps.utils import run_script
class NominatimEnvironment:
self.website_dir.cleanup()
self.website_dir = tempfile.TemporaryDirectory()
- self.run_setup_script('setup-website')
+ cfg = Configuration(None, self.src_dir / 'settings', environ=self.test_env)
+ refresh.setup_website(Path(self.website_dir.name) / 'website', self.src_dir / 'lib-php', cfg)
def db_drop_database(self, name):
try:
self.run_setup_script('all', osm_file=self.api_test_file)
self.run_setup_script('import-tiger-data')
+ self.run_setup_script('drop')
phrase_file = str((testdata / 'specialphrases_testdb.sql').resolve())
run_script(['psql', '-d', self.api_test_db, '-f', phrase_file])
# Tables, Indices
{
- $this->assertEmpty($oDB->getListOfTables());
$oDB->exec('CREATE TABLE table1 (id integer, city varchar, country varchar)');
- $oDB->exec('CREATE TABLE table2 (id integer, city varchar, country varchar)');
- $this->assertEquals(
- array('table1', 'table2'),
- $oDB->getListOfTables()
- );
- $this->assertTrue($oDB->deleteTable('table2'));
- $this->assertTrue($oDB->deleteTable('table99'));
- $this->assertEquals(
- array('table1'),
- $oDB->getListOfTables()
- );
$this->assertTrue($oDB->tableExists('table1'));
$this->assertFalse($oDB->tableExists('table99'));
$this->assertFalse($oDB->tableExists(null));
-
- $this->assertEmpty($oDB->getListOfIndices());
- $oDB->exec('CREATE UNIQUE INDEX table1_index ON table1 (id)');
- $this->assertEquals(
- array('table1_index'),
- $oDB->getListOfIndices()
- );
- $this->assertEmpty($oDB->getListOfIndices('table2'));
}
# select queries
$this->assertSame("''", addQuotes(''));
}
-
- public function testCreatePointsAroundCenter()
- {
- // you might say we're creating a circle
- $aPoints = createPointsAroundCenter(0, 0, 2);
-
- $this->assertEquals(
- 101,
- count($aPoints)
- );
- $this->assertEquals(
- array(
- array('', 0, 2),
- array('', 0.12558103905863, 1.9960534568565),
- array('', 0.25066646712861, 1.984229402629)
- ),
- array_splice($aPoints, 0, 3)
- );
- }
-
public function testParseLatLon()
{
// no coordinates expected
// start == end
$this->closestHouseNumberEvenOddOther(50, 50, 0.5, array('even' => 50, 'odd' => 50, 'other' => 50));
}
-
- public function testGetSearchRankLabel()
- {
- $this->assertEquals('unknown', getSearchRankLabel(null));
- $this->assertEquals('continent', getSearchRankLabel(0));
- $this->assertEquals('continent', getSearchRankLabel(1));
- $this->assertEquals('other: 30', getSearchRankLabel(30));
- }
}
return set((tuple(row) for row in self))
+ def table_exists(self, table):
+ """ Check that a table with the given name exists in the database.
+ """
+ num = self.scalar("""SELECT count(*) FROM pg_tables
+ WHERE tablename = %s""", (table, ))
+ return num == 1
+
+
@pytest.fixture
def temp_db(monkeypatch):
""" Create an empty database for the test. The database name is also
import nominatim.clicmd.refresh
import nominatim.clicmd.admin
import nominatim.indexer.indexer
+import nominatim.tools.admin
+import nominatim.tools.check_database
+import nominatim.tools.freeze
import nominatim.tools.refresh
import nominatim.tools.replication
from nominatim.errors import UsageError
monkeypatch.setattr(nominatim.cli, 'run_legacy_script', mock)
return mock
+@pytest.fixture
+def mock_func_factory(monkeypatch):
+ def get_mock(module, func):
+ mock = MockParamCapture()
+ monkeypatch.setattr(module, func, mock)
+ return mock
+
+ return get_mock
def test_cli_help(capsys):
""" Running nominatim tool without arguments prints help.
@pytest.mark.parametrize("command,script", [
(('import', '--continue', 'load-data'), 'setup'),
- (('freeze',), 'setup'),
(('special-phrases',), 'specialphrases'),
(('add-data', '--tiger-data', 'tiger'), 'setup'),
(('add-data', '--file', 'foo.osm'), 'update'),
assert mock_run_legacy.last_args[0] == script + '.php'
+def test_freeze_command(mock_func_factory, temp_db):
+ mock_drop = mock_func_factory(nominatim.tools.freeze, 'drop_update_tables')
+ mock_flatnode = mock_func_factory(nominatim.tools.freeze, 'drop_flatnode_file')
+
+ assert 0 == call_nominatim('freeze')
+
+ assert mock_drop.called == 1
+ assert mock_flatnode.called == 1
+
+
@pytest.mark.parametrize("params", [('--warm', ),
('--warm', '--reverse-only'),
- ('--warm', '--search-only'),
- ('--check-database', )])
-def test_admin_command_legacy(monkeypatch, params):
- mock_run_legacy = MockParamCapture()
- monkeypatch.setattr(nominatim.clicmd.admin, 'run_legacy_script', mock_run_legacy)
+ ('--warm', '--search-only')])
+def test_admin_command_legacy(mock_func_factory, params):
+ mock_run_legacy = mock_func_factory(nominatim.clicmd.admin, 'run_legacy_script')
assert 0 == call_nominatim('admin', *params)
assert mock_run_legacy.called == 1
+
@pytest.mark.parametrize("func, params", [('analyse_indexing', ('--analyse-indexing', ))])
-def test_admin_command_tool(temp_db, monkeypatch, func, params):
- mock = MockParamCapture()
- monkeypatch.setattr(nominatim.tools.admin, func, mock)
+def test_admin_command_tool(temp_db, mock_func_factory, func, params):
+ mock = mock_func_factory(nominatim.tools.admin, func)
assert 0 == call_nominatim('admin', *params)
assert mock.called == 1
+
+def test_admin_command_check_database(mock_func_factory):
+ mock = mock_func_factory(nominatim.tools.check_database, 'check_database')
+
+ assert 0 == call_nominatim('admin', '--check-database')
+ assert mock.called == 1
+
+
@pytest.mark.parametrize("name,oid", [('file', 'foo.osm'), ('diff', 'foo.osc'),
('node', 12), ('way', 8), ('relation', 32)])
def test_add_data_command(mock_run_legacy, name, oid):
(['--boundaries-only'], 1, 0),
(['--no-boundaries'], 0, 1),
(['--boundaries-only', '--no-boundaries'], 0, 0)])
-def test_index_command(monkeypatch, temp_db_cursor, params, do_bnds, do_ranks):
+def test_index_command(mock_func_factory, temp_db_cursor, params, do_bnds, do_ranks):
temp_db_cursor.execute("CREATE TABLE import_status (indexed bool)")
- bnd_mock = MockParamCapture()
- monkeypatch.setattr(nominatim.indexer.indexer.Indexer, 'index_boundaries', bnd_mock)
- rank_mock = MockParamCapture()
- monkeypatch.setattr(nominatim.indexer.indexer.Indexer, 'index_by_rank', rank_mock)
+ bnd_mock = mock_func_factory(nominatim.indexer.indexer.Indexer, 'index_boundaries')
+ rank_mock = mock_func_factory(nominatim.indexer.indexer.Indexer, 'index_by_rank')
assert 0 == call_nominatim('index', *params)
@pytest.mark.parametrize("command,params", [
('wiki-data', ('setup.php', '--import-wikipedia-articles')),
('importance', ('update.php', '--recompute-importance')),
- ('website', ('setup.php', '--setup-website')),
])
-def test_refresh_legacy_command(monkeypatch, temp_db, command, params):
- mock_run_legacy = MockParamCapture()
- monkeypatch.setattr(nominatim.clicmd.refresh, 'run_legacy_script', mock_run_legacy)
+def test_refresh_legacy_command(mock_func_factory, temp_db, command, params):
+ mock_run_legacy = mock_func_factory(nominatim.clicmd.refresh, 'run_legacy_script')
assert 0 == call_nominatim('refresh', '--' + command)
('word-counts', 'recompute_word_counts'),
('address-levels', 'load_address_levels_from_file'),
('functions', 'create_functions'),
+ ('website', 'setup_website'),
])
-def test_refresh_command(monkeypatch, temp_db, command, func):
- func_mock = MockParamCapture()
- monkeypatch.setattr(nominatim.tools.refresh, func, func_mock)
+def test_refresh_command(mock_func_factory, temp_db, command, func):
+ func_mock = mock_func_factory(nominatim.tools.refresh, func)
assert 0 == call_nominatim('refresh', '--' + command)
assert func_mock.called == 1
-def test_refresh_importance_computed_after_wiki_import(monkeypatch, temp_db):
- mock_run_legacy = MockParamCapture()
- monkeypatch.setattr(nominatim.clicmd.refresh, 'run_legacy_script', mock_run_legacy)
+def test_refresh_importance_computed_after_wiki_import(mock_func_factory, temp_db):
+ mock_run_legacy = mock_func_factory(nominatim.clicmd.refresh, 'run_legacy_script')
assert 0 == call_nominatim('refresh', '--importance', '--wiki-data')
(('--init', '--no-update-functions'), 'init_replication'),
(('--check-for-updates',), 'check_for_updates')
])
-def test_replication_command(monkeypatch, temp_db, params, func):
- func_mock = MockParamCapture()
- monkeypatch.setattr(nominatim.tools.replication, func, func_mock)
+def test_replication_command(mock_func_factory, temp_db, params, func):
+ func_mock = mock_func_factory(nominatim.tools.replication, func)
assert 0 == call_nominatim('replication', *params)
assert func_mock.called == 1
@pytest.mark.parametrize("state", [nominatim.tools.replication.UpdateState.UP_TO_DATE,
nominatim.tools.replication.UpdateState.NO_CHANGES])
-def test_replication_update_once_no_index(monkeypatch, temp_db, temp_db_conn,
+def test_replication_update_once_no_index(mock_func_factory, temp_db, temp_db_conn,
status_table, state):
status.set_status(temp_db_conn, date=dt.datetime.now(dt.timezone.utc), seq=1)
- func_mock = MockParamCapture(retval=state)
- monkeypatch.setattr(nominatim.tools.replication, 'update', func_mock)
+ func_mock = mock_func_factory(nominatim.tools.replication, 'update')
assert 0 == call_nominatim('replication', '--once', '--no-index')
assert sleep_mock.last_args[0] == 60
-def test_serve_command(monkeypatch):
- func = MockParamCapture()
- monkeypatch.setattr(nominatim.cli, 'run_php_server', func)
+def test_serve_command(mock_func_factory):
+ func = mock_func_factory(nominatim.cli, 'run_php_server')
call_nominatim('serve')
('details', '--place_id', '10001'),
('status',)
])
-def test_api_commands_simple(monkeypatch, params):
- mock_run_api = MockParamCapture()
- monkeypatch.setattr(nominatim.clicmd.api, 'run_api_script', mock_run_api)
+def test_api_commands_simple(mock_func_factory, params):
+ mock_run_api = mock_func_factory(nominatim.clicmd.api, 'run_api_script')
assert 0 == call_nominatim(*params)
assert db.table_exists('foobar') == True
+def test_connection_index_exists(db, temp_db_cursor):
+ assert db.index_exists('some_index') == False
+
+ temp_db_cursor.execute('CREATE TABLE foobar (id INT)')
+ temp_db_cursor.execute('CREATE INDEX some_index ON foobar(id)')
+
+ assert db.index_exists('some_index') == True
+ assert db.index_exists('some_index', table='foobar') == True
+ assert db.index_exists('some_index', table='bar') == False
+
+
+def test_connection_server_version_tuple(db):
+ ver = db.server_version_tuple()
+
+ assert isinstance(ver, tuple)
+ assert len(ver) == 2
+ assert ver[0] > 8
+
def test_cursor_scalar(db, temp_db_cursor):
temp_db_cursor.execute('CREATE TABLE dummy (id INT)')
with db.cursor() as cur:
assert cur.scalar('SELECT count(*) FROM dummy') == 0
+
def test_cursor_scalar_many_rows(db):
with db.cursor() as cur:
with pytest.raises(RuntimeError):
--- /dev/null
+"""
+Tests for database integrity checks.
+"""
+import pytest
+
+from nominatim.tools import check_database as chkdb
+
+def test_check_database_unknown_db(def_config, monkeypatch):
+ monkeypatch.setenv('NOMINATIM_DATABASE_DSN', 'pgsql:dbname=fjgkhughwgh2423gsags')
+ assert 1 == chkdb.check_database(def_config)
+
+
+def test_check_conection_good(temp_db_conn, def_config):
+ assert chkdb.check_connection(temp_db_conn, def_config) == chkdb.CheckState.OK
+
+
+def test_check_conection_bad(def_config):
+ badconn = chkdb._BadConnection('Error')
+ assert chkdb.check_connection(badconn, def_config) == chkdb.CheckState.FATAL
+
+
+def test_check_placex_table_good(temp_db_cursor, temp_db_conn, def_config):
+ temp_db_cursor.execute('CREATE TABLE placex (place_id int)')
+ assert chkdb.check_placex_table(temp_db_conn, def_config) == chkdb.CheckState.OK
+
+
+def test_check_placex_table_bad(temp_db_conn, def_config):
+ assert chkdb.check_placex_table(temp_db_conn, def_config) == chkdb.CheckState.FATAL
+
+
+def test_check_placex_table_size_good(temp_db_cursor, temp_db_conn, def_config):
+ temp_db_cursor.execute('CREATE TABLE placex (place_id int)')
+ temp_db_cursor.execute('INSERT INTO placex VALUES (1), (2)')
+ assert chkdb.check_placex_size(temp_db_conn, def_config) == chkdb.CheckState.OK
+
+
+def test_check_placex_table_size_bad(temp_db_cursor, temp_db_conn, def_config):
+ temp_db_cursor.execute('CREATE TABLE placex (place_id int)')
+ assert chkdb.check_placex_size(temp_db_conn, def_config) == chkdb.CheckState.FATAL
+
+
+def test_check_module_bad(temp_db_conn, def_config):
+ assert chkdb.check_module(temp_db_conn, def_config) == chkdb.CheckState.FAIL
+
+
+def test_check_indexing_good(temp_db_cursor, temp_db_conn, def_config):
+ temp_db_cursor.execute('CREATE TABLE placex (place_id int, indexed_status smallint)')
+ temp_db_cursor.execute('INSERT INTO placex VALUES (1, 0), (2, 0)')
+ assert chkdb.check_indexing(temp_db_conn, def_config) == chkdb.CheckState.OK
+
+
+def test_check_indexing_bad(temp_db_cursor, temp_db_conn, def_config):
+ temp_db_cursor.execute('CREATE TABLE placex (place_id int, indexed_status smallint)')
+ temp_db_cursor.execute('INSERT INTO placex VALUES (1, 0), (2, 2)')
+ assert chkdb.check_indexing(temp_db_conn, def_config) == chkdb.CheckState.FAIL
+
+
+def test_check_database_indexes_bad(temp_db_conn, def_config):
+ assert chkdb.check_database_indexes(temp_db_conn, def_config) == chkdb.CheckState.FAIL
+
+
+def test_check_tiger_table_disabled(temp_db_conn, def_config, monkeypatch):
+ monkeypatch.setenv('NOMINATIM_USE_US_TIGER_DATA' , 'no')
+ assert chkdb.check_tiger_table(temp_db_conn, def_config) == chkdb.CheckState.NOT_APPLICABLE
+
+
+def test_check_tiger_table_enabled(temp_db_cursor, temp_db_conn, def_config, monkeypatch):
+ monkeypatch.setenv('NOMINATIM_USE_US_TIGER_DATA' , 'yes')
+ assert chkdb.check_tiger_table(temp_db_conn, def_config) == chkdb.CheckState.FAIL
+
+ temp_db_cursor.execute('CREATE TABLE location_property_tiger (place_id int)')
+ assert chkdb.check_tiger_table(temp_db_conn, def_config) == chkdb.CheckState.FAIL
+
+ temp_db_cursor.execute('INSERT INTO location_property_tiger VALUES (1), (2)')
+ assert chkdb.check_tiger_table(temp_db_conn, def_config) == chkdb.CheckState.OK
+
--- /dev/null
+"""
+Tests for freeze functions (removing unused database parts).
+"""
+import pytest
+
+from nominatim.tools import freeze
+
+NOMINATIM_RUNTIME_TABLES = [
+ 'country_name', 'country_osm_grid',
+ 'location_postcode', 'location_property_osmline', 'location_property_tiger',
+ 'placex', 'place_adressline',
+ 'search_name',
+ 'word'
+]
+
+NOMINATIM_DROP_TABLES = [
+ 'address_levels',
+ 'location_area', 'location_area_country', 'location_area_large_100',
+ 'location_road_1',
+ 'place', 'planet_osm_nodes', 'planet_osm_rels', 'planet_osm_ways',
+ 'search_name_111',
+ 'wikipedia_article', 'wikipedia_redirect'
+]
+
+def test_drop_tables(temp_db_conn, temp_db_cursor):
+ for table in NOMINATIM_RUNTIME_TABLES + NOMINATIM_DROP_TABLES:
+ temp_db_cursor.execute('CREATE TABLE {} (id int)'.format(table))
+
+ freeze.drop_update_tables(temp_db_conn)
+
+ for table in NOMINATIM_RUNTIME_TABLES:
+ assert temp_db_cursor.table_exists(table)
+
+ for table in NOMINATIM_DROP_TABLES:
+ assert not temp_db_cursor.table_exists(table)
+
+def test_drop_flatnode_file_no_file():
+ freeze.drop_flatnode_file('')
+
+
+def test_drop_flatnode_file_file_already_gone(tmp_path):
+ freeze.drop_flatnode_file(str(tmp_path / 'something.store'))
+
+
+def test_drop_flatnode_file_delte(tmp_path):
+ flatfile = tmp_path / 'flatnode.store'
+ flatfile.write_text('Some content')
+
+ freeze.drop_flatnode_file(str(flatfile))
+
+ assert not flatfile.exists()
--- /dev/null
+"""
+Tests for setting up the website scripts.
+"""
+from pathlib import Path
+import subprocess
+
+import pytest
+
+from nominatim.tools import refresh
+
+@pytest.fixture
+def envdir(tmpdir):
+ (tmpdir / 'php').mkdir()
+ (tmpdir / 'php' / 'website').mkdir()
+ return tmpdir
+
+
+@pytest.fixture
+def test_script(envdir):
+ def _create_file(code):
+ outfile = envdir / 'php' / 'website' / 'search.php'
+ outfile.write_text('<?php\n{}\n'.format(code), 'utf-8')
+
+ return _create_file
+
+
+def run_website_script(envdir, config):
+ refresh.setup_website(envdir, envdir / 'php', config)
+
+ proc = subprocess.run(['/usr/bin/env', 'php', '-Cq',
+ envdir / 'search.php'], check=False)
+
+ return proc.returncode
+
+
+@pytest.mark.parametrize("setting,retval", (('yes', 10), ('no', 20)))
+def test_setup_website_check_bool(def_config, monkeypatch, envdir, test_script,
+ setting, retval):
+ monkeypatch.setenv('NOMINATIM_CORS_NOACCESSCONTROL', setting)
+
+ test_script('exit(CONST_NoAccessControl ? 10 : 20);')
+
+ assert run_website_script(envdir, def_config) == retval
+
+
+@pytest.mark.parametrize("setting", (0, 10, 99067))
+def test_setup_website_check_int(def_config, monkeypatch, envdir, test_script, setting):
+ monkeypatch.setenv('NOMINATIM_LOOKUP_MAX_COUNT', str(setting))
+
+ test_script('exit(CONST_Places_Max_ID_count == {} ? 10 : 20);'.format(setting))
+
+ assert run_website_script(envdir, def_config) == 10
+
+
+def test_setup_website_check_empty_str(def_config, monkeypatch, envdir, test_script):
+ monkeypatch.setenv('NOMINATIM_DEFAULT_LANGUAGE', '')
+
+ test_script('exit(CONST_Default_Language === false ? 10 : 20);')
+
+ assert run_website_script(envdir, def_config) == 10
+
+
+def test_setup_website_check_str(def_config, monkeypatch, envdir, test_script):
+ monkeypatch.setenv('NOMINATIM_DEFAULT_LANGUAGE', 'ffde 2')
+
+ test_script('exit(CONST_Default_Language === "ffde 2" ? 10 : 20);')
+
+ assert run_website_script(envdir, def_config) == 10
+
+
projects / nominatim.git / commitdiff