From: Sarah Hoffmann Date: Fri, 12 Feb 2021 14:55:27 +0000 (+0100) Subject: Merge remote-tracking branch 'upstream/master' X-Git-Tag: deploy~183 X-Git-Url: https://git.openstreetmap.org./nominatim.git/commitdiff_plain/80f6aca0c22984b79e24a19bc602258dd3caeb34?hp=9afebddb16b09e09fa9a74624dcd79475b476f10 Merge remote-tracking branch 'upstream/master' --- diff --git a/.github/actions/build-nominatim/action.yml b/.github/actions/build-nominatim/action.yml index 3cd826af..d62ecf86 100644 --- a/.github/actions/build-nominatim/action.yml +++ b/.github/actions/build-nominatim/action.yml @@ -9,21 +9,21 @@ runs: sudo apt-get install -y -qq libboost-system-dev libboost-filesystem-dev libexpat1-dev zlib1g-dev libbz2-dev libpq-dev libproj-dev python3-psycopg2 python3-pyosmium python3-dotenv shell: bash + - name: Download dependencies + run: | + if [ ! -f country_grid.sql.gz ]; then + wget --no-verbose https://www.nominatim.org/data/country_grid.sql.gz + fi + cp country_grid.sql.gz Nominatim/data/country_osm_grid.sql.gz + shell: bash + - name: Configure - run: mkdir build && cd build && cmake .. + run: mkdir build && cd build && cmake ../Nominatim shell: bash - name: Build run: | make -j2 all - ./nominatim refresh --website + sudo make install shell: bash working-directory: build - - - name: Download dependencies - run: | - if [ ! -f data/country_osm_grid.sql.gz ]; then - wget --no-verbose -O data/country_osm_grid.sql.gz https://www.nominatim.org/data/country_grid.sql.gz - fi - shell: bash - diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 1fa7e19d..e0e68a9c 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -19,6 +19,7 @@ jobs: - uses: actions/checkout@v2 with: submodules: true + path: Nominatim - name: Setup PHP uses: shivammathur/setup-php@v2 @@ -35,35 +36,37 @@ jobs: - uses: actions/cache@v2 with: path: | - data/country_osm_grid.sql.gz - monaco-latest.osm.pbf - key: nominatim-data-${{ steps.get-date.outputs.date }} + country_grid.sql.gz + key: nominatim-country-data-${{ steps.get-date.outputs.date }} - - uses: ./.github/actions/setup-postgresql + - uses: ./Nominatim/.github/actions/setup-postgresql with: postgresql-version: ${{ matrix.postgresql }} postgis-version: ${{ matrix.postgis }} - - uses: ./.github/actions/build-nominatim + - uses: ./Nominatim/.github/actions/build-nominatim - name: Install test prerequsites run: sudo apt-get install -y -qq php-codesniffer pylint python3-pytest python3-behave - name: PHP linting run: phpcs --report-width=120 . + working-directory: Nominatim - name: Python linting run: pylint --extension-pkg-whitelist=osmium nominatim + working-directory: Nominatim - name: PHP unit tests run: phpunit ./ - working-directory: test/php + working-directory: Nominatim/test/php - name: Python unit tests run: py.test-3 test/python + working-directory: Nominatim - name: BDD tests - run: behave -DREMOVE_TEMPLATE=1 --format=progress3 - working-directory: test/bdd + run: behave -DREMOVE_TEMPLATE=1 -DBUILDDIR=$GITHUB_WORKSPACE/build --format=progress3 + working-directory: Nominatim/test/bdd import: runs-on: ubuntu-20.04 @@ -72,6 +75,7 @@ jobs: - uses: actions/checkout@v2 with: submodules: true + path: Nominatim - name: Get Date id: get-date @@ -82,46 +86,55 @@ jobs: - uses: actions/cache@v2 with: path: | - data/country_osm_grid.sql.gz + country_grid.sql.gz + key: nominatim-country-data-${{ steps.get-date.outputs.date }} + + - uses: actions/cache@v2 + with: + path: | monaco-latest.osm.pbf - key: nominatim-data-${{ steps.get-date.outputs.date }} + key: nominatim-test-data-${{ steps.get-date.outputs.date }} - - uses: ./.github/actions/setup-postgresql + - uses: ./Nominatim/.github/actions/setup-postgresql with: postgresql-version: 13 postgis-version: 3 - - uses: ./.github/actions/build-nominatim + - uses: ./Nominatim/.github/actions/build-nominatim + + - name: Clean installation + run: rm -rf Nominatim build + shell: bash - - name: Download import data + - name: Prepare import environment run: | if [ ! -f monaco-latest.osm.pbf ]; then wget --no-verbose https://download.geofabrik.de/europe/monaco-latest.osm.pbf fi + mkdir data-env + cd data-env shell: bash - name: Import - run: | - mkdir data-env - cd data-env - ../build/nominatim import --osm-file ../monaco-latest.osm.pbf + run: nominatim import --osm-file ../monaco-latest.osm.pbf shell: bash + working-directory: data-env - name: Import special phrases - run: ../build/nominatim special-phrases --from-wiki | psql -d nominatim + run: nominatim special-phrases --from-wiki | psql -d nominatim working-directory: data-env - name: Check import - run: ../build/nominatim check-database + run: nominatim admin --check-database working-directory: data-env - name: Run update run: | - ../build/nominatim replication --init - ../build/nominatim replication --once + nominatim replication --init + nominatim replication --once working-directory: data-env - name: Run reverse-only import - run : | - echo 'NOMINATIM_DATABASE_DSN="pgsql:dbname=reverse"' > .env - ../build/nominatim import --osm-file ../monaco-latest.osm.pbf --reverse-only + run : nominatim import --osm-file ../monaco-latest.osm.pbf --reverse-only working-directory: data-env + env: + NOMINATIM_DATABASE_DSN: pgsql:dbname=reverse diff --git a/CMakeLists.txt b/CMakeLists.txt index 45b205fd..7794a50b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -97,6 +97,17 @@ endif() #----------------------------------------------------------------------------- if (BUILD_IMPORTER) + find_file(COUNTRY_GRID_FILE country_osm_grid.sql.gz + PATHS ${PROJECT_SOURCE_DIR}/data + NO_DEFAULT_PATH + DOC "Location of the country grid file." + ) + + if (NOT COUNTRY_GRID_FILE) + message(FATAL_ERROR "\nYou need to download the country_osm_grid first:\n" + " wget -O ${PROJECT_SOURCE_DIR}/data/country_osm_grid.sql.gz https://www.nominatim.org/data/country_grid.sql.gz") + endif() + set(CUSTOMSCRIPTS check_import_finished.php country_languages.php @@ -218,3 +229,54 @@ endif() if (BUILD_DOCS) add_subdirectory(docs) endif() + +#----------------------------------------------------------------------------- +# Installation +#----------------------------------------------------------------------------- + + +include(GNUInstallDirs) +set(NOMINATIM_DATADIR ${CMAKE_INSTALL_FULL_DATADIR}/${PROJECT_NAME}) +set(NOMINATIM_LIBDIR ${CMAKE_INSTALL_FULL_LIBDIR}/${PROJECT_NAME}) +set(NOMINATIM_CONFIGDIR ${CMAKE_INSTALL_FULL_SYSCONFDIR}/${PROJECT_NAME}) + +if (BUILD_IMPORTER) + configure_file(${PROJECT_SOURCE_DIR}/cmake/tool-installed.tmpl installed.bin) + install(PROGRAMS ${PROJECT_BINARY_DIR}/installed.bin + DESTINATION ${CMAKE_INSTALL_BINDIR} + RENAME nominatim) + + install(DIRECTORY nominatim + DESTINATION ${NOMINATIM_LIBDIR}/lib-python + FILES_MATCHING PATTERN "*.py" + PATTERN __pycache__ EXCLUDE) + install(DIRECTORY lib-sql DESTINATION ${NOMINATIM_LIBDIR}) + + install(FILES data/country_name.sql + ${COUNTRY_GRID_FILE} + data/words.sql + DESTINATION ${NOMINATIM_DATADIR}) +endif() + +if (BUILD_OSM2PGSQL) + install(TARGETS osm2pgsql RUNTIME DESTINATION ${NOMINATIM_LIBDIR}) +endif() + +if (BUILD_MODULE) + install(PROGRAMS ${PROJECT_BINARY_DIR}/module/nominatim.so + DESTINATION ${NOMINATIM_LIBDIR}/module) +endif() + +if (BUILD_API) + install(DIRECTORY lib-php DESTINATION ${NOMINATIM_LIBDIR}) +endif() + +install(FILES settings/env.defaults + settings/address-levels.json + settings/phrase_settings.php + settings/import-admin.style + settings/import-street.style + settings/import-address.style + settings/import-full.style + settings/import-extratags.style + DESTINATION ${NOMINATIM_CONFIGDIR}) diff --git a/README.md b/README.md index d4bb0936..6fd0cd45 100644 --- a/README.md +++ b/README.md @@ -41,12 +41,13 @@ A quick summary of the necessary steps: cd build cmake .. make + sudo make install 2. Create a project directory, get OSM data and import: mkdir nominatim-project cd nominatim-project - ~/build/nominatim import --osm-file + nominatim import --osm-file 3. Point your webserver to the nominatim-project/website directory. diff --git a/cmake/script.tmpl b/cmake/script.tmpl index aa25a124..3fbe535e 100755 --- a/cmake/script.tmpl +++ b/cmake/script.tmpl @@ -1,13 +1,14 @@ #!@PHP_BIN@ -Cq ` to the +cmake command. Make sure that the `bin` directory is available in your path +in that case, e.g. + +``` +export PATH=/bin:$PATH ``` Now continue with [importing the database](Import.md). diff --git a/docs/admin/Migration.md b/docs/admin/Migration.md index 333c2477..dc94310b 100644 --- a/docs/admin/Migration.md +++ b/docs/admin/Migration.md @@ -37,8 +37,8 @@ functionality of each script: * ./utils/setup.php: `import`, `freeze`, `refresh` * ./utils/update.php: `replication`, `add-data`, `index`, `refresh` * ./utils/specialphrases.php: `special-phrases` -* ./utils/check_import_finished.php: `check-database` -* ./utils/warm.php: `warm` +* ./utils/check_import_finished.php: `admin` +* ./utils/warm.php: `admin` * ./utils/export.php: `export` Try `nominatim --help` for more information about each subcommand. diff --git a/lib/AddressDetails.php b/lib-php/AddressDetails.php similarity index 100% rename from lib/AddressDetails.php rename to lib-php/AddressDetails.php diff --git a/lib/ClassTypes.php b/lib-php/ClassTypes.php similarity index 100% rename from lib/ClassTypes.php rename to lib-php/ClassTypes.php diff --git a/lib/DB.php b/lib-php/DB.php similarity index 100% rename from lib/DB.php rename to lib-php/DB.php diff --git a/lib/DatabaseError.php b/lib-php/DatabaseError.php similarity index 100% rename from lib/DatabaseError.php rename to lib-php/DatabaseError.php diff --git a/lib/DebugHtml.php b/lib-php/DebugHtml.php similarity index 100% rename from lib/DebugHtml.php rename to lib-php/DebugHtml.php diff --git a/lib/DebugNone.php b/lib-php/DebugNone.php similarity index 100% rename from lib/DebugNone.php rename to lib-php/DebugNone.php diff --git a/lib/Geocode.php b/lib-php/Geocode.php similarity index 100% rename from lib/Geocode.php rename to lib-php/Geocode.php diff --git a/lib/ParameterParser.php b/lib-php/ParameterParser.php similarity index 100% rename from lib/ParameterParser.php rename to lib-php/ParameterParser.php diff --git a/lib/Phrase.php b/lib-php/Phrase.php similarity index 100% rename from lib/Phrase.php rename to lib-php/Phrase.php diff --git a/lib/PlaceLookup.php b/lib-php/PlaceLookup.php similarity index 100% rename from lib/PlaceLookup.php rename to lib-php/PlaceLookup.php diff --git a/lib/Result.php b/lib-php/Result.php similarity index 100% rename from lib/Result.php rename to lib-php/Result.php diff --git a/lib/ReverseGeocode.php b/lib-php/ReverseGeocode.php similarity index 100% rename from lib/ReverseGeocode.php rename to lib-php/ReverseGeocode.php diff --git a/lib/SearchContext.php b/lib-php/SearchContext.php similarity index 100% rename from lib/SearchContext.php rename to lib-php/SearchContext.php diff --git a/lib/SearchDescription.php b/lib-php/SearchDescription.php similarity index 100% rename from lib/SearchDescription.php rename to lib-php/SearchDescription.php diff --git a/lib/Shell.php b/lib-php/Shell.php similarity index 100% rename from lib/Shell.php rename to lib-php/Shell.php diff --git a/lib/SpecialSearchOperator.php b/lib-php/SpecialSearchOperator.php similarity index 100% rename from lib/SpecialSearchOperator.php rename to lib-php/SpecialSearchOperator.php diff --git a/lib/Status.php b/lib-php/Status.php similarity index 100% rename from lib/Status.php rename to lib-php/Status.php diff --git a/lib/TokenCountry.php b/lib-php/TokenCountry.php similarity index 100% rename from lib/TokenCountry.php rename to lib-php/TokenCountry.php diff --git a/lib/TokenHousenumber.php b/lib-php/TokenHousenumber.php similarity index 100% rename from lib/TokenHousenumber.php rename to lib-php/TokenHousenumber.php diff --git a/lib/TokenList.php b/lib-php/TokenList.php similarity index 100% rename from lib/TokenList.php rename to lib-php/TokenList.php diff --git a/lib/TokenPostcode.php b/lib-php/TokenPostcode.php similarity index 100% rename from lib/TokenPostcode.php rename to lib-php/TokenPostcode.php diff --git a/lib/TokenSpecialTerm.php b/lib-php/TokenSpecialTerm.php similarity index 100% rename from lib/TokenSpecialTerm.php rename to lib-php/TokenSpecialTerm.php diff --git a/lib/TokenWord.php b/lib-php/TokenWord.php similarity index 100% rename from lib/TokenWord.php rename to lib-php/TokenWord.php diff --git a/lib/admin/check_import_finished.php b/lib-php/admin/check_import_finished.php similarity index 100% rename from lib/admin/check_import_finished.php rename to lib-php/admin/check_import_finished.php diff --git a/lib/admin/country_languages.php b/lib-php/admin/country_languages.php similarity index 100% rename from lib/admin/country_languages.php rename to lib-php/admin/country_languages.php diff --git a/lib/admin/export.php b/lib-php/admin/export.php similarity index 100% rename from lib/admin/export.php rename to lib-php/admin/export.php diff --git a/lib/admin/query.php b/lib-php/admin/query.php similarity index 100% rename from lib/admin/query.php rename to lib-php/admin/query.php diff --git a/lib/admin/setup.php b/lib-php/admin/setup.php similarity index 100% rename from lib/admin/setup.php rename to lib-php/admin/setup.php diff --git a/lib/admin/specialphrases.php b/lib-php/admin/specialphrases.php similarity index 100% rename from lib/admin/specialphrases.php rename to lib-php/admin/specialphrases.php diff --git a/lib/admin/update.php b/lib-php/admin/update.php similarity index 100% rename from lib/admin/update.php rename to lib-php/admin/update.php diff --git a/lib/admin/warm.php b/lib-php/admin/warm.php similarity index 100% rename from lib/admin/warm.php rename to lib-php/admin/warm.php diff --git a/lib/cmd.php b/lib-php/cmd.php similarity index 100% rename from lib/cmd.php rename to lib-php/cmd.php diff --git a/lib/dotenv_loader.php b/lib-php/dotenv_loader.php similarity index 77% rename from lib/dotenv_loader.php rename to lib-php/dotenv_loader.php index 919891a0..35471fdc 100644 --- a/lib/dotenv_loader.php +++ b/lib-php/dotenv_loader.php @@ -5,7 +5,7 @@ require('Symfony/Component/Dotenv/autoload.php'); function loadDotEnv() { $dotenv = new \Symfony\Component\Dotenv\Dotenv(); - $dotenv->load(CONST_DataDir.'/settings/env.defaults'); + $dotenv->load(CONST_ConfigDir.'/env.defaults'); if (file_exists('.env')) { $dotenv->load('.env'); diff --git a/lib/init-cmd.php b/lib-php/init-cmd.php similarity index 100% rename from lib/init-cmd.php rename to lib-php/init-cmd.php diff --git a/lib/init-website.php b/lib-php/init-website.php similarity index 100% rename from lib/init-website.php rename to lib-php/init-website.php diff --git a/lib/init.php b/lib-php/init.php similarity index 100% rename from lib/init.php rename to lib-php/init.php diff --git a/lib/lib.php b/lib-php/lib.php similarity index 97% rename from lib/lib.php rename to lib-php/lib.php index a02fefd0..6798e749 100644 --- a/lib/lib.php +++ b/lib-php/lib.php @@ -7,7 +7,8 @@ function loadSettings($sProjectDir) // the installed scripts. Neither setting is part of the official // set of settings. defined('CONST_DataDir') or define('CONST_DataDir', $_SERVER['NOMINATIM_DATADIR']); - defined('CONST_BinDir') or define('CONST_BinDir', $_SERVER['NOMINATIM_BINDIR']); + defined('CONST_SqlDir') or define('CONST_SqlDir', $_SERVER['NOMINATIM_SQLDIR']); + defined('CONST_ConfigDir') or define('CONST_ConfigDir', $_SERVER['NOMINATIM_CONFIGDIR']); defined('CONST_Default_ModulePath') or define('CONST_Default_ModulePath', $_SERVER['NOMINATIM_DATABASE_MODULE_SRC_PATH']); } @@ -36,7 +37,7 @@ function getSettingConfig($sConfName, $sSystemConfig) $sValue = $_SERVER['NOMINATIM_'.$sConfName]; if (!$sValue) { - return CONST_DataDir.'/settings/'.$sSystemConfig; + return CONST_ConfigDir.'/'.$sSystemConfig; } return $sValue; diff --git a/lib/log.php b/lib-php/log.php similarity index 100% rename from lib/log.php rename to lib-php/log.php diff --git a/lib/output.php b/lib-php/output.php similarity index 100% rename from lib/output.php rename to lib-php/output.php diff --git a/lib/setup/SetupClass.php b/lib-php/setup/SetupClass.php similarity index 95% rename from lib/setup/SetupClass.php rename to lib-php/setup/SetupClass.php index dda49160..fedbb644 100755 --- a/lib/setup/SetupClass.php +++ b/lib-php/setup/SetupClass.php @@ -166,29 +166,8 @@ class SetupFunctions // Try accessing the C module, so we know early if something is wrong $this->checkModulePresence(); // raises exception on failure - if (!file_exists(CONST_DataDir.'/data/country_osm_grid.sql.gz')) { - echo 'Error: you need to download the country_osm_grid first:'; - echo "\n wget -O ".CONST_DataDir."/data/country_osm_grid.sql.gz https://www.nominatim.org/data/country_grid.sql.gz\n"; - exit(1); - } - $this->pgsqlRunScriptFile(CONST_DataDir.'/data/country_name.sql'); - $this->pgsqlRunScriptFile(CONST_DataDir.'/data/country_osm_grid.sql.gz'); - $this->pgsqlRunScriptFile(CONST_DataDir.'/data/gb_postcode_table.sql'); - $this->pgsqlRunScriptFile(CONST_DataDir.'/data/us_postcode_table.sql'); - - $sPostcodeFilename = CONST_InstallDir.'/gb_postcode_data.sql.gz'; - if (file_exists($sPostcodeFilename)) { - $this->pgsqlRunScriptFile($sPostcodeFilename); - } else { - warn('optional external GB postcode table file ('.$sPostcodeFilename.') not found. Skipping.'); - } - - $sPostcodeFilename = CONST_InstallDir.'/us_postcode_data.sql.gz'; - if (file_exists($sPostcodeFilename)) { - $this->pgsqlRunScriptFile($sPostcodeFilename); - } else { - warn('optional external US postcode table file ('.$sPostcodeFilename.') not found. Skipping.'); - } + $this->pgsqlRunScriptFile(CONST_DataDir.'/country_name.sql'); + $this->pgsqlRunScriptFile(CONST_DataDir.'/country_osm_grid.sql.gz'); if ($this->bNoPartitions) { $this->pgsqlRunScript('update country_name set partition = 0'); @@ -269,7 +248,7 @@ class SetupFunctions { info('Create Tables'); - $sTemplate = file_get_contents(CONST_DataDir.'/sql/tables.sql'); + $sTemplate = file_get_contents(CONST_SqlDir.'/tables.sql'); $sTemplate = $this->replaceSqlPatterns($sTemplate); $this->pgsqlRunScript($sTemplate, false); @@ -285,7 +264,7 @@ class SetupFunctions { info('Create Tables'); - $sTemplate = file_get_contents(CONST_DataDir.'/sql/table-triggers.sql'); + $sTemplate = file_get_contents(CONST_SqlDir.'/table-triggers.sql'); $sTemplate = $this->replaceSqlPatterns($sTemplate); $this->pgsqlRunScript($sTemplate, false); @@ -295,7 +274,7 @@ class SetupFunctions { info('Create Partition Tables'); - $sTemplate = file_get_contents(CONST_DataDir.'/sql/partition-tables.src.sql'); + $sTemplate = file_get_contents(CONST_SqlDir.'/partition-tables.src.sql'); $sTemplate = $this->replaceSqlPatterns($sTemplate); $this->pgsqlRunPartitionScript($sTemplate); @@ -366,7 +345,7 @@ class SetupFunctions // pre-create the word list if (!$bDisableTokenPrecalc) { info('Loading word list'); - $this->pgsqlRunScriptFile(CONST_DataDir.'/data/words.sql'); + $this->pgsqlRunScriptFile(CONST_DataDir.'/words.sql'); } info('Load Data'); @@ -458,7 +437,7 @@ class SetupFunctions warn('Tiger data import selected but no files found in path '.$sTigerPath); return; } - $sTemplate = file_get_contents(CONST_DataDir.'/sql/tiger_import_start.sql'); + $sTemplate = file_get_contents(CONST_SqlDir.'/tiger_import_start.sql'); $sTemplate = $this->replaceSqlPatterns($sTemplate); $this->pgsqlRunScript($sTemplate, false); @@ -512,7 +491,7 @@ class SetupFunctions } info('Creating indexes on Tiger data'); - $sTemplate = file_get_contents(CONST_DataDir.'/sql/tiger_import_finish.sql'); + $sTemplate = file_get_contents(CONST_SqlDir.'/tiger_import_finish.sql'); $sTemplate = $this->replaceSqlPatterns($sTemplate); $this->pgsqlRunScript($sTemplate, false); @@ -521,6 +500,23 @@ class SetupFunctions public function calculatePostcodes($bCMDResultAll) { info('Calculate Postcodes'); + $this->pgsqlRunScriptFile(CONST_SqlDir.'/postcode_tables.sql'); + + $sPostcodeFilename = CONST_InstallDir.'/gb_postcode_data.sql.gz'; + if (file_exists($sPostcodeFilename)) { + $this->pgsqlRunScriptFile($sPostcodeFilename); + } else { + warn('optional external GB postcode table file ('.$sPostcodeFilename.') not found. Skipping.'); + } + + $sPostcodeFilename = CONST_InstallDir.'/us_postcode_data.sql.gz'; + if (file_exists($sPostcodeFilename)) { + $this->pgsqlRunScriptFile($sPostcodeFilename); + } else { + warn('optional external US postcode table file ('.$sPostcodeFilename.') not found. Skipping.'); + } + + $this->db()->exec('TRUNCATE location_postcode'); $sSQL = 'INSERT INTO location_postcode'; @@ -620,12 +616,12 @@ class SetupFunctions $this->db()->exec("DROP INDEX $sIndexName;"); } - $sTemplate = file_get_contents(CONST_DataDir.'/sql/indices.src.sql'); + $sTemplate = file_get_contents(CONST_SqlDir.'/indices.src.sql'); if (!$this->bDrop) { - $sTemplate .= file_get_contents(CONST_DataDir.'/sql/indices_updates.src.sql'); + $sTemplate .= file_get_contents(CONST_SqlDir.'/indices_updates.src.sql'); } if (!$this->dbReverseOnly()) { - $sTemplate .= file_get_contents(CONST_DataDir.'/sql/indices_search.src.sql'); + $sTemplate .= file_get_contents(CONST_SqlDir.'/indices_search.src.sql'); } $sTemplate = $this->replaceSqlPatterns($sTemplate); @@ -736,8 +732,6 @@ class SetupFunctions fwrite($rFile, '@define(\'CONST_Debug\', $_GET[\'debug\'] ?? false);'."\n\n"); fwriteConstDef($rFile, 'LibDir', CONST_LibDir); - fwriteConstDef($rFile, 'DataDir', CONST_DataDir); - fwriteConstDef($rFile, 'InstallDir', CONST_InstallDir); fwriteConstDef($rFile, 'Database_DSN', getSetting('DATABASE_DSN')); fwriteConstDef($rFile, 'Default_Language', getSetting('DEFAULT_LANGUAGE')); fwriteConstDef($rFile, 'Log_DB', getSettingBool('LOG_DB')); @@ -753,8 +747,7 @@ class SetupFunctions fwriteConstDef($rFile, 'Use_US_Tiger_Data', getSettingBool('USE_US_TIGER_DATA')); fwriteConstDef($rFile, 'MapIcon_URL', getSetting('MAPICON_URL')); - // XXX scripts should go into the library. - fwrite($rFile, 'require_once(\''.CONST_DataDir.'/website/'.$sScript."');\n"); + fwrite($rFile, 'require_once(\''.CONST_LibDir.'/website/'.$sScript."');\n"); fclose($rFile); chmod(CONST_InstallDir.'/website/'.$sScript, 0755); diff --git a/lib/setup_functions.php b/lib-php/setup_functions.php similarity index 91% rename from lib/setup_functions.php rename to lib-php/setup_functions.php index dc84cf92..c89db534 100755 --- a/lib/setup_functions.php +++ b/lib-php/setup_functions.php @@ -27,7 +27,7 @@ function getImportStyle() $sStyle = getSetting('IMPORT_STYLE'); if (in_array($sStyle, array('admin', 'street', 'address', 'full', 'extratags'))) { - return CONST_DataDir.'/settings/import-'.$sStyle.'.style'; + return CONST_ConfigDir.'/import-'.$sStyle.'.style'; } return $sStyle; diff --git a/lib/template/address-geocodejson.php b/lib-php/template/address-geocodejson.php similarity index 100% rename from lib/template/address-geocodejson.php rename to lib-php/template/address-geocodejson.php diff --git a/lib/template/address-geojson.php b/lib-php/template/address-geojson.php similarity index 100% rename from lib/template/address-geojson.php rename to lib-php/template/address-geojson.php diff --git a/lib/template/address-json.php b/lib-php/template/address-json.php similarity index 100% rename from lib/template/address-json.php rename to lib-php/template/address-json.php diff --git a/lib/template/address-xml.php b/lib-php/template/address-xml.php similarity index 100% rename from lib/template/address-xml.php rename to lib-php/template/address-xml.php diff --git a/lib/template/details-json.php b/lib-php/template/details-json.php similarity index 100% rename from lib/template/details-json.php rename to lib-php/template/details-json.php diff --git a/lib/template/error-json.php b/lib-php/template/error-json.php similarity index 100% rename from lib/template/error-json.php rename to lib-php/template/error-json.php diff --git a/lib/template/error-xml.php b/lib-php/template/error-xml.php similarity index 100% rename from lib/template/error-xml.php rename to lib-php/template/error-xml.php diff --git a/lib/template/search-batch-json.php b/lib-php/template/search-batch-json.php similarity index 100% rename from lib/template/search-batch-json.php rename to lib-php/template/search-batch-json.php diff --git a/lib/template/search-geocodejson.php b/lib-php/template/search-geocodejson.php similarity index 100% rename from lib/template/search-geocodejson.php rename to lib-php/template/search-geocodejson.php diff --git a/lib/template/search-geojson.php b/lib-php/template/search-geojson.php similarity index 100% rename from lib/template/search-geojson.php rename to lib-php/template/search-geojson.php diff --git a/lib/template/search-json.php b/lib-php/template/search-json.php similarity index 100% rename from lib/template/search-json.php rename to lib-php/template/search-json.php diff --git a/lib/template/search-xml.php b/lib-php/template/search-xml.php similarity index 100% rename from lib/template/search-xml.php rename to lib-php/template/search-xml.php diff --git a/website/403.html b/lib-php/website/403.html similarity index 100% rename from website/403.html rename to lib-php/website/403.html diff --git a/website/509.html b/lib-php/website/509.html similarity index 100% rename from website/509.html rename to lib-php/website/509.html diff --git a/website/crossdomain.xml b/lib-php/website/crossdomain.xml similarity index 100% rename from website/crossdomain.xml rename to lib-php/website/crossdomain.xml diff --git a/website/deletable.php b/lib-php/website/deletable.php similarity index 100% rename from website/deletable.php rename to lib-php/website/deletable.php diff --git a/website/details.php b/lib-php/website/details.php similarity index 100% rename from website/details.php rename to lib-php/website/details.php diff --git a/website/favicon.ico b/lib-php/website/favicon.ico similarity index 100% rename from website/favicon.ico rename to lib-php/website/favicon.ico diff --git a/website/lookup.php b/lib-php/website/lookup.php similarity index 100% rename from website/lookup.php rename to lib-php/website/lookup.php diff --git a/website/nominatim.xml b/lib-php/website/nominatim.xml similarity index 100% rename from website/nominatim.xml rename to lib-php/website/nominatim.xml diff --git a/website/polygons.php b/lib-php/website/polygons.php similarity index 100% rename from website/polygons.php rename to lib-php/website/polygons.php diff --git a/website/reverse.php b/lib-php/website/reverse.php similarity index 100% rename from website/reverse.php rename to lib-php/website/reverse.php diff --git a/website/robots.txt b/lib-php/website/robots.txt similarity index 100% rename from website/robots.txt rename to lib-php/website/robots.txt diff --git a/website/search.php b/lib-php/website/search.php similarity index 100% rename from website/search.php rename to lib-php/website/search.php diff --git a/website/status.php b/lib-php/website/status.php similarity index 100% rename from website/status.php rename to lib-php/website/status.php diff --git a/website/taginfo.json b/lib-php/website/taginfo.json similarity index 100% rename from website/taginfo.json rename to lib-php/website/taginfo.json diff --git a/sql/aux_tables.sql b/lib-sql/aux_tables.sql similarity index 100% rename from sql/aux_tables.sql rename to lib-sql/aux_tables.sql diff --git a/sql/functions/address_lookup.sql b/lib-sql/functions/address_lookup.sql similarity index 100% rename from sql/functions/address_lookup.sql rename to lib-sql/functions/address_lookup.sql diff --git a/sql/functions/aux_property.sql b/lib-sql/functions/aux_property.sql similarity index 100% rename from sql/functions/aux_property.sql rename to lib-sql/functions/aux_property.sql diff --git a/sql/functions/importance.sql b/lib-sql/functions/importance.sql similarity index 100% rename from sql/functions/importance.sql rename to lib-sql/functions/importance.sql diff --git a/sql/functions/interpolation.sql b/lib-sql/functions/interpolation.sql similarity index 100% rename from sql/functions/interpolation.sql rename to lib-sql/functions/interpolation.sql diff --git a/sql/functions/normalization.sql b/lib-sql/functions/normalization.sql similarity index 100% rename from sql/functions/normalization.sql rename to lib-sql/functions/normalization.sql diff --git a/sql/functions/place_triggers.sql b/lib-sql/functions/place_triggers.sql similarity index 100% rename from sql/functions/place_triggers.sql rename to lib-sql/functions/place_triggers.sql diff --git a/sql/functions/placex_triggers.sql b/lib-sql/functions/placex_triggers.sql similarity index 100% rename from sql/functions/placex_triggers.sql rename to lib-sql/functions/placex_triggers.sql diff --git a/sql/functions/postcode_triggers.sql b/lib-sql/functions/postcode_triggers.sql similarity index 100% rename from sql/functions/postcode_triggers.sql rename to lib-sql/functions/postcode_triggers.sql diff --git a/sql/functions/ranking.sql b/lib-sql/functions/ranking.sql similarity index 100% rename from sql/functions/ranking.sql rename to lib-sql/functions/ranking.sql diff --git a/sql/functions/utils.sql b/lib-sql/functions/utils.sql similarity index 100% rename from sql/functions/utils.sql rename to lib-sql/functions/utils.sql diff --git a/sql/indices.src.sql b/lib-sql/indices.src.sql similarity index 100% rename from sql/indices.src.sql rename to lib-sql/indices.src.sql diff --git a/sql/indices_search.src.sql b/lib-sql/indices_search.src.sql similarity index 100% rename from sql/indices_search.src.sql rename to lib-sql/indices_search.src.sql diff --git a/sql/indices_updates.src.sql b/lib-sql/indices_updates.src.sql similarity index 100% rename from sql/indices_updates.src.sql rename to lib-sql/indices_updates.src.sql diff --git a/sql/partition-functions.src.sql b/lib-sql/partition-functions.src.sql similarity index 100% rename from sql/partition-functions.src.sql rename to lib-sql/partition-functions.src.sql diff --git a/sql/partition-tables.src.sql b/lib-sql/partition-tables.src.sql similarity index 100% rename from sql/partition-tables.src.sql rename to lib-sql/partition-tables.src.sql diff --git a/lib-sql/postcode_tables.sql b/lib-sql/postcode_tables.sql new file mode 100644 index 00000000..c445d6af --- /dev/null +++ b/lib-sql/postcode_tables.sql @@ -0,0 +1,15 @@ +DROP TABLE IF EXISTS gb_postcode; +CREATE TABLE gb_postcode ( + id integer, + postcode character varying(9), + geometry geometry, + CONSTRAINT enforce_dims_geometry CHECK ((st_ndims(geometry) = 2)), + CONSTRAINT enforce_srid_geometry CHECK ((st_srid(geometry) = 4326)) +); + +DROP TABLE IF EXISTS us_postcode; +CREATE TABLE us_postcode ( + postcode text, + x double precision, + y double precision +); diff --git a/sql/table-triggers.sql b/lib-sql/table-triggers.sql similarity index 100% rename from sql/table-triggers.sql rename to lib-sql/table-triggers.sql diff --git a/sql/tables.sql b/lib-sql/tables.sql similarity index 98% rename from sql/tables.sql rename to lib-sql/tables.sql index 8647e304..d15e42c4 100644 --- a/sql/tables.sql +++ b/lib-sql/tables.sql @@ -35,8 +35,6 @@ GRANT UPDATE ON new_query_log TO "{www-user}" ; GRANT SELECT ON new_query_log TO "{www-user}" ; GRANT SELECT ON TABLE country_name TO "{www-user}"; -GRANT SELECT ON TABLE gb_postcode TO "{www-user}"; -GRANT SELECT ON TABLE us_postcode TO "{www-user}"; drop table IF EXISTS word; CREATE TABLE word ( diff --git a/sql/tiger_import_finish.sql b/lib-sql/tiger_import_finish.sql similarity index 100% rename from sql/tiger_import_finish.sql rename to lib-sql/tiger_import_finish.sql diff --git a/sql/tiger_import_start.sql b/lib-sql/tiger_import_start.sql similarity index 100% rename from sql/tiger_import_start.sql rename to lib-sql/tiger_import_start.sql diff --git a/sql/update-postcodes.sql b/lib-sql/update-postcodes.sql similarity index 100% rename from sql/update-postcodes.sql rename to lib-sql/update-postcodes.sql diff --git a/sql/words.sql b/lib-sql/words.sql similarity index 100% rename from sql/words.sql rename to lib-sql/words.sql diff --git a/sql/words_from_search_name.sql b/lib-sql/words_from_search_name.sql similarity index 100% rename from sql/words_from_search_name.sql rename to lib-sql/words_from_search_name.sql diff --git a/nominatim/cli.py b/nominatim/cli.py index 37bcaffb..8cb73a8e 100644 --- a/nominatim/cli.py +++ b/nominatim/cli.py @@ -2,31 +2,19 @@ Command-line interface to the Nominatim functions for import, update, database administration and querying. """ -import datetime as dt +import logging import os -import socket import sys -import time import argparse -import logging from pathlib import Path from .config import Configuration -from .tools.exec_utils import run_legacy_script, run_api_script, run_php_server -from .db.connection import connect -from .db import status +from .tools.exec_utils import run_legacy_script, run_php_server from .errors import UsageError +from . import clicmd LOG = logging.getLogger() -def _num_system_cpus(): - try: - cpus = len(os.sched_getaffinity(0)) - except NotImplementedError: - cpus = None - - return cpus or os.cpu_count() - class CommandlineParser: """ Wraps some of the common functions for parsing the command line @@ -80,7 +68,8 @@ class CommandlineParser: self.parser.print_help() return 1 - for arg in ('module_dir', 'osm2pgsql_path', 'phplib_dir', 'data_dir', 'phpcgi_path'): + for arg in ('module_dir', 'osm2pgsql_path', 'phplib_dir', 'sqllib_dir', + 'data_dir', 'config_dir', 'phpcgi_path'): setattr(args, arg, Path(kwargs[arg])) args.project_dir = Path(args.project_dir).resolve() @@ -89,7 +78,7 @@ class CommandlineParser: datefmt='%Y-%m-%d %H:%M:%S', level=max(4 - args.verbose, 1) * 10) - args.config = Configuration(args.project_dir, args.data_dir / 'settings') + args.config = Configuration(args.project_dir, args.config_dir) log = logging.getLogger() log.warning('Using project directory: %s', str(args.project_dir)) @@ -105,16 +94,6 @@ class CommandlineParser: return 1 -def _osm2pgsql_options_from_args(args, default_cache, default_threads): - """ Set up the stanadrd osm2pgsql from the command line arguments. - """ - return dict(osm2pgsql=args.osm2pgsql_path, - osm2pgsql_cache=args.osm2pgsql_cache or default_cache, - osm2pgsql_style=args.config.get_import_style_file(), - threads=args.threads or default_threads, - dsn=args.config.get_libpq_dsn(), - flatnode_file=args.config.FLATNODE_FILE) - ##### Subcommand classes # # Each class needs to implement two functions: add_args() adds the CLI parameters @@ -237,153 +216,6 @@ class SetupSpecialPhrases: return run_legacy_script('specialphrases.php', '--wiki-import', nominatim_env=args) -class UpdateReplication: - """\ - Update the database using an online replication service. - """ - - @staticmethod - def add_args(parser): - group = parser.add_argument_group('Arguments for initialisation') - group.add_argument('--init', action='store_true', - help='Initialise the update process') - group.add_argument('--no-update-functions', dest='update_functions', - action='store_false', - help="""Do not update the trigger function to - support differential updates.""") - group = parser.add_argument_group('Arguments for updates') - group.add_argument('--check-for-updates', action='store_true', - help='Check if new updates are available and exit') - group.add_argument('--once', action='store_true', - help="""Download and apply updates only once. When - not set, updates are continuously applied""") - group.add_argument('--no-index', action='store_false', dest='do_index', - help="""Do not index the new data. Only applicable - together with --once""") - group.add_argument('--osm2pgsql-cache', metavar='SIZE', type=int, - help='Size of cache to be used by osm2pgsql (in MB)') - group = parser.add_argument_group('Download parameters') - group.add_argument('--socket-timeout', dest='socket_timeout', type=int, default=60, - help='Set timeout for file downloads.') - - @staticmethod - def _init_replication(args): - from .tools import replication, refresh - - socket.setdefaulttimeout(args.socket_timeout) - - LOG.warning("Initialising replication updates") - conn = connect(args.config.get_libpq_dsn()) - replication.init_replication(conn, base_url=args.config.REPLICATION_URL) - if args.update_functions: - LOG.warning("Create functions") - refresh.create_functions(conn, args.config, args.data_dir, - True, False) - conn.close() - return 0 - - - @staticmethod - def _check_for_updates(args): - from .tools import replication - - conn = connect(args.config.get_libpq_dsn()) - ret = replication.check_for_updates(conn, base_url=args.config.REPLICATION_URL) - conn.close() - return ret - - @staticmethod - def _report_update(batchdate, start_import, start_index): - def round_time(delta): - return dt.timedelta(seconds=int(delta.total_seconds())) - - end = dt.datetime.now(dt.timezone.utc) - LOG.warning("Update completed. Import: %s. %sTotal: %s. Remaining backlog: %s.", - round_time((start_index or end) - start_import), - "Indexing: {} ".format(round_time(end - start_index)) - if start_index else '', - round_time(end - start_import), - round_time(end - batchdate)) - - @staticmethod - def _update(args): - from .tools import replication - from .indexer.indexer import Indexer - - params = _osm2pgsql_options_from_args(args, 2000, 1) - params.update(base_url=args.config.REPLICATION_URL, - update_interval=args.config.get_int('REPLICATION_UPDATE_INTERVAL'), - import_file=args.project_dir / 'osmosischange.osc', - max_diff_size=args.config.get_int('REPLICATION_MAX_DIFF'), - indexed_only=not args.once) - - # Sanity check to not overwhelm the Geofabrik servers. - if 'download.geofabrik.de'in params['base_url']\ - and params['update_interval'] < 86400: - LOG.fatal("Update interval too low for download.geofabrik.de.\n" - "Please check install documentation " - "(https://nominatim.org/release-docs/latest/admin/Import-and-Update#" - "setting-up-the-update-process).") - raise UsageError("Invalid replication update interval setting.") - - if not args.once: - if not args.do_index: - LOG.fatal("Indexing cannot be disabled when running updates continuously.") - raise UsageError("Bad argument '--no-index'.") - recheck_interval = args.config.get_int('REPLICATION_RECHECK_INTERVAL') - - while True: - conn = connect(args.config.get_libpq_dsn()) - start = dt.datetime.now(dt.timezone.utc) - state = replication.update(conn, params) - if state is not replication.UpdateState.NO_CHANGES: - status.log_status(conn, start, 'import') - batchdate, _, _ = status.get_status(conn) - conn.close() - - if state is not replication.UpdateState.NO_CHANGES and args.do_index: - index_start = dt.datetime.now(dt.timezone.utc) - indexer = Indexer(args.config.get_libpq_dsn(), - args.threads or 1) - indexer.index_boundaries(0, 30) - indexer.index_by_rank(0, 30) - - conn = connect(args.config.get_libpq_dsn()) - status.set_indexed(conn, True) - status.log_status(conn, index_start, 'index') - conn.close() - else: - index_start = None - - if LOG.isEnabledFor(logging.WARNING): - UpdateReplication._report_update(batchdate, start, index_start) - - if args.once: - break - - if state is replication.UpdateState.NO_CHANGES: - LOG.warning("No new changes. Sleeping for %d sec.", recheck_interval) - time.sleep(recheck_interval) - - return state.value - - @staticmethod - def run(args): - try: - import osmium # pylint: disable=W0611 - except ModuleNotFoundError: - LOG.fatal("pyosmium not installed. Replication functions not available.\n" - "To install pyosmium via pip: pip3 install osmium") - return 1 - - if args.init: - return UpdateReplication._init_replication(args) - - if args.check_for_updates: - return UpdateReplication._check_for_updates(args) - - return UpdateReplication._update(args) - class UpdateAddData: """\ Add additional data from a file or an online source. @@ -434,157 +266,6 @@ class UpdateAddData: return run_legacy_script(*params, nominatim_env=args) -class UpdateIndex: - """\ - Reindex all new and modified data. - """ - - @staticmethod - def add_args(parser): - group = parser.add_argument_group('Filter arguments') - group.add_argument('--boundaries-only', action='store_true', - help="""Index only administrative boundaries.""") - group.add_argument('--no-boundaries', action='store_true', - help="""Index everything except administrative boundaries.""") - group.add_argument('--minrank', '-r', type=int, metavar='RANK', default=0, - help='Minimum/starting rank') - group.add_argument('--maxrank', '-R', type=int, metavar='RANK', default=30, - help='Maximum/finishing rank') - - @staticmethod - def run(args): - from .indexer.indexer import Indexer - - indexer = Indexer(args.config.get_libpq_dsn(), - args.threads or _num_system_cpus() or 1) - - if not args.no_boundaries: - indexer.index_boundaries(args.minrank, args.maxrank) - if not args.boundaries_only: - indexer.index_by_rank(args.minrank, args.maxrank) - - if not args.no_boundaries and not args.boundaries_only \ - and args.minrank == 0 and args.maxrank == 30: - conn = connect(args.config.get_libpq_dsn()) - status.set_indexed(conn, True) - conn.close() - - return 0 - - -class UpdateRefresh: - """\ - Recompute auxiliary data used by the indexing process. - - These functions must not be run in parallel with other update commands. - """ - - @staticmethod - def add_args(parser): - group = parser.add_argument_group('Data arguments') - group.add_argument('--postcodes', action='store_true', - help='Update postcode centroid table') - group.add_argument('--word-counts', action='store_true', - help='Compute frequency of full-word search terms') - group.add_argument('--address-levels', action='store_true', - help='Reimport address level configuration') - group.add_argument('--functions', action='store_true', - help='Update the PL/pgSQL functions in the database') - group.add_argument('--wiki-data', action='store_true', - help='Update Wikipedia/data importance numbers.') - group.add_argument('--importance', action='store_true', - help='Recompute place importances (expensive!)') - group.add_argument('--website', action='store_true', - help='Refresh the directory that serves the scripts for the web API') - group = parser.add_argument_group('Arguments for function refresh') - group.add_argument('--no-diff-updates', action='store_false', dest='diffs', - help='Do not enable code for propagating updates') - group.add_argument('--enable-debug-statements', action='store_true', - help='Enable debug warning statements in functions') - - @staticmethod - def run(args): - from .tools import refresh - - if args.postcodes: - LOG.warning("Update postcodes centroid") - conn = connect(args.config.get_libpq_dsn()) - refresh.update_postcodes(conn, args.data_dir) - conn.close() - - if args.word_counts: - LOG.warning('Recompute frequency of full-word search terms') - conn = connect(args.config.get_libpq_dsn()) - refresh.recompute_word_counts(conn, args.data_dir) - conn.close() - - if args.address_levels: - cfg = Path(args.config.ADDRESS_LEVEL_CONFIG) - LOG.warning('Updating address levels from %s', cfg) - conn = connect(args.config.get_libpq_dsn()) - refresh.load_address_levels_from_file(conn, cfg) - conn.close() - - if args.functions: - LOG.warning('Create functions') - conn = connect(args.config.get_libpq_dsn()) - refresh.create_functions(conn, args.config, args.data_dir, - args.diffs, args.enable_debug_statements) - conn.close() - - if args.wiki_data: - run_legacy_script('setup.php', '--import-wikipedia-articles', - nominatim_env=args, throw_on_fail=True) - # Attention: importance MUST come after wiki data import. - if args.importance: - run_legacy_script('update.php', '--recompute-importance', - nominatim_env=args, throw_on_fail=True) - if args.website: - run_legacy_script('setup.php', '--setup-website', - nominatim_env=args, throw_on_fail=True) - - return 0 - - -class AdminCheckDatabase: - """\ - Check that the database is complete and operational. - """ - - @staticmethod - def add_args(parser): - pass # No options - - @staticmethod - def run(args): - return run_legacy_script('check_import_finished.php', nominatim_env=args) - - -class AdminWarm: - """\ - Warm database caches for search and reverse queries. - """ - - @staticmethod - def add_args(parser): - group = parser.add_argument_group('Target arguments') - group.add_argument('--search-only', action='store_const', dest='target', - const='search', - help="Only pre-warm tables for search queries") - group.add_argument('--reverse-only', action='store_const', dest='target', - const='reverse', - help="Only pre-warm tables for reverse queries") - - @staticmethod - def run(args): - params = ['warm.php'] - if args.target == 'reverse': - params.append('--reverse-only') - if args.target == 'search': - params.append('--search-only') - return run_legacy_script(*params, nominatim_env=args) - - class QueryExport: """\ Export addresses as CSV file from the database. @@ -662,246 +343,6 @@ class AdminServe: def run(args): run_php_server(args.server, args.project_dir / 'website') -STRUCTURED_QUERY = ( - ('street', 'housenumber and street'), - ('city', 'city, town or village'), - ('county', 'county'), - ('state', 'state'), - ('country', 'country'), - ('postalcode', 'postcode') -) - -EXTRADATA_PARAMS = ( - ('addressdetails', 'Include a breakdown of the address into elements.'), - ('extratags', """Include additional information if available - (e.g. wikipedia link, opening hours)."""), - ('namedetails', 'Include a list of alternative names.') -) - -DETAILS_SWITCHES = ( - ('addressdetails', 'Include a breakdown of the address into elements.'), - ('keywords', 'Include a list of name keywords and address keywords.'), - ('linkedplaces', 'Include a details of places that are linked with this one.'), - ('hierarchy', 'Include details of places lower in the address hierarchy.'), - ('group_hierarchy', 'Group the places by type.'), - ('polygon_geojson', 'Include geometry of result.') -) - -def _add_api_output_arguments(parser): - group = parser.add_argument_group('Output arguments') - group.add_argument('--format', default='jsonv2', - choices=['xml', 'json', 'jsonv2', 'geojson', 'geocodejson'], - help='Format of result') - for name, desc in EXTRADATA_PARAMS: - group.add_argument('--' + name, action='store_true', help=desc) - - group.add_argument('--lang', '--accept-language', metavar='LANGS', - help='Preferred language order for presenting search results') - group.add_argument('--polygon-output', - choices=['geojson', 'kml', 'svg', 'text'], - help='Output geometry of results as a GeoJSON, KML, SVG or WKT.') - group.add_argument('--polygon-threshold', type=float, metavar='TOLERANCE', - help="""Simplify output geometry. - Parameter is difference tolerance in degrees.""") - - -class APISearch: - """\ - Execute API search query. - """ - - @staticmethod - def add_args(parser): - group = parser.add_argument_group('Query arguments') - group.add_argument('--query', - help='Free-form query string') - for name, desc in STRUCTURED_QUERY: - group.add_argument('--' + name, help='Structured query: ' + desc) - - _add_api_output_arguments(parser) - - group = parser.add_argument_group('Result limitation') - group.add_argument('--countrycodes', metavar='CC,..', - help='Limit search results to one or more countries.') - group.add_argument('--exclude_place_ids', metavar='ID,..', - help='List of search object to be excluded') - group.add_argument('--limit', type=int, - help='Limit the number of returned results') - group.add_argument('--viewbox', metavar='X1,Y1,X2,Y2', - help='Preferred area to find search results') - group.add_argument('--bounded', action='store_true', - help='Strictly restrict results to viewbox area') - - group = parser.add_argument_group('Other arguments') - group.add_argument('--no-dedupe', action='store_false', dest='dedupe', - help='Do not remove duplicates from the result list') - - - @staticmethod - def run(args): - if args.query: - params = dict(q=args.query) - else: - params = {k : getattr(args, k) for k, _ in STRUCTURED_QUERY if getattr(args, k)} - - for param, _ in EXTRADATA_PARAMS: - if getattr(args, param): - params[param] = '1' - for param in ('format', 'countrycodes', 'exclude_place_ids', 'limit', 'viewbox'): - if getattr(args, param): - params[param] = getattr(args, param) - if args.lang: - params['accept-language'] = args.lang - if args.polygon_output: - params['polygon_' + args.polygon_output] = '1' - if args.polygon_threshold: - params['polygon_threshold'] = args.polygon_threshold - if args.bounded: - params['bounded'] = '1' - if not args.dedupe: - params['dedupe'] = '0' - - return run_api_script('search', args.project_dir, - phpcgi_bin=args.phpcgi_path, params=params) - -class APIReverse: - """\ - Execute API reverse query. - """ - - @staticmethod - def add_args(parser): - group = parser.add_argument_group('Query arguments') - group.add_argument('--lat', type=float, required=True, - help='Latitude of coordinate to look up (in WGS84)') - group.add_argument('--lon', type=float, required=True, - help='Longitude of coordinate to look up (in WGS84)') - group.add_argument('--zoom', type=int, - help='Level of detail required for the address') - - _add_api_output_arguments(parser) - - - @staticmethod - def run(args): - params = dict(lat=args.lat, lon=args.lon) - if args.zoom is not None: - params['zoom'] = args.zoom - - for param, _ in EXTRADATA_PARAMS: - if getattr(args, param): - params[param] = '1' - if args.format: - params['format'] = args.format - if args.lang: - params['accept-language'] = args.lang - if args.polygon_output: - params['polygon_' + args.polygon_output] = '1' - if args.polygon_threshold: - params['polygon_threshold'] = args.polygon_threshold - - return run_api_script('reverse', args.project_dir, - phpcgi_bin=args.phpcgi_path, params=params) - - -class APILookup: - """\ - Execute API reverse query. - """ - - @staticmethod - def add_args(parser): - group = parser.add_argument_group('Query arguments') - group.add_argument('--id', metavar='OSMID', - action='append', required=True, dest='ids', - help='OSM id to lookup in format (may be repeated)') - - _add_api_output_arguments(parser) - - - @staticmethod - def run(args): - params = dict(osm_ids=','.join(args.ids)) - - for param, _ in EXTRADATA_PARAMS: - if getattr(args, param): - params[param] = '1' - if args.format: - params['format'] = args.format - if args.lang: - params['accept-language'] = args.lang - if args.polygon_output: - params['polygon_' + args.polygon_output] = '1' - if args.polygon_threshold: - params['polygon_threshold'] = args.polygon_threshold - - return run_api_script('lookup', args.project_dir, - phpcgi_bin=args.phpcgi_path, params=params) - - -class APIDetails: - """\ - Execute API lookup query. - """ - - @staticmethod - def add_args(parser): - group = parser.add_argument_group('Query arguments') - objs = group.add_mutually_exclusive_group(required=True) - objs.add_argument('--node', '-n', type=int, - help="Look up the OSM node with the given ID.") - objs.add_argument('--way', '-w', type=int, - help="Look up the OSM way with the given ID.") - objs.add_argument('--relation', '-r', type=int, - help="Look up the OSM relation with the given ID.") - objs.add_argument('--place_id', '-p', type=int, - help='Database internal identifier of the OSM object to look up.') - group.add_argument('--class', dest='object_class', - help="""Class type to disambiguated multiple entries - of the same object.""") - - group = parser.add_argument_group('Output arguments') - for name, desc in DETAILS_SWITCHES: - group.add_argument('--' + name, action='store_true', help=desc) - group.add_argument('--lang', '--accept-language', metavar='LANGS', - help='Preferred language order for presenting search results') - - @staticmethod - def run(args): - if args.node: - params = dict(osmtype='N', osmid=args.node) - elif args.way: - params = dict(osmtype='W', osmid=args.node) - elif args.relation: - params = dict(osmtype='R', osmid=args.node) - else: - params = dict(place_id=args.place_id) - if args.object_class: - params['class'] = args.object_class - for name, _ in DETAILS_SWITCHES: - params[name] = '1' if getattr(args, name) else '0' - - return run_api_script('details', args.project_dir, - phpcgi_bin=args.phpcgi_path, params=params) - - -class APIStatus: - """\ - Execute API status query. - """ - - @staticmethod - def add_args(parser): - group = parser.add_argument_group('API parameters') - group.add_argument('--format', default='text', choices=['text', 'json'], - help='Format of result') - - @staticmethod - def run(args): - return run_api_script('status', args.project_dir, - phpcgi_bin=args.phpcgi_path, - params=dict(format=args.format)) - def nominatim(**kwargs): """\ @@ -912,26 +353,25 @@ def nominatim(**kwargs): parser.add_subcommand('import', SetupAll) parser.add_subcommand('freeze', SetupFreeze) - parser.add_subcommand('replication', UpdateReplication) - - parser.add_subcommand('check-database', AdminCheckDatabase) - parser.add_subcommand('warm', AdminWarm) + parser.add_subcommand('replication', clicmd.UpdateReplication) parser.add_subcommand('special-phrases', SetupSpecialPhrases) parser.add_subcommand('add-data', UpdateAddData) - parser.add_subcommand('index', UpdateIndex) - parser.add_subcommand('refresh', UpdateRefresh) + parser.add_subcommand('index', clicmd.UpdateIndex) + parser.add_subcommand('refresh', clicmd.UpdateRefresh) + + parser.add_subcommand('admin', clicmd.AdminFuncs) parser.add_subcommand('export', QueryExport) parser.add_subcommand('serve', AdminServe) if kwargs.get('phpcgi_path'): - parser.add_subcommand('search', APISearch) - parser.add_subcommand('reverse', APIReverse) - parser.add_subcommand('lookup', APILookup) - parser.add_subcommand('details', APIDetails) - parser.add_subcommand('status', APIStatus) + parser.add_subcommand('search', clicmd.APISearch) + parser.add_subcommand('reverse', clicmd.APIReverse) + parser.add_subcommand('lookup', clicmd.APILookup) + parser.add_subcommand('details', clicmd.APIDetails) + parser.add_subcommand('status', clicmd.APIStatus) else: parser.parser.epilog = 'php-cgi not found. Query commands not available.' diff --git a/nominatim/clicmd/__init__.py b/nominatim/clicmd/__init__.py new file mode 100644 index 00000000..9a686df2 --- /dev/null +++ b/nominatim/clicmd/__init__.py @@ -0,0 +1,9 @@ +""" +Subcommand definitions for the command-line tool. +""" + +from .replication import UpdateReplication +from .api import APISearch, APIReverse, APILookup, APIDetails, APIStatus +from .index import UpdateIndex +from .refresh import UpdateRefresh +from .admin import AdminFuncs diff --git a/nominatim/clicmd/admin.py b/nominatim/clicmd/admin.py new file mode 100644 index 00000000..8d34f386 --- /dev/null +++ b/nominatim/clicmd/admin.py @@ -0,0 +1,64 @@ +""" +Implementation of the 'admin' subcommand. +""" +from ..tools.exec_utils import run_legacy_script +from ..db.connection import connect + +# Do not repeat documentation of subcommand classes. +# pylint: disable=C0111 +# Using non-top-level imports to avoid eventually unused imports. +# pylint: disable=E0012,C0415 + +class AdminFuncs: + """\ + Analyse and maintain the database. + """ + + @staticmethod + def add_args(parser): + group = parser.add_argument_group('Admin task arguments') + group.add_argument('--warm', action='store_true', + help='Warm database caches for search and reverse queries.') + group.add_argument('--check-database', action='store_true', + help='Check that the database is complete and operational.') + group.add_argument('--analyse-indexing', action='store_true', + help='Print performance analysis of the indexing process.') + group = parser.add_argument_group('Arguments for cache warming') + group.add_argument('--search-only', action='store_const', dest='target', + const='search', + help="Only pre-warm tables for search queries") + group.add_argument('--reverse-only', action='store_const', dest='target', + const='reverse', + help="Only pre-warm tables for reverse queries") + group = parser.add_argument_group('Arguments for index anaysis') + mgroup = group.add_mutually_exclusive_group() + mgroup.add_argument('--osm-id', type=str, + help='Analyse indexing of the given OSM object') + mgroup.add_argument('--place-id', type=int, + help='Analyse indexing of the given Nominatim object') + + @staticmethod + def run(args): + from ..tools import admin + if args.warm: + AdminFuncs._warm(args) + + if args.check_database: + run_legacy_script('check_import_finished.php', nominatim_env=args) + + if args.analyse_indexing: + conn = connect(args.config.get_libpq_dsn()) + admin.analyse_indexing(conn, osm_id=args.osm_id, place_id=args.place_id) + conn.close() + + return 0 + + + @staticmethod + def _warm(args): + params = ['warm.php'] + if args.target == 'reverse': + params.append('--reverse-only') + if args.target == 'search': + params.append('--search-only') + return run_legacy_script(*params, nominatim_env=args) diff --git a/nominatim/clicmd/api.py b/nominatim/clicmd/api.py new file mode 100644 index 00000000..e50c00dc --- /dev/null +++ b/nominatim/clicmd/api.py @@ -0,0 +1,251 @@ +""" +Subcommand definitions for API calls from the command line. +""" +import logging + +from ..tools.exec_utils import run_api_script + +# Do not repeat documentation of subcommand classes. +# pylint: disable=C0111 + +LOG = logging.getLogger() + +STRUCTURED_QUERY = ( + ('street', 'housenumber and street'), + ('city', 'city, town or village'), + ('county', 'county'), + ('state', 'state'), + ('country', 'country'), + ('postalcode', 'postcode') +) + +EXTRADATA_PARAMS = ( + ('addressdetails', 'Include a breakdown of the address into elements.'), + ('extratags', """Include additional information if available + (e.g. wikipedia link, opening hours)."""), + ('namedetails', 'Include a list of alternative names.') +) + +DETAILS_SWITCHES = ( + ('addressdetails', 'Include a breakdown of the address into elements.'), + ('keywords', 'Include a list of name keywords and address keywords.'), + ('linkedplaces', 'Include a details of places that are linked with this one.'), + ('hierarchy', 'Include details of places lower in the address hierarchy.'), + ('group_hierarchy', 'Group the places by type.'), + ('polygon_geojson', 'Include geometry of result.') +) + +def _add_api_output_arguments(parser): + group = parser.add_argument_group('Output arguments') + group.add_argument('--format', default='jsonv2', + choices=['xml', 'json', 'jsonv2', 'geojson', 'geocodejson'], + help='Format of result') + for name, desc in EXTRADATA_PARAMS: + group.add_argument('--' + name, action='store_true', help=desc) + + group.add_argument('--lang', '--accept-language', metavar='LANGS', + help='Preferred language order for presenting search results') + group.add_argument('--polygon-output', + choices=['geojson', 'kml', 'svg', 'text'], + help='Output geometry of results as a GeoJSON, KML, SVG or WKT.') + group.add_argument('--polygon-threshold', type=float, metavar='TOLERANCE', + help="""Simplify output geometry. + Parameter is difference tolerance in degrees.""") + + +class APISearch: + """\ + Execute API search query. + """ + + @staticmethod + def add_args(parser): + group = parser.add_argument_group('Query arguments') + group.add_argument('--query', + help='Free-form query string') + for name, desc in STRUCTURED_QUERY: + group.add_argument('--' + name, help='Structured query: ' + desc) + + _add_api_output_arguments(parser) + + group = parser.add_argument_group('Result limitation') + group.add_argument('--countrycodes', metavar='CC,..', + help='Limit search results to one or more countries.') + group.add_argument('--exclude_place_ids', metavar='ID,..', + help='List of search object to be excluded') + group.add_argument('--limit', type=int, + help='Limit the number of returned results') + group.add_argument('--viewbox', metavar='X1,Y1,X2,Y2', + help='Preferred area to find search results') + group.add_argument('--bounded', action='store_true', + help='Strictly restrict results to viewbox area') + + group = parser.add_argument_group('Other arguments') + group.add_argument('--no-dedupe', action='store_false', dest='dedupe', + help='Do not remove duplicates from the result list') + + + @staticmethod + def run(args): + if args.query: + params = dict(q=args.query) + else: + params = {k : getattr(args, k) for k, _ in STRUCTURED_QUERY if getattr(args, k)} + + for param, _ in EXTRADATA_PARAMS: + if getattr(args, param): + params[param] = '1' + for param in ('format', 'countrycodes', 'exclude_place_ids', 'limit', 'viewbox'): + if getattr(args, param): + params[param] = getattr(args, param) + if args.lang: + params['accept-language'] = args.lang + if args.polygon_output: + params['polygon_' + args.polygon_output] = '1' + if args.polygon_threshold: + params['polygon_threshold'] = args.polygon_threshold + if args.bounded: + params['bounded'] = '1' + if not args.dedupe: + params['dedupe'] = '0' + + return run_api_script('search', args.project_dir, + phpcgi_bin=args.phpcgi_path, params=params) + +class APIReverse: + """\ + Execute API reverse query. + """ + + @staticmethod + def add_args(parser): + group = parser.add_argument_group('Query arguments') + group.add_argument('--lat', type=float, required=True, + help='Latitude of coordinate to look up (in WGS84)') + group.add_argument('--lon', type=float, required=True, + help='Longitude of coordinate to look up (in WGS84)') + group.add_argument('--zoom', type=int, + help='Level of detail required for the address') + + _add_api_output_arguments(parser) + + + @staticmethod + def run(args): + params = dict(lat=args.lat, lon=args.lon) + if args.zoom is not None: + params['zoom'] = args.zoom + + for param, _ in EXTRADATA_PARAMS: + if getattr(args, param): + params[param] = '1' + if args.format: + params['format'] = args.format + if args.lang: + params['accept-language'] = args.lang + if args.polygon_output: + params['polygon_' + args.polygon_output] = '1' + if args.polygon_threshold: + params['polygon_threshold'] = args.polygon_threshold + + return run_api_script('reverse', args.project_dir, + phpcgi_bin=args.phpcgi_path, params=params) + + +class APILookup: + """\ + Execute API reverse query. + """ + + @staticmethod + def add_args(parser): + group = parser.add_argument_group('Query arguments') + group.add_argument('--id', metavar='OSMID', + action='append', required=True, dest='ids', + help='OSM id to lookup in format (may be repeated)') + + _add_api_output_arguments(parser) + + + @staticmethod + def run(args): + params = dict(osm_ids=','.join(args.ids)) + + for param, _ in EXTRADATA_PARAMS: + if getattr(args, param): + params[param] = '1' + if args.format: + params['format'] = args.format + if args.lang: + params['accept-language'] = args.lang + if args.polygon_output: + params['polygon_' + args.polygon_output] = '1' + if args.polygon_threshold: + params['polygon_threshold'] = args.polygon_threshold + + return run_api_script('lookup', args.project_dir, + phpcgi_bin=args.phpcgi_path, params=params) + + +class APIDetails: + """\ + Execute API lookup query. + """ + + @staticmethod + def add_args(parser): + group = parser.add_argument_group('Query arguments') + objs = group.add_mutually_exclusive_group(required=True) + objs.add_argument('--node', '-n', type=int, + help="Look up the OSM node with the given ID.") + objs.add_argument('--way', '-w', type=int, + help="Look up the OSM way with the given ID.") + objs.add_argument('--relation', '-r', type=int, + help="Look up the OSM relation with the given ID.") + objs.add_argument('--place_id', '-p', type=int, + help='Database internal identifier of the OSM object to look up.') + group.add_argument('--class', dest='object_class', + help="""Class type to disambiguated multiple entries + of the same object.""") + + group = parser.add_argument_group('Output arguments') + for name, desc in DETAILS_SWITCHES: + group.add_argument('--' + name, action='store_true', help=desc) + group.add_argument('--lang', '--accept-language', metavar='LANGS', + help='Preferred language order for presenting search results') + + @staticmethod + def run(args): + if args.node: + params = dict(osmtype='N', osmid=args.node) + elif args.way: + params = dict(osmtype='W', osmid=args.node) + elif args.relation: + params = dict(osmtype='R', osmid=args.node) + else: + params = dict(place_id=args.place_id) + if args.object_class: + params['class'] = args.object_class + for name, _ in DETAILS_SWITCHES: + params[name] = '1' if getattr(args, name) else '0' + + return run_api_script('details', args.project_dir, + phpcgi_bin=args.phpcgi_path, params=params) + + +class APIStatus: + """\ + Execute API status query. + """ + + @staticmethod + def add_args(parser): + group = parser.add_argument_group('API parameters') + group.add_argument('--format', default='text', choices=['text', 'json'], + help='Format of result') + + @staticmethod + def run(args): + return run_api_script('status', args.project_dir, + phpcgi_bin=args.phpcgi_path, + params=dict(format=args.format)) diff --git a/nominatim/clicmd/index.py b/nominatim/clicmd/index.py new file mode 100644 index 00000000..ca3f9dee --- /dev/null +++ b/nominatim/clicmd/index.py @@ -0,0 +1,58 @@ +""" +Implementation of the 'index' subcommand. +""" +import os + +from ..db import status +from ..db.connection import connect + +# Do not repeat documentation of subcommand classes. +# pylint: disable=C0111 +# Using non-top-level imports to avoid eventually unused imports. +# pylint: disable=E0012,C0415 + +def _num_system_cpus(): + try: + cpus = len(os.sched_getaffinity(0)) + except NotImplementedError: + cpus = None + + return cpus or os.cpu_count() + + +class UpdateIndex: + """\ + Reindex all new and modified data. + """ + + @staticmethod + def add_args(parser): + group = parser.add_argument_group('Filter arguments') + group.add_argument('--boundaries-only', action='store_true', + help="""Index only administrative boundaries.""") + group.add_argument('--no-boundaries', action='store_true', + help="""Index everything except administrative boundaries.""") + group.add_argument('--minrank', '-r', type=int, metavar='RANK', default=0, + help='Minimum/starting rank') + group.add_argument('--maxrank', '-R', type=int, metavar='RANK', default=30, + help='Maximum/finishing rank') + + @staticmethod + def run(args): + from ..indexer.indexer import Indexer + + indexer = Indexer(args.config.get_libpq_dsn(), + args.threads or _num_system_cpus() or 1) + + if not args.no_boundaries: + indexer.index_boundaries(args.minrank, args.maxrank) + if not args.boundaries_only: + indexer.index_by_rank(args.minrank, args.maxrank) + + if not args.no_boundaries and not args.boundaries_only \ + and args.minrank == 0 and args.maxrank == 30: + conn = connect(args.config.get_libpq_dsn()) + status.set_indexed(conn, True) + conn.close() + + return 0 diff --git a/nominatim/clicmd/refresh.py b/nominatim/clicmd/refresh.py new file mode 100644 index 00000000..8e69caca --- /dev/null +++ b/nominatim/clicmd/refresh.py @@ -0,0 +1,88 @@ +""" +Implementation of 'refresh' subcommand. +""" +import logging +from pathlib import Path + +from ..db.connection import connect +from ..tools.exec_utils import run_legacy_script + +# Do not repeat documentation of subcommand classes. +# pylint: disable=C0111 +# Using non-top-level imports to avoid eventually unused imports. +# pylint: disable=E0012,C0415 + +LOG = logging.getLogger() + +class UpdateRefresh: + """\ + Recompute auxiliary data used by the indexing process. + + These functions must not be run in parallel with other update commands. + """ + + @staticmethod + def add_args(parser): + group = parser.add_argument_group('Data arguments') + group.add_argument('--postcodes', action='store_true', + help='Update postcode centroid table') + group.add_argument('--word-counts', action='store_true', + help='Compute frequency of full-word search terms') + group.add_argument('--address-levels', action='store_true', + help='Reimport address level configuration') + group.add_argument('--functions', action='store_true', + help='Update the PL/pgSQL functions in the database') + group.add_argument('--wiki-data', action='store_true', + help='Update Wikipedia/data importance numbers.') + group.add_argument('--importance', action='store_true', + help='Recompute place importances (expensive!)') + group.add_argument('--website', action='store_true', + help='Refresh the directory that serves the scripts for the web API') + group = parser.add_argument_group('Arguments for function refresh') + group.add_argument('--no-diff-updates', action='store_false', dest='diffs', + help='Do not enable code for propagating updates') + group.add_argument('--enable-debug-statements', action='store_true', + help='Enable debug warning statements in functions') + + @staticmethod + def run(args): + from ..tools import refresh + + if args.postcodes: + LOG.warning("Update postcodes centroid") + conn = connect(args.config.get_libpq_dsn()) + refresh.update_postcodes(conn, args.sqllib_dir) + conn.close() + + if args.word_counts: + LOG.warning('Recompute frequency of full-word search terms') + conn = connect(args.config.get_libpq_dsn()) + refresh.recompute_word_counts(conn, args.sqllib_dir) + conn.close() + + if args.address_levels: + cfg = Path(args.config.ADDRESS_LEVEL_CONFIG) + LOG.warning('Updating address levels from %s', cfg) + conn = connect(args.config.get_libpq_dsn()) + refresh.load_address_levels_from_file(conn, cfg) + conn.close() + + if args.functions: + LOG.warning('Create functions') + conn = connect(args.config.get_libpq_dsn()) + refresh.create_functions(conn, args.config, args.sqllib_dir, + args.diffs, args.enable_debug_statements) + conn.close() + + if args.wiki_data: + run_legacy_script('setup.php', '--import-wikipedia-articles', + nominatim_env=args, throw_on_fail=True) + # Attention: importance MUST come after wiki data import. + if args.importance: + run_legacy_script('update.php', '--recompute-importance', + nominatim_env=args, throw_on_fail=True) + if args.website: + run_legacy_script('setup.php', '--setup-website', + nominatim_env=args, throw_on_fail=True) + + return 0 diff --git a/nominatim/clicmd/replication.py b/nominatim/clicmd/replication.py new file mode 100644 index 00000000..2a19e6cd --- /dev/null +++ b/nominatim/clicmd/replication.py @@ -0,0 +1,169 @@ +""" +Implementation of the 'replication' sub-command. +""" +import datetime as dt +import logging +import socket +import time + +from ..db import status +from ..db.connection import connect +from ..errors import UsageError + +LOG = logging.getLogger() + +# Do not repeat documentation of subcommand classes. +# pylint: disable=C0111 +# Using non-top-level imports to make pyosmium optional for replication only. +# pylint: disable=E0012,C0415 + +def _osm2pgsql_options_from_args(args, default_cache, default_threads): + """ Set up the standard osm2pgsql from the command line arguments. + """ + return dict(osm2pgsql=args.osm2pgsql_path, + osm2pgsql_cache=args.osm2pgsql_cache or default_cache, + osm2pgsql_style=args.config.get_import_style_file(), + threads=args.threads or default_threads, + dsn=args.config.get_libpq_dsn(), + flatnode_file=args.config.FLATNODE_FILE) + + +class UpdateReplication: + """\ + Update the database using an online replication service. + """ + + @staticmethod + def add_args(parser): + group = parser.add_argument_group('Arguments for initialisation') + group.add_argument('--init', action='store_true', + help='Initialise the update process') + group.add_argument('--no-update-functions', dest='update_functions', + action='store_false', + help="""Do not update the trigger function to + support differential updates.""") + group = parser.add_argument_group('Arguments for updates') + group.add_argument('--check-for-updates', action='store_true', + help='Check if new updates are available and exit') + group.add_argument('--once', action='store_true', + help="""Download and apply updates only once. When + not set, updates are continuously applied""") + group.add_argument('--no-index', action='store_false', dest='do_index', + help="""Do not index the new data. Only applicable + together with --once""") + group.add_argument('--osm2pgsql-cache', metavar='SIZE', type=int, + help='Size of cache to be used by osm2pgsql (in MB)') + group = parser.add_argument_group('Download parameters') + group.add_argument('--socket-timeout', dest='socket_timeout', type=int, default=60, + help='Set timeout for file downloads.') + + @staticmethod + def _init_replication(args): + from ..tools import replication, refresh + + LOG.warning("Initialising replication updates") + conn = connect(args.config.get_libpq_dsn()) + replication.init_replication(conn, base_url=args.config.REPLICATION_URL) + if args.update_functions: + LOG.warning("Create functions") + refresh.create_functions(conn, args.config, args.sqllib_dir, + True, False) + conn.close() + return 0 + + + @staticmethod + def _check_for_updates(args): + from ..tools import replication + + conn = connect(args.config.get_libpq_dsn()) + ret = replication.check_for_updates(conn, base_url=args.config.REPLICATION_URL) + conn.close() + return ret + + @staticmethod + def _report_update(batchdate, start_import, start_index): + def round_time(delta): + return dt.timedelta(seconds=int(delta.total_seconds())) + + end = dt.datetime.now(dt.timezone.utc) + LOG.warning("Update completed. Import: %s. %sTotal: %s. Remaining backlog: %s.", + round_time((start_index or end) - start_import), + "Indexing: {} ".format(round_time(end - start_index)) + if start_index else '', + round_time(end - start_import), + round_time(end - batchdate)) + + @staticmethod + def _update(args): + from ..tools import replication + from ..indexer.indexer import Indexer + + params = _osm2pgsql_options_from_args(args, 2000, 1) + params.update(base_url=args.config.REPLICATION_URL, + update_interval=args.config.get_int('REPLICATION_UPDATE_INTERVAL'), + import_file=args.project_dir / 'osmosischange.osc', + max_diff_size=args.config.get_int('REPLICATION_MAX_DIFF'), + indexed_only=not args.once) + + # Sanity check to not overwhelm the Geofabrik servers. + if 'download.geofabrik.de'in params['base_url']\ + and params['update_interval'] < 86400: + LOG.fatal("Update interval too low for download.geofabrik.de.\n" + "Please check install documentation " + "(https://nominatim.org/release-docs/latest/admin/Import-and-Update#" + "setting-up-the-update-process).") + raise UsageError("Invalid replication update interval setting.") + + if not args.once: + if not args.do_index: + LOG.fatal("Indexing cannot be disabled when running updates continuously.") + raise UsageError("Bad argument '--no-index'.") + recheck_interval = args.config.get_int('REPLICATION_RECHECK_INTERVAL') + + while True: + conn = connect(args.config.get_libpq_dsn()) + start = dt.datetime.now(dt.timezone.utc) + state = replication.update(conn, params) + if state is not replication.UpdateState.NO_CHANGES: + status.log_status(conn, start, 'import') + batchdate, _, _ = status.get_status(conn) + conn.close() + + if state is not replication.UpdateState.NO_CHANGES and args.do_index: + index_start = dt.datetime.now(dt.timezone.utc) + indexer = Indexer(args.config.get_libpq_dsn(), + args.threads or 1) + indexer.index_boundaries(0, 30) + indexer.index_by_rank(0, 30) + + conn = connect(args.config.get_libpq_dsn()) + status.set_indexed(conn, True) + status.log_status(conn, index_start, 'index') + conn.close() + else: + index_start = None + + if LOG.isEnabledFor(logging.WARNING): + UpdateReplication._report_update(batchdate, start, index_start) + + if args.once: + break + + if state is replication.UpdateState.NO_CHANGES: + LOG.warning("No new changes. Sleeping for %d sec.", recheck_interval) + time.sleep(recheck_interval) + + + @staticmethod + def run(args): + socket.setdefaulttimeout(args.socket_timeout) + + if args.init: + return UpdateReplication._init_replication(args) + + if args.check_for_updates: + return UpdateReplication._check_for_updates(args) + + UpdateReplication._update(args) + return 0 diff --git a/nominatim/tools/admin.py b/nominatim/tools/admin.py new file mode 100644 index 00000000..119adf37 --- /dev/null +++ b/nominatim/tools/admin.py @@ -0,0 +1,49 @@ +""" +Functions for database analysis and maintenance. +""" +import logging + +from ..errors import UsageError + +LOG = logging.getLogger() + +def analyse_indexing(conn, osm_id=None, place_id=None): + """ Analyse indexing of a single Nominatim object. + """ + with conn.cursor() as cur: + if osm_id: + osm_type = osm_id[0].upper() + if osm_type not in 'NWR' or not osm_id[1:].isdigit(): + LOG.fatal('OSM ID must be of form . Got: %s', osm_id) + raise UsageError("OSM ID parameter badly formatted") + cur.execute('SELECT place_id FROM placex WHERE osm_type = %s AND osm_id = %s', + (osm_type, osm_id[1:])) + + if cur.rowcount < 1: + LOG.fatal("OSM object %s not found in database.", osm_id) + raise UsageError("OSM object not found") + + place_id = cur.fetchone()[0] + + if place_id is None: + LOG.fatal("No OSM object given to index.") + raise UsageError("OSM object not found") + + cur.execute("update placex set indexed_status = 2 where place_id = %s", + (place_id, )) + + cur.execute("""SET auto_explain.log_min_duration = '0'; + SET auto_explain.log_analyze = 'true'; + SET auto_explain.log_nested_statements = 'true'; + LOAD 'auto_explain'; + SET client_min_messages = LOG; + SET log_min_messages = FATAL""") + + cur.execute("update placex set indexed_status = 0 where place_id = %s", + (place_id, )) + + # we do not want to keep the results + conn.rollback() + + for msg in conn.notices: + print(msg) diff --git a/nominatim/tools/exec_utils.py b/nominatim/tools/exec_utils.py index 45853163..541a2b08 100644 --- a/nominatim/tools/exec_utils.py +++ b/nominatim/tools/exec_utils.py @@ -25,7 +25,8 @@ def run_legacy_script(script, *args, nominatim_env=None, throw_on_fail=False): env = nominatim_env.config.get_os_env() env['NOMINATIM_DATADIR'] = str(nominatim_env.data_dir) - env['NOMINATIM_BINDIR'] = str(nominatim_env.data_dir / 'utils') + env['NOMINATIM_SQLDIR'] = str(nominatim_env.sqllib_dir) + env['NOMINATIM_CONFIGDIR'] = str(nominatim_env.config_dir) env['NOMINATIM_DATABASE_MODULE_SRC_PATH'] = nominatim_env.module_dir if not env['NOMINATIM_OSM2PGSQL_BINARY']: env['NOMINATIM_OSM2PGSQL_BINARY'] = nominatim_env.osm2pgsql_path diff --git a/nominatim/tools/refresh.py b/nominatim/tools/refresh.py index 5fbb07f8..1fcb1577 100644 --- a/nominatim/tools/refresh.py +++ b/nominatim/tools/refresh.py @@ -8,17 +8,17 @@ from psycopg2.extras import execute_values from ..db.utils import execute_file -def update_postcodes(conn, datadir): +def update_postcodes(conn, sql_dir): """ Recalculate postcode centroids and add, remove and update entries in the location_postcode table. `conn` is an opne connection to the database. """ - execute_file(conn, datadir / 'sql' / 'update-postcodes.sql') + execute_file(conn, sql_dir / 'update-postcodes.sql') -def recompute_word_counts(conn, datadir): +def recompute_word_counts(conn, sql_dir): """ Compute the frequency of full-word search terms. """ - execute_file(conn, datadir / 'sql' / 'words_from_search_name.sql') + execute_file(conn, sql_dir / 'words_from_search_name.sql') def _add_address_level_rows_from_entry(rows, entry): @@ -153,12 +153,10 @@ def _get_partition_function_sql(conn, sql_dir): return replace_partition_string(sql, sorted(partitions)) -def create_functions(conn, config, data_dir, +def create_functions(conn, config, sql_dir, enable_diff_updates=True, enable_debug=False): """ (Re)create the PL/pgSQL functions. """ - sql_dir = data_dir / 'sql' - sql = _get_standard_function_sql(conn, config, sql_dir, enable_diff_updates, enable_debug) sql += _get_partition_function_sql(conn, sql_dir) diff --git a/nominatim/tools/replication.py b/nominatim/tools/replication.py index afc1af47..cb201b1e 100644 --- a/nominatim/tools/replication.py +++ b/nominatim/tools/replication.py @@ -6,13 +6,18 @@ from enum import Enum import logging import time -from osmium.replication.server import ReplicationServer -from osmium import WriteHandler - from ..db import status from .exec_utils import run_osm2pgsql from ..errors import UsageError +try: + from osmium.replication.server import ReplicationServer + from osmium import WriteHandler +except ModuleNotFoundError as exc: + logging.getLogger().fatal("pyosmium not installed. Replication functions not available.\n" + "To install pyosmium via pip: pip3 install osmium") + raise UsageError("replication tools not available") from exc + LOG = logging.getLogger() def init_replication(conn, base_url): diff --git a/osm2pgsql b/osm2pgsql index a65ab49f..497476d5 160000 --- a/osm2pgsql +++ b/osm2pgsql @@ -1 +1 @@ -Subproject commit a65ab49f777b9785726117971d3a4140436d70aa +Subproject commit 497476d56f7c1fcbbdb95b363293de6ce0feac00 diff --git a/sql/hstore_compatability_9_0.sql b/sql/hstore_compatability_9_0.sql deleted file mode 100644 index 088dd792..00000000 --- a/sql/hstore_compatability_9_0.sql +++ /dev/null @@ -1,8 +0,0 @@ -CREATE OR REPLACE FUNCTION hstore(k text, v text) RETURNS HSTORE - AS $$ -DECLARE -BEGIN - RETURN k => v; -END; -$$ -LANGUAGE plpgsql IMMUTABLE; diff --git a/test/bdd/steps/cgi-with-coverage.php b/test/bdd/steps/cgi-with-coverage.php index 80f898a3..6f0d79bb 100644 --- a/test/bdd/steps/cgi-with-coverage.php +++ b/test/bdd/steps/cgi-with-coverage.php @@ -10,7 +10,7 @@ function coverage_shutdown($oCoverage) } $covfilter = new SebastianBergmann\CodeCoverage\Filter(); -$covfilter->addDirectoryToWhitelist($_SERVER['COV_PHP_DIR'].'/lib'); +$covfilter->addDirectoryToWhitelist($_SERVER['COV_PHP_DIR'].'/lib-php'); $covfilter->addDirectoryToWhitelist($_SERVER['COV_PHP_DIR'].'/website'); $coverage = new SebastianBergmann\CodeCoverage\CodeCoverage(null, $covfilter); $coverage->start($_SERVER['COV_TEST_NAME']); diff --git a/test/bdd/steps/nominatim_environment.py b/test/bdd/steps/nominatim_environment.py index 0ee92137..dd76dee3 100644 --- a/test/bdd/steps/nominatim_environment.py +++ b/test/bdd/steps/nominatim_environment.py @@ -87,14 +87,18 @@ class NominatimEnvironment: self.test_env['NOMINATIM_FLATNODE_FILE'] = '' self.test_env['NOMINATIM_IMPORT_STYLE'] = 'full' self.test_env['NOMINATIM_USE_US_TIGER_DATA'] = 'yes' - self.test_env['NOMINATIM_DATADIR'] = self.src_dir - self.test_env['NOMINATIM_BINDIR'] = self.src_dir / 'utils' - self.test_env['NOMINATIM_DATABASE_MODULE_PATH'] = self.build_dir / 'module' + self.test_env['NOMINATIM_DATADIR'] = self.src_dir / 'data' + self.test_env['NOMINATIM_SQLDIR'] = self.src_dir / 'lib-sql' + self.test_env['NOMINATIM_CONFIGDIR'] = self.src_dir / 'settings' + self.test_env['NOMINATIM_DATABASE_MODULE_SRC_PATH'] = self.build_dir / 'module' self.test_env['NOMINATIM_OSM2PGSQL_BINARY'] = self.build_dir / 'osm2pgsql' / 'osm2pgsql' self.test_env['NOMINATIM_NOMINATIM_TOOL'] = self.build_dir / 'nominatim' if self.server_module_path: self.test_env['NOMINATIM_DATABASE_MODULE_PATH'] = self.server_module_path + else: + # avoid module being copied into the temporary environment + self.test_env['NOMINATIM_DATABASE_MODULE_PATH'] = self.build_dir / 'module' if self.website_dir is not None: self.website_dir.cleanup() @@ -262,7 +266,7 @@ class NominatimEnvironment: """ Run one of the Nominatim utility scripts with the given arguments. """ cmd = ['/usr/bin/env', 'php', '-Cq'] - cmd.append((Path(self.src_dir) / 'lib' / 'admin' / '{}.php'.format(script)).resolve()) + cmd.append((Path(self.src_dir) / 'lib-php' / 'admin' / '{}.php'.format(script)).resolve()) cmd.extend(['--' + x for x in args]) for k, v in kwargs.items(): cmd.extend(('--' + k.replace('_', '-'), str(v))) diff --git a/test/bdd/steps/steps_api_queries.py b/test/bdd/steps/steps_api_queries.py index ad4a8515..1b3da08b 100644 --- a/test/bdd/steps/steps_api_queries.py +++ b/test/bdd/steps/steps_api_queries.py @@ -60,7 +60,7 @@ def query_cmd(context, query, dups): """ Query directly via PHP script. """ cmd = ['/usr/bin/env', 'php'] - cmd.append(context.nominatim.src_dir / 'lib' / 'admin' / 'query.php') + cmd.append(context.nominatim.src_dir / 'lib-php' / 'admin' / 'query.php') if query: cmd.extend(['--search', query]) # add more parameters in table form diff --git a/test/php/bootstrap.php b/test/php/bootstrap.php index 3a36db38..bfdbbf05 100644 --- a/test/php/bootstrap.php +++ b/test/php/bootstrap.php @@ -1,5 +1,5 @@ - ../../lib/ + ../../lib-php/ diff --git a/test/python/conftest.py b/test/python/conftest.py index 8b0ba145..ecd40d7c 100644 --- a/test/python/conftest.py +++ b/test/python/conftest.py @@ -153,3 +153,39 @@ def place_row(place_table, temp_db_cursor): geom or 'SRID=4326;POINT(0 0 )')) return _insert + +@pytest.fixture +def placex_table(temp_db_with_extensions, temp_db_conn): + """ Create an empty version of the place table. + """ + with temp_db_conn.cursor() as cur: + cur.execute("""CREATE TABLE placex ( + place_id BIGINT NOT NULL, + parent_place_id BIGINT, + linked_place_id BIGINT, + importance FLOAT, + indexed_date TIMESTAMP, + geometry_sector INTEGER, + rank_address SMALLINT, + rank_search SMALLINT, + partition SMALLINT, + indexed_status SMALLINT, + osm_id int8, + osm_type char(1), + class text, + type text, + name hstore, + admin_level smallint, + address hstore, + extratags hstore, + geometry Geometry(Geometry,4326), + wikipedia TEXT, + country_code varchar(2), + housenumber TEXT, + postcode TEXT, + centroid GEOMETRY(Geometry, 4326)) + """) + temp_db_conn.commit() + + + diff --git a/test/python/test_cli.py b/test/python/test_cli.py index 702a4b74..0c0a689e 100644 --- a/test/python/test_cli.py +++ b/test/python/test_cli.py @@ -11,6 +11,9 @@ import pytest import time import nominatim.cli +import nominatim.clicmd.api +import nominatim.clicmd.refresh +import nominatim.clicmd.admin import nominatim.indexer.indexer import nominatim.tools.refresh import nominatim.tools.replication @@ -20,9 +23,11 @@ from nominatim.db import status def call_nominatim(*args): return nominatim.cli.nominatim(module_dir='build/module', osm2pgsql_path='build/osm2pgsql/osm2pgsql', - phplib_dir='lib', + phplib_dir='lib-php', data_dir='.', phpcgi_path='/usr/bin/php-cgi', + sqllib_dir='lib-sql', + config_dir='settings', cli_args=args) class MockParamCapture: @@ -45,12 +50,6 @@ def mock_run_legacy(monkeypatch): monkeypatch.setattr(nominatim.cli, 'run_legacy_script', mock) return mock -@pytest.fixture -def mock_run_api(monkeypatch): - mock = MockParamCapture() - monkeypatch.setattr(nominatim.cli, 'run_api_script', mock) - return mock - def test_cli_help(capsys): """ Running nominatim tool without arguments prints help. @@ -67,8 +66,6 @@ def test_cli_help(capsys): (('special-phrases',), 'specialphrases'), (('add-data', '--tiger-data', 'tiger'), 'setup'), (('add-data', '--file', 'foo.osm'), 'update'), - (('check-database',), 'check_import_finished'), - (('warm',), 'warm'), (('export',), 'export') ]) def test_legacy_commands_simple(mock_run_legacy, command, script): @@ -78,6 +75,26 @@ def test_legacy_commands_simple(mock_run_legacy, command, script): assert mock_run_legacy.last_args[0] == script + '.php' +@pytest.mark.parametrize("params", [('--warm', ), + ('--warm', '--reverse-only'), + ('--warm', '--search-only'), + ('--check-database', )]) +def test_admin_command_legacy(monkeypatch, params): + mock_run_legacy = MockParamCapture() + monkeypatch.setattr(nominatim.clicmd.admin, 'run_legacy_script', mock_run_legacy) + + assert 0 == call_nominatim('admin', *params) + + assert mock_run_legacy.called == 1 + +@pytest.mark.parametrize("func, params", [('analyse_indexing', ('--analyse-indexing', ))]) +def test_admin_command_tool(temp_db, monkeypatch, func, params): + mock = MockParamCapture() + monkeypatch.setattr(nominatim.tools.admin, func, mock) + + assert 0 == call_nominatim('admin', *params) + assert mock.called == 1 + @pytest.mark.parametrize("name,oid", [('file', 'foo.osm'), ('diff', 'foo.osc'), ('node', 12), ('way', 8), ('relation', 32)]) def test_add_data_command(mock_run_legacy, name, oid): @@ -110,7 +127,10 @@ def test_index_command(monkeypatch, temp_db_cursor, params, do_bnds, do_ranks): ('importance', ('update.php', '--recompute-importance')), ('website', ('setup.php', '--setup-website')), ]) -def test_refresh_legacy_command(mock_run_legacy, temp_db, command, params): +def test_refresh_legacy_command(monkeypatch, temp_db, command, params): + mock_run_legacy = MockParamCapture() + monkeypatch.setattr(nominatim.clicmd.refresh, 'run_legacy_script', mock_run_legacy) + assert 0 == call_nominatim('refresh', '--' + command) assert mock_run_legacy.called == 1 @@ -131,7 +151,10 @@ def test_refresh_command(monkeypatch, temp_db, command, func): assert func_mock.called == 1 -def test_refresh_importance_computed_after_wiki_import(mock_run_legacy, temp_db): +def test_refresh_importance_computed_after_wiki_import(monkeypatch, temp_db): + mock_run_legacy = MockParamCapture() + monkeypatch.setattr(nominatim.clicmd.refresh, 'run_legacy_script', mock_run_legacy) + assert 0 == call_nominatim('refresh', '--importance', '--wiki-data') assert mock_run_legacy.called == 2 @@ -163,17 +186,15 @@ def test_replication_update_bad_interval_for_geofabrik(monkeypatch, temp_db): assert call_nominatim('replication') == 1 -@pytest.mark.parametrize("state, retval", [ - (nominatim.tools.replication.UpdateState.UP_TO_DATE, 0), - (nominatim.tools.replication.UpdateState.NO_CHANGES, 3) - ]) +@pytest.mark.parametrize("state", [nominatim.tools.replication.UpdateState.UP_TO_DATE, + nominatim.tools.replication.UpdateState.NO_CHANGES]) def test_replication_update_once_no_index(monkeypatch, temp_db, temp_db_conn, - status_table, state, retval): + status_table, state): status.set_status(temp_db_conn, date=dt.datetime.now(dt.timezone.utc), seq=1) func_mock = MockParamCapture(retval=state) monkeypatch.setattr(nominatim.tools.replication, 'update', func_mock) - assert retval == call_nominatim('replication', '--once', '--no-index') + assert 0 == call_nominatim('replication', '--once', '--no-index') def test_replication_update_continuous(monkeypatch, temp_db_conn, status_table): @@ -233,7 +254,10 @@ def test_serve_command(monkeypatch): ('details', '--place_id', '10001'), ('status',) ]) -def test_api_commands_simple(mock_run_api, params): +def test_api_commands_simple(monkeypatch, params): + mock_run_api = MockParamCapture() + monkeypatch.setattr(nominatim.clicmd.api, 'run_api_script', mock_run_api) + assert 0 == call_nominatim(*params) assert mock_run_api.called == 1 diff --git a/test/python/test_tools_admin.py b/test/python/test_tools_admin.py new file mode 100644 index 00000000..a40a17db --- /dev/null +++ b/test/python/test_tools_admin.py @@ -0,0 +1,42 @@ +""" +Tests for maintenance and analysis functions. +""" +import pytest + +from nominatim.db.connection import connect +from nominatim.errors import UsageError +from nominatim.tools import admin + +@pytest.fixture +def db(temp_db, placex_table): + conn = connect('dbname=' + temp_db) + yield conn + conn.close() + +def test_analyse_indexing_no_objects(db): + with pytest.raises(UsageError): + admin.analyse_indexing(db) + + +@pytest.mark.parametrize("oid", ['1234', 'N123a', 'X123']) +def test_analyse_indexing_bad_osmid(db, oid): + with pytest.raises(UsageError): + admin.analyse_indexing(db, osm_id=oid) + + +def test_analyse_indexing_unknown_osmid(db): + with pytest.raises(UsageError): + admin.analyse_indexing(db, osm_id='W12345674') + + +def test_analyse_indexing_with_place_id(db, temp_db_cursor): + temp_db_cursor.execute("INSERT INTO placex (place_id) VALUES(12345)") + + admin.analyse_indexing(db, place_id=12345) + + +def test_analyse_indexing_with_osm_id(db, temp_db_cursor): + temp_db_cursor.execute("""INSERT INTO placex (place_id, osm_type, osm_id) + VALUES(9988, 'N', 10000)""") + + admin.analyse_indexing(db, osm_id='N10000') diff --git a/test/python/test_tools_exec_utils.py b/test/python/test_tools_exec_utils.py index ef1b46e2..283f486a 100644 --- a/test/python/test_tools_exec_utils.py +++ b/test/python/test_tools_exec_utils.py @@ -23,6 +23,8 @@ def nominatim_env(tmp_phplib_dir, def_config): phplib_dir = tmp_phplib_dir data_dir = Path('data') project_dir = Path('.') + sqllib_dir = Path('lib-sql') + config_dir = Path('settings') module_dir = 'module' osm2pgsql_path = 'osm2pgsql' diff --git a/test/python/test_tools_refresh_create_functions.py b/test/python/test_tools_refresh_create_functions.py index 4807e64f..d219d748 100644 --- a/test/python/test_tools_refresh_create_functions.py +++ b/test/python/test_tools_refresh_create_functions.py @@ -7,7 +7,7 @@ import pytest from nominatim.db.connection import connect from nominatim.tools.refresh import _get_standard_function_sql, _get_partition_function_sql -SQL_DIR = (Path(__file__) / '..' / '..' / '..' / 'sql').resolve() +SQL_DIR = (Path(__file__) / '..' / '..' / '..' / 'lib-sql').resolve() @pytest.fixture def db(temp_db): diff --git a/utils/analyse_indexing.py b/utils/analyse_indexing.py deleted file mode 100755 index 97cb6843..00000000 --- a/utils/analyse_indexing.py +++ /dev/null @@ -1,119 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: GPL-2.0-only -# -# This file is part of Nominatim. -# Copyright (C) 2020 Sarah Hoffmann - -""" -Script for analysing the indexing process. - -The script enables detailed logging for nested statements and then -runs the indexing process for teh given object. Detailed 'EXPLAIN ANALYSE' -information is printed for each executed query in the trigger. The -transaction is then rolled back, so that no actual changes to the database -happen. It also disables logging into the system log, so that the -log files are not cluttered. -""" - -from argparse import ArgumentParser, RawDescriptionHelpFormatter, ArgumentTypeError -import psycopg2 -import getpass -import re - -class Analyser(object): - - def __init__(self, options): - password = None - if options.password_prompt: - password = getpass.getpass("Database password: ") - - self.options = options - self.conn = psycopg2.connect(dbname=options.dbname, - user=options.user, - password=password, - host=options.host, - port=options.port) - - - - def run(self): - c = self.conn.cursor() - - if self.options.placeid: - place_id = self.options.placeid - else: - if self.options.rank: - c.execute(f"""select place_id from placex - where rank_address = {self.options.rank} - and linked_place_id is null - limit 1""") - objinfo = f"rank {self.options.rank}" - - if self.options.osmid: - osm_type = self.options.osmid[0].upper() - if osm_type not in ('N', 'W', 'R'): - raise RuntimeError("OSM ID must be of form ") - try: - osm_id = int(self.options.osmid[1:]) - except ValueError: - raise RuntimeError("OSM ID must be of form ") - - c.execute(f"""SELECT place_id FROM placex - WHERE osm_type = '{osm_type}' AND osm_id = {osm_id}""") - objinfo = f"OSM object {self.options.osmid}" - - - if c.rowcount < 1: - raise RuntimeError(f"Cannot find a place for {objinfo}.") - place_id = c.fetchone()[0] - - c.execute(f"""update placex set indexed_status = 2 where - place_id = {place_id}""") - - c.execute("""SET auto_explain.log_min_duration = '0'; - SET auto_explain.log_analyze = 'true'; - SET auto_explain.log_nested_statements = 'true'; - LOAD 'auto_explain'; - SET client_min_messages = LOG; - SET log_min_messages = FATAL"""); - - c.execute(f"""update placex set indexed_status = 0 where - place_id = {place_id}""") - - c.close() # automatic rollback - - for l in self.conn.notices: - print(l) - - -if __name__ == '__main__': - def h(s): - return re.sub("\s\s+" , " ", s) - - p = ArgumentParser(description=__doc__, - formatter_class=RawDescriptionHelpFormatter) - - group = p.add_mutually_exclusive_group(required=True) - group.add_argument('--rank', dest='rank', type=int, - help='Analyse indexing of the given address rank') - group.add_argument('--osm-id', dest='osmid', type=str, - help='Analyse indexing of the given OSM object') - group.add_argument('--place-id', dest='placeid', type=int, - help='Analyse indexing of the given Nominatim object') - p.add_argument('-d', '--database', - dest='dbname', action='store', default='nominatim', - help='Name of the PostgreSQL database to connect to.') - p.add_argument('-U', '--username', - dest='user', action='store', - help='PostgreSQL user name.') - p.add_argument('-W', '--password', - dest='password_prompt', action='store_true', - help='Force password prompt.') - p.add_argument('-H', '--host', - dest='host', action='store', - help='PostgreSQL server hostname or socket location.') - p.add_argument('-P', '--port', - dest='port', action='store', - help='PostgreSQL server port') - - Analyser(p.parse_args()).run()