shell: bash
- name: Import
- run: ./nominatim import --osm-file ../monaco-latest.osm.pbf
- working-directory: build
+ run: |
+ mkdir data-env
+ cd data-env
+ ../build/nominatim import --osm-file ../monaco-latest.osm.pbf
+ shell: bash
- name: Import special phrases
- run: ./nominatim special-phrases --from-wiki | psql -d nominatim
- working-directory: build
+ run: ../build/nominatim special-phrases --from-wiki | psql -d nominatim
+ working-directory: data-env
- name: Check import
- run: ./nominatim check-database
- working-directory: build
+ run: ../build/nominatim check-database
+ working-directory: data-env
- name: Run update
run: |
- ./nominatim replication --init
- ./nominatim replication --once
- working-directory: build
+ ../build/nominatim replication --init
+ ../build/nominatim replication --once
+ working-directory: data-env
- name: Run reverse-only import
run : |
- dropdb nominatim
- php ./nominatim import --osm-file ../monaco-latest.osm.pbf --reverse-only
- working-directory: build
+ echo 'NOMINATIM_DATABASE_DSN="pgsql:dbname=reverse"' > .env
+ ../build/nominatim import --osm-file ../monaco-latest.osm.pbf --reverse-only
+ working-directory: data-env
${PROJECT_BINARY_DIR}/nominatim)
endif()
-#-----------------------------------------------------------------------------
-# Targets for running a development webserver from the build directory.
-#-----------------------------------------------------------------------------
-
-if (BUILD_API)
- add_custom_target(serve
- php -S 127.0.0.1:8088
- WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/website
- )
-
- add_custom_target(serve-global
- php -S 0.0.0.0:8088
- WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/website
- )
-endif()
-
#-----------------------------------------------------------------------------
# Tests
#-----------------------------------------------------------------------------
administrators only who can do some trouble-shooting themselves. We are sorry,
but we can not provide installation support. We are all doing this in our free
time and there is just so much of that time to go around. Do not open issues in
-our bug tracker if you need help. You can ask questions on the mailing list
-(see below) or on [help.openstreetmap.org](https://help.openstreetmap.org/).**
+our bug tracker if you need help. Use the discussions forum
+or ask for help on [help.openstreetmap.org](https://help.openstreetmap.org/).**
The latest stable release can be downloaded from https://nominatim.org.
There you can also find [installation instructions for the release](https://nominatim.org/release-docs/latest/admin/Installation), as well as an extensive [Troubleshooting/FAQ section](https://nominatim.org/release-docs/latest/admin/Faq/).
-Detailed installation instructions for the development version can be
-found at [nominatim.org](https://nominatim.org/release-docs/develop/admin/Installation)
-as well.
+[Detailed installation instructions for current master](https://nominatim.org/release-docs/develop/admin/Installation)
+can be found at nominatim.org as well.
A quick summary of the necessary steps:
cmake ..
make
-2. Get OSM data and import:
+2. Create a project directory, get OSM data and import:
- ./build/utils/setup.php --osm-file <your planet file> --all
+ mkdir nominatim-project
+ cd nominatim-project
+ ~/build/nominatim import --osm-file <your planet file>
-3. Point your webserver to the ./build/website directory.
+3. Point your webserver to the nominatim-project/website directory.
License
Contributing
============
-Contributions are welcome. For details see [contribution guide](CONTRIBUTING.md).
+Contributions, bugreport and pull requests are welcome.
+For details see [contribution guide](CONTRIBUTING.md).
-Both bug reports and pull requests are welcome.
+Questions and help
+==================
-Mailing list
-============
-
-For questions you can join the geocoding mailing list, see
-https://lists.openstreetmap.org/listinfo/geocoding
+For questions, community help and discussions you can use the
+[Github discussions forum](https://github.com/osm-search/Nominatim/discussions)
+or join the
+[geocoding mailing list](https://lists.openstreetmap.org/listinfo/geocoding).
sys.path.insert(1, '@CMAKE_SOURCE_DIR@')
-os.environ['NOMINATIM_NOMINATIM_TOOL'] = __file__
+os.environ['NOMINATIM_NOMINATIM_TOOL'] = os.path.abspath(__file__)
from nominatim import cli
# Importing the Database
The following instructions explain how to create a Nominatim database
-from an OSM planet file and how to keep the database up to date. It
-is assumed that you have already successfully installed the Nominatim
-software itself, if not return to the [installation page](Installation.md).
+from an OSM planet file. It is assumed that you have already successfully
+installed the Nominatim software itself. If this is not the case, return to the
+[installation page](Installation.md).
-## Configuration setup in `.env`
+## Creating the project directory
-The Nominatim server can be customized via a `.env` in the build directory.
+Before you start the import, you should create a project directory for your
+new database installation. This directory receives all data that is related
+to a single Nominatim setup: configuration, extra data, etc. Create a project
+directory apart from the Nominatim software:
+
+```
+mkdir ~/nominatim-planet
+```
+
+In the following, we refer to the project directory as `$PROJECT_DIR`. To be
+able to copy&paste instructions, you can export the appropriate variable:
+
+```
+export PROJECT_DIR=~/nominatim-planet
+```
+
+The Nominatim tool assumes per default that the current working directory is
+the project directory but you may explicitly state a different directory using
+the `--project-dir` parameter. The following instructions assume that you have
+added the Nominatim build directory to your PATH and run all directories from
+the project directory. If you haven't done yet, add the build directory to your
+path and change to the new project directory:
+
+```
+export PATH=~/Nominatim/build:$PATH
+cd $PROJECT_DIR
+```
+
+Of course, you have to replace the path above with the location of your build
+directory.
+
+!!! tip "Migration Tip"
+
+ Nominatim used to be run directly from the build directory until version 3.6.
+ Essentially, the build directory functioned as the project directory
+ for the database installation. This setup still works and can be useful for
+ development purposes. It is not recommended anymore for production setups.
+ Create a project directory that is separate from the Nominatim software.
+
+### Configuration setup in `.env`
+
+The Nominatim server can be customized via a `.env` in the project directory.
This is a file in [dotenv](https://github.com/theskumar/python-dotenv) format
which looks the same as variable settings in a standard shell environment.
You can also set the same configuration via environment variables. All
Wikipedia can be used as an optional auxiliary data source to help indicate
the importance of OSM features. Nominatim will work without this information
but it will improve the quality of the results if this is installed.
-This data is available as a binary download:
+This data is available as a binary download. Put it into your project directory:
- cd $NOMINATIM_SOURCE_DIR/data
+ cd $PROJECT_DIR
wget https://www.nominatim.org/data/wikimedia-importance.sql.gz
The file is about 400MB and adds around 4GB to the Nominatim database.
!!! tip
If you forgot to download the wikipedia rankings, you can also add
importances after the import. Download the files, then run
- `./nominatim refresh --wiki-data --importance`.
+ `nominatim refresh --wiki-data --importance`. Updating importances for
+ a planet can take a couple of hours.
### Great Britain, USA postcodes
Nominatim can use postcodes from an external source to improve searches that
-involve a GB or US postcode. This data can be optionally downloaded:
+involve a GB or US postcode. This data can be optionally downloaded into the
+project directory:
- cd $NOMINATIM_SOURCE_DIR/data
+ cd $PROJECT_DIR
wget https://www.nominatim.org/data/gb_postcode_data.sql.gz
wget https://www.nominatim.org/data/us_postcode_data.sql.gz
You can also drop the dynamic part later using the following command:
```
-./nominatim freeze
+nominatim freeze
```
Note that you still need to provide for sufficient disk space for the initial
from the **build directory** to start the import:
```sh
-./nominatim import --osm-file <data file> 2>&1 | tee setup.log
+nominatim import --osm-file <data file> 2>&1 | tee setup.log
```
### Notes on full planet imports
above. If the system starts swapping or you are getting out-of-memory errors,
reduce the cache size or even consider using a flatnode file.
-### Verify the import
+
+### Testing the installation
Run this script to verify all required tables and indices got created successfully.
```sh
-./nominatim check-database
+nominatim check-database
```
-### Testing the installation
-
Now you can try out your installation by running:
```sh
-make serve
+nominatim serve
```
This runs a small test server normally used for development. You can use it
of forward geocoding in particular under high load. To recompute word counts run:
```sh
-./nominatim refresh --word-counts
+nominatim refresh --word-counts
```
This will take a couple of hours for a full planet installation. You can
[special key phrases](https://wiki.openstreetmap.org/wiki/Nominatim/Special_Phrases)
you also need to enable these key phrases like this:
- ./nominatim special-phrases --from-wiki > specialphrases.sql
+ nominatim special-phrases --from-wiki > specialphrases.sql
psql -d nominatim -f specialphrases.sql
Note that this command downloads the phrases from the wiki link above. You
TIGER data to your own Nominatim instance by following these steps. The
entire US adds about 10GB to your database.
- 1. Get preprocessed TIGER 2019 data and unpack it into the
- data directory in your Nominatim sources:
+ 1. Get preprocessed TIGER 2019 data and unpack it into your project
+ directory:
+ cd $PROJECT_DIR
wget https://nominatim.org/data/tiger2019-nominatim-preprocessed.tar.gz
tar xf tiger2019-nominatim-preprocessed.tar.gz
2. Import the data into your Nominatim database:
- ./nominatim add-data --tiger-data tiger
+ nominatim add-data --tiger-data tiger
3. Enable use of the Tiger data in your `.env` by adding:
- NOMINATIM_USE_US_TIGER_DATA=yes
+ echo NOMINATIM_USE_US_TIGER_DATA=yes >> .env
4. Apply the new settings:
-```sh
- ./nominatim refresh --functions
-```
+ nominatim refresh --functions
See the [developer's guide](../develop/data-sources.md#us-census-tiger) for more
## 3.6.0 -> master
+### Status table contains now time zone information
+
+The `import_status` table has been changed to include timezone information
+with the time stamp. You need to alter an existing table before running
+any replication functions with:
+
+```sql
+ALTER TABLE import_status ALTER COLUMN lastimportdate TYPE timestamp with time zone;
+```
+
+### New location for data files
+
+External data files for Wikipedia importance, postcodes etc. are no longer
+expected to reside in the source tree by default. Instead they will be searched
+in the project directory. If you have an automated setup script you must
+either adapt the download location or explicitly set the location of the
+files to the old place in your `.env`.
+
### Introducing `nominatim` command line tool
The various php utilities have been replaced with a single `nominatim`
Try `nominatim <command> --help` for more information about each subcommand.
`./utils/query.php` no longer exists in its old form. `nominatim search`
-provides a replacement.
+provides a replacement but returns different output.
## 3.5.0 -> 3.6.0
To set up the update process now run the following command:
- ./nominatim replication --init
+ nominatim replication --init
It outputs the date where updates will start. Recheck that this date is
what you expect.
The following command will keep your database constantly up to date:
- ./nominatim replication
+ nominatim replication
If you have imported multiple country extracts and want to keep them
up-to-date, [Advanced installations section](Advanced-Installations.md) contains instructions
if ($aCMDResult['import-tiger-data']) {
$bDidSomething = true;
- $sTigerPath = getSetting('TIGER_DATA_PATH');
- if (!$sTigerPath) {
- $sTigerPath = CONST_DataDir.'/data/tiger';
- }
+ $sTigerPath = getSetting('TIGER_DATA_PATH', CONST_InstallDir.'/tiger');
$oSetup->importTigerData($sTigerPath);
}
// set of settings.
defined('CONST_DataDir') or define('CONST_DataDir', $_SERVER['NOMINATIM_DATADIR']);
defined('CONST_BinDir') or define('CONST_BinDir', $_SERVER['NOMINATIM_BINDIR']);
+ defined('CONST_Default_ModulePath') or define('CONST_Default_ModulePath', $_SERVER['NOMINATIM_DATABASE_MODULE_SRC_PATH']);
}
function getSetting($sConfName, $sDefault = null)
{
protected $iCacheMemory;
protected $iInstances;
- protected $sModulePath;
protected $aDSNInfo;
protected $bQuiet;
protected $bVerbose;
$this->iCacheMemory = getCacheMemoryMB();
}
- $this->sModulePath = getSetting('DATABASE_MODULE_PATH');
- if (!$this->sModulePath) {
- $this->sModulePath = CONST_Default_ModulePath;
- }
- info('module path: ' . $this->sModulePath);
-
// parse database string
$this->aDSNInfo = \Nominatim\DB::parseDSN(getSetting('DATABASE_DSN'));
if (!isset($this->aDSNInfo['port'])) {
exit(1);
}
+ if (!getSetting('DATABASE_MODULE_PATH')) {
+ // If no custom module path is set then copy the module into the
+ // project directory, but only if it is not the same file already
+ // (aka we are running from the build dir).
+ $sDest = CONST_InstallDir.'/module';
+ if ($sDest != CONST_Default_ModulePath) {
+ if (!file_exists($sDest)) {
+ mkdir($sDest);
+ }
+ if (!copy(CONST_Default_ModulePath.'/nominatim.so', $sDest.'/nominatim.so')) {
+ echo "Failed to copy database module to $sDest.";
+ exit(1);
+ }
+ chmod($sDest.'/nominatim.so', 0755);
+ info("Database module installed at $sDest.");
+ } else {
+ info('Running from build directory. Leaving database module as is.');
+ }
+ } else {
+ info('Using database module from DATABASE_MODULE_PATH ('.getSetting('DATABASE_MODULE_PATH').').');
+ }
// Try accessing the C module, so we know early if something is wrong
$this->checkModulePresence(); // raises exception on failure
$this->pgsqlRunScriptFile(CONST_DataDir.'/data/gb_postcode_table.sql');
$this->pgsqlRunScriptFile(CONST_DataDir.'/data/us_postcode_table.sql');
- $sPostcodeFilename = CONST_DataDir.'/data/gb_postcode_data.sql.gz';
+ $sPostcodeFilename = CONST_InstallDir.'/gb_postcode_data.sql.gz';
if (file_exists($sPostcodeFilename)) {
$this->pgsqlRunScriptFile($sPostcodeFilename);
} else {
warn('optional external GB postcode table file ('.$sPostcodeFilename.') not found. Skipping.');
}
- $sPostcodeFilename = CONST_DataDir.'/data/us_postcode_data.sql.gz';
+ $sPostcodeFilename = CONST_InstallDir.'/us_postcode_data.sql.gz';
if (file_exists($sPostcodeFilename)) {
$this->pgsqlRunScriptFile($sPostcodeFilename);
} else {
public function importWikipediaArticles()
{
- $sWikiArticlePath = getSetting('WIKIPEDIA_DATA_PATH', CONST_DataDir.'/data');
+ $sWikiArticlePath = getSetting('WIKIPEDIA_DATA_PATH', CONST_InstallDir);
$sWikiArticlesFile = $sWikiArticlePath.'/wikimedia-importance.sql.gz';
if (file_exists($sWikiArticlesFile)) {
info('Importing wikipedia articles and redirects');
*/
private function checkModulePresence()
{
+ $sModulePath = getSetting('DATABASE_MODULE_PATH', CONST_InstallDir.'/module');
$sSQL = "CREATE FUNCTION nominatim_test_import_func(text) RETURNS text AS '";
- $sSQL .= $this->sModulePath . "/nominatim.so', 'transliteration' LANGUAGE c IMMUTABLE STRICT";
+ $sSQL .= $sModulePath . "/nominatim.so', 'transliteration' LANGUAGE c IMMUTABLE STRICT";
$sSQL .= ';DROP FUNCTION nominatim_test_import_func(text);';
$oDB = new \Nominatim\DB();
$oDB->connect();
- $oDB->exec($sSQL, null, 'Database server failed to load '.$this->sModulePath.'/nominatim.so module');
+ $oDB->exec($sSQL, null, 'Database server failed to load '.$sModulePath.'/nominatim.so module');
}
}
from pathlib import Path
from .config import Configuration
-from .tools.exec_utils import run_legacy_script, run_api_script
+from .tools.exec_utils import run_legacy_script, run_api_script, run_php_server
from .db.connection import connect
from .db import status
from .errors import UsageError
for arg in ('module_dir', 'osm2pgsql_path', 'phplib_dir', 'data_dir', 'phpcgi_path'):
setattr(args, arg, Path(kwargs[arg]))
- args.project_dir = Path(args.project_dir)
+ args.project_dir = Path(args.project_dir).resolve()
logging.basicConfig(stream=sys.stderr,
format='%(asctime)s: %(message)s',
args.config = Configuration(args.project_dir, args.data_dir / 'settings')
+ log = logging.getLogger()
+ log.warning('Using project directory: %s', str(args.project_dir))
+
try:
return args.command.run(args)
except UsageError as exception:
- log = logging.getLogger()
if log.isEnabledFor(logging.DEBUG):
raise # use Python's exception printing
log.fatal('FATAL: %s', exception)
return run_legacy_script(*params, nominatim_env=args)
+
+class AdminServe:
+ """\
+ Start a simple web server for serving the API.
+
+ This command starts the built-in PHP webserver to serve the website
+ from the current project directory. This webserver is only suitable
+ for testing and develop. Do not use it in production setups!
+
+ By the default, the webserver can be accessed at: http://127.0.0.1:8088
+ """
+
+ @staticmethod
+ def add_args(parser):
+ group = parser.add_argument_group('Server arguments')
+ group.add_argument('--server', default='127.0.0.1:8088',
+ help='The address the server will listen to.')
+
+ @staticmethod
+ def run(args):
+ run_php_server(args.server, args.project_dir / 'website')
+
STRUCTURED_QUERY = (
('street', 'housenumber and street'),
('city', 'city, town or village'),
parser.add_subcommand('refresh', UpdateRefresh)
parser.add_subcommand('export', QueryExport)
+ parser.add_subcommand('serve', AdminServe)
if kwargs.get('phpcgi_path'):
parser.add_subcommand('search', APISearch)
env = nominatim_env.config.get_os_env()
env['NOMINATIM_DATADIR'] = str(nominatim_env.data_dir)
env['NOMINATIM_BINDIR'] = str(nominatim_env.data_dir / 'utils')
- if not env['NOMINATIM_DATABASE_MODULE_PATH']:
- env['NOMINATIM_DATABASE_MODULE_PATH'] = nominatim_env.module_dir
+ env['NOMINATIM_DATABASE_MODULE_SRC_PATH'] = nominatim_env.module_dir
if not env['NOMINATIM_OSM2PGSQL_BINARY']:
env['NOMINATIM_OSM2PGSQL_BINARY'] = nominatim_env.osm2pgsql_path
return 0
+def run_php_server(server_address, base_dir):
+ """ Run the built-in server from the given directory.
+ """
+ subprocess.run(['/usr/bin/env', 'php', '-S', server_address],
+ cwd=str(base_dir), check=True)
+
+
def run_osm2pgsql(options):
""" Run osm2pgsql with the given options.
"""
# Database connection string.
# Add host, port, user etc through additional semicolon-separated attributes.
# e.g. ;host=...;port=...;user=...;password=...
-# Changing this variable requires to run 'setup.php --setup-website'.
+# Changing this variable requires to run 'nominatim refresh --website'.
NOMINATIM_DATABASE_DSN="pgsql:dbname=nominatim"
# Database web user.
# Directory where to find the PostgreSQL server module.
# When empty the module is expected to be located in the 'module' subdirectory
# in the project directory.
-# Changing this value requires to run ./utils/setup --create-functions.
+# Changing this value requires to run 'nominatim refresh --functions'.
NOMINATIM_DATABASE_MODULE_PATH=
# Number of occurances of a word before it is considered frequent.
NOMINATIM_OSM2PGSQL_BINARY=
# Directory where to find US Tiger data files to import.
-# Used with setup.php --import-tiger-data. When unset, the data is expected
-# to be located under 'data/tiger' in the source tree.
+# OBSOLETE: use `nominatim add-data --tiger-data <dir>` to explicitly state
+# the directory on import
NOMINATIM_TIGER_DATA_PATH=
# Directory where to find pre-computed Wikipedia importance files.
-# When unset, the data is expected to be located in the 'data' directory
-# in the source tree.
+# When unset, the data is expected to be located in the project directory.
NOMINATIM_WIKIPEDIA_DATA_PATH=
# Configuration file for special phrase import.
# These are used to keep the database up to date. Per default it points to
# the minutely updates for the main OSM database. There are other services
# geared towards larger update intervals or data extracts.
-# Changing this value requires to rerun 'update/php --init-updates'.
+# Changing this value requires to rerun 'nominatim replication --init'.
NOMINATIM_REPLICATION_URL="https://planet.openstreetmap.org/replication/minute"
# Maximum amount of data to download per batch.
### API settings
#
# The following settings configure the API responses. You must rerun
-# setup.php --setup-website after changing any of them.
+# 'nominatim refresh --website' after changing any of them.
# Send permissive CORS access headers.
# When enabled, send CORS headers to allow access to everybody.
### Log settings
#
# The following options allow to enable logging of API requests.
-# You must rerun setup.php --setup-website after changing any of them.
+# You must rerun 'nominatim refresh --website' after changing any of them.
#
# Enable logging of requests into the DB.
# The request will be logged into the new_query_log table.
assert sleep_mock.last_args[0] == 60
+def test_serve_command(monkeypatch):
+ func = MockParamCapture()
+ monkeypatch.setattr(nominatim.cli, 'run_php_server', func)
+
+ call_nominatim('serve')
+
+ assert func.called == 1
+
@pytest.mark.parametrize("params", [
('search', '--query', 'new'),
('reverse', '--lat', '0', '--lon', '0'),
assert 0 == exec_utils.run_legacy_script(fname, nominatim_env=nominatim_env,
throw_on_fail=True)
-def test_run_legacy_use_given__module_path(nominatim_env, test_script):
- fname = test_script("exit($_SERVER['NOMINATIM_DATABASE_MODULE_PATH'] == 'module' ? 0 : 23);")
+def test_run_legacy_use_given_module_path(nominatim_env, test_script):
+ fname = test_script("exit($_SERVER['NOMINATIM_DATABASE_MODULE_PATH'] == '' ? 0 : 23);")
assert 0 == exec_utils.run_legacy_script(fname, nominatim_env=nominatim_env)
projects / nominatim.git / commitdiff