The file is about 400MB and adds around 4GB to the Nominatim database.
+### OSM views
+OSM publishes aggregate map access numbers that are generated based on the users’
+behavior when viewing locations on the map. This data is also optional and
+it complements wikipedia/wikidata rankings to further enhance the search results
+if added.
+OSM views data is avalaible as a GeoTIFF file. Put it into your project directory:
+
+ cd $PROJECT_DIR
+ wget https://qrank.wmcloud.org/download/osmviews.tiff
+
+The file is about 380MB and adds around 4GB to the Nominatim database. Importing
+OSM views into Nominatim takes a little over 3 hours.
+
+!!! warning
+ Importing OSM views is currently an experimental feature. OSM views data are
+ not yet included in the importance values calculations.
+
!!! tip
- If you forgot to download the wikipedia rankings, you can also add
- importances after the import. Download the files, then run
- `nominatim refresh --wiki-data --importance`. Updating importances for
- a planet can take a couple of hours.
+ If you forgot to download the wikipedia rankings or OSM views, then you can
+ also add importances after the import. To add both, download their files, then
+ run `nominatim refresh --wiki-data --osm-views --importance`. Updating
+ importances for a planet will take a couple of hours.
### External postcodes
database or reuse the space later.
!!! warning
- The datastructure for updates are also required when adding additional data
+ The data structure for updates are also required when adding additional data
after the import, for example [TIGER housenumber data](../customize/Tiger.md).
If you plan to use those, you must not use the `--no-updates` parameter.
Do a normal import, add the external data and once you are done with
address_levels: bool
functions: bool
wiki_data: bool
+ osm_views: bool
importance: bool
website: bool
diffs: bool
help='Enable debug warning statements in functions')
- def run(self, args: NominatimArgs) -> int: #pylint: disable=too-many-branches
+ def run(self, args: NominatimArgs) -> int: #pylint: disable=too-many-branches, too-many-statements
from ..tools import refresh, postcodes
from ..indexer.indexer import Indexer
data_path) > 0:
LOG.fatal('FATAL: Wikipedia importance dump file not found')
return 1
-
+
if args.osm_views:
- data_path = Path(args.config.OSM_VIEWS_DATA_PATH
- or args.project_dir)
+ data_path = Path(args.project_dir)
LOG.warning('Import OSM views GeoTIFF data from %s', data_path)
- if refresh.import_osm_views_geotiff(args.config.get_libpq_dsn(),
- data_path) > 0:
- LOG.fatal('FATAL: OSM views GeoTIFF file not found')
- return 1
+ with connect(args.config.get_libpq_dsn()) as conn:
+ if refresh.import_osm_views_geotiff(conn, data_path) == 1:
+ LOG.fatal('FATAL: OSM views GeoTIFF file not found')
+ return 1
+ if refresh.import_osm_views_geotiff(conn, data_path) == 2:
+ LOG.fatal('FATAL: PostGIS version number is less than 3')
+ return 1
# Attention: importance MUST come after wiki data import.
if args.importance:
if refresh.import_wikipedia_articles(args.config.get_libpq_dsn(),
data_path) > 0:
LOG.error('Wikipedia importance dump file not found. '
- 'Calculating importance values of locations will not use Wikipedia importance data.')
-
+ 'Calculating importance values of locations will not \
+ use Wikipedia importance data.')
+
LOG.warning('Importing OSM views GeoTIFF data')
- database_import.import_osm_views_geotiff()
- data_path = Path(args.config.OSM_VIEWS_DATA_PATH or args.project_dir)
- if refresh.import_osm_views_geotiff(args.config.get_libpq_dsn(),
- data_path) > 0:
- LOG.error('OSM views GeoTIFF file not found. '
+ data_path = Path(args.project_dir)
+ with connect(args.config.get_libpq_dsn()) as conn:
+ if refresh.import_osm_views_geotiff(conn, data_path) == 1:
+ LOG.error('OSM views GeoTIFF file not found. '
+ 'Calculating importance values of locations will not use OSM views data.')
+ elif refresh.import_osm_views_geotiff(conn, data_path) == 2:
+ LOG.error('PostGIS version number is less than 3. '
'Calculating importance values of locations will not use OSM views data.')
if args.continue_at is None or args.continue_at == 'load-data':
with conn.cursor() as cur:
cur.execute('CREATE EXTENSION IF NOT EXISTS hstore')
cur.execute('CREATE EXTENSION IF NOT EXISTS postgis')
- cur.execute('CREATE EXTENSION IF NOT EXISTS postgis_raster')
+
+ postgis_version = conn.postgis_version_tuple()
+ if postgis_version[0] >= 3:
+ cur.execute('CREATE EXTENSION IF NOT EXISTS postgis_raster')
+
conn.commit()
_require_version('PostGIS',
sql.run_parallel_sql_file(config.get_libpq_dsn(),
'indices.sql', min(8, threads), drop=drop)
-
-
-def import_osm_views_geotiff():
- """Import OSM views GeoTIFF file"""
- subprocess.run("raster2pgsql -s 4326 -I -C -t 100x100 -e osmviews.tiff public.osmviews | psql nominatim", shell=True, check=True)
"""
from typing import MutableSequence, Tuple, Any, Type, Mapping, Sequence, List, cast
import logging
+import subprocess
from textwrap import dedent
from pathlib import Path
return 0
-def import_osm_views_geotiff(dsn, data_path, ignore_errors=False):
+def import_osm_views_geotiff(conn: Connection, data_path: Path) -> int:
""" Replaces the OSM views table with new data.
-
- Returns 0 if all was well and 1 if the GeoTIFF file could not
+
+ Returns 0 if all was well and 1 if the OSM views GeoTIFF file could not
be found. Throws an exception if there was an error reading the file.
"""
datafile = data_path / 'osmviews.tiff'
if not datafile.exists():
return 1
- pre_code = """BEGIN;
- DROP TABLE IF EXISTS "osmviews";
- """
- post_code = "COMMIT"
- execute_file(dsn, datafile, ignore_errors=ignore_errors,
- pre_code=pre_code, post_code=post_code)
+ postgis_version = conn.postgis_version_tuple()
+ if postgis_version[0] < 3:
+ return 2
+
+ with conn.cursor() as cur:
+ cur.execute('DROP TABLE IF EXISTS "osm_views"')
+ conn.commit()
+
+ cmd = f"raster2pgsql -s 4326 -I -C -t 100x100 {datafile} \
+ public.osm_views | psql nominatim > /dev/null"
+ subprocess.run(["/bin/bash", "-c" , cmd], check=True)
return 0
# When unset, the data is expected to be located in the project directory.
NOMINATIM_WIKIPEDIA_DATA_PATH=
-# Directory where to find OSM views GeoTIFF file.
-# When unset, the data is expected to be located in the project directory.
-NOMINATIM_OSM_VIEWS_DATA_PATH=
-
# Configuration file for special phrase import.
# OBSOLETE: use `nominatim special-phrases --config <file>` or simply put
# a custom phrase-settings.json into your project directory.
assert cf_mock.called > 1
for mock in mocks:
- assert mock.called == 1, "Mock '{}' not called".format(mock.func_name)
+ assert mock.called > 0, "Mock '{}' not called".format(mock.func_name)
def test_import_continue_load_data(self, mock_func_factory):
@pytest.mark.parametrize("command,func", [
('address-levels', 'load_address_levels_from_config'),
('wiki-data', 'import_wikipedia_articles'),
- ('osm-views', 'import_osm_views_geotiff')
+ ('osm-views', 'import_osm_views_geotiff'),
('importance', 'recompute_importance'),
('website', 'setup_website'),
])
func_mock = mock_func_factory(nominatim.tools.refresh, func)
assert self.call_nominatim('refresh', '--' + command) == 0
- assert func_mock.called == 1
+ assert func_mock.called > 0
def test_refresh_word_count(self):
assert self.call_nominatim('refresh', '--wiki-data') == 1
- def test_refresh_osm_views_geotiff_file_not_found(self, monkeypatch):
- monkeypatch.setenv('NOMINATIM_OSM_VIEWS_DATA_PATH', 'gjoiergjeroi345Q')
-
+ def test_refresh_osm_views_geotiff_file_not_found(self):
assert self.call_nominatim('refresh', '--osm-views') == 1
- def test_refresh_importance_computed_after_wiki_and_osm_views_import(self, monkeypatch):
+ def test_refresh_importance_computed_after_wiki_import(self, monkeypatch):
calls = []
monkeypatch.setattr(nominatim.tools.refresh, 'import_wikipedia_articles',
lambda *args, **kwargs: calls.append('import') or 0)
- monkeypatch.setattr(nominatim.tools.refresh, 'import_osm_views_geotiff',
- lambda *args, **kwargs: calls.append('import') or 0)
monkeypatch.setattr(nominatim.tools.refresh, 'recompute_importance',
lambda *args, **kwargs: calls.append('update'))
- assert self.call_nominatim('refresh', '--importance', '--wiki-data', '--osm-views') == 0
+ assert self.call_nominatim('refresh', '--importance', '--wiki-data') == 0
assert calls == ['import', 'update']
'address_levels',
'location_area', 'location_area_country', 'location_area_large_100',
'location_road_1',
- 'osmviews'
'place', 'planet_osm_nodes', 'planet_osm_rels', 'planet_osm_ways',
'search_name_111',
'wikipedia_article', 'wikipedia_redirect'
assert temp_db_cursor.table_rows('wikipedia_redirect') > 0
-@pytest.mark.parametrize("replace", (True, False))
-def test_refresh_import_osm_views_geotiff(dsn, src_dir, table_factory, temp_db_cursor, replace):
- if replace:
- table_factory('osmviews')
-
- # use the small osm views GeoTIFF file for the API testdb
- assert refresh.import_osm_views_geotiff(dsn, src_dir / 'test' / 'testdb') == 0
-
- assert temp_db_cursor.table_rows('osmviews') > 0
-
-
def test_recompute_importance(placex_table, table_factory, temp_db_conn, temp_db_cursor):
temp_db_cursor.execute("""CREATE OR REPLACE FUNCTION compute_importance(extratags HSTORE,
country_code varchar(2),
projects / nominatim.git / commitdiff