description: 'Additional options to hand to cmake'
required: false
default: ''
+ lua:
+ description: 'Version of Lua to use'
+ required: false
+ default: '5.3'
runs:
using: "composite"
shell: bash
- name: Install prerequisites
run: |
- sudo apt-get install -y -qq libboost-system-dev libboost-filesystem-dev libexpat1-dev zlib1g-dev libbz2-dev libpq-dev libproj-dev libicu-dev
+ sudo apt-get install -y -qq libboost-system-dev libboost-filesystem-dev libexpat1-dev zlib1g-dev libbz2-dev libpq-dev libproj-dev libicu-dev liblua${LUA_VERSION}-dev lua${LUA_VERSION}
if [ "x$UBUNTUVER" == "x18" ]; then
pip3 install python-dotenv psycopg2==2.7.7 jinja2==2.8 psutil==5.4.2 pyicu==2.9 osmium PyYAML==5.1 datrie
else
env:
UBUNTUVER: ${{ inputs.ubuntu }}
CMAKE_ARGS: ${{ inputs.cmake-args }}
+ LUA_VERSION: ${{ inputs.lua }}
- name: Configure
run: mkdir build && cd build && cmake $CMAKE_ARGS ../Nominatim
endif()
set(BUILD_TESTS_SAVED "${BUILD_TESTS}")
set(BUILD_TESTS off)
- set(WITH_LUA off CACHE BOOL "")
add_subdirectory(osm2pgsql)
set(BUILD_TESTS ${BUILD_TESTS_SAVED})
endif()
nominatim add-data --tiger-data tiger-nominatim-preprocessed-latest.csv.tar.gz
- 3. Enable use of the Tiger data in your `.env` by adding:
+ 3. Enable use of the Tiger data in your existing `.env` file by adding:
echo NOMINATIM_USE_US_TIGER_DATA=yes >> .env
4. Apply the new settings:
- nominatim refresh --functions
+ nominatim refresh --functions --website
See the [TIGER-data project](https://github.com/osm-search/TIGER-data) for more
RETURN null;
END IF;
+ -- Remove the place from the list of places to be deleted
+ DELETE FROM place_to_be_deleted pdel
+ WHERE pdel.osm_type = NEW.osm_type and pdel.osm_id = NEW.osm_id
+ and pdel.class = NEW.class;
+
-- Have we already done this place?
SELECT * INTO existing
FROM place
{% if debug %}RAISE WARNING 'Existing: %',existing.osm_id;{% endif %}
- -- Handle a place changing type by removing the old data.
- -- (This trigger is executed BEFORE INSERT of the NEW tuple.)
IF existing.osm_type IS NULL THEN
DELETE FROM place where osm_type = NEW.osm_type and osm_id = NEW.osm_id and class = NEW.class;
END IF;
END IF;
{% endif %}
- IF existing.osm_type IS NOT NULL THEN
- -- Pathological case caused by the triggerless copy into place during initial import
- -- force delete even for large areas, it will be reinserted later
- UPDATE place SET geometry = ST_SetSRID(ST_Point(0,0), 4326)
- WHERE osm_type = NEW.osm_type and osm_id = NEW.osm_id
- and class = NEW.class and type = NEW.type;
- DELETE FROM place
- WHERE osm_type = NEW.osm_type and osm_id = NEW.osm_id
- and class = NEW.class and type = NEW.type;
+ IF existingplacex.osm_type is not NULL THEN
+ -- Mark any existing place for delete in the placex table
+ UPDATE placex SET indexed_status = 100
+ WHERE placex.osm_type = NEW.osm_type and placex.osm_id = NEW.osm_id
+ and placex.class = NEW.class and placex.type = NEW.type;
END IF;
-- Process it as a new insertion
{% if debug %}RAISE WARNING 'insert done % % % % %',NEW.osm_type,NEW.osm_id,NEW.class,NEW.type,NEW.name;{% endif %}
+ IF existing.osm_type is not NULL THEN
+ -- If there is already an entry in place, just update that, if necessary.
+ IF coalesce(existing.name, ''::hstore) != coalesce(NEW.name, ''::hstore)
+ or coalesce(existing.address, ''::hstore) != coalesce(NEW.address, ''::hstore)
+ or coalesce(existing.extratags, ''::hstore) != coalesce(NEW.extratags, ''::hstore)
+ or coalesce(existing.admin_level, 15) != coalesce(NEW.admin_level, 15)
+ or existing.geometry::text != NEW.geometry::text
+ THEN
+ UPDATE place
+ SET name = NEW.name,
+ address = NEW.address,
+ extratags = NEW.extratags,
+ admin_level = NEW.admin_level,
+ geometry = NEW.geometry
+ WHERE osm_type = NEW.osm_type and osm_id = NEW.osm_id
+ and class = NEW.class and type = NEW.type;
+ END IF;
+
+ RETURN NULL;
+ END IF;
+
RETURN NEW;
END IF;
END;
$$ LANGUAGE plpgsql;
-
CREATE OR REPLACE FUNCTION place_delete()
RETURNS TRIGGER
AS $$
DECLARE
- has_rank BOOLEAN;
+ deferred BOOLEAN;
BEGIN
-
- {% if debug %}RAISE WARNING 'delete: % % % %',OLD.osm_type,OLD.osm_id,OLD.class,OLD.type;{% endif %}
-
- -- deleting large polygons can have a massive effect on the system - require manual intervention to let them through
- IF st_area(OLD.geometry) > 2 and st_isvalid(OLD.geometry) THEN
- SELECT bool_or(not (rank_address = 0 or rank_address > 25)) as ranked FROM placex WHERE osm_type = OLD.osm_type and osm_id = OLD.osm_id and class = OLD.class and type = OLD.type INTO has_rank;
- IF has_rank THEN
- insert into import_polygon_delete (osm_type, osm_id, class, type) values (OLD.osm_type,OLD.osm_id,OLD.class,OLD.type);
- RETURN NULL;
- END IF;
+ {% if debug %}RAISE WARNING 'Delete for % % %/%', OLD.osm_type, OLD.osm_id, OLD.class, OLD.type;{% endif %}
+
+ deferred := ST_IsValid(OLD.geometry) and ST_Area(OLD.geometry) > 2;
+ IF deferred THEN
+ SELECT bool_or(not (rank_address = 0 or rank_address > 25)) INTO deferred
+ FROM placex
+ WHERE osm_type = OLD.osm_type and osm_id = OLD.osm_id
+ and class = OLD.class and type = OLD.type;
END IF;
- -- mark for delete
- UPDATE placex set indexed_status = 100 where osm_type = OLD.osm_type and osm_id = OLD.osm_id and class = OLD.class and type = OLD.type;
+ INSERT INTO place_to_be_deleted (osm_type, osm_id, class, type, deferred)
+ VALUES(OLD.osm_type, OLD.osm_id, OLD.class, OLD.type, deferred);
- -- interpolations are special
- IF OLD.osm_type='W' and OLD.class = 'place' and OLD.type = 'houses' THEN
- UPDATE location_property_osmline set indexed_status = 100 where osm_id = OLD.osm_id; -- osm_id = wayid (=old.osm_id)
- END IF;
+ RETURN NULL;
+END;
+$$ LANGUAGE plpgsql;
- RETURN OLD;
+CREATE OR REPLACE FUNCTION flush_deleted_places()
+ RETURNS INTEGER
+ AS $$
+BEGIN
+ -- deleting large polygons can have a massive effect on the system - require manual intervention to let them through
+ INSERT INTO import_polygon_delete (osm_type, osm_id, class, type)
+ SELECT osm_type, osm_id, class, type FROM place_to_be_deleted WHERE deferred;
+
+ -- delete from place table
+ ALTER TABLE place DISABLE TRIGGER place_before_delete;
+ DELETE FROM place USING place_to_be_deleted
+ WHERE place.osm_type = place_to_be_deleted.osm_type
+ and place.osm_id = place_to_be_deleted.osm_id
+ and place.class = place_to_be_deleted.class
+ and place.type = place_to_be_deleted.type
+ and not deferred;
+ ALTER TABLE place ENABLE TRIGGER place_before_delete;
+
+ -- Mark for delete in the placex table
+ UPDATE placex SET indexed_status = 100 FROM place_to_be_deleted
+ WHERE placex.osm_type = place_to_be_deleted.osm_type
+ and placex.osm_id = place_to_be_deleted.osm_id
+ and placex.class = place_to_be_deleted.class
+ and placex.type = place_to_be_deleted.type
+ and not deferred;
+
+ -- Mark for delete in interpolations
+ UPDATE location_property_osmline SET indexed_status = 100 FROM place_to_be_deleted
+ WHERE place_to_be_deleted.osm_type = 'W'
+ and place_to_be_deleted.class = 'place'
+ and place_to_be_deleted.type = 'houses'
+ and location_property_osmline.osm_id = place_to_be_deleted.osm_id
+ and not deferred;
+
+ -- Clear todo list.
+ TRUNCATE TABLE place_to_be_deleted;
+
+ RETURN NULL;
END;
-$$
-LANGUAGE plpgsql;
+$$ LANGUAGE plpgsql;
INCLUDE (startnumber, endnumber) {{db.tablespace.search_index}}
WHERE startnumber is not null;
{% endif %}
+---
+-- Table needed for running updates with osm2pgsql on place.
+ CREATE TABLE IF NOT EXISTS place_to_be_deleted (
+ osm_type CHAR(1),
+ osm_id BIGINT,
+ class TEXT,
+ type TEXT,
+ deferred BOOLEAN
+ );
+
{% endif %}
return dict(osm2pgsql=self.config.OSM2PGSQL_BINARY or self.osm2pgsql_path,
osm2pgsql_cache=self.osm2pgsql_cache or default_cache,
osm2pgsql_style=self.config.get_import_style_file(),
+ osm2pgsql_style_path=self.config.config_dir,
threads=self.threads or default_threads,
dsn=self.config.get_libpq_dsn(),
flatnode_file=str(self.config.get_path('FLATNODE_FILE') or ''),
from typing import Any, Union, Optional, Mapping, IO
from pathlib import Path
import logging
+import os
import subprocess
import urllib.request as urlrequest
from urllib.parse import urlencode
'--log-progress', 'true',
'--number-processes', str(options['threads']),
'--cache', str(options['osm2pgsql_cache']),
- '--output', 'gazetteer',
'--style', str(options['osm2pgsql_style'])
]
+
+ if str(options['osm2pgsql_style']).endswith('.lua'):
+ env['LUA_PATH'] = ';'.join((str(options['osm2pgsql_style_path'] / 'flex-base.lua'),
+ os.environ.get('LUAPATH', ';')))
+ cmd.extend(('--output', 'flex'))
+ else:
+ cmd.extend(('--output', 'gazetteer'))
+
if options['append']:
cmd.append('--append')
else:
names = {}
names['countrycode'] = country_code
analyzer.add_country_names(country_code, names)
+
+
+@_migration(4, 1, 99, 0)
+def add_place_deletion_todo_table(conn: Connection, **_: Any) -> None:
+ """ Add helper table for deleting data on updates.
+
+ The table is only necessary when updates are possible, i.e.
+ the database is not in freeze mode.
+ """
+ if conn.table_exists('place'):
+ with conn.cursor() as cur:
+ cur.execute("""CREATE TABLE IF NOT EXISTS place_to_be_deleted (
+ osm_type CHAR(1),
+ osm_id BIGINT,
+ class TEXT,
+ type TEXT,
+ deferred BOOLEAN)""")
if endseq is None:
return UpdateState.NO_CHANGES
- # Consume updates with osm2pgsql.
- options['append'] = True
- options['disable_jit'] = conn.server_version_tuple() >= (11, 0)
- run_osm2pgsql(options)
+ run_osm2pgsql_updates(conn, options)
# Write the current status to the file
endstate = repl.get_state_info(endseq)
return UpdateState.UP_TO_DATE
+def run_osm2pgsql_updates(conn: Connection, options: MutableMapping[str, Any]) -> None:
+ """ Run osm2pgsql in append mode.
+ """
+ # Remove any stale deletion marks.
+ with conn.cursor() as cur:
+ cur.execute('TRUNCATE place_to_be_deleted')
+ conn.commit()
+
+ # Consume updates with osm2pgsql.
+ options['append'] = True
+ options['disable_jit'] = conn.server_version_tuple() >= (11, 0)
+ run_osm2pgsql(options)
+
+ # Handle deletions
+ with conn.cursor() as cur:
+ cur.execute('SELECT flush_deleted_places()')
+ conn.commit()
+
+
def _make_replication_server(url: str, timeout: int) -> ContextManager[ReplicationServer]:
""" Returns a ReplicationServer in form of a context manager.
""" Download a resource from the given URL and return a byte sequence
of the content.
"""
- get_params = {
- 'headers': {"User-Agent" : f"Nominatim (pyosmium/{pyo_version.pyosmium_release})"},
- 'timeout': timeout or None,
- 'stream': True
- }
+ headers = {"User-Agent" : f"Nominatim (pyosmium/{pyo_version.pyosmium_release})"}
if self.session is not None:
- return self.session.get(url.get_full_url(), **get_params)
+ return self.session.get(url.get_full_url(),
+ headers=headers, timeout=timeout or None,
+ stream=True)
@contextmanager
def _get_url_with_session() -> Iterator[requests.Response]:
with requests.Session() as session:
- request = session.get(url.get_full_url(), **get_params) # type: ignore
+ request = session.get(url.get_full_url(),
+ headers=headers, timeout=timeout or None,
+ stream=True)
yield request
return _get_url_with_session()
repl = ReplicationServer(url)
- repl.open_url = types.MethodType(patched_open_url, repl)
+ setattr(repl, 'open_url', types.MethodType(patched_open_url, repl))
return cast(ContextManager[ReplicationServer], repl)
# patch level when cherry-picking the commit with the migration.
#
# Released versions always have a database patch level of 0.
-NOMINATIM_VERSION = (4, 1, 0, 0)
+NOMINATIM_VERSION = (4, 1, 99, 0)
POSTGRESQL_REQUIRED_VERSION = (9, 6)
POSTGIS_REQUIRED_VERSION = (2, 2)
--- /dev/null
+-- Core functions for Nominatim import flex style.
+--
+
+
+-- The single place table.
+place_table = osm2pgsql.define_table{
+ name = "place",
+ ids = { type = 'any', id_column = 'osm_id', type_column = 'osm_type' },
+ columns = {
+ { column = 'class', type = 'text', not_null = true },
+ { column = 'type', type = 'text', not_null = true },
+ { column = 'admin_level', type = 'smallint' },
+ { column = 'name', type = 'hstore' },
+ { column = 'address', type = 'hstore' },
+ { column = 'extratags', type = 'hstore' },
+ { column = 'geometry', type = 'geometry', projection = 'WGS84', not_null = true },
+ }
+}
+
+------------- Place class ------------------------------------------
+
+local Place = {}
+Place.__index = Place
+
+function Place.new(object, geom_func)
+ local self = setmetatable({}, Place)
+ self.object = object
+ self.geom_func = geom_func
+
+ self.admin_level = tonumber(self.object:grab_tag('admin_level'))
+ if self.admin_level == nil
+ or self.admin_level <= 0 or self.admin_level > 15
+ or math.floor(self.admin_level) ~= self.admin_level then
+ self.admin_level = 15
+ end
+
+ self.num_entries = 0
+ self.has_name = false
+ self.names = {}
+ self.address = {}
+ self.extratags = {}
+
+ return self
+end
+
+function Place:delete(data)
+ if data.match ~= nil then
+ for k, v in pairs(self.object.tags) do
+ if data.match(k, v) then
+ self.object.tags[k] = nil
+ end
+ end
+ end
+end
+
+function Place:grab_extratags(data)
+ local count = 0
+
+ if data.match ~= nil then
+ for k, v in pairs(self.object.tags) do
+ if data.match(k, v) then
+ self.object.tags[k] = nil
+ self.extratags[k] = v
+ count = count + 1
+ end
+ end
+ end
+
+ return count
+end
+
+function Place:grab_address(data)
+ local count = 0
+
+ if data.match ~= nil then
+ for k, v in pairs(self.object.tags) do
+ if data.match(k, v) then
+ self.object.tags[k] = nil
+
+ if data.include_on_name == true then
+ self.has_name = true
+ end
+
+ if data.out_key ~= nil then
+ self.address[data.out_key] = v
+ return 1
+ end
+
+ if k:sub(1, 5) == 'addr:' then
+ self.address[k:sub(6)] = v
+ elseif k:sub(1, 6) == 'is_in:' then
+ self.address[k:sub(7)] = v
+ else
+ self.address[k] = v
+ end
+ count = count + 1
+ end
+ end
+ end
+
+ return count
+end
+
+function Place:set_address(key, value)
+ self.address[key] = value
+end
+
+function Place:grab_name(data)
+ local count = 0
+
+ if data.match ~= nil then
+ for k, v in pairs(self.object.tags) do
+ if data.match(k, v) then
+ self.object.tags[k] = nil
+ self.names[k] = v
+ if data.include_on_name ~= false then
+ self.has_name = true
+ end
+ count = count + 1
+ end
+ end
+ end
+
+ return count
+end
+
+function Place:grab_tag(key)
+ return self.object:grab_tag(key)
+end
+
+function Place:tags()
+ return self.object.tags
+end
+
+function Place:write_place(k, v, mtype, save_extra_mains)
+ if mtype == nil then
+ return 0
+ end
+
+ v = v or self.object.tags[k]
+ if v == nil then
+ return 0
+ end
+
+ if type(mtype) == 'table' then
+ mtype = mtype[v] or mtype[1]
+ end
+
+ if mtype == 'always' or (self.has_name and mtype == 'named') then
+ return self:write_row(k, v, save_extra_mains)
+ end
+
+ if mtype == 'named_with_key' then
+ local names = {}
+ local prefix = k .. ':name'
+ for namek, namev in pairs(self.object.tags) do
+ if namek:sub(1, #prefix) == prefix
+ and (#namek == #prefix
+ or namek:sub(#prefix + 1, #prefix + 1) == ':') then
+ names[namek:sub(#k + 2)] = namev
+ end
+ end
+
+ if next(names) ~= nil then
+ local saved_names = self.names
+ self.names = names
+
+ local results = self:write_row(k, v, save_extra_mains)
+
+ self.names = saved_names
+
+ return results
+ end
+ end
+
+ return 0
+end
+
+function Place:write_row(k, v, save_extra_mains)
+ if self.geometry == nil then
+ self.geometry = self.geom_func(self.object)
+ end
+ if self.geometry:is_null() then
+ return 0
+ end
+
+ if save_extra_mains then
+ for extra_k, extra_v in pairs(self.object.tags) do
+ if extra_k ~= k then
+ self.extratags[extra_k] = extra_v
+ end
+ end
+ end
+
+ place_table:insert{
+ class = k,
+ type = v,
+ admin_level = self.admin_level,
+ name = next(self.names) and self.names,
+ address = next(self.address) and self.address,
+ extratags = next(self.extratags) and self.extratags,
+ geometry = self.geometry
+ }
+
+ if save_extra_mains then
+ for k, v in pairs(self.object.tags) do
+ self.extratags[k] = nil
+ end
+ end
+
+ self.num_entries = self.num_entries + 1
+
+ return 1
+end
+
+
+function tag_match(data)
+ if data == nil or next(data) == nil then
+ return nil
+ end
+
+ local fullmatches = {}
+ local key_prefixes = {}
+ local key_suffixes = {}
+
+ if data.keys ~= nil then
+ for _, key in pairs(data.keys) do
+ if key:sub(1, 1) == '*' then
+ if #key > 1 then
+ if key_suffixes[#key - 1] == nil then
+ key_suffixes[#key - 1] = {}
+ end
+ key_suffixes[#key - 1][key:sub(2)] = true
+ end
+ elseif key:sub(#key, #key) == '*' then
+ if key_prefixes[#key - 1] == nil then
+ key_prefixes[#key - 1] = {}
+ end
+ key_prefixes[#key - 1][key:sub(1, #key - 1)] = true
+ else
+ fullmatches[key] = true
+ end
+ end
+ end
+
+ if data.tags ~= nil then
+ for k, vlist in pairs(data.tags) do
+ if fullmatches[k] == nil then
+ fullmatches[k] = {}
+ for _, v in pairs(vlist) do
+ fullmatches[k][v] = true
+ end
+ end
+ end
+ end
+
+ return function (k, v)
+ if fullmatches[k] ~= nil and (fullmatches[k] == true or fullmatches[k][v] ~= nil) then
+ return true
+ end
+
+ for slen, slist in pairs(key_suffixes) do
+ if #k >= slen and slist[k:sub(-slen)] ~= nil then
+ return true
+ end
+ end
+
+ for slen, slist in pairs(key_prefixes) do
+ if #k >= slen and slist[k:sub(1, slen)] ~= nil then
+ return true
+ end
+ end
+
+ return false
+ end
+end
+
+
+-- Process functions for all data types
+function osm2pgsql.process_node(object)
+
+ local function geom_func(o)
+ return o:as_point()
+ end
+
+ process_tags(Place.new(object, geom_func))
+end
+
+function osm2pgsql.process_way(object)
+
+ local function geom_func(o)
+ local geom = o:as_polygon()
+
+ if geom:is_null() then
+ geom = o:as_linestring()
+ end
+
+ return geom
+ end
+
+ process_tags(Place.new(object, geom_func))
+end
+
+function relation_as_multipolygon(o)
+ return o:as_multipolygon()
+end
+
+function relation_as_multiline(o)
+ return o:as_multilinestring():line_merge()
+end
+
+function osm2pgsql.process_relation(object)
+ local geom_func = RELATION_TYPES[object.tags.type]
+
+ if geom_func ~= nil then
+ process_tags(Place.new(object, geom_func))
+ end
+end
+
+function process_tags(o)
+ local fallback
+
+ o:delete{match = PRE_DELETE}
+ o:grab_extratags{match = PRE_EXTRAS}
+
+ -- Exception for boundary/place double tagging
+ if o.object.tags.boundary == 'administrative' then
+ o:grab_extratags{match = function (k, v)
+ return k == 'place' and v:sub(1,3) ~= 'isl'
+ end}
+ end
+
+ -- address keys
+ o:grab_address{match=COUNTRY_TAGS, out_key='country'}
+ if o.address.country ~= nil and #o.address.country ~= 2 then
+ o.address['country'] = nil
+ end
+ if o:grab_name{match=HOUSENAME_TAGS} > 0 then
+ fallback = {'place', 'house'}
+ end
+ if o:grab_address{match=HOUSENUMBER_TAGS, include_on_name = true} > 0 and fallback == nil then
+ fallback = {'place', 'house'}
+ end
+ if o:grab_address{match=POSTCODES, out_key='postcode'} > 0 and fallback == nil then
+ fallback = {'place', 'postcode'}
+ end
+
+ local is_interpolation = o:grab_address{match=INTERPOLATION_TAGS} > 0
+
+ if ADD_TIGER_COUNTY then
+ local v = o:grab_tag('tiger:county')
+ if v ~= nil then
+ v, num = v:gsub(',.*', ' county')
+ if num == 0 then
+ v = v .. ' county'
+ end
+ o:set_address('tiger:county', v)
+ end
+ end
+ o:grab_address{match=ADDRESS_TAGS}
+
+ if is_interpolation then
+ o:write_place('place', 'houses', 'always', SAVE_EXTRA_MAINS)
+ return
+ end
+
+ -- name keys
+ o:grab_name{match = NAMES}
+ o:grab_name{match = REFS, include_on_name = false}
+
+ o:delete{match = POST_DELETE}
+ o:grab_extratags{match = POST_EXTRAS}
+
+ -- collect main keys
+ local num_mains = 0
+ for k, v in pairs(o:tags()) do
+ num_mains = num_mains + o:write_place(k, v, MAIN_KEYS[k], SAVE_EXTRA_MAINS)
+ end
+
+ if num_mains == 0 then
+ for tag, mtype in pairs(MAIN_FALLBACK_KEYS) do
+ if o:write_place(tag, nil, mtype, SAVE_EXTRA_MAINS) > 0 then
+ return
+ end
+ end
+
+ if fallback ~= nil then
+ o:write_place(fallback[1], fallback[2], 'always', SAVE_EXTRA_MAINS)
+ end
+ end
+end
+
+
--- /dev/null
+require('flex-base')
+
+RELATION_TYPES = {
+ multipolygon = relation_as_multipolygon,
+ boundary = relation_as_multipolygon,
+ waterway = relation_as_multiline
+}
+
+MAIN_KEYS = {
+ emergency = 'always',
+ historic = 'always',
+ military = 'always',
+ natural = 'named',
+ landuse = 'named',
+ highway = {'always',
+ street_lamp = 'named',
+ traffic_signals = 'named',
+ service = 'named',
+ cycleway = 'named',
+ path = 'named',
+ footway = 'named',
+ steps = 'named',
+ bridleway = 'named',
+ track = 'named',
+ motorway_link = 'named',
+ trunk_link = 'named',
+ primary_link = 'named',
+ secondary_link = 'named',
+ tertiary_link = 'named'},
+ railway = 'named',
+ man_made = 'always',
+ aerialway = 'always',
+ boundary = {'named',
+ postal_code = 'named'},
+ aeroway = 'always',
+ amenity = 'always',
+ club = 'always',
+ craft = 'always',
+ leisure = 'always',
+ office = 'always',
+ mountain_pass = 'always',
+ shop = 'always',
+ tourism = 'always',
+ bridge = 'named_with_key',
+ tunnel = 'named_with_key',
+ waterway = 'named',
+ place = 'always'
+}
+
+MAIN_FALLBACK_KEYS = {
+ building = 'named',
+ landuse = 'named',
+ junction = 'named',
+ healthcare = 'named'
+}
+
+
+PRE_DELETE = tag_match{keys = {'note', 'note:*', 'source', 'source*', 'attribution',
+ 'comment', 'fixme', 'FIXME', 'created_by', 'NHD:*',
+ 'nhd:*', 'gnis:*', 'geobase:*', 'KSJ2:*', 'yh:*',
+ 'osak:*', 'naptan:*', 'CLC:*', 'import', 'it:fvg:*',
+ 'type', 'lacounty:*', 'ref:ruian:*', 'building:ruian:type',
+ 'ref:linz:*', 'is_in:postcode'},
+ tags = {emergency = {'yes', 'no', 'fire_hydrant'},
+ historic = {'yes', 'no'},
+ military = {'yes', 'no'},
+ natural = {'yes', 'no', 'coastline'},
+ highway = {'no', 'turning_circle', 'mini_roundabout',
+ 'noexit', 'crossing', 'give_way', 'stop'},
+ railway = {'level_crossing', 'no', 'rail'},
+ man_made = {'survey_point', 'cutline'},
+ aerialway = {'pylon', 'no'},
+ aeroway = {'no'},
+ amenity = {'no'},
+ club = {'no'},
+ craft = {'no'},
+ leisure = {'no'},
+ office = {'no'},
+ mountain_pass = {'no'},
+ shop = {'no'},
+ tourism = {'yes', 'no'},
+ bridge = {'no'},
+ tunnel = {'no'},
+ waterway = {'riverbank'},
+ building = {'no'},
+ boundary = {'place'}}
+ }
+
+POST_DELETE = tag_match{keys = {'tiger:*'}}
+
+PRE_EXTRAS = tag_match{keys = {'*:prefix', '*:suffix', 'name:prefix:*', 'name:suffix:*',
+ 'name:etymology', 'name:signed', 'name:botanical',
+ 'wikidata', '*:wikidata',
+ 'addr:street:name', 'addr:street:type'}
+ }
+
+
+NAMES = tag_match{keys = {'name', 'name:*',
+ 'int_name', 'int_name:*',
+ 'nat_name', 'nat_name:*',
+ 'reg_name', 'reg_name:*',
+ 'loc_name', 'loc_name:*',
+ 'old_name', 'old_name:*',
+ 'alt_name', 'alt_name:*', 'alt_name_*',
+ 'official_name', 'official_name:*',
+ 'place_name', 'place_name:*',
+ 'short_name', 'short_name:*', 'brand'}}
+
+REFS = tag_match{keys = {'ref', 'int_ref', 'nat_ref', 'reg_ref', 'loc_ref', 'old_ref',
+ 'iata', 'icao', 'pcode', 'pcode:*', 'ISO3166-2'}}
+
+POSTCODES = tag_match{keys = {'postal_code', 'postcode', 'addr:postcode',
+ 'tiger:zip_left', 'tiger:zip_right'}}
+
+COUNTRY_TAGS = tag_match{keys = {'country_code', 'ISO3166-1',
+ 'addr:country_code', 'is_in:country_code',
+ 'addr:country', 'is_in:country'}}
+
+HOUSENAME_TAGS = tag_match{keys = {'addr:housename'}}
+
+HOUSENUMBER_TAGS = tag_match{keys = {'addr:housenumber', 'addr:conscriptionnumber',
+ 'addr:streetnumber'}}
+
+INTERPOLATION_TAGS = tag_match{keys = {'addr:interpolation'}}
+
+ADDRESS_TAGS = tag_match{keys = {'addr:*', 'is_in:*'}}
+ADD_TIGER_COUNTY = true
+
+SAVE_EXTRA_MAINS = true
+
'API_TEST_FILE' : (TEST_BASE_DIR / 'testdb' / 'apidb-test-data.pbf').resolve(),
'SERVER_MODULE_PATH' : None,
'TOKENIZER' : None, # Test with a custom tokenizer
+ 'STYLE' : 'extratags',
'PHPCOV' : False, # set to output directory to enable code coverage
}
--- /dev/null
+@DB
+Feature: Tag evaluation
+ Tests if tags are correctly imported into the place table
+
+ Scenario: Main tags as fallback
+ When loading osm data
+ """
+ n100 Tjunction=yes,highway=bus_stop
+ n101 Tjunction=yes,name=Bar
+ n200 Tbuilding=yes,amenity=cafe
+ n201 Tbuilding=yes,name=Intersting
+ n202 Tbuilding=yes
+ """
+ Then place contains exactly
+ | object | class | type |
+ | N100 | highway | bus_stop |
+ | N101 | junction | yes |
+ | N200 | amenity | cafe |
+ | N201 | building | yes |
+
+
+ Scenario: Name and reg tags
+ When loading osm data
+ """
+ n2001 Thighway=road,name=Foo,alt_name:de=Bar,ref=45
+ n2002 Thighway=road,name:prefix=Pre,name:suffix=Post,ref:de=55
+ n2003 Thighway=yes,name:%20%de=Foo,name=real1
+ n2004 Thighway=yes,name:%a%de=Foo,name=real2
+ n2005 Thighway=yes,name:%9%de=Foo,name:\\=real3
+ n2006 Thighway=yes,name:%9%de=Foo,name=rea\l3
+ """
+ Then place contains exactly
+ | object | class | type | name |
+ | N2001 | highway | road | 'name': 'Foo', 'alt_name:de': 'Bar', 'ref': '45' |
+ | N2002 | highway | road | - |
+ | N2003 | highway | yes | 'name: de': 'Foo', 'name': 'real1' |
+ | N2004 | highway | yes | 'name:\nde': 'Foo', 'name': 'real2' |
+ | N2005 | highway | yes | 'name:\tde': 'Foo', 'name:\\\\': 'real3' |
+ | N2006 | highway | yes | 'name:\tde': 'Foo', 'name': 'rea\\l3' |
+
+ And place contains
+ | object | extratags |
+ | N2002 | 'name:prefix': 'Pre', 'name:suffix': 'Post', 'ref:de': '55' |
+
+
+ Scenario: Name when using with_name flag
+ When loading osm data
+ """
+ n3001 Tbridge=yes,bridge:name=GoldenGate
+ n3002 Tbridge=yes,bridge:name:en=Rainbow
+ """
+ Then place contains exactly
+ | object | class | type | name |
+ | N3001 | bridge | yes | 'name': 'GoldenGate' |
+ | N3002 | bridge | yes | 'name:en': 'Rainbow' |
+
+
+ Scenario: Address tags
+ When loading osm data
+ """
+ n4001 Taddr:housenumber=34,addr:city=Esmarald,addr:county=Land
+ n4002 Taddr:streetnumber=10,is_in:city=Rootoo,is_in=Gold
+ """
+ Then place contains exactly
+ | object | class | address |
+ | N4001 | place | 'housenumber': '34', 'city': 'Esmarald', 'county': 'Land' |
+ | N4002 | place | 'streetnumber': '10', 'city': 'Rootoo' |
+
+
+ Scenario: Country codes
+ When loading osm data
+ """
+ n5001 Tshop=yes,country_code=DE
+ n5002 Tshop=yes,country_code=toolong
+ n5003 Tshop=yes,country_code=x
+ n5004 Tshop=yes,addr:country=us
+ n5005 Tshop=yes,country=be
+ n5006 Tshop=yes,addr:country=France
+ """
+ Then place contains exactly
+ | object | class | address |
+ | N5001 | shop | 'country': 'DE' |
+ | N5002 | shop | - |
+ | N5003 | shop | - |
+ | N5004 | shop | 'country': 'us' |
+ | N5005 | shop | - |
+ | N5006 | shop | - |
+
+
+ Scenario: Postcodes
+ When loading osm data
+ """
+ n6001 Tshop=bank,addr:postcode=12345
+ n6002 Tshop=bank,tiger:zip_left=34343
+ n6003 Tshop=bank,is_in:postcode=9009
+ """
+ Then place contains exactly
+ | object | class | address |
+ | N6001 | shop | 'postcode': '12345' |
+ | N6002 | shop | 'postcode': '34343' |
+ | N6003 | shop | - |
+
+
+ Scenario: Main with extra
+ When loading osm data
+ """
+ n7001 Thighway=primary,bridge=yes,name=1
+ n7002 Thighway=primary,bridge=yes,bridge:name=1
+ """
+ Then place contains exactly
+ | object | class | type | name | extratags+bridge:name |
+ | N7001 | highway | primary | 'name': '1' | - |
+ | N7002:highway | highway | primary | - | 1 |
+ | N7002:bridge | bridge | yes | 'name': '1' | 1 |
+
+
+ Scenario: Global fallback and skipping
+ When loading osm data
+ """
+ n8001 Tshop=shoes,note:de=Nein,xx=yy
+ n8002 Tshop=shoes,building=no,ele=234
+ n8003 Tshop=shoes,name:source=survey
+ """
+ Then place contains exactly
+ | object | class | extratags |
+ | N8001 | shop | 'xx': 'yy' |
+ | N8002 | shop | 'ele': '234' |
+ | N8003 | shop | - |
+
+
+ Scenario: Admin levels
+ When loading osm data
+ """
+ n9001 Tplace=city
+ n9002 Tplace=city,admin_level=16
+ n9003 Tplace=city,admin_level=x
+ n9004 Tplace=city,admin_level=1
+ n9005 Tplace=city,admin_level=0
+ n9006 Tplace=city,admin_level=2.5
+ """
+ Then place contains exactly
+ | object | class | admin_level |
+ | N9001 | place | 15 |
+ | N9002 | place | 15 |
+ | N9003 | place | 15 |
+ | N9004 | place | 1 |
+ | N9005 | place | 15 |
+ | N9006 | place | 15 |
+
+
+ Scenario: Administrative boundaries with place tags
+ When loading osm data
+ """
+ n10001 Tboundary=administrative,place=city,name=A
+ n10002 Tboundary=natural,place=city,name=B
+ n10003 Tboundary=administrative,place=island,name=C
+ """
+ Then place contains
+ | object | class | type | extratags |
+ | N10001 | boundary | administrative | 'place': 'city' |
+ And place contains
+ | object | class | type |
+ | N10002:boundary | boundary | natural |
+ | N10002:place | place | city |
+ | N10003:boundary | boundary | administrative |
+ | N10003:place | place | island |
+
+
+ Scenario: Shorten tiger:county tags
+ When loading osm data
+ """
+ n11001 Tplace=village,tiger:county=Feebourgh%2c%%20%AL
+ n11002 Tplace=village,addr:state=Alabama,tiger:county=Feebourgh%2c%%20%AL
+ n11003 Tplace=village,tiger:county=Feebourgh
+ """
+ Then place contains exactly
+ | object | class | address |
+ | N11001 | place | 'tiger:county': 'Feebourgh county' |
+ | N11002 | place | 'tiger:county': 'Feebourgh county', 'state': 'Alabama' |
+ | N11003 | place | 'tiger:county': 'Feebourgh county' |
+
+
+ Scenario: Building fallbacks
+ When loading osm data
+ """
+ n12001 Ttourism=hotel,building=yes
+ n12002 Tbuilding=house
+ n12003 Tbuilding=shed,addr:housenumber=1
+ n12004 Tbuilding=yes,name=Das-Haus
+ n12005 Tbuilding=yes,addr:postcode=12345
+ """
+ Then place contains exactly
+ | object | class | type |
+ | N12001 | tourism | hotel |
+ | N12003 | building | shed |
+ | N12004 | building | yes |
+ | N12005 | place | postcode |
+
+
+ Scenario: Address interpolations
+ When loading osm data
+ """
+ n13001 Taddr:interpolation=odd
+ n13002 Taddr:interpolation=even,place=city
+ """
+ Then place contains exactly
+ | object | class | type | address |
+ | N13001 | place | houses | 'interpolation': 'odd' |
+ | N13002 | place | houses | 'interpolation': 'even' |
--- /dev/null
+@DB
+Feature: Updates of address interpolation objects
+ Test that changes to address interpolation objects are correctly
+ propagated.
+
+ Background:
+ Given the grid
+ | 1 | 2 |
+
+
+ Scenario: Adding a new interpolation
+ When loading osm data
+ """
+ n1 Taddr:housenumber=3
+ n2 Taddr:housenumber=17
+ """
+ Then place contains
+ | object | type |
+ | N1:place | house |
+ | N2:place | house |
+
+ When updating osm data
+ """
+ w99 Taddr:interpolation=odd Nn1,n2
+ """
+ Then place contains
+ | object | type |
+ | N1:place | house |
+ | N2:place | house |
+ | W99:place | houses |
+ When indexing
+ Then placex contains exactly
+ | object | type |
+ | N1:place | house |
+ | N2:place | house |
+ Then location_property_osmline contains exactly
+ | object |
+ | 99:5 |
+
+
+ Scenario: Delete an existing interpolation
+ When loading osm data
+ """
+ n1 Taddr:housenumber=2
+ n2 Taddr:housenumber=7
+ w99 Taddr:interpolation=odd Nn1,n2
+ """
+ Then place contains
+ | object | type |
+ | N1:place | house |
+ | N2:place | house |
+ | W99:place | houses |
+
+ When updating osm data
+ """
+ w99 v2 dD
+ """
+ Then place contains
+ | object | type |
+ | N1:place | house |
+ | N2:place | house |
+ When indexing
+ Then placex contains exactly
+ | object | type |
+ | N1:place | house |
+ | N2:place | house |
+ Then location_property_osmline contains exactly
+ | object | indexed_status |
+
+
+ Scenario: Changing an object to an interpolation
+ When loading osm data
+ """
+ n1 Taddr:housenumber=3
+ n2 Taddr:housenumber=17
+ w99 Thighway=residential Nn1,n2
+ """
+ Then place contains
+ | object | type |
+ | N1:place | house |
+ | N2:place | house |
+ | W99:highway | residential |
+
+ When updating osm data
+ """
+ w99 Taddr:interpolation=odd Nn1,n2
+ """
+ Then place contains
+ | object | type |
+ | N1:place | house |
+ | N2:place | house |
+ | W99:place | houses |
+ When indexing
+ Then placex contains exactly
+ | object | type |
+ | N1:place | house |
+ | N2:place | house |
+ And location_property_osmline contains exactly
+ | object |
+ | 99:5 |
+
+
+ Scenario: Changing an interpolation to something else
+ When loading osm data
+ """
+ n1 Taddr:housenumber=3
+ n2 Taddr:housenumber=17
+ w99 Taddr:interpolation=odd Nn1,n2
+ """
+ Then place contains
+ | object | type |
+ | N1:place | house |
+ | N2:place | house |
+ | W99:place | houses |
+
+ When updating osm data
+ """
+ w99 Thighway=residential Nn1,n2
+ """
+ Then place contains
+ | object | type |
+ | N1:place | house |
+ | N2:place | house |
+ | W99:highway | residential |
+ When indexing
+ Then placex contains exactly
+ | object | type |
+ | N1:place | house |
+ | N2:place | house |
+ | W99:highway | residential |
+ And location_property_osmline contains exactly
+ | object |
+
--- /dev/null
+@DB
+Feature: Update of postcode only objects
+ Tests that changes to objects containing only a postcode are
+ propagated correctly.
+
+
+ Scenario: Adding a postcode-only node
+ When loading osm data
+ """
+ """
+ Then place contains exactly
+ | object |
+
+ When updating osm data
+ """
+ n34 Tpostcode=4456
+ """
+ Then place contains exactly
+ | object | type |
+ | N34:place | postcode |
+ When indexing
+ Then placex contains exactly
+ | object |
+
+
+ Scenario: Deleting a postcode-only node
+ When loading osm data
+ """
+ n34 Tpostcode=4456
+ """
+ Then place contains exactly
+ | object | type |
+ | N34:place | postcode |
+
+ When updating osm data
+ """
+ n34 v2 dD
+ """
+ Then place contains exactly
+ | object |
+ When indexing
+ Then placex contains exactly
+ | object |
+
+
+ Scenario Outline: Converting a regular object into a postcode-only node
+ When loading osm data
+ """
+ n34 T<class>=<type>
+ """
+ Then place contains exactly
+ | object | type |
+ | N34:<class> | <type> |
+
+ When updating osm data
+ """
+ n34 Tpostcode=4456
+ """
+ Then place contains exactly
+ | object | type |
+ | N34:place | postcode |
+ When indexing
+ Then placex contains exactly
+ | object |
+
+ Examples:
+ | class | type |
+ | amenity | restaurant |
+ | place | hamlet |
+
+
+ Scenario Outline: Converting a postcode-only node into a regular object
+ When loading osm data
+ """
+ n34 Tpostcode=4456
+ """
+ Then place contains exactly
+ | object | type |
+ | N34:place | postcode |
+
+ When updating osm data
+ """
+ n34 T<class>=<type>
+ """
+ Then place contains exactly
+ | object | type |
+ | N34:<class> | <type> |
+ When indexing
+ Then placex contains exactly
+ | object | type |
+ | N34:<class> | <type> |
+
+ Examples:
+ | class | type |
+ | amenity | restaurant |
+ | place | hamlet |
+
+
+ Scenario: Converting na interpolation into a postcode-only node
+ Given the grid
+ | 1 | 2 |
+ When loading osm data
+ """
+ n1 Taddr:housenumber=3
+ n2 Taddr:housenumber=17
+ w34 Taddr:interpolation=odd Nn1,n2
+ """
+ Then place contains exactly
+ | object | type |
+ | N1:place | house |
+ | N2:place | house |
+ | W34:place | houses |
+
+ When updating osm data
+ """
+ w34 Tpostcode=4456 Nn1,n2
+ """
+ Then place contains exactly
+ | object | type |
+ | N1:place | house |
+ | N2:place | house |
+ | W34:place | postcode |
+ When indexing
+ Then location_property_osmline contains exactly
+ | object |
+ And placex contains exactly
+ | object | type |
+ | N1:place | house |
+ | N2:place | house |
+
+
+ Scenario: Converting a postcode-only node into an interpolation
+ Given the grid
+ | 1 | 2 |
+ When loading osm data
+ """
+ n1 Taddr:housenumber=3
+ n2 Taddr:housenumber=17
+ w34 Tpostcode=4456 Nn1,n2
+ """
+ Then place contains exactly
+ | object | type |
+ | N1:place | house |
+ | N2:place | house |
+ | W34:place | postcode |
+
+ When updating osm data
+ """
+ w34 Taddr:interpolation=odd Nn1,n2
+ """
+ Then place contains exactly
+ | object | type |
+ | N1:place | house |
+ | N2:place | house |
+ | W34:place | houses |
+ When indexing
+ Then location_property_osmline contains exactly
+ | object |
+ | 34:5 |
+ And placex contains exactly
+ | object | type |
+ | N1:place | house |
+ | N2:place | house |
Feature: Update of simple objects by osm2pgsql
Testing basic update functions of osm2pgsql.
- Scenario: Import object with two main tags
+ Scenario: Adding a new object
When loading osm data
"""
- n1 Ttourism=hotel,amenity=restaurant,name=foo
- n2 Tplace=locality,name=spotty
+ n1 Tplace=town,name=Middletown
"""
- Then place contains
- | object | type | name+name |
- | N1:tourism | hotel | foo |
- | N1:amenity | restaurant | foo |
- | N2:place | locality | spotty |
- When updating osm data
- """
- n1 dV Ttourism=hotel,name=foo
- n2 dD
- """
- Then place has no entry for N1:amenity
- And place has no entry for N2
- And place contains
- | object | class | type | name |
- | N1:tourism | tourism | hotel | 'name' : 'foo' |
+ Then place contains exactly
+ | object | type | name+name |
+ | N1:place | town | Middletown |
+
+ When updating osm data
+ """
+ n2 Tamenity=hotel,name=Posthotel
+ """
+ Then place contains exactly
+ | object | type | name+name |
+ | N1:place | town | Middletown |
+ | N2:amenity | hotel | Posthotel |
+ And placex contains exactly
+ | object | type | name+name | indexed_status |
+ | N1:place | town | Middletown | 0 |
+ | N2:amenity | hotel | Posthotel | 1 |
- Scenario: Downgrading a highway to one that is dropped without name
- When loading osm data
- """
- n100 x0 y0
- n101 x0.0001 y0.0001
- w1 Thighway=residential Nn100,n101
- """
- Then place contains
- | object |
- | W1:highway |
- When updating osm data
- """
- w1 Thighway=service Nn100,n101
- """
- Then place has no entry for W1
- Scenario: Downgrading a highway when a second tag is present
+ Scenario: Deleting an existing object
When loading osm data
"""
- n100 x0 y0
- n101 x0.0001 y0.0001
- w1 Thighway=residential,tourism=hotel Nn100,n101
- """
- Then place contains
- | object |
- | W1:highway |
- | W1:tourism |
- When updating osm data
+ n1 Tplace=town,name=Middletown
+ n2 Tamenity=hotel,name=Posthotel
"""
- w1 Thighway=service,tourism=hotel Nn100,n101
- """
- Then place has no entry for W1:highway
- And place contains
- | object |
- | W1:tourism |
+ Then place contains exactly
+ | object | type | name+name |
+ | N1:place | town | Middletown |
+ | N2:amenity | hotel | Posthotel |
+
+ When updating osm data
+ """
+ n2 dD
+ """
+ Then place contains exactly
+ | object | type | name+name |
+ | N1:place | town | Middletown |
+ And placex contains exactly
+ | object | type | name+name | indexed_status |
+ | N1:place | town | Middletown | 0 |
+ | N2:amenity | hotel | Posthotel | 100 |
--- /dev/null
+@DB
+Feature: Tag evaluation
+ Tests if tags are correctly updated in the place table
+
+ Background:
+ Given the grid
+ | 1 | 2 | 3 |
+ | 10 | 11 | |
+ | 45 | 46 | |
+
+ Scenario: Main tag deleted
+ When loading osm data
+ """
+ n1 Tamenity=restaurant
+ n2 Thighway=bus_stop,railway=stop,name=X
+ n3 Tamenity=prison
+ """
+ Then place contains exactly
+ | object | class | type |
+ | N1 | amenity | restaurant |
+ | N2:highway | highway | bus_stop |
+ | N2:railway | railway | stop |
+ | N3 | amenity | prison |
+
+ When updating osm data
+ """
+ n1 Tnot_a=restaurant
+ n2 Thighway=bus_stop,name=X
+ """
+ Then place contains exactly
+ | object | class | type |
+ | N2:highway | highway | bus_stop |
+ | N3 | amenity | prison |
+ And placex contains
+ | object | indexed_status |
+ | N3:amenity | 0 |
+ When indexing
+ Then placex contains exactly
+ | object | type | name |
+ | N2:highway | bus_stop | 'name': 'X' |
+ | N3:amenity | prison | - |
+
+
+ Scenario: Main tag added
+ When loading osm data
+ """
+ n1 Tatity=restaurant
+ n2 Thighway=bus_stop,name=X
+ """
+ Then place contains exactly
+ | object | class | type |
+ | N2:highway | highway | bus_stop |
+
+ When updating osm data
+ """
+ n1 Tamenity=restaurant
+ n2 Thighway=bus_stop,railway=stop,name=X
+ """
+ Then place contains exactly
+ | object | class | type |
+ | N1 | amenity | restaurant |
+ | N2:highway | highway | bus_stop |
+ | N2:railway | railway | stop |
+ When indexing
+ Then placex contains exactly
+ | object | type | name |
+ | N1:amenity | restaurant | - |
+ | N2:highway | bus_stop | 'name': 'X' |
+ | N2:railway | stop | 'name': 'X' |
+
+
+ Scenario: Main tag modified
+ When loading osm data
+ """
+ n10 Thighway=footway,name=X
+ n11 Tamenity=atm
+ """
+ Then place contains exactly
+ | object | class | type |
+ | N10 | highway | footway |
+ | N11 | amenity | atm |
+
+ When updating osm data
+ """
+ n10 Thighway=path,name=X
+ n11 Thighway=primary
+ """
+ Then place contains exactly
+ | object | class | type |
+ | N10 | highway | path |
+ | N11 | highway | primary |
+ When indexing
+ Then placex contains exactly
+ | object | type | name |
+ | N10:highway | path | 'name': 'X' |
+ | N11:highway | primary | - |
+
+
+ Scenario: Main tags with name, name added
+ When loading osm data
+ """
+ n45 Tlanduse=cemetry
+ n46 Tbuilding=yes
+ """
+ Then place contains exactly
+ | object | class | type |
+
+ When updating osm data
+ """
+ n45 Tlanduse=cemetry,name=TODO
+ n46 Tbuilding=yes,addr:housenumber=1
+ """
+ Then place contains exactly
+ | object | class | type |
+ | N45 | landuse | cemetry |
+ | N46 | building| yes |
+ When indexing
+ Then placex contains exactly
+ | object | type | name | address |
+ | N45:landuse | cemetry | 'name': 'TODO' | - |
+ | N46:building| yes | - | 'housenumber': '1' |
+
+
+ Scenario: Main tags with name, name removed
+ When loading osm data
+ """
+ n45 Tlanduse=cemetry,name=TODO
+ n46 Tbuilding=yes,addr:housenumber=1
+ """
+ Then place contains exactly
+ | object | class | type |
+ | N45 | landuse | cemetry |
+ | N46 | building| yes |
+
+ When updating osm data
+ """
+ n45 Tlanduse=cemetry
+ n46 Tbuilding=yes
+ """
+ Then place contains exactly
+ | object | class | type |
+ When indexing
+ Then placex contains exactly
+ | object |
+
+ Scenario: Main tags with name, name modified
+ When loading osm data
+ """
+ n45 Tlanduse=cemetry,name=TODO
+ n46 Tbuilding=yes,addr:housenumber=1
+ """
+ Then place contains exactly
+ | object | class | type | name | address |
+ | N45 | landuse | cemetry | 'name' : 'TODO' | - |
+ | N46 | building| yes | - | 'housenumber': '1'|
+
+ When updating osm data
+ """
+ n45 Tlanduse=cemetry,name=DONE
+ n46 Tbuilding=yes,addr:housenumber=10
+ """
+ Then place contains exactly
+ | object | class | type | name | address |
+ | N45 | landuse | cemetry | 'name' : 'DONE' | - |
+ | N46 | building| yes | - | 'housenumber': '10'|
+ When indexing
+ Then placex contains exactly
+ | object | class | type | name | address |
+ | N45 | landuse | cemetry | 'name' : 'DONE' | - |
+ | N46 | building| yes | - | 'housenumber': '10'|
+
+
+ Scenario: Main tag added to address only node
+ When loading osm data
+ """
+ n1 Taddr:housenumber=345
+ """
+ Then place contains exactly
+ | object | class | type | address |
+ | N1 | place | house | 'housenumber': '345'|
+
+ When updating osm data
+ """
+ n1 Taddr:housenumber=345,building=yes
+ """
+ Then place contains exactly
+ | object | class | type | address |
+ | N1 | building | yes | 'housenumber': '345'|
+ When indexing
+ Then placex contains exactly
+ | object | class | type | address |
+ | N1 | building | yes | 'housenumber': '345'|
+
+
+ Scenario: Main tag removed from address only node
+ When loading osm data
+ """
+ n1 Taddr:housenumber=345,building=yes
+ """
+ Then place contains exactly
+ | object | class | type | address |
+ | N1 | building | yes | 'housenumber': '345'|
+
+ When updating osm data
+ """
+ n1 Taddr:housenumber=345
+ """
+ Then place contains exactly
+ | object | class | type | address |
+ | N1 | place | house | 'housenumber': '345'|
+ When indexing
+ Then placex contains exactly
+ | object | class | type | address |
+ | N1 | place | house | 'housenumber': '345'|
+
+
+ Scenario: Main tags with name key, adding key name
+ When loading osm data
+ """
+ n2 Tbridge=yes
+ """
+ Then place contains exactly
+ | object | class | type |
+
+ When updating osm data
+ """
+ n2 Tbridge=yes,bridge:name=high
+ """
+ Then place contains exactly
+ | object | class | type | name |
+ | N2 | bridge | yes | 'name': 'high' |
+ When indexing
+ Then placex contains exactly
+ | object | class | type | name |
+ | N2 | bridge | yes | 'name': 'high' |
+
+
+ Scenario: Main tags with name key, deleting key name
+ When loading osm data
+ """
+ n2 Tbridge=yes,bridge:name=high
+ """
+ Then place contains exactly
+ | object | class | type | name |
+ | N2 | bridge | yes | 'name': 'high' |
+
+ When updating osm data
+ """
+ n2 Tbridge=yes
+ """
+ Then place contains exactly
+ | object |
+ When indexing
+ Then placex contains exactly
+ | object |
+
+
+ Scenario: Main tags with name key, changing key name
+ When loading osm data
+ """
+ n2 Tbridge=yes,bridge:name=high
+ """
+ Then place contains exactly
+ | object | class | type | name |
+ | N2 | bridge | yes | 'name': 'high' |
+
+ When updating osm data
+ """
+ n2 Tbridge=yes,bridge:name:en=high
+ """
+ Then place contains exactly
+ | object | class | type | name |
+ | N2 | bridge | yes | 'name:en': 'high' |
+ When indexing
+ Then placex contains exactly
+ | object | class | type | name |
+ | N2 | bridge | yes | 'name:en': 'high' |
+
+
+ Scenario: Downgrading a highway to one that is dropped without name
+ When loading osm data
+ """
+ n100 x0 y0
+ n101 x0.0001 y0.0001
+ w1 Thighway=residential Nn100,n101
+ """
+ Then place contains exactly
+ | object |
+ | W1:highway |
+
+ When updating osm data
+ """
+ w1 Thighway=service Nn100,n101
+ """
+ Then place contains exactly
+ | object |
+ When indexing
+ Then placex contains exactly
+ | object |
+
+
+ Scenario: Upgrading a highway to one that is not dropped without name
+ When loading osm data
+ """
+ n100 x0 y0
+ n101 x0.0001 y0.0001
+ w1 Thighway=service Nn100,n101
+ """
+ Then place contains exactly
+ | object |
+
+ When updating osm data
+ """
+ w1 Thighway=unclassified Nn100,n101
+ """
+ Then place contains exactly
+ | object |
+ | W1:highway |
+ When indexing
+ Then placex contains exactly
+ | object |
+ | W1:highway |
+
+
+ Scenario: Downgrading a highway when a second tag is present
+ When loading osm data
+ """
+ n100 x0 y0
+ n101 x0.0001 y0.0001
+ w1 Thighway=residential,tourism=hotel Nn100,n101
+ """
+ Then place contains exactly
+ | object | type |
+ | W1:highway | residential |
+ | W1:tourism | hotel |
+
+ When updating osm data
+ """
+ w1 Thighway=service,tourism=hotel Nn100,n101
+ """
+ Then place contains exactly
+ | object | type |
+ | W1:tourism | hotel |
+ When indexing
+ Then placex contains exactly
+ | object | type |
+ | W1:tourism | hotel |
+
+
+ Scenario: Upgrading a highway when a second tag is present
+ When loading osm data
+ """
+ n100 x0 y0
+ n101 x0.0001 y0.0001
+ w1 Thighway=service,tourism=hotel Nn100,n101
+ """
+ Then place contains exactly
+ | object | type |
+ | W1:tourism | hotel |
+
+ When updating osm data
+ """
+ w1 Thighway=residential,tourism=hotel Nn100,n101
+ """
+ Then place contains exactly
+ | object | type |
+ | W1:highway | residential |
+ | W1:tourism | hotel |
+ When indexing
+ Then placex contains exactly
+ | object | type |
+ | W1:highway | residential |
+ | W1:tourism | hotel |
+
+
+ Scenario: Replay on administrative boundary
+ When loading osm data
+ """
+ n10 x34.0 y-4.23
+ n11 x34.1 y-4.23
+ n12 x34.2 y-4.13
+ w10 Tboundary=administrative,waterway=river,name=Border,admin_level=2 Nn12,n11,n10
+ """
+ Then place contains exactly
+ | object | type | admin_level | name |
+ | W10:waterway | river | 2 | 'name': 'Border' |
+ | W10:boundary | administrative | 2 | 'name': 'Border' |
+
+ When updating osm data
+ """
+ w10 Tboundary=administrative,waterway=river,name=Border,admin_level=2 Nn12,n11,n10
+ """
+ Then place contains exactly
+ | object | type | admin_level | name |
+ | W10:waterway | river | 2 | 'name': 'Border' |
+ | W10:boundary | administrative | 2 | 'name': 'Border' |
+ When indexing
+ Then placex contains exactly
+ | object | type | admin_level | name |
+ | W10:waterway | river | 2 | 'name': 'Border' |
+
+
+ Scenario: Change admin_level on administrative boundary
+ Given the grid
+ | 10 | 11 |
+ | 13 | 12 |
+ When loading osm data
+ """
+ n10
+ n11
+ n12
+ n13
+ w10 Nn10,n11,n12,n13,n10
+ r10 Ttype=multipolygon,boundary=administrative,name=Border,admin_level=2 Mw10@
+ """
+ Then place contains exactly
+ | object | admin_level |
+ | R10:boundary | 2 |
+
+ When updating osm data
+ """
+ r10 Ttype=multipolygon,boundary=administrative,name=Border,admin_level=4 Mw10@
+ """
+ Then place contains exactly
+ | object | type | admin_level |
+ | R10:boundary | administrative | 4 |
+ When indexing
+ Then placex contains exactly
+ | object | type | admin_level |
+ | R10:boundary | administrative | 4 |
+
+
+ Scenario: Change boundary to administrative
+ Given the grid
+ | 10 | 11 |
+ | 13 | 12 |
+ When loading osm data
+ """
+ n10
+ n11
+ n12
+ n13
+ w10 Nn10,n11,n12,n13,n10
+ r10 Ttype=multipolygon,boundary=informal,name=Border,admin_level=4 Mw10@
+ """
+ Then place contains exactly
+ | object | type | admin_level |
+ | R10:boundary | informal | 4 |
+
+ When updating osm data
+ """
+ r10 Ttype=multipolygon,boundary=administrative,name=Border,admin_level=4 Mw10@
+ """
+ Then place contains exactly
+ | object | type | admin_level |
+ | R10:boundary | administrative | 4 |
+ When indexing
+ Then placex contains exactly
+ | object | type | admin_level |
+ | R10:boundary | administrative | 4 |
+
+
+ Scenario: Change boundary away from administrative
+ Given the grid
+ | 10 | 11 |
+ | 13 | 12 |
+ When loading osm data
+ """
+ n10
+ n11
+ n12
+ n13
+ w10 Nn10,n11,n12,n13,n10
+ r10 Ttype=multipolygon,boundary=administrative,name=Border,admin_level=4 Mw10@
+ """
+ Then place contains exactly
+ | object | type | admin_level |
+ | R10:boundary | administrative | 4 |
+
+ When updating osm data
+ """
+ r10 Ttype=multipolygon,boundary=informal,name=Border,admin_level=4 Mw10@
+ """
+ Then place contains exactly
+ | object | type | admin_level |
+ | R10:boundary | informal | 4 |
+ When indexing
+ Then placex contains exactly
+ | object | type | admin_level |
+ | R10:boundary | informal | 4 |
self.api_test_db = config['API_TEST_DB']
self.api_test_file = config['API_TEST_FILE']
self.tokenizer = config['TOKENIZER']
+ self.import_style = config['STYLE']
self.server_module_path = config['SERVER_MODULE_PATH']
self.reuse_template = not config['REMOVE_TEMPLATE']
self.keep_scenario_db = config['KEEP_TEST_DB']
self.test_env['NOMINATIM_NOMINATIM_TOOL'] = str((self.build_dir / 'nominatim').resolve())
if self.tokenizer is not None:
self.test_env['NOMINATIM_TOKENIZER'] = self.tokenizer
+ if self.import_style is not None:
+ self.test_env['NOMINATIM_IMPORT_STYLE'] = self.import_style
if self.server_module_path:
self.test_env['NOMINATIM_DATABASE_MODULE_PATH'] = self.server_module_path
else:
self.columns[column] = {key: value}
+ def db_delete(self, cursor):
+ """ Issue a delete for the given OSM object.
+ """
+ cursor.execute('DELETE FROM place WHERE osm_type = %s and osm_id = %s',
+ (self.columns['osm_type'] , self.columns['osm_id']))
+
def db_insert(self, cursor):
""" Insert the collected data into the database.
"""
context.nominatim.run_nominatim('refresh', '--functions')
with context.db.cursor() as cur:
for row in context.table:
- PlaceColumn(context).add_row(row, False).db_insert(cur)
+ col = PlaceColumn(context).add_row(row, False)
+ col.db_delete(cur)
+ col.db_insert(cur)
+ cur.execute('SELECT flush_deleted_places()')
context.nominatim.reindex_placex(context.db)
check_database_integrity(context)
"""
context.nominatim.run_nominatim('refresh', '--functions')
with context.db.cursor() as cur:
+ cur.execute('TRUNCATE place_to_be_deleted')
for oid in oids.split(','):
NominatimID(oid).query_osm_id(cur, 'DELETE FROM place WHERE {}')
+ cur.execute('SELECT flush_deleted_places()')
context.nominatim.reindex_placex(context.db)
if exact:
cur.execute('SELECT osm_type, osm_id, class from {}'.format(table))
- assert expected_content == set([(r[0], r[1], r[2]) for r in cur])
+ actual = set([(r[0], r[1], r[2]) for r in cur])
+ assert expected_content == actual, \
+ f"Missing entries: {expected_content - actual}\n" \
+ f"Not expected in table: {actual - expected_content}"
@then("(?P<table>placex|place) has no entry for (?P<oid>.*)")
assert not todo, f"Unmatched lines in table: {list(context.table[i] for i in todo)}"
+@then("location_property_osmline contains(?P<exact> exactly)?")
+def check_place_contents(context, exact):
+ """ Check contents of the interpolation table. Each row represents a table row
+ and all data must match. Data not present in the expected table, may
+ be arbitry. The rows are identified via the 'object' column which must
+ have an identifier of the form '<osm id>[:<startnumber>]'. When multiple
+ rows match (for example because 'startnumber' was left out and there are
+ multiple entries for the given OSM object) then all must match. All
+ expected rows are expected to be present with at least one database row.
+ When 'exactly' is given, there must not be additional rows in the database.
+ """
+ with context.db.cursor(cursor_factory=psycopg2.extras.DictCursor) as cur:
+ expected_content = set()
+ for row in context.table:
+ if ':' in row['object']:
+ nid, start = row['object'].split(':', 2)
+ start = int(start)
+ else:
+ nid, start = row['object'], None
+
+ query = """SELECT *, ST_AsText(linegeo) as geomtxt,
+ ST_GeometryType(linegeo) as geometrytype
+ FROM location_property_osmline WHERE osm_id=%s"""
+
+ if ':' in row['object']:
+ query += ' and startnumber = %s'
+ params = [int(val) for val in row['object'].split(':', 2)]
+ else:
+ params = (int(row['object']), )
+
+ cur.execute(query, params)
+ assert cur.rowcount > 0, "No rows found for " + row['object']
+
+ for res in cur:
+ if exact:
+ expected_content.add((res['osm_id'], res['startnumber']))
+
+ DBRow(nid, res, context).assert_row(row, ['object'])
+
+ if exact:
+ cur.execute('SELECT osm_id, startnumber from location_property_osmline')
+ actual = set([(r[0], r[1]) for r in cur])
+ assert expected_content == actual, \
+ f"Missing entries: {expected_content - actual}\n" \
+ f"Not expected in table: {actual - expected_content}"
from pathlib import Path
from nominatim.tools.exec_utils import run_osm2pgsql
+from nominatim.tools.replication import run_osm2pgsql_updates
from geometry_alias import ALIASES
return dict(import_file=fname,
osm2pgsql=str(nominatim_env.build_dir / 'osm2pgsql' / 'osm2pgsql'),
osm2pgsql_cache=50,
- osm2pgsql_style=str(nominatim_env.src_dir / 'settings' / 'import-extratags.style'),
+ osm2pgsql_style=str(nominatim_env.get_test_config().get_import_style_file()),
+ osm2pgsql_style_path=nominatim_env.get_test_config().config_dir,
threads=1,
dsn=nominatim_env.get_libpq_dsn(),
flatnode_file='',
# create an OSM file and import it
fname = write_opl_file(context.text, context.osm)
try:
- run_osm2pgsql(get_osm2pgsql_options(context.nominatim, fname, append=True))
+ run_osm2pgsql_updates(context.db,
+ get_osm2pgsql_options(context.nominatim, fname, append=True))
finally:
os.remove(fname)
+
+@when('indexing')
+def index_database(context):
+ """
+ Run the Nominatim indexing step. This will process data previously
+ loaded with 'updating osm data'
+ """
+ context.nominatim.run_nominatim('index')
sudo apt install -y php-cgi
sudo apt install -y build-essential cmake g++ libboost-dev libboost-system-dev \
libboost-filesystem-dev libexpat1-dev zlib1g-dev\
- libbz2-dev libpq-dev \
+ libbz2-dev libpq-dev liblua5.3-dev lua5.3\
postgresql-10-postgis-2.4 \
postgresql-contrib-10 postgresql-10-postgis-scripts \
php-cli php-pgsql php-intl libicu-dev python3-pip \
sudo apt install -y php-cgi
sudo apt install -y build-essential cmake g++ libboost-dev libboost-system-dev \
libboost-filesystem-dev libexpat1-dev zlib1g-dev \
- libbz2-dev libpq-dev \
+ libbz2-dev libpq-dev liblua5.3-dev lua5.3 \
postgresql-12-postgis-3 \
postgresql-contrib-12 postgresql-12-postgis-3-scripts \
php-cli php-pgsql php-intl libicu-dev python3-dotenv \
sudo apt install -y php-cgi
sudo apt install -y build-essential cmake g++ libboost-dev libboost-system-dev \
libboost-filesystem-dev libexpat1-dev zlib1g-dev \
- libbz2-dev libpq-dev \
+ libbz2-dev libpq-dev liblua5.3-dev lua5.3 \
postgresql-server-dev-14 postgresql-14-postgis-3 \
postgresql-contrib-14 postgresql-14-postgis-3-scripts \
php-cli php-pgsql php-intl libicu-dev python3-dotenv \
projects / nominatim.git / commitdiff