X-Git-Url: https://git.openstreetmap.org./nominatim.git/blobdiff_plain/51ed55cc32580644544b8e38c570bbfdaf09b5a2..253dc7d7cd6cae88b8f2d777c1b2996390b37e54:/settings/flex-base.lua?ds=sidebyside diff --git a/settings/flex-base.lua b/settings/flex-base.lua index d91299ad..4d960d72 100644 --- a/settings/flex-base.lua +++ b/settings/flex-base.lua @@ -1,9 +1,25 @@ -- Core functions for Nominatim import flex style. -- +local module = {} + +local PRE_DELETE = nil +local PRE_EXTRAS = nil +local POST_DELETE = nil +local MAIN_KEYS = nil +local NAMES = nil +local ADDRESS_TAGS = nil +local SAVE_EXTRA_MAINS = false +local POSTCODE_FALLBACK = true + +-- tables required for taginfo +module.TAGINFO_MAIN = {keys = {}, delete_tags = {}} +module.TAGINFO_NAME_KEYS = {} +module.TAGINFO_ADDRESS_KEYS = {} + -- The single place table. -place_table = osm2pgsql.define_table{ +local place_table = osm2pgsql.define_table{ name = "place", ids = { type = 'any', id_column = 'osm_id', type_column = 'osm_type' }, columns = { @@ -14,7 +30,27 @@ place_table = osm2pgsql.define_table{ { column = 'address', type = 'hstore' }, { column = 'extratags', type = 'hstore' }, { column = 'geometry', type = 'geometry', projection = 'WGS84', not_null = true }, - } + }, + data_tablespace = os.getenv("NOMINATIM_TABLESPACE_PLACE_DATA"), + index_tablespace = os.getenv("NOMINATIM_TABLESPACE_PLACE_INDEX"), + indexes = {} +} + +------------ Geometry functions for relations --------------------- + +function module.relation_as_multipolygon(o) + return o:as_multipolygon() +end + +function module.relation_as_multiline(o) + return o:as_multilinestring():line_merge() +end + + +module.RELATION_TYPES = { + multipolygon = module.relation_as_multipolygon, + boundary = module.relation_as_multipolygon, + waterway = module.relation_as_multiline } ------------- Place class ------------------------------------------ @@ -43,6 +79,17 @@ function Place.new(object, geom_func) return self end +function Place:clean(data) + for k, v in pairs(self.object.tags) do + if data.delete ~= nil and data.delete(k, v) then + self.object.tags[k] = nil + elseif data.extra ~= nil and data.extra(k, v) then + self.extratags[k] = v + self.object.tags[k] = nil + end + end +end + function Place:delete(data) if data.match ~= nil then for k, v in pairs(self.object.tags) do @@ -69,31 +116,37 @@ function Place:grab_extratags(data) return count end -function Place:grab_address(data) +local function strip_address_prefix(k) + if k:sub(1, 5) == 'addr:' then + return k:sub(6) + end + + if k:sub(1, 6) == 'is_in:' then + return k:sub(7) + end + + return k +end + + +function Place:grab_address_parts(data) local count = 0 - if data.match ~= nil then + if data.groups ~= nil then for k, v in pairs(self.object.tags) do - if data.match(k, v) then - self.object.tags[k] = nil + local atype = data.groups(k, v) - if data.include_on_name == true then + if atype ~= nil then + if atype == 'main' then self.has_name = true - end - - if data.out_key ~= nil then - self.address[data.out_key] = v - return 1 - end - - if k:sub(1, 5) == 'addr:' then - self.address[k:sub(6)] = v - elseif k:sub(1, 6) == 'is_in:' then - self.address[k:sub(7)] = v + self.address[strip_address_prefix(k)] = v + count = count + 1 + elseif atype == 'extra' then + self.address[strip_address_prefix(k)] = v else - self.address[k] = v + self.address[atype] = v end - count = count + 1 + self.object.tags[k] = nil end end end @@ -101,36 +154,30 @@ function Place:grab_address(data) return count end -function Place:set_address(key, value) - self.address[key] = value -end -function Place:grab_name(data) - local count = 0 +function Place:grab_name_parts(data) + local fallback = nil - if data.match ~= nil then + if data.groups ~= nil then for k, v in pairs(self.object.tags) do - if data.match(k, v) then - self.object.tags[k] = nil + local atype = data.groups(k, v) + + if atype ~= nil then self.names[k] = v - if data.include_on_name ~= false then + self.object.tags[k] = nil + if atype == 'main' then + self.has_name = true + elseif atype == 'house' then self.has_name = true + fallback = {'place', 'house', 'always'} end - count = count + 1 end end end - return count -end - -function Place:grab_tag(key) - return self.object:grab_tag(key) + return fallback end -function Place:tags() - return self.object.tags -end function Place:write_place(k, v, mtype, save_extra_mains) if mtype == nil then @@ -184,9 +231,9 @@ function Place:write_row(k, v, save_extra_mains) return 0 end - if save_extra_mains then + if save_extra_mains ~= nil then for extra_k, extra_v in pairs(self.object.tags) do - if extra_k ~= k then + if extra_k ~= k and save_extra_mains(extra_k, extra_v) then self.extratags[extra_k] = extra_v end end @@ -203,8 +250,10 @@ function Place:write_row(k, v, save_extra_mains) } if save_extra_mains then - for k, v in pairs(self.object.tags) do - self.extratags[k] = nil + for tk, tv in pairs(self.object.tags) do + if save_extra_mains(tk, tv) then + self.extratags[tk] = nil + end end end @@ -214,70 +263,145 @@ function Place:write_row(k, v, save_extra_mains) end -function tag_match(data) +function module.tag_match(data) if data == nil or next(data) == nil then return nil end - local tests = {} + local fullmatches = {} + local key_prefixes = {} + local key_suffixes = {} if data.keys ~= nil then for _, key in pairs(data.keys) do if key:sub(1, 1) == '*' then if #key > 1 then - local suffix = key:sub(2) - tests[#tests + 1] = function (k, v) - return k:sub(-#suffix) == suffix + if key_suffixes[#key - 1] == nil then + key_suffixes[#key - 1] = {} end + key_suffixes[#key - 1][key:sub(2)] = true end elseif key:sub(#key, #key) == '*' then - local prefix = key:sub(1, #key - 1) - tests[#tests + 1] = function (k, v) - return k:sub(1, #prefix) == prefix + if key_prefixes[#key - 1] == nil then + key_prefixes[#key - 1] = {} end + key_prefixes[#key - 1][key:sub(1, #key - 1)] = true else - tests[#tests + 1] = function (k, v) - return k == key - end + fullmatches[key] = true end end end if data.tags ~= nil then - local tags = {} for k, vlist in pairs(data.tags) do - tags[k] = {} - for _, v in pairs(vlist) do - tags[k][v] = true + if fullmatches[k] == nil then + fullmatches[k] = {} + for _, v in pairs(vlist) do + fullmatches[k][v] = true + end end end - tests[#tests + 1] = function (k, v) - return tags[k] ~= nil and tags[k][v] ~= nil - end end return function (k, v) - for _, func in pairs(tests) do - if func(k, v) then + if fullmatches[k] ~= nil and (fullmatches[k] == true or fullmatches[k][v] ~= nil) then + return true + end + + for slen, slist in pairs(key_suffixes) do + if #k >= slen and slist[k:sub(-slen)] ~= nil then + return true + end + end + + for slen, slist in pairs(key_prefixes) do + if #k >= slen and slist[k:sub(1, slen)] ~= nil then return true end end + return false end end +function module.tag_group(data) + if data == nil or next(data) == nil then + return nil + end + + local fullmatches = {} + local key_prefixes = {} + local key_suffixes = {} + + for group, tags in pairs(data) do + for _, key in pairs(tags) do + if key:sub(1, 1) == '*' then + if #key > 1 then + if key_suffixes[#key - 1] == nil then + key_suffixes[#key - 1] = {} + end + key_suffixes[#key - 1][key:sub(2)] = group + end + elseif key:sub(#key, #key) == '*' then + if key_prefixes[#key - 1] == nil then + key_prefixes[#key - 1] = {} + end + key_prefixes[#key - 1][key:sub(1, #key - 1)] = group + else + fullmatches[key] = group + end + end + end + + return function (k, v) + local val = fullmatches[k] + if val ~= nil then + return val + end + + for slen, slist in pairs(key_suffixes) do + if #k >= slen then + val = slist[k:sub(-slen)] + if val ~= nil then + return val + end + end + end + + for slen, slist in pairs(key_prefixes) do + if #k >= slen then + val = slist[k:sub(1, slen)] + if val ~= nil then + return val + end + end + end + end +end + +-- Returns prefix part of the keys, and reject suffix matching keys +local function process_key(key) + if key:sub(1, 1) == '*' then + return nil + end + if key:sub(#key, #key) == '*' then + return key:sub(1, #key - 2) + end + return key +end + -- Process functions for all data types -function osm2pgsql.process_node(object) +function module.process_node(object) local function geom_func(o) return o:as_point() end - process_tags(Place.new(object, geom_func)) + module.process_tags(Place.new(object, geom_func)) end -function osm2pgsql.process_way(object) +function module.process_way(object) local function geom_func(o) local geom = o:as_polygon() @@ -289,30 +413,24 @@ function osm2pgsql.process_way(object) return geom end - process_tags(Place.new(object, geom_func)) + module.process_tags(Place.new(object, geom_func)) end -function relation_as_multipolygon(o) - return o:as_multipolygon() -end - -function relation_as_multiline(o) - return o:as_multilinestring():line_merge() -end - -function osm2pgsql.process_relation(object) - local geom_func = RELATION_TYPES[object.tags.type] +function module.process_relation(object) + local geom_func = module.RELATION_TYPES[object.tags.type] if geom_func ~= nil then - process_tags(Place.new(object, geom_func)) + module.process_tags(Place.new(object, geom_func)) end end -function process_tags(o) - local fallback +-- The process functions are used by default by osm2pgsql. +osm2pgsql.process_node = module.process_node +osm2pgsql.process_way = module.process_way +osm2pgsql.process_relation = module.process_relation - o:delete{match = PRE_DELETE} - o:grab_extratags{match = PRE_EXTRAS} +function module.process_tags(o) + o:clean{delete = PRE_DELETE, extra = PRE_EXTRAS} -- Exception for boundary/place double tagging if o.object.tags.boundary == 'administrative' then @@ -321,62 +439,116 @@ function process_tags(o) end} end + -- name keys + local fallback = o:grab_name_parts{groups=NAMES} + -- address keys - o:grab_address{match=function (k, v) return COUNTRY_TAGS(k, v) and #v == 2 end, - out_key='country'} - if o:grab_name{match=HOUSENAME_TAGS} > 0 then - fallback = {'place', 'house'} + if o:grab_address_parts{groups=ADDRESS_TAGS} > 0 and fallback == nil then + fallback = {'place', 'house', 'always'} end - if o:grab_address{match=HOUSENUMBER_TAGS, include_on_name = true} > 0 and fallback == nil then - fallback = {'place', 'house'} + if o.address.country ~= nil and #o.address.country ~= 2 then + o.address['country'] = nil end - if o:grab_address{match=POSTCODES, out_key='postcode'} > 0 and fallback == nil then - fallback = {'place', 'postcode'} + if POSTCODE_FALLBACK and fallback == nil and o.address.postcode ~= nil then + fallback = {'place', 'postcode', 'always'} + end + + if o.address.interpolation ~= nil then + o:write_place('place', 'houses', 'always', SAVE_EXTRA_MAINS) + return end - local is_interpolation = o:grab_address{match=INTERPOLATION_TAGS} > 0 + o:clean{delete = POST_DELETE} - if ADD_TIGER_COUNTY then - local v = o:grab_tag('tiger:county') - if v ~= nil then - v, num = v:gsub(',.*', ' county') - if num == 0 then - v = v .. ' county' + -- collect main keys + for k, v in pairs(o.object.tags) do + local ktype = MAIN_KEYS[k] + if ktype == 'fallback' then + if o.has_name then + fallback = {k, v, 'named'} end - o:set_address('tiger:county', v) + elseif ktype ~= nil then + o:write_place(k, v, MAIN_KEYS[k], SAVE_EXTRA_MAINS) end end - o:grab_address{match=ADDRESS_TAGS} - if is_interpolation then - o:write_place('place', 'houses', 'always', SAVE_EXTRA_MAINS) - return + if fallback ~= nil and o.num_entries == 0 then + o:write_place(fallback[1], fallback[2], fallback[3], SAVE_EXTRA_MAINS) end +end - -- name keys - o:grab_name{match = NAMES} - o:grab_name{match = REFS, include_on_name = false} +--------- Convenience functions for simple style configuration ----------------- - o:delete{match = POST_DELETE} - o:grab_extratags{match = POST_EXTRAS} - -- collect main keys - local num_mains = 0 - for k, v in pairs(o:tags()) do - num_mains = num_mains + o:write_place(k, v, MAIN_KEYS[k], SAVE_EXTRA_MAINS) +function module.set_prefilters(data) + PRE_DELETE = module.tag_match{keys = data.delete_keys, tags = data.delete_tags} + PRE_EXTRAS = module.tag_match{keys = data.extra_keys, + tags = data.extra_tags} + module.TAGINFO_MAIN.delete_tags = data.delete_tags +end + +function module.set_main_tags(data) + MAIN_KEYS = data + local keys = {} + for k, _ in pairs(data) do + table.insert(keys, k) end + module.TAGINFO_MAIN.keys = keys +end + +function module.set_name_tags(data) + NAMES = module.tag_group(data) - if num_mains == 0 then - for tag, mtype in pairs(MAIN_FALLBACK_KEYS) do - if o:write_place(tag, nil, mtype, SAVE_EXTRA_MAINS) > 0 then - return + for _, lst in pairs(data) do + for _, k in ipairs(lst) do + local key = process_key(k) + if key ~= nil then + module.TAGINFO_NAME_KEYS[key] = true end end + end +end - if fallback ~= nil then - o:write_place(fallback[1], fallback[2], 'always', SAVE_EXTRA_MAINS) +function module.set_address_tags(data) + if data.postcode_fallback ~= nil then + POSTCODE_FALLBACK = data.postcode_fallback + data.postcode_fallback = nil + end + ADDRESS_TAGS = module.tag_group(data) + + for _, lst in pairs(data) do + if lst ~= nil then + for _, k in ipairs(lst) do + local key = process_key(k) + if key ~= nil then + module.TAGINFO_ADDRESS_KEYS[key] = true + end + end end end end +function module.set_unused_handling(data) + if data.extra_keys == nil and data.extra_tags == nil then + POST_DELETE = module.tag_match{keys = data.delete_keys, tags = data.delete_tags} + SAVE_EXTRA_MAINS = function() return true end + elseif data.delete_keys == nil and data.delete_tags == nil then + POST_DELETE = nil + SAVE_EXTRA_MAINS = module.tag_match{keys = data.extra_keys, tags = data.extra_tags} + else + error("unused handler can have only 'extra_keys' or 'delete_keys' set.") + end +end + +function module.set_relation_types(data) + module.RELATION_TYPES = {} + for k, v in data do + if v == 'multipolygon' then + module.RELATION_TYPES[k] = module.relation_as_multipolygon + elseif v == 'multiline' then + module.RELATION_TYPES[k] = module.relation_as_multiline + end + end +end +return module