also switch unit tests for cli

[nominatim.git] / settings / flex-base.lua

diff --git a/settings/flex-base.lua b/settings/flex-base.lua
index fe3ce32ae6d85cd18fe0217900ba324da58c12e8..dc2c12eef1f2d4bb4f2c67b075376ebcd2f4323c 100644 (file)
--- a/settings/flex-base.lua
+++ b/settings/flex-base.lua
@@ -1,9 +1,24 @@
 -- Core functions for Nominatim import flex style.
 --
 
 -- Core functions for Nominatim import flex style.
 --
 

+local module = {}
+
+local PRE_DELETE = nil
+local PRE_EXTRAS = nil
+local MAIN_KEYS = nil
+local NAMES = nil
+local ADDRESS_TAGS = nil
+local SAVE_EXTRA_MAINS = false
+local POSTCODE_FALLBACK = true
+
+-- tables required for taginfo
+module.TAGINFO_MAIN = {keys = {}, delete_tags = {}}
+module.TAGINFO_NAME_KEYS = {}
+module.TAGINFO_ADDRESS_KEYS = {}
+

 
 -- The single place table.
 
 -- The single place table.

-place_table = osm2pgsql.define_table{
+local place_table = osm2pgsql.define_table{

     name = "place",
     ids = { type = 'any', id_column = 'osm_id', type_column = 'osm_type' },
     columns = {
     name = "place",
     ids = { type = 'any', id_column = 'osm_id', type_column = 'osm_type' },
     columns = {


@@ -14,7 +29,27 @@ place_table = osm2pgsql.define_table{
         { column = 'address', type = 'hstore' },
         { column = 'extratags', type = 'hstore' },
         { column = 'geometry', type = 'geometry', projection = 'WGS84', not_null = true },
         { column = 'address', type = 'hstore' },
         { column = 'extratags', type = 'hstore' },
         { column = 'geometry', type = 'geometry', projection = 'WGS84', not_null = true },

-    }
+    },
+    data_tablespace = os.getenv("NOMINATIM_TABLESPACE_PLACE_DATA"),
+    index_tablespace = os.getenv("NOMINATIM_TABLESPACE_PLACE_INDEX"),
+    indexes = {}
+}
+
+------------ Geometry functions for relations ---------------------
+
+function module.relation_as_multipolygon(o)
+    return o:as_multipolygon()
+end
+
+function module.relation_as_multiline(o)
+    return o:as_multilinestring():line_merge()
+end
+
+
+module.RELATION_TYPES = {
+    multipolygon = module.relation_as_multipolygon,
+    boundary = module.relation_as_multipolygon,
+    waterway = module.relation_as_multiline

 }
 
 ------------- Place class ------------------------------------------
 }
 
 ------------- Place class ------------------------------------------


@@ -43,6 +78,17 @@ function Place.new(object, geom_func)
     return self
 end
 
     return self
 end
 

+function Place:clean(data)
+    for k, v in pairs(self.object.tags) do
+        if data.delete ~= nil and data.delete(k, v) then
+            self.object.tags[k] = nil
+        elseif data.extra ~= nil and data.extra(k, v) then
+            self.extratags[k] = v
+            self.object.tags[k] = nil
+        end
+    end
+end
+

 function Place:delete(data)
     if data.match ~= nil then
         for k, v in pairs(self.object.tags) do
 function Place:delete(data)
     if data.match ~= nil then
         for k, v in pairs(self.object.tags) do


@@ -69,31 +115,37 @@ function Place:grab_extratags(data)
     return count
 end
 
     return count
 end
 

-function Place:grab_address(data)
+local function strip_address_prefix(k)
+    if k:sub(1, 5) == 'addr:' then
+        return k:sub(6)
+    end
+
+    if k:sub(1, 6) == 'is_in:' then
+        return k:sub(7)
+    end
+
+    return k
+end
+
+
+function Place:grab_address_parts(data)

     local count = 0
 
     local count = 0
 

-    if data.match ~= nil then
+    if data.groups ~= nil then

         for k, v in pairs(self.object.tags) do
         for k, v in pairs(self.object.tags) do

-            if data.match(k, v) then
-                self.object.tags[k] = nil
+            local atype = data.groups(k, v)

 
 

-                if data.include_on_name == true then
+            if atype ~= nil then
+                if atype == 'main' then

                     self.has_name = true
                     self.has_name = true

-                end
-
-                if data.out_key ~= nil then
-                    self.address[data.out_key] = v
-                    return 1
-                end
-
-                if k:sub(1, 5) == 'addr:' then
-                    self.address[k:sub(6)] = v
-                elseif k:sub(1, 6) == 'is_in:' then
-                    self.address[k:sub(7)] = v
+                    self.address[strip_address_prefix(k)] = v
+                    count = count + 1
+                elseif atype == 'extra' then
+                    self.address[strip_address_prefix(k)] = v

                 else
                 else

-                    self.address[k] = v
+                    self.address[atype] = v

                 end
                 end

-                count = count + 1
+                self.object.tags[k] = nil

             end
         end
     end
             end
         end
     end


@@ -101,36 +153,30 @@ function Place:grab_address(data)
     return count
 end
 
     return count
 end
 

-function Place:set_address(key, value)
-    self.address[key] = value
-end

 
 

-function Place:grab_name(data)
-    local count = 0
+function Place:grab_name_parts(data)
+    local fallback = nil

 
 

-    if data.match ~= nil then
+    if data.groups ~= nil then

         for k, v in pairs(self.object.tags) do
         for k, v in pairs(self.object.tags) do

-            if data.match(k, v) then
-                self.object.tags[k] = nil
+            local atype = data.groups(k, v)
+
+            if atype ~= nil then

                 self.names[k] = v
                 self.names[k] = v

-                if data.include_on_name ~= false then
+                self.object.tags[k] = nil
+                if atype == 'main' then

                     self.has_name = true
                     self.has_name = true

+                elseif atype == 'house' then
+                    self.has_name = true
+                    fallback = {'place', 'house', 'always'}

                 end
                 end

-                count = count + 1

             end
         end
     end
 
             end
         end
     end
 

-    return count
-end
-
-function Place:grab_tag(key)
-    return self.object:grab_tag(key)
+    return fallback

 end
 
 end
 

-function Place:tags()
-    return self.object.tags
-end

 
 function Place:write_place(k, v, mtype, save_extra_mains)
     if mtype == nil then
 
 function Place:write_place(k, v, mtype, save_extra_mains)
     if mtype == nil then


@@ -184,9 +230,9 @@ function Place:write_row(k, v, save_extra_mains)
         return 0
     end
 
         return 0
     end
 

-    if save_extra_mains then
+    if save_extra_mains ~= nil then

         for extra_k, extra_v in pairs(self.object.tags) do
         for extra_k, extra_v in pairs(self.object.tags) do

-            if extra_k ~= k then
+            if extra_k ~= k and save_extra_mains(extra_k, extra_v) then

                 self.extratags[extra_k] = extra_v
             end
         end
                 self.extratags[extra_k] = extra_v
             end
         end


@@ -204,7 +250,9 @@ function Place:write_row(k, v, save_extra_mains)
 
     if save_extra_mains then
         for k, v in pairs(self.object.tags) do
 
     if save_extra_mains then
         for k, v in pairs(self.object.tags) do

-            self.extratags[k] = nil
+            if save_extra_mains(k, v) then
+                self.extratags[k] = nil
+            end

         end
     end
 
         end
     end
 


@@ -214,7 +262,7 @@ function Place:write_row(k, v, save_extra_mains)
 end
 
 
 end
 
 

-function tag_match(data)
+function module.tag_match(data)

     if data == nil or next(data) == nil then
         return nil
     end
     if data == nil or next(data) == nil then
         return nil
     end


@@ -276,17 +324,83 @@ function tag_match(data)
 end
 
 
 end
 
 

+function module.tag_group(data)
+    if data == nil or next(data) == nil then
+        return nil
+    end
+
+    local fullmatches = {}
+    local key_prefixes = {}
+    local key_suffixes = {}
+
+    for group, tags in pairs(data) do
+        for _, key in pairs(tags) do
+            if key:sub(1, 1) == '*' then
+                if #key > 1 then
+                    if key_suffixes[#key - 1] == nil then
+                        key_suffixes[#key - 1] = {}
+                    end
+                    key_suffixes[#key - 1][key:sub(2)] = group
+                end
+            elseif key:sub(#key, #key) == '*' then
+                if key_prefixes[#key - 1] == nil then
+                    key_prefixes[#key - 1] = {}
+                end
+                key_prefixes[#key - 1][key:sub(1, #key - 1)] = group
+            else
+                fullmatches[key] = group
+            end
+        end
+    end
+
+    return function (k, v)
+        local val = fullmatches[k]
+        if val ~= nil then
+            return val
+        end
+
+        for slen, slist in pairs(key_suffixes) do
+            if #k >= slen then
+                val = slist[k:sub(-slen)]
+                if val ~= nil then
+                    return val
+                end
+            end
+        end
+
+        for slen, slist in pairs(key_prefixes) do
+            if #k >= slen then
+                val = slist[k:sub(1, slen)]
+                if val ~= nil then
+                    return val
+                end
+            end
+        end
+    end
+end
+
+-- Returns prefix part of the keys, and reject suffix matching keys
+local function process_key(key)
+    if key:sub(1, 1) == '*' then
+        return nil
+    end
+    if key:sub(#key, #key) == '*' then
+        return key:sub(1, #key - 2)
+    end
+    return key
+end
+

 -- Process functions for all data types
 -- Process functions for all data types

-function osm2pgsql.process_node(object)
+function module.process_node(object)

 
     local function geom_func(o)
         return o:as_point()
     end
 
 
     local function geom_func(o)
         return o:as_point()
     end
 

-    process_tags(Place.new(object, geom_func))
+    module.process_tags(Place.new(object, geom_func))

 end
 
 end
 

-function osm2pgsql.process_way(object)
+function module.process_way(object)

 
     local function geom_func(o)
         local geom = o:as_polygon()
 
     local function geom_func(o)
         local geom = o:as_polygon()


@@ -298,30 +412,24 @@ function osm2pgsql.process_way(object)
         return geom
     end
 
         return geom
     end
 

-    process_tags(Place.new(object, geom_func))
-end
-
-function relation_as_multipolygon(o)
-    return o:as_multipolygon()
-end
-
-function relation_as_multiline(o)
-    return o:as_multilinestring():line_merge()
+    module.process_tags(Place.new(object, geom_func))

 end
 
 end
 

-function osm2pgsql.process_relation(object)
-    local geom_func = RELATION_TYPES[object.tags.type]
+function module.process_relation(object)
+    local geom_func = module.RELATION_TYPES[object.tags.type]

 
     if geom_func ~= nil then
 
     if geom_func ~= nil then

-        process_tags(Place.new(object, geom_func))
+        module.process_tags(Place.new(object, geom_func))

     end
 end
 
     end
 end
 

-function process_tags(o)
-    local fallback
+-- The process functions are used by default by osm2pgsql.
+osm2pgsql.process_node = module.process_node
+osm2pgsql.process_way = module.process_way
+osm2pgsql.process_relation = module.process_relation

 
 

-    o:delete{match = PRE_DELETE}
-    o:grab_extratags{match = PRE_EXTRAS}
+function module.process_tags(o)
+    o:clean{delete = PRE_DELETE, extra = PRE_EXTRAS}

 
     -- Exception for boundary/place double tagging
     if o.object.tags.boundary == 'administrative' then
 
     -- Exception for boundary/place double tagging
     if o.object.tags.boundary == 'administrative' then


@@ -330,54 +438,116 @@ function process_tags(o)
         end}
     end
 
         end}
     end
 

+    -- name keys
+    local fallback = o:grab_name_parts{groups=NAMES}
+

     -- address keys
     -- address keys

-    o:grab_address{match=COUNTRY_TAGS, out_key='country'}
+    if o:grab_address_parts{groups=ADDRESS_TAGS} > 0 and fallback == nil then
+        fallback = {'place', 'house', 'always'}
+    end

     if o.address.country ~= nil and #o.address.country ~= 2 then
         o.address['country'] = nil
     end
     if o.address.country ~= nil and #o.address.country ~= 2 then
         o.address['country'] = nil
     end

-    if o:grab_name{match=HOUSENAME_TAGS} > 0 then
-        fallback = {'place', 'house'}
+    if POSTCODE_FALLBACK and fallback == nil and o.address.postcode ~= nil then
+        fallback = {'place', 'postcode', 'always'}

     end
     end

-    if o:grab_address{match=HOUSENUMBER_TAGS, include_on_name = true} > 0 and fallback == nil then
-        fallback = {'place', 'house'}
-    end
-    if o:grab_address{match=POSTCODES, out_key='postcode'} > 0 and fallback == nil then
-        fallback = {'place', 'postcode'}
+
+    if o.address.interpolation ~= nil then
+        o:write_place('place', 'houses', 'always', SAVE_EXTRA_MAINS)
+        return

     end
 
     end
 

-    local is_interpolation = o:grab_address{match=INTERPOLATION_TAGS} > 0
+    o:clean{delete = POST_DELETE}

 
 

-    o:grab_address{match=ADDRESS_TAGS}
+    -- collect main keys
+    for k, v in pairs(o.object.tags) do
+        local ktype = MAIN_KEYS[k]
+        if ktype == 'fallback' then
+            if o.has_name then
+                fallback = {k, v, 'named'}
+            end
+        elseif ktype ~= nil then
+            o:write_place(k, v, MAIN_KEYS[k], SAVE_EXTRA_MAINS)
+        end
+    end

 
 

-    if is_interpolation then
-        o:write_place('place', 'houses', 'always', SAVE_EXTRA_MAINS)
-        return
+    if fallback ~= nil and o.num_entries == 0 then
+        o:write_place(fallback[1], fallback[2], fallback[3], SAVE_EXTRA_MAINS)

     end
     end

+end

 
 

-    -- name keys
-    o:grab_name{match = NAMES}
-    o:grab_name{match = REFS, include_on_name = false}
+--------- Convenience functions for simple style configuration -----------------

 
 

-    o:delete{match = POST_DELETE}
-    o:grab_extratags{match = POST_EXTRAS}

 
 

-    -- collect main keys
-    local num_mains = 0
-    for k, v in pairs(o:tags()) do
-        num_mains = num_mains + o:write_place(k, v, MAIN_KEYS[k], SAVE_EXTRA_MAINS)
+function module.set_prefilters(data)
+    PRE_DELETE = module.tag_match{keys = data.delete_keys, tags = data.delete_tags}
+    PRE_EXTRAS = module.tag_match{keys = data.extra_keys,
+                                  tags = data.extra_tags}
+    module.TAGINFO_MAIN.delete_tags = data.delete_tags
+end
+
+function module.set_main_tags(data)
+    MAIN_KEYS = data
+    local keys = {}
+    for k, _ in pairs(data) do
+        table.insert(keys, k)

     end
     end

+    module.TAGINFO_MAIN.keys = keys
+end

 
 

-    if num_mains == 0 then
-        for tag, mtype in pairs(MAIN_FALLBACK_KEYS) do
-            if o:write_place(tag, nil, mtype, SAVE_EXTRA_MAINS) > 0 then
-                return
+function module.set_name_tags(data)
+    NAMES = module.tag_group(data)
+
+    for _, lst in pairs(data) do
+        for _, k in ipairs(lst) do
+            local key = process_key(k)
+            if key ~= nil then
+                module.TAGINFO_NAME_KEYS[key] = true

             end
         end
             end
         end

+    end
+end

 
 

-        if fallback ~= nil then
-            o:write_place(fallback[1], fallback[2], 'always', SAVE_EXTRA_MAINS)
+function module.set_address_tags(data)
+    if data.postcode_fallback ~= nil then
+        POSTCODE_FALLBACK = data.postcode_fallback
+        data.postcode_fallback = nil
+    end
+    ADDRESS_TAGS = module.tag_group(data)
+
+    for _, lst in pairs(data) do
+        if lst ~= nil then
+            for _, k in ipairs(lst) do
+                local key = process_key(k)
+                if key ~= nil then
+                    module.TAGINFO_ADDRESS_KEYS[key] = true
+                end
+            end

         end
     end
 end
 
         end
     end
 end
 

+function module.set_unused_handling(data)
+    if data.extra_keys == nil and data.extra_tags == nil then
+        POST_DELETE = module.tag_match{keys = data.delete_keys, tags = data.delete_tags}
+        SAVE_EXTRA_MAINS = function() return true end
+    elseif data.delete_keys == nil and data.delete_tags == nil then
+        POST_DELETE = nil
+        SAVE_EXTRA_MAINS = module.tag_match{keys = data.extra_keys, tags = data.extra_tags}
+    else
+        error("unused handler can have only 'extra_keys' or 'delete_keys' set.")
+    end
+end
+
+function set_relation_types(data)
+    module.RELATION_TYPES = {}
+    for k, v in data do
+        if v == 'multipolygon' then
+            module.RELATION_TYPES[k] = module.relation_as_multipolygon
+        elseif v == 'multiline' then
+            module.RELATION_TYPES[k] = module.relation_as_multiline
+        end
+    end
+end

 
 

+return module