flex: add combining clean function

[nominatim.git] / settings / flex-base.lua

diff --git a/settings/flex-base.lua b/settings/flex-base.lua
index d91299ad08475db6fbc911f87271997fb2faf4bf..98887705b6b4c3b526da61dbe48dcdbbd479fb56 100644 (file)
--- a/settings/flex-base.lua
+++ b/settings/flex-base.lua
@@ -14,7 +14,8 @@ place_table = osm2pgsql.define_table{
         { column = 'address', type = 'hstore' },
         { column = 'extratags', type = 'hstore' },
         { column = 'geometry', type = 'geometry', projection = 'WGS84', not_null = true },
         { column = 'address', type = 'hstore' },
         { column = 'extratags', type = 'hstore' },
         { column = 'geometry', type = 'geometry', projection = 'WGS84', not_null = true },

-    }
+    },
+    indexes = {}

 }
 
 ------------- Place class ------------------------------------------
 }
 
 ------------- Place class ------------------------------------------


@@ -43,6 +44,19 @@ function Place.new(object, geom_func)
     return self
 end
 
     return self
 end
 

+function Place:clean(data)
+    if data.delete ~= nil or data.extra ~= nil then
+        for k, v in pairs(self.object.tags) do
+            if data.delete ~= nil and data.delete(k, v) then
+                self.object.tags[k] = nil
+            elseif data.extra ~= nil and data.extra(k, v) then
+                self.extratags[k] = v
+                self.object.tags[k] = nil
+            end
+        end
+    end
+end
+

 function Place:delete(data)
     if data.match ~= nil then
         for k, v in pairs(self.object.tags) do
 function Place:delete(data)
     if data.match ~= nil then
         for k, v in pairs(self.object.tags) do


@@ -101,8 +115,39 @@ function Place:grab_address(data)
     return count
 end
 
     return count
 end
 

-function Place:set_address(key, value)
-    self.address[key] = value
+local function strip_address_prefix(k)
+    if k:sub(1, 5) == 'addr:' then
+        return k:sub(6)
+    end
+
+    if k:sub(1, 6) == 'is_in:' then
+        return k:sub(7)
+    end
+
+    return k
+end
+
+
+function Place:grab_address_parts(data)
+    local count = 0
+
+    if data.groups ~= nil then
+        for k, v in pairs(self.object.tags) do
+            local atype = data.groups(k, v)
+
+            if atype == 'main' then
+                self.has_name = true
+                self.address[strip_address_prefix(k)] = v
+                count = count + 1
+            elseif atype == 'extra' then
+                self.address[strip_address_prefix(k)] = v
+            elseif atype ~= nil then
+                self.address[atype] = v
+            end
+        end
+    end
+
+    return count

 end
 
 function Place:grab_name(data)
 end
 
 function Place:grab_name(data)


@@ -124,12 +169,30 @@ function Place:grab_name(data)
     return count
 end
 
     return count
 end
 

-function Place:grab_tag(key)
-    return self.object:grab_tag(key)
+function Place:grab_name_parts(data)
+    local fallback = nil
+
+    if data.groups ~= nil then
+        for k, v in pairs(self.object.tags) do
+            local atype = data.groups(k, v)
+
+            if atype ~= nil then
+                self.names[k] = v
+                if atype == 'main' then
+                    self.has_name = true
+                elseif atype == 'house' then
+                    self.has_name = true
+                    fallback = {'place', 'house', 'always'}
+                end
+            end
+        end
+    end
+
+    return fallback

 end
 
 end
 

-function Place:tags()
-    return self.object.tags
+function Place:grab_tag(key)
+    return self.object:grab_tag(key)

 end
 
 function Place:write_place(k, v, mtype, save_extra_mains)
 end
 
 function Place:write_place(k, v, mtype, save_extra_mains)


@@ -219,54 +282,118 @@ function tag_match(data)
         return nil
     end
 
         return nil
     end
 

-    local tests = {}
+    local fullmatches = {}
+    local key_prefixes = {}
+    local key_suffixes = {}

 
     if data.keys ~= nil then
         for _, key in pairs(data.keys) do
             if key:sub(1, 1) == '*' then
                 if #key > 1 then
 
     if data.keys ~= nil then
         for _, key in pairs(data.keys) do
             if key:sub(1, 1) == '*' then
                 if #key > 1 then

-                    local suffix = key:sub(2)
-                    tests[#tests + 1] = function (k, v)
-                        return k:sub(-#suffix) == suffix
+                    if key_suffixes[#key - 1] == nil then
+                        key_suffixes[#key - 1] = {}

                     end
                     end

+                    key_suffixes[#key - 1][key:sub(2)] = true

                 end
             elseif key:sub(#key, #key) == '*' then
                 end
             elseif key:sub(#key, #key) == '*' then

-                local prefix = key:sub(1, #key - 1)
-                tests[#tests + 1] = function (k, v)
-                    return k:sub(1, #prefix) == prefix
+                if key_prefixes[#key - 1] == nil then
+                    key_prefixes[#key - 1] = {}

                 end
                 end

+                key_prefixes[#key - 1][key:sub(1, #key - 1)] = true

             else
             else

-                tests[#tests + 1] = function (k, v)
-                    return k == key
-                end
+                fullmatches[key] = true

             end
         end
     end
 
     if data.tags ~= nil then
             end
         end
     end
 
     if data.tags ~= nil then

-        local tags = {}

         for k, vlist in pairs(data.tags) do
         for k, vlist in pairs(data.tags) do

-            tags[k] = {}
-            for _, v in pairs(vlist) do
-                tags[k][v] = true
+            if fullmatches[k] == nil then
+                fullmatches[k] = {}
+                for _, v in pairs(vlist) do
+                    fullmatches[k][v] = true
+                end

             end
         end
             end
         end

-        tests[#tests + 1] = function (k, v)
-            return tags[k] ~= nil and tags[k][v] ~= nil
-        end

     end
 
     return function (k, v)
     end
 
     return function (k, v)

-        for _, func in pairs(tests) do
-            if func(k, v) then
+        if fullmatches[k] ~= nil and (fullmatches[k] == true or fullmatches[k][v] ~= nil) then
+            return true
+        end
+
+        for slen, slist in pairs(key_suffixes) do
+            if #k >= slen and slist[k:sub(-slen)] ~= nil then

                 return true
             end
         end
                 return true
             end
         end

+
+        for slen, slist in pairs(key_prefixes) do
+            if #k >= slen and slist[k:sub(1, slen)] ~= nil then
+                return true
+            end
+        end
+

         return false
     end
 end
 
 
         return false
     end
 end
 
 

+function key_group(data)
+    if data == nil or next(data) == nil then
+        return nil
+    end
+
+    local fullmatches = {}
+    local key_prefixes = {}
+    local key_suffixes = {}
+
+    for group, tags in pairs(data) do
+        for _, key in pairs(tags) do
+            if key:sub(1, 1) == '*' then
+                if #key > 1 then
+                    if key_suffixes[#key - 1] == nil then
+                        key_suffixes[#key - 1] = {}
+                    end
+                    key_suffixes[#key - 1][key:sub(2)] = group
+                end
+            elseif key:sub(#key, #key) == '*' then
+                if key_prefixes[#key - 1] == nil then
+                    key_prefixes[#key - 1] = {}
+                end
+                key_prefixes[#key - 1][key:sub(1, #key - 1)] = group
+            else
+                fullmatches[key] = group
+            end
+        end
+    end
+
+    return function (k, v)
+        local val = fullmatches[k]
+        if val ~= nil then
+            return val
+        end
+
+        for slen, slist in pairs(key_suffixes) do
+            if #k >= slen then
+                val = slist[k:sub(-slen)]
+                if val ~= nil then
+                    return val
+                end
+            end
+        end
+
+        for slen, slist in pairs(key_prefixes) do
+            if #k >= slen then
+                val = slist[k:sub(1, slen)]
+                if val ~= nil then
+                    return val
+                end
+            end
+        end
+    end
+end
+

 -- Process functions for all data types
 function osm2pgsql.process_node(object)
 
 -- Process functions for all data types
 function osm2pgsql.process_node(object)
 


@@ -309,10 +436,7 @@ function osm2pgsql.process_relation(object)
 end
 
 function process_tags(o)
 end
 
 function process_tags(o)

-    local fallback
-
-    o:delete{match = PRE_DELETE}
-    o:grab_extratags{match = PRE_EXTRAS}
+    o:clean{delete = PRE_DELETE, extra = PRE_EXTRAS}

 
     -- Exception for boundary/place double tagging
     if o.object.tags.boundary == 'administrative' then
 
     -- Exception for boundary/place double tagging
     if o.object.tags.boundary == 'administrative' then


@@ -321,61 +445,41 @@ function process_tags(o)
         end}
     end
 
         end}
     end
 

+    -- name keys
+    local fallback = o:grab_name_parts{groups=NAMES}
+

     -- address keys
     -- address keys

-    o:grab_address{match=function (k, v) return COUNTRY_TAGS(k, v) and #v == 2 end,
-                   out_key='country'}
-    if o:grab_name{match=HOUSENAME_TAGS} > 0 then
-        fallback = {'place', 'house'}
+    if o:grab_address_parts{groups=ADDRESS_TAGS} > 0 and fallback == nil then
+        fallback = {'place', 'house', 'always'}

     end
     end

-    if o:grab_address{match=HOUSENUMBER_TAGS, include_on_name = true} > 0 and fallback == nil then
-        fallback = {'place', 'house'}
+    if o.address.country ~= nil and #o.address.country ~= 2 then
+        o.address['country'] = nil

     end
     end

-    if o:grab_address{match=POSTCODES, out_key='postcode'} > 0 and fallback == nil then
-        fallback = {'place', 'postcode'}
+    if fallback == nil and o.address.postcode ~= nil then
+        fallback = {'place', 'postcode', 'always'}

     end
 
     end
 

-    local is_interpolation = o:grab_address{match=INTERPOLATION_TAGS} > 0
-
-    if ADD_TIGER_COUNTY then
-        local v = o:grab_tag('tiger:county')
-        if v ~= nil then
-            v, num = v:gsub(',.*', ' county')
-            if num == 0 then
-                v = v .. ' county'
-            end
-            o:set_address('tiger:county', v)
-        end
-    end
-    o:grab_address{match=ADDRESS_TAGS}
-
-    if is_interpolation then
+    if o.address.interpolation ~= nil then

         o:write_place('place', 'houses', 'always', SAVE_EXTRA_MAINS)
         return
     end
 
         o:write_place('place', 'houses', 'always', SAVE_EXTRA_MAINS)
         return
     end
 

-    -- name keys
-    o:grab_name{match = NAMES}
-    o:grab_name{match = REFS, include_on_name = false}
-
-    o:delete{match = POST_DELETE}
-    o:grab_extratags{match = POST_EXTRAS}
+    o:clean{delete = POST_DELETE, extra = POST_EXTRAS}

 
     -- collect main keys
 
     -- collect main keys

-    local num_mains = 0
-    for k, v in pairs(o:tags()) do
-        num_mains = num_mains + o:write_place(k, v, MAIN_KEYS[k], SAVE_EXTRA_MAINS)
-    end
-
-    if num_mains == 0 then
-        for tag, mtype in pairs(MAIN_FALLBACK_KEYS) do
-            if o:write_place(tag, nil, mtype, SAVE_EXTRA_MAINS) > 0 then
-                return
+    for k, v in pairs(o.object.tags) do
+        local ktype = MAIN_KEYS[k]
+        if ktype == 'fallback' then
+            if o.has_name then
+                fallback = {k, v, 'named'}

             end
             end

+        elseif ktype ~= nil then
+            o:write_place(k, v, MAIN_KEYS[k], SAVE_EXTRA_MAINS)

         end
         end

+    end

 
 

-        if fallback ~= nil then
-            o:write_place(fallback[1], fallback[2], 'always', SAVE_EXTRA_MAINS)
-        end
+    if fallback ~= nil and o.num_entries == 0 then
+        o:write_place(fallback[1], fallback[2], fallback[3], SAVE_EXTRA_MAINS)

     end
 end
 
     end
 end