Merge pull request #3626 from lonvia/import-performance

[nominatim.git] / lib-lua / themes / nominatim / init.lua

diff --git a/lib-lua/themes/nominatim/init.lua b/lib-lua/themes/nominatim/init.lua
index 55634227f4ba2185f427a682966d13968e2b1185..fef86f919eb8611136d08d5cf6e2c96afc930f31 100644 (file)
--- a/lib-lua/themes/nominatim/init.lua
+++ b/lib-lua/themes/nominatim/init.lua
@@ -22,12 +22,13 @@
 
 local module = {}
 
 
 local module = {}
 

-local POST_DELETE = nil

 local MAIN_KEYS = {admin_level = {'delete'}}
 local PRE_FILTER = {prefix = {}, suffix = {}}
 local MAIN_KEYS = {admin_level = {'delete'}}
 local PRE_FILTER = {prefix = {}, suffix = {}}

-local NAMES = nil
-local ADDRESS_TAGS = nil
-local SAVE_EXTRA_MAINS = false
+local NAMES = {}
+local NAME_FILTER = nil
+local ADDRESS_TAGS = {}
+local ADDRESS_FILTER = nil
+local EXTRATAGS_FILTER

 local POSTCODE_FALLBACK = true
 
 -- This file can also be directly require'd instead of running it under
 local POSTCODE_FALLBACK = true
 
 -- This file can also be directly require'd instead of running it under


@@ -39,12 +40,6 @@ if type(themepark) ~= 'table' then
     themepark = nil
 end
 
     themepark = nil
 end
 

--- tables required for taginfo
-module.TAGINFO_MAIN = {keys = {}, delete_tags = {}}
-module.TAGINFO_NAME_KEYS = {}
-module.TAGINFO_ADDRESS_KEYS = {}
-
-

 -- The single place table.
 local place_table_definition = {
     name = "place",
 -- The single place table.
 local place_table_definition = {
     name = "place",


@@ -64,6 +59,8 @@ local place_table_definition = {
 }
 
 local insert_row
 }
 
 local insert_row

+local script_path = debug.getinfo(1, "S").source:match("@?(.*/)")
+local PRESETS = loadfile(script_path .. 'presets.lua')()

 
 if themepark then
     themepark:add_table(place_table_definition)
 
 if themepark then
     themepark:add_table(place_table_definition)


@@ -137,6 +134,40 @@ function PlaceTransform.named_with_key(place, k)
     end
 end
 
     end
 end
 

+-- Special transform used with address fallbacks: ignore all names
+-- except for those marked as being part of the address.
+local function address_fallback(place)
+    if next(place.names) == nil or NAMES.house == nil then
+        return place
+    end
+
+    local names = {}
+    for k, v in pairs(place.names) do
+        if NAME_FILTER(k, v) == 'house' then
+            names[k] = v
+        end
+    end
+    return place:clone{names=names}
+end
+
+--------- Built-in extratags transformation functions ---------------
+
+local function default_extratags_filter(p, k)
+    -- Default handling is to copy over place tag for boundaries.
+    -- Nominatim needs this.
+    if k ~= 'boundary' or p.intags.place == nil then
+        return p.extratags
+    end
+
+    local extra = { place = p.intags.place }
+    for kin, vin in pairs(p.extratags) do
+        extra[kin] = vin
+    end
+
+    return extra
+end
+EXTRATAGS_FILTER = default_extratags_filter
+

 ----------------- other helper functions -----------------------------
 
 local function lookup_prefilter_classification(k, v)
 ----------------- other helper functions -----------------------------
 
 local function lookup_prefilter_classification(k, v)


@@ -364,7 +395,7 @@ function Place:grab_name_parts(data)
                     self.has_name = true
                 elseif atype == 'house' then
                     self.has_name = true
                     self.has_name = true
                 elseif atype == 'house' then
                     self.has_name = true

-                    fallback = {'place', 'house', PlaceTransform.always}
+                    fallback = {'place', 'house', address_fallback}

                 end
             end
         end
                 end
             end
         end


@@ -374,7 +405,7 @@ function Place:grab_name_parts(data)
 end
 
 
 end
 
 

-function Place:write_place(k, v, mfunc, save_extra_mains)
+function Place:write_place(k, v, mfunc)

     v = v or self.intags[k]
     if v == nil then
         return 0
     v = v or self.intags[k]
     if v == nil then
         return 0


@@ -382,7 +413,7 @@ function Place:write_place(k, v, mfunc, save_extra_mains)
 
     local place = mfunc(self, k, v)
     if place then
 
     local place = mfunc(self, k, v)
     if place then

-        local res = place:write_row(k, v, save_extra_mains)
+        local res = place:write_row(k, v)

         self.num_entries = self.num_entries + res
         return res
     end
         self.num_entries = self.num_entries + res
         return res
     end


@@ -390,20 +421,17 @@ function Place:write_place(k, v, mfunc, save_extra_mains)
     return 0
 end
 
     return 0
 end
 

-function Place:write_row(k, v, save_extra_mains)
+function Place:write_row(k, v)

     if self.geometry == nil then
         self.geometry = self.geom_func(self.object)
     end
     if self.geometry == nil then
         self.geometry = self.geom_func(self.object)
     end

-    if self.geometry:is_null() then
+    if self.geometry == nil or self.geometry:is_null() then

         return 0
     end
 
         return 0
     end
 

-    if save_extra_mains ~= nil then
-        for extra_k, extra_v in pairs(self.intags) do
-            if extra_k ~= k and save_extra_mains(extra_k, extra_v) then
-                self.extratags[extra_k] = extra_v
-            end
-        end
+    local extratags = EXTRATAGS_FILTER(self, k, v)
+    if not (extratags and next(extratags)) then
+        extratags = nil

     end
 
     insert_row{
     end
 
     insert_row{


@@ -412,18 +440,10 @@ function Place:write_row(k, v, save_extra_mains)
         admin_level = self.admin_level,
         name = next(self.names) and self.names,
         address = next(self.address) and self.address,
         admin_level = self.admin_level,
         name = next(self.names) and self.names,
         address = next(self.address) and self.address,

-        extratags = next(self.extratags) and self.extratags,
+        extratags = extratags,

         geometry = self.geometry
     }
 
         geometry = self.geometry
     }
 

-    if save_extra_mains then
-        for tk, tv in pairs(self.intags) do
-            if save_extra_mains(tk, tv) then
-                self.extratags[tk] = nil
-            end
-        end
-    end
-

     return 1
 end
 
     return 1
 end
 


@@ -534,7 +554,7 @@ function module.tag_group(data)
         end
     end
 
         end
     end
 

-    return function (k, v)
+    return function (k)

         local val = fullmatches[k]
         if val ~= nil then
             return val
         local val = fullmatches[k]
         if val ~= nil then
             return val


@@ -588,6 +608,9 @@ function module.process_way(object)
 
         if geom:is_null() then
             geom = o:as_linestring()
 
         if geom:is_null() then
             geom = o:as_linestring()

+            if geom:is_null() or geom:length() > 30 then
+                return nil
+            end

         end
 
         return geom
         end
 
         return geom


@@ -628,11 +651,11 @@ function module.process_tags(o)
     end
 
     -- name keys
     end
 
     -- name keys

-    local fallback = o:grab_name_parts{groups=NAMES}
+    local fallback = o:grab_name_parts{groups=NAME_FILTER}

 
     -- address keys
 
     -- address keys

-    if o:grab_address_parts{groups=ADDRESS_TAGS} > 0 and fallback == nil then
-        fallback = {'place', 'house', PlaceTransform.always}
+    if o:grab_address_parts{groups=ADDRESS_FILTER} > 0 and fallback == nil then
+        fallback = {'place', 'house', address_fallback}

     end
     if o.address.country ~= nil and #o.address.country ~= 2 then
         o.address['country'] = nil
     end
     if o.address.country ~= nil and #o.address.country ~= 2 then
         o.address['country'] = nil


@@ -642,19 +665,17 @@ function module.process_tags(o)
     end
 
     if o.address.interpolation ~= nil then
     end
 
     if o.address.interpolation ~= nil then

-        o:write_place('place', 'houses', PlaceTransform.always, SAVE_EXTRA_MAINS)
+        o:write_place('place', 'houses', PlaceTransform.always)

         return
     end
 
         return
     end
 

-    o:clean{delete = POST_DELETE}
-

     -- collect main keys
     for k, v in pairs(o.intags) do
         local ktable = MAIN_KEYS[k]
         if ktable then
             local ktype = ktable[v] or ktable[1]
             if type(ktype) == 'function' then
     -- collect main keys
     for k, v in pairs(o.intags) do
         local ktable = MAIN_KEYS[k]
         if ktable then
             local ktype = ktable[v] or ktable[1]
             if type(ktype) == 'function' then

-                o:write_place(k, v, ktype, SAVE_EXTRA_MAINS)
+                o:write_place(k, v, ktype)

             elseif ktype == 'fallback' and o.has_name then
                 fallback = {k, v, PlaceTransform.named}
             end
             elseif ktype == 'fallback' and o.has_name then
                 fallback = {k, v, PlaceTransform.named}
             end


@@ -662,7 +683,7 @@ function module.process_tags(o)
     end
 
     if fallback ~= nil and o.num_entries == 0 then
     end
 
     if fallback ~= nil and o.num_entries == 0 then

-        o:write_place(fallback[1], fallback[2], fallback[3], SAVE_EXTRA_MAINS)
+        o:write_place(fallback[1], fallback[2], fallback[3])

     end
 end
 
     end
 end
 


@@ -680,13 +701,27 @@ function module.set_prefilters(data)
 end
 
 
 end
 
 

-function module.ignore_tags(data)
+function module.ignore_keys(data)
+    if type(data) == 'string' then
+        local preset = data
+        data = PRESETS.IGNORE_KEYS[data]
+        if data == nil then
+            error('Unknown preset for ignored keys: ' .. preset)
+        end
+    end

     merge_filters_into_main('delete', data)
     add_pre_filter{delete = data}
 end
 
 
 function module.add_for_extratags(data)
     merge_filters_into_main('delete', data)
     add_pre_filter{delete = data}
 end
 
 
 function module.add_for_extratags(data)

+    if type(data) == 'string' then
+        local preset = data
+        data = PRESETS.EXTRATAGS[data] or PRESETS.IGNORE_KEYS[data]
+        if data == nil then
+            error('Unknown preset for extratags: ' .. preset)
+        end
+    end

     merge_filters_into_main('extra', data)
     add_pre_filter{extra = data}
 end
     merge_filters_into_main('extra', data)
     add_pre_filter{extra = data}
 end


@@ -703,11 +738,25 @@ function module.set_main_tags(data)
             MAIN_KEYS[key] = nil
         end
     end
             MAIN_KEYS[key] = nil
         end
     end

-    module.add_main_tags(data)
+    module.modify_main_tags(data)

 end
 
 
 end
 
 

-function module.add_main_tags(data)
+function module.modify_main_tags(data)
+    if type(data) == 'string' then
+        local preset = data
+        if data:sub(1, 7) == 'street/' then
+            data = PRESETS.MAIN_TAGS_STREETS[data:sub(8)]
+        elseif data:sub(1, 4) == 'poi/' then
+            data = PRESETS.MAIN_TAGS_POIS(data:sub(5))
+        else
+            data = PRESETS.MAIN_TAGS[data]
+        end
+        if data == nil then
+            error('Unknown preset for main tags: ' .. preset)
+        end
+    end
+

     for k, v in pairs(data) do
         if MAIN_KEYS[k] == nil then
             MAIN_KEYS[k] = {}
     for k, v in pairs(data) do
         if MAIN_KEYS[k] == nil then
             MAIN_KEYS[k] = {}


@@ -729,57 +778,129 @@ function module.add_main_tags(data)
 end
 
 
 end
 
 

-function module.set_name_tags(data)
-    NAMES = module.tag_group(data)
+function module.modify_name_tags(data)
+    if type(data) == 'string' then
+        local preset = data
+        data = PRESETS.NAME_TAGS[data]
+        if data == nil then
+            error('Unknown preset for name keys: ' .. preset)
+        end
+    end

 
 

-    for _, lst in pairs(data) do
-        for _, k in ipairs(lst) do
-            local key = process_key(k)
-            if key ~= nil then
-                module.TAGINFO_NAME_KEYS[key] = true
-            end
+    for k,v in pairs(data) do
+        if next(v) then
+            NAMES[k] = v
+        else
+            NAMES[k] = nil

         end
     end
         end
     end

+    NAME_FILTER = module.tag_group(NAMES)

     remove_group_from_main('fallback:name')
     remove_group_from_main('fallback:name')

-    merge_filters_into_main('fallback:name', data.house)
+    if data.house ~= nil then
+        merge_filters_into_main('fallback:name', data.house)
+    end
+end
+
+
+function module.set_name_tags(data)
+    NAMES = {}
+    module.modify_name_tags(data)

 end
 
 
 function module.set_address_tags(data)
 end
 
 
 function module.set_address_tags(data)

-    if data.postcode_fallback ~= nil then
-        POSTCODE_FALLBACK = data.postcode_fallback
-        data.postcode_fallback = nil
-    end
-    ADDRESS_TAGS = module.tag_group(data)
-
-    for _, lst in pairs(data) do
-        if lst ~= nil then
-            for _, k in ipairs(lst) do
-                local key = process_key(k)
-                if key ~= nil then
-                    module.TAGINFO_ADDRESS_KEYS[key] = true
-                end
-            end
+    ADDRESS_TAGS = {}
+    module.modify_address_tags(data)
+end
+
+
+function module.modify_address_tags(data)
+    if type(data) == 'string' then
+        local preset = data
+        data = PRESETS.ADDRESS_TAGS[data]
+        if data == nil then
+            error('Unknown preset for address keys: ' .. preset)
+        end
+    end
+
+    for k, v in pairs(data) do
+        if k == 'postcode_fallback' then
+            POSTCODE_FALLBACK = v
+        elseif next(v) == nil then
+            ADDRESS_TAGS[k] = nil
+        else
+            ADDRESS_TAGS[k] = v

         end
     end
 
         end
     end
 

+    ADDRESS_FILTER = module.tag_group(ADDRESS_TAGS)
+

     remove_group_from_main('fallback:address')
     remove_group_from_main('fallback:address')

-    remove_group_from_main('fallback:postcode')

     merge_filters_into_main('fallback:address', data.main)
     merge_filters_into_main('fallback:address', data.main)

+    merge_filters_into_main('fallback:address', data.interpolation)
+    remove_group_from_main('fallback:postcode')

     if POSTCODE_FALLBACK then
         merge_filters_into_main('fallback:postcode', data.postcode)
     end
     if POSTCODE_FALLBACK then
         merge_filters_into_main('fallback:postcode', data.postcode)
     end

-    merge_filters_into_main('fallback:address', data.interpolation)
+end
+
+
+function module.set_address_tags(data)
+    ADDRESS_TAGS_SOURCE = {}
+    module.modify_address_tags(data)
+end
+
+
+function module.set_postcode_fallback(enable)
+    if POSTCODE_FALLBACK ~= enable then
+        remove_group_from_main('fallback:postcode')
+        if enable then
+            merge_filters_into_main('fallback:postcode', ADDRESS_TAGS.postcode)
+        end
+    end
+    POSTCODE_FALLBACK = enable

 end
 
 
 function module.set_unused_handling(data)
 end
 
 
 function module.set_unused_handling(data)

-    if data.extra_keys == nil and data.extra_tags == nil then
-        POST_DELETE = module.tag_match{keys = data.delete_keys, tags = data.delete_tags}
-        SAVE_EXTRA_MAINS = function() return true end
+    if type(data) == 'function' then
+        EXTRATAGS_FILTER = data
+    elseif data == nil then
+        EXTRATAGS_FILTER = default_extratags_filter
+    elseif data.extra_keys == nil and data.extra_tags == nil then
+        local delfilter = module.tag_match{keys = data.delete_keys, tags = data.delete_tags}
+        EXTRATAGS_FILTER = function (p, k)
+            local extra = {}
+            for kin, vin in pairs(p.intags) do
+                if kin ~= k and not delfilter(kin, vin) then
+                    extra[kin] = vin
+                end
+            end
+            if next(extra) == nil then
+                return p.extratags
+            end
+            for kextra, vextra in pairs(p.extratags) do
+                extra[kextra] = vextra
+            end
+            return extra
+        end

     elseif data.delete_keys == nil and data.delete_tags == nil then
     elseif data.delete_keys == nil and data.delete_tags == nil then

-        POST_DELETE = nil
-        SAVE_EXTRA_MAINS = module.tag_match{keys = data.extra_keys, tags = data.extra_tags}
+        local incfilter = module.tag_match{keys = data.extra_keys, tags = data.extra_tags}
+        EXTRATAGS_FILTER = function (p, k)
+            local extra = {}
+            for kin, vin in pairs(p.intags) do
+                if kin ~= k and incfilter(kin, vin) then
+                    extra[kin] = vin
+                end
+            end
+            if next(extra) == nil then
+                return p.extratags
+            end
+            for kextra, vextra in pairs(p.extratags) do
+                extra[kextra] = vextra
+            end
+            return extra
+        end

     else
         error("unused handler can have only 'extra_keys' or 'delete_keys' set.")
     end
     else
         error("unused handler can have only 'extra_keys' or 'delete_keys' set.")
     end


@@ -796,4 +917,9 @@ function module.set_relation_types(data)
     end
 end
 
     end
 end
 

+
+function module.get_taginfo()
+    return {main = MAIN_KEYS, name = NAMES, address = ADDRESS_TAGS}
+end
+

 return module
 return module