indexing: precompute row counts

[nominatim.git] / settings / flex-base.lua

diff --git a/settings/flex-base.lua b/settings/flex-base.lua
index fb39a32ddbaf71fa43d25858a151c8e1caa4a2b4..4d960d7267a3cc502c57938a2ebb12057ca95b31 100644 (file)
--- a/settings/flex-base.lua
+++ b/settings/flex-base.lua
@@ -5,13 +5,21 @@ local module = {}
 
 local PRE_DELETE = nil
 local PRE_EXTRAS = nil
 
 local PRE_DELETE = nil
 local PRE_EXTRAS = nil

+local POST_DELETE = nil

 local MAIN_KEYS = nil
 local NAMES = nil
 local ADDRESS_TAGS = nil
 local MAIN_KEYS = nil
 local NAMES = nil
 local ADDRESS_TAGS = nil

+local SAVE_EXTRA_MAINS = false
+local POSTCODE_FALLBACK = true
+
+-- tables required for taginfo
+module.TAGINFO_MAIN = {keys = {}, delete_tags = {}}
+module.TAGINFO_NAME_KEYS = {}
+module.TAGINFO_ADDRESS_KEYS = {}

 
 
 -- The single place table.
 
 
 -- The single place table.

-place_table = osm2pgsql.define_table{
+local place_table = osm2pgsql.define_table{

     name = "place",
     ids = { type = 'any', id_column = 'osm_id', type_column = 'osm_type' },
     columns = {
     name = "place",
     ids = { type = 'any', id_column = 'osm_id', type_column = 'osm_type' },
     columns = {


@@ -23,6 +31,8 @@ place_table = osm2pgsql.define_table{
         { column = 'extratags', type = 'hstore' },
         { column = 'geometry', type = 'geometry', projection = 'WGS84', not_null = true },
     },
         { column = 'extratags', type = 'hstore' },
         { column = 'geometry', type = 'geometry', projection = 'WGS84', not_null = true },
     },

+    data_tablespace = os.getenv("NOMINATIM_TABLESPACE_PLACE_DATA"),
+    index_tablespace = os.getenv("NOMINATIM_TABLESPACE_PLACE_INDEX"),

     indexes = {}
 }
 
     indexes = {}
 }
 


@@ -106,38 +116,6 @@ function Place:grab_extratags(data)
     return count
 end
 
     return count
 end
 

-function Place:grab_address(data)
-    local count = 0
-
-    if data.match ~= nil then
-        for k, v in pairs(self.object.tags) do
-            if data.match(k, v) then
-                self.object.tags[k] = nil
-
-                if data.include_on_name == true then
-                    self.has_name = true
-                end
-
-                if data.out_key ~= nil then
-                    self.address[data.out_key] = v
-                    return 1
-                end
-
-                if k:sub(1, 5) == 'addr:' then
-                    self.address[k:sub(6)] = v
-                elseif k:sub(1, 6) == 'is_in:' then
-                    self.address[k:sub(7)] = v
-                else
-                    self.address[k] = v
-                end
-                count = count + 1
-            end
-        end
-    end
-
-    return count
-end
-

 local function strip_address_prefix(k)
     if k:sub(1, 5) == 'addr:' then
         return k:sub(6)
 local function strip_address_prefix(k)
     if k:sub(1, 5) == 'addr:' then
         return k:sub(6)


@@ -176,24 +154,6 @@ function Place:grab_address_parts(data)
     return count
 end
 
     return count
 end
 

-function Place:grab_name(data)
-    local count = 0
-
-    if data.match ~= nil then
-        for k, v in pairs(self.object.tags) do
-            if data.match(k, v) then
-                self.object.tags[k] = nil
-                self.names[k] = v
-                if data.include_on_name ~= false then
-                    self.has_name = true
-                end
-                count = count + 1
-            end
-        end
-    end
-
-    return count
-end

 
 function Place:grab_name_parts(data)
     local fallback = nil
 
 function Place:grab_name_parts(data)
     local fallback = nil


@@ -218,9 +178,6 @@ function Place:grab_name_parts(data)
     return fallback
 end
 
     return fallback
 end
 

-function Place:grab_tag(key)
-    return self.object:grab_tag(key)
-end

 
 function Place:write_place(k, v, mtype, save_extra_mains)
     if mtype == nil then
 
 function Place:write_place(k, v, mtype, save_extra_mains)
     if mtype == nil then


@@ -274,9 +231,9 @@ function Place:write_row(k, v, save_extra_mains)
         return 0
     end
 
         return 0
     end
 

-    if save_extra_mains then
+    if save_extra_mains ~= nil then

         for extra_k, extra_v in pairs(self.object.tags) do
         for extra_k, extra_v in pairs(self.object.tags) do

-            if extra_k ~= k then
+            if extra_k ~= k and save_extra_mains(extra_k, extra_v) then

                 self.extratags[extra_k] = extra_v
             end
         end
                 self.extratags[extra_k] = extra_v
             end
         end


@@ -293,8 +250,10 @@ function Place:write_row(k, v, save_extra_mains)
     }
 
     if save_extra_mains then
     }
 
     if save_extra_mains then

-        for k, v in pairs(self.object.tags) do
-            self.extratags[k] = nil
+        for tk, tv in pairs(self.object.tags) do
+            if save_extra_mains(tk, tv) then
+                self.extratags[tk] = nil
+            end

         end
     end
 
         end
     end
 


@@ -421,8 +380,19 @@ function module.tag_group(data)
     end
 end
 
     end
 end
 

+-- Returns prefix part of the keys, and reject suffix matching keys
+local function process_key(key)
+    if key:sub(1, 1) == '*' then
+        return nil
+    end
+    if key:sub(#key, #key) == '*' then
+        return key:sub(1, #key - 2)
+    end
+    return key
+end
+

 -- Process functions for all data types
 -- Process functions for all data types

-function osm2pgsql.process_node(object)
+function module.process_node(object)

 
     local function geom_func(o)
         return o:as_point()
 
     local function geom_func(o)
         return o:as_point()


@@ -431,7 +401,7 @@ function osm2pgsql.process_node(object)
     module.process_tags(Place.new(object, geom_func))
 end
 
     module.process_tags(Place.new(object, geom_func))
 end
 

-function osm2pgsql.process_way(object)
+function module.process_way(object)

 
     local function geom_func(o)
         local geom = o:as_polygon()
 
     local function geom_func(o)
         local geom = o:as_polygon()


@@ -446,7 +416,7 @@ function osm2pgsql.process_way(object)
     module.process_tags(Place.new(object, geom_func))
 end
 
     module.process_tags(Place.new(object, geom_func))
 end
 

-function osm2pgsql.process_relation(object)
+function module.process_relation(object)

     local geom_func = module.RELATION_TYPES[object.tags.type]
 
     if geom_func ~= nil then
     local geom_func = module.RELATION_TYPES[object.tags.type]
 
     if geom_func ~= nil then


@@ -454,6 +424,11 @@ function osm2pgsql.process_relation(object)
     end
 end
 
     end
 end
 

+-- The process functions are used by default by osm2pgsql.
+osm2pgsql.process_node = module.process_node
+osm2pgsql.process_way = module.process_way
+osm2pgsql.process_relation = module.process_relation
+

 function module.process_tags(o)
     o:clean{delete = PRE_DELETE, extra = PRE_EXTRAS}
 
 function module.process_tags(o)
     o:clean{delete = PRE_DELETE, extra = PRE_EXTRAS}
 


@@ -474,7 +449,7 @@ function module.process_tags(o)
     if o.address.country ~= nil and #o.address.country ~= 2 then
         o.address['country'] = nil
     end
     if o.address.country ~= nil and #o.address.country ~= 2 then
         o.address['country'] = nil
     end

-    if fallback == nil and o.address.postcode ~= nil then
+    if POSTCODE_FALLBACK and fallback == nil and o.address.postcode ~= nil then

         fallback = {'place', 'postcode', 'always'}
     end
 
         fallback = {'place', 'postcode', 'always'}
     end
 


@@ -483,7 +458,7 @@ function module.process_tags(o)
         return
     end
 
         return
     end
 

-    o:clean{delete = POST_DELETE, extra = POST_EXTRAS}
+    o:clean{delete = POST_DELETE}

 
     -- collect main keys
     for k, v in pairs(o.object.tags) do
 
     -- collect main keys
     for k, v in pairs(o.object.tags) do


@@ -507,37 +482,65 @@ end
 
 function module.set_prefilters(data)
     PRE_DELETE = module.tag_match{keys = data.delete_keys, tags = data.delete_tags}
 
 function module.set_prefilters(data)
     PRE_DELETE = module.tag_match{keys = data.delete_keys, tags = data.delete_tags}

-    PRE_EXTRAS = module.tag_match{keys = data.extratag_keys,
-                                  tags = data.extratag_tags}
+    PRE_EXTRAS = module.tag_match{keys = data.extra_keys,
+                                  tags = data.extra_tags}
+    module.TAGINFO_MAIN.delete_tags = data.delete_tags

 end
 
 function module.set_main_tags(data)
     MAIN_KEYS = data
 end
 
 function module.set_main_tags(data)
     MAIN_KEYS = data

+    local keys = {}
+    for k, _ in pairs(data) do
+        table.insert(keys, k)
+    end
+    module.TAGINFO_MAIN.keys = keys

 end
 
 function module.set_name_tags(data)
     NAMES = module.tag_group(data)
 end
 
 function module.set_name_tags(data)
     NAMES = module.tag_group(data)

+
+    for _, lst in pairs(data) do
+        for _, k in ipairs(lst) do
+            local key = process_key(k)
+            if key ~= nil then
+                module.TAGINFO_NAME_KEYS[key] = true
+            end
+        end
+    end

 end
 
 function module.set_address_tags(data)
 end
 
 function module.set_address_tags(data)

+    if data.postcode_fallback ~= nil then
+        POSTCODE_FALLBACK = data.postcode_fallback
+        data.postcode_fallback = nil
+    end

     ADDRESS_TAGS = module.tag_group(data)
     ADDRESS_TAGS = module.tag_group(data)

+
+    for _, lst in pairs(data) do
+        if lst ~= nil then
+            for _, k in ipairs(lst) do
+                local key = process_key(k)
+                if key ~= nil then
+                    module.TAGINFO_ADDRESS_KEYS[key] = true
+                end
+            end
+        end
+    end

 end
 
 function module.set_unused_handling(data)
     if data.extra_keys == nil and data.extra_tags == nil then
 end
 
 function module.set_unused_handling(data)
     if data.extra_keys == nil and data.extra_tags == nil then

-        POST_DELETE = module.tag_match{data.delete_keys, tags = data.delete_tags}
-        POST_EXTRAS = nil
-        SAVE_EXTRA_MAINS = true
+        POST_DELETE = module.tag_match{keys = data.delete_keys, tags = data.delete_tags}
+        SAVE_EXTRA_MAINS = function() return true end

     elseif data.delete_keys == nil and data.delete_tags == nil then
         POST_DELETE = nil
     elseif data.delete_keys == nil and data.delete_tags == nil then
         POST_DELETE = nil

-        POST_EXTRAS = module.tag_match{data.extra_keys, tags = data.extra_tags}
-        SAVE_EXTRA_MAINS = false
+        SAVE_EXTRA_MAINS = module.tag_match{keys = data.extra_keys, tags = data.extra_tags}

     else
         error("unused handler can have only 'extra_keys' or 'delete_keys' set.")
     end
 end
 
     else
         error("unused handler can have only 'extra_keys' or 'delete_keys' set.")
     end
 end
 

-function set_relation_types(data)
+function module.set_relation_types(data)

     module.RELATION_TYPES = {}
     for k, v in data do
         if v == 'multipolygon' then
     module.RELATION_TYPES = {}
     for k, v in data do
         if v == 'multipolygon' then