regularly close connection while indexing

Postgres sooner or later runs out of memory when the connection
is used for too long.

diff --git a/nominatim/nominatim.py b/nominatim/nominatim.py
index 54d9b2085db36c3e5fe840228253a45d690ff2e8..848846ae45123b28b69b53513c378d2ab0435231 100755 (executable)
--- a/nominatim/nominatim.py
+++ b/nominatim/nominatim.py
@@ -104,14 +104,22 @@ class DBConnection(object):
     """
 
     def __init__(self, options):
+        self.current_query = None
+        self.current_params = None
+
+        self.conn = None
+        self.connect()
+
+    def connect(self):
+        if self.conn is not None:
+            self.cursor.close()
+            self.conn.close()
+
         self.conn = make_connection(options, asynchronous=True)
         self.wait()
 
         self.cursor = self.conn.cursor()
 
-        self.current_query = None
-        self.current_params = None
-
     def wait(self):
         """ Block until any pending operation is done.
         """
@@ -257,13 +265,25 @@ class Indexer(object):
             sending a query.
         """
         ready = self.threads
+        command_stat = 0
 
         while True:
             for thread in ready:
                 if thread.is_done():
+                    command_stat += 1
                     yield thread
 
-            ready, _, _ = select.select(self.threads, [], [])
+            # refresh the connections occasionaly to avoid potential
+            # memory leaks in Postgresql.
+            if command_stat > 100000:
+                for t in self.threads:
+                    while not t.is_done():
+                        wait_select(t.conn)
+                    t.connect()
+                command_stat = 0
+                ready = self.threads
+            else:
+                ready, _, _ = select.select(self.threads, [], [])
 
         assert(False, "Unreachable code")