From 8b8aa1b4e611a4f2bfce2d64a283e14e84403c05 Mon Sep 17 00:00:00 2001
From: Sarah Hoffmann <lonvia@denofr.de>
Date: Sun, 9 Feb 2020 13:57:36 +0100
Subject: [PATCH] regularly close connection while indexing

Postgres sooner or later runs out of memory when the connection
is used for too long.
---
 nominatim/nominatim.py | 28 ++++++++++++++++++++++++----
 1 file changed, 24 insertions(+), 4 deletions(-)

diff --git a/nominatim/nominatim.py b/nominatim/nominatim.py
index 54d9b208..848846ae 100755
--- a/nominatim/nominatim.py
+++ b/nominatim/nominatim.py
@@ -104,14 +104,22 @@ class DBConnection(object):
     """
 
     def __init__(self, options):
+        self.current_query = None
+        self.current_params = None
+
+        self.conn = None
+        self.connect()
+
+    def connect(self):
+        if self.conn is not None:
+            self.cursor.close()
+            self.conn.close()
+
         self.conn = make_connection(options, asynchronous=True)
         self.wait()
 
         self.cursor = self.conn.cursor()
 
-        self.current_query = None
-        self.current_params = None
-
     def wait(self):
         """ Block until any pending operation is done.
         """
@@ -257,13 +265,25 @@ class Indexer(object):
             sending a query.
         """
         ready = self.threads
+        command_stat = 0
 
         while True:
             for thread in ready:
                 if thread.is_done():
+                    command_stat += 1
                     yield thread
 
-            ready, _, _ = select.select(self.threads, [], [])
+            # refresh the connections occasionaly to avoid potential
+            # memory leaks in Postgresql.
+            if command_stat > 100000:
+                for t in self.threads:
+                    while not t.is_done():
+                        wait_select(t.conn)
+                    t.connect()
+                command_stat = 0
+                ready = self.threads
+            else:
+                ready, _, _ = select.select(self.threads, [], [])
 
         assert(False, "Unreachable code")
 
-- 
2.39.5