Merge remote-tracking branch 'upstream/master'

[nominatim.git] / nominatim / nominatim.py

diff --git a/nominatim/nominatim.py b/nominatim/nominatim.py
old mode 100644 (file)
new mode 100755 (executable)
index 6190706..b20673d
--- a/nominatim/nominatim.py
+++ b/nominatim/nominatim.py
@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#! /usr/bin/env python3

 #-----------------------------------------------------------------------------
 # nominatim - [description]
 #-----------------------------------------------------------------------------
 #-----------------------------------------------------------------------------
 # nominatim - [description]
 #-----------------------------------------------------------------------------


@@ -28,212 +28,186 @@ import sys
 import re
 import getpass
 from datetime import datetime
 import re
 import getpass
 from datetime import datetime

-import psycopg2
-from psycopg2.extras import wait_select
-import threading
-from queue import Queue
+import select

 
 

-log = logging.getLogger()
+from indexer.progress import ProgressLogger
+from indexer.db import DBConnection, make_connection

 
 

-def make_connection(options, asynchronous=False):
-    return psycopg2.connect(dbname=options.dbname, user=options.user,
-                            password=options.password, host=options.host,
-                            port=options.port, async_=asynchronous)
+log = logging.getLogger()

 
 

-class IndexingThread(threading.Thread):
+class RankRunner(object):
+    """ Returns SQL commands for indexing one rank within the placex table.
+    """

 
 

-    def __init__(self, queue, barrier, options):
-        super().__init__()
-        self.conn = make_connection(options)
-        self.conn.autocommit = True
+    def __init__(self, rank):
+        self.rank = rank

 
 

-        self.cursor = self.conn.cursor()
-        self.perform("SET lc_messages TO 'C'")
-        self.perform(InterpolationRunner.prepare())
-        self.perform(RankRunner.prepare())
-        self.queue = queue
-        self.barrier = barrier
+    def name(self):
+        return "rank {}".format(self.rank)

 
 

-    def run(self):
-        sql = None
-        while True:
-            item = self.queue.get()
-            if item is None:
-                break
-            elif isinstance(item, str):
-                sql = item
-                self.barrier.wait()
-            else:
-                self.perform(sql, (item,))
+    def sql_count_objects(self):
+        return """SELECT count(*) FROM placex
+                  WHERE rank_address = {} and indexed_status > 0
+               """.format(self.rank)

 
 

-    def perform(self, sql, args=None):
-        while True:
-            try:
-                self.cursor.execute(sql, args)
-                return
-            except psycopg2.extensions.TransactionRollbackError as e:
-                if e.pgcode is None:
-                    raise RuntimeError("Postgres exception has no error code")
-                if e.pgcode == '40P01':
-                    log.info("Deadlock detected, retry.")
-                else:
-                    raise
+    def sql_get_objects(self):
+        return """SELECT place_id FROM placex
+                  WHERE indexed_status > 0 and rank_address = {}
+                  ORDER BY geometry_sector""".format(self.rank)

 
 

+    def sql_index_place(self, ids):
+        return "UPDATE placex SET indexed_status = 0 WHERE place_id IN ({})"\
+               .format(','.join((str(i) for i in ids)))

 
 
 
 

-class Indexer(object):
+class InterpolationRunner(object):
+    """ Returns SQL commands for indexing the address interpolation table
+        location_property_osmline.
+    """

 
 

-    def __init__(self, options):
-        self.options = options
-        self.conn = make_connection(options)
+    def name(self):
+        return "interpolation lines (location_property_osmline)"

 
 

-        self.threads = []
-        self.queue = Queue(maxsize=1000)
-        self.barrier = threading.Barrier(options.threads + 1)
-        for i in range(options.threads):
-            t = IndexingThread(self.queue, self.barrier, options)
-            self.threads.append(t)
-            t.start()
+    def sql_count_objects(self):
+        return """SELECT count(*) FROM location_property_osmline
+                  WHERE indexed_status > 0"""

 
 

-    def run(self):
-        log.info("Starting indexing rank ({} to {}) using {} threads".format(
-                 self.options.minrank, self.options.maxrank,
-                 self.options.threads))
+    def sql_get_objects(self):
+        return """SELECT place_id FROM location_property_osmline
+                  WHERE indexed_status > 0
+                  ORDER BY geometry_sector"""

 
 

-        for rank in range(self.options.minrank, 30):
-            self.index(RankRunner(rank))
+    def sql_index_place(self, ids):
+        return """UPDATE location_property_osmline
+                  SET indexed_status = 0 WHERE place_id IN ({})"""\
+               .format(','.join((str(i) for i in ids)))

 
 

-        if self.options.maxrank >= 30:
-            self.index(InterpolationRunner())
-            self.index(RankRunner(30))
+class BoundaryRunner(object):
+    """ Returns SQL commands for indexing the administrative boundaries
+        of a certain rank.
+    """

 
 

-        self.queue_all(None)
-        for t in self.threads:
-            t.join()
+    def __init__(self, rank):
+        self.rank = rank

 
 

-    def queue_all(self, item):
-        for t in self.threads:
-            self.queue.put(item)
+    def name(self):
+        return "boundaries rank {}".format(self.rank)

 
 

-    def index(self, obj):
-        log.info("Starting {}".format(obj.name()))
+    def sql_count_objects(self):
+        return """SELECT count(*) FROM placex
+                  WHERE indexed_status > 0
+                    AND rank_search = {}
+                    AND class = 'boundary' and type = 'administrative'""".format(self.rank)

 
 

-        self.queue_all(obj.sql_index_place())
-        self.barrier.wait()
+    def sql_get_objects(self):
+        return """SELECT place_id FROM placex
+                  WHERE indexed_status > 0 and rank_search = {}
+                        and class = 'boundary' and type = 'administrative'
+                  ORDER BY partition, admin_level""".format(self.rank)

 
 

-        cur = self.conn.cursor(name="main")
-        cur.execute(obj.sql_index_sectors())
+    def sql_index_place(self, ids):
+        return "UPDATE placex SET indexed_status = 0 WHERE place_id IN ({})"\
+               .format(','.join((str(i) for i in ids)))

 
 

-        total_tuples = 0
-        for r in cur:
-            total_tuples += r[1]
-        log.debug("Total number of rows; {}".format(total_tuples))
+class Indexer(object):
+    """ Main indexing routine.
+    """

 
 

-        cur.scroll(0, mode='absolute')
+    def __init__(self, options):
+        self.minrank = max(1, options.minrank)
+        self.maxrank = min(30, options.maxrank)
+        self.conn = make_connection(options)
+        self.threads = [DBConnection(options) for i in range(options.threads)]

 
 

-        done_tuples = 0
-        rank_start_time = datetime.now()
-        for r in cur:
-            sector = r[0]
+    def index_boundaries(self):
+        log.warning("Starting indexing boundaries using {} threads".format(
+                      len(self.threads)))

 
 

-            # Should we do the remaining ones together?
-            do_all = total_tuples - done_tuples < len(self.threads) * 1000
+        for rank in range(max(self.minrank, 5), min(self.maxrank, 26)):
+            self.index(BoundaryRunner(rank))

 
 

-            pcur = self.conn.cursor(name='places')
+    def index_by_rank(self):
+        """ Run classic indexing by rank.
+        """
+        log.warning("Starting indexing rank ({} to {}) using {} threads".format(
+                 self.minrank, self.maxrank, len(self.threads)))

 
 

-            if do_all:
-                pcur.execute(obj.sql_nosector_places())
-            else:
-                pcur.execute(obj.sql_sector_places(), (sector, ))
+        for rank in range(max(1, self.minrank), self.maxrank):
+            self.index(RankRunner(rank))

 
 

-            for place in pcur:
-                place_id = place[0]
-                log.debug("Processing place {}".format(place_id))
+        if self.maxrank == 30:
+            self.index(RankRunner(0))
+            self.index(InterpolationRunner(), 20)
+            self.index(RankRunner(self.maxrank), 20)
+        else:
+            self.index(RankRunner(self.maxrank))

 
 

-                self.queue.put(place_id)
-                done_tuples += 1
+    def index(self, obj, batch=1):
+        """ Index a single rank or table. `obj` describes the SQL to use
+            for indexing. `batch` describes the number of objects that
+            should be processed with a single SQL statement
+        """
+        log.warning("Starting %s (using batch size %s)", obj.name(), batch)

 
 

-            pcur.close()
+        cur = self.conn.cursor()
+        cur.execute(obj.sql_count_objects())

 
 

-            if do_all:
-                break
+        total_tuples = cur.fetchone()[0]
+        log.debug("Total number of rows: {}".format(total_tuples))

 
         cur.close()
 
 
         cur.close()
 

-        self.queue_all("")
-        self.barrier.wait()
-
-        rank_end_time = datetime.now()
-        diff_seconds = (rank_end_time-rank_start_time).total_seconds()
-
-        log.info("Done {} in {} @ {} per second - FINISHED {}\n".format(
-                 done_tuples, int(diff_seconds),
-                 done_tuples/diff_seconds, obj.name()))
+        progress = ProgressLogger(obj.name(), total_tuples)

 
 

+        if total_tuples > 0:
+            cur = self.conn.cursor(name='places')
+            cur.execute(obj.sql_get_objects())

 
 

-class RankRunner(object):
-
-    def __init__(self, rank):
-        self.rank = rank
-
-    def name(self):
-        return "rank {}".format(self.rank)
+            next_thread = self.find_free_thread()
+            while True:
+                places = [p[0] for p in cur.fetchmany(batch)]
+                if len(places) == 0:
+                    break

 
 

-    @classmethod
-    def prepare(cls):
-        return """PREPARE rnk_index AS
-                  UPDATE placex
-                  SET indexed_status = 0 WHERE place_id = $1"""
+                log.debug("Processing places: {}".format(places))
+                thread = next(next_thread)

 
 

-    def sql_index_sectors(self):
-        return """SELECT geometry_sector, count(*) FROM placex
-                  WHERE rank_search = {} and indexed_status > 0
-                  GROUP BY geometry_sector
-                  ORDER BY geometry_sector""".format(self.rank)
+                thread.perform(obj.sql_index_place(places))
+                progress.add(len(places))

 
 

-    def sql_nosector_places(self):
-        return """SELECT place_id FROM placex
-                  WHERE indexed_status > 0 and rank_search = {}
-                  ORDER BY geometry_sector""".format(self.rank)
+            cur.close()

 
 

-    def sql_sector_places(self):
-        return """SELECT place_id FROM placex
-                  WHERE indexed_status > 0 and geometry_sector = %s
-                  ORDER BY geometry_sector"""
+            for t in self.threads:
+                t.wait()

 
 

-    def sql_index_place(self):
-        return "EXECUTE rnk_index(%s)"
+        progress.done()

 
 

+    def find_free_thread(self):
+        """ Generator that returns the next connection that is free for
+            sending a query.
+        """
+        ready = self.threads
+        command_stat = 0

 
 

-class InterpolationRunner(object):
-
-    def name(self):
-        return "interpolation lines (location_property_osmline)"
-
-    @classmethod
-    def prepare(cls):
-        return """PREPARE ipl_index AS
-                  UPDATE location_property_osmline
-                  SET indexed_status = 0 WHERE place_id = $1"""
-
-    def sql_index_sectors(self):
-        return """SELECT geometry_sector, count(*) FROM location_property_osmline
-                  WHERE indexed_status > 0
-                  GROUP BY geometry_sector
-                  ORDER BY geometry_sector"""
-
-    def sql_nosector_places(self):
-        return """SELECT place_id FROM location_property_osmline
-                  WHERE indexed_status > 0
-                  ORDER BY geometry_sector"""
-
-    def sql_sector_places(self):
-        return """SELECT place_id FROM location_property_osmline
-                  WHERE indexed_status > 0 and geometry_sector = %s
-                  ORDER BY geometry_sector"""
+        while True:
+            for thread in ready:
+                if thread.is_done():
+                    command_stat += 1
+                    yield thread
+
+            # refresh the connections occasionaly to avoid potential
+            # memory leaks in Postgresql.
+            if command_stat > 100000:
+                for t in self.threads:
+                    while not t.is_done():
+                        t.wait()
+                    t.connect()
+                command_stat = 0
+                ready = self.threads
+            else:
+                ready, _, _ = select.select(self.threads, [], [])

 
 

-    def sql_index_place(self):
-        return "EXECUTE ipl_index(%s)"
+        assert False, "Unreachable code"

 
 
 def nominatim_arg_parser():
 
 
 def nominatim_arg_parser():


@@ -242,7 +216,7 @@ def nominatim_arg_parser():
     def h(s):
         return re.sub("\s\s+" , " ", s)
 
     def h(s):
         return re.sub("\s\s+" , " ", s)
 

-    p = ArgumentParser(description=__doc__,
+    p = ArgumentParser(description="Indexing tool for Nominatim.",

                        formatter_class=RawDescriptionHelpFormatter)
 
     p.add_argument('-d', '--database',
                        formatter_class=RawDescriptionHelpFormatter)
 
     p.add_argument('-d', '--database',


@@ -260,6 +234,9 @@ def nominatim_arg_parser():
     p.add_argument('-P', '--port',
                    dest='port', action='store',
                    help='PostgreSQL server port')
     p.add_argument('-P', '--port',
                    dest='port', action='store',
                    help='PostgreSQL server port')

+    p.add_argument('-b', '--boundary-only',
+                   dest='boundary_only', action='store_true',
+                   help='Only index administrative boundaries (ignores min/maxrank).')

     p.add_argument('-r', '--minrank',
                    dest='minrank', type=int, metavar='RANK', default=0,
                    help='Minimum/starting rank.')
     p.add_argument('-r', '--minrank',
                    dest='minrank', type=int, metavar='RANK', default=0,
                    help='Minimum/starting rank.')


@@ -287,4 +264,7 @@ if __name__ == '__main__':
         password = getpass.getpass("Database password: ")
         options.password = password
 
         password = getpass.getpass("Database password: ")
         options.password = password
 

-    Indexer(options).run()
+    if options.boundary_only:
+        Indexer(options).index_boundaries()
+    else:
+        Indexer(options).index_by_rank()