Merge pull request #2655 from lonvia/migration-internal-country-name

[nominatim.git] / nominatim / indexer / runners.py

diff --git a/nominatim/indexer/runners.py b/nominatim/indexer/runners.py
index 75429fe427428a9bc79b7c9ec9a4f491a873028c..ac7a0015a78a27469f6edaa85f3ae3dbfe70fff5 100644 (file)
--- a/nominatim/indexer/runners.py
+++ b/nominatim/indexer/runners.py
@@ -1,17 +1,30 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# This file is part of Nominatim. (https://nominatim.org)
+#
+# Copyright (C) 2022 by the Nominatim developer community.
+# For a full list of authors see the git log.

 """
 Mix-ins that provide the actual commands for the indexer for various indexing
 tasks.
 """
 import functools
 
 """
 Mix-ins that provide the actual commands for the indexer for various indexing
 tasks.
 """
 import functools
 

-import psycopg2.extras
+from psycopg2 import sql as pysql
+
+from nominatim.indexer.place_info import PlaceInfo

 
 # pylint: disable=C0111
 
 
 # pylint: disable=C0111
 

+def _mk_valuelist(template, num):
+    return pysql.SQL(',').join([pysql.SQL(template)] * num)
+
+

 class AbstractPlacexRunner:
     """ Returns SQL commands for indexing of the placex table.
     """
 class AbstractPlacexRunner:
     """ Returns SQL commands for indexing of the placex table.
     """

-    SELECT_SQL = 'SELECT place_id, (placex_prepare_update(placex)).* FROM placex'
+    SELECT_SQL = pysql.SQL('SELECT place_id FROM placex ')
+    UPDATE_LINE = "(%s, %s::hstore, %s::hstore, %s::int, %s::jsonb)"

 
     def __init__(self, rank, analyzer):
         self.rank = rank
 
     def __init__(self, rank, analyzer):
         self.rank = rank


@@ -21,18 +34,29 @@ class AbstractPlacexRunner:
     @staticmethod
     @functools.lru_cache(maxsize=1)
     def _index_sql(num_places):
     @staticmethod
     @functools.lru_cache(maxsize=1)
     def _index_sql(num_places):

-        return """ UPDATE placex
-                   SET indexed_status = 0, address = v.addr, token_info = v.ti
-                   FROM (VALUES {}) as v(id, addr, ti)
-                   WHERE place_id = v.id
-               """.format(','.join(["(%s, %s::hstore, %s::jsonb)"]  * num_places))
+        return pysql.SQL(
+            """ UPDATE placex
+                SET indexed_status = 0, address = v.addr, token_info = v.ti,
+                    name = v.name, linked_place_id = v.linked_place_id
+                FROM (VALUES {}) as v(id, name, addr, linked_place_id, ti)
+                WHERE place_id = v.id
+            """).format(_mk_valuelist(AbstractPlacexRunner.UPDATE_LINE, num_places))
+
+
+    @staticmethod
+    def get_place_details(worker, ids):
+        worker.perform("""SELECT place_id, extra.*
+                          FROM placex, LATERAL placex_indexing_prepare(placex) as extra
+                          WHERE place_id IN %s""",
+                       (tuple((p[0] for p in ids)), ))

 
 
     def index_places(self, worker, places):
         values = []
         for place in places:
 
 
     def index_places(self, worker, places):
         values = []
         for place in places:

-            values.extend((place[x] for x in ('place_id', 'address')))
-            values.append(psycopg2.extras.Json(self.analyzer.process_place(place)))
+            for field in ('place_id', 'name', 'address', 'linked_place_id'):
+                values.append(place[field])
+            values.append(PlaceInfo(place).analyze(self.analyzer))

 
         worker.perform(self._index_sql(len(places)), values)
 
 
         worker.perform(self._index_sql(len(places)), values)
 


@@ -45,14 +69,15 @@ class RankRunner(AbstractPlacexRunner):
         return "rank {}".format(self.rank)
 
     def sql_count_objects(self):
         return "rank {}".format(self.rank)
 
     def sql_count_objects(self):

-        return """SELECT count(*) FROM placex
-                  WHERE rank_address = {} and indexed_status > 0
-               """.format(self.rank)
+        return pysql.SQL("""SELECT count(*) FROM placex
+                            WHERE rank_address = {} and indexed_status > 0
+                         """).format(pysql.Literal(self.rank))

 
     def sql_get_objects(self):
 
     def sql_get_objects(self):

-        return """{} WHERE indexed_status > 0 and rank_address = {}
-                     ORDER BY geometry_sector
-               """.format(self.SELECT_SQL, self.rank)
+        return self.SELECT_SQL + pysql.SQL(
+            """WHERE indexed_status > 0 and rank_address = {}
+               ORDER BY geometry_sector
+            """).format(pysql.Literal(self.rank))

 
 
 class BoundaryRunner(AbstractPlacexRunner):
 
 
 class BoundaryRunner(AbstractPlacexRunner):


@@ -64,17 +89,18 @@ class BoundaryRunner(AbstractPlacexRunner):
         return "boundaries rank {}".format(self.rank)
 
     def sql_count_objects(self):
         return "boundaries rank {}".format(self.rank)
 
     def sql_count_objects(self):

-        return """SELECT count(*) FROM placex
-                  WHERE indexed_status > 0
-                    AND rank_search = {}
-                    AND class = 'boundary' and type = 'administrative'
-               """.format(self.rank)
+        return pysql.SQL("""SELECT count(*) FROM placex
+                            WHERE indexed_status > 0
+                              AND rank_search = {}
+                              AND class = 'boundary' and type = 'administrative'
+                         """).format(pysql.Literal(self.rank))

 
     def sql_get_objects(self):
 
     def sql_get_objects(self):

-        return """{} WHERE indexed_status > 0 and rank_search = {}
-                           and class = 'boundary' and type = 'administrative'
-                     ORDER BY partition, admin_level
-               """.format(self.SELECT_SQL, self.rank)
+        return self.SELECT_SQL + pysql.SQL(
+            """WHERE indexed_status > 0 and rank_search = {}
+                     and class = 'boundary' and type = 'administrative'
+               ORDER BY partition, admin_level
+            """).format(pysql.Literal(self.rank))

 
 
 class InterpolationRunner:
 
 
 class InterpolationRunner:


@@ -97,27 +123,34 @@ class InterpolationRunner:
 
     @staticmethod
     def sql_get_objects():
 
     @staticmethod
     def sql_get_objects():

-        return """SELECT place_id, get_interpolation_address(address, osm_id) as address
+        return """SELECT place_id

                   FROM location_property_osmline
                   WHERE indexed_status > 0
                   ORDER BY geometry_sector"""
 
 
                   FROM location_property_osmline
                   WHERE indexed_status > 0
                   ORDER BY geometry_sector"""
 
 

+    @staticmethod
+    def get_place_details(worker, ids):
+        worker.perform("""SELECT place_id, get_interpolation_address(address, osm_id) as address
+                          FROM location_property_osmline WHERE place_id IN %s""",
+                       (tuple((p[0] for p in ids)), ))
+
+

     @staticmethod
     @functools.lru_cache(maxsize=1)
     def _index_sql(num_places):
     @staticmethod
     @functools.lru_cache(maxsize=1)
     def _index_sql(num_places):

-        return """ UPDATE location_property_osmline
-                   SET indexed_status = 0, address = v.addr, token_info = v.ti
-                   FROM (VALUES {}) as v(id, addr, ti)
-                   WHERE place_id = v.id
-               """.format(','.join(["(%s, %s::hstore, %s::jsonb)"]  * num_places))
+        return pysql.SQL("""UPDATE location_property_osmline
+                            SET indexed_status = 0, address = v.addr, token_info = v.ti
+                            FROM (VALUES {}) as v(id, addr, ti)
+                            WHERE place_id = v.id
+                         """).format(_mk_valuelist("(%s, %s::hstore, %s::jsonb)", num_places))

 
 
     def index_places(self, worker, places):
         values = []
         for place in places:
             values.extend((place[x] for x in ('place_id', 'address')))
 
 
     def index_places(self, worker, places):
         values = []
         for place in places:
             values.extend((place[x] for x in ('place_id', 'address')))

-            values.append(psycopg2.extras.Json(self.analyzer.process_place(place)))
+            values.append(PlaceInfo(place).analyze(self.analyzer))

 
         worker.perform(self._index_sql(len(places)), values)
 
 
         worker.perform(self._index_sql(len(places)), values)
 


@@ -143,6 +176,6 @@ class PostcodeRunner:
 
     @staticmethod
     def index_places(worker, ids):
 
     @staticmethod
     def index_places(worker, ids):

-        worker.perform(""" UPDATE location_postcode SET indexed_status = 0
-                           WHERE place_id IN ({})
-                       """.format(','.join((str(i[0]) for i in ids))))
+        worker.perform(pysql.SQL("""UPDATE location_postcode SET indexed_status = 0
+                                    WHERE place_id IN ({})""")
+                       .format(pysql.SQL(',').join((pysql.Literal(i[0]) for i in ids))))