nominatim/indexer/runners.py

   1 # SPDX-License-Identifier: GPL-2.0-only
   2 #
   3 # This file is part of Nominatim. (https://nominatim.org)
   4 #
   5 # Copyright (C) 2022 by the Nominatim developer community.
   6 # For a full list of authors see the git log.
   7 """
   8 Mix-ins that provide the actual commands for the indexer for various indexing
   9 tasks.
  10 """
  11 import functools
  12
  13 from psycopg2 import sql as pysql
  14
  15 from nominatim.indexer.place_info import PlaceInfo
  16
  17 # pylint: disable=C0111
  18
  19 def _mk_valuelist(template, num):
  20     return pysql.SQL(',').join([pysql.SQL(template)] * num)
  21
  22
  23 class AbstractPlacexRunner:
  24     """ Returns SQL commands for indexing of the placex table.
  25     """
  26     SELECT_SQL = pysql.SQL('SELECT place_id FROM placex ')
  27     UPDATE_LINE = "(%s, %s::hstore, %s::hstore, %s::int, %s::jsonb)"
  28
  29     def __init__(self, rank, analyzer):
  30         self.rank = rank
  31         self.analyzer = analyzer
  32
  33
  34     @staticmethod
  35     @functools.lru_cache(maxsize=1)
  36     def _index_sql(num_places):
  37         return pysql.SQL(
  38             """ UPDATE placex
  39                 SET indexed_status = 0, address = v.addr, token_info = v.ti,
  40                     name = v.name, linked_place_id = v.linked_place_id
  41                 FROM (VALUES {}) as v(id, name, addr, linked_place_id, ti)
  42                 WHERE place_id = v.id
  43             """).format(_mk_valuelist(AbstractPlacexRunner.UPDATE_LINE, num_places))
  44
  45
  46     @staticmethod
  47     def get_place_details(worker, ids):
  48         worker.perform("""SELECT place_id, (placex_indexing_prepare(placex)).*
  49                           FROM placex WHERE place_id IN %s""",
  50                        (tuple((p[0] for p in ids)), ))
  51
  52
  53     def index_places(self, worker, places):
  54         values = []
  55         for place in places:
  56             for field in ('place_id', 'name', 'address', 'linked_place_id'):
  57                 values.append(place[field])
  58             values.append(PlaceInfo(place).analyze(self.analyzer))
  59
  60         worker.perform(self._index_sql(len(places)), values)
  61
  62
  63 class RankRunner(AbstractPlacexRunner):
  64     """ Returns SQL commands for indexing one rank within the placex table.
  65     """
  66
  67     def name(self):
  68         return "rank {}".format(self.rank)
  69
  70     def sql_count_objects(self):
  71         return pysql.SQL("""SELECT count(*) FROM placex
  72                             WHERE rank_address = {} and indexed_status > 0
  73                          """).format(pysql.Literal(self.rank))
  74
  75     def sql_get_objects(self):
  76         return self.SELECT_SQL + pysql.SQL(
  77             """WHERE indexed_status > 0 and rank_address = {}
  78                ORDER BY geometry_sector
  79             """).format(pysql.Literal(self.rank))
  80
  81
  82 class BoundaryRunner(AbstractPlacexRunner):
  83     """ Returns SQL commands for indexing the administrative boundaries
  84         of a certain rank.
  85     """
  86
  87     def name(self):
  88         return "boundaries rank {}".format(self.rank)
  89
  90     def sql_count_objects(self):
  91         return pysql.SQL("""SELECT count(*) FROM placex
  92                             WHERE indexed_status > 0
  93                               AND rank_search = {}
  94                               AND class = 'boundary' and type = 'administrative'
  95                          """).format(pysql.Literal(self.rank))
  96
  97     def sql_get_objects(self):
  98         return self.SELECT_SQL + pysql.SQL(
  99             """WHERE indexed_status > 0 and rank_search = {}
 100                      and class = 'boundary' and type = 'administrative'
 101                ORDER BY partition, admin_level
 102             """).format(pysql.Literal(self.rank))
 103
 104
 105 class InterpolationRunner:
 106     """ Returns SQL commands for indexing the address interpolation table
 107         location_property_osmline.
 108     """
 109
 110     def __init__(self, analyzer):
 111         self.analyzer = analyzer
 112
 113
 114     @staticmethod
 115     def name():
 116         return "interpolation lines (location_property_osmline)"
 117
 118     @staticmethod
 119     def sql_count_objects():
 120         return """SELECT count(*) FROM location_property_osmline
 121                   WHERE indexed_status > 0"""
 122
 123     @staticmethod
 124     def sql_get_objects():
 125         return """SELECT place_id
 126                   FROM location_property_osmline
 127                   WHERE indexed_status > 0
 128                   ORDER BY geometry_sector"""
 129
 130
 131     @staticmethod
 132     def get_place_details(worker, ids):
 133         worker.perform("""SELECT place_id, get_interpolation_address(address, osm_id) as address
 134                           FROM location_property_osmline WHERE place_id IN %s""",
 135                        (tuple((p[0] for p in ids)), ))
 136
 137
 138     @staticmethod
 139     @functools.lru_cache(maxsize=1)
 140     def _index_sql(num_places):
 141         return pysql.SQL("""UPDATE location_property_osmline
 142                             SET indexed_status = 0, address = v.addr, token_info = v.ti
 143                             FROM (VALUES {}) as v(id, addr, ti)
 144                             WHERE place_id = v.id
 145                          """).format(_mk_valuelist("(%s, %s::hstore, %s::jsonb)", num_places))
 146
 147
 148     def index_places(self, worker, places):
 149         values = []
 150         for place in places:
 151             values.extend((place[x] for x in ('place_id', 'address')))
 152             values.append(PlaceInfo(place).analyze(self.analyzer))
 153
 154         worker.perform(self._index_sql(len(places)), values)
 155
 156
 157
 158 class PostcodeRunner:
 159     """ Provides the SQL commands for indexing the location_postcode table.
 160     """
 161
 162     @staticmethod
 163     def name():
 164         return "postcodes (location_postcode)"
 165
 166     @staticmethod
 167     def sql_count_objects():
 168         return 'SELECT count(*) FROM location_postcode WHERE indexed_status > 0'
 169
 170     @staticmethod
 171     def sql_get_objects():
 172         return """SELECT place_id FROM location_postcode
 173                   WHERE indexed_status > 0
 174                   ORDER BY country_code, postcode"""
 175
 176     @staticmethod
 177     def index_places(worker, ids):
 178         worker.perform(pysql.SQL("""UPDATE location_postcode SET indexed_status = 0
 179                                     WHERE place_id IN ({})""")
 180                        .format(pysql.SQL(',').join((pysql.Literal(i[0]) for i in ids))))