Merge pull request #2681 from lonvia/improve-geocodejson

[nominatim.git] / nominatim / tools / special_phrases / sp_importer.py

diff --git a/nominatim/tools/special_phrases/sp_importer.py b/nominatim/tools/special_phrases/sp_importer.py
index 681990fa6175be2935f6b59e763fe4dab6239792..195f8387d3d98d33a3339ac5920203d3955e1262 100644 (file)
--- a/nominatim/tools/special_phrases/sp_importer.py
+++ b/nominatim/tools/special_phrases/sp_importer.py
@@ -1,3 +1,9 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# This file is part of Nominatim. (https://nominatim.org)
+#
+# Copyright (C) 2022 by the Nominatim developer community.
+# For a full list of authors see the git log.

 """
     Module containing the class handling the import
     of the special phrases.
 """
     Module containing the class handling the import
     of the special phrases.


@@ -8,15 +14,9 @@
     valids anymore are removed.
 """
 import logging
     valids anymore are removed.
 """
 import logging

-import os
-from os.path import isfile
-from pathlib import Path

 import re
 import re

-import subprocess
-import json

 
 from psycopg2.sql import Identifier, Literal, SQL
 
 from psycopg2.sql import Identifier, Literal, SQL

-from nominatim.errors import UsageError

 from nominatim.tools.special_phrases.importer_statistics import SpecialPhrasesImporterStatistics
 
 LOG = logging.getLogger()
 from nominatim.tools.special_phrases.importer_statistics import SpecialPhrasesImporterStatistics
 
 LOG = logging.getLogger()


@@ -33,9 +33,8 @@ class SPImporter():
 
         Take a sp loader which load the phrases from an external source.
     """
 
         Take a sp loader which load the phrases from an external source.
     """

-    def __init__(self, config, phplib_dir, db_connection, sp_loader) -> None:
+    def __init__(self, config, db_connection, sp_loader) -> None:

         self.config = config
         self.config = config

-        self.phplib_dir = phplib_dir

         self.db_connection = db_connection
         self.sp_loader = sp_loader
         self.statistics_handler = SpecialPhrasesImporterStatistics()
         self.db_connection = db_connection
         self.sp_loader = sp_loader
         self.statistics_handler = SpecialPhrasesImporterStatistics()


@@ -44,8 +43,8 @@ class SPImporter():
         # This set will contain all existing phrases to be added.
         # It contains tuples with the following format: (lable, class, type, operator)
         self.word_phrases = set()
         # This set will contain all existing phrases to be added.
         # It contains tuples with the following format: (lable, class, type, operator)
         self.word_phrases = set()

-        #This set will contain all existing place_classtype tables which doesn't match any
-        #special phrases class/type on the wiki.
+        # This set will contain all existing place_classtype tables which doesn't match any
+        # special phrases class/type on the wiki.

         self.table_phrases_to_delete = set()
 
     def import_phrases(self, tokenizer, should_replace):
         self.table_phrases_to_delete = set()
 
     def import_phrases(self, tokenizer, should_replace):


@@ -60,7 +59,7 @@ class SPImporter():
         LOG.warning('Special phrases importation starting')
         self._fetch_existing_place_classtype_tables()
 
         LOG.warning('Special phrases importation starting')
         self._fetch_existing_place_classtype_tables()
 

-        #Store pairs of class/type for further processing
+        # Store pairs of class/type for further processing

         class_type_pairs = set()
 
         for loaded_phrases in self.sp_loader:
         class_type_pairs = set()
 
         for loaded_phrases in self.sp_loader:


@@ -101,13 +100,8 @@ class SPImporter():
         """
             Load white and black lists from phrases-settings.json.
         """
         """
             Load white and black lists from phrases-settings.json.
         """

-        settings_path = (self.config.config_dir / 'phrase-settings.json').resolve()
+        settings = self.config.load_sub_configuration('phrase-settings.json')

 
 

-        if self.config.PHRASE_CONFIG:
-            settings_path = self._convert_php_settings_if_needed(self.config.PHRASE_CONFIG)
-
-        with settings_path.open("r") as json_settings:
-            settings = json.load(json_settings)

         return settings['blackList'], settings['whiteList']
 
     def _check_sanity(self, phrase):
         return settings['blackList'], settings['whiteList']
 
     def _check_sanity(self, phrase):


@@ -131,17 +125,17 @@ class SPImporter():
             Return the class/type pair corresponding to the phrase.
         """
 
             Return the class/type pair corresponding to the phrase.
         """
 

-        #blacklisting: disallow certain class/type combinations
+        # blacklisting: disallow certain class/type combinations

         if phrase.p_class in self.black_list.keys() \
            and phrase.p_type in self.black_list[phrase.p_class]:
             return None
 
         if phrase.p_class in self.black_list.keys() \
            and phrase.p_type in self.black_list[phrase.p_class]:
             return None
 

-        #whitelisting: if class is in whitelist, allow only tags in the list
+        # whitelisting: if class is in whitelist, allow only tags in the list

         if phrase.p_class in self.white_list.keys() \
            and phrase.p_type not in self.white_list[phrase.p_class]:
             return None
 
         if phrase.p_class in self.white_list.keys() \
            and phrase.p_type not in self.white_list[phrase.p_class]:
             return None
 

-        #sanity check, in case somebody added garbage in the wiki
+        # sanity check, in case somebody added garbage in the wiki

         if not self._check_sanity(phrase):
             self.statistics_handler.notify_one_phrase_invalid()
             return None
         if not self._check_sanity(phrase):
             self.statistics_handler.notify_one_phrase_invalid()
             return None


@@ -161,7 +155,7 @@ class SPImporter():
 
         sql_tablespace = self.config.TABLESPACE_AUX_DATA
         if sql_tablespace:
 
         sql_tablespace = self.config.TABLESPACE_AUX_DATA
         if sql_tablespace:

-            sql_tablespace = ' TABLESPACE '+sql_tablespace
+            sql_tablespace = ' TABLESPACE ' + sql_tablespace

 
         with self.db_connection.cursor() as db_cursor:
             db_cursor.execute("CREATE INDEX idx_placex_classtype ON placex (class, type)")
 
         with self.db_connection.cursor() as db_cursor:
             db_cursor.execute("CREATE INDEX idx_placex_classtype ON placex (class, type)")


@@ -174,19 +168,19 @@ class SPImporter():
 
             if table_name in self.table_phrases_to_delete:
                 self.statistics_handler.notify_one_table_ignored()
 
             if table_name in self.table_phrases_to_delete:
                 self.statistics_handler.notify_one_table_ignored()

-                #Remove this table from the ones to delete as it match a class/type
-                #still existing on the special phrases of the wiki.
+                # Remove this table from the ones to delete as it match a
+                # class/type still existing on the special phrases of the wiki.

                 self.table_phrases_to_delete.remove(table_name)
                 self.table_phrases_to_delete.remove(table_name)

-                #So dont need to create the table and indexes.
+                # So don't need to create the table and indexes.

                 continue
 
                 continue
 

-            #Table creation
+            # Table creation

             self._create_place_classtype_table(sql_tablespace, phrase_class, phrase_type)
 
             self._create_place_classtype_table(sql_tablespace, phrase_class, phrase_type)
 

-            #Indexes creation
+            # Indexes creation

             self._create_place_classtype_indexes(sql_tablespace, phrase_class, phrase_type)
 
             self._create_place_classtype_indexes(sql_tablespace, phrase_class, phrase_type)
 

-            #Grant access on read to the web user.
+            # Grant access on read to the web user.

             self._grant_access_to_webuser(phrase_class, phrase_type)
 
             self.statistics_handler.notify_one_table_created()
             self._grant_access_to_webuser(phrase_class, phrase_type)
 
             self.statistics_handler.notify_one_table_created()


@@ -202,8 +196,8 @@ class SPImporter():
         table_name = _classtype_table(phrase_class, phrase_type)
         with self.db_connection.cursor() as db_cursor:
             db_cursor.execute(SQL("""
         table_name = _classtype_table(phrase_class, phrase_type)
         with self.db_connection.cursor() as db_cursor:
             db_cursor.execute(SQL("""

-                    CREATE TABLE IF NOT EXISTS {{}} {} 
-                    AS SELECT place_id AS place_id,st_centroid(geometry) AS centroid FROM placex 
+                    CREATE TABLE IF NOT EXISTS {{}} {}
+                    AS SELECT place_id AS place_id,st_centroid(geometry) AS centroid FROM placex

                     WHERE class = {{}} AND type = {{}}""".format(sql_tablespace))
                               .format(Identifier(table_name), Literal(phrase_class),
                                       Literal(phrase_type)))
                     WHERE class = {{}} AND type = {{}}""".format(sql_tablespace))
                               .format(Identifier(table_name), Literal(phrase_class),
                                       Literal(phrase_type)))


@@ -215,7 +209,7 @@ class SPImporter():
         """
         index_prefix = 'idx_place_classtype_{}_{}_'.format(phrase_class, phrase_type)
         base_table = _classtype_table(phrase_class, phrase_type)
         """
         index_prefix = 'idx_place_classtype_{}_{}_'.format(phrase_class, phrase_type)
         base_table = _classtype_table(phrase_class, phrase_type)

-        #Index on centroid
+        # Index on centroid

         if not self.db_connection.index_exists(index_prefix + 'centroid'):
             with self.db_connection.cursor() as db_cursor:
                 db_cursor.execute(SQL("""
         if not self.db_connection.index_exists(index_prefix + 'centroid'):
             with self.db_connection.cursor() as db_cursor:
                 db_cursor.execute(SQL("""


@@ -223,7 +217,7 @@ class SPImporter():
                                   .format(Identifier(index_prefix + 'centroid'),
                                           Identifier(base_table)), sql_tablespace)
 
                                   .format(Identifier(index_prefix + 'centroid'),
                                           Identifier(base_table)), sql_tablespace)
 

-        #Index on place_id
+        # Index on place_id

         if not self.db_connection.index_exists(index_prefix + 'place_id'):
             with self.db_connection.cursor() as db_cursor:
                 db_cursor.execute(SQL(
         if not self.db_connection.index_exists(index_prefix + 'place_id'):
             with self.db_connection.cursor() as db_cursor:
                 db_cursor.execute(SQL(


@@ -248,40 +242,10 @@ class SPImporter():
             Delete the place_classtype tables.
         """
         LOG.warning('Cleaning database...')
             Delete the place_classtype tables.
         """
         LOG.warning('Cleaning database...')

-        #Array containing all queries to execute. Contain tuples of format (query, parameters)
-        queries_parameters = []
-
-        #Delete place_classtype tables corresponding to class/type which are not on the wiki anymore
-        for table in self.table_phrases_to_delete:
-            self.statistics_handler.notify_one_table_deleted()
-            query = SQL('DROP TABLE IF EXISTS {}').format(Identifier(table))
-            queries_parameters.append((query, ()))

 
 

+        # Delete place_classtype tables corresponding to class/type which
+        # are not on the wiki anymore.

         with self.db_connection.cursor() as db_cursor:
         with self.db_connection.cursor() as db_cursor:

-            for query, parameters in queries_parameters:
-                db_cursor.execute(query, parameters)
-
-    def _convert_php_settings_if_needed(self, file_path):
-        """
-            Convert php settings file of special phrases to json file if it is still in php format.
-        """
-        if not isfile(file_path):
-            raise UsageError(str(file_path) + ' is not a valid file.')
-
-        file, extension = os.path.splitext(file_path)
-        json_file_path = Path(file + '.json').resolve()
-
-        if extension not in('.php', '.json'):
-            raise UsageError('The custom NOMINATIM_PHRASE_CONFIG file has not a valid extension.')
-
-        if extension == '.php' and not isfile(json_file_path):
-            try:
-                subprocess.run(['/usr/bin/env', 'php', '-Cq',
-                                (self.phplib_dir / 'migration/PhraseSettingsToJson.php').resolve(),
-                                file_path], check=True)
-                LOG.warning('special_phrase configuration file has been converted to json.')
-            except subprocess.CalledProcessError:
-                LOG.error('Error while converting %s to json.', file_path)
-                raise
-
-        return json_file_path
+            for table in self.table_phrases_to_delete:
+                self.statistics_handler.notify_one_table_deleted()
+                db_cursor.drop_table(table)