From: AntoJvlt Date: Sun, 16 May 2021 14:59:12 +0000 (+0200) Subject: Code cleaning and SPLoader deleted X-Git-Tag: v4.0.0~79^2~4 X-Git-Url: https://git.openstreetmap.org./nominatim.git/commitdiff_plain/06aab389ede19c9d2a63630ddf0bae3bdc173f63?ds=sidebyside;hp=-c Code cleaning and SPLoader deleted --- 06aab389ede19c9d2a63630ddf0bae3bdc173f63 diff --git a/nominatim/clicmd/special_phrases.py b/nominatim/clicmd/special_phrases.py index ecd01c91..66fb41ed 100644 --- a/nominatim/clicmd/special_phrases.py +++ b/nominatim/clicmd/special_phrases.py @@ -46,7 +46,7 @@ class ImportSpecialPhrases: def start_import(args, loader): """ Create the SPImporter object containing the right - SPLoader and then start the import of special phrases. + sp loader and then start the import of special phrases. """ from ..tokenizer import factory as tokenizer_factory diff --git a/nominatim/tools/special_phrases/sp_csv_loader.py b/nominatim/tools/special_phrases/sp_csv_loader.py index cd0c2a84..b7b24a7d 100644 --- a/nominatim/tools/special_phrases/sp_csv_loader.py +++ b/nominatim/tools/special_phrases/sp_csv_loader.py @@ -5,14 +5,13 @@ """ import csv import os +from collections.abc import Iterator from nominatim.tools.special_phrases.special_phrase import SpecialPhrase -from nominatim.tools.special_phrases.sp_loader import SPLoader from nominatim.errors import UsageError -class SPCsvLoader(SPLoader): +class SPCsvLoader(Iterator): """ - Base class for special phrases loaders. - Handle the loading of special phrases from external sources. + Handles loading of special phrases from external csv file. """ def __init__(self, csv_path): super().__init__() @@ -24,18 +23,17 @@ class SPCsvLoader(SPLoader): raise StopIteration() self.has_been_read = True - SPCsvLoader.check_csv_validity(self.csv_path) - return SPCsvLoader.parse_csv(self.csv_path) + self.check_csv_validity() + return self.parse_csv() - @staticmethod - def parse_csv(csv_path): + def parse_csv(self): """ Open and parse the given csv file. Create the corresponding SpecialPhrases. """ phrases = set() - with open(csv_path) as file: + with open(self.csv_path) as file: reader = csv.DictReader(file, delimiter=',') for row in reader: phrases.add( @@ -43,12 +41,11 @@ class SPCsvLoader(SPLoader): ) return phrases - @staticmethod - def check_csv_validity(csv_path): + def check_csv_validity(self): """ Check that the csv file has the right extension. """ - _, extension = os.path.splitext(csv_path) + _, extension = os.path.splitext(self.csv_path) if extension != '.csv': - raise UsageError('The file {} is not a csv file.'.format(csv_path)) + raise UsageError('The file {} is not a csv file.'.format(self.csv_path)) diff --git a/nominatim/tools/special_phrases/sp_importer.py b/nominatim/tools/special_phrases/sp_importer.py index 1b42cb00..e82f721e 100644 --- a/nominatim/tools/special_phrases/sp_importer.py +++ b/nominatim/tools/special_phrases/sp_importer.py @@ -25,7 +25,7 @@ class SPImporter(): """ Class handling the process of special phrases importations into the database. - Take a SPLoader which load the phrases from an external source. + Take a sp loader which load the phrases from an external source. """ def __init__(self, config, phplib_dir, db_connection, sp_loader) -> None: self.config = config @@ -121,16 +121,14 @@ class SPImporter(): """ #blacklisting: disallow certain class/type combinations - if ( - phrase.p_class in self.black_list.keys() and - phrase.p_type in self.black_list[phrase.p_class] - ): return None + if phrase.p_class in self.black_list.keys() \ + and phrase.p_type in self.black_list[phrase.p_class]: + return None #whitelisting: if class is in whitelist, allow only tags in the list - if ( - phrase.p_class in self.white_list.keys() and - phrase.p_type not in self.white_list[phrase.p_class] - ): return None + if phrase.p_class in self.white_list.keys() \ + and phrase.p_type not in self.white_list[phrase.p_class]: + return None #sanity check, in case somebody added garbage in the wiki if not self._check_sanity(phrase): diff --git a/nominatim/tools/special_phrases/sp_loader.py b/nominatim/tools/special_phrases/sp_loader.py deleted file mode 100644 index e86d5221..00000000 --- a/nominatim/tools/special_phrases/sp_loader.py +++ /dev/null @@ -1,16 +0,0 @@ -""" - Module containing the SPLoader class. -""" -from abc import ABC, abstractmethod - -class SPLoader(ABC): - """ - Base class for special phrases loaders. - Handle the loading of special phrases from external sources. - """ - def __iter__(self): - return self - - @abstractmethod - def __next__(self): - pass diff --git a/nominatim/tools/special_phrases/sp_wiki_loader.py b/nominatim/tools/special_phrases/sp_wiki_loader.py index 4990eef2..e9cbfdb2 100644 --- a/nominatim/tools/special_phrases/sp_wiki_loader.py +++ b/nominatim/tools/special_phrases/sp_wiki_loader.py @@ -3,12 +3,12 @@ """ import re import logging +from collections.abc import Iterator from nominatim.tools.special_phrases.special_phrase import SpecialPhrase -from nominatim.tools.special_phrases.sp_loader import SPLoader from nominatim.tools.exec_utils import get_url LOG = logging.getLogger() -class SPWikiLoader(SPLoader): +class SPWikiLoader(Iterator): """ Handles loading of special phrases from the wiki. """ @@ -28,7 +28,7 @@ class SPWikiLoader(SPLoader): raise StopIteration lang = self.languages.pop(0) - loaded_xml = SPWikiLoader._get_wiki_content(lang) + loaded_xml = self._get_wiki_content(lang) LOG.warning('Importing phrases for lang: %s...', lang) return self.parse_xml(loaded_xml) diff --git a/test/python/test_tools_sp_csv_loader.py b/test/python/test_tools_sp_csv_loader.py index aac8593b..628dfd16 100644 --- a/test/python/test_tools_sp_csv_loader.py +++ b/test/python/test_tools_sp_csv_loader.py @@ -13,7 +13,7 @@ def test_parse_csv(sp_csv_loader): Test method parse_csv() Should return the right SpecialPhrase objects. """ - phrases = sp_csv_loader.parse_csv(sp_csv_loader.csv_path) + phrases = sp_csv_loader.parse_csv() assert check_phrases_content(phrases) @@ -33,10 +33,10 @@ def test_check_csv_validity(sp_csv_loader): different exception than .csv is given. """ sp_csv_loader.csv_path = 'test.csv' - sp_csv_loader.check_csv_validity(sp_csv_loader.csv_path) + sp_csv_loader.check_csv_validity() sp_csv_loader.csv_path = 'test.wrong' with pytest.raises(UsageError): - assert sp_csv_loader.check_csv_validity(sp_csv_loader.csv_path) + assert sp_csv_loader.check_csv_validity() def check_phrases_content(phrases): """