From 83054af46fb21ea40c0bebb2c8a528937e5a49a0 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Sun, 17 Jul 2022 23:18:55 +0200 Subject: [PATCH 1/1] remove typing_extensions requirement The typing_extensions package is only necessary now when running mypy. It won't be used at runtime anymore. --- docs/admin/Installation.md | 1 - docs/develop/Development-Environment.md | 1 + nominatim/clicmd/args.py | 3 +-- nominatim/db/status.py | 3 +-- nominatim/indexer/runners.py | 3 +-- nominatim/tokenizer/base.py | 3 +-- nominatim/tokenizer/icu_rule_loader.py | 10 +++++----- nominatim/tokenizer/icu_token_analysis.py | 2 +- nominatim/tokenizer/sanitizers/base.py | 2 +- nominatim/tokenizer/token_analysis/base.py | 13 +++++-------- nominatim/tools/special_phrases/sp_importer.py | 3 +-- nominatim/typing.py | 16 +++++++++++++++- vagrant/Install-on-Ubuntu-18.sh | 2 +- vagrant/Install-on-Ubuntu-20.sh | 5 ----- vagrant/Install-on-Ubuntu-22.sh | 1 - 15 files changed, 34 insertions(+), 34 deletions(-) diff --git a/docs/admin/Installation.md b/docs/admin/Installation.md index d7d35b7e..96546cf3 100644 --- a/docs/admin/Installation.md +++ b/docs/admin/Installation.md @@ -45,7 +45,6 @@ For running Nominatim: * [PostgreSQL](https://www.postgresql.org) (9.6+ will work, 11+ strongly recommended) * [PostGIS](https://postgis.net) (2.2+ will work, 3.0+ strongly recommended) * [Python 3](https://www.python.org/) (3.6+) - * [Python Typing Extensions](https://github.com/python/typing_extensions) * [Psycopg2](https://www.psycopg.org) (2.7+) * [Python Dotenv](https://github.com/theskumar/python-dotenv) * [psutil](https://github.com/giampaolo/psutil) diff --git a/docs/develop/Development-Environment.md b/docs/develop/Development-Environment.md index 05bd50de..65dc7990 100644 --- a/docs/develop/Development-Environment.md +++ b/docs/develop/Development-Environment.md @@ -34,6 +34,7 @@ It has the following additional requirements: * [PHP CodeSniffer](https://github.com/squizlabs/PHP_CodeSniffer) * [Pylint](https://pylint.org/) (CI always runs the latest version from pip) * [mypy](http://mypy-lang.org/) (plus typing information for external libs) +* [Python Typing Extensions](https://github.com/python/typing_extensions) (for Python < 3.9) * [pytest](https://pytest.org) The documentation is built with mkdocs: diff --git a/nominatim/clicmd/args.py b/nominatim/clicmd/args.py index f5f1b98e..c976f394 100644 --- a/nominatim/clicmd/args.py +++ b/nominatim/clicmd/args.py @@ -12,10 +12,9 @@ import argparse import logging from pathlib import Path -from typing_extensions import Protocol - from nominatim.errors import UsageError from nominatim.config import Configuration +from nominatim.typing import Protocol LOG = logging.getLogger() diff --git a/nominatim/db/status.py b/nominatim/db/status.py index f0039546..aea25a97 100644 --- a/nominatim/db/status.py +++ b/nominatim/db/status.py @@ -12,11 +12,10 @@ import datetime as dt import logging import re -from typing_extensions import TypedDict - from nominatim.db.connection import Connection from nominatim.tools.exec_utils import get_url from nominatim.errors import UsageError +from nominatim.typing import TypedDict LOG = logging.getLogger() ISODATE_FORMAT = '%Y-%m-%dT%H:%M:%S' diff --git a/nominatim/indexer/runners.py b/nominatim/indexer/runners.py index 973c6ea9..bbadd282 100644 --- a/nominatim/indexer/runners.py +++ b/nominatim/indexer/runners.py @@ -11,14 +11,13 @@ tasks. from typing import Any, List import functools -from typing_extensions import Protocol from psycopg2 import sql as pysql import psycopg2.extras from nominatim.data.place_info import PlaceInfo from nominatim.tokenizer.base import AbstractAnalyzer from nominatim.db.async_connection import DBConnection -from nominatim.typing import Query, DictCursorResult, DictCursorResults +from nominatim.typing import Query, DictCursorResult, DictCursorResults, Protocol # pylint: disable=C0111 diff --git a/nominatim/tokenizer/base.py b/nominatim/tokenizer/base.py index 1c1ca9f7..dbc4cfad 100644 --- a/nominatim/tokenizer/base.py +++ b/nominatim/tokenizer/base.py @@ -12,10 +12,9 @@ from abc import ABC, abstractmethod from typing import List, Tuple, Dict, Any, Optional, Iterable from pathlib import Path -from typing_extensions import Protocol - from nominatim.config import Configuration from nominatim.data.place_info import PlaceInfo +from nominatim.typing import Protocol class AbstractAnalyzer(ABC): """ The analyzer provides the functions for analysing names and building diff --git a/nominatim/tokenizer/icu_rule_loader.py b/nominatim/tokenizer/icu_rule_loader.py index 7199f5f5..84040ddc 100644 --- a/nominatim/tokenizer/icu_rule_loader.py +++ b/nominatim/tokenizer/icu_rule_loader.py @@ -7,7 +7,7 @@ """ Helper class to create ICU rules from a configuration file. """ -from typing import Mapping, Any, Generic, Dict, Optional +from typing import Mapping, Any, Dict, Optional import importlib import io import json @@ -19,7 +19,7 @@ from nominatim.db.connection import Connection from nominatim.errors import UsageError from nominatim.tokenizer.place_sanitizer import PlaceSanitizer from nominatim.tokenizer.icu_token_analysis import ICUTokenAnalysis -from nominatim.tokenizer.token_analysis.base import AnalysisModule, Analyser, T_config +from nominatim.tokenizer.token_analysis.base import AnalysisModule, Analyser import nominatim.data.country_info LOG = logging.getLogger() @@ -130,7 +130,7 @@ class ICURuleLoader: def _setup_analysis(self) -> None: """ Process the rules used for creating the various token analyzers. """ - self.analysis: Dict[Optional[str], TokenAnalyzerRule[Any]] = {} + self.analysis: Dict[Optional[str], TokenAnalyzerRule] = {} if not isinstance(self.analysis_rules, list): raise UsageError("Configuration section 'token-analysis' must be a list.") @@ -163,7 +163,7 @@ class ICURuleLoader: return ';'.join(flatten_config_list(content, section)) + ';' -class TokenAnalyzerRule(Generic[T_config]): +class TokenAnalyzerRule: """ Factory for a single analysis module. The class saves the configuration and creates a new token analyzer on request. """ @@ -172,7 +172,7 @@ class TokenAnalyzerRule(Generic[T_config]): # Find the analysis module module_name = 'nominatim.tokenizer.token_analysis.' \ + _get_section(rules, 'analyzer').replace('-', '_') - self._analysis_mod: AnalysisModule[T_config] = importlib.import_module(module_name) + self._analysis_mod: AnalysisModule = importlib.import_module(module_name) # Load the configuration. self.config = self._analysis_mod.configure(rules, normalization_rules) diff --git a/nominatim/tokenizer/icu_token_analysis.py b/nominatim/tokenizer/icu_token_analysis.py index 62e6a80a..3c4d7298 100644 --- a/nominatim/tokenizer/icu_token_analysis.py +++ b/nominatim/tokenizer/icu_token_analysis.py @@ -23,7 +23,7 @@ class ICUTokenAnalysis: """ def __init__(self, norm_rules: str, trans_rules: str, - analysis_rules: Mapping[Optional[str], 'TokenAnalyzerRule[Any]']): + analysis_rules: Mapping[Optional[str], 'TokenAnalyzerRule']): self.normalizer = Transliterator.createFromRules("icu_normalization", norm_rules) trans_rules += ";[:Space:]+ > ' '" diff --git a/nominatim/tokenizer/sanitizers/base.py b/nominatim/tokenizer/sanitizers/base.py index f2e1bc41..692c6d5f 100644 --- a/nominatim/tokenizer/sanitizers/base.py +++ b/nominatim/tokenizer/sanitizers/base.py @@ -9,9 +9,9 @@ Common data types and protocols for sanitizers. """ from typing import Optional, Dict, List, Mapping, Callable -from typing_extensions import Protocol, Final from nominatim.tokenizer.sanitizers.config import SanitizerConfig from nominatim.data.place_info import PlaceInfo +from nominatim.typing import Protocol, Final class PlaceName: """ A searchable name for a place together with properties. diff --git a/nominatim/tokenizer/token_analysis/base.py b/nominatim/tokenizer/token_analysis/base.py index b55b4f7c..b2a4386c 100644 --- a/nominatim/tokenizer/token_analysis/base.py +++ b/nominatim/tokenizer/token_analysis/base.py @@ -7,12 +7,9 @@ """ Common data types and protocols for analysers. """ -from typing import TypeVar, Mapping, List, Any +from typing import Mapping, List, Any -from typing_extensions import Protocol - - -T_config = TypeVar('T_config') # pylint: disable=invalid-name +from nominatim.typing import Protocol class Analyser(Protocol): """ Instance of the token analyser. @@ -28,17 +25,17 @@ class Analyser(Protocol): and transliterate the result. """ -class AnalysisModule(Protocol[T_config]): +class AnalysisModule(Protocol): """ Protocol for analysis modules. """ - def configure(self, rules: Mapping[str, Any], normalization_rules: str) -> T_config: + def configure(self, rules: Mapping[str, Any], normalization_rules: str) -> Any: """ Prepare the configuration of the analysis module. This function should prepare all data that can be shared between instances of this analyser. """ - def create(self, normalizer: Any, transliterator: Any, config: T_config) -> Analyser: + def create(self, normalizer: Any, transliterator: Any, config: Any) -> Analyser: """ Create a new instance of the analyser. A separate instance of the analyser is created for each thread when used in multi-threading context. diff --git a/nominatim/tools/special_phrases/sp_importer.py b/nominatim/tools/special_phrases/sp_importer.py index 6ca6a1e1..8906e03e 100644 --- a/nominatim/tools/special_phrases/sp_importer.py +++ b/nominatim/tools/special_phrases/sp_importer.py @@ -17,8 +17,6 @@ from typing import Iterable, Tuple, Mapping, Sequence, Optional, Set import logging import re -from typing_extensions import Protocol - from psycopg2.sql import Identifier, SQL from nominatim.config import Configuration @@ -26,6 +24,7 @@ from nominatim.db.connection import Connection from nominatim.tools.special_phrases.importer_statistics import SpecialPhrasesImporterStatistics from nominatim.tools.special_phrases.special_phrase import SpecialPhrase from nominatim.tokenizer.base import AbstractTokenizer +from nominatim.typing import Protocol LOG = logging.getLogger() diff --git a/nominatim/typing.py b/nominatim/typing.py index 36bde834..308f3e6a 100644 --- a/nominatim/typing.py +++ b/nominatim/typing.py @@ -12,7 +12,8 @@ Complex type definitions are moved here, to keep the source files readable. from typing import Any, Union, Mapping, TypeVar, Sequence, TYPE_CHECKING # Generics varaible names do not confirm to naming styles, ignore globally here. -# pylint: disable=invalid-name,abstract-method,multiple-statements,missing-class-docstring +# pylint: disable=invalid-name,abstract-method,multiple-statements +# pylint: disable=missing-class-docstring,useless-import-alias if TYPE_CHECKING: import psycopg2.sql @@ -36,3 +37,16 @@ class DictCursorResult(Mapping[str, Any]): DictCursorResults = Sequence[DictCursorResult] T_cursor = TypeVar('T_cursor', bound='psycopg2.extensions.cursor') + +# The following typing features require typing_extensions to work +# on all supported Python versions. +# Only require this for type checking but not for normal operations. + +if TYPE_CHECKING: + from typing_extensions import (Protocol as Protocol, + Final as Final, + TypedDict as TypedDict) +else: + Protocol = object + Final = 'Final' + TypedDict = dict diff --git a/vagrant/Install-on-Ubuntu-18.sh b/vagrant/Install-on-Ubuntu-18.sh index b853cdb5..3537bcf4 100755 --- a/vagrant/Install-on-Ubuntu-18.sh +++ b/vagrant/Install-on-Ubuntu-18.sh @@ -33,7 +33,7 @@ export DEBIAN_FRONTEND=noninteractive #DOCS: # Some of the Python packages that come with Ubuntu 18.04 are too old, so # install the latest version from pip: - pip3 install --user python-dotenv datrie pyyaml psycopg2-binary typing-extensions + pip3 install --user python-dotenv datrie pyyaml psycopg2-binary # # System Configuration diff --git a/vagrant/Install-on-Ubuntu-20.sh b/vagrant/Install-on-Ubuntu-20.sh index 365f609f..8dcc2fef 100755 --- a/vagrant/Install-on-Ubuntu-20.sh +++ b/vagrant/Install-on-Ubuntu-20.sh @@ -30,11 +30,6 @@ export DEBIAN_FRONTEND=noninteractive #DOCS: python3-psycopg2 python3-psutil python3-jinja2 \ python3-icu python3-datrie python3-yaml python3-pip git -# Nominatim uses some typing features only available in later Python versions. -# Install the latest version of the backport package: - - pip3 install --user typing-extensions - # # System Configuration # ==================== diff --git a/vagrant/Install-on-Ubuntu-22.sh b/vagrant/Install-on-Ubuntu-22.sh index b86f1f65..dbb70ffe 100755 --- a/vagrant/Install-on-Ubuntu-22.sh +++ b/vagrant/Install-on-Ubuntu-22.sh @@ -28,7 +28,6 @@ export DEBIAN_FRONTEND=noninteractive #DOCS: postgresql-contrib-14 postgresql-14-postgis-3-scripts \ php php-pgsql php-intl libicu-dev python3-dotenv \ python3-psycopg2 python3-psutil python3-jinja2 \ - python3-typing-extensions \ python3-icu python3-datrie git # -- 2.39.5