From 4da4cbfe27a576ae011430b2de205c74435e241b Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Thu, 27 Jun 2024 21:26:12 +0200 Subject: [PATCH] reduce from 3 to 2 packages --- CMakeLists.txt | 20 +++---- Makefile | 8 +-- docs/admin/Installation.md | 2 +- docs/library/Configuration.md | 2 +- packaging/nominatim-api/extra_src/paths.py | 15 +++++ packaging/nominatim-api/pyproject.toml | 18 +++++- .../settings | 0 packaging/nominatim-core/COPYING | 1 - packaging/nominatim-core/README.md | 24 -------- packaging/nominatim-core/data | 1 - packaging/nominatim-core/pyproject.toml | 60 ------------------- packaging/nominatim-core/src | 1 - packaging/nominatim-db/data | 1 + .../extra_src/nominatim_db}/paths.py | 0 .../{nominatim-core => nominatim-db}/lib-sql | 0 packaging/nominatim-db/pyproject.toml | 30 +++++++++- packaging/nominatim-db/settings | 1 + src/nominatim_api/__init__.py | 4 +- src/nominatim_api/config.py | 12 ++++ src/nominatim_api/connection.py | 6 +- src/nominatim_api/core.py | 8 +-- .../errors.py | 0 src/nominatim_api/lookup.py | 2 +- src/nominatim_api/results.py | 4 +- src/nominatim_api/reverse.py | 6 +- src/nominatim_api/search/db_search_fields.py | 4 +- src/nominatim_api/search/db_search_lookups.py | 4 +- src/nominatim_api/search/db_searches.py | 6 +- src/nominatim_api/search/icu_tokenizer.py | 4 +- src/nominatim_api/search/legacy_tokenizer.py | 2 +- src/nominatim_api/server/falcon/server.py | 2 +- src/nominatim_api/server/starlette/server.py | 2 +- .../sql}/async_core_library.py | 0 src/nominatim_api/sql/sqlalchemy_functions.py | 2 +- .../sql}/sqlalchemy_schema.py | 0 .../sql}/sqlalchemy_types/__init__.py | 0 .../sql}/sqlalchemy_types/geometry.py | 0 .../sql}/sqlalchemy_types/int_array.py | 0 .../sql}/sqlalchemy_types/json.py | 0 .../sql}/sqlalchemy_types/key_value.py | 0 src/nominatim_api/status.py | 6 +- src/nominatim_api/types.py | 2 +- src/nominatim_api/typing.py | 40 +++++++++++++ .../utils}/__init__.py | 0 .../utils/json_writer.py | 0 src/nominatim_api/v1/format.py | 2 +- src/nominatim_api/v1/format_json.py | 2 +- src/nominatim_api/v1/server_glue.py | 4 +- src/nominatim_api/version.py | 6 -- src/nominatim_core/py.typed | 0 src/nominatim_core/version.py | 57 ------------------ src/nominatim_db/cli.py | 4 +- src/nominatim_db/clicmd/admin.py | 4 +- src/nominatim_db/clicmd/api.py | 2 +- src/nominatim_db/clicmd/args.py | 6 +- src/nominatim_db/clicmd/convert.py | 2 +- src/nominatim_db/clicmd/export.py | 7 ++- src/nominatim_db/clicmd/freeze.py | 2 +- src/nominatim_db/clicmd/index.py | 4 +- src/nominatim_db/clicmd/refresh.py | 4 +- src/nominatim_db/clicmd/replication.py | 6 +- src/nominatim_db/clicmd/setup.py | 8 +-- src/nominatim_db/clicmd/special_phrases.py | 4 +- .../config.py | 6 +- src/nominatim_db/data/country_info.py | 8 +-- src/nominatim_db/data/postcode_format.py | 2 +- .../db/__init__.py | 0 .../db/async_connection.py | 0 .../db/connection.py | 0 .../db/properties.py | 0 .../db/sql_preprocessor.py | 0 .../db/status.py | 0 .../db/utils.py | 0 src/nominatim_db/errors.py | 14 +++++ src/nominatim_db/indexer/indexer.py | 6 +- src/nominatim_db/indexer/runners.py | 4 +- src/{nominatim_core => nominatim_db}/paths.py | 0 src/nominatim_db/tokenizer/base.py | 6 +- src/nominatim_db/tokenizer/factory.py | 8 +-- src/nominatim_db/tokenizer/icu_rule_loader.py | 8 +-- src/nominatim_db/tokenizer/icu_tokenizer.py | 8 +-- .../tokenizer/legacy_tokenizer.py | 12 ++-- src/nominatim_db/tokenizer/place_sanitizer.py | 4 +- src/nominatim_db/tokenizer/sanitizers/base.py | 2 +- .../tokenizer/sanitizers/config.py | 2 +- .../tokenizer/token_analysis/base.py | 2 +- .../token_analysis/config_variants.py | 4 +- .../tokenizer/token_analysis/generic.py | 2 +- .../token_analysis/generic_mutation.py | 2 +- src/nominatim_db/tools/add_osm_data.py | 4 +- src/nominatim_db/tools/admin.py | 8 +-- src/nominatim_db/tools/check_database.py | 8 +-- src/nominatim_db/tools/collect_os_info.py | 4 +- src/nominatim_db/tools/convert_sqlite.py | 4 +- src/nominatim_db/tools/database_import.py | 10 ++-- src/nominatim_db/tools/exec_utils.py | 4 +- src/nominatim_db/tools/freeze.py | 2 +- src/nominatim_db/tools/migration.py | 8 +-- src/nominatim_db/tools/postcodes.py | 4 +- src/nominatim_db/tools/refresh.py | 8 +-- src/nominatim_db/tools/replication.py | 6 +- .../tools/special_phrases/sp_csv_loader.py | 2 +- .../tools/special_phrases/sp_importer.py | 6 +- .../tools/special_phrases/sp_wiki_loader.py | 4 +- src/nominatim_db/tools/tiger_data.py | 10 ++-- .../typing.py | 23 ------- .../utils/__init__.py | 0 .../utils/centroid.py | 0 .../utils/url_utils.py | 4 +- src/nominatim_db/version.py | 48 ++++++++++++++- test/bdd/steps/nominatim_environment.py | 4 +- test/python/api/conftest.py | 2 +- test/python/api/fake_adaptor.py | 2 +- test/python/api/test_api_status.py | 2 +- test/python/api/test_api_types.py | 2 +- test/python/cli/test_cmd_import.py | 8 +-- test/python/cli/test_cmd_replication.py | 2 +- test/python/config/test_config.py | 4 +- test/python/config/test_config_load_module.py | 2 +- test/python/conftest.py | 6 +- test/python/db/test_async_connection.py | 2 +- test/python/db/test_connection.py | 2 +- test/python/db/test_properties.py | 2 +- test/python/db/test_sql_preprocessor.py | 2 +- test/python/db/test_status.py | 46 +++++++------- test/python/db/test_utils.py | 4 +- test/python/dummy_tokenizer.py | 2 +- test/python/mocks.py | 2 +- .../sanitizers/test_sanitizer_config.py | 2 +- .../sanitizers/test_split_name_list.py | 2 +- test/python/tokenizer/test_factory.py | 4 +- test/python/tokenizer/test_icu.py | 4 +- test/python/tokenizer/test_icu_rule_loader.py | 2 +- test/python/tokenizer/test_legacy.py | 4 +- test/python/tokenizer/test_place_sanitizer.py | 2 +- .../token_analysis/test_analysis_postcodes.py | 2 +- .../tokenizer/token_analysis/test_generic.py | 2 +- .../token_analysis/test_generic_mutation.py | 2 +- test/python/tools/test_admin.py | 4 +- test/python/tools/test_database_import.py | 2 +- test/python/tools/test_exec_utils.py | 2 +- .../tools/test_import_special_phrases.py | 2 +- test/python/tools/test_migration.py | 2 +- test/python/tools/test_replication.py | 4 +- test/python/tools/test_sp_csv_loader.py | 2 +- test/python/tools/test_tiger_data.py | 2 +- test/python/utils/test_centroid.py | 2 +- test/python/utils/test_json_writer.py | 2 +- vagrant/Install-on-Ubuntu-24.sh | 2 +- 149 files changed, 418 insertions(+), 421 deletions(-) create mode 100644 packaging/nominatim-api/extra_src/paths.py rename packaging/{nominatim-core => nominatim-api}/settings (100%) delete mode 120000 packaging/nominatim-core/COPYING delete mode 100644 packaging/nominatim-core/README.md delete mode 120000 packaging/nominatim-core/data delete mode 100644 packaging/nominatim-core/pyproject.toml delete mode 120000 packaging/nominatim-core/src create mode 120000 packaging/nominatim-db/data rename packaging/{nominatim-core/extra_src/nominatim_core => nominatim-db/extra_src/nominatim_db}/paths.py (100%) rename packaging/{nominatim-core => nominatim-db}/lib-sql (100%) create mode 120000 packaging/nominatim-db/settings create mode 100644 src/nominatim_api/config.py rename src/{nominatim_core => nominatim_api}/errors.py (100%) rename src/{nominatim_core/db => nominatim_api/sql}/async_core_library.py (100%) rename src/{nominatim_core/db => nominatim_api/sql}/sqlalchemy_schema.py (100%) rename src/{nominatim_core/db => nominatim_api/sql}/sqlalchemy_types/__init__.py (100%) rename src/{nominatim_core/db => nominatim_api/sql}/sqlalchemy_types/geometry.py (100%) rename src/{nominatim_core/db => nominatim_api/sql}/sqlalchemy_types/int_array.py (100%) rename src/{nominatim_core/db => nominatim_api/sql}/sqlalchemy_types/json.py (100%) rename src/{nominatim_core/db => nominatim_api/sql}/sqlalchemy_types/key_value.py (100%) create mode 100644 src/nominatim_api/typing.py rename src/{nominatim_core => nominatim_api/utils}/__init__.py (100%) rename src/{nominatim_core => nominatim_api}/utils/json_writer.py (100%) delete mode 100644 src/nominatim_core/py.typed delete mode 100644 src/nominatim_core/version.py rename src/{nominatim_core => nominatim_db}/config.py (98%) rename src/{nominatim_core => nominatim_db}/db/__init__.py (100%) rename src/{nominatim_core => nominatim_db}/db/async_connection.py (100%) rename src/{nominatim_core => nominatim_db}/db/connection.py (100%) rename src/{nominatim_core => nominatim_db}/db/properties.py (100%) rename src/{nominatim_core => nominatim_db}/db/sql_preprocessor.py (100%) rename src/{nominatim_core => nominatim_db}/db/status.py (100%) rename src/{nominatim_core => nominatim_db}/db/utils.py (100%) create mode 100644 src/nominatim_db/errors.py rename src/{nominatim_core => nominatim_db}/paths.py (100%) rename src/{nominatim_core => nominatim_db}/typing.py (65%) rename src/{nominatim_core => nominatim_db}/utils/__init__.py (100%) rename src/{nominatim_core => nominatim_db}/utils/centroid.py (100%) rename src/{nominatim_core => nominatim_db}/utils/url_utils.py (87%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1fa4d1b4..01a71a3c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -221,23 +221,23 @@ if (BUILD_IMPORTER) DESTINATION ${CMAKE_INSTALL_BINDIR} RENAME nominatim) - foreach (submodule nominatim_core nominatim_db nominatim_api) + if (EXISTS ${PHP_BIN}) + configure_file(${PROJECT_SOURCE_DIR}/cmake/paths-py.tmpl paths-py.installed) + else() + configure_file(${PROJECT_SOURCE_DIR}/cmake/paths-py-no-php.tmpl paths-py.installed) + endif() + + foreach (submodule nominatim_db nominatim_api) install(DIRECTORY src/${submodule} DESTINATION ${NOMINATIM_LIBDIR}/lib-python FILES_MATCHING PATTERN "*.py" PATTERN "paths.py" EXCLUDE PATTERN __pycache__ EXCLUDE) + install(FILES ${PROJECT_BINARY_DIR}/paths-py.installed + DESTINATION ${NOMINATIM_LIBDIR}/lib-python/${submodule} + RENAME paths.py) endforeach() - if (EXISTS ${PHP_BIN}) - configure_file(${PROJECT_SOURCE_DIR}/cmake/paths-py.tmpl paths-py.installed) - else() - configure_file(${PROJECT_SOURCE_DIR}/cmake/paths-py-no-php.tmpl paths-py.installed) - endif() - install(FILES ${PROJECT_BINARY_DIR}/paths-py.installed - DESTINATION ${NOMINATIM_LIBDIR}/lib-python/nominatim_core - RENAME paths.py) - install(DIRECTORY lib-sql DESTINATION ${NOMINATIM_LIBDIR}) install(FILES ${COUNTRY_GRID_FILE} diff --git a/Makefile b/Makefile index 0e09a884..dae45322 100644 --- a/Makefile +++ b/Makefile @@ -2,10 +2,10 @@ all: # Building of wheels -build: build-core build-db build-api +build: clean-build build-db build-api -build-core: - python3 -m build packaging/nominatim-core --outdir dist/ +clean-build: + rm -f dist/* build-db: python3 -m build packaging/nominatim-db --outdir dist/ @@ -29,4 +29,4 @@ lint: bdd: cd test/bdd; behave -DREMOVE_TEMPLATE=1 -.PHONY: tests mypy pytest lint bdd build build-core build-db build-api +.PHONY: tests mypy pytest lint bdd build clean-build build-db build-api diff --git a/docs/admin/Installation.md b/docs/admin/Installation.md index cb0df5c1..a2f1a084 100644 --- a/docs/admin/Installation.md +++ b/docs/admin/Installation.md @@ -162,7 +162,7 @@ Nominatim is easiest to run from its own virtual environment. To create one, run To install Nominatim directly from the source tree into the virtual environment, run: - /srv/nominatim-venv/bin/pip install packaging/nominatim-{core,db,api} + /srv/nominatim-venv/bin/pip install packaging/nominatim-{db,api} #### Building in legacy CMake mode diff --git a/docs/library/Configuration.md b/docs/library/Configuration.md index bf093166..e13470e9 100644 --- a/docs/library/Configuration.md +++ b/docs/library/Configuration.md @@ -20,7 +20,7 @@ configuration parameters, see the [Configuration page](../customize/Settings.md) ## `Configuration` class -::: nominatim_core.config.Configuration +::: nominatim_api.Configuration options: members: - get_bool diff --git a/packaging/nominatim-api/extra_src/paths.py b/packaging/nominatim-api/extra_src/paths.py new file mode 100644 index 00000000..6131319c --- /dev/null +++ b/packaging/nominatim-api/extra_src/paths.py @@ -0,0 +1,15 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +# +# This file is part of Nominatim. (https://nominatim.org) +# +# Copyright (C) 2024 by the Nominatim developer community. +# For a full list of authors see the git log. +""" +Path settings for extra data used by Nominatim. +""" +from pathlib import Path + +PHPLIB_DIR = None +DATA_DIR = None +SQLLIB_DIR = None +CONFIG_DIR = (Path(__file__) / '..' / 'resources' / 'settings').resolve() diff --git a/packaging/nominatim-api/pyproject.toml b/packaging/nominatim-api/pyproject.toml index 7d270021..4b268b16 100644 --- a/packaging/nominatim-api/pyproject.toml +++ b/packaging/nominatim-api/pyproject.toml @@ -14,7 +14,8 @@ classifiers = [ "Operating System :: OS Independent", ] dependencies = [ - "nominatim-core", + "python-dotenv", + "pyYAML>=5.1", "SQLAlchemy>=1.4.31", "psycopg", "PyICU" @@ -34,8 +35,21 @@ path = "src/nominatim_api/version.py" pattern = "NOMINATIM_API_VERSION = '(?P[^']+)'" [tool.hatch.build.targets.sdist] -include = ["src/nominatim_api"] +include = [ + "src/nominatim_api", + "src/nominatim_db/config.py", + "settings", + "extra_src/paths.py" +] + +exclude = [ + "src/nominatim_api/config.py" +] [tool.hatch.build.targets.wheel] packages = ["src/nominatim_api"] +[tool.hatch.build.targets.wheel.force-include] +"src/nominatim_db/config.py" = "nominatim_api/config.py" +"extra_src/paths.py" = "nominatim_api/paths.py" +"settings" = "nominatim_api/resources/settings" diff --git a/packaging/nominatim-core/settings b/packaging/nominatim-api/settings similarity index 100% rename from packaging/nominatim-core/settings rename to packaging/nominatim-api/settings diff --git a/packaging/nominatim-core/COPYING b/packaging/nominatim-core/COPYING deleted file mode 120000 index 7d29222e..00000000 --- a/packaging/nominatim-core/COPYING +++ /dev/null @@ -1 +0,0 @@ -../../COPYING \ No newline at end of file diff --git a/packaging/nominatim-core/README.md b/packaging/nominatim-core/README.md deleted file mode 100644 index 127303fc..00000000 --- a/packaging/nominatim-core/README.md +++ /dev/null @@ -1,24 +0,0 @@ -Nominatim - Core Package -========= - -Nominatim is a tool to search OpenStreetMap data -by name and address (geocoding) and to generate synthetic addresses of -OSM points (reverse geocoding). - -This is the core pacakage containing resources and code shared by -Nominatim's frontend `nominatim-api` and backend `nominatim-db`. You -usually don't want to install this package directly. - -Documentation -============= - -The documentation of the latest development version is in the -`docs/` subdirectory. A HTML version can be found at -https://nominatim.org/release-docs/develop/ . - -License -======= - -The Python source code is available under a GPL license version 3 or later. -The Lua configuration files for osm2pgsql are released under the -Apache License, Version 2.0. All other files are under a GPLv2 license. diff --git a/packaging/nominatim-core/data b/packaging/nominatim-core/data deleted file mode 120000 index e67b4559..00000000 --- a/packaging/nominatim-core/data +++ /dev/null @@ -1 +0,0 @@ -../../data \ No newline at end of file diff --git a/packaging/nominatim-core/pyproject.toml b/packaging/nominatim-core/pyproject.toml deleted file mode 100644 index 9c7af651..00000000 --- a/packaging/nominatim-core/pyproject.toml +++ /dev/null @@ -1,60 +0,0 @@ -[project] -name = "nominatim-core" -description = "A tool for building a database of OpenStreetMap for geocoding and for searching the database. Base package for common resources for the project." -readme = "README.md" -requires-python = ">=3.7" -license = 'GPL-3.0-or-later' -maintainers = [ - { name = "Sarah Hoffmann", email = "lonvia@denofr.de" } -] -keywords = [ "geocoding", "OpenStreetMap", "search" ] -classifiers = [ - "Programming Language :: Python :: 3", - "License :: OSI Approved :: GNU General Public License (GPL)", - "Operating System :: OS Independent", -] -dependencies = [ - "python-dotenv", - "jinja2", - "pyYAML>=5.1", - "datrie" -] -dynamic = ["version"] - -[project.urls] -Homepage = "https://nominatim.org" -Issues = "https://github.com/osm-search/Nominatim/issues" - -[build-system] -requires = ["hatchling"] -build-backend = "hatchling.build" - -[tool.hatch.version] -source = "code" -path = "src/nominatim_core/version.py" -expression = "NOMINATIM_CORE_VERSION" - -[tool.hatch.build.targets.sdist] -include = [ - "src/nominatim_core", - "lib-sql/**.sql", - "settings", - "data/words.sql", - "extra_src/nominatim_core/paths.py" -] -artifacts = [ - "data/country_osm_grid.sql.gz" -] -exclude = [ - "src/nominatim_core/paths.py" -] - -[tool.hatch.build.targets.wheel] -packages = ["src/nominatim_core"] - -[tool.hatch.build.targets.wheel.force-include] -"lib-sql" = "nominatim_core/resources/lib-sql" -"settings" = "nominatim_core/resources/settings" -"data/country_osm_grid.sql.gz" = "nominatim_core/resources/country_osm_grid.sql.gz" -"data/words.sql" = "nominatim_core/resources/words.sql" -"extra_src/nominatim_core/paths.py" = "nominatim_core/paths.py" diff --git a/packaging/nominatim-core/src b/packaging/nominatim-core/src deleted file mode 120000 index 929cb3dc..00000000 --- a/packaging/nominatim-core/src +++ /dev/null @@ -1 +0,0 @@ -../../src \ No newline at end of file diff --git a/packaging/nominatim-db/data b/packaging/nominatim-db/data new file mode 120000 index 00000000..b8fb52e7 --- /dev/null +++ b/packaging/nominatim-db/data @@ -0,0 +1 @@ +../../data/ \ No newline at end of file diff --git a/packaging/nominatim-core/extra_src/nominatim_core/paths.py b/packaging/nominatim-db/extra_src/nominatim_db/paths.py similarity index 100% rename from packaging/nominatim-core/extra_src/nominatim_core/paths.py rename to packaging/nominatim-db/extra_src/nominatim_db/paths.py diff --git a/packaging/nominatim-core/lib-sql b/packaging/nominatim-db/lib-sql similarity index 100% rename from packaging/nominatim-core/lib-sql rename to packaging/nominatim-db/lib-sql diff --git a/packaging/nominatim-db/pyproject.toml b/packaging/nominatim-db/pyproject.toml index 312f1c81..69b863c2 100644 --- a/packaging/nominatim-db/pyproject.toml +++ b/packaging/nominatim-db/pyproject.toml @@ -14,8 +14,11 @@ classifiers = [ "Operating System :: OS Independent", ] dependencies = [ - "nominatim-core", "psycopg2-binary", + "python-dotenv", + "jinja2", + "pyYAML>=5.1", + "datrie", "psutil", "PyICU" ] @@ -24,6 +27,7 @@ dynamic = ["version"] [project.urls] Homepage = "https://nominatim.org" Issues = "https://github.com/osm-search/Nominatim/issues" +Documentation = "https://nominatim.org/release-docs/latest/" [build-system] requires = ["hatchling"] @@ -34,10 +38,32 @@ path = "src/nominatim_db/version.py" pattern = "NOMINATIM_VERSION = parse_version.'(?P[^-]+)" [tool.hatch.build.targets.sdist] -include = ["src/nominatim_db", "scripts"] +include = [ + "src/nominatim_db", + "scripts", + "lib-sql/**/*.sql", + "settings", + "data/words.sql", + "extra_src/nominatim_db/paths.py" +] + +artifacts = [ + "data/country_osm_grid.sql.gz" +] + +exclude = [ + "src/nominatim_db/paths.py" +] [tool.hatch.build.targets.wheel] packages = ["src/nominatim_db"] [tool.hatch.build.targets.wheel.shared-scripts] "scripts" = "/" + +[tool.hatch.build.targets.wheel.force-include] +"lib-sql" = "nominatim_db/resources/lib-sql" +"settings" = "nominatim_db/resources/settings" +"data/country_osm_grid.sql.gz" = "nominatim_db/resources/country_osm_grid.sql.gz" +"data/words.sql" = "nominatim_db/resources/words.sql" +"extra_src/nominatim_db/paths.py" = "nominatim_db/paths.py" diff --git a/packaging/nominatim-db/settings b/packaging/nominatim-db/settings new file mode 120000 index 00000000..51326e66 --- /dev/null +++ b/packaging/nominatim-db/settings @@ -0,0 +1 @@ +../../settings/ \ No newline at end of file diff --git a/src/nominatim_api/__init__.py b/src/nominatim_api/__init__.py index 0f02f979..4f471447 100644 --- a/src/nominatim_api/__init__.py +++ b/src/nominatim_api/__init__.py @@ -14,8 +14,8 @@ import from this file, not from the source files directly. # See also https://github.com/PyCQA/pylint/issues/6006 # pylint: disable=useless-import-alias -from nominatim_core.errors import (UsageError as UsageError) -from nominatim_core.config import (Configuration as Configuration) +from .errors import (UsageError as UsageError) +from .config import (Configuration as Configuration) from .core import (NominatimAPI as NominatimAPI, NominatimAPIAsync as NominatimAPIAsync) diff --git a/src/nominatim_api/config.py b/src/nominatim_api/config.py new file mode 100644 index 00000000..18afda66 --- /dev/null +++ b/src/nominatim_api/config.py @@ -0,0 +1,12 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +# +# This file is part of Nominatim. (https://nominatim.org) +# +# Copyright (C) 2024 by the Nominatim developer community. +# For a full list of authors see the git log. + +# This file is just a placeholder to make the config module available +# during development. It will be replaced by nominatim_db/config.py on +# installation. +# pylint: skip-file +from nominatim_db.config import * diff --git a/src/nominatim_api/connection.py b/src/nominatim_api/connection.py index 66b00ac8..167ffaa4 100644 --- a/src/nominatim_api/connection.py +++ b/src/nominatim_api/connection.py @@ -14,9 +14,9 @@ import asyncio import sqlalchemy as sa from sqlalchemy.ext.asyncio import AsyncConnection -from nominatim_core.typing import SaFromClause -from nominatim_core.db.sqlalchemy_schema import SearchTables -from nominatim_core.db.sqlalchemy_types import Geometry +from .typing import SaFromClause +from .sql.sqlalchemy_schema import SearchTables +from .sql.sqlalchemy_types import Geometry from .logging import log T = TypeVar('T') diff --git a/src/nominatim_api/core.py b/src/nominatim_api/core.py index 6f91e7a8..632c97a7 100644 --- a/src/nominatim_api/core.py +++ b/src/nominatim_api/core.py @@ -16,10 +16,10 @@ from pathlib import Path import sqlalchemy as sa import sqlalchemy.ext.asyncio as sa_asyncio -from nominatim_core.errors import UsageError -from nominatim_core.db.sqlalchemy_schema import SearchTables -from nominatim_core.db.async_core_library import PGCORE_LIB, PGCORE_ERROR -from nominatim_core.config import Configuration +from .errors import UsageError +from .sql.sqlalchemy_schema import SearchTables +from .sql.async_core_library import PGCORE_LIB, PGCORE_ERROR +from .config import Configuration from .sql import sqlite_functions, sqlalchemy_functions #pylint: disable=unused-import from .connection import SearchConnection from .status import get_status, StatusResult diff --git a/src/nominatim_core/errors.py b/src/nominatim_api/errors.py similarity index 100% rename from src/nominatim_core/errors.py rename to src/nominatim_api/errors.py diff --git a/src/nominatim_api/lookup.py b/src/nominatim_api/lookup.py index e451edbe..34739171 100644 --- a/src/nominatim_api/lookup.py +++ b/src/nominatim_api/lookup.py @@ -12,7 +12,7 @@ import datetime as dt import sqlalchemy as sa -from nominatim_core.typing import SaColumn, SaRow, SaSelect +from .typing import SaColumn, SaRow, SaSelect from .connection import SearchConnection from .logging import log from . import types as ntyp diff --git a/src/nominatim_api/results.py b/src/nominatim_api/results.py index 4256e0b7..805527a7 100644 --- a/src/nominatim_api/results.py +++ b/src/nominatim_api/results.py @@ -18,8 +18,8 @@ import datetime as dt import sqlalchemy as sa -from nominatim_core.typing import SaSelect, SaRow -from nominatim_core.db.sqlalchemy_types import Geometry +from .typing import SaSelect, SaRow +from .sql.sqlalchemy_types import Geometry from .types import Point, Bbox, LookupDetails from .connection import SearchConnection from .logging import log diff --git a/src/nominatim_api/reverse.py b/src/nominatim_api/reverse.py index 83b0983a..20270656 100644 --- a/src/nominatim_api/reverse.py +++ b/src/nominatim_api/reverse.py @@ -12,9 +12,9 @@ import functools import sqlalchemy as sa -from nominatim_core.typing import SaColumn, SaSelect, SaFromClause, SaLabel, SaRow,\ - SaBind, SaLambdaSelect -from nominatim_core.db.sqlalchemy_types import Geometry +from .typing import SaColumn, SaSelect, SaFromClause, SaLabel, SaRow,\ + SaBind, SaLambdaSelect +from .sql.sqlalchemy_types import Geometry from .connection import SearchConnection from . import results as nres from .logging import log diff --git a/src/nominatim_api/search/db_search_fields.py b/src/nominatim_api/search/db_search_fields.py index 8f674ecc..95b2b4a7 100644 --- a/src/nominatim_api/search/db_search_fields.py +++ b/src/nominatim_api/search/db_search_fields.py @@ -12,8 +12,8 @@ import dataclasses import sqlalchemy as sa -from nominatim_core.typing import SaFromClause, SaColumn, SaExpression -from nominatim_core.utils.json_writer import JsonWriter +from ..typing import SaFromClause, SaColumn, SaExpression +from ..utils.json_writer import JsonWriter from .query import Token from . import db_search_lookups as lookups diff --git a/src/nominatim_api/search/db_search_lookups.py b/src/nominatim_api/search/db_search_lookups.py index faabd827..712cd894 100644 --- a/src/nominatim_api/search/db_search_lookups.py +++ b/src/nominatim_api/search/db_search_lookups.py @@ -12,8 +12,8 @@ from typing import List, Any import sqlalchemy as sa from sqlalchemy.ext.compiler import compiles -from nominatim_core.typing import SaFromClause -from nominatim_core.db.sqlalchemy_types import IntArray +from ..typing import SaFromClause +from ..sql.sqlalchemy_types import IntArray # pylint: disable=consider-using-f-string diff --git a/src/nominatim_api/search/db_searches.py b/src/nominatim_api/search/db_searches.py index f5c43146..35c063fc 100644 --- a/src/nominatim_api/search/db_searches.py +++ b/src/nominatim_api/search/db_searches.py @@ -12,9 +12,9 @@ import abc import sqlalchemy as sa -from nominatim_core.typing import SaFromClause, SaScalarSelect, SaColumn, \ - SaExpression, SaSelect, SaLambdaSelect, SaRow, SaBind -from nominatim_core.db.sqlalchemy_types import Geometry, IntArray +from ..typing import SaFromClause, SaScalarSelect, SaColumn, \ + SaExpression, SaSelect, SaLambdaSelect, SaRow, SaBind +from ..sql.sqlalchemy_types import Geometry, IntArray from ..connection import SearchConnection from ..types import SearchDetails, DataLayer, GeometryFormat, Bbox from .. import results as nres diff --git a/src/nominatim_api/search/icu_tokenizer.py b/src/nominatim_api/search/icu_tokenizer.py index 42665679..971e95be 100644 --- a/src/nominatim_api/search/icu_tokenizer.py +++ b/src/nominatim_api/search/icu_tokenizer.py @@ -16,8 +16,8 @@ from icu import Transliterator import sqlalchemy as sa -from nominatim_core.typing import SaRow -from nominatim_core.db.sqlalchemy_types import Json +from ..typing import SaRow +from ..sql.sqlalchemy_types import Json from ..connection import SearchConnection from ..logging import log from ..search import query as qmod diff --git a/src/nominatim_api/search/legacy_tokenizer.py b/src/nominatim_api/search/legacy_tokenizer.py index b49d3ae6..ecb0bbfe 100644 --- a/src/nominatim_api/search/legacy_tokenizer.py +++ b/src/nominatim_api/search/legacy_tokenizer.py @@ -14,7 +14,7 @@ import dataclasses import sqlalchemy as sa -from nominatim_core.typing import SaRow +from ..typing import SaRow from ..connection import SearchConnection from ..logging import log from . import query as qmod diff --git a/src/nominatim_api/server/falcon/server.py b/src/nominatim_api/server/falcon/server.py index 851b0964..bc9850b2 100644 --- a/src/nominatim_api/server/falcon/server.py +++ b/src/nominatim_api/server/falcon/server.py @@ -14,7 +14,7 @@ import asyncio from falcon.asgi import App, Request, Response -from nominatim_core.config import Configuration +from ...config import Configuration from ...core import NominatimAPIAsync from ... import v1 as api_impl from ... import logging as loglib diff --git a/src/nominatim_api/server/starlette/server.py b/src/nominatim_api/server/starlette/server.py index dd35cd6e..5f5cf055 100644 --- a/src/nominatim_api/server/starlette/server.py +++ b/src/nominatim_api/server/starlette/server.py @@ -21,7 +21,7 @@ from starlette.middleware import Middleware from starlette.middleware.base import BaseHTTPMiddleware, RequestResponseEndpoint from starlette.middleware.cors import CORSMiddleware -from nominatim_core.config import Configuration +from ...config import Configuration from ...core import NominatimAPIAsync from ... import v1 as api_impl from ... import logging as loglib diff --git a/src/nominatim_core/db/async_core_library.py b/src/nominatim_api/sql/async_core_library.py similarity index 100% rename from src/nominatim_core/db/async_core_library.py rename to src/nominatim_api/sql/async_core_library.py diff --git a/src/nominatim_api/sql/sqlalchemy_functions.py b/src/nominatim_api/sql/sqlalchemy_functions.py index 40202749..7abe7d3f 100644 --- a/src/nominatim_api/sql/sqlalchemy_functions.py +++ b/src/nominatim_api/sql/sqlalchemy_functions.py @@ -13,7 +13,7 @@ from typing import Any import sqlalchemy as sa from sqlalchemy.ext.compiler import compiles -from nominatim_core.typing import SaColumn +from ..typing import SaColumn # pylint: disable=all diff --git a/src/nominatim_core/db/sqlalchemy_schema.py b/src/nominatim_api/sql/sqlalchemy_schema.py similarity index 100% rename from src/nominatim_core/db/sqlalchemy_schema.py rename to src/nominatim_api/sql/sqlalchemy_schema.py diff --git a/src/nominatim_core/db/sqlalchemy_types/__init__.py b/src/nominatim_api/sql/sqlalchemy_types/__init__.py similarity index 100% rename from src/nominatim_core/db/sqlalchemy_types/__init__.py rename to src/nominatim_api/sql/sqlalchemy_types/__init__.py diff --git a/src/nominatim_core/db/sqlalchemy_types/geometry.py b/src/nominatim_api/sql/sqlalchemy_types/geometry.py similarity index 100% rename from src/nominatim_core/db/sqlalchemy_types/geometry.py rename to src/nominatim_api/sql/sqlalchemy_types/geometry.py diff --git a/src/nominatim_core/db/sqlalchemy_types/int_array.py b/src/nominatim_api/sql/sqlalchemy_types/int_array.py similarity index 100% rename from src/nominatim_core/db/sqlalchemy_types/int_array.py rename to src/nominatim_api/sql/sqlalchemy_types/int_array.py diff --git a/src/nominatim_core/db/sqlalchemy_types/json.py b/src/nominatim_api/sql/sqlalchemy_types/json.py similarity index 100% rename from src/nominatim_core/db/sqlalchemy_types/json.py rename to src/nominatim_api/sql/sqlalchemy_types/json.py diff --git a/src/nominatim_core/db/sqlalchemy_types/key_value.py b/src/nominatim_api/sql/sqlalchemy_types/key_value.py similarity index 100% rename from src/nominatim_core/db/sqlalchemy_types/key_value.py rename to src/nominatim_api/sql/sqlalchemy_types/key_value.py diff --git a/src/nominatim_api/status.py b/src/nominatim_api/status.py index 614789ed..ea88a467 100644 --- a/src/nominatim_api/status.py +++ b/src/nominatim_api/status.py @@ -14,7 +14,7 @@ import dataclasses import sqlalchemy as sa from .connection import SearchConnection -from .version import NOMINATIM_API_VERSION, NominatimVersion, parse_version +from .version import NOMINATIM_API_VERSION @dataclasses.dataclass class StatusResult: @@ -24,7 +24,7 @@ class StatusResult: message: str software_version = NOMINATIM_API_VERSION data_updated: Optional[dt.datetime] = None - database_version: Optional[NominatimVersion] = None + database_version: Optional[str] = None async def get_status(conn: SearchConnection) -> StatusResult: @@ -44,7 +44,7 @@ async def get_status(conn: SearchConnection) -> StatusResult: # Database version try: - status.database_version = parse_version(await conn.get_property('database_version')) + status.database_version = await conn.get_property('database_version') except ValueError: pass diff --git a/src/nominatim_api/types.py b/src/nominatim_api/types.py index f2ad4f35..6c8adeb8 100644 --- a/src/nominatim_api/types.py +++ b/src/nominatim_api/types.py @@ -16,7 +16,7 @@ import math from struct import unpack from binascii import unhexlify -from nominatim_core.errors import UsageError +from .errors import UsageError from .localization import Locales # pylint: disable=no-member,too-many-boolean-expressions,too-many-instance-attributes diff --git a/src/nominatim_api/typing.py b/src/nominatim_api/typing.py new file mode 100644 index 00000000..ba0d95bf --- /dev/null +++ b/src/nominatim_api/typing.py @@ -0,0 +1,40 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +# +# This file is part of Nominatim. (https://nominatim.org) +# +# Copyright (C) 2024 by the Nominatim developer community. +# For a full list of authors see the git log. +""" +Type definitions for typing annotations. + +Complex type definitions are moved here, to keep the source files readable. +""" +from typing import Union, TYPE_CHECKING + +# pylint: disable=missing-class-docstring,useless-import-alias + +# SQLAlchemy introduced generic types in version 2.0 making typing +# incompatible with older versions. Add wrappers here so we don't have +# to litter the code with bare-string types. + +if TYPE_CHECKING: + from typing import Any + import sqlalchemy as sa + import os + from typing_extensions import (TypeAlias as TypeAlias) +else: + TypeAlias = str + +StrPath = Union[str, 'os.PathLike[str]'] + +SaLambdaSelect: TypeAlias = 'Union[sa.Select[Any], sa.StatementLambdaElement]' +SaSelect: TypeAlias = 'sa.Select[Any]' +SaScalarSelect: TypeAlias = 'sa.ScalarSelect[Any]' +SaRow: TypeAlias = 'sa.Row[Any]' +SaColumn: TypeAlias = 'sa.ColumnElement[Any]' +SaExpression: TypeAlias = 'sa.ColumnElement[bool]' +SaLabel: TypeAlias = 'sa.Label[Any]' +SaFromClause: TypeAlias = 'sa.FromClause' +SaSelectable: TypeAlias = 'sa.Selectable' +SaBind: TypeAlias = 'sa.BindParameter[Any]' +SaDialect: TypeAlias = 'sa.Dialect' diff --git a/src/nominatim_core/__init__.py b/src/nominatim_api/utils/__init__.py similarity index 100% rename from src/nominatim_core/__init__.py rename to src/nominatim_api/utils/__init__.py diff --git a/src/nominatim_core/utils/json_writer.py b/src/nominatim_api/utils/json_writer.py similarity index 100% rename from src/nominatim_core/utils/json_writer.py rename to src/nominatim_api/utils/json_writer.py diff --git a/src/nominatim_api/v1/format.py b/src/nominatim_api/v1/format.py index d8faa482..e74b61e1 100644 --- a/src/nominatim_api/v1/format.py +++ b/src/nominatim_api/v1/format.py @@ -11,7 +11,7 @@ from typing import List, Dict, Mapping, Any import collections import datetime as dt -from nominatim_core.utils.json_writer import JsonWriter +from ..utils.json_writer import JsonWriter from ..status import StatusResult from ..results import DetailedResult, ReverseResults, SearchResults, \ AddressLines, AddressLine diff --git a/src/nominatim_api/v1/format_json.py b/src/nominatim_api/v1/format_json.py index 2c50dec4..34bb777a 100644 --- a/src/nominatim_api/v1/format_json.py +++ b/src/nominatim_api/v1/format_json.py @@ -9,7 +9,7 @@ Helper functions for output of results in json formats. """ from typing import Mapping, Any, Optional, Tuple, Union -from nominatim_core.utils.json_writer import JsonWriter +from ..utils.json_writer import JsonWriter from ..results import AddressLines, ReverseResults, SearchResults from . import classtypes as cl diff --git a/src/nominatim_api/v1/server_glue.py b/src/nominatim_api/v1/server_glue.py index 67def74e..c00b580b 100644 --- a/src/nominatim_api/v1/server_glue.py +++ b/src/nominatim_api/v1/server_glue.py @@ -17,8 +17,8 @@ from urllib.parse import urlencode import sqlalchemy as sa -from nominatim_core.errors import UsageError -from nominatim_core.config import Configuration +from ..errors import UsageError +from ..config import Configuration from .. import logging as loglib from ..core import NominatimAPIAsync from .format import dispatch as formatting diff --git a/src/nominatim_api/version.py b/src/nominatim_api/version.py index 5dd084c7..d275f4fc 100644 --- a/src/nominatim_api/version.py +++ b/src/nominatim_api/version.py @@ -8,10 +8,4 @@ Version information for the Nominatim API. """ -# See also https://github.com/PyCQA/pylint/issues/6006 -# pylint: disable=useless-import-alias,unused-import - -from nominatim_core.version import (NominatimVersion as NominatimVersion, - parse_version as parse_version) - NOMINATIM_API_VERSION = '4.4.99' diff --git a/src/nominatim_core/py.typed b/src/nominatim_core/py.typed deleted file mode 100644 index e69de29b..00000000 diff --git a/src/nominatim_core/version.py b/src/nominatim_core/version.py deleted file mode 100644 index 91193fa4..00000000 --- a/src/nominatim_core/version.py +++ /dev/null @@ -1,57 +0,0 @@ -# SPDX-License-Identifier: GPL-3.0-or-later -# -# This file is part of Nominatim. (https://nominatim.org) -# -# Copyright (C) 2024 by the Nominatim developer community. -# For a full list of authors see the git log. -""" -Version information for the Nominatim core package. -""" -from typing import NamedTuple, Optional - -__version__ = '4.4.99' -NOMINATIM_CORE_VERSION = __version__ - -class NominatimVersion(NamedTuple): - """ Version information for Nominatim. We follow semantic versioning. - - Major, minor and patch_level refer to the last released version. - The database patch level tracks important changes between releases - and must always be increased when there is a change to the database or code - that requires a migration. - - When adding a migration on the development branch, raise the patch level - to 99 to make sure that the migration is applied when updating from a - patch release to the next minor version. Patch releases usually shouldn't - have migrations in them. When they are needed, then make sure that the - migration can be reapplied and set the migration version to the appropriate - patch level when cherry-picking the commit with the migration. - """ - - major: int - minor: int - patch_level: int - db_patch_level: Optional[int] - - def __str__(self) -> str: - if self.db_patch_level is None: - return f"{self.major}.{self.minor}.{self.patch_level}" - - return f"{self.major}.{self.minor}.{self.patch_level}-{self.db_patch_level}" - - def release_version(self) -> str: - """ Return the release version in semantic versioning format. - - The release version does not include the database patch version. - """ - return f"{self.major}.{self.minor}.{self.patch_level}" - - -def parse_version(version: str) -> NominatimVersion: - """ Parse a version string into a version consisting of a tuple of - four ints: major, minor, patch level, database patch level - - This is the reverse operation of `version_str()`. - """ - parts = version.split('.') - return NominatimVersion(*[int(x) for x in parts[:2] + parts[2].split('-')]) diff --git a/src/nominatim_db/cli.py b/src/nominatim_db/cli.py index 8a85e0e2..41684fa1 100644 --- a/src/nominatim_db/cli.py +++ b/src/nominatim_db/cli.py @@ -16,8 +16,8 @@ import sys import argparse from pathlib import Path -from nominatim_core.config import Configuration -from nominatim_core.errors import UsageError +from .config import Configuration +from .errors import UsageError from .tools.exec_utils import run_php_server from . import clicmd from . import version diff --git a/src/nominatim_db/clicmd/admin.py b/src/nominatim_db/clicmd/admin.py index 7b97557d..7744595b 100644 --- a/src/nominatim_db/clicmd/admin.py +++ b/src/nominatim_db/clicmd/admin.py @@ -11,8 +11,8 @@ import logging import argparse import random -from nominatim_core.errors import UsageError -from nominatim_core.db.connection import connect +from ..errors import UsageError +from ..db.connection import connect from .args import NominatimArgs # Do not repeat documentation of subcommand classes. diff --git a/src/nominatim_db/clicmd/api.py b/src/nominatim_db/clicmd/api.py index 25b4bff3..fac88bdd 100644 --- a/src/nominatim_db/clicmd/api.py +++ b/src/nominatim_db/clicmd/api.py @@ -14,12 +14,12 @@ import json import sys from functools import reduce -from nominatim_core.errors import UsageError import nominatim_api as napi import nominatim_api.v1 as api_output from nominatim_api.v1.helpers import zoom_to_rank, deduplicate_results from nominatim_api.v1.format import dispatch as formatting import nominatim_api.logging as loglib +from ..errors import UsageError from .args import NominatimArgs # Do not repeat documentation of subcommand classes. diff --git a/src/nominatim_db/clicmd/args.py b/src/nominatim_db/clicmd/args.py index 857cb55e..6a11b089 100644 --- a/src/nominatim_db/clicmd/args.py +++ b/src/nominatim_db/clicmd/args.py @@ -12,9 +12,9 @@ import argparse import logging from pathlib import Path -from nominatim_core.errors import UsageError -from nominatim_core.config import Configuration -from nominatim_core.typing import Protocol +from ..errors import UsageError +from ..config import Configuration +from ..typing import Protocol LOG = logging.getLogger() diff --git a/src/nominatim_db/clicmd/convert.py b/src/nominatim_db/clicmd/convert.py index 51db848c..1468b782 100644 --- a/src/nominatim_db/clicmd/convert.py +++ b/src/nominatim_db/clicmd/convert.py @@ -12,7 +12,7 @@ import argparse import asyncio from pathlib import Path -from nominatim_core.errors import UsageError +from ..errors import UsageError from .args import NominatimArgs # Do not repeat documentation of subcommand classes. diff --git a/src/nominatim_db/clicmd/export.py b/src/nominatim_db/clicmd/export.py index 460d27d6..93f63980 100644 --- a/src/nominatim_db/clicmd/export.py +++ b/src/nominatim_db/clicmd/export.py @@ -14,12 +14,13 @@ import asyncio import csv import sys -import sqlalchemy as sa - import nominatim_api as napi from nominatim_api.results import create_from_placex_row, ReverseResult, add_result_details from nominatim_api.types import LookupDetails -from nominatim_core.errors import UsageError + +import sqlalchemy as sa # pylint: disable=C0411 + +from ..errors import UsageError from .args import NominatimArgs # Do not repeat documentation of subcommand classes. diff --git a/src/nominatim_db/clicmd/freeze.py b/src/nominatim_db/clicmd/freeze.py index 83b84b17..27562ccc 100644 --- a/src/nominatim_db/clicmd/freeze.py +++ b/src/nominatim_db/clicmd/freeze.py @@ -9,7 +9,7 @@ Implementation of the 'freeze' subcommand. """ import argparse -from nominatim_core.db.connection import connect +from ..db.connection import connect from .args import NominatimArgs # Do not repeat documentation of subcommand classes. diff --git a/src/nominatim_db/clicmd/index.py b/src/nominatim_db/clicmd/index.py index 7766a1d1..87e0fc03 100644 --- a/src/nominatim_db/clicmd/index.py +++ b/src/nominatim_db/clicmd/index.py @@ -11,8 +11,8 @@ import argparse import psutil -from nominatim_core.db import status -from nominatim_core.db.connection import connect +from ..db import status +from ..db.connection import connect from .args import NominatimArgs # Do not repeat documentation of subcommand classes. diff --git a/src/nominatim_db/clicmd/refresh.py b/src/nominatim_db/clicmd/refresh.py index ceead258..d5acf54b 100644 --- a/src/nominatim_db/clicmd/refresh.py +++ b/src/nominatim_db/clicmd/refresh.py @@ -12,8 +12,8 @@ import argparse import logging from pathlib import Path -from nominatim_core.config import Configuration -from nominatim_core.db.connection import connect +from ..config import Configuration +from ..db.connection import connect from ..tokenizer.base import AbstractTokenizer from .args import NominatimArgs diff --git a/src/nominatim_db/clicmd/replication.py b/src/nominatim_db/clicmd/replication.py index 6fc4dc30..f04c730f 100644 --- a/src/nominatim_db/clicmd/replication.py +++ b/src/nominatim_db/clicmd/replication.py @@ -14,9 +14,9 @@ import logging import socket import time -from nominatim_core.db import status -from nominatim_core.db.connection import connect -from nominatim_core.errors import UsageError +from ..db import status +from ..db.connection import connect +from ..errors import UsageError from .args import NominatimArgs LOG = logging.getLogger() diff --git a/src/nominatim_db/clicmd/setup.py b/src/nominatim_db/clicmd/setup.py index af586d49..f516ba0c 100644 --- a/src/nominatim_db/clicmd/setup.py +++ b/src/nominatim_db/clicmd/setup.py @@ -14,10 +14,10 @@ from pathlib import Path import psutil -from nominatim_core.errors import UsageError -from nominatim_core.config import Configuration -from nominatim_core.db.connection import connect -from nominatim_core.db import status, properties +from ..errors import UsageError +from ..config import Configuration +from ..db.connection import connect +from ..db import status, properties from ..tokenizer.base import AbstractTokenizer from ..version import NOMINATIM_VERSION from .args import NominatimArgs diff --git a/src/nominatim_db/clicmd/special_phrases.py b/src/nominatim_db/clicmd/special_phrases.py index 1e0f2d76..66085942 100644 --- a/src/nominatim_db/clicmd/special_phrases.py +++ b/src/nominatim_db/clicmd/special_phrases.py @@ -11,8 +11,8 @@ import argparse import logging from pathlib import Path -from nominatim_core.errors import UsageError -from nominatim_core.db.connection import connect +from ..errors import UsageError +from ..db.connection import connect from ..tools.special_phrases.sp_importer import SPImporter, SpecialPhraseLoader from ..tools.special_phrases.sp_wiki_loader import SPWikiLoader from ..tools.special_phrases.sp_csv_loader import SPCsvLoader diff --git a/src/nominatim_core/config.py b/src/nominatim_db/config.py similarity index 98% rename from src/nominatim_core/config.py rename to src/nominatim_db/config.py index f4c925eb..c4264f0d 100644 --- a/src/nominatim_core/config.py +++ b/src/nominatim_db/config.py @@ -17,7 +17,11 @@ import json import yaml from dotenv import dotenv_values -from psycopg2.extensions import parse_dsn + +try: + from psycopg2.extensions import parse_dsn +except ModuleNotFoundError: + from psycopg.conninfo import conninfo_to_dict as parse_dsn # type: ignore[assignment] from .typing import StrPath from .errors import UsageError diff --git a/src/nominatim_db/data/country_info.py b/src/nominatim_db/data/country_info.py index 35943a50..c8002ee7 100644 --- a/src/nominatim_db/data/country_info.py +++ b/src/nominatim_db/data/country_info.py @@ -11,10 +11,10 @@ from typing import Dict, Any, Iterable, Tuple, Optional, Container, overload from pathlib import Path import psycopg2.extras -from nominatim_core.db import utils as db_utils -from nominatim_core.db.connection import connect, Connection -from nominatim_core.errors import UsageError -from nominatim_core.config import Configuration +from ..db import utils as db_utils +from ..db.connection import connect, Connection +from ..errors import UsageError +from ..config import Configuration from ..tokenizer.base import AbstractTokenizer def _flatten_name_list(names: Any) -> Dict[str, str]: diff --git a/src/nominatim_db/data/postcode_format.py b/src/nominatim_db/data/postcode_format.py index 670827ee..0d04826d 100644 --- a/src/nominatim_db/data/postcode_format.py +++ b/src/nominatim_db/data/postcode_format.py @@ -11,7 +11,7 @@ format. from typing import Any, Mapping, Optional, Set, Match import re -from nominatim_core.errors import UsageError +from ..errors import UsageError from . import country_info class CountryPostcodeMatcher: diff --git a/src/nominatim_core/db/__init__.py b/src/nominatim_db/db/__init__.py similarity index 100% rename from src/nominatim_core/db/__init__.py rename to src/nominatim_db/db/__init__.py diff --git a/src/nominatim_core/db/async_connection.py b/src/nominatim_db/db/async_connection.py similarity index 100% rename from src/nominatim_core/db/async_connection.py rename to src/nominatim_db/db/async_connection.py diff --git a/src/nominatim_core/db/connection.py b/src/nominatim_db/db/connection.py similarity index 100% rename from src/nominatim_core/db/connection.py rename to src/nominatim_db/db/connection.py diff --git a/src/nominatim_core/db/properties.py b/src/nominatim_db/db/properties.py similarity index 100% rename from src/nominatim_core/db/properties.py rename to src/nominatim_db/db/properties.py diff --git a/src/nominatim_core/db/sql_preprocessor.py b/src/nominatim_db/db/sql_preprocessor.py similarity index 100% rename from src/nominatim_core/db/sql_preprocessor.py rename to src/nominatim_db/db/sql_preprocessor.py diff --git a/src/nominatim_core/db/status.py b/src/nominatim_db/db/status.py similarity index 100% rename from src/nominatim_core/db/status.py rename to src/nominatim_db/db/status.py diff --git a/src/nominatim_core/db/utils.py b/src/nominatim_db/db/utils.py similarity index 100% rename from src/nominatim_core/db/utils.py rename to src/nominatim_db/db/utils.py diff --git a/src/nominatim_db/errors.py b/src/nominatim_db/errors.py new file mode 100644 index 00000000..c7331a89 --- /dev/null +++ b/src/nominatim_db/errors.py @@ -0,0 +1,14 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +# +# This file is part of Nominatim. (https://nominatim.org) +# +# Copyright (C) 2024 by the Nominatim developer community. +# For a full list of authors see the git log. +""" +Custom exception and error classes for Nominatim. +""" + +class UsageError(Exception): + """ An error raised because of bad user input. This error will usually + not cause a stack trace to be printed unless debugging is enabled. + """ diff --git a/src/nominatim_db/indexer/indexer.py b/src/nominatim_db/indexer/indexer.py index 4f9c0575..5a219f6b 100644 --- a/src/nominatim_db/indexer/indexer.py +++ b/src/nominatim_db/indexer/indexer.py @@ -13,9 +13,9 @@ import time import psycopg2.extras -from nominatim_core.typing import DictCursorResults -from nominatim_core.db.async_connection import DBConnection, WorkerPool -from nominatim_core.db.connection import connect, Connection, Cursor +from ..typing import DictCursorResults +from ..db.async_connection import DBConnection, WorkerPool +from ..db.connection import connect, Connection, Cursor from ..tokenizer.base import AbstractTokenizer from .progress import ProgressLogger from . import runners diff --git a/src/nominatim_db/indexer/runners.py b/src/nominatim_db/indexer/runners.py index 4d6a28e4..7b98e240 100644 --- a/src/nominatim_db/indexer/runners.py +++ b/src/nominatim_db/indexer/runners.py @@ -14,8 +14,8 @@ import functools from psycopg2 import sql as pysql import psycopg2.extras -from nominatim_core.typing import Query, DictCursorResult, DictCursorResults, Protocol -from nominatim_core.db.async_connection import DBConnection +from ..typing import Query, DictCursorResult, DictCursorResults, Protocol +from ..db.async_connection import DBConnection from ..data.place_info import PlaceInfo from ..tokenizer.base import AbstractAnalyzer diff --git a/src/nominatim_core/paths.py b/src/nominatim_db/paths.py similarity index 100% rename from src/nominatim_core/paths.py rename to src/nominatim_db/paths.py diff --git a/src/nominatim_db/tokenizer/base.py b/src/nominatim_db/tokenizer/base.py index 0ca7e1d1..d3aeeaca 100644 --- a/src/nominatim_db/tokenizer/base.py +++ b/src/nominatim_db/tokenizer/base.py @@ -12,9 +12,9 @@ from abc import ABC, abstractmethod from typing import List, Tuple, Dict, Any, Optional, Iterable from pathlib import Path -from nominatim_core.typing import Protocol -from nominatim_core.config import Configuration -from nominatim_core.db.connection import Connection +from ..typing import Protocol +from ..config import Configuration +from ..db.connection import Connection from ..data.place_info import PlaceInfo class AbstractAnalyzer(ABC): diff --git a/src/nominatim_db/tokenizer/factory.py b/src/nominatim_db/tokenizer/factory.py index 1193f499..b9022d8d 100644 --- a/src/nominatim_db/tokenizer/factory.py +++ b/src/nominatim_db/tokenizer/factory.py @@ -24,10 +24,10 @@ import logging import importlib from pathlib import Path -from nominatim_core.errors import UsageError -from nominatim_core.db import properties -from nominatim_core.db.connection import connect -from nominatim_core.config import Configuration +from ..errors import UsageError +from ..db import properties +from ..db.connection import connect +from ..config import Configuration from ..tokenizer.base import AbstractTokenizer, TokenizerModule LOG = logging.getLogger() diff --git a/src/nominatim_db/tokenizer/icu_rule_loader.py b/src/nominatim_db/tokenizer/icu_rule_loader.py index 500683e5..0aca2921 100644 --- a/src/nominatim_db/tokenizer/icu_rule_loader.py +++ b/src/nominatim_db/tokenizer/icu_rule_loader.py @@ -14,10 +14,10 @@ import logging from icu import Transliterator -from nominatim_core.config import flatten_config_list, Configuration -from nominatim_core.db.properties import set_property, get_property -from nominatim_core.db.connection import Connection -from nominatim_core.errors import UsageError +from ..config import flatten_config_list, Configuration +from ..db.properties import set_property, get_property +from ..db.connection import Connection +from ..errors import UsageError from .place_sanitizer import PlaceSanitizer from .icu_token_analysis import ICUTokenAnalysis from .token_analysis.base import AnalysisModule, Analyzer diff --git a/src/nominatim_db/tokenizer/icu_tokenizer.py b/src/nominatim_db/tokenizer/icu_tokenizer.py index af03bd39..22e2d048 100644 --- a/src/nominatim_db/tokenizer/icu_tokenizer.py +++ b/src/nominatim_db/tokenizer/icu_tokenizer.py @@ -16,10 +16,10 @@ import logging from pathlib import Path from textwrap import dedent -from nominatim_core.db.connection import connect, Connection, Cursor -from nominatim_core.config import Configuration -from nominatim_core.db.utils import CopyBuffer -from nominatim_core.db.sql_preprocessor import SQLPreprocessor +from ..db.connection import connect, Connection, Cursor +from ..config import Configuration +from ..db.utils import CopyBuffer +from ..db.sql_preprocessor import SQLPreprocessor from ..data.place_info import PlaceInfo from ..data.place_name import PlaceName from .icu_rule_loader import ICURuleLoader diff --git a/src/nominatim_db/tokenizer/legacy_tokenizer.py b/src/nominatim_db/tokenizer/legacy_tokenizer.py index 5c6ba743..136a7331 100644 --- a/src/nominatim_db/tokenizer/legacy_tokenizer.py +++ b/src/nominatim_db/tokenizer/legacy_tokenizer.py @@ -20,12 +20,12 @@ from icu import Transliterator import psycopg2 import psycopg2.extras -from nominatim_core.errors import UsageError -from nominatim_core.db.connection import connect, Connection -from nominatim_core.config import Configuration -from nominatim_core.db import properties -from nominatim_core.db import utils as db_utils -from nominatim_core.db.sql_preprocessor import SQLPreprocessor +from ..errors import UsageError +from ..db.connection import connect, Connection +from ..config import Configuration +from ..db import properties +from ..db import utils as db_utils +from ..db.sql_preprocessor import SQLPreprocessor from ..data.place_info import PlaceInfo from .base import AbstractAnalyzer, AbstractTokenizer diff --git a/src/nominatim_db/tokenizer/place_sanitizer.py b/src/nominatim_db/tokenizer/place_sanitizer.py index 546d0da3..68322f9f 100644 --- a/src/nominatim_db/tokenizer/place_sanitizer.py +++ b/src/nominatim_db/tokenizer/place_sanitizer.py @@ -10,8 +10,8 @@ is handed to the token analysis. """ from typing import Optional, List, Mapping, Sequence, Callable, Any, Tuple -from nominatim_core.errors import UsageError -from nominatim_core.config import Configuration +from ..errors import UsageError +from ..config import Configuration from .sanitizers.config import SanitizerConfig from .sanitizers.base import SanitizerHandler, ProcessInfo from ..data.place_name import PlaceName diff --git a/src/nominatim_db/tokenizer/sanitizers/base.py b/src/nominatim_db/tokenizer/sanitizers/base.py index a28f86b5..2dbc4482 100644 --- a/src/nominatim_db/tokenizer/sanitizers/base.py +++ b/src/nominatim_db/tokenizer/sanitizers/base.py @@ -9,7 +9,7 @@ Common data types and protocols for sanitizers. """ from typing import Optional, List, Mapping, Callable -from nominatim_core.typing import Protocol, Final +from ...typing import Protocol, Final from ...data.place_info import PlaceInfo from ...data.place_name import PlaceName from .config import SanitizerConfig diff --git a/src/nominatim_db/tokenizer/sanitizers/config.py b/src/nominatim_db/tokenizer/sanitizers/config.py index 1ce1a1b7..034d0791 100644 --- a/src/nominatim_db/tokenizer/sanitizers/config.py +++ b/src/nominatim_db/tokenizer/sanitizers/config.py @@ -11,7 +11,7 @@ from typing import Sequence, Union, Optional, Pattern, Callable, Any, TYPE_CHECK from collections import UserDict import re -from nominatim_core.errors import UsageError +from ...errors import UsageError # working around missing generics in Python < 3.8 # See https://github.com/python/typing/issues/60#issuecomment-869757075 diff --git a/src/nominatim_db/tokenizer/token_analysis/base.py b/src/nominatim_db/tokenizer/token_analysis/base.py index fc673491..9435edb3 100644 --- a/src/nominatim_db/tokenizer/token_analysis/base.py +++ b/src/nominatim_db/tokenizer/token_analysis/base.py @@ -9,7 +9,7 @@ Common data types and protocols for analysers. """ from typing import Mapping, List, Any -from nominatim_core.typing import Protocol +from ...typing import Protocol from ...data.place_name import PlaceName class Analyzer(Protocol): diff --git a/src/nominatim_db/tokenizer/token_analysis/config_variants.py b/src/nominatim_db/tokenizer/token_analysis/config_variants.py index b455a8a5..aff360af 100644 --- a/src/nominatim_db/tokenizer/token_analysis/config_variants.py +++ b/src/nominatim_db/tokenizer/token_analysis/config_variants.py @@ -12,8 +12,8 @@ from collections import defaultdict import itertools import re -from nominatim_core.config import flatten_config_list -from nominatim_core.errors import UsageError +from ...config import flatten_config_list +from ...errors import UsageError class ICUVariant(NamedTuple): """ A single replacement rule for variant creation. diff --git a/src/nominatim_db/tokenizer/token_analysis/generic.py b/src/nominatim_db/tokenizer/token_analysis/generic.py index cd649e62..30f1944e 100644 --- a/src/nominatim_db/tokenizer/token_analysis/generic.py +++ b/src/nominatim_db/tokenizer/token_analysis/generic.py @@ -12,7 +12,7 @@ import itertools import datrie -from nominatim_core.errors import UsageError +from ...errors import UsageError from ...data.place_name import PlaceName from .config_variants import get_variant_config from .generic_mutation import MutationVariantGenerator diff --git a/src/nominatim_db/tokenizer/token_analysis/generic_mutation.py b/src/nominatim_db/tokenizer/token_analysis/generic_mutation.py index 03cc63db..be70b49d 100644 --- a/src/nominatim_db/tokenizer/token_analysis/generic_mutation.py +++ b/src/nominatim_db/tokenizer/token_analysis/generic_mutation.py @@ -12,7 +12,7 @@ import itertools import logging import re -from nominatim_core.errors import UsageError +from ...errors import UsageError LOG = logging.getLogger() diff --git a/src/nominatim_db/tools/add_osm_data.py b/src/nominatim_db/tools/add_osm_data.py index 44ff2197..f9e2e648 100644 --- a/src/nominatim_db/tools/add_osm_data.py +++ b/src/nominatim_db/tools/add_osm_data.py @@ -12,8 +12,8 @@ from pathlib import Path import logging import urllib -from nominatim_core.db.connection import connect -from nominatim_core.utils.url_utils import get_url +from ..db.connection import connect +from ..utils.url_utils import get_url from .exec_utils import run_osm2pgsql LOG = logging.getLogger() diff --git a/src/nominatim_db/tools/admin.py b/src/nominatim_db/tools/admin.py index 7cc0f047..cea2ad66 100644 --- a/src/nominatim_db/tools/admin.py +++ b/src/nominatim_db/tools/admin.py @@ -13,10 +13,10 @@ import logging from psycopg2.extras import Json, register_hstore from psycopg2 import DataError -from nominatim_core.typing import DictCursorResult -from nominatim_core.config import Configuration -from nominatim_core.db.connection import connect, Cursor -from nominatim_core.errors import UsageError +from ..typing import DictCursorResult +from ..config import Configuration +from ..db.connection import connect, Cursor +from ..errors import UsageError from ..tokenizer import factory as tokenizer_factory from ..data.place_info import PlaceInfo diff --git a/src/nominatim_db/tools/check_database.py b/src/nominatim_db/tools/check_database.py index d2659877..ef28a0e5 100644 --- a/src/nominatim_db/tools/check_database.py +++ b/src/nominatim_db/tools/check_database.py @@ -11,10 +11,10 @@ from typing import Callable, Optional, Any, Union, Tuple, Mapping, List from enum import Enum from textwrap import dedent -from nominatim_core.config import Configuration -from nominatim_core.db.connection import connect, Connection -from nominatim_core.db import properties -from nominatim_core.errors import UsageError +from ..config import Configuration +from ..db.connection import connect, Connection +from ..db import properties +from ..errors import UsageError from ..tokenizer import factory as tokenizer_factory from . import freeze from ..version import NOMINATIM_VERSION, parse_version diff --git a/src/nominatim_db/tools/collect_os_info.py b/src/nominatim_db/tools/collect_os_info.py index 5096ce69..e1f8b166 100644 --- a/src/nominatim_db/tools/collect_os_info.py +++ b/src/nominatim_db/tools/collect_os_info.py @@ -17,8 +17,8 @@ from typing import List, Optional, Tuple, Union import psutil from psycopg2.extensions import make_dsn, parse_dsn -from nominatim_core.config import Configuration -from nominatim_core.db.connection import connect +from ..config import Configuration +from ..db.connection import connect from ..version import NOMINATIM_VERSION diff --git a/src/nominatim_db/tools/convert_sqlite.py b/src/nominatim_db/tools/convert_sqlite.py index 47ab63b7..2377abc0 100644 --- a/src/nominatim_db/tools/convert_sqlite.py +++ b/src/nominatim_db/tools/convert_sqlite.py @@ -16,8 +16,8 @@ import sqlalchemy as sa import nominatim_api as napi from nominatim_api.search.query_analyzer_factory import make_query_analyzer -from nominatim_core.typing import SaSelect, SaRow -from nominatim_core.db.sqlalchemy_types import Geometry, IntArray +from nominatim_api.typing import SaSelect, SaRow +from nominatim_api.sql.sqlalchemy_types import Geometry, IntArray LOG = logging.getLogger() diff --git a/src/nominatim_db/tools/database_import.py b/src/nominatim_db/tools/database_import.py index 84f2f325..d07febc8 100644 --- a/src/nominatim_db/tools/database_import.py +++ b/src/nominatim_db/tools/database_import.py @@ -17,11 +17,11 @@ from pathlib import Path import psutil from psycopg2 import sql as pysql -from nominatim_core.errors import UsageError -from nominatim_core.config import Configuration -from nominatim_core.db.connection import connect, get_pg_env, Connection -from nominatim_core.db.async_connection import DBConnection -from nominatim_core.db.sql_preprocessor import SQLPreprocessor +from ..errors import UsageError +from ..config import Configuration +from ..db.connection import connect, get_pg_env, Connection +from ..db.async_connection import DBConnection +from ..db.sql_preprocessor import SQLPreprocessor from .exec_utils import run_osm2pgsql from ..version import POSTGRESQL_REQUIRED_VERSION, POSTGIS_REQUIRED_VERSION diff --git a/src/nominatim_db/tools/exec_utils.py b/src/nominatim_db/tools/exec_utils.py index 467347f8..406e2511 100644 --- a/src/nominatim_db/tools/exec_utils.py +++ b/src/nominatim_db/tools/exec_utils.py @@ -13,8 +13,8 @@ import os import subprocess import shutil -from nominatim_core.typing import StrPath -from nominatim_core.db.connection import get_pg_env +from ..typing import StrPath +from ..db.connection import get_pg_env LOG = logging.getLogger() diff --git a/src/nominatim_db/tools/freeze.py b/src/nominatim_db/tools/freeze.py index 2023bb8a..bd52ba9a 100644 --- a/src/nominatim_db/tools/freeze.py +++ b/src/nominatim_db/tools/freeze.py @@ -12,7 +12,7 @@ from pathlib import Path from psycopg2 import sql as pysql -from nominatim_core.db.connection import Connection +from ..db.connection import Connection UPDATE_TABLES = [ 'address_levels', diff --git a/src/nominatim_db/tools/migration.py b/src/nominatim_db/tools/migration.py index 0712f187..e6803c7d 100644 --- a/src/nominatim_db/tools/migration.py +++ b/src/nominatim_db/tools/migration.py @@ -12,10 +12,10 @@ import logging from psycopg2 import sql as pysql -from nominatim_core.errors import UsageError -from nominatim_core.config import Configuration -from nominatim_core.db import properties -from nominatim_core.db.connection import connect, Connection +from ..errors import UsageError +from ..config import Configuration +from ..db import properties +from ..db.connection import connect, Connection from ..version import NominatimVersion, NOMINATIM_VERSION, parse_version from ..tokenizer import factory as tokenizer_factory from . import refresh diff --git a/src/nominatim_db/tools/postcodes.py b/src/nominatim_db/tools/postcodes.py index 772651b3..8dc5bdbd 100644 --- a/src/nominatim_db/tools/postcodes.py +++ b/src/nominatim_db/tools/postcodes.py @@ -18,8 +18,8 @@ from math import isfinite from psycopg2 import sql as pysql -from nominatim_core.db.connection import connect, Connection -from nominatim_core.utils.centroid import PointsCentroid +from ..db.connection import connect, Connection +from ..utils.centroid import PointsCentroid from ..data.postcode_format import PostcodeFormatter, CountryPostcodeMatcher from ..tokenizer.base import AbstractAnalyzer, AbstractTokenizer diff --git a/src/nominatim_db/tools/refresh.py b/src/nominatim_db/tools/refresh.py index 96608097..6946a41a 100644 --- a/src/nominatim_db/tools/refresh.py +++ b/src/nominatim_db/tools/refresh.py @@ -16,10 +16,10 @@ from pathlib import Path from psycopg2 import sql as pysql -from nominatim_core.config import Configuration -from nominatim_core.db.connection import Connection, connect -from nominatim_core.db.utils import execute_file, CopyBuffer -from nominatim_core.db.sql_preprocessor import SQLPreprocessor +from ..config import Configuration +from ..db.connection import Connection, connect +from ..db.utils import execute_file, CopyBuffer +from ..db.sql_preprocessor import SQLPreprocessor from ..version import NOMINATIM_VERSION LOG = logging.getLogger() diff --git a/src/nominatim_db/tools/replication.py b/src/nominatim_db/tools/replication.py index f9421bb8..bf1189df 100644 --- a/src/nominatim_db/tools/replication.py +++ b/src/nominatim_db/tools/replication.py @@ -18,9 +18,9 @@ import urllib.request as urlrequest import requests -from nominatim_core.errors import UsageError -from nominatim_core.db import status -from nominatim_core.db.connection import Connection, connect +from ..errors import UsageError +from ..db import status +from ..db.connection import Connection, connect from .exec_utils import run_osm2pgsql try: diff --git a/src/nominatim_db/tools/special_phrases/sp_csv_loader.py b/src/nominatim_db/tools/special_phrases/sp_csv_loader.py index 9f472e68..db4806cd 100644 --- a/src/nominatim_db/tools/special_phrases/sp_csv_loader.py +++ b/src/nominatim_db/tools/special_phrases/sp_csv_loader.py @@ -13,7 +13,7 @@ from typing import Iterable import csv import os -from nominatim_core.errors import UsageError +from ...errors import UsageError from .special_phrase import SpecialPhrase class SPCsvLoader: diff --git a/src/nominatim_db/tools/special_phrases/sp_importer.py b/src/nominatim_db/tools/special_phrases/sp_importer.py index 9749db3e..1bdcdaf1 100644 --- a/src/nominatim_db/tools/special_phrases/sp_importer.py +++ b/src/nominatim_db/tools/special_phrases/sp_importer.py @@ -19,9 +19,9 @@ import re from psycopg2.sql import Identifier, SQL -from nominatim_core.typing import Protocol -from nominatim_core.config import Configuration -from nominatim_core.db.connection import Connection +from ...typing import Protocol +from ...config import Configuration +from ...db.connection import Connection from .importer_statistics import SpecialPhrasesImporterStatistics from .special_phrase import SpecialPhrase from ...tokenizer.base import AbstractTokenizer diff --git a/src/nominatim_db/tools/special_phrases/sp_wiki_loader.py b/src/nominatim_db/tools/special_phrases/sp_wiki_loader.py index c7e38336..0fe7c0aa 100644 --- a/src/nominatim_db/tools/special_phrases/sp_wiki_loader.py +++ b/src/nominatim_db/tools/special_phrases/sp_wiki_loader.py @@ -11,8 +11,8 @@ from typing import Iterable import re import logging -from nominatim_core.config import Configuration -from nominatim_core.utils.url_utils import get_url +from ...config import Configuration +from ...utils.url_utils import get_url from .special_phrase import SpecialPhrase LOG = logging.getLogger() diff --git a/src/nominatim_db/tools/tiger_data.py b/src/nominatim_db/tools/tiger_data.py index 6030a38a..7c52b710 100644 --- a/src/nominatim_db/tools/tiger_data.py +++ b/src/nominatim_db/tools/tiger_data.py @@ -16,11 +16,11 @@ import tarfile from psycopg2.extras import Json -from nominatim_core.config import Configuration -from nominatim_core.db.connection import connect -from nominatim_core.db.async_connection import WorkerPool -from nominatim_core.db.sql_preprocessor import SQLPreprocessor -from nominatim_core.errors import UsageError +from ..config import Configuration +from ..db.connection import connect +from ..db.async_connection import WorkerPool +from ..db.sql_preprocessor import SQLPreprocessor +from ..errors import UsageError from ..data.place_info import PlaceInfo from ..tokenizer.base import AbstractAnalyzer, AbstractTokenizer from . import freeze diff --git a/src/nominatim_core/typing.py b/src/nominatim_db/typing.py similarity index 65% rename from src/nominatim_core/typing.py rename to src/nominatim_db/typing.py index 1f4f1fd7..f1abee82 100644 --- a/src/nominatim_core/typing.py +++ b/src/nominatim_db/typing.py @@ -50,26 +50,3 @@ else: Protocol = object Final = 'Final' TypedDict = dict - - -# SQLAlchemy introduced generic types in version 2.0 making typing -# incompatible with older versions. Add wrappers here so we don't have -# to litter the code with bare-string types. - -if TYPE_CHECKING: - import sqlalchemy as sa - from typing_extensions import (TypeAlias as TypeAlias) -else: - TypeAlias = str - -SaLambdaSelect: TypeAlias = 'Union[sa.Select[Any], sa.StatementLambdaElement]' -SaSelect: TypeAlias = 'sa.Select[Any]' -SaScalarSelect: TypeAlias = 'sa.ScalarSelect[Any]' -SaRow: TypeAlias = 'sa.Row[Any]' -SaColumn: TypeAlias = 'sa.ColumnElement[Any]' -SaExpression: TypeAlias = 'sa.ColumnElement[bool]' -SaLabel: TypeAlias = 'sa.Label[Any]' -SaFromClause: TypeAlias = 'sa.FromClause' -SaSelectable: TypeAlias = 'sa.Selectable' -SaBind: TypeAlias = 'sa.BindParameter[Any]' -SaDialect: TypeAlias = 'sa.Dialect' diff --git a/src/nominatim_core/utils/__init__.py b/src/nominatim_db/utils/__init__.py similarity index 100% rename from src/nominatim_core/utils/__init__.py rename to src/nominatim_db/utils/__init__.py diff --git a/src/nominatim_core/utils/centroid.py b/src/nominatim_db/utils/centroid.py similarity index 100% rename from src/nominatim_core/utils/centroid.py rename to src/nominatim_db/utils/centroid.py diff --git a/src/nominatim_core/utils/url_utils.py b/src/nominatim_db/utils/url_utils.py similarity index 87% rename from src/nominatim_core/utils/url_utils.py rename to src/nominatim_db/utils/url_utils.py index d7689187..eb56f72e 100644 --- a/src/nominatim_core/utils/url_utils.py +++ b/src/nominatim_db/utils/url_utils.py @@ -11,7 +11,7 @@ from typing import IO import logging import urllib.request as urlrequest -from ..version import NOMINATIM_CORE_VERSION +from ..version import NOMINATIM_VERSION LOG = logging.getLogger() @@ -20,7 +20,7 @@ def get_url(url: str) -> str: This version makes sure that an appropriate user agent is sent. """ - headers = {"User-Agent": f"Nominatim/{NOMINATIM_CORE_VERSION!s}"} + headers = {"User-Agent": f"Nominatim/{NOMINATIM_VERSION!s}"} try: request = urlrequest.Request(url, headers=headers) diff --git a/src/nominatim_db/version.py b/src/nominatim_db/version.py index 34e3b432..70e1ac14 100644 --- a/src/nominatim_db/version.py +++ b/src/nominatim_db/version.py @@ -7,13 +7,55 @@ """ Version information for Nominatim. """ -from typing import Optional +from typing import NamedTuple, Optional # See also https://github.com/PyCQA/pylint/issues/6006 # pylint: disable=useless-import-alias,unused-import -from nominatim_core.version import (NominatimVersion as NominatimVersion, - parse_version as parse_version) +class NominatimVersion(NamedTuple): + """ Version information for Nominatim. We follow semantic versioning. + + Major, minor and patch_level refer to the last released version. + The database patch level tracks important changes between releases + and must always be increased when there is a change to the database or code + that requires a migration. + + When adding a migration on the development branch, raise the patch level + to 99 to make sure that the migration is applied when updating from a + patch release to the next minor version. Patch releases usually shouldn't + have migrations in them. When they are needed, then make sure that the + migration can be reapplied and set the migration version to the appropriate + patch level when cherry-picking the commit with the migration. + """ + + major: int + minor: int + patch_level: int + db_patch_level: Optional[int] + + def __str__(self) -> str: + if self.db_patch_level is None: + return f"{self.major}.{self.minor}.{self.patch_level}" + + return f"{self.major}.{self.minor}.{self.patch_level}-{self.db_patch_level}" + + def release_version(self) -> str: + """ Return the release version in semantic versioning format. + + The release version does not include the database patch version. + """ + return f"{self.major}.{self.minor}.{self.patch_level}" + + +def parse_version(version: str) -> NominatimVersion: + """ Parse a version string into a version consisting of a tuple of + four ints: major, minor, patch level, database patch level + + This is the reverse operation of `version_str()`. + """ + parts = version.split('.') + return NominatimVersion(*[int(x) for x in parts[:2] + parts[2].split('-')]) + NOMINATIM_VERSION = parse_version('4.4.99-1') diff --git a/test/bdd/steps/nominatim_environment.py b/test/bdd/steps/nominatim_environment.py index 46559720..dfbbee28 100644 --- a/test/bdd/steps/nominatim_environment.py +++ b/test/bdd/steps/nominatim_environment.py @@ -15,8 +15,8 @@ import psycopg2.extras sys.path.insert(1, str((Path(__file__) / '..' / '..' / '..' / '..'/ 'src').resolve())) from nominatim_db import cli -from nominatim_core.config import Configuration -from nominatim_core.db.connection import Connection +from nominatim_db.config import Configuration +from nominatim_db.db.connection import Connection from nominatim_db.tools import refresh from nominatim_db.tokenizer import factory as tokenizer_factory from steps.utils import run_script diff --git a/test/python/api/conftest.py b/test/python/api/conftest.py index 08b7422c..a902e264 100644 --- a/test/python/api/conftest.py +++ b/test/python/api/conftest.py @@ -15,7 +15,7 @@ import datetime as dt import sqlalchemy as sa import nominatim_api as napi -from nominatim_core.db.sql_preprocessor import SQLPreprocessor +from nominatim_db.db.sql_preprocessor import SQLPreprocessor from nominatim_api.search.query_analyzer_factory import make_query_analyzer from nominatim_db.tools import convert_sqlite import nominatim_api.logging as loglib diff --git a/test/python/api/fake_adaptor.py b/test/python/api/fake_adaptor.py index 50fbbbbe..9caa9226 100644 --- a/test/python/api/fake_adaptor.py +++ b/test/python/api/fake_adaptor.py @@ -10,7 +10,7 @@ Provides dummy implementations of ASGIAdaptor for testing. from collections import namedtuple import nominatim_api.v1.server_glue as glue -from nominatim_core.config import Configuration +from nominatim_api.config import Configuration class FakeError(BaseException): diff --git a/test/python/api/test_api_status.py b/test/python/api/test_api_status.py index 2b211611..5412ca6e 100644 --- a/test/python/api/test_api_status.py +++ b/test/python/api/test_api_status.py @@ -39,7 +39,7 @@ def test_status_full(apiobj, frontend): assert result.status == 0 assert result.message == 'OK' assert result.software_version == NOMINATIM_API_VERSION - assert result.database_version == NominatimVersion(99, 5, 4, 2) + assert result.database_version == '99.5.4-2' assert result.data_updated == import_date diff --git a/test/python/api/test_api_types.py b/test/python/api/test_api_types.py index 2faba33d..fbb9b682 100644 --- a/test/python/api/test_api_types.py +++ b/test/python/api/test_api_types.py @@ -9,7 +9,7 @@ Tests for loading of parameter dataclasses. """ import pytest -from nominatim_core.errors import UsageError +from nominatim_api.errors import UsageError import nominatim_api.types as typ def test_no_params_defaults(): diff --git a/test/python/cli/test_cmd_import.py b/test/python/cli/test_cmd_import.py index 9c2400c5..85235e1e 100644 --- a/test/python/cli/test_cmd_import.py +++ b/test/python/cli/test_cmd_import.py @@ -14,7 +14,7 @@ import nominatim_db.data.country_info import nominatim_db.tools.refresh import nominatim_db.tools.postcodes import nominatim_db.indexer.indexer -import nominatim_core.db.properties +import nominatim_db.db.properties class TestCliImportWithDb: @@ -81,7 +81,7 @@ class TestCliImportWithDb: mock_func_factory(nominatim_db.tools.postcodes, 'update_postcodes'), mock_func_factory(nominatim_db.indexer.indexer.Indexer, 'index_full'), mock_func_factory(nominatim_db.tools.refresh, 'setup_website'), - mock_func_factory(nominatim_core.db.properties, 'set_property') + mock_func_factory(nominatim_db.db.properties, 'set_property') ] assert self.call_nominatim('import', '--continue', 'load-data') == 0 @@ -98,7 +98,7 @@ class TestCliImportWithDb: mock_func_factory(nominatim_db.data.country_info, 'create_country_names'), mock_func_factory(nominatim_db.indexer.indexer.Indexer, 'index_full'), mock_func_factory(nominatim_db.tools.refresh, 'setup_website'), - mock_func_factory(nominatim_core.db.properties, 'set_property') + mock_func_factory(nominatim_db.db.properties, 'set_property') ] assert self.call_nominatim('import', '--continue', 'indexing') == 0 @@ -115,7 +115,7 @@ class TestCliImportWithDb: mock_func_factory(nominatim_db.tools.database_import, 'create_search_indices'), mock_func_factory(nominatim_db.data.country_info, 'create_country_names'), mock_func_factory(nominatim_db.tools.refresh, 'setup_website'), - mock_func_factory(nominatim_core.db.properties, 'set_property') + mock_func_factory(nominatim_db.db.properties, 'set_property') ] assert self.call_nominatim('import', '--continue', 'db-postprocess') == 0 diff --git a/test/python/cli/test_cmd_replication.py b/test/python/cli/test_cmd_replication.py index ff900e58..8c1e8ea6 100644 --- a/test/python/cli/test_cmd_replication.py +++ b/test/python/cli/test_cmd_replication.py @@ -16,7 +16,7 @@ import nominatim_db.cli import nominatim_db.indexer.indexer import nominatim_db.tools.replication import nominatim_db.tools.refresh -from nominatim_core.db import status +from nominatim_db.db import status @pytest.fixture def tokenizer_mock(monkeypatch): diff --git a/test/python/config/test_config.py b/test/python/config/test_config.py index bb649d23..5c9393ec 100644 --- a/test/python/config/test_config.py +++ b/test/python/config/test_config.py @@ -10,8 +10,8 @@ Test for loading dotenv configuration. from pathlib import Path import pytest -from nominatim_core.config import Configuration, flatten_config_list -from nominatim_core.errors import UsageError +from nominatim_db.config import Configuration, flatten_config_list +from nominatim_db.errors import UsageError @pytest.fixture def make_config(): diff --git a/test/python/config/test_config_load_module.py b/test/python/config/test_config_load_module.py index 06bfff06..c2912180 100644 --- a/test/python/config/test_config_load_module.py +++ b/test/python/config/test_config_load_module.py @@ -12,7 +12,7 @@ import sys import pytest -from nominatim_core.config import Configuration +from nominatim_db.config import Configuration @pytest.fixture def test_config(src_dir, tmp_path): diff --git a/test/python/conftest.py b/test/python/conftest.py index c9e8f048..d301c973 100644 --- a/test/python/conftest.py +++ b/test/python/conftest.py @@ -15,9 +15,9 @@ import pytest SRC_DIR = (Path(__file__) / '..' / '..' / '..').resolve() sys.path.insert(0, str(SRC_DIR / 'src')) -from nominatim_core.config import Configuration -from nominatim_core.db import connection -from nominatim_core.db.sql_preprocessor import SQLPreprocessor +from nominatim_db.config import Configuration +from nominatim_db.db import connection +from nominatim_db.db.sql_preprocessor import SQLPreprocessor import nominatim_db.tokenizer.factory import dummy_tokenizer diff --git a/test/python/db/test_async_connection.py b/test/python/db/test_async_connection.py index 019f53c1..fff695e5 100644 --- a/test/python/db/test_async_connection.py +++ b/test/python/db/test_async_connection.py @@ -13,7 +13,7 @@ import concurrent.futures import pytest import psycopg2 -from nominatim_core.db.async_connection import DBConnection, DeadlockHandler +from nominatim_db.db.async_connection import DBConnection, DeadlockHandler @pytest.fixture diff --git a/test/python/db/test_connection.py b/test/python/db/test_connection.py index 7ecf3bb4..8b4cc62f 100644 --- a/test/python/db/test_connection.py +++ b/test/python/db/test_connection.py @@ -10,7 +10,7 @@ Tests for specialised connection and cursor classes. import pytest import psycopg2 -from nominatim_core.db.connection import connect, get_pg_env +from nominatim_db.db.connection import connect, get_pg_env @pytest.fixture def db(dsn): diff --git a/test/python/db/test_properties.py b/test/python/db/test_properties.py index aca1b1f8..e55bb973 100644 --- a/test/python/db/test_properties.py +++ b/test/python/db/test_properties.py @@ -9,7 +9,7 @@ Tests for property table manpulation. """ import pytest -from nominatim_core.db import properties +from nominatim_db.db import properties @pytest.fixture def property_factory(property_table, temp_db_cursor): diff --git a/test/python/db/test_sql_preprocessor.py b/test/python/db/test_sql_preprocessor.py index 1ef6bc88..114c5244 100644 --- a/test/python/db/test_sql_preprocessor.py +++ b/test/python/db/test_sql_preprocessor.py @@ -9,7 +9,7 @@ Tests for SQL preprocessing. """ import pytest -from nominatim_core.db.sql_preprocessor import SQLPreprocessor +from nominatim_db.db.sql_preprocessor import SQLPreprocessor @pytest.fixture def sql_factory(tmp_path): diff --git a/test/python/db/test_status.py b/test/python/db/test_status.py index 67cd22e9..77135a8c 100644 --- a/test/python/db/test_status.py +++ b/test/python/db/test_status.py @@ -11,8 +11,8 @@ import datetime as dt import pytest -import nominatim_core.db.status -from nominatim_core.errors import UsageError +import nominatim_db.db.status +from nominatim_db.errors import UsageError OSM_NODE_DATA = """\ @@ -22,7 +22,7 @@ OSM_NODE_DATA = """\ """ def iso_date(date): - return dt.datetime.strptime(date, nominatim_core.db.status.ISODATE_FORMAT)\ + return dt.datetime.strptime(date, nominatim_db.db.status.ISODATE_FORMAT)\ .replace(tzinfo=dt.timezone.utc) @@ -36,7 +36,7 @@ def test_compute_database_date_from_osm2pgsql(table_factory, temp_db_conn, offli table_factory('osm2pgsql_properties', 'property TEXT, value TEXT', content=(('current_timestamp', '2024-01-03T23:45:54Z'), )) - date = nominatim_core.db.status.compute_database_date(temp_db_conn, offline=offline) + date = nominatim_db.db.status.compute_database_date(temp_db_conn, offline=offline) assert date == iso_date('2024-01-03T23:45:54') @@ -44,12 +44,12 @@ def test_compute_database_date_from_osm2pgsql_nodata(table_factory, temp_db_conn table_factory('osm2pgsql_properties', 'property TEXT, value TEXT') with pytest.raises(UsageError, match='Cannot determine database date from data in offline mode'): - nominatim_core.db.status.compute_database_date(temp_db_conn, offline=True) + nominatim_db.db.status.compute_database_date(temp_db_conn, offline=True) def test_compute_database_date_place_empty(place_table, temp_db_conn): with pytest.raises(UsageError): - nominatim_core.db.status.compute_database_date(temp_db_conn) + nominatim_db.db.status.compute_database_date(temp_db_conn) def test_compute_database_date_valid(monkeypatch, place_row, temp_db_conn): @@ -60,9 +60,9 @@ def test_compute_database_date_valid(monkeypatch, place_row, temp_db_conn): requested_url.append(url) return OSM_NODE_DATA - monkeypatch.setattr(nominatim_core.db.status, "get_url", mock_url) + monkeypatch.setattr(nominatim_db.db.status, "get_url", mock_url) - date = nominatim_core.db.status.compute_database_date(temp_db_conn) + date = nominatim_db.db.status.compute_database_date(temp_db_conn) assert requested_url == ['https://www.openstreetmap.org/api/0.6/node/45673/1'] assert date == iso_date('2006-01-27T22:09:10') @@ -76,15 +76,15 @@ def test_compute_database_broken_api(monkeypatch, place_row, temp_db_conn): requested_url.append(url) return ' ' '; diff --git a/test/python/tokenizer/token_analysis/test_generic.py b/test/python/tokenizer/token_analysis/test_generic.py index 277758e1..191f551f 100644 --- a/test/python/tokenizer/token_analysis/test_generic.py +++ b/test/python/tokenizer/token_analysis/test_generic.py @@ -12,7 +12,7 @@ import pytest from icu import Transliterator import nominatim_db.tokenizer.token_analysis.generic as module -from nominatim_core.errors import UsageError +from nominatim_db.errors import UsageError DEFAULT_NORMALIZATION = """ :: NFD (); '🜳' > ' '; diff --git a/test/python/tokenizer/token_analysis/test_generic_mutation.py b/test/python/tokenizer/token_analysis/test_generic_mutation.py index ea813bdc..7d0db925 100644 --- a/test/python/tokenizer/token_analysis/test_generic_mutation.py +++ b/test/python/tokenizer/token_analysis/test_generic_mutation.py @@ -12,7 +12,7 @@ import pytest from icu import Transliterator import nominatim_db.tokenizer.token_analysis.generic as module -from nominatim_core.errors import UsageError +from nominatim_db.errors import UsageError DEFAULT_NORMALIZATION = """ '🜳' > ' '; [[:Nonspacing Mark:] [:Cf:]] >; diff --git a/test/python/tools/test_admin.py b/test/python/tools/test_admin.py index a59f3d9d..1e1f0e29 100644 --- a/test/python/tools/test_admin.py +++ b/test/python/tools/test_admin.py @@ -9,10 +9,10 @@ Tests for maintenance and analysis functions. """ import pytest -from nominatim_core.errors import UsageError +from nominatim_db.errors import UsageError from nominatim_db.tools import admin from nominatim_db.tokenizer import factory -from nominatim_core.db.sql_preprocessor import SQLPreprocessor +from nominatim_db.db.sql_preprocessor import SQLPreprocessor @pytest.fixture(autouse=True) def create_placex_table(project_env, tokenizer_mock, temp_db_cursor, placex_table): diff --git a/test/python/tools/test_database_import.py b/test/python/tools/test_database_import.py index a95f3d98..548ec800 100644 --- a/test/python/tools/test_database_import.py +++ b/test/python/tools/test_database_import.py @@ -14,7 +14,7 @@ import pytest import psycopg2 from nominatim_db.tools import database_import -from nominatim_core.errors import UsageError +from nominatim_db.errors import UsageError class TestDatabaseSetup: DBNAME = 'test_nominatim_python_unittest' diff --git a/test/python/tools/test_exec_utils.py b/test/python/tools/test_exec_utils.py index 50bec7e6..666ef0b8 100644 --- a/test/python/tools/test_exec_utils.py +++ b/test/python/tools/test_exec_utils.py @@ -12,7 +12,7 @@ import subprocess import pytest -from nominatim_core.config import Configuration +from nominatim_db.config import Configuration import nominatim_db.tools.exec_utils as exec_utils def test_run_osm2pgsql(osm2pgsql_options): diff --git a/test/python/tools/test_import_special_phrases.py b/test/python/tools/test_import_special_phrases.py index 91c9fc15..64eb7b18 100644 --- a/test/python/tools/test_import_special_phrases.py +++ b/test/python/tools/test_import_special_phrases.py @@ -13,7 +13,7 @@ import pytest from nominatim_db.tools.special_phrases.sp_importer import SPImporter from nominatim_db.tools.special_phrases.sp_wiki_loader import SPWikiLoader from nominatim_db.tools.special_phrases.special_phrase import SpecialPhrase -from nominatim_core.errors import UsageError +from nominatim_db.errors import UsageError from cursor import CursorForTesting diff --git a/test/python/tools/test_migration.py b/test/python/tools/test_migration.py index d33ff422..3a849adb 100644 --- a/test/python/tools/test_migration.py +++ b/test/python/tools/test_migration.py @@ -11,7 +11,7 @@ import pytest import psycopg2.extras from nominatim_db.tools import migration -from nominatim_core.errors import UsageError +from nominatim_db.errors import UsageError import nominatim_db.version from mock_legacy_word_table import MockLegacyWordTable diff --git a/test/python/tools/test_replication.py b/test/python/tools/test_replication.py index 4cd53ae3..392ea075 100644 --- a/test/python/tools/test_replication.py +++ b/test/python/tools/test_replication.py @@ -14,8 +14,8 @@ import pytest from osmium.replication.server import OsmosisState import nominatim_db.tools.replication -import nominatim_core.db.status as status -from nominatim_core.errors import UsageError +import nominatim_db.db.status as status +from nominatim_db.errors import UsageError OSM_NODE_DATA = """\ diff --git a/test/python/tools/test_sp_csv_loader.py b/test/python/tools/test_sp_csv_loader.py index e6185fd6..9d0ad9cc 100644 --- a/test/python/tools/test_sp_csv_loader.py +++ b/test/python/tools/test_sp_csv_loader.py @@ -9,7 +9,7 @@ """ import pytest -from nominatim_core.errors import UsageError +from nominatim_db.errors import UsageError from nominatim_db.tools.special_phrases.sp_csv_loader import SPCsvLoader from nominatim_db.tools.special_phrases.special_phrase import SpecialPhrase diff --git a/test/python/tools/test_tiger_data.py b/test/python/tools/test_tiger_data.py index 396fe069..7ef6a1e6 100644 --- a/test/python/tools/test_tiger_data.py +++ b/test/python/tools/test_tiger_data.py @@ -13,7 +13,7 @@ from textwrap import dedent import pytest from nominatim_db.tools import tiger_data, freeze -from nominatim_core.errors import UsageError +from nominatim_db.errors import UsageError class MockTigerTable: diff --git a/test/python/utils/test_centroid.py b/test/python/utils/test_centroid.py index d91edae2..bac0edb3 100644 --- a/test/python/utils/test_centroid.py +++ b/test/python/utils/test_centroid.py @@ -9,7 +9,7 @@ Tests for centroid computation. """ import pytest -from nominatim_core.utils.centroid import PointsCentroid +from nominatim_db.utils.centroid import PointsCentroid def test_empty_set(): c = PointsCentroid() diff --git a/test/python/utils/test_json_writer.py b/test/python/utils/test_json_writer.py index ff542f04..53e3f4d3 100644 --- a/test/python/utils/test_json_writer.py +++ b/test/python/utils/test_json_writer.py @@ -11,7 +11,7 @@ import json import pytest -from nominatim_core.utils.json_writer import JsonWriter +from nominatim_api.utils.json_writer import JsonWriter @pytest.mark.parametrize("inval,outstr", [(None, 'null'), (True, 'true'), (False, 'false'), diff --git a/vagrant/Install-on-Ubuntu-24.sh b/vagrant/Install-on-Ubuntu-24.sh index 3411c2ba..05dc38a4 100644 --- a/vagrant/Install-on-Ubuntu-24.sh +++ b/vagrant/Install-on-Ubuntu-24.sh @@ -119,7 +119,7 @@ fi #DOCS: # Now install Nominatim using pip: cd $USERHOME/Nominatim - $USERHOME/nominatim-venv/bin/pip install packaging/nominatim-{core,db} + $USERHOME/nominatim-venv/bin/pip install packaging/nominatim-db # Nominatim is now ready to use. The nominatim binary is available at # `$USERHOME/venv/bin/nominatim`. If you want to have 'nominatim' in your -- 2.39.5