From a72e2ecb3f4c2d0127e4a8e9cba1b1414976854d Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Tue, 13 Dec 2022 12:12:05 +0100 Subject: [PATCH] update dependencies for Actions --- .github/actions/build-nominatim/action.yml | 5 ++- .github/workflows/ci-tests.yml | 47 ++++++++++++---------- docs/admin/Installation.md | 3 +- test/bdd/steps/nominatim_environment.py | 5 +-- vagrant/Install-on-Ubuntu-18.sh | 2 +- vagrant/Install-on-Ubuntu-20.sh | 8 +++- vagrant/Install-on-Ubuntu-22.sh | 3 +- 7 files changed, 42 insertions(+), 31 deletions(-) diff --git a/.github/actions/build-nominatim/action.yml b/.github/actions/build-nominatim/action.yml index acc9b8b6..cc500c5d 100644 --- a/.github/actions/build-nominatim/action.yml +++ b/.github/actions/build-nominatim/action.yml @@ -27,9 +27,10 @@ runs: run: | sudo apt-get install -y -qq libboost-system-dev libboost-filesystem-dev libexpat1-dev zlib1g-dev libbz2-dev libpq-dev libproj-dev libicu-dev liblua${LUA_VERSION}-dev lua${LUA_VERSION} if [ "x$UBUNTUVER" == "x18" ]; then - pip3 install python-dotenv psycopg2==2.7.7 jinja2==2.8 psutil==5.4.2 pyicu==2.9 osmium PyYAML==5.1 datrie + pip3 install python-dotenv psycopg2==2.7.7 jinja2==2.8 psutil==5.4.2 pyicu==2.9 osmium PyYAML==5.1 sqlalchemy==1.4 datrie asyncpg else - sudo apt-get install -y -qq python3-icu python3-datrie python3-pyosmium python3-jinja2 python3-psutil python3-psycopg2 python3-dotenv python3-yaml + sudo apt-get install -y -qq python3-icu python3-datrie python3-pyosmium python3-jinja2 python3-psutil python3-psycopg2 python3-dotenv python3-yaml python3-asyncpg + pip3 install sqlalchemy fi shell: bash env: diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index cdc7ea1e..9b51a90b 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -42,17 +42,14 @@ jobs: - ubuntu: 18 postgresql: 9.6 postgis: 2.5 - pytest: pytest php: 7.2 - ubuntu: 20 postgresql: 13 postgis: 3 - pytest: py.test-3 php: 7.4 - ubuntu: 22 postgresql: 15 postgis: 3 - pytest: py.test-3 php: 8.1 runs-on: ubuntu-${{ matrix.ubuntu }}.04 @@ -86,50 +83,56 @@ jobs: with: ubuntu: ${{ matrix.ubuntu }} - - name: Install test prerequsites - run: sudo apt-get install -y -qq python3-pytest python3-behave + - name: Install test prerequsites (behave from apt) + run: sudo apt-get install -y -qq python3-behave if: matrix.ubuntu == 20 - - name: Install test prerequsites - run: pip3 install pylint pytest behave==1.2.6 + - name: Install test prerequsites (behave from pip) + run: pip3 install behave==1.2.6 if: ${{ (matrix.ubuntu == 18) || (matrix.ubuntu == 22) }} - - name: Install test prerequsites - run: sudo apt-get install -y -qq python3-pytest - if: matrix.ubuntu == 22 + - name: Install test prerequsites (from apt for Ununtu 2x) + run: sudo apt-get install -y -qq python3-pytest uvicorn + if: matrix.ubuntu >= 20 + + - name: Install test prerequsites (from pip for Ubuntu 18) + run: pip3 install pytest uvicorn + if: matrix.ubuntu == 18 + + - name: Install Python webservers + run: pip3 install falcon sanic sanic-testing starlette - name: Install latest pylint/mypy - run: pip3 install -U pylint mypy types-PyYAML types-jinja2 types-psycopg2 types-psutil types-requests typing-extensions + run: pip3 install -U pylint mypy types-PyYAML types-jinja2 types-psycopg2 types-psutil types-requests typing-extensions asgi_lifespan sqlalchemy2-stubs - name: PHP linting run: phpcs --report-width=120 . working-directory: Nominatim - name: Python linting - run: pylint nominatim + run: python3 -m pylint nominatim working-directory: Nominatim - - name: Python static typechecking - run: mypy --strict nominatim - working-directory: Nominatim - - - name: PHP unit tests run: phpunit ./ working-directory: Nominatim/test/php if: ${{ (matrix.ubuntu == 20) || (matrix.ubuntu == 22) }} - name: Python unit tests - run: $PYTEST test/python + run: python3 -m pytest test/python working-directory: Nominatim - env: - PYTEST: ${{ matrix.pytest }} - name: BDD tests run: | - behave -DREMOVE_TEMPLATE=1 -DBUILDDIR=$GITHUB_WORKSPACE/build --format=progress3 + python3 -m behave -DREMOVE_TEMPLATE=1 -DBUILDDIR=$GITHUB_WORKSPACE/build --format=progress3 working-directory: Nominatim/test/bdd + - name: Install newer Python packages (for typechecking info) + run: pip3 install -U osmium uvicorn + + - name: Python static typechecking + run: python3 -m mypy --strict nominatim + working-directory: Nominatim legacy-test: needs: create-archive @@ -166,7 +169,7 @@ jobs: - name: BDD tests (legacy tokenizer) run: | - behave -DREMOVE_TEMPLATE=1 -DBUILDDIR=$GITHUB_WORKSPACE/build -DTOKENIZER=legacy --format=progress3 + python3 -m behave -DREMOVE_TEMPLATE=1 -DBUILDDIR=$GITHUB_WORKSPACE/build -DTOKENIZER=legacy --format=progress3 working-directory: Nominatim/test/bdd diff --git a/docs/admin/Installation.md b/docs/admin/Installation.md index 4d78fd1b..36b8eeca 100644 --- a/docs/admin/Installation.md +++ b/docs/admin/Installation.md @@ -49,6 +49,8 @@ For running Nominatim: * [Python Dotenv](https://github.com/theskumar/python-dotenv) * [psutil](https://github.com/giampaolo/psutil) * [Jinja2](https://palletsprojects.com/p/jinja/) + * [SQLAlchemy](https://www.sqlalchemy.org/) (1.4+ with greenlet support) + * [asyncpg](https://magicstack.github.io/asyncpg) * [PyICU](https://pypi.org/project/PyICU/) * [PyYaml](https://pyyaml.org/) (5.1+) * [datrie](https://github.com/pytries/datrie) @@ -63,7 +65,6 @@ For running continuous updates: For running the experimental Python frontend: - * [SQLAlchemy](https://www.sqlalchemy.org/) (1.4+ with greenlet support) * one of the following web frameworks: * [falcon](https://falconframework.org/) (3.0+) * [sanic](https://sanic.dev) diff --git a/test/bdd/steps/nominatim_environment.py b/test/bdd/steps/nominatim_environment.py index 158b7974..e156c60c 100644 --- a/test/bdd/steps/nominatim_environment.py +++ b/test/bdd/steps/nominatim_environment.py @@ -9,9 +9,6 @@ import importlib import sys import tempfile -from asgi_lifespan import LifespanManager -import httpx - import psycopg2 import psycopg2.extras @@ -337,6 +334,8 @@ class NominatimEnvironment: def create_api_request_func_starlette(self): import nominatim.server.starlette.server + from asgi_lifespan import LifespanManager + import httpx async def _request(endpoint, params, project_dir, environ): app = nominatim.server.starlette.server.get_application(project_dir, environ) diff --git a/vagrant/Install-on-Ubuntu-18.sh b/vagrant/Install-on-Ubuntu-18.sh index 09de9747..f1b6c3c4 100755 --- a/vagrant/Install-on-Ubuntu-18.sh +++ b/vagrant/Install-on-Ubuntu-18.sh @@ -33,7 +33,7 @@ export DEBIAN_FRONTEND=noninteractive #DOCS: # Some of the Python packages that come with Ubuntu 18.04 are too old, so # install the latest version from pip: - pip3 install --user python-dotenv datrie pyyaml psycopg2-binary + pip3 install --user python-dotenv datrie pyyaml psycopg2-binary sqlalchemy asyncpg # # System Configuration diff --git a/vagrant/Install-on-Ubuntu-20.sh b/vagrant/Install-on-Ubuntu-20.sh index 2d4eaa71..34e81637 100755 --- a/vagrant/Install-on-Ubuntu-20.sh +++ b/vagrant/Install-on-Ubuntu-20.sh @@ -27,9 +27,15 @@ export DEBIAN_FRONTEND=noninteractive #DOCS: postgresql-12-postgis-3 \ postgresql-contrib-12 postgresql-12-postgis-3-scripts \ php-cli php-pgsql php-intl libicu-dev python3-dotenv \ - python3-psycopg2 python3-psutil python3-jinja2 \ + python3-psycopg2 python3-psutil python3-jinja2 python3-pip \ python3-icu python3-datrie python3-yaml git +# Some of the Python packages that come with Ubuntu 20.04 are too old, so +# install the latest version from pip: + + pip3 install --user sqlalchemy asyncpg + + # # System Configuration # ==================== diff --git a/vagrant/Install-on-Ubuntu-22.sh b/vagrant/Install-on-Ubuntu-22.sh index c45ac55e..82e706c9 100755 --- a/vagrant/Install-on-Ubuntu-22.sh +++ b/vagrant/Install-on-Ubuntu-22.sh @@ -28,7 +28,8 @@ export DEBIAN_FRONTEND=noninteractive #DOCS: postgresql-contrib-14 postgresql-14-postgis-3-scripts \ php-cli php-pgsql php-intl libicu-dev python3-dotenv \ python3-psycopg2 python3-psutil python3-jinja2 \ - python3-icu python3-datrie git + python3-icu python3-datrie python3-sqlalchemy \ + python3-asyncpg git # # System Configuration -- 2.39.5