.github/workflows/ci-tests.yml

   1 name: CI Tests
   2
   3 on: [ push, pull_request ]
   4
   5 jobs:
   6     create-archive:
   7         runs-on: ubuntu-latest
   8
   9         steps:
  10             - uses: actions/checkout@v3
  11               with:
  12                 submodules: true
  13
  14             - uses: actions/cache@v3
  15               with:
  16                   path: |
  17                      data/country_osm_grid.sql.gz
  18                   key: nominatim-country-data-1
  19
  20             - name: Package tarball
  21               run: |
  22                   if [ ! -f data/country_osm_grid.sql.gz ]; then
  23                       wget --no-verbose -O data/country_osm_grid.sql.gz https://www.nominatim.org/data/country_grid.sql.gz
  24                   fi
  25                   cd ..
  26                   tar czf nominatim-src.tar.bz2 Nominatim
  27                   mv nominatim-src.tar.bz2 Nominatim
  28
  29             - name: 'Upload Artifact'
  30               uses: actions/upload-artifact@v3
  31               with:
  32                   name: full-source
  33                   path: nominatim-src.tar.bz2
  34                   retention-days: 1
  35
  36     tests:
  37         needs: create-archive
  38         strategy:
  39             matrix:
  40                 flavour: [oldstuff, "ubuntu-20", "ubuntu-22"]
  41                 include:
  42                     - flavour: oldstuff
  43                       ubuntu: 20
  44                       postgresql: '9.6'
  45                       postgis: '2.5'
  46                       php: '7.3'
  47                       lua: '5.1'
  48                     - flavour: ubuntu-20
  49                       ubuntu: 20
  50                       postgresql: 13
  51                       postgis: 3
  52                       php: '7.4'
  53                       lua: '5.3'
  54                     - flavour: ubuntu-22
  55                       ubuntu: 22
  56                       postgresql: 15
  57                       postgis: 3
  58                       php: '8.1'
  59                       lua: '5.3'
  60
  61         runs-on: ubuntu-${{ matrix.ubuntu }}.04
  62
  63         steps:
  64             - uses: actions/download-artifact@v3
  65               with:
  66                   name: full-source
  67
  68             - name: Unpack Nominatim
  69               run: tar xf nominatim-src.tar.bz2
  70
  71             - name: Setup PHP
  72               uses: shivammathur/setup-php@v2
  73               with:
  74                   php-version: ${{ matrix.php }}
  75                   tools: phpunit, phpcs, composer
  76                   ini-values: opcache.jit=disable
  77
  78             - uses: actions/setup-python@v4
  79               with:
  80                 python-version: 3.7
  81               if: matrix.flavour == 'oldstuff'
  82
  83             - uses: ./Nominatim/.github/actions/setup-postgresql
  84               with:
  85                   postgresql-version: ${{ matrix.postgresql }}
  86                   postgis-version: ${{ matrix.postgis }}
  87
  88             - uses: ./Nominatim/.github/actions/build-nominatim
  89               with:
  90                   flavour: ${{ matrix.flavour }}
  91                   lua: ${{ matrix.lua }}
  92
  93             - name: Install test prerequsites (behave from apt)
  94               run: sudo apt-get install -y -qq python3-behave
  95               if: matrix.flavour == 'ubuntu-20'
  96
  97             - name: Install test prerequsites (behave from pip)
  98               run: pip3 install behave==1.2.6
  99               if: (matrix.flavour == 'oldstuff') || (matrix.flavour == 'ubuntu-22')
 100
 101             - name: Install test prerequsites (from apt for Ununtu 2x)
 102               run: sudo apt-get install -y -qq python3-pytest uvicorn
 103               if: matrix.flavour != 'oldstuff'
 104
 105             - name: Install test prerequsites (from pip for Ubuntu 18)
 106               run: pip3 install pytest uvicorn
 107               if: matrix.flavour == 'oldstuff'
 108
 109             - name: Install Python webservers
 110               run: pip3 install falcon sanic sanic-testing sanic-cors starlette
 111
 112             - name: Install latest pylint/mypy
 113               run: pip3 install -U pylint mypy types-PyYAML types-jinja2 types-psycopg2 types-psutil types-requests typing-extensions asgi_lifespan sqlalchemy2-stubs
 114
 115             - name: PHP linting
 116               run: phpcs --report-width=120 .
 117               working-directory: Nominatim
 118
 119             - name: Python linting
 120               run: python3 -m pylint nominatim
 121               working-directory: Nominatim
 122
 123             - name: PHP unit tests
 124               run: phpunit ./
 125               working-directory: Nominatim/test/php
 126               if: ${{ (matrix.ubuntu == 20) || (matrix.ubuntu == 22) }}
 127
 128             - name: Python unit tests
 129               run: python3 -m pytest test/python
 130               working-directory: Nominatim
 131
 132             - name: BDD tests
 133               run: |
 134                   python3 -m behave -DREMOVE_TEMPLATE=1 -DBUILDDIR=$GITHUB_WORKSPACE/build --format=progress3
 135               working-directory: Nominatim/test/bdd
 136
 137             - name: Install newer Python packages (for typechecking info)
 138               run: pip3 install -U osmium uvicorn
 139               if: matrix.flavour != 'oldstuff'
 140
 141             - name: Python static typechecking
 142               run: python3 -m mypy --strict nominatim
 143               working-directory: Nominatim
 144               if: matrix.flavour != 'oldstuff'
 145
 146     legacy-test:
 147         needs: create-archive
 148         runs-on: ubuntu-20.04
 149
 150         steps:
 151             - uses: actions/download-artifact@v3
 152               with:
 153                   name: full-source
 154
 155             - name: Unpack Nominatim
 156               run: tar xf nominatim-src.tar.bz2
 157
 158             - name: Setup PHP
 159               uses: shivammathur/setup-php@v2
 160               with:
 161                   php-version: '7.4'
 162
 163             - uses: ./Nominatim/.github/actions/setup-postgresql
 164               with:
 165                   postgresql-version: 13
 166                   postgis-version: 3
 167
 168             - name: Install Postgresql server dev
 169               run: sudo apt-get install postgresql-server-dev-13
 170
 171             - uses: ./Nominatim/.github/actions/build-nominatim
 172               with:
 173                   cmake-args: -DBUILD_MODULE=on
 174
 175             - name: Install test prerequsites
 176               run: sudo apt-get install -y -qq python3-behave
 177
 178             - name: BDD tests (legacy tokenizer)
 179               run: |
 180                   python3 -m behave -DREMOVE_TEMPLATE=1 -DBUILDDIR=$GITHUB_WORKSPACE/build -DTOKENIZER=legacy --format=progress3
 181               working-directory: Nominatim/test/bdd
 182
 183
 184     install:
 185         runs-on: ubuntu-latest
 186         needs: create-archive
 187
 188         strategy:
 189             matrix:
 190                 name: [Ubuntu-20, Ubuntu-22]
 191                 include:
 192                     - name: Ubuntu-20
 193                       image: "ubuntu:20.04"
 194                       ubuntu: 20
 195                       install_mode: install-apache
 196                     - name: Ubuntu-22
 197                       image: "ubuntu:22.04"
 198                       ubuntu: 22
 199                       install_mode: install-apache
 200
 201         container:
 202             image: ${{ matrix.image }}
 203             env:
 204                 LANG: en_US.UTF-8
 205
 206         defaults:
 207             run:
 208                 shell: sudo -Hu nominatim bash --noprofile --norc -eo pipefail {0}
 209
 210         steps:
 211             - name: Prepare container (Ubuntu)
 212               run: |
 213                   export APT_LISTCHANGES_FRONTEND=none
 214                   export DEBIAN_FRONTEND=noninteractive
 215                   apt-get update -qq
 216                   apt-get install -y git sudo wget
 217                   ln -snf /usr/share/zoneinfo/$CONTAINER_TIMEZONE /etc/localtime && echo $CONTAINER_TIMEZONE > /etc/timezone
 218               shell: bash
 219
 220             - name: Setup import user
 221               run: |
 222                   useradd -m nominatim
 223                   echo 'nominatim   ALL=(ALL:ALL) NOPASSWD: ALL' > /etc/sudoers.d/nominiatim
 224                   echo "/home/nominatim/Nominatim/vagrant/Install-on-${OS}.sh no $INSTALL_MODE" > /home/nominatim/vagrant.sh
 225               shell: bash
 226               env:
 227                 OS: ${{ matrix.name }}
 228                 INSTALL_MODE: ${{ matrix.install_mode }}
 229
 230             - uses: actions/download-artifact@v3
 231               with:
 232                   name: full-source
 233                   path: /home/nominatim
 234
 235             - name: Install Nominatim
 236               run: |
 237                 export USERNAME=nominatim
 238                 export USERHOME=/home/nominatim
 239                 export NOSYSTEMD=yes
 240                 export HAVE_SELINUX=no
 241                 tar xf nominatim-src.tar.bz2
 242                 . vagrant.sh
 243               working-directory: /home/nominatim
 244
 245             - name: Prepare import environment
 246               run: |
 247                   mv Nominatim/test/testdb/apidb-test-data.pbf test.pbf
 248                   rm -rf Nominatim
 249                   mkdir data-env-reverse
 250               working-directory: /home/nominatim
 251
 252             - name: Print version
 253               run: nominatim --version
 254               working-directory: /home/nominatim/nominatim-project
 255
 256             - name: Collect host OS information
 257               run: nominatim admin --collect-os-info
 258               working-directory: /home/nominatim/nominatim-project
 259
 260             - name: Import
 261               run: nominatim import --osm-file ../test.pbf
 262               working-directory: /home/nominatim/nominatim-project
 263
 264             - name: Import special phrases
 265               run: nominatim special-phrases --import-from-wiki
 266               working-directory: /home/nominatim/nominatim-project
 267
 268             - name: Check full import
 269               run: nominatim admin --check-database
 270               working-directory: /home/nominatim/nominatim-project
 271
 272             - name: Warm up database
 273               run: nominatim admin --warm
 274               working-directory: /home/nominatim/nominatim-project
 275
 276             - name: Prepare update (Ubuntu)
 277               run: apt-get install -y python3-pip
 278               shell: bash
 279
 280             - name: Run update
 281               run: |
 282                   pip3 install --user osmium
 283                   nominatim replication --init
 284                   NOMINATIM_REPLICATION_MAX_DIFF=1 nominatim replication --once
 285               working-directory: /home/nominatim/nominatim-project
 286
 287             - name: Clean up database
 288               run: nominatim refresh --postcodes --word-tokens
 289               working-directory: /home/nominatim/nominatim-project
 290
 291             - name: Run reverse-only import
 292               run : |
 293                   echo 'NOMINATIM_DATABASE_DSN="pgsql:dbname=reverse"' >> .env
 294                   nominatim import --osm-file ../test.pbf --reverse-only --no-updates
 295               working-directory: /home/nominatim/data-env-reverse
 296
 297             - name: Check reverse-only import
 298               run: nominatim admin --check-database
 299               working-directory: /home/nominatim/data-env-reverse
 300
 301             - name: Clean up database (reverse-only import)
 302               run: nominatim refresh --postcodes --word-tokens
 303               working-directory: /home/nominatim/nominatim-project