.github/workflows/ci-tests.yml

   1 name: CI Tests
   2
   3 on: [ push, pull_request ]
   4
   5 jobs:
   6     create-archive:
   7         runs-on: ubuntu-latest
   8
   9         steps:
  10             - uses: actions/checkout@v4
  11               with:
  12                 submodules: true
  13
  14             - uses: actions/cache@v3
  15               with:
  16                   path: |
  17                      data/country_osm_grid.sql.gz
  18                   key: nominatim-country-data-1
  19
  20             - name: Package tarball
  21               run: |
  22                   if [ ! -f data/country_osm_grid.sql.gz ]; then
  23                       wget --no-verbose -O data/country_osm_grid.sql.gz https://www.nominatim.org/data/country_grid.sql.gz
  24                   fi
  25                   cd ..
  26                   tar czf nominatim-src.tar.bz2 Nominatim
  27                   mv nominatim-src.tar.bz2 Nominatim
  28
  29             - name: 'Upload Artifact'
  30               uses: actions/upload-artifact@v3
  31               with:
  32                   name: full-source
  33                   path: nominatim-src.tar.bz2
  34                   retention-days: 1
  35
  36     tests:
  37         needs: create-archive
  38         strategy:
  39             matrix:
  40                 flavour: [oldstuff, "ubuntu-20", "ubuntu-22"]
  41                 include:
  42                     - flavour: oldstuff
  43                       ubuntu: 20
  44                       postgresql: '9.6'
  45                       postgis: '2.5'
  46                       php: '7.3'
  47                       lua: '5.1'
  48                     - flavour: ubuntu-20
  49                       ubuntu: 20
  50                       postgresql: 13
  51                       postgis: 3
  52                       php: '7.4'
  53                       lua: '5.3'
  54                     - flavour: ubuntu-22
  55                       ubuntu: 22
  56                       postgresql: 15
  57                       postgis: 3
  58                       php: '8.1'
  59                       lua: '5.3'
  60
  61         runs-on: ubuntu-${{ matrix.ubuntu }}.04
  62
  63         steps:
  64             - uses: actions/download-artifact@v3
  65               with:
  66                   name: full-source
  67
  68             - name: Unpack Nominatim
  69               run: tar xf nominatim-src.tar.bz2
  70
  71             - name: Setup PHP
  72               uses: shivammathur/setup-php@v2
  73               with:
  74                   php-version: ${{ matrix.php }}
  75                   tools: phpunit:9, phpcs, composer
  76                   ini-values: opcache.jit=disable
  77               env:
  78                   GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
  79
  80             - uses: actions/setup-python@v4
  81               with:
  82                 python-version: 3.7
  83               if: matrix.flavour == 'oldstuff'
  84
  85             - uses: ./Nominatim/.github/actions/setup-postgresql
  86               with:
  87                   postgresql-version: ${{ matrix.postgresql }}
  88                   postgis-version: ${{ matrix.postgis }}
  89
  90             - uses: ./Nominatim/.github/actions/build-nominatim
  91               with:
  92                   flavour: ${{ matrix.flavour }}
  93                   lua: ${{ matrix.lua }}
  94
  95             - name: Install test prerequsites (behave from apt)
  96               run: sudo apt-get install -y -qq python3-behave
  97               if: matrix.flavour == 'ubuntu-20'
  98
  99             - name: Install test prerequsites (behave from pip)
 100               run: pip3 install behave==1.2.6
 101               if: (matrix.flavour == 'oldstuff') || (matrix.flavour == 'ubuntu-22')
 102
 103             - name: Install test prerequsites (from apt for Ununtu 2x)
 104               run: sudo apt-get install -y -qq python3-pytest python3-pytest-asyncio uvicorn
 105               if: matrix.flavour != 'oldstuff'
 106
 107             - name: Install newer pytest-asyncio
 108               run: pip3 install -U pytest-asyncio==0.21.1
 109               if: matrix.flavour == 'ubuntu-20'
 110
 111             - name: Install test prerequsites (from pip for Ubuntu 18)
 112               run: pip3 install pytest pytest-asyncio uvicorn
 113               if: matrix.flavour == 'oldstuff'
 114
 115             - name: Install Python webservers
 116               run: pip3 install falcon starlette asgi_lifespan
 117
 118             - name: Install latest pylint
 119               run: pip3 install -U pylint
 120               if: matrix.flavour != 'oldstuff'
 121
 122             - name: PHP linting
 123               run: phpcs --report-width=120 .
 124               working-directory: Nominatim
 125               if: matrix.flavour != 'oldstuff'
 126
 127             - name: Python linting
 128               run: python3 -m pylint nominatim
 129               working-directory: Nominatim
 130               if: matrix.flavour != 'oldstuff'
 131
 132             - name: PHP unit tests
 133               run: phpunit ./
 134               working-directory: Nominatim/test/php
 135
 136             - name: Python unit tests
 137               run: python3 -m pytest test/python
 138               working-directory: Nominatim
 139
 140             - name: BDD tests
 141               run: |
 142                   python3 -m behave -DREMOVE_TEMPLATE=1 -DBUILDDIR=$GITHUB_WORKSPACE/build --format=progress3
 143               working-directory: Nominatim/test/bdd
 144
 145             - name: Install mypy and typechecking info
 146               run: pip3 install -U mypy osmium uvicorn types-PyYAML types-jinja2 types-psycopg2 types-psutil types-requests types-ujson types-Pygments typing-extensions
 147               if: matrix.flavour != 'oldstuff'
 148
 149             - name: Python static typechecking
 150               run: python3 -m mypy --strict nominatim
 151               working-directory: Nominatim
 152               if: matrix.flavour != 'oldstuff'
 153
 154     legacy-test:
 155         needs: create-archive
 156         runs-on: ubuntu-20.04
 157
 158         steps:
 159             - uses: actions/download-artifact@v3
 160               with:
 161                   name: full-source
 162
 163             - name: Unpack Nominatim
 164               run: tar xf nominatim-src.tar.bz2
 165
 166             - name: Setup PHP
 167               uses: shivammathur/setup-php@v2
 168               with:
 169                   php-version: '7.4'
 170
 171             - uses: ./Nominatim/.github/actions/setup-postgresql
 172               with:
 173                   postgresql-version: 13
 174                   postgis-version: 3
 175
 176             - name: Install Postgresql server dev
 177               run: sudo apt-get install postgresql-server-dev-13
 178
 179             - uses: ./Nominatim/.github/actions/build-nominatim
 180               with:
 181                   cmake-args: -DBUILD_MODULE=on
 182
 183             - name: Install test prerequsites
 184               run: sudo apt-get install -y -qq python3-behave
 185
 186             - name: BDD tests (legacy tokenizer)
 187               run: |
 188                   python3 -m behave -DREMOVE_TEMPLATE=1 -DBUILDDIR=$GITHUB_WORKSPACE/build -DTOKENIZER=legacy --format=progress3
 189               working-directory: Nominatim/test/bdd
 190
 191
 192     python-api-test:
 193         needs: create-archive
 194         runs-on: ubuntu-22.04
 195
 196         steps:
 197             - uses: actions/download-artifact@v3
 198               with:
 199                   name: full-source
 200
 201             - name: Unpack Nominatim
 202               run: tar xf nominatim-src.tar.bz2
 203
 204             - uses: ./Nominatim/.github/actions/setup-postgresql
 205               with:
 206                   postgresql-version: 15
 207                   postgis-version: 3
 208
 209             - uses: ./Nominatim/.github/actions/build-nominatim
 210               with:
 211                   flavour: 'ubuntu-22'
 212
 213             - name: Install test prerequsites
 214               run: sudo apt-get install -y -qq python3-behave
 215
 216             - name: Install Python webservers
 217               run: pip3 install starlette asgi_lifespan httpx
 218
 219             - name: BDD tests (starlette)
 220               run: |
 221                   python3 -m behave -DREMOVE_TEMPLATE=1 -DBUILDDIR=$GITHUB_WORKSPACE/build -DAPI_ENGINE=starlette --format=progress3
 222               working-directory: Nominatim/test/bdd
 223
 224
 225     install:
 226         runs-on: ubuntu-latest
 227         needs: create-archive
 228
 229         strategy:
 230             matrix:
 231                 name: [Ubuntu-20, Ubuntu-22]
 232                 include:
 233                     - name: Ubuntu-20
 234                       image: "ubuntu:20.04"
 235                       ubuntu: 20
 236                       install_mode: install-apache
 237                     - name: Ubuntu-22
 238                       image: "ubuntu:22.04"
 239                       ubuntu: 22
 240                       install_mode: install-apache
 241
 242         container:
 243             image: ${{ matrix.image }}
 244             env:
 245                 LANG: en_US.UTF-8
 246
 247         defaults:
 248             run:
 249                 shell: sudo -Hu nominatim bash --noprofile --norc -eo pipefail {0}
 250
 251         steps:
 252             - name: Prepare container (Ubuntu)
 253               run: |
 254                   export APT_LISTCHANGES_FRONTEND=none
 255                   export DEBIAN_FRONTEND=noninteractive
 256                   apt-get update -qq
 257                   apt-get install -y git sudo wget
 258                   ln -snf /usr/share/zoneinfo/$CONTAINER_TIMEZONE /etc/localtime && echo $CONTAINER_TIMEZONE > /etc/timezone
 259               shell: bash
 260
 261             - name: Setup import user
 262               run: |
 263                   useradd -m nominatim
 264                   echo 'nominatim   ALL=(ALL:ALL) NOPASSWD: ALL' > /etc/sudoers.d/nominiatim
 265                   echo "/home/nominatim/Nominatim/vagrant/Install-on-${OS}.sh no $INSTALL_MODE" > /home/nominatim/vagrant.sh
 266               shell: bash
 267               env:
 268                 OS: ${{ matrix.name }}
 269                 INSTALL_MODE: ${{ matrix.install_mode }}
 270
 271             - uses: actions/download-artifact@v3
 272               with:
 273                   name: full-source
 274                   path: /home/nominatim
 275
 276             - name: Install Nominatim
 277               run: |
 278                 export USERNAME=nominatim
 279                 export USERHOME=/home/nominatim
 280                 export NOSYSTEMD=yes
 281                 export HAVE_SELINUX=no
 282                 tar xf nominatim-src.tar.bz2
 283                 . vagrant.sh
 284               working-directory: /home/nominatim
 285
 286             - name: Prepare import environment
 287               run: |
 288                   mv Nominatim/test/testdb/apidb-test-data.pbf test.pbf
 289                   mv Nominatim/settings/flex-base.lua flex-base.lua
 290                   mv Nominatim/settings/import-extratags.lua import-extratags.lua
 291                   mv Nominatim/settings/taginfo.lua taginfo.lua
 292                   rm -rf Nominatim
 293                   mkdir data-env-reverse
 294               working-directory: /home/nominatim
 295
 296             - name: Print version
 297               run: nominatim --version
 298               working-directory: /home/nominatim/nominatim-project
 299
 300             - name: Print taginfo
 301               run: lua taginfo.lua
 302               working-directory: /home/nominatim
 303
 304             - name: Collect host OS information
 305               run: nominatim admin --collect-os-info
 306               working-directory: /home/nominatim/nominatim-project
 307
 308             - name: Import
 309               run: nominatim import --osm-file ../test.pbf
 310               working-directory: /home/nominatim/nominatim-project
 311
 312             - name: Import special phrases
 313               run: nominatim special-phrases --import-from-wiki
 314               working-directory: /home/nominatim/nominatim-project
 315
 316             - name: Check full import
 317               run: nominatim admin --check-database
 318               working-directory: /home/nominatim/nominatim-project
 319
 320             - name: Warm up database
 321               run: nominatim admin --warm
 322               working-directory: /home/nominatim/nominatim-project
 323
 324             - name: Prepare update (Ubuntu)
 325               run: apt-get install -y python3-pip
 326               shell: bash
 327
 328             - name: Run update
 329               run: |
 330                   pip3 install --user osmium
 331                   nominatim replication --init
 332                   NOMINATIM_REPLICATION_MAX_DIFF=1 nominatim replication --once
 333               working-directory: /home/nominatim/nominatim-project
 334
 335             - name: Clean up database
 336               run: nominatim refresh --postcodes --word-tokens
 337               working-directory: /home/nominatim/nominatim-project
 338
 339             - name: Run reverse-only import
 340               run : |
 341                   echo 'NOMINATIM_DATABASE_DSN="pgsql:dbname=reverse"' >> .env
 342                   nominatim import --osm-file ../test.pbf --reverse-only --no-updates
 343               working-directory: /home/nominatim/data-env-reverse
 344
 345             - name: Check reverse-only import
 346               run: nominatim admin --check-database
 347               working-directory: /home/nominatim/data-env-reverse
 348
 349             - name: Clean up database (reverse-only import)
 350               run: nominatim refresh --postcodes --word-tokens
 351               working-directory: /home/nominatim/nominatim-project
 352
 353     install-no-superuser:
 354       runs-on: ubuntu-latest
 355       needs: create-archive
 356
 357       strategy:
 358           matrix:
 359               name: [Ubuntu-22]
 360               include:
 361                   - name: Ubuntu-22
 362                     image: "ubuntu:22.04"
 363                     ubuntu: 22
 364                     install_mode: install-apache
 365
 366       container:
 367           image: ${{ matrix.image }}
 368           env:
 369               LANG: en_US.UTF-8
 370
 371       defaults:
 372           run:
 373               shell: sudo -Hu nominatim bash --noprofile --norc -eo pipefail {0}
 374
 375       steps:
 376           - name: Prepare container (Ubuntu)
 377             run: |
 378                 export APT_LISTCHANGES_FRONTEND=none
 379                 export DEBIAN_FRONTEND=noninteractive
 380                 apt-get update -qq
 381                 apt-get install -y git sudo wget
 382                 ln -snf /usr/share/zoneinfo/$CONTAINER_TIMEZONE /etc/localtime && echo $CONTAINER_TIMEZONE > /etc/timezone
 383             shell: bash
 384
 385           - name: Setup import user
 386             run: |
 387                 useradd -m nominatim
 388                 echo 'nominatim   ALL=(ALL:ALL) NOPASSWD: ALL' > /etc/sudoers.d/nominiatim
 389                 echo "/home/nominatim/Nominatim/vagrant/Install-on-${OS}.sh no $INSTALL_MODE" > /home/nominatim/vagrant.sh
 390             shell: bash
 391             env:
 392               OS: ${{ matrix.name }}
 393               INSTALL_MODE: ${{ matrix.install_mode }}
 394
 395           - uses: actions/download-artifact@v3
 396             with:
 397                 name: full-source
 398                 path: /home/nominatim
 399
 400           - name: Install Nominatim
 401             run: |
 402               export USERNAME=nominatim
 403               export USERHOME=/home/nominatim
 404               export NOSYSTEMD=yes
 405               export HAVE_SELINUX=no
 406               tar xf nominatim-src.tar.bz2
 407               . vagrant.sh
 408             working-directory: /home/nominatim
 409
 410           - name: Prepare import environment
 411             run: |
 412                 mv Nominatim/test/testdb/apidb-test-data.pbf test.pbf
 413                 mv Nominatim/settings/flex-base.lua flex-base.lua
 414                 mv Nominatim/settings/import-extratags.lua import-extratags.lua
 415                 mv Nominatim/settings/taginfo.lua taginfo.lua
 416                 rm -rf Nominatim
 417                 mkdir data-env-reverse
 418             working-directory: /home/nominatim
 419
 420           - name: Prepare Database
 421             run: |
 422                 nominatim import --prepare-database
 423             working-directory: /home/nominatim/nominatim-project
 424
 425           - name: Create import user
 426             run: |
 427                 sudo -u postgres createuser -S osm-import
 428                 sudo -u postgres psql -c "ALTER USER \"osm-import\" WITH PASSWORD 'osm-import';"
 429             working-directory: /home/nominatim/nominatim-project
 430
 431           - name: Grant import user rights
 432             run: |
 433                 sudo -u postgres psql -c "GRANT INSERT, UPDATE ON ALL TABLES IN SCHEMA public TO \"osm-import\";"
 434             working-directory: /home/nominatim/nominatim-project
 435
 436           - name: Run import
 437             run: |
 438                 NOMINATIM_DATABASE_DSN="pgsql:host=127.0.0.1;dbname=nominatim;user=osm-import;password=osm-import" nominatim import --continue import-from-file --osm-file ../test.pbf
 439             working-directory: /home/nominatim/nominatim-project
 440
 441           - name: Check full import
 442             run: nominatim admin --check-database
 443             working-directory: /home/nominatim/nominatim-project