]> git.openstreetmap.org Git - nominatim.git/blob - .github/workflows/ci-tests.yml
Merge remote-tracking branch 'upstream/master'
[nominatim.git] / .github / workflows / ci-tests.yml
1 name: CI Tests
2
3 on: [ push, pull_request ]
4
5 jobs:
6     create-archive:
7         runs-on: ubuntu-latest
8
9         steps:
10             - uses: actions/checkout@v4
11               with:
12                 submodules: true
13
14             - uses: actions/cache@v4
15               with:
16                   path: |
17                      data/country_osm_grid.sql.gz
18                   key: nominatim-country-data-1
19
20             - name: Package tarball
21               run: |
22                   if [ ! -f data/country_osm_grid.sql.gz ]; then
23                       wget --no-verbose -O data/country_osm_grid.sql.gz https://www.nominatim.org/data/country_grid.sql.gz
24                   fi
25                   cd ..
26                   tar czf nominatim-src.tar.bz2 Nominatim
27                   mv nominatim-src.tar.bz2 Nominatim
28
29             - name: 'Upload Artifact'
30               uses: actions/upload-artifact@v4
31               with:
32                   name: full-source
33                   path: nominatim-src.tar.bz2
34                   retention-days: 1
35
36     tests:
37         needs: create-archive
38         strategy:
39             matrix:
40                 flavour: [oldstuff, "ubuntu-20", "ubuntu-22"]
41                 include:
42                     - flavour: oldstuff
43                       ubuntu: 20
44                       postgresql: '9.6'
45                       postgis: '2.5'
46                       lua: '5.1'
47                     - flavour: ubuntu-20
48                       ubuntu: 20
49                       postgresql: 13
50                       postgis: 3
51                       lua: '5.3'
52                     - flavour: ubuntu-22
53                       ubuntu: 22
54                       postgresql: 15
55                       postgis: 3
56                       lua: '5.3'
57
58         runs-on: ubuntu-${{ matrix.ubuntu }}.04
59
60         steps:
61             - uses: actions/download-artifact@v4
62               with:
63                   name: full-source
64
65             - name: Unpack Nominatim
66               run: tar xf nominatim-src.tar.bz2
67
68             - uses: actions/setup-python@v5
69               with:
70                 python-version: 3.7
71               if: matrix.flavour == 'oldstuff'
72
73             - uses: ./Nominatim/.github/actions/setup-postgresql
74               with:
75                   postgresql-version: ${{ matrix.postgresql }}
76                   postgis-version: ${{ matrix.postgis }}
77
78             - uses: ./Nominatim/.github/actions/build-nominatim
79               with:
80                   flavour: ${{ matrix.flavour }}
81                   lua: ${{ matrix.lua }}
82
83             - name: Install test prerequisites (behave from apt)
84               run: sudo apt-get install -y -qq python3-behave
85               if: matrix.flavour == 'ubuntu-20'
86
87             - name: Install test prerequisites (behave from pip)
88               run: pip3 install behave==1.2.6
89               if: (matrix.flavour == 'oldstuff') || (matrix.flavour == 'ubuntu-22')
90
91             - name: Install test prerequisites (from apt for Ununtu 2x)
92               run: sudo apt-get install -y -qq python3-pytest python3-pytest-asyncio uvicorn
93               if: matrix.flavour != 'oldstuff'
94
95             - name: Install newer pytest-asyncio
96               run: pip3 install -U pytest-asyncio
97               if: matrix.flavour == 'ubuntu-20'
98
99             - name: Install test prerequisites (from pip for Ubuntu 18)
100               run: pip3 install pytest pytest-asyncio uvicorn
101               if: matrix.flavour == 'oldstuff'
102
103             - name: Install Python webservers
104               run: pip3 install falcon starlette asgi_lifespan
105
106             - name: Install latest pylint
107               run: pip3 install -U pylint
108               if: matrix.flavour == 'ubuntu-22'
109
110             - name: Python linting
111               run: python3 -m pylint src
112               working-directory: Nominatim
113               if: matrix.flavour == 'ubuntu-22'
114
115             - name: Python unit tests
116               run: python3 -m pytest test/python
117               working-directory: Nominatim
118
119             - name: BDD tests
120               run: |
121                   export PATH=$GITHUB_WORKSPACE/build/osm2pgsql:$PATH
122                   python3 -m behave -DREMOVE_TEMPLATE=1 --format=progress3
123               working-directory: Nominatim/test/bdd
124
125             - name: Install mypy and typechecking info
126               run: pip3 install -U mypy osmium uvicorn types-PyYAML types-jinja2 types-psycopg2 types-psutil types-requests types-ujson types-Pygments typing-extensions
127               if: matrix.flavour != 'oldstuff'
128
129             - name: Python static typechecking
130               run: python3 -m mypy --strict src
131               working-directory: Nominatim
132               if: matrix.flavour != 'oldstuff'
133
134     install:
135         runs-on: ubuntu-latest
136         needs: create-archive
137
138         strategy:
139             matrix:
140                 name: [Ubuntu-22, Ubuntu-24]
141                 include:
142                     - name: Ubuntu-22
143                       image: "ubuntu:22.04"
144                       ubuntu: 22
145                       install_mode: install-apache
146                     - name: Ubuntu-24
147                       image: "ubuntu:24.04"
148                       ubuntu: 24
149                       install_mode: install-apache
150
151         container:
152             image: ${{ matrix.image }}
153             env:
154                 LANG: en_US.UTF-8
155
156         defaults:
157             run:
158                 shell: sudo -Hu nominatim bash --noprofile --norc -eo pipefail {0}
159
160         steps:
161             - name: Prepare container (Ubuntu)
162               run: |
163                   export APT_LISTCHANGES_FRONTEND=none
164                   export DEBIAN_FRONTEND=noninteractive
165                   apt-get update -qq
166                   apt-get install -y git sudo wget
167                   ln -snf /usr/share/zoneinfo/$CONTAINER_TIMEZONE /etc/localtime && echo $CONTAINER_TIMEZONE > /etc/timezone
168               shell: bash
169
170             - name: Setup import user
171               run: |
172                   useradd -m nominatim
173                   echo 'nominatim   ALL=(ALL:ALL) NOPASSWD: ALL' > /etc/sudoers.d/nominiatim
174                   echo "/home/nominatim/Nominatim/vagrant/Install-on-${OS}.sh no $INSTALL_MODE" > /home/nominatim/vagrant.sh
175               shell: bash
176               env:
177                 OS: ${{ matrix.name }}
178                 INSTALL_MODE: ${{ matrix.install_mode }}
179
180             - uses: actions/download-artifact@v4
181               with:
182                   name: full-source
183                   path: /home/nominatim
184
185             - name: Install Nominatim
186               run: |
187                 export USERNAME=nominatim
188                 export USERHOME=/home/nominatim
189                 export NOSYSTEMD=yes
190                 export HAVE_SELINUX=no
191                 tar xf nominatim-src.tar.bz2
192                 . vagrant.sh
193               working-directory: /home/nominatim
194
195             - name: Prepare import environment
196               run: |
197                   mv Nominatim/test/testdb/apidb-test-data.pbf test.pbf
198                   mv Nominatim/settings/flex-base.lua flex-base.lua
199                   mv Nominatim/settings/import-extratags.lua import-extratags.lua
200                   mv Nominatim/settings/taginfo.lua taginfo.lua
201                   rm -rf Nominatim
202                   mkdir data-env-reverse
203               working-directory: /home/nominatim
204
205             - name: Add nominatim to path
206               run: |
207                 sudo ln -s /home/nominatim/nominatim-venv/bin/nominatim /usr/local/bin/nominatim
208               if: matrix.ubuntu == 24
209
210             - name: Need lua binary
211               run: |
212                 sudo apt-get install -y lua5.4 lua-dkjson
213               if: matrix.ubuntu == 24
214
215             - name: Print version
216               run: nominatim --version
217               working-directory: /home/nominatim/nominatim-project
218
219             - name: Print taginfo
220               run: lua taginfo.lua
221               working-directory: /home/nominatim
222
223             - name: Collect host OS information
224               run: nominatim admin --collect-os-info
225               working-directory: /home/nominatim/nominatim-project
226
227             - name: Import
228               run: nominatim import --osm-file ../test.pbf
229               working-directory: /home/nominatim/nominatim-project
230
231             - name: Import special phrases
232               run: nominatim special-phrases --import-from-wiki
233               working-directory: /home/nominatim/nominatim-project
234
235             - name: Check full import
236               run: nominatim admin --check-database
237               working-directory: /home/nominatim/nominatim-project
238
239             - name: Warm up database
240               run: nominatim admin --warm
241               working-directory: /home/nominatim/nominatim-project
242
243             - name: Prepare update (Ubuntu)
244               run: apt-get install -y python3-pip
245               shell: bash
246
247             - name: Install osmium (Ubuntu 22)
248               run: |
249                   pip3 install --user osmium
250               if: matrix.ubuntu == 22
251
252             - name: Install osmium (Ubuntu 24)
253               run: |
254                   /home/nominatim/nominatim-venv/bin/pip install osmium
255               if: matrix.ubuntu == 24
256
257             - name: Run update
258               run: |
259                   nominatim replication --init
260                   NOMINATIM_REPLICATION_MAX_DIFF=1 nominatim replication --once
261               working-directory: /home/nominatim/nominatim-project
262
263             - name: Clean up database
264               run: nominatim refresh --postcodes --word-tokens
265               working-directory: /home/nominatim/nominatim-project
266
267             - name: Run reverse-only import
268               run : |
269                   echo 'NOMINATIM_DATABASE_DSN="pgsql:dbname=reverse"' >> .env
270                   nominatim import --osm-file ../test.pbf --reverse-only --no-updates
271               working-directory: /home/nominatim/data-env-reverse
272
273             - name: Check reverse-only import
274               run: nominatim admin --check-database
275               working-directory: /home/nominatim/data-env-reverse
276
277             - name: Clean up database (reverse-only import)
278               run: nominatim refresh --postcodes --word-tokens
279               working-directory: /home/nominatim/nominatim-project
280
281     install-no-superuser:
282       runs-on: ubuntu-latest
283       needs: create-archive
284
285       steps:
286           - uses: actions/download-artifact@v4
287             with:
288                 name: full-source
289
290           - name: Unpack Nominatim
291             run: tar xf nominatim-src.tar.bz2
292
293           - uses: ./Nominatim/.github/actions/setup-postgresql
294             with:
295                 postgresql-version: 16
296                 postgis-version: 3
297
298           - uses: ./Nominatim/.github/actions/build-nominatim
299             with:
300                 flavour: ubuntu-22
301                 lua: 5.3
302
303           - name: Prepare import environment
304             run: |
305                 mv Nominatim/test/testdb/apidb-test-data.pbf test.pbf
306                 rm -rf Nominatim
307
308           - name: Prepare Database
309             run: |
310                 nominatim import --prepare-database
311
312           - name: Create import user
313             run: |
314                 sudo -u postgres createuser osm-import
315                 psql -d nominatim -c "ALTER USER \"osm-import\" WITH PASSWORD 'osm-import'"
316                 psql -d nominatim -c 'GRANT CREATE ON SCHEMA public TO "osm-import"'
317
318           - name: Run import
319             run: |
320                 NOMINATIM_DATABASE_DSN="pgsql:host=127.0.0.1;dbname=nominatim;user=osm-import;password=osm-import" nominatim import --continue import-from-file --osm-file test.pbf
321
322           - name: Check full import
323             run: nominatim admin --check-database
324
325     migrate:
326       runs-on: ubuntu-24.04
327       needs: create-archive
328
329       steps:
330           - uses: actions/download-artifact@v4
331             with:
332                 name: full-source
333
334           - name: Unpack Nominatim
335             run: tar xf nominatim-src.tar.bz2
336
337           - uses: ./Nominatim/.github/actions/setup-postgresql
338             with:
339                 postgresql-version: 17
340                 postgis-version: 3
341
342           - name: Install Python dependencies
343             run: |
344               sudo apt-get install --no-install-recommends virtualenv osm2pgsql
345
346           - name: Install Nominatim master version
347             run: |
348               virtualenv master
349               cd Nominatim
350               ../master/bin/pip install packaging/nominatim-db
351
352           - name: Install Nominatim from pypi
353             run: |
354               virtualenv release
355               ./release/bin/pip install nominatim-db
356
357           - name: Import Nominatim database using release
358             run: |
359               ./release/bin/nominatim import --osm-file Nominatim/test/testdb/apidb-test-data.pbf
360               ./release/bin/nominatim add-data --file Nominatim/test/testdb/additional_api_test.data.osm
361
362           - name: Migrate to master version
363             run: |
364               ./master/bin/nominatim admin --migrate
365               ./release/bin/nominatim add-data --file Nominatim/test/testdb/additional_api_test.data.osm
366
367     codespell:
368       runs-on: ubuntu-latest
369       steps:
370           - uses: codespell-project/actions-codespell@v2
371             with:
372                 only_warn: 1