From 4e779695455cdb44bdc5e1bb2d093d22ef461d13 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Sun, 22 Aug 2021 18:42:20 +0200 Subject: [PATCH] add workflow for centos 8 --- .github/workflows/vagrant-scripts.yml | 126 +++++++++++++++++++++++--- vagrant/Install-on-Centos-8.sh | 59 ++++++++---- vagrant/Install-on-Ubuntu-18.sh | 2 +- vagrant/Install-on-Ubuntu-20.sh | 4 +- 4 files changed, 155 insertions(+), 36 deletions(-) diff --git a/.github/workflows/vagrant-scripts.yml b/.github/workflows/vagrant-scripts.yml index 5b2df2f4..4a8c1cc5 100644 --- a/.github/workflows/vagrant-scripts.yml +++ b/.github/workflows/vagrant-scripts.yml @@ -1,4 +1,4 @@ -name: Install scripts +name: Install and Import on: [ push, pull_request ] @@ -11,25 +11,19 @@ jobs: with: submodules: true - - name: Get Date - id: get-date - run: | - echo "::set-output name=date::$(/bin/date -u "+%Y%W")" - shell: bash - - uses: actions/cache@v2 with: path: | - data/country_grid.sql.gz - key: nominatim-country-data-${{ steps.get-date.outputs.date }} + data/country_osm_grid.sql.gz + key: nominatim-country-data-1 - name: Package tarball run: | - if [ ! -f data/country_grid.sql.gz ]; then - wget --no-verbose -O data/country_grid.sql.gz https://www.nominatim.org/data/country_grid.sql.gz + if [ ! -f data/country_osm_grid.sql.gz ]; then + wget --no-verbose -O data/country_osm_grid.sql.gz https://www.nominatim.org/data/country_grid.sql.gz fi cd .. - tar cjf nominatim-src.tar.bz2 Nominatim + tar czf nominatim-src.tar.bz2 Nominatim mv nominatim-src.tar.bz2 Nominatim - name: 'Upload Artifact' @@ -39,8 +33,8 @@ jobs: path: nominatim-src.tar.bz2 retention-days: 1 - ubuntu: + if: false runs-on: ubuntu-latest needs: create-archive @@ -100,6 +94,7 @@ jobs: . vagrant.sh working-directory: /home/nominatim + - uses: actions/cache@v2 with: path: | @@ -150,3 +145,108 @@ jobs: - name: Check reverse import run: nominatim admin --check-database working-directory: data-env-reverse + + + centos: + runs-on: ubuntu-latest + needs: create-archive + + strategy: + matrix: + centos: [8] + + container: + image: centos:${{ matrix.centos }} + env: + LANG: en_US.UTF-8 + + defaults: + run: + shell: sudo -Hu nominatim bash --noprofile --norc -eo pipefail {0} + + steps: + - name: Prepare container + run: | + dnf update -y + dnf install -y sudo + shell: bash + + - name: Setup import user + run: | + useradd -m nominatim + echo 'nominatim ALL=(ALL:ALL) NOPASSWD: ALL' > /etc/sudoers.d/nominiatim + echo "/home/nominatim/Nominatim/vagrant/Install-on-Centos-$CENTOS.sh no" > /home/nominatim/vagrant.sh + shell: bash + env: + CENTOS: ${{ matrix.centos }} + + - uses: actions/download-artifact@v2 + with: + name: full-source + path: /home/nominatim + + - name: Get Date + id: get-date + run: | + echo "::set-output name=date::$(/bin/date -u "+%Y%W")" + shell: bash + + - name: Install Nominatim + run: | + export USERNAME=nominatim + export USERHOME=/home/nominatim + export NOSYSTEMD=yes + export HAVE_SELINUX=no + tar xf nominatim-src.tar.bz2 + ls -al /home/nominatim/Nominatim/data + . vagrant.sh + working-directory: /home/nominatim + + - name: Prepare import environment + run: | + mv /home/nominatim/Nominatim/test/testdb/apidb-test-data.pbf test.pbf + rm -rf /home/nominatim/Nominatim + mkdir data-env data-env-reverse + chown nominatim. data-env data-env-reverse + shell: bash + + - name: Prepare import environment (CentOS) + run: | + sudo ln -s /usr/local/bin/nominatim /usr/bin/nominatim + echo NOMINATIM_DATABASE_WEBUSER="apache" > data-env/.env + cp data-env/.env data-env-reverse/.env + + - name: Import + run: nominatim import --osm-file ../test.pbf + working-directory: data-env + + - name: Import special phrases + run: nominatim special-phrases --import-from-wiki + working-directory: data-env + + - name: Check full import + run: nominatim admin --check-database + working-directory: data-env + + - name: Warm up database + run: nominatim admin --warm + working-directory: data-env + + - name: Run update + run: | + pip3 install --user osmium + nominatim replication --init + NOMINATIM_REPLICATION_MAX_DIFF=1 nominatim replication --once + working-directory: data-env + + - name: Run reverse-only import + run : | + echo 'NOMINATIM_DATABASE_DSN="pgsql:dbname=reverse"' >> .env + nominatim import --osm-file ../test.pbf --reverse-only --no-updates + working-directory: data-env-reverse + env: + NOMINATIM_DATABASE_DSN: pgsql:dbname=reverse + + - name: Check reverse import + run: nominatim admin --check-database + working-directory: data-env-reverse diff --git a/vagrant/Install-on-Centos-8.sh b/vagrant/Install-on-Centos-8.sh index a41e846c..18991364 100755 --- a/vagrant/Install-on-Centos-8.sh +++ b/vagrant/Install-on-Centos-8.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/bash -ex # # *Note:* these installation instructions are also available in executable # form for use with vagrant under `vagrant/Install-on-Centos-8.sh`. @@ -18,10 +18,12 @@ sudo dnf install -y epel-release redhat-rpm-config # EPEL contains Postgres 9.6 and 10, but not PostGIS. Postgres 9.4+/10/11/12 -# and PostGIS 2.4/2.5/3.0 are availble from postgresql.org +# and PostGIS 2.4/2.5/3.0 are availble from postgresql.org. Enable these +# repositories and make sure, the binaries can be found: sudo dnf -qy module disable postgresql sudo dnf install -y https://download.postgresql.org/pub/repos/yum/reporpms/EL-8-x86_64/pgdg-redhat-repo-latest.noarch.rpm + export PATH=/usr/pgsql-12/bin:$PATH # Now you can install all packages needed for Nominatim: @@ -33,9 +35,10 @@ php-pgsql php php-intl php-json libpq-devel \ bzip2-devel proj-devel boost-devel \ python3-pip python3-setuptools python3-devel \ - expat-devel zlib-devel libicu-dev + python3-psycopg2 \ + expat-devel zlib-devel libicu-devel - pip3 install --user psycopg2 python-dotenv psutil Jinja2 PyICU datrie pyyaml + pip3 install --user python-dotenv psutil Jinja2 PyICU datrie pyyaml # @@ -54,16 +57,19 @@ # we assume this user is called nominatim and the installation will be in # /srv/nominatim. To create the user and directory run: # -sudo mkdir -p /srv/nominatim #DOCS: sudo useradd -d /srv/nominatim -s /bin/bash -m nominatim -sudo chown vagrant /srv/nominatim #DOCS: +# sudo useradd -d /srv/nominatim -s /bin/bash -m nominatim # # You may find a more suitable location if you wish. # # To be able to copy and paste instructions from this manual, export # user name and home directory now like this: # - export USERNAME=vagrant #DOCS: export USERNAME=nominatim +if [ "x$USERNAME" == "x" ]; then #DOCS: + export USERNAME=vagrant #DOCS: export USERNAME=nominatim export USERHOME=/srv/nominatim + sudo mkdir -p /srv/nominatim #DOCS: + sudo chown vagrant /srv/nominatim #DOCS: +fi #DOCS: # # **Never, ever run the installation as a root user.** You have been warned. # @@ -75,21 +81,29 @@ sudo chown vagrant /srv/nominatim #DOCS: # --------------------- # # CentOS does not automatically create a database cluster. Therefore, start -# with initializing the database, then enable the server to start at boot: - +# with initializing the database: +if [ "x$NOSYSTEMD" == "xyes" ]; then #DOCS: + sudo -u postgres /usr/pgsql-12/bin/pg_ctl initdb -D /var/lib/pgsql/12/data #DOCS: + sudo mkdir /var/log/postgresql + sudo chown postgres. /var/log/postgresql +else #DOCS: sudo /usr/pgsql-12/bin/postgresql-12-setup initdb - sudo systemctl enable postgresql-12 - +fi #DOCS: # -# Next tune the postgresql configuration, which is located in -# `/var/lib/pgsql/data/postgresql.conf`. See section *Postgres Tuning* in +# Next tune the postgresql configuration, which is located in +# `/var/lib/pgsql/12/data/postgresql.conf`. See section *Postgres Tuning* in # [the installation page](../admin/Installation.md#postgresql-tuning) # for the parameters to change. # -# Now start the postgresql service after updating this config file. +# Now start the postgresql service after updating this config file: +if [ "x$NOSYSTEMD" == "xyes" ]; then #DOCS: + sudo -u postgres /usr/pgsql-12/bin/pg_ctl -D /var/lib/pgsql/12/data -l /var/log/postgresql/postgresql-12.log start #DOCS: +else #DOCS: + sudo systemctl enable postgresql-12 sudo systemctl restart postgresql-12 +fi #DOCS: # # Finally, we need to add two postgres users: one for the user that does @@ -140,11 +154,11 @@ fi #DOCS: # # The webserver should serve the php scripts from the website directory of your # [project directory](../admin/Import.md#creating-the-project-directory). -# Therefore set up a project directory and populate the website directory: +# This directory needs to exist when the webserver is configured. +# Therefore set up a project directory and create the website directory: # mkdir $USERHOME/nominatim-project - cd $USERHOME/nominatim-project - nominatim refresh --website + mkdir $USERHOME/nominatim-project/website # # You need to create an alias to the website directory in your apache # configuration. Add a separate nominatim configuration to your webserver: @@ -165,11 +179,15 @@ EOFAPACHECONF sudo sed -i 's:#.*::' /etc/httpd/conf.d/nominatim.conf #DOCS: # -# Then reload apache +# Then reload apache: # +if [ "x$NOSYSTEMD" == "xyes" ]; then #DOCS: + sudo httpd #DOCS: +else #DOCS: sudo systemctl enable httpd sudo systemctl restart httpd +fi #DOCS: # # Adding SELinux Security Settings @@ -179,18 +197,19 @@ sudo sed -i 's:#.*::' /etc/httpd/conf.d/nominatim.conf #DOCS: # with a web server accessible from the Internet. At a minimum the # following SELinux labeling should be done for Nominatim: +if [ "x$HAVE_SELINUX" != "xno" ]; then #DOCS: sudo semanage fcontext -a -t httpd_sys_content_t "/usr/local/nominatim/lib/lib-php(/.*)?" sudo semanage fcontext -a -t httpd_sys_content_t "$USERHOME/nominatim-project/website(/.*)?" sudo semanage fcontext -a -t lib_t "$USERHOME/nominatim-project/module/nominatim.so" sudo restorecon -R -v /usr/local/lib/nominatim sudo restorecon -R -v $USERHOME/nominatim-project - +fi #DOCS: # You need to create a minimal configuration file that tells nominatim # the name of your webserver user: #DOCS:```sh -echo NOMINATIM_DATABASE_WEBUSER="apache" | tee .env +echo NOMINATIM_DATABASE_WEBUSER="apache" | tee $USERHOME/nominatim-project/.env #DOCS:``` diff --git a/vagrant/Install-on-Ubuntu-18.sh b/vagrant/Install-on-Ubuntu-18.sh index 9818c2a0..1d839b36 100755 --- a/vagrant/Install-on-Ubuntu-18.sh +++ b/vagrant/Install-on-Ubuntu-18.sh @@ -33,7 +33,7 @@ export DEBIAN_FRONTEND=noninteractive #DOCS: # Some of the Python packages that come with Ubuntu 18.04 are too old, so # install the latest version from pip: - pip3 install --user python-dotenv datrie pyyaml psycopg2 + pip3 install --user python-dotenv datrie pyyaml psycopg2-binary # # System Configuration diff --git a/vagrant/Install-on-Ubuntu-20.sh b/vagrant/Install-on-Ubuntu-20.sh index 0850644d..1fbabf24 100755 --- a/vagrant/Install-on-Ubuntu-20.sh +++ b/vagrant/Install-on-Ubuntu-20.sh @@ -75,9 +75,9 @@ fi #DOCS: # Restart the postgresql service after updating this config file. if [ "x$NOSYSTEMD" == "xyes" ]; then #DOCS: - sudo pg_ctlcluster 12 main start + sudo pg_ctlcluster 12 main start #DOCS: else #DOCS: - sudo systemctl restart postgresql #DOCS: + sudo systemctl restart postgresql fi #DOCS: # # Finally, we need to add two postgres users: one for the user that does -- 2.39.5