cookbooks/planet/templates/default/planet-notes-dump.erb

   1 #!/bin/bash
   2 ulimit -m 4194304 -v 4194304
   3 set -e
   4
   5 if [ -f /tmp/planet-notes-dump.lock ]; then
   6         if [ "$(ps -p `cat /tmp/planet-notes-dump.lock` | wc -l)" -gt 1 ]; then
   7                 # process is still running
   8                 echo Error: Another planet-notes-dump is running
   9                 exit 1
  10         else
  11                 # process not running, but lock file not deleted?
  12                 rm /tmp/planet-notes-dump.lock
  13         fi
  14 fi
  15
  16 #Create Lock
  17 echo $$ > /tmp/planet-notes-dump.lock
  18
  19
  20 cur_date=$(date +%y%m%d)
  21 cur_year=$(date +%Y)
  22 cur_planet_notes="planet-notes-${cur_date}.osn"
  23 planet_dir=/store/planet/
  24
  25 export PATH='/usr/local/bin:/usr/bin:/bin:/usr/bin/X11'
  26
  27
  28 if [ \! -d "${planet_dir}/notes/" ]
  29         then
  30                 echo "${planet_dir}notes/ does not exist"
  31                 exit 1
  32         fi
  33 if [ \! -d "${planet_dir}/notes/${cur_year}/" ]; then mkdir "${planet_dir}/notes/${cur_year}/"; fi
  34 cd "${planet_dir}/notes/${cur_year}/"
  35
  36 /usr/bin/python3 /opt/planet-notes-dump/dump.py --quiet --database openstreetmap --host <%= node[:web][:readonly_database_host] %> --user planetdump --password '<%= @password %>' ".${cur_planet_notes}"
  37 pbzip2 -p6 -9 ".${cur_planet_notes}"
  38
  39 planet_notes_size=$(du -sb ".${cur_planet_notes}.bz2" | awk '{ print $1 }')
  40 if ((planet_notes_size<12000000)); then
  41         echo "Planet .${cur_planet_notes}.bz2 too small"
  42         exit 1
  43 fi
  44
  45 mv ".${cur_planet_notes}.bz2" "${cur_planet_notes}.bz2"
  46 md5sum "${cur_planet_notes}.bz2" > "${cur_planet_notes}.bz2.md5"
  47
  48 #link planet latest to the new file
  49 cd "${planet_dir}/notes/"
  50
  51 ln -fs "${cur_year}/${cur_planet_notes}.bz2" planet-notes-latest.osn.bz2
  52
  53 # mangle md5 files for 'latest' ones
  54 rm -f planet-notes-latest.osn.bz2.md5
  55
  56 sed -e "s/${cur_planet_notes}.bz2/planet-notes-latest.osn.bz2/" "${cur_year}/${cur_planet_notes}.bz2.md5" > planet-notes-latest.osn.bz2.md5
  57
  58 /opt/awscli/v2/current/bin/aws --profile osm-pds-upload s3 cp --storage-class INTELLIGENT_TIERING "${cur_year}/${cur_planet_notes}.bz2.md5" "s3://osm-planet-eu-central-1/notes/osn/${cur_year}/${cur_planet_notes}.bz2.md5"
  59 /opt/awscli/v2/current/bin/aws --profile osm-pds-upload s3 cp --storage-class INTELLIGENT_TIERING "${cur_year}/${cur_planet_notes}.bz2" "s3://osm-planet-eu-central-1/notes/osn/${cur_year}/${cur_planet_notes}.bz2"
  60
  61 rm /tmp/planet-notes-dump.lock