]> git.openstreetmap.org Git - chef.git/blob - cookbooks/planet/templates/default/planet-notes-dump.erb
Add otrs test role from naga
[chef.git] / cookbooks / planet / templates / default / planet-notes-dump.erb
1 #!/bin/bash
2 ulimit -m 4194304 -v 4194304
3 set -e
4
5 if [ -f /tmp/planet-notes-dump.lock ]; then
6         if [ "$(ps -p `cat /tmp/planet-notes-dump.lock` | wc -l)" -gt 1 ]; then
7                 # process is still running
8                 echo Error: Another planet-notes-dump is running
9                 exit 1
10         else
11                 # process not running, but lock file not deleted?
12                 rm /tmp/planet-notes-dump.lock
13         fi
14 fi
15
16 #Create Lock
17 echo $$ > /tmp/planet-notes-dump.lock
18
19
20 cur_date=$(date +%y%m%d)
21 cur_year=$(date +%Y)
22 cur_planet_notes="planet-notes-${cur_date}.osn"
23 planet_dir=/store/planet/
24
25 export PATH='/usr/local/bin:/usr/bin:/bin:/usr/bin/X11'
26
27
28 if [ \! -d "${planet_dir}/notes/" ]
29         then
30                 echo "${planet_dir}notes/ does not exist"
31                 exit 1
32         fi
33 if [ \! -d "${planet_dir}/notes/${cur_year}/" ]; then mkdir "${planet_dir}/notes/${cur_year}/"; fi
34 cd "${planet_dir}/notes/${cur_year}/"
35
36 /usr/bin/python3 /opt/planet-notes-dump/dump.py --quiet --database openstreetmap --host <%= node[:web][:readonly_database_host] %> --user planetdump --password '<%= @password %>' ".${cur_planet_notes}"
37 pbzip2 -p6 -9 ".${cur_planet_notes}"
38
39 planet_notes_size=$(du -sb ".${cur_planet_notes}.bz2" | awk '{ print $1 }')
40 if ((planet_notes_size<12000000)); then
41         echo "Planet .${cur_planet_notes}.bz2 too small"
42         exit 1
43 fi
44
45 mv ".${cur_planet_notes}.bz2" "${cur_planet_notes}.bz2"
46 md5sum "${cur_planet_notes}.bz2" > "${cur_planet_notes}.bz2.md5"
47
48 #link planet latest to the new file
49 cd "${planet_dir}/notes/"
50
51 ln -fs "${cur_year}/${cur_planet_notes}.bz2" planet-notes-latest.osn.bz2
52
53 # mangle md5 files for 'latest' ones
54 rm -f planet-notes-latest.osn.bz2.md5
55
56 sed -e "s/${cur_planet_notes}.bz2/planet-notes-latest.osn.bz2/" "${cur_year}/${cur_planet_notes}.bz2.md5" > planet-notes-latest.osn.bz2.md5
57
58 /opt/awscli/v2/current/bin/aws --profile osm-pds-upload s3 cp --storage-class INTELLIGENT_TIERING  --no-progress "${cur_year}/${cur_planet_notes}.bz2.md5" "s3://osm-planet-eu-central-1/notes/osn/${cur_year}/${cur_planet_notes}.bz2.md5"
59 /opt/awscli/v2/current/bin/aws --profile osm-pds-upload s3 cp --storage-class INTELLIGENT_TIERING  --no-progress "${cur_year}/${cur_planet_notes}.bz2" "s3://osm-planet-eu-central-1/notes/osn/${cur_year}/${cur_planet_notes}.bz2"
60
61 rm /tmp/planet-notes-dump.lock