make sleep interval dependent on date of last batch

[nominatim.git] / utils / import_wikipedia.sh

diff --git a/utils/import_wikipedia.sh b/utils/import_wikipedia.sh
index 7dd5ab08269cfc6b802c64d0af4a3765fe9f826a..0a15e2d016909a7bbd5657a4e08dac28a3b40151 100755 (executable)
--- a/utils/import_wikipedia.sh
+++ b/utils/import_wikipedia.sh
@@ -39,8 +39,11 @@ do
   echo "insert into wikipedia_article select '${i}', title, count, othercount, count+othercount from ${i}pagelinkcount;" | $psqlcmd
 done
 
+echo "update wikipedia_article set importance = log(totalcount)/log((select max(totalcount) from wikipedia_article))" | $psqlcmd
+
 # precalculated lat,lon from dbpedia
-cat geo_coordinates_en.nq | grep http://www.georss.org/georss/point | sed 's|<http://dbpedia.org/resource/[^>]*> *<http://www.georss.org/georss/point> "\(-\?[-0-9.E]\+\) \(-\?[-0-9.E]\+\)"@en <http://\([a-z][a-z]\).wikipedia.org/wiki/\([^#]\+\)#> .|update pagelinks set lat=\1, lon=\2 where language = '"'"'\3'"'"' and title = decode_url_part('"'"'\4'"'"');|g' | $psqlcmd
+wget http://downloads.dbpedia.org/current/en/geo_coordinates_en.nq.bz2
+bzip2 -dc geo_coordinates_en.nq.bz2 | grep http://www.georss.org/georss/point | sed 's|<http://dbpedia.org/resource/[^>]*> *<http://www.georss.org/georss/point> "\(-\?[-0-9.E]\+\) \(-\?[-0-9.E]\+\)"@en <http://\([a-z][a-z]\).wikipedia.org/wiki/\([^#]\+\)#> .|update pagelinks set lat=\1, lon=\2 where language = '"'"'\3'"'"' and title = decode_url_part('"'"'\4'"'"');|g' | $psqlcmd
 
 # media wiki dumper
 wget https://github.com/bcollier/mwdumper/blob/master/build/mwdumper.jar