From: Sarah Hoffmann <lonvia@denofr.de>
Date: Fri, 9 Jun 2017 19:54:05 +0000 (+0200)
Subject: Merge remote-tracking branch 'upstream/master'
X-Git-Tag: deploy~381
X-Git-Url: https://git.openstreetmap.org./nominatim.git/commitdiff_plain/c504ec523cd815d920528bcdc9b915a83f2fd64a?hp=ebbd03efe08c33fe81fdcf17e498d1d14546a314

Merge remote-tracking branch 'upstream/master'
---

diff --git a/docs/install-on-centos-7.md b/docs/install-on-centos-7.md
index 40e8a404..481bd2d0 100644
--- a/docs/install-on-centos-7.md
+++ b/docs/install-on-centos-7.md
@@ -6,7 +6,7 @@ Installing the Required Software
 ================================
 
 These instructions expect that you have a freshly installed CentOS version 7.
-Make sure all packages are are up-to-date by running:
+Make sure all packages are up-to-date by running:
 
     sudo yum update -y
 
@@ -20,7 +20,7 @@ Now you can install all packages needed for Nominatim:
 
     sudo yum install -y postgresql-server postgresql-contrib postgresql-devel postgis postgis-utils \
                         git cmake make gcc gcc-c++ libtool policycoreutils-python \
-                        php-pgsql php php-pear php-pear-DB libpqxx-devel proj-epsg \
+                        php-pgsql php php-pear php-pear-DB php-intl libpqxx-devel proj-epsg \
                         bzip2-devel proj-devel geos-devel libxml2-devel boost-devel expat-devel zlib-devel
 
 If you want to run the test suite, you need to install the following
diff --git a/docs/install-on-ubuntu-16.md b/docs/install-on-ubuntu-16.md
index 66d82cf0..de8468d4 100644
--- a/docs/install-on-ubuntu-16.md
+++ b/docs/install-on-ubuntu-16.md
@@ -27,7 +27,7 @@ Now you can install all packages needed for Nominatim:
                             libbz2-dev libpq-dev libgeos-dev libgeos++-dev libproj-dev \
                             postgresql-server-dev-9.5 postgresql-9.5-postgis-2.2 postgresql-contrib-9.5 \
                             apache2 php php-pgsql libapache2-mod-php php-pear php-db \
-                            git
+                            php-intl git
 
 If you want to run the test suite, you need to install the following
 additional packages:
diff --git a/lib/lib.php b/lib/lib.php
index 941f6a1d..c6eadc6c 100644
--- a/lib/lib.php
+++ b/lib/lib.php
@@ -38,8 +38,13 @@ function getDatabaseDate(&$oDB)
     // Find the newest node in the DB
     $iLastOSMID = $oDB->getOne("select max(osm_id) from place where osm_type = 'N'");
     // Lookup the timestamp that node was created
-    $sLastNodeURL = 'http://www.openstreetmap.org/api/0.6/node/'.$iLastOSMID."/1";
+    $sLastNodeURL = 'https://www.openstreetmap.org/api/0.6/node/'.$iLastOSMID."/1";
     $sLastNodeXML = file_get_contents($sLastNodeURL);
+
+    if ($sLastNodeXML === false) {
+        return false;
+    }
+
     preg_match('#timestamp="(([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2}):([0-9]{2}):([0-9]{2})Z)"#', $sLastNodeXML, $aLastNodeDate);
 
     return $aLastNodeDate[1];
diff --git a/utils/osm_file_date.py b/utils/osm_file_date.py
new file mode 100755
index 00000000..6418efb0
--- /dev/null
+++ b/utils/osm_file_date.py
@@ -0,0 +1,33 @@
+#!/usr/bin/python
+
+import osmium
+import sys
+import datetime
+
+
+class Datecounter(osmium.SimpleHandler):
+
+    filedate = None
+
+    def date(self, o):
+        ts = o.timestamp
+        if self.filedate is None or ts > self.filedate:
+            self.filedate = ts
+
+    node = date
+    way = date
+    relation = date
+
+
+if __name__ == '__main__':
+    if len(sys.argv) != 2:
+        print("Usage: python osm_file_date.py <osmfile>")
+        sys.exit(-1)
+
+    h = Datecounter()
+
+    h.apply_file(sys.argv[1])
+
+    print(h.filedate)
+
+
diff --git a/utils/setup.php b/utils/setup.php
index eeba0a28..7490d820 100755
--- a/utils/setup.php
+++ b/utils/setup.php
@@ -407,9 +407,13 @@ if ($aCMDResult['load-data'] || $aCMDResult['all']) {
 
     $sDatabaseDate = getDatabaseDate($oDB);
     pg_query($oDB->connection, 'TRUNCATE import_status');
-    $sSQL = "INSERT INTO import_status (lastimportdate) VALUES('".$sDatabaseDate."')";
-    pg_query($oDB->connection, $sSQL);
-    echo "Latest data imported from $sDatabaseDate.\n";
+    if ($sDatabaseDate === false) {
+        echo "WARNING: could not determine database date.\n";
+    } else {
+        $sSQL = "INSERT INTO import_status (lastimportdate) VALUES('".$sDatabaseDate."')";
+        pg_query($oDB->connection, $sSQL);
+        echo "Latest data imported from $sDatabaseDate.\n";
+    }
 }
 
 if ($aCMDResult['import-tiger-data']) {
diff --git a/utils/update.php b/utils/update.php
index f7f20aa3..099ca0b3 100755
--- a/utils/update.php
+++ b/utils/update.php
@@ -68,6 +68,9 @@ if ($aResult['init-updates']) {
     }
 
     $sDatabaseDate = getDatabaseDate($oDB);
+    if ($sDatabaseDate === false) {
+        fail("Cannot determine date of database.");
+    }
     $sWindBack = strftime('%Y-%m-%dT%H:%M:%SZ',
                           strtotime($sDatabaseDate) - (3*60*60));
 
@@ -294,7 +297,14 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
 
             // write the update logs
             $iFileSize = filesize($sImportFile);
-            $sBatchEnd = getDatabaseDate($oDB);
+            // get the newest object from the diff file
+            $sBatchEnd = 0;
+            $iRet = 0;
+            exec(CONST_BasePath.'/utils/osm_file_date.py '.$sImportFile, $sBatchEnd, $iRet);
+            if ($iRet != 0) {
+                fail('Error getting date from diff file.');
+            }
+            $sBatchEnd = $sBatchEnd[0];
             $sSQL = "INSERT INTO import_osmosis_log (batchend, batchseq, batchsize, starttime, endtime, event) values ('$sBatchEnd',$iEndSequence,$iFileSize,'".date('Y-m-d H:i:s', $fCMDStartTime)."','".date('Y-m-d H:i:s')."','import')";
             var_Dump($sSQL);
             chksql($oDB->query($sSQL));