Use check_import_finished in test suite, document

[nominatim.git] / nominatim / import.c

diff --git a/nominatim/import.c b/nominatim/import.c
index c4058f9e205702b2f9d990f3ded5ee613e43214f..419ab6a5db3ab9c43d2af7f87148f73347ad9627 100644 (file)
--- a/nominatim/import.c
+++ b/nominatim/import.c
@@ -74,6 +74,8 @@ xmlHashTablePtr               partionTableTagsHashDelete;
 char                                   featureNameString[MAX_FEATURENAMESTRING];
 char                                   featureExtraTagString[MAX_FEATUREEXTRATAGSTRING];
 
 char                                   featureNameString[MAX_FEATURENAMESTRING];
 char                                   featureExtraTagString[MAX_FEATUREEXTRATAGSTRING];
 

+extern int verbose;
+

 void StartElement(xmlTextReaderPtr reader, const xmlChar *name)
 {
     char * value;
 void StartElement(xmlTextReaderPtr reader, const xmlChar *name)
 {
     char * value;


@@ -367,6 +369,7 @@ void EndElement(xmlTextReaderPtr reader, const xmlChar *name)
         if (fileMode == FILEMODE_UPDATE || fileMode == FILEMODE_DELETE || fileMode == FILEMODE_ADD)
         {
             paramValues[0] = (const char *)place_id;
         if (fileMode == FILEMODE_UPDATE || fileMode == FILEMODE_DELETE || fileMode == FILEMODE_ADD)
         {
             paramValues[0] = (const char *)place_id;

+            if (verbose) fprintf(stderr, "placex_delete: %s\n", paramValues[0]);

             res = PQexecPrepared(conn, "placex_delete", 1, paramValues, NULL, NULL, 0);
             if (PQresultStatus(res) != PGRES_COMMAND_OK)
             {
             res = PQexecPrepared(conn, "placex_delete", 1, paramValues, NULL, NULL, 0);
             if (PQresultStatus(res) != PGRES_COMMAND_OK)
             {


@@ -376,6 +379,7 @@ void EndElement(xmlTextReaderPtr reader, const xmlChar *name)
             }
             PQclear(res);
 
             }
             PQclear(res);
 

+            if (verbose) fprintf(stderr, "search_name_delete: %s\n", paramValues[0]);

             res = PQexecPrepared(conn, "search_name_delete", 1, paramValues, NULL, NULL, 0);
             if (PQresultStatus(res) != PGRES_COMMAND_OK)
             {
             res = PQexecPrepared(conn, "search_name_delete", 1, paramValues, NULL, NULL, 0);
             if (PQresultStatus(res) != PGRES_COMMAND_OK)
             {


@@ -385,6 +389,7 @@ void EndElement(xmlTextReaderPtr reader, const xmlChar *name)
             }
             PQclear(res);
 
             }
             PQclear(res);
 

+            if (verbose) fprintf(stderr, "place_addressline_delete: %s\n", paramValues[0]);

             res = PQexecPrepared(conn, "place_addressline_delete", 1, paramValues, NULL, NULL, 0);
             if (PQresultStatus(res) != PGRES_COMMAND_OK)
             {
             res = PQexecPrepared(conn, "place_addressline_delete", 1, paramValues, NULL, NULL, 0);
             if (PQresultStatus(res) != PGRES_COMMAND_OK)
             {


@@ -441,6 +446,9 @@ void EndElement(xmlTextReaderPtr reader, const xmlChar *name)
                     strcpy(featureNameString+namePos, (char *) featureName[i].value);
                     namePos += lineValueLen;
                     strcpy(featureNameString+(namePos++), "\"");
                     strcpy(featureNameString+namePos, (char *) featureName[i].value);
                     namePos += lineValueLen;
                     strcpy(featureNameString+(namePos++), "\"");

+
+                    xmlFree(featureName[i].type);
+                    xmlFree(featureName[i].value);

                 }
             }
             paramValues[5] = (const char *)featureNameString;
                 }
             }
             paramValues[5] = (const char *)featureNameString;


@@ -471,11 +479,14 @@ void EndElement(xmlTextReaderPtr reader, const xmlChar *name)
                     strcpy(featureExtraTagString+namePos, (char *) featureExtraTag[i].value);
                     namePos += lineValueLen;
                     strcpy(featureExtraTagString+(namePos++), "\"");
                     strcpy(featureExtraTagString+namePos, (char *) featureExtraTag[i].value);
                     namePos += lineValueLen;
                     strcpy(featureExtraTagString+(namePos++), "\"");

+
+                    xmlFree(featureExtraTag[i].type);
+                    xmlFree(featureExtraTag[i].value);

                 }
             }
             paramValues[7] = (const char *)featureExtraTagString;
 
                 }
             }
             paramValues[7] = (const char *)featureExtraTagString;
 

-            if (strlen(feature.parentPlaceID) == 0)
+            if (xmlStrlen(feature.parentPlaceID) == 0)

                 paramValues[8] = "0";
             else
                 paramValues[8] = (const char *)feature.parentPlaceID;
                 paramValues[8] = "0";
             else
                 paramValues[8] = (const char *)feature.parentPlaceID;


@@ -485,8 +496,9 @@ void EndElement(xmlTextReaderPtr reader, const xmlChar *name)
             paramValues[11] = (const char *)feature.rankAddress;
             paramValues[12] = (const char *)feature.rankSearch;
             paramValues[13] = (const char *)feature.geometry;
             paramValues[11] = (const char *)feature.rankAddress;
             paramValues[12] = (const char *)feature.rankSearch;
             paramValues[13] = (const char *)feature.geometry;

-            if (strlen(paramValues[3]))
+            if (strlen(paramValues[3]) && strlen(paramValues[13]))

             {
             {

+                if (verbose) fprintf(stderr, "placex_insert: %s\n", paramValues[0]);

                 res = PQexecPrepared(conn, "placex_insert", 14, paramValues, NULL, NULL, 0);
                 if (PQresultStatus(res) != PGRES_COMMAND_OK)
                 {
                 res = PQexecPrepared(conn, "placex_insert", 14, paramValues, NULL, NULL, 0);
                 if (PQresultStatus(res) != PGRES_COMMAND_OK)
                 {


@@ -503,15 +515,18 @@ void EndElement(xmlTextReaderPtr reader, const xmlChar *name)
                 // insert into place_address
                 paramValues[0] = (const char *)place_id;
                 paramValues[1] = (const char *)featureAddress[i].distance;
                 // insert into place_address
                 paramValues[0] = (const char *)place_id;
                 paramValues[1] = (const char *)featureAddress[i].distance;

+                if (paramValues[1] == NULL || strlen(paramValues[1]) == 0) paramValues[1] = "0";

                 paramValues[2] = (const char *)featureAddress[i].type;
                 paramValues[3] = (const char *)featureAddress[i].id;
                 paramValues[4] = (const char *)featureAddress[i].key;
                 paramValues[5] = (const char *)featureAddress[i].value;
                 paramValues[6] = (const char *)featureAddress[i].isAddress;
                 paramValues[2] = (const char *)featureAddress[i].type;
                 paramValues[3] = (const char *)featureAddress[i].id;
                 paramValues[4] = (const char *)featureAddress[i].key;
                 paramValues[5] = (const char *)featureAddress[i].value;
                 paramValues[6] = (const char *)featureAddress[i].isAddress;

+                if (verbose) fprintf(stderr, "placex_insert: %s %s\n", paramValues[2], paramValues[3]);

                 res = PQexecPrepared(conn, "place_addressline_insert", 7, paramValues, NULL, NULL, 0);
                 if (PQresultStatus(res) != PGRES_COMMAND_OK)
                 {
                     fprintf(stderr, "place_addressline_insert: INSERT failed: %s", PQerrorMessage(conn));
                 res = PQexecPrepared(conn, "place_addressline_insert", 7, paramValues, NULL, NULL, 0);
                 if (PQresultStatus(res) != PGRES_COMMAND_OK)
                 {
                     fprintf(stderr, "place_addressline_insert: INSERT failed: %s", PQerrorMessage(conn));

+                    fprintf(stderr, "(%s,%s,%s,%s,%s,%s,%s)",paramValues[0],paramValues[1],paramValues[2],paramValues[3],paramValues[4],paramValues[5],paramValues[6]);

                     PQclear(res);
                     exit(EXIT_FAILURE);
                 }
                     PQclear(res);
                     exit(EXIT_FAILURE);
                 }


@@ -526,15 +541,33 @@ void EndElement(xmlTextReaderPtr reader, const xmlChar *name)
 
             if (featureNameLines)
             {
 
             if (featureNameLines)
             {

-                paramValues[0] = (const char *)place_id;
-                res = PQexecPrepared(conn, "search_name_insert", 1, paramValues, NULL, NULL, 0);
-                if (PQresultStatus(res) != PGRES_COMMAND_OK)
-                {
-                    fprintf(stderr, "search_name_insert: INSERT failed: %s", PQerrorMessage(conn));
+                if (xmlStrlen(feature.parentPlaceID) > 0 && featureAddressLines == 0)
+               {
+                    paramValues[0] = (const char *)place_id;
+                    paramValues[1] = (const char *)feature.parentPlaceID;
+                    if (verbose) fprintf(stderr, "search_name_from_parent_insert: INSERT %s %s\n", paramValues[0], paramValues[1]);
+                    res = PQexecPrepared(conn, "search_name_from_parent_insert", 2, paramValues, NULL, NULL, 0);
+                    if (PQresultStatus(res) != PGRES_COMMAND_OK)
+                    {
+                        fprintf(stderr, "search_name_from_parent_insert: INSERT failed: %s", PQerrorMessage(conn));
+                        PQclear(res);
+                        exit(EXIT_FAILURE);
+                    }
+                    PQclear(res);
+               }
+               else
+               {
+                    paramValues[0] = (const char *)place_id;
+                    if (verbose) fprintf(stderr, "search_name_insert: INSERT %s\n", paramValues[0]);
+                    res = PQexecPrepared(conn, "search_name_insert", 1, paramValues, NULL, NULL, 0);
+                    if (PQresultStatus(res) != PGRES_COMMAND_OK)
+                    {
+                        fprintf(stderr, "search_name_insert: INSERT failed: %s", PQerrorMessage(conn));
+                        PQclear(res);
+                        exit(EXIT_FAILURE);
+                    }

                     PQclear(res);
                     PQclear(res);

-                    exit(EXIT_FAILURE);

                 }
                 }

-                PQclear(res);

             }
 
             partionQueryName = xmlHashLookup2(partionTableTagsHash, feature.key, feature.value);
             }
 
             partionQueryName = xmlHashLookup2(partionTableTagsHash, feature.key, feature.value);


@@ -573,11 +606,11 @@ void EndElement(xmlTextReaderPtr reader, const xmlChar *name)
         xmlFree(feature.value);
         xmlFree(feature.rankAddress);
         xmlFree(feature.rankSearch);
         xmlFree(feature.value);
         xmlFree(feature.rankAddress);
         xmlFree(feature.rankSearch);

+        if (feature.countryCode) xmlFree(feature.countryCode);

        if (feature.parentPlaceID) xmlFree(feature.parentPlaceID);
        if (feature.parentType) xmlFree(feature.parentType);
        if (feature.parentID) xmlFree(feature.parentID);
 //             if (feature.name) xmlFree(feature.name);
        if (feature.parentPlaceID) xmlFree(feature.parentPlaceID);
        if (feature.parentType) xmlFree(feature.parentType);
        if (feature.parentID) xmlFree(feature.parentID);
 //             if (feature.name) xmlFree(feature.name);

-        if (feature.countryCode) xmlFree(feature.countryCode);

         if (feature.adminLevel) xmlFree(feature.adminLevel);
         if (feature.houseNumber) xmlFree(feature.houseNumber);
         if (feature.geometry) xmlFree(feature.geometry);
         if (feature.adminLevel) xmlFree(feature.adminLevel);
         if (feature.houseNumber) xmlFree(feature.houseNumber);
         if (feature.geometry) xmlFree(feature.geometry);


@@ -731,10 +764,25 @@ int nominatim_import(const char *conninfo, const char *partionTagsFilename, cons
     res = PQprepare(conn, "search_name_insert",
                     "insert into search_name (place_id, search_rank, address_rank, country_code, name_vector, nameaddress_vector, centroid) "
                     "select place_id, rank_search, rank_address, country_code, make_keywords(name), "
     res = PQprepare(conn, "search_name_insert",
                     "insert into search_name (place_id, search_rank, address_rank, country_code, name_vector, nameaddress_vector, centroid) "
                     "select place_id, rank_search, rank_address, country_code, make_keywords(name), "

-                    "(select uniq(sort(array_agg(name_vector))) from place_addressline join search_name on "
-                    "(address_place_id = search_name.place_id) where place_addressline.place_id = $1 ), st_centroid(geometry) from placex "
+                    "(select uniq(sort(array_agg(parent_search_name.name_vector))) from search_name as parent_search_name where place_id in "
+                     "(select distinct address_place_id from place_addressline where place_addressline.place_id = $1 limit 1000)"
+                    "), st_centroid(geometry) from placex "

                     "where place_id = $1",
                     1, NULL);
                     "where place_id = $1",
                     1, NULL);

+
+    if (PQresultStatus(res) != PGRES_COMMAND_OK)
+    {
+        fprintf(stderr, "Failed to prepare search_name_insert: %s\n", PQerrorMessage(conn));
+        exit(EXIT_FAILURE);
+    }
+
+    res = PQprepare(conn, "search_name_from_parent_insert",
+                    "insert into search_name (place_id, search_rank, address_rank, country_code, name_vector, nameaddress_vector, centroid) "
+                    "select place_id, rank_search, rank_address, country_code, make_keywords(name), "
+                    "(select uniq(sort(name_vector+nameaddress_vector)) from search_name as parent_search_name "
+                    "where parent_search_name.place_id = $2 ), st_centroid(geometry) from placex "
+                    "where place_id = $1",
+                    2, NULL);

     if (PQresultStatus(res) != PGRES_COMMAND_OK)
     {
         fprintf(stderr, "Failed to prepare search_name_insert: %s\n", PQerrorMessage(conn));
     if (PQresultStatus(res) != PGRES_COMMAND_OK)
     {
         fprintf(stderr, "Failed to prepare search_name_insert: %s\n", PQerrorMessage(conn));