deduplicate search term graph slightly earlier

[nominatim.git] / nominatim / index.c

diff --git a/nominatim/index.c b/nominatim/index.c
index bee8b3dabf68a1ebbccc3af393bac45982d9f879..0b87abdc635681df6c5007246b840a52ff17c28a 100644 (file)
--- a/nominatim/index.c
+++ b/nominatim/index.c
@@ -40,6 +40,7 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co
     int i;
     int iSector;
     int iResult;
     int i;
     int iSector;
     int iResult;

+    int bSkip;

 
     const char *paramValues[2];
     int         paramLengths[2];
 
     const char *paramValues[2];
     int         paramLengths[2];


@@ -149,7 +150,7 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co
 
     for (rank = rank_min; rank <= rank_max; rank++)
     {
 
     for (rank = rank_min; rank <= rank_max; rank++)
     {

-        printf("Starting rank %d\n", rank);
+        fprintf(stderr, "Starting rank %d\n", rank);

         rankCountTuples = 0;
         rankPerSecond = 0;
 
         rankCountTuples = 0;
         rankPerSecond = 0;
 


@@ -160,7 +161,8 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co
 //        if (rank < 16)
 //            resSectors = PQexecPrepared(conn, "index_nosectors", 1, paramValues, paramLengths, paramFormats, 1);
 //        else
 //        if (rank < 16)
 //            resSectors = PQexecPrepared(conn, "index_nosectors", 1, paramValues, paramLengths, paramFormats, 1);
 //        else

-            resSectors = PQexecPrepared(conn, "index_sectors", 1, paramValues, paramLengths, paramFormats, 1);
+        resSectors = PQexecPrepared(conn, "index_sectors", 1, paramValues, paramLengths, paramFormats, 1);
+

         if (PQresultStatus(resSectors) != PGRES_TUPLES_OK)
         {
             fprintf(stderr, "index_sectors: SELECT failed: %s", PQerrorMessage(conn));
         if (PQresultStatus(resSectors) != PGRES_TUPLES_OK)
         {
             fprintf(stderr, "index_sectors: SELECT failed: %s", PQerrorMessage(conn));


@@ -215,7 +217,7 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co
             if (iSector < PQntuples(resSectors))
             {
                 sector = PGint32(*((uint32_t *)PQgetvalue(resSectors, iSector, 0)));
             if (iSector < PQntuples(resSectors))
             {
                 sector = PGint32(*((uint32_t *)PQgetvalue(resSectors, iSector, 0)));

-                //printf("\n Starting sector %d size %ld\n", sector, PGint64(*((uint64_t *)PQgetvalue(resSectors, iSector, 1))));
+//                fprintf(stderr, "\n Starting sector %d size %ld\n", sector, PGint64(*((uint64_t *)PQgetvalue(resSectors, iSector, 1))));

 
                 // Get all the place_id's for this sector
                 paramRank = PGint32(rank);
 
                 // Get all the place_id's for this sector
                 paramRank = PGint32(rank);


@@ -226,9 +228,9 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co
                 paramValues[1] = (char *)&paramSector;
                 paramLengths[1] = sizeof(paramSector);
                 paramFormats[1] = 1;
                 paramValues[1] = (char *)&paramSector;
                 paramLengths[1] = sizeof(paramSector);
                 paramFormats[1] = 1;

-                if (rankTotalTuples-rankCountTuples < num_threads*20)
+                if (rankTotalTuples-rankCountTuples < num_threads*1000)

                {
                {

-                       iResult = PQsendQueryPrepared(conn, "index_nosector_places", 1, paramValues, paramLengths, paramFormats, 1);
+                    iResult = PQsendQueryPrepared(conn, "index_nosector_places", 1, paramValues, paramLengths, paramFormats, 1);

                }
                 else
                {
                }
                 else
                {


@@ -272,7 +274,7 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co
                         if (sleepcount++ > 500)
                         {
                             rankPerSecond = ((float)rankCountTuples + (float)count) / MAX(difftime(time(0), rankStartTime),1);
                         if (sleepcount++ > 500)
                         {
                             rankPerSecond = ((float)rankCountTuples + (float)count) / MAX(difftime(time(0), rankStartTime),1);

-                            printf("  Done %i in %i @ %f per second - Rank %i ETA (seconds): %f\n", (rankCountTuples + count), (int)(difftime(time(0), rankStartTime)), rankPerSecond, rank, ((float)(rankTotalTuples - (rankCountTuples + count)))/rankPerSecond);
+                            fprintf(stderr, "  Done %i in %i @ %f per second - Rank %i ETA (seconds): %f\n", (rankCountTuples + count), (int)(difftime(time(0), rankStartTime)), rankPerSecond, rank, ((float)(rankTotalTuples - (rankCountTuples + count)))/rankPerSecond);

                             sleepcount = 0;
                         }
                     }
                             sleepcount = 0;
                         }
                     }


@@ -288,14 +290,17 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co
 
                 // Finished sector
                 rankPerSecond = (float)rankCountTuples / MAX(difftime(time(0), rankStartTime),1);
 
                 // Finished sector
                 rankPerSecond = (float)rankCountTuples / MAX(difftime(time(0), rankStartTime),1);

-                printf("  Done %i in %i @ %f per second - ETA (seconds): %f\n", rankCountTuples, (int)(difftime(time(0), rankStartTime)), rankPerSecond, ((float)(rankTotalTuples - rankCountTuples))/rankPerSecond);
+                fprintf(stderr, "  Done %i in %i @ %f per second - ETA (seconds): %f\n", rankCountTuples, (int)(difftime(time(0), rankStartTime)), rankPerSecond, ((float)(rankTotalTuples - rankCountTuples))/rankPerSecond);

 
                 PQclear(resPlaces);
             }
 
                 PQclear(resPlaces);
             }

-            if (rankTotalTuples-rankCountTuples < num_threads*20) iSector = PQntuples(resSectors);
+            if (rankTotalTuples-rankCountTuples < num_threads*20 && iSector < PQntuples(resSectors))
+            {
+                iSector = PQntuples(resSectors) - 1;
+            }

         }
         // Finished rank
         }
         // Finished rank

-        printf("\r  Done %i in %i @ %f per second - FINISHED                      \n\n", rankCountTuples, (int)(difftime(time(0), rankStartTime)), rankPerSecond);
+        fprintf(stderr, "\r  Done %i in %i @ %f per second - FINISHED                      \n\n", rankCountTuples, (int)(difftime(time(0), rankStartTime)), rankPerSecond);

 
         PQclear(resSectors);
     }
 
         PQclear(resSectors);
     }


@@ -309,6 +314,7 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co
 void *nominatim_indexThread(void * thread_data_in)
 {
     struct index_thread_data * thread_data = (struct index_thread_data * )thread_data_in;
 void *nominatim_indexThread(void * thread_data_in)
 {
     struct index_thread_data * thread_data = (struct index_thread_data * )thread_data_in;

+    struct export_data querySet;

 
     PGresult   *res;
 
 
     PGresult   *res;
 


@@ -333,12 +339,19 @@ void *nominatim_indexThread(void * thread_data_in)
 
         pthread_mutex_unlock( thread_data->count_mutex );
 
 
         pthread_mutex_unlock( thread_data->count_mutex );
 

-        if (verbose) printf("  Processing place_id %d\n", place_id);
+        if (verbose) fprintf(stderr, "  Processing place_id %d\n", place_id);

 
         updateStartTime = time(0);
        int done = 0;
 
         updateStartTime = time(0);
        int done = 0;

+
+        if (thread_data->writer)
+        {
+             nominatim_exportPlaceQueries(place_id, thread_data->conn, &querySet);
+        }
+

        while(!done)
        {
        while(!done)
        {

+

                paramPlaceID = PGint32(place_id);
                paramValues[0] = (char *)&paramPlaceID;
                paramLengths[0] = sizeof(paramPlaceID);
                paramPlaceID = PGint32(place_id);
                paramValues[0] = (char *)&paramPlaceID;
                paramLengths[0] = sizeof(paramPlaceID);


@@ -351,22 +364,25 @@ void *nominatim_indexThread(void * thread_data_in)
                        if (strncmp(PQerrorMessage(thread_data->conn), "ERROR:  deadlock detected", 25))
                        {
                            fprintf(stderr, "index_placex: UPDATE failed - deadlock, retrying\n");
                        if (strncmp(PQerrorMessage(thread_data->conn), "ERROR:  deadlock detected", 25))
                        {
                            fprintf(stderr, "index_placex: UPDATE failed - deadlock, retrying\n");

+                           PQclear(res);
+                            sleep(rand() % 10);

                        }
                        else
                        {
                            fprintf(stderr, "index_placex: UPDATE failed: %s", PQerrorMessage(thread_data->conn));
                            PQclear(res);
                        }
                        else
                        {
                            fprintf(stderr, "index_placex: UPDATE failed: %s", PQerrorMessage(thread_data->conn));
                            PQclear(res);

-                               sleep(5);
+                            sleep(rand() % 10);

 //                         exit(EXIT_FAILURE);
                        }
                }
         }
         PQclear(res);
 //                         exit(EXIT_FAILURE);
                        }
                }
         }
         PQclear(res);

-        if (difftime(time(0), updateStartTime) > 1) printf("  Slow place_id %d\n", place_id);
+        if (difftime(time(0), updateStartTime) > 1) fprintf(stderr, "  Slow place_id %d\n", place_id);

 
         if (thread_data->writer)
         {
 
         if (thread_data->writer)
         {

-            nominatim_exportPlace(place_id, thread_data->conn, thread_data->writer, thread_data->writer_mutex);
+            nominatim_exportPlace(place_id, thread_data->conn, thread_data->writer, thread_data->writer_mutex, &querySet);
+            nominatim_exportFreeQueries(&querySet);

         }
     }
 
         }
     }