X-Git-Url: https://git.openstreetmap.org./nominatim.git/blobdiff_plain/52cdaba0f1f0725dad32f64326bcc3af5accc694..95d50f35d046e2ae540eadf7ab9af40ac119e06f:/nominatim/index.c diff --git a/nominatim/index.c b/nominatim/index.c index 07c8daba..0b87abdc 100644 --- a/nominatim/index.c +++ b/nominatim/index.c @@ -40,6 +40,7 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co int i; int iSector; int iResult; + int bSkip; const char *paramValues[2]; int paramLengths[2]; @@ -149,7 +150,7 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co for (rank = rank_min; rank <= rank_max; rank++) { - printf("Starting rank %d\n", rank); + fprintf(stderr, "Starting rank %d\n", rank); rankCountTuples = 0; rankPerSecond = 0; @@ -157,10 +158,11 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co paramValues[0] = (char *)¶mRank; paramLengths[0] = sizeof(paramRank); paramFormats[0] = 1; - if (rank < 16) - resSectors = PQexecPrepared(conn, "index_nosectors", 1, paramValues, paramLengths, paramFormats, 1); - else - resSectors = PQexecPrepared(conn, "index_sectors", 1, paramValues, paramLengths, paramFormats, 1); +// if (rank < 16) +// resSectors = PQexecPrepared(conn, "index_nosectors", 1, paramValues, paramLengths, paramFormats, 1); +// else + resSectors = PQexecPrepared(conn, "index_sectors", 1, paramValues, paramLengths, paramFormats, 1); + if (PQresultStatus(resSectors) != PGRES_TUPLES_OK) { fprintf(stderr, "index_sectors: SELECT failed: %s", PQerrorMessage(conn)); @@ -215,7 +217,7 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co if (iSector < PQntuples(resSectors)) { sector = PGint32(*((uint32_t *)PQgetvalue(resSectors, iSector, 0))); - //printf("\n Starting sector %d size %ld\n", sector, PGint64(*((uint64_t *)PQgetvalue(resSectors, iSector, 1)))); +// fprintf(stderr, "\n Starting sector %d size %ld\n", sector, PGint64(*((uint64_t *)PQgetvalue(resSectors, iSector, 1)))); // Get all the place_id's for this sector paramRank = PGint32(rank); @@ -226,10 +228,14 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co paramValues[1] = (char *)¶mSector; paramLengths[1] = sizeof(paramSector); paramFormats[1] = 1; - if (rank < 16) + if (rankTotalTuples-rankCountTuples < num_threads*1000) + { iResult = PQsendQueryPrepared(conn, "index_nosector_places", 1, paramValues, paramLengths, paramFormats, 1); + } else + { iResult = PQsendQueryPrepared(conn, "index_sector_places", 2, paramValues, paramLengths, paramFormats, 1); + } if (!iResult) { fprintf(stderr, "index_sector_places: SELECT failed: %s", PQerrorMessage(conn)); @@ -268,7 +274,7 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co if (sleepcount++ > 500) { rankPerSecond = ((float)rankCountTuples + (float)count) / MAX(difftime(time(0), rankStartTime),1); - printf(" Done %i in %i @ %f per second - Rank %i ETA (seconds): %f\n", (rankCountTuples + count), (int)(difftime(time(0), rankStartTime)), rankPerSecond, rank, ((float)(rankTotalTuples - (rankCountTuples + count)))/rankPerSecond); + fprintf(stderr, " Done %i in %i @ %f per second - Rank %i ETA (seconds): %f\n", (rankCountTuples + count), (int)(difftime(time(0), rankStartTime)), rankPerSecond, rank, ((float)(rankTotalTuples - (rankCountTuples + count)))/rankPerSecond); sleepcount = 0; } } @@ -284,13 +290,17 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co // Finished sector rankPerSecond = (float)rankCountTuples / MAX(difftime(time(0), rankStartTime),1); - printf(" Done %i in %i @ %f per second - ETA (seconds): %f\n", rankCountTuples, (int)(difftime(time(0), rankStartTime)), rankPerSecond, ((float)(rankTotalTuples - rankCountTuples))/rankPerSecond); + fprintf(stderr, " Done %i in %i @ %f per second - ETA (seconds): %f\n", rankCountTuples, (int)(difftime(time(0), rankStartTime)), rankPerSecond, ((float)(rankTotalTuples - rankCountTuples))/rankPerSecond); PQclear(resPlaces); } + if (rankTotalTuples-rankCountTuples < num_threads*20 && iSector < PQntuples(resSectors)) + { + iSector = PQntuples(resSectors) - 1; + } } // Finished rank - printf("\r Done %i in %i @ %f per second - FINISHED \n\n", rankCountTuples, (int)(difftime(time(0), rankStartTime)), rankPerSecond); + fprintf(stderr, "\r Done %i in %i @ %f per second - FINISHED \n\n", rankCountTuples, (int)(difftime(time(0), rankStartTime)), rankPerSecond); PQclear(resSectors); } @@ -304,6 +314,7 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co void *nominatim_indexThread(void * thread_data_in) { struct index_thread_data * thread_data = (struct index_thread_data * )thread_data_in; + struct export_data querySet; PGresult *res; @@ -328,26 +339,50 @@ void *nominatim_indexThread(void * thread_data_in) pthread_mutex_unlock( thread_data->count_mutex ); - if (verbose) printf(" Processing place_id %d\n", place_id); + if (verbose) fprintf(stderr, " Processing place_id %d\n", place_id); updateStartTime = time(0); - paramPlaceID = PGint32(place_id); - paramValues[0] = (char *)¶mPlaceID; - paramLengths[0] = sizeof(paramPlaceID); - paramFormats[0] = 1; - res = PQexecPrepared(thread_data->conn, "index_placex", 1, paramValues, paramLengths, paramFormats, 1); - if (PQresultStatus(res) != PGRES_COMMAND_OK) + int done = 0; + + if (thread_data->writer) { - fprintf(stderr, "index_placex: UPDATE failed: %s", PQerrorMessage(thread_data->conn)); - PQclear(res); - exit(EXIT_FAILURE); + nominatim_exportPlaceQueries(place_id, thread_data->conn, &querySet); + } + + while(!done) + { + + paramPlaceID = PGint32(place_id); + paramValues[0] = (char *)¶mPlaceID; + paramLengths[0] = sizeof(paramPlaceID); + paramFormats[0] = 1; + res = PQexecPrepared(thread_data->conn, "index_placex", 1, paramValues, paramLengths, paramFormats, 1); + if (PQresultStatus(res) == PGRES_COMMAND_OK) + done = 1; + else + { + if (strncmp(PQerrorMessage(thread_data->conn), "ERROR: deadlock detected", 25)) + { + fprintf(stderr, "index_placex: UPDATE failed - deadlock, retrying\n"); + PQclear(res); + sleep(rand() % 10); + } + else + { + fprintf(stderr, "index_placex: UPDATE failed: %s", PQerrorMessage(thread_data->conn)); + PQclear(res); + sleep(rand() % 10); +// exit(EXIT_FAILURE); + } + } } PQclear(res); - if (difftime(time(0), updateStartTime) > 1) printf(" Slow place_id %d\n", place_id); + if (difftime(time(0), updateStartTime) > 1) fprintf(stderr, " Slow place_id %d\n", place_id); if (thread_data->writer) { - nominatim_exportPlace(place_id, thread_data->conn, thread_data->writer, thread_data->writer_mutex); + nominatim_exportPlace(place_id, thread_data->conn, thread_data->writer, thread_data->writer_mutex, &querySet); + nominatim_exportFreeQueries(&querySet); } }