exit(EXIT_FAILURE);
}
- pg_prepare_params[0] = PG_OID_INT4;
+ pg_prepare_params[0] = PG_OID_INT8;
res = PQprepare(thread_data[i].conn, "index_placex",
"update placex set indexed_status = 0 where place_id = $1",
1, pg_prepare_params);
}
PQclear(res);
- res = PQexec(thread_data[i].conn, "set enable_seqscan = false");
+ /*res = PQexec(thread_data[i].conn, "set enable_seqscan = false");
if (PQresultStatus(res) != PGRES_COMMAND_OK)
{
fprintf(stderr, "Failed disabling sequential scan: %s\n", PQerrorMessage(conn));
exit(EXIT_FAILURE);
}
- PQclear(res);
+ PQclear(res);*/
nominatim_exportCreatePreparedQueries(thread_data[i].conn);
}
writer = nominatim_exportXMLStart(structuredoutputfile);
}
- fprintf(stderr, "Starting indexing rank (%i to %i) using %i treads\n", rank_min, rank_max, num_threads);
+ fprintf(stderr, "Starting indexing rank (%i to %i) using %i threads\n", rank_min, rank_max, num_threads);
for (rank = rank_min; rank <= rank_max; rank++)
{
- printf("Starting rank %d\n", rank);
+ fprintf(stderr, "Starting rank %d\n", rank);
rankCountTuples = 0;
rankPerSecond = 0;
paramValues[0] = (char *)¶mRank;
paramLengths[0] = sizeof(paramRank);
paramFormats[0] = 1;
- if (rank < 16)
- resSectors = PQexecPrepared(conn, "index_nosectors", 1, paramValues, paramLengths, paramFormats, 1);
- else
- resSectors = PQexecPrepared(conn, "index_sectors", 1, paramValues, paramLengths, paramFormats, 1);
+// if (rank < 16)
+// resSectors = PQexecPrepared(conn, "index_nosectors", 1, paramValues, paramLengths, paramFormats, 1);
+// else
+ resSectors = PQexecPrepared(conn, "index_sectors", 1, paramValues, paramLengths, paramFormats, 1);
+
if (PQresultStatus(resSectors) != PGRES_TUPLES_OK)
{
fprintf(stderr, "index_sectors: SELECT failed: %s", PQerrorMessage(conn));
PQclear(resPlaces);
exit(EXIT_FAILURE);
}
- if (PQftype(resPlaces, 0) != PG_OID_INT4)
+ if (PQftype(resPlaces, 0) != PG_OID_INT8)
{
fprintf(stderr, "Place_id value has unexpected type\n");
PQclear(resPlaces);
if (iSector < PQntuples(resSectors))
{
sector = PGint32(*((uint32_t *)PQgetvalue(resSectors, iSector, 0)));
- //printf("\n Starting sector %d size %ld\n", sector, PGint64(*((uint64_t *)PQgetvalue(resSectors, iSector, 1))));
+// fprintf(stderr, "\n Starting sector %d size %ld\n", sector, PGint64(*((uint64_t *)PQgetvalue(resSectors, iSector, 1))));
// Get all the place_id's for this sector
paramRank = PGint32(rank);
paramValues[1] = (char *)¶mSector;
paramLengths[1] = sizeof(paramSector);
paramFormats[1] = 1;
- if (rank < 16)
+ if (rankTotalTuples-rankCountTuples < num_threads*1000)
+ {
iResult = PQsendQueryPrepared(conn, "index_nosector_places", 1, paramValues, paramLengths, paramFormats, 1);
+ }
else
+ {
iResult = PQsendQueryPrepared(conn, "index_sector_places", 2, paramValues, paramLengths, paramFormats, 1);
+ }
if (!iResult)
{
fprintf(stderr, "index_sector_places: SELECT failed: %s", PQerrorMessage(conn));
if (sleepcount++ > 500)
{
rankPerSecond = ((float)rankCountTuples + (float)count) / MAX(difftime(time(0), rankStartTime),1);
- printf(" Done %i in %i @ %f per second - Rank %i ETA (seconds): %f\n", (rankCountTuples + count), (int)(difftime(time(0), rankStartTime)), rankPerSecond, rank, ((float)(rankTotalTuples - (rankCountTuples + count)))/rankPerSecond);
+ fprintf(stderr, " Done %i in %i @ %f per second - Rank %i ETA (seconds): %f\n", (rankCountTuples + count), (int)(difftime(time(0), rankStartTime)), rankPerSecond, rank, ((float)(rankTotalTuples - (rankCountTuples + count)))/rankPerSecond);
sleepcount = 0;
}
}
// Finished sector
rankPerSecond = (float)rankCountTuples / MAX(difftime(time(0), rankStartTime),1);
- printf(" Done %i in %i @ %f per second - ETA (seconds): %f\n", rankCountTuples, (int)(difftime(time(0), rankStartTime)), rankPerSecond, ((float)(rankTotalTuples - rankCountTuples))/rankPerSecond);
+ fprintf(stderr, " Done %i in %i @ %f per second - ETA (seconds): %f\n", rankCountTuples, (int)(difftime(time(0), rankStartTime)), rankPerSecond, ((float)(rankTotalTuples - rankCountTuples))/rankPerSecond);
PQclear(resPlaces);
}
+ if (rankTotalTuples-rankCountTuples < num_threads*20 && iSector < PQntuples(resSectors))
+ {
+ iSector = PQntuples(resSectors) - 1;
+ }
}
// Finished rank
- printf("\r Done %i in %i @ %f per second - FINISHED \n\n", rankCountTuples, (int)(difftime(time(0), rankStartTime)), rankPerSecond);
+ fprintf(stderr, "\r Done %i in %i @ %f per second - FINISHED \n\n", rankCountTuples, (int)(difftime(time(0), rankStartTime)), rankPerSecond);
PQclear(resSectors);
}
{
nominatim_exportXMLEnd(writer);
}
+
+ // Close all connections
+ for (i = 0; i < num_threads; i++)
+ {
+ PQfinish(thread_data[i].conn);
+ }
+ PQfinish(conn);
}
void *nominatim_indexThread(void * thread_data_in)
{
struct index_thread_data * thread_data = (struct index_thread_data * )thread_data_in;
+ struct export_data querySet;
PGresult *res;
const char *paramValues[1];
int paramLengths[1];
int paramFormats[1];
- uint32_t paramPlaceID;
- uint32_t place_id;
+ uint64_t paramPlaceID;
+ uint64_t place_id;
time_t updateStartTime;
while (1)
break;
}
- place_id = PGint32(*((uint32_t *)PQgetvalue(thread_data->res, *thread_data->count, 0)));
+ place_id = PGint64(*((uint64_t *)PQgetvalue(thread_data->res, *thread_data->count, 0)));
(*thread_data->count)++;
pthread_mutex_unlock( thread_data->count_mutex );
- if (verbose) printf(" Processing place_id %d\n", place_id);
+ if (verbose) fprintf(stderr, " Processing place_id %ld\n", place_id);
updateStartTime = time(0);
- paramPlaceID = PGint32(place_id);
- paramValues[0] = (char *)¶mPlaceID;
- paramLengths[0] = sizeof(paramPlaceID);
- paramFormats[0] = 1;
- res = PQexecPrepared(thread_data->conn, "index_placex", 1, paramValues, paramLengths, paramFormats, 1);
- if (PQresultStatus(res) != PGRES_COMMAND_OK)
+ int done = 0;
+
+ if (thread_data->writer)
{
- fprintf(stderr, "index_placex: UPDATE failed: %s", PQerrorMessage(thread_data->conn));
- PQclear(res);
- exit(EXIT_FAILURE);
+ nominatim_exportPlaceQueries(place_id, thread_data->conn, &querySet);
+ }
+
+ while(!done)
+ {
+ paramPlaceID = PGint64(place_id);
+ paramValues[0] = (char *)¶mPlaceID;
+ paramLengths[0] = sizeof(paramPlaceID);
+ paramFormats[0] = 1;
+ res = PQexecPrepared(thread_data->conn, "index_placex", 1, paramValues, paramLengths, paramFormats, 1);
+ if (PQresultStatus(res) == PGRES_COMMAND_OK)
+ done = 1;
+ else
+ {
+ if (!strncmp(PQerrorMessage(thread_data->conn), "ERROR: deadlock detected", 25))
+ {
+ fprintf(stderr, "index_placex: UPDATE failed - deadlock, retrying (%ld)\n", place_id);
+ PQclear(res);
+ sleep(rand() % 10);
+ }
+ else
+ {
+ fprintf(stderr, "index_placex: UPDATE failed: %s", PQerrorMessage(thread_data->conn));
+ PQclear(res);
+ exit(EXIT_FAILURE);
+ }
+ }
}
PQclear(res);
- if (difftime(time(0), updateStartTime) > 1) printf(" Slow place_id %d\n", place_id);
+ if (difftime(time(0), updateStartTime) > 1) fprintf(stderr, " Slow place_id %ld\n", place_id);
if (thread_data->writer)
{
- nominatim_exportPlace(place_id, thread_data->conn, thread_data->writer, thread_data->writer_mutex);
+ nominatim_exportPlace(place_id, thread_data->conn, thread_data->writer, thread_data->writer_mutex, &querySet);
+ nominatim_exportFreeQueries(&querySet);
}
}
projects / nominatim.git / blobdiff