15 #include "nominatim.h"
17 #include "postgresql.h"
23 void nominatim_export(int rank_min, int rank_max, const char *conninfo, const char *structuredoutputfile)
25 xmlTextWriterPtr writer;
31 PGresult * resSectors;
39 const char *paramValues[2];
46 Oid pg_prepare_params[2];
48 conn = PQconnectdb(conninfo);
49 if (PQstatus(conn) != CONNECTION_OK)
51 fprintf(stderr, "Connection to database failed: %s\n", PQerrorMessage(conn));
55 pg_prepare_params[0] = PG_OID_INT4;
56 res = PQprepare(conn, "index_sectors",
57 "select geometry_sector,count(*) from placex where rank_search = $1 and indexed_status = 0 group by geometry_sector order by geometry_sector",
58 1, pg_prepare_params);
59 if (PQresultStatus(res) != PGRES_COMMAND_OK) exit(EXIT_FAILURE);
62 pg_prepare_params[0] = PG_OID_INT4;
63 pg_prepare_params[1] = PG_OID_INT4;
64 res = PQprepare(conn, "index_sector_places",
65 "select place_id from placex where rank_search = $1 and geometry_sector = $2",
66 2, pg_prepare_params);
67 if (PQresultStatus(res) != PGRES_COMMAND_OK) exit(EXIT_FAILURE);
70 nominatim_exportCreatePreparedQueries(conn);
72 // Create the output file
73 writer = nominatim_exportXMLStart(structuredoutputfile);
75 for (rank = rank_min; rank <= rank_max; rank++)
77 printf("Starting rank %d\n", rank);
79 paramRank = PGint32(rank);
80 paramValues[0] = (char *)¶mRank;
81 paramLengths[0] = sizeof(paramRank);
83 resSectors = PQexecPrepared(conn, "index_sectors", 1, paramValues, paramLengths, paramFormats, 1);
84 if (PQresultStatus(resSectors) != PGRES_TUPLES_OK)
86 fprintf(stderr, "index_sectors: SELECT failed: %s", PQerrorMessage(conn));
90 if (PQftype(resSectors, 0) != PG_OID_INT4)
92 fprintf(stderr, "Sector value has unexpected type\n");
96 if (PQftype(resSectors, 1) != PG_OID_INT8)
98 fprintf(stderr, "Sector value has unexpected type\n");
104 for (iSector = 0; iSector < PQntuples(resSectors); iSector++)
106 sector = PGint32(*((uint32_t *)PQgetvalue(resSectors, iSector, 0)));
108 // Get all the place_id's for this sector
109 paramRank = PGint32(rank);
110 paramValues[0] = (char *)¶mRank;
111 paramLengths[0] = sizeof(paramRank);
113 paramSector = PGint32(sector);
114 paramValues[1] = (char *)¶mSector;
115 paramLengths[1] = sizeof(paramSector);
117 resPlaces = PQexecPrepared(conn, "index_sector_places", 2, paramValues, paramLengths, paramFormats, 1);
118 if (PQresultStatus(resPlaces) != PGRES_TUPLES_OK)
120 fprintf(stderr, "index_sector_places: SELECT failed: %s", PQerrorMessage(conn));
124 if (PQftype(resPlaces, 0) != PG_OID_INT4)
126 fprintf(stderr, "Place_id value has unexpected type\n");
131 tuples = PQntuples(resPlaces);
132 for (i = 0; i < tuples; i++)
134 nominatim_exportPlace(PGint32(*((uint32_t *)PQgetvalue(resPlaces, i, 0))), conn, writer, NULL, NULL);
136 if (rankTotalDone%1000 == 0) printf("Done %i (k)\n", rankTotalDone/1000);
143 nominatim_exportXMLEnd(writer);
148 void nominatim_exportCreatePreparedQueries(PGconn * conn)
150 Oid pg_prepare_params[2];
153 pg_prepare_params[0] = PG_OID_INT8;
154 res = PQprepare(conn, "placex_details",
155 "select placex.osm_type, placex.osm_id, placex.class, placex.type, placex.name, placex.housenumber, placex.country_code, ST_AsText(placex.geometry), placex.admin_level, placex.rank_address, placex.rank_search, placex.parent_place_id, parent.osm_type, parent.osm_id, placex.indexed_status from placex left outer join placex as parent on (placex.parent_place_id = parent.place_id) where placex.place_id = $1",
156 1, pg_prepare_params);
157 if (PQresultStatus(res) != PGRES_COMMAND_OK)
159 fprintf(stderr, "Error preparing placex_details: %s", PQerrorMessage(conn));
164 pg_prepare_params[0] = PG_OID_INT8;
165 res = PQprepare(conn, "placex_address",
166 "select osm_type,osm_id,class,type,distance,cached_rank_address,isaddress from place_addressline join placex on (address_place_id = placex.place_id) where place_addressline.place_id = $1 and address_place_id != place_addressline.place_id order by cached_rank_address asc,osm_type,osm_id",
167 1, pg_prepare_params);
168 if (PQresultStatus(res) != PGRES_COMMAND_OK)
170 fprintf(stderr, "Error preparing placex_address: %s", PQerrorMessage(conn));
175 pg_prepare_params[0] = PG_OID_INT8;
176 res = PQprepare(conn, "placex_names",
177 "select (each(name)).key,(each(name)).value from (select name from placex where place_id = $1) as x order by (each(name)).key",
178 1, pg_prepare_params);
179 if (PQresultStatus(res) != PGRES_COMMAND_OK)
181 fprintf(stderr, "Error preparing placex_names: %s", PQerrorMessage(conn));
186 pg_prepare_params[0] = PG_OID_INT8;
187 res = PQprepare(conn, "placex_extratags",
188 "select (each(extratags)).key,(each(extratags)).value from (select extratags from placex where place_id = $1) as x order by (each(extratags)).key",
189 1, pg_prepare_params);
190 if (PQresultStatus(res) != PGRES_COMMAND_OK)
192 fprintf(stderr, "Error preparing placex_extratags: %s", PQerrorMessage(conn));
198 xmlTextWriterPtr nominatim_exportXMLStart(const char *structuredoutputfile)
200 xmlTextWriterPtr writer;
202 writer = xmlNewTextWriterFilename(structuredoutputfile, 0);
205 fprintf(stderr, "Unable to open %s\n", structuredoutputfile);
208 xmlTextWriterSetIndent(writer, 1);
209 if (xmlTextWriterStartDocument(writer, NULL, "UTF8", NULL) < 0)
211 fprintf(stderr, "xmlTextWriterStartDocument failed\n");
214 if (xmlTextWriterStartElement(writer, BAD_CAST "osmStructured") < 0)
216 fprintf(stderr, "xmlTextWriterStartElement failed\n");
219 if (xmlTextWriterWriteAttribute(writer, BAD_CAST "version", BAD_CAST "0.1") < 0)
221 fprintf(stderr, "xmlTextWriterWriteAttribute failed\n");
224 if (xmlTextWriterWriteAttribute(writer, BAD_CAST "generator", BAD_CAST "Nominatim") < 0)
226 fprintf(stderr, "xmlTextWriterWriteAttribute failed\n");
235 void nominatim_exportXMLEnd(xmlTextWriterPtr writer)
237 nominatim_exportEndMode(writer);
239 // End <osmStructured>
240 if (xmlTextWriterEndElement(writer) < 0)
242 fprintf(stderr, "xmlTextWriterEndElement failed\n");
245 if (xmlTextWriterEndDocument(writer) < 0)
247 fprintf(stderr, "xmlTextWriterEndDocument failed\n");
250 xmlFreeTextWriter(writer);
253 void nominatim_exportStartMode(xmlTextWriterPtr writer, int newMode)
255 if (mode == newMode) return;
257 nominatim_exportEndMode(writer);
265 if (xmlTextWriterStartElement(writer, BAD_CAST "add") < 0)
267 fprintf(stderr, "xmlTextWriterStartElement failed\n");
273 if (xmlTextWriterStartElement(writer, BAD_CAST "update") < 0)
275 fprintf(stderr, "xmlTextWriterStartElement failed\n");
281 if (xmlTextWriterStartElement(writer, BAD_CAST "delete") < 0)
283 fprintf(stderr, "xmlTextWriterStartElement failed\n");
291 void nominatim_exportEndMode(xmlTextWriterPtr writer)
295 if (xmlTextWriterEndElement(writer) < 0)
297 fprintf(stderr, "xmlTextWriterEndElement failed\n");
302 void nominatim_exportPlaceQueries(uint64_t place_id, PGconn * conn, struct export_data * querySet)
304 const char * paramValues[1];
307 uint64_t paramPlaceID;
309 paramPlaceID = PGint64(place_id);
310 paramValues[0] = (char *)¶mPlaceID;
311 paramLengths[0] = sizeof(paramPlaceID);
314 querySet->res = PQexecPrepared(conn, "placex_details", 1, paramValues, paramLengths, paramFormats, 0);
315 if (PQresultStatus(querySet->res) != PGRES_TUPLES_OK)
317 fprintf(stderr, "placex_details: SELECT failed: %s", PQerrorMessage(conn));
318 PQclear(querySet->res);
322 querySet->resNames = PQexecPrepared(conn, "placex_names", 1, paramValues, paramLengths, paramFormats, 0);
323 if (PQresultStatus(querySet->resNames) != PGRES_TUPLES_OK)
325 fprintf(stderr, "placex_names: SELECT failed: %s", PQerrorMessage(conn));
326 PQclear(querySet->resNames);
330 querySet->resAddress = PQexecPrepared(conn, "placex_address", 1, paramValues, paramLengths, paramFormats, 0);
331 if (PQresultStatus(querySet->resAddress) != PGRES_TUPLES_OK)
333 fprintf(stderr, "placex_address: SELECT failed: %s", PQerrorMessage(conn));
334 PQclear(querySet->resAddress);
338 querySet->resExtraTags = PQexecPrepared(conn, "placex_extratags", 1, paramValues, paramLengths, paramFormats, 0);
339 if (PQresultStatus(querySet->resExtraTags) != PGRES_TUPLES_OK)
341 fprintf(stderr, "placex_extratags: SELECT failed: %s", PQerrorMessage(conn));
342 PQclear(querySet->resExtraTags);
347 void nominatim_exportFreeQueries(struct export_data * querySet)
349 PQclear(querySet->res);
350 PQclear(querySet->resNames);
351 PQclear(querySet->resAddress);
352 PQclear(querySet->resExtraTags);
356 * Requirements: the prepared queries must exist
358 void nominatim_exportPlace(uint64_t place_id, PGconn * conn,
359 xmlTextWriterPtr writer, pthread_mutex_t * writer_mutex, struct export_data * prevQuerySet)
361 struct export_data querySet;
365 nominatim_exportPlaceQueries(place_id, conn, &querySet);
367 // Add, modify or delete?
370 if ((PQgetvalue(prevQuerySet->res, 0, 14) && strcmp(PQgetvalue(prevQuerySet->res, 0, 14), "100") == 0) || PQntuples(querySet.res) == 0)
373 if (writer_mutex) pthread_mutex_lock( writer_mutex );
374 nominatim_exportStartMode(writer, 3);
375 xmlTextWriterStartElement(writer, BAD_CAST "feature");
376 xmlTextWriterWriteFormatAttribute(writer, BAD_CAST "place_id", "%li", place_id);
377 xmlTextWriterWriteAttribute(writer, BAD_CAST "type", BAD_CAST PQgetvalue(prevQuerySet->res, 0, 0));
378 xmlTextWriterWriteAttribute(writer, BAD_CAST "id", BAD_CAST PQgetvalue(prevQuerySet->res, 0, 1));
379 xmlTextWriterWriteAttribute(writer, BAD_CAST "key", BAD_CAST PQgetvalue(prevQuerySet->res, 0, 2));
380 xmlTextWriterWriteAttribute(writer, BAD_CAST "value", BAD_CAST PQgetvalue(prevQuerySet->res, 0, 3));
381 xmlTextWriterEndElement(writer);
382 if (writer_mutex) pthread_mutex_unlock( writer_mutex );
383 nominatim_exportFreeQueries(&querySet);
386 if (PQgetvalue(prevQuerySet->res, 0, 14) && strcmp(PQgetvalue(prevQuerySet->res, 0, 14), "1") == 0)
389 if (writer_mutex) pthread_mutex_lock( writer_mutex );
390 nominatim_exportStartMode(writer, 1);
394 // Update, but only if something has changed
396 // TODO: detect changes
398 if (writer_mutex) pthread_mutex_lock( writer_mutex );
399 nominatim_exportStartMode(writer, 2);
405 if (writer_mutex) pthread_mutex_lock( writer_mutex );
406 nominatim_exportStartMode(writer, 1);
409 xmlTextWriterStartElement(writer, BAD_CAST "feature");
410 xmlTextWriterWriteFormatAttribute(writer, BAD_CAST "place_id", "%li", place_id);
411 xmlTextWriterWriteAttribute(writer, BAD_CAST "type", BAD_CAST PQgetvalue(querySet.res, 0, 0));
412 xmlTextWriterWriteAttribute(writer, BAD_CAST "id", BAD_CAST PQgetvalue(querySet.res, 0, 1));
413 xmlTextWriterWriteAttribute(writer, BAD_CAST "key", BAD_CAST PQgetvalue(querySet.res, 0, 2));
414 xmlTextWriterWriteAttribute(writer, BAD_CAST "value", BAD_CAST PQgetvalue(querySet.res, 0, 3));
415 xmlTextWriterWriteAttribute(writer, BAD_CAST "rank", BAD_CAST PQgetvalue(querySet.res, 0, 9));
416 xmlTextWriterWriteAttribute(writer, BAD_CAST "importance", BAD_CAST PQgetvalue(querySet.res, 0, 10));
417 xmlTextWriterWriteAttribute(writer, BAD_CAST "parent_place_id", BAD_CAST PQgetvalue(querySet.res, 0, 11));
418 xmlTextWriterWriteAttribute(writer, BAD_CAST "parent_type", BAD_CAST PQgetvalue(querySet.res, 0, 12));
419 xmlTextWriterWriteAttribute(writer, BAD_CAST "parent_id", BAD_CAST PQgetvalue(querySet.res, 0, 13));
421 if (PQntuples(querySet.resNames))
423 xmlTextWriterStartElement(writer, BAD_CAST "names");
425 for (i = 0; i < PQntuples(querySet.resNames); i++)
427 xmlTextWriterStartElement(writer, BAD_CAST "name");
428 xmlTextWriterWriteAttribute(writer, BAD_CAST "type", BAD_CAST PQgetvalue(querySet.resNames, i, 0));
429 xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(querySet.resNames, i, 1));
430 xmlTextWriterEndElement(writer);
433 xmlTextWriterEndElement(writer);
436 if (PQgetvalue(querySet.res, 0, 5) && strlen(PQgetvalue(querySet.res, 0, 5)))
438 xmlTextWriterStartElement(writer, BAD_CAST "houseNumber");
439 xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(querySet.res, 0, 5));
440 xmlTextWriterEndElement(writer);
443 if (PQgetvalue(querySet.res, 0, 8) && strlen(PQgetvalue(querySet.res, 0, 8)))
445 xmlTextWriterStartElement(writer, BAD_CAST "adminLevel");
446 xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(querySet.res, 0, 8));
447 xmlTextWriterEndElement(writer);
450 if (PQgetvalue(querySet.res, 0, 6) && strlen(PQgetvalue(querySet.res, 0, 6)))
452 xmlTextWriterStartElement(writer, BAD_CAST "countryCode");
453 xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(querySet.res, 0, 6));
454 xmlTextWriterEndElement(writer);
457 if (PQntuples(querySet.resAddress) > 0)
459 xmlTextWriterStartElement(writer, BAD_CAST "address");
460 for (i = 0; i < PQntuples(querySet.resAddress); i++)
462 xmlTextWriterStartElement(writer, BAD_CAST getRankLabel(atoi(PQgetvalue(querySet.resAddress, i, 5))));
463 xmlTextWriterWriteAttribute(writer, BAD_CAST "rank", BAD_CAST PQgetvalue(querySet.resAddress, i, 5));
464 xmlTextWriterWriteAttribute(writer, BAD_CAST "type", BAD_CAST PQgetvalue(querySet.resAddress, i, 0));
465 xmlTextWriterWriteAttribute(writer, BAD_CAST "id", BAD_CAST PQgetvalue(querySet.resAddress, i, 1));
466 xmlTextWriterWriteAttribute(writer, BAD_CAST "key", BAD_CAST PQgetvalue(querySet.resAddress, i, 2));
467 xmlTextWriterWriteAttribute(writer, BAD_CAST "value", BAD_CAST PQgetvalue(querySet.resAddress, i, 3));
468 xmlTextWriterWriteAttribute(writer, BAD_CAST "distance", BAD_CAST PQgetvalue(querySet.resAddress, i, 4));
469 xmlTextWriterWriteAttribute(writer, BAD_CAST "isaddress", BAD_CAST PQgetvalue(querySet.resAddress, i, 6));
470 xmlTextWriterEndElement(writer);
472 xmlTextWriterEndElement(writer);
475 if (PQntuples(querySet.resExtraTags))
477 xmlTextWriterStartElement(writer, BAD_CAST "tags");
479 for (i = 0; i < PQntuples(querySet.resExtraTags); i++)
481 xmlTextWriterStartElement(writer, BAD_CAST "tag");
482 xmlTextWriterWriteAttribute(writer, BAD_CAST "type", BAD_CAST PQgetvalue(querySet.resExtraTags, i, 0));
483 xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(querySet.resExtraTags, i, 1));
484 xmlTextWriterEndElement(writer);
487 xmlTextWriterEndElement(writer);
491 xmlTextWriterStartElement(writer, BAD_CAST "osmGeometry");
492 xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(querySet.res, 0, 7));
493 xmlTextWriterEndElement(writer);
495 xmlTextWriterEndElement(writer); // </feature>
497 if (writer_mutex) pthread_mutex_unlock( writer_mutex );
499 nominatim_exportFreeQueries(&querySet);
502 const char * getRankLabel(int rank)
540 return "neighborhood";