else
{
if (is_bool($xVal)) return $xVal?'true':'false';
- if (is_numeric($xVal)) return $xVal;
+// if (is_numeric($xVal)) return $xVal;
return '"'.str_replace('>','\\>',str_replace(array("\n","\r"),'\\n',str_replace(array("\n\r","\r\n"),'\\n',str_replace('"','\\"',$xVal)))).'"';
}
}
extern int verbose;
+int mode = 0;
+
void nominatim_export(int rank_min, int rank_max, const char *conninfo, const char *structuredoutputfile)
{
xmlTextWriterPtr writer;
tuples = PQntuples(resPlaces);
for (i = 0; i < tuples; i++)
{
- nominatim_exportPlace(PGint32(*((uint32_t *)PQgetvalue(resPlaces, i, 0))), conn, writer, NULL);
+ nominatim_exportPlace(PGint32(*((uint32_t *)PQgetvalue(resPlaces, i, 0))), conn, writer, NULL, NULL);
rankTotalDone++;
if (rankTotalDone%1000 == 0) printf("Done %i (k)\n", rankTotalDone/1000);
}
pg_prepare_params[0] = PG_OID_INT8;
res = PQprepare(conn, "placex_details",
- "select osm_type, osm_id, class, type, name, housenumber, country_code, ST_AsText(geometry), admin_level, rank_address, rank_search from placex where place_id = $1",
+ "select placex.osm_type, placex.osm_id, placex.class, placex.type, placex.name, placex.housenumber, placex.country_code, ST_AsText(placex.geometry), placex.admin_level, placex.rank_address, placex.rank_search, placex.parent_place_id, parent.osm_type, parent.osm_id, placex.indexed_status from placex left outer join placex as parent on (placex.parent_place_id = parent.place_id) where placex.place_id = $1",
1, pg_prepare_params);
if (PQresultStatus(res) != PGRES_COMMAND_OK)
{
fprintf(stderr, "xmlTextWriterWriteAttribute failed\n");
exit(EXIT_FAILURE);
}
- if (xmlTextWriterStartElement(writer, BAD_CAST "add") < 0)
- {
- fprintf(stderr, "xmlTextWriterStartElement failed\n");
- exit(EXIT_FAILURE);
- }
+
+ mode = 0;
return writer;
}
void nominatim_exportXMLEnd(xmlTextWriterPtr writer)
{
- // End <add>
- if (xmlTextWriterEndElement(writer) < 0)
- {
- fprintf(stderr, "xmlTextWriterEndElement failed\n");
- exit(EXIT_FAILURE);
- }
+ nominatim_exportEndMode(writer);
+
// End <osmStructured>
if (xmlTextWriterEndElement(writer) < 0)
{
xmlFreeTextWriter(writer);
}
-/*
- * Requirements: the prepared queries must exist
- */
-void nominatim_exportPlace(uint64_t place_id, PGconn * conn, xmlTextWriterPtr writer, pthread_mutex_t * writer_mutex)
+void nominatim_exportStartMode(xmlTextWriterPtr writer, int newMode)
{
- PGresult * res;
- PGresult * resNames;
- PGresult * resAddress;
- PGresult * resExtraTags;
+ if (mode == newMode) return;
- int i;
+ nominatim_exportEndMode(writer);
+
+ switch(newMode)
+ {
+ case 0:
+ break;
+ case 1:
+ if (xmlTextWriterStartElement(writer, BAD_CAST "add") < 0)
+ {
+ fprintf(stderr, "xmlTextWriterStartElement failed\n");
+ exit(EXIT_FAILURE);
+ }
+ break;
+
+ case 2:
+ if (xmlTextWriterStartElement(writer, BAD_CAST "update") < 0)
+ {
+ fprintf(stderr, "xmlTextWriterStartElement failed\n");
+ exit(EXIT_FAILURE);
+ }
+ break;
+
+ case 3:
+ if (xmlTextWriterStartElement(writer, BAD_CAST "delete") < 0)
+ {
+ fprintf(stderr, "xmlTextWriterStartElement failed\n");
+ exit(EXIT_FAILURE);
+ }
+ break;
+ }
+ mode = newMode;
+}
+
+void nominatim_exportEndMode(xmlTextWriterPtr writer)
+{
+ if (!mode) return;
+
+ if (xmlTextWriterEndElement(writer) < 0)
+ {
+ fprintf(stderr, "xmlTextWriterEndElement failed\n");
+ exit(EXIT_FAILURE);
+ }
+}
+
+void nominatim_exportPlaceQueries(uint64_t place_id, PGconn * conn, struct export_data * querySet)
+{
const char * paramValues[1];
int paramLengths[1];
int paramFormats[1];
uint64_t paramPlaceID;
-
paramPlaceID = PGint64(place_id);
paramValues[0] = (char *)¶mPlaceID;
paramLengths[0] = sizeof(paramPlaceID);
paramFormats[0] = 1;
- res = PQexecPrepared(conn, "placex_details", 1, paramValues, paramLengths, paramFormats, 0);
- if (PQresultStatus(res) != PGRES_TUPLES_OK)
+ querySet->res = PQexecPrepared(conn, "placex_details", 1, paramValues, paramLengths, paramFormats, 0);
+ if (PQresultStatus(querySet->res) != PGRES_TUPLES_OK)
{
fprintf(stderr, "placex_details: SELECT failed: %s", PQerrorMessage(conn));
- PQclear(res);
+ PQclear(querySet->res);
exit(EXIT_FAILURE);
}
- resNames = PQexecPrepared(conn, "placex_names", 1, paramValues, paramLengths, paramFormats, 0);
- if (PQresultStatus(resNames) != PGRES_TUPLES_OK)
+ querySet->resNames = PQexecPrepared(conn, "placex_names", 1, paramValues, paramLengths, paramFormats, 0);
+ if (PQresultStatus(querySet->resNames) != PGRES_TUPLES_OK)
{
fprintf(stderr, "placex_names: SELECT failed: %s", PQerrorMessage(conn));
- PQclear(resNames);
+ PQclear(querySet->resNames);
exit(EXIT_FAILURE);
}
- resAddress = PQexecPrepared(conn, "placex_address", 1, paramValues, paramLengths, paramFormats, 0);
- if (PQresultStatus(resAddress) != PGRES_TUPLES_OK)
+ querySet->resAddress = PQexecPrepared(conn, "placex_address", 1, paramValues, paramLengths, paramFormats, 0);
+ if (PQresultStatus(querySet->resAddress) != PGRES_TUPLES_OK)
{
fprintf(stderr, "placex_address: SELECT failed: %s", PQerrorMessage(conn));
- PQclear(resAddress);
+ PQclear(querySet->resAddress);
exit(EXIT_FAILURE);
}
- resExtraTags = PQexecPrepared(conn, "placex_extratags", 1, paramValues, paramLengths, paramFormats, 0);
- if (PQresultStatus(resExtraTags) != PGRES_TUPLES_OK)
+ querySet->resExtraTags = PQexecPrepared(conn, "placex_extratags", 1, paramValues, paramLengths, paramFormats, 0);
+ if (PQresultStatus(querySet->resExtraTags) != PGRES_TUPLES_OK)
{
fprintf(stderr, "placex_extratags: SELECT failed: %s", PQerrorMessage(conn));
- PQclear(resExtraTags);
+ PQclear(querySet->resExtraTags);
exit(EXIT_FAILURE);
}
+}
+
+void nominatim_exportFreeQueries(struct export_data * querySet)
+{
+ PQclear(querySet->res);
+ PQclear(querySet->resNames);
+ PQclear(querySet->resAddress);
+ PQclear(querySet->resExtraTags);
+}
+
+/*
+ * Requirements: the prepared queries must exist
+ */
+void nominatim_exportPlace(uint64_t place_id, PGconn * conn,
+ xmlTextWriterPtr writer, pthread_mutex_t * writer_mutex, struct export_data * prevQuerySet)
+{
+ struct export_data querySet;
+
+ int i;
+
+ nominatim_exportPlaceQueries(place_id, conn, &querySet);
+
+ // Add, modify or delete?
+ if (prevQuerySet)
+ {
+ if ((PQgetvalue(prevQuerySet->res, 0, 14) && strcmp(PQgetvalue(prevQuerySet->res, 0, 14), "100") == 0) || PQntuples(querySet.res))
+ {
+ // Delete
+ if (writer_mutex) pthread_mutex_lock( writer_mutex );
+ nominatim_exportStartMode(writer, 3);
+ xmlTextWriterStartElement(writer, BAD_CAST "feature");
+ xmlTextWriterWriteFormatAttribute(writer, BAD_CAST "place_id", "%li", place_id);
+ xmlTextWriterWriteAttribute(writer, BAD_CAST "type", BAD_CAST PQgetvalue(prevQuerySet->res, 0, 0));
+ xmlTextWriterWriteAttribute(writer, BAD_CAST "id", BAD_CAST PQgetvalue(prevQuerySet->res, 0, 1));
+ xmlTextWriterWriteAttribute(writer, BAD_CAST "key", BAD_CAST PQgetvalue(prevQuerySet->res, 0, 2));
+ xmlTextWriterWriteAttribute(writer, BAD_CAST "value", BAD_CAST PQgetvalue(prevQuerySet->res, 0, 3));
+ xmlTextWriterEndElement(writer);
+ if (writer_mutex) pthread_mutex_unlock( writer_mutex );
+ nominatim_exportFreeQueries(&querySet);
+ return;
+ }
+ if (PQgetvalue(prevQuerySet->res, 0, 14) && strcmp(PQgetvalue(prevQuerySet->res, 0, 14), "1") == 0)
+ {
+ // Add
+ if (writer_mutex) pthread_mutex_lock( writer_mutex );
+ nominatim_exportStartMode(writer, 1);
+ }
+ else
+ {
+ // Update, but only if something has changed
- if (writer_mutex) pthread_mutex_lock( writer_mutex );
+ // TODO: detect changes
+
+ if (writer_mutex) pthread_mutex_lock( writer_mutex );
+ nominatim_exportStartMode(writer, 2);
+ }
+ }
+ else
+ {
+ // Add
+ if (writer_mutex) pthread_mutex_lock( writer_mutex );
+ nominatim_exportStartMode(writer, 1);
+ }
xmlTextWriterStartElement(writer, BAD_CAST "feature");
xmlTextWriterWriteFormatAttribute(writer, BAD_CAST "place_id", "%li", place_id);
- xmlTextWriterWriteAttribute(writer, BAD_CAST "type", BAD_CAST PQgetvalue(res, 0, 0));
- xmlTextWriterWriteAttribute(writer, BAD_CAST "id", BAD_CAST PQgetvalue(res, 0, 1));
- xmlTextWriterWriteAttribute(writer, BAD_CAST "key", BAD_CAST PQgetvalue(res, 0, 2));
- xmlTextWriterWriteAttribute(writer, BAD_CAST "value", BAD_CAST PQgetvalue(res, 0, 3));
- xmlTextWriterWriteAttribute(writer, BAD_CAST "rank", BAD_CAST PQgetvalue(res, 0, 9));
- xmlTextWriterWriteAttribute(writer, BAD_CAST "importance", BAD_CAST PQgetvalue(res, 0, 10));
-
- if (PQntuples(resNames))
+ xmlTextWriterWriteAttribute(writer, BAD_CAST "type", BAD_CAST PQgetvalue(querySet.res, 0, 0));
+ xmlTextWriterWriteAttribute(writer, BAD_CAST "id", BAD_CAST PQgetvalue(querySet.res, 0, 1));
+ xmlTextWriterWriteAttribute(writer, BAD_CAST "key", BAD_CAST PQgetvalue(querySet.res, 0, 2));
+ xmlTextWriterWriteAttribute(writer, BAD_CAST "value", BAD_CAST PQgetvalue(querySet.res, 0, 3));
+ xmlTextWriterWriteAttribute(writer, BAD_CAST "rank", BAD_CAST PQgetvalue(querySet.res, 0, 9));
+ xmlTextWriterWriteAttribute(writer, BAD_CAST "importance", BAD_CAST PQgetvalue(querySet.res, 0, 10));
+ xmlTextWriterWriteAttribute(writer, BAD_CAST "parent_place_id", BAD_CAST PQgetvalue(querySet.res, 0, 11));
+ xmlTextWriterWriteAttribute(writer, BAD_CAST "parent_type", BAD_CAST PQgetvalue(querySet.res, 0, 12));
+ xmlTextWriterWriteAttribute(writer, BAD_CAST "parent_id", BAD_CAST PQgetvalue(querySet.res, 0, 13));
+
+ if (PQntuples(querySet.resNames))
{
xmlTextWriterStartElement(writer, BAD_CAST "names");
- for (i = 0; i < PQntuples(resNames); i++)
+ for (i = 0; i < PQntuples(querySet.resNames); i++)
{
xmlTextWriterStartElement(writer, BAD_CAST "name");
- xmlTextWriterWriteAttribute(writer, BAD_CAST "type", BAD_CAST PQgetvalue(resNames, i, 0));
- xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(resNames, i, 1));
+ xmlTextWriterWriteAttribute(writer, BAD_CAST "type", BAD_CAST PQgetvalue(querySet.resNames, i, 0));
+ xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(querySet.resNames, i, 1));
xmlTextWriterEndElement(writer);
}
xmlTextWriterEndElement(writer);
}
- if (PQgetvalue(res, 0, 5) && strlen(PQgetvalue(res, 0, 5)))
+ if (PQgetvalue(querySet.res, 0, 5) && strlen(PQgetvalue(querySet.res, 0, 5)))
{
xmlTextWriterStartElement(writer, BAD_CAST "houseNumber");
- xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(res, 0, 5));
+ xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(querySet.res, 0, 5));
xmlTextWriterEndElement(writer);
}
- if (PQgetvalue(res, 0, 8) && strlen(PQgetvalue(res, 0, 8)))
+ if (PQgetvalue(querySet.res, 0, 8) && strlen(PQgetvalue(querySet.res, 0, 8)))
{
xmlTextWriterStartElement(writer, BAD_CAST "adminLevel");
- xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(res, 0, 8));
+ xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(querySet.res, 0, 8));
xmlTextWriterEndElement(writer);
}
- if (PQgetvalue(res, 0, 6) && strlen(PQgetvalue(res, 0, 6)))
+ if (PQgetvalue(querySet.res, 0, 6) && strlen(PQgetvalue(querySet.res, 0, 6)))
{
xmlTextWriterStartElement(writer, BAD_CAST "countryCode");
- xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(res, 0, 6));
+ xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(querySet.res, 0, 6));
xmlTextWriterEndElement(writer);
}
- if (PQntuples(resAddress) > 0)
+ if (PQntuples(querySet.resAddress) > 0)
{
xmlTextWriterStartElement(writer, BAD_CAST "address");
- for (i = 0; i < PQntuples(resAddress); i++)
+ for (i = 0; i < PQntuples(querySet.resAddress); i++)
{
- xmlTextWriterStartElement(writer, BAD_CAST getRankLabel(atoi(PQgetvalue(resAddress, i, 5))));
- xmlTextWriterWriteAttribute(writer, BAD_CAST "rank", BAD_CAST PQgetvalue(resAddress, i, 5));
- xmlTextWriterWriteAttribute(writer, BAD_CAST "type", BAD_CAST PQgetvalue(resAddress, i, 0));
- xmlTextWriterWriteAttribute(writer, BAD_CAST "id", BAD_CAST PQgetvalue(resAddress, i, 1));
- xmlTextWriterWriteAttribute(writer, BAD_CAST "key", BAD_CAST PQgetvalue(resAddress, i, 2));
- xmlTextWriterWriteAttribute(writer, BAD_CAST "value", BAD_CAST PQgetvalue(resAddress, i, 3));
- xmlTextWriterWriteAttribute(writer, BAD_CAST "distance", BAD_CAST PQgetvalue(resAddress, i, 4));
- xmlTextWriterWriteAttribute(writer, BAD_CAST "isaddress", BAD_CAST PQgetvalue(resAddress, i, 6));
+ xmlTextWriterStartElement(writer, BAD_CAST getRankLabel(atoi(PQgetvalue(querySet.resAddress, i, 5))));
+ xmlTextWriterWriteAttribute(writer, BAD_CAST "rank", BAD_CAST PQgetvalue(querySet.resAddress, i, 5));
+ xmlTextWriterWriteAttribute(writer, BAD_CAST "type", BAD_CAST PQgetvalue(querySet.resAddress, i, 0));
+ xmlTextWriterWriteAttribute(writer, BAD_CAST "id", BAD_CAST PQgetvalue(querySet.resAddress, i, 1));
+ xmlTextWriterWriteAttribute(writer, BAD_CAST "key", BAD_CAST PQgetvalue(querySet.resAddress, i, 2));
+ xmlTextWriterWriteAttribute(writer, BAD_CAST "value", BAD_CAST PQgetvalue(querySet.resAddress, i, 3));
+ xmlTextWriterWriteAttribute(writer, BAD_CAST "distance", BAD_CAST PQgetvalue(querySet.resAddress, i, 4));
+ xmlTextWriterWriteAttribute(writer, BAD_CAST "isaddress", BAD_CAST PQgetvalue(querySet.resAddress, i, 6));
xmlTextWriterEndElement(writer);
}
xmlTextWriterEndElement(writer);
}
- if (PQntuples(resExtraTags))
+ if (PQntuples(querySet.resExtraTags))
{
xmlTextWriterStartElement(writer, BAD_CAST "tags");
- for (i = 0; i < PQntuples(resExtraTags); i++)
+ for (i = 0; i < PQntuples(querySet.resExtraTags); i++)
{
xmlTextWriterStartElement(writer, BAD_CAST "tag");
- xmlTextWriterWriteAttribute(writer, BAD_CAST "type", BAD_CAST PQgetvalue(resExtraTags, i, 0));
- xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(resExtraTags, i, 1));
+ xmlTextWriterWriteAttribute(writer, BAD_CAST "type", BAD_CAST PQgetvalue(querySet.resExtraTags, i, 0));
+ xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(querySet.resExtraTags, i, 1));
xmlTextWriterEndElement(writer);
}
xmlTextWriterStartElement(writer, BAD_CAST "osmGeometry");
- xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(res, 0, 7));
+ xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(querySet.res, 0, 7));
xmlTextWriterEndElement(writer);
xmlTextWriterEndElement(writer); // </feature>
if (writer_mutex) pthread_mutex_unlock( writer_mutex );
- PQclear(res);
- PQclear(resNames);
- PQclear(resAddress);
- PQclear(resExtraTags);
+ nominatim_exportFreeQueries(&querySet);
}
const char * getRankLabel(int rank)
#include <libxml/xmlwriter.h>
#include <stdint.h>
+struct export_data
+{
+ PGresult * res;
+ PGresult * resNames;
+ PGresult * resAddress;
+ PGresult * resExtraTags;
+};
+
void nominatim_export(int rank_min, int rank_max, const char *conninfo, const char *structuredoutputfile);
void nominatim_exportCreatePreparedQueries(PGconn * conn);
+
xmlTextWriterPtr nominatim_exportXMLStart(const char *structuredoutputfile);
void nominatim_exportXMLEnd(xmlTextWriterPtr writer);
-void nominatim_exportPlace(uint64_t place_id, PGconn * conn, xmlTextWriterPtr writer, pthread_mutex_t * writer_mutex);
+
+void nominatim_exportEndMode(xmlTextWriterPtr writer);
+
+void nominatim_exportPlaceQueries(uint64_t place_id, PGconn * conn, struct export_data * querySet);
+void nominatim_exportFreeQueries(struct export_data * querySet);
+
+void nominatim_exportPlace(uint64_t place_id, PGconn * conn,
+ xmlTextWriterPtr writer, pthread_mutex_t * writer_mutex, struct export_data * prevQuerySet);
const char * getRankLabel(int rank);
#endif
xmlChar * rankAddress;
xmlChar * rankSearch;
xmlChar * countryCode;
+ xmlChar * parentPlaceID;
+ xmlChar * parentType;
+ xmlChar * parentID;
xmlChar * adminLevel;
xmlChar * houseNumber;
xmlChar * geometry;
int featureExtraTagLines = 0;
int featureCount = 0;
xmlHashTablePtr partionTableTagsHash;
+xmlHashTablePtr partionTableTagsHashDelete;
char featureNameString[MAX_FEATURENAMESTRING];
char featureExtraTagString[MAX_FEATUREEXTRATAGSTRING];
feature.rankAddress = xmlTextReaderGetAttribute(reader, BAD_CAST "rank");
feature.rankSearch = xmlTextReaderGetAttribute(reader, BAD_CAST "importance");
+ feature.parentPlaceID = xmlTextReaderGetAttribute(reader, BAD_CAST "parent_place_id");
+ feature.parentType = xmlTextReaderGetAttribute(reader, BAD_CAST "parent_type");
+ feature.parentID = xmlTextReaderGetAttribute(reader, BAD_CAST "parent_id");
+
feature.countryCode = NULL;
feature.adminLevel = NULL;
feature.houseNumber = NULL;
feature.geometry = NULL;
featureAddressLines = 0;
featureNameLines = 0;
- featureExtraTagLines = 0;
+ featureExtraTagLines = 0;
return;
}
void EndElement(xmlTextReaderPtr reader, const xmlChar *name)
{
PGresult * res;
- PGresult * resPlaceID;
const char * paramValues[11];
char * place_id;
char * partionQueryName;
}
}
*/
- place_id = feature.placeID;
+ place_id = (char *)feature.placeID;
- if (fileMode == FILEMODE_UPDATE || fileMode == FILEMODE_DELETE)
+ if (fileMode == FILEMODE_UPDATE || fileMode == FILEMODE_DELETE || fileMode == FILEMODE_ADD)
{
paramValues[0] = (const char *)place_id;
res = PQexecPrepared(conn, "placex_delete", 1, paramValues, NULL, NULL, 0);
exit(EXIT_FAILURE);
}
PQclear(res);
+
+ partionQueryName = xmlHashLookup2(partionTableTagsHashDelete, feature.key, feature.value);
+ if (partionQueryName)
+ {
+ res = PQexecPrepared(conn, partionQueryName, 1, paramValues, NULL, NULL, 0);
+ if (PQresultStatus(res) != PGRES_COMMAND_OK)
+ {
+ fprintf(stderr, "%s: DELETE failed: %s", partionQueryName, PQerrorMessage(conn));
+ PQclear(res);
+ exit(EXIT_FAILURE);
+ }
+ PQclear(res);
+ }
}
if (fileMode == FILEMODE_UPDATE || fileMode == FILEMODE_ADD)
lineValueLen = 0;
for (i = 0; i < featureNameLines; i++)
{
- lineTypeLen = strlen(BAD_CAST featureName[i].type);
- lineValueLen = strlen(BAD_CAST featureName[i].value);
+ lineTypeLen = (int)strlen((char *) featureName[i].type);
+ lineValueLen = (int)strlen((char *) featureName[i].value);
if (namePos+lineTypeLen+lineValueLen+7 > MAX_FEATURENAMESTRING)
{
fprintf(stderr, "feature name too long: %s", (const char *)featureName[i].value);
}
if (namePos) strcpy(featureNameString+(namePos++), ",");
strcpy(featureNameString+(namePos++), "\"");
- strcpy(featureNameString+namePos, BAD_CAST featureName[i].type);
+ strcpy(featureNameString+namePos, (char*) featureName[i].type);
namePos += lineTypeLen;
strcpy(featureNameString+namePos, "\"=>\"");
namePos += 4;
- strcpy(featureNameString+namePos, BAD_CAST featureName[i].value);
+ strcpy(featureNameString+namePos, (char *) featureName[i].value);
namePos += lineValueLen;
strcpy(featureNameString+(namePos++), "\"");
}
lineValueLen = 0;
for (i = 0; i < featureExtraTagLines; i++)
{
- lineTypeLen = strlen(BAD_CAST featureExtraTag[i].type);
- lineValueLen = strlen(BAD_CAST featureExtraTag[i].value);
+ lineTypeLen = strlen((char *) featureExtraTag[i].type);
+ lineValueLen = strlen((char *) featureExtraTag[i].value);
if (namePos+lineTypeLen+lineValueLen+7 > MAX_FEATUREEXTRATAGSTRING)
{
fprintf(stderr, "feature extra tag too long: %s", (const char *)featureExtraTag[i].value);
}
if (namePos) strcpy(featureExtraTagString+(namePos++),",");
strcpy(featureExtraTagString+(namePos++), "\"");
- strcpy(featureExtraTagString+namePos, BAD_CAST featureExtraTag[i].type);
+ strcpy(featureExtraTagString+namePos, (char *) featureExtraTag[i].type);
namePos += lineTypeLen;
strcpy(featureExtraTagString+namePos, "\"=>\"");
namePos += 4;
- strcpy(featureExtraTagString+namePos, BAD_CAST featureExtraTag[i].value);
+ strcpy(featureExtraTagString+namePos, (char *) featureExtraTag[i].value);
namePos += lineValueLen;
strcpy(featureExtraTagString+(namePos++), "\"");
}
}
paramValues[6] = (const char *)featureExtraTagString;
- paramValues[7] = (const char *)feature.adminLevel;
- paramValues[8] = (const char *)feature.houseNumber;
- paramValues[9] = (const char *)feature.rankAddress;
- paramValues[10] = (const char *)feature.rankSearch;
- paramValues[11] = (const char *)feature.geometry;
- res = PQexecPrepared(conn, "placex_insert", 12, paramValues, NULL, NULL, 0);
- if (PQresultStatus(res) != PGRES_COMMAND_OK)
+ paramValues[7] = (const char *)feature.parentPlaceID;
+
+ paramValues[8] = (const char *)feature.adminLevel;
+ paramValues[9] = (const char *)feature.houseNumber;
+ paramValues[10] = (const char *)feature.rankAddress;
+ paramValues[11] = (const char *)feature.rankSearch;
+ paramValues[12] = (const char *)feature.geometry;
+ if (strlen(paramValues[3]))
{
- fprintf(stderr, "index_placex: INSERT failed: %s", PQerrorMessage(conn));
- PQclear(res);
- exit(EXIT_FAILURE);
+ res = PQexecPrepared(conn, "placex_insert", 13, paramValues, NULL, NULL, 0);
+ if (PQresultStatus(res) != PGRES_COMMAND_OK)
+ {
+ fprintf(stderr, "index_placex: INSERT failed: %s", PQerrorMessage(conn));
+ fprintf(stderr, "index_placex: INSERT failed: %s %s %s", paramValues[0], paramValues[1], paramValues[2]);
+ PQclear(res);
+ exit(EXIT_FAILURE);
+ }
+ PQclear(res);
}
- PQclear(res);
for (i = 0; i < featureAddressLines; i++)
{
exit(EXIT_FAILURE);
}
PQclear(res);
-
}
}
}
partionTableTagsHash = xmlHashCreate(200);
+ partionTableTagsHashDelete = xmlHashCreate(200);
partionTagsFile = fopen(partionTagsFilename, "rt");
if (!partionTagsFile)
}
xmlHashAddEntry2(partionTableTagsHash, BAD_CAST osmkey, BAD_CAST osmvalue, BAD_CAST partionQueryName);
+
+ partionQueryName = malloc(strlen("partition_delete_")+strlen(osmkey)+strlen(osmvalue)+2);
+ strcpy(partionQueryName, "partition_delete_");
+ strcat(partionQueryName, osmkey);
+ strcat(partionQueryName, "_");
+ strcat(partionQueryName, osmvalue);
+
+ strcpy(partionQuerySQL, "delete from place_classtype_");
+ strcat(partionQuerySQL, osmkey);
+ strcat(partionQuerySQL, "_");
+ strcat(partionQuerySQL, osmvalue);
+ strcat(partionQuerySQL, " where place_id = $1::integer");
+
+ res = PQprepare(conn, partionQueryName, partionQuerySQL, 1, NULL);
+ if (PQresultStatus(res) != PGRES_COMMAND_OK)
+ {
+ fprintf(stderr, "Failed to prepare %s: %s\n", partionQueryName, PQerrorMessage(conn));
+ exit(EXIT_FAILURE);
+ }
+
+ xmlHashAddEntry2(partionTableTagsHashDelete, BAD_CAST osmkey, BAD_CAST osmvalue, BAD_CAST partionQueryName);
}
res = PQprepare(conn, "get_new_place_id",
}
res = PQprepare(conn, "placex_insert",
- "insert into placex (place_id,osm_type,osm_id,class,type,name,extratags,admin_level,housenumber,rank_address,rank_search,geometry) "
- "values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, st_setsrid($12, 4326))",
+ "insert into placex (place_id,osm_type,osm_id,class,type,name,extratags,parent_place_id,admin_level,housenumber,rank_address,rank_search,geometry) "
+ "values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, st_setsrid($13, 4326))",
12, NULL);
if (PQresultStatus(res) != PGRES_COMMAND_OK)
{
xmlFreeTextReader(reader);
xmlHashFree(partionTableTagsHash, NULL);
+ xmlHashFree(partionTableTagsHashDelete, NULL);
return 0;
}
int i;
int iSector;
int iResult;
+ int bSkip;
const char *paramValues[2];
int paramLengths[2];
// if (rank < 16)
// resSectors = PQexecPrepared(conn, "index_nosectors", 1, paramValues, paramLengths, paramFormats, 1);
// else
- resSectors = PQexecPrepared(conn, "index_sectors", 1, paramValues, paramLengths, paramFormats, 1);
+ resSectors = PQexecPrepared(conn, "index_sectors", 1, paramValues, paramLengths, paramFormats, 1);
+
if (PQresultStatus(resSectors) != PGRES_TUPLES_OK)
{
fprintf(stderr, "index_sectors: SELECT failed: %s", PQerrorMessage(conn));
if (iSector < PQntuples(resSectors))
{
sector = PGint32(*((uint32_t *)PQgetvalue(resSectors, iSector, 0)));
- //printf("\n Starting sector %d size %ld\n", sector, PGint64(*((uint64_t *)PQgetvalue(resSectors, iSector, 1))));
+// printf("\n Starting sector %d size %ld\n", sector, PGint64(*((uint64_t *)PQgetvalue(resSectors, iSector, 1))));
// Get all the place_id's for this sector
paramRank = PGint32(rank);
paramValues[1] = (char *)¶mSector;
paramLengths[1] = sizeof(paramSector);
paramFormats[1] = 1;
- if (rankTotalTuples-rankCountTuples < num_threads*20)
+ if (rankTotalTuples-rankCountTuples < num_threads*1000)
{
- iResult = PQsendQueryPrepared(conn, "index_nosector_places", 1, paramValues, paramLengths, paramFormats, 1);
+ iResult = PQsendQueryPrepared(conn, "index_nosector_places", 1, paramValues, paramLengths, paramFormats, 1);
}
else
{
PQclear(resPlaces);
}
- if (rankTotalTuples-rankCountTuples < num_threads*20) iSector = PQntuples(resSectors);
+ if (rankTotalTuples-rankCountTuples < num_threads*20 && iSector < PQntuples(resSectors))
+ {
+ iSector = PQntuples(resSectors) - 1;
+ }
}
// Finished rank
printf("\r Done %i in %i @ %f per second - FINISHED \n\n", rankCountTuples, (int)(difftime(time(0), rankStartTime)), rankPerSecond);
void *nominatim_indexThread(void * thread_data_in)
{
struct index_thread_data * thread_data = (struct index_thread_data * )thread_data_in;
+ struct export_data querySet;
PGresult *res;
int done = 0;
while(!done)
{
+ if (thread_data->writer)
+ {
+ nominatim_exportPlaceQueries(place_id, thread_data->conn, &querySet);
+ }
+
+
paramPlaceID = PGint32(place_id);
paramValues[0] = (char *)¶mPlaceID;
paramLengths[0] = sizeof(paramPlaceID);
if (thread_data->writer)
{
- nominatim_exportPlace(place_id, thread_data->conn, thread_data->writer, thread_data->writer_mutex);
+ nominatim_exportPlace(place_id, thread_data->conn, thread_data->writer, thread_data->writer_mutex, &querySet);
+ nominatim_exportFreeQueries(&querySet);
}
}
-- RETURN NULL;
-- END IF;
--- RETURN NEW; -- The following is not needed until doing diff updates, and slows the main index process down
+ RETURN NEW; -- The following is not needed until doing diff updates, and slows the main index process down
IF (ST_GeometryType(NEW.geometry) in ('ST_Polygon','ST_MultiPolygon') AND ST_IsValid(NEW.geometry)) THEN
-- Performance: We just can't handle re-indexing for country level changes
result := deleteSearchName(NEW.partition, NEW.place_id);
DELETE FROM place_addressline WHERE place_id = NEW.place_id;
DELETE FROM place_boundingbox where place_id = NEW.place_id;
+ result := deleteRoad(NEW.partition, NEW.place_id);
+ result := deleteLocationArea(NEW.partition, NEW.place_id);
+
+ NEW.country_code := lower(get_country_code(NEW.geometry, NEW.country_code));
+ NEW.partition := get_partition(NEW.geometry, NEW.country_code);
+ NEW.geometry_sector := geometry_sector(NEW.partition, NEW.geometry);
-- Adding ourselves to the list simplifies address calculations later
INSERT INTO place_addressline VALUES (NEW.place_id, NEW.place_id, true, true, 0, NEW.rank_address);
END LOOP;
END IF;
---RAISE WARNING 'x3';
+--RAISE WARNING 'x3 %',NEW.parent_place_id;
IF NEW.parent_place_id IS NULL AND NEW.street IS NOT NULL THEN
address_street_word_id := get_name_id(make_standard_name(NEW.street));
END IF;
END IF;
---RAISE WARNING 'x4';
+--RAISE WARNING 'x4 %',NEW.parent_place_id;
-- Still nothing, just use the nearest road
- FOR location IN SELECT place_id FROM getNearestRoadFeature(NEW.partition, place_centroid) LOOP
- NEW.parent_place_id := location.place_id;
- END LOOP;
+ IF NEW.parent_place_id IS NULL THEN
+ FOR location IN SELECT place_id FROM getNearestRoadFeature(NEW.partition, place_centroid) LOOP
+ NEW.parent_place_id := location.place_id;
+ END LOOP;
+ END IF;
--return NEW;
--RAISE WARNING 'x6 %',NEW.parent_place_id;
DELETE FROM place_addressline where address_place_id = OLD.place_id;
+ b := deleteRoad(OLD.partition, OLD.place_id);
+
+ update placex set indexed_status = 2 where parent_place_id = OLD.place_id and indexed_status = 0;
+
END IF;
IF OLD.rank_address < 26 THEN
partition INTEGER;
BEGIN
- IF FALSE AND NEW.osm_type = 'R' THEN
+ IF FALSE and NEW.osm_type = 'R' THEN
RAISE WARNING '-----------------------------------------------------------------------------------';
RAISE WARNING 'place_insert: % % % % %',NEW.osm_type,NEW.osm_id,NEW.class,NEW.type,st_area(NEW.geometry);
select * from placex where osm_type = NEW.osm_type and osm_id = NEW.osm_id and class = NEW.class and type = NEW.type INTO existingplacex;
END IF;
IF ST_IsEmpty(NEW.geometry) OR NOT ST_IsValid(NEW.geometry) OR ST_X(ST_Centroid(NEW.geometry))::text in ('NaN','Infinity','-Infinity') OR ST_Y(ST_Centroid(NEW.geometry))::text in ('NaN','Infinity','-Infinity') THEN
+ INSERT INTO import_polygon_error values (NEW.osm_type, NEW.osm_id, NEW.class, NEW.type, NEW.name, NEW.country_code,
+ now(), ST_IsValidReason(NEW.geometry), null, NEW.geometry);
-- RAISE WARNING 'Invalid Geometry: % % % %',NEW.osm_type,NEW.osm_id,NEW.class,NEW.type;
RETURN null;
END IF;
-- Handle a place changing type by removing the old data
-- My generated 'place' types are causing havok because they overlap with real tags
- -- TODO: move them to their own special purpose tag to avoid collisions
- IF existing.osm_type IS NULL AND (NEW.type not in ('postcode','house','houses')) THEN
- DELETE FROM place where osm_type = NEW.osm_type and osm_id = NEW.osm_id and class = NEW.class and type not in ('postcode','house','houses');
- END IF;
+ -- TODO: move them to their own special purpose key/class to avoid collisions
+-- IF existing.osm_type IS NULL AND (NEW.type not in ('postcode','house','houses')) THEN
+-- DELETE FROM place where osm_type = NEW.osm_type and osm_id = NEW.osm_id and class = NEW.class and type not in ('postcode','house','houses');
+-- END IF;
-- RAISE WARNING 'Existing: %',existing.place_id;
+ -- Log and discard
+ IF existing.geometry is not null AND st_isvalid(existing.geometry)
+ AND st_area(existing.geometry) > 0.02
+ AND ST_GeometryType(NEW.geometry) in ('ST_Polygon','ST_MultiPolygon')
+ AND st_area(NEW.geometry) < st_area(existing.geometry)*0.5
+ THEN
+ INSERT INTO import_polygon_error values (NEW.osm_type, NEW.osm_id, NEW.class, NEW.type, NEW.name, NEW.country_code, now(),
+ 'Area reduced from '||st_area(existing.geometry)||' to '||st_area(NEW.geometry), existing.geometry, NEW.geometry);
+ RETURN null;
+ END IF;
+
+ DELETE from import_polygon_error where osm_type = NEW.osm_type and osm_id = NEW.osm_id;
+
-- To paraphrase, if there isn't an existing item, OR if the admin level has changed, OR if it is a major change in geometry
IF existing.osm_type IS NULL
OR existingplacex.osm_type IS NULL
-- RAISE WARNING 'no existing placex %', existingplacex;
-- END IF;
-
-- RAISE WARNING 'delete and replace';
IF existing.osm_type IS NOT NULL THEN
select geometry from placex where osm_type = NEW.osm_type and osm_id = NEW.osm_id and class = NEW.class and type = NEW.type into existinggeometry;
-- Performance limit
- IF st_area(NEW.geometry) < 1 AND st_area(existinggeometry) < 1 THEN
+ IF st_area(NEW.geometry) < 0.000000001 AND st_area(existinggeometry) < 1 THEN
-- re-index points that have moved in / out of the polygon, could be done as a single query but postgres gets the index usage wrong
update placex set indexed_status = 2 where indexed_status = 0 and
-- performance, can't take the load of re-indexing a whole country / huge area
IF st_area(NEW.geometry) < 0.5 THEN
- UPDATE placex set indexed_status = 2 from place_addressline where address_place_id = existingplacex.place_id
- and placex.place_id = place_addressline.place_id and indexed_status = 0;
+-- UPDATE placex set indexed_status = 2 from place_addressline where address_place_id = existingplacex.place_id
+-- and placex.place_id = place_addressline.place_id and indexed_status = 0;
END IF;
END IF;
geometry = NEW.geometry
where place_id = existingplacex.place_id;
--- now done as part of indexing
+-- now done as part of insert
-- partition := get_partition(NEW.geometry, existingplacex.country_code);
-- result := update_location(partition, existingplacex.place_id, existingplacex.country_code, NEW.name, existingplacex.rank_search, existingplacex.rank_address, NEW.geometry);
name HSTORE,
class TEXT,
type TEXT,
+ admin_level INTEGER,
fromarea BOOLEAN,
isaddress BOOLEAN,
rank_address INTEGER,
search TEXT[];
found INTEGER;
location RECORD;
+ countrylocation RECORD;
searchcountrycode varchar(2);
searchhousenumber TEXT;
searchhousename HSTORE;
searchrankaddress INTEGER;
searchpostcode TEXT;
+ searchclass TEXT;
+ searchtype TEXT;
countryname HSTORE;
+ hadcountry BOOLEAN;
BEGIN
- select parent_place_id,'us', housenumber, 30, postcode, null from location_property_tiger
+ select parent_place_id,'us', housenumber, 30, postcode, null, 'place', 'house' from location_property_tiger
WHERE place_id = in_place_id
- INTO for_place_id,searchcountrycode, searchhousenumber, searchrankaddress, searchpostcode, searchhousename;
+ INTO for_place_id,searchcountrycode, searchhousenumber, searchrankaddress, searchpostcode, searchhousename, searchclass, searchtype;
IF for_place_id IS NULL THEN
- select parent_place_id,'us', housenumber, 30, postcode, null from location_property_aux
+ select parent_place_id,'us', housenumber, 30, postcode, null, 'place', 'house' from location_property_aux
WHERE place_id = in_place_id
- INTO for_place_id,searchcountrycode, searchhousenumber, searchrankaddress, searchpostcode, searchhousename;
+ INTO for_place_id,searchcountrycode, searchhousenumber, searchrankaddress, searchpostcode, searchhousename, searchclass, searchtype;
END IF;
IF for_place_id IS NULL THEN
- select parent_place_id, country_code, housenumber, rank_address, postcode, name from placex
+ select parent_place_id, country_code, housenumber, rank_address, postcode, name, class, type from placex
WHERE place_id = in_place_id and rank_address = 30
- INTO for_place_id, searchcountrycode, searchhousenumber, searchrankaddress, searchpostcode, searchhousename;
+ INTO for_place_id, searchcountrycode, searchhousenumber, searchrankaddress, searchpostcode, searchhousename, searchclass, searchtype;
END IF;
IF for_place_id IS NULL THEN
--RAISE WARNING '% % % %',searchcountrycode, searchhousenumber, searchrankaddress, searchpostcode;
found := 1000;
+ hadcountry := false;
FOR location IN
- select placex.place_id, osm_type, osm_id,
+ select placex.place_id, osm_type, osm_id,
CASE WHEN class = 'place' and type = 'postcode' THEN 'name' => postcode ELSE name END as name,
- class, type, fromarea, isaddress,
+ class, type, admin_level, fromarea, isaddress,
CASE WHEN address_place_id = for_place_id AND rank_address = 0 THEN 100 WHEN rank_address = 11 THEN 5 ELSE rank_address END as rank_address,
- distance
+ distance,country_code
from place_addressline join placex on (address_place_id = placex.place_id)
where place_addressline.place_id = for_place_id
and ((cached_rank_address > 0 AND cached_rank_address < searchrankaddress) OR address_place_id = for_place_id)
order by rank_address desc,isaddress desc,fromarea desc,distance asc,rank_search desc
LOOP
--RAISE WARNING '%',location;
+ IF searchcountrycode IS NULL AND location.country_code IS NOT NULL THEN
+ searchcountrycode := location.country_code;
+ END IF;
IF searchpostcode IS NOT NULL and location.type = 'postcode' THEN
location.isaddress := FALSE;
END IF;
- RETURN NEXT location;
+ IF location.rank_address = 4 AND location.isaddress THEN
+ hadcountry := true;
+ END IF;
+ IF location.rank_address < 4 AND NOT hadcountry THEN
+ select name from country_name where country_code = searchcountrycode limit 1 INTO countryname;
+ IF countryname IS NOT NULL THEN
+ countrylocation := ROW(null, null, null, countryname, 'place', 'country', null, true, true, 4, 0)::addressline;
+ RETURN NEXT countrylocation;
+ END IF;
+ END IF;
+ countrylocation := ROW(location.place_id, location.osm_type, location.osm_id, location.name, location.class,
+ location.type, location.admin_level, location.fromarea, location.isaddress, location.rank_address,
+ location.distance)::addressline;
+ RETURN NEXT countrylocation;
found := location.rank_address;
END LOOP;
select name from country_name where country_code = searchcountrycode limit 1 INTO countryname;
--RAISE WARNING '% % %',found,searchcountrycode,countryname;
IF countryname IS NOT NULL THEN
- location := ROW(null, null, null, countryname, 'place', 'country', true, true, 4, 0)::addressline;
+ location := ROW(null, null, null, countryname, 'place', 'country', null, true, true, 4, 0)::addressline;
RETURN NEXT location;
END IF;
END IF;
IF searchcountrycode IS NOT NULL THEN
- location := ROW(null, null, null, 'ref'=>searchcountrycode, 'place', 'country_code', true, false, 4, 0)::addressline;
+ location := ROW(null, null, null, 'ref'=>searchcountrycode, 'place', 'country_code', null, true, false, 4, 0)::addressline;
RETURN NEXT location;
END IF;
IF searchhousename IS NOT NULL THEN
- location := ROW(in_place_id, null, null, searchhousename, 'place', 'house_name', true, true, 29, 0)::addressline;
+ location := ROW(in_place_id, null, null, searchhousename, searchclass, searchtype, null, true, true, 29, 0)::addressline;
+-- location := ROW(in_place_id, null, null, searchhousename, 'place', 'house_name', null, true, true, 29, 0)::addressline;
RETURN NEXT location;
END IF;
IF searchhousenumber IS NOT NULL THEN
- location := ROW(in_place_id, null, null, 'ref'=>searchhousenumber, 'place', 'house_number', true, true, 28, 0)::addressline;
+ location := ROW(in_place_id, null, null, 'ref'=>searchhousenumber, 'place', 'house_number', null, true, true, 28, 0)::addressline;
RETURN NEXT location;
END IF;
IF searchpostcode IS NOT NULL THEN
- location := ROW(null, null, null, 'ref'=>searchpostcode, 'place', 'postcode', true, true, 5, 0)::addressline;
+ location := ROW(null, null, null, 'ref'=>searchpostcode, 'place', 'postcode', null, true, true, 5, 0)::addressline;
RETURN NEXT location;
END IF;
DROP SEQUENCE seq_postcodes;
CREATE SEQUENCE seq_postcodes start 1;
+
+drop table import_polygon_error;
+CREATE TABLE import_polygon_error (
+ osm_type char(1),
+ osm_id INTEGER,
+ class TEXT NOT NULL,
+ type TEXT NOT NULL,
+ name HSTORE,
+ country_code varchar(2),
+ updated timestamp,
+ errormessage text
+ );
+SELECT AddGeometryColumn('import_polygon_error', 'prevgeometry', 4326, 'GEOMETRY', 2);
+SELECT AddGeometryColumn('import_polygon_error', 'newgeometry', 4326, 'GEOMETRY', 2);
+CREATE INDEX idx_import_polygon_error_osmid ON import_polygon_error USING BTREE (osm_type, osm_id);
{
echo "Partitions\n";
$bDidSomething = true;
-echo "here";
$oDB =& getDB();
-echo "there";
$sSQL = 'select partition from country_name order by country_code';
-var_dump($sSQL);
$aPartitions = $oDB->getCol($sSQL);
-var_dump($aPartitions);
if (PEAR::isError($aPartitions))
{
fail($aPartitions->getMessage());
preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
foreach($aMatches as $aMatch)
{
-var_dump($aMatch);
$sResult = '';
foreach($aPartitions as $sPartitionName)
{
-var_dump($sPartitionName);
$sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
}
$sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
{
// Convert database DSN to psql paramaters
$aDSNInfo = DB::parseDSN(CONST_Database_DSN);
- $sCMD = 'psql '.$aDSNInfo['database'];
-
+ if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
+ $sCMD = 'psql -p '.$aDSNInfo['port'].' '.$aDSNInfo['database'];
$aDescriptors = array(
0 => array('pipe', 'r'),
1 => array('pipe', 'w'),
projects / nominatim.git / commitdiff