nominatim/import.c

   1 /*
   2 */
   3 #include <stdlib.h>
   4 #include <string.h>
   5
   6 #include <libpq-fe.h>
   7
   8 #include <libxml/xmlstring.h>
   9 #include <libxml/xmlreader.h>
  10 #include <libxml/hash.h>
  11
  12 #include "nominatim.h"
  13 #include "import.h"
  14 #include "input.h"
  15
  16 typedef enum { FILETYPE_NONE, FILETYPE_STRUCTUREDV0P1 } filetypes_t;
  17 typedef enum { FILEMODE_NONE, FILEMODE_ADD, FILEMODE_UPDATE, FILEMODE_DELETE } filemodes_t;
  18
  19 #define MAX_FEATUREADDRESS 5000
  20 #define MAX_FEATURENAMES 10000
  21 #define MAX_FEATUREEXTRATAGS 10000
  22 #define MAX_FEATURENAMESTRING 1000000
  23 #define MAX_FEATUREEXTRATAGSTRING 500000
  24
  25 struct feature_address
  26 {
  27     int                 place_id;
  28     int                 rankAddress;
  29     char                        isAddress[2];
  30     xmlChar *   type;
  31     xmlChar *   id;
  32     xmlChar *   key;
  33     xmlChar *   value;
  34     xmlChar *   distance;
  35 };
  36
  37 struct feature_tag
  38 {
  39     xmlChar *   type;
  40     xmlChar *   value;
  41 };
  42
  43 struct feature
  44 {
  45     xmlChar *   placeID;
  46     xmlChar *   type;
  47     xmlChar *   id;
  48     xmlChar *   key;
  49     xmlChar *   value;
  50     xmlChar *   rankAddress;
  51     xmlChar *   rankSearch;
  52     xmlChar *   countryCode;
  53     xmlChar *   parentPlaceID;
  54     xmlChar *   parentType;
  55     xmlChar *   parentID;
  56     xmlChar *   adminLevel;
  57     xmlChar *   houseNumber;
  58     xmlChar *   geometry;
  59 } feature;
  60
  61 int                                     fileType = FILETYPE_NONE;
  62 int                                     fileMode = FILEMODE_ADD;
  63 PGconn *                                conn;
  64 struct feature_address  featureAddress[MAX_FEATUREADDRESS];
  65 struct feature_tag              featureName[MAX_FEATURENAMES];
  66 struct feature_tag              featureExtraTag[MAX_FEATUREEXTRATAGS];
  67 struct feature                  feature;
  68 int                                     featureAddressLines = 0;
  69 int                                     featureNameLines = 0;
  70 int                                     featureExtraTagLines = 0;
  71 int                                     featureCount = 0;
  72 xmlHashTablePtr                 partionTableTagsHash;
  73 xmlHashTablePtr                 partionTableTagsHashDelete;
  74 char                                    featureNameString[MAX_FEATURENAMESTRING];
  75 char                                    featureExtraTagString[MAX_FEATUREEXTRATAGSTRING];
  76
  77 void StartElement(xmlTextReaderPtr reader, const xmlChar *name)
  78 {
  79     char * value;
  80     float version;
  81     int isAddressLine;
  82
  83     if (fileType == FILETYPE_NONE)
  84     {
  85         // Potential to handle other file types in the future / versions
  86         if (xmlStrEqual(name, BAD_CAST "osmStructured"))
  87         {
  88             value = (char*)xmlTextReaderGetAttribute(reader, BAD_CAST "version");
  89             version = strtof(value, NULL);
  90             xmlFree(value);
  91
  92             if (version == (float)0.1)
  93             {
  94                 fileType = FILETYPE_STRUCTUREDV0P1;
  95                 fileMode = FILEMODE_ADD;
  96             }
  97             else
  98             {
  99                 fprintf( stderr, "Unknown osmStructured version %f (%s)\n", version, value );
 100                 exit_nicely();
 101             }
 102         }
 103         else
 104         {
 105             fprintf( stderr, "Unknown XML document type: %s\n", name );
 106             exit_nicely();
 107         }
 108         return;
 109     }
 110
 111     if (xmlStrEqual(name, BAD_CAST "add"))
 112     {
 113         fileMode = FILEMODE_ADD;
 114         return;
 115     }
 116     if (xmlStrEqual(name, BAD_CAST "update"))
 117     {
 118         fileMode = FILEMODE_UPDATE;
 119         return;
 120     }
 121     if (xmlStrEqual(name, BAD_CAST "delete"))
 122     {
 123         fileMode = FILEMODE_DELETE;
 124         return;
 125     }
 126     if (fileMode == FILEMODE_NONE)
 127     {
 128         fprintf( stderr, "Unknown import mode in: %s\n", name );
 129         exit_nicely();
 130     }
 131
 132     if (xmlStrEqual(name, BAD_CAST "feature"))
 133     {
 134         feature.placeID = xmlTextReaderGetAttribute(reader, BAD_CAST "place_id");
 135         feature.type = xmlTextReaderGetAttribute(reader, BAD_CAST "type");
 136         feature.id = xmlTextReaderGetAttribute(reader, BAD_CAST "id");
 137         feature.key = xmlTextReaderGetAttribute(reader, BAD_CAST "key");
 138         feature.value = xmlTextReaderGetAttribute(reader, BAD_CAST "value");
 139         feature.rankAddress = xmlTextReaderGetAttribute(reader, BAD_CAST "rank");
 140         feature.rankSearch = xmlTextReaderGetAttribute(reader, BAD_CAST "importance");
 141
 142         feature.parentPlaceID = xmlTextReaderGetAttribute(reader, BAD_CAST "parent_place_id");
 143         feature.parentType = xmlTextReaderGetAttribute(reader, BAD_CAST "parent_type");
 144         feature.parentID = xmlTextReaderGetAttribute(reader, BAD_CAST "parent_id");
 145
 146         feature.countryCode = NULL;
 147         feature.adminLevel = NULL;
 148         feature.houseNumber = NULL;
 149         feature.geometry = NULL;
 150         featureAddressLines = 0;
 151         featureNameLines = 0;
 152         featureExtraTagLines = 0;
 153
 154         return;
 155     }
 156     if (xmlStrEqual(name, BAD_CAST "names")) return;
 157     if (xmlStrEqual(name, BAD_CAST "name"))
 158     {
 159         if (featureNameLines < MAX_FEATURENAMES)
 160         {
 161                 featureName[featureNameLines].type = xmlTextReaderGetAttribute(reader, BAD_CAST "type");
 162             featureName[featureNameLines].value = xmlTextReaderReadString(reader);
 163                 featureNameLines++;
 164                 }
 165                 else
 166                 {
 167             fprintf( stderr, "Too many name elements (%s%s)\n", feature.type, feature.id);
 168 //            exit_nicely();
 169         }
 170         return;
 171     }
 172     if (xmlStrEqual(name, BAD_CAST "tags")) return;
 173     if (xmlStrEqual(name, BAD_CAST "tag"))
 174     {
 175         if (featureExtraTagLines < MAX_FEATUREEXTRATAGS)
 176                 {
 177                 featureExtraTag[featureExtraTagLines].type = xmlTextReaderGetAttribute(reader, BAD_CAST "type");
 178             featureExtraTag[featureExtraTagLines].value = xmlTextReaderReadString(reader);
 179                 featureExtraTagLines++;
 180                 }
 181                 else
 182         {
 183             fprintf( stderr, "Too many extra tag elements (%s%s)\n", feature.type, feature.id);
 184 //            exit_nicely();
 185         }
 186         return;
 187     }
 188     if (xmlStrEqual(name, BAD_CAST "osmGeometry"))
 189     {
 190         feature.geometry = xmlTextReaderReadString(reader);
 191         return;
 192     }
 193     if (xmlStrEqual(name, BAD_CAST "adminLevel"))
 194     {
 195         feature.adminLevel = xmlTextReaderReadString(reader);
 196         return;
 197     }
 198     if (xmlStrEqual(name, BAD_CAST "countryCode"))
 199     {
 200         feature.countryCode = xmlTextReaderReadString(reader);
 201         return;
 202     }
 203     if (xmlStrEqual(name, BAD_CAST "houseNumber"))
 204     {
 205         feature.houseNumber = xmlTextReaderReadString(reader);
 206         return;
 207     }
 208     if (xmlStrEqual(name, BAD_CAST "address"))
 209     {
 210         featureAddressLines = 0;
 211         return;
 212     }
 213     isAddressLine = 0;
 214     if (xmlStrEqual(name, BAD_CAST "continent"))
 215     {
 216         isAddressLine = 1;
 217     }
 218     else if (xmlStrEqual(name, BAD_CAST "sea"))
 219     {
 220         isAddressLine = 1;
 221     }
 222     else if (xmlStrEqual(name, BAD_CAST "country"))
 223     {
 224         isAddressLine = 1;
 225     }
 226     else if (xmlStrEqual(name, BAD_CAST "state"))
 227     {
 228         isAddressLine = 1;
 229     }
 230     else if (xmlStrEqual(name, BAD_CAST "county"))
 231     {
 232         isAddressLine = 1;
 233     }
 234     else if (xmlStrEqual(name, BAD_CAST "city"))
 235     {
 236         isAddressLine = 1;
 237     }
 238     else if (xmlStrEqual(name, BAD_CAST "town"))
 239     {
 240         isAddressLine = 1;
 241     }
 242     else if (xmlStrEqual(name, BAD_CAST "village"))
 243     {
 244         isAddressLine = 1;
 245     }
 246     else if (xmlStrEqual(name, BAD_CAST "unknown"))
 247     {
 248         isAddressLine = 1;
 249     }
 250     else if (xmlStrEqual(name, BAD_CAST "suburb"))
 251     {
 252         isAddressLine = 1;
 253     }
 254     else if (xmlStrEqual(name, BAD_CAST "postcode"))
 255     {
 256         isAddressLine = 1;
 257     }
 258     else if (xmlStrEqual(name, BAD_CAST "neighborhood"))
 259     {
 260         isAddressLine = 1;
 261     }
 262     else if (xmlStrEqual(name, BAD_CAST "street"))
 263     {
 264         isAddressLine = 1;
 265     }
 266     else if (xmlStrEqual(name, BAD_CAST "access"))
 267     {
 268         isAddressLine = 1;
 269     }
 270     else if (xmlStrEqual(name, BAD_CAST "building"))
 271     {
 272         isAddressLine = 1;
 273     }
 274     else if (xmlStrEqual(name, BAD_CAST "other"))
 275     {
 276         isAddressLine = 1;
 277     }
 278     if (isAddressLine)
 279     {
 280         if (featureAddressLines < MAX_FEATUREADDRESS)
 281                 {
 282                 value = (char*)xmlTextReaderGetAttribute(reader, BAD_CAST "rank");
 283             if (!value)
 284                 {
 285                 fprintf( stderr, "Address element missing rank\n");
 286                     exit_nicely();
 287                 }
 288             featureAddress[featureAddressLines].rankAddress =  atoi(value);
 289                 xmlFree(value);
 290
 291                 value = (char*)xmlTextReaderGetAttribute(reader, BAD_CAST "isaddress");
 292             if (!value)
 293                 {
 294                     fprintf( stderr, "Address element missing rank\n");
 295                 exit_nicely();
 296                 }
 297                 if (*value == 't') strcpy(featureAddress[featureAddressLines].isAddress, "t");
 298             else strcpy(featureAddress[featureAddressLines].isAddress, "f");
 299                 xmlFree(value);
 300
 301                 featureAddress[featureAddressLines].type = xmlTextReaderGetAttribute(reader, BAD_CAST "type");
 302             featureAddress[featureAddressLines].id = xmlTextReaderGetAttribute(reader, BAD_CAST "id");
 303                 featureAddress[featureAddressLines].key = xmlTextReaderGetAttribute(reader, BAD_CAST "key");
 304                 featureAddress[featureAddressLines].value = xmlTextReaderGetAttribute(reader, BAD_CAST "value");
 305             featureAddress[featureAddressLines].distance = xmlTextReaderGetAttribute(reader, BAD_CAST "distance");
 306
 307             featureAddressLines++;
 308                 }
 309                 else
 310         {
 311             fprintf( stderr, "Too many address elements (%s%s)\n", feature.type, feature.id);
 312 //            exit_nicely();
 313         }
 314
 315         return;
 316     }
 317     fprintf(stderr, "%s: Unknown element name: %s\n", __FUNCTION__, name);
 318 }
 319
 320 void EndElement(xmlTextReaderPtr reader, const xmlChar *name)
 321 {
 322     PGresult *          res;
 323     const char *        paramValues[11];
 324     char *                      place_id;
 325     char *                      partionQueryName;
 326     int i, namePos, lineTypeLen, lineValueLen;
 327
 328     if (xmlStrEqual(name, BAD_CAST "feature"))
 329     {
 330         featureCount++;
 331         if (featureCount % 1000 == 0) printf("feature %i(k)\n", featureCount/1000);
 332 /*
 333         if (fileMode == FILEMODE_ADD)
 334         {
 335             resPlaceID = PQexecPrepared(conn, "get_new_place_id", 0, NULL, NULL, NULL, 0);
 336             if (PQresultStatus(resPlaceID) != PGRES_TUPLES_OK)
 337             {
 338                 fprintf(stderr, "get_place_id: INSERT failed: %s", PQerrorMessage(conn));
 339                 PQclear(resPlaceID);
 340                 exit(EXIT_FAILURE);
 341             }
 342         }
 343         else
 344         {
 345             paramValues[0] = (const char *)feature.type;
 346             paramValues[1] = (const char *)feature.id;
 347             paramValues[2] = (const char *)feature.key;
 348             paramValues[3] = (const char *)feature.value;
 349             resPlaceID = PQexecPrepared(conn, "get_new_place_id", 4, paramValues, NULL, NULL, 0);
 350             if (PQresultStatus(resPlaceID) != PGRES_TUPLES_OK)
 351             {
 352                 fprintf(stderr, "index_placex: INSERT failed: %s", PQerrorMessage(conn));
 353                 PQclear(resPlaceID);
 354                 exit(EXIT_FAILURE);
 355             }
 356         }
 357 */
 358         place_id = (char *)feature.placeID;
 359
 360         if (fileMode == FILEMODE_UPDATE || fileMode == FILEMODE_DELETE || fileMode == FILEMODE_ADD)
 361         {
 362             paramValues[0] = (const char *)place_id;
 363             res = PQexecPrepared(conn, "placex_delete", 1, paramValues, NULL, NULL, 0);
 364             if (PQresultStatus(res) != PGRES_COMMAND_OK)
 365             {
 366                 fprintf(stderr, "placex_delete: DELETE failed: %s", PQerrorMessage(conn));
 367                 PQclear(res);
 368                 exit(EXIT_FAILURE);
 369             }
 370             PQclear(res);
 371
 372             res = PQexecPrepared(conn, "search_name_delete", 1, paramValues, NULL, NULL, 0);
 373             if (PQresultStatus(res) != PGRES_COMMAND_OK)
 374             {
 375                 fprintf(stderr, "search_name_delete: DELETE failed: %s", PQerrorMessage(conn));
 376                 PQclear(res);
 377                 exit(EXIT_FAILURE);
 378             }
 379             PQclear(res);
 380
 381             res = PQexecPrepared(conn, "place_addressline_delete", 1, paramValues, NULL, NULL, 0);
 382             if (PQresultStatus(res) != PGRES_COMMAND_OK)
 383             {
 384                 fprintf(stderr, "place_addressline_delete: DELETE failed: %s", PQerrorMessage(conn));
 385                 PQclear(res);
 386                 exit(EXIT_FAILURE);
 387             }
 388             PQclear(res);
 389
 390             partionQueryName = xmlHashLookup2(partionTableTagsHashDelete, feature.key, feature.value);
 391             if (partionQueryName)
 392             {
 393                 res = PQexecPrepared(conn, partionQueryName, 1, paramValues, NULL, NULL, 0);
 394                 if (PQresultStatus(res) != PGRES_COMMAND_OK)
 395                 {
 396                     fprintf(stderr, "%s: DELETE failed: %s", partionQueryName, PQerrorMessage(conn));
 397                     PQclear(res);
 398                     exit(EXIT_FAILURE);
 399                 }
 400                 PQclear(res);
 401             }
 402         }
 403
 404         if (fileMode == FILEMODE_UPDATE || fileMode == FILEMODE_ADD)
 405         {
 406             // Insert into placex
 407             paramValues[0] = (const char *)place_id;
 408             paramValues[1] = (const char *)feature.type;
 409             paramValues[2] = (const char *)feature.id;
 410             paramValues[3] = (const char *)feature.key;
 411             paramValues[4] = (const char *)feature.value;
 412
 413             featureNameString[0] = 0;
 414             if (featureNameLines)
 415             {
 416                 namePos = 0;
 417                 lineTypeLen = 0;
 418                 lineValueLen = 0;
 419                 for (i = 0; i < featureNameLines; i++)
 420                 {
 421                     lineTypeLen = (int)strlen((char *) featureName[i].type);
 422                     lineValueLen = (int)strlen((char *) featureName[i].value);
 423                     if (namePos+lineTypeLen+lineValueLen+7 > MAX_FEATURENAMESTRING)
 424                     {
 425                         fprintf(stderr, "feature name too long: %s", (const char *)featureName[i].value);
 426                         break;
 427                     }
 428                     if (namePos) strcpy(featureNameString+(namePos++), ",");
 429                     strcpy(featureNameString+(namePos++), "\"");
 430                     strcpy(featureNameString+namePos, (char*) featureName[i].type);
 431                     namePos += lineTypeLen;
 432                     strcpy(featureNameString+namePos, "\"=>\"");
 433                     namePos += 4;
 434                     strcpy(featureNameString+namePos, (char *) featureName[i].value);
 435                     namePos += lineValueLen;
 436                     strcpy(featureNameString+(namePos++), "\"");
 437                 }
 438             }
 439             paramValues[5] = (const char *)featureNameString;
 440
 441             featureExtraTagString[0] = 0;
 442             if (featureExtraTagLines)
 443             {
 444                 namePos = 0;
 445                 lineTypeLen = 0;
 446                 lineValueLen = 0;
 447                 for (i = 0; i < featureExtraTagLines; i++)
 448                 {
 449                     lineTypeLen = strlen((char *) featureExtraTag[i].type);
 450                     lineValueLen = strlen((char *) featureExtraTag[i].value);
 451                     if (namePos+lineTypeLen+lineValueLen+7 > MAX_FEATUREEXTRATAGSTRING)
 452                     {
 453                         fprintf(stderr, "feature extra tag too long: %s", (const char *)featureExtraTag[i].value);
 454                         break;
 455                     }
 456                     if (namePos) strcpy(featureExtraTagString+(namePos++),",");
 457                     strcpy(featureExtraTagString+(namePos++), "\"");
 458                     strcpy(featureExtraTagString+namePos, (char *) featureExtraTag[i].type);
 459                     namePos += lineTypeLen;
 460                     strcpy(featureExtraTagString+namePos, "\"=>\"");
 461                     namePos += 4;
 462                     strcpy(featureExtraTagString+namePos, (char *) featureExtraTag[i].value);
 463                     namePos += lineValueLen;
 464                     strcpy(featureExtraTagString+(namePos++), "\"");
 465                 }
 466             }
 467             paramValues[6] = (const char *)featureExtraTagString;
 468
 469             paramValues[7] = (const char *)feature.parentPlaceID;
 470
 471             paramValues[8] = (const char *)feature.adminLevel;
 472             paramValues[9] = (const char *)feature.houseNumber;
 473             paramValues[10] = (const char *)feature.rankAddress;
 474             paramValues[11] = (const char *)feature.rankSearch;
 475             paramValues[12] = (const char *)feature.geometry;
 476             if (strlen(paramValues[3]))
 477             {
 478                 res = PQexecPrepared(conn, "placex_insert", 13, paramValues, NULL, NULL, 0);
 479                 if (PQresultStatus(res) != PGRES_COMMAND_OK)
 480                 {
 481                     fprintf(stderr, "index_placex: INSERT failed: %s", PQerrorMessage(conn));
 482                     fprintf(stderr, "index_placex: INSERT failed: %s %s %s", paramValues[0], paramValues[1], paramValues[2]);
 483                     PQclear(res);
 484                     exit(EXIT_FAILURE);
 485                }
 486                PQclear(res);
 487             }
 488
 489             for (i = 0; i < featureAddressLines; i++)
 490             {
 491                 // insert into place_address
 492                 paramValues[0] = (const char *)place_id;
 493                 paramValues[1] = (const char *)featureAddress[i].distance;
 494                 paramValues[2] = (const char *)featureAddress[i].type;
 495                 paramValues[3] = (const char *)featureAddress[i].id;
 496                 paramValues[4] = (const char *)featureAddress[i].key;
 497                 paramValues[5] = (const char *)featureAddress[i].value;
 498                 paramValues[6] = (const char *)featureAddress[i].isAddress;
 499                 res = PQexecPrepared(conn, "place_addressline_insert", 7, paramValues, NULL, NULL, 0);
 500                 if (PQresultStatus(res) != PGRES_COMMAND_OK)
 501                 {
 502                     fprintf(stderr, "place_addressline_insert: INSERT failed: %s", PQerrorMessage(conn));
 503                     PQclear(res);
 504                     exit(EXIT_FAILURE);
 505                 }
 506                 PQclear(res);
 507
 508                 xmlFree(featureAddress[i].type);
 509                 xmlFree(featureAddress[i].id);
 510                 xmlFree(featureAddress[i].key);
 511                 xmlFree(featureAddress[i].value);
 512                 xmlFree(featureAddress[i].distance);
 513             }
 514
 515             if (featureNameLines)
 516             {
 517                 paramValues[0] = (const char *)place_id;
 518                 res = PQexecPrepared(conn, "search_name_insert", 1, paramValues, NULL, NULL, 0);
 519                 if (PQresultStatus(res) != PGRES_COMMAND_OK)
 520                 {
 521                     fprintf(stderr, "search_name_insert: INSERT failed: %s", PQerrorMessage(conn));
 522                     PQclear(res);
 523                     exit(EXIT_FAILURE);
 524                 }
 525                 PQclear(res);
 526             }
 527
 528             partionQueryName = xmlHashLookup2(partionTableTagsHash, feature.key, feature.value);
 529             if (partionQueryName)
 530             {
 531                 // insert into partition table
 532                 paramValues[0] = (const char *)place_id;
 533                 paramValues[1] = (const char *)feature.geometry;
 534                 res = PQexecPrepared(conn, partionQueryName, 2, paramValues, NULL, NULL, 0);
 535                 if (PQresultStatus(res) != PGRES_COMMAND_OK)
 536                 {
 537                     fprintf(stderr, "%s: INSERT failed: %s", partionQueryName, PQerrorMessage(conn));
 538                     PQclear(res);
 539                     exit(EXIT_FAILURE);
 540                 }
 541                 PQclear(res);
 542             }
 543
 544         }
 545         else
 546         {
 547             for (i = 0; i < featureAddressLines; i++)
 548             {
 549                 xmlFree(featureAddress[i].type);
 550                 xmlFree(featureAddress[i].id);
 551                 xmlFree(featureAddress[i].key);
 552                 xmlFree(featureAddress[i].value);
 553                 xmlFree(featureAddress[i].distance);
 554             }
 555         }
 556
 557         xmlFree(feature.placeID);
 558         xmlFree(feature.type);
 559         xmlFree(feature.id);
 560         xmlFree(feature.key);
 561         xmlFree(feature.value);
 562         xmlFree(feature.rankAddress);
 563         xmlFree(feature.rankSearch);
 564 //              if (feature.name) xmlFree(feature.name);
 565         if (feature.countryCode) xmlFree(feature.countryCode);
 566         if (feature.adminLevel) xmlFree(feature.adminLevel);
 567         if (feature.houseNumber) xmlFree(feature.houseNumber);
 568         if (feature.geometry) xmlFree(feature.geometry);
 569
 570 //        PQclear(resPlaceID);
 571     }
 572 }
 573
 574 static void processNode(xmlTextReaderPtr reader)
 575 {
 576     xmlChar *name;
 577     name = xmlTextReaderName(reader);
 578     if (name == NULL)
 579     {
 580         name = xmlStrdup(BAD_CAST "--");
 581     }
 582
 583     switch (xmlTextReaderNodeType(reader))
 584     {
 585     case XML_READER_TYPE_ELEMENT:
 586         StartElement(reader, name);
 587         if (xmlTextReaderIsEmptyElement(reader))
 588             EndElement(reader, name); /* No end_element for self closing tags! */
 589         break;
 590     case XML_READER_TYPE_END_ELEMENT:
 591         EndElement(reader, name);
 592         break;
 593     case XML_READER_TYPE_TEXT:
 594     case XML_READER_TYPE_CDATA:
 595     case XML_READER_TYPE_SIGNIFICANT_WHITESPACE:
 596         /* Ignore */
 597         break;
 598     default:
 599         fprintf(stderr, "Unknown node type %d\n", xmlTextReaderNodeType(reader));
 600         break;
 601     }
 602
 603     xmlFree(name);
 604 }
 605
 606 int nominatim_import(const char *conninfo, const char *partionTagsFilename, const char *filename)
 607 {
 608     xmlTextReaderPtr    reader;
 609     int                                 ret = 0;
 610     PGresult *                  res;
 611     FILE *                              partionTagsFile;
 612     char *                              partionQueryName;
 613     char                                partionQuerySQL[1024];
 614
 615     conn = PQconnectdb(conninfo);
 616     if (PQstatus(conn) != CONNECTION_OK)
 617     {
 618         fprintf(stderr, "Connection to database failed: %s\n", PQerrorMessage(conn));
 619         exit(EXIT_FAILURE);
 620     }
 621
 622     partionTableTagsHash = xmlHashCreate(200);
 623     partionTableTagsHashDelete = xmlHashCreate(200);
 624
 625     partionTagsFile = fopen(partionTagsFilename, "rt");
 626     if (!partionTagsFile)
 627     {
 628         fprintf(stderr, "Unable to read partition tags file: %s\n", partionTagsFilename);
 629         exit(EXIT_FAILURE);
 630     }
 631
 632     char buffer[1024], osmkey[256], osmvalue[256];
 633     int fields;
 634     while (fgets(buffer, sizeof(buffer), partionTagsFile) != NULL)
 635     {
 636         fields = sscanf( buffer, "%23s %63s", osmkey, osmvalue );
 637
 638         if ( fields <= 0 ) continue;
 639
 640         if ( fields != 2  )
 641         {
 642             fprintf( stderr, "Error partition file\n");
 643             exit_nicely();
 644         }
 645         partionQueryName = malloc(strlen("partition_insert_")+strlen(osmkey)+strlen(osmvalue)+2);
 646         strcpy(partionQueryName, "partition_insert_");
 647         strcat(partionQueryName, osmkey);
 648         strcat(partionQueryName, "_");
 649         strcat(partionQueryName, osmvalue);
 650
 651         strcpy(partionQuerySQL, "insert into place_classtype_");
 652         strcat(partionQuerySQL, osmkey);
 653         strcat(partionQuerySQL, "_");
 654         strcat(partionQuerySQL, osmvalue);
 655         strcat(partionQuerySQL, " (place_id, centroid) values ($1, ST_Centroid(st_setsrid($2, 4326)))");
 656
 657         res = PQprepare(conn, partionQueryName, partionQuerySQL, 2, NULL);
 658         if (PQresultStatus(res) != PGRES_COMMAND_OK)
 659         {
 660             fprintf(stderr, "Failed to prepare %s: %s\n", partionQueryName, PQerrorMessage(conn));
 661             exit(EXIT_FAILURE);
 662         }
 663
 664         xmlHashAddEntry2(partionTableTagsHash, BAD_CAST osmkey, BAD_CAST osmvalue, BAD_CAST partionQueryName);
 665
 666         partionQueryName = malloc(strlen("partition_delete_")+strlen(osmkey)+strlen(osmvalue)+2);
 667         strcpy(partionQueryName, "partition_delete_");
 668         strcat(partionQueryName, osmkey);
 669         strcat(partionQueryName, "_");
 670         strcat(partionQueryName, osmvalue);
 671
 672         strcpy(partionQuerySQL, "delete from place_classtype_");
 673         strcat(partionQuerySQL, osmkey);
 674         strcat(partionQuerySQL, "_");
 675         strcat(partionQuerySQL, osmvalue);
 676         strcat(partionQuerySQL, " where place_id = $1::integer");
 677
 678         res = PQprepare(conn, partionQueryName, partionQuerySQL, 1, NULL);
 679         if (PQresultStatus(res) != PGRES_COMMAND_OK)
 680         {
 681             fprintf(stderr, "Failed to prepare %s: %s\n", partionQueryName, PQerrorMessage(conn));
 682             exit(EXIT_FAILURE);
 683         }
 684
 685         xmlHashAddEntry2(partionTableTagsHashDelete, BAD_CAST osmkey, BAD_CAST osmvalue, BAD_CAST partionQueryName);
 686     }
 687
 688     res = PQprepare(conn, "get_new_place_id",
 689                     "select nextval('seq_place')",
 690                     0, NULL);
 691     if (PQresultStatus(res) != PGRES_COMMAND_OK)
 692     {
 693         fprintf(stderr, "Failed to prepare get_new_place_id: %s\n", PQerrorMessage(conn));
 694         exit(EXIT_FAILURE);
 695     }
 696
 697     res = PQprepare(conn, "get_place_id",
 698                     "select place_id from placex where osm_type = $1 and osm_id = $2 and class = $3 and type = $4",
 699                     4, NULL);
 700     if (PQresultStatus(res) != PGRES_COMMAND_OK)
 701     {
 702         fprintf(stderr, "Failed to prepare get_place_id: %s\n", PQerrorMessage(conn));
 703         exit(EXIT_FAILURE);
 704     }
 705
 706     res = PQprepare(conn, "placex_insert",
 707                     "insert into placex (place_id,osm_type,osm_id,class,type,name,extratags,parent_place_id,admin_level,housenumber,rank_address,rank_search,geometry) "
 708                     "values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, st_setsrid($13, 4326))",
 709                     12, NULL);
 710     if (PQresultStatus(res) != PGRES_COMMAND_OK)
 711     {
 712         fprintf(stderr, "Failed to prepare placex_insert: %s\n", PQerrorMessage(conn));
 713         exit(EXIT_FAILURE);
 714     }
 715
 716     res = PQprepare(conn, "search_name_insert",
 717                     "insert into search_name (place_id, search_rank, address_rank, country_code, name_vector, nameaddress_vector, centroid) "
 718                     "select place_id, rank_address, rank_search, country_code, make_keywords(name), "
 719                     "(select uniq(sort(array_agg(name_vector))) from place_addressline join search_name on "
 720                     "(address_place_id = search_name.place_id) where place_addressline.place_id = $1 ), st_centroid(geometry) from placex "
 721                     "where place_id = $1",
 722                     1, NULL);
 723     if (PQresultStatus(res) != PGRES_COMMAND_OK)
 724     {
 725         fprintf(stderr, "Failed to prepare search_name_insert: %s\n", PQerrorMessage(conn));
 726         exit(EXIT_FAILURE);
 727     }
 728
 729     res = PQprepare(conn, "place_addressline_insert",
 730                     "insert into place_addressline (place_id, address_place_id, fromarea, isaddress, distance, cached_rank_address) "
 731                     "select $1, place_id, false, $7, $2, rank_address from placex where osm_type = $3 and osm_id = $4 and class = $5 and type = $6",
 732                     7, NULL);
 733     if (PQresultStatus(res) != PGRES_COMMAND_OK)
 734     {
 735         fprintf(stderr, "Failed to prepare place_addressline_insert: %s\n", PQerrorMessage(conn));
 736         exit(EXIT_FAILURE);
 737     }
 738
 739     res = PQprepare(conn, "placex_delete",
 740                     "delete from placex where place_id = $1",
 741                     1, NULL);
 742     if (PQresultStatus(res) != PGRES_COMMAND_OK)
 743     {
 744         fprintf(stderr, "Failed to prepare placex_delete: %s\n", PQerrorMessage(conn));
 745         exit(EXIT_FAILURE);
 746     }
 747
 748     res = PQprepare(conn, "search_name_delete",
 749                     "delete from search_name where place_id = $1",
 750                     1, NULL);
 751     if (PQresultStatus(res) != PGRES_COMMAND_OK)
 752     {
 753         fprintf(stderr, "Failed to prepare search_name_delete: %s\n", PQerrorMessage(conn));
 754         exit(EXIT_FAILURE);
 755     }
 756
 757     res = PQprepare(conn, "place_addressline_delete",
 758                     "delete from place_addressline where place_id = $1",
 759                     1, NULL);
 760     if (PQresultStatus(res) != PGRES_COMMAND_OK)
 761     {
 762         fprintf(stderr, "Failed to prepare place_addressline_delete: %s\n", PQerrorMessage(conn));
 763         exit(EXIT_FAILURE);
 764     }
 765
 766     featureCount = 0;
 767
 768     reader = inputUTF8(filename);
 769
 770     if (reader == NULL)
 771     {
 772         fprintf(stderr, "Unable to open %s\n", filename);
 773         return 1;
 774     }
 775
 776     ret = xmlTextReaderRead(reader);
 777     while (ret == 1)
 778     {
 779         processNode(reader);
 780         ret = xmlTextReaderRead(reader);
 781     }
 782     if (ret != 0)
 783     {
 784         fprintf(stderr, "%s : failed to parse\n", filename);
 785         return ret;
 786     }
 787
 788     xmlFreeTextReader(reader);
 789     xmlHashFree(partionTableTagsHash, NULL);
 790     xmlHashFree(partionTableTagsHashDelete, NULL);
 791
 792     return 0;
 793 }