nominatim/import.c

   1 /*
   2 */
   3 #include <stdlib.h>
   4 #include <string.h>
   5
   6 #include <libpq-fe.h>
   7
   8 #include <libxml/xmlstring.h>
   9 #include <libxml/xmlreader.h>
  10 #include <libxml/hash.h>
  11
  12 #include "nominatim.h"
  13 #include "import.h"
  14 #include "input.h"
  15
  16 typedef enum { FILETYPE_NONE, FILETYPE_STRUCTUREDV0P1 } filetypes_t;
  17 typedef enum { FILEMODE_NONE, FILEMODE_ADD, FILEMODE_UPDATE, FILEMODE_DELETE } filemodes_t;
  18
  19 #define MAX_FEATUREADDRESS 5000
  20 #define MAX_FEATURENAMES 10000
  21 #define MAX_FEATUREEXTRATAGS 10000
  22 #define MAX_FEATURENAMESTRING 1000000
  23 #define MAX_FEATUREEXTRATAGSTRING 500000
  24
  25 struct feature_address
  26 {
  27     int                 place_id;
  28     int                 rankAddress;
  29     char                        isAddress[2];
  30     xmlChar *   type;
  31     xmlChar *   id;
  32     xmlChar *   key;
  33     xmlChar *   value;
  34     xmlChar *   distance;
  35 };
  36
  37 struct feature_tag
  38 {
  39     xmlChar *   type;
  40     xmlChar *   value;
  41 };
  42
  43 struct feature
  44 {
  45     xmlChar *   placeID;
  46     xmlChar *   type;
  47     xmlChar *   id;
  48     xmlChar *   key;
  49     xmlChar *   value;
  50     xmlChar *   rankAddress;
  51     xmlChar *   rankSearch;
  52     xmlChar *   countryCode;
  53     xmlChar *   parentPlaceID;
  54     xmlChar *   parentType;
  55     xmlChar *   parentID;
  56     xmlChar *   adminLevel;
  57     xmlChar *   houseNumber;
  58     xmlChar *   geometry;
  59 } feature;
  60
  61 int                                     fileType = FILETYPE_NONE;
  62 int                                     fileMode = FILEMODE_ADD;
  63 PGconn *                                conn;
  64 struct feature_address  featureAddress[MAX_FEATUREADDRESS];
  65 struct feature_tag              featureName[MAX_FEATURENAMES];
  66 struct feature_tag              featureExtraTag[MAX_FEATUREEXTRATAGS];
  67 struct feature                  feature;
  68 int                                     featureAddressLines = 0;
  69 int                                     featureNameLines = 0;
  70 int                                     featureExtraTagLines = 0;
  71 int                                     featureCount = 0;
  72 xmlHashTablePtr                 partionTableTagsHash;
  73 xmlHashTablePtr                 partionTableTagsHashDelete;
  74 char                                    featureNameString[MAX_FEATURENAMESTRING];
  75 char                                    featureExtraTagString[MAX_FEATUREEXTRATAGSTRING];
  76
  77 extern int verbose;
  78
  79 void StartElement(xmlTextReaderPtr reader, const xmlChar *name)
  80 {
  81     char * value;
  82     float version;
  83     int isAddressLine;
  84
  85     if (fileType == FILETYPE_NONE)
  86     {
  87         // Potential to handle other file types in the future / versions
  88         if (xmlStrEqual(name, BAD_CAST "osmStructured"))
  89         {
  90             value = (char*)xmlTextReaderGetAttribute(reader, BAD_CAST "version");
  91             version = strtof(value, NULL);
  92             xmlFree(value);
  93
  94             if (version == (float)0.1)
  95             {
  96                 fileType = FILETYPE_STRUCTUREDV0P1;
  97                 fileMode = FILEMODE_ADD;
  98             }
  99             else
 100             {
 101                 fprintf( stderr, "Unknown osmStructured version %f (%s)\n", version, value );
 102                 exit_nicely();
 103             }
 104         }
 105         else
 106         {
 107             fprintf( stderr, "Unknown XML document type: %s\n", name );
 108             exit_nicely();
 109         }
 110         return;
 111     }
 112
 113     if (xmlStrEqual(name, BAD_CAST "add"))
 114     {
 115         fileMode = FILEMODE_ADD;
 116         return;
 117     }
 118     if (xmlStrEqual(name, BAD_CAST "update"))
 119     {
 120         fileMode = FILEMODE_UPDATE;
 121         return;
 122     }
 123     if (xmlStrEqual(name, BAD_CAST "delete"))
 124     {
 125         fileMode = FILEMODE_DELETE;
 126         return;
 127     }
 128     if (fileMode == FILEMODE_NONE)
 129     {
 130         fprintf( stderr, "Unknown import mode in: %s\n", name );
 131         exit_nicely();
 132     }
 133
 134     if (xmlStrEqual(name, BAD_CAST "feature"))
 135     {
 136         feature.placeID = xmlTextReaderGetAttribute(reader, BAD_CAST "place_id");
 137         feature.type = xmlTextReaderGetAttribute(reader, BAD_CAST "type");
 138         feature.id = xmlTextReaderGetAttribute(reader, BAD_CAST "id");
 139         feature.key = xmlTextReaderGetAttribute(reader, BAD_CAST "key");
 140         feature.value = xmlTextReaderGetAttribute(reader, BAD_CAST "value");
 141         feature.rankAddress = xmlTextReaderGetAttribute(reader, BAD_CAST "rank");
 142         feature.rankSearch = xmlTextReaderGetAttribute(reader, BAD_CAST "importance");
 143
 144         feature.parentPlaceID = xmlTextReaderGetAttribute(reader, BAD_CAST "parent_place_id");
 145 /*
 146         if (strlen(feature.parentPlaceID) == 0)
 147         {
 148                 xmlFree(feature.parentPlaceID);
 149                 feature.parentPlaceID = NULL;
 150         }
 151 */
 152         feature.parentType = xmlTextReaderGetAttribute(reader, BAD_CAST "parent_type");
 153         feature.parentID = xmlTextReaderGetAttribute(reader, BAD_CAST "parent_id");
 154
 155         feature.countryCode = NULL;
 156         feature.adminLevel = NULL;
 157         feature.houseNumber = NULL;
 158         feature.geometry = NULL;
 159         featureAddressLines = 0;
 160         featureNameLines = 0;
 161         featureExtraTagLines = 0;
 162
 163         return;
 164     }
 165     if (xmlStrEqual(name, BAD_CAST "names")) return;
 166     if (xmlStrEqual(name, BAD_CAST "name"))
 167     {
 168         if (featureNameLines < MAX_FEATURENAMES)
 169         {
 170                 featureName[featureNameLines].type = xmlTextReaderGetAttribute(reader, BAD_CAST "type");
 171             featureName[featureNameLines].value = xmlTextReaderReadString(reader);
 172                 featureNameLines++;
 173                 }
 174                 else
 175                 {
 176             fprintf( stderr, "Too many name elements (%s%s)\n", feature.type, feature.id);
 177 //            exit_nicely();
 178         }
 179         return;
 180     }
 181     if (xmlStrEqual(name, BAD_CAST "tags")) return;
 182     if (xmlStrEqual(name, BAD_CAST "tag"))
 183     {
 184         if (featureExtraTagLines < MAX_FEATUREEXTRATAGS)
 185                 {
 186                 featureExtraTag[featureExtraTagLines].type = xmlTextReaderGetAttribute(reader, BAD_CAST "type");
 187             featureExtraTag[featureExtraTagLines].value = xmlTextReaderReadString(reader);
 188                 featureExtraTagLines++;
 189                 }
 190                 else
 191         {
 192             fprintf( stderr, "Too many extra tag elements (%s%s)\n", feature.type, feature.id);
 193 //            exit_nicely();
 194         }
 195         return;
 196     }
 197     if (xmlStrEqual(name, BAD_CAST "osmGeometry"))
 198     {
 199         feature.geometry = xmlTextReaderReadString(reader);
 200         return;
 201     }
 202     if (xmlStrEqual(name, BAD_CAST "adminLevel"))
 203     {
 204         feature.adminLevel = xmlTextReaderReadString(reader);
 205         return;
 206     }
 207     if (xmlStrEqual(name, BAD_CAST "countryCode"))
 208     {
 209         feature.countryCode = xmlTextReaderReadString(reader);
 210         return;
 211     }
 212     if (xmlStrEqual(name, BAD_CAST "houseNumber"))
 213     {
 214         feature.houseNumber = xmlTextReaderReadString(reader);
 215         return;
 216     }
 217     if (xmlStrEqual(name, BAD_CAST "address"))
 218     {
 219         featureAddressLines = 0;
 220         return;
 221     }
 222     isAddressLine = 0;
 223     if (xmlStrEqual(name, BAD_CAST "continent"))
 224     {
 225         isAddressLine = 1;
 226     }
 227     else if (xmlStrEqual(name, BAD_CAST "sea"))
 228     {
 229         isAddressLine = 1;
 230     }
 231     else if (xmlStrEqual(name, BAD_CAST "country"))
 232     {
 233         isAddressLine = 1;
 234     }
 235     else if (xmlStrEqual(name, BAD_CAST "state"))
 236     {
 237         isAddressLine = 1;
 238     }
 239     else if (xmlStrEqual(name, BAD_CAST "county"))
 240     {
 241         isAddressLine = 1;
 242     }
 243     else if (xmlStrEqual(name, BAD_CAST "city"))
 244     {
 245         isAddressLine = 1;
 246     }
 247     else if (xmlStrEqual(name, BAD_CAST "town"))
 248     {
 249         isAddressLine = 1;
 250     }
 251     else if (xmlStrEqual(name, BAD_CAST "village"))
 252     {
 253         isAddressLine = 1;
 254     }
 255     else if (xmlStrEqual(name, BAD_CAST "unknown"))
 256     {
 257         isAddressLine = 1;
 258     }
 259     else if (xmlStrEqual(name, BAD_CAST "suburb"))
 260     {
 261         isAddressLine = 1;
 262     }
 263     else if (xmlStrEqual(name, BAD_CAST "postcode"))
 264     {
 265         isAddressLine = 1;
 266     }
 267     else if (xmlStrEqual(name, BAD_CAST "neighborhood"))
 268     {
 269         isAddressLine = 1;
 270     }
 271     else if (xmlStrEqual(name, BAD_CAST "street"))
 272     {
 273         isAddressLine = 1;
 274     }
 275     else if (xmlStrEqual(name, BAD_CAST "access"))
 276     {
 277         isAddressLine = 1;
 278     }
 279     else if (xmlStrEqual(name, BAD_CAST "building"))
 280     {
 281         isAddressLine = 1;
 282     }
 283     else if (xmlStrEqual(name, BAD_CAST "other"))
 284     {
 285         isAddressLine = 1;
 286     }
 287     if (isAddressLine)
 288     {
 289         if (featureAddressLines < MAX_FEATUREADDRESS)
 290                 {
 291                 value = (char*)xmlTextReaderGetAttribute(reader, BAD_CAST "rank");
 292             if (!value)
 293                 {
 294                 fprintf( stderr, "Address element missing rank\n");
 295                     exit_nicely();
 296                 }
 297             featureAddress[featureAddressLines].rankAddress =  atoi(value);
 298                 xmlFree(value);
 299
 300                 value = (char*)xmlTextReaderGetAttribute(reader, BAD_CAST "isaddress");
 301             if (!value)
 302                 {
 303                     fprintf( stderr, "Address element missing rank\n");
 304                 exit_nicely();
 305                 }
 306                 if (*value == 't') strcpy(featureAddress[featureAddressLines].isAddress, "t");
 307             else strcpy(featureAddress[featureAddressLines].isAddress, "f");
 308                 xmlFree(value);
 309
 310                 featureAddress[featureAddressLines].type = xmlTextReaderGetAttribute(reader, BAD_CAST "type");
 311             featureAddress[featureAddressLines].id = xmlTextReaderGetAttribute(reader, BAD_CAST "id");
 312                 featureAddress[featureAddressLines].key = xmlTextReaderGetAttribute(reader, BAD_CAST "key");
 313                 featureAddress[featureAddressLines].value = xmlTextReaderGetAttribute(reader, BAD_CAST "value");
 314             featureAddress[featureAddressLines].distance = xmlTextReaderGetAttribute(reader, BAD_CAST "distance");
 315
 316             featureAddressLines++;
 317                 }
 318                 else
 319         {
 320             fprintf( stderr, "Too many address elements (%s%s)\n", feature.type, feature.id);
 321 //            exit_nicely();
 322         }
 323
 324         return;
 325     }
 326     fprintf(stderr, "%s: Unknown element name: %s\n", __FUNCTION__, name);
 327 }
 328
 329 void EndElement(xmlTextReaderPtr reader, const xmlChar *name)
 330 {
 331     PGresult *          res;
 332     const char *        paramValues[14];
 333     char *                      place_id;
 334     char *                      partionQueryName;
 335     int i, namePos, lineTypeLen, lineValueLen;
 336
 337     if (xmlStrEqual(name, BAD_CAST "feature"))
 338     {
 339         featureCount++;
 340         if (featureCount % 1000 == 0) printf("feature %i(k)\n", featureCount/1000);
 341 /*
 342         if (fileMode == FILEMODE_ADD)
 343         {
 344             resPlaceID = PQexecPrepared(conn, "get_new_place_id", 0, NULL, NULL, NULL, 0);
 345             if (PQresultStatus(resPlaceID) != PGRES_TUPLES_OK)
 346             {
 347                 fprintf(stderr, "get_place_id: INSERT failed: %s", PQerrorMessage(conn));
 348                 PQclear(resPlaceID);
 349                 exit(EXIT_FAILURE);
 350             }
 351         }
 352         else
 353         {
 354             paramValues[0] = (const char *)feature.type;
 355             paramValues[1] = (const char *)feature.id;
 356             paramValues[2] = (const char *)feature.key;
 357             paramValues[3] = (const char *)feature.value;
 358             resPlaceID = PQexecPrepared(conn, "get_new_place_id", 4, paramValues, NULL, NULL, 0);
 359             if (PQresultStatus(resPlaceID) != PGRES_TUPLES_OK)
 360             {
 361                 fprintf(stderr, "index_placex: INSERT failed: %s", PQerrorMessage(conn));
 362                 PQclear(resPlaceID);
 363                 exit(EXIT_FAILURE);
 364             }
 365         }
 366 */
 367         place_id = (char *)feature.placeID;
 368
 369         if (fileMode == FILEMODE_UPDATE || fileMode == FILEMODE_DELETE || fileMode == FILEMODE_ADD)
 370         {
 371             paramValues[0] = (const char *)place_id;
 372             if (verbose) fprintf(stderr, "placex_delete: %s\n", paramValues[0]);
 373             res = PQexecPrepared(conn, "placex_delete", 1, paramValues, NULL, NULL, 0);
 374             if (PQresultStatus(res) != PGRES_COMMAND_OK)
 375             {
 376                 fprintf(stderr, "placex_delete: DELETE failed: %s", PQerrorMessage(conn));
 377                 PQclear(res);
 378                 exit(EXIT_FAILURE);
 379             }
 380             PQclear(res);
 381
 382             if (verbose) fprintf(stderr, "search_name_delete: %s\n", paramValues[0]);
 383             res = PQexecPrepared(conn, "search_name_delete", 1, paramValues, NULL, NULL, 0);
 384             if (PQresultStatus(res) != PGRES_COMMAND_OK)
 385             {
 386                 fprintf(stderr, "search_name_delete: DELETE failed: %s", PQerrorMessage(conn));
 387                 PQclear(res);
 388                 exit(EXIT_FAILURE);
 389             }
 390             PQclear(res);
 391
 392             if (verbose) fprintf(stderr, "place_addressline_delete: %s\n", paramValues[0]);
 393             res = PQexecPrepared(conn, "place_addressline_delete", 1, paramValues, NULL, NULL, 0);
 394             if (PQresultStatus(res) != PGRES_COMMAND_OK)
 395             {
 396                 fprintf(stderr, "place_addressline_delete: DELETE failed: %s", PQerrorMessage(conn));
 397                 PQclear(res);
 398                 exit(EXIT_FAILURE);
 399             }
 400             PQclear(res);
 401
 402             partionQueryName = xmlHashLookup2(partionTableTagsHashDelete, feature.key, feature.value);
 403             if (partionQueryName)
 404             {
 405                 res = PQexecPrepared(conn, partionQueryName, 1, paramValues, NULL, NULL, 0);
 406                 if (PQresultStatus(res) != PGRES_COMMAND_OK)
 407                 {
 408                     fprintf(stderr, "%s: DELETE failed: %s", partionQueryName, PQerrorMessage(conn));
 409                     PQclear(res);
 410                     exit(EXIT_FAILURE);
 411                 }
 412                 PQclear(res);
 413             }
 414         }
 415
 416         if (fileMode == FILEMODE_UPDATE || fileMode == FILEMODE_ADD)
 417         {
 418             // Insert into placex
 419             paramValues[0] = (const char *)place_id;
 420             paramValues[1] = (const char *)feature.type;
 421             paramValues[2] = (const char *)feature.id;
 422             paramValues[3] = (const char *)feature.key;
 423             paramValues[4] = (const char *)feature.value;
 424
 425             featureNameString[0] = 0;
 426             if (featureNameLines)
 427             {
 428                 namePos = 0;
 429                 lineTypeLen = 0;
 430                 lineValueLen = 0;
 431                 for (i = 0; i < featureNameLines; i++)
 432                 {
 433                     lineTypeLen = (int)strlen((char *) featureName[i].type);
 434                     lineValueLen = (int)strlen((char *) featureName[i].value);
 435                     if (namePos+lineTypeLen+lineValueLen+7 > MAX_FEATURENAMESTRING)
 436                     {
 437                         fprintf(stderr, "feature name too long: %s", (const char *)featureName[i].value);
 438                         break;
 439                     }
 440                     if (namePos) strcpy(featureNameString+(namePos++), ",");
 441                     strcpy(featureNameString+(namePos++), "\"");
 442                     strcpy(featureNameString+namePos, (char*) featureName[i].type);
 443                     namePos += lineTypeLen;
 444                     strcpy(featureNameString+namePos, "\"=>\"");
 445                     namePos += 4;
 446                     strcpy(featureNameString+namePos, (char *) featureName[i].value);
 447                     namePos += lineValueLen;
 448                     strcpy(featureNameString+(namePos++), "\"");
 449
 450                     xmlFree(featureName[i].type);
 451                     xmlFree(featureName[i].value);
 452                 }
 453             }
 454             paramValues[5] = (const char *)featureNameString;
 455
 456             paramValues[6] = (const char *)feature.countryCode;
 457
 458             featureExtraTagString[0] = 0;
 459             if (featureExtraTagLines)
 460             {
 461                 namePos = 0;
 462                 lineTypeLen = 0;
 463                 lineValueLen = 0;
 464                 for (i = 0; i < featureExtraTagLines; i++)
 465                 {
 466                     lineTypeLen = strlen((char *) featureExtraTag[i].type);
 467                     lineValueLen = strlen((char *) featureExtraTag[i].value);
 468                     if (namePos+lineTypeLen+lineValueLen+7 > MAX_FEATUREEXTRATAGSTRING)
 469                     {
 470                         fprintf(stderr, "feature extra tag too long: %s", (const char *)featureExtraTag[i].value);
 471                         break;
 472                     }
 473                     if (namePos) strcpy(featureExtraTagString+(namePos++),",");
 474                     strcpy(featureExtraTagString+(namePos++), "\"");
 475                     strcpy(featureExtraTagString+namePos, (char *) featureExtraTag[i].type);
 476                     namePos += lineTypeLen;
 477                     strcpy(featureExtraTagString+namePos, "\"=>\"");
 478                     namePos += 4;
 479                     strcpy(featureExtraTagString+namePos, (char *) featureExtraTag[i].value);
 480                     namePos += lineValueLen;
 481                     strcpy(featureExtraTagString+(namePos++), "\"");
 482
 483                     xmlFree(featureExtraTag[i].type);
 484                     xmlFree(featureExtraTag[i].value);
 485                 }
 486             }
 487             paramValues[7] = (const char *)featureExtraTagString;
 488
 489             if (strlen(feature.parentPlaceID) == 0)
 490                 paramValues[8] = "0";
 491             else
 492                 paramValues[8] = (const char *)feature.parentPlaceID;
 493
 494             paramValues[9] = (const char *)feature.adminLevel;
 495             paramValues[10] = (const char *)feature.houseNumber;
 496             paramValues[11] = (const char *)feature.rankAddress;
 497             paramValues[12] = (const char *)feature.rankSearch;
 498             paramValues[13] = (const char *)feature.geometry;
 499             if (strlen(paramValues[3]) && strlen(paramValues[13]))
 500             {
 501                 if (verbose) fprintf(stderr, "placex_insert: %s\n", paramValues[0]);
 502                 res = PQexecPrepared(conn, "placex_insert", 14, paramValues, NULL, NULL, 0);
 503                 if (PQresultStatus(res) != PGRES_COMMAND_OK)
 504                 {
 505                     fprintf(stderr, "index_placex: INSERT failed: %s", PQerrorMessage(conn));
 506                     fprintf(stderr, "index_placex: INSERT failed: %s %s %s", paramValues[0], paramValues[1], paramValues[2]);
 507                     PQclear(res);
 508                     exit(EXIT_FAILURE);
 509                }
 510                PQclear(res);
 511             }
 512
 513             for (i = 0; i < featureAddressLines; i++)
 514             {
 515                 // insert into place_address
 516                 paramValues[0] = (const char *)place_id;
 517                 paramValues[1] = (const char *)featureAddress[i].distance;
 518                 paramValues[2] = (const char *)featureAddress[i].type;
 519                 paramValues[3] = (const char *)featureAddress[i].id;
 520                 paramValues[4] = (const char *)featureAddress[i].key;
 521                 paramValues[5] = (const char *)featureAddress[i].value;
 522                 paramValues[6] = (const char *)featureAddress[i].isAddress;
 523                 res = PQexecPrepared(conn, "place_addressline_insert", 7, paramValues, NULL, NULL, 0);
 524                 if (PQresultStatus(res) != PGRES_COMMAND_OK)
 525                 {
 526                     fprintf(stderr, "place_addressline_insert: INSERT failed: %s", PQerrorMessage(conn));
 527                     PQclear(res);
 528                     exit(EXIT_FAILURE);
 529                 }
 530                 PQclear(res);
 531
 532                 xmlFree(featureAddress[i].type);
 533                 xmlFree(featureAddress[i].id);
 534                 xmlFree(featureAddress[i].key);
 535                 xmlFree(featureAddress[i].value);
 536                 xmlFree(featureAddress[i].distance);
 537             }
 538
 539             if (featureNameLines)
 540             {
 541                 if (strlen(feature.parentPlaceID) > 0 && featureAddressLines == 0)
 542                 {
 543                     paramValues[0] = (const char *)place_id;
 544                     paramValues[1] = feature.parentPlaceID;
 545                     if (verbose) fprintf(stderr, "search_name_from_parent_insert: INSERT %s %s\n", paramValues[0], paramValues[1]);
 546                     res = PQexecPrepared(conn, "search_name_from_parent_insert", 2, paramValues, NULL, NULL, 0);
 547                     if (PQresultStatus(res) != PGRES_COMMAND_OK)
 548                     {
 549                         fprintf(stderr, "search_name_from_parent_insert: INSERT failed: %s", PQerrorMessage(conn));
 550                         PQclear(res);
 551                         exit(EXIT_FAILURE);
 552                     }
 553                     PQclear(res);
 554                 }
 555                 else
 556                 {
 557                     paramValues[0] = (const char *)place_id;
 558                     if (verbose) fprintf(stderr, "search_name_insert: INSERT %s\n", paramValues[0]);
 559                     res = PQexecPrepared(conn, "search_name_insert", 1, paramValues, NULL, NULL, 0);
 560                     if (PQresultStatus(res) != PGRES_COMMAND_OK)
 561                     {
 562                         fprintf(stderr, "search_name_insert: INSERT failed: %s", PQerrorMessage(conn));
 563                         PQclear(res);
 564                         exit(EXIT_FAILURE);
 565                     }
 566                     PQclear(res);
 567                 }
 568             }
 569
 570             partionQueryName = xmlHashLookup2(partionTableTagsHash, feature.key, feature.value);
 571             if (partionQueryName)
 572             {
 573                 // insert into partition table
 574                 paramValues[0] = (const char *)place_id;
 575                 paramValues[1] = (const char *)feature.geometry;
 576                 res = PQexecPrepared(conn, partionQueryName, 2, paramValues, NULL, NULL, 0);
 577                 if (PQresultStatus(res) != PGRES_COMMAND_OK)
 578                 {
 579                     fprintf(stderr, "%s: INSERT failed: %s", partionQueryName, PQerrorMessage(conn));
 580                     PQclear(res);
 581                     exit(EXIT_FAILURE);
 582                 }
 583                 PQclear(res);
 584             }
 585
 586         }
 587         else
 588         {
 589             for (i = 0; i < featureAddressLines; i++)
 590             {
 591                 xmlFree(featureAddress[i].type);
 592                 xmlFree(featureAddress[i].id);
 593                 xmlFree(featureAddress[i].key);
 594                 xmlFree(featureAddress[i].value);
 595                 xmlFree(featureAddress[i].distance);
 596             }
 597         }
 598
 599         xmlFree(feature.placeID);
 600         xmlFree(feature.type);
 601         xmlFree(feature.id);
 602         xmlFree(feature.key);
 603         xmlFree(feature.value);
 604         xmlFree(feature.rankAddress);
 605         xmlFree(feature.rankSearch);
 606         if (feature.countryCode) xmlFree(feature.countryCode);
 607         if (feature.parentPlaceID) xmlFree(feature.parentPlaceID);
 608         if (feature.parentType) xmlFree(feature.parentType);
 609         if (feature.parentID) xmlFree(feature.parentID);
 610 //              if (feature.name) xmlFree(feature.name);
 611         if (feature.adminLevel) xmlFree(feature.adminLevel);
 612         if (feature.houseNumber) xmlFree(feature.houseNumber);
 613         if (feature.geometry) xmlFree(feature.geometry);
 614
 615 //        PQclear(resPlaceID);
 616     }
 617 }
 618
 619 static void processNode(xmlTextReaderPtr reader)
 620 {
 621     xmlChar *name;
 622     name = xmlTextReaderName(reader);
 623     if (name == NULL)
 624     {
 625         name = xmlStrdup(BAD_CAST "--");
 626     }
 627
 628     switch (xmlTextReaderNodeType(reader))
 629     {
 630     case XML_READER_TYPE_ELEMENT:
 631         StartElement(reader, name);
 632         if (xmlTextReaderIsEmptyElement(reader))
 633             EndElement(reader, name); /* No end_element for self closing tags! */
 634         break;
 635     case XML_READER_TYPE_END_ELEMENT:
 636         EndElement(reader, name);
 637         break;
 638     case XML_READER_TYPE_TEXT:
 639     case XML_READER_TYPE_CDATA:
 640     case XML_READER_TYPE_SIGNIFICANT_WHITESPACE:
 641         /* Ignore */
 642         break;
 643     default:
 644         fprintf(stderr, "Unknown node type %d\n", xmlTextReaderNodeType(reader));
 645         break;
 646     }
 647
 648     xmlFree(name);
 649 }
 650
 651 int nominatim_import(const char *conninfo, const char *partionTagsFilename, const char *filename)
 652 {
 653     xmlTextReaderPtr    reader;
 654     int                                 ret = 0;
 655     PGresult *                  res;
 656     FILE *                              partionTagsFile;
 657     char *                              partionQueryName;
 658     char                                partionQuerySQL[1024];
 659
 660     conn = PQconnectdb(conninfo);
 661     if (PQstatus(conn) != CONNECTION_OK)
 662     {
 663         fprintf(stderr, "Connection to database failed: %s\n", PQerrorMessage(conn));
 664         exit(EXIT_FAILURE);
 665     }
 666
 667     partionTableTagsHash = xmlHashCreate(200);
 668     partionTableTagsHashDelete = xmlHashCreate(200);
 669
 670     partionTagsFile = fopen(partionTagsFilename, "rt");
 671     if (!partionTagsFile)
 672     {
 673         fprintf(stderr, "Unable to read partition tags file: %s\n", partionTagsFilename);
 674         exit(EXIT_FAILURE);
 675     }
 676
 677     char buffer[1024], osmkey[256], osmvalue[256];
 678     int fields;
 679     while (fgets(buffer, sizeof(buffer), partionTagsFile) != NULL)
 680     {
 681         fields = sscanf( buffer, "%23s %63s", osmkey, osmvalue );
 682
 683         if ( fields <= 0 ) continue;
 684
 685         if ( fields != 2  )
 686         {
 687             fprintf( stderr, "Error partition file\n");
 688             exit_nicely();
 689         }
 690         partionQueryName = malloc(strlen("partition_insert_")+strlen(osmkey)+strlen(osmvalue)+2);
 691         strcpy(partionQueryName, "partition_insert_");
 692         strcat(partionQueryName, osmkey);
 693         strcat(partionQueryName, "_");
 694         strcat(partionQueryName, osmvalue);
 695
 696         strcpy(partionQuerySQL, "insert into place_classtype_");
 697         strcat(partionQuerySQL, osmkey);
 698         strcat(partionQuerySQL, "_");
 699         strcat(partionQuerySQL, osmvalue);
 700         strcat(partionQuerySQL, " (place_id, centroid) values ($1, ST_Centroid(st_setsrid($2, 4326)))");
 701
 702         res = PQprepare(conn, partionQueryName, partionQuerySQL, 2, NULL);
 703         if (PQresultStatus(res) != PGRES_COMMAND_OK)
 704         {
 705             fprintf(stderr, "Failed to prepare %s: %s\n", partionQueryName, PQerrorMessage(conn));
 706             exit(EXIT_FAILURE);
 707         }
 708
 709         xmlHashAddEntry2(partionTableTagsHash, BAD_CAST osmkey, BAD_CAST osmvalue, BAD_CAST partionQueryName);
 710
 711         partionQueryName = malloc(strlen("partition_delete_")+strlen(osmkey)+strlen(osmvalue)+2);
 712         strcpy(partionQueryName, "partition_delete_");
 713         strcat(partionQueryName, osmkey);
 714         strcat(partionQueryName, "_");
 715         strcat(partionQueryName, osmvalue);
 716
 717         strcpy(partionQuerySQL, "delete from place_classtype_");
 718         strcat(partionQuerySQL, osmkey);
 719         strcat(partionQuerySQL, "_");
 720         strcat(partionQuerySQL, osmvalue);
 721         strcat(partionQuerySQL, " where place_id = $1::integer");
 722
 723         res = PQprepare(conn, partionQueryName, partionQuerySQL, 1, NULL);
 724         if (PQresultStatus(res) != PGRES_COMMAND_OK)
 725         {
 726             fprintf(stderr, "Failed to prepare %s: %s\n", partionQueryName, PQerrorMessage(conn));
 727             exit(EXIT_FAILURE);
 728         }
 729
 730         xmlHashAddEntry2(partionTableTagsHashDelete, BAD_CAST osmkey, BAD_CAST osmvalue, BAD_CAST partionQueryName);
 731     }
 732
 733     res = PQprepare(conn, "get_new_place_id",
 734                     "select nextval('seq_place')",
 735                     0, NULL);
 736     if (PQresultStatus(res) != PGRES_COMMAND_OK)
 737     {
 738         fprintf(stderr, "Failed to prepare get_new_place_id: %s\n", PQerrorMessage(conn));
 739         exit(EXIT_FAILURE);
 740     }
 741
 742     res = PQprepare(conn, "get_place_id",
 743                     "select place_id from placex where osm_type = $1 and osm_id = $2 and class = $3 and type = $4",
 744                     4, NULL);
 745     if (PQresultStatus(res) != PGRES_COMMAND_OK)
 746     {
 747         fprintf(stderr, "Failed to prepare get_place_id: %s\n", PQerrorMessage(conn));
 748         exit(EXIT_FAILURE);
 749     }
 750
 751     res = PQprepare(conn, "placex_insert",
 752                     "insert into placex (place_id,osm_type,osm_id,class,type,name,country_code,extratags,parent_place_id,admin_level,housenumber,rank_address,rank_search,geometry) "
 753                     "values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, st_setsrid($14, 4326))",
 754                     12, NULL);
 755     if (PQresultStatus(res) != PGRES_COMMAND_OK)
 756     {
 757         fprintf(stderr, "Failed to prepare placex_insert: %s\n", PQerrorMessage(conn));
 758         exit(EXIT_FAILURE);
 759     }
 760
 761     res = PQprepare(conn, "search_name_insert",
 762                     "insert into search_name (place_id, search_rank, address_rank, country_code, name_vector, nameaddress_vector, centroid) "
 763                     "select place_id, rank_search, rank_address, country_code, make_keywords(name), "
 764                     "(select uniq(sort(array_agg(parent_search_name.name_vector))) from search_name as parent_search_name where place_id in "
 765                      "(select distinct address_place_id from place_addressline where place_addressline.place_id = $1 limit 1000)"
 766                     "), st_centroid(geometry) from placex "
 767                     "where place_id = $1",
 768                     1, NULL);
 769
 770     if (PQresultStatus(res) != PGRES_COMMAND_OK)
 771     {
 772         fprintf(stderr, "Failed to prepare search_name_insert: %s\n", PQerrorMessage(conn));
 773         exit(EXIT_FAILURE);
 774     }
 775
 776     res = PQprepare(conn, "search_name_from_parent_insert",
 777                     "insert into search_name (place_id, search_rank, address_rank, country_code, name_vector, nameaddress_vector, centroid) "
 778                     "select place_id, rank_search, rank_address, country_code, make_keywords(name), "
 779                     "(select uniq(sort(name_vector+nameaddress_vector)) from search_name as parent_search_name "
 780                     "where parent_search_name.place_id = $2 ), st_centroid(geometry) from placex "
 781                     "where place_id = $1",
 782                     2, NULL);
 783     if (PQresultStatus(res) != PGRES_COMMAND_OK)
 784     {
 785         fprintf(stderr, "Failed to prepare search_name_insert: %s\n", PQerrorMessage(conn));
 786         exit(EXIT_FAILURE);
 787     }
 788
 789     res = PQprepare(conn, "place_addressline_insert",
 790                     "insert into place_addressline (place_id, address_place_id, fromarea, isaddress, distance, cached_rank_address) "
 791                     "select $1, place_id, false, $7, $2, rank_address from placex where osm_type = $3 and osm_id = $4 and class = $5 and type = $6",
 792                     7, NULL);
 793     if (PQresultStatus(res) != PGRES_COMMAND_OK)
 794     {
 795         fprintf(stderr, "Failed to prepare place_addressline_insert: %s\n", PQerrorMessage(conn));
 796         exit(EXIT_FAILURE);
 797     }
 798
 799     res = PQprepare(conn, "placex_delete",
 800                     "delete from placex where place_id = $1",
 801                     1, NULL);
 802     if (PQresultStatus(res) != PGRES_COMMAND_OK)
 803     {
 804         fprintf(stderr, "Failed to prepare placex_delete: %s\n", PQerrorMessage(conn));
 805         exit(EXIT_FAILURE);
 806     }
 807
 808     res = PQprepare(conn, "search_name_delete",
 809                     "delete from search_name where place_id = $1",
 810                     1, NULL);
 811     if (PQresultStatus(res) != PGRES_COMMAND_OK)
 812     {
 813         fprintf(stderr, "Failed to prepare search_name_delete: %s\n", PQerrorMessage(conn));
 814         exit(EXIT_FAILURE);
 815     }
 816
 817     res = PQprepare(conn, "place_addressline_delete",
 818                     "delete from place_addressline where place_id = $1",
 819                     1, NULL);
 820     if (PQresultStatus(res) != PGRES_COMMAND_OK)
 821     {
 822         fprintf(stderr, "Failed to prepare place_addressline_delete: %s\n", PQerrorMessage(conn));
 823         exit(EXIT_FAILURE);
 824     }
 825
 826     featureCount = 0;
 827
 828     reader = inputUTF8(filename);
 829
 830     if (reader == NULL)
 831     {
 832         fprintf(stderr, "Unable to open %s\n", filename);
 833         return 1;
 834     }
 835
 836     ret = xmlTextReaderRead(reader);
 837     while (ret == 1)
 838     {
 839         processNode(reader);
 840         ret = xmlTextReaderRead(reader);
 841     }
 842     if (ret != 0)
 843     {
 844         fprintf(stderr, "%s : failed to parse\n", filename);
 845         return ret;
 846     }
 847
 848     xmlFreeTextReader(reader);
 849     xmlHashFree(partionTableTagsHash, NULL);
 850     xmlHashFree(partionTableTagsHashDelete, NULL);
 851
 852     return 0;
 853 }