nominatim/import.c

   1 /*
   2 */
   3 #include <stdlib.h>
   4 #include <string.h>
   5
   6 #include <libpq-fe.h>
   7
   8 #include <libxml/xmlstring.h>
   9 #include <libxml/xmlreader.h>
  10 #include <libxml/hash.h>
  11
  12 #include "nominatim.h"
  13 #include "import.h"
  14 #include "input.h"
  15
  16 typedef enum { FILETYPE_NONE, FILETYPE_STRUCTUREDV0P1 } filetypes_t;
  17 typedef enum { FILEMODE_NONE, FILEMODE_ADD, FILEMODE_UPDATE, FILEMODE_DELETE } filemodes_t;
  18
  19 #define MAX_FEATUREADDRESS 5000
  20 #define MAX_FEATURENAMES 10000
  21 #define MAX_FEATUREEXTRATAGS 10000
  22 #define MAX_FEATURENAMESTRING 1000000
  23 #define MAX_FEATUREEXTRATAGSTRING 500000
  24
  25 struct feature_address
  26 {
  27     int                 place_id;
  28     int                 rankAddress;
  29     char                        isAddress[2];
  30     xmlChar *   type;
  31     xmlChar *   id;
  32     xmlChar *   key;
  33     xmlChar *   value;
  34     xmlChar *   distance;
  35 };
  36
  37 struct feature_tag
  38 {
  39     xmlChar *   type;
  40     xmlChar *   value;
  41 };
  42
  43 struct feature
  44 {
  45     xmlChar *   placeID;
  46     xmlChar *   type;
  47     xmlChar *   id;
  48     xmlChar *   key;
  49     xmlChar *   value;
  50     xmlChar *   rankAddress;
  51     xmlChar *   rankSearch;
  52     xmlChar *   countryCode;
  53     xmlChar *   parentPlaceID;
  54     xmlChar *   parentType;
  55     xmlChar *   parentID;
  56     xmlChar *   adminLevel;
  57     xmlChar *   houseNumber;
  58     xmlChar *   geometry;
  59 } feature;
  60
  61 int                                     fileType = FILETYPE_NONE;
  62 int                                     fileMode = FILEMODE_ADD;
  63 PGconn *                                conn;
  64 struct feature_address  featureAddress[MAX_FEATUREADDRESS];
  65 struct feature_tag              featureName[MAX_FEATURENAMES];
  66 struct feature_tag              featureExtraTag[MAX_FEATUREEXTRATAGS];
  67 struct feature                  feature;
  68 int                                     featureAddressLines = 0;
  69 int                                     featureNameLines = 0;
  70 int                                     featureExtraTagLines = 0;
  71 int                                     featureCount = 0;
  72 xmlHashTablePtr                 partionTableTagsHash;
  73 xmlHashTablePtr                 partionTableTagsHashDelete;
  74 char                                    featureNameString[MAX_FEATURENAMESTRING];
  75 char                                    featureExtraTagString[MAX_FEATUREEXTRATAGSTRING];
  76
  77 extern int verbose;
  78
  79 void StartElement(xmlTextReaderPtr reader, const xmlChar *name)
  80 {
  81     char * value;
  82     float version;
  83     int isAddressLine;
  84
  85     if (fileType == FILETYPE_NONE)
  86     {
  87         // Potential to handle other file types in the future / versions
  88         if (xmlStrEqual(name, BAD_CAST "osmStructured"))
  89         {
  90             value = (char*)xmlTextReaderGetAttribute(reader, BAD_CAST "version");
  91             version = strtof(value, NULL);
  92             xmlFree(value);
  93
  94             if (version == (float)0.1)
  95             {
  96                 fileType = FILETYPE_STRUCTUREDV0P1;
  97                 fileMode = FILEMODE_ADD;
  98             }
  99             else
 100             {
 101                 fprintf( stderr, "Unknown osmStructured version %f (%s)\n", version, value );
 102                 exit_nicely();
 103             }
 104         }
 105         else
 106         {
 107             fprintf( stderr, "Unknown XML document type: %s\n", name );
 108             exit_nicely();
 109         }
 110         return;
 111     }
 112
 113     if (xmlStrEqual(name, BAD_CAST "add"))
 114     {
 115         fileMode = FILEMODE_ADD;
 116         return;
 117     }
 118     if (xmlStrEqual(name, BAD_CAST "update"))
 119     {
 120         fileMode = FILEMODE_UPDATE;
 121         return;
 122     }
 123     if (xmlStrEqual(name, BAD_CAST "delete"))
 124     {
 125         fileMode = FILEMODE_DELETE;
 126         return;
 127     }
 128     if (fileMode == FILEMODE_NONE)
 129     {
 130         fprintf( stderr, "Unknown import mode in: %s\n", name );
 131         exit_nicely();
 132     }
 133
 134     if (xmlStrEqual(name, BAD_CAST "feature"))
 135     {
 136         feature.placeID = xmlTextReaderGetAttribute(reader, BAD_CAST "place_id");
 137         feature.type = xmlTextReaderGetAttribute(reader, BAD_CAST "type");
 138         feature.id = xmlTextReaderGetAttribute(reader, BAD_CAST "id");
 139         feature.key = xmlTextReaderGetAttribute(reader, BAD_CAST "key");
 140         feature.value = xmlTextReaderGetAttribute(reader, BAD_CAST "value");
 141         feature.rankAddress = xmlTextReaderGetAttribute(reader, BAD_CAST "rank");
 142         feature.rankSearch = xmlTextReaderGetAttribute(reader, BAD_CAST "importance");
 143
 144         feature.parentPlaceID = xmlTextReaderGetAttribute(reader, BAD_CAST "parent_place_id");
 145 /*
 146         if (strlen(feature.parentPlaceID) == 0)
 147         {
 148                 xmlFree(feature.parentPlaceID);
 149                 feature.parentPlaceID = NULL;
 150         }
 151 */
 152         feature.parentType = xmlTextReaderGetAttribute(reader, BAD_CAST "parent_type");
 153         feature.parentID = xmlTextReaderGetAttribute(reader, BAD_CAST "parent_id");
 154
 155         feature.countryCode = NULL;
 156         feature.adminLevel = NULL;
 157         feature.houseNumber = NULL;
 158         feature.geometry = NULL;
 159         featureAddressLines = 0;
 160         featureNameLines = 0;
 161         featureExtraTagLines = 0;
 162
 163         return;
 164     }
 165     if (xmlStrEqual(name, BAD_CAST "names")) return;
 166     if (xmlStrEqual(name, BAD_CAST "name"))
 167     {
 168         if (featureNameLines < MAX_FEATURENAMES)
 169         {
 170                 featureName[featureNameLines].type = xmlTextReaderGetAttribute(reader, BAD_CAST "type");
 171             featureName[featureNameLines].value = xmlTextReaderReadString(reader);
 172                 featureNameLines++;
 173                 }
 174                 else
 175                 {
 176             fprintf( stderr, "Too many name elements (%s%s)\n", feature.type, feature.id);
 177 //            exit_nicely();
 178         }
 179         return;
 180     }
 181     if (xmlStrEqual(name, BAD_CAST "tags")) return;
 182     if (xmlStrEqual(name, BAD_CAST "tag"))
 183     {
 184         if (featureExtraTagLines < MAX_FEATUREEXTRATAGS)
 185                 {
 186                 featureExtraTag[featureExtraTagLines].type = xmlTextReaderGetAttribute(reader, BAD_CAST "type");
 187             featureExtraTag[featureExtraTagLines].value = xmlTextReaderReadString(reader);
 188                 featureExtraTagLines++;
 189                 }
 190                 else
 191         {
 192             fprintf( stderr, "Too many extra tag elements (%s%s)\n", feature.type, feature.id);
 193 //            exit_nicely();
 194         }
 195         return;
 196     }
 197     if (xmlStrEqual(name, BAD_CAST "osmGeometry"))
 198     {
 199         feature.geometry = xmlTextReaderReadString(reader);
 200         return;
 201     }
 202     if (xmlStrEqual(name, BAD_CAST "adminLevel"))
 203     {
 204         feature.adminLevel = xmlTextReaderReadString(reader);
 205         return;
 206     }
 207     if (xmlStrEqual(name, BAD_CAST "countryCode"))
 208     {
 209         feature.countryCode = xmlTextReaderReadString(reader);
 210         return;
 211     }
 212     if (xmlStrEqual(name, BAD_CAST "houseNumber"))
 213     {
 214         feature.houseNumber = xmlTextReaderReadString(reader);
 215         return;
 216     }
 217     if (xmlStrEqual(name, BAD_CAST "address"))
 218     {
 219         featureAddressLines = 0;
 220         return;
 221     }
 222     isAddressLine = 0;
 223     if (xmlStrEqual(name, BAD_CAST "continent"))
 224     {
 225         isAddressLine = 1;
 226     }
 227     else if (xmlStrEqual(name, BAD_CAST "sea"))
 228     {
 229         isAddressLine = 1;
 230     }
 231     else if (xmlStrEqual(name, BAD_CAST "country"))
 232     {
 233         isAddressLine = 1;
 234     }
 235     else if (xmlStrEqual(name, BAD_CAST "state"))
 236     {
 237         isAddressLine = 1;
 238     }
 239     else if (xmlStrEqual(name, BAD_CAST "county"))
 240     {
 241         isAddressLine = 1;
 242     }
 243     else if (xmlStrEqual(name, BAD_CAST "city"))
 244     {
 245         isAddressLine = 1;
 246     }
 247     else if (xmlStrEqual(name, BAD_CAST "town"))
 248     {
 249         isAddressLine = 1;
 250     }
 251     else if (xmlStrEqual(name, BAD_CAST "village"))
 252     {
 253         isAddressLine = 1;
 254     }
 255     else if (xmlStrEqual(name, BAD_CAST "unknown"))
 256     {
 257         isAddressLine = 1;
 258     }
 259     else if (xmlStrEqual(name, BAD_CAST "suburb"))
 260     {
 261         isAddressLine = 1;
 262     }
 263     else if (xmlStrEqual(name, BAD_CAST "postcode"))
 264     {
 265         isAddressLine = 1;
 266     }
 267     else if (xmlStrEqual(name, BAD_CAST "neighborhood"))
 268     {
 269         isAddressLine = 1;
 270     }
 271     else if (xmlStrEqual(name, BAD_CAST "street"))
 272     {
 273         isAddressLine = 1;
 274     }
 275     else if (xmlStrEqual(name, BAD_CAST "access"))
 276     {
 277         isAddressLine = 1;
 278     }
 279     else if (xmlStrEqual(name, BAD_CAST "building"))
 280     {
 281         isAddressLine = 1;
 282     }
 283     else if (xmlStrEqual(name, BAD_CAST "other"))
 284     {
 285         isAddressLine = 1;
 286     }
 287     if (isAddressLine)
 288     {
 289         if (featureAddressLines < MAX_FEATUREADDRESS)
 290                 {
 291                 value = (char*)xmlTextReaderGetAttribute(reader, BAD_CAST "rank");
 292             if (!value)
 293                 {
 294                 fprintf( stderr, "Address element missing rank\n");
 295                     exit_nicely();
 296                 }
 297             featureAddress[featureAddressLines].rankAddress =  atoi(value);
 298                 xmlFree(value);
 299
 300                 value = (char*)xmlTextReaderGetAttribute(reader, BAD_CAST "isaddress");
 301             if (!value)
 302                 {
 303                     fprintf( stderr, "Address element missing rank\n");
 304                 exit_nicely();
 305                 }
 306                 if (*value == 't') strcpy(featureAddress[featureAddressLines].isAddress, "t");
 307             else strcpy(featureAddress[featureAddressLines].isAddress, "f");
 308                 xmlFree(value);
 309
 310                 featureAddress[featureAddressLines].type = xmlTextReaderGetAttribute(reader, BAD_CAST "type");
 311             featureAddress[featureAddressLines].id = xmlTextReaderGetAttribute(reader, BAD_CAST "id");
 312                 featureAddress[featureAddressLines].key = xmlTextReaderGetAttribute(reader, BAD_CAST "key");
 313                 featureAddress[featureAddressLines].value = xmlTextReaderGetAttribute(reader, BAD_CAST "value");
 314             featureAddress[featureAddressLines].distance = xmlTextReaderGetAttribute(reader, BAD_CAST "distance");
 315
 316             featureAddressLines++;
 317                 }
 318                 else
 319         {
 320             fprintf( stderr, "Too many address elements (%s%s)\n", feature.type, feature.id);
 321 //            exit_nicely();
 322         }
 323
 324         return;
 325     }
 326     fprintf(stderr, "%s: Unknown element name: %s\n", __FUNCTION__, name);
 327 }
 328
 329 void EndElement(xmlTextReaderPtr reader, const xmlChar *name)
 330 {
 331     PGresult *          res;
 332     const char *        paramValues[14];
 333     char *                      place_id;
 334     char *                      partionQueryName;
 335     int i, namePos, lineTypeLen, lineValueLen;
 336
 337     if (xmlStrEqual(name, BAD_CAST "feature"))
 338     {
 339         featureCount++;
 340         if (featureCount % 1000 == 0) printf("feature %i(k)\n", featureCount/1000);
 341 /*
 342         if (fileMode == FILEMODE_ADD)
 343         {
 344             resPlaceID = PQexecPrepared(conn, "get_new_place_id", 0, NULL, NULL, NULL, 0);
 345             if (PQresultStatus(resPlaceID) != PGRES_TUPLES_OK)
 346             {
 347                 fprintf(stderr, "get_place_id: INSERT failed: %s", PQerrorMessage(conn));
 348                 PQclear(resPlaceID);
 349                 exit(EXIT_FAILURE);
 350             }
 351         }
 352         else
 353         {
 354             paramValues[0] = (const char *)feature.type;
 355             paramValues[1] = (const char *)feature.id;
 356             paramValues[2] = (const char *)feature.key;
 357             paramValues[3] = (const char *)feature.value;
 358             resPlaceID = PQexecPrepared(conn, "get_new_place_id", 4, paramValues, NULL, NULL, 0);
 359             if (PQresultStatus(resPlaceID) != PGRES_TUPLES_OK)
 360             {
 361                 fprintf(stderr, "index_placex: INSERT failed: %s", PQerrorMessage(conn));
 362                 PQclear(resPlaceID);
 363                 exit(EXIT_FAILURE);
 364             }
 365         }
 366 */
 367         place_id = (char *)feature.placeID;
 368
 369         if (fileMode == FILEMODE_UPDATE || fileMode == FILEMODE_DELETE || fileMode == FILEMODE_ADD)
 370         {
 371             paramValues[0] = (const char *)place_id;
 372             if (verbose) fprintf(stderr, "placex_delete: %s\n", paramValues[0]);
 373             res = PQexecPrepared(conn, "placex_delete", 1, paramValues, NULL, NULL, 0);
 374             if (PQresultStatus(res) != PGRES_COMMAND_OK)
 375             {
 376                 fprintf(stderr, "placex_delete: DELETE failed: %s", PQerrorMessage(conn));
 377                 PQclear(res);
 378                 exit(EXIT_FAILURE);
 379             }
 380             PQclear(res);
 381
 382             if (verbose) fprintf(stderr, "search_name_delete: %s\n", paramValues[0]);
 383             res = PQexecPrepared(conn, "search_name_delete", 1, paramValues, NULL, NULL, 0);
 384             if (PQresultStatus(res) != PGRES_COMMAND_OK)
 385             {
 386                 fprintf(stderr, "search_name_delete: DELETE failed: %s", PQerrorMessage(conn));
 387                 PQclear(res);
 388                 exit(EXIT_FAILURE);
 389             }
 390             PQclear(res);
 391
 392             if (verbose) fprintf(stderr, "place_addressline_delete: %s\n", paramValues[0]);
 393             res = PQexecPrepared(conn, "place_addressline_delete", 1, paramValues, NULL, NULL, 0);
 394             if (PQresultStatus(res) != PGRES_COMMAND_OK)
 395             {
 396                 fprintf(stderr, "place_addressline_delete: DELETE failed: %s", PQerrorMessage(conn));
 397                 PQclear(res);
 398                 exit(EXIT_FAILURE);
 399             }
 400             PQclear(res);
 401
 402             partionQueryName = xmlHashLookup2(partionTableTagsHashDelete, feature.key, feature.value);
 403             if (partionQueryName)
 404             {
 405                 res = PQexecPrepared(conn, partionQueryName, 1, paramValues, NULL, NULL, 0);
 406                 if (PQresultStatus(res) != PGRES_COMMAND_OK)
 407                 {
 408                     fprintf(stderr, "%s: DELETE failed: %s", partionQueryName, PQerrorMessage(conn));
 409                     PQclear(res);
 410                     exit(EXIT_FAILURE);
 411                 }
 412                 PQclear(res);
 413             }
 414         }
 415
 416         if (fileMode == FILEMODE_UPDATE || fileMode == FILEMODE_ADD)
 417         {
 418             // Insert into placex
 419             paramValues[0] = (const char *)place_id;
 420             paramValues[1] = (const char *)feature.type;
 421             paramValues[2] = (const char *)feature.id;
 422             paramValues[3] = (const char *)feature.key;
 423             paramValues[4] = (const char *)feature.value;
 424
 425             featureNameString[0] = 0;
 426             if (featureNameLines)
 427             {
 428                 namePos = 0;
 429                 lineTypeLen = 0;
 430                 lineValueLen = 0;
 431                 for (i = 0; i < featureNameLines; i++)
 432                 {
 433                     lineTypeLen = (int)strlen((char *) featureName[i].type);
 434                     lineValueLen = (int)strlen((char *) featureName[i].value);
 435                     if (namePos+lineTypeLen+lineValueLen+7 > MAX_FEATURENAMESTRING)
 436                     {
 437                         fprintf(stderr, "feature name too long: %s", (const char *)featureName[i].value);
 438                         break;
 439                     }
 440                     if (namePos) strcpy(featureNameString+(namePos++), ",");
 441                     strcpy(featureNameString+(namePos++), "\"");
 442                     strcpy(featureNameString+namePos, (char*) featureName[i].type);
 443                     namePos += lineTypeLen;
 444                     strcpy(featureNameString+namePos, "\"=>\"");
 445                     namePos += 4;
 446                     strcpy(featureNameString+namePos, (char *) featureName[i].value);
 447                     namePos += lineValueLen;
 448                     strcpy(featureNameString+(namePos++), "\"");
 449
 450                     xmlFree(featureName[i].type);
 451                     xmlFree(featureName[i].value);
 452                 }
 453             }
 454             paramValues[5] = (const char *)featureNameString;
 455
 456             paramValues[6] = (const char *)feature.countryCode;
 457
 458             featureExtraTagString[0] = 0;
 459             if (featureExtraTagLines)
 460             {
 461                 namePos = 0;
 462                 lineTypeLen = 0;
 463                 lineValueLen = 0;
 464                 for (i = 0; i < featureExtraTagLines; i++)
 465                 {
 466                     lineTypeLen = strlen((char *) featureExtraTag[i].type);
 467                     lineValueLen = strlen((char *) featureExtraTag[i].value);
 468                     if (namePos+lineTypeLen+lineValueLen+7 > MAX_FEATUREEXTRATAGSTRING)
 469                     {
 470                         fprintf(stderr, "feature extra tag too long: %s", (const char *)featureExtraTag[i].value);
 471                         break;
 472                     }
 473                     if (namePos) strcpy(featureExtraTagString+(namePos++),",");
 474                     strcpy(featureExtraTagString+(namePos++), "\"");
 475                     strcpy(featureExtraTagString+namePos, (char *) featureExtraTag[i].type);
 476                     namePos += lineTypeLen;
 477                     strcpy(featureExtraTagString+namePos, "\"=>\"");
 478                     namePos += 4;
 479                     strcpy(featureExtraTagString+namePos, (char *) featureExtraTag[i].value);
 480                     namePos += lineValueLen;
 481                     strcpy(featureExtraTagString+(namePos++), "\"");
 482
 483                     xmlFree(featureExtraTag[i].type);
 484                     xmlFree(featureExtraTag[i].value);
 485                 }
 486             }
 487             paramValues[7] = (const char *)featureExtraTagString;
 488
 489             if (strlen(feature.parentPlaceID) == 0)
 490                 paramValues[8] = "0";
 491             else
 492                 paramValues[8] = (const char *)feature.parentPlaceID;
 493
 494             paramValues[9] = (const char *)feature.adminLevel;
 495             paramValues[10] = (const char *)feature.houseNumber;
 496             paramValues[11] = (const char *)feature.rankAddress;
 497             paramValues[12] = (const char *)feature.rankSearch;
 498             paramValues[13] = (const char *)feature.geometry;
 499             if (strlen(paramValues[3]) && strlen(paramValues[13]))
 500             {
 501                 if (verbose) fprintf(stderr, "placex_insert: %s\n", paramValues[0]);
 502                 res = PQexecPrepared(conn, "placex_insert", 14, paramValues, NULL, NULL, 0);
 503                 if (PQresultStatus(res) != PGRES_COMMAND_OK)
 504                 {
 505                     fprintf(stderr, "index_placex: INSERT failed: %s", PQerrorMessage(conn));
 506                     fprintf(stderr, "index_placex: INSERT failed: %s %s %s", paramValues[0], paramValues[1], paramValues[2]);
 507                     PQclear(res);
 508                     exit(EXIT_FAILURE);
 509                }
 510                PQclear(res);
 511             }
 512
 513             for (i = 0; i < featureAddressLines; i++)
 514             {
 515                 // insert into place_address
 516                 paramValues[0] = (const char *)place_id;
 517                 paramValues[1] = (const char *)featureAddress[i].distance;
 518                 if (paramValues[1] == NULL || strlen(paramValues[1]) == 0) paramValues[1] = "0";
 519                 paramValues[2] = (const char *)featureAddress[i].type;
 520                 paramValues[3] = (const char *)featureAddress[i].id;
 521                 paramValues[4] = (const char *)featureAddress[i].key;
 522                 paramValues[5] = (const char *)featureAddress[i].value;
 523                 paramValues[6] = (const char *)featureAddress[i].isAddress;
 524                 if (verbose) fprintf(stderr, "placex_insert: %s %s\n", paramValues[2], paramValues[3]);
 525                 res = PQexecPrepared(conn, "place_addressline_insert", 7, paramValues, NULL, NULL, 0);
 526                 if (PQresultStatus(res) != PGRES_COMMAND_OK)
 527                 {
 528                     fprintf(stderr, "place_addressline_insert: INSERT failed: %s", PQerrorMessage(conn));
 529                     fprintf(stderr, "(%s,%s,%s,%s,%s,%s,%s)",paramValues[0],paramValues[1],paramValues[2],paramValues[3],paramValues[4],paramValues[5],paramValues[6]);
 530                     PQclear(res);
 531                     exit(EXIT_FAILURE);
 532                 }
 533                 PQclear(res);
 534
 535                 xmlFree(featureAddress[i].type);
 536                 xmlFree(featureAddress[i].id);
 537                 xmlFree(featureAddress[i].key);
 538                 xmlFree(featureAddress[i].value);
 539                 xmlFree(featureAddress[i].distance);
 540             }
 541
 542             if (featureNameLines)
 543             {
 544                 if (strlen(feature.parentPlaceID) > 0 && featureAddressLines == 0)
 545                 {
 546                     paramValues[0] = (const char *)place_id;
 547                     paramValues[1] = feature.parentPlaceID;
 548                     if (verbose) fprintf(stderr, "search_name_from_parent_insert: INSERT %s %s\n", paramValues[0], paramValues[1]);
 549                     res = PQexecPrepared(conn, "search_name_from_parent_insert", 2, paramValues, NULL, NULL, 0);
 550                     if (PQresultStatus(res) != PGRES_COMMAND_OK)
 551                     {
 552                         fprintf(stderr, "search_name_from_parent_insert: INSERT failed: %s", PQerrorMessage(conn));
 553                         PQclear(res);
 554                         exit(EXIT_FAILURE);
 555                     }
 556                     PQclear(res);
 557                 }
 558                 else
 559                 {
 560                     paramValues[0] = (const char *)place_id;
 561                     if (verbose) fprintf(stderr, "search_name_insert: INSERT %s\n", paramValues[0]);
 562                     res = PQexecPrepared(conn, "search_name_insert", 1, paramValues, NULL, NULL, 0);
 563                     if (PQresultStatus(res) != PGRES_COMMAND_OK)
 564                     {
 565                         fprintf(stderr, "search_name_insert: INSERT failed: %s", PQerrorMessage(conn));
 566                         PQclear(res);
 567                         exit(EXIT_FAILURE);
 568                     }
 569                     PQclear(res);
 570                 }
 571             }
 572
 573             partionQueryName = xmlHashLookup2(partionTableTagsHash, feature.key, feature.value);
 574             if (partionQueryName)
 575             {
 576                 // insert into partition table
 577                 paramValues[0] = (const char *)place_id;
 578                 paramValues[1] = (const char *)feature.geometry;
 579                 res = PQexecPrepared(conn, partionQueryName, 2, paramValues, NULL, NULL, 0);
 580                 if (PQresultStatus(res) != PGRES_COMMAND_OK)
 581                 {
 582                     fprintf(stderr, "%s: INSERT failed: %s", partionQueryName, PQerrorMessage(conn));
 583                     PQclear(res);
 584                     exit(EXIT_FAILURE);
 585                 }
 586                 PQclear(res);
 587             }
 588
 589         }
 590         else
 591         {
 592             for (i = 0; i < featureAddressLines; i++)
 593             {
 594                 xmlFree(featureAddress[i].type);
 595                 xmlFree(featureAddress[i].id);
 596                 xmlFree(featureAddress[i].key);
 597                 xmlFree(featureAddress[i].value);
 598                 xmlFree(featureAddress[i].distance);
 599             }
 600         }
 601
 602         xmlFree(feature.placeID);
 603         xmlFree(feature.type);
 604         xmlFree(feature.id);
 605         xmlFree(feature.key);
 606         xmlFree(feature.value);
 607         xmlFree(feature.rankAddress);
 608         xmlFree(feature.rankSearch);
 609         if (feature.countryCode) xmlFree(feature.countryCode);
 610         if (feature.parentPlaceID) xmlFree(feature.parentPlaceID);
 611         if (feature.parentType) xmlFree(feature.parentType);
 612         if (feature.parentID) xmlFree(feature.parentID);
 613 //              if (feature.name) xmlFree(feature.name);
 614         if (feature.adminLevel) xmlFree(feature.adminLevel);
 615         if (feature.houseNumber) xmlFree(feature.houseNumber);
 616         if (feature.geometry) xmlFree(feature.geometry);
 617
 618 //        PQclear(resPlaceID);
 619     }
 620 }
 621
 622 static void processNode(xmlTextReaderPtr reader)
 623 {
 624     xmlChar *name;
 625     name = xmlTextReaderName(reader);
 626     if (name == NULL)
 627     {
 628         name = xmlStrdup(BAD_CAST "--");
 629     }
 630
 631     switch (xmlTextReaderNodeType(reader))
 632     {
 633     case XML_READER_TYPE_ELEMENT:
 634         StartElement(reader, name);
 635         if (xmlTextReaderIsEmptyElement(reader))
 636             EndElement(reader, name); /* No end_element for self closing tags! */
 637         break;
 638     case XML_READER_TYPE_END_ELEMENT:
 639         EndElement(reader, name);
 640         break;
 641     case XML_READER_TYPE_TEXT:
 642     case XML_READER_TYPE_CDATA:
 643     case XML_READER_TYPE_SIGNIFICANT_WHITESPACE:
 644         /* Ignore */
 645         break;
 646     default:
 647         fprintf(stderr, "Unknown node type %d\n", xmlTextReaderNodeType(reader));
 648         break;
 649     }
 650
 651     xmlFree(name);
 652 }
 653
 654 int nominatim_import(const char *conninfo, const char *partionTagsFilename, const char *filename)
 655 {
 656     xmlTextReaderPtr    reader;
 657     int                                 ret = 0;
 658     PGresult *                  res;
 659     FILE *                              partionTagsFile;
 660     char *                              partionQueryName;
 661     char                                partionQuerySQL[1024];
 662
 663     conn = PQconnectdb(conninfo);
 664     if (PQstatus(conn) != CONNECTION_OK)
 665     {
 666         fprintf(stderr, "Connection to database failed: %s\n", PQerrorMessage(conn));
 667         exit(EXIT_FAILURE);
 668     }
 669
 670     partionTableTagsHash = xmlHashCreate(200);
 671     partionTableTagsHashDelete = xmlHashCreate(200);
 672
 673     partionTagsFile = fopen(partionTagsFilename, "rt");
 674     if (!partionTagsFile)
 675     {
 676         fprintf(stderr, "Unable to read partition tags file: %s\n", partionTagsFilename);
 677         exit(EXIT_FAILURE);
 678     }
 679
 680     char buffer[1024], osmkey[256], osmvalue[256];
 681     int fields;
 682     while (fgets(buffer, sizeof(buffer), partionTagsFile) != NULL)
 683     {
 684         fields = sscanf( buffer, "%23s %63s", osmkey, osmvalue );
 685
 686         if ( fields <= 0 ) continue;
 687
 688         if ( fields != 2  )
 689         {
 690             fprintf( stderr, "Error partition file\n");
 691             exit_nicely();
 692         }
 693         partionQueryName = malloc(strlen("partition_insert_")+strlen(osmkey)+strlen(osmvalue)+2);
 694         strcpy(partionQueryName, "partition_insert_");
 695         strcat(partionQueryName, osmkey);
 696         strcat(partionQueryName, "_");
 697         strcat(partionQueryName, osmvalue);
 698
 699         strcpy(partionQuerySQL, "insert into place_classtype_");
 700         strcat(partionQuerySQL, osmkey);
 701         strcat(partionQuerySQL, "_");
 702         strcat(partionQuerySQL, osmvalue);
 703         strcat(partionQuerySQL, " (place_id, centroid) values ($1, ST_Centroid(st_setsrid($2, 4326)))");
 704
 705         res = PQprepare(conn, partionQueryName, partionQuerySQL, 2, NULL);
 706         if (PQresultStatus(res) != PGRES_COMMAND_OK)
 707         {
 708             fprintf(stderr, "Failed to prepare %s: %s\n", partionQueryName, PQerrorMessage(conn));
 709             exit(EXIT_FAILURE);
 710         }
 711
 712         xmlHashAddEntry2(partionTableTagsHash, BAD_CAST osmkey, BAD_CAST osmvalue, BAD_CAST partionQueryName);
 713
 714         partionQueryName = malloc(strlen("partition_delete_")+strlen(osmkey)+strlen(osmvalue)+2);
 715         strcpy(partionQueryName, "partition_delete_");
 716         strcat(partionQueryName, osmkey);
 717         strcat(partionQueryName, "_");
 718         strcat(partionQueryName, osmvalue);
 719
 720         strcpy(partionQuerySQL, "delete from place_classtype_");
 721         strcat(partionQuerySQL, osmkey);
 722         strcat(partionQuerySQL, "_");
 723         strcat(partionQuerySQL, osmvalue);
 724         strcat(partionQuerySQL, " where place_id = $1::integer");
 725
 726         res = PQprepare(conn, partionQueryName, partionQuerySQL, 1, NULL);
 727         if (PQresultStatus(res) != PGRES_COMMAND_OK)
 728         {
 729             fprintf(stderr, "Failed to prepare %s: %s\n", partionQueryName, PQerrorMessage(conn));
 730             exit(EXIT_FAILURE);
 731         }
 732
 733         xmlHashAddEntry2(partionTableTagsHashDelete, BAD_CAST osmkey, BAD_CAST osmvalue, BAD_CAST partionQueryName);
 734     }
 735
 736     res = PQprepare(conn, "get_new_place_id",
 737                     "select nextval('seq_place')",
 738                     0, NULL);
 739     if (PQresultStatus(res) != PGRES_COMMAND_OK)
 740     {
 741         fprintf(stderr, "Failed to prepare get_new_place_id: %s\n", PQerrorMessage(conn));
 742         exit(EXIT_FAILURE);
 743     }
 744
 745     res = PQprepare(conn, "get_place_id",
 746                     "select place_id from placex where osm_type = $1 and osm_id = $2 and class = $3 and type = $4",
 747                     4, NULL);
 748     if (PQresultStatus(res) != PGRES_COMMAND_OK)
 749     {
 750         fprintf(stderr, "Failed to prepare get_place_id: %s\n", PQerrorMessage(conn));
 751         exit(EXIT_FAILURE);
 752     }
 753
 754     res = PQprepare(conn, "placex_insert",
 755                     "insert into placex (place_id,osm_type,osm_id,class,type,name,country_code,extratags,parent_place_id,admin_level,housenumber,rank_address,rank_search,geometry) "
 756                     "values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, st_setsrid($14, 4326))",
 757                     12, NULL);
 758     if (PQresultStatus(res) != PGRES_COMMAND_OK)
 759     {
 760         fprintf(stderr, "Failed to prepare placex_insert: %s\n", PQerrorMessage(conn));
 761         exit(EXIT_FAILURE);
 762     }
 763
 764     res = PQprepare(conn, "search_name_insert",
 765                     "insert into search_name (place_id, search_rank, address_rank, country_code, name_vector, nameaddress_vector, centroid) "
 766                     "select place_id, rank_search, rank_address, country_code, make_keywords(name), "
 767                     "(select uniq(sort(array_agg(parent_search_name.name_vector))) from search_name as parent_search_name where place_id in "
 768                      "(select distinct address_place_id from place_addressline where place_addressline.place_id = $1 limit 1000)"
 769                     "), st_centroid(geometry) from placex "
 770                     "where place_id = $1",
 771                     1, NULL);
 772
 773     if (PQresultStatus(res) != PGRES_COMMAND_OK)
 774     {
 775         fprintf(stderr, "Failed to prepare search_name_insert: %s\n", PQerrorMessage(conn));
 776         exit(EXIT_FAILURE);
 777     }
 778
 779     res = PQprepare(conn, "search_name_from_parent_insert",
 780                     "insert into search_name (place_id, search_rank, address_rank, country_code, name_vector, nameaddress_vector, centroid) "
 781                     "select place_id, rank_search, rank_address, country_code, make_keywords(name), "
 782                     "(select uniq(sort(name_vector+nameaddress_vector)) from search_name as parent_search_name "
 783                     "where parent_search_name.place_id = $2 ), st_centroid(geometry) from placex "
 784                     "where place_id = $1",
 785                     2, NULL);
 786     if (PQresultStatus(res) != PGRES_COMMAND_OK)
 787     {
 788         fprintf(stderr, "Failed to prepare search_name_insert: %s\n", PQerrorMessage(conn));
 789         exit(EXIT_FAILURE);
 790     }
 791
 792     res = PQprepare(conn, "place_addressline_insert",
 793                     "insert into place_addressline (place_id, address_place_id, fromarea, isaddress, distance, cached_rank_address) "
 794                     "select $1, place_id, false, $7, $2, rank_address from placex where osm_type = $3 and osm_id = $4 and class = $5 and type = $6",
 795                     7, NULL);
 796     if (PQresultStatus(res) != PGRES_COMMAND_OK)
 797     {
 798         fprintf(stderr, "Failed to prepare place_addressline_insert: %s\n", PQerrorMessage(conn));
 799         exit(EXIT_FAILURE);
 800     }
 801
 802     res = PQprepare(conn, "placex_delete",
 803                     "delete from placex where place_id = $1",
 804                     1, NULL);
 805     if (PQresultStatus(res) != PGRES_COMMAND_OK)
 806     {
 807         fprintf(stderr, "Failed to prepare placex_delete: %s\n", PQerrorMessage(conn));
 808         exit(EXIT_FAILURE);
 809     }
 810
 811     res = PQprepare(conn, "search_name_delete",
 812                     "delete from search_name where place_id = $1",
 813                     1, NULL);
 814     if (PQresultStatus(res) != PGRES_COMMAND_OK)
 815     {
 816         fprintf(stderr, "Failed to prepare search_name_delete: %s\n", PQerrorMessage(conn));
 817         exit(EXIT_FAILURE);
 818     }
 819
 820     res = PQprepare(conn, "place_addressline_delete",
 821                     "delete from place_addressline where place_id = $1",
 822                     1, NULL);
 823     if (PQresultStatus(res) != PGRES_COMMAND_OK)
 824     {
 825         fprintf(stderr, "Failed to prepare place_addressline_delete: %s\n", PQerrorMessage(conn));
 826         exit(EXIT_FAILURE);
 827     }
 828
 829     featureCount = 0;
 830
 831     reader = inputUTF8(filename);
 832
 833     if (reader == NULL)
 834     {
 835         fprintf(stderr, "Unable to open %s\n", filename);
 836         return 1;
 837     }
 838
 839     ret = xmlTextReaderRead(reader);
 840     while (ret == 1)
 841     {
 842         processNode(reader);
 843         ret = xmlTextReaderRead(reader);
 844     }
 845     if (ret != 0)
 846     {
 847         fprintf(stderr, "%s : failed to parse\n", filename);
 848         return ret;
 849     }
 850
 851     xmlFreeTextReader(reader);
 852     xmlHashFree(partionTableTagsHash, NULL);
 853     xmlHashFree(partionTableTagsHashDelete, NULL);
 854
 855     return 0;
 856 }