nominatim/import.c

   1 /*
   2 */
   3 #include <stdlib.h>
   4 #include <string.h>
   5
   6 #include <libpq-fe.h>
   7
   8 #include <libxml/xmlstring.h>
   9 #include <libxml/xmlreader.h>
  10 #include <libxml/hash.h>
  11
  12 #include "nominatim.h"
  13 #include "import.h"
  14 #include "input.h"
  15
  16 typedef enum { FILETYPE_NONE, FILETYPE_STRUCTUREDV0P1 } filetypes_t;
  17 typedef enum { FILEMODE_NONE, FILEMODE_ADD, FILEMODE_UPDATE, FILEMODE_DELETE } filemodes_t;
  18
  19 #define MAX_FEATUREADDRESS 5000
  20 #define MAX_FEATURENAMES 10000
  21 #define MAX_FEATUREEXTRATAGS 10000
  22 #define MAX_FEATURENAMESTRING 1000000
  23 #define MAX_FEATUREEXTRATAGSTRING 500000
  24
  25 struct feature_address
  26 {
  27     int                 place_id;
  28     int                 rankAddress;
  29     char                        isAddress[2];
  30     xmlChar *   type;
  31     xmlChar *   id;
  32     xmlChar *   key;
  33     xmlChar *   value;
  34     xmlChar *   distance;
  35 };
  36
  37 struct feature_tag
  38 {
  39     xmlChar *   type;
  40     xmlChar *   value;
  41 };
  42
  43 struct feature
  44 {
  45     xmlChar *   placeID;
  46     xmlChar *   type;
  47     xmlChar *   id;
  48     xmlChar *   key;
  49     xmlChar *   value;
  50     xmlChar *   rankAddress;
  51     xmlChar *   rankSearch;
  52     xmlChar *   countryCode;
  53     xmlChar *   parentPlaceID;
  54     xmlChar *   parentType;
  55     xmlChar *   parentID;
  56     xmlChar *   adminLevel;
  57     xmlChar *   houseNumber;
  58     xmlChar *   geometry;
  59 } feature;
  60
  61 int                                     fileType = FILETYPE_NONE;
  62 int                                     fileMode = FILEMODE_ADD;
  63 PGconn *                                conn;
  64 struct feature_address  featureAddress[MAX_FEATUREADDRESS];
  65 struct feature_tag              featureName[MAX_FEATURENAMES];
  66 struct feature_tag              featureExtraTag[MAX_FEATUREEXTRATAGS];
  67 struct feature                  feature;
  68 int                                     featureAddressLines = 0;
  69 int                                     featureNameLines = 0;
  70 int                                     featureExtraTagLines = 0;
  71 int                                     featureCount = 0;
  72 xmlHashTablePtr                 partionTableTagsHash;
  73 xmlHashTablePtr                 partionTableTagsHashDelete;
  74 char                                    featureNameString[MAX_FEATURENAMESTRING];
  75 char                                    featureExtraTagString[MAX_FEATUREEXTRATAGSTRING];
  76
  77 void StartElement(xmlTextReaderPtr reader, const xmlChar *name)
  78 {
  79     char * value;
  80     float version;
  81     int isAddressLine;
  82
  83     if (fileType == FILETYPE_NONE)
  84     {
  85         // Potential to handle other file types in the future / versions
  86         if (xmlStrEqual(name, BAD_CAST "osmStructured"))
  87         {
  88             value = (char*)xmlTextReaderGetAttribute(reader, BAD_CAST "version");
  89             version = strtof(value, NULL);
  90             xmlFree(value);
  91
  92             if (version == (float)0.1)
  93             {
  94                 fileType = FILETYPE_STRUCTUREDV0P1;
  95                 fileMode = FILEMODE_ADD;
  96             }
  97             else
  98             {
  99                 fprintf( stderr, "Unknown osmStructured version %f (%s)\n", version, value );
 100                 exit_nicely();
 101             }
 102         }
 103         else
 104         {
 105             fprintf( stderr, "Unknown XML document type: %s\n", name );
 106             exit_nicely();
 107         }
 108         return;
 109     }
 110
 111     if (xmlStrEqual(name, BAD_CAST "add"))
 112     {
 113         fileMode = FILEMODE_ADD;
 114         return;
 115     }
 116     if (xmlStrEqual(name, BAD_CAST "update"))
 117     {
 118         fileMode = FILEMODE_UPDATE;
 119         return;
 120     }
 121     if (xmlStrEqual(name, BAD_CAST "delete"))
 122     {
 123         fileMode = FILEMODE_DELETE;
 124         return;
 125     }
 126     if (fileMode == FILEMODE_NONE)
 127     {
 128         fprintf( stderr, "Unknown import mode in: %s\n", name );
 129         exit_nicely();
 130     }
 131
 132     if (xmlStrEqual(name, BAD_CAST "feature"))
 133     {
 134         feature.placeID = xmlTextReaderGetAttribute(reader, BAD_CAST "place_id");
 135         feature.type = xmlTextReaderGetAttribute(reader, BAD_CAST "type");
 136         feature.id = xmlTextReaderGetAttribute(reader, BAD_CAST "id");
 137         feature.key = xmlTextReaderGetAttribute(reader, BAD_CAST "key");
 138         feature.value = xmlTextReaderGetAttribute(reader, BAD_CAST "value");
 139         feature.rankAddress = xmlTextReaderGetAttribute(reader, BAD_CAST "rank");
 140         feature.rankSearch = xmlTextReaderGetAttribute(reader, BAD_CAST "importance");
 141
 142         feature.parentPlaceID = xmlTextReaderGetAttribute(reader, BAD_CAST "parent_place_id");
 143 /*
 144         if (strlen(feature.parentPlaceID) == 0)
 145         {
 146                 xmlFree(feature.parentPlaceID);
 147                 feature.parentPlaceID = NULL;
 148         }
 149 */
 150         feature.parentType = xmlTextReaderGetAttribute(reader, BAD_CAST "parent_type");
 151         feature.parentID = xmlTextReaderGetAttribute(reader, BAD_CAST "parent_id");
 152
 153         feature.countryCode = NULL;
 154         feature.adminLevel = NULL;
 155         feature.houseNumber = NULL;
 156         feature.geometry = NULL;
 157         featureAddressLines = 0;
 158         featureNameLines = 0;
 159         featureExtraTagLines = 0;
 160
 161         return;
 162     }
 163     if (xmlStrEqual(name, BAD_CAST "names")) return;
 164     if (xmlStrEqual(name, BAD_CAST "name"))
 165     {
 166         if (featureNameLines < MAX_FEATURENAMES)
 167         {
 168                 featureName[featureNameLines].type = xmlTextReaderGetAttribute(reader, BAD_CAST "type");
 169             featureName[featureNameLines].value = xmlTextReaderReadString(reader);
 170                 featureNameLines++;
 171                 }
 172                 else
 173                 {
 174             fprintf( stderr, "Too many name elements (%s%s)\n", feature.type, feature.id);
 175 //            exit_nicely();
 176         }
 177         return;
 178     }
 179     if (xmlStrEqual(name, BAD_CAST "tags")) return;
 180     if (xmlStrEqual(name, BAD_CAST "tag"))
 181     {
 182         if (featureExtraTagLines < MAX_FEATUREEXTRATAGS)
 183                 {
 184                 featureExtraTag[featureExtraTagLines].type = xmlTextReaderGetAttribute(reader, BAD_CAST "type");
 185             featureExtraTag[featureExtraTagLines].value = xmlTextReaderReadString(reader);
 186                 featureExtraTagLines++;
 187                 }
 188                 else
 189         {
 190             fprintf( stderr, "Too many extra tag elements (%s%s)\n", feature.type, feature.id);
 191 //            exit_nicely();
 192         }
 193         return;
 194     }
 195     if (xmlStrEqual(name, BAD_CAST "osmGeometry"))
 196     {
 197         feature.geometry = xmlTextReaderReadString(reader);
 198         return;
 199     }
 200     if (xmlStrEqual(name, BAD_CAST "adminLevel"))
 201     {
 202         feature.adminLevel = xmlTextReaderReadString(reader);
 203         return;
 204     }
 205     if (xmlStrEqual(name, BAD_CAST "countryCode"))
 206     {
 207         feature.countryCode = xmlTextReaderReadString(reader);
 208         return;
 209     }
 210     if (xmlStrEqual(name, BAD_CAST "houseNumber"))
 211     {
 212         feature.houseNumber = xmlTextReaderReadString(reader);
 213         return;
 214     }
 215     if (xmlStrEqual(name, BAD_CAST "address"))
 216     {
 217         featureAddressLines = 0;
 218         return;
 219     }
 220     isAddressLine = 0;
 221     if (xmlStrEqual(name, BAD_CAST "continent"))
 222     {
 223         isAddressLine = 1;
 224     }
 225     else if (xmlStrEqual(name, BAD_CAST "sea"))
 226     {
 227         isAddressLine = 1;
 228     }
 229     else if (xmlStrEqual(name, BAD_CAST "country"))
 230     {
 231         isAddressLine = 1;
 232     }
 233     else if (xmlStrEqual(name, BAD_CAST "state"))
 234     {
 235         isAddressLine = 1;
 236     }
 237     else if (xmlStrEqual(name, BAD_CAST "county"))
 238     {
 239         isAddressLine = 1;
 240     }
 241     else if (xmlStrEqual(name, BAD_CAST "city"))
 242     {
 243         isAddressLine = 1;
 244     }
 245     else if (xmlStrEqual(name, BAD_CAST "town"))
 246     {
 247         isAddressLine = 1;
 248     }
 249     else if (xmlStrEqual(name, BAD_CAST "village"))
 250     {
 251         isAddressLine = 1;
 252     }
 253     else if (xmlStrEqual(name, BAD_CAST "unknown"))
 254     {
 255         isAddressLine = 1;
 256     }
 257     else if (xmlStrEqual(name, BAD_CAST "suburb"))
 258     {
 259         isAddressLine = 1;
 260     }
 261     else if (xmlStrEqual(name, BAD_CAST "postcode"))
 262     {
 263         isAddressLine = 1;
 264     }
 265     else if (xmlStrEqual(name, BAD_CAST "neighborhood"))
 266     {
 267         isAddressLine = 1;
 268     }
 269     else if (xmlStrEqual(name, BAD_CAST "street"))
 270     {
 271         isAddressLine = 1;
 272     }
 273     else if (xmlStrEqual(name, BAD_CAST "access"))
 274     {
 275         isAddressLine = 1;
 276     }
 277     else if (xmlStrEqual(name, BAD_CAST "building"))
 278     {
 279         isAddressLine = 1;
 280     }
 281     else if (xmlStrEqual(name, BAD_CAST "other"))
 282     {
 283         isAddressLine = 1;
 284     }
 285     if (isAddressLine)
 286     {
 287         if (featureAddressLines < MAX_FEATUREADDRESS)
 288                 {
 289                 value = (char*)xmlTextReaderGetAttribute(reader, BAD_CAST "rank");
 290             if (!value)
 291                 {
 292                 fprintf( stderr, "Address element missing rank\n");
 293                     exit_nicely();
 294                 }
 295             featureAddress[featureAddressLines].rankAddress =  atoi(value);
 296                 xmlFree(value);
 297
 298                 value = (char*)xmlTextReaderGetAttribute(reader, BAD_CAST "isaddress");
 299             if (!value)
 300                 {
 301                     fprintf( stderr, "Address element missing rank\n");
 302                 exit_nicely();
 303                 }
 304                 if (*value == 't') strcpy(featureAddress[featureAddressLines].isAddress, "t");
 305             else strcpy(featureAddress[featureAddressLines].isAddress, "f");
 306                 xmlFree(value);
 307
 308                 featureAddress[featureAddressLines].type = xmlTextReaderGetAttribute(reader, BAD_CAST "type");
 309             featureAddress[featureAddressLines].id = xmlTextReaderGetAttribute(reader, BAD_CAST "id");
 310                 featureAddress[featureAddressLines].key = xmlTextReaderGetAttribute(reader, BAD_CAST "key");
 311                 featureAddress[featureAddressLines].value = xmlTextReaderGetAttribute(reader, BAD_CAST "value");
 312             featureAddress[featureAddressLines].distance = xmlTextReaderGetAttribute(reader, BAD_CAST "distance");
 313
 314             featureAddressLines++;
 315                 }
 316                 else
 317         {
 318             fprintf( stderr, "Too many address elements (%s%s)\n", feature.type, feature.id);
 319 //            exit_nicely();
 320         }
 321
 322         return;
 323     }
 324     fprintf(stderr, "%s: Unknown element name: %s\n", __FUNCTION__, name);
 325 }
 326
 327 void EndElement(xmlTextReaderPtr reader, const xmlChar *name)
 328 {
 329     PGresult *          res;
 330     const char *        paramValues[14];
 331     char *                      place_id;
 332     char *                      partionQueryName;
 333     int i, namePos, lineTypeLen, lineValueLen;
 334
 335     if (xmlStrEqual(name, BAD_CAST "feature"))
 336     {
 337         featureCount++;
 338         if (featureCount % 1000 == 0) printf("feature %i(k)\n", featureCount/1000);
 339 /*
 340         if (fileMode == FILEMODE_ADD)
 341         {
 342             resPlaceID = PQexecPrepared(conn, "get_new_place_id", 0, NULL, NULL, NULL, 0);
 343             if (PQresultStatus(resPlaceID) != PGRES_TUPLES_OK)
 344             {
 345                 fprintf(stderr, "get_place_id: INSERT failed: %s", PQerrorMessage(conn));
 346                 PQclear(resPlaceID);
 347                 exit(EXIT_FAILURE);
 348             }
 349         }
 350         else
 351         {
 352             paramValues[0] = (const char *)feature.type;
 353             paramValues[1] = (const char *)feature.id;
 354             paramValues[2] = (const char *)feature.key;
 355             paramValues[3] = (const char *)feature.value;
 356             resPlaceID = PQexecPrepared(conn, "get_new_place_id", 4, paramValues, NULL, NULL, 0);
 357             if (PQresultStatus(resPlaceID) != PGRES_TUPLES_OK)
 358             {
 359                 fprintf(stderr, "index_placex: INSERT failed: %s", PQerrorMessage(conn));
 360                 PQclear(resPlaceID);
 361                 exit(EXIT_FAILURE);
 362             }
 363         }
 364 */
 365         place_id = (char *)feature.placeID;
 366
 367         if (fileMode == FILEMODE_UPDATE || fileMode == FILEMODE_DELETE || fileMode == FILEMODE_ADD)
 368         {
 369             paramValues[0] = (const char *)place_id;
 370             res = PQexecPrepared(conn, "placex_delete", 1, paramValues, NULL, NULL, 0);
 371             if (PQresultStatus(res) != PGRES_COMMAND_OK)
 372             {
 373                 fprintf(stderr, "placex_delete: DELETE failed: %s", PQerrorMessage(conn));
 374                 PQclear(res);
 375                 exit(EXIT_FAILURE);
 376             }
 377             PQclear(res);
 378
 379             res = PQexecPrepared(conn, "search_name_delete", 1, paramValues, NULL, NULL, 0);
 380             if (PQresultStatus(res) != PGRES_COMMAND_OK)
 381             {
 382                 fprintf(stderr, "search_name_delete: DELETE failed: %s", PQerrorMessage(conn));
 383                 PQclear(res);
 384                 exit(EXIT_FAILURE);
 385             }
 386             PQclear(res);
 387
 388             res = PQexecPrepared(conn, "place_addressline_delete", 1, paramValues, NULL, NULL, 0);
 389             if (PQresultStatus(res) != PGRES_COMMAND_OK)
 390             {
 391                 fprintf(stderr, "place_addressline_delete: DELETE failed: %s", PQerrorMessage(conn));
 392                 PQclear(res);
 393                 exit(EXIT_FAILURE);
 394             }
 395             PQclear(res);
 396
 397             partionQueryName = xmlHashLookup2(partionTableTagsHashDelete, feature.key, feature.value);
 398             if (partionQueryName)
 399             {
 400                 res = PQexecPrepared(conn, partionQueryName, 1, paramValues, NULL, NULL, 0);
 401                 if (PQresultStatus(res) != PGRES_COMMAND_OK)
 402                 {
 403                     fprintf(stderr, "%s: DELETE failed: %s", partionQueryName, PQerrorMessage(conn));
 404                     PQclear(res);
 405                     exit(EXIT_FAILURE);
 406                 }
 407                 PQclear(res);
 408             }
 409         }
 410
 411         if (fileMode == FILEMODE_UPDATE || fileMode == FILEMODE_ADD)
 412         {
 413             // Insert into placex
 414             paramValues[0] = (const char *)place_id;
 415             paramValues[1] = (const char *)feature.type;
 416             paramValues[2] = (const char *)feature.id;
 417             paramValues[3] = (const char *)feature.key;
 418             paramValues[4] = (const char *)feature.value;
 419
 420             featureNameString[0] = 0;
 421             if (featureNameLines)
 422             {
 423                 namePos = 0;
 424                 lineTypeLen = 0;
 425                 lineValueLen = 0;
 426                 for (i = 0; i < featureNameLines; i++)
 427                 {
 428                     lineTypeLen = (int)strlen((char *) featureName[i].type);
 429                     lineValueLen = (int)strlen((char *) featureName[i].value);
 430                     if (namePos+lineTypeLen+lineValueLen+7 > MAX_FEATURENAMESTRING)
 431                     {
 432                         fprintf(stderr, "feature name too long: %s", (const char *)featureName[i].value);
 433                         break;
 434                     }
 435                     if (namePos) strcpy(featureNameString+(namePos++), ",");
 436                     strcpy(featureNameString+(namePos++), "\"");
 437                     strcpy(featureNameString+namePos, (char*) featureName[i].type);
 438                     namePos += lineTypeLen;
 439                     strcpy(featureNameString+namePos, "\"=>\"");
 440                     namePos += 4;
 441                     strcpy(featureNameString+namePos, (char *) featureName[i].value);
 442                     namePos += lineValueLen;
 443                     strcpy(featureNameString+(namePos++), "\"");
 444
 445                     xmlFree(featureName[i].type);
 446                     xmlFree(featureName[i].value);
 447                 }
 448             }
 449             paramValues[5] = (const char *)featureNameString;
 450
 451             paramValues[6] = (const char *)feature.countryCode;
 452
 453             featureExtraTagString[0] = 0;
 454             if (featureExtraTagLines)
 455             {
 456                 namePos = 0;
 457                 lineTypeLen = 0;
 458                 lineValueLen = 0;
 459                 for (i = 0; i < featureExtraTagLines; i++)
 460                 {
 461                     lineTypeLen = strlen((char *) featureExtraTag[i].type);
 462                     lineValueLen = strlen((char *) featureExtraTag[i].value);
 463                     if (namePos+lineTypeLen+lineValueLen+7 > MAX_FEATUREEXTRATAGSTRING)
 464                     {
 465                         fprintf(stderr, "feature extra tag too long: %s", (const char *)featureExtraTag[i].value);
 466                         break;
 467                     }
 468                     if (namePos) strcpy(featureExtraTagString+(namePos++),",");
 469                     strcpy(featureExtraTagString+(namePos++), "\"");
 470                     strcpy(featureExtraTagString+namePos, (char *) featureExtraTag[i].type);
 471                     namePos += lineTypeLen;
 472                     strcpy(featureExtraTagString+namePos, "\"=>\"");
 473                     namePos += 4;
 474                     strcpy(featureExtraTagString+namePos, (char *) featureExtraTag[i].value);
 475                     namePos += lineValueLen;
 476                     strcpy(featureExtraTagString+(namePos++), "\"");
 477
 478                     xmlFree(featureExtraTag[i].type);
 479                     xmlFree(featureExtraTag[i].value);
 480                 }
 481             }
 482             paramValues[7] = (const char *)featureExtraTagString;
 483
 484             if (strlen(feature.parentPlaceID) == 0)
 485                 paramValues[8] = "0";
 486             else
 487                 paramValues[8] = (const char *)feature.parentPlaceID;
 488
 489             paramValues[9] = (const char *)feature.adminLevel;
 490             paramValues[10] = (const char *)feature.houseNumber;
 491             paramValues[11] = (const char *)feature.rankAddress;
 492             paramValues[12] = (const char *)feature.rankSearch;
 493             paramValues[13] = (const char *)feature.geometry;
 494             if (strlen(paramValues[3]) && strlen(paramValues[13]))
 495             {
 496                 res = PQexecPrepared(conn, "placex_insert", 14, paramValues, NULL, NULL, 0);
 497                 if (PQresultStatus(res) != PGRES_COMMAND_OK)
 498                 {
 499                     fprintf(stderr, "index_placex: INSERT failed: %s", PQerrorMessage(conn));
 500                     fprintf(stderr, "index_placex: INSERT failed: %s %s %s", paramValues[0], paramValues[1], paramValues[2]);
 501                     PQclear(res);
 502                     exit(EXIT_FAILURE);
 503                }
 504                PQclear(res);
 505             }
 506
 507             for (i = 0; i < featureAddressLines; i++)
 508             {
 509                 // insert into place_address
 510                 paramValues[0] = (const char *)place_id;
 511                 paramValues[1] = (const char *)featureAddress[i].distance;
 512                 paramValues[2] = (const char *)featureAddress[i].type;
 513                 paramValues[3] = (const char *)featureAddress[i].id;
 514                 paramValues[4] = (const char *)featureAddress[i].key;
 515                 paramValues[5] = (const char *)featureAddress[i].value;
 516                 paramValues[6] = (const char *)featureAddress[i].isAddress;
 517                 res = PQexecPrepared(conn, "place_addressline_insert", 7, paramValues, NULL, NULL, 0);
 518                 if (PQresultStatus(res) != PGRES_COMMAND_OK)
 519                 {
 520                     fprintf(stderr, "place_addressline_insert: INSERT failed: %s", PQerrorMessage(conn));
 521                     PQclear(res);
 522                     exit(EXIT_FAILURE);
 523                 }
 524                 PQclear(res);
 525
 526                 xmlFree(featureAddress[i].type);
 527                 xmlFree(featureAddress[i].id);
 528                 xmlFree(featureAddress[i].key);
 529                 xmlFree(featureAddress[i].value);
 530                 xmlFree(featureAddress[i].distance);
 531             }
 532
 533             if (featureNameLines)
 534             {
 535                 if (strlen(feature.parentPlaceID) > 0 && featureAddressLines == 0)
 536                 {
 537                     paramValues[0] = (const char *)place_id;
 538                     paramValues[1] = feature.parentPlaceID;
 539                     res = PQexecPrepared(conn, "search_name_from_parent_insert", 2, paramValues, NULL, NULL, 0);
 540                     if (PQresultStatus(res) != PGRES_COMMAND_OK)
 541                     {
 542                         fprintf(stderr, "search_name_from_parent_insert: INSERT failed: %s", PQerrorMessage(conn));
 543                         PQclear(res);
 544                         exit(EXIT_FAILURE);
 545                     }
 546                     PQclear(res);
 547                 }
 548                 else
 549                 {
 550                     paramValues[0] = (const char *)place_id;
 551                     res = PQexecPrepared(conn, "search_name_insert", 1, paramValues, NULL, NULL, 0);
 552                     if (PQresultStatus(res) != PGRES_COMMAND_OK)
 553                     {
 554                         fprintf(stderr, "search_name_insert: INSERT failed: %s", PQerrorMessage(conn));
 555                         PQclear(res);
 556                         exit(EXIT_FAILURE);
 557                     }
 558                     PQclear(res);
 559                 }
 560             }
 561
 562             partionQueryName = xmlHashLookup2(partionTableTagsHash, feature.key, feature.value);
 563             if (partionQueryName)
 564             {
 565                 // insert into partition table
 566                 paramValues[0] = (const char *)place_id;
 567                 paramValues[1] = (const char *)feature.geometry;
 568                 res = PQexecPrepared(conn, partionQueryName, 2, paramValues, NULL, NULL, 0);
 569                 if (PQresultStatus(res) != PGRES_COMMAND_OK)
 570                 {
 571                     fprintf(stderr, "%s: INSERT failed: %s", partionQueryName, PQerrorMessage(conn));
 572                     PQclear(res);
 573                     exit(EXIT_FAILURE);
 574                 }
 575                 PQclear(res);
 576             }
 577
 578         }
 579         else
 580         {
 581             for (i = 0; i < featureAddressLines; i++)
 582             {
 583                 xmlFree(featureAddress[i].type);
 584                 xmlFree(featureAddress[i].id);
 585                 xmlFree(featureAddress[i].key);
 586                 xmlFree(featureAddress[i].value);
 587                 xmlFree(featureAddress[i].distance);
 588             }
 589         }
 590
 591         xmlFree(feature.placeID);
 592         xmlFree(feature.type);
 593         xmlFree(feature.id);
 594         xmlFree(feature.key);
 595         xmlFree(feature.value);
 596         xmlFree(feature.rankAddress);
 597         xmlFree(feature.rankSearch);
 598         if (feature.countryCode) xmlFree(feature.countryCode);
 599         if (feature.parentPlaceID) xmlFree(feature.parentPlaceID);
 600         if (feature.parentType) xmlFree(feature.parentType);
 601         if (feature.parentID) xmlFree(feature.parentID);
 602 //              if (feature.name) xmlFree(feature.name);
 603         if (feature.adminLevel) xmlFree(feature.adminLevel);
 604         if (feature.houseNumber) xmlFree(feature.houseNumber);
 605         if (feature.geometry) xmlFree(feature.geometry);
 606
 607 //        PQclear(resPlaceID);
 608     }
 609 }
 610
 611 static void processNode(xmlTextReaderPtr reader)
 612 {
 613     xmlChar *name;
 614     name = xmlTextReaderName(reader);
 615     if (name == NULL)
 616     {
 617         name = xmlStrdup(BAD_CAST "--");
 618     }
 619
 620     switch (xmlTextReaderNodeType(reader))
 621     {
 622     case XML_READER_TYPE_ELEMENT:
 623         StartElement(reader, name);
 624         if (xmlTextReaderIsEmptyElement(reader))
 625             EndElement(reader, name); /* No end_element for self closing tags! */
 626         break;
 627     case XML_READER_TYPE_END_ELEMENT:
 628         EndElement(reader, name);
 629         break;
 630     case XML_READER_TYPE_TEXT:
 631     case XML_READER_TYPE_CDATA:
 632     case XML_READER_TYPE_SIGNIFICANT_WHITESPACE:
 633         /* Ignore */
 634         break;
 635     default:
 636         fprintf(stderr, "Unknown node type %d\n", xmlTextReaderNodeType(reader));
 637         break;
 638     }
 639
 640     xmlFree(name);
 641 }
 642
 643 int nominatim_import(const char *conninfo, const char *partionTagsFilename, const char *filename)
 644 {
 645     xmlTextReaderPtr    reader;
 646     int                                 ret = 0;
 647     PGresult *                  res;
 648     FILE *                              partionTagsFile;
 649     char *                              partionQueryName;
 650     char                                partionQuerySQL[1024];
 651
 652     conn = PQconnectdb(conninfo);
 653     if (PQstatus(conn) != CONNECTION_OK)
 654     {
 655         fprintf(stderr, "Connection to database failed: %s\n", PQerrorMessage(conn));
 656         exit(EXIT_FAILURE);
 657     }
 658
 659     partionTableTagsHash = xmlHashCreate(200);
 660     partionTableTagsHashDelete = xmlHashCreate(200);
 661
 662     partionTagsFile = fopen(partionTagsFilename, "rt");
 663     if (!partionTagsFile)
 664     {
 665         fprintf(stderr, "Unable to read partition tags file: %s\n", partionTagsFilename);
 666         exit(EXIT_FAILURE);
 667     }
 668
 669     char buffer[1024], osmkey[256], osmvalue[256];
 670     int fields;
 671     while (fgets(buffer, sizeof(buffer), partionTagsFile) != NULL)
 672     {
 673         fields = sscanf( buffer, "%23s %63s", osmkey, osmvalue );
 674
 675         if ( fields <= 0 ) continue;
 676
 677         if ( fields != 2  )
 678         {
 679             fprintf( stderr, "Error partition file\n");
 680             exit_nicely();
 681         }
 682         partionQueryName = malloc(strlen("partition_insert_")+strlen(osmkey)+strlen(osmvalue)+2);
 683         strcpy(partionQueryName, "partition_insert_");
 684         strcat(partionQueryName, osmkey);
 685         strcat(partionQueryName, "_");
 686         strcat(partionQueryName, osmvalue);
 687
 688         strcpy(partionQuerySQL, "insert into place_classtype_");
 689         strcat(partionQuerySQL, osmkey);
 690         strcat(partionQuerySQL, "_");
 691         strcat(partionQuerySQL, osmvalue);
 692         strcat(partionQuerySQL, " (place_id, centroid) values ($1, ST_Centroid(st_setsrid($2, 4326)))");
 693
 694         res = PQprepare(conn, partionQueryName, partionQuerySQL, 2, NULL);
 695         if (PQresultStatus(res) != PGRES_COMMAND_OK)
 696         {
 697             fprintf(stderr, "Failed to prepare %s: %s\n", partionQueryName, PQerrorMessage(conn));
 698             exit(EXIT_FAILURE);
 699         }
 700
 701         xmlHashAddEntry2(partionTableTagsHash, BAD_CAST osmkey, BAD_CAST osmvalue, BAD_CAST partionQueryName);
 702
 703         partionQueryName = malloc(strlen("partition_delete_")+strlen(osmkey)+strlen(osmvalue)+2);
 704         strcpy(partionQueryName, "partition_delete_");
 705         strcat(partionQueryName, osmkey);
 706         strcat(partionQueryName, "_");
 707         strcat(partionQueryName, osmvalue);
 708
 709         strcpy(partionQuerySQL, "delete from place_classtype_");
 710         strcat(partionQuerySQL, osmkey);
 711         strcat(partionQuerySQL, "_");
 712         strcat(partionQuerySQL, osmvalue);
 713         strcat(partionQuerySQL, " where place_id = $1::integer");
 714
 715         res = PQprepare(conn, partionQueryName, partionQuerySQL, 1, NULL);
 716         if (PQresultStatus(res) != PGRES_COMMAND_OK)
 717         {
 718             fprintf(stderr, "Failed to prepare %s: %s\n", partionQueryName, PQerrorMessage(conn));
 719             exit(EXIT_FAILURE);
 720         }
 721
 722         xmlHashAddEntry2(partionTableTagsHashDelete, BAD_CAST osmkey, BAD_CAST osmvalue, BAD_CAST partionQueryName);
 723     }
 724
 725     res = PQprepare(conn, "get_new_place_id",
 726                     "select nextval('seq_place')",
 727                     0, NULL);
 728     if (PQresultStatus(res) != PGRES_COMMAND_OK)
 729     {
 730         fprintf(stderr, "Failed to prepare get_new_place_id: %s\n", PQerrorMessage(conn));
 731         exit(EXIT_FAILURE);
 732     }
 733
 734     res = PQprepare(conn, "get_place_id",
 735                     "select place_id from placex where osm_type = $1 and osm_id = $2 and class = $3 and type = $4",
 736                     4, NULL);
 737     if (PQresultStatus(res) != PGRES_COMMAND_OK)
 738     {
 739         fprintf(stderr, "Failed to prepare get_place_id: %s\n", PQerrorMessage(conn));
 740         exit(EXIT_FAILURE);
 741     }
 742
 743     res = PQprepare(conn, "placex_insert",
 744                     "insert into placex (place_id,osm_type,osm_id,class,type,name,country_code,extratags,parent_place_id,admin_level,housenumber,rank_address,rank_search,geometry) "
 745                     "values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, st_setsrid($14, 4326))",
 746                     12, NULL);
 747     if (PQresultStatus(res) != PGRES_COMMAND_OK)
 748     {
 749         fprintf(stderr, "Failed to prepare placex_insert: %s\n", PQerrorMessage(conn));
 750         exit(EXIT_FAILURE);
 751     }
 752
 753     res = PQprepare(conn, "search_name_insert",
 754                     "insert into search_name (place_id, search_rank, address_rank, country_code, name_vector, nameaddress_vector, centroid) "
 755                     "select place_id, rank_search, rank_address, country_code, make_keywords(name), "
 756                     "(select uniq(sort(array_agg(parent_search_name.name_vector))) from place_addressline join search_name as parent_search_name on "
 757                     "(address_place_id = parent_search_name.place_id) where place_addressline.place_id = $1 ), st_centroid(geometry) from placex "
 758                     "where place_id = $1",
 759                     1, NULL);
 760     if (PQresultStatus(res) != PGRES_COMMAND_OK)
 761     {
 762         fprintf(stderr, "Failed to prepare search_name_insert: %s\n", PQerrorMessage(conn));
 763         exit(EXIT_FAILURE);
 764     }
 765
 766     res = PQprepare(conn, "search_name_from_parent_insert",
 767                     "insert into search_name (place_id, search_rank, address_rank, country_code, name_vector, nameaddress_vector, centroid) "
 768                     "select place_id, rank_search, rank_address, country_code, make_keywords(name), "
 769                     "(select uniq(sort(name_vector+nameaddress_vector)) from search_name as parent_search_name "
 770                     "where parent_search_name.place_id = $2 ), st_centroid(geometry) from placex "
 771                     "where place_id = $1",
 772                     2, NULL);
 773     if (PQresultStatus(res) != PGRES_COMMAND_OK)
 774     {
 775         fprintf(stderr, "Failed to prepare search_name_insert: %s\n", PQerrorMessage(conn));
 776         exit(EXIT_FAILURE);
 777     }
 778
 779     res = PQprepare(conn, "place_addressline_insert",
 780                     "insert into place_addressline (place_id, address_place_id, fromarea, isaddress, distance, cached_rank_address) "
 781                     "select $1, place_id, false, $7, $2, rank_address from placex where osm_type = $3 and osm_id = $4 and class = $5 and type = $6",
 782                     7, NULL);
 783     if (PQresultStatus(res) != PGRES_COMMAND_OK)
 784     {
 785         fprintf(stderr, "Failed to prepare place_addressline_insert: %s\n", PQerrorMessage(conn));
 786         exit(EXIT_FAILURE);
 787     }
 788
 789     res = PQprepare(conn, "placex_delete",
 790                     "delete from placex where place_id = $1",
 791                     1, NULL);
 792     if (PQresultStatus(res) != PGRES_COMMAND_OK)
 793     {
 794         fprintf(stderr, "Failed to prepare placex_delete: %s\n", PQerrorMessage(conn));
 795         exit(EXIT_FAILURE);
 796     }
 797
 798     res = PQprepare(conn, "search_name_delete",
 799                     "delete from search_name where place_id = $1",
 800                     1, NULL);
 801     if (PQresultStatus(res) != PGRES_COMMAND_OK)
 802     {
 803         fprintf(stderr, "Failed to prepare search_name_delete: %s\n", PQerrorMessage(conn));
 804         exit(EXIT_FAILURE);
 805     }
 806
 807     res = PQprepare(conn, "place_addressline_delete",
 808                     "delete from place_addressline where place_id = $1",
 809                     1, NULL);
 810     if (PQresultStatus(res) != PGRES_COMMAND_OK)
 811     {
 812         fprintf(stderr, "Failed to prepare place_addressline_delete: %s\n", PQerrorMessage(conn));
 813         exit(EXIT_FAILURE);
 814     }
 815
 816     featureCount = 0;
 817
 818     reader = inputUTF8(filename);
 819
 820     if (reader == NULL)
 821     {
 822         fprintf(stderr, "Unable to open %s\n", filename);
 823         return 1;
 824     }
 825
 826     ret = xmlTextReaderRead(reader);
 827     while (ret == 1)
 828     {
 829         processNode(reader);
 830         ret = xmlTextReaderRead(reader);
 831     }
 832     if (ret != 0)
 833     {
 834         fprintf(stderr, "%s : failed to parse\n", filename);
 835         return ret;
 836     }
 837
 838     xmlFreeTextReader(reader);
 839     xmlHashFree(partionTableTagsHash, NULL);
 840     xmlHashFree(partionTableTagsHashDelete, NULL);
 841
 842     return 0;
 843 }