nominatim/import.c

   1 /*
   2 */
   3 #include <stdlib.h>
   4 #include <string.h>
   5
   6 #include <libpq-fe.h>
   7
   8 #include <libxml/xmlstring.h>
   9 #include <libxml/xmlreader.h>
  10 #include <libxml/hash.h>
  11
  12 #include "nominatim.h"
  13 #include "import.h"
  14 #include "input.h"
  15
  16 typedef enum { FILETYPE_NONE, FILETYPE_STRUCTUREDV0P1 } filetypes_t;
  17 typedef enum { FILEMODE_NONE, FILEMODE_ADD, FILEMODE_UPDATE, FILEMODE_DELETE } filemodes_t;
  18
  19 #define MAX_FEATUREADDRESS 5000
  20 #define MAX_FEATURENAMES 10000
  21 #define MAX_FEATUREEXTRATAGS 10000
  22 #define MAX_FEATURENAMESTRING 1000000
  23 #define MAX_FEATUREEXTRATAGSTRING 500000
  24
  25 struct feature_address
  26 {
  27     int                 place_id;
  28     int                 rankAddress;
  29     char                        isAddress[2];
  30     xmlChar *   type;
  31     xmlChar *   id;
  32     xmlChar *   key;
  33     xmlChar *   value;
  34     xmlChar *   distance;
  35 };
  36
  37 struct feature_tag
  38 {
  39     xmlChar *   type;
  40     xmlChar *   value;
  41 };
  42
  43 struct feature
  44 {
  45     xmlChar *   placeID;
  46     xmlChar *   type;
  47     xmlChar *   id;
  48     xmlChar *   key;
  49     xmlChar *   value;
  50     xmlChar *   rankAddress;
  51     xmlChar *   rankSearch;
  52     xmlChar *   countryCode;
  53     xmlChar *   parentPlaceID;
  54     xmlChar *   parentType;
  55     xmlChar *   parentID;
  56     xmlChar *   adminLevel;
  57     xmlChar *   houseNumber;
  58     xmlChar *   geometry;
  59 } feature;
  60
  61 int                                     fileType = FILETYPE_NONE;
  62 int                                     fileMode = FILEMODE_ADD;
  63 PGconn *                                conn;
  64 struct feature_address  featureAddress[MAX_FEATUREADDRESS];
  65 struct feature_tag              featureName[MAX_FEATURENAMES];
  66 struct feature_tag              featureExtraTag[MAX_FEATUREEXTRATAGS];
  67 struct feature                  feature;
  68 int                                     featureAddressLines = 0;
  69 int                                     featureNameLines = 0;
  70 int                                     featureExtraTagLines = 0;
  71 int                                     featureCount = 0;
  72 xmlHashTablePtr                 partionTableTagsHash;
  73 xmlHashTablePtr                 partionTableTagsHashDelete;
  74 char                                    featureNameString[MAX_FEATURENAMESTRING];
  75 char                                    featureExtraTagString[MAX_FEATUREEXTRATAGSTRING];
  76
  77 void StartElement(xmlTextReaderPtr reader, const xmlChar *name)
  78 {
  79     char * value;
  80     float version;
  81     int isAddressLine;
  82
  83     if (fileType == FILETYPE_NONE)
  84     {
  85         // Potential to handle other file types in the future / versions
  86         if (xmlStrEqual(name, BAD_CAST "osmStructured"))
  87         {
  88             value = (char*)xmlTextReaderGetAttribute(reader, BAD_CAST "version");
  89             version = strtof(value, NULL);
  90             xmlFree(value);
  91
  92             if (version == (float)0.1)
  93             {
  94                 fileType = FILETYPE_STRUCTUREDV0P1;
  95                 fileMode = FILEMODE_ADD;
  96             }
  97             else
  98             {
  99                 fprintf( stderr, "Unknown osmStructured version %f (%s)\n", version, value );
 100                 exit_nicely();
 101             }
 102         }
 103         else
 104         {
 105             fprintf( stderr, "Unknown XML document type: %s\n", name );
 106             exit_nicely();
 107         }
 108         return;
 109     }
 110
 111     if (xmlStrEqual(name, BAD_CAST "add"))
 112     {
 113         fileMode = FILEMODE_ADD;
 114         return;
 115     }
 116     if (xmlStrEqual(name, BAD_CAST "update"))
 117     {
 118         fileMode = FILEMODE_UPDATE;
 119         return;
 120     }
 121     if (xmlStrEqual(name, BAD_CAST "delete"))
 122     {
 123         fileMode = FILEMODE_DELETE;
 124         return;
 125     }
 126     if (fileMode == FILEMODE_NONE)
 127     {
 128         fprintf( stderr, "Unknown import mode in: %s\n", name );
 129         exit_nicely();
 130     }
 131
 132     if (xmlStrEqual(name, BAD_CAST "feature"))
 133     {
 134         feature.placeID = xmlTextReaderGetAttribute(reader, BAD_CAST "place_id");
 135         feature.type = xmlTextReaderGetAttribute(reader, BAD_CAST "type");
 136         feature.id = xmlTextReaderGetAttribute(reader, BAD_CAST "id");
 137         feature.key = xmlTextReaderGetAttribute(reader, BAD_CAST "key");
 138         feature.value = xmlTextReaderGetAttribute(reader, BAD_CAST "value");
 139         feature.rankAddress = xmlTextReaderGetAttribute(reader, BAD_CAST "rank");
 140         feature.rankSearch = xmlTextReaderGetAttribute(reader, BAD_CAST "importance");
 141
 142         feature.parentPlaceID = xmlTextReaderGetAttribute(reader, BAD_CAST "parent_place_id");
 143 /*
 144         if (strlen(feature.parentPlaceID) == 0)
 145         {
 146                 xmlFree(feature.parentPlaceID);
 147                 feature.parentPlaceID = NULL;
 148         }
 149 */
 150         feature.parentType = xmlTextReaderGetAttribute(reader, BAD_CAST "parent_type");
 151         feature.parentID = xmlTextReaderGetAttribute(reader, BAD_CAST "parent_id");
 152
 153         feature.countryCode = NULL;
 154         feature.adminLevel = NULL;
 155         feature.houseNumber = NULL;
 156         feature.geometry = NULL;
 157         featureAddressLines = 0;
 158         featureNameLines = 0;
 159         featureExtraTagLines = 0;
 160
 161         return;
 162     }
 163     if (xmlStrEqual(name, BAD_CAST "names")) return;
 164     if (xmlStrEqual(name, BAD_CAST "name"))
 165     {
 166         if (featureNameLines < MAX_FEATURENAMES)
 167         {
 168                 featureName[featureNameLines].type = xmlTextReaderGetAttribute(reader, BAD_CAST "type");
 169             featureName[featureNameLines].value = xmlTextReaderReadString(reader);
 170                 featureNameLines++;
 171                 }
 172                 else
 173                 {
 174             fprintf( stderr, "Too many name elements (%s%s)\n", feature.type, feature.id);
 175 //            exit_nicely();
 176         }
 177         return;
 178     }
 179     if (xmlStrEqual(name, BAD_CAST "tags")) return;
 180     if (xmlStrEqual(name, BAD_CAST "tag"))
 181     {
 182         if (featureExtraTagLines < MAX_FEATUREEXTRATAGS)
 183                 {
 184                 featureExtraTag[featureExtraTagLines].type = xmlTextReaderGetAttribute(reader, BAD_CAST "type");
 185             featureExtraTag[featureExtraTagLines].value = xmlTextReaderReadString(reader);
 186                 featureExtraTagLines++;
 187                 }
 188                 else
 189         {
 190             fprintf( stderr, "Too many extra tag elements (%s%s)\n", feature.type, feature.id);
 191 //            exit_nicely();
 192         }
 193         return;
 194     }
 195     if (xmlStrEqual(name, BAD_CAST "osmGeometry"))
 196     {
 197         feature.geometry = xmlTextReaderReadString(reader);
 198         return;
 199     }
 200     if (xmlStrEqual(name, BAD_CAST "adminLevel"))
 201     {
 202         feature.adminLevel = xmlTextReaderReadString(reader);
 203         return;
 204     }
 205     if (xmlStrEqual(name, BAD_CAST "countryCode"))
 206     {
 207         feature.countryCode = xmlTextReaderReadString(reader);
 208         return;
 209     }
 210     if (xmlStrEqual(name, BAD_CAST "houseNumber"))
 211     {
 212         feature.houseNumber = xmlTextReaderReadString(reader);
 213         return;
 214     }
 215     if (xmlStrEqual(name, BAD_CAST "address"))
 216     {
 217         featureAddressLines = 0;
 218         return;
 219     }
 220     isAddressLine = 0;
 221     if (xmlStrEqual(name, BAD_CAST "continent"))
 222     {
 223         isAddressLine = 1;
 224     }
 225     else if (xmlStrEqual(name, BAD_CAST "sea"))
 226     {
 227         isAddressLine = 1;
 228     }
 229     else if (xmlStrEqual(name, BAD_CAST "country"))
 230     {
 231         isAddressLine = 1;
 232     }
 233     else if (xmlStrEqual(name, BAD_CAST "state"))
 234     {
 235         isAddressLine = 1;
 236     }
 237     else if (xmlStrEqual(name, BAD_CAST "county"))
 238     {
 239         isAddressLine = 1;
 240     }
 241     else if (xmlStrEqual(name, BAD_CAST "city"))
 242     {
 243         isAddressLine = 1;
 244     }
 245     else if (xmlStrEqual(name, BAD_CAST "town"))
 246     {
 247         isAddressLine = 1;
 248     }
 249     else if (xmlStrEqual(name, BAD_CAST "village"))
 250     {
 251         isAddressLine = 1;
 252     }
 253     else if (xmlStrEqual(name, BAD_CAST "unknown"))
 254     {
 255         isAddressLine = 1;
 256     }
 257     else if (xmlStrEqual(name, BAD_CAST "suburb"))
 258     {
 259         isAddressLine = 1;
 260     }
 261     else if (xmlStrEqual(name, BAD_CAST "postcode"))
 262     {
 263         isAddressLine = 1;
 264     }
 265     else if (xmlStrEqual(name, BAD_CAST "neighborhood"))
 266     {
 267         isAddressLine = 1;
 268     }
 269     else if (xmlStrEqual(name, BAD_CAST "street"))
 270     {
 271         isAddressLine = 1;
 272     }
 273     else if (xmlStrEqual(name, BAD_CAST "access"))
 274     {
 275         isAddressLine = 1;
 276     }
 277     else if (xmlStrEqual(name, BAD_CAST "building"))
 278     {
 279         isAddressLine = 1;
 280     }
 281     else if (xmlStrEqual(name, BAD_CAST "other"))
 282     {
 283         isAddressLine = 1;
 284     }
 285     if (isAddressLine)
 286     {
 287         if (featureAddressLines < MAX_FEATUREADDRESS)
 288                 {
 289                 value = (char*)xmlTextReaderGetAttribute(reader, BAD_CAST "rank");
 290             if (!value)
 291                 {
 292                 fprintf( stderr, "Address element missing rank\n");
 293                     exit_nicely();
 294                 }
 295             featureAddress[featureAddressLines].rankAddress =  atoi(value);
 296                 xmlFree(value);
 297
 298                 value = (char*)xmlTextReaderGetAttribute(reader, BAD_CAST "isaddress");
 299             if (!value)
 300                 {
 301                     fprintf( stderr, "Address element missing rank\n");
 302                 exit_nicely();
 303                 }
 304                 if (*value == 't') strcpy(featureAddress[featureAddressLines].isAddress, "t");
 305             else strcpy(featureAddress[featureAddressLines].isAddress, "f");
 306                 xmlFree(value);
 307
 308                 featureAddress[featureAddressLines].type = xmlTextReaderGetAttribute(reader, BAD_CAST "type");
 309             featureAddress[featureAddressLines].id = xmlTextReaderGetAttribute(reader, BAD_CAST "id");
 310                 featureAddress[featureAddressLines].key = xmlTextReaderGetAttribute(reader, BAD_CAST "key");
 311                 featureAddress[featureAddressLines].value = xmlTextReaderGetAttribute(reader, BAD_CAST "value");
 312             featureAddress[featureAddressLines].distance = xmlTextReaderGetAttribute(reader, BAD_CAST "distance");
 313
 314             featureAddressLines++;
 315                 }
 316                 else
 317         {
 318             fprintf( stderr, "Too many address elements (%s%s)\n", feature.type, feature.id);
 319 //            exit_nicely();
 320         }
 321
 322         return;
 323     }
 324     fprintf(stderr, "%s: Unknown element name: %s\n", __FUNCTION__, name);
 325 }
 326
 327 void EndElement(xmlTextReaderPtr reader, const xmlChar *name)
 328 {
 329     PGresult *          res;
 330     const char *        paramValues[14];
 331     char *                      place_id;
 332     char *                      partionQueryName;
 333     int i, namePos, lineTypeLen, lineValueLen;
 334
 335     if (xmlStrEqual(name, BAD_CAST "feature"))
 336     {
 337         featureCount++;
 338         if (featureCount % 1000 == 0) printf("feature %i(k)\n", featureCount/1000);
 339 /*
 340         if (fileMode == FILEMODE_ADD)
 341         {
 342             resPlaceID = PQexecPrepared(conn, "get_new_place_id", 0, NULL, NULL, NULL, 0);
 343             if (PQresultStatus(resPlaceID) != PGRES_TUPLES_OK)
 344             {
 345                 fprintf(stderr, "get_place_id: INSERT failed: %s", PQerrorMessage(conn));
 346                 PQclear(resPlaceID);
 347                 exit(EXIT_FAILURE);
 348             }
 349         }
 350         else
 351         {
 352             paramValues[0] = (const char *)feature.type;
 353             paramValues[1] = (const char *)feature.id;
 354             paramValues[2] = (const char *)feature.key;
 355             paramValues[3] = (const char *)feature.value;
 356             resPlaceID = PQexecPrepared(conn, "get_new_place_id", 4, paramValues, NULL, NULL, 0);
 357             if (PQresultStatus(resPlaceID) != PGRES_TUPLES_OK)
 358             {
 359                 fprintf(stderr, "index_placex: INSERT failed: %s", PQerrorMessage(conn));
 360                 PQclear(resPlaceID);
 361                 exit(EXIT_FAILURE);
 362             }
 363         }
 364 */
 365         place_id = (char *)feature.placeID;
 366
 367         if (fileMode == FILEMODE_UPDATE || fileMode == FILEMODE_DELETE || fileMode == FILEMODE_ADD)
 368         {
 369             paramValues[0] = (const char *)place_id;
 370             res = PQexecPrepared(conn, "placex_delete", 1, paramValues, NULL, NULL, 0);
 371             if (PQresultStatus(res) != PGRES_COMMAND_OK)
 372             {
 373                 fprintf(stderr, "placex_delete: DELETE failed: %s", PQerrorMessage(conn));
 374                 PQclear(res);
 375                 exit(EXIT_FAILURE);
 376             }
 377             PQclear(res);
 378
 379             res = PQexecPrepared(conn, "search_name_delete", 1, paramValues, NULL, NULL, 0);
 380             if (PQresultStatus(res) != PGRES_COMMAND_OK)
 381             {
 382                 fprintf(stderr, "search_name_delete: DELETE failed: %s", PQerrorMessage(conn));
 383                 PQclear(res);
 384                 exit(EXIT_FAILURE);
 385             }
 386             PQclear(res);
 387
 388             res = PQexecPrepared(conn, "place_addressline_delete", 1, paramValues, NULL, NULL, 0);
 389             if (PQresultStatus(res) != PGRES_COMMAND_OK)
 390             {
 391                 fprintf(stderr, "place_addressline_delete: DELETE failed: %s", PQerrorMessage(conn));
 392                 PQclear(res);
 393                 exit(EXIT_FAILURE);
 394             }
 395             PQclear(res);
 396
 397             partionQueryName = xmlHashLookup2(partionTableTagsHashDelete, feature.key, feature.value);
 398             if (partionQueryName)
 399             {
 400                 res = PQexecPrepared(conn, partionQueryName, 1, paramValues, NULL, NULL, 0);
 401                 if (PQresultStatus(res) != PGRES_COMMAND_OK)
 402                 {
 403                     fprintf(stderr, "%s: DELETE failed: %s", partionQueryName, PQerrorMessage(conn));
 404                     PQclear(res);
 405                     exit(EXIT_FAILURE);
 406                 }
 407                 PQclear(res);
 408             }
 409         }
 410
 411         if (fileMode == FILEMODE_UPDATE || fileMode == FILEMODE_ADD)
 412         {
 413             // Insert into placex
 414             paramValues[0] = (const char *)place_id;
 415             paramValues[1] = (const char *)feature.type;
 416             paramValues[2] = (const char *)feature.id;
 417             paramValues[3] = (const char *)feature.key;
 418             paramValues[4] = (const char *)feature.value;
 419
 420             featureNameString[0] = 0;
 421             if (featureNameLines)
 422             {
 423                 namePos = 0;
 424                 lineTypeLen = 0;
 425                 lineValueLen = 0;
 426                 for (i = 0; i < featureNameLines; i++)
 427                 {
 428                     lineTypeLen = (int)strlen((char *) featureName[i].type);
 429                     lineValueLen = (int)strlen((char *) featureName[i].value);
 430                     if (namePos+lineTypeLen+lineValueLen+7 > MAX_FEATURENAMESTRING)
 431                     {
 432                         fprintf(stderr, "feature name too long: %s", (const char *)featureName[i].value);
 433                         break;
 434                     }
 435                     if (namePos) strcpy(featureNameString+(namePos++), ",");
 436                     strcpy(featureNameString+(namePos++), "\"");
 437                     strcpy(featureNameString+namePos, (char*) featureName[i].type);
 438                     namePos += lineTypeLen;
 439                     strcpy(featureNameString+namePos, "\"=>\"");
 440                     namePos += 4;
 441                     strcpy(featureNameString+namePos, (char *) featureName[i].value);
 442                     namePos += lineValueLen;
 443                     strcpy(featureNameString+(namePos++), "\"");
 444                 }
 445             }
 446             paramValues[5] = (const char *)featureNameString;
 447
 448             paramValues[6] = (const char *)feature.countryCode;
 449
 450             featureExtraTagString[0] = 0;
 451             if (featureExtraTagLines)
 452             {
 453                 namePos = 0;
 454                 lineTypeLen = 0;
 455                 lineValueLen = 0;
 456                 for (i = 0; i < featureExtraTagLines; i++)
 457                 {
 458                     lineTypeLen = strlen((char *) featureExtraTag[i].type);
 459                     lineValueLen = strlen((char *) featureExtraTag[i].value);
 460                     if (namePos+lineTypeLen+lineValueLen+7 > MAX_FEATUREEXTRATAGSTRING)
 461                     {
 462                         fprintf(stderr, "feature extra tag too long: %s", (const char *)featureExtraTag[i].value);
 463                         break;
 464                     }
 465                     if (namePos) strcpy(featureExtraTagString+(namePos++),",");
 466                     strcpy(featureExtraTagString+(namePos++), "\"");
 467                     strcpy(featureExtraTagString+namePos, (char *) featureExtraTag[i].type);
 468                     namePos += lineTypeLen;
 469                     strcpy(featureExtraTagString+namePos, "\"=>\"");
 470                     namePos += 4;
 471                     strcpy(featureExtraTagString+namePos, (char *) featureExtraTag[i].value);
 472                     namePos += lineValueLen;
 473                     strcpy(featureExtraTagString+(namePos++), "\"");
 474                 }
 475             }
 476             paramValues[7] = (const char *)featureExtraTagString;
 477
 478             if (strlen(feature.parentPlaceID) == 0)
 479                 paramValues[8] = "0";
 480             else
 481                 paramValues[8] = (const char *)feature.parentPlaceID;
 482
 483             paramValues[9] = (const char *)feature.adminLevel;
 484             paramValues[10] = (const char *)feature.houseNumber;
 485             paramValues[11] = (const char *)feature.rankAddress;
 486             paramValues[12] = (const char *)feature.rankSearch;
 487             paramValues[13] = (const char *)feature.geometry;
 488             if (strlen(paramValues[3]) && strlen(paramValues[13]))
 489             {
 490                 res = PQexecPrepared(conn, "placex_insert", 14, paramValues, NULL, NULL, 0);
 491                 if (PQresultStatus(res) != PGRES_COMMAND_OK)
 492                 {
 493                     fprintf(stderr, "index_placex: INSERT failed: %s", PQerrorMessage(conn));
 494                     fprintf(stderr, "index_placex: INSERT failed: %s %s %s", paramValues[0], paramValues[1], paramValues[2]);
 495                     PQclear(res);
 496                     exit(EXIT_FAILURE);
 497                }
 498                PQclear(res);
 499             }
 500
 501             for (i = 0; i < featureAddressLines; i++)
 502             {
 503                 // insert into place_address
 504                 paramValues[0] = (const char *)place_id;
 505                 paramValues[1] = (const char *)featureAddress[i].distance;
 506                 paramValues[2] = (const char *)featureAddress[i].type;
 507                 paramValues[3] = (const char *)featureAddress[i].id;
 508                 paramValues[4] = (const char *)featureAddress[i].key;
 509                 paramValues[5] = (const char *)featureAddress[i].value;
 510                 paramValues[6] = (const char *)featureAddress[i].isAddress;
 511                 res = PQexecPrepared(conn, "place_addressline_insert", 7, paramValues, NULL, NULL, 0);
 512                 if (PQresultStatus(res) != PGRES_COMMAND_OK)
 513                 {
 514                     fprintf(stderr, "place_addressline_insert: INSERT failed: %s", PQerrorMessage(conn));
 515                     PQclear(res);
 516                     exit(EXIT_FAILURE);
 517                 }
 518                 PQclear(res);
 519
 520                 xmlFree(featureAddress[i].type);
 521                 xmlFree(featureAddress[i].id);
 522                 xmlFree(featureAddress[i].key);
 523                 xmlFree(featureAddress[i].value);
 524                 xmlFree(featureAddress[i].distance);
 525             }
 526
 527             if (featureNameLines)
 528             {
 529                 if (strlen(feature.parentPlaceID) > 0 && featureAddressLines == 0)
 530                 {
 531                     paramValues[0] = (const char *)place_id;
 532                     paramValues[1] = feature.parentPlaceID;
 533                     res = PQexecPrepared(conn, "search_name_from_parent_insert", 2, paramValues, NULL, NULL, 0);
 534                     if (PQresultStatus(res) != PGRES_COMMAND_OK)
 535                     {
 536                         fprintf(stderr, "search_name_from_parent_insert: INSERT failed: %s", PQerrorMessage(conn));
 537                         PQclear(res);
 538                         exit(EXIT_FAILURE);
 539                     }
 540                     PQclear(res);
 541                 }
 542                 else
 543                 {
 544                     paramValues[0] = (const char *)place_id;
 545                     res = PQexecPrepared(conn, "search_name_insert", 1, paramValues, NULL, NULL, 0);
 546                     if (PQresultStatus(res) != PGRES_COMMAND_OK)
 547                     {
 548                         fprintf(stderr, "search_name_insert: INSERT failed: %s", PQerrorMessage(conn));
 549                         PQclear(res);
 550                         exit(EXIT_FAILURE);
 551                     }
 552                     PQclear(res);
 553                 }
 554             }
 555
 556             partionQueryName = xmlHashLookup2(partionTableTagsHash, feature.key, feature.value);
 557             if (partionQueryName)
 558             {
 559                 // insert into partition table
 560                 paramValues[0] = (const char *)place_id;
 561                 paramValues[1] = (const char *)feature.geometry;
 562                 res = PQexecPrepared(conn, partionQueryName, 2, paramValues, NULL, NULL, 0);
 563                 if (PQresultStatus(res) != PGRES_COMMAND_OK)
 564                 {
 565                     fprintf(stderr, "%s: INSERT failed: %s", partionQueryName, PQerrorMessage(conn));
 566                     PQclear(res);
 567                     exit(EXIT_FAILURE);
 568                 }
 569                 PQclear(res);
 570             }
 571
 572         }
 573         else
 574         {
 575             for (i = 0; i < featureAddressLines; i++)
 576             {
 577                 xmlFree(featureAddress[i].type);
 578                 xmlFree(featureAddress[i].id);
 579                 xmlFree(featureAddress[i].key);
 580                 xmlFree(featureAddress[i].value);
 581                 xmlFree(featureAddress[i].distance);
 582             }
 583         }
 584
 585         xmlFree(feature.placeID);
 586         xmlFree(feature.type);
 587         xmlFree(feature.id);
 588         xmlFree(feature.key);
 589         xmlFree(feature.value);
 590         xmlFree(feature.rankAddress);
 591         xmlFree(feature.rankSearch);
 592         if (feature.parentPlaceID) xmlFree(feature.parentPlaceID);
 593         if (feature.parentType) xmlFree(feature.parentType);
 594         if (feature.parentID) xmlFree(feature.parentID);
 595 //              if (feature.name) xmlFree(feature.name);
 596         if (feature.countryCode) xmlFree(feature.countryCode);
 597         if (feature.adminLevel) xmlFree(feature.adminLevel);
 598         if (feature.houseNumber) xmlFree(feature.houseNumber);
 599         if (feature.geometry) xmlFree(feature.geometry);
 600
 601 //        PQclear(resPlaceID);
 602     }
 603 }
 604
 605 static void processNode(xmlTextReaderPtr reader)
 606 {
 607     xmlChar *name;
 608     name = xmlTextReaderName(reader);
 609     if (name == NULL)
 610     {
 611         name = xmlStrdup(BAD_CAST "--");
 612     }
 613
 614     switch (xmlTextReaderNodeType(reader))
 615     {
 616     case XML_READER_TYPE_ELEMENT:
 617         StartElement(reader, name);
 618         if (xmlTextReaderIsEmptyElement(reader))
 619             EndElement(reader, name); /* No end_element for self closing tags! */
 620         break;
 621     case XML_READER_TYPE_END_ELEMENT:
 622         EndElement(reader, name);
 623         break;
 624     case XML_READER_TYPE_TEXT:
 625     case XML_READER_TYPE_CDATA:
 626     case XML_READER_TYPE_SIGNIFICANT_WHITESPACE:
 627         /* Ignore */
 628         break;
 629     default:
 630         fprintf(stderr, "Unknown node type %d\n", xmlTextReaderNodeType(reader));
 631         break;
 632     }
 633
 634     xmlFree(name);
 635 }
 636
 637 int nominatim_import(const char *conninfo, const char *partionTagsFilename, const char *filename)
 638 {
 639     xmlTextReaderPtr    reader;
 640     int                                 ret = 0;
 641     PGresult *                  res;
 642     FILE *                              partionTagsFile;
 643     char *                              partionQueryName;
 644     char                                partionQuerySQL[1024];
 645
 646     conn = PQconnectdb(conninfo);
 647     if (PQstatus(conn) != CONNECTION_OK)
 648     {
 649         fprintf(stderr, "Connection to database failed: %s\n", PQerrorMessage(conn));
 650         exit(EXIT_FAILURE);
 651     }
 652
 653     partionTableTagsHash = xmlHashCreate(200);
 654     partionTableTagsHashDelete = xmlHashCreate(200);
 655
 656     partionTagsFile = fopen(partionTagsFilename, "rt");
 657     if (!partionTagsFile)
 658     {
 659         fprintf(stderr, "Unable to read partition tags file: %s\n", partionTagsFilename);
 660         exit(EXIT_FAILURE);
 661     }
 662
 663     char buffer[1024], osmkey[256], osmvalue[256];
 664     int fields;
 665     while (fgets(buffer, sizeof(buffer), partionTagsFile) != NULL)
 666     {
 667         fields = sscanf( buffer, "%23s %63s", osmkey, osmvalue );
 668
 669         if ( fields <= 0 ) continue;
 670
 671         if ( fields != 2  )
 672         {
 673             fprintf( stderr, "Error partition file\n");
 674             exit_nicely();
 675         }
 676         partionQueryName = malloc(strlen("partition_insert_")+strlen(osmkey)+strlen(osmvalue)+2);
 677         strcpy(partionQueryName, "partition_insert_");
 678         strcat(partionQueryName, osmkey);
 679         strcat(partionQueryName, "_");
 680         strcat(partionQueryName, osmvalue);
 681
 682         strcpy(partionQuerySQL, "insert into place_classtype_");
 683         strcat(partionQuerySQL, osmkey);
 684         strcat(partionQuerySQL, "_");
 685         strcat(partionQuerySQL, osmvalue);
 686         strcat(partionQuerySQL, " (place_id, centroid) values ($1, ST_Centroid(st_setsrid($2, 4326)))");
 687
 688         res = PQprepare(conn, partionQueryName, partionQuerySQL, 2, NULL);
 689         if (PQresultStatus(res) != PGRES_COMMAND_OK)
 690         {
 691             fprintf(stderr, "Failed to prepare %s: %s\n", partionQueryName, PQerrorMessage(conn));
 692             exit(EXIT_FAILURE);
 693         }
 694
 695         xmlHashAddEntry2(partionTableTagsHash, BAD_CAST osmkey, BAD_CAST osmvalue, BAD_CAST partionQueryName);
 696
 697         partionQueryName = malloc(strlen("partition_delete_")+strlen(osmkey)+strlen(osmvalue)+2);
 698         strcpy(partionQueryName, "partition_delete_");
 699         strcat(partionQueryName, osmkey);
 700         strcat(partionQueryName, "_");
 701         strcat(partionQueryName, osmvalue);
 702
 703         strcpy(partionQuerySQL, "delete from place_classtype_");
 704         strcat(partionQuerySQL, osmkey);
 705         strcat(partionQuerySQL, "_");
 706         strcat(partionQuerySQL, osmvalue);
 707         strcat(partionQuerySQL, " where place_id = $1::integer");
 708
 709         res = PQprepare(conn, partionQueryName, partionQuerySQL, 1, NULL);
 710         if (PQresultStatus(res) != PGRES_COMMAND_OK)
 711         {
 712             fprintf(stderr, "Failed to prepare %s: %s\n", partionQueryName, PQerrorMessage(conn));
 713             exit(EXIT_FAILURE);
 714         }
 715
 716         xmlHashAddEntry2(partionTableTagsHashDelete, BAD_CAST osmkey, BAD_CAST osmvalue, BAD_CAST partionQueryName);
 717     }
 718
 719     res = PQprepare(conn, "get_new_place_id",
 720                     "select nextval('seq_place')",
 721                     0, NULL);
 722     if (PQresultStatus(res) != PGRES_COMMAND_OK)
 723     {
 724         fprintf(stderr, "Failed to prepare get_new_place_id: %s\n", PQerrorMessage(conn));
 725         exit(EXIT_FAILURE);
 726     }
 727
 728     res = PQprepare(conn, "get_place_id",
 729                     "select place_id from placex where osm_type = $1 and osm_id = $2 and class = $3 and type = $4",
 730                     4, NULL);
 731     if (PQresultStatus(res) != PGRES_COMMAND_OK)
 732     {
 733         fprintf(stderr, "Failed to prepare get_place_id: %s\n", PQerrorMessage(conn));
 734         exit(EXIT_FAILURE);
 735     }
 736
 737     res = PQprepare(conn, "placex_insert",
 738                     "insert into placex (place_id,osm_type,osm_id,class,type,name,country_code,extratags,parent_place_id,admin_level,housenumber,rank_address,rank_search,geometry) "
 739                     "values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, st_setsrid($14, 4326))",
 740                     12, NULL);
 741     if (PQresultStatus(res) != PGRES_COMMAND_OK)
 742     {
 743         fprintf(stderr, "Failed to prepare placex_insert: %s\n", PQerrorMessage(conn));
 744         exit(EXIT_FAILURE);
 745     }
 746
 747     res = PQprepare(conn, "search_name_insert",
 748                     "insert into search_name (place_id, search_rank, address_rank, country_code, name_vector, nameaddress_vector, centroid) "
 749                     "select place_id, rank_search, rank_address, country_code, make_keywords(name), "
 750                     "(select uniq(sort(array_agg(parent_search_name.name_vector))) from place_addressline join search_name as parent_search_name on "
 751                     "(address_place_id = parent_search_name.place_id) where place_addressline.place_id = $1 ), st_centroid(geometry) from placex "
 752                     "where place_id = $1",
 753                     1, NULL);
 754     if (PQresultStatus(res) != PGRES_COMMAND_OK)
 755     {
 756         fprintf(stderr, "Failed to prepare search_name_insert: %s\n", PQerrorMessage(conn));
 757         exit(EXIT_FAILURE);
 758     }
 759
 760     res = PQprepare(conn, "search_name_from_parent_insert",
 761                     "insert into search_name (place_id, search_rank, address_rank, country_code, name_vector, nameaddress_vector, centroid) "
 762                     "select place_id, rank_search, rank_address, country_code, make_keywords(name), "
 763                     "(select uniq(sort(name_vector+nameaddress_vector)) from search_name as parent_search_name "
 764                     "where parent_search_name.place_id = $2 ), st_centroid(geometry) from placex "
 765                     "where place_id = $1",
 766                     2, NULL);
 767     if (PQresultStatus(res) != PGRES_COMMAND_OK)
 768     {
 769         fprintf(stderr, "Failed to prepare search_name_insert: %s\n", PQerrorMessage(conn));
 770         exit(EXIT_FAILURE);
 771     }
 772
 773     res = PQprepare(conn, "place_addressline_insert",
 774                     "insert into place_addressline (place_id, address_place_id, fromarea, isaddress, distance, cached_rank_address) "
 775                     "select $1, place_id, false, $7, $2, rank_address from placex where osm_type = $3 and osm_id = $4 and class = $5 and type = $6",
 776                     7, NULL);
 777     if (PQresultStatus(res) != PGRES_COMMAND_OK)
 778     {
 779         fprintf(stderr, "Failed to prepare place_addressline_insert: %s\n", PQerrorMessage(conn));
 780         exit(EXIT_FAILURE);
 781     }
 782
 783     res = PQprepare(conn, "placex_delete",
 784                     "delete from placex where place_id = $1",
 785                     1, NULL);
 786     if (PQresultStatus(res) != PGRES_COMMAND_OK)
 787     {
 788         fprintf(stderr, "Failed to prepare placex_delete: %s\n", PQerrorMessage(conn));
 789         exit(EXIT_FAILURE);
 790     }
 791
 792     res = PQprepare(conn, "search_name_delete",
 793                     "delete from search_name where place_id = $1",
 794                     1, NULL);
 795     if (PQresultStatus(res) != PGRES_COMMAND_OK)
 796     {
 797         fprintf(stderr, "Failed to prepare search_name_delete: %s\n", PQerrorMessage(conn));
 798         exit(EXIT_FAILURE);
 799     }
 800
 801     res = PQprepare(conn, "place_addressline_delete",
 802                     "delete from place_addressline where place_id = $1",
 803                     1, NULL);
 804     if (PQresultStatus(res) != PGRES_COMMAND_OK)
 805     {
 806         fprintf(stderr, "Failed to prepare place_addressline_delete: %s\n", PQerrorMessage(conn));
 807         exit(EXIT_FAILURE);
 808     }
 809
 810     featureCount = 0;
 811
 812     reader = inputUTF8(filename);
 813
 814     if (reader == NULL)
 815     {
 816         fprintf(stderr, "Unable to open %s\n", filename);
 817         return 1;
 818     }
 819
 820     ret = xmlTextReaderRead(reader);
 821     while (ret == 1)
 822     {
 823         processNode(reader);
 824         ret = xmlTextReaderRead(reader);
 825     }
 826     if (ret != 0)
 827     {
 828         fprintf(stderr, "%s : failed to parse\n", filename);
 829         return ret;
 830     }
 831
 832     xmlFreeTextReader(reader);
 833     xmlHashFree(partionTableTagsHash, NULL);
 834     xmlHashFree(partionTableTagsHashDelete, NULL);
 835
 836     return 0;
 837 }