more multi-processor improvements

diff --git a/nominatim/index.c b/nominatim/index.c
index 1ae414c06f88344ebca4a7abc71aba004b7f2faf..b81d0c050a305726eaa7226f3c18ac25592afc3b 100644 (file)
--- a/nominatim/index.c
+++ b/nominatim/index.c
@@ -68,6 +68,17 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co
     }
     PQclear(res);
 
     }
     PQclear(res);
 

+    pg_prepare_params[0] = PG_OID_INT4;
+    res = PQprepare(conn, "index_nosectors",
+       "select 0::integer,count(*) from placex where rank_search = $1 and indexed_status > 0",
+       1, pg_prepare_params);
+    if (PQresultStatus(res) != PGRES_COMMAND_OK)
+    {
+        fprintf(stderr, "Failed preparing index_sectors: %s\n", PQerrorMessage(conn));
+        exit(EXIT_FAILURE);
+    }
+    PQclear(res);
+

     pg_prepare_params[0] = PG_OID_INT4;
     pg_prepare_params[1] = PG_OID_INT4;
     res = PQprepare(conn, "index_sector_places",
     pg_prepare_params[0] = PG_OID_INT4;
     pg_prepare_params[1] = PG_OID_INT4;
     res = PQprepare(conn, "index_sector_places",


@@ -80,6 +91,17 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co
     }
     PQclear(res);
 
     }
     PQclear(res);
 

+    pg_prepare_params[0] = PG_OID_INT4;
+    res = PQprepare(conn, "index_nosector_places",
+       "select place_id from placex where rank_search = $1 and indexed_status > 0 order by geometry_sector",
+       1, pg_prepare_params);
+    if (PQresultStatus(res) != PGRES_COMMAND_OK)
+    {
+        fprintf(stderr, "Failed preparing index_nosector_places: %s\n", PQerrorMessage(conn));
+        exit(EXIT_FAILURE);
+    }
+    PQclear(res);
+

     // Build the data for each thread
     thread_data = (struct index_thread_data *)malloc(sizeof(struct index_thread_data)*num_threads);
        for (i = 0; i < num_threads; i++)
     // Build the data for each thread
     thread_data = (struct index_thread_data *)malloc(sizeof(struct index_thread_data)*num_threads);
        for (i = 0; i < num_threads; i++)


@@ -124,7 +146,10 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co
         paramValues[0] = (char *)&paramRank;
         paramLengths[0] = sizeof(paramRank);
         paramFormats[0] = 1;
         paramValues[0] = (char *)&paramRank;
         paramLengths[0] = sizeof(paramRank);
         paramFormats[0] = 1;

-        resSectors = PQexecPrepared(conn, "index_sectors", 1, paramValues, paramLengths, paramFormats, 1);
+               if (rank < 16)
+               resSectors = PQexecPrepared(conn, "index_nosectors", 1, paramValues, paramLengths, paramFormats, 1);
+               else
+               resSectors = PQexecPrepared(conn, "index_sectors", 1, paramValues, paramLengths, paramFormats, 1);

         if (PQresultStatus(resSectors) != PGRES_TUPLES_OK)
         {
             fprintf(stderr, "index_sectors: SELECT failed: %s", PQerrorMessage(conn));
         if (PQresultStatus(resSectors) != PGRES_TUPLES_OK)
         {
             fprintf(stderr, "index_sectors: SELECT failed: %s", PQerrorMessage(conn));


@@ -156,15 +181,18 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co
                //printf("\n Starting sector %d size %ld\n", sector, PGint64(*((uint64_t *)PQgetvalue(resSectors, iSector, 1))));
 
                        // Get all the place_id's for this sector
                //printf("\n Starting sector %d size %ld\n", sector, PGint64(*((uint64_t *)PQgetvalue(resSectors, iSector, 1))));
 
                        // Get all the place_id's for this sector

-               paramRank = PGint32(rank);
-               paramValues[0] = (char *)&paramRank;
-               paramLengths[0] = sizeof(paramRank);
-               paramFormats[0] = 1;
-               paramSector = PGint32(sector);
+                   paramRank = PGint32(rank);
+               paramValues[0] = (char *)&paramRank;
+                   paramLengths[0] = sizeof(paramRank);
+                   paramFormats[0] = 1;
+               paramSector = PGint32(sector);

                paramValues[1] = (char *)&paramSector;
                paramValues[1] = (char *)&paramSector;

-               paramLengths[1] = sizeof(paramSector);
-               paramFormats[1] = 1;
-               resPlaces = PQexecPrepared(conn, "index_sector_places", 2, paramValues, paramLengths, paramFormats, 1);
+                       paramLengths[1] = sizeof(paramSector);
+                   paramFormats[1] = 1;
+                       if (rank < 16)
+                       resPlaces = PQexecPrepared(conn, "index_nosector_places", 1, paramValues, paramLengths, paramFormats, 1);
+                       else
+                       resPlaces = PQexecPrepared(conn, "index_sector_places", 2, paramValues, paramLengths, paramFormats, 1);

                if (PQresultStatus(resPlaces) != PGRES_TUPLES_OK)
                {
                    fprintf(stderr, "index_sector_places: SELECT failed: %s", PQerrorMessage(conn));
                if (PQresultStatus(resPlaces) != PGRES_TUPLES_OK)
                {
                    fprintf(stderr, "index_sector_places: SELECT failed: %s", PQerrorMessage(conn));


@@ -265,7 +293,7 @@ void *nominatim_indexThread(void * thread_data_in)
 
                pthread_mutex_unlock( thread_data->count_mutex );
 
 
                pthread_mutex_unlock( thread_data->count_mutex );
 

-               //printf("  Processing place_id %ld\n", place_id);
+//             printf("  Processing place_id %d\n", place_id);

                paramPlaceID = PGint32(place_id);
         paramValues[0] = (char *)&paramPlaceID;
         paramLengths[0] = sizeof(paramPlaceID);
                paramPlaceID = PGint32(place_id);
         paramValues[0] = (char *)&paramPlaceID;
         paramLengths[0] = sizeof(paramPlaceID);

diff --git a/sql/functions.sql b/sql/functions.sql
index c2de656c1950dd921705c86e4b61469c9d77c72c..5b2f716a4dbefe5afce5d3511257e8bee722c1c6 100644 (file)
--- a/sql/functions.sql
+++ b/sql/functions.sql
@@ -508,6 +508,20 @@ END;
 $$
 LANGUAGE plpgsql IMMUTABLE;
 
 $$
 LANGUAGE plpgsql IMMUTABLE;
 

+CREATE OR REPLACE FUNCTION get_country_code(place geometry, in_country_code VARCHAR(2)) RETURNS TEXT
+  AS $$
+DECLARE
+  nearcountry RECORD;
+BEGIN
+  FOR nearcountry IN select country_code from country_name where country_code = lower(in_country_code)
+  LOOP
+    RETURN nearcountry.country_code;
+  END LOOP;
+  RETURN get_country_code(place);
+END;
+$$
+LANGUAGE plpgsql IMMUTABLE;
+

 CREATE OR REPLACE FUNCTION get_country_language_code(search_country_code VARCHAR(2)) RETURNS TEXT
   AS $$
 DECLARE
 CREATE OR REPLACE FUNCTION get_country_language_code(search_country_code VARCHAR(2)) RETURNS TEXT
   AS $$
 DECLARE


@@ -882,9 +896,7 @@ BEGIN
   NEW.place_id := nextval('seq_place');
   NEW.indexed_status := 1; --STATUS_NEW
 
   NEW.place_id := nextval('seq_place');
   NEW.indexed_status := 1; --STATUS_NEW
 

-  IF NEW.country_code is null THEN
-    NEW.country_code := get_country_code(NEW.geometry);
-  END IF;
+  NEW.country_code := get_country_code(NEW.geometry, NEW.country_code);

   NEW.geometry_sector := geometry_sector(NEW.geometry);
   NEW.partition := get_partition(NEW.geometry, NEW.country_code);
 
   NEW.geometry_sector := geometry_sector(NEW.geometry);
   NEW.partition := get_partition(NEW.geometry, NEW.country_code);
 


@@ -1159,15 +1171,6 @@ BEGIN
     RETURN NEW;
   END IF;
 
     RETURN NEW;
   END IF;
 

-  IF NEW.country_code is null THEN
-    NEW.country_code := get_country_code(NEW.geometry);
-  END IF;
-  NEW.country_code := lower(NEW.country_code);
-  NEW.partition := NEW.country_code;
-  IF NEW.partition is null THEN
-    NEW.partition := 'none';
-  END IF;
-

   IF NEW.indexed_status = 0 and OLD.indexed_status != 0 THEN
 
     NEW.indexed_date = now();
   IF NEW.indexed_status = 0 and OLD.indexed_status != 0 THEN
 
     NEW.indexed_date = now();


@@ -1395,6 +1398,7 @@ BEGIN
     -- Process area matches
     location_rank_search := 100;
     location_distance := 0;
     -- Process area matches
     location_rank_search := 100;
     location_distance := 0;

+--RAISE WARNING '%', NEW.partition;

     FOR location IN SELECT * from getNearFeatures(NEW.partition, place_centroid, search_maxrank, isin_tokens) LOOP
 
 --RAISE WARNING '  AREA: %',location;
     FOR location IN SELECT * from getNearFeatures(NEW.partition, place_centroid, search_maxrank, isin_tokens) LOOP
 
 --RAISE WARNING '  AREA: %',location;