simplify token precomputation

Rename function to reflect that it is only used for precomputation.
The token IDs are not really needed, so don't bother to compute
the array of tokens.

diff --git a/data/words.sql b/data/words.sql
index ac250739563e9a104c5be0fb41f09d79b69d69eb..5613d927334d21e0053090f2afea6d6c8c675e9b 100644 (file)
--- a/data/words.sql
+++ b/data/words.sql
@@ -29787,7 +29787,7 @@ st      5557484
 
 -- prefill word table
 
 
 -- prefill word table
 

-select count(make_keywords(v)) from (select distinct svals(name) as v from place) as w where v is not null;
+select count(precompute_words(v)) from (select distinct svals(name) as v from place) as w where v is not null;

 select count(getorcreate_housenumber_id(make_standard_name(v))) from (select distinct address->'housenumber' as v from place where address ? 'housenumber') as w;
 
 -- copy the word frequencies
 select count(getorcreate_housenumber_id(make_standard_name(v))) from (select distinct address->'housenumber' as v from place where address ? 'housenumber') as w;
 
 -- copy the word frequencies

diff --git a/lib-sql/functions/normalization.sql b/lib-sql/functions/normalization.sql
index f283f9165d9e324590ecbdaaa5fc9fc872651e16..c7bd2fc5f83391b7746bbcad03f462e5a133c866 100644 (file)
--- a/lib-sql/functions/normalization.sql
+++ b/lib-sql/functions/normalization.sql
@@ -377,40 +377,26 @@ $$
 LANGUAGE plpgsql;
 
 
 LANGUAGE plpgsql;
 
 

-CREATE OR REPLACE FUNCTION make_keywords(src TEXT)
-  RETURNS INTEGER[]
+CREATE OR REPLACE FUNCTION precompute_words(src TEXT)
+  RETURNS INTEGER

   AS $$
 DECLARE
   AS $$
 DECLARE

-  result INTEGER[];

   s TEXT;
   w INTEGER;
   words TEXT[];
   i INTEGER;
   j INTEGER;
 BEGIN
   s TEXT;
   w INTEGER;
   words TEXT[];
   i INTEGER;
   j INTEGER;
 BEGIN

-  result := '{}'::INTEGER[];
-

   s := make_standard_name(src);
   w := getorcreate_name_id(s, src);
 
   s := make_standard_name(src);
   w := getorcreate_name_id(s, src);
 

-  IF NOT (ARRAY[w] <@ result) THEN
-    result := result || w;
-  END IF;
-

   w := getorcreate_word_id(s);
 
   w := getorcreate_word_id(s);
 

-  IF w IS NOT NULL AND NOT (ARRAY[w] <@ result) THEN
-    result := result || w;
-  END IF;
-

   words := string_to_array(s, ' ');
   IF array_upper(words, 1) IS NOT NULL THEN
     FOR j IN 1..array_upper(words, 1) LOOP
       IF (words[j] != '') THEN
   words := string_to_array(s, ' ');
   IF array_upper(words, 1) IS NOT NULL THEN
     FOR j IN 1..array_upper(words, 1) LOOP
       IF (words[j] != '') THEN

-        w = getorcreate_word_id(words[j]);
-        IF w IS NOT NULL AND NOT (ARRAY[w] <@ result) THEN
-          result := result || w;
-        END IF;
+        w := getorcreate_word_id(words[j]);

       END IF;
     END LOOP;
   END IF;
       END IF;
     END LOOP;
   END IF;


@@ -421,9 +407,6 @@ BEGIN
       s := make_standard_name(words[j]);
       IF s != '' THEN
         w := getorcreate_word_id(s);
       s := make_standard_name(words[j]);
       IF s != '' THEN
         w := getorcreate_word_id(s);

-        IF w IS NOT NULL AND NOT (ARRAY[w] <@ result) THEN
-          result := result || w;
-        END IF;

       END IF;
     END LOOP;
   END IF;
       END IF;
     END LOOP;
   END IF;


@@ -433,13 +416,10 @@ BEGIN
     s := make_standard_name(s);
     IF s != '' THEN
       w := getorcreate_name_id(s, src);
     s := make_standard_name(s);
     IF s != '' THEN
       w := getorcreate_name_id(s, src);

-      IF NOT (ARRAY[w] <@ result) THEN
-        result := result || w;
-      END IF;

     END IF;
   END IF;
 
     END IF;
   END IF;
 

-  RETURN result;
+  RETURN 1;

 END;
 $$
 LANGUAGE plpgsql;
 END;
 $$
 LANGUAGE plpgsql;

diff --git a/test/python/test_tools_database_import.py b/test/python/test_tools_database_import.py
index e2852acb45adae34d5761b0e70ac6636341c9ea2..1311ef5dc052dc3168f5ba7e9789a639adcce61c 100644 (file)
--- a/test/python/test_tools_database_import.py
+++ b/test/python/test_tools_database_import.py
@@ -187,7 +187,7 @@ def test_truncate_database_tables(temp_db_conn, temp_db_cursor, table_factory):
 @pytest.mark.parametrize("threads", (1, 5))
 def test_load_data(dsn, src_dir, place_row, placex_table, osmline_table, word_table,
                    temp_db_cursor, threads):
 @pytest.mark.parametrize("threads", (1, 5))
 def test_load_data(dsn, src_dir, place_row, placex_table, osmline_table, word_table,
                    temp_db_cursor, threads):

-    for func in ('make_keywords', 'getorcreate_housenumber_id', 'make_standard_name'):
+    for func in ('precompute_words', 'getorcreate_housenumber_id', 'make_standard_name'):

         temp_db_cursor.execute("""CREATE FUNCTION {} (src TEXT)
                                   RETURNS TEXT AS $$ SELECT 'a'::TEXT $$ LANGUAGE SQL
                                """.format(func))
         temp_db_cursor.execute("""CREATE FUNCTION {} (src TEXT)
                                   RETURNS TEXT AS $$ SELECT 'a'::TEXT $$ LANGUAGE SQL
                                """.format(func))