remove legacy tokenizer and direct tests

[nominatim.git] / module / nominatim.c

diff --git a/module/nominatim.c b/module/nominatim.c
index 75238e2207ff172aad46771569b091d7e73980d5..54632f76d03e4bffe9a7ddebc2cb8ced6f3413e0 100644 (file)
--- a/module/nominatim.c
+++ b/module/nominatim.c
@@ -1,12 +1,22 @@
+/**
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This file is part of Nominatim. (https://nominatim.org)
+ *
+ * Copyright (C) 2022 by the Nominatim developer community.
+ * For a full list of authors see the git log.
+ */
 #include "postgres.h"
 #include "fmgr.h"
 #include "mb/pg_wchar.h"
 #include <utfasciitable.h>
 
-#ifdef PG_MODULE_MAGIC
-PG_MODULE_MAGIC;
+#if PG_MAJORVERSION_NUM > 15
+#include "varatt.h"
 #endif
 
+PG_MODULE_MAGIC;
+
 Datum transliteration( PG_FUNCTION_ARGS );
 Datum gettokenstring( PG_FUNCTION_ARGS );
 void str_replace(char* buffer, int* len, int* changes, char* from, int fromlen, char* to, int tolen, int);
@@ -157,17 +167,18 @@ transliteration( PG_FUNCTION_ARGS )
        PG_RETURN_TEXT_P(result);
 }
 
+// Set isspace=1 if the replacement _only_ adds a space before the search string.  I.e. to == " " + from
 void str_replace(char* buffer, int* len, int* changes, char* from, int fromlen, char* to, int tolen, int isspace)
 {
         char *p;
 
-        // Search string is too long to be pressent
+        // Search string is too long to be present
         if (fromlen > *len) return;
 
         p = strstr(buffer, from);
         while(p)
         {
-                if (!isspace || *(p-1) != ' ')
+                if (!isspace || (p > buffer && *(p-1) != ' '))
                 {
                         (*changes)++;
                         if (tolen != fromlen) memmove(p+tolen, p+fromlen, *len-(p-buffer)+1);
@@ -230,7 +241,7 @@ gettokenstring( PG_FUNCTION_ARGS )
        sourcedata = (unsigned char *)VARDATA(source);
        sourcedatalength = VARSIZE(source) - VARHDRSZ;
 
-       // Buffer for doing the replace in - string could get slightly longer (double is mastive overkill)
+       // Buffer for doing the replace in - string could get slightly longer (double is massive overkill)
        buffer = (char *)palloc((sourcedatalength*2)*sizeof(char));
        memcpy(buffer+1, sourcedata, sourcedatalength);
        buffer[0] = 32;