ignore Unicode format characters for normalization

Also adds tests.

Fixes #1007.

diff --git a/settings/defaults.php b/settings/defaults.php
index 81c19c74307491aeb9b502bb5d6d0c652e98f5b3..2d8f47d097887a79e84951d359f6e44c1a4ee36e 100644 (file)
--- a/settings/defaults.php
+++ b/settings/defaults.php
@@ -20,7 +20,7 @@ if (isset($_GET['debug']) && $_GET['debug']) @define('CONST_Debug', true);
 // Rules for normalizing terms for comparison before doing comparisons.
 // The default is to remove accents and punctuation and to lower-case the
 // term. Spaces are kept but collapsed to one standard space.
-@define('CONST_Term_Normalization_Rules', ":: NFD (); [:Nonspacing Mark:] >;  :: lower (); [[:Punctuation:][:Space:]]+ > ' '; :: NFC ();");
+@define('CONST_Term_Normalization_Rules', ":: NFD (); [[:Nonspacing Mark:] [:Cf:]] >;  :: lower (); [[:Punctuation:][:Space:]]+ > ' '; :: NFC ();");
 
 // Set to false to avoid importing extra postcodes for the US.
 @define('CONST_Use_Extra_US_Postcodes', true);

diff --git a/test/bdd/db/import/postcodes.feature b/test/bdd/db/import/postcodes.feature
index 4c49dc5bf7d812a310241f1c6279b8480e054de0..7fde34d328e5c873be1ea85b17217c5cd190697d 100644 (file)
--- a/test/bdd/db/import/postcodes.feature
+++ b/test/bdd/db/import/postcodes.feature
@@ -95,7 +95,6 @@ Feature: Import of postcodes
             | object | postcode |
             | W93    | 445023   |
 
-    @wip
     Scenario: Postcodes from admin boundaries are preferred over estimated postcodes
         Given the scene admin-areas
         And the named places

diff --git a/test/bdd/db/query/normalization.feature b/test/bdd/db/query/normalization.feature
index 1ef1fcbe6b223aee279caba92b6cd79855fb05d1..3205264753f1c67cef5a0beefb2af0e5425df293 100644 (file)
--- a/test/bdd/db/query/normalization.feature
+++ b/test/bdd/db/query/normalization.feature
@@ -136,3 +136,13 @@ Feature: Import and search of names
         Then results contain
          | ID | osm_type | osm_id |
          | 0  | R        | 1 |
+
+     Scenario: Unprintable characters in postcodes are ignored
+        Given the named places
+            | osm  | class   | type   | address |
+            | N234 | amenity | prison | 'postcode' : u'1234\u200e' |
+        When importing
+        And searching for "1234"
+        Then results contain
+         | ID | osm_type |
+         | 0  | P        |

diff --git a/test/bdd/osm2pgsql/import/tags.feature b/test/bdd/osm2pgsql/import/tags.feature
index 7db8d629cd867a5b8784560b5b3991d1491d3cb3..e25943439bfd738b70f978501ff38420107d67eb 100644 (file)
--- a/test/bdd/osm2pgsql/import/tags.feature
+++ b/test/bdd/osm2pgsql/import/tags.feature
@@ -96,6 +96,15 @@ Feature: Tag evaluation
          | N3     | 'name: de' : 'Foo', 'name:\\\\' : 'real3' |
          | N4     | 'name: de' : 'Foo', 'name' : 'rea\\l3' |
 
+    Scenario: Unprintable character in address tag are maintained
+        When loading osm data
+         """
+         n23 Tamenity=yes,name=foo,addr:postcode=1234%200e%
+         """
+        Then place contains
+         | object | address |
+         | N23    | 'postcode' : u'1234\u200e' |
+
     Scenario Outline: Included places
         When loading osm data
          """

diff --git a/test/bdd/steps/db_ops.py b/test/bdd/steps/db_ops.py
index be2211fad9fb17c5ee65eba5ca3350106fd15c12..87babdad88158a1f42cb26f136d5957bfc4ad0b9 100644 (file)
--- a/test/bdd/steps/db_ops.py
+++ b/test/bdd/steps/db_ops.py
@@ -22,6 +22,8 @@ class PlaceColumn:
             self.add_hstore('extratags', key[6:], value)
         elif key.startswith('addr+'):
             self.add_hstore('address', key[5:], value)
+        elif key in ('name', 'address', 'extratags'):
+            self.columns[key] = eval('{' + value + '}')
         else:
             assert_in(key, ('class', 'type'))
             self.columns[key] = None if value == '' else value