X-Git-Url: https://git.openstreetmap.org./rails.git/blobdiff_plain/1facda11d6a1dbcbc1290eb447fcb0df7c24d6d5..2a44ff581f4c547a3637ea52567a3398b1d8bfe0:/script/locale/merge-from-translatewiki

diff --git a/script/locale/merge-from-translatewiki b/script/locale/merge-from-translatewiki
old mode 100644
new mode 100755
index 83a66b246..706ec45b3
--- a/script/locale/merge-from-translatewiki
+++ b/script/locale/merge-from-translatewiki
@@ -23,12 +23,16 @@ merge-from-translatewiki - Get new translations from L<http://translatewiki.net>
 
 =head1 SYNOPSIS
 
+    # Run this normally, hopefully...
     merge-from-translatewiki --locales-dir=config/locales
 
     # Diff the existing files:
     config/locales$ for i in $(ls *yml | grep -v en.yml); do perl ../../script/locale/diff --dump-flat $i > $i.0 ;done
 
     # Merge and find out what changed:
+    rails_port$ perl script/locale/merge-from-translatewiki --locales-dir config/locales
+
+    # Or, more complexy:
     rails_port$ for i in $(svn st config/locales/ | egrep '^M|\\?' | awk '{print $2}' | grep 'yml$'); do rm -v $i; done && svn up config/locales && perl script/locale/merge-from-translatewiki --locales-dir config/locales && svn st config/locales
  
     # Diff:
@@ -50,7 +54,16 @@ Print this help message.
 
 =item --locales-dir
 
-The locales dir we'll merge stuff into. E.g. C<config/locales>.
+The locales dir we'll merge stuff into. F<config/locales> by default.
+
+=item --only-new
+
+Only import translations that don't exists for us yet.
+
+=item --cache
+
+Write a L<Storable> cache for things downloaded from Translatewiki and
+use it if it exists.
 
 =back
 
@@ -65,13 +78,15 @@ Getopt::Long::Parser->new(
     config => [ qw< bundling no_ignore_case no_require_order pass_through > ],
 )->getoptions(
     'h|help' => \my $help,
-    'locales-dir=s' => \my $locales_dir,
+    'locales-dir=s' => \(my $locales_dir = 'config/locales'),
+    'only-new' => \my $only_new,
+    'cache' => \my $cache,
 ) or help();
 
 # On --help
 help() if $help;
 
-help() unless $locales_dir;
+help() unless $locales_dir and -d $locales_dir;
 
 ###
 ### Main
@@ -104,7 +119,7 @@ for my $my_yaml_file (@my_yaml_files) {
 say "loaded my translations";
 
 ## Write out merged data
-for my $translatewiki_lang (@translatewiki_languages_codes) {
+for my $translatewiki_lang (sort @translatewiki_languages_codes) {
     my $rails_lang = $translatewiki_lang; $rails_lang =~ s/(?<=-)(\w+)/\U$1\E/;
     my $out_file = catfile($locales_dir, $rails_lang . '.yml');
 
@@ -114,19 +129,32 @@ for my $translatewiki_lang (@translatewiki_languages_codes) {
         my $expanded = expand_hash($translatewiki_translations{$translatewiki_lang});
         my $out = +{ $rails_lang => $expanded };
         spit_out($out_file, $out);
-    } elsif (ref $my_translations{$translatewiki_lang} eq 'HASH') {
-        say STDERR "$rails_lang has existing translations. Merging";
+    } elsif (ref $my_translations{$translatewiki_lang} eq 'HASH' and not $only_new) {
+        say STDERR "$rails_lang has existing translations. Merging the old translation with the new Translatewiki one";
 
         # Get the data
         my %tw = %{ $translatewiki_translations{$translatewiki_lang} };
         my %me = %{ $my_translations{$translatewiki_lang} };
+        my %en = %{ $my_translations{en} };
         # Use %tw to start with
         my %new = %tw;
 
         ### Merge stuff
 
-        # These keys shouldn't be removed
+        ## These keys shouldn't be removed but are due to
+        ## Translatewiki fail (they were missing in the original
+        ## import)
         my @url_keys = qw(
+                             browse.relation_member.entry
+                             changeset.changeset.id
+                             geocoder.search_osm_namefinder.suffix_suburb
+                             html.dir
+                             layouts.intro_3_bytemark
+                             layouts.intro_3_ucl
+                             layouts.project_name.h1
+                             layouts.project_name.title
+                             printable_name.with_version
+                             site.edit.anon_edits
                              layouts.help_wiki_url
                              layouts.shop_url
                              notifier.gpx_notification.failure.import_failures_url
@@ -139,14 +167,55 @@ for my $translatewiki_lang (@translatewiki_languages_codes) {
 
         for my $key (@url_keys) {
             if ( exists $me{$key} and not exists $new{$key} ) {
-                $new{$key} = $me{$key};
+                $new{$key} = $me{$key} if $me{$key} ne $en{$key};
+            }
+        }
+
+        ## When foo exists in this file but only foo.one, foo,other
+        ## etc in English or the original file we don't want to throw away what we have
+        my @plural_keys = qw( zero one many few other two );
+
+        while (my ($me_k, $me_v) = each %me) {
+            if (not exists $tw{ $me_k } and 
+                not exists $en{ $me_k } and
+                (
+                    exists $en{ $me_k . '.zero' } or
+                    exists $en{ $me_k . '.one' } or
+                    exists $en{ $me_k . '.many' } or
+                    exists $en{ $me_k . '.few' } or
+                    exists $en{ $me_k . '.other' } or
+                    exists $en{ $me_k . '.two' })) {
+                #say STDERR "Bringing back nuked plural form '$me_k' Setting it to '$me{ $me_k }'";
+                $new{ $me_k } = $me{ $me_k };
+            }
+        }
+
+        # Both arrays and strings are supported in the site key. Avoid removing e.g.:
+        #   -site.key.table.entry.school: 學校；大學
+        # Just because en.yml has site.key.table.entry.school.0 and site.key.table.entry.school.1
+        while (my ($me_k, $me_v) = each %me) {
+            next unless $me_k =~ /^site\.key\.table\.entry/;
+            next if $me_k =~ /\.\d+$/;
+
+            if (ref $en{ $me_k } eq 'ARRAY' and not ref $me{ $me_k }) {
+                $new{ $me_k } = $me{ $me_k };
+            }
+        }
+
+        # There are a bunch of keys on Translatewiki that are
+        # equivalent to English for some reason. Probably because they
+        # were there at import time. Nuke them.
+        while (my ($new_k, $new_v) = each %new) {
+            if (exists $en{ $new_k } and $en{ $new_k } eq $new_v) {
+                #say "Purging dupe in $rails_lang: $new_k=$new_v";
+                delete $new{ $new_k };
             }
         }
 
         my $expanded = expand_hash( \%new );
         my $out = +{ $rails_lang => $expanded };
         spit_out($out_file, $out);
-    } else {
+    } elsif (not $only_new) {
         die "Internal error on $translatewiki_lang";
     }
 }
@@ -157,7 +226,6 @@ sub spit_out
     my $yaml_out = Dump $data;
     
     open my $fh, ">", $file;
-    say $fh "# Imported at " . (scalar localtime) . " from Translatewiki.net";
     print $fh $yaml_out;
     close $fh;
 }
@@ -199,16 +267,46 @@ sub expand_hash
     my ($flat_hash) = @_;
     my %new_hash;
     while (my ($k, $v) = each %$flat_hash) {
+        #say "Inserting $k=$v";
         insert_string_deep(\%new_hash, $k, $v);
     }
 
     \%new_hash;
 }
 
-sub insert_string_deep {
-    my ($h, $ks, $v) = @_;
-    my $p = \$h; $p = \$$p->{$_} for split /\./, $ks;
-    $$p = $v;
+# Fails under strict in certain cases:
+## Inserting browse.start_rjs.object_list.history.type.way=Vía [[id]]
+## Inserting activerecord.models.relation_tag=Etiqueta de la relación
+## Inserting browse.changeset_details.has_nodes.one=Tiene el siguiente {{count}} nodo:
+## Can't use string ("Tiene {{count}} nodos:") as a HASH ref while "strict refs" in use at script/locale/merge-from-translatewiki line 234.
+# Line 234 = my $p = \$h; $p = \$$p->{$_} for split /\./, $ks;
+
+# sub insert_string_deep_X {
+#     my ($h, $ks, $v) = @_;
+#     my $p = \$h; $p = \$$p->{$_} for split /\./, $ks;
+#     $$p = $v;
+# }
+
+sub insert_string_deep
+{
+    my ($hash, $key, $value) = @_;
+
+    my @key = split /\./, $key;
+    my $h = $hash;
+
+    my $i = 0;
+    for my $k (@key) {
+        $i ++;
+        if ($i == @key) {
+            $h->{$k} = $value;
+        } else {
+            if (ref $h->{$k}) {
+                $h = $h->{$k};
+            } else {
+                $h = $h->{$k} = {};
+            }
+        }
+    }
 }
 
 #
@@ -220,15 +318,20 @@ sub get_translatewiki_translations
     my @languages = @_;
 
     my $cache_file = "/tmp/merge-from-translatewiki.storable";
-    if (-f $cache_file) {
-        my $cache = retrieve($cache_file);
-        return %$cache;
+    if ($cache) {
+        if (-f $cache_file) {
+            my $c = retrieve($cache_file);
+            return %$c;
+        }
     }
 
     my %translatewiki_languages;
-    say "All languages are: @languages";
+    my $all_count = scalar @languages;
+    say "Translatewiki has $all_count languages I'm about to get";
+    my $count = 0;
     for my $lang (@languages) {
-        say STDERR "Getting language $lang from Translatewiki";
+        $count ++;
+        say STDERR "Getting language $count/$all_count ($lang) from Translatewiki";
         my $yaml = get_language_from_translatewiki($lang);
 
         my $flat_data = load_and_flatten_yaml($yaml);
@@ -236,7 +339,9 @@ sub get_translatewiki_translations
         $translatewiki_languages{$lang} = $flat_data;
     }
 
-    store \%translatewiki_languages, $cache_file;
+    if ($cache) {
+        store \%translatewiki_languages, $cache_file;
+    }
 
     return %translatewiki_languages;
 }
@@ -246,7 +351,7 @@ sub get_language_from_translatewiki
     my ($lang) = @_;
     my $mech = WWW::Mechanize->new;
 
-    $mech->get("http://translatewiki.net/w/i.php?title=Special%3ATranslate&task=export-to-file&group=out-osm&language=$lang");
+    $mech->get("http://translatewiki.net/w/i.php?title=Special%3ATranslate&task=export-to-file&group=out-osm-site&language=$lang");
 
     die "Couldn't get lang $lang lang from Translatewiki" unless $mech->success;
 
@@ -261,7 +366,7 @@ sub translatewiki_languages
 {
     my $mech = WWW::Mechanize->new;
 
-    $mech->get('http://translatewiki.net/wiki/Translating:OpenStreetMap/stats/trunk');
+    $mech->get('http://translatewiki.net/wiki/Translating:OpenStreetMap/stats/trunk/site');
 
     die "Couldn't get translatewiki table" unless $mech->success;