=head1 SYNOPSIS
+ # Run this normally, hopefully...
merge-from-translatewiki --locales-dir=config/locales
# Diff the existing files:
config/locales$ for i in $(ls *yml | grep -v en.yml); do perl ../../script/locale/diff --dump-flat $i > $i.0 ;done
# Merge and find out what changed:
+ rails_port$ perl script/locale/merge-from-translatewiki --locales-dir config/locales
+
+ # Or, more complexy:
rails_port$ for i in $(svn st config/locales/ | egrep '^M|\\?' | awk '{print $2}' | grep 'yml$'); do rm -v $i; done && svn up config/locales && perl script/locale/merge-from-translatewiki --locales-dir config/locales && svn st config/locales
# Diff:
=item --locales-dir
-The locales dir we'll merge stuff into. E.g. C<config/locales>.
+The locales dir we'll merge stuff into. F<config/locales> by default.
+
+=item --only-new
+
+Only import translations that don't exists for us yet.
+
+=item --cache
+
+Write a L<Storable> cache for things downloaded from Translatewiki and
+use it if it exists.
=back
config => [ qw< bundling no_ignore_case no_require_order pass_through > ],
)->getoptions(
'h|help' => \my $help,
- 'locales-dir=s' => \my $locales_dir,
+ 'locales-dir=s' => \(my $locales_dir = 'config/locales'),
+ 'only-new' => \my $only_new,
+ 'cache' => \my $cache,
) or help();
# On --help
help() if $help;
-help() unless $locales_dir;
+help() unless $locales_dir and -d $locales_dir;
###
### Main
say "loaded my translations";
## Write out merged data
-for my $translatewiki_lang (@translatewiki_languages_codes) {
+for my $translatewiki_lang (sort @translatewiki_languages_codes) {
my $rails_lang = $translatewiki_lang; $rails_lang =~ s/(?<=-)(\w+)/\U$1\E/;
my $out_file = catfile($locales_dir, $rails_lang . '.yml');
my $expanded = expand_hash($translatewiki_translations{$translatewiki_lang});
my $out = +{ $rails_lang => $expanded };
spit_out($out_file, $out);
- } elsif (ref $my_translations{$translatewiki_lang} eq 'HASH') {
- say STDERR "$rails_lang has existing translations. Merging";
+ } elsif (ref $my_translations{$translatewiki_lang} eq 'HASH' and not $only_new) {
+ say STDERR "$rails_lang has existing translations. Merging the old translation with the new Translatewiki one";
# Get the data
my %tw = %{ $translatewiki_translations{$translatewiki_lang} };
my %me = %{ $my_translations{$translatewiki_lang} };
+ my %en = %{ $my_translations{en} };
# Use %tw to start with
my %new = %tw;
### Merge stuff
- # These keys shouldn't be removed
+ ## These keys shouldn't be removed but are due to
+ ## Translatewiki fail (they were missing in the original
+ ## import)
my @url_keys = qw(
+ browse.relation_member.entry
+ changeset.changeset.id
+ geocoder.search_osm_namefinder.suffix_suburb
+ html.dir
+ layouts.intro_3_bytemark
+ layouts.intro_3_ucl
+ layouts.project_name.h1
+ layouts.project_name.title
+ printable_name.with_version
+ site.edit.anon_edits
layouts.help_wiki_url
layouts.shop_url
notifier.gpx_notification.failure.import_failures_url
for my $key (@url_keys) {
if ( exists $me{$key} and not exists $new{$key} ) {
- $new{$key} = $me{$key};
+ $new{$key} = $me{$key} if $me{$key} ne $en{$key};
+ }
+ }
+
+ ## When foo exists in this file but only foo.one, foo,other
+ ## etc in English or the original file we don't want to throw away what we have
+ my @plural_keys = qw( zero one many few other two );
+
+ while (my ($me_k, $me_v) = each %me) {
+ if (not exists $tw{ $me_k } and
+ not exists $en{ $me_k } and
+ (
+ exists $en{ $me_k . '.zero' } or
+ exists $en{ $me_k . '.one' } or
+ exists $en{ $me_k . '.many' } or
+ exists $en{ $me_k . '.few' } or
+ exists $en{ $me_k . '.other' } or
+ exists $en{ $me_k . '.two' })) {
+ #say STDERR "Bringing back nuked plural form '$me_k' Setting it to '$me{ $me_k }'";
+ $new{ $me_k } = $me{ $me_k };
+ }
+ }
+
+ # Both arrays and strings are supported in the site key. Avoid removing e.g.:
+ # -site.key.table.entry.school: 學校；大學
+ # Just because en.yml has site.key.table.entry.school.0 and site.key.table.entry.school.1
+ while (my ($me_k, $me_v) = each %me) {
+ next unless $me_k =~ /^site\.key\.table\.entry/;
+ next if $me_k =~ /\.\d+$/;
+
+ if (ref $en{ $me_k } eq 'ARRAY' and not ref $me{ $me_k }) {
+ $new{ $me_k } = $me{ $me_k };
+ }
+ }
+
+ # There are a bunch of keys on Translatewiki that are
+ # equivalent to English for some reason. Probably because they
+ # were there at import time. Nuke them.
+ while (my ($new_k, $new_v) = each %new) {
+ if (exists $en{ $new_k } and $en{ $new_k } eq $new_v) {
+ #say "Purging dupe in $rails_lang: $new_k=$new_v";
+ delete $new{ $new_k };
}
}
my $expanded = expand_hash( \%new );
my $out = +{ $rails_lang => $expanded };
spit_out($out_file, $out);
- } else {
+ } elsif (not $only_new) {
die "Internal error on $translatewiki_lang";
}
}
my $yaml_out = Dump $data;
open my $fh, ">", $file;
- say $fh "# Imported at " . (scalar localtime) . " from Translatewiki.net";
print $fh $yaml_out;
close $fh;
}
my ($flat_hash) = @_;
my %new_hash;
while (my ($k, $v) = each %$flat_hash) {
+ #say "Inserting $k=$v";
insert_string_deep(\%new_hash, $k, $v);
}
\%new_hash;
}
-sub insert_string_deep {
- my ($h, $ks, $v) = @_;
- my $p = \$h; $p = \$$p->{$_} for split /\./, $ks;
- $$p = $v;
+# Fails under strict in certain cases:
+## Inserting browse.start_rjs.object_list.history.type.way=Vía [[id]]
+## Inserting activerecord.models.relation_tag=Etiqueta de la relación
+## Inserting browse.changeset_details.has_nodes.one=Tiene el siguiente {{count}} nodo:
+## Can't use string ("Tiene {{count}} nodos:") as a HASH ref while "strict refs" in use at script/locale/merge-from-translatewiki line 234.
+# Line 234 = my $p = \$h; $p = \$$p->{$_} for split /\./, $ks;
+
+# sub insert_string_deep_X {
+# my ($h, $ks, $v) = @_;
+# my $p = \$h; $p = \$$p->{$_} for split /\./, $ks;
+# $$p = $v;
+# }
+
+sub insert_string_deep
+{
+ my ($hash, $key, $value) = @_;
+
+ my @key = split /\./, $key;
+ my $h = $hash;
+
+ my $i = 0;
+ for my $k (@key) {
+ $i ++;
+ if ($i == @key) {
+ $h->{$k} = $value;
+ } else {
+ if (ref $h->{$k}) {
+ $h = $h->{$k};
+ } else {
+ $h = $h->{$k} = {};
+ }
+ }
+ }
}
#
my @languages = @_;
my $cache_file = "/tmp/merge-from-translatewiki.storable";
- if (-f $cache_file) {
- my $cache = retrieve($cache_file);
- return %$cache;
+ if ($cache) {
+ if (-f $cache_file) {
+ my $c = retrieve($cache_file);
+ return %$c;
+ }
}
my %translatewiki_languages;
- say "All languages are: @languages";
+ my $all_count = scalar @languages;
+ say "Translatewiki has $all_count languages I'm about to get";
+ my $count = 0;
for my $lang (@languages) {
- say STDERR "Getting language $lang from Translatewiki";
+ $count ++;
+ say STDERR "Getting language $count/$all_count ($lang) from Translatewiki";
my $yaml = get_language_from_translatewiki($lang);
my $flat_data = load_and_flatten_yaml($yaml);
$translatewiki_languages{$lang} = $flat_data;
}
- store \%translatewiki_languages, $cache_file;
+ if ($cache) {
+ store \%translatewiki_languages, $cache_file;
+ }
return %translatewiki_languages;
}
my ($lang) = @_;
my $mech = WWW::Mechanize->new;
- $mech->get("http://translatewiki.net/w/i.php?title=Special%3ATranslate&task=export-to-file&group=out-osm&language=$lang");
+ $mech->get("http://translatewiki.net/w/i.php?title=Special%3ATranslate&task=export-to-file&group=out-osm-site&language=$lang");
die "Couldn't get lang $lang lang from Translatewiki" unless $mech->success;
{
my $mech = WWW::Mechanize->new;
- $mech->get('http://translatewiki.net/wiki/Translating:OpenStreetMap/stats/trunk');
+ $mech->get('http://translatewiki.net/wiki/Translating:OpenStreetMap/stats/trunk/site');
die "Couldn't get translatewiki table" unless $mech->success;
projects / rails.git / blobdiff