Skip to content

Commit

Permalink
Weblate plugin
Browse files Browse the repository at this point in the history
  • Loading branch information
@dragosv
dragosv committed Mar 26, 2020
1 parent ee24a17 commit 6d784d6
Show file tree
Hide file tree
Showing 10 changed files with 383 additions and 0 deletions.
62 changes: 62 additions & 0 deletions docs/pod/weblate.pod
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
=encoding utf8

=head1 NAME

Serge::Sync::Plugin::TranslationService::Weblate - L<Weblate|https://weblate.org/> synchronization plugin for L<Serge|https://serge.io/> based on L<Weblate CLI|https://docs.weblate.org/en/latest/wlc.html#>

=head1 SYNOPSIS

ts
{
plugin weblate

data
{
project WeblateProject
root_directory ./files
config_file ./.weblate
languages de es fr-ca
}
}

=head1 DESCRIPTION

Integration between Serge (Free, Open Source Solution for Continuous Localization) and Weblate (Web-based continuous localisation),
implemented using the command-line tool for the Weblate API.

=head1 ATTRIBUTES

=over

=item I<project>

Weblate project.

=item I<root_directory>

Weblate root directory for files.

=item I<config_file>

Path to the Weblate configuration file.

=item I<languages>

List of languages.

=back

=head1 INSTALLATION

cpanm Serge::Sync::Plugin::TranslationService::weblate

=head1 AUTHOR

Dragos Varovici <dvarovici.work@gmail.com>

=head1 COPYRIGHT AND LICENSE

This software is copyright (c) 2020 by Dragos Varovici.

This is free software; you can redistribute it and/or modify it under
the same terms as the Perl 5 programming language system itself.
253 changes: 253 additions & 0 deletions lib/Serge/Sync/Plugin/TranslationService/weblate.pm
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,253 @@
# ABSTRACT: Weblate (https://weblate.org/) synchronization plugin for Serge

package Serge::Sync::Plugin::TranslationService::weblate;
use parent Serge::Sync::Plugin::Base::TranslationService, Serge::Interface::SysCmdRunner;

use strict;

use File::chdir;
use File::Find qw(find);
use File::Spec::Functions qw(catfile abs2rel);
use JSON -support_by_pp; # -support_by_pp is used to make Perl on Mac happy
use Serge::Util qw(subst_macros);
use version;
use Scalar::Util qw(reftype);
use File::Path qw(make_path);
use File::Basename;

our $VERSION = qv('0.900.0');

sub name {
return 'Weblate translation software (https://weblate.org/) synchronization plugin';
}

sub init {
my $self = shift;

$self->SUPER::init(@_);

$self->{optimizations} = 1; # set to undef to disable optimizations

$self->merge_schema({
root_directory => 'STRING',
project => 'STRING',
config_file => 'STRING',
languages => 'ARRAY'
});
}

sub validate_data {
my ($self) = @_;

$self->SUPER::validate_data;

$self->{data}->{root_directory} = subst_macros($self->{data}->{root_directory});
$self->{data}->{project} = subst_macros($self->{data}->{project});
$self->{data}->{config_file} = subst_macros($self->{data}->{config_file});
$self->{data}->{languages} = subst_macros($self->{data}->{languages});

die "'root_directory', which is set to '$self->{data}->{root_directory}', does not point to a valid folder." unless -d $self->{data}->{root_directory};
die "'config_file' not defined" unless defined $self->{data}->{config_file};
die "'config_file', which is set to '$self->{data}->{config_file}', does not point to a valid file.\n" unless -f $self->{data}->{config_file};
die "'project' not defined" unless defined $self->{data}->{project};

if (!exists $self->{data}->{languages} or scalar(@{$self->{data}->{languages}}) == 0) {
die "the list of destination languages is empty";
}
}

sub pull_ts {
my ($self, $langs) = @_;

my $langs_to_push = $self->get_all_langs($langs);
my %files = $self->translation_files($langs_to_push);

foreach my $key (sort keys %files) {
my $file = $files{$key};
my $full_path = catfile($self->{data}->{root_directory}, $file);

my ($file_name,$folder_path,$file_suffix) = fileparse($full_path);

if (!(-d $folder_path)) {
make_path($folder_path);
}

my $cli_return = $self->run_weblate_cli('download --output "'.$file.'" '.$key, 0);

if ($cli_return != 0) {
return $cli_return;
}
}

return 0;
}

sub push_ts {
my ($self, $langs) = @_;

my $langs_to_push = $self->get_all_langs($langs);
my %files = $self->translation_files($langs_to_push);

foreach my $key (sort keys %files) {
my $file = $files{$key};
my $full_path = catfile($self->{data}->{root_directory}, $file);

if (-f $full_path) {
my $command = 'upload --overwrite --input "' . $file . '"';
$command .= ' --method replace';
if ($self->{data}->{fuzzy}) {
$command .= ' --fuzzy '.$self->{data}->{fuzzy};
}
$command .= ' '.$key;

my $cli_return = $self->run_weblate_cli($command, 0);

if ($cli_return != 0) {
return $cli_return;
}
}
}

return 0;
}

sub translation_files {
my ($self, $langs) = @_;

my $json_components = $self->run_weblate_cli('--format json list-components '.$self->{data}->{project}, 1);

my $json_components_tree = $self->parse_json($json_components);
my $json_components_list = $self->parse_list($json_components_tree);
my @components = map { $_->{slug} } @$json_components_list;

my %translations = ();

foreach my $component (@components) {
my $json_translations = $self->run_weblate_cli('--format json list-translations '.$self->{data}->{project}.'/'.$component, 1);

my $json_translations_tree = $self->parse_json($json_translations);
my $json_translations_list = $self->parse_list($json_translations_tree);

foreach my $translation (@$json_translations_list) {
my $language = $translation->{language}->{code};
my $filename = $translation->{filename};
my $language_code = $translation->{language_code};

if ( $language_code ~~ @$langs ) {
$translations{$self->{data}->{project}.'/'.$component.'/'.$language} = $filename;
}
}
}

return %translations;
}

sub get_all_langs {
my ($self, $langs) = @_;

if (!$langs) {
$langs = $self->{data}->{languages};
}

my @all_langs = ();

push @all_langs, @$langs;

return \@all_langs;
}

sub run_weblate_cli {
my ($self, $action, $capture) = @_;

my $cli_return = 0;

my $command = 'wlc --config '.$self->{data}->{config_file}.' '.$action;

print "Running $action ...\n";

my $json_response = $self->run_in($self->{data}->{root_directory}, $command, 1);

if ($capture) {
return $json_response;
}

if ($json_response == '') {
return 0;
}

return $self->parse_result($json_response);
}

sub parse_result {
my ($self, $json_response) = @_;

my $json_tree = $self->parse_json($json_response);

if (reftype($json_tree) == 'ARRAY') {
return 0;
}

my $result = $json_tree->{result};

if ($result == 'true') {
return 0;
}

return 1;
}

sub parse_list {
my ($self, $json_tree) = @_;

if (reftype($json_tree) == 'ARRAY') {
return $json_tree;
}

return $json_tree->{results};
}

sub find_lang_files {
my ($self, $directory) = @_;

my @files = ();

find(sub {
push @files, abs2rel($File::Find::name, $directory) if(-f $_);
}, $directory);

return @files;
}

sub parse_json {
my ($self, $json) = @_;

my $tree;
eval {
($tree) = from_json($json, {relaxed => 1});
};
if ($@ || !$tree) {
my $error_text = $@;
if ($error_text) {
$error_text =~ s/\t/ /g;
$error_text =~ s/^\s+//s;
} else {
$error_text = "from_json() returned empty data structure";
}

die $error_text;
}

return $tree;
}

sub get_langs {
my ($self, $langs) = @_;

if (!$langs) {
$langs = $self->{data}->{languages};
}

return $langs;
}

1;
18 changes: 18 additions & 0 deletions t/data/plugins/weblate/00/commands/pull.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
[
{
"command" : "wlc --config ./files/.weblate --format json list-components test",
"output" : "[{\"url\":\"http://localhost:8080/api/components/test/main/\",\"web_url\":\"http://localhost:8080/projects/test/main/\",\"name\":\"Main\",\"slug\":\"main\",\"project\":{\"url\":\"http://localhost:8080/api/projects/test/\",\"web_url\":\"http://localhost:8080/projects/test/\",\"name\":\"Test\",\"slug\":\"test\",\"web\":\"https://www.test.com/\",\"source_language\":{\"url\":\"http://localhost:8080/api/languages/en/\",\"web_url\":\"http://localhost:8080/languages/en/\",\"code\":\"en\",\"name\":\"English\",\"direction\":\"ltr\"}},\"vcs\":\"local\",\"repo\":\"local:\",\"git_export\":\"http://localhost:8080/git/test/main/\",\"branch\":\"master\",\"filemask\":\"test/*/testen.xliff\",\"template\":\"\",\"new_base\":\"test/en-us/testen.xliff\",\"file_format\":\"xliff\",\"license\":\"\",\"license_url\":null}]"
},
{
"command" : "wlc --config ./files/.weblate --format json list-translations test/main",
"output" : "[{\"url\":\"http://localhost/api/translations/test/main/en/\",\"web_url\":\"http://localhost/projects/test/main/en/\",\"language\":{\"url\":\"http://localhost/api/languages/en/\",\"web_url\":\"http://localhost/languages/en/\",\"code\":\"en\",\"name\":\"English\",\"direction\":\"ltr\"},\"component\":{\"url\":\"http://localhost/api/components/test/main/\",\"web_url\":\"http://localhost/projects/test/main/\",\"name\":\"Main\",\"slug\":\"main\",\"project\":{\"url\":\"http://localhost/api/projects/test/\",\"web_url\":\"http://localhost/projects/test/\",\"name\":\"Mojito\",\"slug\":\"test\",\"web\":\"https://www.test.global/\",\"source_language\":{\"url\":\"http://localhost/api/languages/en/\",\"web_url\":\"http://localhost/languages/en/\",\"code\":\"en\",\"name\":\"English\",\"direction\":\"ltr\"}},\"vcs\":\"local\",\"repo\":\"local:\",\"git_export\":\"http://localhost/git/test/main/\",\"branch\":\"master\",\"filemask\":\"test/*/testen.xliff\",\"template\":\"\",\"new_base\":\"test/en-us/testen.xliff\",\"file_format\":\"xliff\",\"license\":\"\",\"license_url\":null},\"translated\":141,\"fuzzy\":0,\"total\":141,\"translated_words\":454,\"fuzzy_words\":0,\"failing_checks_words\":425,\"total_words\":454,\"failing_checks\":117,\"have_suggestion\":0,\"have_comment\":0,\"language_code\":\"en\",\"filename\":\"\",\"revision\":\"\",\"share_url\":\"http://localhost/engage/test/en/\",\"translate_url\":\"http://localhost/translate/test/main/en/\",\"is_template\":true,\"translated_percent\":100.0,\"fuzzy_percent\":0.0,\"failing_checks_percent\":82.9,\"last_change\":null,\"last_author\":null},{\"url\":\"http://localhost/api/translations/test/main/de/\",\"web_url\":\"http://localhost/projects/test/main/de/\",\"language\":{\"url\":\"http://localhost/api/languages/de/\",\"web_url\":\"http://localhost/languages/de/\",\"code\":\"de\",\"name\":\"German\",\"direction\":\"ltr\"},\"component\":{\"url\":\"http://localhost/api/components/test/main/\",\"web_url\":\"http://localhost/projects/test/main/\",\"name\":\"Main\",\"slug\":\"main\",\"project\":{\"url\":\"http://localhost/api/projects/test/\",\"web_url\":\"http://localhost/projects/test/\",\"name\":\"Mojito\",\"slug\":\"test\",\"web\":\"https://www.test.global/\",\"source_language\":{\"url\":\"http://localhost/api/languages/en/\",\"web_url\":\"http://localhost/languages/en/\",\"code\":\"en\",\"name\":\"English\",\"direction\":\"ltr\"}},\"vcs\":\"local\",\"repo\":\"local:\",\"git_export\":\"http://localhost/git/test/main/\",\"branch\":\"master\",\"filemask\":\"test/*/testen.xliff\",\"template\":\"\",\"new_base\":\"test/en-us/testen.xliff\",\"file_format\":\"xliff\",\"license\":\"\",\"license_url\":null},\"translated\":0,\"fuzzy\":141,\"total\":141,\"translated_words\":0,\"fuzzy_words\":454,\"failing_checks_words\":454,\"total_words\":454,\"failing_checks\":141,\"have_suggestion\":0,\"have_comment\":0,\"language_code\":\"de\",\"filename\":\"test/de/testen.xliff\",\"revision\":\"0f135d859a814c5ba230dbc03df8e31c000fd498\",\"share_url\":\"http://localhost/engage/test/de/\",\"translate_url\":\"http://localhost/translate/test/main/de/\",\"is_template\":false,\"translated_percent\":0.0,\"fuzzy_percent\":100.0,\"failing_checks_percent\":100.0,\"last_change\":\"2020-03-25T18:24:13.835815+00:00\",\"last_author\":\"UserNameLastName\"},{\"url\":\"http://localhost/api/translations/test/main/es/\",\"web_url\":\"http://localhost/projects/test/main/es/\",\"language\":{\"url\":\"http://localhost/api/languages/es/\",\"web_url\":\"http://localhost/languages/es/\",\"code\":\"es\",\"name\":\"Spanish\",\"direction\":\"ltr\"},\"component\":{\"url\":\"http://localhost/api/components/test/main/\",\"web_url\":\"http://localhost/projects/test/main/\",\"name\":\"Main\",\"slug\":\"main\",\"project\":{\"url\":\"http://localhost/api/projects/test/\",\"web_url\":\"http://localhost/projects/test/\",\"name\":\"Mojito\",\"slug\":\"test\",\"web\":\"https://www.test.global/\",\"source_language\":{\"url\":\"http://localhost/api/languages/en/\",\"web_url\":\"http://localhost/languages/en/\",\"code\":\"en\",\"name\":\"English\",\"direction\":\"ltr\"}},\"vcs\":\"local\",\"repo\":\"local:\",\"git_export\":\"http://localhost/git/test/main/\",\"branch\":\"master\",\"filemask\":\"test/*/testen.xliff\",\"template\":\"\",\"new_base\":\"test/en-us/testen.xliff\",\"file_format\":\"xliff\",\"license\":\"\",\"license_url\":null},\"translated\":0,\"fuzzy\":141,\"total\":141,\"translated_words\":0,\"fuzzy_words\":454,\"failing_checks_words\":454,\"total_words\":454,\"failing_checks\":141,\"have_suggestion\":0,\"have_comment\":0,\"language_code\":\"es\",\"filename\":\"test/es/testen.xliff\",\"revision\":\"8e0c83738db390ec2c0ee36ce1aa01880c9b3224\",\"share_url\":\"http://localhost/engage/test/es/\",\"translate_url\":\"http://localhost/translate/test/main/es/\",\"is_template\":false,\"translated_percent\":0.0,\"fuzzy_percent\":100.0,\"failing_checks_percent\":100.0,\"last_change\":\"2020-03-25T18:23:49.981599+00:00\",\"last_author\":\"UserNameLastName\"},{\"url\":\"http://localhost/api/translations/test/main/fr/\",\"web_url\":\"http://localhost/projects/test/main/fr/\",\"language\":{\"url\":\"http://localhost/api/languages/fr/\",\"web_url\":\"http://localhost/languages/fr/\",\"code\":\"fr\",\"name\":\"French\",\"direction\":\"ltr\"},\"component\":{\"url\":\"http://localhost/api/components/test/main/\",\"web_url\":\"http://localhost/projects/test/main/\",\"name\":\"Main\",\"slug\":\"main\",\"project\":{\"url\":\"http://localhost/api/projects/test/\",\"web_url\":\"http://localhost/projects/test/\",\"name\":\"Mojito\",\"slug\":\"test\",\"web\":\"https://www.test.global/\",\"source_language\":{\"url\":\"http://localhost/api/languages/en/\",\"web_url\":\"http://localhost/languages/en/\",\"code\":\"en\",\"name\":\"English\",\"direction\":\"ltr\"}},\"vcs\":\"local\",\"repo\":\"local:\",\"git_export\":\"http://localhost/git/test/main/\",\"branch\":\"master\",\"filemask\":\"test/*/testen.xliff\",\"template\":\"\",\"new_base\":\"test/en-us/testen.xliff\",\"file_format\":\"xliff\",\"license\":\"\",\"license_url\":null},\"translated\":0,\"fuzzy\":141,\"total\":141,\"translated_words\":0,\"fuzzy_words\":454,\"failing_checks_words\":454,\"total_words\":454,\"failing_checks\":141,\"have_suggestion\":0,\"have_comment\":0,\"language_code\":\"fr-fr\",\"filename\":\"test/fr-fr/testen.xliff\",\"revision\":\"16b7f2e5b1e30d685d5705dba2bfcb6d9336106e\",\"share_url\":\"http://localhost/engage/test/fr/\",\"translate_url\":\"http://localhost/translate/test/main/fr/\",\"is_template\":false,\"translated_percent\":0.0,\"fuzzy_percent\":100.0,\"failing_checks_percent\":100.0,\"last_change\":\"2020-03-25T18:24:02.855819+00:00\",\"last_author\":\"UserNameLastName\"},{\"url\":\"http://localhost/api/translations/test/main/it/\",\"web_url\":\"http://localhost/projects/test/main/it/\",\"language\":{\"url\":\"http://localhost/api/languages/it/\",\"web_url\":\"http://localhost/languages/it/\",\"code\":\"it\",\"name\":\"Italian\",\"direction\":\"ltr\"},\"component\":{\"url\":\"http://localhost/api/components/test/main/\",\"web_url\":\"http://localhost/projects/test/main/\",\"name\":\"Main\",\"slug\":\"main\",\"project\":{\"url\":\"http://localhost/api/projects/test/\",\"web_url\":\"http://localhost/projects/test/\",\"name\":\"Mojito\",\"slug\":\"test\",\"web\":\"https://www.test.global/\",\"source_language\":{\"url\":\"http://localhost/api/languages/en/\",\"web_url\":\"http://localhost/languages/en/\",\"code\":\"en\",\"name\":\"English\",\"direction\":\"ltr\"}},\"vcs\":\"local\",\"repo\":\"local:\",\"git_export\":\"http://localhost/git/test/main/\",\"branch\":\"master\",\"filemask\":\"test/*/testen.xliff\",\"template\":\"\",\"new_base\":\"test/en-us/testen.xliff\",\"file_format\":\"xliff\",\"license\":\"\",\"license_url\":null},\"translated\":141,\"fuzzy\":0,\"total\":141,\"translated_words\":454,\"fuzzy_words\":0,\"failing_checks_words\":425,\"total_words\":454,\"failing_checks\":117,\"have_suggestion\":0,\"have_comment\":0,\"language_code\":\"it\",\"filename\":\"test/it/testen.xliff\",\"revision\":\"3e5620e454bd38cce19c8601cc44aa89af981948\",\"share_url\":\"http://localhost/engage/test/it/\",\"translate_url\":\"http://localhost/translate/test/main/it/\",\"is_template\":false,\"translated_percent\":100.0,\"fuzzy_percent\":0.0,\"failing_checks_percent\":82.9,\"last_change\":null,\"last_author\":null},{\"url\":\"http://localhost/api/translations/test/main/nl/\",\"web_url\":\"http://localhost/projects/test/main/nl/\",\"language\":{\"url\":\"http://localhost/api/languages/nl/\",\"web_url\":\"http://localhost/languages/nl/\",\"code\":\"nl\",\"name\":\"Dutch\",\"direction\":\"ltr\"},\"component\":{\"url\":\"http://localhost/api/components/test/main/\",\"web_url\":\"http://localhost/projects/test/main/\",\"name\":\"Main\",\"slug\":\"main\",\"project\":{\"url\":\"http://localhost/api/projects/test/\",\"web_url\":\"http://localhost/projects/test/\",\"name\":\"Mojito\",\"slug\":\"test\",\"web\":\"https://www.test.global/\",\"source_language\":{\"url\":\"http://localhost/api/languages/en/\",\"web_url\":\"http://localhost/languages/en/\",\"code\":\"en\",\"name\":\"English\",\"direction\":\"ltr\"}},\"vcs\":\"local\",\"repo\":\"local:\",\"git_export\":\"http://localhost/git/test/main/\",\"branch\":\"master\",\"filemask\":\"test/*/testen.xliff\",\"template\":\"\",\"new_base\":\"test/en-us/testen.xliff\",\"file_format\":\"xliff\",\"license\":\"\",\"license_url\":null},\"translated\":141,\"fuzzy\":0,\"total\":141,\"translated_words\":454,\"fuzzy_words\":0,\"failing_checks_words\":425,\"total_words\":454,\"failing_checks\":117,\"have_suggestion\":0,\"have_comment\":0,\"language_code\":\"nl\",\"filename\":\"test/nl/testen.xliff\",\"revision\":\"d407d9a6ee7658cb4a3cbd6fb2c228eaed29d36f\",\"share_url\":\"http://localhost/engage/test/nl/\",\"translate_url\":\"http://localhost/translate/test/main/nl/\",\"is_template\":false,\"translated_percent\":100.0,\"fuzzy_percent\":0.0,\"failing_checks_percent\":82.9,\"last_change\":null,\"last_author\":null}]"
},
{
"command" : "wlc --config ./files/.weblate download --output \"test/es/testen.xliff\" test/main/es",
"output" : "{ \"result\" : \"true\" }"
},
{
"command" : "wlc --config ./files/.weblate download --output \"test/fr-fr/testen.xliff\" test/main/fr",
"output" : "{ \"result\" : \"true\" }"
}
]
Loading

0 comments on commit 6d784d6

Please sign in to comment.