feat: update langs and monica:localize command. Add 3 new languages. (#…

…6917)

app/Actions/Jetstream/UserProfile.php

@@ -35,10 +35,10 @@ public function __invoke(Request $request, array $data): array
         $data['userTokens'] = $request->user()->userTokens()->get();
         $data['webauthnKeys'] = $webauthnKeys;
 
-        $data['locales'] = collect(config('lang-detector.languages'))->map(fn (string $locale) => [
+        $data['locales'] = collect(config('localizer.supported_locales'))->map(fn (string $locale) => [
             'id' => $locale,
             'name' => __('auth.lang', [], $locale),
-        ]);
+        ])->sortByCollator('name');
 
         $data['layoutData'] = VaultIndexViewHelper::layoutData();
 

app/Console/Commands/Local/MonicaLocalize.php

@@ -3,9 +3,11 @@
 namespace App\Console\Commands\Local;
 
 use Illuminate\Console\Command;
+use Illuminate\Support\Facades\File;
 use Illuminate\Support\Facades\Storage;
+use Illuminate\Support\Str;
+use Illuminate\Translation\MessageSelector;
 use Stichoza\GoogleTranslate\GoogleTranslate;
-use Symfony\Component\Finder\Finder;
 
 use function Safe\json_decode;
 use function Safe\json_encode;
@@ -19,7 +21,10 @@ class MonicaLocalize extends Command
      *
      * @var string
      */
-    protected $signature = 'monica:localize';
+    protected $signature = 'monica:localize
+                            {--update : Update the current locales.}
+                            {--remove-missing : Remove missing translations.}
+                            {--restart : Restart translation of all messages.}';
 
     /**
      * The console command description.
@@ -33,52 +38,167 @@ class MonicaLocalize extends Command
      */
     public function handle(): void
     {
-        $locales = config('localizer.supported_locales');
-        array_shift($locales);
-        $this->call('localize', ['lang' => implode(',', $locales)]);
+        $this->googleTranslate = (new GoogleTranslate())->setSource('en');
+
+        $locales = $langs = config('localizer.supported_locales');
+
+        $this->updateLocales($locales);
+
+        array_shift($langs);
+        $this->call('localize', [
+            'lang' => implode(',', $langs),
+            '--remove-missing' => $this->option('remove-missing'),
+        ]);
 
         $this->loadTranslations($locales);
+
+        $this->fixPagination($locales);
+    }
+
+    private function updateLocales(array $locales): void
+    {
+        $currentLocales = Storage::disk('lang')->directories();
+
+        if ($this->option('update')) {
+            $this->info('Updating locales...');
+            $this->call('lang:update');
+        }
+
+        $newLocales = collect($locales)->diff($currentLocales);
+
+        foreach ($newLocales as $locale) {
+            try {
+                $this->info('Adding locale: '.$locale);
+                $this->call('lang:add', ['locales' => Str::replace('-', '_', $locale)]);
+            } catch (\LaravelLang\Publisher\Exceptions\UnknownLocaleCodeException) {
+                $this->warn('Locale not recognize: '.$locale);
+            }
+        }
     }
 
     /**
      * Heavily inspired by https://stevensteel.com/blog/automatically-find-translate-and-save-missing-translation-keys.
      */
     private function loadTranslations(array $locales): void
     {
-        $path = lang_path();
-        $finder = new Finder();
-        $finder->in($path)->name(['*.json'])->files();
-        $this->googleTranslate = new GoogleTranslate();
+        foreach ($locales as $locale) {
+            $this->info('Loading locale: '.$locale);
 
-        foreach ($finder as $file) {
-            $locale = $file->getFilenameWithoutExtension();
+            $content = Storage::disk('lang')->get($locale.'.json');
+            $strings = json_decode($content, true);
+            $this->translateStrings($locale, $strings);
+        }
+    }
 
-            if (! in_array($locale, $locales)) {
-                continue;
+    private function translateStrings(string $locale, array $strings)
+    {
+        try {
+            if ($locale !== 'en') {
+                $this->googleTranslate->setTarget($this->getTarget($locale));
+
+                foreach ($strings as $index => $value) {
+                    if ($value === '' || $this->option('restart')) {
+                        // we store the translated string in the array
+                        $strings[$index] = $this->translate($locale, $index);
+                    }
+                }
             }
+        } finally {
 
-            $this->info('loading locale: '.$locale);
-            $jsonString = $file->getContents();
-            $strings = json_decode($jsonString, true);
+            $strings = collect($strings)->map(fn ($value) => Str::replace(['\''], ['’'], $value))->all();
 
-            $this->translateStrings($locale, $strings);
+            // now we need to save the array back to the file
+            ksort($strings, SORT_NATURAL | SORT_FLAG_CASE);
+
+            Storage::disk('lang')->put($locale.'.json', json_encode($strings, JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES));
         }
     }
 
-    private function translateStrings(string $locale, array $strings)
+    private function getTarget($locale)
+    {
+        // Google Translate knows Norwegian locale as 'no' instead of 'nn'
+        return $locale === 'nn' ? 'no' : $locale;
+    }
+
+    private function translate(string $locale, string $text): string
+    {
+        $result = Str::contains($text, '|')
+            ? $this->translateCount($locale, $text)
+            : $this->translateText($text);
+
+        $this->info('translating: `'.$text.'` to `'.$result.'`');
+
+        return $result;
+    }
+
+    private function translateText(string $text): string
+    {
+        $str = Str::of($text);
+        $match = $str->matchAll('/(?<pattern>:\w+)/');
+
+        // replace the placeholders with a generic string
+        if ($match->count() > 0) {
+            $replacements = $match->map(fn ($item, $index) => "{{#$index}}");
+            $str = $str->replace($match->toArray(), $replacements->toArray());
+        }
+
+        $translated = $this->googleTranslate->translate((string) $str);
+
+        // replace the generic string with the placeholders
+        if ($match->count() > 0) {
+            $translated = Str::replace($replacements->toArray(), $match->toArray(), $translated);
+        }
+
+        $translated = Str::replace(['\''], ['’'], $translated);
+
+        return $translated;
+    }
+
+    private function translateCount(string $locale, string $text): string
     {
-        foreach ($strings as $index => $value) {
-            if ($value === '') {
-                $this->googleTranslate->setTarget($locale);
-                $translated = $this->googleTranslate->translate($index);
-                $this->info('translating: `'.$index.'` to `'.$translated.'`');
-
-                // we store the translated string in the array
-                $strings[$index] = $translated;
+        $strings = collect(explode('|', $text));
+        $result = collect([]);
+
+        for ($i = 0; $i < 100; $i++) {
+
+            // Get the plural index for the given locale and count.
+            $j = app(MessageSelector::class)->getPluralIndex($locale, $i);
+
+            if (! $result->has($j)) {
+                // Update the translation for the given plural index.
+                if ($j >= $strings->count()) {
+                    $message = $strings[$strings->count() - 1];
+                } elseif (! $result->has($j)) {
+                    $message = $strings[$j];
+                }
+
+                // Replace the count placeholder with the actual count.
+                $replaced = Str::replace(':count', (string) $i, $message);
+
+                $translated = $this->translateText($replaced);
+
+                // Replace back with the placeholder
+                if (Str::contains($translated, (string) $i)) {
+                    $translated = Str::replace((string) $i, ':count', $translated);
+                } else {
+                    $translated = $this->translateText($message);
+                }
+
+                $result->put($j, $translated);
             }
         }
 
-        // now we need to save the array back to the file
-        Storage::disk('lang')->put($locale.'.json', json_encode($strings, JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE));
+        return $result->sortKeys()->implode('|');
+    }
+
+    private function fixPagination(array $locales): void
+    {
+        foreach ($locales as $locale) {
+            $pagination = Storage::disk('lang')->get($locale.DIRECTORY_SEPARATOR.'pagination.php');
+
+            $pagination = Str::replace(['&laquo; ', ' &raquo;'], ['❮ ', ' ❯'], $pagination);
+
+            Storage::disk('lang')->put($locale.DIRECTORY_SEPARATOR.'pagination.php', $pagination);
+        }
     }
 }

app/Domains/Settings/ManageUserPreferences/Web/ViewHelpers/UserPreferencesIndexViewHelper.php

@@ -196,11 +196,11 @@ public static function dtoLocale(User $user): array
             'name' => self::language($user->locale),
             'dir' => htmldir(),
             'locales' => collect(config('localizer.supported_locales'))
-                ->map(fn ($locale) => [
+                ->map(fn (string $locale) => [
                     'id' => $locale,
                     'name' => self::language($locale),
                 ])
-                ->sortByCollator(fn ($value) => $value['name']),
+                ->sortByCollator('name'),
             'url' => [
                 'store' => route('settings.preferences.locale.store'),
             ],

config/localizer.php

@@ -6,9 +6,9 @@
      * The locales you wish to support.
      * English HAS TO be the first language of the array.
      */
-    'supported_locales' => ['en', 'bn', 'ca', 'da', 'de', 'es', 'el', 'fr',
-        'he', 'hi', 'it', 'ja', 'ml', 'nl', 'no', 'pa', 'pl', 'pt', 'ro',
-        'ru', 'sv', 'te', 'tr', 'ur', 'vi', 'zh',
+    'supported_locales' => ['en', 'ar', 'bn', 'ca', 'da', 'de', 'es', 'el',
+        'fr', 'he', 'hi', 'it', 'ja', 'ml', 'nl', 'nn', 'pa', 'pl', 'pt',
+        'pt_BR', 'ro', 'ru', 'sv', 'te', 'tr', 'ur', 'vi', 'zh_CN', 'zh_TW',
     ],
 
     /**

database/migrations/2023_10_06_064814_rename_locales.php

@@ -0,0 +1,16 @@
+<?php
+
+use App\Models\User;
+use Illuminate\Database\Migrations\Migration;
+
+return new class extends Migration
+{
+    /**
+     * Run the migrations.
+     */
+    public function up(): void
+    {
+        User::where('locale', 'no')->update(['locale' => 'nn']);
+        User::where('locale', 'zh')->update(['locale' => 'zh_CN']);
+    }
+};