From cbc87b5dab8daf0cd2217b2a2525dfd12dad7272 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Hoste?= Date: Tue, 7 Sep 2021 20:01:42 +0200 Subject: [PATCH] feat: Cloud service providers & Translation.io (#1107) Now with Lingui, you could integrate your favorite SaSS for syncing your translations using an internal process. Translation.io already introduced themselves and now you can sync Translation.io translations in your React codebases without anything else than `lingui extract` --- packages/cli/package.json | 1 + packages/cli/src/lingui-extract.ts | 7 + packages/cli/src/services/translationIO.ts | 293 ++++++++++++++++++ packages/conf/index.d.ts | 11 +- .../conf/src/__snapshots__/index.test.ts.snap | 4 + packages/conf/src/index.ts | 7 + 6 files changed, 321 insertions(+), 2 deletions(-) create mode 100644 packages/cli/src/services/translationIO.ts diff --git a/packages/cli/package.json b/packages/cli/package.json index 6a4f48929..ebd610cc7 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -29,6 +29,7 @@ "LICENSE", "README.md", "api", + "services", "lingui.js", "lingui-*.js" ], diff --git a/packages/cli/src/lingui-extract.ts b/packages/cli/src/lingui-extract.ts index 7eb4ca9b4..3acc97f2d 100644 --- a/packages/cli/src/lingui-extract.ts +++ b/packages/cli/src/lingui-extract.ts @@ -68,6 +68,13 @@ export default function command( ) } + // If service key is present in configuration, synchronize with cloud translation platform + if (typeof config.service === 'object' && config.service.name && config.service.name.length) { + import(`./services/${config.service.name}`) + .then(module => module.default(config, options)) + .catch(err => console.error(`Can't load service module ${config.service.name}`, err)) + } + return true } diff --git a/packages/cli/src/services/translationIO.ts b/packages/cli/src/services/translationIO.ts new file mode 100644 index 000000000..07c391c37 --- /dev/null +++ b/packages/cli/src/services/translationIO.ts @@ -0,0 +1,293 @@ +import fs from "fs" +import { dirname } from "path" +import PO from "pofile" +import https from "https" +import glob from "glob" +import { format as formatDate } from "date-fns" + +const getCreateHeaders = (language) => ({ + "POT-Creation-Date": formatDate(new Date(), "yyyy-MM-dd HH:mmxxxx"), + "MIME-Version": "1.0", + "Content-Type": "text/plain; charset=utf-8", + "Content-Transfer-Encoding": "8bit", + "X-Generator": "@lingui/cli", + Language: language, +}) + +// Main sync method, call "Init" or "Sync" depending on the project context +export default function syncProcess(config, options) { + if (config.format != 'po') { + console.error(`\n----------\nTranslation.io service is only compatible with the "po" format. Please update your Lingui configuration accordingly.\n----------`) + process.exit(1) + } + + const successCallback = (project) => { + console.log(`\n----------\nProject successfully synchronized. Please use this URL to translate: ${project.url}\n----------`) + } + + const failCallback = (errors) => { + console.error(`\n----------\nSynchronization with Translation.io failed: ${errors.join(', ')}\n----------`) + } + + init(config, options, successCallback, (errors) => { + if (errors.length && errors[0] === 'This project has already been initialized.') { + sync(config, options, successCallback, failCallback) + } + else { + failCallback(errors) + } + }) +} + +// Initialize project with source and existing translations (only first time!) +// Cf. https://translation.io/docs/create-library#initialization +function init(config, options, successCallback, failCallback) { + const sourceLocale = config.sourceLocale || 'en' + const targetLocales = config.locales.filter((value) => value != sourceLocale) + const paths = poPathsPerLocale(config) + + let segments = {} + + targetLocales.forEach((targetLocale) => { + segments[targetLocale] = [] + }) + + // Create segments from source locale PO items + paths[sourceLocale].forEach((path) => { + let raw = fs.readFileSync(path).toString() + let po = PO.parse(raw) + + po.items.filter((item) => !item['obsolete']).forEach((item) => { + targetLocales.forEach((targetLocale) => { + let newSegment = createSegmentFromPoItem(item) + + segments[targetLocale].push(newSegment) + }) + }) + }) + + // Add translations to segments from target locale PO items + targetLocales.forEach((targetLocale) => { + paths[targetLocale].forEach((path) => { + let raw = fs.readFileSync(path).toString() + let po = PO.parse(raw) + + po.items.filter((item) => !item['obsolete']).forEach((item, index) => { + segments[targetLocale][index].target = item.msgstr[0] + }) + }) + }) + + let request = { + "client": "lingui", + "version": require('@lingui/core/package.json').version, + "source_language": sourceLocale, + "target_languages": targetLocales, + "segments": segments + } + + postTio("init", request, config.service.apiKey, (response) => { + if (response.errors) { + failCallback(response.errors) + } + else { + saveSegmentsToTargetPos(config, paths, response.segments) + successCallback(response.project) + } + }, (error) => { + console.error(`\n----------\nSynchronization with Translation.io failed: ${error}\n----------`) + }) +} + +// Send all source text from PO to Translation.io and create new PO based on received translations +// Cf. https://translation.io/docs/create-library#synchronization +function sync(config, options, successCallback, failCallback) { + const sourceLocale = config.sourceLocale || 'en' + const targetLocales = config.locales.filter((value) => value != sourceLocale) + const paths = poPathsPerLocale(config) + + let segments = [] + + // Create segments with correct source + paths[sourceLocale].forEach((path) => { + let raw = fs.readFileSync(path).toString() + let po = PO.parse(raw) + + po.items.filter((item) => !item['obsolete']).forEach((item) => { + let newSegment = createSegmentFromPoItem(item) + + segments.push(newSegment) + }) + }) + + let request = { + "client": "lingui", + "version": require('@lingui/core/package.json').version, + "source_language": sourceLocale, + "target_languages": targetLocales, + "segments": segments + } + + // Sync and then remove unused segments (not present in the local application) from Translation.io + if (options.clean) { + request['purge'] = true + } + + postTio("sync", request, config.service.apiKey, (response) => { + if (response.errors) { + failCallback(response.errors) + } + else { + saveSegmentsToTargetPos(config, paths, response.segments) + successCallback(response.project) + } + }, (error) => { + console.error(`\n----------\nSynchronization with Translation.io failed: ${error}\n----------`) + }) +} + +function createSegmentFromPoItem(item) { + let itemHasId = item.msgid != item.msgstr[0] && item.msgstr[0].length + + let segment = { + type: 'source', // No way to edit text for source language (inside code), so not using "key" here + source: itemHasId ? item.msgstr[0] : item.msgid, // msgstr may be empty if --overwrite is used and no ID is used + context: '', + references: [], + comment: '' + } + + if (itemHasId) { + segment.context = item.msgid + } + + if (item.references.length) { + segment.references = item.references + } + + if (item.extractedComments.length) { + segment.comment = item.extractedComments.join(' | ') + } + + return segment +} + +function createPoItemFromSegment(segment) { + let item = new PO.Item() + + item.msgid = segment.context ? segment.context : segment.source + item.msgstr = [segment.target] + item.references = (segment.references && segment.references.length) ? segment.references : [] + item.extractedComments = segment.comment ? segment.comment.split(' | ') : [] + + return item +} + +function saveSegmentsToTargetPos(config, paths, segmentsPerLocale) { + const NAME = "{name}" + const LOCALE = "{locale}" + + Object.keys(segmentsPerLocale).forEach((targetLocale) => { + // Remove existing target POs and JS for this target locale + paths[targetLocale].forEach((path) => { + const jsPath = path.replace(/\.po?$/, "") + ".js" + const dirPath = dirname(path) + + // Remove PO, JS and empty dir + if (fs.existsSync(path)) { fs.unlinkSync(path) } + if (fs.existsSync(jsPath)) { fs.unlinkSync(jsPath) } + if (fs.existsSync(dirPath) && fs.readdirSync(dirPath).length === 0) { fs.rmdirSync(dirPath) } + }) + + // Find target path (ignoring {name}) + const localePath = "".concat(config.catalogs[0].path.replace(LOCALE, targetLocale).replace(NAME, ''), ".po") + const segments = segmentsPerLocale[targetLocale] + + let po = new PO() + po.headers = getCreateHeaders(targetLocale) + + let items = [] + + segments.forEach((segment) => { + let item = createPoItemFromSegment(segment) + items.push(item) + }) + + // Sort items by messageId + po.items = items.sort((a, b) => { + if (a.msgid < b.msgid) { return -1 } + if (a.msgid > b.msgid) { return 1 } + return 0 + }) + + // Check that localePath directory exists and save PO file + fs.promises.mkdir(dirname(localePath), {recursive: true}).then(() => { + po.save(localePath, (err) => { + if (err) { + console.error('Error while saving target PO files:') + console.error(err) + process.exit(1) + } + }) + }) + }) +} + +function poPathsPerLocale(config) { + const NAME = "{name}" + const LOCALE = "{locale}" + const paths = [] + + config.locales.forEach((locale) => { + paths[locale] = [] + + config.catalogs.forEach((catalog) => { + const path = "".concat(catalog.path.replace(LOCALE, locale).replace(NAME, "*"), ".po") + + // If {name} is present (replaced by *), list all the existing POs + if (path.includes('*')) { + paths[locale] = paths[locale].concat(glob.sync(path)) + } + else { + paths[locale].push(path) + } + }) + }) + + return paths +} + +function postTio(action, request, apiKey, successCallback, failCallback) { + let jsonRequest = JSON.stringify(request) + + let options = { + hostname: 'translation.io', + path: '/api/v1/segments/' + action + '.json?api_key=' + apiKey, + method: 'POST', + headers: { + 'Content-Type': 'application/json', + } + } + + let req = https.request(options, (res) => { + res.setEncoding('utf8') + + let body = "" + + res.on('data', (chunk) => { + body = body.concat(chunk) + }) + + res.on('end', () => { + let response = JSON.parse(body) + successCallback(response) + }) + }) + + req.on('error', (e) => { + failCallback(e) + }) + + req.write(jsonRequest) + req.end() +} diff --git a/packages/conf/index.d.ts b/packages/conf/index.d.ts index fe47871e6..227fb8dbb 100644 --- a/packages/conf/index.d.ts +++ b/packages/conf/index.d.ts @@ -1,6 +1,6 @@ import type { GeneratorOptions } from "@babel/core"; -export declare type CatalogFormat = "lingui" | "minimal" | "po" | "csv" | "po-gettext"; +export declare type CatalogFormat = "lingui" | "minimal" | "po" | "csv" | "po-gettext"; export type CatalogFormatOptions = { origins?: boolean; lineNumbers?: boolean; @@ -23,6 +23,11 @@ export type DefaultLocaleObject = { export declare type FallbackLocales = LocaleObject | DefaultLocaleObject +declare type CatalogService = { + name: string; + apiKey: string; +} + declare type ExtractorType = { match(filename: string): boolean; extract(filename: string, targetDir: string, options?: any): void; @@ -46,6 +51,7 @@ export declare type LinguiConfig = { rootDir: string; runtimeConfigModule: [string, string?]; sourceLocale: string; + service: CatalogService; }; export declare const defaultConfig: LinguiConfig; export declare function getConfig({ cwd, configPath, skipValidation, }?: { @@ -63,7 +69,7 @@ export declare const configValidation: { }; compilerBabelOptions: GeneratorOptions; catalogs: CatalogConfig[]; - compileNamespace: "es" | "ts" | "cjs" | string; + compileNamespace: "es" | "ts" | "cjs" | string; fallbackLocales: FallbackLocales; format: CatalogFormat; formatOptions: CatalogFormatOptions; @@ -73,6 +79,7 @@ export declare const configValidation: { rootDir: string; runtimeConfigModule: [string, string?]; sourceLocale: string; + service: CatalogService; }; deprecatedConfig: { fallbackLocale: (config: LinguiConfig & DeprecatedFallbackLanguage) => string; diff --git a/packages/conf/src/__snapshots__/index.test.ts.snap b/packages/conf/src/__snapshots__/index.test.ts.snap index 27bdca3f4..08bba6b97 100644 --- a/packages/conf/src/__snapshots__/index.test.ts.snap +++ b/packages/conf/src/__snapshots__/index.test.ts.snap @@ -150,6 +150,10 @@ Object { @lingui/core, i18n, ], + service: Object { + apiKey: , + name: , + }, sourceLocale: , } `; diff --git a/packages/conf/src/index.ts b/packages/conf/src/index.ts index 4d8277986..a8c895863 100644 --- a/packages/conf/src/index.ts +++ b/packages/conf/src/index.ts @@ -32,6 +32,11 @@ export type FallbackLocales = LocaleObject | DefaultLocaleObject | false type ModuleSource = [string, string?] +type CatalogService = { + name: string + apiKey: string +} + type ExtractorType = { match(filename: string): boolean extract(filename: string, targetDir: string, options?: any): void @@ -53,6 +58,7 @@ export type LinguiConfig = { rootDir: string runtimeConfigModule: ModuleSource | { [symbolName: string]: ModuleSource } sourceLocale: string + service: CatalogService } // Enforce posix path delimiters internally @@ -98,6 +104,7 @@ export const defaultConfig: LinguiConfig = { rootDir: ".", runtimeConfigModule: ["@lingui/core", "i18n"], sourceLocale: "", + service: { name: "", apiKey: "" } } function configExists(path) {