diff --git a/packages/cli/package.json b/packages/cli/package.json index 6a4f48929..ebd610cc7 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -29,6 +29,7 @@ "LICENSE", "README.md", "api", + "services", "lingui.js", "lingui-*.js" ], diff --git a/packages/cli/src/lingui-extract.ts b/packages/cli/src/lingui-extract.ts index 7eb4ca9b4..3acc97f2d 100644 --- a/packages/cli/src/lingui-extract.ts +++ b/packages/cli/src/lingui-extract.ts @@ -68,6 +68,13 @@ export default function command( ) } + // If service key is present in configuration, synchronize with cloud translation platform + if (typeof config.service === 'object' && config.service.name && config.service.name.length) { + import(`./services/${config.service.name}`) + .then(module => module.default(config, options)) + .catch(err => console.error(`Can't load service module ${config.service.name}`, err)) + } + return true } diff --git a/packages/cli/src/services/translationIO.ts b/packages/cli/src/services/translationIO.ts new file mode 100644 index 000000000..07c391c37 --- /dev/null +++ b/packages/cli/src/services/translationIO.ts @@ -0,0 +1,293 @@ +import fs from "fs" +import { dirname } from "path" +import PO from "pofile" +import https from "https" +import glob from "glob" +import { format as formatDate } from "date-fns" + +const getCreateHeaders = (language) => ({ + "POT-Creation-Date": formatDate(new Date(), "yyyy-MM-dd HH:mmxxxx"), + "MIME-Version": "1.0", + "Content-Type": "text/plain; charset=utf-8", + "Content-Transfer-Encoding": "8bit", + "X-Generator": "@lingui/cli", + Language: language, +}) + +// Main sync method, call "Init" or "Sync" depending on the project context +export default function syncProcess(config, options) { + if (config.format != 'po') { + console.error(`\n----------\nTranslation.io service is only compatible with the "po" format. Please update your Lingui configuration accordingly.\n----------`) + process.exit(1) + } + + const successCallback = (project) => { + console.log(`\n----------\nProject successfully synchronized. Please use this URL to translate: ${project.url}\n----------`) + } + + const failCallback = (errors) => { + console.error(`\n----------\nSynchronization with Translation.io failed: ${errors.join(', ')}\n----------`) + } + + init(config, options, successCallback, (errors) => { + if (errors.length && errors[0] === 'This project has already been initialized.') { + sync(config, options, successCallback, failCallback) + } + else { + failCallback(errors) + } + }) +} + +// Initialize project with source and existing translations (only first time!) +// Cf. https://translation.io/docs/create-library#initialization +function init(config, options, successCallback, failCallback) { + const sourceLocale = config.sourceLocale || 'en' + const targetLocales = config.locales.filter((value) => value != sourceLocale) + const paths = poPathsPerLocale(config) + + let segments = {} + + targetLocales.forEach((targetLocale) => { + segments[targetLocale] = [] + }) + + // Create segments from source locale PO items + paths[sourceLocale].forEach((path) => { + let raw = fs.readFileSync(path).toString() + let po = PO.parse(raw) + + po.items.filter((item) => !item['obsolete']).forEach((item) => { + targetLocales.forEach((targetLocale) => { + let newSegment = createSegmentFromPoItem(item) + + segments[targetLocale].push(newSegment) + }) + }) + }) + + // Add translations to segments from target locale PO items + targetLocales.forEach((targetLocale) => { + paths[targetLocale].forEach((path) => { + let raw = fs.readFileSync(path).toString() + let po = PO.parse(raw) + + po.items.filter((item) => !item['obsolete']).forEach((item, index) => { + segments[targetLocale][index].target = item.msgstr[0] + }) + }) + }) + + let request = { + "client": "lingui", + "version": require('@lingui/core/package.json').version, + "source_language": sourceLocale, + "target_languages": targetLocales, + "segments": segments + } + + postTio("init", request, config.service.apiKey, (response) => { + if (response.errors) { + failCallback(response.errors) + } + else { + saveSegmentsToTargetPos(config, paths, response.segments) + successCallback(response.project) + } + }, (error) => { + console.error(`\n----------\nSynchronization with Translation.io failed: ${error}\n----------`) + }) +} + +// Send all source text from PO to Translation.io and create new PO based on received translations +// Cf. https://translation.io/docs/create-library#synchronization +function sync(config, options, successCallback, failCallback) { + const sourceLocale = config.sourceLocale || 'en' + const targetLocales = config.locales.filter((value) => value != sourceLocale) + const paths = poPathsPerLocale(config) + + let segments = [] + + // Create segments with correct source + paths[sourceLocale].forEach((path) => { + let raw = fs.readFileSync(path).toString() + let po = PO.parse(raw) + + po.items.filter((item) => !item['obsolete']).forEach((item) => { + let newSegment = createSegmentFromPoItem(item) + + segments.push(newSegment) + }) + }) + + let request = { + "client": "lingui", + "version": require('@lingui/core/package.json').version, + "source_language": sourceLocale, + "target_languages": targetLocales, + "segments": segments + } + + // Sync and then remove unused segments (not present in the local application) from Translation.io + if (options.clean) { + request['purge'] = true + } + + postTio("sync", request, config.service.apiKey, (response) => { + if (response.errors) { + failCallback(response.errors) + } + else { + saveSegmentsToTargetPos(config, paths, response.segments) + successCallback(response.project) + } + }, (error) => { + console.error(`\n----------\nSynchronization with Translation.io failed: ${error}\n----------`) + }) +} + +function createSegmentFromPoItem(item) { + let itemHasId = item.msgid != item.msgstr[0] && item.msgstr[0].length + + let segment = { + type: 'source', // No way to edit text for source language (inside code), so not using "key" here + source: itemHasId ? item.msgstr[0] : item.msgid, // msgstr may be empty if --overwrite is used and no ID is used + context: '', + references: [], + comment: '' + } + + if (itemHasId) { + segment.context = item.msgid + } + + if (item.references.length) { + segment.references = item.references + } + + if (item.extractedComments.length) { + segment.comment = item.extractedComments.join(' | ') + } + + return segment +} + +function createPoItemFromSegment(segment) { + let item = new PO.Item() + + item.msgid = segment.context ? segment.context : segment.source + item.msgstr = [segment.target] + item.references = (segment.references && segment.references.length) ? segment.references : [] + item.extractedComments = segment.comment ? segment.comment.split(' | ') : [] + + return item +} + +function saveSegmentsToTargetPos(config, paths, segmentsPerLocale) { + const NAME = "{name}" + const LOCALE = "{locale}" + + Object.keys(segmentsPerLocale).forEach((targetLocale) => { + // Remove existing target POs and JS for this target locale + paths[targetLocale].forEach((path) => { + const jsPath = path.replace(/\.po?$/, "") + ".js" + const dirPath = dirname(path) + + // Remove PO, JS and empty dir + if (fs.existsSync(path)) { fs.unlinkSync(path) } + if (fs.existsSync(jsPath)) { fs.unlinkSync(jsPath) } + if (fs.existsSync(dirPath) && fs.readdirSync(dirPath).length === 0) { fs.rmdirSync(dirPath) } + }) + + // Find target path (ignoring {name}) + const localePath = "".concat(config.catalogs[0].path.replace(LOCALE, targetLocale).replace(NAME, ''), ".po") + const segments = segmentsPerLocale[targetLocale] + + let po = new PO() + po.headers = getCreateHeaders(targetLocale) + + let items = [] + + segments.forEach((segment) => { + let item = createPoItemFromSegment(segment) + items.push(item) + }) + + // Sort items by messageId + po.items = items.sort((a, b) => { + if (a.msgid < b.msgid) { return -1 } + if (a.msgid > b.msgid) { return 1 } + return 0 + }) + + // Check that localePath directory exists and save PO file + fs.promises.mkdir(dirname(localePath), {recursive: true}).then(() => { + po.save(localePath, (err) => { + if (err) { + console.error('Error while saving target PO files:') + console.error(err) + process.exit(1) + } + }) + }) + }) +} + +function poPathsPerLocale(config) { + const NAME = "{name}" + const LOCALE = "{locale}" + const paths = [] + + config.locales.forEach((locale) => { + paths[locale] = [] + + config.catalogs.forEach((catalog) => { + const path = "".concat(catalog.path.replace(LOCALE, locale).replace(NAME, "*"), ".po") + + // If {name} is present (replaced by *), list all the existing POs + if (path.includes('*')) { + paths[locale] = paths[locale].concat(glob.sync(path)) + } + else { + paths[locale].push(path) + } + }) + }) + + return paths +} + +function postTio(action, request, apiKey, successCallback, failCallback) { + let jsonRequest = JSON.stringify(request) + + let options = { + hostname: 'translation.io', + path: '/api/v1/segments/' + action + '.json?api_key=' + apiKey, + method: 'POST', + headers: { + 'Content-Type': 'application/json', + } + } + + let req = https.request(options, (res) => { + res.setEncoding('utf8') + + let body = "" + + res.on('data', (chunk) => { + body = body.concat(chunk) + }) + + res.on('end', () => { + let response = JSON.parse(body) + successCallback(response) + }) + }) + + req.on('error', (e) => { + failCallback(e) + }) + + req.write(jsonRequest) + req.end() +} diff --git a/packages/conf/index.d.ts b/packages/conf/index.d.ts index fe47871e6..227fb8dbb 100644 --- a/packages/conf/index.d.ts +++ b/packages/conf/index.d.ts @@ -1,6 +1,6 @@ import type { GeneratorOptions } from "@babel/core"; -export declare type CatalogFormat = "lingui" | "minimal" | "po" | "csv" | "po-gettext"; +export declare type CatalogFormat = "lingui" | "minimal" | "po" | "csv" | "po-gettext"; export type CatalogFormatOptions = { origins?: boolean; lineNumbers?: boolean; @@ -23,6 +23,11 @@ export type DefaultLocaleObject = { export declare type FallbackLocales = LocaleObject | DefaultLocaleObject +declare type CatalogService = { + name: string; + apiKey: string; +} + declare type ExtractorType = { match(filename: string): boolean; extract(filename: string, targetDir: string, options?: any): void; @@ -46,6 +51,7 @@ export declare type LinguiConfig = { rootDir: string; runtimeConfigModule: [string, string?]; sourceLocale: string; + service: CatalogService; }; export declare const defaultConfig: LinguiConfig; export declare function getConfig({ cwd, configPath, skipValidation, }?: { @@ -63,7 +69,7 @@ export declare const configValidation: { }; compilerBabelOptions: GeneratorOptions; catalogs: CatalogConfig[]; - compileNamespace: "es" | "ts" | "cjs" | string; + compileNamespace: "es" | "ts" | "cjs" | string; fallbackLocales: FallbackLocales; format: CatalogFormat; formatOptions: CatalogFormatOptions; @@ -73,6 +79,7 @@ export declare const configValidation: { rootDir: string; runtimeConfigModule: [string, string?]; sourceLocale: string; + service: CatalogService; }; deprecatedConfig: { fallbackLocale: (config: LinguiConfig & DeprecatedFallbackLanguage) => string; diff --git a/packages/conf/src/__snapshots__/index.test.ts.snap b/packages/conf/src/__snapshots__/index.test.ts.snap index 27bdca3f4..08bba6b97 100644 --- a/packages/conf/src/__snapshots__/index.test.ts.snap +++ b/packages/conf/src/__snapshots__/index.test.ts.snap @@ -150,6 +150,10 @@ Object { @lingui/core, i18n, ], + service: Object { + apiKey: , + name: , + }, sourceLocale: , } `; diff --git a/packages/conf/src/index.ts b/packages/conf/src/index.ts index 4d8277986..a8c895863 100644 --- a/packages/conf/src/index.ts +++ b/packages/conf/src/index.ts @@ -32,6 +32,11 @@ export type FallbackLocales = LocaleObject | DefaultLocaleObject | false type ModuleSource = [string, string?] +type CatalogService = { + name: string + apiKey: string +} + type ExtractorType = { match(filename: string): boolean extract(filename: string, targetDir: string, options?: any): void @@ -53,6 +58,7 @@ export type LinguiConfig = { rootDir: string runtimeConfigModule: ModuleSource | { [symbolName: string]: ModuleSource } sourceLocale: string + service: CatalogService } // Enforce posix path delimiters internally @@ -98,6 +104,7 @@ export const defaultConfig: LinguiConfig = { rootDir: ".", runtimeConfigModule: ["@lingui/core", "i18n"], sourceLocale: "", + service: { name: "", apiKey: "" } } function configExists(path) {