feat: Cloud service providers & Translation.io (#1107)

Now with Lingui, you could integrate your favorite SaSS for syncing your translations using an internal process. Translation.io already introduced themselves and now you can sync Translation.io translations in your React codebases without anything else than `lingui extract`

packages/cli/package.json

@@ -29,6 +29,7 @@
     "LICENSE",
     "README.md",
     "api",
+    "services",
     "lingui.js",
     "lingui-*.js"
   ],

packages/cli/src/lingui-extract.ts

@@ -68,6 +68,13 @@ export default function command(
     )
   }
 
+  // If service key is present in configuration, synchronize with cloud translation platform
+  if (typeof config.service === 'object' && config.service.name && config.service.name.length) {
+    import(`./services/${config.service.name}`)
+      .then(module => module.default(config, options))
+      .catch(err => console.error(`Can't load service module ${config.service.name}`, err))
+  }
+
   return true
 }
 

packages/cli/src/services/translationIO.ts

@@ -0,0 +1,293 @@
+import fs from "fs"
+import { dirname } from "path"
+import PO from "pofile"
+import https from "https"
+import glob from "glob"
+import { format as formatDate } from "date-fns"
+
+const getCreateHeaders = (language) => ({
+  "POT-Creation-Date": formatDate(new Date(), "yyyy-MM-dd HH:mmxxxx"),
+  "MIME-Version": "1.0",
+  "Content-Type": "text/plain; charset=utf-8",
+  "Content-Transfer-Encoding": "8bit",
+  "X-Generator": "@lingui/cli",
+  Language: language,
+})
+
+// Main sync method, call "Init" or "Sync" depending on the project context
+export default function syncProcess(config, options) {
+  if (config.format != 'po') {
+    console.error(`\n----------\nTranslation.io service is only compatible with the "po" format. Please update your Lingui configuration accordingly.\n----------`)
+    process.exit(1)
+  }
+
+  const successCallback = (project) => {
+    console.log(`\n----------\nProject successfully synchronized. Please use this URL to translate: ${project.url}\n----------`)
+  }
+
+  const failCallback = (errors) => {
+    console.error(`\n----------\nSynchronization with Translation.io failed: ${errors.join(', ')}\n----------`)
+  }
+
+  init(config, options, successCallback, (errors) => {
+    if (errors.length && errors[0] === 'This project has already been initialized.') {
+      sync(config, options, successCallback, failCallback)
+    }
+    else {
+      failCallback(errors)
+    }
+  })
+}
+
+// Initialize project with source and existing translations (only first time!)
+// Cf. https://translation.io/docs/create-library#initialization
+function init(config, options, successCallback, failCallback) {
+  const sourceLocale  = config.sourceLocale || 'en'
+  const targetLocales = config.locales.filter((value) => value != sourceLocale)
+  const paths         = poPathsPerLocale(config)
+
+  let segments = {}
+
+  targetLocales.forEach((targetLocale) => {
+    segments[targetLocale] = []
+  })
+
+  // Create segments from source locale PO items
+  paths[sourceLocale].forEach((path) => {
+    let raw = fs.readFileSync(path).toString()
+    let po = PO.parse(raw)
+
+    po.items.filter((item) => !item['obsolete']).forEach((item) => {
+      targetLocales.forEach((targetLocale) => {
+        let newSegment = createSegmentFromPoItem(item)
+
+        segments[targetLocale].push(newSegment)
+      })
+    })
+  })
+
+  // Add translations to segments from target locale PO items
+  targetLocales.forEach((targetLocale) => {
+    paths[targetLocale].forEach((path) => {
+      let raw = fs.readFileSync(path).toString()
+      let po = PO.parse(raw)
+
+      po.items.filter((item) => !item['obsolete']).forEach((item, index) => {
+        segments[targetLocale][index].target = item.msgstr[0]
+      })
+    })
+  })
+
+  let request = {
+    "client": "lingui",
+    "version": require('@lingui/core/package.json').version,
+    "source_language": sourceLocale,
+    "target_languages": targetLocales,
+    "segments": segments
+  }
+
+  postTio("init", request, config.service.apiKey, (response) => {
+    if (response.errors) {
+      failCallback(response.errors)
+    }
+    else {
+      saveSegmentsToTargetPos(config, paths, response.segments)
+      successCallback(response.project)
+    }
+  }, (error) => {
+    console.error(`\n----------\nSynchronization with Translation.io failed: ${error}\n----------`)
+  })
+}
+
+// Send all source text from PO to Translation.io and create new PO based on received translations
+// Cf. https://translation.io/docs/create-library#synchronization
+function sync(config, options, successCallback, failCallback) {
+  const sourceLocale  = config.sourceLocale || 'en'
+  const targetLocales = config.locales.filter((value) => value != sourceLocale)
+  const paths         = poPathsPerLocale(config)
+
+  let segments = []
+
+  // Create segments with correct source
+  paths[sourceLocale].forEach((path) => {
+    let raw = fs.readFileSync(path).toString()
+    let po = PO.parse(raw)
+
+    po.items.filter((item) => !item['obsolete']).forEach((item) => {
+      let newSegment = createSegmentFromPoItem(item)
+
+      segments.push(newSegment)
+    })
+  })
+
+  let request = {
+    "client": "lingui",
+    "version": require('@lingui/core/package.json').version,
+    "source_language": sourceLocale,
+    "target_languages": targetLocales,
+    "segments": segments
+  }
+
+  // Sync and then remove unused segments (not present in the local application) from Translation.io
+  if (options.clean) {
+    request['purge'] = true
+  }
+
+  postTio("sync", request, config.service.apiKey, (response) => {
+    if (response.errors) {
+      failCallback(response.errors)
+    }
+    else {
+      saveSegmentsToTargetPos(config, paths, response.segments)
+      successCallback(response.project)
+    }
+  }, (error) => {
+    console.error(`\n----------\nSynchronization with Translation.io failed: ${error}\n----------`)
+  })
+}
+
+function createSegmentFromPoItem(item) {
+  let itemHasId = item.msgid != item.msgstr[0] && item.msgstr[0].length
+
+  let segment = {
+    type: 'source',                                 // No way to edit text for source language (inside code), so not using "key" here
+    source: itemHasId ? item.msgstr[0] : item.msgid, // msgstr may be empty if --overwrite is used and no ID is used
+    context: '',
+    references: [],
+    comment: ''
+  }
+
+  if (itemHasId) {
+    segment.context = item.msgid
+  }
+
+  if (item.references.length) {
+    segment.references = item.references
+  }
+
+  if (item.extractedComments.length) {
+    segment.comment = item.extractedComments.join(' | ')
+  }
+
+  return segment
+}
+
+function createPoItemFromSegment(segment) {
+  let item = new PO.Item()
+
+  item.msgid             = segment.context ? segment.context : segment.source
+  item.msgstr            = [segment.target]
+  item.references        = (segment.references && segment.references.length) ? segment.references : []
+  item.extractedComments = segment.comment ? segment.comment.split(' | ') : []
+
+  return item
+}
+
+function saveSegmentsToTargetPos(config, paths, segmentsPerLocale) {
+  const NAME = "{name}"
+  const LOCALE = "{locale}"
+
+  Object.keys(segmentsPerLocale).forEach((targetLocale) => {
+    // Remove existing target POs and JS for this target locale
+    paths[targetLocale].forEach((path) => {
+      const jsPath  = path.replace(/\.po?$/, "") + ".js"
+      const dirPath = dirname(path)
+
+      // Remove PO, JS and empty dir
+      if (fs.existsSync(path)) { fs.unlinkSync(path) }
+      if (fs.existsSync(jsPath)) { fs.unlinkSync(jsPath) }
+      if (fs.existsSync(dirPath) && fs.readdirSync(dirPath).length === 0) { fs.rmdirSync(dirPath) }
+    })
+
+    // Find target path (ignoring {name})
+    const localePath = "".concat(config.catalogs[0].path.replace(LOCALE, targetLocale).replace(NAME, ''), ".po")
+    const segments = segmentsPerLocale[targetLocale]
+
+    let po = new PO()
+    po.headers = getCreateHeaders(targetLocale)
+
+    let items = []
+
+    segments.forEach((segment) => {
+      let item = createPoItemFromSegment(segment)
+      items.push(item)
+    })
+
+    // Sort items by messageId
+    po.items = items.sort((a, b) => {
+      if (a.msgid < b.msgid) { return -1 }
+      if (a.msgid > b.msgid) { return 1 }
+      return 0
+    })
+
+    // Check that localePath directory exists and save PO file
+    fs.promises.mkdir(dirname(localePath), {recursive: true}).then(() => {
+      po.save(localePath, (err) => {
+        if (err) {
+          console.error('Error while saving target PO files:')
+          console.error(err)
+          process.exit(1)
+        }
+      })
+    })
+  })
+}
+
+function poPathsPerLocale(config) {
+  const NAME = "{name}"
+  const LOCALE = "{locale}"
+  const paths = []
+
+  config.locales.forEach((locale) => {
+    paths[locale] = []
+
+    config.catalogs.forEach((catalog) => {
+      const path = "".concat(catalog.path.replace(LOCALE, locale).replace(NAME, "*"), ".po")
+
+      // If {name} is present (replaced by *), list all the existing POs
+      if (path.includes('*')) {
+        paths[locale] = paths[locale].concat(glob.sync(path))
+      }
+      else {
+        paths[locale].push(path)
+      }
+    })
+  })
+
+  return paths
+}
+
+function postTio(action, request, apiKey, successCallback, failCallback) {
+  let jsonRequest = JSON.stringify(request)
+
+  let options = {
+    hostname: 'translation.io',
+    path: '/api/v1/segments/' + action + '.json?api_key=' + apiKey,
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/json',
+    }
+  }
+
+  let req = https.request(options, (res) => {
+    res.setEncoding('utf8')
+
+    let body = ""
+
+    res.on('data', (chunk) => {
+      body = body.concat(chunk)
+    })
+
+    res.on('end', () => {
+      let response = JSON.parse(body)
+      successCallback(response)
+    })
+  })
+
+  req.on('error', (e) => {
+    failCallback(e)
+  })
+
+  req.write(jsonRequest)
+  req.end()
+}

packages/conf/index.d.ts

@@ -1,6 +1,6 @@
 import type { GeneratorOptions } from "@babel/core";
 
-export declare type CatalogFormat = "lingui" | "minimal" | "po" | "csv" | "po-gettext";
+export declare type CatalogFormat = "lingui" | "minimal" | "po" | "csv" | "po-gettext";
 export type CatalogFormatOptions = {
     origins?: boolean;
     lineNumbers?: boolean;
@@ -23,6 +23,11 @@ export type DefaultLocaleObject = {
 
 export declare type FallbackLocales = LocaleObject | DefaultLocaleObject
 
+declare type CatalogService = {
+    name: string;
+    apiKey: string;
+}
+
 declare type ExtractorType = {
     match(filename: string): boolean;
     extract(filename: string, targetDir: string, options?: any): void;
@@ -46,6 +51,7 @@ export declare type LinguiConfig = {
     rootDir: string;
     runtimeConfigModule: [string, string?];
     sourceLocale: string;
+    service: CatalogService;
 };
 export declare const defaultConfig: LinguiConfig;
 export declare function getConfig({ cwd, configPath, skipValidation, }?: {
@@ -63,7 +69,7 @@ export declare const configValidation: {
         };
         compilerBabelOptions: GeneratorOptions;
         catalogs: CatalogConfig[];
-        compileNamespace: "es" | "ts" | "cjs" | string;
+        compileNamespace: "es" | "ts" | "cjs" | string;
         fallbackLocales: FallbackLocales;
         format: CatalogFormat;
         formatOptions: CatalogFormatOptions;
@@ -73,6 +79,7 @@ export declare const configValidation: {
         rootDir: string;
         runtimeConfigModule: [string, string?];
         sourceLocale: string;
+        service: CatalogService;
     };
     deprecatedConfig: {
         fallbackLocale: (config: LinguiConfig & DeprecatedFallbackLanguage) => string;

packages/conf/src/__snapshots__/index.test.ts.snap

@@ -150,6 +150,10 @@ Object {
     @lingui/core,
     i18n,
   ],
+  service: Object {
+    apiKey: ,
+    name: ,
+  },
   sourceLocale: ,
 }
 `;

packages/conf/src/index.ts

@@ -32,6 +32,11 @@ export type FallbackLocales = LocaleObject | DefaultLocaleObject | false
 
 type ModuleSource = [string, string?]
 
+type CatalogService = {
+  name: string
+  apiKey: string
+}
+
 type ExtractorType = {
   match(filename: string): boolean
   extract(filename: string, targetDir: string, options?: any): void
@@ -53,6 +58,7 @@ export type LinguiConfig = {
   rootDir: string
   runtimeConfigModule: ModuleSource | { [symbolName: string]: ModuleSource }
   sourceLocale: string
+  service: CatalogService
 }
 
 // Enforce posix path delimiters internally
@@ -98,6 +104,7 @@ export const defaultConfig: LinguiConfig = {
   rootDir: ".",
   runtimeConfigModule: ["@lingui/core", "i18n"],
   sourceLocale: "",
+  service: { name: "", apiKey: "" }
 }
 
 function configExists(path) {