feat: load remote catalogs with remoteLoader() (#1080)

packages/cli/src/api/formats/csv.ts

@@ -48,6 +48,10 @@ const csv: CatalogFormatter = {
       throw new Error(`Cannot read ${filename}: ${e.message}`)
     }
   },
+
+  parse(content) {
+    return deserialize(content)
+  }
 }
 
 export default csv

packages/cli/src/api/formats/index.ts

@@ -28,6 +28,7 @@ export type CatalogFormatter = {
     options?: CatalogFormatOptionsInternal
   ): void
   read(filename: string): CatalogType | null
+  parse(content): any
 }
 
 export default function getFormat(name: CatalogFormat): CatalogFormatter {

packages/cli/src/api/formats/lingui.ts

@@ -49,6 +49,10 @@ const lingui: CatalogFormatter = {
       throw new Error(`Cannot read ${filename}: ${e.message}`)
     }
   },
+
+  parse(content) {
+    return content
+  }
 }
 
 export default lingui

packages/cli/src/api/formats/minimal.ts

@@ -36,6 +36,10 @@ const minimal: CatalogFormatter = {
       throw new Error(`Cannot read ${filename}: ${e.message}`)
     }
   },
+
+  parse(content: Record<string, any>) {
+    return deserialize(content)
+  }
 }
 
 export default minimal

packages/cli/src/api/formats/po-gettext.ts

@@ -332,7 +332,7 @@ const poGettext: CatalogFormatter & PoFormatter = {
     return this.parse(raw)
   },
 
-  parse(raw) {
+  parse(raw: string) {
     const po = PO.parse(raw)
     convertPluralsToICU(po.items, po.headers.Language)
     return deserialize(indexItems(po.items))

packages/cli/src/api/formats/po.ts

@@ -106,7 +106,7 @@ const po: CatalogFormatter & PoFormatter = {
     return this.parse(raw)
   },
 
-  parse(raw) {
+  parse(raw: string) {
     const po = PO.parse(raw)
     validateItems(po.items)
     return deserialize(indexItems(po.items))

packages/loader/index.d.ts

@@ -0,0 +1,3 @@
+declare type RemoteLoaderMessages<T> = string | Record<string, any> | T;
+export declare function remoteLoader<T>(locale: string, messages: RemoteLoaderMessages<T>, fallbackMessages?: RemoteLoaderMessages<T>): any;
+export {};

packages/loader/index.js

@@ -1 +1,2 @@
 export { default } from "./src"
+export { remoteLoader } from "./src/remoteLoader"

packages/loader/package.json

@@ -2,6 +2,7 @@
   "name": "@lingui/loader",
   "version": "3.9.0",
   "description": "webpack loader for lingui message catalogs",
+  "types": "index.d.ts",
   "main": "index.js",
   "author": {
     "name": "Tomáš Ehrlich",

packages/loader/src/index.ts

@@ -0,0 +1,2 @@
+export { default } from "./webpackLoader"
+export { remoteLoader } from "./remoteLoader"

packages/loader/src/remoteLoader.test.ts

@@ -0,0 +1,95 @@
+import { remoteLoader } from "./remoteLoader"
+import fs from "fs"
+import path from "path"
+
+describe("remote-loader", () => {
+  it("should compile correctly JSON messages coming from the fly", async () => {
+    const unlink = createConfig("minimal")
+    const messages = await simulatedJsonResponse()
+    expect(remoteLoader("en", messages)).toMatchInlineSnapshot(
+      `/*eslint-disable*/module.exports={messages:{"property.key":"value","{0} Deposited":[["0"]," Deposited"],"{0} Strategy":[["0"]," Strategy"]}};`
+    )
+    unlink()
+  })
+
+  it("should compile correctly .po messages coming from the fly", async () => {
+    const unlink = createConfig("po")
+    const messages = await simulatedPoResponse()
+    expect(remoteLoader("en", messages)).toMatchInlineSnapshot(
+      `/*eslint-disable*/module.exports={messages:{"Hello World":"Hello World","My name is {name}":["My name is ",["name"]]}};`
+    )
+    unlink()
+  })
+
+  describe("fallbacks", () => {
+    it("should fallback correctly to the fallback collection", async () => {
+      const unlink = createConfig("minimal")
+      const messages = await simulatedJsonResponse(true)
+      const fallbackMessages = await simulatedJsonResponse()
+
+      expect(
+        remoteLoader("en", messages, fallbackMessages)
+      ).toMatchInlineSnapshot(
+        `/*eslint-disable*/module.exports={messages:{"property.key":"value","{0} Deposited":[["0"]," Deposited"],"{0} Strategy":[["0"]," Strategy"]}};`
+      )
+      unlink()
+    })
+
+    it("should fallback to compiled fallback", async () => {
+      const unlink = createConfig("po")
+      const messages = await simulatedPoResponse("es")
+      const fallbackMessages = await simulatedPoCompiledFile()
+
+      expect(
+        remoteLoader("en", messages, fallbackMessages)
+      ).toMatchInlineSnapshot(
+        `/*eslint-disable*/module.exports={messages:{"Hello World":"Hello World","My name is {name}":["My name is ",["name"]]}};`
+      )
+      unlink()
+    })
+  })
+})
+
+function simulatedJsonResponse(nully?: boolean) {
+  return new Promise((resolve) => {
+    resolve({
+      "property.key": nully ? "" : "value",
+      "{0} Deposited": "{0} Deposited",
+      "{0} Strategy": "{0} Strategy",
+    })
+  })
+}
+
+function simulatedPoResponse(locale = "en") {
+  return new Promise((resolve) => {
+    const file = fs.readFileSync(
+      path.join(__dirname, "..", "test/locale/" + locale + "/messages.po"),
+      "utf-8"
+    )
+    resolve(file)
+  })
+}
+
+function simulatedPoCompiledFile() {
+  return new Promise((resolve) => {
+    resolve({
+      "Hello World": "Hello World",
+      "My name is {name}": ["My name is ", ["name"]],
+    })
+  })
+}
+
+function createConfig(format: string) {
+  const filename = `${process.cwd()}/.linguirc`
+  const config = `
+  {
+    'locales': ['en'],
+    'catalogs': [{
+      'path': 'locale/{locale}/messages'
+    }],
+    'format': '${format}'
+  }
+  `
+  fs.writeFileSync(filename, config)
+  return () => fs.unlinkSync(filename)
+}

packages/loader/src/remoteLoader.ts

@@ -0,0 +1,57 @@
+import R from "ramda"
+import { getConfig } from "@lingui/conf"
+import { createCompiledCatalog, getFormat } from "@lingui/cli/api"
+
+type RemoteLoaderMessages<T> = string | Record<string, any> | T
+
+export function remoteLoader<T>(locale: string, messages: RemoteLoaderMessages<T>, fallbackMessages?: RemoteLoaderMessages<T>) {
+  const config = getConfig()
+
+  // When format is minimal, everything works fine because are .json files,
+  // but when is csv, .po, .po-gettext needs to be parsed to something interpretable
+  let parsedMessages;
+  let parsedFallbackMessages;
+  if (config.format) {
+    const formatter = getFormat(config.format)
+    if (fallbackMessages) {
+      // we do this because, people could just import the fallback and import the ./en/messages.js
+      // generated by lingui and the use case of format .po but fallback as .json module could be perfectly valid
+      parsedFallbackMessages = typeof fallbackMessages === "object" ? getFormat("minimal").parse(fallbackMessages) : formatter.parse(fallbackMessages)
+    }
+
+    parsedMessages = formatter.parse(messages)
+  } else {
+    throw new Error(`
+        *format* value in the Lingui configuration is required to make this loader 100% functional
+        Read more about this here: https://lingui.js.org/ref/conf.html#format
+      `)
+  }
+
+
+  const mapTranslationsToInterporlatedString = R.mapObjIndexed(
+    (_, key) => {
+      // if there's fallback and translation is empty, return the fallback
+      if (parsedMessages[key].translation === "" && parsedFallbackMessages?.[key]?.translation) {
+        return parsedFallbackMessages[key].translation
+      }
+
+      return parsedMessages[key].translation
+    },
+    parsedMessages
+  )
+
+  // In production we don't want untranslated strings. It's better to use message
+  // keys as a last resort.
+  // In development, however, we want to catch missing strings with `missing` parameter
+  // of I18nProvider (React) or setupI18n (core) and therefore we need to get
+  // empty translations if missing.
+  const strict = process.env.NODE_ENV !== "production"
+  return createCompiledCatalog(locale, mapTranslationsToInterporlatedString, {
+    strict,
+    ...config,
+    namespace: config.compileNamespace,
+    pseudoLocale: config.pseudoLocale,
+  })
+
+}
+

packages/loader/src/index.js → packages/loader/src/webpackLoader.ts

@@ -18,7 +18,7 @@ try {
   JavascriptGenerator = require("webpack/lib/JavascriptGenerator")
 } catch (error) {
   if (error.code !== "MODULE_NOT_FOUND") {
-    throw e
+    throw error
   }
 }
 

packages/loader/test/locale/es/messages.po

@@ -0,0 +1,5 @@
+msgid "Hello World"
+msgstr ""
+
+msgid "My name is {name}"
+msgstr "My name is {name}"