From e73a4b34cf8d83a45044c220148761d79b4fd8a9 Mon Sep 17 00:00:00 2001 From: Sergio Moreno Date: Tue, 8 Jun 2021 18:15:53 +0200 Subject: [PATCH] feat: load remote catalogs with remoteLoader() (#1080) --- packages/cli/src/api/formats/csv.ts | 4 + packages/cli/src/api/formats/index.ts | 1 + packages/cli/src/api/formats/lingui.ts | 4 + packages/cli/src/api/formats/minimal.ts | 4 + packages/cli/src/api/formats/po-gettext.ts | 2 +- packages/cli/src/api/formats/po.ts | 2 +- packages/loader/index.d.ts | 3 + packages/loader/index.js | 1 + packages/loader/package.json | 1 + packages/loader/src/index.ts | 2 + packages/loader/src/remoteLoader.test.ts | 95 +++++++++++++++++++ packages/loader/src/remoteLoader.ts | 57 +++++++++++ .../loader/src/{index.js => webpackLoader.ts} | 2 +- packages/loader/test/locale/es/messages.po | 5 + 14 files changed, 180 insertions(+), 3 deletions(-) create mode 100644 packages/loader/index.d.ts create mode 100644 packages/loader/src/index.ts create mode 100644 packages/loader/src/remoteLoader.test.ts create mode 100644 packages/loader/src/remoteLoader.ts rename packages/loader/src/{index.js => webpackLoader.ts} (99%) create mode 100644 packages/loader/test/locale/es/messages.po diff --git a/packages/cli/src/api/formats/csv.ts b/packages/cli/src/api/formats/csv.ts index 73f8901ed..c2c114196 100644 --- a/packages/cli/src/api/formats/csv.ts +++ b/packages/cli/src/api/formats/csv.ts @@ -48,6 +48,10 @@ const csv: CatalogFormatter = { throw new Error(`Cannot read ${filename}: ${e.message}`) } }, + + parse(content) { + return deserialize(content) + } } export default csv diff --git a/packages/cli/src/api/formats/index.ts b/packages/cli/src/api/formats/index.ts index 020015508..e75d4f68c 100644 --- a/packages/cli/src/api/formats/index.ts +++ b/packages/cli/src/api/formats/index.ts @@ -28,6 +28,7 @@ export type CatalogFormatter = { options?: CatalogFormatOptionsInternal ): void read(filename: string): CatalogType | null + parse(content): any } export default function getFormat(name: CatalogFormat): CatalogFormatter { diff --git a/packages/cli/src/api/formats/lingui.ts b/packages/cli/src/api/formats/lingui.ts index b010d8a43..6b97fb6ff 100644 --- a/packages/cli/src/api/formats/lingui.ts +++ b/packages/cli/src/api/formats/lingui.ts @@ -49,6 +49,10 @@ const lingui: CatalogFormatter = { throw new Error(`Cannot read ${filename}: ${e.message}`) } }, + + parse(content) { + return content + } } export default lingui diff --git a/packages/cli/src/api/formats/minimal.ts b/packages/cli/src/api/formats/minimal.ts index e59cc0cf1..635da9934 100644 --- a/packages/cli/src/api/formats/minimal.ts +++ b/packages/cli/src/api/formats/minimal.ts @@ -36,6 +36,10 @@ const minimal: CatalogFormatter = { throw new Error(`Cannot read ${filename}: ${e.message}`) } }, + + parse(content: Record) { + return deserialize(content) + } } export default minimal diff --git a/packages/cli/src/api/formats/po-gettext.ts b/packages/cli/src/api/formats/po-gettext.ts index 2fc75906e..9f99017a1 100644 --- a/packages/cli/src/api/formats/po-gettext.ts +++ b/packages/cli/src/api/formats/po-gettext.ts @@ -332,7 +332,7 @@ const poGettext: CatalogFormatter & PoFormatter = { return this.parse(raw) }, - parse(raw) { + parse(raw: string) { const po = PO.parse(raw) convertPluralsToICU(po.items, po.headers.Language) return deserialize(indexItems(po.items)) diff --git a/packages/cli/src/api/formats/po.ts b/packages/cli/src/api/formats/po.ts index 914cb1ee7..7ed556814 100644 --- a/packages/cli/src/api/formats/po.ts +++ b/packages/cli/src/api/formats/po.ts @@ -106,7 +106,7 @@ const po: CatalogFormatter & PoFormatter = { return this.parse(raw) }, - parse(raw) { + parse(raw: string) { const po = PO.parse(raw) validateItems(po.items) return deserialize(indexItems(po.items)) diff --git a/packages/loader/index.d.ts b/packages/loader/index.d.ts new file mode 100644 index 000000000..d19498dd8 --- /dev/null +++ b/packages/loader/index.d.ts @@ -0,0 +1,3 @@ +declare type RemoteLoaderMessages = string | Record | T; +export declare function remoteLoader(locale: string, messages: RemoteLoaderMessages, fallbackMessages?: RemoteLoaderMessages): any; +export {}; diff --git a/packages/loader/index.js b/packages/loader/index.js index c14e96c06..39ac9ca5c 100644 --- a/packages/loader/index.js +++ b/packages/loader/index.js @@ -1 +1,2 @@ export { default } from "./src" +export { remoteLoader } from "./src/remoteLoader" \ No newline at end of file diff --git a/packages/loader/package.json b/packages/loader/package.json index 72b307b97..5f926a65b 100644 --- a/packages/loader/package.json +++ b/packages/loader/package.json @@ -2,6 +2,7 @@ "name": "@lingui/loader", "version": "3.9.0", "description": "webpack loader for lingui message catalogs", + "types": "index.d.ts", "main": "index.js", "author": { "name": "Tomáš Ehrlich", diff --git a/packages/loader/src/index.ts b/packages/loader/src/index.ts new file mode 100644 index 000000000..a4df3dc36 --- /dev/null +++ b/packages/loader/src/index.ts @@ -0,0 +1,2 @@ +export { default } from "./webpackLoader" +export { remoteLoader } from "./remoteLoader" \ No newline at end of file diff --git a/packages/loader/src/remoteLoader.test.ts b/packages/loader/src/remoteLoader.test.ts new file mode 100644 index 000000000..e2c08748e --- /dev/null +++ b/packages/loader/src/remoteLoader.test.ts @@ -0,0 +1,95 @@ +import { remoteLoader } from "./remoteLoader" +import fs from "fs" +import path from "path" + +describe("remote-loader", () => { + it("should compile correctly JSON messages coming from the fly", async () => { + const unlink = createConfig("minimal") + const messages = await simulatedJsonResponse() + expect(remoteLoader("en", messages)).toMatchInlineSnapshot( + `/*eslint-disable*/module.exports={messages:{"property.key":"value","{0} Deposited":[["0"]," Deposited"],"{0} Strategy":[["0"]," Strategy"]}};` + ) + unlink() + }) + + it("should compile correctly .po messages coming from the fly", async () => { + const unlink = createConfig("po") + const messages = await simulatedPoResponse() + expect(remoteLoader("en", messages)).toMatchInlineSnapshot( + `/*eslint-disable*/module.exports={messages:{"Hello World":"Hello World","My name is {name}":["My name is ",["name"]]}};` + ) + unlink() + }) + + describe("fallbacks", () => { + it("should fallback correctly to the fallback collection", async () => { + const unlink = createConfig("minimal") + const messages = await simulatedJsonResponse(true) + const fallbackMessages = await simulatedJsonResponse() + + expect( + remoteLoader("en", messages, fallbackMessages) + ).toMatchInlineSnapshot( + `/*eslint-disable*/module.exports={messages:{"property.key":"value","{0} Deposited":[["0"]," Deposited"],"{0} Strategy":[["0"]," Strategy"]}};` + ) + unlink() + }) + + it("should fallback to compiled fallback", async () => { + const unlink = createConfig("po") + const messages = await simulatedPoResponse("es") + const fallbackMessages = await simulatedPoCompiledFile() + + expect( + remoteLoader("en", messages, fallbackMessages) + ).toMatchInlineSnapshot( + `/*eslint-disable*/module.exports={messages:{"Hello World":"Hello World","My name is {name}":["My name is ",["name"]]}};` + ) + unlink() + }) + }) +}) + +function simulatedJsonResponse(nully?: boolean) { + return new Promise((resolve) => { + resolve({ + "property.key": nully ? "" : "value", + "{0} Deposited": "{0} Deposited", + "{0} Strategy": "{0} Strategy", + }) + }) +} + +function simulatedPoResponse(locale = "en") { + return new Promise((resolve) => { + const file = fs.readFileSync( + path.join(__dirname, "..", "test/locale/" + locale + "/messages.po"), + "utf-8" + ) + resolve(file) + }) +} + +function simulatedPoCompiledFile() { + return new Promise((resolve) => { + resolve({ + "Hello World": "Hello World", + "My name is {name}": ["My name is ", ["name"]], + }) + }) +} + +function createConfig(format: string) { + const filename = `${process.cwd()}/.linguirc` + const config = ` + { + 'locales': ['en'], + 'catalogs': [{ + 'path': 'locale/{locale}/messages' + }], + 'format': '${format}' + } + ` + fs.writeFileSync(filename, config) + return () => fs.unlinkSync(filename) +} diff --git a/packages/loader/src/remoteLoader.ts b/packages/loader/src/remoteLoader.ts new file mode 100644 index 000000000..e89bd0420 --- /dev/null +++ b/packages/loader/src/remoteLoader.ts @@ -0,0 +1,57 @@ +import R from "ramda" +import { getConfig } from "@lingui/conf" +import { createCompiledCatalog, getFormat } from "@lingui/cli/api" + +type RemoteLoaderMessages = string | Record | T + +export function remoteLoader(locale: string, messages: RemoteLoaderMessages, fallbackMessages?: RemoteLoaderMessages) { + const config = getConfig() + + // When format is minimal, everything works fine because are .json files, + // but when is csv, .po, .po-gettext needs to be parsed to something interpretable + let parsedMessages; + let parsedFallbackMessages; + if (config.format) { + const formatter = getFormat(config.format) + if (fallbackMessages) { + // we do this because, people could just import the fallback and import the ./en/messages.js + // generated by lingui and the use case of format .po but fallback as .json module could be perfectly valid + parsedFallbackMessages = typeof fallbackMessages === "object" ? getFormat("minimal").parse(fallbackMessages) : formatter.parse(fallbackMessages) + } + + parsedMessages = formatter.parse(messages) + } else { + throw new Error(` + *format* value in the Lingui configuration is required to make this loader 100% functional + Read more about this here: https://lingui.js.org/ref/conf.html#format + `) + } + + + const mapTranslationsToInterporlatedString = R.mapObjIndexed( + (_, key) => { + // if there's fallback and translation is empty, return the fallback + if (parsedMessages[key].translation === "" && parsedFallbackMessages?.[key]?.translation) { + return parsedFallbackMessages[key].translation + } + + return parsedMessages[key].translation + }, + parsedMessages + ) + + // In production we don't want untranslated strings. It's better to use message + // keys as a last resort. + // In development, however, we want to catch missing strings with `missing` parameter + // of I18nProvider (React) or setupI18n (core) and therefore we need to get + // empty translations if missing. + const strict = process.env.NODE_ENV !== "production" + return createCompiledCatalog(locale, mapTranslationsToInterporlatedString, { + strict, + ...config, + namespace: config.compileNamespace, + pseudoLocale: config.pseudoLocale, + }) + +} + diff --git a/packages/loader/src/index.js b/packages/loader/src/webpackLoader.ts similarity index 99% rename from packages/loader/src/index.js rename to packages/loader/src/webpackLoader.ts index 5a2445907..b6cf178c4 100644 --- a/packages/loader/src/index.js +++ b/packages/loader/src/webpackLoader.ts @@ -18,7 +18,7 @@ try { JavascriptGenerator = require("webpack/lib/JavascriptGenerator") } catch (error) { if (error.code !== "MODULE_NOT_FOUND") { - throw e + throw error } } diff --git a/packages/loader/test/locale/es/messages.po b/packages/loader/test/locale/es/messages.po new file mode 100644 index 000000000..6647c1086 --- /dev/null +++ b/packages/loader/test/locale/es/messages.po @@ -0,0 +1,5 @@ +msgid "Hello World" +msgstr "" + +msgid "My name is {name}" +msgstr "My name is {name}"