From fc7dc5e2755906f20e800f2cf71e82b1fb11d776 Mon Sep 17 00:00:00 2001 From: John Resig Date: Mon, 22 Jul 2024 11:15:28 -0400 Subject: [PATCH] Add in loadTranslations/clearTranslations to i18n, use stored messages if they exist. (#2277) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary: This is adding in functionality that's similar to LinguiJS, to the wonder-blocks-i18n package, so that we can have a store of translated strings and use them for our translations instead of assuming that the strings will already be translated for us. Issue: FEI-5682 ## Test plan: Tests should pass! I'll be wiring this up in Webapp in a separate PR, once this lands. Author: jeresig Reviewers: jeresig, jandrade Required Reviewers: Approved By: jandrade Checks: ✅ codecov/project, ✅ Chromatic - Get results on regular PRs (ubuntu-latest, 20.x), ✅ Test (ubuntu-latest, 20.x, 2/2), ✅ Check build sizes (ubuntu-latest, 20.x), ✅ Test (ubuntu-latest, 20.x, 1/2), ✅ Lint (ubuntu-latest, 20.x), ✅ Chromatic - Build on regular PRs / chromatic (ubuntu-latest, 20.x), ✅ Publish npm snapshot (ubuntu-latest, 20.x), ⏭️ Chromatic - Skip on Release PR (changesets), ✅ Prime node_modules cache for primary configuration (ubuntu-latest, 20.x), ✅ Check for .changeset entries for all changed files (ubuntu-latest, 20.x), ✅ gerald, ⏭️ dependabot Pull Request URL: https://github.com/Khan/wonder-blocks/pull/2277 --- .changeset/dull-stingrays-mate.md | 5 + .vscode/settings.json | 5 +- .vscode/tasks.json | 4 +- .../__tests__/field-heading.test.tsx | 12 +- .../functions/__tests__/i18n-store.test.ts | 147 ++++++++++++++++++ .../src/functions/__tests__/i18n.test.ts | 82 +++++++++- .../src/functions/i18n-store.ts | 134 ++++++++++++++++ .../wonder-blocks-i18n/src/functions/i18n.ts | 87 ++++------- .../wonder-blocks-i18n/src/functions/types.ts | 5 + packages/wonder-blocks-i18n/src/index.ts | 1 + 10 files changed, 417 insertions(+), 65 deletions(-) create mode 100644 .changeset/dull-stingrays-mate.md create mode 100644 packages/wonder-blocks-i18n/src/functions/__tests__/i18n-store.test.ts create mode 100644 packages/wonder-blocks-i18n/src/functions/i18n-store.ts diff --git a/.changeset/dull-stingrays-mate.md b/.changeset/dull-stingrays-mate.md new file mode 100644 index 000000000..048d5b680 --- /dev/null +++ b/.changeset/dull-stingrays-mate.md @@ -0,0 +1,5 @@ +--- +"@khanacademy/wonder-blocks-i18n": minor +--- + +Add loadTranslations/clearTranslations methods to wonder-blocks-i18n. diff --git a/.vscode/settings.json b/.vscode/settings.json index 516bea859..841a5c646 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -13,5 +13,6 @@ "files.associations": { "*.flow": "plaintext", }, - "flow.enabled": false -} \ No newline at end of file + "flow.enabled": false, + "jest.jestCommandLine": "yarn jest" +} diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 471e7cbe6..2883ec81c 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -44,11 +44,11 @@ "focus": false, "panel": "dedicated" }, - "script": "test", + "script": "jest", "group": { "kind": "test", "isDefault": true } } ] -} \ No newline at end of file +} diff --git a/packages/wonder-blocks-form/src/components/__tests__/field-heading.test.tsx b/packages/wonder-blocks-form/src/components/__tests__/field-heading.test.tsx index a5fd1a090..81804f0f7 100644 --- a/packages/wonder-blocks-form/src/components/__tests__/field-heading.test.tsx +++ b/packages/wonder-blocks-form/src/components/__tests__/field-heading.test.tsx @@ -187,7 +187,11 @@ describe("FieldHeading", () => { render( {}} />} - label={Hello, world!} + label={ + {s}}> + {"Test Hello, world!"} + + } />, ); @@ -205,7 +209,11 @@ describe("FieldHeading", () => { {}} />} label={Hello, world} - description={description} + description={ + {s}}> + {"Test description"} + + } />, ); diff --git a/packages/wonder-blocks-i18n/src/functions/__tests__/i18n-store.test.ts b/packages/wonder-blocks-i18n/src/functions/__tests__/i18n-store.test.ts new file mode 100644 index 000000000..603ed8306 --- /dev/null +++ b/packages/wonder-blocks-i18n/src/functions/__tests__/i18n-store.test.ts @@ -0,0 +1,147 @@ +import { + clearTranslations, + getPluralTranslation, + getSingularTranslation, + loadTranslations, +} from "../i18n-store"; +import {setLocale} from "../locale"; + +describe("getSingularTranslation", () => { + const TEST_LOCALE = "en-pt"; + + afterEach(() => { + clearTranslations(TEST_LOCALE); + }); + + it("should return the translated string", () => { + // Arrange + loadTranslations(TEST_LOCALE, { + test: "arrrr matey", + }); + setLocale(TEST_LOCALE); + + // Act + const result = getSingularTranslation("test"); + + // Assert + expect(result).toMatchInlineSnapshot(`"arrrr matey"`); + }); + + it("should return the original string if no translation found", () => { + // Arrange + loadTranslations(TEST_LOCALE, { + test2: "arrrr matey", + }); + setLocale(TEST_LOCALE); + + // Act + const result = getSingularTranslation("test"); + + // Assert + expect(result).toMatchInlineSnapshot(`"test"`); + }); + + it("should return the translated string even if it's plural", () => { + // Arrange + loadTranslations(TEST_LOCALE, { + test: ["arrrr matey", "arrrr mateys"], + }); + setLocale(TEST_LOCALE); + + // Act + const result = getSingularTranslation("test"); + + // Assert + expect(result).toMatchInlineSnapshot(`"arrrr matey"`); + }); + + it("should return a fake translation", () => { + // Arrange + setLocale("boxes"); + + // Act + const result = getSingularTranslation("test"); + + // Assert + expect(result).toMatchInlineSnapshot(`"□□□□"`); + }); +}); + +describe("getPluralTranslation", () => { + const TEST_LOCALE = "en-pt"; + + afterEach(() => { + clearTranslations(TEST_LOCALE); + }); + + it("should return the translated plural singular string", () => { + // Arrange + loadTranslations(TEST_LOCALE, { + test: "arrrr matey", + }); + setLocale(TEST_LOCALE); + + // Act + const result = getPluralTranslation( + { + lang: TEST_LOCALE, + messages: ["test singular", "test plural"], + }, + 0, + ); + + // Assert + expect(result).toMatchInlineSnapshot(`"test singular"`); + }); + + it("should return the translated plural string", () => { + // Arrange + loadTranslations(TEST_LOCALE, { + test: "arrrr matey", + }); + setLocale(TEST_LOCALE); + + // Act + const result = getPluralTranslation( + { + lang: TEST_LOCALE, + messages: ["test singular", "test plural"], + }, + 1, + ); + + // Assert + expect(result).toMatchInlineSnapshot(`"test plural"`); + }); + + it("should return the original string if no translation found", () => { + // Arrange + loadTranslations(TEST_LOCALE, { + test: "arrrr matey", + }); + setLocale(TEST_LOCALE); + + // Act + const result = getSingularTranslation("test2"); + + // Assert + expect(result).toMatchInlineSnapshot(`"test2"`); + }); + + it("should return a fake translation", () => { + // Arrange + setLocale("boxes"); + + // Act + const result = getPluralTranslation( + { + lang: TEST_LOCALE, + messages: ["test singular", "test plural"], + }, + 0, + ); + + // Assert + expect(result).toMatchInlineSnapshot(`"□□□□ □□□□□□□□"`); + }); +}); diff --git a/packages/wonder-blocks-i18n/src/functions/__tests__/i18n.test.ts b/packages/wonder-blocks-i18n/src/functions/__tests__/i18n.test.ts index d30752e48..e5b2a757b 100644 --- a/packages/wonder-blocks-i18n/src/functions/__tests__/i18n.test.ts +++ b/packages/wonder-blocks-i18n/src/functions/__tests__/i18n.test.ts @@ -3,6 +3,7 @@ import * as React from "react"; import * as Locale from "../locale"; import * as FakeTranslate from "../i18n-faketranslate"; import {_, $_, ngettext, doNotTranslate, doNotTranslateYet} from "../i18n"; +import {clearTranslations, loadTranslations} from "../i18n-store"; jest.mock("react", () => { return { @@ -27,7 +28,7 @@ describe("i18n", () => { jest.clearAllMocks(); }); - describe("integration tests", () => { + describe("FakeTranslate integration tests", () => { beforeEach(() => { jest.spyOn(Locale, "getLocale").mockImplementation(() => "boxes"); }); @@ -89,6 +90,85 @@ describe("i18n", () => { }); }); + describe("I18nStore integration tests", () => { + const TEST_LOCALE = "en-pt"; + + afterEach(() => { + clearTranslations(TEST_LOCALE); + }); + + beforeEach(() => { + jest.spyOn(Locale, "getLocale").mockImplementation( + () => TEST_LOCALE, + ); + }); + + it("_ should translate", () => { + // Arrange + loadTranslations(TEST_LOCALE, { + test: "arrrr matey", + }); + + // Act + const result = _("test"); + + // Assert + expect(result).toMatchInlineSnapshot(`"arrrr matey"`); + }); + + it("$_ should translate", () => { + // Arrange + loadTranslations(TEST_LOCALE, { + test: "arrrr matey", + }); + + // Act + const result = $_("test"); + + // Assert + expect(result).toMatchInlineSnapshot(`"arrrr matey"`); + }); + + it("ngettext should translate", () => { + // Arrange + loadTranslations(TEST_LOCALE, { + Singular: ["arrrr matey", "arrrr mateys"], + }); + + // Act + const result = ngettext("Singular", "Plural", 0); + + // Assert + expect(result).toMatchInlineSnapshot(`"Plural"`); + }); + + it("doNotTranslate should not translate", () => { + // Arrange + loadTranslations(TEST_LOCALE, { + test: "arrrr matey", + }); + + // Act + const result = doNotTranslate("test"); + + // Assert + expect(result).toEqual("test"); + }); + + it("doNotTranslateYet should not translate", () => { + // Arrange + loadTranslations(TEST_LOCALE, { + test: "arrrr matey", + }); + + // Act + const result = doNotTranslateYet("test"); + + // Assert + expect(result).toEqual("test"); + }); + }); + describe("# _", () => { it("returns input when no translation nor substitutions", () => { // Arrange diff --git a/packages/wonder-blocks-i18n/src/functions/i18n-store.ts b/packages/wonder-blocks-i18n/src/functions/i18n-store.ts new file mode 100644 index 000000000..ce2e31f7c --- /dev/null +++ b/packages/wonder-blocks-i18n/src/functions/i18n-store.ts @@ -0,0 +1,134 @@ +/** + * Functions for storing and retrieving translations. + * + * The i18n store is a simple cache that stores translations for a given locale. + */ +import FakeTranslate from "./i18n-faketranslate"; +import {getLocale} from "./locale"; +import {PluralConfigurationObject} from "./types"; + +// The cache of strings that have been translated, by locale. +const localeMessageStore = new Map< + string, + // Singular strings are stored as a string, plural strings are stored as + // an arrays of strings. + Record> +>(); + +// Create a fake translate object to use if we can't find a translation. +const {translate: fakeTranslate} = new FakeTranslate(); + +/** + * Get the translation for a given id and locale. + * + * @param id the id of the string to translate + * @param locale the locale to translate to + * @returns the translated string, or null if no translation is found + */ +const getTranslationFromStore = (id: string, locale: string) => { + // See if we have a cache for the locale. + const messageCache = localeMessageStore.get(locale); + + if (!messageCache) { + return null; + } + + // See if we have a cached message for the id. + const cachedMessage = messageCache[id]; + + if (!cachedMessage) { + return null; + } + + // We found the translated string (or strings) so we can return it. + return cachedMessage; +}; + +/** + * Get the translation for a given message. If no translation is found, attempt + * to get the translation using FakeTranslate. If that fails, return the message. + * + * @param strOrPluralConfig the id of the string to translate, or a plural + * configuration object + * @returns the translated string + */ +export const getSingularTranslation = ( + strOrPluralConfig: string | PluralConfigurationObject, +) => { + // Sometimes we're given an argument that's meant for ngettext(). This + // happens if the same string is used in both i18n._() and i18n.ngettext() + // (.g. a = i18n._(foo); b = i18n.ngettext("foo", "bar", count); + // In such cases, only the plural form ends up in the .po file, and + // then it gets sent to us for the i18n._() case too. No problem, though: + // we'll just take the singular arg. + const id = + typeof strOrPluralConfig === "string" + ? strOrPluralConfig + : strOrPluralConfig.messages[0]; + + // We try to find the translation in the cache. + const message = getTranslationFromStore(id, getLocale()); + + // We found the translation so we can return it. + // We need to make sure that we only return the first message, in the case + // where there is a plural form for the same message. + if (message) { + return Array.isArray(message) ? message[0] : message; + } + + // Otherwise, there's no translation, so we try to do fake translation. + return fakeTranslate(id); +}; + +/** + * Get the plural translation for a given plural configuration object. + * + * @param pluralConfig the plural configuration object + * @param idx the index of the plural form to use + * @returns the translated string + */ +export const getPluralTranslation = ( + pluralConfig: PluralConfigurationObject, + idx: number, +) => { + const {lang, messages} = pluralConfig; + + // We try to find the translation in the cache. + const translatedMessages = getTranslationFromStore(messages[0], lang); + + // We found the translation so we can return the right plural form. + if (translatedMessages) { + if (!Array.isArray(translatedMessages)) { + // NOTE(john): This should never happen, but we should handle it + // just in case. + return translatedMessages; + } + return translatedMessages[idx]; + } + + // Otherwise, there's no translation, so we try to do fake translation. + return fakeTranslate(messages[idx]); +}; + +/** + * Load locale data into the cache. + * + * @param locale the locale to load data for + * @param data the id-message pairs to load + */ +export const loadTranslations = ( + locale: string, + data: Record>, +) => { + const messageCache = localeMessageStore.get(locale); + localeMessageStore.set(locale, {...messageCache, ...data}); +}; + +/** + * Clear locale data from the cache. + * + * @param locale the locale to clear data for + */ +export const clearTranslations = (locale: string) => { + localeMessageStore.delete(locale); +}; diff --git a/packages/wonder-blocks-i18n/src/functions/i18n.ts b/packages/wonder-blocks-i18n/src/functions/i18n.ts index 77b45a7a1..cd3899ea0 100644 --- a/packages/wonder-blocks-i18n/src/functions/i18n.ts +++ b/packages/wonder-blocks-i18n/src/functions/i18n.ts @@ -3,9 +3,10 @@ /* To fix, remove an entry above, run ka-lint, and fix errors. */ import * as React from "react"; -import FakeTranslate from "./i18n-faketranslate"; import {allPluralForms} from "./plural-forms"; import {getLocale} from "./locale"; +import {PluralConfigurationObject} from "./types"; +import {getPluralTranslation, getSingularTranslation} from "./i18n-store"; type InterpolationOptions = { [key: string]: T; @@ -15,11 +16,6 @@ type NGetOptions = { [key: string]: any; }; -type PluralConfigurationObject = { - lang: string; - messages: Array; -}; - interface ngettextOverloads { ( config: PluralConfigurationObject, @@ -51,30 +47,10 @@ interface _Overloads { ): string; } -interface internalTranslateOverloads { - ( - str: string, - options: - | InterpolationOptions - | null - | undefined, - additionalTranslation: (arg1: string) => string, - ): string; - ( - pluralConfig: PluralConfigurationObject, - options: - | InterpolationOptions - | null - | undefined, - additionalTranslation: (arg1: string) => string, - ): string; -} - -const {translate: fakeTranslate} = new FakeTranslate(); - type Language = keyof typeof allPluralForms; const interpolationMarker = /%\(([\w_]+)\)s/g; + /** * Performs sprintf-like %(name)s replacement on str, and returns a React * fragment of the string interleaved with those replacements. The replacements @@ -95,7 +71,7 @@ const interpolateStringToFragment = function ( options = options || {}; // Split the string into its language fragments and substitutions - const split = fakeTranslate(str).split(interpolationMarker); + const split = getSingularTranslation(str).split(interpolationMarker); const result: { [key: string]: React.ReactNode; @@ -149,32 +125,22 @@ const interpolateStringToFragment = function ( }; /** - * This is the real worker for handling substitution and fake translation. + * Handle string interpolation (for plain strings, not React fragments). */ -const internalTranslate: internalTranslateOverloads = ( - strOrPluralConfig, - options, - additionalTranslation: (arg1: string) => string, +const interpolateString = ( + message: string, + options: + | InterpolationOptions + | null + | undefined, ) => { - // Sometimes we're given an argument that's meant for ngettext(). This - // happens if the same string is used in both i18n._() and i18n.ngettext() - // (.g. a = i18n._(foo); b = i18n.ngettext("foo", "bar", count); - // In such cases, only the plural form ends up in the .po file, and - // then it gets sent to us for the i18n._() case too. No problem, though: - // we'll just take the singular arg. - if (typeof strOrPluralConfig === "object" && strOrPluralConfig.messages) { - strOrPluralConfig = strOrPluralConfig.messages[0]; - } - - // @ts-expect-error [FEI-5019] - TS2345 - Argument of type 'string | PluralConfigurationObject' is not assignable to parameter of type 'string'. - const translated = additionalTranslation(strOrPluralConfig); // Options are optional, if we don't have any, just return the string. if (options == null) { - return translated; + return message; } // Otherwise, let's see if our string has anything to be replaced. - return translated.replace(interpolationMarker, (match, key) => { + return message.replace(interpolationMarker, (match, key) => { const replaceWith = options[key]; return replaceWith != null ? String(replaceWith) : match; }); @@ -186,9 +152,10 @@ const internalTranslate: internalTranslateOverloads = ( * i18n._("Some string") * i18n._("Hello %(name)s", {name: "John"}) */ -export const _: _Overloads = (strOrPluralConfig, options) => - // @ts-expect-error [FEI-5019] - TS2769 - No overload matches this call. - internalTranslate(strOrPluralConfig, options, fakeTranslate); +export const _: _Overloads = (strOrPluralConfig, options) => { + const message = getSingularTranslation(strOrPluralConfig); + return interpolateString(message, options); +}; /** * i18n method that supports sprintf-like %(name)s replacement for React nodes @@ -239,7 +206,7 @@ export const ngettext: ngettextOverloads = ( num?: number | null | undefined | NGetOptions, options?: NGetOptions, ) => { - const {messages, lang}: PluralConfigurationObject = + const pluralConfObj: PluralConfigurationObject = typeof singular === "object" ? singular : { @@ -254,17 +221,17 @@ export const ngettext: ngettextOverloads = ( (typeof singular === "object" ? num : (options as any)) || {}; // Get the translated string - const idx = ngetpos(actualNum, lang); + const idx = ngetpos(actualNum, pluralConfObj.lang); // The common (non-error) case is messages[idx]. - const translation = idx < messages.length ? messages[idx] : ""; + const translation = getPluralTranslation(pluralConfObj, idx); // Get the options to substitute into the string. // We automatically add in the 'magic' option-variable 'num'. actualOptions.num = formatNumber(actualNum); - // Then pass into i18n._ for the actual substitution - return _(translation, actualOptions); + // Then do the actual substitution + return interpolateString(translation, actualOptions); }; /** @@ -300,9 +267,13 @@ export const ngetpos = function (num: number, lang?: Language): number { * they shouldn't complain that this text isn't translated.) * Use it like so: 'tag.author = i18n.doNotTranslate("Jim");' */ -export const doNotTranslate: _Overloads = (s, o) => - // @ts-expect-error [FEI-5019] - TS2769 - No overload matches this call. - internalTranslate(s, o, (t) => t); +export const doNotTranslate: _Overloads = (strOrPluralConfig, options) => { + const id = + typeof strOrPluralConfig === "string" + ? strOrPluralConfig + : strOrPluralConfig.messages[0]; + return interpolateString(id, options); +}; /* * A dummy identity function, like i18n.doNotTranslate. It's used to diff --git a/packages/wonder-blocks-i18n/src/functions/types.ts b/packages/wonder-blocks-i18n/src/functions/types.ts index 29c90242e..a30acfb0e 100644 --- a/packages/wonder-blocks-i18n/src/functions/types.ts +++ b/packages/wonder-blocks-i18n/src/functions/types.ts @@ -14,3 +14,8 @@ export interface IProvideTranslation { */ translate(input: string): string; } + +export type PluralConfigurationObject = { + lang: string; + messages: Array; +}; diff --git a/packages/wonder-blocks-i18n/src/index.ts b/packages/wonder-blocks-i18n/src/index.ts index 7fc6604ad..2ef40d750 100644 --- a/packages/wonder-blocks-i18n/src/index.ts +++ b/packages/wonder-blocks-i18n/src/index.ts @@ -6,6 +6,7 @@ export { doNotTranslateYet, // used by handlebars translation functions in webapp ngetpos, } from "./functions/i18n"; +export {loadTranslations, clearTranslations} from "./functions/i18n-store"; export {localeToFixed, getDecimalSeparator} from "./functions/l10n"; export {getLocale, setLocale} from "./functions/locale";