From 06e459735d93d561f2bd112c730562546782d22d Mon Sep 17 00:00:00 2001 From: Joshua Hawxwell Date: Tue, 16 Jan 2024 13:53:00 +0000 Subject: [PATCH] Escape data passed to templated translation messages --- .../cloud_watch_event_handler.go | 2 +- cmd/mlpa/main.go | 5 +- internal/localize/localizer.go | 189 ++++++++++++++++-- internal/localize/localizer_test.go | 26 ++- internal/localize/testdata/bad/cy.json | 5 + internal/localize/testdata/bad/en.json | 5 + internal/localize/testdata/zz.json | 1 + internal/page/common.go | 12 +- internal/templatefn/fn_test.go | 45 +++-- 9 files changed, 242 insertions(+), 48 deletions(-) create mode 100644 internal/localize/testdata/bad/cy.json create mode 100644 internal/localize/testdata/bad/en.json create mode 100644 internal/localize/testdata/zz.json diff --git a/cmd/event-received/cloud_watch_event_handler.go b/cmd/event-received/cloud_watch_event_handler.go index 8f146f4a4b..47c97a20ac 100644 --- a/cmd/event-received/cloud_watch_event_handler.go +++ b/cmd/event-received/cloud_watch_event_handler.go @@ -45,7 +45,7 @@ func (h *cloudWatchEventHandler) Handle(ctx context.Context, event events.CloudW return handleEvidenceReceived(ctx, h.dynamoClient, event) case "reduced-fee-approved": - bundle := localize.NewBundle("./lang/en.json", "./lang/cy.json") + bundle, _ := localize.NewBundle("./lang/en.json", "./lang/cy.json") //TODO do this in handleFeeApproved when/if we save lang preference in LPA appData := page.AppData{Localizer: bundle.For(localize.En)} diff --git a/cmd/mlpa/main.go b/cmd/mlpa/main.go index 17398022d1..335efdb7b8 100644 --- a/cmd/mlpa/main.go +++ b/cmd/mlpa/main.go @@ -117,7 +117,10 @@ func main() { logger.Fatal(err) } - bundle := localize.NewBundle("lang/en.json", "lang/cy.json") + bundle, err := localize.NewBundle("lang/en.json", "lang/cy.json") + if err != nil { + logger.Fatal(err) + } cfg, err := config.LoadDefaultConfig(ctx) if err != nil { diff --git a/internal/localize/localizer.go b/internal/localize/localizer.go index 9d75776b23..8229aec700 100644 --- a/internal/localize/localizer.go +++ b/internal/localize/localizer.go @@ -1,50 +1,179 @@ package localize import ( + "bytes" "encoding/json" + "errors" "fmt" + "html/template" + "os" + "path" "strings" "time" "unicode" "unicode/utf8" "github.com/ministryofjustice/opg-modernising-lpa/internal/date" - "github.com/nicksnyder/go-i18n/v2/i18n" - "golang.org/x/text/language" ) +type Message struct { + S string + + // when plural + One string + Other string + + // for Welsh only + Two string + Few string + Many string +} + +func (m *Message) UnmarshalJSON(text []byte) error { + var s string + if err := json.Unmarshal(text, &s); err == nil { + m.S = s + return nil + } + + var v map[string]string + if err := json.Unmarshal(text, &v); err == nil { + m.One = v["one"] + m.Other = v["other"] + m.Two = v["two"] + m.Few = v["few"] + m.Many = v["many"] + return nil + } + + return errors.New("message malformed") +} + +type Messages map[string]Message + +func (m Messages) Find(key string) (string, bool) { + if msg, ok := m[key]; ok { + return msg.S, true + } + + return "", false +} + +func (m Messages) FindPlural(key string, count int) (string, bool) { + msg, ok := m[key] + if !ok { + return "", false + } + + if count == 1 { + return msg.One, true + } + + if count == 2 && msg.Two != "" { + return msg.Two, true + } + + if count == 3 && msg.Few != "" { + return msg.Few, true + } + + if count == 6 && msg.Many != "" { + return msg.Many, true + } + + return msg.Other, true +} + type Bundle struct { - *i18n.Bundle + messages map[string]Messages } -func NewBundle(paths ...string) Bundle { - bundle := i18n.NewBundle(language.English) - bundle.RegisterUnmarshalFunc("json", json.Unmarshal) +func NewBundle(paths ...string) (*Bundle, error) { + bundle := &Bundle{messages: map[string]Messages{}} + for _, path := range paths { - bundle.LoadMessageFile(path) + if err := bundle.LoadMessageFile(path); err != nil { + return nil, err + } } - return Bundle{bundle} + return bundle, nil } -func (b Bundle) For(lang Lang) *Localizer { - return &Localizer{ - i18n.NewLocalizer(b.Bundle, lang.String()), - false, - lang, +func (b *Bundle) LoadMessageFile(p string) error { + data, err := os.ReadFile(p) + if err != nil { + return err + } + + var v map[string]Message + if err := json.Unmarshal(data, &v); err != nil { + return err } + + lang, _ := strings.CutSuffix(path.Base(p), ".json") + + if lang == "en" { + if err := verifyEn(v); err != nil { + return err + } + } else if lang == "cy" { + if err := verifyCy(v); err != nil { + return err + } + } else { + return errors.New("only supports en or cy") + } + + b.messages[lang] = v + return nil +} + +func verifyEn(v map[string]Message) error { + for key, message := range v { + if message.S != "" { + continue + } + + if message.One != "" && message.Other != "" && message.Two == "" && message.Few == "" && message.Many == "" { + continue + } + + return fmt.Errorf("problem with key: %s", key) + } + + return nil +} + +func verifyCy(v map[string]Message) error { + for key, message := range v { + if message.S != "" { + continue + } + + if message.One != "" && message.Other != "" && message.Two != "" && message.Few != "" && message.Many != "" { + continue + } + + return fmt.Errorf("problem with key: %s", key) + } + + return nil +} + +func (b *Bundle) For(lang Lang) *Localizer { + return &Localizer{b.messages[lang.String()], false, lang} } type Localizer struct { - *i18n.Localizer + messages Messages showTranslationKeys bool Lang Lang } -func (l Localizer) T(messageID string) string { - msg, err := l.Localize(&i18n.LocalizeConfig{MessageID: messageID}) - - if err != nil { +func (l *Localizer) T(messageID string) string { + msg, ok := l.messages.Find(messageID) + if !ok { return l.translate(messageID, messageID) } @@ -52,16 +181,32 @@ func (l Localizer) T(messageID string) string { } func (l Localizer) Format(messageID string, data map[string]interface{}) string { - return l.translate(l.MustLocalize(&i18n.LocalizeConfig{MessageID: messageID, TemplateData: data}), messageID) + msg, ok := l.messages.Find(messageID) + if !ok { + return l.translate(messageID, messageID) + } + + return l.translateFormat(msg, messageID, data) } func (l Localizer) Count(messageID string, count int) string { - return l.translate(l.MustLocalize(&i18n.LocalizeConfig{MessageID: messageID, PluralCount: count}), messageID) + return l.FormatCount(messageID, count, map[string]any{}) } -func (l Localizer) FormatCount(messageID string, count int, data map[string]interface{}) string { +func (l Localizer) FormatCount(messageID string, count int, data map[string]any) string { + msg, ok := l.messages.FindPlural(messageID, count) + if !ok { + return l.translate(messageID, messageID) + } + data["PluralCount"] = count - return l.translate(l.MustLocalize(&i18n.LocalizeConfig{MessageID: messageID, PluralCount: count, TemplateData: data}), messageID) + return l.translateFormat(msg, messageID, data) +} + +func (l Localizer) translateFormat(msg, messageID string, data any) string { + var buf bytes.Buffer + template.Must(template.New("").Parse(msg)).Execute(&buf, data) + return l.translate(buf.String(), messageID) } func (l Localizer) translate(translation, messageID string) string { diff --git a/internal/localize/localizer_test.go b/internal/localize/localizer_test.go index 8129caf92e..53e161fbcd 100644 --- a/internal/localize/localizer_test.go +++ b/internal/localize/localizer_test.go @@ -10,7 +10,8 @@ import ( func TestNewBundle(t *testing.T) { assert := assert.New(t) - bundle := NewBundle("testdata/en.json", "testdata/cy.json") + bundle, err := NewBundle("testdata/en.json", "testdata/cy.json") + assert.Nil(err) en := bundle.For(En) assert.Equal("A", en.T("a")) @@ -20,6 +21,7 @@ func TestNewBundle(t *testing.T) { assert.Equal("1 ONE", en.Count("c", 1)) assert.Equal("2 OTHER", en.Count("c", 2)) + assert.Equal("key does not exist", en.Count("key does not exist", 3)) assert.Equal("1 ONE FORMATTED", en.FormatCount("d", 1, map[string]interface{}{"x": "FORMATTED"})) assert.Equal("2 OTHER FORMATTED", en.FormatCount("d", 2, map[string]interface{}{"x": "FORMATTED"})) @@ -46,9 +48,27 @@ func TestNewBundle(t *testing.T) { assert.Equal("7 other formatted", cy.FormatCount("d", 7, map[string]interface{}{"x": "formatted"})) } +func TestNewBundleWhenBadFormat(t *testing.T) { + _, err := NewBundle("testdata/bad/en.json") + assert.NotNil(t, err) + + _, err = NewBundle("testdata/bad/cy.json") + assert.NotNil(t, err) +} + +func TestNewBundleWhenMissingFile(t *testing.T) { + _, err := NewBundle("testdata/a.json") + assert.NotNil(t, err) +} + +func TestNewBundleWhenOtherLang(t *testing.T) { + _, err := NewBundle("testdata/zz.json") + assert.NotNil(t, err) +} + func TestNewBundleWithTransKeys(t *testing.T) { assert := assert.New(t) - bundle := NewBundle("testdata/en.json", "testdata/cy.json") + bundle, _ := NewBundle("testdata/en.json", "testdata/cy.json") en := bundle.For(En) en.SetShowTranslationKeys(true) @@ -121,7 +141,7 @@ func TestPossessive(t *testing.T) { } func TestConcat(t *testing.T) { - bundle := NewBundle("testdata/en.json", "testdata/cy.json") + bundle, _ := NewBundle("testdata/en.json", "testdata/cy.json") en := bundle.For(En) assert.Equal(t, "Bob Smith, Alice Jones, John Doe or Paul Compton", en.Concat([]string{"Bob Smith", "Alice Jones", "John Doe", "Paul Compton"}, "or")) diff --git a/internal/localize/testdata/bad/cy.json b/internal/localize/testdata/bad/cy.json new file mode 100644 index 0000000000..3d6d566967 --- /dev/null +++ b/internal/localize/testdata/bad/cy.json @@ -0,0 +1,5 @@ +{ + "x": { + "other": "other" + } +} diff --git a/internal/localize/testdata/bad/en.json b/internal/localize/testdata/bad/en.json new file mode 100644 index 0000000000..cb15ab0b99 --- /dev/null +++ b/internal/localize/testdata/bad/en.json @@ -0,0 +1,5 @@ +{ + "s": { + "one": "one" + } +} diff --git a/internal/localize/testdata/zz.json b/internal/localize/testdata/zz.json new file mode 100644 index 0000000000..0967ef424b --- /dev/null +++ b/internal/localize/testdata/zz.json @@ -0,0 +1 @@ +{} diff --git a/internal/page/common.go b/internal/page/common.go index acaa3a7579..f9d997ddf3 100644 --- a/internal/page/common.go +++ b/internal/page/common.go @@ -57,16 +57,16 @@ type Bundle interface { } type Localizer interface { - Format(string, map[string]any) string - T(string) string + Concat([]string, string) string Count(string, int) string + Format(string, map[string]any) string FormatCount(string, int, map[string]interface{}) string - ShowTranslationKeys() bool - SetShowTranslationKeys(bool) - Possessive(string) string - Concat([]string, string) string FormatDate(date.TimeOrDate) string FormatDateTime(time.Time) string + Possessive(string) string + SetShowTranslationKeys(bool) + ShowTranslationKeys() bool + T(string) string } func PostFormString(r *http.Request, name string) string { diff --git a/internal/templatefn/fn_test.go b/internal/templatefn/fn_test.go index 34f24e0f96..d690be0936 100644 --- a/internal/templatefn/fn_test.go +++ b/internal/templatefn/fn_test.go @@ -152,8 +152,9 @@ func TestContains(t *testing.T) { } func TestTr(t *testing.T) { + bundle, _ := localize.NewBundle("testdata/en.json") app := page.AppData{ - Localizer: localize.NewBundle("testdata/en.json").For(localize.En), + Localizer: bundle.For(localize.En), } assert.Equal(t, "hi", tr(app, "message-id")) @@ -161,8 +162,9 @@ func TestTr(t *testing.T) { } func TestTrFormat(t *testing.T) { + bundle, _ := localize.NewBundle("testdata/en.json") app := page.AppData{ - Localizer: localize.NewBundle("testdata/en.json").For(localize.En), + Localizer: bundle.For(localize.En), } assert.Equal(t, "hi Person", trFormat(app, "with-format", "name", "Person")) @@ -170,8 +172,9 @@ func TestTrFormat(t *testing.T) { } func TestTrHtml(t *testing.T) { + bundle, _ := localize.NewBundle("testdata/en.json") app := page.AppData{ - Localizer: localize.NewBundle("testdata/en.json").For(localize.En), + Localizer: bundle.For(localize.En), } assert.Equal(t, template.HTML("hi"), trHtml(app, "message-id")) @@ -179,25 +182,30 @@ func TestTrHtml(t *testing.T) { } func TestTrFormatHtml(t *testing.T) { + bundle, _ := localize.NewBundle("testdata/en.json") app := page.AppData{ - Localizer: localize.NewBundle("testdata/en.json").For(localize.En), + Localizer: bundle.For(localize.En), } assert.Equal(t, template.HTML("hi Person"), trFormatHtml(app, "with-format", "name", "Person")) assert.Equal(t, template.HTML(""), trFormatHtml(app, "", "name", "Person")) + + assert.Equal(t, template.HTML("hi <script>alert('hi');</script>"), trFormatHtml(app, "with-format", "name", "")) } func TestTrCount(t *testing.T) { + bundle, _ := localize.NewBundle("testdata/en.json") enApp := page.AppData{ - Localizer: localize.NewBundle("testdata/en.json").For(localize.En), + Localizer: bundle.For(localize.En), } assert.Equal(t, "hi one", trCount(enApp, "with-count", 1)) assert.Equal(t, "hi other", trCount(enApp, "with-count", 2)) assert.Equal(t, "", trCount(enApp, "", 2)) + bundle, _ = localize.NewBundle("testdata/cy.json") cyApp := page.AppData{ - Localizer: localize.NewBundle("testdata/cy.json").For(localize.Cy), + Localizer: bundle.For(localize.Cy), } assert.Equal(t, "cy one", trCount(cyApp, "with-count", 1)) @@ -211,16 +219,18 @@ func TestTrCount(t *testing.T) { } func TestTrFormatCount(t *testing.T) { + bundle, _ := localize.NewBundle("testdata/en.json") enApp := page.AppData{ - Localizer: localize.NewBundle("testdata/en.json").For(localize.En), + Localizer: bundle.For(localize.En), } assert.Equal(t, "hi 1 one Person", trFormatCount(enApp, "with-format-count", 1, "name", "Person")) assert.Equal(t, "hi 2 other Person", trFormatCount(enApp, "with-format-count", 2, "name", "Person")) assert.Equal(t, "", trFormatCount(enApp, "", 2, "name", "Person")) + bundle, _ = localize.NewBundle("testdata/cy.json") cyApp := page.AppData{ - Localizer: localize.NewBundle("testdata/cy.json").For(localize.Cy), + Localizer: bundle.For(localize.Cy), } assert.Equal(t, "cy hi 1 one Person", trFormatCount(cyApp, "with-format-count", 1, "name", "Person")) @@ -242,8 +252,9 @@ func TestAddDays(t *testing.T) { } func TestFormatDate(t *testing.T) { - appEn := page.AppData{Localizer: localize.NewBundle("testdata/en.json").For(localize.En)} - appCy := page.AppData{Localizer: localize.NewBundle("testdata/cy.json").For(localize.Cy)} + bundle, _ := localize.NewBundle("testdata/en.json", "testdata/cy.json") + appEn := page.AppData{Localizer: bundle.For(localize.En)} + appCy := page.AppData{Localizer: bundle.For(localize.Cy)} assert.Equal(t, "7 March 2020", formatDate(appEn, time.Date(2020, time.March, 7, 3, 4, 5, 6, time.UTC))) assert.Equal(t, "7 March 2020", formatDate(appEn, date.New("2020", "3", "7"))) @@ -253,8 +264,9 @@ func TestFormatDate(t *testing.T) { } func TestFormatDateTime(t *testing.T) { - appEn := page.AppData{Localizer: localize.NewBundle("testdata/en.json").For(localize.En)} - appCy := page.AppData{Localizer: localize.NewBundle("testdata/cy.json").For(localize.Cy)} + bundle, _ := localize.NewBundle("testdata/en.json", "testdata/cy.json") + appEn := page.AppData{Localizer: bundle.For(localize.En)} + appCy := page.AppData{Localizer: bundle.For(localize.Cy)} assert.Equal(t, "7 March 2020 at 3:04am", formatDateTime(appEn, time.Date(2020, time.March, 7, 3, 4, 0, 0, time.UTC))) @@ -372,16 +384,18 @@ func TestPrintStruct(t *testing.T) { } func TestPossessive(t *testing.T) { + bundle, _ := localize.NewBundle("testdata/en.json") app := page.AppData{ - Localizer: localize.NewBundle("testdata/en.json").For(localize.En), + Localizer: bundle.For(localize.En), } assert.Equal(t, "John’s", possessive(app, "John")) } func TestConcatAnd(t *testing.T) { + bundle, _ := localize.NewBundle("testdata/en.json") app := page.AppData{ - Localizer: localize.NewBundle("testdata/en.json").For(localize.En), + Localizer: bundle.For(localize.En), } assert.Equal(t, "", concatAnd(app, []string{})) @@ -391,8 +405,9 @@ func TestConcatAnd(t *testing.T) { } func TestConcatOr(t *testing.T) { + bundle, _ := localize.NewBundle("testdata/en.json") app := page.AppData{ - Localizer: localize.NewBundle("testdata/en.json").For(localize.En), + Localizer: bundle.For(localize.En), } assert.Equal(t, "", concatOr(app, []string{}))