From 8561e69648f879e0579565feac6ea54410f6ab1d Mon Sep 17 00:00:00 2001 From: Kevin Gimbel Date: Sun, 15 Oct 2017 12:35:25 +0200 Subject: [PATCH 1/6] i18n: Allow custom language codes Use the new `RegisterPluralSpec` function to register all defined languages. This allows the usage of language identifiers which are not part of the Unicode CLDR standard. See #3564 --- i18n/translationProvider.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/i18n/translationProvider.go b/i18n/translationProvider.go index 9947d3ce521..1a1cba4c80e 100644 --- a/i18n/translationProvider.go +++ b/i18n/translationProvider.go @@ -19,6 +19,7 @@ import ( "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/source" "github.com/nicksnyder/go-i18n/i18n/bundle" + "github.com/nicksnyder/go-i18n/i18n/language" ) // TranslationProvider provides translation handling, i.e. loading @@ -39,6 +40,7 @@ func (tp *TranslationProvider) Update(d *deps.Deps) error { sources := []source.Input{sp.NewFilesystem(dir)} themeI18nDir, err := d.PathSpec.GetThemeI18nDirPath() + ps := &language.PluralSpec{} if err == nil { sources = []source.Input{sp.NewFilesystem(themeI18nDir), sources[0]} @@ -50,6 +52,7 @@ func (tp *TranslationProvider) Update(d *deps.Deps) error { for _, currentSource := range sources { for _, r := range currentSource.Files() { + language.RegisterPluralSpec([]string{r.BaseFileName()}, ps) err := i18nBundle.ParseTranslationFileBytes(r.LogicalName(), r.Bytes()) if err != nil { return fmt.Errorf("Failed to load translations in file %q: %s", r.LogicalName(), err) From 1742fe9ded5418901629d760929434d5eafb9d06 Mon Sep 17 00:00:00 2001 From: Kevin Gimbel Date: Wed, 18 Oct 2017 16:01:31 +0200 Subject: [PATCH 2/6] i18n: Add test for custom language code --- i18n/i18n_test.go | 21 +++++++++++++++++++++ i18n/translationProvider.go | 6 +++--- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/i18n/i18n_test.go b/i18n/i18n_test.go index 6a9d362b01f..71b4aba4556 100644 --- a/i18n/i18n_test.go +++ b/i18n/i18n_test.go @@ -23,9 +23,11 @@ import ( "github.com/gohugoio/hugo/config" "github.com/nicksnyder/go-i18n/i18n/bundle" + "github.com/nicksnyder/go-i18n/i18n/language" jww "github.com/spf13/jwalterweatherman" "github.com/spf13/viper" "github.com/stretchr/testify/require" + "strings" ) var logger = jww.NewNotepad(jww.LevelError, jww.LevelError, os.Stdout, ioutil.Discard, "", log.Ldate|log.Ltime) @@ -137,10 +139,29 @@ var i18nTests = []i18nTest{ expected: "hello", expectedFlag: "[i18n] hello", }, + // Non Unicode CLDR language code + { + data: map[string][]byte{ + "dk.toml": []byte("[hello]\nother = \"hej\""), + }, + args: nil, + lang: "dk", + id: "hello", + expected: "hej", + expectedFlag: "hej", + }, } func doTestI18nTranslate(t *testing.T, test i18nTest, cfg config.Provider) string { i18nBundle := bundle.New() + ids := []string{} + + for file := range test.data { + id := strings.Split(file, ".toml") + ids = append(ids, id[0]) + } + + language.RegisterPluralSpec(ids, &language.PluralSpec{}) for file, content := range test.data { err := i18nBundle.ParseTranslationFileBytes(file, content) diff --git a/i18n/translationProvider.go b/i18n/translationProvider.go index 1a1cba4c80e..f39c616b010 100644 --- a/i18n/translationProvider.go +++ b/i18n/translationProvider.go @@ -19,7 +19,7 @@ import ( "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/source" "github.com/nicksnyder/go-i18n/i18n/bundle" - "github.com/nicksnyder/go-i18n/i18n/language" + //"github.com/nicksnyder/go-i18n/i18n/language" ) // TranslationProvider provides translation handling, i.e. loading @@ -40,7 +40,7 @@ func (tp *TranslationProvider) Update(d *deps.Deps) error { sources := []source.Input{sp.NewFilesystem(dir)} themeI18nDir, err := d.PathSpec.GetThemeI18nDirPath() - ps := &language.PluralSpec{} + //ps := &language.PluralSpec{} if err == nil { sources = []source.Input{sp.NewFilesystem(themeI18nDir), sources[0]} @@ -52,7 +52,7 @@ func (tp *TranslationProvider) Update(d *deps.Deps) error { for _, currentSource := range sources { for _, r := range currentSource.Files() { - language.RegisterPluralSpec([]string{r.BaseFileName()}, ps) + //language.RegisterPluralSpec([]string{r.BaseFileName()}, ps) err := i18nBundle.ParseTranslationFileBytes(r.LogicalName(), r.Bytes()) if err != nil { return fmt.Errorf("Failed to load translations in file %q: %s", r.LogicalName(), err) From 74c91345b99a638af0a05df34ce37d7891768461 Mon Sep 17 00:00:00 2001 From: Kevin Gimbel Date: Tue, 24 Oct 2017 16:18:57 +0200 Subject: [PATCH 3/6] i18n: Un-comment code --- i18n/translationProvider.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/i18n/translationProvider.go b/i18n/translationProvider.go index f39c616b010..32f700a6398 100644 --- a/i18n/translationProvider.go +++ b/i18n/translationProvider.go @@ -19,7 +19,7 @@ import ( "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/source" "github.com/nicksnyder/go-i18n/i18n/bundle" - //"github.com/nicksnyder/go-i18n/i18n/language" + "github.com/nicksnyder/go-i18n/i18n/language" ) // TranslationProvider provides translation handling, i.e. loading @@ -38,9 +38,10 @@ func (tp *TranslationProvider) Update(d *deps.Deps) error { dir := d.PathSpec.AbsPathify(d.Cfg.GetString("i18nDir")) sp := source.NewSourceSpec(d.Cfg, d.Fs) sources := []source.Input{sp.NewFilesystem(dir)} + ps := &language.PluralSpec{} + langs := []string{} themeI18nDir, err := d.PathSpec.GetThemeI18nDirPath() - //ps := &language.PluralSpec{} if err == nil { sources = []source.Input{sp.NewFilesystem(themeI18nDir), sources[0]} @@ -52,7 +53,14 @@ func (tp *TranslationProvider) Update(d *deps.Deps) error { for _, currentSource := range sources { for _, r := range currentSource.Files() { - //language.RegisterPluralSpec([]string{r.BaseFileName()}, ps) + langs = append(langs, r.BaseFileName()) + } + } + + language.RegisterPluralSpec(langs, ps) + + for _, currentSource := range sources { + for _, r := range currentSource.Files() { err := i18nBundle.ParseTranslationFileBytes(r.LogicalName(), r.Bytes()) if err != nil { return fmt.Errorf("Failed to load translations in file %q: %s", r.LogicalName(), err) From 55686ec01bd52af8c5913c6d2a5cd5bfad0378a5 Mon Sep 17 00:00:00 2001 From: Kevin Gimbel Date: Tue, 31 Oct 2017 16:33:29 +0100 Subject: [PATCH 4/6] i18n: Add comments about language specs; Move variable initialization --- i18n/i18n_test.go | 4 ++-- i18n/translationProvider.go | 8 +++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/i18n/i18n_test.go b/i18n/i18n_test.go index 71b4aba4556..9e7aed0398b 100644 --- a/i18n/i18n_test.go +++ b/i18n/i18n_test.go @@ -157,8 +157,8 @@ func doTestI18nTranslate(t *testing.T, test i18nTest, cfg config.Provider) strin ids := []string{} for file := range test.data { - id := strings.Split(file, ".toml") - ids = append(ids, id[0]) + id := strings.TrimSuffix(file, ".toml") + ids = append(ids, id) } language.RegisterPluralSpec(ids, &language.PluralSpec{}) diff --git a/i18n/translationProvider.go b/i18n/translationProvider.go index 32f700a6398..221ebb9b1fa 100644 --- a/i18n/translationProvider.go +++ b/i18n/translationProvider.go @@ -38,8 +38,6 @@ func (tp *TranslationProvider) Update(d *deps.Deps) error { dir := d.PathSpec.AbsPathify(d.Cfg.GetString("i18nDir")) sp := source.NewSourceSpec(d.Cfg, d.Fs) sources := []source.Input{sp.NewFilesystem(dir)} - ps := &language.PluralSpec{} - langs := []string{} themeI18nDir, err := d.PathSpec.GetThemeI18nDirPath() @@ -51,12 +49,16 @@ func (tp *TranslationProvider) Update(d *deps.Deps) error { i18nBundle := bundle.New() + langs := []string{} for _, currentSource := range sources { for _, r := range currentSource.Files() { langs = append(langs, r.BaseFileName()) } } - + // we need to register all language codes as "plural spec" to prevent errors + // with unknown language codes + // see https://github.com/gohugoio/hugo/issues/3564 + ps := &language.PluralSpec{} language.RegisterPluralSpec(langs, ps) for _, currentSource := range sources { From 79343bf327ec2621fc2e137d80029c1737312ca7 Mon Sep 17 00:00:00 2001 From: Kevin Gimbel Date: Wed, 1 Nov 2017 09:38:26 +0100 Subject: [PATCH 5/6] i18n: Capitalize first letter in comment and add dot at the end --- i18n/translationProvider.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/i18n/translationProvider.go b/i18n/translationProvider.go index 221ebb9b1fa..f262b2cc97c 100644 --- a/i18n/translationProvider.go +++ b/i18n/translationProvider.go @@ -55,8 +55,7 @@ func (tp *TranslationProvider) Update(d *deps.Deps) error { langs = append(langs, r.BaseFileName()) } } - // we need to register all language codes as "plural spec" to prevent errors - // with unknown language codes + // We need to register all language codes as "plural spec" to prevent errors with unknown language codes. // see https://github.com/gohugoio/hugo/issues/3564 ps := &language.PluralSpec{} language.RegisterPluralSpec(langs, ps) From 7803972cdb61f7937feff084d7cb31f621735445 Mon Sep 17 00:00:00 2001 From: Kevin Gimbel Date: Thu, 2 Nov 2017 16:09:17 +0100 Subject: [PATCH 6/6] i18n: Add sync to prevent data race --- i18n/translationProvider.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/i18n/translationProvider.go b/i18n/translationProvider.go index f262b2cc97c..663a8dd5e0d 100644 --- a/i18n/translationProvider.go +++ b/i18n/translationProvider.go @@ -15,6 +15,7 @@ package i18n import ( "fmt" + "sync" "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/source" @@ -22,6 +23,10 @@ import ( "github.com/nicksnyder/go-i18n/i18n/language" ) +// Unfortunately this needs to be global, see +// https://github.com/nicksnyder/go-i18n/issues/82 +var tpMu sync.Mutex + // TranslationProvider provides translation handling, i.e. loading // of bundles etc. type TranslationProvider struct { @@ -35,6 +40,9 @@ func NewTranslationProvider() *TranslationProvider { // Update updates the i18n func in the provided Deps. func (tp *TranslationProvider) Update(d *deps.Deps) error { + tpMu.Lock() + defer tpMu.Unlock() + dir := d.PathSpec.AbsPathify(d.Cfg.GetString("i18nDir")) sp := source.NewSourceSpec(d.Cfg, d.Fs) sources := []source.Input{sp.NewFilesystem(dir)}