feat: Add pseudolocalization (#309)

docs/tutorials/cli.rst

@@ -193,6 +193,48 @@ If you use natural language for message IDs (that's the default),
 set :conf:`sourceLocale`. You shouldn't use this config if you're using custom
 IDs (e.g: ``Component.title``).
 
+Pseudolocalization
+=========================
+
+There is built in support for `pseudolocalization <https://en.wikipedia.org/wiki/Pseudolocalization>`. 
+Pseudolocalization is a method for testing the internationalization aspects 
+of your application by replacing your strings with altered versions 
+and maintaining string readability. It also makes hard coded strings 
+and improperly concatenated strings easy to spot so that they can be properly localized.
+
+  Example:
+  Ţĥĩś ţēxţ ĩś ƥśēũďōĺōćàĺĩźēď
+
+To setup pseudolocalization add :conf:`pseudoLocale` in ``package.json``::
+
+{
+   "lingui": {
+      "pseudoLocale": "pseudo-LOCALE"
+   }
+}
+
+:conf:`pseudoLocale` option can be any string 
+examples: :conf:`en-PL`, :conf:`pseudo-LOCALE`, :conf:`pseudolocalization` or :conf:`en-UK`
+
+PseudoLocale folder is automatically created based on configuration when running 
+ ``lingui extract`` command. Pseudolocalized text is created on  ``lingui compile`` command.
+The pseudolocalization is automatically created from default messages. 
+It can also be changed by setting translation in :conf:`message.json` into non-pseudolocalized text.
+
+How to switch your browser into specified pseudoLocale
+We can use browsers settings or extensions. Extensions allow to use any locale.
+Browsers are usually limited into valid language tags (BCP 47). 
+In that case, the locale for pseudolocalization has to be standard locale,
+which is not used in your application for example :conf:`zu_ZA` Zulu - SOUTH AFRICA
+
+Chrome:
+a) With extension (any string) - https://chrome.google.com/webstore/detail/quick-language-switcher/pmjbhfmaphnpbehdanbjphdcniaelfie
+b) Without extension - chrome://settings/?search=languages
+
+Firefox:
+a) With extension (any string) - https://addons.mozilla.org/en-GB/firefox/addon/quick-accept-language-switc/?src=search
+b) Without extension - about:preferences#general > Language
+
 Catalogs in VCS and CI
 ======================
 

packages/cli/package.json

@@ -54,8 +54,10 @@
     "make-plural": "^4.1.1",
     "messageformat-parser": "^2.0.0",
     "mkdirp": "^0.5.1",
+    "opencollective": "^1.0.3",
     "ora": "^3.0.0",
     "pofile": "^1.0.11",
+    "pseudolocale": "^1.1.0",
     "ramda": "^0.25.0",
     "typescript": "^2.9.2"
   },

packages/cli/src/api/catalog.js

@@ -155,7 +155,7 @@ export default (config: LinguiConfig): CatalogApi => {
     },
 
     addLocale(locale) {
-      if (!locales.isValid(locale)) {
+      if (!locales.isValid(locale) && locale !== config.pseudoLocale) {
         return [false, null]
       }
 

packages/cli/src/api/catalog.test.js

@@ -38,8 +38,24 @@ describe("Catalog API", function() {
       ])
     })
 
+    it("should add pseudoLocale", () => {
+      const config = createConfig({
+        pseudoLocale: "pseudo-LOCALE"
+      })
+      const catalog = configureCatalog(config)
+
+      expect(catalog.addLocale("pseudo-LOCALE")).toEqual([
+        true,
+        expect.stringMatching(
+          escapeRegExp(path.join("pseudo-LOCALE", "messages.json")) + "$"
+        )
+      ])
+    })
+
     it("shouldn't add invalid locale", function() {
-      const config = createConfig()
+      const config = createConfig({
+        pseudoLocale: "pseudo-LOCALE"
+      })
       const catalog = configureCatalog(config)
       expect(catalog.addLocale("xyz")).toEqual([false, null])
     })

packages/cli/src/api/compile.js

@@ -7,6 +7,7 @@ import plurals from "make-plural"
 import R from "ramda"
 
 import type { CatalogType } from "./types"
+import pseudoLocalize from "./pseudoLocalize"
 
 const isString = s => typeof s === "string"
 
@@ -126,13 +127,17 @@ export function createCompiledCatalog(
   locale: string,
   messages: CatalogType,
   strict: boolean = false,
-  namespace: string = "cjs"
+  namespace: string = "cjs",
+  pseudoLocale: string
 ) {
   const [language] = locale.split(/[_-]/)
   const pluralRules = plurals[language]
 
   const compiledMessages = R.keys(messages).map(key => {
-    const translation = messages[key] || (!strict ? key : "")
+    let translation = messages[key] || (!strict ? key : "")
+    if (locale === pseudoLocale) {
+      translation = pseudoLocalize(translation)
+    }
     return t.objectProperty(t.stringLiteral(key), compile(translation))
   })
 

packages/cli/src/api/pseudoLocalize.js

@@ -0,0 +1,63 @@
+// @flow
+
+import R from "ramda"
+import pseudolocale from "pseudolocale"
+
+const delimiter = "%&&&%"
+
+pseudolocale.option.delimiter = delimiter
+// We do not want prepending and appending because of Plurals structure
+pseudolocale.option.prepend = ""
+pseudolocale.option.append = ""
+
+/*
+Regex should match HTML tags
+It was taken from https://haacked.com/archive/2004/10/25/usingregularexpressionstomatchhtml.aspx/
+Example: https://regex101.com/r/bDHD9z/3
+*/
+const HTMLRegex = /<\/?\w+((\s+\w+(\s*=\s*(?:".*?"|'.*?'|[^'">\s]+))?)+\s*|\s*)\/?>/g
+/*
+Regex should match js-lingui plurals
+Example: https://regex101.com/r/zXWiQR/3
+*/
+const PluralRegex = /{\w*,\s*plural,\s*\w*\s*{|}\s*\w*\s*({|})/g
+/*
+Regex should match js-lingui variables
+Example: https://regex101.com/r/kD7K2b/1
+*/
+const VariableRegex = /({|})/g
+
+function addDelimitersHTMLTags(message) {
+  return message.replace(HTMLRegex, matchedString => {
+    return `${delimiter}${matchedString}${delimiter}`
+  })
+}
+
+function addDelimitersPlural(message) {
+  return message.replace(PluralRegex, matchedString => {
+    return `${delimiter}${matchedString}${delimiter}`
+  })
+}
+
+function addDelimitersVariables(message) {
+  return message.replace(VariableRegex, matchedString => {
+    return `${delimiter}${matchedString}${delimiter}`
+  })
+}
+
+const addDelimiters = R.compose(
+  addDelimitersVariables,
+  addDelimitersPlural,
+  addDelimitersHTMLTags
+)
+
+function removeDelimiters(message) {
+  return message.replace(new RegExp(delimiter, "g"), "")
+}
+
+export default function(message: string) {
+  message = addDelimiters(message)
+  message = pseudolocale.str(message)
+
+  return removeDelimiters(message)
+}

packages/cli/src/api/pseudoLocalize.test.js

@@ -0,0 +1,38 @@
+import pseudoLocalize from "./pseudoLocalize"
+
+describe("PseudoLocalization", () => {
+  it("should pseudolocalize strings", () => {
+    expect(pseudoLocalize("Martin Černý")).toEqual("Màŕţĩń Čēŕńý")
+  })
+
+  it("should not pseudolocalize HTML tags", () => {
+    expect(pseudoLocalize('Martin <span id="spanId">Černý</span>')).toEqual(
+      'Màŕţĩń <span id="spanId">Čēŕńý</span>'
+    )
+    expect(
+      pseudoLocalize("Martin Cerny  123a<span id='id'>Černý</span>")
+    ).toEqual("Màŕţĩń Ćēŕńŷ  123à<span id='id'>Čēŕńý</span>")
+    expect(pseudoLocalize("Martin <a title='>>'>a</a>")).toEqual(
+      "Màŕţĩń <a title='>>'>à</a>"
+    )
+    expect(pseudoLocalize("<a title=TITLE>text</a>")).toEqual(
+      "<a title=TITLE>ţēxţ</a>"
+    )
+  })
+
+  it("should pseudlocalize plurals with HTML tags", () => {
+    expect(
+      pseudoLocalize(
+        "{messagesCount, plural, zero {There's # <span>message</span>} other {There're # messages}"
+      )
+    ).toEqual(
+      "{messagesCount, plural, zero {Ţĥēŕē'ś # <span>mēśśàĝē</span>} other {Ţĥēŕē'ŕē # mēśśàĝēś}"
+    )
+  })
+
+  it("should pseudolocalize plurals", () => {
+    expect(
+      pseudoLocalize("{value, plural, one {# book} other {# books}}")
+    ).toEqual("{value, plural, one {# ƀōōķ} other {# ƀōōķś}}")
+  })
+})

packages/cli/src/api/types.js

@@ -5,6 +5,7 @@ export type LinguiConfig = {|
   localeDir: string,
   sourceLocale: string,
   fallbackLocale: string,
+  pseudoLocale: string,
   srcPathDirs: Array<string>,
   srcPathIgnorePatterns: Array<string>,
   format: "lingui" | "minimal" | "po"

packages/cli/src/lingui-compile.js

@@ -91,7 +91,8 @@ function command(config, options) {
       locale,
       messages,
       false,
-      options.namespace || config.compileNamespace
+      options.namespace || config.compileNamespace,
+      config.pseudoLocale
     )
     const compiledPath = catalog.writeCompiled(locale, compiledCatalog)
     if (options.typescript) {

packages/cli/src/lingui-extract.js

@@ -34,6 +34,11 @@ export default function command(
   }
 
   const catalog = configureCatalog(config)
+  const pseudoLocale = config.pseudoLocale
+  if (pseudoLocale) {
+    catalog.addLocale(pseudoLocale)
+  }
+
   const locales = catalog.getLocales()
 
   if (!locales.length) {

packages/cli/src/lingui-extract.test.js

@@ -1,9 +1,18 @@
-// @flow
+import fs from "fs"
 import { mockConsole, mockConfig } from "./mocks"
 import command from "./lingui-extract"
+import configureCatalog from "./api/catalog"
+import { detect } from "./api/detect"
+
+import { extract, collect, cleanObsolete, order } from "./api/extract"
+
+jest.mock("fs")
+jest.mock("./api/catalog")
+jest.mock("./api/extract")
+jest.mock("./api/detect")
 
 describe("lingui extract", function() {
-  function mockExtractOptions(options?: Object = {}) {
+  function mockExtractOptions(options) {
     return {
       verbose: false,
       clean: false,
@@ -12,10 +21,25 @@ describe("lingui extract", function() {
       ...options
     }
   }
+
+  beforeEach(() => {
+    detect.mockClear()
+    extract.mockClear()
+    collect.mockClear()
+    cleanObsolete.mockClear()
+    order.mockClear()
+  })
+
   it("should exit when there aren't any locales", function() {
     const config = mockConfig()
     const options = mockExtractOptions()
 
+    configureCatalog.mockImplementation(() => {
+      return {
+        getLocales: jest.fn().mockReturnValue([])
+      }
+    })
+
     mockConsole(console => {
       command(config, options)
       expect(console.log).toBeCalledWith(
@@ -26,4 +50,28 @@ describe("lingui extract", function() {
       )
     })
   })
+  it("should add pseudoLocale when defined", () => {
+    const config = mockConfig({
+      pseudoLocale: "pseudo-LOCALE"
+    })
+    const options = mockExtractOptions()
+
+    const addLocale = jest.fn()
+    order.mockImplementation(() => ["pseudo-LOCALE"])
+    configureCatalog.mockImplementation(() => {
+      return {
+        addLocale: addLocale,
+        getLocales: jest.fn().mockReturnValue(["pseudo-LOCALE"]),
+        readAll: jest.fn(),
+        merge: jest.fn(),
+        write: jest.fn().mockReturnValue([true, "messages.json"])
+      }
+    })
+
+    mockConsole(console => {
+      command(config, options)
+    })
+
+    expect(addLocale).toBeCalledWith("pseudo-LOCALE")
+  })
 })

packages/cli/test/fixtures/extract/App.js

@@ -1,5 +1,5 @@
 import * as React from "react"
-import { Trans } from "@lingui/react"
+import { Trans, Plural } from "@lingui/react"
 
 class App extends React.Component {
   render() {
@@ -17,6 +17,15 @@ class App extends React.Component {
         </p>
 
         <Trans>Value of {value}</Trans>
+
+        <p>
+          <Plural
+            value={messagesCount}
+            zero="There're no messages"
+            one="There's # message <span>in</span> your inbox"
+            other="There're # messages in your inbox"
+          />
+        </p>
       </div>
     )
   }

packages/cli/test/fixtures/extract/package.json

@@ -1,7 +1,8 @@
 {
   "lingui": {
-    "format": "po",
+    "format": "lingui",
     "localeDir": "./locale",
-    "sourceLocale": "en"
+    "sourceLocale": "en",
+    "pseudoLocale": "en-PT"
   }
 }

packages/conf/src/index.js

@@ -29,6 +29,7 @@ export const defaultConfig = {
   localeDir: "./locale",
   sourceLocale: "",
   fallbackLocale: "",
+  pseudoLocale: "",
   srcPathDirs: ["<rootDir>"],
   srcPathIgnorePatterns: [NODE_MODULES],
   format: "lingui",

packages/conf/src/index.test.js

@@ -7,6 +7,7 @@ describe("lingui-conf", function() {
     expect(config.localeDir).toBeDefined()
     expect(config.sourceLocale).toBeDefined()
     expect(config.fallbackLocale).toBeDefined()
+    expect(config.pseudoLocale).toBeDefined()
     expect(config.srcPathDirs).toBeDefined()
     expect(config.srcPathIgnorePatterns).toBeDefined()
     expect(config.extractBabelOptions).toBeDefined()