#296: Add pseudolocalization

packages/cli/package.json

@@ -48,14 +48,15 @@
     "cli-table": "^0.3.1",
     "commander": "^2.17.1",
     "date-fns": "^1.29.0",
-    "opencollective": "^1.0.3",
-    "pofile": "^1.0.11",
     "glob": "^7.1.2",
     "inquirer": "^6.2.0",
     "make-plural": "^4.1.1",
     "messageformat-parser": "^2.0.0",
     "mkdirp": "^0.5.1",
+    "opencollective": "^1.0.3",
     "ora": "^3.0.0",
+    "pofile": "^1.0.11",
+    "pseudolocale": "^1.1.0",
     "ramda": "^0.25.0",
     "typescript": "^2.9.2"
   },

packages/cli/src/api/catalog.js

@@ -155,7 +155,7 @@ export default (config: LinguiConfig): CatalogApi => {
     },
 
     addLocale(locale) {
-      if (!locales.isValid(locale)) {
+      if (!locales.isValid(locale) && locale !== config.pseudoLocale) {
         return [false, null]
       }
 

packages/cli/src/api/catalog.test.js

@@ -38,8 +38,24 @@ describe("Catalog API", function() {
       ])
     })
 
+    it("should add pseudoLocale", () => {
+      const config = createConfig({
+        pseudoLocale: "pseudo-LOCALE"
+      })
+      const catalog = configureCatalog(config)
+
+      expect(catalog.addLocale("pseudo-LOCALE")).toEqual([
+        true,
+        expect.stringMatching(
+          escapeRegExp(path.join("pseudo-LOCALE", "messages.json")) + "$"
+        )
+      ])
+    })
+
     it("shouldn't add invalid locale", function() {
-      const config = createConfig()
+      const config = createConfig({
+        pseudoLocale: "pseudo-LOCALE"
+      })
       const catalog = configureCatalog(config)
       expect(catalog.addLocale("xyz")).toEqual([false, null])
     })

packages/cli/src/api/compile.js

@@ -7,6 +7,7 @@ import plurals from "make-plural"
 import R from "ramda"
 
 import type { CatalogType } from "./types"
+import pseudoLocalize from "./pseudoLocalize"
 
 const isString = s => typeof s === "string"
 
@@ -126,13 +127,17 @@ export function createCompiledCatalog(
   locale: string,
   messages: CatalogType,
   strict: boolean = false,
-  namespace: string = "cjs"
+  namespace: string = "cjs",
+  pseudoLocale: string
 ) {
   const [language] = locale.split(/[_-]/)
   const pluralRules = plurals[language]
 
   const compiledMessages = R.keys(messages).map(key => {
-    const translation = messages[key] || (!strict ? key : "")
+    let translation = messages[key] || (!strict ? key : "")
+    if (locale === pseudoLocale) {
+      translation = pseudoLocalize(translation)
+    }
     return t.objectProperty(t.stringLiteral(key), compile(translation))
   })
 

packages/cli/src/api/pseudoLocalize.js

@@ -0,0 +1,70 @@
+// @flow
+
+import R from "ramda"
+import pseudolocale from "pseudolocale"
+
+const delimiter = "%&&&%"
+/*
+Regex should match HTML tags
+It was taken from https://haacked.com/archive/2004/10/25/usingregularexpressionstomatchhtml.aspx/
+Example: https://regex101.com/r/bDHD9z/3
+*/
+const HTMLRegex = /<\/?\w+((\s+\w+(\s*=\s*(?:".*?"|'.*?'|[^'">\s]+))?)+\s*|\s*)\/?>/g
+/*
+Regex should match js-lingui plurals
+Example: https://regex101.com/r/zXWiQR/3
+*/
+const PluralRegex = /{\w*,\s*plural,\s*\w*\s*{|}\s*\w*\s*({|})/g
+/*
+Regex should match js-lingui variables
+Example: https://regex101.com/r/kD7K2b/1
+*/
+const VariableRegex = /({|})/g
+
+let isPseudoLocalizeOptionSet = false
+
+function addDelimitersHTMLTags(message) {
+  return message.replace(HTMLRegex, matchedString => {
+    return `${delimiter}${matchedString}${delimiter}`
+  })
+}
+
+function addDelimitersPlural(message) {
+  return message.replace(PluralRegex, matchedString => {
+    return `${delimiter}${matchedString}${delimiter}`
+  })
+}
+
+function addDelimitersVariables(message) {
+  return message.replace(VariableRegex, matchedString => {
+    return `${delimiter}${matchedString}${delimiter}`
+  })
+}
+
+const addDelimiters = R.compose(
+  addDelimitersVariables,
+  addDelimitersPlural,
+  addDelimitersHTMLTags
+)
+
+function removeDelimiters(message) {
+  return message.replace(new RegExp(delimiter, "g"), "")
+}
+
+export function setPseudoLocalizeOption() {
+  pseudolocale.option.delimiter = delimiter
+  // We do not want prepending and appending because of Plurals structure
+  pseudolocale.option.prepend = ""
+  pseudolocale.option.append = ""
+  isPseudoLocalizeOptionSet = true
+}
+
+export default function(message: string) {
+  if (!isPseudoLocalizeOptionSet) {
+    setPseudoLocalizeOption()
+  }
+  message = addDelimiters(message)
+  message = pseudolocale.str(message)
+
+  return removeDelimiters(message)
+}

packages/cli/src/api/pseudoLocalize.test.js

@@ -0,0 +1,38 @@
+import pseudoLocalize from "./pseudoLocalize"
+
+describe("PseudoLocalization", () => {
+  it("should pseudolocalize strings", () => {
+    expect(pseudoLocalize("Martin Černý")).toEqual("Màŕţĩń Čēŕńý")
+  })
+
+  it("should not pseudolocalize HTML tags", () => {
+    expect(pseudoLocalize('Martin <span id="spanId">Černý</span>')).toEqual(
+      'Màŕţĩń <span id="spanId">Čēŕńý</span>'
+    )
+    expect(
+      pseudoLocalize("Martin Cerny  123a<span id='id'>Černý</span>")
+    ).toEqual("Màŕţĩń Ćēŕńŷ  123à<span id='id'>Čēŕńý</span>")
+    expect(pseudoLocalize("Martin <a title='>>'>a</a>")).toEqual(
+      "Màŕţĩń <a title='>>'>à</a>"
+    )
+    expect(pseudoLocalize("<a title=TITLE>text</a>")).toEqual(
+      "<a title=TITLE>ţēxţ</a>"
+    )
+  })
+
+  it("should pseudlocalize plurals with HTML tags", () => {
+    expect(
+      pseudoLocalize(
+        "{messagesCount, plural, zero {There's # <span>message</span>} other {There're # messages}"
+      )
+    ).toEqual(
+      "{messagesCount, plural, zero {Ţĥēŕē'ś # <span>mēśśàĝē</span>} other {Ţĥēŕē'ŕē # mēśśàĝēś}"
+    )
+  })
+
+  it("should pseudolocalize plurals", () => {
+    expect(
+      pseudoLocalize("{value, plural, one {# book} other {# books}}")
+    ).toEqual("{value, plural, one {# ƀōōķ} other {# ƀōōķś}}")
+  })
+})

packages/cli/src/api/types.js

@@ -5,6 +5,7 @@ export type LinguiConfig = {|
   localeDir: string,
   sourceLocale: string,
   fallbackLocale: string,
+  pseudoLocale: string,
   srcPathDirs: Array<string>,
   srcPathIgnorePatterns: Array<string>,
   format: "lingui" | "minimal" | "po"

packages/cli/src/lingui-compile.js

@@ -91,7 +91,8 @@ function command(config, options) {
       locale,
       messages,
       false,
-      options.namespace || config.compileNamespace
+      options.namespace || config.compileNamespace,
+      config.pseudoLocale
     )
     const compiledPath = catalog.writeCompiled(locale, compiledCatalog)
     if (options.typescript) {

packages/cli/src/lingui-extract.js

@@ -34,6 +34,11 @@ export default function command(
   }
 
   const catalog = configureCatalog(config)
+  const pseudoLocale = config.pseudoLocale
+  if (pseudoLocale) {
+    catalog.addLocale(pseudoLocale)
+  }
+
   const locales = catalog.getLocales()
 
   if (!locales.length) {

packages/cli/src/lingui-extract.test.js

@@ -1,9 +1,18 @@
-// @flow
+import fs from "fs"
 import { mockConsole, mockConfig } from "./mocks"
 import command from "./lingui-extract"
+import configureCatalog from "./api/catalog"
+import { detect } from "./api/detect"
+
+import { extract, collect, cleanObsolete, order } from "./api/extract"
+
+jest.mock("fs")
+jest.mock("./api/catalog")
+jest.mock("./api/extract")
+jest.mock("./api/detect")
 
 describe("lingui extract", function() {
-  function mockExtractOptions(options?: Object = {}) {
+  function mockExtractOptions(options) {
     return {
       verbose: false,
       clean: false,
@@ -12,10 +21,25 @@ describe("lingui extract", function() {
       ...options
     }
   }
+
+  beforeEach(() => {
+    detect.mockClear()
+    extract.mockClear()
+    collect.mockClear()
+    cleanObsolete.mockClear()
+    order.mockClear()
+  })
+
   it("should exit when there aren't any locales", function() {
     const config = mockConfig()
     const options = mockExtractOptions()
 
+    configureCatalog.mockImplementation(() => {
+      return {
+        getLocales: jest.fn().mockReturnValue([])
+      }
+    })
+
     mockConsole(console => {
       command(config, options)
       expect(console.log).toBeCalledWith(
@@ -26,4 +50,28 @@ describe("lingui extract", function() {
       )
     })
   })
+  it("should add pseudoLocale when defined", () => {
+    const config = mockConfig({
+      pseudoLocale: "pseudo-LOCALE"
+    })
+    const options = mockExtractOptions()
+
+    const addLocale = jest.fn()
+    order.mockImplementation(() => ["pseudo-LOCALE"])
+    configureCatalog.mockImplementation(() => {
+      return {
+        addLocale: addLocale,
+        getLocales: jest.fn().mockReturnValue(["pseudo-LOCALE"]),
+        readAll: jest.fn(),
+        merge: jest.fn(),
+        write: jest.fn().mockReturnValue([true, "messages.json"])
+      }
+    })
+
+    mockConsole(console => {
+      command(config, options)
+    })
+
+    expect(addLocale).toBeCalledWith("pseudo-LOCALE")
+  })
 })

packages/cli/test/fixtures/extract/App.js

@@ -1,5 +1,5 @@
 import * as React from "react"
-import { Trans } from "@lingui/react"
+import { Trans, Plural } from "@lingui/react"
 
 class App extends React.Component {
   render() {
@@ -17,6 +17,15 @@ class App extends React.Component {
         </p>
 
         <Trans>Value of {value}</Trans>
+
+        <p>
+          <Plural
+            value={messagesCount}
+            zero="There're no messages"
+            one="There's # message <span>in</span> your inbox"
+            other="There're # messages in your inbox"
+          />
+        </p>
       </div>
     )
   }

packages/cli/test/fixtures/extract/package.json

@@ -1,7 +1,8 @@
 {
   "lingui": {
-    "format": "po",
+    "format": "lingui",
     "localeDir": "./locale",
-    "sourceLocale": "en"
+    "sourceLocale": "en",
+    "pseudoLocale": "en-PT"
   }
 }

packages/conf/src/index.js

@@ -29,6 +29,7 @@ export const defaultConfig = {
   localeDir: "./locale",
   sourceLocale: "",
   fallbackLocale: "",
+  pseudoLocale: "",
   srcPathDirs: ["<rootDir>"],
   srcPathIgnorePatterns: [NODE_MODULES],
   format: "lingui",

packages/conf/src/index.test.js

@@ -7,6 +7,7 @@ describe("lingui-conf", function() {
     expect(config.localeDir).toBeDefined()
     expect(config.sourceLocale).toBeDefined()
     expect(config.fallbackLocale).toBeDefined()
+    expect(config.pseudoLocale).toBeDefined()
     expect(config.srcPathDirs).toBeDefined()
     expect(config.srcPathIgnorePatterns).toBeDefined()
     expect(config.extractBabelOptions).toBeDefined()

yarn.lock

@@ -1613,7 +1613,7 @@ combined-stream@1.0.6, combined-stream@~1.0.5:
   dependencies:
     delayed-stream "~1.0.0"
 
-commander@^2.17.1:
+commander@*, commander@^2.17.1:
   version "2.17.1"
   resolved "https://registry.yarnpkg.com/commander/-/commander-2.17.1.tgz#bd77ab7de6de94205ceacc72f1716d29f20a77bf"
 
@@ -4949,6 +4949,12 @@ prr@~1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/prr/-/prr-1.0.1.tgz#d3fc114ba06995a45ec6893f484ceb1d78f5f476"
 
+pseudolocale@^1.1.0:
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/pseudolocale/-/pseudolocale-1.1.0.tgz#f333f229433d2c586ec384d021e81f0cf7ca8dc7"
+  dependencies:
+    commander "*"
+
 pseudomap@^1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/pseudomap/-/pseudomap-1.0.2.tgz#f052a28da70e618917ef0a8ac34c1ae5a68286b3"