SAP · tobiasso85 · Jul 29, 2019 · Jul 16, 2019 · Jul 16, 2019 · Jul 16, 2019
diff --git a/index.js b/index.js
@@ -19,6 +19,7 @@ module.exports = {
  bootstrapHtmlTransformer: require("./lib/processors/bootstrapHtmlTransformer"),
  debugFileCreator: require("./lib/processors/debugFileCreator"),
  resourceCopier: require("./lib/processors/resourceCopier"),
+ nonAsciiEscaper: require("./lib/processors/nonAsciiEscaper"),
  stringReplacer: require("./lib/processors/stringReplacer"),
  themeBuilder: require("./lib/processors/themeBuilder"),
  uglifier: require("./lib/processors/uglifier"),
@@ -43,6 +44,7 @@ module.exports = {
  generateApiIndex: require("./lib/tasks/jsdoc/generateApiIndex"),
  generateJsdoc: require("./lib/tasks/jsdoc/generateJsdoc"),
  generateVersionInfo: require("./lib/tasks/generateVersionInfo"),
+ escapeNonAsciiCharacters: require("./lib/tasks/escapeNonAsciiCharacters"),
  replaceCopyright: require("./lib/tasks/replaceCopyright"),
  replaceVersion: require("./lib/tasks/replaceVersion"),
  transformBootstrapHtml: require("./lib/tasks/transformBootstrapHtml"),

diff --git a/lib/lbt/bundle/AutoSplitter.js b/lib/lbt/bundle/AutoSplitter.js
@@ -2,6 +2,9 @@
 
 const uglify = require("uglify-es");
 const {pd} = require("pretty-data");
+
+const nonAsciiEscaper = require("../../processors/nonAsciiEscaper");
+
 const ModuleName = require("../utils/ModuleName");
 const {SectionType} = require("./BundleDefinition");
 const log = require("@ui5/logger").getLogger("lbt:bundle:AutoSplitter");
@@ -217,7 +220,6 @@ class AutoSplitter {
  // trace.debug("analyzed %s:%d%n", module, mw.getTargetLength());
  return fileContent.length;
  } else if ( /\.properties$/.test(module) ) {
- const fileContent = await resource.buffer();
  /* NODE-TODO minimize *.properties
  Properties props = new Properties();
  props.load(in);
@@ -226,7 +228,27 @@ class AutoSplitter {
  props.store(out, "");
  return out.toString().length();
  */
- return fileContent.toString("latin1").length;
+
+ // input encoding is read from project configuration, default is "ISO-8859-1"
+ // since AutoSplitter is also used when splitting non-project resources (e.g. dependencies)
+ // *.properties files should be escaped if encoding option is specified
+ const propertiesFileEncoding = resource.getProject()
+ && resource.getProject().resources
+ && resource.getProject().resources.configuration
+ && resource.getProject().resources.configuration.propertiesFileEncoding;
+ if (propertiesFileEncoding) {
+ const encoding = nonAsciiEscaper.getEncodingFromAlias(propertiesFileEncoding);
+ await nonAsciiEscaper({
+ resources: [resource.resource],
+ options: {
+ encoding
+ }
+ });
+ }
+
+ const fileContent = await resource.buffer();
+
+ return fileContent.toString().length;
  } else if ( this.optimizeXMLViews && /\.view.xml$/.test(module) ) {
  // needs to be activated when it gets activated in JSMergedModuleBuilderExt
  let fileContent = await resource.buffer();

diff --git a/lib/lbt/bundle/Builder.js b/lib/lbt/bundle/Builder.js
@@ -9,6 +9,8 @@ const escodegen = require("escodegen");
 const {Syntax} = esprima;
 // const MOZ_SourceMap = require("source-map");
 
+const nonAsciiEscaper = require("../../processors/nonAsciiEscaper");
+
 const {isMethodCall} = require("../utils/ASTUtils");
 const ModuleName = require("../utils/ModuleName");
 const UI5ClientConstants = require("../UI5ClientConstants");
@@ -442,9 +444,26 @@ class BundleBuilder {
  }
  outW.write( makeStringLiteral( fileContent ) );
  } else if ( /\.properties$/.test(module) ) {
- // same as for other text files, but input encoding is ISO_8859_1
+ // same as for other text files, input encoding is read from project configuration, default is ISO-8859-1
+ // since Builder is also used when building non-project resources (e.g. dependencies)
+ // *.properties files should be escaped if encoding option is specified
+ const propertiesFileEncoding = resource.getProject()
+ && resource.getProject().resources
+ && resource.getProject().resources.configuration
+ && resource.getProject().resources.configuration.propertiesFileEncoding;
+ if (propertiesFileEncoding) {
+ const encoding = nonAsciiEscaper.getEncodingFromAlias(propertiesFileEncoding);
+ await nonAsciiEscaper({
+ resources: [resource.resource],
+ options: {
+ encoding
+ }
+ });
+ }
+
  const fileContent = await resource.buffer();
- outW.write( makeStringLiteral( fileContent.toString("latin1") ) );
+
+ outW.write( makeStringLiteral( fileContent.toString() ) );
  } else {
  log.error("don't know how to embed module " + module); // TODO throw?
  }

diff --git a/lib/processors/bundlers/moduleBundler.js b/lib/processors/bundlers/moduleBundler.js
@@ -16,6 +16,10 @@ class LocatorResource extends Resource {
  buffer() {
  return this.resource.getBuffer();
  }
+
+ getProject() {
+ return this.resource._project;
+ }
 }
 
 class LocatorResourcePool extends ResourcePool {

diff --git a/lib/processors/nonAsciiEscaper.js b/lib/processors/nonAsciiEscaper.js
@@ -0,0 +1,103 @@
+const escapeUnicode = require("escape-unicode");
+
+/**
+ * @see https://en.wikipedia.org/wiki/ASCII
+ * ascii contains 128 characters.
+ * its char codes reach from 0 to 127.
+ * @type {number}
+ */
+const CHAR_CODE_OF_LAST_ASCII_CHARACTER = 127;
+
+// use memoization for escapeUnicode function for performance
+const memoizeEscapeUnicodeMap = {};
+const memoizeEscapeUnicode = function(sChar) {
+ if (memoizeEscapeUnicodeMap[sChar]) {
+ return memoizeEscapeUnicodeMap[sChar];
+ }
+ memoizeEscapeUnicodeMap[sChar] = escapeUnicode(sChar);
+ return memoizeEscapeUnicodeMap[sChar];
+};
+
+/**
+ * Escapes non ASCII characters with unicode escape sequences.
+ *
+ * @see https://en.wikipedia.org/wiki/ASCII
+ * @see https://tools.ietf.org/html/rfc5137#section-6.1
+ *
+ *
+ * @param {string} string input string with non ascii characters, e.g. L♥VE
+ * @returns {{string: (string), modified: boolean}} output string with all non ascii
+ * characters being escaped by unicode sequence, e.g. L\u2665VE
+ */
+const escapeNonAscii = function(string) {
+ let result = "";
+ let modified = false;
+ for (let i = 0; i < string.length; i++) {
+ const char = string[i];
+ // check for non ascii characters (characters which have a char code
+ // greater than the ascii character code range)
+ if (string.charCodeAt(i) > CHAR_CODE_OF_LAST_ASCII_CHARACTER) {
+ result += memoizeEscapeUnicode(char);
+ modified = true;
+ } else {
+ result += char;
+ }
+ }
+ return {
+ modified,
+ string: result
+ };
+};
+
+/**
+ * Escapes non ASCII characters with unicode escape sequences.
+ *
+ * @example
+ * const encoding = nonAsciiEscaper.getEncodingFromAlias("ISO-8859-1");
+ * nonAsciiEscaper({resources, options: {encoding}});
+ *
+ *
+ * @public
+ * @alias module:@ui5/builder.processors.nonAsciiEscaper
+ * @param {Object} parameters Parameters
+ * @param {module:@ui5/fs.Resource[]} parameters.resources List of resources to be processed
+ * @param {Object} [parameters.options] Options
+ * @param {string} [parameters.options.encoding="utf8"] resource file encoding (node.js based encodings). Use #getEncodingFromAlias to get the encoding string
+ * {@link https://nodejs.org/api/buffer.html#buffer_buffers_and_character_encodings Node.js character encodings};
+ * @returns {Promise<module:@ui5/fs.Resource[]>} Promise resolving with the processed resources
+ */
+module.exports = async function nonAsciiEscaper({resources, options={}}) {
+ const encoding = options.encoding || "utf8";
+
+ async function processResource(resource) {
+ const resourceString = (await resource.getBuffer()).toString(encoding);
+ const escaped = escapeNonAscii(resourceString);
+ // only modify the resource's string if it was changed
+ if (escaped.modified) {
+ resource.setString(escaped.string);
+ }
+ return resource;
+ }
+
+ return Promise.all(resources.map(processResource));
+};
+
+const encodingMap = {
+ "UTF-8": "utf8",
+ "ISO-8859-1": "latin1",
+};
+
+/**
+ * Provides a mapping from user-friendly encoding name (alias) such as "UTF-8" and "ISO-8859-1" to node
+ * specific encoding name such as "utf8" or "latin1". Simplifies usage of nonAsciiEscaper encoding
+ * option such that it can be used standalone without the respective task (e.g. in Splitter, Bundler and related projects).
+ *
+ * @param {string} encoding encoding labels: "UTF-8" and "ISO-8859-1"
+ * @returns {string} node.js character encoding string, e.g. utf8 and latin1
+ */
+module.exports.getEncodingFromAlias = function(encoding) {
+ if (!encodingMap[encoding]) {
+ throw new Error(`Encoding "${encoding}" is not supported. Only ${Object.keys(encodingMap).join(",")} are allowed values` );
+ }
+ return encodingMap[encoding];
+};
diff --git a/lib/tasks/escapeNonAsciiCharacters.js b/lib/tasks/escapeNonAsciiCharacters.js
@@ -0,0 +1,30 @@
+const nonAsciiEscaper = require("../processors/nonAsciiEscaper");
+
+/**
+ * Task to escape non ascii characters in properties files resources.
+ *
+ * @public
+ * @alias module:@ui5/builder.tasks.escapePropertiesFiles
+ * @param {Object} parameters Parameters
+ * @param {module:@ui5/fs.DuplexCollection} parameters.workspace DuplexCollection to read and write files
+ * @param {Object} parameters.options Options
+ * @param {string} parameters.options.pattern Glob pattern to locate the files to be processed
+ * @param {string} parameters.options.encoding source file encoding either "UTF-8" or "ISO-8859-1"
+ * @returns {Promise<undefined>} Promise resolving with <code>undefined</code> once data has been written
+ */
+module.exports = async function({workspace, options}) {
+ if (!options.encoding) {
+ throw new Error("[escapeNonAsciiCharacters] Mandatory option 'encoding' not provided");
+ }
+
+ const allResources = await workspace.byGlob(options.pattern);
+
+ const processedResources = await nonAsciiEscaper({
+ resources: allResources,
+ options: {
+ encoding: nonAsciiEscaper.getEncodingFromAlias(options.encoding)
+ }
+ });
+
+ await Promise.all(processedResources.map((resource) => workspace.write(resource)));
+};
diff --git a/lib/tasks/taskRepository.js b/lib/tasks/taskRepository.js
@@ -2,6 +2,7 @@ const tasks = {
  replaceCopyright: require("./replaceCopyright"),
  replaceVersion: require("./replaceVersion"),
  createDebugFiles: require("./createDebugFiles"),
+ escapeNonAsciiCharacters: require("./escapeNonAsciiCharacters"),
  executeJsdocSdkTransformation: require("./jsdoc/executeJsdocSdkTransformation"),
  generateApiIndex: require("./jsdoc/generateApiIndex"),
  generateJsdoc: require("./jsdoc/generateJsdoc"),

diff --git a/lib/types/application/ApplicationBuilder.js b/lib/types/application/ApplicationBuilder.js
@@ -8,6 +8,7 @@ const tasks = { // can't require index.js due to circular dependency
  generateStandaloneAppBundle: require("../../tasks/bundlers/generateStandaloneAppBundle"),
  generateBundle: require("../../tasks/bundlers/generateBundle"),
  generateCachebusterInfo: require("../../tasks/generateCachebusterInfo"),
+ escapeNonAsciiCharacters: require("../../tasks/escapeNonAsciiCharacters"),
  buildThemes: require("../../tasks/buildThemes"),
  createDebugFiles: require("../../tasks/createDebugFiles"),
  generateVersionInfo: require("../../tasks/generateVersionInfo"),
@@ -26,6 +27,19 @@ class ApplicationBuilder extends AbstractBuilder {
  "Also see: https://github.com/SAP/ui5-builder#application");
  }
 
+ this.addTask("escapeNonAsciiCharacters", () => {
+ const propertiesFileEncoding = project.resources
+ && project.resources.configuration
+ && project.resources.configuration.propertiesFileEncoding;
+ return tasks.escapeNonAsciiCharacters({
+ workspace: resourceCollections.workspace,
+ options: {
+ encoding: propertiesFileEncoding,
+ pattern: "/**/*.properties"
+ }
+ });
+ });
+
  this.addTask("replaceCopyright", () => {
  return tasks.replaceCopyright({
  workspace: resourceCollections.workspace,

diff --git a/lib/types/application/ApplicationFormatter.js b/lib/types/application/ApplicationFormatter.js
@@ -133,6 +133,15 @@ class ApplicationFormatter extends AbstractUi5Formatter {
  project.resources.configuration.paths.webapp = "webapp";
  }
 
+ // default encoding to "ISO-8859-1" if not specified
+ if (!project.resources.configuration.propertiesFileEncoding) {
+ project.resources.configuration.propertiesFileEncoding = "ISO-8859-1";
+ }
+ if (!["ISO-8859-1", "UTF-8"].includes(project.resources.configuration.propertiesFileEncoding)) {
+ throw new Error(`Invalid properties file encoding specified for project ${project.id}: ` +
+ `encoding provided: ${project.resources.configuration.propertiesFileEncoding}. Must be either "ISO-8859-1" or "UTF-8".`);
+ }
+
  const absolutePath = path.join(project.path, project.resources.configuration.paths.webapp);
  return this.dirExists(absolutePath).then((bExists) => {
  if (!bExists) {

diff --git a/lib/types/library/LibraryBuilder.js b/lib/types/library/LibraryBuilder.js
@@ -6,6 +6,7 @@ const tasks = { // can't require index.js due to circular dependency
  generateLibraryPreload: require("../../tasks/bundlers/generateLibraryPreload"),
  generateManifestBundle: require("../../tasks/bundlers/generateManifestBundle"),
  generateStandaloneAppBundle: require("../../tasks/bundlers/generateStandaloneAppBundle"),
+ escapeNonAsciiCharacters: require("../../tasks/escapeNonAsciiCharacters"),
  buildThemes: require("../../tasks/buildThemes"),
  createDebugFiles: require("../../tasks/createDebugFiles"),
  generateJsdoc: require("../../tasks/jsdoc/generateJsdoc"),
@@ -25,6 +26,19 @@ class LibraryBuilder extends AbstractBuilder {
  "Also see: https://github.com/SAP/ui5-builder#library");
  }
 
+ this.addTask("escapeNonAsciiCharacters", () => {
+ const propertiesFileEncoding = project.resources
+ && project.resources.configuration
+ && project.resources.configuration.propertiesFileEncoding;
+ return tasks.escapeNonAsciiCharacters({
+ workspace: resourceCollections.workspace,
+ options: {
+ encoding: propertiesFileEncoding,
+ pattern: "/**/*.properties"
+ }
+ });
+ });
+
  this.addTask("replaceCopyright", () => {
  const replaceCopyright = tasks.replaceCopyright;
  return replaceCopyright({

diff --git a/lib/types/library/LibraryFormatter.js b/lib/types/library/LibraryFormatter.js
@@ -340,6 +340,15 @@ class LibraryFormatter extends AbstractUi5Formatter {
  project.resources.configuration.paths.test = "test";
  }
 
+ // default encoding to "ISO-8859-1" if not specified
+ if (!project.resources.configuration.propertiesFileEncoding) {
+ project.resources.configuration.propertiesFileEncoding = "ISO-8859-1";
+ }
+ if (!["ISO-8859-1", "UTF-8"].includes(project.resources.configuration.propertiesFileEncoding)) {
+ throw new Error(`Invalid properties file encoding specified for project ${project.id}: ` +
+ `encoding provided: ${project.resources.configuration.propertiesFileEncoding}. Must be either "ISO-8859-1" or "UTF-8".`);
+ }
+
  const absoluteSrcPath = path.join(project.path, project.resources.configuration.paths.src);
  const absoluteTestPath = path.join(project.path, project.resources.configuration.paths.test);
  return Promise.all([

diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -99,6 +99,7 @@
  "@ui5/fs": "^1.1.2",
  "@ui5/logger": "^1.0.1",
  "cheerio": "^0.22.0",
+ "escape-unicode": "^0.2.0",
  "escodegen": "^1.11.1",
  "escope": "^3.6.0",
  "esprima": "^4.0.1",

diff --git a/test/expected/build/application.b/dest/i18n/i18n.properties b/test/expected/build/application.b/dest/i18n/i18n.properties
@@ -1 +1,2 @@
-title=app-i18n
+title=app-i18n
+fame=Stra\u00dfe
diff --git a/test/expected/build/application.b/dest/i18n/i18n_de.properties b/test/expected/build/application.b/dest/i18n/i18n_de.properties
@@ -1 +1,2 @@
-title=app-i18n_de
+title=app-i18n_de
+fame=Stra\u00dfe
diff --git a/test/expected/build/application.b/dest/manifest-bundle/i18n/i18n.properties b/test/expected/build/application.b/dest/manifest-bundle/i18n/i18n.properties
@@ -1 +1,2 @@
-title=app-i18n
+title=app-i18n
+fame=Stra�e
diff --git a/test/expected/build/application.b/dest/manifest-bundle/i18n/i18n_de.properties b/test/expected/build/application.b/dest/manifest-bundle/i18n/i18n_de.properties
@@ -1 +1,2 @@
-title=app-i18n_de
+title=app-i18n_de
+fame=Stra�e
diff --git a/test/expected/build/application.b/standalone/i18n.properties b/test/expected/build/application.b/standalone/i18n.properties
@@ -1 +1,2 @@
-title=app-i18n-wrong
+title=app-i18n-wrong
+fame=Stra�e
diff --git a/test/expected/build/application.b/standalone/i18n/i18n.properties b/test/expected/build/application.b/standalone/i18n/i18n.properties
@@ -1 +1,2 @@
-title=app-i18n
+title=app-i18n
+fame=Stra�e