[FEATURE] Properties File Escaping

index.js

@@ -19,6 +19,7 @@ module.exports = {
 		bootstrapHtmlTransformer: require("./lib/processors/bootstrapHtmlTransformer"),
 		debugFileCreator: require("./lib/processors/debugFileCreator"),
 		resourceCopier: require("./lib/processors/resourceCopier"),
+		nonAsciiEscaper: require("./lib/processors/nonAsciiEscaper"),
 		stringReplacer: require("./lib/processors/stringReplacer"),
 		themeBuilder: require("./lib/processors/themeBuilder"),
 		uglifier: require("./lib/processors/uglifier"),
@@ -43,6 +44,7 @@ module.exports = {
 		generateApiIndex: require("./lib/tasks/jsdoc/generateApiIndex"),
 		generateJsdoc: require("./lib/tasks/jsdoc/generateJsdoc"),
 		generateVersionInfo: require("./lib/tasks/generateVersionInfo"),
+		escapeNonAsciiCharacters: require("./lib/tasks/escapeNonAsciiCharacters"),
 		replaceCopyright: require("./lib/tasks/replaceCopyright"),
 		replaceVersion: require("./lib/tasks/replaceVersion"),
 		transformBootstrapHtml: require("./lib/tasks/transformBootstrapHtml"),

lib/lbt/bundle/AutoSplitter.js

@@ -2,8 +2,10 @@
 
 const uglify = require("uglify-es");
 const {pd} = require("pretty-data");
+
 const ModuleName = require("../utils/ModuleName");
 const {SectionType} = require("./BundleDefinition");
+const escapePropertiesFiles = require("../utils/escapePropertiesFiles");
 const log = require("@ui5/logger").getLogger("lbt:bundle:AutoSplitter");
 
 const copyrightCommentsPattern = /copyright|\(c\)(?:[0-9]+|\s+[0-9A-za-z])|released under|license|\u00a9/i;
@@ -217,7 +219,6 @@ class AutoSplitter {
 				// trace.debug("analyzed %s:%d%n", module, mw.getTargetLength());
 				return fileContent.length;
 			} else if ( /\.properties$/.test(module) ) {
-				const fileContent = await resource.buffer();
 				/* NODE-TODO minimize *.properties
 				Properties props = new Properties();
 				props.load(in);
@@ -226,7 +227,12 @@ class AutoSplitter {
 				props.store(out, "");
 				return out.toString().length();
 				*/
-				return fileContent.toString("latin1").length;
+
+				// Since AutoSplitter is also used when splitting non-project resources (e.g. dependencies)
+				// *.properties files should be escaped if encoding option is specified
+				const fileContent = await escapePropertiesFiles(resource);
+
+				return fileContent.length;
 			} else if ( this.optimizeXMLViews && /\.view.xml$/.test(module) ) {
 				// needs to be activated when it gets activated in JSMergedModuleBuilderExt
 				let fileContent = await resource.buffer();

lib/lbt/bundle/Builder.js

@@ -12,6 +12,7 @@ const {Syntax} = esprima;
 const {isMethodCall} = require("../utils/ASTUtils");
 const ModuleName = require("../utils/ModuleName");
 const UI5ClientConstants = require("../UI5ClientConstants");
+const escapePropertiesFiles = require("../utils/escapePropertiesFiles");
 
 const BundleResolver = require("./Resolver");
 const BundleSplitter = require("./AutoSplitter");
@@ -442,9 +443,11 @@ class BundleBuilder {
 			}
 			outW.write( makeStringLiteral( fileContent ) );
 		} else if ( /\.properties$/.test(module) ) {
-			// same as for other text files, but input encoding is ISO_8859_1
-			const fileContent = await resource.buffer();
-			outW.write( makeStringLiteral( fileContent.toString("latin1") ) );
+			// Since the Builder is also used when building non-project resources (e.g. dependencies)
+			// *.properties files should be escaped if encoding option is specified
+			const fileContent = await escapePropertiesFiles(resource);
+
+			outW.write( makeStringLiteral( fileContent ) );
 		} else {
 			log.error("don't know how to embed module " + module); // TODO throw?
 		}

lib/lbt/utils/escapePropertiesFiles.js

@@ -0,0 +1,29 @@
+const nonAsciiEscaper = require("../../processors/nonAsciiEscaper");
+
+/**
+ * Can be used to escape *.properties files.
+ *
+ * Input encoding is read from project configuration.
+ * In case the resource belongs to no project (e.g. bundler is used standalone) the default is "ISO-8859-1".
+ *
+ * @private
+ * @param {Resource} resource the resource for which the content will be escaped
+ * @returns {Promise<string>} resolves with the escaped string content of the given Resource
+ */
+module.exports = async function(resource) {
+	const propertiesFileEncoding = resource.getProject()
+			&& resource.getProject().resources
+			&& resource.getProject().resources.configuration
+			&& resource.getProject().resources.configuration.propertiesFileEncoding;
+	const encoding = nonAsciiEscaper.getEncodingFromAlias(propertiesFileEncoding || "ISO-8859-1");
+	await nonAsciiEscaper({
+		resources: [resource.resource],
+		options: {
+			encoding
+		}
+	});
+
+	const fileContent = await resource.buffer();
+
+	return fileContent.toString(encoding);
+};

lib/processors/bundlers/moduleBundler.js

@@ -16,6 +16,10 @@ class LocatorResource extends Resource {
 	buffer() {
 		return this.resource.getBuffer();
 	}
+
+	getProject() {
+		return this.resource._project;
+	}
 }
 
 class LocatorResourcePool extends ResourcePool {

lib/processors/nonAsciiEscaper.js

@@ -0,0 +1,103 @@
+const escapeUnicode = require("escape-unicode");
+
+/**
+ * @see https://en.wikipedia.org/wiki/ASCII
+ * ascii contains 128 characters.
+ * its char codes reach from 0 to 127.
+ * @type {number}
+ */
+const CHAR_CODE_OF_LAST_ASCII_CHARACTER = 127;
+
+// use memoization for escapeUnicode function for performance
+const memoizeEscapeUnicodeMap = {};
+const memoizeEscapeUnicode = function(sChar) {
+	if (memoizeEscapeUnicodeMap[sChar]) {
+		return memoizeEscapeUnicodeMap[sChar];
+	}
+	memoizeEscapeUnicodeMap[sChar] = escapeUnicode(sChar);
+	return memoizeEscapeUnicodeMap[sChar];
+};
+
+/**
+ * Escapes non ASCII characters with unicode escape sequences.
+ *
+ * @see https://en.wikipedia.org/wiki/ASCII
+ * @see https://tools.ietf.org/html/rfc5137#section-6.1
+ *
+ *
+ * @param {string} string input string with non ascii characters, e.g. L♥VE
+ * @returns {{string: (string), modified: boolean}} output string with all non ascii
+ * characters being escaped by unicode sequence, e.g. L\u2665VE
+ */
+const escapeNonAscii = function(string) {
+	let result = "";
+	let modified = false;
+	for (let i = 0; i < string.length; i++) {
+		const char = string[i];
+		// check for non ascii characters (characters which have a char code
+		// greater than the ascii character code range)
+		if (string.charCodeAt(i) > CHAR_CODE_OF_LAST_ASCII_CHARACTER) {
+			result += memoizeEscapeUnicode(char);
+			modified = true;
+		} else {
+			result += char;
+		}
+	}
+	return {
+		modified,
+		string: result
+	};
+};
+
+/**
+ * Escapes non ASCII characters with unicode escape sequences.
+ *
+ * @example
+ * const encoding = nonAsciiEscaper.getEncodingFromAlias("ISO-8859-1");
+ * nonAsciiEscaper({resources, options: {encoding}});
+ *
+ *
+ * @public
+ * @alias module:@ui5/builder.processors.nonAsciiEscaper
+ * @param {Object} parameters Parameters
+ * @param {module:@ui5/fs.Resource[]} parameters.resources List of resources to be processed
+ * @param {Object} [parameters.options] Options
+ * @param {string} [parameters.options.encoding="utf8"] resource file encoding (node.js based encodings). Use #getEncodingFromAlias to get the encoding string
+ * {@link https://nodejs.org/api/buffer.html#buffer_buffers_and_character_encodings Node.js character encodings};
+ * @returns {Promise<module:@ui5/fs.Resource[]>} Promise resolving with the processed resources
+ */
+module.exports = async function nonAsciiEscaper({resources, options={}}) {
+	const encoding = options.encoding || "utf8";
+
+	async function processResource(resource) {
+		const resourceString = (await resource.getBuffer()).toString(encoding);
+		const escaped = escapeNonAscii(resourceString);
+		// only modify the resource's string if it was changed
+		if (escaped.modified) {
+			resource.setString(escaped.string);
+		}
+		return resource;
+	}
+
+	return Promise.all(resources.map(processResource));
+};
+
+const encodingMap = {
+	"UTF-8": "utf8",
+	"ISO-8859-1": "latin1",
+};
+
+/**
+ * Provides a mapping from user-friendly encoding name (alias) such as "UTF-8" and "ISO-8859-1" to node
+ * specific encoding name such as "utf8" or "latin1". Simplifies usage of nonAsciiEscaper encoding
+ * option such that it can be used standalone without the respective task (e.g. in Splitter, Bundler and related projects).
+ *
+ * @param {string} encoding encoding labels: "UTF-8" and "ISO-8859-1"
+ * @returns {string} node.js character encoding string, e.g. utf8 and latin1
+ */
+module.exports.getEncodingFromAlias = function(encoding) {
+	if (!encodingMap[encoding]) {
+		throw new Error(`Encoding "${encoding}" is not supported. Only ${Object.keys(encodingMap).join(", ")} are allowed values` );
+	}
+	return encodingMap[encoding];
+};

lib/tasks/escapeNonAsciiCharacters.js

@@ -0,0 +1,30 @@
+const nonAsciiEscaper = require("../processors/nonAsciiEscaper");
+
+/**
+ * Task to escape non ascii characters in properties files resources.
+ *
+ * @public
+ * @alias module:@ui5/builder.tasks.escapePropertiesFiles
+ * @param {Object} parameters Parameters
+ * @param {module:@ui5/fs.DuplexCollection} parameters.workspace DuplexCollection to read and write files
+ * @param {Object} parameters.options Options
+ * @param {string} parameters.options.pattern Glob pattern to locate the files to be processed
+ * @param {string} parameters.options.encoding source file encoding either "UTF-8" or "ISO-8859-1"
+ * @returns {Promise<undefined>} Promise resolving with <code>undefined</code> once data has been written
+ */
+module.exports = async function({workspace, options}) {
+	if (!options.encoding) {
+		throw new Error("[escapeNonAsciiCharacters] Mandatory option 'encoding' not provided");
+	}
+
+	const allResources = await workspace.byGlob(options.pattern);
+
+	const processedResources = await nonAsciiEscaper({
+		resources: allResources,
+		options: {
+			encoding: nonAsciiEscaper.getEncodingFromAlias(options.encoding)
+		}
+	});
+
+	await Promise.all(processedResources.map((resource) => workspace.write(resource)));
+};

lib/tasks/taskRepository.js

@@ -2,6 +2,7 @@ const tasks = {
 	replaceCopyright: require("./replaceCopyright"),
 	replaceVersion: require("./replaceVersion"),
 	createDebugFiles: require("./createDebugFiles"),
+	escapeNonAsciiCharacters: require("./escapeNonAsciiCharacters"),
 	executeJsdocSdkTransformation: require("./jsdoc/executeJsdocSdkTransformation"),
 	generateApiIndex: require("./jsdoc/generateApiIndex"),
 	generateJsdoc: require("./jsdoc/generateJsdoc"),

lib/types/application/ApplicationBuilder.js

@@ -8,6 +8,7 @@ const tasks = { // can't require index.js due to circular dependency
 	generateStandaloneAppBundle: require("../../tasks/bundlers/generateStandaloneAppBundle"),
 	generateBundle: require("../../tasks/bundlers/generateBundle"),
 	generateCachebusterInfo: require("../../tasks/generateCachebusterInfo"),
+	escapeNonAsciiCharacters: require("../../tasks/escapeNonAsciiCharacters"),
 	buildThemes: require("../../tasks/buildThemes"),
 	createDebugFiles: require("../../tasks/createDebugFiles"),
 	generateVersionInfo: require("../../tasks/generateVersionInfo"),
@@ -26,6 +27,19 @@ class ApplicationBuilder extends AbstractBuilder {
 					"Also see: https://github.com/SAP/ui5-builder#application");
 		}
 
+		this.addTask("escapeNonAsciiCharacters", () => {
+			const propertiesFileEncoding = project.resources
+				&& project.resources.configuration
+				&& project.resources.configuration.propertiesFileEncoding;
+			return tasks.escapeNonAsciiCharacters({
+				workspace: resourceCollections.workspace,
+				options: {
+					encoding: propertiesFileEncoding,
+					pattern: "/**/*.properties"
+				}
+			});
+		});
+
 		this.addTask("replaceCopyright", () => {
 			return tasks.replaceCopyright({
 				workspace: resourceCollections.workspace,

lib/types/application/ApplicationFormatter.js

@@ -133,6 +133,15 @@ class ApplicationFormatter extends AbstractUi5Formatter {
 				project.resources.configuration.paths.webapp = "webapp";
 			}
 
+			// default encoding to "ISO-8859-1" if not specified
+			if (!project.resources.configuration.propertiesFileEncoding) {
+				project.resources.configuration.propertiesFileEncoding = "ISO-8859-1";
+			}
+			if (!["ISO-8859-1", "UTF-8"].includes(project.resources.configuration.propertiesFileEncoding)) {
+				throw new Error(`Invalid properties file encoding specified for project ${project.id}: ` +
+					`encoding provided: ${project.resources.configuration.propertiesFileEncoding}. Must be either "ISO-8859-1" or "UTF-8".`);
+			}
+
 			const absolutePath = path.join(project.path, project.resources.configuration.paths.webapp);
 			return this.dirExists(absolutePath).then((bExists) => {
 				if (!bExists) {

lib/types/library/LibraryBuilder.js

@@ -6,6 +6,7 @@ const tasks = { // can't require index.js due to circular dependency
 	generateLibraryPreload: require("../../tasks/bundlers/generateLibraryPreload"),
 	generateManifestBundle: require("../../tasks/bundlers/generateManifestBundle"),
 	generateStandaloneAppBundle: require("../../tasks/bundlers/generateStandaloneAppBundle"),
+	escapeNonAsciiCharacters: require("../../tasks/escapeNonAsciiCharacters"),
 	buildThemes: require("../../tasks/buildThemes"),
 	createDebugFiles: require("../../tasks/createDebugFiles"),
 	generateJsdoc: require("../../tasks/jsdoc/generateJsdoc"),
@@ -25,6 +26,19 @@ class LibraryBuilder extends AbstractBuilder {
 				"Also see: https://github.com/SAP/ui5-builder#library");
 		}
 
+		this.addTask("escapeNonAsciiCharacters", () => {
+			const propertiesFileEncoding = project.resources
+				&& project.resources.configuration
+				&& project.resources.configuration.propertiesFileEncoding;
+			return tasks.escapeNonAsciiCharacters({
+				workspace: resourceCollections.workspace,
+				options: {
+					encoding: propertiesFileEncoding,
+					pattern: "/**/*.properties"
+				}
+			});
+		});
+
 		this.addTask("replaceCopyright", () => {
 			const replaceCopyright = tasks.replaceCopyright;
 			return replaceCopyright({

lib/types/library/LibraryFormatter.js

@@ -340,6 +340,15 @@ class LibraryFormatter extends AbstractUi5Formatter {
 				project.resources.configuration.paths.test = "test";
 			}
 
+			// default encoding to "ISO-8859-1" if not specified
+			if (!project.resources.configuration.propertiesFileEncoding) {
+				project.resources.configuration.propertiesFileEncoding = "ISO-8859-1";
+			}
+			if (!["ISO-8859-1", "UTF-8"].includes(project.resources.configuration.propertiesFileEncoding)) {
+				throw new Error(`Invalid properties file encoding specified for project ${project.id}: ` +
+					`encoding provided: ${project.resources.configuration.propertiesFileEncoding}. Must be either "ISO-8859-1" or "UTF-8".`);
+			}
+
 			const absoluteSrcPath = path.join(project.path, project.resources.configuration.paths.src);
 			const absoluteTestPath = path.join(project.path, project.resources.configuration.paths.test);
 			return Promise.all([

package.json

@@ -99,6 +99,7 @@
 		"@ui5/fs": "^1.1.2",
 		"@ui5/logger": "^1.0.1",
 		"cheerio": "^0.22.0",
+		"escape-unicode": "^0.2.0",
 		"escodegen": "^1.11.1",
 		"escope": "^3.6.0",
 		"esprima": "^4.0.1",

test/expected/build/application.b/dest/i18n/i18n.properties

@@ -1 +1,2 @@
-title=app-i18n
+title=app-i18n
+fame=Stra\u00dfe

test/expected/build/application.b/dest/i18n/i18n_de.properties

@@ -1 +1,2 @@
-title=app-i18n_de
+title=app-i18n_de
+fame=Stra\u00dfe

test/expected/build/application.b/dest/manifest-bundle/i18n/i18n.properties

@@ -1 +1,2 @@
-title=app-i18n
+title=app-i18n
+fame=Stra\u00dfe

test/expected/build/application.b/dest/manifest-bundle/i18n/i18n_de.properties

@@ -1 +1,2 @@
-title=app-i18n_de
+title=app-i18n_de
+fame=Stra\u00dfe

test/expected/build/application.b/standalone/i18n.properties

@@ -1 +1,2 @@
-title=app-i18n-wrong
+title=app-i18n-wrong
+fame=Stra�e

test/expected/build/application.b/standalone/i18n/i18n.properties

@@ -1 +1,2 @@
-title=app-i18n
+title=app-i18n
+fame=Stra�e