-
Notifications
You must be signed in to change notification settings - Fork 23
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[FEATURE] Properties File Escaping (#293)
Add processor nonAsciiEscaper which escapes non ASCII characters in resources. Add task escapeNonAsciiCharacters which uses nonAsciiEscaper to escape non ascii characters.
- Loading branch information
1 parent
2ba75af
commit 9d213ce
Showing
37 changed files
with
660 additions
and
54 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,29 @@ | ||
| const nonAsciiEscaper = require("../../processors/nonAsciiEscaper"); | ||
|
|
||
| /** | ||
| * Can be used to escape *.properties files. | ||
| * | ||
| * Input encoding is read from project configuration. | ||
| * In case the resource belongs to no project (e.g. bundler is used standalone) the default is "ISO-8859-1". | ||
| * | ||
| * @private | ||
| * @param {Resource} resource the resource for which the content will be escaped | ||
| * @returns {Promise<string>} resolves with the escaped string content of the given Resource | ||
| */ | ||
| module.exports = async function(resource) { | ||
| const propertiesFileSourceEncoding = resource.getProject() | ||
| && resource.getProject().resources | ||
| && resource.getProject().resources.configuration | ||
| && resource.getProject().resources.configuration.propertiesFileSourceEncoding; | ||
| const encoding = nonAsciiEscaper.getEncodingFromAlias(propertiesFileSourceEncoding || "ISO-8859-1"); | ||
| await nonAsciiEscaper({ | ||
| resources: [resource.resource], | ||
| options: { | ||
| encoding | ||
| } | ||
| }); | ||
|
|
||
| const fileContent = await resource.buffer(); | ||
|
|
||
| return fileContent.toString(); | ||
| }; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,103 @@ | ||
| const escapeUnicode = require("escape-unicode"); | ||
|
|
||
| /** | ||
| * @see https://en.wikipedia.org/wiki/ASCII | ||
| * ascii contains 128 characters. | ||
| * its char codes reach from 0 to 127. | ||
| * @type {number} | ||
| */ | ||
| const CHAR_CODE_OF_LAST_ASCII_CHARACTER = 127; | ||
|
|
||
| // use memoization for escapeUnicode function for performance | ||
| const memoizeEscapeUnicodeMap = {}; | ||
| const memoizeEscapeUnicode = function(sChar) { | ||
| if (memoizeEscapeUnicodeMap[sChar]) { | ||
| return memoizeEscapeUnicodeMap[sChar]; | ||
| } | ||
| memoizeEscapeUnicodeMap[sChar] = escapeUnicode(sChar); | ||
| return memoizeEscapeUnicodeMap[sChar]; | ||
| }; | ||
|
|
||
| /** | ||
| * Escapes non ASCII characters with unicode escape sequences. | ||
| * | ||
| * @see https://en.wikipedia.org/wiki/ASCII | ||
| * @see https://tools.ietf.org/html/rfc5137#section-6.1 | ||
| * | ||
| * | ||
| * @param {string} string input string with non ascii characters, e.g. L♥VE | ||
| * @returns {{string: (string), modified: boolean}} output string with all non ascii | ||
| * characters being escaped by unicode sequence, e.g. L\u2665VE | ||
| */ | ||
| const escapeNonAscii = function(string) { | ||
| let result = ""; | ||
| let modified = false; | ||
| for (let i = 0; i < string.length; i++) { | ||
| const char = string[i]; | ||
| // check for non ascii characters (characters which have a char code | ||
| // greater than the ascii character code range) | ||
| if (string.charCodeAt(i) > CHAR_CODE_OF_LAST_ASCII_CHARACTER) { | ||
| result += memoizeEscapeUnicode(char); | ||
| modified = true; | ||
| } else { | ||
| result += char; | ||
| } | ||
| } | ||
| return { | ||
| modified, | ||
| string: result | ||
| }; | ||
| }; | ||
|
|
||
| /** | ||
| * Escapes non ASCII characters with unicode escape sequences. | ||
| * | ||
| * @example | ||
| * const encoding = nonAsciiEscaper.getEncodingFromAlias("ISO-8859-1"); | ||
| * nonAsciiEscaper({resources, options: {encoding}}); | ||
| * | ||
| * | ||
| * @public | ||
| * @alias module:@ui5/builder.processors.nonAsciiEscaper | ||
| * @param {Object} parameters Parameters | ||
| * @param {module:@ui5/fs.Resource[]} parameters.resources List of resources to be processed | ||
| * @param {Object} [parameters.options] Options | ||
| * @param {string} [parameters.options.encoding="utf8"] resource file encoding (node.js based encodings). Use #getEncodingFromAlias to get the encoding string | ||
| * {@link https://nodejs.org/api/buffer.html#buffer_buffers_and_character_encodings Node.js character encodings}; | ||
| * @returns {Promise<module:@ui5/fs.Resource[]>} Promise resolving with the processed resources | ||
| */ | ||
| module.exports = async function nonAsciiEscaper({resources, options={}}) { | ||
| const encoding = options.encoding || "utf8"; | ||
|
|
||
| async function processResource(resource) { | ||
| const resourceString = (await resource.getBuffer()).toString(encoding); | ||
| const escaped = escapeNonAscii(resourceString); | ||
| // only modify the resource's string if it was changed | ||
| if (escaped.modified) { | ||
| resource.setString(escaped.string); | ||
| } | ||
| return resource; | ||
| } | ||
|
|
||
| return Promise.all(resources.map(processResource)); | ||
| }; | ||
|
|
||
| const encodingMap = { | ||
| "UTF-8": "utf8", | ||
| "ISO-8859-1": "latin1", | ||
| }; | ||
|
|
||
| /** | ||
| * Provides a mapping from user-friendly encoding name (alias) such as "UTF-8" and "ISO-8859-1" to node | ||
| * specific encoding name such as "utf8" or "latin1". Simplifies usage of nonAsciiEscaper encoding | ||
| * option such that it can be used standalone without the respective task (e.g. in Splitter, Bundler and related projects). | ||
| * | ||
| * @param {string} encoding encoding labels: "UTF-8" and "ISO-8859-1" | ||
| * @returns {string} node.js character encoding string, e.g. utf8 and latin1 | ||
| */ | ||
| module.exports.getEncodingFromAlias = function(encoding) { | ||
| if (!encodingMap[encoding]) { | ||
| throw new Error(`Encoding "${encoding}" is not supported. Only ${Object.keys(encodingMap).join(", ")} are allowed values` ); | ||
| } | ||
| return encodingMap[encoding]; | ||
| }; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,30 @@ | ||
| const nonAsciiEscaper = require("../processors/nonAsciiEscaper"); | ||
|
|
||
| /** | ||
| * Task to escape non ascii characters in properties files resources. | ||
| * | ||
| * @public | ||
| * @alias module:@ui5/builder.tasks.escapeNonAsciiCharacters | ||
| * @param {Object} parameters Parameters | ||
| * @param {module:@ui5/fs.DuplexCollection} parameters.workspace DuplexCollection to read and write files | ||
| * @param {Object} parameters.options Options | ||
| * @param {string} parameters.options.pattern Glob pattern to locate the files to be processed | ||
| * @param {string} parameters.options.encoding source file encoding either "UTF-8" or "ISO-8859-1" | ||
| * @returns {Promise<undefined>} Promise resolving with <code>undefined</code> once data has been written | ||
| */ | ||
| module.exports = async function({workspace, options}) { | ||
| if (!options.encoding) { | ||
| throw new Error("[escapeNonAsciiCharacters] Mandatory option 'encoding' not provided"); | ||
| } | ||
|
|
||
| const allResources = await workspace.byGlob(options.pattern); | ||
|
|
||
| const processedResources = await nonAsciiEscaper({ | ||
| resources: allResources, | ||
| options: { | ||
| encoding: nonAsciiEscaper.getEncodingFromAlias(options.encoding) | ||
| } | ||
| }); | ||
|
|
||
| await Promise.all(processedResources.map((resource) => workspace.write(resource))); | ||
| }; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
Oops, something went wrong.