From fe652e410bd0eab506fc42036ad2cfa374fa5a6c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Matthias=20O=C3=9Fwald?=
 <1410947+matz3@users.noreply.github.com>
Date: Mon, 10 Feb 2020 14:17:16 +0100
Subject: [PATCH] [FIX] Ensure proper handling of multi-byte characters in
 streams (#280)

The stream encoding needs to be set in order to properly handle
multi-byte utf8 characters that are split between different chunks.

See:
https://nodejs.org/api/stream.html#stream_readable_setencoding_encoding

Streams in Node 8 doesn't seem to support a proper handling when utf8 encoding is set.
Therefore the test is skipped.
---
 lib/middleware/serveResources.js             |   1 +
 test/lib/server/middleware/serveResources.js | 141 +++++++++++++++++++
 2 files changed, 142 insertions(+)

diff --git a/lib/middleware/serveResources.js b/lib/middleware/serveResources.js
index 4f7d27bb..a557796f 100644
--- a/lib/middleware/serveResources.js
+++ b/lib/middleware/serveResources.js
@@ -76,6 +76,7 @@ function createMiddleware({resources}) {
 			//	and LibraryBuilder
 			if ((!charset || charset === "UTF-8") && rReplaceVersion.test(resourcePath)) {
 				if (resource._project) {
+					stream.setEncoding("utf8");
 					stream = stream.pipe(replaceStream("${version}", resource._project.version));
 				} else {
 					log.verbose("Project missing from resource %s", pathname);
diff --git a/test/lib/server/middleware/serveResources.js b/test/lib/server/middleware/serveResources.js
index 65b60ee3..f4c03e64 100644
--- a/test/lib/server/middleware/serveResources.js
+++ b/test/lib/server/middleware/serveResources.js
@@ -1,5 +1,6 @@
 const test = require("ava");
 const sinon = require("sinon");
+const {Readable, Writable} = require("stream");
 const resourceFactory = require("@ui5/fs").resourceFactory;
 const serveResourcesMiddleware = require("../../../../lib/middleware/serveResources");
 const writeResource = function(writer, path, size, stringContent, project) {
@@ -157,3 +158,143 @@ fame=stra\\u00dfe`);
 		});
 	});
 });
+
+test.serial.cb("Check if version replacement is done", (t) => {
+	const input = "foo ${version} bar";
+	const expected = "foo 1.0.0 bar";
+
+	const resource = {
+		getPath: sinon.stub().returns("/foo.js"),
+		getStatInfo: sinon.stub().returns({
+			ino: 0,
+			ctime: new Date(),
+			mtime: new Date(),
+			size: 1024 * 1024,
+			isDirectory: function() {
+				return false;
+			}
+		}),
+		getStream: () => {
+			const stream = new Readable();
+			stream.push(Buffer.from(input));
+			stream.push(null);
+			return stream;
+		},
+		_project: {
+			version: "1.0.0"
+		}
+	};
+
+	const resources = {
+		all: {
+			byPath: sinon.stub()
+		}
+	};
+	const middleware = serveResourcesMiddleware({
+		resources
+	});
+
+	resources.all.byPath.withArgs("/foo.js").resolves(resource);
+
+	const req = {
+		url: "/foo.js",
+		headers: {}
+	};
+
+	const res = new Writable();
+	const buffers = [];
+	res.setHeader = sinon.stub();
+	res.getHeader = sinon.stub();
+	res._write = function(chunk, encoding, callback) {
+		buffers.push(chunk);
+		callback();
+	};
+	res.end = function() {
+		t.is(Buffer.concat(buffers).toString(), expected);
+		t.end();
+	},
+
+	middleware(req, res, function(err) {
+		if (err) {
+			t.fail("Unexpected error passed to next function: " + err);
+		} else {
+			t.fail("Unexpected call of next function");
+		}
+		t.end();
+	});
+});
+
+// Skip test in Node v8 as unicode handling of streams seems to be broken
+test.serial[
+	process.version.startsWith("v8.") ? "skip" : "cb"
+]("Check if utf8 characters are correctly processed in version replacement", (t) => {
+	const utf8string = "Κυ";
+	const expected = utf8string;
+
+	const resource = {
+		getPath: sinon.stub().returns("/foo.js"),
+		getStatInfo: sinon.stub().returns({
+			ino: 0,
+			ctime: new Date(),
+			mtime: new Date(),
+			size: 1024 * 1024,
+			isDirectory: function() {
+				return false;
+			}
+		}),
+		getStream: () => {
+			const stream = new Readable();
+			const utf8stringAsBuffer = Buffer.from(utf8string, "utf8");
+			// Pushing each byte separately makes content unreadable
+			// if stream encoding is not set to utf8
+			// This might happen when reading large files with utf8 characters
+			stream.push(Buffer.from([utf8stringAsBuffer[0]]));
+			stream.push(Buffer.from([utf8stringAsBuffer[1]]));
+			stream.push(Buffer.from([utf8stringAsBuffer[2]]));
+			stream.push(Buffer.from([utf8stringAsBuffer[3]]));
+			stream.push(null);
+			return stream;
+		},
+		_project: {
+			version: "1.0.0"
+		}
+	};
+
+	const resources = {
+		all: {
+			byPath: sinon.stub()
+		}
+	};
+	const middleware = serveResourcesMiddleware({
+		resources
+	});
+
+	resources.all.byPath.withArgs("/foo.js").resolves(resource);
+
+	const req = {
+		url: "/foo.js",
+		headers: {}
+	};
+
+	const res = new Writable();
+	const buffers = [];
+	res.setHeader = sinon.stub();
+	res.getHeader = sinon.stub();
+	res._write = function(chunk, encoding, callback) {
+		buffers.push(chunk);
+		callback();
+	};
+	res.end = function() {
+		t.is(Buffer.concat(buffers).toString(), expected);
+		t.end();
+	},
+
+	middleware(req, res, function(err) {
+		if (err) {
+			t.fail("Unexpected error passed to next function: " + err);
+		} else {
+			t.fail("Unexpected call of next function");
+		}
+		t.end();
+	});
+});