diff --git a/html5/index.html b/html5/index.html
index 5c5c9c06..82115a5e 100644
--- a/html5/index.html
+++ b/html5/index.html
@@ -54,6 +54,9 @@
+
+
+
diff --git a/html5/js/Client.js b/html5/js/Client.js
index 2d73db84..187d8f67 100644
--- a/html5/js/Client.js
+++ b/html5/js/Client.js
@@ -66,7 +66,7 @@ XpraClient.prototype.init_settings = function(container) {
this.supported_encodings = ["jpeg", "png", "rgb", "rgb32", "rgb24"];
//extra encodings we enable if validated via the decode worker:
//(we also validate jpeg and png as a sanity check)
- this.check_encodings = ["webp", "jpeg", "png"]; //"h264", "vp8+webm", "h264+mp4", "mpeg4+mp4"];
+ this.check_encodings = []; //"webp", "jpeg", "png"]; //"h264", "vp8+webm", "h264+mp4", "mpeg4+mp4"];
this.debug_categories = [];
this.start_new_session = null;
this.clipboard_enabled = false;
@@ -91,6 +91,7 @@ XpraClient.prototype.init_settings = function(container) {
this.PING_FREQUENCY = 5000;
this.INFO_FREQUENCY = 1000;
this.uuid = Utilities.getHexUUID();
+ this.offscreen_api = XpraOffscreenWorker.isAvailable();
};
XpraClient.prototype.init_state = function(container) {
@@ -431,7 +432,12 @@ XpraClient.prototype.initialize_workers = function() {
this.decode_worker = false;
return;
}
- const decode_worker = new Worker('js/DecodeWorker.js');
+ let decode_worker;
+ if (this.offscreen_api) {
+ decode_worker = new Worker('js/OffscreenDecodeWorker.js');
+ } else {
+ decode_worker = new Worker('js/DecodeWorker.js');
+ }
decode_worker.addEventListener('message', function(e) {
const data = e.data;
if (data['draw']) {
@@ -3245,31 +3251,35 @@ XpraClient.prototype.do_process_draw = function(packet, start) {
function send_damage_sequence(decode_time, message) {
me.do_send_damage_sequence(packet_sequence, wid, width, height, decode_time, message);
}
+ function decode_result(error) {
+ const flush = options["flush"] || 0;
+ let decode_time = -1;
+ if(flush==0) {
+ me.request_redraw(win);
+ }
+ if (error || start==0) {
+ me.request_redraw(win);
+ }
+ else {
+ decode_time = Math.round(1000*performance.now() - 1000*start);
+ }
+ me.debug("draw", "decode time for ", coding, " sequence ", packet_sequence, ": ", decode_time, ", flush=", flush);
+ send_damage_sequence(decode_time, error || "");
+ }
if (!win) {
this.debug("draw", 'cannot paint, window not found:', wid);
send_damage_sequence(-1, "window "+wid+" not found");
return;
}
+ if (coding=="offscreen-painted") {
+ //we're done!
+ decode_result(0);
+ return;
+ }
try {
win.paint(x, y,
width, height,
- coding, data, packet_sequence, rowstride, options,
- function (error) {
- const flush = options["flush"] || 0;
- let decode_time = -1;
- if(flush==0) {
- me.request_redraw(win);
- }
- if (error || start==0) {
- me.request_redraw(win);
- }
- else {
- decode_time = Math.round(1000*performance.now() - 1000*start);
- }
- me.debug("draw", "decode time for ", coding, " sequence ", packet_sequence, ": ", decode_time, ", flush=", flush);
- send_damage_sequence(decode_time, error || "");
- }
- );
+ coding, data, packet_sequence, rowstride, options, decode_result);
}
catch(e) {
me.exc(e, "error painting", coding, "sequence no", packet_sequence);
diff --git a/html5/js/ImageDecoder.js b/html5/js/ImageDecoder.js
new file mode 100644
index 00000000..301f0f84
--- /dev/null
+++ b/html5/js/ImageDecoder.js
@@ -0,0 +1,50 @@
+/*
+ * This file is part of Xpra.
+ * Copyright (C) 2021 Tijs van der Zwaan
+ * Licensed under MPL 2.0, see:
+ * http://www.mozilla.org/MPL/2.0/
+ *
+ */
+
+/*
+ * Receives native image packages and decode them via ImageDecoder.
+ * https://developer.mozilla.org/en-US/docs/Web/API/ImageDecoder
+ * ImageDecoder is only working in Chrome 94+ and Android
+ *
+ */
+
+const XpraImageDecoderLoader = {
+ hasNativeDecoder: function () {
+ return typeof ImageDecoder !== "undefined";
+ }
+}
+
+function XpraImageDecoder() {
+ this.on_frame_decoded = null;
+}
+
+XpraImageDecoder.prototype.queue_frame = function (packet, start) {
+ const width = packet[4];
+ const height = packet[5];
+ const coding = packet[6];
+ if (coding.startsWith("rgb")) {
+ // TODO: Figure out how to decode rgb with ImageDecoder API;
+ const data = decode_rgb(packet);
+ createImageBitmap(new ImageData(new Uint8ClampedArray(data.buffer), width, height), 0, 0, width, height).then((bitmap) => {
+ packet[6] = "bitmap";
+ packet[7] = bitmap;
+ this.on_frame_decoded(packet, start);
+ });
+ } else {
+ const decoder = new ImageDecoder({
+ type: "image/" + coding,
+ data: packet[7]
+ });
+ decoder.decode({ frameIndex: 0 }).then((result) => {
+ packet[6] = "frame";
+ packet[7] = result;
+ decoder.close();
+ this.on_frame_decoded(packet, start);
+ });
+ }
+};
diff --git a/html5/js/OffscreenDecodeWorker.js b/html5/js/OffscreenDecodeWorker.js
new file mode 100644
index 00000000..fce60a1e
--- /dev/null
+++ b/html5/js/OffscreenDecodeWorker.js
@@ -0,0 +1,185 @@
+/*
+ * This file is part of Xpra.
+ * Copyright (C) 2021 Tijs van der Zwaan
+ * Licensed under MPL 2.0, see:
+ * http://www.mozilla.org/MPL/2.0/
+ *
+ */
+
+/*
+ * Worker for offscreen decoding and painting.
+ * Requires Chrome 94+ or Android and a secure (SSL or localhost) context.
+ */
+
+importScripts("./lib/zlib.js");
+importScripts("./lib/lz4.js");
+importScripts("./lib/broadway/Decoder.js");
+importScripts("./VideoDecoder.js");
+importScripts("./ImageDecoder.js");
+importScripts("./RgbHelpers.js");
+
+// Array of offscreen canvases and video decoders we have control over
+const offscreen_canvas = [];
+const image_decoders = [];
+const video_decoders = [];
+
+function add_decoder_for_window(wid, canvas) {
+ // Canvas
+ offscreen_canvas[wid] = [];
+ offscreen_canvas[wid]["c"] = canvas;
+ offscreen_canvas[wid]["ctx"] = canvas.getContext("2d");
+ offscreen_canvas[wid]["ctx"].imageSmoothingEnabled = false;
+
+ // Decoders
+ image_decoders[wid] = new XpraImageDecoder();
+ image_decoders[wid].on_frame_decoded = ((packet, start) => {
+ const wid = packet[1],
+ x = packet[2],
+ y = packet[3],
+ width = packet[4],
+ height = packet[5],
+ coding = packet[6],
+ data = packet[7];
+
+ let ctx = offscreen_canvas[wid]["ctx"];
+ if (coding == "bitmap") {
+ // RGB is transformed to bitmap
+ ctx.clearRect(x, y, width, height);
+ ctx.drawImage(data, x, y, width, height);
+ } else {
+ // All others are transformed to VideoFrame
+ ctx.clearRect(x, y, width, height);
+ ctx.drawImage(data.image, x, y, width, height);
+ data.image.close();
+ }
+ // Replace the coding & drop data
+ packet[6] = "offscreen-painted";
+ packet[7] = null;
+ self.postMessage({ 'draw': packet, 'start': start }, []);
+ });
+
+ video_decoders[wid] = new XpraVideoDecoder();
+ video_decoders[wid].on_frame_decoded = ((packet, start) => {
+ const wid = packet[1],
+ x = packet[2],
+ y = packet[3],
+ w = packet[4],
+ h = packet[5],
+ coding = packet[6],
+ data = packet[7];
+ let options = packet[10].length > 10 ? packet[10] : {};
+
+ let enc_width = w;
+ let enc_height = h;
+ const scaled_size = options["scaled_size"];
+ if (scaled_size) {
+ enc_width = scaled_size[0];
+ enc_height = scaled_size[1];
+ }
+
+ let ctx = offscreen_canvas[wid]["ctx"];
+ if (coding == "frame") {
+ ctx.drawImage(data, x, y, enc_width, enc_height);
+ data.close();
+ packet[6] = "offscreen-painted";
+ packet[7] = null;
+ self.postMessage({ 'draw': packet, 'start': start }, []);
+ } else {
+ // Encoding throttle is used to slow down frame input
+ // TODO: Relal error handling
+ const timeout = coding == "throttle" ? 500 : 0;
+ setTimeout(() => {
+ packet[6] = "offscreen-painted";
+ packet[7] = null;
+ self.postMessage({ 'draw': packet, 'start': start }, []);
+ }, timeout);
+ }
+ });
+
+}
+
+function decode_draw_packet(packet, start) {
+ const image_coding = ["rgb", "rgb32", "rgb24", "jpeg", "png", "webp"];
+ const video_coding = ["h264"];
+ const wid = packet[1];
+ const coding = packet[6];
+
+ if (image_coding.includes(coding)) {
+ // Add to image queue
+ let decoder = image_decoders[wid];
+ decoder.queue_frame(packet, start);
+
+ } else if (video_coding.includes(coding)) {
+ // Add to video queue
+ let decoder = video_decoders[wid];
+ if (!decoder.initialized) {
+ // Init with width and heigth of this packet.
+ // TODO: Use video max-size? It does not seem to matter.
+ decoder.init(packet[4], packet[5]);
+ }
+ decoder.queue_frame(packet, start);
+ }
+ else if (coding == "scroll") {
+ const data = packet[7];
+ const canvas = offscreen_canvas[wid]["c"];;
+ const ctx = offscreen_canvas[wid]["ctx"];;
+ for (let i = 0, j = data.length; i < j; ++i) {
+ const scroll_data = data[i];
+ const sx = scroll_data[0],
+ sy = scroll_data[1],
+ sw = scroll_data[2],
+ sh = scroll_data[3],
+ xdelta = scroll_data[4],
+ ydelta = scroll_data[5];
+
+ ctx.drawImage(canvas, sx, sy, sw, sh, sx + xdelta, sy + ydelta, sw, sh);
+ }
+ packet[6] = "offscreen-painted";
+ packet[7] = null;
+ self.postMessage({ 'draw': packet, 'start': start }, []);
+ }
+ else {
+ // We dont know, pass trough
+ self.postMessage({ 'draw': packet, 'start': start }, []);
+ }
+}
+
+function close_video(wid) {
+ try {
+ video_decoders[wid]._close();
+ } catch {
+ // TODO: Handle error.
+ }
+}
+
+onmessage = function (e) {
+ const data = e.data;
+ switch (data.cmd) {
+ case 'check':
+ // We do not check. We are here because we support native decoding.
+ // TODO: Reconsider this. It might be a good thing to do some testing, just for sanity??
+ const encodings = data.encodings;
+ self.postMessage({ 'result': true, 'formats': encodings });
+ break;
+ case 'eos':
+ close_video(data.wid);
+ break;
+ case 'decode':
+ decode_draw_packet(data.packet, data.start);
+ break
+ case 'canvas':
+ add_decoder_for_window(data.wid, data.canvas)
+ break;
+ case 'canvas-geo':
+ if (offscreen_canvas[data.wid]) {
+ const canvas = offscreen_canvas[data.wid]["c"];
+ if (canvas.width != data.w || canvas.height != data.h) {
+ canvas.width = data.w;
+ canvas.height = data.h;
+ }
+ }
+ break;
+ default:
+ console.error("Offscreen decode worker got unknown message: " + data.cmd);
+ }
+}
\ No newline at end of file
diff --git a/html5/js/OffscreenDecodeWorkerHelper.js b/html5/js/OffscreenDecodeWorkerHelper.js
new file mode 100644
index 00000000..709a5143
--- /dev/null
+++ b/html5/js/OffscreenDecodeWorkerHelper.js
@@ -0,0 +1,23 @@
+/*
+ * This file is part of Xpra.
+ * Copyright (C) 2021 Tijs van der Zwaan
+ * Licensed under MPL 2.0, see:
+ * http://www.mozilla.org/MPL/2.0/
+ *
+ */
+
+/*
+ * Helper for offscreen decoding and painting.
+ * Requires Chrome 94+ or Android and a secure (SSL or localhost) context.
+ */
+
+const XpraOffscreenWorker = {
+ isAvailable: function () {
+ if (XpraImageDecoderLoader.hasNativeDecoder() && XpraVideoDecoderLoader.hasNativeDecoder && typeof OffscreenCanvas !== "undefined") {
+ return true;
+ } else {
+ console.warn("Offscreen decoding is not available. Please consider using Google Chrome 94+ in a secure (SSL or localhost) context for better performance.");
+ return false;
+ }
+ }
+}
\ No newline at end of file
diff --git a/html5/js/VideoDecoder.js b/html5/js/VideoDecoder.js
new file mode 100644
index 00000000..c53e8b51
--- /dev/null
+++ b/html5/js/VideoDecoder.js
@@ -0,0 +1,158 @@
+/*
+ * This file is part of Xpra.
+ * Copyright (C) 2021 Tijs van der Zwaan
+ * Licensed under MPL 2.0, see:
+ * http://www.mozilla.org/MPL/2.0/
+ *
+ */
+
+/*
+ * Receives native video packages and decode them via VideoDecoder.
+ * https://developer.mozilla.org/en-US/docs/Web/API/VideoDecoder
+ * VideoDecoder is only working in Chrome 94+ and Android
+ *
+ * Example taken from: https://github.com/w3c/webcodecs/blob/main/explainer.md
+ *
+ */
+
+const XpraVideoDecoderLoader = {
+ hasNativeDecoder: function () {
+ return typeof VideoDecoder !== "undefined";
+ }
+}
+
+function XpraVideoDecoder() {
+ this.initialized = false;
+ this.had_first_key = false;
+ this.draining = false;
+
+ this.decoder_queue = [];
+ this.frame_threshold = 250;
+ this.on_frame_decoded = {}; //callback
+}
+
+XpraVideoDecoder.prototype.init = function (width, height) {
+ this.videoDecoder = new VideoDecoder({
+ output: this._on_decoded_frame.bind(this),
+ error: this._on_decoder_error.bind(this)
+ });
+
+ this.videoDecoder.configure({
+ codec: "avc1.42C01E",
+ // hardwareAcceleration: "prefer-hardware",
+ optimizeForLatency: true,
+ codedWidth: width,
+ codedHeight: height
+ });
+ this.last_timestamp = 0;
+ this.initialized = true;
+};
+
+XpraVideoDecoder.prototype._on_decoded_frame = function (videoFrame) {
+ if (this.decoder_queue.length == 0) {
+ videoFrame.close();
+ return;
+ }
+
+ // Find the frame
+ const frame_timestamp = videoFrame.timestamp;
+ let current_frame = this.decoder_queue.filter(q => q.p[10]["frame"] == frame_timestamp);
+ if (current_frame.length == 1) {
+ // We found our frame!
+ this.decoder_queue = this.decoder_queue.filter(q => q.p[10]["frame"] != frame_timestamp);
+ current_frame = current_frame[0];
+ } else {
+ // We decoded a frame the is no longer queued??
+ // TODO: handle error??
+ videoFrame.close();
+ return;
+ }
+
+ if (frame_timestamp == 0) {
+ this.last_timestamp = 0;
+ }
+
+ if ((this.decoder_queue.length > this.frame_threshold) || this.last_timestamp > frame_timestamp) {
+ // Skip if the decoders queue is growing too big or this frames timestamp is smaller then the last one painted.
+ videoFrame.close();
+
+ const packet = current_frame.p;
+ packet[6] = "throttle";
+ packet[7] = null;
+ this.on_frame_decoded(packet, current_frame.start);
+ return;
+ }
+
+ this.last_timestamp = frame_timestamp;
+ const packet = current_frame.p;
+
+ // Latest possible check for draining
+ if (this.draining) {
+ videoFrame.close();
+ return;
+ }
+
+ packet[6] = "frame";
+ packet[7] = videoFrame;
+ this.on_frame_decoded(packet, current_frame.start);
+}
+
+XpraVideoDecoder.prototype._on_decoder_error = function (err) {
+ // TODO: Handle err? Or just assume we will catch up?
+ this._close();
+};
+
+XpraVideoDecoder.prototype.queue_frame = function (packet, start) {
+ let options = packet.length > 10 ? packet[10] : {};
+ const data = packet[7];
+
+ if (!this.had_first_key && options["type"] != "IDR" )
+ {
+ // Need first frame to be a key frame
+ packet[6] = "failed";
+ packet[7] = null;
+ this.on_frame_decoded(packet, start);
+ return;
+ }
+
+ if (this.videoDecoder.state == "closed" || this.draining) {
+ packet[6] = "failed";
+ packet[7] = null;
+ this.on_frame_decoded(packet, start);
+ this._close();
+ return;
+ }
+
+ this.had_first_key = true;
+ this.decoder_queue.push({"p" : packet, start});
+ const init = {
+ type: options["type"] == "IDR" ? "key" : "delta",
+ data: data,
+ timestamp: options["frame"],
+ };
+ const chunk = new EncodedVideoChunk(init);
+ this.videoDecoder.decode(chunk);
+};
+
+XpraVideoDecoder.prototype._close = function () {
+ if (this.initialized) {
+ if (this.videoDecoder.state != "closed")
+ this.videoDecoder.close();
+ this.had_first_key = false;
+
+ // Callback on all frames (bail out)
+ this.draining = true;
+ let drain_queue = this.decoder_queue;
+ this.decoder_queue = [];
+
+ for (let frame of drain_queue) {
+ const packet = frame.p;
+ packet[6] = "failed";
+ packet[7] = null;
+ this.on_frame_decoded(packet, frame.start);
+ };
+ this.draining = false;
+ }
+ this.initialized = false;
+
+};
\ No newline at end of file
diff --git a/html5/js/Window.js b/html5/js/Window.js
index 3a6a7733..6ea83dc3 100644
--- a/html5/js/Window.js
+++ b/html5/js/Window.js
@@ -33,7 +33,7 @@ function XpraWindow(client, canvas_state, wid, x, y, w, h, metadata, override_re
this.debug_categories = client.debug_categories;
//keep reference both the internal canvas and screen drawn canvas:
this.canvas = canvas_state;
- this.canvas_ctx = this.canvas.getContext('2d');
+ this.canvas_ctx = this.client.offscreen_api ? {} : this.canvas.getContext('2d');
this.canvas_ctx.imageSmoothingEnabled = false;
this.offscreen_canvas = null;
this.offscreen_canvas_ctx = null;
@@ -42,6 +42,12 @@ function XpraWindow(client, canvas_state, wid, x, y, w, h, metadata, override_re
this.paint_queue = [];
this.paint_pending = 0;
+ if (this.client.offscreen_api) {
+ // Transfer canvas control. Syntax postMessage({message_obj}, [transferlist])
+ const offscreen_handle = this.canvas.transferControlToOffscreen();
+ this.client.decode_worker.postMessage({'cmd': 'canvas', 'wid' : String(wid), 'canvas': offscreen_handle}, [offscreen_handle]);
+ }
+
//enclosing div in page DOM
this.div = jQuery("#" + String(wid));
@@ -347,6 +353,10 @@ XpraWindow.prototype.ensure_visible = function() {
};
XpraWindow.prototype.updateCanvasGeometry = function() {
+ if (this.client.offscreen_api) {
+ this.client.decode_worker.postMessage({'cmd': 'canvas-geo', 'wid': this.wid, 'w' : this.w, 'h' : this.h});
+ return;
+ }
// set size of both canvas if needed
if(this.canvas.width != this.w) {
this.canvas.width = this.w;