From cf6ed110291565e8b7b931582b7552fe6db23f89 Mon Sep 17 00:00:00 2001 From: Valkryst Date: Tue, 5 Dec 2023 10:04:42 -0400 Subject: [PATCH] Rewrite code to render `video` stream on `canvas` --- css/styles.css | 62 +++--------------------------------- index.html | 21 +++++------- js/camera.js | 10 ++---- js/detector/body_detector.js | 8 +---- js/mesh_renderer.js | 51 +++++++++++++++++------------ 5 files changed, 47 insertions(+), 105 deletions(-) diff --git a/css/styles.css b/css/styles.css index be752aa..433f134 100644 --- a/css/styles.css +++ b/css/styles.css @@ -3,12 +3,6 @@ html { & > body { height: 100%; - - & > main { - & > canvas, & > video { - position: absolute; - } - } } } @@ -28,61 +22,13 @@ table, th, td { #jellron-display { & > canvas, & > video { position: absolute; - - transform-origin: top left; - transform: scale(-1, 1) translateX(-100%); - } -} - -@media only screen and (min-width: 320px) { - #jellron-display { - & > video { - height: 240px; - width: 320px; - } } -} -@media only screen and (min-width: 640px) { - #jellron-display { - & > video { - height: 480px; - width: 640px; - } + & > canvas { + transform: scaleX(-1); } -} - -@media only screen and (min-width: 800px) { - #jellron-display > video { - height: 600px; - width: 800px; - } -} - -@media only screen and (min-width: 1024px) { - #jellron-display > video { - height: 768px; - width: 1024px; - } -} - -@media only screen and (min-width: 1280px) { - #jellron-display > video { - height: 960px; - width: 1280px; - } -} - -@media only screen and (min-width: 1600px) { - #jellron-display > video { - height: 1200px; - width: 1600px; - } -} -@media only screen and (min-width: 1920px) { - #jellron-display > video { - height: 1440px; - width: 1920px; + & > video { + visibility: hidden; } } \ No newline at end of file diff --git a/index.html b/index.html index d1cace6..acea139 100644 --- a/index.html +++ b/index.html @@ -4,7 +4,7 @@ Jellron - + @@ -224,11 +224,10 @@
+ - -
@@ -274,6 +273,9 @@ }); glContext.autoClear = true; + const deviceSelect = await Camera.getSelectElement(); + document.getElementById("video-input-device-select-container").appendChild(deviceSelect); + const mesh = new Mesh(); const faceDetector = new FaceDetector(); const bodyDetector = new BodyDetector(); @@ -336,7 +338,7 @@ await camera.setVideoElement(document.getElementsByTagName("video")[0]); const videoElement = await camera.getVideoElement(); - glContext.setSize(videoElement.scrollWidth, videoElement.scrollHeight, false); + glContext.setSize(videoElement.width, videoElement.height, false); if (!bodyDetector.isRunning()) { bodyDetector.start(15, videoElement, mesh); @@ -367,20 +369,13 @@ window.onresize = () => updateDisplay(); - const deviceSelect = await Camera.getSelectElement(() => { + deviceSelect.onchange = () => { try { updateDisplay(); } catch (error) { console.error(error); - - bodyDetector.stop(); - faceDetector.stop(); - handDetector.stop(); - meshRenderer.stop(); } - }); - document.getElementById("video-input-device-select-container").appendChild(deviceSelect); - + }; // Register Listeners for Body Settings diff --git a/js/camera.js b/js/camera.js index bb14c2a..7d2989a 100644 --- a/js/camera.js +++ b/js/camera.js @@ -20,13 +20,10 @@ export class Camera { * The select element is configured to automatically update as the set of available devices changes, and when the * camera permissions are changed. * - * @param {function} onChange Function to call when the value of the select element changes. This will replace the existing onChange function. - * * @returns {Promise} A promise that resolves to a select element. */ - static async getSelectElement(onChange = null) { + static async getSelectElement() { if (Camera.selectElement) { - Camera.selectElement.onchange = () => onChange?.(); await Camera.updateSelectElement(); return Camera.selectElement; } @@ -34,7 +31,6 @@ export class Camera { const select = document.createElement("select"); select.disabled = true; select.id = "video-input-device-select"; - select.onchange = () => onChange?.(); Camera.selectElement = select; await Camera.updateSelectElement(); @@ -129,8 +125,8 @@ export class Camera { * Both FaceDetector and HandDetector use the video element's height and width properties to scale the input * MediaStream before processing it. This is why we need to set them. */ - this.videoElement.height = this.videoElement.scrollHeight; - this.videoElement.width = this.videoElement.scrollWidth; + this.videoElement.height = await this.getMediaStreamHeight(); + this.videoElement.width = await this.getMediaStreamWidth(); } /** diff --git a/js/detector/body_detector.js b/js/detector/body_detector.js index e948cf3..e453de4 100644 --- a/js/detector/body_detector.js +++ b/js/detector/body_detector.js @@ -40,20 +40,14 @@ export class BodyDetector extends Detector { this.intervalId = setInterval(async () => { const currentTime = performance.now(); - let frame = null; let rawBodies = []; try { - frame = tf.browser.fromPixels(videoElement); - frame = tf.image.resizeBilinear(frame, [videoElement.height, videoElement.width]); - - rawBodies = await this.detector.estimatePoses(frame); + rawBodies = await this.detector.estimatePoses(videoElement); } catch (e) { /* * Depending on the state of the video element, this can throw a "Requested texture size [0x0] is * invalid." error. It doesn't seem to cause any issues, so we ignore it. */ - } finally { - frame?.dispose(); } if (rawBodies.length === 0) { diff --git a/js/mesh_renderer.js b/js/mesh_renderer.js index 799f429..ef3ef24 100644 --- a/js/mesh_renderer.js +++ b/js/mesh_renderer.js @@ -1,6 +1,7 @@ import {Keypoint} from "./keypoint.js"; import {Mesh} from "./mesh.js"; -import {OrthographicCamera, Scene, WebGLRenderer} from "three"; +import {FrontSide, MeshBasicMaterial, OrthographicCamera, PlaneGeometry, Scene, SRGBColorSpace, VideoTexture, WebGLRenderer} from "three"; +import {Mesh as ThreeMesh} from "three"; import { validateBoolean, validateInstanceOf, validateNonEmptyString, @@ -40,8 +41,9 @@ export class MeshRenderer { const camera = this.createCamera(glContext); const scene = new Scene(); + scene.add(this.createVideoMesh()); - this.intervalId = setInterval(() => { + this.intervalId = setInterval(async () => { const currentTime = performance.now(); for (const keypoint of mesh.getBodyKeypoints()) { @@ -160,6 +162,33 @@ export class MeshRenderer { return camera; } + // todo Cleanup & Document, also pass in params. + createVideoMesh() { + const canvasElement = document.getElementsByTagName("canvas")[0]; + const videoElement = document.getElementsByTagName("video")[0]; + + const videoTexture = new VideoTexture(videoElement); + videoTexture.colorSpace = SRGBColorSpace; + videoTexture.needsUpdate = true; + + const videoMaterial = new MeshBasicMaterial({ + map: videoTexture, + side: FrontSide, + toneMapped: false + }); + videoMaterial.needsUpdate = true; + + const videoMesh = new ThreeMesh( + new PlaneGeometry(canvasElement.scrollWidth, canvasElement.scrollHeight), + videoMaterial + ); + videoMesh.position.x += canvasElement.scrollWidth / 2; + videoMesh.position.y -= canvasElement.scrollHeight / 2; + videoMesh.position.z -= 100; + + return videoMesh; + } + /** * Displays a 2D necklace on the necklace Keypoint. * @@ -188,24 +217,6 @@ export class MeshRenderer { mesh.getEarlobeKeypoints()[isLeft ? 0 : 1].display2DAsset(url); } - /** - * Calculates the distance between two Keypoints. - * - * @param {Keypoint} keypoint1 First Keypoint. - * @param {Keypoint} keypoint2 Second Keypoint. - * - * @returns {number} Distance between the Keypoints. - */ - distanceBetweenKeypoints(keypoint1, keypoint2) { - validateInstanceOf(keypoint1, Keypoint); - validateInstanceOf(keypoint2, Keypoint); - - return Math.sqrt( - Math.pow(keypoint1.getX() - keypoint2.getX(), 2) + - Math.pow(keypoint1.getY() - keypoint2.getY(), 2) - ); - } - /** * Places a Keypoint within a Scene. *