diff --git a/libraries/Camera/examples/CameraCaptureWebSerial/CameraCaptureWebSerial.ino b/libraries/Camera/examples/CameraCaptureWebSerial/CameraCaptureWebSerial.ino new file mode 100644 index 000000000..8347f725a --- /dev/null +++ b/libraries/Camera/examples/CameraCaptureWebSerial/CameraCaptureWebSerial.ino @@ -0,0 +1,160 @@ +/* + * This example shows how to capture images from the camera and send them over Web Serial. + * + * There is a companion web app that receives the images and displays them in a canvas. + * It can be found in the "extras" folder of this library. + * The on-board LED lights up while the image is being sent over serial. + * + * Instructions: + * 1. Make sure the correct camera is selected in the #include section below by uncommenting the correct line. + * 2. Upload this sketch to your camera-equipped board. + * 3. Open the web app in a browser (Chrome or Edge) by opening the index.html file + * in the "WebSerialCamera" folder which is located in the "extras" folder. + * + * Initial author: Sebastian Romero @sebromero + */ + +#include "camera.h" + +#ifdef ARDUINO_NICLA_VISION + #include "gc2145.h" + GC2145 galaxyCore; + Camera cam(galaxyCore); + #define IMAGE_MODE CAMERA_RGB565 +#elif defined(ARDUINO_PORTENTA_H7_M7) + // uncomment the correct camera in use + #include "hm0360.h" + HM0360 himax; + // #include "himax.h"; + // HM01B0 himax; + Camera cam(himax); + #define IMAGE_MODE CAMERA_GRAYSCALE +#elif defined(ARDUINO_GIGA) + #include "ov767x.h" + // uncomment the correct camera in use + OV7670 ov767x; + // OV7675 ov767x; + Camera cam(ov767x); + #define IMAGE_MODE CAMERA_RGB565 +#else +#error "This board is unsupported." +#endif + +/* +Other buffer instantiation options: + FrameBuffer fb(0x30000000); + FrameBuffer fb(320,240,2); + +If resolution higher than 320x240 is required, please use external RAM via + #include "SDRAM.h" + FrameBuffer fb(SDRAM_START_ADDRESS); + ... + // and adding in setup() + SDRAM.begin(); +*/ +constexpr uint16_t CHUNK_SIZE = 512; // Size of chunks in bytes +constexpr uint8_t RESOLUTION = CAMERA_R320x240; // CAMERA_R160x120 +constexpr uint8_t CONFIG_SEND_REQUEST = 2; +constexpr uint8_t IMAGE_SEND_REQUEST = 1; + +uint8_t START_SEQUENCE[4] = { 0xfa, 0xce, 0xfe, 0xed }; +uint8_t STOP_SEQUENCE[4] = { 0xda, 0xbb, 0xad, 0x00 }; +FrameBuffer fb; + +/** + * Blinks the LED a specified number of times. + * @param ledPin The pin number of the LED. + * @param count The number of times to blink the LED. Default is 0xFFFFFFFF. + */ +void blinkLED(int ledPin, uint32_t count = 0xFFFFFFFF) { + while (count--) { + digitalWrite(ledPin, LOW); // turn the LED on (HIGH is the voltage level) + delay(50); // wait for a second + digitalWrite(ledPin, HIGH); // turn the LED off by making the voltage LOW + delay(50); // wait for a second + } +} + +void setup() { + pinMode(LED_BUILTIN, OUTPUT); + pinMode(LEDR, OUTPUT); + digitalWrite(LED_BUILTIN, HIGH); + digitalWrite(LEDR, HIGH); + + // Init the cam QVGA, 30FPS + if (!cam.begin(RESOLUTION, IMAGE_MODE, 30)) { + blinkLED(LEDR); + } + + blinkLED(LED_BUILTIN, 5); +} + +/** + * Sends a chunk of data over a serial connection. + * + * @param buffer The buffer containing the data to be sent. + * @param bufferSize The size of the buffer. + */ +void sendChunk(uint8_t* buffer, size_t bufferSize){ + Serial.write(buffer, bufferSize); + Serial.flush(); + delay(1); // Optional: Add a small delay to allow the receiver to process the chunk +} + +/** + * Sends a frame of camera image data over a serial connection. + */ +void sendFrame(){ + // Grab frame and write to serial + if (cam.grabFrame(fb, 3000) == 0) { + byte* buffer = fb.getBuffer(); + size_t bufferSize = cam.frameSize(); + digitalWrite(LED_BUILTIN, LOW); + + sendChunk(START_SEQUENCE, sizeof(START_SEQUENCE)); + + // Split buffer into chunks + for(size_t i = 0; i < bufferSize; i += CHUNK_SIZE) { + size_t chunkSize = min(bufferSize - i, CHUNK_SIZE); + sendChunk(buffer + i, chunkSize); + } + + sendChunk(STOP_SEQUENCE, sizeof(STOP_SEQUENCE)); + + digitalWrite(LED_BUILTIN, HIGH); + } else { + blinkLED(20); + } +} + +/** + * Sends the camera configuration over a serial connection. + * This is used to configure the web app to display the image correctly. + */ +void sendCameraConfig(){ + Serial.write(IMAGE_MODE); + Serial.write(RESOLUTION); + Serial.flush(); + delay(1); +} + +void loop() { + if(!Serial) { + Serial.begin(115200); + while(!Serial); + } + + if(!Serial.available()) return; + + byte request = Serial.read(); + + switch(request){ + case IMAGE_SEND_REQUEST: + sendFrame(); + break; + case CONFIG_SEND_REQUEST: + sendCameraConfig(); + break; + } + +} diff --git a/libraries/Camera/extras/WebSerialCamera/README.md b/libraries/Camera/extras/WebSerialCamera/README.md new file mode 100644 index 000000000..04888bbcc --- /dev/null +++ b/libraries/Camera/extras/WebSerialCamera/README.md @@ -0,0 +1,12 @@ +# 📹 WebSerial Camera Stream + +This folder contains a web application that provides a camera stream over WebSerial. +This is an experimental feature not supported in all browsers. It's recommended to use Google Chrome. +See [Browser Compatibility](https://developer.mozilla.org/en-US/docs/Web/API/Web_Serial_API#browser_compatibility) + +## Instructions + +1. Upload the companion [Arduino sketch](../../examples/CameraCaptureWebSerial/CameraCaptureWebSerial.ino) to your board. +2. Open the web app either by directly opening the index.html file or serving it via a webserver and opening the URL provided by the webserver. +3. Click "Connect". Your board's serial port should show up in the popup. Select it. Click once again "Connect". The camera feed should start. If the board has been previously connected to the browser, it will connect automatically. +4. (Optional) click "Save Image" if you want to save individual camera frames to your computer. \ No newline at end of file diff --git a/libraries/Camera/extras/WebSerialCamera/app.js b/libraries/Camera/extras/WebSerialCamera/app.js new file mode 100644 index 000000000..68823b347 --- /dev/null +++ b/libraries/Camera/extras/WebSerialCamera/app.js @@ -0,0 +1,135 @@ +/** + * @fileoverview This file contains the main application logic. + * + * The application uses the Web Serial API to connect to the serial port. + * Check the following links for more information on the Web Serial API: + * https://developer.chrome.com/articles/serial/ + * https://wicg.github.io/serial/ + * + * The flow of the application is as follows: + * 1. The user clicks the "Connect" button or the browser automatically connects + * to the serial port if it has been previously connected. + * 2. The application requests the camera configuration (mode and resolution) from the board. + * 3. The application starts reading the image data stream from the serial port. + * It waits until the expected amount of bytes have been read and then processes the data. + * 4. The processed image data is rendered on the canvas. + * + * @author Sebastian Romero + */ + +const connectButton = document.getElementById('connect'); +const refreshButton = document.getElementById('refresh'); +const startButton = document.getElementById('start'); +const saveImageButton = document.getElementById('save-image'); +const canvas = document.getElementById('bitmapCanvas'); +const ctx = canvas.getContext('2d'); + +const imageDataTransfomer = new ImageDataTransformer(ctx); +imageDataTransfomer.setStartSequence([0xfa, 0xce, 0xfe, 0xed]); +imageDataTransfomer.setStopSequence([0xda, 0xbb, 0xad, 0x00]); + +// 🐣 Uncomment one of the following lines to apply a filter to the image data +// imageDataTransfomer.filter = new GrayScaleFilter(); +// imageDataTransfomer.filter = new BlackAndWhiteFilter(); +// imageDataTransfomer.filter = new SepiaColorFilter(); +// imageDataTransfomer.filter = new PixelateFilter(8); +// imageDataTransfomer.filter = new BlurFilter(8); +const connectionHandler = new SerialConnectionHandler(); + + +// Connection handler event listeners + +connectionHandler.onConnect = async () => { + connectButton.textContent = 'Disconnect'; + cameraConfig = await connectionHandler.getConfig(); + if(!cameraConfig){ + console.error('🚫 Could not read camera configuration. Aborting...'); + return; + } + const imageMode = CAMERA_MODES[cameraConfig[0]]; + const imageResolution = CAMERA_RESOLUTIONS[cameraConfig[1]]; + if(!imageMode || !imageResolution){ + console.error(`🚫 Invalid camera configuration: ${cameraConfig[0]}, ${cameraConfig[1]}. Aborting...`); + return; + } + imageDataTransfomer.setImageMode(imageMode); + imageDataTransfomer.setResolution(imageResolution.width, imageResolution.height); + renderStream(); +}; + +connectionHandler.onDisconnect = () => { + connectButton.textContent = 'Connect'; + imageDataTransfomer.reset(); +}; + + +// Rendering logic + +async function renderStream(){ + while(connectionHandler.isConnected()){ + if(imageDataTransfomer.isConfigured()) await renderFrame(); + } +} + +/** + * Renders the image data for one frame from the board and renders it. + * @returns {Promise} True if a frame was rendered, false otherwise. + */ +async function renderFrame(){ + if(!connectionHandler.isConnected()) return; + const imageData = await connectionHandler.getFrame(imageDataTransfomer); + if(!imageData) return false; // Nothing to render + if(!(imageData instanceof ImageData)) throw new Error('🚫 Image data is not of type ImageData'); + renderBitmap(ctx, imageData); + return true; +} + +/** + * Renders the image data on the canvas. + * @param {CanvasRenderingContext2D} context The canvas context to render on. + * @param {ImageData} imageData The image data to render. + */ +function renderBitmap(context, imageData) { + context.canvas.width = imageData.width; + context.canvas.height = imageData.height; + context.clearRect(0, 0, canvas.width, canvas.height); + context.putImageData(imageData, 0, 0); +} + + +// UI Event listeners + +startButton.addEventListener('click', renderStream); + +connectButton.addEventListener('click', async () => { + if(connectionHandler.isConnected()){ + connectionHandler.disconnectSerial(); + } else { + await connectionHandler.requestSerialPort(); + await connectionHandler.connectSerial(); + } +}); + +refreshButton.addEventListener('click', () => { + if(imageDataTransfomer.isConfigured()) renderFrame(); +}); + +saveImageButton.addEventListener('click', () => { + const link = document.createElement('a'); + link.download = 'image.png'; + link.href = canvas.toDataURL(); + link.click(); + link.remove(); +}); + +// On page load event, try to connect to the serial port +window.addEventListener('load', async () => { + console.log('🚀 Page loaded. Trying to connect to serial port...'); + setTimeout(() => { + connectionHandler.autoConnect(); + }, 1000); +}); + +if (!("serial" in navigator)) { + alert("The Web Serial API is not supported in your browser."); +} \ No newline at end of file diff --git a/libraries/Camera/extras/WebSerialCamera/cameraConfig.js b/libraries/Camera/extras/WebSerialCamera/cameraConfig.js new file mode 100644 index 000000000..1c0474473 --- /dev/null +++ b/libraries/Camera/extras/WebSerialCamera/cameraConfig.js @@ -0,0 +1,51 @@ +/** + * @fileoverview This file contains the configuration for the camera. + * @author Sebastian Romero + */ + +/** + * The available camera (color) modes. + * The Arduino sketch uses the same values to communicate which mode should be used. + **/ +const CAMERA_MODES = { + 0: "GRAYSCALE", + 1: "BAYER", + 2: "RGB565" +}; + +/** + * The available camera resolutions. + * The Arduino sketch uses the same values to communicate which resolution should be used. + */ +const CAMERA_RESOLUTIONS = { + 0: { + "name": "QQVGA", + "width": 160, + "height": 120 + }, + 1: { + "name": "QVGA", + "width": 320, + "height": 240 + }, + 2: { + "name": "320x320", + "width": 320, + "height": 320 + }, + 3: { + "name": "VGA", + "width": 640, + "height": 480 + }, + 5: { + "name": "SVGA", + "width": 800, + "height": 600 + }, + 6: { + "name": "UXGA", + "width": 1600, + "height": 1200 + } +}; \ No newline at end of file diff --git a/libraries/Camera/extras/WebSerialCamera/filters.js b/libraries/Camera/extras/WebSerialCamera/filters.js new file mode 100644 index 000000000..0b7622898 --- /dev/null +++ b/libraries/Camera/extras/WebSerialCamera/filters.js @@ -0,0 +1,207 @@ +/** + * @fileoverview This file contains the filters that can be applied to an image. + * @author Sebastian Romero + */ + +/** + * Represents an image filter interface. This class is meant to be extended by subclasses. + */ +class ImageFilter { +/** + * Applies a filter to the given pixel data. + * @param {Uint8Array} pixelData - The pixel data to apply the filter to. The pixel data gets modified in place. + * @param {number} [width=null] - The width of the image. Defaults to null. + * @param {number} [height=null] - The height of the image. Defaults to null. + * @throws {Error} - Throws an error if the applyFilter method is not implemented. + */ + applyFilter(pixelData, width = null, height = null) { + throw new Error('applyFilter not implemented'); + } +} + +/** + * Represents a grayscale filter that converts an image to grayscale. + * @extends ImageFilter + */ +class GrayScaleFilter extends ImageFilter { + /** + * Applies the grayscale filter to the given pixel data. + * @param {Uint8ClampedArray} pixelData - The pixel data to apply the filter to. + * @param {number} [width=null] - The width of the image. + * @param {number} [height=null] - The height of the image. + */ + applyFilter(pixelData, width = null, height = null) { + for (let i = 0; i < pixelData.length; i += 4) { + const r = pixelData[i]; + const g = pixelData[i + 1]; + const b = pixelData[i + 2]; + const gray = (r + g + b) / 3; + pixelData[i] = gray; + pixelData[i + 1] = gray; + pixelData[i + 2] = gray; + } + } +} + +/** + * A class representing a black and white image filter. + * @extends ImageFilter + */ +class BlackAndWhiteFilter extends ImageFilter { + applyFilter(pixelData, width = null, height = null) { + for (let i = 0; i < pixelData.length; i += 4) { + const r = pixelData[i]; + const g = pixelData[i + 1]; + const b = pixelData[i + 2]; + const gray = (r + g + b) / 3; + const bw = gray > 127 ? 255 : 0; + pixelData[i] = bw; + pixelData[i + 1] = bw; + pixelData[i + 2] = bw; + } + } +} + +/** + * Represents a color filter that applies a sepia tone effect to an image. + * @extends ImageFilter + */ +class SepiaColorFilter extends ImageFilter { + applyFilter(pixelData, width = null, height = null) { + for (let i = 0; i < pixelData.length; i += 4) { + const r = pixelData[i]; + const g = pixelData[i + 1]; + const b = pixelData[i + 2]; + const gray = (r + g + b) / 3; + pixelData[i] = gray + 100; + pixelData[i + 1] = gray + 50; + pixelData[i + 2] = gray; + } + } +} + +/** + * Represents a filter that applies a pixelation effect to an image. + * @extends ImageFilter + */ +class PixelateFilter extends ImageFilter { + + constructor(blockSize = 8){ + super(); + this.blockSize = blockSize; + } + + applyFilter(pixelData, width, height) { + for (let y = 0; y < height; y += this.blockSize) { + for (let x = 0; x < width; x += this.blockSize) { + const blockAverage = this.getBlockAverage(x, y, width, height, pixelData, this.blockSize); + + // Set all pixels in the block to the calculated average color + for (let blockY = 0; blockY < this.blockSize && y + blockY < height; blockY++) { + for (let blockX = 0; blockX < this.blockSize && x + blockX < width; blockX++) { + const pixelIndex = ((y + blockY) * width + (x + blockX)) * 4; + pixelData[pixelIndex] = blockAverage.red; + pixelData[pixelIndex + 1] = blockAverage.green; + pixelData[pixelIndex + 2] = blockAverage.blue; + } + } + } + } + } + + /** + * Calculates the average RGB values of a block of pixels. + * + * @param {number} x - The x-coordinate of the top-left corner of the block. + * @param {number} y - The y-coordinate of the top-left corner of the block. + * @param {number} width - The width of the image. + * @param {number} height - The height of the image. + * @param {Uint8ClampedArray} pixels - The array of pixel data. + * @returns {Object} - An object containing the average red, green, and blue values. + */ + getBlockAverage(x, y, width, height, pixels) { + let totalRed = 0; + let totalGreen = 0; + let totalBlue = 0; + const blockSizeSquared = this.blockSize * this.blockSize; + + for (let blockY = 0; blockY < this.blockSize && y + blockY < height; blockY++) { + for (let blockX = 0; blockX < this.blockSize && x + blockX < width; blockX++) { + const pixelIndex = ((y + blockY) * width + (x + blockX)) * 4; + totalRed += pixels[pixelIndex]; + totalGreen += pixels[pixelIndex + 1]; + totalBlue += pixels[pixelIndex + 2]; + } + } + + return { + red: totalRed / blockSizeSquared, + green: totalGreen / blockSizeSquared, + blue: totalBlue / blockSizeSquared, + }; + } + +} + +/** + * Represents a filter that applies a blur effect to an image. + * @extends ImageFilter + */ +class BlurFilter extends ImageFilter { + constructor(radius = 8) { + super(); + this.radius = radius; + } + + applyFilter(pixelData, width, height) { + for (let y = 0; y < height; y++) { + for (let x = 0; x < width; x++) { + const pixelIndex = (y * width + x) * 4; + + const averageColor = this.getAverageColor(x, y, width, height, pixelData, this.radius); + pixelData[pixelIndex] = averageColor.red; + pixelData[pixelIndex + 1] = averageColor.green; + pixelData[pixelIndex + 2] = averageColor.blue; + } + } + } + + /** + * Calculates the average color of a rectangular region in an image. + * + * @param {number} x - The x-coordinate of the top-left corner of the region. + * @param {number} y - The y-coordinate of the top-left corner of the region. + * @param {number} width - The width of the region. + * @param {number} height - The height of the region. + * @param {Uint8ClampedArray} pixels - The pixel data of the image. + * @param {number} radius - The radius of the neighborhood to consider for each pixel. + * @returns {object} - An object representing the average color of the region, with red, green, and blue components. + */ + getAverageColor(x, y, width, height, pixels, radius) { + let totalRed = 0; + let totalGreen = 0; + let totalBlue = 0; + let pixelCount = 0; + + for (let offsetY = -radius; offsetY <= radius; offsetY++) { + for (let offsetX = -radius; offsetX <= radius; offsetX++) { + const neighborX = x + offsetX; + const neighborY = y + offsetY; + + if (neighborX >= 0 && neighborX < width && neighborY >= 0 && neighborY < height) { + const pixelIndex = (neighborY * width + neighborX) * 4; + totalRed += pixels[pixelIndex]; + totalGreen += pixels[pixelIndex + 1]; + totalBlue += pixels[pixelIndex + 2]; + pixelCount++; + } + } + } + + return { + red: totalRed / pixelCount, + green: totalGreen / pixelCount, + blue: totalBlue / pixelCount, + }; + } +} diff --git a/libraries/Camera/extras/WebSerialCamera/imageDataProcessor.js b/libraries/Camera/extras/WebSerialCamera/imageDataProcessor.js new file mode 100644 index 000000000..83ff2a650 --- /dev/null +++ b/libraries/Camera/extras/WebSerialCamera/imageDataProcessor.js @@ -0,0 +1,162 @@ +/** + * Represents an image data processor that converts raw image data to a specified pixel format. + * + * @author Sebastian Romero + */ +class ImageDataProcessor { + pixelFormatInfo = { + "RGB565": { + "convert": this.convertRGB565ToRGB888, + "bytesPerPixel": 2 + }, + "GRAYSCALE": { + "convert": this.convertGrayScaleToRGB888, + "bytesPerPixel": 1 + }, + "RGB888": { + "convert": this.convertToRGB888, + "bytesPerPixel": 3 + }, + "BAYER": { + "convert": () => {throw new Error("BAYER conversion not implemented.")}, + "bytesPerPixel": 1 + } + }; + + /** + * Creates a new instance of the imageDataProcessor class. + * @param {string|null} mode - The image mode of the image data processor. (Optional) + * Possible values: RGB565, GRAYSCALE, RGB888, BAYER + * @param {number|null} width - The width of the image data processor. (Optional) + * @param {number|null} height - The height of the image data processor. (Optional) + */ + constructor(mode = null, width = null, height = null) { + if(mode) this.setImageMode(mode); + if(width && height) this.setResolution(width, height); + } + + /** + * Sets the image mode of the image data processor. + * Possible values: RGB565, GRAYSCALE, RGB888, BAYER + * + * @param {string} mode - The image mode of the image data processor. + */ + setImageMode(mode) { + this.mode = mode; + this.bytesPerPixel = this.pixelFormatInfo[mode].bytesPerPixel; + } + + /** + * Sets the resolution of the target image. + * @param {number} width - The width of the resolution. + * @param {number} height - The height of the resolution. + */ + setResolution(width, height) { + this.width = width; + this.height = height; + } + + /** + * Calculates the total number of bytes in the image data + * based on the current image mode and resolution. + * + * @returns {number} The total number of bytes. + */ + getTotalBytes() { + return this.width * this.height * this.bytesPerPixel; + } + + /** + * Resets the state of the imageDataProcessor. + * This resets the image mode, resolution, and bytes per pixel. + */ + reset() { + this.mode = null; + this.bytesPerPixel = null; + this.width = null; + this.height = null; + } + + /** + * Converts a pixel value from RGB565 format to RGB888 format. + * @param {number} pixelValue - The pixel value in RGB565 format. + * @returns {number[]} - The RGB888 pixel value as an array of three values [R, G, B]. + */ + convertRGB565ToRGB888(pixelValue) { + // RGB565 + let r = (pixelValue >> (6 + 5)) & 0x1F; + let g = (pixelValue >> 5) & 0x3F; + let b = pixelValue & 0x1F; + // RGB888 - amplify + r <<= 3; + g <<= 2; + b <<= 3; + return [r, g, b]; + } + + /** + * Converts a grayscale pixel value to RGB888 format. + * @param {number} pixelValue - The grayscale pixel value. + * @returns {number[]} - The RGB888 pixel value as an array of three values [R, G, B]. + */ + convertGrayScaleToRGB888(pixelValue) { + return [pixelValue, pixelValue, pixelValue]; + } + + /** + * Converts a pixel value to RGB888 format. + * @param {number} pixelValue - The pixel value to convert. + * @returns {number[]} - The RGB888 pixel value as an array of three values [R, G, B]. + */ + convertToRGB888(pixelValue){ + return pixelValue; + } + + /** + * Retrieves the pixel value from the source data at the specified index + * using big endian: the most significant byte comes first. + * + * @param {Uint8Array} sourceData - The source data array. + * @param {number} index - The index of the pixel value in the source data array. + * @returns {number} The pixel value. + */ + getPixelValue(sourceData, index) { + if (this.bytesPerPixel == 1) { + return sourceData[index]; + } else if (this.bytesPerPixel == 2) { + return (sourceData[index] << 8) | sourceData[index + 1]; + } else if (this.bytesPerPixel == 3) { + return (sourceData[index] << 16) | (sourceData[index + 1] << 8) | sourceData[index + 2]; + } else if (this.bytesPerPixel == 4) { + return (sourceData[index] << 24) | (sourceData[index + 1] << 16) | (sourceData[index + 2] << 8) | sourceData[index + 3]; + } + + return 0; + } + + /** + * Retrieves the image data from the given bytes by converting each pixel value. + * + * @param {Uint8Array} bytes - The raw byte array containing the image data. + * @returns {Uint8ClampedArray} The image data as a Uint8ClampedArray containing RGBA values. + */ + convertToPixelData(bytes) { + const BYTES_PER_ROW = this.width * this.bytesPerPixel; + const dataContainer = new Uint8ClampedArray(this.width * this.height * 4); // 4 channels: R, G, B, A + + for (let row = 0; row < this.height; row++) { + for (let col = 0; col < this.width; col++) { + const sourceDataIndex = (row * BYTES_PER_ROW) + (col * this.bytesPerPixel); + const pixelValue = this.getPixelValue(bytes, sourceDataIndex, this.bytesPerPixel); + const [r, g, b] = this.pixelFormatInfo[this.mode].convert(pixelValue); + + const pixelIndex = ((row * this.width) + col) * 4; // 4 channels: R, G, B, A + dataContainer[pixelIndex] = r; // Red channel + dataContainer[pixelIndex + 1] = g; // Green channel + dataContainer[pixelIndex + 2] = b; // Blue channel + dataContainer[pixelIndex + 3] = 255; // Alpha channel (opacity) + } + } + return dataContainer; + } + } \ No newline at end of file diff --git a/libraries/Camera/extras/WebSerialCamera/index.html b/libraries/Camera/extras/WebSerialCamera/index.html new file mode 100644 index 000000000..3f677a2ff --- /dev/null +++ b/libraries/Camera/extras/WebSerialCamera/index.html @@ -0,0 +1,26 @@ + + + + + + Web Serial Bitmap Reader + + + +
+ +
+ + + + +
+
+ + + + + + + + diff --git a/libraries/Camera/extras/WebSerialCamera/serialConnectionHandler.js b/libraries/Camera/extras/WebSerialCamera/serialConnectionHandler.js new file mode 100644 index 000000000..6f23c644f --- /dev/null +++ b/libraries/Camera/extras/WebSerialCamera/serialConnectionHandler.js @@ -0,0 +1,285 @@ +/** + * @fileoverview This file contains the SerialConnectionHandler class. + * It handles the connection between the browser and the Arduino board via Web Serial. + * @author Sebastian Romero + */ + +const ArduinoUSBVendorId = 0x2341; +const UserActionAbortError = 8; + +/** + * Handles the connection between the browser and the Arduino board via Web Serial. + * Please note that for board with software serial over USB, the baud rate and other serial settings have no effect. + */ +class SerialConnectionHandler { + /** + * Represents a serial connection handler. + * @constructor + * @param {number} [baudRate=115200] - The baud rate of the serial connection. + * @param {number} [dataBits=8] - The number of data bits. + * @param {number} [stopBits=1] - The number of stop bits. + * @param {string} [parity="none"] - The parity setting. + * @param {string} [flowControl="none"] - The flow control setting. + * @param {number} [bufferSize=2097152] - The size of the buffer in bytes. The default value is 2 MB. Max buffer size is 16MB. + * @param {number} [timeout=2000] - The connection timeout value in milliseconds. The default value is 2000 ms. + */ + constructor(baudRate = 115200, dataBits = 8, stopBits = 1, parity = "none", flowControl = "none", bufferSize = 2 * 1024 * 1024, timeout = 2000) { + this.baudRate = baudRate; + this.dataBits = dataBits; + this.stopBits = stopBits; + this.flowControl = flowControl; + this.bufferSize = bufferSize; + this.parity = parity; + this.timeout = timeout; + this.currentPort = null; + this.currentReader = null; + this.currentTransformer = null; + this.readableStreamClosed = null; + this.registerEvents(); + } + + /** + * Sets the connection timeout for the serial connection. + * @param {number} timeout - The timeout value in milliseconds. + */ + setConnectionTimeout(timeout) { + this.timeout = timeout; + } + + /** + * Prompts the user to select a serial port. + * @returns {Promise} The serial port that the user has selected. + */ + async requestSerialPort() { + try { + const port = await navigator.serial.requestPort({ filters: [{ usbVendorId: ArduinoUSBVendorId }] }); + this.currentPort = port; + return port; + } catch (error) { + if (error.code != UserActionAbortError) { + console.log(error); + } + return null; + } + } + + /** + * Checks if the browser is connected to a serial port. + * @returns {boolean} True if the browser is connected, false otherwise. + */ + isConnected() { + return this.currentPort?.readable != null; + } + + /** + * Opens a connection to the given serial port by using the settings specified in the constructor. + * If the port is already open, it will be closed first. + * This method will call the `onConnect` callback before it returns. + * @returns {boolean} True if the connection was successfully opened, false otherwise. + */ + async connectSerial() { + try { + // If the port is already open, close it + if (this.isConnected()) await this.currentPort.close(); + await this.currentPort.open({ + baudRate: this.baudRate, + parity: this.parity, + dataBits: this.dataBits, + stopBits: this.stopBits, + bufferSize: this.bufferSize, + flowControl: this.flowControl + }); + console.log('✅ Connected to serial port.'); + if(this.onConnect) this.onConnect(); + return true; + } catch (error) { + return false; + } + } + + /** + * Disconnects from the current serial port. + * If a reading operation is in progress, it will be canceled. + * This function will call the `onDisconnect` callback before it returns. + * @returns {Promise} A promise that resolves when the port has been closed. + */ + async disconnectSerial() { + if (!this.currentPort) return; + try { + const port = this.currentPort; + this.currentPort = null; + await this.currentReader?.cancel(); + await this.readableStreamClosed.catch(() => { }); // Ignores the error + this.currentTransformer?.flush(); + await port.close(); + console.log('🔌 Disconnected from serial port.'); + if(this.onDisconnect) this.onDisconnect(); + } catch (error) { + console.error('💣 Error occurred while disconnecting: ' + error.message); + }; + } + + /** + * Tries to establish a connection to the first available serial port that has the Arduino USB vendor ID. + * This only works if the user has previously granted the website access to that serial port. + * @returns {Promise} True if the connection was successfully opened, false otherwise. + */ + async autoConnect() { + if (this.currentPort) { + console.log('🔌 Already connected to a serial port.'); + return false; + } + + // Get all serial ports the user has previously granted the website access to. + const ports = await navigator.serial.getPorts(); + + for (const port of ports) { + console.log('👀 Serial port found with VID: 0x' + port.getInfo().usbVendorId.toString(16)); + if (port.getInfo().usbVendorId === ArduinoUSBVendorId) { + this.currentPort = port; + return await this.connectSerial(this.currentPort); + } + } + return false; + } + + + /** + * Reads a specified number of bytes from the serial connection. + * @param {number} numBytes - The number of bytes to read. + * @returns {Promise} - A promise that resolves to a Uint8Array containing the read bytes. + */ + async readBytes(numBytes) { + return await this.readData(new BytesWaitTransformer(numBytes)); + } + + /** + * Reads the specified number of bytes from the serial port. + * @param {Transformer} transformer The transformer that is used to process the bytes. + * If the timeout is reached, the reader will be canceled and the read lock will be released. + */ + async readData(transformer) { + if(!transformer) throw new Error('Transformer is null'); + if(!this.currentPort) return null; + if(this.currentPort.readable.locked) { + console.log('🔒 Stream is already locked. Ignoring request...'); + return null; + } + + const transformStream = new TransformStream(transformer); + this.currentTransformer = transformer; + // pipeThrough() cannot be used because we need a promise that resolves when the stream is closed + // to be able to close the port. pipeTo() returns such a promise. + // SEE: https://stackoverflow.com/questions/71262432/how-can-i-close-a-web-serial-port-that-ive-piped-through-a-transformstream + this.readableStreamClosed = this.currentPort.readable.pipeTo(transformStream.writable); + const reader = transformStream.readable.getReader(); + this.currentReader = reader; + let timeoutID = null; + + try { + if (this.timeout) { + timeoutID = setTimeout(() => { + console.log('⌛️ Timeout occurred while reading.'); + if (this.currentPort?.readable) reader?.cancel(); + this.currentTransformer.flush(); + }, this.timeout); + } + const { value, done } = await reader.read(); + if (timeoutID) clearTimeout(timeoutID); + + if (done) { + console.log('🚫 Reader has been canceled'); + return null; + } + return value; + } catch (error) { + console.error('💣 Error occurred while reading: ' + error.message); + } finally { + // console.log('🔓 Releasing reader lock...'); + await reader?.cancel(); // Discards any enqueued data + await this.readableStreamClosed.catch(() => { }); // Ignores the error + reader?.releaseLock(); + this.currentReader = null; + this.currentTransformer = null; + } + } + + /** + * Sends the provided byte array data through the current serial port. + * + * @param {ArrayBuffer} byteArray - The byte array data to send. + * @returns {Promise} - A promise that resolves when the data has been sent. + */ + async sendData(byteArray) { + if (!this.currentPort?.writable) { + console.log('🚫 Port is not writable. Ignoring request...'); + return; + } + const writer = this.currentPort.writable.getWriter(); + await writer.write(new Uint8Array(byteArray)); + await writer.close(); + } + + /** + * Reqests an image frame from the Arduino board by writing a 1 to the serial port. + * @returns {Promise} A promise that resolves when the frame has been requested and the write stream has been closed. + */ + async requestFrame() { + // console.log('Writing 1 to the serial port...'); + // Write a 1 to the serial port + return this.sendData([1]); + } + + /** + * Requests the camera configuration from the board by writing a 2 to the serial port. + * @returns {Promise} A promise that resolves with the configuration data. + */ + async requestConfig() { + return this.sendData([2]); + } + + /** + * Requests the camera resolution from the board and reads it back from the serial port. + * The configuration simply consists of two bytes: the mode and the resolution. + * @returns {Promise} The raw configuration data as an ArrayBuffer. + */ + async getConfig() { + if (!this.currentPort) return; + + await this.requestConfig(); + // console.log(`Trying to read 2 bytes...`); + return await this.readBytes(2, this.timeout); + } + + /** + * Requests a frame from the Arduino board and reads the specified number of bytes from the serial port afterwards. + * Times out after the timeout in milliseconds specified in the constructor. + * @param {Transformer} transformer The transformer that is used to process the bytes. + */ + async getFrame(transformer) { + if (!this.currentPort) return; + await this.requestFrame(); + return await this.readData(transformer, this.timeout); + } + + /** + * Registers event listeners for the `connect` and `disconnect` events of the serial port. + * The `connect` event is fired when a serial port becomes available not when it is opened. + * When the `connect` event is fired, `autoConnect()` is called. + * The `disconnect` event is fired when a serial port is lost. + * When the `disconnect` event is fired, the `onDisconnect` callback is called. + **/ + registerEvents() { + navigator.serial.addEventListener("connect", (e) => { + // Connect to `e.target` or add it to a list of available ports. + console.log('🔌 Serial port became available. VID: 0x' + e.target.getInfo().usbVendorId.toString(16)); + this.autoConnect(); + }); + + navigator.serial.addEventListener("disconnect", (e) => { + console.log('❌ Serial port lost. VID: 0x' + e.target.getInfo().usbVendorId.toString(16)); + this.currentPort = null; + if(this.onDisconnect) this.onDisconnect(); + }); + } +} \ No newline at end of file diff --git a/libraries/Camera/extras/WebSerialCamera/style.css b/libraries/Camera/extras/WebSerialCamera/style.css new file mode 100644 index 000000000..ef9f7832f --- /dev/null +++ b/libraries/Camera/extras/WebSerialCamera/style.css @@ -0,0 +1,67 @@ +:root { + --main-control-color: #008184; + --main-control-color-hover: #005c5f; + --main-flexbox-gap: 16px; + --secondary-text-color: #87898b; +} + +html { + font-size: 14px; +} + +body { + font-family: 'Open Sans', sans-serif; + text-align: center; +} + +#main-container { + display: flex; + flex-direction: column; + align-items: center; + gap: 1rem; + margin-top: 20px; +} + +#controls { + display: flex; + flex-direction: row; + align-items: center; + gap: 1rem; + margin-top: 20px; +} + +canvas { + border-radius: 5px; +} + +button { + font-family: 'Open Sans', sans-serif; + font-weight: 700; + font-size: 1rem; + justify-content: center; + background-color: var(--main-control-color); + color: #fff; + cursor: pointer; + letter-spacing: 1.28px; + line-height: normal; + outline: none; + padding: 8px 18px; + text-align: center; + text-decoration: none; + border: 2px solid transparent; + border-radius: 32px; + text-transform: uppercase; + box-sizing: border-box; +} + +button:hover { + background-color: var(--main-control-color-hover); +} + +#refresh { + display: none; +} + +#start { + display: none; +} \ No newline at end of file diff --git a/libraries/Camera/extras/WebSerialCamera/transformers.js b/libraries/Camera/extras/WebSerialCamera/transformers.js new file mode 100644 index 000000000..722353c20 --- /dev/null +++ b/libraries/Camera/extras/WebSerialCamera/transformers.js @@ -0,0 +1,329 @@ +/** + * @fileoverview This file contains classes that transform incoming data into higher-level data types. + * @author Sebastian Romero + */ + + +/** + * Represents a transformer that processes incoming data between start and stop sequences. + */ +class StartStopSequenceTransformer { + constructor(startSequence = null, stopSequence = null, expectedBytes = null) { + this.startSequence = new Uint8Array(startSequence); + this.stopSequence = new Uint8Array(stopSequence); + this.expectedBytes = expectedBytes; + this.buffer = new Uint8Array(0); + this.controller = undefined; + this.waitingForStart = true; + } + + /** + * Sets the start sequence for the received data. + * This is used to disregard any data before the start sequence. + * @param {Array} startSequence - The start sequence as an array of numbers. + */ + setStartSequence(startSequence) { + this.startSequence = new Uint8Array(startSequence); + } + + /** + * Sets the stop sequence for the received data. + * This is used to know when the data has finished being sent and should be processed. + * @param {Array} stopSequence - The stop sequence as an array of numbers. + */ + setStopSequence(stopSequence) { + this.stopSequence = new Uint8Array(stopSequence); + } + + /** + * Sets the expected number of bytes for the received data. + * This is used to check if the number of bytes matches the expected amount + * and discard the data if it doesn't. + * + * @param {number} expectedBytes - The expected number of bytes. + */ + setExpectedBytes(expectedBytes) { + this.expectedBytes = expectedBytes; + } + + /** + * Transforms the incoming chunk of data and enqueues the processed bytes to the controller + * between start and stop sequences. + * + * @param {Uint8Array} chunk - The incoming chunk of data. + * @param {TransformStreamDefaultController} controller - The controller for enqueuing processed bytes. + * @returns {Promise} - A promise that resolves when the transformation is complete. + */ + async transform(chunk, controller) { + this.controller = controller; + + // Concatenate incoming chunk with existing buffer + this.buffer = new Uint8Array([...this.buffer, ...chunk]); + let startIndex = 0; + + // Only process data if at least one start and stop sequence is present in the buffer + const minimumRequiredBytes = Math.min(this.startSequence.length, this.stopSequence.length); + + while (this.buffer.length >= minimumRequiredBytes) { + if (this.waitingForStart) { + // Look for the start sequence + startIndex = this.indexOfSequence(this.buffer, this.startSequence, startIndex); + + if (startIndex === -1) { + // No start sequence found, discard the buffer + this.buffer = new Uint8Array(0); + return; + } + + // Remove bytes before the start sequence including the start sequence + this.buffer = this.buffer.slice(startIndex + this.startSequence.length); + startIndex = 0; // Reset startIndex after removing bytes + this.waitingForStart = false; + } + + // Look for the stop sequence + const stopIndex = this.indexOfSequence(this.buffer, this.stopSequence, startIndex); + + if (stopIndex === -1) { + // No stop sequence found, wait for more data + return; + } + + // Extract bytes between start and stop sequences + const bytesToProcess = this.buffer.slice(startIndex, stopIndex); + // Remove processed bytes from the buffer including the stop sequence. + this.buffer = this.buffer.slice(stopIndex + this.stopSequence.length); + + // Check if the number of bytes matches the expected amount + if (this.expectedBytes !== null && bytesToProcess.length !== this.expectedBytes) { + // Skip processing the bytes, but keep the remaining data in the buffer + console.error(`🚫 Expected ${this.expectedBytes} bytes, but got ${bytesToProcess.length} bytes instead. Dropping data.`); + this.waitingForStart = true; + return; + } + + // Notify the controller that bytes have been processed + controller.enqueue(this.convertBytes(bytesToProcess)); + this.waitingForStart = true; + } + } + + /** + * Flushes the buffer and discards any remaining bytes when the stream is closed. + * + * @param {WritableStreamDefaultController} controller - The controller for the writable stream. + */ + flush(controller) { + // Discard the remaining data in the buffer + this.buffer = new Uint8Array(0); + } + + + /** + * Finds the index of the given sequence in the buffer. + * + * @param {Uint8Array} buffer - The buffer to search. + * @param {Uint8Array} sequence - The sequence to find. + * @param {number} startIndex - The index to start searching from. + * @returns {number} - The index of the sequence in the buffer, or -1 if not found. + */ + indexOfSequence(buffer, sequence, startIndex) { + for (let i = startIndex; i <= buffer.length - sequence.length; i++) { + if (this.isSubarray(buffer, sequence, i)) { + return i; + } + } + return -1; + } + + /** + * Checks if a subarray is present at a given index in the buffer. + * + * @param {Uint8Array} buffer - The buffer to check. + * @param {Uint8Array} subarray - The subarray to check. + * @param {number} index - The index to start checking from. + * @returns {boolean} - True if the subarray is present at the given index, false otherwise. + */ + isSubarray(buffer, subarray, index) { + for (let i = 0; i < subarray.length; i++) { + if (buffer[index + i] !== subarray[i]) { + return false; + } + } + return true; + } + + /** + * Converts bytes into higher-level data types. + * This method is meant to be overridden by subclasses. + * @param {Uint8Array} bytes + * @returns + */ + convertBytes(bytes) { + return bytes; + } + +} + + +/** + * A transformer class that waits for a specific number of bytes before processing them. + */ +class BytesWaitTransformer { + constructor(waitBytes = 1) { + this.waitBytes = waitBytes; + this.buffer = new Uint8Array(0); + this.controller = undefined; + } + + /** + * Sets the number of bytes to wait before processing the data. + * @param {number} waitBytes - The number of bytes to wait. + */ + setBytesToWait(waitBytes) { + this.waitBytes = waitBytes; + } + + /** + * Converts bytes into higher-level data types. + * This method is meant to be overridden by subclasses. + * @param {Uint8Array} bytes + * @returns + */ + convertBytes(bytes) { + return bytes; + } + + /** + * Transforms the incoming chunk of data and enqueues the processed bytes to the controller. + * It does so when the buffer contains at least the specified number of bytes. + * @param {Uint8Array} chunk - The incoming chunk of data. + * @param {TransformStreamDefaultController} controller - The controller for enqueuing processed bytes. + * @returns {Promise} - A promise that resolves when the transformation is complete. + */ + async transform(chunk, controller) { + this.controller = controller; + + // Concatenate incoming chunk with existing buffer + this.buffer = new Uint8Array([...this.buffer, ...chunk]); + + while (this.buffer.length >= this.waitBytes) { + // Extract the required number of bytes + const bytesToProcess = this.buffer.slice(0, this.waitBytes); + + // Remove processed bytes from the buffer + this.buffer = this.buffer.slice(this.waitBytes); + + // Notify the controller that bytes have been processed + controller.enqueue(this.convertBytes(bytesToProcess)); + } + } + + /** + * Flushes the buffer and processes any remaining bytes when the stream is closed. + * + * @param {WritableStreamDefaultController} controller - The controller for the writable stream. + */ + flush(controller) { + if (this.buffer.length > 0) { + // Handle remaining bytes (if any) when the stream is closed + const remainingBytes = this.buffer.slice(); + console.log("Remaining bytes:", remainingBytes); + + // Notify the controller that remaining bytes have been processed + controller?.enqueue(remainingBytes); + } + } +} + +/** + * Represents an Image Data Transformer that converts bytes into image data. + * See other example for PNGs here: https://github.com/mdn/dom-examples/blob/main/streams/png-transform-stream/png-transform-stream.js + * @extends StartStopSequenceTransformer + */ +class ImageDataTransformer extends StartStopSequenceTransformer { + /** + * Creates a new instance of the Transformer class. + * @param {CanvasRenderingContext2D} context - The canvas rendering context. + * @param {number} [width=null] - The width of the image. + * @param {number} [height=null] - The height of the image. + * @param {string} [imageMode=null] - The image mode. + */ + constructor(context, width = null, height = null, imageMode = null) { + super(); + this.context = context; + this.imageDataProcessor = new ImageDataProcessor(); + if (width && height){ + this.setResolution(width, height); + } + if (imageMode){ + this.setImageMode(imageMode); + } + } + + /** + * Sets the resolution of the camera image that is being processed. + * + * @param {number} width - The width of the resolution. + * @param {number} height - The height of the resolution. + */ + setResolution(width, height) { + this.width = width; + this.height = height; + this.imageDataProcessor.setResolution(width, height); + if(this.isConfigured()){ + this.setExpectedBytes(this.imageDataProcessor.getTotalBytes()); + } + } + + /** + * Sets the image mode of the camera image that is being processed. + * Possible values: RGB565, GRAYSCALE, RGB888, BAYER + * + * @param {string} imageMode - The image mode to set. + */ + setImageMode(imageMode) { + this.imageMode = imageMode; + this.imageDataProcessor.setImageMode(imageMode); + if(this.isConfigured()){ + this.setBytesToWait(this.imageDataProcessor.getTotalBytes()); + } + } + + /** + * Checks if the image data processor is configured. + * This is true if the image mode and resolution are set. + * @returns {boolean} True if the image data processor is configured, false otherwise. + */ + isConfigured() { + return this.imageMode && this.width && this.height; + } + + /** + * Resets the state of the transformer. + */ + reset() { + this.imageMode = null; + this.width = null; + this.height = null; + this.imageDataProcessor.reset(); + } + + /** + * Converts the given raw bytes into an ImageData object by using the ImageDataProcessor. + * + * @param {Uint8Array} bytes - The bytes to convert. + * @returns {ImageData} The converted ImageData object. + */ + convertBytes(bytes) { + let pixelData = this.imageDataProcessor.convertToPixelData(bytes); + + if(this.filter){ + this.filter.applyFilter(pixelData, imageDataTransfomer.width, imageDataTransfomer.height); + } + + const imageData = this.context.createImageData(imageDataTransfomer.width, imageDataTransfomer.height); + imageData.data.set(pixelData); + return imageData; + } +} \ No newline at end of file