Merge pull request #56 from jdibenes/strmcnt

strmcnt

.gitignore

@@ -18,3 +18,5 @@ tools/__pycache__/
 bbn/__pycache__/
 bbn/tests/__pycache__/
 calibration
+extensions/build/
+extensions/.vscode/

3rdparty/Zdepth/LICENSE

@@ -0,0 +1,29 @@
+BSD 3-Clause License
+
+Copyright (c) 2019, Chris Taylor
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

3rdparty/Zdepth/include/zdepth.hpp

@@ -0,0 +1,344 @@
+// Copyright 2019 (c) Christopher A. Taylor.  All rights reserved.
+
+/*
+    Zdepth
+
+    Lossless depth buffer compression designed and tested for Azure Kinect DK.
+    Based on the Facebook Zstd library for compression.
+
+    The compressor defines a file format and performs full input checking.
+    Supports temporal back-references similar to other video formats.
+
+    We do not use H.264 or other video codecs because the main goal is to
+    measure the limits of lossless real-time compression of depth data.
+    Another goal is to be 2x smaller than the best published software (RVL).
+
+    Algorithm:
+
+    (1) Quantize depth based on sensor accuracy at range.
+    (2) Compress run-lengths of zeroes with Zstd.
+    (3) For each 8x8 block of the depth image, determine the best predictor.
+    (4) Zig-zag encode and compress the residuals with Zstd.
+
+    The predictors are similar to the way the PNG format works.
+*/
+
+#pragma once
+
+#include <stdint.h>
+#include <vector>
+
+// Compiler-specific force inline keyword
+#if defined(_MSC_VER)
+    #define DEPTH_INLINE inline __forceinline
+#else // _MSC_VER
+    #define DEPTH_INLINE inline __attribute__((always_inline))
+#endif // _MSC_VER
+
+// Architecture check
+#if defined(__arm__) || defined(_M_ARM)
+    // Use aligned accesses on ARM
+    #define DEPTH_ALIGNED_ACCESSES
+#endif // ANDROID
+
+namespace zdepth {
+
+
+//------------------------------------------------------------------------------
+// Constants
+
+// First byte of the file format
+static const uint8_t kDepthFormatMagic = 202; // 0xCA
+
+// Number of bytes in header
+static const int kDepthHeaderBytes = 40;
+
+/*
+    File format:
+
+    Format Magic is used to quickly check that the file is of this format.
+    Words are stored in little-endian byte order.
+
+    0: <Format Magic = 202 (1 byte)>
+    1: <Flags (1 byte)>
+    2: <Frame Number (2 bytes)>
+    4: <Width (2 bytes)>
+    6: <Height (2 bytes)>
+    8: <Zeroes Uncompressed Bytes (4 bytes)>
+    12: <Zeroes Compressed Bytes (4 bytes)>
+    16: <Blocks Uncompressed Bytes (4 bytes)>
+    20: <Blocks Compressed Bytes (4 bytes)>
+    24: <Edges Uncompressed Bytes (4 bytes)>
+    28: <Edges Compressed Bytes (4 bytes)>
+    32: <Surfaces Uncompressed Bytes (4 bytes)>
+    36: <Surfaces Compressed Bytes (4 bytes)>
+
+    Followed by compressed Zeroes, then Blocks, then Edges, then Surfaces.
+
+    The compressed and uncompressed sizes are of packed data for Zstd.
+
+    Flags = 1 for I-frames and 0 for P-frames.
+    The P-frames are able to use predictors that reference the previous frame.
+    The decoder keeps track of the previously decoded Frame Number and rejects
+    frames that cannot be decoded due to a missing previous frame.
+*/
+
+// Size of a block for predictor selection purposes
+static const int kBlockSize = 8;
+
+enum class DepthResult
+{
+    FileTruncated,
+    WrongFormat,
+    Corrupted,
+    MissingPFrame, // Missing previous referenced frame
+    BadDimensions, // Width % kBlockSize != 0 and/or Height % kBlockSize != 0
+    Success
+};
+
+const char* DepthResultString(DepthResult result);
+
+
+//------------------------------------------------------------------------------
+// Tools
+
+// Little-endian 16-bit read
+DEPTH_INLINE uint16_t ReadU16_LE(const void* data)
+{
+#ifdef DEPTH_ALIGNED_ACCESSES
+    const uint8_t* u8p = reinterpret_cast<const uint8_t*>(data);
+    return ((uint16_t)u8p[1] << 8) | u8p[0];
+#else
+    const uint16_t* word_ptr = reinterpret_cast<const uint16_t*>(data);
+    return *word_ptr;
+#endif
+}
+
+// Little-endian 32-bit read
+DEPTH_INLINE uint32_t ReadU32_LE(const void* data)
+{
+#ifdef DEPTH_ALIGNED_ACCESSES
+    const uint8_t* u8p = reinterpret_cast<const uint8_t*>(data);
+    return ((uint32_t)u8p[3] << 24) | ((uint32_t)u8p[2] << 16) | ((uint32_t)u8p[1] << 8) | u8p[0];
+#else
+    const uint32_t* u32p = reinterpret_cast<const uint32_t*>(data);
+    return *u32p;
+#endif
+}
+
+// Little-endian 16-bit write
+DEPTH_INLINE void WriteU16_LE(void* data, uint16_t value)
+{
+#ifdef DEPTH_ALIGNED_ACCESSES
+    uint8_t* u8p = reinterpret_cast<uint8_t*>(data);
+    u8p[1] = static_cast<uint8_t>(value >> 8);
+    u8p[0] = static_cast<uint8_t>(value);
+#else
+    uint16_t* word_ptr = reinterpret_cast<uint16_t*>(data);
+    *word_ptr = value;
+#endif
+}
+
+// Little-endian 32-bit write
+DEPTH_INLINE void WriteU32_LE(void* data, uint32_t value)
+{
+#ifdef DEPTH_ALIGNED_ACCESSES
+    uint8_t* u8p = reinterpret_cast<uint8_t*>(data);
+    u8p[3] = (uint8_t)(value >> 24);
+    u8p[2] = static_cast<uint8_t>(value >> 16);
+    u8p[1] = static_cast<uint8_t>(value >> 8);
+    u8p[0] = static_cast<uint8_t>(value);
+#else
+    uint32_t* word_ptr = reinterpret_cast<uint32_t*>(data);
+    *word_ptr = value;
+#endif
+}
+
+
+bool IsDepthFrame(const uint8_t* file_data, unsigned file_bytes);
+bool IsKeyFrame(const uint8_t* file_data, unsigned file_bytes);
+
+
+//------------------------------------------------------------------------------
+// Depth Quantization
+
+/*
+    Azure Kinect DK sensor whitepaper:
+    https://docs.microsoft.com/en-us/windows/mixed-reality/ISSCC-2018
+
+    Minimum operating range = 200 mm.
+
+    The Kinect has 1-2mm accuracy up to about 4 meters.
+    Depth uncertainty < 0.2% of range:
+
+        <750 mm  : 1.5 mm precision (or better)
+        <1500 mm : 3 mm precision (or better)
+        <3000 mm : 6 mm precision (or better)
+        <6000 mm : 12 mm precision (or better)
+        <12000 mm : 24 mm precision (or better)
+
+    Our quantization table:
+
+        [0, 200] mm      -> 0            (no depth data)
+        [201, 750) mm    -> [1, 550)     (lossless)
+        [750, 1500) mm   -> [550, 925)   (quantized 2x)
+        [1500, 3000) mm  -> [925, 1300)  (quantized 4x)
+        [3000, 6000) mm  -> [1300, 1675) (quantized 8x)
+        [6000, 11840) mm -> [1675, 2040) (quantized 16x)
+        Larger depth     -> 0            (no depth data)
+
+    Reverse quantization table:
+
+        0            -> 0                (no depth data)
+        [1, 550)     -> [201, 750) mm    (lossless)
+        [550, 925)   -> [750, 1500) mm   (quantized 2x)
+        [925, 1300)  -> [1500, 3000) mm  (quantized 4x)
+        [1300, 1675) -> [3000, 6000) mm  (quantized 8x)
+        [1675, 2040) -> [6000, 11840) mm (quantized 16x)
+        Larger values are invalid.
+*/
+
+// Quantize depth from 200..11840 mm to a value from 0..2040
+uint16_t AzureKinectQuantizeDepth(uint16_t depth);
+
+// Reverse quantization back to original depth
+uint16_t AzureKinectDequantizeDepth(uint16_t quantized);
+
+// Quantize depth for a whole image
+void QuantizeDepthImage(
+    int width,
+    int height,
+    const uint16_t* depth,
+    std::vector<uint16_t>& quantized);
+void DequantizeDepthImage(
+    int width,
+    int height,
+    const uint16_t* quantized,
+    std::vector<uint16_t>& depth);
+
+
+//------------------------------------------------------------------------------
+// Zstd
+
+void ZstdCompress(
+    const std::vector<uint8_t>& uncompressed,
+    std::vector<uint8_t>& compressed);
+
+bool ZstdDecompress(
+    const uint8_t* compressed_data,
+    int compressed_bytes,
+    int uncompressed_bytes,
+    std::vector<uint8_t>& uncompressed);
+
+
+//------------------------------------------------------------------------------
+// Pack12
+
+/*
+    Pack12 Format Description:
+
+    This packing format is optimized for depth data compression.
+    Precondition: There are an even number of 12-bit values to write.
+
+    For N inputs the first N output bytes form the `packed0` plane.
+    The next N/2 output bytes form the `packed1` plane.
+
+    The `packed0` plane consists of the high 8 bytes of all the inputs.
+    The `packed1` plane consists of low bits of pairs of inputs.
+*/
+
+// Pad data with a zero entry to make its length even
+void Pad12(std::vector<uint16_t>& data);
+
+// Pack 12-bit fields into bytes for Zstd compression.
+// Input must be a multiple of two in size
+void Pack12(
+    const std::vector<uint16_t>& data,
+    std::vector<uint8_t>& packed);
+
+void Unpack12(
+    const std::vector<uint8_t>& packed,
+    std::vector<uint16_t>& data);
+
+
+//------------------------------------------------------------------------------
+// DepthCompressor
+
+class DepthCompressor
+{
+public:
+    // Compress depth array to buffer
+    // Set keyframe to indicate this frame should not reference the previous one
+    // Returns DepthResult::Success if the depth can be compressed
+    DepthResult Compress(
+        int width,
+        int height,
+        const uint16_t* unquantized_depth,
+        std::vector<uint8_t>& compressed,
+        bool keyframe);
+
+    // Decompress buffer to depth array.
+    // Resulting depth buffer is row-first, stride=width*2 (no surprises).
+    // Returns DepthResult::Success if original depth can be recovered
+    DepthResult Decompress(
+        const std::vector<uint8_t>& compressed,
+        int& width,
+        int& height,
+        std::vector<uint16_t>& depth_out);
+
+protected:
+    // Depth values quantized for current and last frame
+    std::vector<uint16_t> QuantizedDepth[2];
+    unsigned CurrentFrameIndex = 0;
+    unsigned CompressedFrameNumber = 0;
+
+    // Accumulated through the end of the filtering then compressed separately
+    std::vector<uint16_t> Edges, Surfaces;
+
+    // Block descriptors
+    std::vector<uint8_t> Zeroes, Blocks;
+
+    int Zeroes_UncompressedBytes = 0;
+    int Surfaces_UncompressedBytes = 0;
+    int Blocks_UncompressedBytes = 0;
+    int Edges_UncompressedBytes = 0;
+
+    // Results of zstd compression
+    std::vector<uint8_t> ZeroesOut, SurfacesOut, BlocksOut, EdgesOut;
+
+    // Packs the 16-bit overruns into 12-bit values and apply Zstd
+    std::vector<uint8_t> Packed;
+
+
+    void CompressImage(
+        int width,
+        int height,
+        const uint16_t* depth,
+        const uint16_t* prev_depth);
+    bool DecompressImage(
+        int width,
+        int height,
+        uint16_t* depth,
+        const uint16_t* prev_depth);
+
+    void WriteCompressedFile(
+        int width,
+        int height,
+        bool keyframe,
+        std::vector<uint8_t>& compressed);
+
+    void EncodeZeroes(
+        int width,
+        int height,
+        const uint16_t* depth);
+
+    // This writes the whole QuantizedDepth image with 0 or 1
+    // before the block decoding starts.
+    void DecodeZeroes(
+        int width,
+        int height,
+        uint16_t* depth);
+};
+
+
+} // namespace zdepth