Squashed 'app/src/cpp/libcimbar/' changes from b39469c..5fbface

5fbface Merge pull request #60 from sz3/ci-package
9d52131 Draft a github release (with bits!) when a new tag is pushed
2010df7 It's still gcc9
1686eca Build on new ubuntu?
4ab275a a script to build cimbar.js
bd87003 Ubuntu 16.04 build for glibc >= 2.23. Or 2.14(!?)
7bb1bc8 -DBUILD_PORTABLE_LINUX for static linking against opencv and libc++
729eb7e Merge pull request #59 from sz3/bugfix-misc
9e0dae0 Compile time check is fine, I guess
c05e3b5 hash difference aside, the binary differences are minor
e698786 Update concurrentqueue (1.0.3)
fec665f Update fmt (8.1.1)
0f2665a Upgrade intx
cc6abee Update picosha2 and stb_image
39b37d9 Update wirehair
82ed9e9 oof
28559cb I think we need calib3d now?
178e02a Make CMakeLists smart enough to find opencv4
b0091d1 new catch2 to make compiler happy
52547ae Include to make g++11 happy
7c545bb cerr
c5f7676 Bugfix (maybe): properly clean up old decodes
f752be4 Merge pull request #58 from sz3/service-worker
19c7904 No PWA for now
ef6df87 Merge pull request #57 from sz3/css-fix
8103bf0 Possibly functional?
74ceae3 add favicon...
8fa1509 Cache more stuff? clear old caches?
e20a3cd WIP? cache please?
99eb349 Merge branch 'css-fix' into sw
8d4cbc9 Try using a button element?
a7f663a Set zoom on the canvas element, not the whole page?
d0dffa2 Slightly more responsive to window size changes
9e8e552 Fix the background css
989796b service worker attempt (doesn't work)

git-subtree-dir: app/src/cpp/libcimbar
git-subtree-split: 5fbface

.github/workflows/ci.yml

@@ -14,6 +14,9 @@ jobs:
  os: ubuntu-18.04
  env: CXX="g++-7" CC="gcc-7"
 
+ - name: "linux gcc9"
+ os: ubuntu-latest
+
  - name: "linux clang"
  os: ubuntu-18.04
  env: CXX="clang++" CC="clang"

.github/workflows/release.yml

@@ -0,0 +1,42 @@
+name: release
+
+on:
+ push:
+ tags:
+ - '*'
+
+jobs:
+ package-cimbar:
+ runs-on: ubuntu-latest
+ steps:
+ - name: Get the code
+ uses: actions/checkout@v2
+ with:
+ submodules: 'recursive'
+
+ - name: Get openCV
+ run: |
+ wget https://github.com/opencv/opencv/archive/refs/tags/4.5.5.zip
+ unzip 4.5.5.zip
+ mv opencv-4.5.5 opencv4
+
+ - name: Run the build process with Docker
+ uses: addnab/docker-run-action@v3
+ with:
+ image: emscripten/emsdk:latest
+ options: -v ${{ github.workspace }}:/usr/src/app
+ shell: bash
+ run: |
+ bash /usr/src/app/package-wasm.sh
+
+ - name: Show results
+ run: ls -l web/
+
+ - name: Create release
+ uses: softprops/action-gh-release@v1
+ with:
+ draft: true
+ files: |
+ web/cimbar.asmjs.zip
+ web/cimbar.wasm.tar.gz
+

CMakeLists.txt

@@ -16,7 +16,7 @@ if("${CMAKE_BUILD_TYPE}" STREQUAL "")
  set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-Wall -g -O2 -fPIC")
 endif()
 
-if(DEFINED USE_WASM)
+if(DEFINED USE_WASM) # wasm build needs OPENCV_DIR defined
  set(DISABLE_TESTS true)
  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DGF256_TARGET_MOBILE")
  set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-Os")
@@ -28,10 +28,22 @@ if(DEFINED USE_WASM)
  ${OPENCV_DIR}/opencv-build-wasm/build_wasm/
  ${opencv_include_modules}
  )
+else() # if not wasm, go find opencv. 3 or 4 should both work
+ find_package(OpenCV REQUIRED)
+ include_directories(${OpenCV_INCLUDE_DIRS})
+endif()
+
+if(DEFINED BUILD_PORTABLE_LINUX)
+ find_package(PkgConfig REQUIRED)
+ pkg_check_modules(OPENCV4 REQUIRED opencv4)
+ link_directories(${OPENCV4_STATIC_LIBRARY_DIRS})
+
+ # statically link c++/gcc (not libc!!)
+ set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -static-libstdc++ -static-libgcc")
 endif()
 
 if(NOT DEFINED OPENCV_LIBS)
- set(OPENCV_LIBS "opencv_core" "opencv_imgcodecs" "opencv_imgproc" "opencv_photo")
+ set(OPENCV_LIBS "opencv_calib3d" "opencv_imgcodecs" "opencv_imgproc" "opencv_photo" "opencv_core" ${OPENCV4_STATIC_LIBRARIES})
 endif()
 
 if(NOT DEFINED CPPFILESYSTEM)

package-portable-linux.sh

@@ -0,0 +1,31 @@
+#!/bin/sh
+## targeting old glibc
+# docker run --mount type=bind,source="$(pwd)",target="/usr/src/app" -it ubuntu:16.04
+
+cd /usr/src/app
+
+# https://gist.github.com/jlblancoc/99521194aba975286c80f93e47966dc5
+apt update
+apt install -y software-properties-common
+add-apt-repository -y ppa:ubuntu-toolchain-r/test
+
+apt update
+apt install -y pkgconf g++-7 python-pip
+apt install -y libgles2-mesa-dev libglfw3-dev
+
+# cmake (via pip)
+python -m pip install cmake
+
+# use gcc7
+update-alternatives --install /usr/bin/cc cc /usr/bin/gcc-7 100
+update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++-7 100
+
+cd opencv4/
+mkdir build-portable/ && cd build-portable/
+/usr/local/bin/cmake .. -DCMAKE_BUILD_TYPE=RELEASE -DBUILD_SHARED_LIBS=OFF -DOPENCV_GENERATE_PKGCONFIG=YES -DOPENCV_FORCE_3RDPARTY_BUILD=YES
+make -j5 install
+
+cd /usr/src/app
+mkdir build-portable/ && cd build-portable/
+/usr/local/bin/cmake .. -DBUILD_PORTABLE_LINUX=1
+make -j5 install

package-wasm.sh

@@ -0,0 +1,23 @@
+#!/bin/sh
+#docker run --mount type=bind,source="$(pwd)",target="/usr/src/app" -it emscripten/emsdk:latest
+
+cd /usr/src/app
+
+apt update
+apt install python -y
+
+cd opencv4/
+mkdir opencv-build-wasm && cd opencv-build-wasm
+python ../platforms/js/build_js.py build_wasm --build_wasm --emscripten_dir=/emsdk/upstream/emscripten
+
+cd /usr/src/app
+mkdir build-wasm && cd build-wasm
+emcmake cmake .. -DUSE_WASM=1 -DOPENCV_DIR=/usr/src/app/opencv4
+make -j5 install
+(cd ../web/ && tar -czvf cimbar.wasm.tar.gz cimbar_js.* index.html main.js)
+
+cd /usr/src/app
+mkdir build-asmjs && cd build-asmjs
+emcmake cmake .. -DUSE_WASM=2 -DOPENCV_DIR=/usr/src/app/opencv4
+make -j5 install
+(cd ../web/ && zip cimbar.asmjs.zip cimbar_js.js index.html main.js)

src/exe/cimbar_send/send.cpp

@@ -70,7 +70,7 @@ int main(int argc, char** argv)
  if (!initialize_GL(window_size, window_size))
  {
  std::cerr << "failed to create window :(" << std::endl;
- return 50;
+ return 70;
  }
 
  configure(colorBits, ecc, compressionLevel);

src/lib/cimb_translator/CellPositions.h

@@ -1,8 +1,9 @@
 /* This code is subject to the terms of the Mozilla Public License, v.2.0. http://mozilla.org/MPL/2.0/. */
 #pragma once
 
-#include <vector>
+#include <cstddef>
 #include <utility>
+#include <vector>
 
 class CellPositions
 {

src/lib/cimb_translator/test/CimbDecoderTest.cpp

@@ -56,7 +56,7 @@ TEST_CASE( "CimbDecoderTest/testPrethresholdDecode", "[unit]" )
  for (unsigned i = 0; i < 16; ++i)
  {
  cv::Mat tile = cimbar::getTile(4, i, true);
- cv::Mat tenxten(10, 10, tile.type());
+ cv::Mat tenxten(10, 10, tile.type(), cv::Scalar(0, 0, 0));
  tile.copyTo(tenxten(cv::Rect(cv::Point(1, 1), tile.size())));
 
  // grayscale and threshold, since that's what average_hash needs

src/lib/compression/zstd_compressor.h

@@ -32,7 +32,7 @@ class zstd_compressor : public STREAM
  size_t compressedBytes = ZSTD_compressCCtx(_cctx, _compBuff.data(), _compBuff.size(), data, writeLen, _compressionLevel);
  if (ZSTD_isError(compressedBytes))
  {
- std::cout << "error? " << ZSTD_getErrorName(compressedBytes) << std::endl;
+ std::cerr << "error? " << ZSTD_getErrorName(compressedBytes) << std::endl;
  return false;
  }
  STREAM::write(_compBuff.data(), compressedBytes);

src/lib/encoder/test/DecoderTest.cpp

@@ -56,5 +56,8 @@ TEST_CASE( "DecoderTest/testDecode.Sample", "[unit]" )
  unsigned bytesDecoded = dec.decode(TestCimbar::getSample("6bit/4_30_f0_627_extract.jpg"), decodedFile);
  assertEquals( 9300, bytesDecoded );
 
- assertEquals( "3de927c8aa0221807a2784210160cdc17567eb587bf01233d166900aadf14bf5", get_hash(decodedFile) );
+ if (CV_VERSION_MAJOR == 3)
+ assertEquals( "3de927c8aa0221807a2784210160cdc17567eb587bf01233d166900aadf14bf5", get_hash(decodedFile) );
+ else // # cv4
+ assertEquals( "59ddb2516b4ff5a528aebe538a22b736a6714263a454d20e146e1ffbba36c5ae", get_hash(decodedFile) );
 }

src/lib/fountain/FountainMetadata.h

@@ -15,7 +15,7 @@ class FountainMetadata
  }
 
 public:
- FountainMetadata(uint64_t id)
+ FountainMetadata(uint32_t id)
  : _data(id)
  {
  }
@@ -33,7 +33,7 @@ class FountainMetadata
  to_uint8_arr(encode_id, size, d);
  }
 
- unsigned id() const
+ uint32_t id() const
  {
  return _data;
  }
@@ -65,5 +65,5 @@ class FountainMetadata
  }
 
 protected:
- uint64_t _data; // might invert this and only generate the uint64_t when we need it
+ uint32_t _data; // might invert this and only generate the uint32_t when we need it
 };

src/lib/fountain/fountain_decoder_sink.h

@@ -39,10 +39,10 @@ class fountain_decoder_sink
  return true;
  }
 
- void mark_done(uint64_t id)
+ void mark_done(const FountainMetadata& md)
  {
- _done.insert(id);
- auto it = _streams.find(id);
+ _done.insert(md.id());
+ auto it = _streams.find(stream_slot(md));
  if (it != _streams.end())
  _streams.erase(it);
  }
@@ -60,7 +60,7 @@ class fountain_decoder_sink
  std::vector<std::string> get_done() const
  {
  std::vector<std::string> done;
- for (uint64_t id : _done)
+ for (uint32_t id : _done)
  done.push_back( get_filename(FountainMetadata(id)) );
  return done;
  }
@@ -77,7 +77,7 @@ class fountain_decoder_sink
  return progress;
  }
 
- bool is_done(uint64_t id) const
+ bool is_done(uint32_t id) const
  {
  return _done.find(id) != _done.end();
  }
@@ -106,7 +106,7 @@ class fountain_decoder_sink
  return false;
 
  if (store(md, *finished))
- mark_done(md.id());
+ mark_done(md);
  return true;
  }
 
@@ -122,7 +122,7 @@ class fountain_decoder_sink
  }
 
 protected:
- // streams is limited to at most 8 decoders at a time. Current, we just use the lower bits of the encode_id.
+ // streams is limited to at most 8 decoders at a time. Currently, we just use the lower bits of the encode_id.
  uint8_t stream_slot(const FountainMetadata& md) const
  {
  return md.encode_id() & 0x7;
@@ -137,7 +137,10 @@ class fountain_decoder_sink
  std::string _dataDir;
  unsigned _chunkSize;
 
+ // maybe instead of unordered_map+set, something where we can "age out" old streams?
+ // e.g. most recent 16/8, or something?
+ // question is what happens to _done/_streams when we wrap for continuous data streaming...
  std::unordered_map<uint8_t, fountain_decoder_stream> _streams;
- // track the uint64_t combo of (encode_id,size) to avoid redundant work
- std::set<uint64_t> _done;
+ // track the uint32_t combo of (encode_id,size) to avoid redundant work
+ std::set<uint32_t> _done;
 };

src/lib/image_hash/ahash_result.h

@@ -2,7 +2,7 @@
 #pragma once
 
 #include "bit_extractor.h"
-#include "intx/int128.hpp"
+#include "intx/intx.hpp"
 #include <array>
 #include <iostream>
 #include <utility>

src/lib/image_hash/average_hash.h

@@ -6,7 +6,7 @@
 #include "bit_file/bitmatrix.h"
 #include "cimb_translator/Cell.h"
 
-#include "intx/int128.hpp"
+#include "intx/intx.hpp"
 #include <opencv2/opencv.hpp>
 
 #include <array>
@@ -51,6 +51,10 @@ namespace image_hash
  uint64_t mval = (*hax) & 0x101010101ULL;
  const uint8_t* cv = reinterpret_cast<const uint8_t*>(&mval);
  uint8_t val = cv[0] << 4 | cv[1] << 3 | cv[2] << 2 | cv[3] << 1 | cv[4];
+ // TODO:
+ /*if (bigEndian)
+ val = cv[7] << 4 | cv[6] << 3 | cv[5] << 2 | cv[4] << 1 | cv[3];*/ // ?
+
  res |= intx::uint128(val) << bitpos;
  }
  }

src/lib/image_hash/test/bitExtractorTest.cpp

@@ -2,7 +2,7 @@
 #include "unittest.h"
 
 #include "bit_extractor.h"
-#include "intx/int128.hpp"
+#include "intx/intx.hpp"
 
 #include <bitset>
 #include <iostream>
@@ -36,7 +36,9 @@ TEST_CASE( "bitExtractorTest/testLargerValue.1", "[unit]" )
 
 TEST_CASE( "bitExtractorTest/testLargerValue.2", "[unit]" )
 {
- intx::uint128 bits{0xFFBFCFE3FULL, 0xF83C0E030080000ULL};
+ intx::uint128 bits{0xF83C0E030080000ULL, 0xFFBFCFE3FULL};
+ assertEquals( "ffbfcfe3f0f83c0e030080000", intx::hex(bits) ); // sanity check
+
  bit_extractor<intx::uint128, 100> be(bits);
  uint64_t res = be.extract(1, 11, 21, 31);
  assertEquals( 0xfffefcf8, res );

src/third_party_lib/PicoSHA2/picosha2.h

@@ -93,7 +93,7 @@ void hash256_block(RaIter1 message_digest, RaIter2 first, RaIter2 last) {
  assert(first + 64 == last);
  static_cast<void>(last); // for avoiding unused-variable warning
  word_t w[64];
- std::fill(w, w + 64, 0);
+ std::fill(w, w + 64, word_t(0));
  for (std::size_t i = 0; i < 16; ++i) {
  w[i] = (static_cast<word_t>(mask_8bit(*(first + i * 4))) << 24) |
  (static_cast<word_t>(mask_8bit(*(first + i * 4 + 1))) << 16) |
@@ -185,7 +185,7 @@ class hash256_one_by_one {
 
  void init() {
  buffer_.clear();
- std::fill(data_length_digits_, data_length_digits_ + 4, 0);
+ std::fill(data_length_digits_, data_length_digits_ + 4, word_t(0));
  std::copy(detail::initial_message_digest,
  detail::initial_message_digest + 8, h_);
  }
@@ -204,17 +204,17 @@ class hash256_one_by_one {
 
  void finish() {
  byte_t temp[64];
- std::fill(temp, temp + 64, 0);
+ std::fill(temp, temp + 64, byte_t(0));
  std::size_t remains = buffer_.size();
  std::copy(buffer_.begin(), buffer_.end(), temp);
  temp[remains] = 0x80;
 
  if (remains > 55) {
- std::fill(temp + remains + 1, temp + 64, 0);
+ std::fill(temp + remains + 1, temp + 64, byte_t(0));
  detail::hash256_block(h_, temp, temp + 64);
- std::fill(temp, temp + 64 - 4, 0);
+ std::fill(temp, temp + 64 - 4, byte_t(0));
  } else {
- std::fill(temp + remains + 1, temp + 64 - 4, 0);
+ std::fill(temp + remains + 1, temp + 64 - 4, byte_t(0));
  }
 
  write_data_bit_length(&(temp[56]));

src/third_party_lib/concurrentqueue/LICENSE.md

@@ -1,6 +1,7 @@
 This license file applies to everything in this repository except that which
 is explicitly annotated as being written by other authors, i.e. the Boost
 queue (included in the benchmarks for comparison), Intel's TBB library (ditto),
+dlib::pipe (ditto),
 the CDSChecker tool (used for verification), the Relacy model checker (ditto),
 and Jeff Preshing's semaphore implementation (used in the blocking queue) which
 has a zlib license (embedded in lightweightsempahore.h).