Merge ktxtools into main (#714)

PR to merge the contents of the ktxtools branch into main. This PR should be functionally identical to the ktxtools branch. Briefs: Rebase ktxtools onto main Update the CTS golden files to pickup the changes in ASTC and ZSTD versions Add CTS testing to MinGW and Windows CI builds Pickup changes in image.hpp from main to ktxtools's image.hpp Fix any remaining warnings
KhronosGroup · Jun 13, 2023 · a6abf2f · a6abf2f
1 parent c6568b7
commit a6abf2f
Show file tree

Hide file tree

Showing 83 changed files with 19,036 additions and 444 deletions.
diff --git a/.gitattributes b/.gitattributes
@@ -75,3 +75,9 @@ tests/testimages/*.ktx2 filter=lfs -text
 tests/srcimages/**/*.jpg filter=lfs -text
 tests/srcimages/**/*.png filter=lfs -text
 tests/webgl/**/*.ktx2 filter=lfs -text
+tests/clitests/input/images/**/*.ktx2 filter=lfs -text
+tests/clitests/input/images/**/*.png filter=lfs -text
+tests/clitests/input/images/**/*.exr filter=lfs -text
+tests/clitests/golden/images/**/*.ktx2 filter=lfs -text
+tests/clitests/golden/images/**/*.png filter=lfs -text
+tests/clitests/golden/images/**/*.exr filter=lfs -text
diff --git a/.github/workflows/mingw.yml b/.github/workflows/mingw.yml
@@ -43,16 +43,18 @@ jobs:
       run: git fetch -f origin ${{ github.ref }}:${{ github.ref }}
     - name: Pull test images from Git LFS
       run: git lfs pull --include=tests/srcimages,tests/testimages
+    - name: Fetch CTS Submodule
+      run: git submodule update --init --recursive tests/cts
     - name: Install Ninja
       run: choco install ninja --no-progress
     - name: Configure Mingw x64
-      run: cmake -B build -G "Ninja Multi-Config" -DCMAKE_C_COMPILER=gcc -DCMAKE_CXX_COMPILER=g++
+      run: cmake -B build -G "Ninja Multi-Config" -DCMAKE_C_COMPILER=gcc -DCMAKE_CXX_COMPILER=g++ -DKTX_FEATURE_TOOLS=TRUE -DKTX_FEATURE_TOOLS_CTS=TRUE
     - name: Build Mingw x64 Debug
       run: cmake --build build --config Debug
     - name: Build Mingw x64 Release
       run: cmake --build build --config Release
     - name: Test Mingw build
-      run: ctest --test-dir build -C Release
+      run: ctest --output-on-failure --test-dir build -C Release
     - name: Upload test log
       if: ${{ failure() }}
       run: ci_scripts/on_failure.ps1

diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml
@@ -33,19 +33,19 @@ jobs:
         arch: [ x64 ]
         options: [
           {config: 'Debug,Release',
-           doc: ON, jni: ON, loadtests: OpenGL+Vulkan, tests: ON, tools: ON,
+           doc: ON, jni: ON, loadtests: OpenGL+Vulkan, tests: ON, tools: ON, tools_cts: ON,
            package: YES,
            sse: ON, opencl: OFF},
           {config: Release,
-           doc: OFF, loadtests: OFF, tests: OFF, tools: OFF,
+           doc: OFF, loadtests: OFF, tests: OFF, tools: OFF, tools_cts: OFF,
            package: NO,
            sse: OFF, opencl: OFF},
           {config: Release,
-           doc: OFF, loadtests: OFF, tests: OFF, tools: OFF,
+           doc: OFF, loadtests: OFF, tests: OFF, tools: OFF, tools_cts: OFF,
            package: NO,
            sse: OFF, opencl: ON},
           {config: Release,
-           doc: OFF, loadtests: OFF, tests: OFF, tools: OFF,
+           doc: OFF, loadtests: OFF, tests: OFF, tools: OFF, tools_cts: OFF,
            package: NO,
            sse: ON, opencl: ON}
         ]
@@ -56,7 +56,7 @@ jobs:
             arch: x64
             options: {
               config: 'Debug,Release',
-              doc: ON, jni: ON, loadtests: OpenGL+Vulkan, tests: ON, tools: ON,
+              doc: ON, jni: ON, loadtests: OpenGL+Vulkan, tests: ON, tools: ON, tools_cts: ON,
               package: NO,
               sse: ON, opencl: OFF
             }
@@ -68,7 +68,7 @@ jobs:
             # built tests
             options: {
               config: 'Debug,Release',
-              doc: ON, jni: ON, loadtests: OFF, tests: ON, tools: ON,
+              doc: ON, jni: ON, loadtests: OFF, tests: ON, tools: ON, tools_cts: ON,
               package: YES
             }
     runs-on: ${{ matrix.os }}
@@ -90,6 +90,7 @@ jobs:
       FEATURE_LOADTESTS: ${{ matrix.options.loadtests }}
       FEATURE_TESTS: ${{ matrix.options.tests }}
       FEATURE_TOOLS: ${{ matrix.options.tools }}
+      FEATURE_TOOLS_CTS: ${{ matrix.options.tools_cts }}
       PACKAGE: ${{ matrix.options.package }}
       SUPPORT_OPENCL: ${{ matrix.options.opencl }}
       SUPPORT_SSE: ${{ matrix.options.sse }}
@@ -189,7 +190,7 @@ jobs:
 
     - name: Test Windows build
       if: matrix.arch == 'x64' && matrix.options.tests == 'ON'
-      run: ctest --test-dir $env:BUILD_DIR -C Release
+      run: ctest --output-on-failure --test-dir $env:BUILD_DIR -C Release
 
     - name: Upload test log
       if: ${{ failure() }}

diff --git a/.gitignore b/.gitignore
@@ -40,6 +40,9 @@ version.h
 .vs/
 CMakeSettings.json
 
+# Visual Studio Code
+.vscode/
+
 # Build bindings & maven
 /interface/java_binding/build/
 /interface/java_binding/target/

diff --git a/.gitmodules b/.gitmodules
@@ -0,0 +1,8 @@
+# Copyright 2022-2023 The Khronos Group Inc.
+# Copyright 2022-2023 RasterGrid Kft.
+# SPDX-License-Identifier: Apache-2.0
+
+[submodule "tests/cts"]
+	path = tests/cts
+	url = https://github.com/KhronosGroup/KTX-Software-CTS.git
+	branch = main
diff --git a/.reuse/dep5 b/.reuse/dep5
@@ -204,3 +204,7 @@ License: Apache-2.0
 Files: other_projects/fmt/*
 Copyright: 2012 - present Victor Zverovich
 License: MIT
+
+Files: other_projects/cxxopts/*
+Copyright: 2014-2022 Jarryd Beck
+License: MIT
diff --git a/.travis.yml b/.travis.yml
@@ -43,35 +43,36 @@ env:
     # add the list of tests to the cmake test runner.
     - CONFIGURATION=Debug,Release PLATFORM=macOS ARCHS=x86_64
       FEATURE_DOC=ON FEATURE_JNI=ON FEATURE_LOADTESTS=OpenGL+Vulkan FEATURE_TOOLS=ON
-      SUPPORT_SSE=ON SUPPORT_OPENCL=OFF DEPLOY_DOCS=YES PACKAGE=YES
+      FEATURE_TOOLS_CTS=ON SUPPORT_SSE=ON SUPPORT_OPENCL=OFF DEPLOY_DOCS=YES PACKAGE=YES
     - CONFIGURATION=Release PLATFORM=macOS ARCHS=arm64
       FEATURE_DOC=ON FEATURE_JNI=ON FEATURE_LOADTESTS=OpenGL+Vulkan FEATURE_TESTS=OFF
-      FEATURE_TOOLS=ON SUPPORT_SSE=ON SUPPORT_OPENCL=OFF PACKAGE=YES
+      FEATURE_TOOLS=ON FEATURE_TOOLS_CTS=OFF SUPPORT_SSE=ON SUPPORT_OPENCL=OFF PACKAGE=YES
 
     - CONFIGURATION=Release PLATFORM=macOS ARCHS=x86_64
       FEATURE_DOC=ON FEATURE_JNI=OFF FEATURE_LOADTESTS=OFF FEATURE_TOOLS=OFF
-      SUPPORT_SSE=ON SUPPORT_OPENCL=ON
+      FEATURE_TOOLS_CTS=OFF SUPPORT_SSE=ON SUPPORT_OPENCL=ON
     - CONFIGURATION=Release PLATFORM=macOS ARCHS=arm64
       FEATURE_DOC=ON FEATURE_JNI=OFF FEATURE_LOADTESTS=OFF FEATURE_TESTS=OFF
-      FEATURE_TOOLS=OFF SUPPORT_SSE=ON SUPPORT_OPENCL=ON
+      FEATURE_TOOLS=OFF FEATURE_TOOLS_CTS=OFF SUPPORT_SSE=ON SUPPORT_OPENCL=ON
 
     - CONFIGURATION=Release PLATFORM=macOS ARCHS=x86_64
       FEATURE_DOC=ON FEATURE_JNI=OFF FEATURE_LOADTESTS=OFF FEATURE_TOOLS=OFF
-      SUPPORT_SSE=OFF SUPPORT_OPENCL=ON
+      FEATURE_TOOLS_CTS=OFF SUPPORT_SSE=OFF SUPPORT_OPENCL=ON
     - CONFIGURATION=Release PLATFORM=macOS ARCHS=arm64
       FEATURE_DOC=ON FEATURE_JNI=OFF FEATURE_LOADTESTS=OFF FEATURE_TESTS=OFF
-      FEATURE_TOOLS=OFF SUPPORT_SSE=OFF SUPPORT_OPENCL=ON
+      FEATURE_TOOLS=OFF FEATURE_TOOLS_CTS=OFF SUPPORT_SSE=OFF SUPPORT_OPENCL=ON
 
     - CONFIGURATION=Release PLATFORM=macOS ARCHS=x86_64
       FEATURE_DOC=ON FEATURE_JNI=OFF FEATURE_LOADTESTS=OFF FEATURE_TOOLS=OFF
-      SUPPORT_SSE=OFF SUPPORT_OPENCL=OFF
+      FEATURE_TOOLS_CTS=OFF SUPPORT_SSE=OFF SUPPORT_OPENCL=OFF
     - CONFIGURATION=Release PLATFORM=macOS ARCHS=arm64
       FEATURE_DOC=ON FEATURE_JNI=OFF FEATURE_LOADTESTS=OFF FEATURE_TESTS=OFF
-      FEATURE_TOOLS=OFF SUPPORT_SSE=OFF SUPPORT_OPENCL=OFF
+      FEATURE_TOOLS=OFF FEATURE_TOOLS_CTS=OFF SUPPORT_SSE=OFF
+      SUPPORT_OPENCL=OFF
 
     - CONFIGURATION=Debug,Release PLATFORM=iOS
       FEATURE_DOC=OFF FEATURE_JNI=OFF FEATURE_LOADTESTS=OpenGL+Vulkan FEATURE_TOOLS=OFF
-      SUPPORT_SSE=OFF SUPPORT_OPENCL=OFF PACKAGE=YES
+      FEATURE_TOOLS_CTS=OFF SUPPORT_SSE=OFF SUPPORT_OPENCL=OFF PACKAGE=YES
 
 jobs:
   include:
@@ -110,15 +111,17 @@ jobs:
         - VULKAN_SDK_VER: "1.3.243"
         - CMAKE_GEN: Ninja
         - CONFIGURATION=Release
-          FEATURE_DOC=ON FEATURE_JNI=ON FEATURE_LOADTESTS=OpenGL FEATURE_TOOLS=ON
+          FEATURE_DOC=ON FEATURE_JNI=ON FEATURE_LOADTESTS=OpenGL
+          FEATURE_TOOLS=ON FEATURE_TOOLS_CTS=OFF
           SUPPORT_SSE=OFF SUPPORT_OPENCL=OFF PACKAGE=YES
     - os: linux
       dist: jammy
       compiler: gcc
       env:
         - CMAKE_GEN: Ninja
         - CONFIGURATION=Release
-          FEATURE_DOC=OFF FEATURE_JNI=OFF FEATURE_LOADTESTS=OFF FEATURE_TOOLS=OFF
+          FEATURE_DOC=OFF FEATURE_JNI=OFF FEATURE_LOADTESTS=OFF
+          FEATURE_TOOLS=OFF FEATURE_TOOLS_CTS=OFF
           SUPPORT_SSE=ON SUPPORT_OPENCL=ON PACKAGE=NO
     - os: linux
       dist: jammy
@@ -127,7 +130,7 @@ jobs:
         - CMAKE_GEN: Ninja
         - CONFIGURATION=Release
           FEATURE_DOC=OFF FEATURE_JNI=OFF FEATURE_LOADTESTS=OFF
-          FEATURE_TOOLS=OFF
+          FEATURE_TOOLS=OFF FEATURE_TOOLS_CTS=OFF
           SUPPORT_SSE=OFF SUPPORT_OPENCL=ON PACKAGE=NO
     - os: linux
       dist: jammy
@@ -136,7 +139,7 @@ jobs:
         - CMAKE_GEN: Ninja
         - CONFIGURATION=Release
           FEATURE_DOC=OFF FEATURE_JNI=OFF FEATURE_LOADTESTS=OFF
-          FEATURE_TOOLS=OFF
+          FEATURE_TOOLS=OFF FEATURE_TOOLS_CTS=OFF
           SUPPORT_SSE=OFF SUPPORT_OPENCL=OFF PACKAGE=NO
     - os: linux
       dist: jammy

diff --git a/BUILDING.md b/BUILDING.md
@@ -51,6 +51,10 @@ If you need the library to be static, add `-D KTX_FEATURE_STATIC_LIBRARY=ON` to
 > define `KHRONOS_STATIC` before including KTX header files.
 > This is especially important on Windows.
 
+If you want to run the CTS tests (recommended only during KTX development)
+add `-D KTX_FEATURE_TOOLS_CTS=ON` to the CMake configure command and fetch
+the CTS submodule. For more information see [Conformance Test Suite](#conformance-test-suite).
+
 If you want the Basis Universal encoders in `libktx` to use OpenCL
 add `-D BASISU_SUPPORT_OPENCL=ON` to the CMake configure command.
 
@@ -421,6 +425,48 @@ cmake --build "build-android"
 
 > Note: SSE has to be disabled currently (for ABIs x86 and x86_64) due to [an issue](https://github.com/BinomialLLC/basis_universal/pull/233).
 
+Conformance Test Suite
+------------
+
+The submodule of [CTS Repository](https://github.com/KhronosGroup/KTX-Software-CTS/) is optional and
+only required for running the CTS tests during KTX development. If the CTS test suit is desired it
+can be fetched during cloning with the additional `--recurse-submodules` git clone flag:
+```bash
+git clone --recurse-submodules git@github.com:KhronosGroup/KTX-Software.git
+```
+If the repository was already cloned or whenever the submodule ref changes the submodule has to be
+updated with:
+```bash
+git submodule update --init --recursive tests/cts
+```
+(For more information on submodules see the [git documentation](https://git-scm.com/book/en/v2/Git-Tools-Submodules).)
+
+Once the submodule is fetched the CTS tests can be enabled with the `KTX_FEATURE_TOOLS_CTS`
+cmake option during cmake configuration. Please note that for `KTX_FEATURE_TOOLS_CTS` to take
+effect both `KTX_FEATURE_TESTS` and `KTX_FEATURE_TOOLS` has to be also enabled.
+The CTS integrates into `ctest` so running `ctest` will also execute the CTS tests too.
+The test cases can be limited to the CTS tests with `ctest -R ktxToolTests`.
+
+Example for development workflow with CTS testing:
+```bash
+# Git clone and submodule fetch 
+git clone git@github.com:KhronosGroup/KTX-Software.git
+cd KTX-Software/
+git submodule update --init --recursive tests/cts
+# Configure 
+mkdir build
+cmake -B build . -DKTX_FEATURE_DOC=ON -DKTX_FEATURE_STATIC_LIBRARY=ON -DKTX_FEATURE_TOOLS_CTS=ON -DKTX_FEATURE_TESTS=ON -DKTX_FEATURE_TOOLS_CTS=ON
+# Build everything (depending on workflow its better to build the specific target like 'ktxtools'):
+cmake --build build --target all 
+# Run every test case:
+ctest --test-dir build
+# Run only the CTS test cases:
+ctest --test-dir build -R ktxToolTests
+```
+
+To create and update CTS test cases and about their specific features and usages
+see the [CTS documentation](https://github.com/KhronosGroup/KTX-Software-CTS/blob/main/README.md).
+
 Dependencies
 ------------