From 7f67af7e0a4c842cc4969acf58aa13daab540330 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cs=C3=A1sz=C3=A1r=20M=C3=A1ty=C3=A1s?= Date: Mon, 31 Jul 2023 11:17:12 +0200 Subject: [PATCH] Improve output determinism and add internal ktxdiff tool for comparing test outputs (#745) * Implement internal ktxdiff tool for comparing test output files. * Fix color conversion overflow. --- CMakeLists.txt | 54 +++- tests/CMakeLists.txt | 2 + tests/cts | 2 +- tests/ktxdiff/CMakeLists.txt | 44 ++++ tests/ktxdiff/ktxdiff_main.cpp | 458 +++++++++++++++++++++++++++++++++ tools/ktx/encode_utils.h | 2 +- tools/ktx/image.hpp | 7 +- 7 files changed, 564 insertions(+), 5 deletions(-) create mode 100644 tests/ktxdiff/CMakeLists.txt create mode 100644 tests/ktxdiff/ktxdiff_main.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index ad9fd5e62a..9cd8855e33 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -254,6 +254,55 @@ else() message(FATAL_ERROR "${CMAKE_CXX_COMPILER_ID} not yet supported.") endif() +# To improve output determinism enable precise floating point operations globally +# This code was based on lib/astc-encoder/Source/cmake_core.cmake + +# For Visual Studio prior to 2022 (compiler < 19.30) /fp:strict +# For Visual Studio 2022 (compiler >= 19.30) /fp:precise +# For Visual Studio 2022 ClangCL seems to have accidentally enabled contraction by default, +# so behaves differently to CL.exe. Use the -Xclang argument to workaround and allow access +# GNU-style switch to control contraction and force disable. + +# On CMake 3.25 or older CXX_COMPILER_FRONTEND_VARIANT is not always set +if(CMAKE_CXX_COMPILER_FRONTEND_VARIANT STREQUAL "") + set(CMAKE_CXX_COMPILER_FRONTEND_VARIANT "${CMAKE_CXX_COMPILER_ID}") +endif() + +# Compiler accepts MSVC-style command line options +set(is_msvc_fe "$") +# Compiler accepts GNU-style command line options +set(is_gnu_fe1 "$") +# Compiler accepts AppleClang-style command line options, which is also GNU-style +set(is_gnu_fe2 "$") +# Compiler accepts GNU-style command line options +set(is_gnu_fe "$") + +# Compiler is Visual Studio cl.exe +set(is_msvccl "$>") +# Compiler is Visual Studio clangcl.exe +set(is_clangcl "$>") +# Compiler is upstream clang with the standard frontend +set(is_clang "$>") + +if(${is_msvccl} AND $,19.30>) + add_compile_options(/fp:strict) +endif() +if(${is_msvccl} AND $,19.30>) + add_compile_options(/fp:precise) +endif() +if(${is_clangcl}) + add_compile_options(/fp:precise) +endif() +if(${is_clangcl} AND $,14.0.0>) + add_compile_options(-Xclang -ffp-contract=off) +endif() +if(${is_clang} AND $,10.0.0>) + add_compile_options(-ffp-model=precise) +endif() +if(${is_gnu_fe}) + add_compile_options(-ffp-contract=off) +endif() + set(KTX_BUILD_DIR "${CMAKE_BINARY_DIR}") set(KTX_MAIN_SRC @@ -361,10 +410,11 @@ if(WIN32) set(CMAKE_RUNTIME_OUTPUT_DIRECTORY $<1:${KTX_BUILD_DIR}/$>) elseif(APPLE) if(NOT IOS) - # Set a common RUNTIME_OUTPUT_DIR for all targets, so that - # INSTALL RPATH is functional in build directory as well. + # Set a common RUNTIME_OUTPUT_DIR and LIBRARY_OUTPUT_DIR for all targets, + # so that INSTALL RPATH is functional in build directory as well. # BUILD_WITH_INSTALL_RPATH is necessary for working code signing. set(CMAKE_RUNTIME_OUTPUT_DIRECTORY $<1:${KTX_BUILD_DIR}/$>) + set(CMAKE_LIBRARY_OUTPUT_DIRECTORY $<1:${KTX_BUILD_DIR}/$>) endif() endif() diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index f510804046..49910cec6f 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -28,7 +28,9 @@ if(KTX_FEATURE_TOOLS) # ktx cli tool tests if(KTX_FEATURE_TOOLS_CTS) + add_subdirectory(ktxdiff) set(KTX_TOOLS_PATH $) + set(KTX_DIFF_PATH $) add_subdirectory(cts/clitests) endif() endif() diff --git a/tests/cts b/tests/cts index d06c0a1697..83b23ce3e2 160000 --- a/tests/cts +++ b/tests/cts @@ -1 +1 @@ -Subproject commit d06c0a1697921cae68f4f59de91f970915e72df9 +Subproject commit 83b23ce3e24728576a67d7e3c8c128f0f5a6927f diff --git a/tests/ktxdiff/CMakeLists.txt b/tests/ktxdiff/CMakeLists.txt new file mode 100644 index 0000000000..f866c49db1 --- /dev/null +++ b/tests/ktxdiff/CMakeLists.txt @@ -0,0 +1,44 @@ +# Copyright 2022-2023 The Khronos Group Inc. +# Copyright 2022-2023 RasterGrid Kft. +# SPDX-License-Identifier: Apache-2.0 + + +add_executable(ktxdiff + ktxdiff_main.cpp +) + +set_target_properties( + ktxdiff + PROPERTIES + CXX_STANDARD 17 + CXX_STANDARD_REQUIRED YES +) + +target_include_directories( + ktxdiff +PRIVATE + . + $ +) + +target_include_directories( + ktxdiff + SYSTEM +PRIVATE + ${PROJECT_SOURCE_DIR}/lib + ${PROJECT_SOURCE_DIR}/other_include +) + +target_link_libraries( + ktxdiff +PRIVATE + ktx + ${ASTCENC_LIB_TARGET} + fmt::fmt +) + +target_compile_definitions( + ktxdiff +PRIVATE + $ +) diff --git a/tests/ktxdiff/ktxdiff_main.cpp b/tests/ktxdiff/ktxdiff_main.cpp new file mode 100644 index 0000000000..192d70b411 --- /dev/null +++ b/tests/ktxdiff/ktxdiff_main.cpp @@ -0,0 +1,458 @@ +// Copyright 2022-2023 The Khronos Group Inc. +// Copyright 2022-2023 RasterGrid Kft. +// SPDX-License-Identifier: Apache-2.0 + +#include "ktx.h" +#include "ktxint.h" +#include "texture2.h" +#include "vkformat_enum.h" + +#include "astc-encoder/Source/astcenc.h" + +#include +#include +#include +#include +#include + +#include +#include +#include + + +template +[[nodiscard]] constexpr inline T ceil_div(const T x, const T y) noexcept { + assert(y != 0); + return (x + y - 1) / y; +} + +// C++20 - std::bit_cast +template +[[nodiscard]] constexpr inline To bit_cast(const From& src) noexcept { + static_assert(sizeof(To) == sizeof(From)); + static_assert(std::is_trivially_copyable_v); + static_assert(std::is_trivially_copyable_v); + static_assert(std::is_trivially_constructible_v); + To dst; + std::memcpy(&dst, &src, sizeof(To)); + return dst; +} + +[[nodiscard]] constexpr inline bool isFormatAstc(VkFormat format) noexcept { + switch (format) { + case VK_FORMAT_ASTC_4x4_UNORM_BLOCK: [[fallthrough]]; + case VK_FORMAT_ASTC_4x4_SRGB_BLOCK: [[fallthrough]]; + case VK_FORMAT_ASTC_5x4_UNORM_BLOCK: [[fallthrough]]; + case VK_FORMAT_ASTC_5x4_SRGB_BLOCK: [[fallthrough]]; + case VK_FORMAT_ASTC_5x5_UNORM_BLOCK: [[fallthrough]]; + case VK_FORMAT_ASTC_5x5_SRGB_BLOCK: [[fallthrough]]; + case VK_FORMAT_ASTC_6x5_UNORM_BLOCK: [[fallthrough]]; + case VK_FORMAT_ASTC_6x5_SRGB_BLOCK: [[fallthrough]]; + case VK_FORMAT_ASTC_6x6_UNORM_BLOCK: [[fallthrough]]; + case VK_FORMAT_ASTC_6x6_SRGB_BLOCK: [[fallthrough]]; + case VK_FORMAT_ASTC_8x5_UNORM_BLOCK: [[fallthrough]]; + case VK_FORMAT_ASTC_8x5_SRGB_BLOCK: [[fallthrough]]; + case VK_FORMAT_ASTC_8x6_UNORM_BLOCK: [[fallthrough]]; + case VK_FORMAT_ASTC_8x6_SRGB_BLOCK: [[fallthrough]]; + case VK_FORMAT_ASTC_8x8_UNORM_BLOCK: [[fallthrough]]; + case VK_FORMAT_ASTC_8x8_SRGB_BLOCK: [[fallthrough]]; + case VK_FORMAT_ASTC_10x5_UNORM_BLOCK: [[fallthrough]]; + case VK_FORMAT_ASTC_10x5_SRGB_BLOCK: [[fallthrough]]; + case VK_FORMAT_ASTC_10x6_UNORM_BLOCK: [[fallthrough]]; + case VK_FORMAT_ASTC_10x6_SRGB_BLOCK: [[fallthrough]]; + case VK_FORMAT_ASTC_10x8_UNORM_BLOCK: [[fallthrough]]; + case VK_FORMAT_ASTC_10x8_SRGB_BLOCK: [[fallthrough]]; + case VK_FORMAT_ASTC_10x10_UNORM_BLOCK: [[fallthrough]]; + case VK_FORMAT_ASTC_10x10_SRGB_BLOCK: [[fallthrough]]; + case VK_FORMAT_ASTC_12x10_UNORM_BLOCK: [[fallthrough]]; + case VK_FORMAT_ASTC_12x10_SRGB_BLOCK: [[fallthrough]]; + case VK_FORMAT_ASTC_12x12_UNORM_BLOCK: [[fallthrough]]; + case VK_FORMAT_ASTC_12x12_SRGB_BLOCK: [[fallthrough]]; + case VK_FORMAT_ASTC_4x4_SFLOAT_BLOCK_EXT: [[fallthrough]]; + case VK_FORMAT_ASTC_5x4_SFLOAT_BLOCK_EXT: [[fallthrough]]; + case VK_FORMAT_ASTC_5x5_SFLOAT_BLOCK_EXT: [[fallthrough]]; + case VK_FORMAT_ASTC_6x5_SFLOAT_BLOCK_EXT: [[fallthrough]]; + case VK_FORMAT_ASTC_6x6_SFLOAT_BLOCK_EXT: [[fallthrough]]; + case VK_FORMAT_ASTC_8x5_SFLOAT_BLOCK_EXT: [[fallthrough]]; + case VK_FORMAT_ASTC_8x6_SFLOAT_BLOCK_EXT: [[fallthrough]]; + case VK_FORMAT_ASTC_8x8_SFLOAT_BLOCK_EXT: [[fallthrough]]; + case VK_FORMAT_ASTC_10x5_SFLOAT_BLOCK_EXT: [[fallthrough]]; + case VK_FORMAT_ASTC_10x6_SFLOAT_BLOCK_EXT: [[fallthrough]]; + case VK_FORMAT_ASTC_10x8_SFLOAT_BLOCK_EXT: [[fallthrough]]; + case VK_FORMAT_ASTC_10x10_SFLOAT_BLOCK_EXT: [[fallthrough]]; + case VK_FORMAT_ASTC_12x10_SFLOAT_BLOCK_EXT: [[fallthrough]]; + case VK_FORMAT_ASTC_12x12_SFLOAT_BLOCK_EXT: [[fallthrough]]; + case VK_FORMAT_ASTC_3x3x3_UNORM_BLOCK_EXT: [[fallthrough]]; + case VK_FORMAT_ASTC_3x3x3_SRGB_BLOCK_EXT: [[fallthrough]]; + case VK_FORMAT_ASTC_3x3x3_SFLOAT_BLOCK_EXT: [[fallthrough]]; + case VK_FORMAT_ASTC_4x3x3_UNORM_BLOCK_EXT: [[fallthrough]]; + case VK_FORMAT_ASTC_4x3x3_SRGB_BLOCK_EXT: [[fallthrough]]; + case VK_FORMAT_ASTC_4x3x3_SFLOAT_BLOCK_EXT: [[fallthrough]]; + case VK_FORMAT_ASTC_4x4x3_UNORM_BLOCK_EXT: [[fallthrough]]; + case VK_FORMAT_ASTC_4x4x3_SRGB_BLOCK_EXT: [[fallthrough]]; + case VK_FORMAT_ASTC_4x4x3_SFLOAT_BLOCK_EXT: [[fallthrough]]; + case VK_FORMAT_ASTC_4x4x4_UNORM_BLOCK_EXT: [[fallthrough]]; + case VK_FORMAT_ASTC_4x4x4_SRGB_BLOCK_EXT: [[fallthrough]]; + case VK_FORMAT_ASTC_4x4x4_SFLOAT_BLOCK_EXT: [[fallthrough]]; + case VK_FORMAT_ASTC_5x4x4_UNORM_BLOCK_EXT: [[fallthrough]]; + case VK_FORMAT_ASTC_5x4x4_SRGB_BLOCK_EXT: [[fallthrough]]; + case VK_FORMAT_ASTC_5x4x4_SFLOAT_BLOCK_EXT: [[fallthrough]]; + case VK_FORMAT_ASTC_5x5x4_UNORM_BLOCK_EXT: [[fallthrough]]; + case VK_FORMAT_ASTC_5x5x4_SRGB_BLOCK_EXT: [[fallthrough]]; + case VK_FORMAT_ASTC_5x5x4_SFLOAT_BLOCK_EXT: [[fallthrough]]; + case VK_FORMAT_ASTC_5x5x5_UNORM_BLOCK_EXT: [[fallthrough]]; + case VK_FORMAT_ASTC_5x5x5_SRGB_BLOCK_EXT: [[fallthrough]]; + case VK_FORMAT_ASTC_5x5x5_SFLOAT_BLOCK_EXT: [[fallthrough]]; + case VK_FORMAT_ASTC_6x5x5_UNORM_BLOCK_EXT: [[fallthrough]]; + case VK_FORMAT_ASTC_6x5x5_SRGB_BLOCK_EXT: [[fallthrough]]; + case VK_FORMAT_ASTC_6x5x5_SFLOAT_BLOCK_EXT: [[fallthrough]]; + case VK_FORMAT_ASTC_6x6x5_UNORM_BLOCK_EXT: [[fallthrough]]; + case VK_FORMAT_ASTC_6x6x5_SRGB_BLOCK_EXT: [[fallthrough]]; + case VK_FORMAT_ASTC_6x6x5_SFLOAT_BLOCK_EXT: [[fallthrough]]; + case VK_FORMAT_ASTC_6x6x6_UNORM_BLOCK_EXT: [[fallthrough]]; + case VK_FORMAT_ASTC_6x6x6_SRGB_BLOCK_EXT: [[fallthrough]]; + case VK_FORMAT_ASTC_6x6x6_SFLOAT_BLOCK_EXT: + return true; + default: + return false; + } +} + +// ------------------------------------------------------------------------------------------------- + +int EXIT_CODE_ERROR = 2; +int EXIT_CODE_MISMATCH = 1; +int EXIT_CODE_MATCH = 0; + +template +void error(int return_code, Args&&... args) { + fmt::print(std::cerr, std::forward(args)...); + std::exit(return_code); +} + +[[nodiscard]] inline std::string errnoMessage() { + return std::make_error_code(static_cast(errno)).message(); +} + +struct Texture { + std::string filepath; + std::vector rawData; + + KTX_header2 header; + std::vector levelIndices; + const std::byte* levelIndexData = nullptr; + size_t levelIndexSize = 0; + const std::byte* dfdData = nullptr; + size_t dfdSize = 0; + const std::byte* kvdData = nullptr; + size_t kvdSize = 0; + const std::byte* sgdData = nullptr; + size_t sgdSize = 0; + + ktxTexture2* handle = nullptr; + bool transcoded = false; + +public: + explicit Texture(std::string filepath) : + filepath(filepath) { + std::memset(&header, 0, sizeof(header)); + + loadFile(); + loadKTX(); + loadMetadata(); + } + ~Texture() { + std::free(handle); + } + void loadFile(); + void loadKTX(); + void loadMetadata(); + inline ktxTexture2* operator->() const { + return handle; + } +}; + +void Texture::loadFile() { + auto file = std::ifstream(filepath, std::ios::binary | std::ios::in | std::ios::ate); + if (!file) + error(EXIT_CODE_ERROR, "ktxdiff error \"{}\": Failed to open file: {}\n", filepath, errnoMessage()); + + const auto fileSize = file.tellg(); + file.seekg(0); + if (file.fail()) + error(EXIT_CODE_ERROR, "ktxdiff error \"{}\": Failed to seek file: {}\n", filepath, errnoMessage()); + + rawData.resize(fileSize); + file.read(reinterpret_cast(rawData.data()), fileSize); + if (file.fail()) + error(EXIT_CODE_ERROR, "ktxdiff error \"{}\": Failed to read file: {}\n", filepath, errnoMessage()); +} + +void Texture::loadKTX() { + KTX_error_code ec = KTX_SUCCESS; + ec = ktxTexture2_CreateFromMemory( + reinterpret_cast(rawData.data()), + rawData.size(), + KTX_TEXTURE_CREATE_LOAD_IMAGE_DATA_BIT, + &handle); + if (ec != KTX_SUCCESS) + error(EXIT_CODE_ERROR, "ktxdiff error \"{}\": ktxTexture2_CreateFromNamedFile: {}\n", filepath, ktxErrorString(ec)); + + if (ktxTexture2_NeedsTranscoding(handle)) { + ec = ktxTexture2_TranscodeBasis(handle, KTX_TTF_RGBA32, 0); + if (ec != KTX_SUCCESS) + error(EXIT_CODE_ERROR, "ktxdiff error \"{}\": ktxTexture2_TranscodeBasis: {}\n", filepath, ktxErrorString(ec)); + transcoded = true; + } +} + +void Texture::loadMetadata() { + const auto headerData = rawData.data(); + const auto headerSize = sizeof(KTX_header2); + std::memcpy(&header, headerData, headerSize); + + const auto numLevels = std::max(header.levelCount, 1u); + levelIndexData = rawData.data() + sizeof(KTX_header2); + levelIndexSize = sizeof(ktxLevelIndexEntry) * numLevels; + levelIndices.resize(numLevels); + std::memcpy(levelIndices.data(), levelIndexData, levelIndexSize); + + if (header.dataFormatDescriptor.byteLength != 0) { + dfdData = rawData.data() + header.dataFormatDescriptor.byteOffset; + dfdSize = header.dataFormatDescriptor.byteLength; + } + if (header.keyValueData.byteLength != 0) { + kvdData = rawData.data() + header.keyValueData.byteOffset; + kvdSize = header.keyValueData.byteLength; + } + if (header.supercompressionGlobalData.byteLength != 0) { + sgdData = rawData.data() + header.dataFormatDescriptor.byteOffset; + sgdSize = header.dataFormatDescriptor.byteLength; + } +} + +// ------------------------------------------------------------------------------------------------- + +struct CompareResult { + bool match = true; + float difference = 0.f; + std::size_t elementIndex = 0; + std::size_t byteOffset = 0; +}; + +CompareResult compareUnorm8(const char* rawLhs, const char* rawRhs, std::size_t rawSize, float tolerance) { + const auto* lhs = reinterpret_cast(rawLhs); + const auto* rhs = reinterpret_cast(rawRhs); + const auto element_size = sizeof(uint8_t); + const auto count = rawSize / element_size; + + for (std::size_t i = 0; i < count; ++i) { + const auto diff = std::abs(static_cast(lhs[i]) / 255.f - static_cast(rhs[i]) / 255.f); + if (diff > tolerance) + return CompareResult{false, diff, i, i * element_size}; + } + + return CompareResult{}; +} + +CompareResult compareSFloat32(const char* rawLhs, const char* rawRhs, std::size_t rawSize, float tolerance) { + const auto* lhs = reinterpret_cast(rawLhs); + const auto* rhs = reinterpret_cast(rawRhs); + const auto element_size = sizeof(float); + const auto count = rawSize / element_size; + + for (std::size_t i = 0; i < count; ++i) { + const auto diff = std::abs(lhs[i] - rhs[i]); + if (diff > tolerance) + return CompareResult{false, diff, i, i * element_size}; + } + + return CompareResult{}; +} + +auto decodeASTC(const char* compressedData, std::size_t compressedSize, uint32_t width, uint32_t height, + const std::string& filepath, bool isFormatSRGB, uint32_t blockSizeX, uint32_t blockSizeY, uint32_t blockSizeZ) { + + const auto threadCount = 1u; + static constexpr astcenc_swizzle swizzle{ASTCENC_SWZ_R, ASTCENC_SWZ_G, ASTCENC_SWZ_B, ASTCENC_SWZ_A}; + + astcenc_error ec = ASTCENC_SUCCESS; + + const astcenc_profile profile = isFormatSRGB ? ASTCENC_PRF_LDR_SRGB : ASTCENC_PRF_LDR; + astcenc_config config{}; + ec = astcenc_config_init(profile, blockSizeX, blockSizeY, blockSizeZ, ASTCENC_PRE_MEDIUM, ASTCENC_FLG_DECOMPRESS_ONLY, &config); + if (ec != ASTCENC_SUCCESS) + error(EXIT_CODE_ERROR, "ktxdiff error \"{}\": astcenc_config_init: {}\n", filepath, astcenc_get_error_string(ec)); + + struct ASTCencStruct { + astcenc_context* context = nullptr; + ~ASTCencStruct() { + astcenc_context_free(context); + } + } astcenc; + astcenc_context*& context = astcenc.context; + + ec = astcenc_context_alloc(&config, threadCount, &context); + if (ec != ASTCENC_SUCCESS) + error(EXIT_CODE_ERROR, "ktxdiff error \"{}\": astcenc_context_alloc: {}\n", filepath, astcenc_get_error_string(ec)); + + astcenc_image image{}; + image.dim_x = width; + image.dim_y = height; + image.dim_z = 1; // 3D ASTC formats are currently not supported + const auto uncompressedSize = width * height * 4 * sizeof(uint8_t); + auto uncompressedBuffer = std::make_unique(uncompressedSize); + auto* bufferPtr = uncompressedBuffer.get(); + image.data = reinterpret_cast(&bufferPtr); + image.data_type = ASTCENC_TYPE_U8; + + ec = astcenc_decompress_image(context, reinterpret_cast(compressedData), compressedSize, &image, &swizzle, 0); + if (ec != ASTCENC_SUCCESS) + error(EXIT_CODE_ERROR, "ktxdiff error \"{}\": astcenc_decompress_image: {}\n", filepath, astcenc_get_error_string(ec)); + + astcenc_decompress_reset(context); + + struct Result { + std::unique_ptr data; + std::size_t size; + }; + return Result{std::move(uncompressedBuffer), uncompressedSize}; +} + +CompareResult compareAstc(const char* lhs, const char* rhs, std::size_t size, uint32_t width, uint32_t height, + const std::string& filepathLhs, const std::string& filepathRhs, + bool isFormatSRGB, uint32_t blockSizeX, uint32_t blockSizeY, uint32_t blockSizeZ, + float tolerance) { + const auto uncompressedLhs = decodeASTC(lhs, size, width, height, filepathLhs, isFormatSRGB, blockSizeX, blockSizeY, blockSizeZ); + const auto uncompressedRhs = decodeASTC(rhs, size, width, height, filepathRhs, isFormatSRGB, blockSizeX, blockSizeY, blockSizeZ); + + return compareUnorm8( + reinterpret_cast(uncompressedLhs.data.get()), + reinterpret_cast(uncompressedRhs.data.get()), + uncompressedLhs.size, + tolerance); +} + +bool compare(Texture& lhs, Texture& rhs, float tolerance) { + const auto vkFormat = static_cast(lhs.header.vkFormat); + const auto* bdfd = reinterpret_cast(lhs.dfdData) + 1; + const auto componentCount = KHR_DFDSAMPLECOUNT(bdfd); + const auto texelBlockDimension0 = static_cast(KHR_DFDVAL(bdfd, TEXELBLOCKDIMENSION0)); + const auto texelBlockDimension1 = static_cast(KHR_DFDVAL(bdfd, TEXELBLOCKDIMENSION1)); + const auto texelBlockDimension2 = static_cast(KHR_DFDVAL(bdfd, TEXELBLOCKDIMENSION2)); + const auto blockSizeX = texelBlockDimension0 + 1u; + const auto blockSizeY = texelBlockDimension1 + 1u; + const auto blockSizeZ = texelBlockDimension2 + 1u; + const bool isFormatSRGB = KHR_DFDVAL(bdfd, TRANSFER) == KHR_DF_TRANSFER_SRGB; + + const bool isSigned = (KHR_DFDSVAL(bdfd, 0, QUALIFIERS) & KHR_DF_SAMPLE_DATATYPE_SIGNED) != 0; + const bool isFloat = (KHR_DFDSVAL(bdfd, 0, QUALIFIERS) & KHR_DF_SAMPLE_DATATYPE_FLOAT) != 0; + const bool isNormalized = KHR_DFDSVAL(bdfd, 0, SAMPLEUPPER) == (isFloat ? bit_cast(1.0f) : 1u); + const bool is32Bit = KHR_DFDSVAL(bdfd, 0, BITLENGTH) + 1 == 32; + const bool is8Bit = KHR_DFDSVAL(bdfd, 0, BITLENGTH) + 1 == 8; + const bool isFormatSFloat32 = isSigned && isFloat && is32Bit && vkFormat != VK_FORMAT_D32_SFLOAT_S8_UINT; + const bool isFormatUNORM8 = !isSigned && !isFloat && is8Bit && isNormalized; + + const auto mismatch = [&](auto&&... args) { + fmt::print("ktxdiff: "); + fmt::print(std::forward(args)...); + fmt::print(" between\n"); + fmt::print(" Expected: {} and\n", lhs.filepath); + fmt::print(" Received: {}\n", rhs.filepath); + return false; + }; + + if (lhs.transcoded) { + // For encoded images the compressed data sizes can differ. + // Skip the related checks for header.supercompressionGlobalData and levelIndex + if (std::memcmp(&lhs.header, &rhs.header, sizeof(lhs.header) - sizeof(ktxIndexEntry64)) != 0) + return mismatch("Mismatching header"); + } else { + if (std::memcmp(&lhs.header, &rhs.header, sizeof(lhs.header)) != 0) + return mismatch("Mismatching header"); + if (lhs.levelIndexSize != rhs.levelIndexSize) + return mismatch("Mismatching levelIndices"); + for (uint32_t i = 0; i < lhs.levelIndices.size(); ++i) + // Offsets and (compressed) sizes can differ, but uncompressedByteLength must match + if (lhs.levelIndices[i].uncompressedByteLength != rhs.levelIndices[i].uncompressedByteLength) + return mismatch("Mismatching levelIndices[{}].uncompressedByteLength", i); + } + if (lhs.dfdSize != rhs.dfdSize || std::memcmp(lhs.dfdData, rhs.dfdData, lhs.dfdSize) != 0) + return mismatch("Mismatching DFD"); + + if (lhs.kvdSize != rhs.kvdSize || std::memcmp(lhs.kvdData, rhs.kvdData, lhs.kvdSize) != 0) + return mismatch("Mismatching KVD"); + + if (!lhs.transcoded) + if (lhs.sgdSize != rhs.sgdSize || std::memcmp(lhs.sgdData, rhs.sgdData, lhs.sgdSize) != 0) + return mismatch("Mismatching SGD"); + + // If the tolerance is 1 or above accept every image data as matching + if (tolerance >= 1.0f) + return true; + + for (uint32_t levelIndex = 0; levelIndex < lhs->numLevels; ++levelIndex) { + const auto imageSize = ktxTexture_GetImageSize(ktxTexture(lhs.handle), levelIndex); + const auto imageWidth = std::max(1u, lhs->baseWidth >> levelIndex); + const auto imageHeight = std::max(1u, lhs->baseHeight >> levelIndex); + const auto imageDepth = std::max(1u, lhs->baseDepth >> levelIndex); + + for (uint32_t faceIndex = 0; faceIndex < lhs->numFaces; ++faceIndex) { + for (uint32_t layerIndex = 0; layerIndex < lhs->numLayers; ++layerIndex) { + for (uint32_t depthIndex = 0; depthIndex < ceil_div(imageDepth, blockSizeZ); ++depthIndex) { + + ktx_size_t imageOffset; + ktxTexture2_GetImageOffset(lhs.handle, levelIndex, layerIndex, faceIndex + depthIndex, &imageOffset); + const char* imageDataLhs = reinterpret_cast(lhs->pData) + imageOffset; + const char* imageDataRhs = reinterpret_cast(rhs->pData) + imageOffset; + + CompareResult result; + if (lhs.transcoded || isFormatUNORM8) { + result = compareUnorm8(imageDataLhs, imageDataRhs, imageSize, tolerance); + } else if (isFormatAstc(vkFormat)) { + result = compareAstc(imageDataLhs, imageDataRhs, imageSize, imageWidth, imageHeight, + lhs.filepath, rhs.filepath, + isFormatSRGB, blockSizeX, blockSizeY, blockSizeZ, + tolerance); + } else if (isFormatSFloat32) { + result = compareSFloat32(imageDataLhs, imageDataRhs, imageSize, tolerance); + } else { + for (std::size_t i = 0; i < imageSize; ++i) { + if (imageDataLhs[i] != imageDataRhs[i]) + return mismatch("Mismatching image data: level {}, face {}, layer {}, depth {}, image byte {}", + levelIndex, faceIndex, layerIndex, depthIndex, i); + } + } + + if (!result.match) { + return mismatch("Mismatching image data (diff: {}): level {}, face {}, layer {}, depth {}, pixel {}, component {}", + result.difference, levelIndex, faceIndex, layerIndex, depthIndex, + result.elementIndex / componentCount, result.elementIndex % componentCount); + } + } + } + } + } + + return true; +} + +/// EXIT CODES: +/// 0 - Matching files +/// 1 - Mismatching files +/// 2 - Error while loading, decoding or processing an input file +int main(int argc, const char* argv[]) { + if (argc < 3) { + fmt::print("Missing input file arguments\n"); + fmt::print("Usage: ktxdiff [tolerance]\n"); + return EXIT_FAILURE; + } + + const float tolerance = argc > 3 ? std::stof(argv[3]) : 0.05f; + + Texture lhs(argv[1]); + Texture rhs(argv[2]); + const auto match = compare(lhs, rhs, tolerance); + + return match ? 0 : 1; +} diff --git a/tools/ktx/encode_utils.h b/tools/ktx/encode_utils.h index 914dc0089b..581c9ec28d 100644 --- a/tools/ktx/encode_utils.h +++ b/tools/ktx/encode_utils.h @@ -107,7 +107,7 @@ enum class EncodeCodec { 4 Very slow 48.24dB - You are strongly encouraged to also specify @b --zcmp to + You are strongly encouraged to also specify @b --zstd to losslessly compress the UASTC data. This and any LZ-style compression can be made more effective by conditioning the UASTC texture data using the Rate Distortion Optimization (RDO) diff --git a/tools/ktx/image.hpp b/tools/ktx/image.hpp index e3ac42dd90..0e602082dd 100644 --- a/tools/ktx/image.hpp +++ b/tools/ktx/image.hpp @@ -1248,7 +1248,12 @@ class ImageT : public Image { // Encode destination transfer function for (uint32_t comp = 0; comp < components; comp++) { brightness[comp] = encode.encode(intensity[comp]); - c.set(comp, roundf(brightness[comp] * static_cast(Color::one()))); + // clamp(value, color::min, color::max) is required as static_cast has platform-specific behaviors + // and on certain platforms can over or underflow + c.set(comp, cclamp( + roundf(brightness[comp] * static_cast(Color::one())), + static_cast(Color::min()), + static_cast(Color::max()))); } } return *this;