Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

gltfpack: Implement support for KTX2 with supercompression #81

Merged
merged 16 commits into from
Nov 27, 2019
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -58,7 +58,7 @@ if(BUILD_DEMO)
endif()

if(BUILD_TOOLS)
add_executable(gltfpack tools/gltfpack.cpp tools/meshloader.cpp)
add_executable(gltfpack tools/gltfpack.cpp tools/meshloader.cpp tools/basistoktx.cpp)
target_link_libraries(gltfpack meshoptimizer)
list(APPEND TARGETS gltfpack)

2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
@@ -12,7 +12,7 @@ LIBRARY_OBJECTS=$(LIBRARY_SOURCES:%=$(BUILD)/%.o)
DEMO_SOURCES=$(wildcard demo/*.c demo/*.cpp) tools/meshloader.cpp
DEMO_OBJECTS=$(DEMO_SOURCES:%=$(BUILD)/%.o)

GLTFPACK_SOURCES=tools/gltfpack.cpp tools/meshloader.cpp
GLTFPACK_SOURCES=tools/gltfpack.cpp tools/meshloader.cpp tools/basistoktx.cpp
GLTFPACK_OBJECTS=$(GLTFPACK_SOURCES:%=$(BUILD)/%.o)

OBJECTS=$(LIBRARY_OBJECTS) $(DEMO_OBJECTS) $(GLTFPACK_OBJECTS)
3 changes: 2 additions & 1 deletion src/vertexcodec.cpp
Original file line number Diff line number Diff line change
@@ -420,7 +420,8 @@ static unsigned char kDecodeBytesGroupCount[256];
#ifdef EMSCRIPTEN
__attribute__((cold)) // this saves 500 bytes in the output binary - we don't need to vectorize this loop!
#endif
static bool decodeBytesGroupBuildTables()
static bool
decodeBytesGroupBuildTables()
{
for (int mask = 0; mask < 256; ++mask)
{
294 changes: 294 additions & 0 deletions tools/basistoktx.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,294 @@
#include <stdexcept>
#include <string>
#include <vector>

#include <assert.h>
#include <stdint.h>
#include <stdio.h>
#include <string.h>

#include "basisu_format.h"
#include "khr_df.h"
#include "ktx2_format.h"

template <typename T>
static void read(const std::string& data, size_t offset, T& result)
{
if (offset + sizeof(T) > data.size())
throw std::out_of_range("read");

memcpy(&result, &data[offset], sizeof(T));
}

template <typename T>
static void write(std::string& data, const T& value)
{
data.append(reinterpret_cast<const char*>(&value), sizeof(value));
}

template <typename T>
static void write(std::string& data, size_t offset, const T& value)
{
if (offset + sizeof(T) > data.size())
throw std::out_of_range("write");

memcpy(&data[offset], &value, sizeof(T));
}

static void createDfd(std::vector<uint32_t>& result, int channels, bool srgb)
{
assert(channels <= 4);

int descriptor_size = KHR_DF_WORD_SAMPLESTART + channels * KHR_DF_WORD_SAMPLEWORDS;

result.clear();
result.resize(1 + descriptor_size);

result[0] = (1 + descriptor_size) * sizeof(uint32_t);

uint32_t* dfd = &result[1];

KHR_DFDSETVAL(dfd, VENDORID, KHR_DF_VENDORID_KHRONOS);
KHR_DFDSETVAL(dfd, DESCRIPTORTYPE, KHR_DF_KHR_DESCRIPTORTYPE_BASICFORMAT);
KHR_DFDSETVAL(dfd, VERSIONNUMBER, KHR_DF_VERSIONNUMBER_1_3);
KHR_DFDSETVAL(dfd, DESCRIPTORBLOCKSIZE, descriptor_size * sizeof(uint32_t));
KHR_DFDSETVAL(dfd, MODEL, KHR_DF_MODEL_RGBSDA);
KHR_DFDSETVAL(dfd, PRIMARIES, KHR_DF_PRIMARIES_BT709);
KHR_DFDSETVAL(dfd, TRANSFER, srgb ? KHR_DF_TRANSFER_SRGB : KHR_DF_TRANSFER_LINEAR);
KHR_DFDSETVAL(dfd, FLAGS, KHR_DF_FLAG_ALPHA_STRAIGHT);

static const khr_df_model_channels_e channel_enums[] = {
KHR_DF_CHANNEL_RGBSDA_R,
KHR_DF_CHANNEL_RGBSDA_G,
KHR_DF_CHANNEL_RGBSDA_B,
KHR_DF_CHANNEL_RGBSDA_A,
};

for (int i = 0; i < channels; ++i)
{
KHR_DFDSETSVAL(dfd, i, CHANNELID, channel_enums[i]);
}
}

std::string basisToKtx(const std::string& basis, bool srgb)
{
std::string ktx;

basist::basis_file_header basis_header;
read(basis, 0, basis_header);

assert(basis_header.m_sig == basist::basis_file_header::cBASISSigValue);

assert(basis_header.m_total_slices > 0);
assert(basis_header.m_total_images == 1);

assert(basis_header.m_format == 0);
assert(basis_header.m_flags & basist::cBASISHeaderFlagETC1S);
assert(!(basis_header.m_flags & basist::cBASISHeaderFlagYFlipped));
assert(basis_header.m_tex_type == basist::cBASISTexType2D);

bool has_alpha = (basis_header.m_flags & basist::cBASISHeaderFlagHasAlphaSlices) != 0;

std::vector<basist::basis_slice_desc> slices(basis_header.m_total_slices);

for (size_t i = 0; i < basis_header.m_total_slices; ++i)
read(basis, basis_header.m_slice_desc_file_ofs + i * sizeof(basist::basis_slice_desc), slices[i]);

assert(slices[0].m_level_index == 0);
uint32_t width = slices[0].m_orig_width;
uint32_t height = slices[0].m_orig_height;
uint32_t levels = has_alpha ? uint32_t(slices.size()) / 2 : uint32_t(slices.size());

KTX_header2 ktx_header = {KTX2_IDENTIFIER_REF};
ktx_header.typeSize = 1;
ktx_header.pixelWidth = width;
ktx_header.pixelHeight = height;
ktx_header.layerCount = 0;
ktx_header.faceCount = 1;
ktx_header.levelCount = levels;
ktx_header.supercompressionScheme = KTX_SUPERCOMPRESSION_BASIS;

size_t header_size = sizeof(KTX_header2) + levels * sizeof(ktxLevelIndexEntry);

std::vector<uint32_t> dfd;
createDfd(dfd, has_alpha ? 4 : 3, srgb);

const char* kvp_data[][2] = {
{"KTXwriter", "gltfpack"},
};

std::string kvp;

for (size_t i = 0; i < sizeof(kvp_data) / sizeof(kvp_data[0]); ++i)
{
const char* key = kvp_data[i][0];
const char* value = kvp_data[i][1];

write(kvp, uint32_t(strlen(key) + strlen(value) + 2));
kvp += key;
kvp += '\0';
kvp += value;
kvp += '\0';

if (i + 1 != kvp.size())
kvp.resize((kvp.size() + 3) & ~3);
}

size_t kvp_size = kvp.size();
size_t dfd_size = dfd.size() * sizeof(uint32_t);

size_t bgd_size =
sizeof(ktxBasisGlobalHeader) + sizeof(ktxBasisSliceDesc) * levels +
basis_header.m_endpoint_cb_file_size + basis_header.m_selector_cb_file_size + basis_header.m_tables_file_size;

ktx_header.dataFormatDescriptor.byteOffset = uint32_t(header_size);
ktx_header.dataFormatDescriptor.byteLength = uint32_t(dfd_size);

ktx_header.keyValueData.byteOffset = uint32_t(header_size + dfd_size);
ktx_header.keyValueData.byteLength = uint32_t(kvp_size);

ktx_header.supercompressionGlobalData.byteOffset = (header_size + dfd_size + kvp_size + 7) & ~7;
ktx_header.supercompressionGlobalData.byteLength = bgd_size;

// KTX2 header
write(ktx, ktx_header);

size_t ktx_level_offset = ktx.size();

for (size_t i = 0; i < levels; ++i)
{
ktxLevelIndexEntry le = {}; // This will be patched later
write(ktx, le);
}

// data format descriptor
for (size_t i = 0; i < dfd.size(); ++i)
write(ktx, dfd[i]);

// key/value pair data
ktx += kvp;
ktx.resize((ktx.size() + 7) & ~7);

// supercompression global data
ktxBasisGlobalHeader sgd_header = {};
sgd_header.globalFlags = basis_header.m_flags;
sgd_header.endpointCount = basis_header.m_total_endpoints;
sgd_header.selectorCount = basis_header.m_total_selectors;
sgd_header.endpointsByteLength = basis_header.m_endpoint_cb_file_size;
sgd_header.selectorsByteLength = basis_header.m_selector_cb_file_size;
sgd_header.tablesByteLength = basis_header.m_tables_file_size;
sgd_header.extendedByteLength = basis_header.m_extended_file_size;

write(ktx, sgd_header);

size_t sgd_level_offset = ktx.size();

for (size_t i = 0; i < levels; ++i)
{
ktxBasisSliceDesc sgd_slice = {}; // This will be patched later
write(ktx, sgd_slice);
}

ktx.append(basis.substr(basis_header.m_endpoint_cb_file_ofs, basis_header.m_endpoint_cb_file_size));
ktx.append(basis.substr(basis_header.m_selector_cb_file_ofs, basis_header.m_selector_cb_file_size));
ktx.append(basis.substr(basis_header.m_tables_file_ofs, basis_header.m_tables_file_size));
ktx.append(basis.substr(basis_header.m_extended_file_ofs, basis_header.m_extended_file_size));

ktx.resize((ktx.size() + 7) & ~7);

// mip levels
for (size_t i = 0; i < levels; ++i)
{
size_t slice_index = (levels - i - 1) * (has_alpha + 1);
const basist::basis_slice_desc& slice = slices[slice_index];
const basist::basis_slice_desc* slice_alpha = has_alpha ? &slices[slice_index + 1] : 0;

assert(slice.m_image_index == 0);
assert(slice.m_level_index == levels - i - 1);

size_t file_offset = ktx.size();

ktx.append(basis.substr(slice.m_file_ofs, slice.m_file_size));

if (slice_alpha)
ktx.append(basis.substr(slice_alpha->m_file_ofs, slice_alpha->m_file_size));

ktxLevelIndexEntry le = {};
le.byteOffset = file_offset;
le.byteLength = ktx.size() - file_offset;
le.uncompressedByteLength = 0;

write(ktx, ktx_level_offset + i * sizeof(ktxLevelIndexEntry), le);

ktxBasisSliceDesc sgd_slice = {};
sgd_slice.sliceByteOffset = 0;
sgd_slice.sliceByteLength = slice.m_file_size;

if (slice_alpha)
{
sgd_slice.alphaSliceByteOffset = slice.m_file_size;
sgd_slice.alphaSliceByteLength = slice_alpha->m_file_size;
}

write(ktx, sgd_level_offset + i * sizeof(ktxBasisSliceDesc), sgd_slice);

if (i + 1 != levels)
ktx.resize((ktx.size() + 7) & ~7);
}

return ktx;
}

#ifdef STANDALONE
bool readFile(const char* path, std::string& data)
{
FILE* file = fopen(path, "rb");
if (!file)
return false;

fseek(file, 0, SEEK_END);
long length = ftell(file);
fseek(file, 0, SEEK_SET);

if (length <= 0)
{
fclose(file);
return false;
}

data.resize(length);
size_t result = fread(&data[0], 1, data.size(), file);
fclose(file);

return result == data.size();
}

bool writeFile(const char* path, const std::string& data)
{
FILE* file = fopen(path, "wb");
if (!file)
return false;

size_t result = fwrite(&data[0], 1, data.size(), file);
fclose(file);

return result == data.size();
}

int main(int argc, const char** argv)
{
if (argc < 2)
return 1;

std::string basis;
if (!readFile(argv[1], basis))
return 1;

std::string ktx = basisToKtx(basis, true);

if (!writeFile(argv[2], ktx))
return 1;

return 0;
}
#endif
138 changes: 138 additions & 0 deletions tools/basisu_format.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,138 @@
// basis_file_headers.h + basisu.h
// Copyright (C) 2019 Binomial LLC. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once

namespace basisu
{
// Always little endian 2-4 byte unsigned int
template<uint32_t NumBytes>
struct packed_uint
{
uint8_t m_bytes[NumBytes];

operator uint32_t() const
{
uint32_t result = 0;
for (uint32_t i = 0; i < NumBytes; i++)
result |= m_bytes[i] << (8 * i);
return result;
}
};
}

namespace basist
{
// Slice desc header flags
enum basis_slice_desc_flags
{
cSliceDescFlagsIsAlphaData = 1,
cSliceDescFlagsFrameIsIFrame = 2 // Video only: Frame doesn't refer to previous frame (no usage of conditional replenishment pred symbols)
};

#pragma pack(push)
#pragma pack(1)
struct basis_slice_desc
{
basisu::packed_uint<3> m_image_index; // The index of the source image provided to the encoder (will always appear in order from first to last, first image index is 0, no skipping allowed)
basisu::packed_uint<1> m_level_index; // The mipmap level index (mipmaps will always appear from largest to smallest)
basisu::packed_uint<1> m_flags; // enum basis_slice_desc_flags

basisu::packed_uint<2> m_orig_width; // The original image width (may not be a multiple of 4 pixels)
basisu::packed_uint<2> m_orig_height; // The original image height (may not be a multiple of 4 pixels)

basisu::packed_uint<2> m_num_blocks_x; // The slice's block X dimensions. Each block is 4x4 pixels. The slice's pixel resolution may or may not be a power of 2.
basisu::packed_uint<2> m_num_blocks_y; // The slice's block Y dimensions.

basisu::packed_uint<4> m_file_ofs; // Offset from the header to the start of the slice's data
basisu::packed_uint<4> m_file_size; // The size of the compressed slice data in bytes

basisu::packed_uint<2> m_slice_data_crc16; // The CRC16 of the compressed slice data, for extra-paranoid use cases
};

// File header files
enum basis_header_flags
{
cBASISHeaderFlagETC1S = 1, // Always set for basis universal files
cBASISHeaderFlagYFlipped = 2, // Set if the texture had to be Y flipped before encoding
cBASISHeaderFlagHasAlphaSlices = 4 // True if the odd slices contain alpha data
};

// The image type field attempts to describe how to interpret the image data in a Basis file.
// The encoder library doesn't really do anything special or different with these texture types, this is mostly here for the benefit of the user.
// We do make sure the various constraints are followed (2DArray/cubemap/videoframes/volume implies that each image has the same resolution and # of mipmap levels, etc., cubemap implies that the # of image slices is a multiple of 6)
enum basis_texture_type
{
cBASISTexType2D = 0, // An arbitrary array of 2D RGB or RGBA images with optional mipmaps, array size = # images, each image may have a different resolution and # of mipmap levels
cBASISTexType2DArray = 1, // An array of 2D RGB or RGBA images with optional mipmaps, array size = # images, each image has the same resolution and mipmap levels
cBASISTexTypeCubemapArray = 2, // an array of cubemap levels, total # of images must be divisable by 6, in X+, X-, Y+, Y-, Z+, Z- order, with optional mipmaps
cBASISTexTypeVideoFrames = 3, // An array of 2D video frames, with optional mipmaps, # frames = # images, each image has the same resolution and # of mipmap levels
cBASISTexTypeVolume = 4, // A 3D texture with optional mipmaps, Z dimension = # images, each image has the same resolution and # of mipmap levels

cBASISTexTypeTotal
};

enum
{
cBASISMaxUSPerFrame = 0xFFFFFF
};

struct basis_file_header
{
enum
{
cBASISSigValue = ('B' << 8) | 's',
cBASISFirstVersion = 0x10
};

basisu::packed_uint<2> m_sig; // 2 byte file signature
basisu::packed_uint<2> m_ver; // Baseline file version
basisu::packed_uint<2> m_header_size; // Header size in bytes, sizeof(basis_file_header)
basisu::packed_uint<2> m_header_crc16; // crc16 of the remaining header data

basisu::packed_uint<4> m_data_size; // The total size of all data after the header
basisu::packed_uint<2> m_data_crc16; // The CRC16 of all data after the header

basisu::packed_uint<3> m_total_slices; // The total # of compressed slices (1 slice per image, or 2 for alpha basis files)

basisu::packed_uint<3> m_total_images; // The total # of images

basisu::packed_uint<1> m_format; // enum basist::block_format
basisu::packed_uint<2> m_flags; // enum basist::header_flags
basisu::packed_uint<1> m_tex_type; // enum basist::basis_texture_type
basisu::packed_uint<3> m_us_per_frame; // Framerate of video, in microseconds per frame

basisu::packed_uint<4> m_reserved; // For future use
basisu::packed_uint<4> m_userdata0; // For client use
basisu::packed_uint<4> m_userdata1; // For client use

basisu::packed_uint<2> m_total_endpoints; // The number of endpoints in the endpoint codebook
basisu::packed_uint<4> m_endpoint_cb_file_ofs; // The compressed endpoint codebook's file offset relative to the header
basisu::packed_uint<3> m_endpoint_cb_file_size; // The compressed endpoint codebook's size in bytes

basisu::packed_uint<2> m_total_selectors; // The number of selectors in the endpoint codebook
basisu::packed_uint<4> m_selector_cb_file_ofs; // The compressed selectors codebook's file offset relative to the header
basisu::packed_uint<3> m_selector_cb_file_size; // The compressed selector codebook's size in bytes

basisu::packed_uint<4> m_tables_file_ofs; // The file offset of the compressed Huffman codelength tables, for decompressing slices
basisu::packed_uint<4> m_tables_file_size; // The file size in bytes of the compressed huffman codelength tables

basisu::packed_uint<4> m_slice_desc_file_ofs; // The file offset to the slice description array, usually follows the header

basisu::packed_uint<4> m_extended_file_ofs; // The file offset of the "extended" header and compressed data, for future use
basisu::packed_uint<4> m_extended_file_size; // The file size in bytes of the "extended" header and compressed data, for future use
};
#pragma pack (pop)

} // namespace basist
330 changes: 190 additions & 140 deletions tools/gltfpack.cpp

Large diffs are not rendered by default.

627 changes: 627 additions & 0 deletions tools/khr_df.h

Large diffs are not rendered by default.

125 changes: 125 additions & 0 deletions tools/ktx2_format.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
/*
* Copyright (c) 2010-2018 The Khronos Group Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/


/*
* Author: Mark Callow from original code by Georg Kolling
*/

/*
* Converted from ktxint.h + basis_sgd.h by extracting meaningful structures for gltfpack
*/

#pragma once

#include <stdint.h>

#define KTX2_IDENTIFIER_REF { 0xAB, 0x4B, 0x54, 0x58, 0x20, 0x32, 0x30, 0xBB, 0x0D, 0x0A, 0x1A, 0x0A }
#define KTX2_HEADER_SIZE (80)

typedef enum ktxSupercmpScheme {
KTX_SUPERCOMPRESSION_NONE = 0, /*!< No supercompression. */
KTX_SUPERCOMPRESSION_BASIS = 1, /*!< Basis Universal supercompression. */
KTX_SUPERCOMPRESSION_LZMA = 2, /*!< LZMA supercompression. */
KTX_SUPERCOMPRESSION_ZLIB = 3, /*!< Zlib supercompression. */
KTX_SUPERCOMPRESSION_ZSTD = 4, /*!< ZStd supercompression. */
KTX_SUPERCOMPRESSION_BEGIN_RANGE = KTX_SUPERCOMPRESSION_NONE,
KTX_SUPERCOMPRESSION_END_RANGE = KTX_SUPERCOMPRESSION_ZSTD,
KTX_SUPERCOMPRESSION_BEGIN_VENDOR_RANGE = 0x10000,
KTX_SUPERCOMPRESSION_END_VENDOR_RANGE = 0x1ffff,
KTX_SUPERCOMPRESSION_BEGIN_RESERVED = 0x20000,
} ktxSupercmpScheme;

/**
* @internal
* @~English
* @brief 32-bit KTX 2 index entry.
*/
typedef struct ktxIndexEntry32 {
uint32_t byteOffset; /*!< Offset of item from start of file. */
uint32_t byteLength; /*!< Number of bytes of data in the item. */
} ktxIndexEntry32;
/**
* @internal
* @~English
* @brief 64-bit KTX 2 index entry.
*/
typedef struct ktxIndexEntry64 {
uint64_t byteOffset; /*!< Offset of item from start of file. */
uint64_t byteLength; /*!< Number of bytes of data in the item. */
} ktxIndexEntry64;

/**
* @internal
* @~English
* @brief KTX 2 file header.
*
* See the KTX 2 specification for descriptions.
*/
typedef struct KTX_header2 {
uint8_t identifier[12];
uint32_t vkFormat;
uint32_t typeSize;
uint32_t pixelWidth;
uint32_t pixelHeight;
uint32_t pixelDepth;
uint32_t layerCount;
uint32_t faceCount;
uint32_t levelCount;
uint32_t supercompressionScheme;
ktxIndexEntry32 dataFormatDescriptor;
ktxIndexEntry32 keyValueData;
ktxIndexEntry64 supercompressionGlobalData;
} KTX_header2;

/* This will cause compilation to fail if the struct size doesn't match */
typedef int KTX_header2_SIZE_ASSERT [sizeof(KTX_header2) == KTX2_HEADER_SIZE];

/**
* @internal
* @~English
* @brief KTX 2 level index entry.
*/
typedef struct ktxLevelIndexEntry {
uint64_t byteOffset; /*!< Offset of level from start of file. */
uint64_t byteLength;
/*!< Number of bytes of compressed image data in the level. */
uint64_t uncompressedByteLength;
/*!< Number of bytes of uncompressed image data in the level. */
} ktxLevelIndexEntry;

typedef struct ktxBasisGlobalHeader {
uint32_t globalFlags;
uint16_t endpointCount;
uint16_t selectorCount;
uint32_t endpointsByteLength;
uint32_t selectorsByteLength;
uint32_t tablesByteLength;
uint32_t extendedByteLength;
} ktxBasisGlobalHeader;

// This header is followed by imageCount "slice" descriptions.

// 1, or 2 slices per image (i.e. layer, face & slice).
// These offsets are relative to start of a mip level as given by the
// main levelIndex.
typedef struct ktxBasisSliceDesc {
uint32_t sliceFlags;
uint32_t sliceByteOffset;
uint32_t sliceByteLength;
uint32_t alphaSliceByteOffset;
uint32_t alphaSliceByteLength;
} ktxBasisSliceDesc;