Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

gltfpack: Deduplicate mesh geometry #782

Merged
merged 8 commits into from
Oct 7, 2024
Merged

gltfpack: Deduplicate mesh geometry #782

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
406a4ad
gltfpack: Implement computation of mesh geometry hash
zeux Oct 5, 2024
31941ea
gltfpack: Implement mesh deduplication using geometry hash
zeux Oct 5, 2024
9f23257
gltfpack: Merge mesh materials before deduplication
zeux Oct 5, 2024
4cc9fb5
gltfpack: Avoid detaching meshes with duplicate geometry
zeux Oct 5, 2024
63cae78
gltfpack: Refactor mesh geometry serialization
zeux Oct 5, 2024
188a66e
gltfpack: Deduplicate primitive accessors/data during writing
zeux Oct 5, 2024
61950f7
gltfpack: Record the fact that mesh geometry might be duplicate
zeux Oct 5, 2024
82e0332
gltfpack: Add temporary command line option to disable mesh deduplica…
zeux Oct 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 39 additions & 24 deletions gltf/gltfpack.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
#include "gltfpack.h"

#include <algorithm>
#include <map>

#include <locale.h>
#include <stdint.h>
Expand Down Expand Up @@ -295,6 +296,10 @@ static void detachMesh(Mesh& mesh, cgltf_data* data, const std::vector<NodeInfo>
if (mesh.nodes.size() > 1 && !settings.mesh_merge && !settings.mesh_instancing)
return;

// mesh has duplicate geometry; detaching it would increase the size due to unique world-space transforms
if (mesh.nodes.size() == 1 && mesh.geometry_duplicate && !settings.mesh_merge)
return;

// prefer instancing if possible, use merging otherwise
if (mesh.nodes.size() > 1 && settings.mesh_instancing)
{
Expand Down Expand Up @@ -341,6 +346,10 @@ static void process(cgltf_data* data, const char* input_path, const char* output
markScenes(data, nodes);
markAnimated(data, nodes, animations);

mergeMeshMaterials(data, meshes, settings);
if (settings.mesh_dedup)
dedupMeshes(meshes);

for (size_t i = 0; i < meshes.size(); ++i)
detachMesh(meshes[i], data, nodes, settings);

Expand Down Expand Up @@ -375,7 +384,6 @@ static void process(cgltf_data* data, const char* input_path, const char* output
filterStreams(mesh, mi);
}

mergeMeshMaterials(data, meshes, settings);
mergeMeshes(meshes, settings);
filterEmptyMeshes(meshes);

Expand Down Expand Up @@ -408,7 +416,13 @@ static void process(cgltf_data* data, const char* input_path, const char* output
#endif

for (size_t i = 0; i < meshes.size(); ++i)
processMesh(meshes[i], settings);
{
Mesh& mesh = meshes[i];
processMesh(mesh, settings);

if (mesh.geometry_duplicate)
hashMesh(mesh);
}

#ifndef NDEBUG
meshes.insert(meshes.end(), debug_meshes.begin(), debug_meshes.end());
Expand Down Expand Up @@ -551,6 +565,8 @@ static void process(cgltf_data* data, const char* input_path, const char* output
ext_texture_transform = ext_texture_transform || mi.uses_texture_transform;
}

std::map<std::pair<uint64_t, uint64_t>, std::pair<size_t, size_t> > primitive_cache;

for (size_t i = 0; i < meshes.size(); ++i)
{
const Mesh& mesh = meshes[i];
Expand Down Expand Up @@ -578,33 +594,26 @@ static void process(cgltf_data* data, const char* input_path, const char* output
const QuantizationTexture& qt = qt_meshes[pi] == size_t(-1) ? qt_dummy : qt_materials[qt_meshes[pi]];

comma(json_meshes);
append(json_meshes, "{\"attributes\":{");
writeMeshAttributes(json_meshes, views, json_accessors, accr_offset, prim, 0, qp, qt, settings);
append(json_meshes, "}");
if (prim.type != cgltf_primitive_type_triangles)
{
append(json_meshes, ",\"mode\":");
append(json_meshes, size_t(prim.type - cgltf_primitive_type_points));
}
if (mesh.targets)

if (prim.geometry_duplicate)
{
append(json_meshes, ",\"targets\":[");
for (size_t j = 0; j < mesh.targets; ++j)
std::pair<size_t, size_t>& primitive_json = primitive_cache[std::make_pair(prim.geometry_hash[0], prim.geometry_hash[1])];

if (primitive_json.second)
{
comma(json_meshes);
append(json_meshes, "{");
writeMeshAttributes(json_meshes, views, json_accessors, accr_offset, prim, int(1 + j), qp, qt, settings);
append(json_meshes, "}");
// reuse previously written accessors
json_meshes.append(json_meshes, primitive_json.first, primitive_json.second);
}
else
{
primitive_json.first = json_meshes.size();
writeMeshGeometry(json_meshes, views, json_accessors, accr_offset, prim, qp, qt, settings);
primitive_json.second = json_meshes.size() - primitive_json.first;
}
append(json_meshes, "]");
}

if (!prim.indices.empty())
else
{
size_t index_accr = writeMeshIndices(views, json_accessors, accr_offset, prim, settings);

append(json_meshes, ",\"indices\":");
append(json_meshes, index_accr);
writeMeshGeometry(json_meshes, views, json_accessors, accr_offset, prim, qp, qt, settings);
}

if (prim.material)
Expand Down Expand Up @@ -1177,6 +1186,7 @@ Settings defaults()
settings.rot_bits = 12;
settings.scl_bits = 16;
settings.anim_freq = 30;
settings.mesh_dedup = true;
settings.simplify_ratio = 1.f;
settings.simplify_error = 1e-2f;
settings.texture_scale = 1.f;
Expand Down Expand Up @@ -1314,6 +1324,11 @@ int main(int argc, char** argv)
{
settings.keep_attributes = true;
}
else if (strcmp(arg, "-mdd") == 0)
{
fprintf(stderr, "Warning: option -mdd disables mesh deduplication and is only provided as a safety measure; it will be removed in the future\n");
settings.mesh_dedup = false;
}
else if (strcmp(arg, "-mm") == 0)
{
settings.mesh_merge = true;
Expand Down
9 changes: 8 additions & 1 deletion gltf/gltfpack.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,9 @@ struct Mesh
std::vector<Stream> streams;
std::vector<unsigned int> indices;

bool geometry_duplicate;
uint64_t geometry_hash[2];

size_t targets;
std::vector<float> target_weights;
std::vector<const char*> target_names;
Expand Down Expand Up @@ -131,6 +134,7 @@ struct Settings
bool keep_extras;
bool keep_attributes;

bool mesh_dedup;
bool mesh_merge;
bool mesh_instancing;

Expand Down Expand Up @@ -314,6 +318,8 @@ bool compareMeshTargets(const Mesh& lhs, const Mesh& rhs);
bool compareMeshVariants(const Mesh& lhs, const Mesh& rhs);
bool compareMeshNodes(const Mesh& lhs, const Mesh& rhs);

void hashMesh(Mesh& mesh);
void dedupMeshes(std::vector<Mesh>& meshes);
void mergeMeshInstances(Mesh& mesh);
void mergeMeshes(std::vector<Mesh>& meshes, const Settings& settings);
void filterEmptyMeshes(std::vector<Mesh>& meshes);
Expand Down Expand Up @@ -376,7 +382,8 @@ void writeSampler(std::string& json, const cgltf_sampler& sampler);
void writeImage(std::string& json, std::vector<BufferView>& views, const cgltf_image& image, const ImageInfo& info, const std::string* encoded, size_t index, const char* input_path, const char* output_path, const Settings& settings);
void writeTexture(std::string& json, const cgltf_texture& texture, const ImageInfo* info, cgltf_data* data, const Settings& settings);
void writeMeshAttributes(std::string& json, std::vector<BufferView>& views, std::string& json_accessors, size_t& accr_offset, const Mesh& mesh, int target, const QuantizationPosition& qp, const QuantizationTexture& qt, const Settings& settings);
size_t writeMeshIndices(std::vector<BufferView>& views, std::string& json_accessors, size_t& accr_offset, const Mesh& mesh, const Settings& settings);
size_t writeMeshIndices(std::vector<BufferView>& views, std::string& json_accessors, size_t& accr_offset, const std::vector<unsigned int>& indices, cgltf_primitive_type type, const Settings& settings);
void writeMeshGeometry(std::string& json, std::vector<BufferView>& views, std::string& json_accessors, size_t& accr_offset, const Mesh& mesh, const QuantizationPosition& qp, const QuantizationTexture& qt, const Settings& settings);
size_t writeJointBindMatrices(std::vector<BufferView>& views, std::string& json_accessors, size_t& accr_offset, const cgltf_skin& skin, const QuantizationPosition& qp, const Settings& settings);
size_t writeInstances(std::vector<BufferView>& views, std::string& json_accessors, size_t& accr_offset, const std::vector<Transform>& transforms, const QuantizationPosition& qp, const Settings& settings);
void writeMeshNode(std::string& json, size_t mesh_offset, cgltf_node* node, cgltf_skin* skin, cgltf_data* data, const QuantizationPosition* qp);
Expand Down
137 changes: 137 additions & 0 deletions gltf/mesh.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -262,6 +262,143 @@ static void mergeMeshes(Mesh& target, const Mesh& mesh)
target.indices[index_offset + i] = unsigned(vertex_offset + mesh.indices[i]);
}

static void hashUpdate(uint64_t hash[2], const void* data, size_t size)
{
#define ROTL64(x, r) (((x) << (r)) | ((x) >> (64 - (r))))

// MurMurHash3 128-bit
const uint64_t c1 = 0x87c37b91114253d5ull;
const uint64_t c2 = 0x4cf5ad432745937full;

uint64_t h1 = hash[0], h2 = hash[1];

size_t offset = 0;

// body
for (; offset + 16 <= size; offset += 16)
{
uint64_t k1, k2;
memcpy(&k1, static_cast<const char*>(data) + offset + 0, 8);
memcpy(&k2, static_cast<const char*>(data) + offset + 8, 8);

k1 *= c1, k1 = ROTL64(k1, 31), k1 *= c2;
h1 ^= k1, h1 = ROTL64(h1, 27), h1 += h2;
h1 = h1 * 5 + 0x52dce729;
k2 *= c2, k2 = ROTL64(k2, 33), k2 *= c1;
h2 ^= k2, h2 = ROTL64(h2, 31), h2 += h1;
h2 = h2 * 5 + 0x38495ab5;
}

// tail
if (offset < size)
{
uint64_t tail[2] = {};
memcpy(tail, static_cast<const char*>(data) + offset, size - offset);

uint64_t k1 = tail[0], k2 = tail[1];

k1 *= c1, k1 = ROTL64(k1, 31), k1 *= c2;
h1 ^= k1;
k2 *= c2, k2 = ROTL64(k2, 33), k2 *= c1;
h2 ^= k2;
}

h1 ^= size;
h2 ^= size;

hash[0] = h1;
hash[1] = h2;

#undef ROTL64
}

void hashMesh(Mesh& mesh)
{
mesh.geometry_hash[0] = mesh.geometry_hash[1] = 41;

for (size_t i = 0; i < mesh.streams.size(); ++i)
{
const Stream& stream = mesh.streams[i];

int meta[3] = {stream.type, stream.index, stream.target};
hashUpdate(mesh.geometry_hash, meta, sizeof(meta));

if (stream.custom_name)
hashUpdate(mesh.geometry_hash, stream.custom_name, strlen(stream.custom_name));

hashUpdate(mesh.geometry_hash, stream.data.data(), stream.data.size() * sizeof(Attr));
}

if (!mesh.indices.empty())
hashUpdate(mesh.geometry_hash, mesh.indices.data(), mesh.indices.size() * sizeof(unsigned int));

int meta[4] = {int(mesh.streams.size()), mesh.streams.empty() ? 0 : int(mesh.streams[0].data.size()), int(mesh.indices.size()), mesh.type};
hashUpdate(mesh.geometry_hash, meta, sizeof(meta));
}

static bool canDedupMesh(const Mesh& mesh)
{
// empty mesh
if (mesh.streams.empty())
return false;

// world-space mesh
if (mesh.nodes.empty() && mesh.instances.empty())
return false;

// to simplify dedup we ignore complex target setups for now
if (!mesh.target_weights.empty() || !mesh.target_names.empty() || !mesh.variants.empty())
return false;

return true;
}

void dedupMeshes(std::vector<Mesh>& meshes)
{
for (size_t i = 0; i < meshes.size(); ++i)
hashMesh(meshes[i]);

for (size_t i = 0; i < meshes.size(); ++i)
{
Mesh& target = meshes[i];

if (!canDedupMesh(target))
continue;

for (size_t j = i + 1; j < meshes.size(); ++j)
{
Mesh& mesh = meshes[j];

if (mesh.geometry_hash[0] != target.geometry_hash[0] || mesh.geometry_hash[1] != target.geometry_hash[1])
continue;

if (!canDedupMesh(mesh))
continue;

if (mesh.scene != target.scene || mesh.material != target.material || mesh.skin != target.skin)
{
// mark both meshes as having duplicate geometry; we don't use this in dedupMeshes but it's useful later in the pipeline
target.geometry_duplicate = true;
mesh.geometry_duplicate = true;
continue;
}

// basic sanity test; these should be included in geometry hash
assert(mesh.streams.size() == target.streams.size());
assert(mesh.streams[0].data.size() == target.streams[0].data.size());
assert(mesh.indices.size() == target.indices.size());

target.nodes.insert(target.nodes.end(), mesh.nodes.begin(), mesh.nodes.end());
target.instances.insert(target.instances.end(), mesh.instances.begin(), mesh.instances.end());

mesh.streams.clear();
mesh.indices.clear();
mesh.nodes.clear();
mesh.instances.clear();
}
}
}

void mergeMeshInstances(Mesh& mesh)
{
if (mesh.nodes.empty())
Expand Down
40 changes: 36 additions & 4 deletions gltf/write.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1046,24 +1046,56 @@ void writeMeshAttributes(std::string& json, std::vector<BufferView>& views, std:
}
}

size_t writeMeshIndices(std::vector<BufferView>& views, std::string& json_accessors, size_t& accr_offset, const Mesh& mesh, const Settings& settings)
size_t writeMeshIndices(std::vector<BufferView>& views, std::string& json_accessors, size_t& accr_offset, const std::vector<unsigned int>& indices, cgltf_primitive_type type, const Settings& settings)
{
std::string scratch;
StreamFormat format = writeIndexStream(scratch, mesh.indices);
BufferView::Compression compression = settings.compress ? (mesh.type == cgltf_primitive_type_triangles ? BufferView::Compression_Index : BufferView::Compression_IndexSequence) : BufferView::Compression_None;
StreamFormat format = writeIndexStream(scratch, indices);
BufferView::Compression compression = settings.compress ? (type == cgltf_primitive_type_triangles ? BufferView::Compression_Index : BufferView::Compression_IndexSequence) : BufferView::Compression_None;

size_t view = getBufferView(views, BufferView::Kind_Index, StreamFormat::Filter_None, compression, format.stride);
size_t offset = views[view].data.size();
views[view].data += scratch;

comma(json_accessors);
writeAccessor(json_accessors, view, offset, format.type, format.component_type, format.normalized, mesh.indices.size());
writeAccessor(json_accessors, view, offset, format.type, format.component_type, format.normalized, indices.size());

size_t index_accr = accr_offset++;

return index_accr;
}

void writeMeshGeometry(std::string& json, std::vector<BufferView>& views, std::string& json_accessors, size_t& accr_offset, const Mesh& mesh, const QuantizationPosition& qp, const QuantizationTexture& qt, const Settings& settings)
{
append(json, "{\"attributes\":{");
writeMeshAttributes(json, views, json_accessors, accr_offset, mesh, 0, qp, qt, settings);
append(json, "}");
if (mesh.type != cgltf_primitive_type_triangles)
{
append(json, ",\"mode\":");
append(json, size_t(mesh.type - cgltf_primitive_type_points));
}
if (mesh.targets)
{
append(json, ",\"targets\":[");
for (size_t j = 0; j < mesh.targets; ++j)
{
comma(json);
append(json, "{");
writeMeshAttributes(json, views, json_accessors, accr_offset, mesh, int(1 + j), qp, qt, settings);
append(json, "}");
}
append(json, "]");
}

if (!mesh.indices.empty())
{
size_t index_accr = writeMeshIndices(views, json_accessors, accr_offset, mesh.indices, mesh.type, settings);

append(json, ",\"indices\":");
append(json, index_accr);
}
}

static size_t writeAnimationTime(std::vector<BufferView>& views, std::string& json_accessors, size_t& accr_offset, float mint, int frames, float period, const Settings& settings)
{
std::vector<float> time(frames);
Expand Down