diff --git a/src/crate-format.hh b/src/crate-format.hh index 0d3bb185f..b4c69cc5b 100644 --- a/src/crate-format.hh +++ b/src/crate-format.hh @@ -35,6 +35,106 @@ constexpr char kTokenVector[] = "TokenVector"; constexpr char kStringVector[] = "StringVector"; constexpr char kPathVector[] = "PathVector"; +// crate data type +// id must be identitical to /pxr/usd/usd/crateDataType.h +enum class CrateDataTypeId { + CRATE_DATA_TYPE_INVALID = 0, + + CRATE_DATA_TYPE_BOOL = 1, + CRATE_DATA_TYPE_UCHAR = 2, + CRATE_DATA_TYPE_INT = 3, + CRATE_DATA_TYPE_UINT = 4, + CRATE_DATA_TYPE_INT64 = 5, + CRATE_DATA_TYPE_UINT64 = 6, + + CRATE_DATA_TYPE_HALF = 7, + CRATE_DATA_TYPE_FLOAT = 8, + CRATE_DATA_TYPE_DOUBLE = 9, + + CRATE_DATA_TYPE_STRING = 10, + CRATE_DATA_TYPE_TOKEN = 11, + CRATE_DATA_TYPE_ASSET_PATH = 12, + + CRATE_DATA_TYPE_MATRIX2D = 13, + CRATE_DATA_TYPE_MATRIX3D = 14, + CRATE_DATA_TYPE_MATRIX4D = 15, + + CRATE_DATA_TYPE_QUATD = 16, + CRATE_DATA_TYPE_QUATF = 17, + CRATE_DATA_TYPE_QUATH = 18, + + CRATE_DATA_TYPE_VEC2D = 19, + CRATE_DATA_TYPE_VEC2F = 20, + CRATE_DATA_TYPE_VEC2H = 21, + CRATE_DATA_TYPE_VEC2I = 22, + + CRATE_DATA_TYPE_VEC3D = 23, + CRATE_DATA_TYPE_VEC3F = 24, + CRATE_DATA_TYPE_VEC3H = 25, + CRATE_DATA_TYPE_VEC3I = 26, + + CRATE_DATA_TYPE_VEC4D = 27, + CRATE_DATA_TYPE_VEC4F = 28, + CRATE_DATA_TYPE_VEC4H = 29, + CRATE_DATA_TYPE_VEC4I = 30, + + CRATE_DATA_TYPE_DICTIONARY = 31, + CRATE_DATA_TYPE_TOKEN_LIST_OP = 32, + CRATE_DATA_TYPE_STRING_LIST_OP = 33, + CRATE_DATA_TYPE_PATH_LIST_OP = 34, + CRATE_DATA_TYPE_REFERENCE_LIST_OP = 35, + CRATE_DATA_TYPE_INT_LIST_OP = 36, + CRATE_DATA_TYPE_INT64_LIST_OP = 37, + CRATE_DATA_TYPE_UINT_LIST_OP = 38, + CRATE_DATA_TYPE_UINT64_LIST_OP = 39, + + CRATE_DATA_TYPE_PATH_VECTOR = 40, + CRATE_DATA_TYPE_TOKEN_VECTOR = 41, + + CRATE_DATA_TYPE_SPECIFIER = 42, + CRATE_DATA_TYPE_PERMISSION = 43, + CRATE_DATA_TYPE_VARIABILITY = 44, + + CRATE_DATA_TYPE_VARIANT_SELECTION_MAP = 45, + CRATE_DATA_TYPE_TIME_SAMPLES = 46, + CRATE_DATA_TYPE_PAYLOAD = 47, + CRATE_DATA_TYPE_DOUBLE_VECTOR = 48, + CRATE_DATA_TYPE_LAYER_OFFSET_VECTOR = 49, + CRATE_DATA_TYPE_STRING_VECTOR = 50, + CRATE_DATA_TYPE_VALUE_BLOCK = 51, + CRATE_DATA_TYPE_VALUE = 52, // Contains ValueRep + CRATE_DATA_TYPE_UNREGISTERED_VALUE = 53, // String or Dict + CRATE_DATA_TYPE_UNREGISTERED_VALUE_LIST_OP = 54, + CRATE_DATA_TYPE_PAYLOAD_LIST_OP = 55, + CRATE_DATA_TYPE_TIME_CODE = 56, + + NumDataTypes // terminator +}; + +class CrateDataType +{ + public: + CrateDataType() = default; + + CrateDataType(const char *s, CrateDataTypeId did, bool a) + : name(s), dtype_id(did), supports_array(a) { + } + + CrateDataType(const CrateDataType &rhs) = default; + CrateDataType &operator=(const CrateDataType&rhs) = default; + + const char *name{nullptr}; // name of CrateDatatType. Constant symbol. TODO: Use string_view. + CrateDataTypeId dtype_id{CrateDataTypeId::CRATE_DATA_TYPE_INVALID}; + bool supports_array{false}; +}; + +std::string GetCrateDataTypeRepr(CrateDataType dty); // for debug cout + +nonstd::expected GetCrateDataType(int32_t type_id); +std::string GetCrateDataTypeName(int32_t type_id); +std::string GetCrateDataTypeName(CrateDataTypeId type_id); + + // -- from USD ---------------------------------------------------------------- @@ -131,7 +231,7 @@ struct ValueRep { std::string GetStringRepr() const { std::stringstream ss; - ss << "ty: " << static_cast(GetType()) << ", isArray: " << IsArray() + ss << "ty: " << static_cast(GetType()) << "(" << GetCrateDataTypeName(GetType()) << "), isArray: " << IsArray() << ", isInlined: " << IsInlined() << ", isCompressed: " << IsCompressed() << ", payload: " << GetPayload(); @@ -277,105 +377,6 @@ struct TableOfContents { std::vector
sections; }; -// crate data type -// id must be identitical to /pxr/usd/usd/crateDataType.h -enum class CrateDataTypeId { - CRATE_DATA_TYPE_INVALID = 0, - - CRATE_DATA_TYPE_BOOL = 1, - CRATE_DATA_TYPE_UCHAR = 2, - CRATE_DATA_TYPE_INT = 3, - CRATE_DATA_TYPE_UINT = 4, - CRATE_DATA_TYPE_INT64 = 5, - CRATE_DATA_TYPE_UINT64 = 6, - - CRATE_DATA_TYPE_HALF = 7, - CRATE_DATA_TYPE_FLOAT = 8, - CRATE_DATA_TYPE_DOUBLE = 9, - - CRATE_DATA_TYPE_STRING = 10, - CRATE_DATA_TYPE_TOKEN = 11, - CRATE_DATA_TYPE_ASSET_PATH = 12, - - CRATE_DATA_TYPE_MATRIX2D = 13, - CRATE_DATA_TYPE_MATRIX3D = 14, - CRATE_DATA_TYPE_MATRIX4D = 15, - - CRATE_DATA_TYPE_QUATD = 16, - CRATE_DATA_TYPE_QUATF = 17, - CRATE_DATA_TYPE_QUATH = 18, - - CRATE_DATA_TYPE_VEC2D = 19, - CRATE_DATA_TYPE_VEC2F = 20, - CRATE_DATA_TYPE_VEC2H = 21, - CRATE_DATA_TYPE_VEC2I = 22, - - CRATE_DATA_TYPE_VEC3D = 23, - CRATE_DATA_TYPE_VEC3F = 24, - CRATE_DATA_TYPE_VEC3H = 25, - CRATE_DATA_TYPE_VEC3I = 26, - - CRATE_DATA_TYPE_VEC4D = 27, - CRATE_DATA_TYPE_VEC4F = 28, - CRATE_DATA_TYPE_VEC4H = 29, - CRATE_DATA_TYPE_VEC4I = 30, - - CRATE_DATA_TYPE_DICTIONARY = 31, - CRATE_DATA_TYPE_TOKEN_LIST_OP = 32, - CRATE_DATA_TYPE_STRING_LIST_OP = 33, - CRATE_DATA_TYPE_PATH_LIST_OP = 34, - CRATE_DATA_TYPE_REFERENCE_LIST_OP = 35, - CRATE_DATA_TYPE_INT_LIST_OP = 36, - CRATE_DATA_TYPE_INT64_LIST_OP = 37, - CRATE_DATA_TYPE_UINT_LIST_OP = 38, - CRATE_DATA_TYPE_UINT64_LIST_OP = 39, - - CRATE_DATA_TYPE_PATH_VECTOR = 40, - CRATE_DATA_TYPE_TOKEN_VECTOR = 41, - - CRATE_DATA_TYPE_SPECIFIER = 42, - CRATE_DATA_TYPE_PERMISSION = 43, - CRATE_DATA_TYPE_VARIABILITY = 44, - - CRATE_DATA_TYPE_VARIANT_SELECTION_MAP = 45, - CRATE_DATA_TYPE_TIME_SAMPLES = 46, - CRATE_DATA_TYPE_PAYLOAD = 47, - CRATE_DATA_TYPE_DOUBLE_VECTOR = 48, - CRATE_DATA_TYPE_LAYER_OFFSET_VECTOR = 49, - CRATE_DATA_TYPE_STRING_VECTOR = 50, - CRATE_DATA_TYPE_VALUE_BLOCK = 51, - CRATE_DATA_TYPE_VALUE = 52, // Contains ValueRep - CRATE_DATA_TYPE_UNREGISTERED_VALUE = 53, // String or Dict - CRATE_DATA_TYPE_UNREGISTERED_VALUE_LIST_OP = 54, - CRATE_DATA_TYPE_PAYLOAD_LIST_OP = 55, - CRATE_DATA_TYPE_TIME_CODE = 56, - - NumDataTypes // terminator -}; - -class CrateDataType -{ - public: - CrateDataType() = default; - - CrateDataType(const char *s, CrateDataTypeId did, bool a) - : name(s), dtype_id(did), supports_array(a) { - } - - CrateDataType(const CrateDataType &rhs) = default; - CrateDataType &operator=(const CrateDataType&rhs) = default; - - const char *name{nullptr}; // name of CrateDatatType. Constant symbol. TODO: Use string_view. - CrateDataTypeId dtype_id{CrateDataTypeId::CRATE_DATA_TYPE_INVALID}; - bool supports_array{false}; -}; - -std::string GetCrateDataTypeRepr(CrateDataType dty); // for debug cout - -nonstd::expected GetCrateDataType(int32_t type_id); -std::string GetCrateDataTypeName(int32_t type_id); -std::string GetCrateDataTypeName(CrateDataTypeId type_id); - // TODO: Use PrimVar? class CrateValue { public: diff --git a/src/crate-reader.cc b/src/crate-reader.cc index 07b6f3e73..6a6a19c82 100644 --- a/src/crate-reader.cc +++ b/src/crate-reader.cc @@ -73,16 +73,8 @@ namespace crate { } while(0) -#if 0 -// -// Impl(TODO) -// -class CrateReader::Impl -{ - public: - Impl(); -}; -#endif + +#define VERSION_LESS_THAN_0_8_0(__version) ((_version[0] == 0) && (_version[1] < 7)) // // -- @@ -304,7 +296,7 @@ template bool CrateReader::ReadCompressedInts(Int *out, size_t num_ints) { if (num_ints > _config.maxInts) { - PUSH_ERROR_AND_RETURN_TAG(kTag, "# of ints too large."); + PUSH_ERROR_AND_RETURN_TAG(kTag, fmt::format("# of ints {} too large. maxInts is set to {}", num_ints, _config.maxInts)); } using Compressor = @@ -356,7 +348,12 @@ bool CrateReader::ReadIntArray(bool is_compressed, std::vector *d) { size_t length{0}; // uncompressed array elements. // < ver 0.7.0 use 32bit - if ((_version[0] == 0) && ((_version[1] < 7))) { + if (VERSION_LESS_THAN_0_8_0(_version)) { + uint32_t shapesize; // not used + if (!_sr->read4(&shapesize)) { + PUSH_ERROR("Failed to read the number of array elements."); + return false; + } uint32_t n; if (!_sr->read4(&n)) { PUSH_ERROR_AND_RETURN_TAG(kTag, "Failed to read the number of array elements."); @@ -416,11 +413,15 @@ bool CrateReader::ReadHalfArray(bool is_compressed, std::vector *d) { size_t length; // < ver 0.7.0 use 32bit - if ((_version[0] == 0) && ((_version[1] < 7))) { + if (VERSION_LESS_THAN_0_8_0(_version)) { + uint32_t shapesize; // not used + if (!_sr->read4(&shapesize)) { + PUSH_ERROR("Failed to read the number of array elements."); + return false; + } uint32_t n; if (!_sr->read4(&n)) { - _err += "Failed to read the number of array elements.\n"; - return false; + PUSH_ERROR_AND_RETURN_TAG(kTag, "Failed to read the number of array elements."); } length = size_t(n); } else { @@ -434,7 +435,7 @@ bool CrateReader::ReadHalfArray(bool is_compressed, } if (length > _config.maxArrayElements) { - PUSH_ERROR_AND_RETURN_TAG(kTag, "Too many array elements."); + PUSH_ERROR_AND_RETURN_TAG(kTag, fmt::format("Too many array elements {}.", length)); } CHECK_MEMORY_USAGE(length * sizeof(uint16_t)); @@ -530,7 +531,12 @@ bool CrateReader::ReadFloatArray(bool is_compressed, std::vector *d) { size_t length; // < ver 0.7.0 use 32bit - if ((_version[0] == 0) && ((_version[1] < 7))) { + if (VERSION_LESS_THAN_0_8_0(_version)) { + uint32_t shapesize; // not used + if (!_sr->read4(&shapesize)) { + PUSH_ERROR("Failed to read the number of array elements."); + return false; + } uint32_t n; if (!_sr->read4(&n)) { _err += "Failed to read the number of array elements.\n"; @@ -639,7 +645,12 @@ bool CrateReader::ReadDoubleArray(bool is_compressed, std::vector *d) { size_t length; // < ver 0.7.0 use 32bit - if ((_version[0] == 0) && ((_version[1] < 7))) { + if (VERSION_LESS_THAN_0_8_0(_version)) { + uint32_t shapesize; // not used + if (!_sr->read4(&shapesize)) { + PUSH_ERROR("Failed to read the number of array elements."); + return false; + } uint32_t n; if (!_sr->read4(&n)) { _err += "Failed to read the number of array elements.\n"; @@ -885,9 +896,9 @@ bool CrateReader::ReadTimeSamples(value::TimeSamples *d) { bool CrateReader::ReadStringArray(std::vector *d) { // array data is not compressed auto ReadFn = [this](std::vector &result) -> bool { - uint64_t n; + uint64_t n{0}; if (!_sr->read8(&n)) { - PUSH_ERROR("Failed to read # of elements."); + PUSH_ERROR_AND_RETURN_TAG(kTag, "Failed to read the number of array elements."); return false; } @@ -1033,9 +1044,9 @@ bool CrateReader::ReadLayerOffset(LayerOffset *d) { bool CrateReader::ReadLayerOffsetArray(std::vector *d) { // array data is not compressed - uint64_t n; + uint64_t n{0}; if (!_sr->read8(&n)) { - PUSH_ERROR("Failed to read # of elements."); + PUSH_ERROR_AND_RETURN_TAG(kTag, "Failed to read the number of array elements."); return false; } @@ -1060,9 +1071,9 @@ bool CrateReader::ReadLayerOffsetArray(std::vector *d) { bool CrateReader::ReadPathArray(std::vector *d) { // array data is not compressed auto ReadFn = [this](std::vector &result) -> bool { - uint64_t n; + uint64_t n{0}; if (!_sr->read8(&n)) { - _err += "Failed to read # of elements in ListOp.\n"; + PUSH_ERROR_AND_RETURN_TAG(kTag, "Failed to read the number of array elements."); return false; } @@ -1233,12 +1244,13 @@ bool CrateReader::ReadStringListOp(ListOp *d) { // array data is not compressed auto ReadFn = [this](std::vector &result) -> bool { - uint64_t n; + uint64_t n{0}; if (!_sr->read8(&n)) { - _err += "Failed to read # of elements in ListOp.\n"; + PUSH_ERROR_AND_RETURN_TAG(kTag, "Failed to read the number of array elements."); return false; } + if (n > _config.maxArrayElements) { _err += "Too many ListOp elements.\n"; return false; @@ -1346,9 +1358,9 @@ bool CrateReader::ReadPathListOp(ListOp *d) { // array data is not compressed auto ReadFn = [this](std::vector &result) -> bool { - uint64_t n; + uint64_t n{0}; if (!_sr->read8(&n)) { - PUSH_ERROR("Failed to read # of elements in ListOp."); + PUSH_ERROR_AND_RETURN_TAG(kTag, "Failed to read the number of array elements."); return false; } @@ -1452,8 +1464,9 @@ bool CrateReader::ReadArray(std::vector *d) { return false; } - uint64_t n; + uint64_t n{0}; if (!_sr->read8(&n)) { + PUSH_ERROR_AND_RETURN_TAG(kTag, "Failed to read the number of array elements."); return false; } @@ -1481,9 +1494,23 @@ bool CrateReader::ReadArray(std::vector *d) { return false; } - uint64_t n; - if (!_sr->read8(&n)) { - return false; + uint64_t n{0}; + if (VERSION_LESS_THAN_0_8_0(_version)) { + uint32_t shapesize; // not used + if (!_sr->read4(&shapesize)) { + PUSH_ERROR("Failed to read the number of array elements."); + return false; + } + uint32_t _n; + if (!_sr->read4(&_n)) { + PUSH_ERROR_AND_RETURN_TAG(kTag, "Failed to read the number of array elements."); + } + n = _n; + } else { + if (!_sr->read8(&n)) { + PUSH_ERROR_AND_RETURN_TAG(kTag, "Failed to read the number of array elements."); + return false; + } } if (n > _config.maxArrayElements) { @@ -1512,9 +1539,23 @@ bool CrateReader::ReadArray(std::vector *d) { return false; } - uint64_t n; - if (!_sr->read8(&n)) { - return false; + uint64_t n{0}; + if (VERSION_LESS_THAN_0_8_0(_version)) { + uint32_t shapesize; // not used + if (!_sr->read4(&shapesize)) { + PUSH_ERROR("Failed to read the number of array elements."); + return false; + } + uint32_t _n; + if (!_sr->read4(&_n)) { + PUSH_ERROR_AND_RETURN_TAG(kTag, "Failed to read the number of array elements."); + } + n = _n; + } else { + if (!_sr->read8(&n)) { + PUSH_ERROR_AND_RETURN_TAG(kTag, "Failed to read the number of array elements."); + return false; + } } if (n > _config.maxArrayElements) { @@ -2286,10 +2327,23 @@ bool CrateReader::UnpackValueRep(const crate::ValueRep &rep, return true; } - uint64_t n; - if (!_sr->read8(&n)) { - PUSH_ERROR("Failed to read the number of array elements."); - return false; + uint64_t n{0}; + if (VERSION_LESS_THAN_0_8_0(_version)) { + uint32_t shapesize; // not used + if (!_sr->read4(&shapesize)) { + PUSH_ERROR("Failed to read the number of array elements."); + return false; + } + uint32_t _n; + if (!_sr->read4(&_n)) { + PUSH_ERROR_AND_RETURN_TAG(kTag, "Failed to read the number of array elements."); + } + n = _n; + } else { + if (!_sr->read8(&n)) { + PUSH_ERROR_AND_RETURN_TAG(kTag, "Failed to read the number of array elements."); + return false; + } } if (n > _config.maxAssetPathElements) { @@ -2362,7 +2416,7 @@ bool CrateReader::UnpackValueRep(const crate::ValueRep &rep, } if (n > _config.maxArrayElements) { - PUSH_ERROR_AND_RETURN_TAG(kTag, fmt::format("Token array too large. TinyUSDZ limites it up to {}", _config.maxArrayElements)); + PUSH_ERROR_AND_RETURN_TAG(kTag, fmt::format("Token array too large. TinyUSDZ limits it up to {}", _config.maxArrayElements)); } CHECK_MEMORY_USAGE(n * sizeof(crate::Index)); @@ -2669,10 +2723,24 @@ bool CrateReader::UnpackValueRep(const crate::ValueRep &rep, return true; } - uint64_t n; - if (!_sr->read8(&n)) { - PUSH_ERROR("Failed to read the number of array elements."); - return false; + uint64_t n{0}; + if (VERSION_LESS_THAN_0_8_0(_version)) { + uint32_t shapesize; // not used + if (!_sr->read4(&shapesize)) { + PUSH_ERROR("Failed to read the number of array elements."); + return false; + } + uint32_t _n; + if (!_sr->read4(&_n)) { + PUSH_ERROR("Failed to read the number of array elements."); + return false; + } + n = _n; + } else { + if (!_sr->read8(&n)) { + PUSH_ERROR("Failed to read the number of array elements."); + return false; + } } if (n == 0) { @@ -2681,7 +2749,7 @@ bool CrateReader::UnpackValueRep(const crate::ValueRep &rep, } if (n > _config.maxArrayElements) { - PUSH_ERROR_AND_RETURN_TAG(kTag, "Array size too large."); + PUSH_ERROR_AND_RETURN_TAG(kTag, fmt::format("Array size {} too large. maxArrayElements is set to {}. Please increase maxArrayElements in CrateReaderConfig.", n, _config.maxArrayElements)); } CHECK_MEMORY_USAGE(n * sizeof(value::matrix2d)); @@ -2726,10 +2794,24 @@ bool CrateReader::UnpackValueRep(const crate::ValueRep &rep, return true; } - uint64_t n; - if (!_sr->read8(&n)) { - PUSH_ERROR("Failed to read the number of array elements."); - return false; + uint64_t n{0}; + if (VERSION_LESS_THAN_0_8_0(_version)) { + uint32_t shapesize; // not used + if (!_sr->read4(&shapesize)) { + PUSH_ERROR("Failed to read the number of array elements."); + return false; + } + uint32_t _n; + if (!_sr->read4(&_n)) { + PUSH_ERROR("Failed to read the number of array elements."); + return false; + } + n = _n; + } else { + if (!_sr->read8(&n)) { + PUSH_ERROR("Failed to read the number of array elements."); + return false; + } } if (n == 0) { @@ -2738,7 +2820,7 @@ bool CrateReader::UnpackValueRep(const crate::ValueRep &rep, } if (n > _config.maxArrayElements) { - PUSH_ERROR_AND_RETURN_TAG(kTag, "Array size too large."); + PUSH_ERROR_AND_RETURN_TAG(kTag, fmt::format("Array size {} too large. maxArrayElements is set to {}. Please increase maxArrayElements in CrateReaderConfig.", n, _config.maxArrayElements)); } CHECK_MEMORY_USAGE(n * sizeof(value::matrix3d)); @@ -2782,11 +2864,25 @@ bool CrateReader::UnpackValueRep(const crate::ValueRep &rep, value->Set(v); return true; } - uint64_t n; - if (!_sr->read8(&n)) { - PUSH_ERROR("Failed to read the number of array elements."); - return false; + uint64_t n{0}; + if (VERSION_LESS_THAN_0_8_0(_version)) { + uint32_t shapesize; // not used + if (!_sr->read4(&shapesize)) { + PUSH_ERROR("Failed to read the number of array elements."); + return false; + } + uint32_t _n; + if (!_sr->read4(&_n)) { + PUSH_ERROR("Failed to read the number of array elements."); + return false; + } + n = _n; + } else { + if (!_sr->read8(&n)) { + PUSH_ERROR("Failed to read the number of array elements."); + return false; + } } if (n == 0) { @@ -2795,7 +2891,7 @@ bool CrateReader::UnpackValueRep(const crate::ValueRep &rep, } if (n > _config.maxArrayElements) { - PUSH_ERROR_AND_RETURN_TAG(kTag, "Array size too large."); + PUSH_ERROR_AND_RETURN_TAG(kTag, fmt::format("Array size {} too large. maxArrayElements is set to {}. Please increase maxArrayElements in CrateReaderConfig.", n, _config.maxArrayElements)); } CHECK_MEMORY_USAGE(n * sizeof(value::matrix4d)); @@ -2836,10 +2932,24 @@ bool CrateReader::UnpackValueRep(const crate::ValueRep &rep, value->Set(v); return true; } - uint64_t n; - if (!_sr->read8(&n)) { - PUSH_ERROR("Failed to read the number of array elements."); - return false; + uint64_t n{0}; + if (VERSION_LESS_THAN_0_8_0(_version)) { + uint32_t shapesize; // not used + if (!_sr->read4(&shapesize)) { + PUSH_ERROR("Failed to read the number of array elements."); + return false; + } + uint32_t _n; + if (!_sr->read4(&_n)) { + PUSH_ERROR("Failed to read the number of array elements."); + return false; + } + n = _n; + } else { + if (!_sr->read8(&n)) { + PUSH_ERROR("Failed to read the number of array elements."); + return false; + } } if (n == 0) { @@ -2848,7 +2958,7 @@ bool CrateReader::UnpackValueRep(const crate::ValueRep &rep, } if (n > _config.maxArrayElements) { - PUSH_ERROR_AND_RETURN_TAG(kTag, "Array size too large."); + PUSH_ERROR_AND_RETURN_TAG(kTag, fmt::format("Array size {} too large. maxArrayElements is set to {}. Please increase maxArrayElements in CrateReaderConfig.", n, _config.maxArrayElements)); } CHECK_MEMORY_USAGE(n * sizeof(value::quatd)); @@ -2890,10 +3000,24 @@ bool CrateReader::UnpackValueRep(const crate::ValueRep &rep, value->Set(v); return true; } - uint64_t n; - if (!_sr->read8(&n)) { - PUSH_ERROR("Failed to read the number of array elements."); - return false; + uint64_t n{0}; + if (VERSION_LESS_THAN_0_8_0(_version)) { + uint32_t shapesize; // not used + if (!_sr->read4(&shapesize)) { + PUSH_ERROR("Failed to read the number of array elements."); + return false; + } + uint32_t _n; + if (!_sr->read4(&_n)) { + PUSH_ERROR("Failed to read the number of array elements."); + return false; + } + n = _n; + } else { + if (!_sr->read8(&n)) { + PUSH_ERROR("Failed to read the number of array elements."); + return false; + } } if (n == 0) { @@ -2902,7 +3026,7 @@ bool CrateReader::UnpackValueRep(const crate::ValueRep &rep, } if (n > _config.maxArrayElements) { - PUSH_ERROR_AND_RETURN_TAG(kTag, "Array size too large."); + PUSH_ERROR_AND_RETURN_TAG(kTag, fmt::format("Array size {} too large. maxArrayElements is set to {}. Please increase maxArrayElements in CrateReaderConfig.", n, _config.maxArrayElements)); } CHECK_MEMORY_USAGE(n * sizeof(value::quatf)); @@ -2944,10 +3068,25 @@ bool CrateReader::UnpackValueRep(const crate::ValueRep &rep, value->Set(v); return true; } - uint64_t n; - if (!_sr->read8(&n)) { - PUSH_ERROR("Failed to read the number of array elements."); - return false; + + uint64_t n{0}; + if (VERSION_LESS_THAN_0_8_0(_version)) { + uint32_t shapesize; // not used + if (!_sr->read4(&shapesize)) { + PUSH_ERROR("Failed to read the number of array elements."); + return false; + } + uint32_t _n; + if (!_sr->read4(&_n)) { + PUSH_ERROR("Failed to read the number of array elements."); + return false; + } + n = _n; + } else { + if (!_sr->read8(&n)) { + PUSH_ERROR("Failed to read the number of array elements."); + return false; + } } if (n == 0) { @@ -2956,7 +3095,7 @@ bool CrateReader::UnpackValueRep(const crate::ValueRep &rep, } if (n > _config.maxArrayElements) { - PUSH_ERROR_AND_RETURN_TAG(kTag, "Array size too large."); + PUSH_ERROR_AND_RETURN_TAG(kTag, fmt::format("Array size {} too large. maxArrayElements is set to {}. Please increase maxArrayElements in CrateReaderConfig.", n, _config.maxArrayElements)); } CHECK_MEMORY_USAGE(n * sizeof(value::quath)); @@ -3001,10 +3140,24 @@ bool CrateReader::UnpackValueRep(const crate::ValueRep &rep, return true; } - uint64_t n; - if (!_sr->read8(&n)) { - PUSH_ERROR("Failed to read the number of array elements."); - return false; + uint64_t n{0}; + if (VERSION_LESS_THAN_0_8_0(_version)) { + uint32_t shapesize; // not used + if (!_sr->read4(&shapesize)) { + PUSH_ERROR("Failed to read the number of array elements."); + return false; + } + uint32_t _n; + if (!_sr->read4(&_n)) { + PUSH_ERROR("Failed to read the number of array elements."); + return false; + } + n = _n; + } else { + if (!_sr->read8(&n)) { + PUSH_ERROR("Failed to read the number of array elements."); + return false; + } } if (n == 0) { @@ -3013,7 +3166,7 @@ bool CrateReader::UnpackValueRep(const crate::ValueRep &rep, } if (n > _config.maxArrayElements) { - PUSH_ERROR_AND_RETURN_TAG(kTag, "Array size too large."); + PUSH_ERROR_AND_RETURN_TAG(kTag, fmt::format("Array size {} too large. maxArrayElements is set to {}. Please increase maxArrayElements in CrateReaderConfig.", n, _config.maxArrayElements)); } CHECK_MEMORY_USAGE(n * sizeof(value::double2)); @@ -3056,14 +3209,28 @@ bool CrateReader::UnpackValueRep(const crate::ValueRep &rep, return true; } - uint64_t n; - if (!_sr->read8(&n)) { - PUSH_ERROR("Failed to read the number of array elements."); - return false; + uint64_t n{0}; + if (VERSION_LESS_THAN_0_8_0(_version)) { + uint32_t shapesize; // not used + if (!_sr->read4(&shapesize)) { + PUSH_ERROR("Failed to read the number of array elements."); + return false; + } + uint32_t _n; + if (!_sr->read4(&_n)) { + PUSH_ERROR("Failed to read the number of array elements."); + return false; + } + n = _n; + } else { + if (!_sr->read8(&n)) { + PUSH_ERROR("Failed to read the number of array elements."); + return false; + } } if (n > _config.maxArrayElements) { - PUSH_ERROR_AND_RETURN_TAG(kTag, "Array size too large."); + PUSH_ERROR_AND_RETURN_TAG(kTag, fmt::format("Array size {} too large. maxArrayElements is set to {}. Please increase maxArrayElements in CrateReaderConfig.", n, _config.maxArrayElements)); } if (n == 0) { @@ -3110,14 +3277,28 @@ bool CrateReader::UnpackValueRep(const crate::ValueRep &rep, value->Set(v); return true; } - uint64_t n; - if (!_sr->read8(&n)) { - PUSH_ERROR("Failed to read the number of array elements."); - return false; + uint64_t n{0}; + if (VERSION_LESS_THAN_0_8_0(_version)) { + uint32_t shapesize; // not used + if (!_sr->read4(&shapesize)) { + PUSH_ERROR("Failed to read the number of array elements."); + return false; + } + uint32_t _n; + if (!_sr->read4(&_n)) { + PUSH_ERROR("Failed to read the number of array elements."); + return false; + } + n = _n; + } else { + if (!_sr->read8(&n)) { + PUSH_ERROR("Failed to read the number of array elements."); + return false; + } } if (n > _config.maxArrayElements) { - PUSH_ERROR_AND_RETURN_TAG(kTag, "Array size too large."); + PUSH_ERROR_AND_RETURN_TAG(kTag, fmt::format("Array size {} too large. maxArrayElements is set to {}. Please increase maxArrayElements in CrateReaderConfig.", n, _config.maxArrayElements)); } CHECK_MEMORY_USAGE(n * sizeof(value::half2)); @@ -3158,14 +3339,29 @@ bool CrateReader::UnpackValueRep(const crate::ValueRep &rep, value->Set(v); return true; } - uint64_t n; - if (!_sr->read8(&n)) { - PUSH_ERROR("Failed to read the number of array elements."); - return false; + + uint64_t n{0}; + if (VERSION_LESS_THAN_0_8_0(_version)) { + uint32_t shapesize; // not used + if (!_sr->read4(&shapesize)) { + PUSH_ERROR("Failed to read the number of array elements."); + return false; + } + uint32_t _n; + if (!_sr->read4(&_n)) { + PUSH_ERROR("Failed to read the number of array elements."); + return false; + } + n = _n; + } else { + if (!_sr->read8(&n)) { + PUSH_ERROR("Failed to read the number of array elements."); + return false; + } } if (n > _config.maxArrayElements) { - PUSH_ERROR_AND_RETURN_TAG(kTag, "Array size too large."); + PUSH_ERROR_AND_RETURN_TAG(kTag, fmt::format("Array size {} too large. maxArrayElements is set to {}. Please increase maxArrayElements in CrateReaderConfig.", n, _config.maxArrayElements)); } CHECK_MEMORY_USAGE(n * sizeof(value::int2)); @@ -3206,14 +3402,29 @@ bool CrateReader::UnpackValueRep(const crate::ValueRep &rep, value->Set(v); return true; } - uint64_t n; - if (!_sr->read8(&n)) { - PUSH_ERROR("Failed to read the number of array elements."); - return false; + + uint64_t n{0}; + if (VERSION_LESS_THAN_0_8_0(_version)) { + uint32_t shapesize; // not used + if (!_sr->read4(&shapesize)) { + PUSH_ERROR("Failed to read the number of array elements."); + return false; + } + uint32_t _n; + if (!_sr->read4(&_n)) { + PUSH_ERROR("Failed to read the number of array elements."); + return false; + } + n = _n; + } else { + if (!_sr->read8(&n)) { + PUSH_ERROR("Failed to read the number of array elements."); + return false; + } } if (n > _config.maxArrayElements) { - PUSH_ERROR_AND_RETURN_TAG(kTag, "Array size too large."); + PUSH_ERROR_AND_RETURN_TAG(kTag, fmt::format("Array size {} too large. maxArrayElements is set to {}. Please increase maxArrayElements in CrateReaderConfig.", n, _config.maxArrayElements)); } CHECK_MEMORY_USAGE(n * sizeof(value::double3)); @@ -3255,14 +3466,28 @@ bool CrateReader::UnpackValueRep(const crate::ValueRep &rep, return true; } - uint64_t n; - if (!_sr->read8(&n)) { - PUSH_ERROR("Failed to read the number of array elements."); - return false; + uint64_t n{0}; + if (VERSION_LESS_THAN_0_8_0(_version)) { + uint32_t shapesize; // not used + if (!_sr->read4(&shapesize)) { + PUSH_ERROR("Failed to read the number of array elements."); + return false; + } + uint32_t _n; + if (!_sr->read4(&_n)) { + PUSH_ERROR("Failed to read the number of array elements."); + return false; + } + n = _n; + } else { + if (!_sr->read8(&n)) { + PUSH_ERROR("Failed to read the number of array elements."); + return false; + } } if (n > _config.maxArrayElements) { - PUSH_ERROR_AND_RETURN_TAG(kTag, "Array size too large."); + PUSH_ERROR_AND_RETURN_TAG(kTag, fmt::format("Array size {} too large. maxArrayElements is set to {}. Please increase maxArrayElements in CrateReaderConfig.", n, _config.maxArrayElements)); } CHECK_MEMORY_USAGE(n * sizeof(value::float3)); @@ -3304,14 +3529,28 @@ bool CrateReader::UnpackValueRep(const crate::ValueRep &rep, value->Set(v); return true; } - uint64_t n; - if (!_sr->read8(&n)) { - PUSH_ERROR("Failed to read the number of array elements."); - return false; + uint64_t n{0}; + if (VERSION_LESS_THAN_0_8_0(_version)) { + uint32_t shapesize; // not used + if (!_sr->read4(&shapesize)) { + PUSH_ERROR("Failed to read the number of array elements."); + return false; + } + uint32_t _n; + if (!_sr->read4(&_n)) { + PUSH_ERROR("Failed to read the number of array elements."); + return false; + } + n = _n; + } else { + if (!_sr->read8(&n)) { + PUSH_ERROR("Failed to read the number of array elements."); + return false; + } } if (n > _config.maxArrayElements) { - PUSH_ERROR_AND_RETURN_TAG(kTag, "Array size too large."); + PUSH_ERROR_AND_RETURN_TAG(kTag, fmt::format("Array size {} too large. maxArrayElements is set to {}. Please increase maxArrayElements in CrateReaderConfig.", n, _config.maxArrayElements)); } CHECK_MEMORY_USAGE(n * sizeof(value::half3)); @@ -3352,14 +3591,28 @@ bool CrateReader::UnpackValueRep(const crate::ValueRep &rep, value->Set(v); return true; } - uint64_t n; - if (!_sr->read8(&n)) { - PUSH_ERROR("Failed to read the number of array elements."); - return false; + uint64_t n{0}; + if (VERSION_LESS_THAN_0_8_0(_version)) { + uint32_t shapesize; // not used + if (!_sr->read4(&shapesize)) { + PUSH_ERROR("Failed to read the number of array elements."); + return false; + } + uint32_t _n; + if (!_sr->read4(&_n)) { + PUSH_ERROR("Failed to read the number of array elements."); + return false; + } + n = _n; + } else { + if (!_sr->read8(&n)) { + PUSH_ERROR("Failed to read the number of array elements."); + return false; + } } if (n > _config.maxArrayElements) { - PUSH_ERROR_AND_RETURN_TAG(kTag, "Array size too large."); + PUSH_ERROR_AND_RETURN_TAG(kTag, fmt::format("Array size {} too large. maxArrayElements is set to {}. Please increase maxArrayElements in CrateReaderConfig.", n, _config.maxArrayElements)); } CHECK_MEMORY_USAGE(n * sizeof(value::int3)); @@ -3400,14 +3653,29 @@ bool CrateReader::UnpackValueRep(const crate::ValueRep &rep, value->Set(v); return true; } - uint64_t n; - if (!_sr->read8(&n)) { - PUSH_ERROR("Failed to read the number of array elements."); - return false; + + uint64_t n{0}; + if (VERSION_LESS_THAN_0_8_0(_version)) { + uint32_t shapesize; // not used + if (!_sr->read4(&shapesize)) { + PUSH_ERROR("Failed to read the number of array elements."); + return false; + } + uint32_t _n; + if (!_sr->read4(&_n)) { + PUSH_ERROR("Failed to read the number of array elements."); + return false; + } + n = _n; + } else { + if (!_sr->read8(&n)) { + PUSH_ERROR("Failed to read the number of array elements."); + return false; + } } if (n > _config.maxArrayElements) { - PUSH_ERROR_AND_RETURN_TAG(kTag, "Array size too large."); + PUSH_ERROR_AND_RETURN_TAG(kTag, fmt::format("Array size {} too large. maxArrayElements is set to {}. Please increase maxArrayElements in CrateReaderConfig.", n, _config.maxArrayElements)); } CHECK_MEMORY_USAGE(n * sizeof(value::double4)); @@ -3448,14 +3716,28 @@ bool CrateReader::UnpackValueRep(const crate::ValueRep &rep, value->Set(v); return true; } - uint64_t n; - if (!_sr->read8(&n)) { - PUSH_ERROR("Failed to read the number of array elements."); - return false; + uint64_t n{0}; + if (VERSION_LESS_THAN_0_8_0(_version)) { + uint32_t shapesize; // not used + if (!_sr->read4(&shapesize)) { + PUSH_ERROR("Failed to read the number of array elements."); + return false; + } + uint32_t _n; + if (!_sr->read4(&_n)) { + PUSH_ERROR("Failed to read the number of array elements."); + return false; + } + n = _n; + } else { + if (!_sr->read8(&n)) { + PUSH_ERROR("Failed to read the number of array elements."); + return false; + } } if (n > _config.maxArrayElements) { - PUSH_ERROR_AND_RETURN_TAG(kTag, "Array size too large."); + PUSH_ERROR_AND_RETURN_TAG(kTag, fmt::format("Array size {} too large. maxArrayElements is set to {}. Please increase maxArrayElements in CrateReaderConfig.", n, _config.maxArrayElements)); } CHECK_MEMORY_USAGE(n * sizeof(value::float4)); @@ -3496,14 +3778,28 @@ bool CrateReader::UnpackValueRep(const crate::ValueRep &rep, value->Set(v); return true; } - uint64_t n; - if (!_sr->read8(&n)) { - PUSH_ERROR("Failed to read the number of array elements."); - return false; + uint64_t n{0}; + if (VERSION_LESS_THAN_0_8_0(_version)) { + uint32_t shapesize; // not used + if (!_sr->read4(&shapesize)) { + PUSH_ERROR("Failed to read the number of array elements."); + return false; + } + uint32_t _n; + if (!_sr->read4(&_n)) { + PUSH_ERROR("Failed to read the number of array elements."); + return false; + } + n = _n; + } else { + if (!_sr->read8(&n)) { + PUSH_ERROR("Failed to read the number of array elements."); + return false; + } } if (n > _config.maxArrayElements) { - PUSH_ERROR_AND_RETURN_TAG(kTag, "Array size too large."); + PUSH_ERROR_AND_RETURN_TAG(kTag, fmt::format("Array size {} too large. maxArrayElements is set to {}. Please increase maxArrayElements in CrateReaderConfig.", n, _config.maxArrayElements)); } CHECK_MEMORY_USAGE(n * sizeof(value::half4)); @@ -3544,14 +3840,28 @@ bool CrateReader::UnpackValueRep(const crate::ValueRep &rep, value->Set(v); return true; } - uint64_t n; - if (!_sr->read8(&n)) { - PUSH_ERROR("Failed to read the number of array elements."); - return false; + uint64_t n{0}; + if (VERSION_LESS_THAN_0_8_0(_version)) { + uint32_t shapesize; // not used + if (!_sr->read4(&shapesize)) { + PUSH_ERROR("Failed to read the number of array elements."); + return false; + } + uint32_t _n; + if (!_sr->read4(&_n)) { + PUSH_ERROR("Failed to read the number of array elements."); + return false; + } + n = _n; + } else { + if (!_sr->read8(&n)) { + PUSH_ERROR("Failed to read the number of array elements."); + return false; + } } if (n > _config.maxArrayElements) { - PUSH_ERROR_AND_RETURN_TAG(kTag, "Array size too large."); + PUSH_ERROR_AND_RETURN_TAG(kTag, fmt::format("Array size {} too large. maxArrayElements is set to {}. Please increase maxArrayElements in CrateReaderConfig.", n, _config.maxArrayElements)); } CHECK_MEMORY_USAGE(n * sizeof(value::int4)); @@ -3657,14 +3967,14 @@ bool CrateReader::UnpackValueRep(const crate::ValueRep &rep, case crate::CrateDataTypeId::CRATE_DATA_TYPE_TOKEN_VECTOR: { COMPRESS_UNSUPPORTED_CHECK(dty) // std::vector - uint64_t n; + uint64_t n{0}; if (!_sr->read8(&n)) { - PUSH_ERROR("Failed to read TokenVector value."); + PUSH_ERROR("Failed to read the number of array elements."); return false; } if (n > _config.maxArrayElements) { - PUSH_ERROR_AND_RETURN_TAG(kTag, "Array size too large."); + PUSH_ERROR_AND_RETURN_TAG(kTag, fmt::format("Array size {} too large. maxArrayElements is set to {}. Please increase maxArrayElements in CrateReaderConfig.", n, _config.maxArrayElements)); } CHECK_MEMORY_USAGE(n * sizeof(crate::Index)); @@ -5327,7 +5637,7 @@ bool CrateReader::ReadTOC() { return false; } if (num_sections >= _config.maxTOCSections) { - PUSH_ERROR_AND_RETURN_TAG(kTag, "# of sections are too large."); + PUSH_ERROR_AND_RETURN_TAG(kTag, fmt::format("# of sections {} are too large. maxTOCSections is set to {}", num_sections, _config.maxTOCSections)); } DCOUT("toc sections = " << num_sections); diff --git a/src/crate-reader.hh b/src/crate-reader.hh index e4a79cf98..f3247ec25 100644 --- a/src/crate-reader.hh +++ b/src/crate-reader.hh @@ -43,7 +43,7 @@ struct CrateReaderConfig { size_t maxValueRecursion = 16; // Prevent recursive Value unpack(e.g. Value encodes itself) // Generic int[] data - size_t maxInts = 1024 * 1024 * 4; + size_t maxInts = 1024 * 1024 * 1024; // Total memory budget for uncompressed USD data(vertices, `tokens`, ...)` in // [bytes]. diff --git a/src/usdc-reader.cc b/src/usdc-reader.cc index 6742eea1f..f3f5221ef 100644 --- a/src/usdc-reader.cc +++ b/src/usdc-reader.cc @@ -2318,7 +2318,7 @@ bool USDCReader::Impl::ReconstructPrimNode(int parent, int current, int level, std::string pTyName; if (!typeName) { - PUSH_WARN("Treat this node as Model(`typeName` field is missing)."); + //PUSH_WARN("Treat this node as Model(`typeName` field is missing)."); pTyName = "Model"; } else { pTyName = typeName.value(); @@ -2478,7 +2478,7 @@ bool USDCReader::Impl::ReconstructPrimNode(int parent, int current, int level, std::string pTyName; // TinyUSDZ' prim typename if (!typeName) { - PUSH_WARN("Treat this node as Model(where `typeName` is missing)."); + //PUSH_WARN("Treat this node as Model(where `typeName` is missing)."); pTyName = "Model"; } else { pTyName = typeName.value(); @@ -2770,7 +2770,7 @@ bool USDCReader::Impl::ReconstructPrimSpecNode(int parent, int current, int leve std::string pTyName; if (!typeName) { - PUSH_WARN("Treat this node as Model(`typeName` field is missing)."); + //PUSH_WARN("Treat this node as Model(`typeName` field is missing)."); pTyName = "Model"; } else { pTyName = typeName.value(); @@ -2945,7 +2945,7 @@ bool USDCReader::Impl::ReconstructPrimSpecNode(int parent, int current, int leve std::string pTyName; // TinyUSDZ' prim typename if (!typeName) { - PUSH_WARN("Treat this node as Model(where `typeName` is missing)."); + //PUSH_WARN("Treat this node as Model(where `typeName` is missing)."); pTyName = "Model"; } else { pTyName = typeName.value(); diff --git a/tests/parse_ascii/runner.py b/tests/parse_ascii/runner.py deleted file mode 100644 index 95eac8b93..000000000 --- a/tests/parse_ascii/runner.py +++ /dev/null @@ -1,53 +0,0 @@ -import argparse -import os -import subprocess -import glob - -def run(config): - cmd = "./usda_parser" - - failure_cases = [] - success_cases = [] - - print("Find USDA files under: ", config["path"]) - fs = glob.glob(os.path.join(config["path"], "*.usda")) - for f in fs: - print(f) - ret = subprocess.run([cmd, f]) - if ret.returncode != 0: - if f.startswith('fail-case'): - # fail expected - success_cases.append(f) - else: - failure_cases.append(f) - else: - if f.startswith('fail-case'): - failure_cases.append(f) - else: - success_cases.append(f) - - print(ret.returncode) - - print("Failure caess: =====================") - for f in failure_cases: - print(f) - -def main(): - - # Assume script is run from /build, e.g.: - # - # python ../tests/parse_ascii/runner.py - - conf = {} - parser = argparse.ArgumentParser(description='USDA parser tester.') - parser.add_argument('usda_path', type=str, nargs='?', default="../tests/usda", - help='Path to USDA source tree') - - args = parser.parse_args() - - conf["path"] = args.usda_path - - run(conf) - -if __name__ == '__main__': - main() diff --git a/tests/parse_usd/runner.py b/tests/parse_usd/runner.py new file mode 100644 index 000000000..9e9a56c78 --- /dev/null +++ b/tests/parse_usd/runner.py @@ -0,0 +1,51 @@ +import argparse +import os +import subprocess +import glob + +def run(config): + cmd = "./test_tinyusdz" + + failure_cases = [] + success_cases = [] + + glob_pattern = ["**/*.usd", "**/*.usda", "**/*.usdc", "**/*.usdz"] + + print("Find USD files under: ", config["path"]) + fs = [] + for pat in glob_pattern: + fs.extend(glob.glob(os.path.join(config["path"], pat), recursive=True)) + + for f in fs: + print(f) + ret = subprocess.run([cmd, f]) + if ret.returncode != 0: + failure_cases.append(f) + else: + success_cases.append(f) + + print(ret.returncode) + + print("Failure cases: =====================") + for f in failure_cases: + print(f) + +def main(): + + # Assume script is run from /build, e.g.: + # + # python ../tests/parse_usd/runner.py + + conf = {} + parser = argparse.ArgumentParser(description='USD parse tester.') + parser.add_argument('usd_path', type=str, nargs='?', default="../tests/usda", + help='Path to USD source tree') + + args = parser.parse_args() + + conf["path"] = args.usd_path + + run(conf) + +if __name__ == '__main__': + main()