Skip to content

Commit

Permalink
bug fixes and more tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@facontidavide
facontidavide committed Nov 28, 2023
1 parent 8ec5b6e commit 1671c9c
Show file tree
Hide file tree
Showing 7 changed files with 343 additions and 67 deletions.
4 changes: 2 additions & 2 deletions data_tamer/include/data_tamer/channel.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -340,8 +340,8 @@ inline RegistrationID LogChannel::registerValue(const std::string& prefix,
}
else
{
auto def = _type_registry.getSerializer<T>();
updateTypeRegistry<T>();
auto def = _type_registry.getSerializer<T>();
return registerValueImpl(prefix, ValuePtr(vect), def);
}
}
Expand All @@ -356,8 +356,8 @@ inline RegistrationID LogChannel::registerValue(const std::string& prefix,
}
else
{
auto def = _type_registry.getSerializer<T>();
updateTypeRegistry<T>();
auto def = _type_registry.getSerializer<T>();
return registerValueImpl(prefix, ValuePtr(vect, def), def);
}
}
Expand Down
4 changes: 2 additions & 2 deletions data_tamer/include/data_tamer/contrib/SerializeMe.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -264,8 +264,8 @@ struct container_info_
typedef _Tp value_type;
};

template <typename>
struct container_info : container_info_<void, false, -1>
template <typename T>
struct container_info : container_info_<T, false, -1>
{
};

Expand Down
36 changes: 29 additions & 7 deletions data_tamer/include/data_tamer/custom_types.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,10 @@ class CustomSerializer
}
// size in bytes of the serialized object.
// Needed to pre-allocate memory in the buffer
virtual uint32_t serializedSize(const void* src_instance) const = 0;
virtual size_t serializedSize(const void* src_instance) const = 0;

virtual bool isFixedSize() const = 0;

// serialize an object into a buffer. Return the size in bytes of the serialized data
virtual void serialize(const void* src_instance, SerializeMe::SpanBytes&) const = 0;
};
Expand Down Expand Up @@ -76,7 +79,9 @@ class CustomSerializerT : public CustomSerializer

const char* typeName() const override;

uint32_t serializedSize(const void* src_instance) const override;
size_t serializedSize(const void* src_instance) const override;

bool isFixedSize() const override;

void serialize(const void* src_instance,
SerializeMe::SpanBytes& dst_buffer) const override;
Expand Down Expand Up @@ -109,6 +114,8 @@ struct TypeDefinition<std::array<T, N>>
}
};

// Recursive function to compute if a type has fixed size (at compile time).
// Used mainly by the CustomSerializerT constructor.
template <typename T>
inline void GetFixedSize(bool& is_fixed_size, size_t &fixed_size)
{
Expand Down Expand Up @@ -150,6 +157,9 @@ inline void GetFixedSize(bool& is_fixed_size, size_t &fixed_size)
template<typename T> inline
CustomSerializerT<T>::CustomSerializerT(const std::string &type_name) : _name(type_name)
{
static_assert(!SerializeMe::container_info<T>().is_container,
"Don't pass containers a template type");

bool is_fixed_size = true;
GetFixedSize<T>(is_fixed_size, _fixed_size);
if(!is_fixed_size)
Expand All @@ -165,14 +175,20 @@ const char *CustomSerializerT<T>::typeName() const
}

template<typename T> inline
uint32_t CustomSerializerT<T>::serializedSize(const void *src_instance) const
size_t CustomSerializerT<T>::serializedSize(const void *src_instance) const
{
if(_fixed_size != 0)
{
return uint32_t(_fixed_size);
return _fixed_size;
}
const auto* obj = static_cast<const T*>(src_instance);
return uint32_t(SerializeMe::BufferSize(*obj));
return SerializeMe::BufferSize(*obj);
}

template<typename T> inline
bool CustomSerializerT<T>::isFixedSize() const
{
return _fixed_size > 0;
}

template<typename T> inline
Expand All @@ -186,8 +202,9 @@ template <typename T>
inline CustomSerializer::Ptr TypesRegistry::getSerializer()
{
static_assert(!IsNumericType<T>(), "You don't need to create a serializer for a "
"numerical type. "
"There might be an error in your code.");
"numerical type.");
static_assert(!SerializeMe::container_info<T>().is_container,
"Don't pass containers as template type");

std::scoped_lock lk(_mutex);
const auto type_name = SerializeMe::TypeDefinition<T>().typeName();
Expand All @@ -206,6 +223,11 @@ template <typename T>
inline CustomSerializer::Ptr TypesRegistry::addType(const std::string& type_name,
bool skip_if_present)
{
static_assert(!IsNumericType<T>(), "You don't need to create a serializer for a "
"numerical type.");
static_assert(!SerializeMe::container_info<T>().is_container,
"Don't pass containers as template type");

std::scoped_lock lk(_mutex);
if (skip_if_present && _types.count(type_name) != 0)
{
Expand Down
56 changes: 46 additions & 10 deletions data_tamer/include/data_tamer/values.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,10 +81,12 @@ inline ValuePtr::ValuePtr(const T* pointer, CustomSerializer::Ptr type_info) :
{
if (type_info)
{
serialize_impl_ = [type_info, pointer](SerializeMe::SpanBytes& buffer) -> void {
serialize_impl_ = [type_info, pointer](SerializeMe::SpanBytes& buffer) -> void
{
type_info->serialize(pointer, buffer);
};
get_size_impl_ = [type_info, pointer]() {
get_size_impl_ = [type_info, pointer]() -> size_t
{
return type_info->serializedSize(pointer);
};
}
Expand All @@ -97,17 +99,37 @@ inline ValuePtr::ValuePtr(const Container<T, TArgs...>* vect,
{
if (type_info)
{
serialize_impl_ = [type_info, vect](SerializeMe::SpanBytes& buffer) -> void {
type_info->serialize(vect, buffer);
serialize_impl_ = [type_info, vect](SerializeMe::SpanBytes& buffer) -> void
{
SerializeMe::SerializeIntoBuffer(buffer, uint32_t(vect->size()));
for(const auto& value: (*vect))
{
type_info->serialize(&value, buffer);
}
};
get_size_impl_ = [type_info, vect]() -> size_t {
if(vect->empty())
{
return 0;
}
if(type_info->isFixedSize())
{
return vect->size() * type_info->serializedSize(&vect->front());
}
size_t tot_size = 0;
for(const auto& value: (*vect))
{
tot_size += type_info->serializedSize(&value);
}
return tot_size;
};
get_size_impl_ = [type_info, vect]() { return type_info->serializedSize(vect); };
}
else
{
serialize_impl_ = [vect](SerializeMe::SpanBytes& buffer) -> void {
SerializeMe::SerializeIntoBuffer(buffer, *vect);
};
get_size_impl_ = [vect]() { return SerializeMe::BufferSize(*vect); };
get_size_impl_ = [vect]() -> size_t { return SerializeMe::BufferSize(*vect); };
}
}

Expand All @@ -116,16 +138,30 @@ inline ValuePtr::ValuePtr(const std::array<T, N>* array,
CustomSerializer::Ptr type_info) :
v_ptr_(array),
type_(GetBasicType<T>()),
memory_size_(N * sizeof(T)),
is_vector_(true),
array_size_(N)
{
if (type_info)
{
serialize_impl_ = [type_info, array](SerializeMe::SpanBytes& buffer) -> void {
type_info->serialize(array, buffer);
serialize_impl_ = [type_info, array](SerializeMe::SpanBytes& buffer) -> void
{
for(const auto& value: (*array))
{
type_info->serialize(&value, buffer);
}
};
get_size_impl_ = [type_info, array]() {
size_t tot_size = 0;
if(type_info->isFixedSize())
{
return N * type_info->serializedSize(&array->front());
}
for(const auto& value: (*array))
{
tot_size += type_info->serializedSize(&value);
}
return tot_size;
};
get_size_impl_ = [type_info, array]() { return type_info->serializedSize(array); };
}
else
{
Expand Down
95 changes: 56 additions & 39 deletions data_tamer/include/data_tamer_parser/data_tamer_parser.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -326,12 +326,15 @@ inline Schema BuilSchemaFromText(const std::string& txt)
}
}

if (field.type == BasicType::OTHER)
auto offset = str_type->find_first_of(" [");
if (field.type != BasicType::OTHER)
{
field.type_name = *str_type;
field.type_name = kNamesNew[static_cast<size_t>(field.type)];
}
else {
field.type_name = str_type->substr(0, offset);
}

auto offset = str_type->find_first_of(" [");
if (offset != std::string::npos && str_type->at(offset) == '[')
{
field.is_vector = true;
Expand Down Expand Up @@ -362,6 +365,55 @@ inline Schema BuilSchemaFromText(const std::string& txt)
return schema;
}

template <typename NumberCallback>
bool ParseSnapshotRecursive(const TypeField& field,
const std::map<std::string, FieldsVector>& types_list,
BufferSpan& buffer,
const NumberCallback& callback_number,
const std::string& prefix)
{

[[maybe_unused]] uint32_t vect_size = field.array_size;
if (field.is_vector && field.array_size == 0)
{
// dynamic vector
vect_size = Deserialize<uint32_t>(buffer);
}

auto new_prefix = (prefix.empty()) ? field.field_name : (prefix + "/" + field.field_name);

auto doParse = [&](const std::string& var_name)
{
if(field.type != BasicType::OTHER)
{
const auto var = DeserializeToVarNumber(field.type, buffer);
callback_number(var_name, var);
}
else {
const FieldsVector& fields = types_list.at(field.type_name);
for(const auto& sub_field: fields)
{
ParseSnapshotRecursive(sub_field, types_list, buffer, callback_number, var_name);
}
}
};

if(!field.is_vector)
{
doParse(new_prefix);
}
else
{
for(uint32_t a=0; a < vect_size; a++)
{
const auto& name = new_prefix + "[" + std::to_string(a) + "]";
doParse(name);
}
}
return true;
}


template <typename NumberCallback, typename CustomCallback>
inline bool ParseSnapshot(const Schema& schema, SnapshotView snapshot,
const NumberCallback& callback_number,
Expand All @@ -378,42 +430,7 @@ inline bool ParseSnapshot(const Schema& schema, SnapshotView snapshot,
const auto& field = schema.fields[i];
if (GetBit(snapshot.active_mask, i))
{
if (!field.is_vector)
{
// regular field, not vector/array
if (field.type == BasicType::OTHER)
{
callback_custom(field.field_name, snapshot.payload, field.type_name);
}
else
{
const auto var = DeserializeToVarNumber(field.type, buffer);
callback_number(field.field_name, var);
}
}
else
{
uint32_t vect_size = field.array_size;
if (field.array_size == 0) // dynamic vector
{
vect_size = Deserialize<uint32_t>(buffer);
}
for (size_t a = 0; a < vect_size; a++)
{
thread_local std::string tmp_name;
tmp_name = field.field_name + "[" + std::to_string(a) + "]";

if (field.type == BasicType::OTHER)
{
callback_custom(tmp_name, snapshot.payload, field.type_name);
}
else
{
const auto var = DeserializeToVarNumber(field.type, buffer);
callback_number(tmp_name, var);
}
}
}
ParseSnapshotRecursive(field, schema.custom_types, buffer, callback_number, "");
}
}
return true;
Expand Down
55 changes: 55 additions & 0 deletions data_tamer/tests/dt_tests.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -338,3 +338,58 @@ TEST(DataTamer, CustomType)
ASSERT_LT(posA, posD);
ASSERT_LT(posA, posE);
}


TEST(DataTamer, CustomType2)
{
ChannelsRegistry registry;
auto channel = registry.getChannel("chan");
auto sink = std::make_shared<DummySink>();
channel->addDataSink(sink);

std::array<Point3D, 2> points;
std::vector<Quaternion> quats(3);

channel->registerValue("points", &points);
channel->registerValue("quats", &quats);

channel->takeSnapshot();
std::this_thread::sleep_for(std::chrono::milliseconds(10));

auto expected_size = 2 * sizeof(Point3D) +
3 * sizeof(Quaternion) + sizeof(uint32_t);

//-------------------------------------------------
const auto schema = channel->getSchema();
std::ostringstream ss;
ss << schema;
const std::string schema_txt = ss.str();

std::cout << schema_txt << std::endl;

ASSERT_EQ(sink->latest_snapshot.payload.size(), expected_size);
ASSERT_EQ(schema.custom_types.count("Point3D"), 1);
ASSERT_EQ(schema.custom_types.count("Quaternion"), 1);

const auto posA = schema_txt.find("Point3D[2] points\n"
"Quaternion[] quats\n");

const auto posB = schema_txt.find("===============\n"
"MSG: Point3D\n"
"float64 x\n"
"float64 y\n"
"float64 z\n");

const auto posC = schema_txt.find("===============\n"
"MSG: Quaternion\n"
"float64 w\n"
"float64 x\n"
"float64 y\n"
"float64 z\n");

ASSERT_TRUE(std::string::npos != posA);
ASSERT_TRUE(std::string::npos != posB);
ASSERT_TRUE(std::string::npos != posC);
ASSERT_LT(posA, posB);
ASSERT_LT(posA, posC);
}
Loading

0 comments on commit 1671c9c

Please sign in to comment.