Bugfix: Empty type

cpp/arcticdb/CMakeLists.txt

@@ -157,7 +157,7 @@ ENDIF()
 
 set(CMAKE_INSTALL_RPATH "\\$ORIGIN/lib")
 
-if (NOT WIN32)
+if (NOT MSVC)
  if(NOT ${ARCTICDB_USING_CONDA})
  # Compilers on conda-forge are relatively strict.
  # For now, we do not want to treat warnings as errors for those builds.
@@ -166,6 +166,13 @@ if (NOT WIN32)
  add_compile_options("-ggdb")
  add_compile_options("-fvisibility=hidden")
  add_compile_options("-fno-omit-frame-pointer")
+else()
+ add_compile_options(
+ /WX
+ /wd4250 # Suppress inherits via dominance warning
+ /we4189 # Unused local variable is an error
+ /we4100 # Unused function parameter is an error
+ )
 endif ()
 
 ## Core library without python bindings ##
@@ -200,6 +207,7 @@ set(arcticdb_srcs
  entity/key.hpp
  entity/metrics.hpp
  entity/data_error.hpp
+ entity/descriptors.hpp
  entity/native_tensor.hpp
  entity/performance_tracing.hpp
  entity/protobuf_mappings.hpp
@@ -208,6 +216,7 @@ set(arcticdb_srcs
  entity/serialized_key.hpp
  entity/type_conversion.hpp
  entity/types.hpp
+ entity/type_utils.hpp
  entity/types-inl.hpp
  entity/variant_key.hpp
  entity/versioned_item.hpp
@@ -233,7 +242,6 @@ set(arcticdb_srcs
  pipeline/read_pipeline.hpp
  pipeline/slicing.hpp
  pipeline/string_pool_utils.hpp
- pipeline/string_pool_utils.hpp
  pipeline/value.hpp
  pipeline/value_set.hpp
  pipeline/write_frame.hpp
@@ -376,7 +384,9 @@ set(arcticdb_srcs
  entity/metrics.cpp
  entity/performance_tracing.cpp
  entity/types.cpp
+ entity/type_utils.cpp
  log/log.cpp
+ pipeline/column_mapping.cpp
  pipeline/column_stats.cpp
  pipeline/frame_slice.cpp
  pipeline/frame_utils.cpp

cpp/arcticdb/codec/codec-inl.hpp

@@ -18,10 +18,6 @@
 
 #include <arcticdb/util/bitset.hpp>
 
-#include <arcticdb/util/buffer.hpp>
-
-#include <google/protobuf/text_format.h>
-
 #include <type_traits>
 
 namespace arcticdb {
@@ -135,6 +131,7 @@ std::size_t decode_ndarray(
  }
 
  if(field.sparse_map_bytes()) {
+ util::check(!is_empty_type(type_desc_tag.data_type()), "Empty typed columns should not have sparse map");
  util::check_magic<util::BitMagicStart>(data_in);
  const auto bitmap_size = field.sparse_map_bytes() - util::combined_bit_magic_delimiters_size();
  bv = util::deserialize_bytes_to_bitmap(data_in, bitmap_size);

cpp/arcticdb/codec/codec.cpp

@@ -310,11 +310,12 @@ void decode_string_pool( const arcticdb::proto::encoding::SegmentHeader& hdr,
  }
 }
 
-void decode_v2(const Segment& segment,
- arcticdb::proto::encoding::SegmentHeader& hdr,
- SegmentInMemory& res,
- const StreamDescriptor& desc)
- {
+void decode_v2(
+ const Segment& segment,
+ arcticdb::proto::encoding::SegmentHeader& hdr,
+ SegmentInMemory& res,
+ const StreamDescriptor& desc
+) {
  ARCTICDB_SAMPLE(DecodeSegment, 0)
  const auto [begin, end] = get_segment_begin_end(segment, hdr);
  auto encoded_fields_ptr = end;
@@ -364,11 +365,12 @@ void decode_v2(const Segment& segment,
  res.set_compacted(segment.header().compacted());
  }}
 
-void decode_v1(const Segment& segment,
- const arcticdb::proto::encoding::SegmentHeader& hdr,
- SegmentInMemory& res,
- StreamDescriptor::Proto& desc)
-{
+void decode_v1(
+ const Segment& segment,
+ const arcticdb::proto::encoding::SegmentHeader& hdr,
+ SegmentInMemory& res,
+ StreamDescriptor::Proto& desc
+) {
  ARCTICDB_SAMPLE(DecodeSegment, 0)
  const uint8_t* data = segment.buffer().data();
  util::check(data != nullptr, "Got null data ptr from segment");
@@ -384,17 +386,29 @@ void decode_v1(const Segment& segment,
  const auto seg_row_count = fields_size ? ssize_t(hdr.fields(0).ndarray().items_count()) : 0LL;
  res.init_column_map();
 
- for (std::size_t i = 0; i < static_cast<size_t>(fields_size); ++i) {
- const auto& field = hdr.fields(static_cast<int>(i));
- const auto& field_name = desc.fields(static_cast<int>(i)).name();
- util::check(data!=end, "Reached end of input block with {} fields to decode", fields_size-i);
- if(auto col_index = res.column_index(field_name)) {
+ for (int i = 0; i < fields_size; ++i) {
+ const auto& field = hdr.fields(i);
+ const auto& field_name = desc.fields(i).name();
+ if (auto col_index = res.column_index(field_name)) {
  auto& col = res.column(static_cast<position_t>(*col_index));
- data += decode_field(res.field(*col_index).type(), field, data, col, col.opt_sparse_map(), to_encoding_version(hdr.encoding_version()));
- } else
+ util::check(
+ data != end || is_empty_type(col.type().data_type()),
+ "Reached end of input block with {} fields to decode",
+ fields_size - i
+ );
+ data += decode_field(
+ res.field(*col_index).type(),
+ field,
+ data,
+ col,
+ col.opt_sparse_map(),
+ to_encoding_version(hdr.encoding_version())
+ );
+ } else {
+ util::check(data != end, "Reached end of input block with {} fields to decode", fields_size - i);
  data += encoding_sizes::field_compressed_size(field);
-
- ARCTICDB_TRACE(log::codec(), "Decoded column {} to position {}", i, data-begin);
+ }
+ ARCTICDB_TRACE(log::codec(), "Decoded column {} to position {}", i, data - begin);
  }
 
  decode_string_pool(hdr, data, begin, end, res);
@@ -500,7 +514,8 @@ void encode_sparse_map(
  Buffer& out,
  std::ptrdiff_t& pos
 ) {
- if (column_data.bit_vector() != nullptr && column_data.bit_vector()->count() > 0) {
+ if (column_data.bit_vector() != nullptr && column_data.bit_vector()->count() > 0) {
+ util::check(!is_empty_type(column_data.type().data_type()), "Empty typed columns should not have sparse maps");
  ARCTICDB_DEBUG(log::codec(), "Sparse map count = {} pos = {}", column_data.bit_vector()->count(), pos);
  const size_t sparse_bm_bytes = encode_bitmap(*column_data.bit_vector(), out, pos);
  util::variant_match(variant_field, [&](auto field) {

cpp/arcticdb/codec/zstd.hpp

@@ -51,28 +51,31 @@ struct ZstdBlockEncoder {
 
 struct ZstdDecoder {
 
-/*
- * encoder_version is here to support multiple versions but won't be used before we have them
- */
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wunused-parameter"
-
- template<typename T>
+ /// @param[in] encoder_version Used to support multiple versions but won't be used before we have them
+ template <typename T>
  static void decode_block(
- std::uint32_t encoder_version,
- const std::uint8_t *in,
+ [[maybe_unused]] std::uint32_t encoder_version,
+ const std::uint8_t* in,
  std::size_t in_bytes,
- T *t_out,
- std::size_t out_bytes) {
+ T* t_out,
+ std::size_t out_bytes
+ ) {
 
  const std::size_t decomp_size = ZSTD_getFrameContentSize(in, in_bytes);
- codec::check<ErrorCode::E_DECODE_ERROR>(decomp_size == out_bytes, "expected out_bytes == zstd deduced bytes, actual {} != {}",
- out_bytes, decomp_size);
+ codec::check<ErrorCode::E_DECODE_ERROR>(
+ decomp_size == out_bytes,
+ "expected out_bytes == zstd deduced bytes, actual {} != {}",
+ out_bytes,
+ decomp_size
+ );
  std::size_t real_decomp = ZSTD_decompress(t_out, out_bytes, in, in_bytes);
- codec::check<ErrorCode::E_DECODE_ERROR>(real_decomp == out_bytes, "expected out_bytes == zstd decompressed bytes, actual {} != {}",
- out_bytes, real_decomp);
+ codec::check<ErrorCode::E_DECODE_ERROR>(
+ real_decomp == out_bytes,
+ "expected out_bytes == zstd decompressed bytes, actual {} != {}",
+ out_bytes,
+ real_decomp
+ );
  }
-#pragma GCC diagnostic pop
 };
 
 } // namespace arcticdb::detail