Skip to content

Commit

Permalink
Merge pull request #8395 from rouault/tiledb_2_17
Browse files Browse the repository at this point in the history 
Add support for TileDB 2.17
  • Loading branch information
@rouault
rouault authored Sep 15, 2023
2 parents e9577a9 + e8ed7b5 commit 9b3d39e
Show file tree
Hide file tree
Showing 3 changed files with 129 additions and 32 deletions.
4 changes: 2 additions & 2 deletions .github/workflows/fedora_rawhide/Dockerfile.ci
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
FROM fedora:rawhide

RUN dnf upgrade -y
RUN dnf5 install -y --setopt=install_weak_deps=False proj-devel
RUN dnf5 install -y clang make diffutils ccache cmake \
RUN dnf install -y --setopt=install_weak_deps=False proj-devel
RUN dnf install -y clang make diffutils ccache cmake \
libxml2-devel libxslt-devel expat-devel xerces-c-devel \
zlib-devel xz-devel libzstd-devel blosc-devel libarchive-devel \
giflib-devel libjpeg-devel libpng-devel \
Expand Down
90 changes: 90 additions & 0 deletions frmts/tiledb/tiledbheaders.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
#ifndef TILEDB_HEADERS_H
#define TILEDB_HEADERS_H

#include <algorithm>
#include <list>
#include <variant>

Expand Down Expand Up @@ -79,6 +80,92 @@
#define HAS_TILEDB_DIMENSION_LABEL
#endif

#if TILEDB_VERSION_MAJOR > 2 || \
(TILEDB_VERSION_MAJOR == 2 && TILEDB_VERSION_MINOR >= 17)
struct gdal_tiledb_vector_of_bool
{
size_t m_size = 0;
size_t m_capacity = 0;
bool *m_v = nullptr;

gdal_tiledb_vector_of_bool() = default;
~gdal_tiledb_vector_of_bool()
{
std::free(m_v);
}
gdal_tiledb_vector_of_bool(gdal_tiledb_vector_of_bool &&other)
: m_size(other.m_size), m_capacity(other.m_capacity),
m_v(std::move(other.m_v))
{
other.m_size = 0;
other.m_capacity = 0;
other.m_v = nullptr;
}
gdal_tiledb_vector_of_bool(const gdal_tiledb_vector_of_bool &) = delete;
gdal_tiledb_vector_of_bool &
operator=(const gdal_tiledb_vector_of_bool &) = delete;
gdal_tiledb_vector_of_bool &
operator=(gdal_tiledb_vector_of_bool &&) = delete;

size_t size() const
{
return m_size;
}
const bool *data() const
{
return m_v;
}
bool *data()
{
return m_v;
}
bool &operator[](size_t idx)
{
return m_v[idx];
}
bool operator[](size_t idx) const
{
return m_v[idx];
}
void resize(size_t new_size)
{
if (new_size > m_capacity)
{
const size_t new_capacity =
std::max<size_t>(new_size, 2 * m_capacity);
bool *new_v = static_cast<bool *>(
std::realloc(m_v, new_capacity * sizeof(bool)));
if (!new_v)
{
throw std::bad_alloc();
}
m_v = new_v;
m_capacity = new_capacity;
}
if (new_size > m_size)
memset(m_v + m_size, 0, (new_size - m_size) * sizeof(bool));
m_size = new_size;
}
void clear()
{
resize(0);
}
size_t capacity() const
{
return m_capacity;
}
void push_back(uint8_t v)
{
resize(size() + 1);
m_v[size() - 1] = static_cast<bool>(v);
}
};
#define VECTOR_OF_BOOL gdal_tiledb_vector_of_bool
#define VECTOR_OF_BOOL_IS_NOT_UINT8_T
#else
#define VECTOR_OF_BOOL std::vector<uint8_t>
#endif

#ifdef HAS_TILEDB_DIMENSION_LABEL
#define HAS_TILEDB_MULTIDIM
#endif
Expand Down Expand Up @@ -228,6 +315,9 @@ class OGRTileDBLayer final : public OGRLayer,
{
public:
typedef std::variant<std::shared_ptr<std::string>,
#ifdef VECTOR_OF_BOOL_IS_NOT_UINT8_T
std::shared_ptr<VECTOR_OF_BOOL>,
#endif
std::shared_ptr<std::vector<uint8_t>>,
std::shared_ptr<std::vector<int16_t>>,
std::shared_ptr<std::vector<uint16_t>>,
Expand Down
67 changes: 37 additions & 30 deletions frmts/tiledb/tiledbsparse.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ template <> struct GetType<TILEDB_UINT16>
#ifdef HAS_TILEDB_BOOL
template <> struct GetType<TILEDB_BOOL>
{
using EltType = std::vector<uint8_t>;
using EltType = VECTOR_OF_BOOL;
};
#endif

Expand Down Expand Up @@ -759,7 +759,7 @@ bool OGRTileDBLayer::InitFromStorage(tiledb::Context *poCtx,
case TILEDB_BOOL:
eType = bIsSingle ? OFTInteger : OFTIntegerList;
eSubType = OFSTBoolean;
fieldValues.push_back(std::make_shared<std::vector<uint8_t>>());
fieldValues.push_back(std::make_shared<VECTOR_OF_BOOL>());
break;
#endif
case TILEDB_DATETIME_DAY:
Expand Down Expand Up @@ -1288,10 +1288,14 @@ void OGRTileDBLayer::SetReadBuffers(bool bGrowVariableSizeArrays)
#ifdef HAS_TILEDB_BOOL
if (m_aeFieldTypes[i] == TILEDB_BOOL)
{
auto &v = *(std::get<std::shared_ptr<std::vector<uint8_t>>>(
fieldValues));
auto &v = *(
std::get<std::shared_ptr<VECTOR_OF_BOOL>>(fieldValues));
v.resize(m_nBatchSize);
#ifdef VECTOR_OF_BOOL_IS_NOT_UINT8_T
m_query->set_buffer(pszFieldName, v.data(), v.size());
#else
m_query->set_buffer(pszFieldName, v);
#endif
}
else
#endif
Expand Down Expand Up @@ -1342,14 +1346,18 @@ void OGRTileDBLayer::SetReadBuffers(bool bGrowVariableSizeArrays)
#ifdef HAS_TILEDB_BOOL
if (m_aeFieldTypes[i] == TILEDB_BOOL)
{
auto &v = *(std::get<std::shared_ptr<std::vector<uint8_t>>>(
fieldValues));
auto &v = *(
std::get<std::shared_ptr<VECTOR_OF_BOOL>>(fieldValues));
v.resize(GetValueSize(
pszFieldName, m_anFieldValuesCapacity[i], nMulFactor));
m_anFieldValuesCapacity[i] = v.capacity();
anOffsets.resize(m_nBatchSize);
m_query->set_offsets_buffer(pszFieldName, anOffsets);
#ifdef VECTOR_OF_BOOL_IS_NOT_UINT8_T
m_query->set_data_buffer(pszFieldName, v.data(), v.size());
#else
m_query->set_data_buffer(pszFieldName, v);
#endif
}
else
#endif
Expand Down Expand Up @@ -1880,9 +1888,8 @@ bool OGRTileDBLayer::SetupQuery(tiledb::QueryCondition *queryCondition)
#ifdef HAS_TILEDB_BOOL
if (m_aeFieldTypes[i] == TILEDB_BOOL)
{
auto &v =
*(std::get<std::shared_ptr<std::vector<uint8_t>>>(
fieldValues));
auto &v = *(std::get<std::shared_ptr<VECTOR_OF_BOOL>>(
fieldValues));
v.resize(result.second);
}
else
Expand Down Expand Up @@ -1927,9 +1934,8 @@ bool OGRTileDBLayer::SetupQuery(tiledb::QueryCondition *queryCondition)
#ifdef HAS_TILEDB_BOOL
if (m_aeFieldTypes[i] == TILEDB_BOOL)
{
auto &v =
*(std::get<std::shared_ptr<std::vector<uint8_t>>>(
fieldValues));
auto &v = *(std::get<std::shared_ptr<VECTOR_OF_BOOL>>(
fieldValues));
if (nRowCount < result.first)
{
v.resize(anOffsets[nRowCount] / sizeof(v[0]));
Expand Down Expand Up @@ -2989,9 +2995,8 @@ OGRFeature *OGRTileDBLayer::TranslateCurrentFeature()
#ifdef HAS_TILEDB_BOOL
if (m_aeFieldTypes[i] == TILEDB_BOOL)
{
const auto &v =
*(std::get<std::shared_ptr<std::vector<uint8_t>>>(
fieldValues));
const auto &v = *(
std::get<std::shared_ptr<VECTOR_OF_BOOL>>(fieldValues));
poFeature->SetFieldSameTypeUnsafe(i,
v[m_nOffsetInResultSet]);
}
Expand Down Expand Up @@ -3041,9 +3046,8 @@ OGRFeature *OGRTileDBLayer::TranslateCurrentFeature()
#ifdef HAS_TILEDB_BOOL
if (m_aeFieldTypes[i] == TILEDB_BOOL)
{
const auto &v =
*(std::get<std::shared_ptr<std::vector<uint8_t>>>(
fieldValues));
const auto &v = *(
std::get<std::shared_ptr<VECTOR_OF_BOOL>>(fieldValues));
const size_t nEltCount = GetEltCount(
anOffsets, sizeof(v[0]), v.size() * sizeof(v[0]));
std::vector<int32_t> tmp;
Expand Down Expand Up @@ -3561,7 +3565,7 @@ void OGRTileDBLayer::InitializeSchemaAndArray()
{
CreateAttr(TILEDB_BOOL, eType == OFTIntegerList);
aFieldValues.push_back(
std::make_shared<std::vector<uint8_t>>());
std::make_shared<VECTOR_OF_BOOL>());
}
else
#endif
Expand Down Expand Up @@ -3939,7 +3943,7 @@ OGRErr OGRTileDBLayer::ICreateFeature(OGRFeature *poFeature)
bFieldIsValid ? poFeature->GetFieldAsIntegerUnsafe(i) : 0;
#ifdef HAS_TILEDB_BOOL
if (m_aeFieldTypes[i] == TILEDB_BOOL)
std::get<std::shared_ptr<std::vector<uint8_t>>>(fieldValues)
std::get<std::shared_ptr<VECTOR_OF_BOOL>>(fieldValues)
->push_back(static_cast<uint8_t>(nVal));
else
#endif
Expand Down Expand Up @@ -3973,8 +3977,8 @@ OGRErr OGRTileDBLayer::ICreateFeature(OGRFeature *poFeature)
#ifdef HAS_TILEDB_BOOL
if (m_aeFieldTypes[i] == TILEDB_BOOL)
{
auto &v = *(std::get<std::shared_ptr<std::vector<uint8_t>>>(
fieldValues));
auto &v = *(
std::get<std::shared_ptr<VECTOR_OF_BOOL>>(fieldValues));
for (int j = 0; j < nCount; ++j)
v.push_back(static_cast<uint8_t>(panVal[j]));
anOffsets.push_back(anOffsets.back() +
Expand Down Expand Up @@ -4299,10 +4303,13 @@ void OGRTileDBLayer::FlushArrays()
#ifdef HAS_TILEDB_BOOL
if (m_aeFieldTypes[i] == TILEDB_BOOL)
{
auto &v =
*(std::get<std::shared_ptr<std::vector<uint8_t>>>(
fieldValues));
query.set_data_buffer(pszFieldName, v);
auto &v = *(std::get<std::shared_ptr<VECTOR_OF_BOOL>>(
fieldValues));
#ifdef VECTOR_OF_BOOL_IS_NOT_UINT8_T
query.set_buffer(pszFieldName, v.data(), v.size());
#else
query.set_buffer(pszFieldName, v);
#endif
}
else
#endif
Expand Down Expand Up @@ -4729,8 +4736,8 @@ void OGRTileDBLayer::FillBoolArray(
psChild->buffers = static_cast<const void **>(CPLCalloc(2, sizeof(void *)));
// TileDB used a std::vector<uint8_t> with 1 element per byte
// whereas Arrow uses a ~ std::vector<bool> with 8 elements per byte
const auto &v_source = *(std::get<std::shared_ptr<std::vector<uint8_t>>>(
m_aFieldValues[iField]));
const auto &v_source =
*(std::get<std::shared_ptr<VECTOR_OF_BOOL>>(m_aFieldValues[iField]));
const size_t nDstSize = abyValidityFromFilters.empty()
? v_source.size()
: static_cast<size_t>(psChild->length);
Expand Down Expand Up @@ -5015,8 +5022,8 @@ void OGRTileDBLayer::FillBoolListArray(
psChild->buffers = static_cast<const void **>(CPLCalloc(2, sizeof(void *)));
auto &offsetsPtr = m_aFieldValueOffsets[iField];
psPrivateData->offsetHolder = offsetsPtr;
auto &v_source = *(std::get<std::shared_ptr<std::vector<uint8_t>>>(
m_aFieldValues[iField]));
auto &v_source =
*(std::get<std::shared_ptr<VECTOR_OF_BOOL>>(m_aFieldValues[iField]));

psChild->n_children = 1;
psChild->children = static_cast<struct ArrowArray **>(
Expand Down

0 comments on commit 9b3d39e

Please sign in to comment.