Skip to content

Commit

Permalink
Merge branch 'dev' into rd/C41-azure
Browse files Browse the repository at this point in the history
  • Loading branch information
teo-tsirpanis committed Dec 18, 2024
2 parents f7534ea + 41c8a10 commit 5631752
Show file tree
Hide file tree
Showing 37 changed files with 928 additions and 140 deletions.
10 changes: 3 additions & 7 deletions .github/workflows/ci-linux_mac.yml
Original file line number Diff line number Diff line change
Expand Up @@ -80,9 +80,6 @@ env:
bootstrap_args: "--enable-ccache --vcpkg-base-triplet=${{ inputs.vcpkg_base_triplet || (startsWith(inputs.matrix_image, 'ubuntu-') && 'x64-linux' || 'arm64-osx') }} ${{ inputs.bootstrap_args }} ${{ inputs.asan && '--enable-sanitizer=address' || '' }}"
VCPKG_BINARY_SOURCES: 'clear;x-gha,readwrite'
SCCACHE_GHA_ENABLED: "true"
# Manylinux does not support Node 20 due to libc incompatibility. Temporarily opt out.
# https://github.blog/changelog/2024-03-07-github-actions-all-actions-will-run-on-node20-instead-of-node16-by-default/
ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: ${{ inputs.manylinux && 'true' || 'false' }}

jobs:
build:
Expand All @@ -91,7 +88,7 @@ jobs:
os:
- ${{ inputs.matrix_image }}
runs-on: ${{matrix.os}}
container: ${{inputs.manylinux && 'quay.io/pypa/manylinux2014_x86_64' || ''}}
container: ${{inputs.manylinux && 'quay.io/pypa/manylinux_2_28_x86_64' || ''}}

if: ${{ startsWith(github.ref , 'refs/tags') != true && startsWith(github.ref , 'build-') != true }}
timeout-minutes: ${{ inputs.timeout || 90 }}
Expand All @@ -117,7 +114,7 @@ jobs:
python-version: '3.12'

- name: Prevent vpckg from building debug variants
run: python ./scripts/ci/patch_vcpkg_triplets.py
run: python3 ./scripts/ci/patch_vcpkg_triplets.py

- name: Setup sccache
uses: mozilla-actions/sccache-action@v0.0.3
Expand All @@ -131,8 +128,7 @@ jobs:
if: inputs.manylinux
run: |
set -e pipefail
yum install -y redhat-lsb-core centos-release-scl devtoolset-7 perl-IPC-Cmd
echo "source /opt/rh/devtoolset-7/enable" >> ~/.bashrc
yum install -y redhat-lsb-core perl-IPC-Cmd curl zip unzip tar
# This must happen after checkout, because checkout would remove the directory.
- name: Install Ninja
Expand Down
1 change: 1 addition & 0 deletions .github/workflows/full-ci.yml
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
name: "Full CI"
on:
workflow_dispatch:
push:
branches:
- dev
Expand Down
16 changes: 5 additions & 11 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -49,19 +49,19 @@ jobs:
triplet: x64-windows-release
- platform: linux-x86_64
os: ubuntu-20.04
manylinux: quay.io/pypa/manylinux2014_x86_64
manylinux: quay.io/pypa/manylinux_2_28_x86_64
triplet: x64-linux-release
- platform: linux-x86_64-noavx2
os: ubuntu-20.04
cmake_args: -DCOMPILER_SUPPORTS_AVX2=OFF
triplet: x64-linux-release
manylinux: quay.io/pypa/manylinux2014_x86_64
manylinux: quay.io/pypa/manylinux_2_28_x86_64
- platform: linux-aarch64
os: linux-arm64-ubuntu24
triplet: arm64-linux-release
manylinux: quay.io/pypa/manylinux2014_aarch64
manylinux: quay.io/pypa/manylinux_2_28_aarch64
- platform: macos-x86_64
os: macos-12
os: macos-13
cmake_args: -DCMAKE_OSX_ARCHITECTURES=x86_64
MACOSX_DEPLOYMENT_TARGET: 11
triplet: x64-osx-release
Expand All @@ -75,9 +75,6 @@ jobs:
env:
MACOSX_DEPLOYMENT_TARGET: ${{ matrix.MACOSX_DEPLOYMENT_TARGET }}
VCPKG_BINARY_SOURCES: 'clear;x-gha,readwrite'
# Manylinux does not support Node 20 due to libc incompatibility. Temporarily opt out.
# https://github.blog/changelog/2024-03-07-github-actions-all-actions-will-run-on-node20-instead-of-node16-by-default/
ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: ${{ matrix.manylinux && 'true' || 'false' }}

steps:
- name: Checkout TileDB
Expand All @@ -103,10 +100,7 @@ jobs:
if: ${{ startsWith(matrix.platform, 'linux') == true }}
run: |
set -e pipefail
yum install -y redhat-lsb-core centos-release-scl devtoolset-7 perl-IPC-Cmd
echo "source /opt/rh/devtoolset-7/enable" >> ~/.bashrc
python3.9 -m pip install ninja
echo "/opt/_internal/cpython-3.9.20/bin" >> $GITHUB_PATH
yum install -y ninja-build perl-IPC-Cmd curl zip unzip tar
echo "VCPKG_FORCE_SYSTEM_BINARIES=YES" >> $GITHUB_ENV
- name: Configure TileDB
run: |
Expand Down
26 changes: 23 additions & 3 deletions examples/cpp_api/using_tiledb_stats.cc
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,25 @@ using namespace tiledb;
// Name of array.
std::string array_name("stats_array");

/**
* Enables tiledb statistics gathering within a block.
*/
class ScopedStats {
public:
ScopedStats()
: enabled_(Stats::is_enabled()) {
Stats::enable();
}
~ScopedStats() {
if (!enabled_) {
Stats::disable();
}
}

private:
bool enabled_;
};

void create_array(uint32_t row_tile_extent, uint32_t col_tile_extent) {
Context ctx;
VFS vfs(ctx);
Expand Down Expand Up @@ -86,10 +105,11 @@ void read_array() {
query.set_subarray(subarray).set_data_buffer("a", values);

// Enable the stats for the read query, and print the report.
Stats::enable();
query.submit();
{
ScopedStats stats;
query.submit();
}
Stats::dump(stdout);
Stats::disable();
}

int main() {
Expand Down
2 changes: 2 additions & 0 deletions test/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,7 @@ set(TILEDB_UNIT_TEST_SOURCES
src/unit-capi-sparse_neg_2.cc
src/unit-capi-sparse_real.cc
src/unit-capi-sparse_real_2.cc
src/unit-capi-stats.cc
src/unit-capi-string.cc
src/unit-capi-string_dims.cc
src/unit-capi-update-queries.cc
Expand Down Expand Up @@ -179,6 +180,7 @@ if (TILEDB_CPP_API)
src/cpp-integration-query-condition.cc
src/unit-cppapi-schema.cc
src/unit-cppapi-schema-evolution.cc
src/unit-cppapi-stats.cc
src/unit-cppapi-string-dims.cc
src/unit-cppapi-subarray.cc
src/unit-cppapi-type.cc
Expand Down
41 changes: 41 additions & 0 deletions test/src/unit-capi-stats.cc
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
/**
* @file unit-capi-stats.cc
*
* @section LICENSE
*
* The MIT License
*
* @copyright Copyright (c) 2023-2024 TileDB Inc.
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE.
*
* @section DESCRIPTION
*
* Tests the C API for stats related functions.
*/

#include "test/support/src/stats.h"
#include "tiledb/sm/cpp_api/tiledb"

#include <test/support/tdb_catch.h>

TEST_CASE("tiledb_stats_is_enabled null arg", "[stats]") {
auto status = tiledb_stats_is_enabled(nullptr);
CHECK(status == TILEDB_ERR);
}
50 changes: 50 additions & 0 deletions test/src/unit-cppapi-consolidation.cc
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
*
* Consolidation tests with the C++ API.
*/
#include "tiledb/sm/cpp_api/tiledb_experimental"

#include <test/support/tdb_catch.h>
#include "test/support/src/helpers.h"
Expand Down Expand Up @@ -488,3 +489,52 @@ TEST_CASE(
if (vfs.is_dir(array_name))
vfs.remove_dir(array_name);
}

TEST_CASE(
"C++ API: Test consolidation that respects the current domain",
"[cppapi][consolidation][current_domain]") {
std::string array_name = "cppapi_consolidation_current_domain";
remove_array(array_name);

Context ctx;

Domain domain(ctx);
auto d1 = Dimension::create<int32_t>(ctx, "d1", {{0, 1000000000}}, 50);
auto d2 = Dimension::create<int32_t>(ctx, "d2", {{0, 1000000000}}, 50);
domain.add_dimensions(d1, d2);

auto a = Attribute::create<int>(ctx, "a");

ArraySchema schema(ctx, TILEDB_DENSE);
schema.set_domain(domain);
schema.add_attributes(a);

tiledb::NDRectangle ndrect(ctx, domain);
int32_t range_one[] = {0, 2};
int32_t range_two[] = {0, 3};
ndrect.set_range(0, range_one[0], range_one[1]);
ndrect.set_range(1, range_two[0], range_two[1]);

tiledb::CurrentDomain current_domain(ctx);
current_domain.set_ndrectangle(ndrect);

tiledb::ArraySchemaExperimental::set_current_domain(
ctx, schema, current_domain);

Array::create(array_name, schema);

std::vector<int> data = {
-60, 79, -8, 100, 88, -19, -100, -111, -72, -85, 58, -41};

// Write it twice so there is something to consolidate
write_array(array_name, {0, 2, 0, 3}, data);
write_array(array_name, {0, 2, 0, 3}, data);

CHECK(tiledb::test::num_fragments(array_name) == 2);
Context ctx2;
Config config;
REQUIRE_NOTHROW(Array::consolidate(ctx2, array_name, &config));
CHECK(tiledb::test::num_fragments(array_name) == 3);

remove_array(array_name);
}
71 changes: 71 additions & 0 deletions test/src/unit-cppapi-stats.cc
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
/**
* @file unit-cppapi-stats.cc
*
* @section LICENSE
*
* The MIT License
*
* @copyright Copyright (c) 2023-2024 TileDB Inc.
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE.
*
* @section DESCRIPTION
*
* Tests the C++ API for stats related functions.
*/

#include "test/support/src/stats.h"
#include "tiledb/sm/cpp_api/tiledb"

#include <test/support/tdb_catch.h>

using namespace tiledb;

TEST_CASE("stats gathering default", "[stats]") {
CHECK(!Stats::is_enabled());
}

TEST_CASE("stats disabled, scoped enable", "[stats]") {
CHECK(!Stats::is_enabled());
{
test::ScopedStats scoped;
CHECK(Stats::is_enabled());
}
CHECK(!Stats::is_enabled());
}

TEST_CASE("stats enabled, scoped enable", "[stats]") {
CHECK(!Stats::is_enabled());

// outer scope disables when exiting
{
test::ScopedStats outer;
CHECK(Stats::is_enabled());

// inner scope does not disable since stats was enabled when entering
{
test::ScopedStats inner;
CHECK(Stats::is_enabled());
}

CHECK(Stats::is_enabled());
}

CHECK(!Stats::is_enabled());
}
1 change: 1 addition & 0 deletions test/support/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ set(TILEDB_TEST_SUPPORT_SOURCES
src/mem_helpers.h
src/mem_helpers.cc
src/serialization_wrappers.cc
src/stats.cc
src/temporary_local_directory.cc
src/vfs_helpers.cc
)
Expand Down
51 changes: 51 additions & 0 deletions test/support/src/stats.cc
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
/**
* @file stats.cc
*
* @section LICENSE
*
* The MIT License
*
* @copyright Copyright (c) 2017-2024 TileDB, Inc.
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE.
*
* @section DESCRIPTION
*
* This file defines some helpers for tests which use stats gathering.
*/

#include "test/support/src/stats.h"
#include "tiledb/sm/cpp_api/tiledb"

using namespace tiledb;

namespace tiledb::test {

ScopedStats::ScopedStats()
: enabled_(Stats::is_enabled()) {
Stats::enable();
}

ScopedStats::~ScopedStats() {
if (!enabled_) {
Stats::disable();
}
}

} // namespace tiledb::test
Loading

0 comments on commit 5631752

Please sign in to comment.