Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Expand Statistics support in the C API #11263

Closed
wants to merge 6 commits into from
Closed

Expand Statistics support in the C API #11263

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
5c6b866
Expand Statistics support in the C API
amatveev-cf Feb 22, 2023
ee915d5
Statistics support in C API, named constants
amatveev-cf Jul 4, 2023
d68bc6f
Statistics support in C API, additional improvements
amatveev-cf Jul 28, 2023
7d16c3e
Merge branch 'facebook:main' into c_api_statistics
amatveev-cf Aug 1, 2023
d21504c
Merge branch 'main' into c_api_statistics
hx235 Aug 2, 2023
9e49832
Fix to nits
hx235 Aug 2, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
114 changes: 108 additions & 6 deletions db/c.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@ using ROCKSDB_NAMESPACE::EnvOptions;
using ROCKSDB_NAMESPACE::FileLock;
using ROCKSDB_NAMESPACE::FilterPolicy;
using ROCKSDB_NAMESPACE::FlushOptions;
using ROCKSDB_NAMESPACE::HistogramData;
using ROCKSDB_NAMESPACE::HyperClockCacheOptions;
using ROCKSDB_NAMESPACE::InfoLogLevel;
using ROCKSDB_NAMESPACE::IngestExternalFileOptions;
Expand Down Expand Up @@ -279,6 +280,11 @@ struct rocksdb_compactionfiltercontext_t {
CompactionFilter::Context rep;
};

struct rocksdb_statistics_histogram_data_t {
rocksdb_statistics_histogram_data_t() : rep() {}
HistogramData rep;
};

struct rocksdb_compactionfilter_t : public CompactionFilter {
void* state_;
void (*destructor_)(void*);
Expand Down Expand Up @@ -3023,6 +3029,29 @@ void rocksdb_options_enable_statistics(rocksdb_options_t* opt) {
opt->rep.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
}

void rocksdb_options_set_statistics_level(rocksdb_options_t* opt, int level) {
if (!opt->rep.statistics) {
return;
}

if (level < rocksdb_statistics_level_disable_all) {
level = rocksdb_statistics_level_disable_all;
}
if (level > rocksdb_statistics_level_all) {
level = rocksdb_statistics_level_all;
}
opt->rep.statistics->set_stats_level(
static_cast<ROCKSDB_NAMESPACE::StatsLevel>(level));
}

int rocksdb_options_get_statistics_level(rocksdb_options_t* opt) {
if (!opt->rep.statistics) {
return ROCKSDB_NAMESPACE::StatsLevel::kDisableAll;
}

return static_cast<int>(opt->rep.statistics->get_stats_level());
}

void rocksdb_options_set_skip_stats_update_on_db_open(rocksdb_options_t* opt,
unsigned char val) {
opt->rep.skip_stats_update_on_db_open = val;
Expand Down Expand Up @@ -3862,6 +3891,26 @@ char* rocksdb_options_statistics_get_string(rocksdb_options_t* opt) {
return nullptr;
}

uint64_t rocksdb_options_statistics_get_ticker_count(rocksdb_options_t* opt,
uint32_t ticker_type) {
ROCKSDB_NAMESPACE::Statistics* statistics = opt->rep.statistics.get();
if (statistics) {
return statistics->getTickerCount(ticker_type);
}
return 0;
}

void rocksdb_options_statistics_get_histogram_data(
rocksdb_options_t* opt, uint32_t type,
rocksdb_statistics_histogram_data_t* const data) {
ROCKSDB_NAMESPACE::Statistics* statistics = opt->rep.statistics.get();
if (statistics) {
statistics->histogramData(type, &data->rep);
} else {
*data = rocksdb_statistics_histogram_data_t{};
hx235 marked this conversation as resolved.
Show resolved Hide resolved
}
}

void rocksdb_options_set_ratelimiter(rocksdb_options_t* opt,
rocksdb_ratelimiter_t* limiter) {
if (limiter) {
Expand Down Expand Up @@ -5194,7 +5243,8 @@ rocksdb_fifo_compaction_options_t* rocksdb_fifo_compaction_options_create() {
}

void rocksdb_fifo_compaction_options_set_allow_compaction(
rocksdb_fifo_compaction_options_t* fifo_opts, unsigned char allow_compaction) {
rocksdb_fifo_compaction_options_t* fifo_opts,
unsigned char allow_compaction) {
fifo_opts->rep.allow_compaction = allow_compaction;
}

Expand Down Expand Up @@ -5623,8 +5673,7 @@ int rocksdb_transactiondb_property_int(rocksdb_transactiondb_t* db,
}
}

rocksdb_t* rocksdb_transactiondb_get_base_db(
rocksdb_transactiondb_t* txn_db) {
rocksdb_t* rocksdb_transactiondb_get_base_db(rocksdb_transactiondb_t* txn_db) {
DB* base_db = txn_db->rep->GetBaseDB();

if (base_db != nullptr) {
Expand All @@ -5636,9 +5685,7 @@ rocksdb_t* rocksdb_transactiondb_get_base_db(
return nullptr;
}

void rocksdb_transactiondb_close_base_db(rocksdb_t* base_db) {
delete base_db;
}
void rocksdb_transactiondb_close_base_db(rocksdb_t* base_db) { delete base_db; }

rocksdb_transaction_t* rocksdb_transaction_begin(
rocksdb_transactiondb_t* txn_db,
Expand Down Expand Up @@ -6617,4 +6664,59 @@ void rocksdb_enable_manual_compaction(rocksdb_t* db) {
db->rep->EnableManualCompaction();
}

rocksdb_statistics_histogram_data_t*
rocksdb_statistics_histogram_data_create() {
return new rocksdb_statistics_histogram_data_t{};
hx235 marked this conversation as resolved.
Show resolved Hide resolved
}
hx235 marked this conversation as resolved.
Show resolved Hide resolved

void rocksdb_statistics_histogram_data_destroy(
rocksdb_statistics_histogram_data_t* data) {
delete data;
}

double rocksdb_statistics_histogram_data_get_median(
rocksdb_statistics_histogram_data_t* data) {
return data->rep.median;
}

double rocksdb_statistics_histogram_data_get_p95(
rocksdb_statistics_histogram_data_t* data) {
return data->rep.percentile95;
}

double rocksdb_statistics_histogram_data_get_p99(
rocksdb_statistics_histogram_data_t* data) {
return data->rep.percentile99;
}

double rocksdb_statistics_histogram_data_get_average(
rocksdb_statistics_histogram_data_t* data) {
return data->rep.average;
}

double rocksdb_statistics_histogram_data_get_std_dev(
rocksdb_statistics_histogram_data_t* data) {
return data->rep.standard_deviation;
}

double rocksdb_statistics_histogram_data_get_max(
rocksdb_statistics_histogram_data_t* data) {
return data->rep.max;
}

uint64_t rocksdb_statistics_histogram_data_get_count(
rocksdb_statistics_histogram_data_t* data) {
return data->rep.count;
}

uint64_t rocksdb_statistics_histogram_data_get_sum(
rocksdb_statistics_histogram_data_t* data) {
return data->rep.sum;
}

double rocksdb_statistics_histogram_data_get_min(
rocksdb_statistics_histogram_data_t* data) {
return data->rep.min;
}

} // end extern "C"
85 changes: 79 additions & 6 deletions db/c_test.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,15 +3,14 @@
found in the LICENSE file. See the AUTHORS file for names of contributors. */
// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.

#include <stdio.h>
#include "rocksdb/c.h"

#include <assert.h>
#include <stddef.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/types.h>

#include "rocksdb/c.h"
#ifndef OS_WIN
#include <unistd.h>
#endif
Expand Down Expand Up @@ -2061,6 +2060,15 @@ int main(int argc, char** argv) {
CheckCondition(29.0 ==
rocksdb_options_get_experimental_mempurge_threshold(o));

CheckCondition(rocksdb_statistics_level_disable_all ==
rocksdb_options_get_statistics_level(o));
rocksdb_options_enable_statistics(o);
CheckCondition(rocksdb_statistics_level_disable_all !=
rocksdb_options_get_statistics_level(o));
rocksdb_options_set_statistics_level(o, rocksdb_statistics_level_all);
CheckCondition(rocksdb_statistics_level_all ==
rocksdb_options_get_statistics_level(o));

/* Blob Options */
rocksdb_options_set_enable_blob_files(o, 1);
CheckCondition(1 == rocksdb_options_get_enable_blob_files(o));
Expand Down Expand Up @@ -3122,12 +3130,12 @@ int main(int argc, char** argv) {
CheckTxnDBGetCF(txn_db, roptions, cfh, "cf_foo", NULL);
CheckTxnDBPinGetCF(txn_db, roptions, cfh, "cf_foo", NULL);


//memory usage
// memory usage
rocksdb_t* base_db = rocksdb_transactiondb_get_base_db(txn_db);
rocksdb_memory_consumers_t* consumers = rocksdb_memory_consumers_create();
rocksdb_memory_consumers_add_db(consumers, base_db);
rocksdb_memory_usage_t* usage = rocksdb_approximate_memory_usage_create(consumers, &err);
rocksdb_memory_usage_t* usage =
rocksdb_approximate_memory_usage_create(consumers, &err);
CheckNoError(err);
rocksdb_approximate_memory_usage_destroy(usage);
rocksdb_memory_consumers_destroy(consumers);
Expand Down Expand Up @@ -3614,6 +3622,71 @@ int main(int argc, char** argv) {
rocksdb_readoptions_destroy(ropts);
}

StartPhase("statistics");
{
const uint32_t BYTES_WRITTEN_TICKER = 40;
const uint32_t DB_WRITE_HIST = 1;
hx235 marked this conversation as resolved.
Show resolved Hide resolved

rocksdb_statistics_histogram_data_t* hist =
rocksdb_statistics_histogram_data_create();
{
// zero by default
CheckCondition(0.0 == rocksdb_statistics_histogram_data_get_median(hist));
CheckCondition(0.0 == rocksdb_statistics_histogram_data_get_p95(hist));
CheckCondition(0.0 == rocksdb_statistics_histogram_data_get_p99(hist));
CheckCondition(0.0 ==
rocksdb_statistics_histogram_data_get_average(hist));
CheckCondition(0.0 ==
rocksdb_statistics_histogram_data_get_std_dev(hist));
CheckCondition(0.0 == rocksdb_statistics_histogram_data_get_max(hist));
CheckCondition(0 == rocksdb_statistics_histogram_data_get_count(hist));
CheckCondition(0 == rocksdb_statistics_histogram_data_get_sum(hist));
CheckCondition(0.0 == rocksdb_statistics_histogram_data_get_min(hist));
}

rocksdb_close(db);
rocksdb_destroy_db(options, dbname, &err);
CheckNoError(err);

rocksdb_options_enable_statistics(options);
rocksdb_options_set_statistics_level(options, rocksdb_statistics_level_all);

db = rocksdb_open(options, dbname, &err);
CheckNoError(err);

CheckCondition(0 == rocksdb_options_statistics_get_ticker_count(
options, BYTES_WRITTEN_TICKER));
rocksdb_options_statistics_get_histogram_data(options, DB_WRITE_HIST, hist);
CheckCondition(0.0 == rocksdb_statistics_histogram_data_get_median(hist));
CheckCondition(0.0 == rocksdb_statistics_histogram_data_get_p95(hist));
CheckCondition(0.0 == rocksdb_statistics_histogram_data_get_p99(hist));
CheckCondition(0.0 == rocksdb_statistics_histogram_data_get_average(hist));
CheckCondition(0.0 == rocksdb_statistics_histogram_data_get_std_dev(hist));
CheckCondition(0.0 == rocksdb_statistics_histogram_data_get_max(hist));
CheckCondition(0 == rocksdb_statistics_histogram_data_get_count(hist));
CheckCondition(0 == rocksdb_statistics_histogram_data_get_sum(hist));

int i;
for (i = 0; i < 10; ++i) {
char key = '0' + (char)i;
rocksdb_put(db, woptions, &key, 1, "", 1, &err);
CheckNoError(err);
}
CheckCondition(0 != rocksdb_options_statistics_get_ticker_count(
options, BYTES_WRITTEN_TICKER));
rocksdb_options_statistics_get_histogram_data(options, DB_WRITE_HIST, hist);
CheckCondition(0.0 != rocksdb_statistics_histogram_data_get_median(hist));
CheckCondition(0.0 != rocksdb_statistics_histogram_data_get_p95(hist));
CheckCondition(0.0 != rocksdb_statistics_histogram_data_get_p99(hist));
CheckCondition(0.0 != rocksdb_statistics_histogram_data_get_average(hist));
CheckCondition(0.0 != rocksdb_statistics_histogram_data_get_std_dev(hist));
CheckCondition(0.0 != rocksdb_statistics_histogram_data_get_max(hist));
CheckCondition(0 != rocksdb_statistics_histogram_data_get_count(hist));
CheckCondition(0 != rocksdb_statistics_histogram_data_get_sum(hist));
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we also check for other fields rocksdb_statistics_histogram_data_get_*(hist) to be non-zero?


rocksdb_statistics_histogram_data_destroy(hist);
}

StartPhase("cancel_all_background_work");
rocksdb_cancel_all_background_work(db, 1);

Expand Down
49 changes: 48 additions & 1 deletion include/rocksdb/c.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,8 @@ typedef struct rocksdb_wal_iterator_t rocksdb_wal_iterator_t;
typedef struct rocksdb_wal_readoptions_t rocksdb_wal_readoptions_t;
typedef struct rocksdb_memory_consumers_t rocksdb_memory_consumers_t;
typedef struct rocksdb_memory_usage_t rocksdb_memory_usage_t;
typedef struct rocksdb_statistics_histogram_data_t
rocksdb_statistics_histogram_data_t;

/* DB operations */

Expand Down Expand Up @@ -1252,6 +1254,22 @@ rocksdb_options_set_max_bytes_for_level_multiplier_additional(
rocksdb_options_t*, int* level_values, size_t num_levels);
extern ROCKSDB_LIBRARY_API void rocksdb_options_enable_statistics(
rocksdb_options_t*);

enum {
rocksdb_statistics_level_disable_all = 0,
rocksdb_statistics_level_except_tickers =
rocksdb_statistics_level_disable_all,
rocksdb_statistics_level_except_histogram_or_timers = 1,
rocksdb_statistics_level_except_timers = 2,
rocksdb_statistics_level_except_detailed_timers = 3,
rocksdb_statistics_level_except_time_for_mutex = 4,
rocksdb_statistics_level_all = 5,
};

extern ROCKSDB_LIBRARY_API void rocksdb_options_set_statistics_level(
rocksdb_options_t*, int level);
hx235 marked this conversation as resolved.
Show resolved Hide resolved
extern ROCKSDB_LIBRARY_API int rocksdb_options_get_statistics_level(
rocksdb_options_t*);
extern ROCKSDB_LIBRARY_API void
rocksdb_options_set_skip_stats_update_on_db_open(rocksdb_options_t* opt,
unsigned char val);
Expand Down Expand Up @@ -1328,6 +1346,11 @@ extern ROCKSDB_LIBRARY_API int rocksdb_options_get_prepopulate_blob_cache(
/* returns a pointer to a malloc()-ed, null terminated string */
extern ROCKSDB_LIBRARY_API char* rocksdb_options_statistics_get_string(
rocksdb_options_t* opt);
extern ROCKSDB_LIBRARY_API uint64_t rocksdb_options_statistics_get_ticker_count(
rocksdb_options_t* opt, uint32_t ticker_type);
extern ROCKSDB_LIBRARY_API void rocksdb_options_statistics_get_histogram_data(
rocksdb_options_t* opt, uint32_t histogram_type,
rocksdb_statistics_histogram_data_t* const data);

extern ROCKSDB_LIBRARY_API void rocksdb_options_set_max_write_buffer_number(
rocksdb_options_t*, int);
Expand Down Expand Up @@ -2257,7 +2280,8 @@ extern ROCKSDB_LIBRARY_API rocksdb_fifo_compaction_options_t*
rocksdb_fifo_compaction_options_create(void);
extern ROCKSDB_LIBRARY_API void
rocksdb_fifo_compaction_options_set_allow_compaction(
rocksdb_fifo_compaction_options_t* fifo_opts, unsigned char allow_compaction);
rocksdb_fifo_compaction_options_t* fifo_opts,
unsigned char allow_compaction);
extern ROCKSDB_LIBRARY_API unsigned char
rocksdb_fifo_compaction_options_get_allow_compaction(
rocksdb_fifo_compaction_options_t* fifo_opts);
Expand Down Expand Up @@ -2880,6 +2904,29 @@ extern ROCKSDB_LIBRARY_API void rocksdb_disable_manual_compaction(

extern ROCKSDB_LIBRARY_API void rocksdb_enable_manual_compaction(rocksdb_t* db);

extern ROCKSDB_LIBRARY_API rocksdb_statistics_histogram_data_t*
rocksdb_statistics_histogram_data_create(void);
extern ROCKSDB_LIBRARY_API void rocksdb_statistics_histogram_data_destroy(
rocksdb_statistics_histogram_data_t* data);
extern ROCKSDB_LIBRARY_API double rocksdb_statistics_histogram_data_get_median(
rocksdb_statistics_histogram_data_t* data);
extern ROCKSDB_LIBRARY_API double rocksdb_statistics_histogram_data_get_p95(
rocksdb_statistics_histogram_data_t* data);
extern ROCKSDB_LIBRARY_API double rocksdb_statistics_histogram_data_get_p99(
rocksdb_statistics_histogram_data_t* data);
extern ROCKSDB_LIBRARY_API double rocksdb_statistics_histogram_data_get_average(
rocksdb_statistics_histogram_data_t* data);
extern ROCKSDB_LIBRARY_API double rocksdb_statistics_histogram_data_get_std_dev(
rocksdb_statistics_histogram_data_t* data);
extern ROCKSDB_LIBRARY_API double rocksdb_statistics_histogram_data_get_max(
rocksdb_statistics_histogram_data_t* data);
extern ROCKSDB_LIBRARY_API uint64_t rocksdb_statistics_histogram_data_get_count(
rocksdb_statistics_histogram_data_t* data);
extern ROCKSDB_LIBRARY_API uint64_t rocksdb_statistics_histogram_data_get_sum(
rocksdb_statistics_histogram_data_t* data);
extern ROCKSDB_LIBRARY_API double rocksdb_statistics_histogram_data_get_min(
rocksdb_statistics_histogram_data_t* data);

#ifdef __cplusplus
} /* end extern "C" */
#endif
Loading