diff --git a/db/c.cc b/db/c.cc index 9864fd3f63c..679f49e8577 100644 --- a/db/c.cc +++ b/db/c.cc @@ -2635,12 +2635,11 @@ void rocksdb_options_set_memtable_huge_page_size(rocksdb_options_t* opt, opt->rep.memtable_huge_page_size = v; } -void rocksdb_options_set_doubly_skip_list_rep(rocksdb_options_t *opt) { +void rocksdb_options_set_doubly_skip_list_rep(rocksdb_options_t* opt) { rocksdb::MemTableRepFactory* factory = new rocksdb::DoublySkipListFactory(); opt->rep.memtable_factory.reset(factory); } - void rocksdb_options_set_hash_skip_list_rep( rocksdb_options_t *opt, size_t bucket_count, int32_t skiplist_height, int32_t skiplist_branching_factor) { diff --git a/include/rocksdb/statistics.h b/include/rocksdb/statistics.h index a8d01e03415..34893111933 100644 --- a/include/rocksdb/statistics.h +++ b/include/rocksdb/statistics.h @@ -539,6 +539,8 @@ class Statistics { }; // Create a concrete DBStatistics object +template std::shared_ptr CreateDBStatistics(); } // namespace rocksdb diff --git a/java/rocksjni/statisticsjni.h b/java/rocksjni/statisticsjni.h index 56186789a9b..0969f839572 100644 --- a/java/rocksjni/statisticsjni.h +++ b/java/rocksjni/statisticsjni.h @@ -14,10 +14,11 @@ #include #include "rocksdb/statistics.h" #include "monitoring/statistics.h" +#include "monitoring/statistics_impl.h" namespace rocksdb { - class StatisticsJni : public StatisticsImpl { + class StatisticsJni : public StatisticsImpl<> { public: StatisticsJni(std::shared_ptr stats); StatisticsJni(std::shared_ptr stats, diff --git a/monitoring/statistics.cc b/monitoring/statistics.cc index 15d702d1f4a..f7b117b6f88 100644 --- a/monitoring/statistics.cc +++ b/monitoring/statistics.cc @@ -3,17 +3,16 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). // -#include "monitoring/statistics.h" -#include #include "rocksdb/statistics.h" -#include "port/likely.h" -#include -#include + +#include + +#include "monitoring/statistics_impl.h" namespace rocksdb { -// The order of items listed in Tickers should be the same as +// The order of items listed in Tickers should be the same as // the order listed in TickersNameMap const std::vector> TickersNameMap = { {BLOCK_CACHE_MISS, "rocksdb.block.cache.miss"}, @@ -228,178 +227,4 @@ const std::vector> HistogramsNameMap = { {SST_BATCH_SIZE, "rocksdb.sst.batch.size"}, }; -std::shared_ptr CreateDBStatistics() { - return std::make_shared(nullptr); -} - -StatisticsImpl::StatisticsImpl(std::shared_ptr stats) - : stats_(std::move(stats)) {} - -StatisticsImpl::~StatisticsImpl() {} - -uint64_t StatisticsImpl::getTickerCount(uint32_t tickerType) const { - MutexLock lock(&aggregate_lock_); - return getTickerCountLocked(tickerType); -} - -uint64_t StatisticsImpl::getTickerCountLocked(uint32_t tickerType) const { - assert(tickerType < TICKER_ENUM_MAX); - uint64_t res = 0; - for (size_t core_idx = 0; core_idx < per_core_stats_.Size(); ++core_idx) { - res += per_core_stats_.AccessAtCore(core_idx)->tickers_[tickerType]; - } - return res; -} - -void StatisticsImpl::histogramData(uint32_t histogramType, - HistogramData* const data) const { - MutexLock lock(&aggregate_lock_); - getHistogramImplLocked(histogramType)->Data(data); -} - -std::unique_ptr StatisticsImpl::getHistogramImplLocked( - uint32_t histogramType) const { - assert(histogramType < HISTOGRAM_ENUM_MAX); - std::unique_ptr res_hist(new HistogramImpl()); - for (size_t core_idx = 0; core_idx < per_core_stats_.Size(); ++core_idx) { - res_hist->Merge( - per_core_stats_.AccessAtCore(core_idx)->histograms_[histogramType]); - } - return res_hist; -} - -std::string StatisticsImpl::getHistogramString(uint32_t histogramType) const { - MutexLock lock(&aggregate_lock_); - return getHistogramImplLocked(histogramType)->ToString(); -} - -void StatisticsImpl::setTickerCount(uint32_t tickerType, uint64_t count) { - { - MutexLock lock(&aggregate_lock_); - setTickerCountLocked(tickerType, count); - } - if (stats_ && tickerType < TICKER_ENUM_MAX) { - stats_->setTickerCount(tickerType, count); - } -} - -void StatisticsImpl::setTickerCountLocked(uint32_t tickerType, uint64_t count) { - assert(tickerType < TICKER_ENUM_MAX); - for (size_t core_idx = 0; core_idx < per_core_stats_.Size(); ++core_idx) { - if (core_idx == 0) { - per_core_stats_.AccessAtCore(core_idx)->tickers_[tickerType] = count; - } else { - per_core_stats_.AccessAtCore(core_idx)->tickers_[tickerType] = 0; - } - } -} - -uint64_t StatisticsImpl::getAndResetTickerCount(uint32_t tickerType) { - uint64_t sum = 0; - { - MutexLock lock(&aggregate_lock_); - assert(tickerType < TICKER_ENUM_MAX); - for (size_t core_idx = 0; core_idx < per_core_stats_.Size(); ++core_idx) { - sum += - per_core_stats_.AccessAtCore(core_idx)->tickers_[tickerType].exchange( - 0, std::memory_order_relaxed); - } - } - if (stats_ && tickerType < TICKER_ENUM_MAX) { - stats_->setTickerCount(tickerType, 0); - } - return sum; -} - -void StatisticsImpl::recordTick(uint32_t tickerType, uint64_t count) { - assert(tickerType < TICKER_ENUM_MAX); - per_core_stats_.Access()->tickers_[tickerType].fetch_add( - count, std::memory_order_relaxed); - if (stats_ && tickerType < TICKER_ENUM_MAX) { - stats_->recordTick(tickerType, count); - } -} - -void StatisticsImpl::recordInHistogram(uint32_t histogramType, uint64_t value) { - assert(histogramType < HISTOGRAM_ENUM_MAX); - if (get_stats_level() <= StatsLevel::kExceptHistogramOrTimers) { - return; - } - per_core_stats_.Access()->histograms_[histogramType].Add(value); - if (stats_ && histogramType < HISTOGRAM_ENUM_MAX) { - stats_->recordInHistogram(histogramType, value); - } -} - -Status StatisticsImpl::Reset() { - MutexLock lock(&aggregate_lock_); - for (uint32_t i = 0; i < TICKER_ENUM_MAX; ++i) { - setTickerCountLocked(i, 0); - } - for (uint32_t i = 0; i < HISTOGRAM_ENUM_MAX; ++i) { - for (size_t core_idx = 0; core_idx < per_core_stats_.Size(); ++core_idx) { - per_core_stats_.AccessAtCore(core_idx)->histograms_[i].Clear(); - } - } - return Status::OK(); -} - -namespace { - -// a buffer size used for temp string buffers -const int kTmpStrBufferSize = 200; - -} // namespace - -std::string StatisticsImpl::ToString() const { - MutexLock lock(&aggregate_lock_); - std::string res; - res.reserve(20000); - for (const auto& t : TickersNameMap) { - assert(t.first < TICKER_ENUM_MAX); - char buffer[kTmpStrBufferSize]; - snprintf(buffer, kTmpStrBufferSize, "%s COUNT : %" PRIu64 "\n", - t.second.c_str(), getTickerCountLocked(t.first)); - res.append(buffer); - } - for (const auto& h : HistogramsNameMap) { - assert(h.first < HISTOGRAM_ENUM_MAX); - char buffer[kTmpStrBufferSize]; - HistogramData hData; - getHistogramImplLocked(h.first)->Data(&hData); - // don't handle failures - buffer should always be big enough and arguments - // should be provided correctly - int ret = - snprintf(buffer, kTmpStrBufferSize, - "%s P50 : %f P95 : %f P99 : %f P100 : %f COUNT : %" PRIu64 - " SUM : %" PRIu64 "\n", - h.second.c_str(), hData.median, hData.percentile95, - hData.percentile99, hData.max, hData.count, hData.sum); - if (ret < 0 || ret >= kTmpStrBufferSize) { - assert(false); - continue; - } - res.append(buffer); - } - res.shrink_to_fit(); - return res; -} - -bool StatisticsImpl::getTickerMap( - std::map* stats_map) const { - assert(stats_map); - if (!stats_map) return false; - stats_map->clear(); - MutexLock lock(&aggregate_lock_); - for (const auto& t : TickersNameMap) { - assert(t.first < TICKER_ENUM_MAX); - (*stats_map)[t.second.c_str()] = getTickerCountLocked(t.first); - } - return true; -} - -bool StatisticsImpl::HistEnabledForType(uint32_t type) const { - return type < HISTOGRAM_ENUM_MAX; -} - } // namespace rocksdb diff --git a/monitoring/statistics.h b/monitoring/statistics.h index 952bf8cb413..49ced896e8a 100644 --- a/monitoring/statistics.h +++ b/monitoring/statistics.h @@ -4,108 +4,14 @@ // (found in the LICENSE.Apache file in the root directory). // #pragma once -#include "rocksdb/statistics.h" - -#include -#include -#include -#include +#include #include "monitoring/histogram.h" #include "port/likely.h" -#include "port/port.h" -#include "util/core_local.h" -#include "util/mutexlock.h" - -#ifdef __clang__ -#define ROCKSDB_FIELD_UNUSED __attribute__((__unused__)) -#else -#define ROCKSDB_FIELD_UNUSED -#endif // __clang__ - -#ifndef STRINGIFY -#define STRINGIFY(x) #x -#define TOSTRING(x) STRINGIFY(x) -#endif +#include "rocksdb/statistics.h" namespace rocksdb { -enum TickersInternal : uint32_t { - INTERNAL_TICKER_ENUM_START = TICKER_ENUM_MAX, - INTERNAL_TICKER_ENUM_MAX -}; - -enum HistogramsInternal : uint32_t { - INTERNAL_HISTOGRAM_START = HISTOGRAM_ENUM_MAX, - INTERNAL_HISTOGRAM_ENUM_MAX -}; - -class StatisticsImpl : public Statistics { - public: - StatisticsImpl(std::shared_ptr stats); - virtual ~StatisticsImpl(); - - virtual uint64_t getTickerCount(uint32_t ticker_type) const override; - virtual void histogramData(uint32_t histogram_type, - HistogramData* const data) const override; - std::string getHistogramString(uint32_t histogram_type) const override; - - virtual void setTickerCount(uint32_t ticker_type, uint64_t count) override; - virtual uint64_t getAndResetTickerCount(uint32_t ticker_type) override; - virtual void recordTick(uint32_t ticker_type, uint64_t count) override; - // The function is implemented for now for backward compatibility reason. - // In case a user explictly calls it, for example, they may have a wrapped - // Statistics object, passing the call to recordTick() into here, nothing - // will break. - void measureTime(uint32_t histogramType, uint64_t time) override { - recordInHistogram(histogramType, time); - } - virtual void recordInHistogram(uint32_t histogram_type, - uint64_t value) override; - - virtual Status Reset() override; - virtual std::string ToString() const override; - virtual bool getTickerMap(std::map*) const override; - virtual bool HistEnabledForType(uint32_t type) const override; - - private: - // If non-nullptr, forwards updates to the object pointed to by `stats_`. - std::shared_ptr stats_; - // Synchronizes anything that operates across other cores' local data, - // such that operations like Reset() can be performed atomically. - mutable port::Mutex aggregate_lock_; - - // The ticker/histogram data are stored in this structure, which we will store - // per-core. It is cache-aligned, so tickers/histograms belonging to different - // cores can never share the same cache line. - // - // Alignment attributes expand to nothing depending on the platform - struct ALIGN_AS(CACHE_LINE_SIZE) StatisticsData { - std::atomic_uint_fast64_t tickers_[INTERNAL_TICKER_ENUM_MAX] = {{0}}; - HistogramImpl histograms_[INTERNAL_HISTOGRAM_ENUM_MAX]; -#ifndef HAVE_ALIGNED_NEW - char - padding[(CACHE_LINE_SIZE - - (INTERNAL_TICKER_ENUM_MAX * sizeof(std::atomic_uint_fast64_t) + - INTERNAL_HISTOGRAM_ENUM_MAX * sizeof(HistogramImpl)) % - CACHE_LINE_SIZE)] ROCKSDB_FIELD_UNUSED; -#endif - void *operator new(size_t s) { return port::cacheline_aligned_alloc(s); } - void *operator new[](size_t s) { return port::cacheline_aligned_alloc(s); } - void operator delete(void *p) { port::cacheline_aligned_free(p); } - void operator delete[](void *p) { port::cacheline_aligned_free(p); } - }; - - static_assert(sizeof(StatisticsData) % CACHE_LINE_SIZE == 0, "Expected " TOSTRING(CACHE_LINE_SIZE) "-byte aligned"); - - CoreLocalArray per_core_stats_; - - uint64_t getTickerCountLocked(uint32_t ticker_type) const; - std::unique_ptr getHistogramImplLocked( - uint32_t histogram_type) const; - void setTickerCountLocked(uint32_t ticker_type, uint64_t count); -}; - // Utility functions inline void RecordInHistogram(Statistics* statistics, uint32_t histogram_type, uint64_t value) { @@ -135,4 +41,4 @@ inline void SetTickerCount(Statistics* statistics, uint32_t ticker_type, } } -} +} // namespace rocksdb diff --git a/monitoring/statistics_impl.h b/monitoring/statistics_impl.h new file mode 100644 index 00000000000..422cc356b3f --- /dev/null +++ b/monitoring/statistics_impl.h @@ -0,0 +1,308 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +#pragma once +#include "rocksdb/statistics.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "monitoring/histogram.h" +#include "port/likely.h" +#include "port/port.h" +#include "util/core_local.h" +#include "util/mutexlock.h" + +#ifdef __clang__ +#define ROCKSDB_FIELD_UNUSED __attribute__((__unused__)) +#else +#define ROCKSDB_FIELD_UNUSED +#endif // __clang__ + +#ifndef STRINGIFY +#define STRINGIFY(x) #x +#define TOSTRING(x) STRINGIFY(x) +#endif + +namespace rocksdb { + +template +class StatisticsImpl : public Statistics { + public: + StatisticsImpl(std::shared_ptr stats); + virtual ~StatisticsImpl(); + + virtual uint64_t getTickerCount(uint32_t ticker_type) const override; + virtual void histogramData(uint32_t histogram_type, + HistogramData* const data) const override; + std::string getHistogramString(uint32_t histogram_type) const override; + + virtual void setTickerCount(uint32_t ticker_type, uint64_t count) override; + virtual uint64_t getAndResetTickerCount(uint32_t ticker_type) override; + virtual void recordTick(uint32_t ticker_type, uint64_t count) override; + // The function is implemented for now for backward compatibility reason. + // In case a user explicitly calls it, for example, they may have a wrapped + // Statistics object, passing the call to recordTick() into here, nothing + // will break. + void measureTime(uint32_t histogramType, uint64_t time) override { + recordInHistogram(histogramType, time); + } + virtual void recordInHistogram(uint32_t histogram_type, + uint64_t value) override; + + virtual Status Reset() override; + virtual std::string ToString() const override; + virtual bool getTickerMap(std::map*) const override; + virtual bool HistEnabledForType(uint32_t type) const override; + + private: + // If non-nullptr, forwards updates to the object pointed to by `stats_`. + std::shared_ptr stats_; + // Synchronizes anything that operates across other cores' local data, + // such that operations like Reset() can be performed atomically. + mutable port::Mutex aggregate_lock_; + + // The ticker/histogram data are stored in this structure, which we will store + // per-core. It is cache-aligned, so tickers/histograms belonging to different + // cores can never share the same cache line. + // + // Alignment attributes expand to nothing depending on the platform + struct ALIGN_AS(CACHE_LINE_SIZE) StatisticsData { + StatisticsData() : tickers_{{0}} {}; + std::atomic_uint_fast64_t tickers_[TICKER_MAX]; + HistogramImpl histograms_[HISTOGRAM_MAX]; +#ifndef HAVE_ALIGNED_NEW + char padding[(CACHE_LINE_SIZE - + (TICKER_MAX * sizeof(std::atomic_uint_fast64_t) + + HISTOGRAM_MAX * sizeof(HistogramImpl)) % + CACHE_LINE_SIZE)] ROCKSDB_FIELD_UNUSED; +#endif + void *operator new(size_t s) { return port::cacheline_aligned_alloc(s); } + void *operator new[](size_t s) { return port::cacheline_aligned_alloc(s); } + void operator delete(void *p) { port::cacheline_aligned_free(p); } + void operator delete[](void *p) { port::cacheline_aligned_free(p); } + }; + + static_assert(sizeof(StatisticsData) % CACHE_LINE_SIZE == 0, "Expected " TOSTRING(CACHE_LINE_SIZE) "-byte aligned"); + + CoreLocalArray per_core_stats_; + + uint64_t getTickerCountLocked(uint32_t ticker_type) const; + std::unique_ptr getHistogramImplLocked( + uint32_t histogram_type) const; + void setTickerCountLocked(uint32_t ticker_type, uint64_t count); +}; + +template +StatisticsImpl::StatisticsImpl( + std::shared_ptr stats) + : stats_(std::move(stats)) {} + +template +StatisticsImpl::~StatisticsImpl() {} + +template +uint64_t StatisticsImpl::getTickerCount( + uint32_t tickerType) const { + MutexLock lock(&aggregate_lock_); + return getTickerCountLocked(tickerType); +} + +template +uint64_t StatisticsImpl::getTickerCountLocked( + uint32_t tickerType) const { + assert(tickerType < TICKER_MAX); + uint64_t res = 0; + for (size_t core_idx = 0; core_idx < per_core_stats_.Size(); ++core_idx) { + res += per_core_stats_.AccessAtCore(core_idx)->tickers_[tickerType]; + } + return res; +} + +template +void StatisticsImpl::histogramData( + uint32_t histogramType, HistogramData* const data) const { + MutexLock lock(&aggregate_lock_); + getHistogramImplLocked(histogramType)->Data(data); +} + +template +std::unique_ptr +StatisticsImpl::getHistogramImplLocked( + uint32_t histogramType) const { + assert(histogramType < HISTOGRAM_MAX); + std::unique_ptr res_hist(new HistogramImpl()); + for (size_t core_idx = 0; core_idx < per_core_stats_.Size(); ++core_idx) { + res_hist->Merge( + per_core_stats_.AccessAtCore(core_idx)->histograms_[histogramType]); + } + return res_hist; +} + +template +std::string StatisticsImpl::getHistogramString( + uint32_t histogramType) const { + MutexLock lock(&aggregate_lock_); + return getHistogramImplLocked(histogramType)->ToString(); +} + +template +void StatisticsImpl::setTickerCount( + uint32_t tickerType, uint64_t count) { + { + MutexLock lock(&aggregate_lock_); + setTickerCountLocked(tickerType, count); + } + if (stats_ && tickerType < TICKER_MAX) { + stats_->setTickerCount(tickerType, count); + } +} + +template +void StatisticsImpl::setTickerCountLocked( + uint32_t tickerType, uint64_t count) { + assert(tickerType < TICKER_MAX); + for (size_t core_idx = 0; core_idx < per_core_stats_.Size(); ++core_idx) { + if (core_idx == 0) { + per_core_stats_.AccessAtCore(core_idx)->tickers_[tickerType] = count; + } else { + per_core_stats_.AccessAtCore(core_idx)->tickers_[tickerType] = 0; + } + } +} + +template +uint64_t StatisticsImpl::getAndResetTickerCount( + uint32_t tickerType) { + uint64_t sum = 0; + { + MutexLock lock(&aggregate_lock_); + assert(tickerType < TICKER_MAX); + for (size_t core_idx = 0; core_idx < per_core_stats_.Size(); ++core_idx) { + sum += + per_core_stats_.AccessAtCore(core_idx)->tickers_[tickerType].exchange( + 0, std::memory_order_relaxed); + } + } + if (stats_ && tickerType < TICKER_MAX) { + stats_->setTickerCount(tickerType, 0); + } + return sum; +} + +template +void StatisticsImpl::recordTick(uint32_t tickerType, + uint64_t count) { + assert(tickerType < TICKER_MAX); + per_core_stats_.Access()->tickers_[tickerType].fetch_add( + count, std::memory_order_relaxed); + if (stats_ && tickerType < TICKER_MAX) { + stats_->recordTick(tickerType, count); + } +} + +template +void StatisticsImpl::recordInHistogram( + uint32_t histogramType, uint64_t value) { + assert(histogramType < HISTOGRAM_MAX); + if (get_stats_level() <= StatsLevel::kExceptHistogramOrTimers) { + return; + } + per_core_stats_.Access()->histograms_[histogramType].Add(value); + if (stats_ && histogramType < HISTOGRAM_MAX) { + stats_->recordInHistogram(histogramType, value); + } +} + +template +Status StatisticsImpl::Reset() { + MutexLock lock(&aggregate_lock_); + for (uint32_t i = 0; i < TICKER_MAX; ++i) { + setTickerCountLocked(i, 0); + } + for (uint32_t i = 0; i < HISTOGRAM_MAX; ++i) { + for (size_t core_idx = 0; core_idx < per_core_stats_.Size(); ++core_idx) { + per_core_stats_.AccessAtCore(core_idx)->histograms_[i].Clear(); + } + } + return Status::OK(); +} + +namespace { + +// a buffer size used for temp string buffers +const int kTmpStrBufferSize = 200; + +} // namespace + +template +std::string StatisticsImpl::ToString() const { + MutexLock lock(&aggregate_lock_); + std::string res; + res.reserve(20000); + for (const auto& t : TickersNameMap) { + assert(t.first < TICKER_MAX); + char buffer[kTmpStrBufferSize]; + snprintf(buffer, kTmpStrBufferSize, "%s COUNT : %" PRIu64 "\n", + t.second.c_str(), getTickerCountLocked(t.first)); + res.append(buffer); + } + for (const auto& h : HistogramsNameMap) { + assert(h.first < HISTOGRAM_MAX); + char buffer[kTmpStrBufferSize]; + HistogramData hData; + getHistogramImplLocked(h.first)->Data(&hData); + // don't handle failures - buffer should always be big enough and arguments + // should be provided correctly + int ret = + snprintf(buffer, kTmpStrBufferSize, + "%s P50 : %f P95 : %f P99 : %f P100 : %f COUNT : %" PRIu64 + " SUM : %" PRIu64 "\n", + h.second.c_str(), hData.median, hData.percentile95, + hData.percentile99, hData.max, hData.count, hData.sum); + if (ret < 0 || ret >= kTmpStrBufferSize) { + assert(false); + continue; + } + res.append(buffer); + } + res.shrink_to_fit(); + return res; +} + +template +bool StatisticsImpl::getTickerMap( + std::map* stats_map) const { + assert(stats_map); + if (!stats_map) return false; + stats_map->clear(); + MutexLock lock(&aggregate_lock_); + for (const auto& t : TickersNameMap) { + assert(t.first < TICKER_MAX); + (*stats_map)[t.second.c_str()] = getTickerCountLocked(t.first); + } + return true; +} + +template +bool StatisticsImpl::HistEnabledForType( + uint32_t type) const { + return type < HISTOGRAM_MAX; +} + +template +std::shared_ptr CreateDBStatistics() { + return std::make_shared>(nullptr); +} +// Explicitly instantiate templates to make it possible to keep the template definitions in this file. +template std::shared_ptr CreateDBStatistics(); + +} \ No newline at end of file