Skip to content

Remove yt specific alloc monitoring. #1202

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
dcherednik merged 1 commit into from
Jan 23, 2024
Merged

Remove yt specific alloc monitoring. #1202

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
115 changes: 0 additions & 115 deletions ydb/core/mon_alloc/monitor.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,6 @@
#include <library/cpp/malloc/api/malloc.h>
#include <library/cpp/monlib/dynamic_counters/counters.h>
#include <library/cpp/monlib/service/pages/templates.h>
#include <library/cpp/ytalloc/api/ytalloc.h>
#include <library/cpp/yt/memory/memory_tag.h>

#include <util/datetime/base.h>
#include <util/generic/hash.h>
Expand Down Expand Up @@ -261,117 +259,6 @@ namespace NKikimr {
}
};

class TYtAllocMonitor: public IAllocMonitor {
private:
TDynamicCountersPtr CounterGroup;
THashMap<TString, TDynamicCounterPtr> PerTag;

public:
TYtAllocMonitor(TDynamicCountersPtr group) {
CounterGroup = group->GetSubgroup("component", "ytalloc_profile");
}

void Update(TDuration interval) override {
Y_UNUSED(interval);
#ifdef PROFILE_MEMORY_ALLOCATIONS
using namespace NYT;

size_t maxTag = NProfiling::GetTagsCount();

TVector<TMemoryTag> tags(maxTag);
std::iota(tags.begin(), tags.end(), 1);

TVector<size_t> usages(maxTag);
GetMemoryUsageForTags(tags.data(), tags.size(), usages.data());

for (size_t tag = 0; tag < maxTag; ++tag) {
if (!usages[tag]) {
continue;
}

auto tagName = NProfiling::GetTag(tag);
if (tagName == nullptr) {
tagName = "__DEFAULT__";
}

TDynamicCounterPtr& perTag = PerTag[tagName];
if (!perTag) {
perTag = CounterGroup->GetCounter(tagName);
}

*perTag = usages[tag];
}
#endif
}

void Dump(IOutputStream& out, const TString& relPath) override {
Y_UNUSED(relPath);
#ifdef PROFILE_MEMORY_ALLOCATIONS
using namespace NYT;

size_t maxTag = NProfiling::GetTagsCount();

TVector<TMemoryTag> tags(maxTag);
std::iota(tags.begin(), tags.end(), 1);

TVector<size_t> usages(maxTag);
GetMemoryUsageForTags(tags.data(), tags.size(), usages.data());

HTML(out) {
TAG(TH3) {
out << "YTAlloc" << Endl;
}
out << "<hr>" << Endl;
TABLE_SORTABLE_CLASS("table") {
TABLEHEAD() {
TABLER() {
TABLEH() {
out << "<span data-toggle='tooltip' "
"title='Allocation Tag'>Tag</span>";
}
TABLEH() {
out << "<span data-toggle='tooltip' "
"title='Total amount of allocated RAM space'>"
"Total Space</span>";
}
}
}

TABLEBODY() {
for (size_t tag = 0; tag < maxTag; ++tag) {
if (!usages[tag]) {
continue;
}

auto tagName = NProfiling::GetTag(tag);
if (tagName == nullptr) {
tagName = "__DEFAULT__";
}

TABLER() {
TABLED() {
out << tagName;
}
TABLED() {
out << usages[tag];
}
}
}
}
}
}
#else
HTML(out) {
TAG(TH3) {
out << "YTAlloc" << Endl;
}
out << "<hr>" << Endl;
out << "PROFILE_MEMORY_ALLOCATIONS is off" << Endl;
}
#endif
}
};

struct TFakeAllocMonitor: public IAllocMonitor {
void Update(TDuration interval) override {
Y_UNUSED(interval);
Expand All @@ -390,8 +277,6 @@ namespace NKikimr {
std::unique_ptr<IAllocMonitor> monitor;
if (name.StartsWith("lf")) {
monitor = std::make_unique<TLfAllocMonitor>(std::move(group));
} else if (name.StartsWith("yt")) {
monitor = std::make_unique<TYtAllocMonitor>(std::move(group));
} else if (name.StartsWith("tc")) {
monitor = std::move(CreateTcMallocMonitor(std::move(group)));
}
Expand Down
102 changes: 0 additions & 102 deletions ydb/core/mon_alloc/profiler.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@

#if defined(PROFILE_MEMORY_ALLOCATIONS)
#include <library/cpp/lfalloc/alloc_profiler/profiler.h>
#include <library/cpp/ytalloc/api/ytalloc.h>
#endif

#if defined(_linux_) && !defined(WITH_VALGRIND)
Expand Down Expand Up @@ -58,105 +57,6 @@ namespace NActors {
}
};

class TYtAllocProfiler: public IProfilerLogic {
public:
TYtAllocProfiler() {
Init();
}

void Start() override {
SetEnabled(true);
}

void Stop(IOutputStream& out, size_t limit, bool forLog) override {
Y_UNUSED(forLog);
DumpStats(out, limit);
SetEnabled(false);
}

private:
static int BacktraceProvider(void** frames, int maxFrames, int skipFrames) {
int count = BackTrace(frames, maxFrames);
if (count > skipFrames) {
std::copy(&frames[skipFrames], &frames[count], frames);
count -= skipFrames;
}
return count;
}

static void Init() {
using namespace NYT::NYTAlloc;

SetBacktraceProvider(BacktraceProvider);
SetProfilingBacktraceDepth(MaxAllocationProfilingBacktraceDepth);

// profile every allocation of any size
SetAllocationProfilingSamplingRate(1);
SetMinProfilingBytesUsedToReport(0);
}

static void SetEnabled(bool enabled) {
using namespace NYT::NYTAlloc;

SetAllocationProfilingEnabled(enabled);

for (size_t rank = 0; rank < SmallRankCount; ++rank) {
SetSmallArenaAllocationProfilingEnabled(rank, enabled);
}

for (size_t rank = 0; rank < LargeRankCount; ++rank) {
SetLargeArenaAllocationProfilingEnabled(rank, enabled);
}
}

static void DumpStats(IOutputStream& out, size_t limit) {
using namespace NYT::NYTAlloc;

struct TLess {
static auto AsTupleRef(const TProfiledAllocation& a) {
return std::forward_as_tuple(
a.Counters[EBasicCounter::BytesUsed],
a.Counters[EBasicCounter::BytesAllocated],
a.Counters[EBasicCounter::BytesFreed]);
}

bool operator()(const TProfiledAllocation& l, const TProfiledAllocation& r) {
return AsTupleRef(r) < AsTupleRef(l);
}
};

auto allocations = GetProfiledAllocationStatistics();
Sort(allocations, TLess());

NAllocProfiler::TStats total;
for (const auto& a : allocations) {
total.Alloc(a.Counters[EBasicCounter::BytesAllocated]);
total.Free(a.Counters[EBasicCounter::BytesFreed]);
}

TAllocDumper dumper(out);
dumper.DumpTotal(total);

size_t printedCount = 0;
for (const auto& a : allocations) {
NAllocProfiler::TAllocationInfo allocInfo;
allocInfo.Tag = 0; // not supported
allocInfo.Stats.Allocs = a.Counters[EBasicCounter::BytesAllocated];
allocInfo.Stats.Frees = a.Counters[EBasicCounter::BytesFreed];
allocInfo.Stats.CurrentSize = a.Counters[EBasicCounter::BytesUsed];

allocInfo.Stack.resize(a.Backtrace.FrameCount);
for (int i = 0; i < a.Backtrace.FrameCount; ++i) {
allocInfo.Stack[i] = a.Backtrace.Frames[i];
}

dumper.DumpEntry(allocInfo);
if (++printedCount >= limit) {
break;
}
}
}
};
#endif // PROFILE_MEMORY_ALLOCATIONS

#if defined(EXEC_PROFILER_ENABLED)
Expand Down Expand Up @@ -206,8 +106,6 @@ namespace NActors {
#if defined(PROFILE_MEMORY_ALLOCATIONS)
if (name.StartsWith("lf")) {
profiler = std::make_unique<TLfAllocProfiler>();
} else if (name.StartsWith("yt")) {
profiler = std::make_unique<TYtAllocProfiler>();
}
#endif // PROFILE_MEMORY_ALLOCATIONS

Expand Down
108 changes: 0 additions & 108 deletions ydb/core/mon_alloc/stats.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@
#include <library/cpp/lfalloc/dbg_info/dbg_info.h>
#include <library/cpp/malloc/api/malloc.h>
#include <library/cpp/monlib/dynamic_counters/counters.h>
#include <library/cpp/ytalloc/api/ytalloc.h>

#include <util/datetime/base.h>

Expand Down Expand Up @@ -91,111 +90,6 @@ namespace NKikimr {
}
};

class TYtAllocStats: public IAllocStats {
struct TTimingEventCounters {
TDynamicCounterPtr Count;
TDynamicCounterPtr Size;
};

private:
TDynamicCountersPtr CounterGroup;

NYT::TEnumIndexedVector<NYT::NYTAlloc::ETotalCounter, TDynamicCounterPtr> TotalAllocationCounters;
NYT::TEnumIndexedVector<NYT::NYTAlloc::ESmallCounter, TDynamicCounterPtr> SmallAllocationCounters;
NYT::TEnumIndexedVector<NYT::NYTAlloc::ELargeCounter, TDynamicCounterPtr> LargeAllocationCounters;
std::array<NYT::TEnumIndexedVector<NYT::NYTAlloc::ESmallArenaCounter, TDynamicCounterPtr>, NYT::NYTAlloc::SmallRankCount> SmallArenaAllocationCounters;
std::array<NYT::TEnumIndexedVector<NYT::NYTAlloc::ELargeArenaCounter, TDynamicCounterPtr>, NYT::NYTAlloc::LargeRankCount> LargeArenaAllocationCounters;
NYT::TEnumIndexedVector<NYT::NYTAlloc::EHugeCounter, TDynamicCounterPtr> HugeAllocationCounters;
NYT::TEnumIndexedVector<NYT::NYTAlloc::ESystemCounter, TDynamicCounterPtr> SystemAllocationCounters;
NYT::TEnumIndexedVector<NYT::NYTAlloc::EUndumpableCounter, TDynamicCounterPtr> UndumpableAllocationCounters;
NYT::TEnumIndexedVector<NYT::NYTAlloc::ETimingEventType, TTimingEventCounters> TimingEventCounters;

public:
TYtAllocStats(TDynamicCountersPtr group) {
CounterGroup = group->GetSubgroup("component", "ytalloc");

InitCounters(TotalAllocationCounters, CounterGroup->GetSubgroup("category", "total"));
InitCounters(SmallAllocationCounters, CounterGroup->GetSubgroup("category", "small"));
InitCounters(LargeAllocationCounters, CounterGroup->GetSubgroup("category", "large"));
InitCounters(SmallArenaAllocationCounters, CounterGroup->GetSubgroup("category", "small_arena"));
InitCounters(LargeArenaAllocationCounters, CounterGroup->GetSubgroup("category", "large_arena"));
InitCounters(HugeAllocationCounters, CounterGroup->GetSubgroup("category", "huge"));
InitCounters(SystemAllocationCounters, CounterGroup->GetSubgroup("category", "system"));
InitCounters(UndumpableAllocationCounters, CounterGroup->GetSubgroup("category", "undumpable"));
InitCounters(TimingEventCounters, CounterGroup->GetSubgroup("category", "timing_event"));
}

void Update() override {
UpdateCounters(TotalAllocationCounters, NYT::NYTAlloc::GetTotalAllocationCounters());
UpdateCounters(SmallAllocationCounters, NYT::NYTAlloc::GetSmallAllocationCounters());
UpdateCounters(LargeAllocationCounters, NYT::NYTAlloc::GetLargeAllocationCounters());
UpdateCounters(SmallArenaAllocationCounters, NYT::NYTAlloc::GetSmallArenaAllocationCounters());
UpdateCounters(LargeArenaAllocationCounters, NYT::NYTAlloc::GetLargeArenaAllocationCounters());
UpdateCounters(HugeAllocationCounters, NYT::NYTAlloc::GetHugeAllocationCounters());
UpdateCounters(SystemAllocationCounters, NYT::NYTAlloc::GetSystemAllocationCounters());
UpdateCounters(UndumpableAllocationCounters, NYT::NYTAlloc::GetUndumpableAllocationCounters());
UpdateCounters(TimingEventCounters, NYT::NYTAlloc::GetTimingEventCounters());
}

private:
template <typename E>
static void InitCounters(
NYT::TEnumIndexedVector<E, TDynamicCounterPtr>& counters,
TDynamicCountersPtr group) {
for (auto c : NYT::TEnumTraits<E>::GetDomainValues()) {
counters[c] = group->GetCounter(NYT::TEnumTraits<E>::ToString(c));
}
}

template <typename E, size_t N>
static void InitCounters(
std::array<NYT::TEnumIndexedVector<E, TDynamicCounterPtr>, N>& counters,
TDynamicCountersPtr group) {
for (size_t i = 0; i < N; ++i) {
InitCounters(counters[i], group->GetSubgroup("rank", ToString(i)));
}
}

template <typename E>
static void InitCounters(
NYT::TEnumIndexedVector<E, TTimingEventCounters>& counters,
TDynamicCountersPtr group) {
for (auto c : NYT::TEnumTraits<E>::GetDomainValues()) {
const auto& name = NYT::TEnumTraits<E>::ToString(c);
counters[c].Count = group->GetCounter(name + "_Count");
counters[c].Size = group->GetCounter(name + "_Size");
}
}

template <typename E>
static void UpdateCounters(
NYT::TEnumIndexedVector<E, TDynamicCounterPtr>& counters,
const NYT::TEnumIndexedVector<E, ssize_t>& source) {
for (auto c : NYT::TEnumTraits<E>::GetDomainValues()) {
*counters[c] = source[c];
}
}

template <typename E, size_t N>
static void UpdateCounters(
std::array<NYT::TEnumIndexedVector<E, TDynamicCounterPtr>, N>& counters,
const std::array<NYT::TEnumIndexedVector<E, ssize_t>, N>& source) {
for (size_t i = 0; i < N; ++i) {
UpdateCounters(counters[i], source[i]);
}
}

template <typename E>
static void UpdateCounters(
NYT::TEnumIndexedVector<E, TTimingEventCounters>& counters,
const NYT::TEnumIndexedVector<E, NYT::NYTAlloc::TTimingEventCounters>& source) {
for (auto c : NYT::TEnumTraits<E>::GetDomainValues()) {
*counters[c].Count = source[c].Count;
*counters[c].Size = source[c].Size;
}
}
};

struct TFakeAllocStats: public IAllocStats {
void Update() override {
}
Expand All @@ -208,8 +102,6 @@ namespace NKikimr {
std::unique_ptr<IAllocStats> stats;
if (name.StartsWith("lf")) {
stats = std::make_unique<TLfAllocStats>(std::move(group));
} else if (name.StartsWith("yt")) {
stats = std::make_unique<TYtAllocStats>(std::move(group));
} else if (name.StartsWith("tc")) {
stats = std::move(CreateTcMallocStats(std::move(group)));
}
Expand Down
2 changes: 0 additions & 2 deletions ydb/core/mon_alloc/ya.make
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,6 @@ PEERDIR(
library/cpp/lfalloc/dbg_info
library/cpp/malloc/api
library/cpp/monlib/service/pages
library/cpp/ytalloc/api
library/cpp/yt/memory
ydb/core/base
ydb/core/control
ydb/library/services
Expand Down