Skip to content

Commit

Permalink
fix memory recording
Browse files Browse the repository at this point in the history
  • Loading branch information
@rainyfly
rainyfly committed Jun 17, 2022
1 parent e3f305b commit 6b3913e
Show file tree
Hide file tree
Showing 3 changed files with 56 additions and 16 deletions.
32 changes: 16 additions & 16 deletions paddle/fluid/memory/allocation/stat_allocator.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,14 +36,14 @@ class StatAllocator : public Allocator {
-allocation->size());
platform::RecordMemEvent(
allocation->ptr(), allocation->place(), allocation->size(),
DEVICE_MEMORY_STAT_CURRENT_VALUE(Allocated,
allocation->place().GetDeviceId()),
DEVICE_MEMORY_STAT_CURRENT_VALUE(Reserved,
allocation->place().GetDeviceId()),
DEVICE_MEMORY_STAT_PEAK_VALUE(Allocated,
allocation->place().GetDeviceId()),
DEVICE_MEMORY_STAT_PEAK_VALUE(Reserved,
allocation->place().GetDeviceId()),
HOST_MEMORY_STAT_CURRENT_VALUE(Allocated,
allocation->place().GetDeviceId()),
HOST_MEMORY_STAT_CURRENT_VALUE(Reserved,
allocation->place().GetDeviceId()),
HOST_MEMORY_STAT_PEAK_VALUE(Allocated,
allocation->place().GetDeviceId()),
HOST_MEMORY_STAT_PEAK_VALUE(Reserved,
allocation->place().GetDeviceId()),
platform::TracerMemEventType::Free);
} else {
DEVICE_MEMORY_STAT_UPDATE(Allocated, allocation->place().GetDeviceId(),
Expand Down Expand Up @@ -74,14 +74,14 @@ class StatAllocator : public Allocator {
allocation->size());
platform::RecordMemEvent(
allocation->ptr(), allocation->place(), allocation->size(),
DEVICE_MEMORY_STAT_CURRENT_VALUE(Allocated,
allocation->place().GetDeviceId()),
DEVICE_MEMORY_STAT_CURRENT_VALUE(Reserved,
allocation->place().GetDeviceId()),
DEVICE_MEMORY_STAT_PEAK_VALUE(Allocated,
allocation->place().GetDeviceId()),
DEVICE_MEMORY_STAT_PEAK_VALUE(Reserved,
allocation->place().GetDeviceId()),
HOST_MEMORY_STAT_CURRENT_VALUE(Allocated,
allocation->place().GetDeviceId()),
HOST_MEMORY_STAT_CURRENT_VALUE(Reserved,
allocation->place().GetDeviceId()),
HOST_MEMORY_STAT_PEAK_VALUE(Allocated,
allocation->place().GetDeviceId()),
HOST_MEMORY_STAT_PEAK_VALUE(Reserved,
allocation->place().GetDeviceId()),
platform::TracerMemEventType::Allocate);
} else {
DEVICE_MEMORY_STAT_UPDATE(Allocated, place.GetDeviceId(),
Expand Down
25 changes: 25 additions & 0 deletions paddle/fluid/memory/detail/system_allocator.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ limitations under the License. */
#endif

#include "paddle/fluid/platform/device/device_wrapper.h"
#include "paddle/fluid/platform/profiler/supplement_tracing.h"

DECLARE_bool(use_pinned_memory);
DECLARE_double(fraction_of_gpu_memory_to_use);
Expand Down Expand Up @@ -95,6 +96,12 @@ void* CPUAllocator::Alloc(size_t* index, size_t size) {
}

HOST_MEMORY_STAT_UPDATE(Reserved, 0, size);
platform::RecordMemEvent(p, CPUPlace(), size,
HOST_MEMORY_STAT_CURRENT_VALUE(Allocated, 0),
HOST_MEMORY_STAT_CURRENT_VALUE(Reserved, 0),
HOST_MEMORY_STAT_PEAK_VALUE(Allocated, 0),
HOST_MEMORY_STAT_PEAK_VALUE(Reserved, 0),
platform::TracerMemEventType::Allocate);

return p;
}
Expand All @@ -114,6 +121,12 @@ void CPUAllocator::Free(void* p, size_t size, size_t index) {
#endif

HOST_MEMORY_STAT_UPDATE(Reserved, 0, -size);
platform::RecordMemEvent(p, CPUPlace(), size,
HOST_MEMORY_STAT_CURRENT_VALUE(Allocated, 0),
HOST_MEMORY_STAT_CURRENT_VALUE(Reserved, 0),
HOST_MEMORY_STAT_PEAK_VALUE(Allocated, 0),
HOST_MEMORY_STAT_PEAK_VALUE(Reserved, 0),
platform::TracerMemEventType::Free);
}

bool CPUAllocator::UseGpu() const { return false; }
Expand Down Expand Up @@ -213,6 +226,12 @@ void* CUDAPinnedAllocator::Alloc(size_t* index, size_t size) {
*index = 1; // PINNED memory
cuda_pinnd_alloc_size_ += size;
HOST_MEMORY_STAT_UPDATE(Reserved, 0, size);
platform::RecordMemEvent(p, CPUPlace(), size,
HOST_MEMORY_STAT_CURRENT_VALUE(Allocated, 0),
HOST_MEMORY_STAT_CURRENT_VALUE(Reserved, 0),
HOST_MEMORY_STAT_PEAK_VALUE(Allocated, 0),
HOST_MEMORY_STAT_PEAK_VALUE(Reserved, 0),
platform::TracerMemEventType::Allocate);
return p;
} else {
LOG(WARNING) << "cudaHostAlloc failed.";
Expand Down Expand Up @@ -259,6 +278,12 @@ void CUDAPinnedAllocator::Free(void* p, size_t size, size_t index) {
}
#endif
HOST_MEMORY_STAT_UPDATE(Reserved, 0, -size);
platform::RecordMemEvent(p, CPUPlace(), size,
HOST_MEMORY_STAT_CURRENT_VALUE(Allocated, 0),
HOST_MEMORY_STAT_CURRENT_VALUE(Reserved, 0),
HOST_MEMORY_STAT_PEAK_VALUE(Allocated, 0),
HOST_MEMORY_STAT_PEAK_VALUE(Reserved, 0),
platform::TracerMemEventType::Free);
}

bool CUDAPinnedAllocator::UseGpu() const { return false; }
Expand Down
15 changes: 15 additions & 0 deletions paddle/fluid/platform/device/gpu/gpu_info.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ limitations under the License. */
#include "paddle/fluid/platform/macros.h"
#include "paddle/fluid/platform/monitor.h"
#include "paddle/fluid/platform/place.h"
#include "paddle/fluid/platform/profiler/supplement_tracing.h"
#include "paddle/fluid/string/split.h"
#include "paddle/phi/backends/gpu/gpu_info.h"

Expand Down Expand Up @@ -236,6 +237,13 @@ class RecordedGpuMallocHelper {
cur_size_.fetch_add(size);
STAT_INT_ADD("STAT_gpu" + std::to_string(dev_id_) + "_mem_size", size);
DEVICE_MEMORY_STAT_UPDATE(Reserved, dev_id_, size);
platform::RecordMemEvent(
ptr, GPUPlace(dev_id_), size,
DEVICE_MEMORY_STAT_CURRENT_VALUE(Allocated, dev_id_),
DEVICE_MEMORY_STAT_CURRENT_VALUE(Reserved, dev_id_),
DEVICE_MEMORY_STAT_PEAK_VALUE(Allocated, dev_id_),
DEVICE_MEMORY_STAT_PEAK_VALUE(Reserved, dev_id_),
platform::TracerMemEventType::ReservedAllocate);

#ifdef PADDLE_WITH_TESTING
gpu_ptrs.insert(*ptr);
Expand Down Expand Up @@ -275,6 +283,13 @@ class RecordedGpuMallocHelper {
cur_size_.fetch_sub(size);
STAT_INT_SUB("STAT_gpu" + std::to_string(dev_id_) + "_mem_size", size);
DEVICE_MEMORY_STAT_UPDATE(Reserved, dev_id_, -size);
platform::RecordMemEvent(
ptr, GPUPlace(dev_id_), size,
DEVICE_MEMORY_STAT_CURRENT_VALUE(Allocated, dev_id_),
DEVICE_MEMORY_STAT_CURRENT_VALUE(Reserved, dev_id_),
DEVICE_MEMORY_STAT_PEAK_VALUE(Allocated, dev_id_),
DEVICE_MEMORY_STAT_PEAK_VALUE(Reserved, dev_id_),
platform::TracerMemEventType::ReservedFree);
} else {
platform::GpuGetLastError(); // clear the error flag when
// cudaErrorCudartUnloading /
Expand Down

0 comments on commit 6b3913e

Please sign in to comment.