diff --git a/mooncake-transfer-engine/src/transport/tcp_transport/tcp_transport.cpp b/mooncake-transfer-engine/src/transport/tcp_transport/tcp_transport.cpp
index 336b63237..71c2176ed 100644
--- a/mooncake-transfer-engine/src/transport/tcp_transport/tcp_transport.cpp
+++ b/mooncake-transfer-engine/src/transport/tcp_transport/tcp_transport.cpp
@@ -160,7 +160,9 @@ struct Session : public std::enable_shared_from_this<Session> {
             [this, addr, dram_buffer, self](const asio::error_code &ec,
                                             std::size_t transferred_bytes) {
 #ifdef USE_CUDA
-                delete[] dram_buffer;
+                if (isCudaMemory(addr)) {
+                    delete[] dram_buffer;
+                }
 #endif
                 if (ec) {
                     LOG(ERROR)