Skip to content

Commit

Permalink
add RingBuffer class; used by HexUserDMA to store descriptors
Browse files Browse the repository at this point in the history
  • Loading branch information
adstraw committed Aug 25, 2022
1 parent c5ac108 commit bd1f7d5
Show file tree
Hide file tree
Showing 4 changed files with 234 additions and 82 deletions.
73 changes: 16 additions & 57 deletions src/runtime/hexagon/hexagon_user_dma.cc
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,6 @@

#include <algorithm>

#include "hexagon_common.h"
#include "hexagon_user_dma_descriptors.h"
#include "hexagon_user_dma_instructions.h"
#include "hexagon_user_dma_registers.h"

namespace tvm {
namespace runtime {
namespace hexagon {
Expand Down Expand Up @@ -56,26 +51,12 @@ int HexagonUserDMA::Copy(void* dst, void* src, uint32_t length) {
uint32_t src32 = static_cast<uint32_t>(src64);
uint32_t dst32 = static_cast<uint32_t>(dst64);

// check if the next DMA descriptor will overwrite an in flight DMA descriptor
// if this is the first DMA there is nothting to check
if (!first_dma_) {
// update the ID of the oldest DMA descriptor in flight
DMAsInFlight();
// calcultate whether there are DMA descriptors in flight
bool dma_desc_in_flight = id_next_dma_desc_ != id_oldest_dma_desc_in_flight_;
// calculate whether the next DMA descriptor will overwrite the oldest DMA descriptor in flight
bool same_ring_buff_index = (id_next_dma_desc_ % dma_desc_ring_buff_size_) ==
(id_oldest_dma_desc_in_flight_ % dma_desc_ring_buff_size_);
// fail if there are DMA descriptors in flight
// and the next DMA descriptor overwrites the oldest DMA descriptor in flight
if (dma_desc_in_flight && same_ring_buff_index) {
return DMA_FAILURE;
}
// get pointer to next descriptor
dma_desc_2d_t* dma_desc = descriptors_->Next();
if (!dma_desc) {
return DMA_FAILURE;
}

// get pointer to next DMA descriptor
void* dma_desc = GetDescriptorAddr(id_next_dma_desc_);

// populate descriptor fields
dma_desc_set_state(dma_desc, DESC_STATE_READY);
dma_desc_set_next(dma_desc, DMA_NULL_PTR);
Expand All @@ -96,13 +77,11 @@ int HexagonUserDMA::Copy(void* dst, void* src, uint32_t length) {
first_dma_ = false;
} else {
// `dmlink` descriptor to tail descriptor
void* tail = GetDescriptorAddr(id_next_dma_desc_ - 1);
dmlink(tail, dma_desc);
dmlink(tail_dma_desc_, dma_desc);
}

// update the ID of the next DMA descriptor
id_next_dma_desc_++;

// update tail
tail_dma_desc_ = dma_desc;
return DMA_SUCCESS;
}

Expand All @@ -115,45 +94,25 @@ void HexagonUserDMA::Wait(uint32_t max_dmas_in_flight) {
uint32_t HexagonUserDMA::Poll() { return DMAsInFlight(); }

uint32_t HexagonUserDMA::DMAsInFlight() {
// poll DMA engine to make sure DMA status is current
dmpoll();

// find the oldest DMA descriptor in flight
// total number of DMA descriptors in flight == ID of the next DMA descriptor
for (; id_oldest_dma_desc_in_flight_ < id_next_dma_desc_; ++id_oldest_dma_desc_in_flight_) {
// read the `done` bit from the DMA descriptor and stop if incomplete
unsigned int done = dma_desc_get_done(GetDescriptorAddr(id_oldest_dma_desc_in_flight_));
if (done == DESC_DONE_INCOMPLETE) {
break;
}
}

// total DMA descriptors in flight = total number DMA desc - ID of the oldest DMA desc in flight
// note that these two IDs are equivalent when no DMA descriptors are in flight
return id_next_dma_desc_ - id_oldest_dma_desc_in_flight_;
}

void* HexagonUserDMA::GetDescriptorAddr(uint32_t dma_desc_id) {
return static_cast<char*>(dma_desc_ring_buff_) +
DMA_DESC_2D_SIZE * (dma_desc_id % dma_desc_ring_buff_size_);
dmpoll(); // update DMA engine status
return descriptors_->InFlight();
}

HexagonUserDMA::HexagonUserDMA() {
// reset DMA engine
unsigned int status = Init();
CHECK_EQ(status, DM0_STATUS_IDLE);

// allocate memory for ring buffer storage for all DMA descriptors
int ret = posix_memalign(&dma_desc_ring_buff_, DMA_DESC_2D_SIZE,
DMA_DESC_2D_SIZE * dma_desc_ring_buff_size_);
CHECK_EQ(ret, 0);
CHECK_NE(dma_desc_ring_buff_, nullptr);
auto desc_in_flight = [](dma_desc_2d_t* dma_desc) {
unsigned int done = dma_desc_get_done(dma_desc);
return (done != DESC_DONE_COMPLETE);
};
descriptors_ = new RingBuffer<dma_desc_2d_t>(100, desc_in_flight);
}

HexagonUserDMA::~HexagonUserDMA() {
// stop the DMA engine
Init();
free(dma_desc_ring_buff_);
Init(); // stop DMA engine
delete descriptors_;
}

int hexagon_user_dma_1d_sync(void* dst, void* src, uint32_t length) {
Expand Down
40 changes: 15 additions & 25 deletions src/runtime/hexagon/hexagon_user_dma.h
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,11 @@
#ifndef TVM_RUNTIME_HEXAGON_HEXAGON_USER_DMA_H_
#define TVM_RUNTIME_HEXAGON_HEXAGON_USER_DMA_H_

#include <stdint.h>
#include "hexagon_common.h"
#include "hexagon_user_dma_descriptors.h"
#include "hexagon_user_dma_instructions.h"
#include "hexagon_user_dma_registers.h"
#include "ring_buffer.h"

namespace tvm {
namespace runtime {
Expand All @@ -36,7 +40,7 @@ class HexagonUserDMA {
* \param dst Destination address
* \param src Source address
* \param length Length in bytes to copy
* \returns Status, either DMA_SUCCESS or DMA_FAILURE
* \returns Status: DMA_SUCCESS or DMA_FAILURE
*/
int Copy(void* dst, void* src, uint32_t length);

Expand All @@ -53,13 +57,14 @@ class HexagonUserDMA {
*/
uint32_t Poll();

//! HexagonUserDMA uses the singleton pattern
//! \brief HexagonUserDMA uses the singleton pattern
static HexagonUserDMA& Get() {
static HexagonUserDMA* hud = new HexagonUserDMA();
return *hud;
}

private:
// HexagonUserDMA uses the singleton pattern
HexagonUserDMA();
~HexagonUserDMA();
HexagonUserDMA(const HexagonUserDMA&) = delete;
Expand All @@ -70,32 +75,17 @@ class HexagonUserDMA {
//! \brief Initializes the Hexagon User DMA engine
unsigned int Init();

//! \brief Calculates and returns the number of DMAs in flight; updates the ID of the oldest
//! descriptor in flight
//! \brief Calculates and returns the number of DMAs in flight
uint32_t DMAsInFlight();

//! \brief Calculates and returns the address of a DMA descriptor in the ring buffer given a
//! descriptor ID
void* GetDescriptorAddr(uint32_t dma_desc_id);

//! \brief Pointer to ring buffer storage for all DMA descriptors
void* dma_desc_ring_buff_{nullptr};

//! \brief Size of ring buffer storage for all DMA descriptors
const uint32_t dma_desc_ring_buff_size_{100};

//! \brief Tracks both the total number of DMA descriptors and the ID of the next DMA descriptor
//! to be added to the ring buffer - modulo ring buffer size to find the ring buffer index for the
//! next DMA descriptor
uint32_t id_next_dma_desc_{0};
//! \brief Tracks whether the very first DMA has been executed
bool first_dma_{true};

//! \brief Tracks the ID of the oldest DMA descriptor in flight OR the ID of the next DMA
//! descriptor if no DMA descriptors are in flight - modulo ring buffer size to find the ring
//! buffer index for the oldest DMA descriptor in flight
uint32_t id_oldest_dma_desc_in_flight_{0};
//! \brief Tracks the tail DMA descriptor
void* tail_dma_desc_{nullptr};

//! \brief Tracks whether (or not) we are executing the very first DMA
bool first_dma_{true};
//! \brief Storage for all DMA descriptors
RingBuffer<dma_desc_2d_t>* descriptors_{nullptr};
};

} // namespace hexagon
Expand Down
94 changes: 94 additions & 0 deletions src/runtime/hexagon/ring_buffer.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

#ifndef TVM_RUNTIME_HEXAGON_RING_BUFFER_H_
#define TVM_RUNTIME_HEXAGON_RING_BUFFER_H_

#include <functional>

#include "hexagon_common.h"

namespace tvm {
namespace runtime {
namespace hexagon {

template <class T>
class RingBuffer {
public:
//! \brief Returns the number of Ts in flight
uint32_t InFlight() {
while (!in_flight_(GetAddr(id_oldest_))) {
id_oldest_++;
}
return id_next_ - id_oldest_;
}

//! \brief Returns pointer to next T; null if ring buffer is full
T* Next() {
if (InFlight() == ring_buff_size_) {
return nullptr;
}
T* next = GetAddr(id_next_);
id_next_++;
return next;
}

/*! \brief Creates a ring buffer for storage items of type T
* \param ring_buff_size Size of the ring buffer in number of Ts
* \param in_flight Function that determines whether a T is in flight
*/
RingBuffer(uint32_t ring_buff_size, std::function<bool(T*)> in_flight)
: ring_buff_size_(ring_buff_size), in_flight_(in_flight) {
CHECK_NE(ring_buff_size, 0);
int ret = posix_memalign(reinterpret_cast<void**>(&ring_buff_ptr_), sizeof(T),
sizeof(T) * ring_buff_size_);
CHECK_EQ(ret, 0);
CHECK_NE(ring_buff_ptr_, nullptr);
}

~RingBuffer() { free(ring_buff_ptr_); }

private:
//! \brief Returns the address of a T given its index
T* GetAddr(uint32_t id) const {
uint32_t ring_buff_index = id % ring_buff_size_;
return ring_buff_ptr_ + ring_buff_index;
}

//! \brief Pointer to the ring buffer
T* ring_buff_ptr_{nullptr};

//! \brief Size of the ring buffer in number of Ts
const uint32_t ring_buff_size_;

//! \brief Function that determines whether a T is in flight
const std::function<bool(T*)> in_flight_;

//! \brief Tracks the ID of the next T to be added to the ring buffer
uint32_t id_next_{0};

//! \brief Tracks the ID of the oldest T in flight
uint32_t id_oldest_{0};
};

} // namespace hexagon
} // namespace runtime
} // namespace tvm

#endif // TVM_RUNTIME_HEXAGON_RING_BUFFER_H_
109 changes: 109 additions & 0 deletions tests/cpp-runtime/hexagon/ring_buffer_tests.cc
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

#include <gtest/gtest.h>

#include "../src/runtime/hexagon/ring_buffer.h"

using namespace tvm::runtime;
using namespace tvm::runtime::hexagon;

class RingBufferTest : public ::testing::Test {
void SetUp() override {
always_in_flight_rb = new RingBuffer<int>(10, always_in_flight);
ring_buff = new RingBuffer<int>(10, check_answer);
}
void TearDown() override { delete always_in_flight_rb; }

public:
std::function<bool(int*)> always_in_flight = [](int* ptr) { return true; };
RingBuffer<int>* always_in_flight_rb;

std::function<bool(int*)> check_answer = [](int* ptr) {
if (*ptr == 42) {
// complete, retired, done
return false;
}
// in flight
return true;
};
RingBuffer<int>* ring_buff;
};

TEST_F(RingBufferTest, zero_size_ring_buffer) {
ASSERT_THROW(RingBuffer<int>(0, always_in_flight), InternalError);
}

TEST_F(RingBufferTest, in_flight) { ASSERT_EQ(always_in_flight_rb->InFlight(), 0); }

TEST_F(RingBufferTest, next) {
int* ptr = always_in_flight_rb->Next();
ASSERT_NE(ptr, nullptr);
ASSERT_EQ(always_in_flight_rb->InFlight(), 1);
}

TEST_F(RingBufferTest, full) {
for (int i = 0; i < 10; ++i) {
int* ptr = always_in_flight_rb->Next();
ASSERT_NE(ptr, nullptr);
}
ASSERT_EQ(always_in_flight_rb->InFlight(), 10);
ASSERT_EQ(always_in_flight_rb->Next(), nullptr);
ASSERT_EQ(always_in_flight_rb->InFlight(), 10);
}

TEST_F(RingBufferTest, half_full) {
// these will complete
for (int i = 0; i < 5; ++i) {
int* ptr = ring_buff->Next();
ASSERT_NE(ptr, nullptr);
*ptr = 42;
}

// these will not complete
for (int i = 0; i < 5; ++i) {
int* ptr = ring_buff->Next();
ASSERT_NE(ptr, nullptr);
*ptr = 43;
}

ASSERT_EQ(ring_buff->InFlight(), 5);
ASSERT_NE(ring_buff->Next(), nullptr);
ASSERT_EQ(ring_buff->InFlight(), 6);
}

TEST_F(RingBufferTest, still_full) {
// these will not complete
for (int i = 0; i < 5; ++i) {
int* ptr = ring_buff->Next();
ASSERT_NE(ptr, nullptr);
*ptr = 43;
}

// these would complete, but they are blocked
for (int i = 0; i < 5; ++i) {
int* ptr = ring_buff->Next();
ASSERT_NE(ptr, nullptr);
*ptr = 42;
}

ASSERT_EQ(ring_buff->InFlight(), 10);
ASSERT_EQ(ring_buff->Next(), nullptr);
ASSERT_EQ(ring_buff->InFlight(), 10);
}

0 comments on commit bd1f7d5

Please sign in to comment.