Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/build-cachelib-docker.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ jobs:
runs-on: ubuntu-latest
env:
REPO: cachelib
GITHUB_REPO: pmem/CacheLib
GITHUB_REPO: intel/CacheLib
CONTAINER_REG: ghcr.io/pmem/cachelib
CONTAINER_REG_USER: ${{ secrets.GH_CR_USER }}
CONTAINER_REG_PASS: ${{ secrets.GH_CR_PAT }}
Expand Down
43 changes: 27 additions & 16 deletions cachelib/allocator/CacheAllocator-inl.h
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,7 @@ ShmSegmentOpts CacheAllocator<CacheTrait>::createShmCacheOpts(TierId tid) {
ShmSegmentOpts opts;
opts.alignment = sizeof(Slab);
opts.typeOpts = memoryTierConfigs[tid].getShmTypeOpts();
opts.memBindNumaNodes = memoryTierConfigs[tid].getMemBind();
if (auto *v = std::get_if<PosixSysVSegmentOpts>(&opts.typeOpts)) {
v->usePosix = config_.usePosixShm;
}
Expand Down Expand Up @@ -1307,15 +1308,15 @@ CacheAllocator<CacheTrait>::moveRegularItemWithSync(
// make sure that no other thread removed it, and only then replaces it.
if (!replaceInMMContainer(oldItem, *newItemHdl)) {
accessContainer_->remove(*newItemHdl);
return {};
return acquire(&oldItem);
}

// Replacing into the MM container was successful, but someone could have
// called insertOrReplace() or remove() before or after the
// replaceInMMContainer() operation, which would invalidate newItemHdl.
if (!newItemHdl->isAccessible()) {
removeFromMMContainer(*newItemHdl);
return {};
return acquire(&oldItem);
}

// no one can add or remove chained items at this point
Expand Down Expand Up @@ -1640,7 +1641,13 @@ typename CacheAllocator<CacheTrait>::WriteHandle
CacheAllocator<CacheTrait>::tryEvictToNextMemoryTier(
TierId tid, PoolId pid, Item& item) {
if(item.isChainedItem()) return {}; // TODO: We do not support ChainedItem yet
if(item.isExpired()) return acquire(&item);
if(item.isExpired()) {
auto handle = removeIf(item, [](const Item& it) {
return it.getRefCount() == 0;
});

if (handle) { return handle; }
}

TierId nextTier = tid; // TODO - calculate this based on some admission policy
while (++nextTier < getNumTiers()) { // try to evict down to the next memory tiers
Expand Down Expand Up @@ -3066,16 +3073,12 @@ CacheAllocator<CacheTrait>::evictNormalItem(Item& item,
// We remove the item from both access and mm containers. It doesn't matter
// if someone else calls remove on the item at this moment, the item cannot
// be freed as long as we have the moving bit set.
auto handle = accessContainer_->removeIf(item, std::move(predicate));

auto handle = removeIf(item, std::move(predicate));
if (!handle) {
return handle;
}

XDCHECK_EQ(reinterpret_cast<uintptr_t>(handle.get()),
reinterpret_cast<uintptr_t>(&item));
XDCHECK_EQ(1u, handle->getRefCount());
removeFromMMContainer(item);

// now that we are the only handle and we actually removed something from
// the RAM cache, we enqueue it to nvmcache.
Expand Down Expand Up @@ -3187,6 +3190,21 @@ CacheAllocator<CacheTrait>::evictChainedItemForSlabRelease(ChainedItem& child) {
return parentHandle;
}

template <typename CacheTrait>
template <typename Fn>
typename CacheAllocator<CacheTrait>::WriteHandle
CacheAllocator<CacheTrait>::removeIf(Item& item, Fn&& predicate) {
auto handle = accessContainer_->removeIf(item, std::forward<Fn>(predicate));

if (handle) {
XDCHECK_EQ(reinterpret_cast<uintptr_t>(handle.get()),
reinterpret_cast<uintptr_t>(&item));
removeFromMMContainer(item);
}

return handle;
}

template <typename CacheTrait>
bool CacheAllocator<CacheTrait>::removeIfExpired(const ReadHandle& handle) {
if (!handle) {
Expand All @@ -3195,14 +3213,7 @@ bool CacheAllocator<CacheTrait>::removeIfExpired(const ReadHandle& handle) {

// We remove the item from both access and mm containers.
// We want to make sure the caller is the only one holding the handle.
auto removedHandle =
accessContainer_->removeIf(*(handle.getInternal()), itemExpiryPredicate);
if (removedHandle) {
removeFromMMContainer(*(handle.getInternal()));
return true;
}

return false;
return (bool)removeIf(*(handle.getInternal()), itemExpiryPredicate);
}

template <typename CacheTrait>
Expand Down
11 changes: 9 additions & 2 deletions cachelib/allocator/CacheAllocator.h
Original file line number Diff line number Diff line change
Expand Up @@ -1496,8 +1496,9 @@ class CacheAllocator : public CacheBase {
// @param oldItem Reference to the item being moved
// @param newItemHdl Reference to the handle of the new item being moved into
//
// @return true If the move was completed, and the containers were updated
// successfully.
// @return the handle to the oldItem if the move was completed
// and the oldItem can be recycled.
// Otherwise an empty handle is returned.
template <typename P>
WriteHandle moveRegularItemWithSync(Item& oldItem, WriteHandle& newItemHdl, P&& predicate);

Expand Down Expand Up @@ -1806,6 +1807,12 @@ class CacheAllocator : public CacheBase {
// handle on failure. caller can retry.
WriteHandle evictChainedItemForSlabRelease(ChainedItem& item);

// Helper function to remove a item if predicates is true.
//
// @return last handle to the item on success. empty handle on failure.
template <typename Fn>
WriteHandle removeIf(Item& item, Fn&& predicate);

// Helper function to remove a item if expired.
//
// @return true if it item expire and removed successfully.
Expand Down
13 changes: 13 additions & 0 deletions cachelib/allocator/MemoryTierCacheConfig.h
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,16 @@ class MemoryTierCacheConfig {

size_t getRatio() const noexcept { return ratio; }

// Allocate memory only from specified NUMA nodes
MemoryTierCacheConfig& setMemBind(const std::vector<size_t>& _numaNodes) {
numaNodes = _numaNodes;
return *this;
}

std::vector<size_t> getMemBind() const {
return numaNodes;
}

size_t calculateTierSize(size_t totalCacheSize, size_t partitionNum) const {
// TODO: Call this method when tiers are enabled in allocator
// to calculate tier sizes in bytes.
Expand Down Expand Up @@ -82,6 +92,9 @@ class MemoryTierCacheConfig {
// Options specific to shm type
ShmTypeOpts shmOpts;

// Numa node(s) to bind the tier
std::vector<size_t> numaNodes;

MemoryTierCacheConfig() = default;
};
} // namespace cachelib
Expand Down
8 changes: 6 additions & 2 deletions cachelib/allocator/tests/AllocatorMemoryTiersTest.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,13 @@ namespace tests {
using LruAllocatorMemoryTiersTest = AllocatorMemoryTiersTest<LruAllocator>;

// TODO(MEMORY_TIER): add more tests with different eviction policies
TEST_F(LruAllocatorMemoryTiersTest, MultiTiersInvalid) { this->testMultiTiersInvalid(); }
TEST_F(LruAllocatorMemoryTiersTest, MultiTiersValid) { this->testMultiTiersValid(); }
TEST_F(LruAllocatorMemoryTiersTest, MultiTiersFromFileInvalid) { this->testMultiTiersFormFileInvalid(); }
TEST_F(LruAllocatorMemoryTiersTest, MultiTiersFromFileValid) { this->testMultiTiersFromFileValid(); }
TEST_F(LruAllocatorMemoryTiersTest, MultiTiersValidMixed) { this->testMultiTiersValidMixed(); }
TEST_F(LruAllocatorMemoryTiersTest, MultiTiersNumaBindingsSysVValid) { this->testMultiTiersNumaBindingsSysVValid(); }
TEST_F(LruAllocatorMemoryTiersTest, MultiTiersNumaBindingsPosixValid) { this->testMultiTiersNumaBindingsPosixValid(); }
TEST_F(LruAllocatorMemoryTiersTest, MultiTiersRemoveDuringEviction) { this->testMultiTiersRemoveDuringEviction(); }
TEST_F(LruAllocatorMemoryTiersTest, MultiTiersReplaceDuringEviction) { this->testMultiTiersReplaceDuringEviction(); }

} // end of namespace tests
} // end of namespace cachelib
Expand Down
139 changes: 137 additions & 2 deletions cachelib/allocator/tests/AllocatorMemoryTiersTest.h
Original file line number Diff line number Diff line change
Expand Up @@ -20,14 +20,44 @@
#include "cachelib/allocator/MemoryTierCacheConfig.h"
#include "cachelib/allocator/tests/TestBase.h"

#include <folly/synchronization/Latch.h>

namespace facebook {
namespace cachelib {
namespace tests {

template <typename AllocatorT>
class AllocatorMemoryTiersTest : public AllocatorTest<AllocatorT> {
private:
template<typename MvCallback>
void testMultiTiersAsyncOpDuringMove(std::unique_ptr<AllocatorT>& alloc,
PoolId& pool, bool& quit, MvCallback&& moveCb) {
typename AllocatorT::Config config;
config.setCacheSize(4 * Slab::kSize);
config.enableCachePersistence("/tmp");
config.configureMemoryTiers({
MemoryTierCacheConfig::fromShm()
.setRatio(1).setMemBind({0}),
MemoryTierCacheConfig::fromShm()
.setRatio(1).setMemBind({0})
});

config.enableMovingOnSlabRelease(moveCb, {} /* ChainedItemsMoveSync */,
-1 /* movingAttemptsLimit */);

alloc = std::make_unique<AllocatorT>(AllocatorT::SharedMemNew, config);
ASSERT(alloc != nullptr);
pool = alloc->addPool("default", alloc->getCacheMemoryStats().cacheSize);

int i = 0;
while(!quit) {
auto handle = alloc->allocate(pool, std::to_string(++i), std::string("value").size());
ASSERT(handle != nullptr);
ASSERT_NO_THROW(alloc->insertOrReplace(handle));
}
}
public:
void testMultiTiersInvalid() {
void testMultiTiersFormFileInvalid() {
typename AllocatorT::Config config;
config.setCacheSize(100 * Slab::kSize);
config.configureMemoryTiers({
Expand All @@ -42,7 +72,7 @@ class AllocatorMemoryTiersTest : public AllocatorTest<AllocatorT> {
std::invalid_argument);
}

void testMultiTiersValid() {
void testMultiTiersFromFileValid() {
typename AllocatorT::Config config;
config.setCacheSize(100 * Slab::kSize);
config.enableCachePersistence("/tmp");
Expand Down Expand Up @@ -83,6 +113,111 @@ class AllocatorMemoryTiersTest : public AllocatorTest<AllocatorT> {
ASSERT(handle != nullptr);
ASSERT_NO_THROW(alloc->insertOrReplace(handle));
}

void testMultiTiersNumaBindingsSysVValid() {
typename AllocatorT::Config config;
config.setCacheSize(100 * Slab::kSize);
config.enableCachePersistence("/tmp");
config.configureMemoryTiers({
MemoryTierCacheConfig::fromShm()
.setRatio(1).setMemBind({0}),
MemoryTierCacheConfig::fromShm()
.setRatio(1).setMemBind({0})
});

auto alloc = std::make_unique<AllocatorT>(AllocatorT::SharedMemNew, config);
ASSERT(alloc != nullptr);

auto pool = alloc->addPool("default", alloc->getCacheMemoryStats().cacheSize);
auto handle = alloc->allocate(pool, "key", std::string("value").size());
ASSERT(handle != nullptr);
ASSERT_NO_THROW(alloc->insertOrReplace(handle));
}

void testMultiTiersNumaBindingsPosixValid() {
typename AllocatorT::Config config;
config.setCacheSize(100 * Slab::kSize);
config.enableCachePersistence("/tmp");
config.usePosixForShm();
config.configureMemoryTiers({
MemoryTierCacheConfig::fromShm()
.setRatio(1).setMemBind({0}),
MemoryTierCacheConfig::fromShm()
.setRatio(1).setMemBind({0})
});

auto alloc = std::make_unique<AllocatorT>(AllocatorT::SharedMemNew, config);
ASSERT(alloc != nullptr);

auto pool = alloc->addPool("default", alloc->getCacheMemoryStats().cacheSize);
auto handle = alloc->allocate(pool, "key", std::string("value").size());
ASSERT(handle != nullptr);
ASSERT_NO_THROW(alloc->insertOrReplace(handle));
}

void testMultiTiersRemoveDuringEviction() {
std::unique_ptr<AllocatorT> alloc;
PoolId pool;
std::unique_ptr<std::thread> t;
folly::Latch latch(1);
bool quit = false;

auto moveCb = [&] (typename AllocatorT::Item& oldItem,
typename AllocatorT::Item& newItem,
typename AllocatorT::Item* /* parentPtr */) {

auto key = oldItem.getKey();
t = std::make_unique<std::thread>([&](){
// remove() function is blocked by wait context
// till item is moved to next tier. So that, we should
// notify latch before calling remove()
latch.count_down();
alloc->remove(key);
});
// wait till async thread is running
latch.wait();
memcpy(newItem.getMemory(), oldItem.getMemory(), oldItem.getSize());
quit = true;
};

testMultiTiersAsyncOpDuringMove(alloc, pool, quit, moveCb);

t->join();
}

void testMultiTiersReplaceDuringEviction() {
std::unique_ptr<AllocatorT> alloc;
PoolId pool;
std::unique_ptr<std::thread> t;
folly::Latch latch(1);
bool quit = false;

auto moveCb = [&] (typename AllocatorT::Item& oldItem,
typename AllocatorT::Item& newItem,
typename AllocatorT::Item* /* parentPtr */) {
auto key = oldItem.getKey();
if(!quit) {
// we need to replace only once because subsequent allocate calls
// will cause evictions recursevly
quit = true;
t = std::make_unique<std::thread>([&](){
auto handle = alloc->allocate(pool, key, std::string("new value").size());
// insertOrReplace() function is blocked by wait context
// till item is moved to next tier. So that, we should
// notify latch before calling insertOrReplace()
latch.count_down();
ASSERT_NO_THROW(alloc->insertOrReplace(handle));
});
// wait till async thread is running
latch.wait();
}
memcpy(newItem.getMemory(), oldItem.getMemory(), oldItem.getSize());
};

testMultiTiersAsyncOpDuringMove(alloc, pool, quit, moveCb);

t->join();
}
};
} // namespace tests
} // namespace cachelib
Expand Down
1 change: 1 addition & 0 deletions cachelib/cachebench/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -89,5 +89,6 @@ if (BUILD_TESTS)
add_test (consistency/tests/ValueHistoryTest.cpp)
add_test (consistency/tests/ValueTrackerTest.cpp)
add_test (util/tests/NandWritesTest.cpp)
add_test (util/tests/MemoryTierConfigTest.cpp)
add_test (cache/tests/TimeStampTickerTest.cpp)
endif()
Loading