diff --git a/.github/workflows/build-cachelib-centos-8-1.yml b/.github/workflows/build-cachelib-centos-8-1.yml index 5eb1090b0a..3983e0c78b 100644 --- a/.github/workflows/build-cachelib-centos-8-1.yml +++ b/.github/workflows/build-cachelib-centos-8-1.yml @@ -14,6 +14,7 @@ name: build-cachelib-centos-8-1 on: # push: + pull_request: schedule: - cron: '0 11 * * 1,3,5' jobs: diff --git a/.github/workflows/build-cachelib-centos-8-5.yml b/.github/workflows/build-cachelib-centos-8-5.yml index 3ffee37765..4e6c2d12e1 100644 --- a/.github/workflows/build-cachelib-centos-8-5.yml +++ b/.github/workflows/build-cachelib-centos-8-5.yml @@ -14,6 +14,7 @@ name: build-cachelib-centos-8.5 on: # push: + pull_request: schedule: - cron: '0 9 * * *' jobs: diff --git a/.github/workflows/build-cachelib-debian-10.yml b/.github/workflows/build-cachelib-debian-10.yml index c7c67e0724..7f0ab29a6c 100644 --- a/.github/workflows/build-cachelib-debian-10.yml +++ b/.github/workflows/build-cachelib-debian-10.yml @@ -14,6 +14,7 @@ name: build-cachelib-debian-10 on: # push: + pull_request: schedule: - cron: '0 13 * * *' jobs: @@ -51,6 +52,9 @@ jobs: g++ - || true - name: "checkout sources" uses: actions/checkout@v2 + - name: "Add Git safe directory" + # Workaround for Docker image bug (GitHub issue #199). + run: git config --system --add safe.directory $GITHUB_WORKSPACE - name: "Install Prerequisites" run: ./contrib/build.sh -S -B - name: "Test: update-submodules" diff --git a/.github/workflows/build-cachelib-fedora-36.yml b/.github/workflows/build-cachelib-fedora-36.yml index 216dbf5841..f8c0424400 100644 --- a/.github/workflows/build-cachelib-fedora-36.yml +++ b/.github/workflows/build-cachelib-fedora-36.yml @@ -14,6 +14,7 @@ name: build-cachelib-fedora-36 on: # push: + pull_request: schedule: - cron: '0 19 * * *' jobs: diff --git a/.github/workflows/build-cachelib-rockylinux-8.yml b/.github/workflows/build-cachelib-rockylinux-8.yml index 879dc27566..c8af12327d 100644 --- a/.github/workflows/build-cachelib-rockylinux-8.yml +++ b/.github/workflows/build-cachelib-rockylinux-8.yml @@ -14,6 +14,7 @@ name: build-cachelib-rockylinux-8.6 on: # push: + pull_request: schedule: - cron: '0 15 * * 2,4,6' jobs: diff --git a/.github/workflows/build-cachelib-rockylinux-9.yml b/.github/workflows/build-cachelib-rockylinux-9.yml index f6a86d75a0..e26eac6ff1 100644 --- a/.github/workflows/build-cachelib-rockylinux-9.yml +++ b/.github/workflows/build-cachelib-rockylinux-9.yml @@ -14,6 +14,7 @@ name: build-cachelib-rockylinux-9.0 on: # push: + pull_request: schedule: - cron: '0 17 * * *' jobs: diff --git a/.github/workflows/build-cachelib-ubuntu-18.yml b/.github/workflows/build-cachelib-ubuntu-18.yml index fad34c0897..ad068278a4 100644 --- a/.github/workflows/build-cachelib-ubuntu-18.yml +++ b/.github/workflows/build-cachelib-ubuntu-18.yml @@ -19,6 +19,7 @@ name: build-cachelib-ubuntu-18 on: # push: + pull_request: schedule: - cron: '0 5 * * 2,4,6' jobs: diff --git a/.github/workflows/build-cachelib-ubuntu-20.yml b/.github/workflows/build-cachelib-ubuntu-20.yml index 35a3f507e2..a8380fdb96 100644 --- a/.github/workflows/build-cachelib-ubuntu-20.yml +++ b/.github/workflows/build-cachelib-ubuntu-20.yml @@ -15,6 +15,7 @@ name: build-cachelib-ubuntu-20 on: # push: + pull_request: schedule: - cron: '0 5 * * 1,3,5' jobs: diff --git a/.github/workflows/build-cachelib-ubuntu-22.yml b/.github/workflows/build-cachelib-ubuntu-22.yml index b4374a5b96..4db194431d 100644 --- a/.github/workflows/build-cachelib-ubuntu-22.yml +++ b/.github/workflows/build-cachelib-ubuntu-22.yml @@ -15,6 +15,7 @@ name: build-cachelib-ubuntu-22 on: # push: + pull_request: schedule: - cron: '0 7 * * *' jobs: diff --git a/.packit.yaml b/.packit.yaml deleted file mode 100644 index bea307d9d0..0000000000 --- a/.packit.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# See the documentation for more information: -# https://packit.dev/docs/configuration - -specfile_path: cachelib.spec - -upstream_package_name: CacheLib -downstream_package_name: cachelib - -actions: - fix-spec-file: - - bash -c "sed -i cachelib.spec -e \"s/%global commit.*/%global commit $(git rev-parse HEAD)/\"" - - bash -c "sed -i cachelib.spec -e \"s/%global date.*/%global date $(git show -s --date=format:'%Y%m%d' --format=%cd)/\"" - create-archive: - - bash -c "COMMIT=$(git rev-parse HEAD); curl -ORL https://github.com/facebook/CacheLib/archive/${COMMIT}/cachelib-${COMMIT}.tar.gz; echo cachelib-${COMMIT}.tar.gz" - post-upstream-clone: "bash -c \"rm -rf cachelib-dist-git; git clone -b packit https://pagure.io/meta/cachelib.git cachelib-dist-git && mv cachelib-dist-git/cachelib*.{spec,patch} .\"" - -jobs: -- job: copr_build - trigger: pull_request - metadata: - targets: - - fedora-rawhide-aarch64 - - fedora-rawhide-x86_64 - - fedora-35-aarch64 - - fedora-35-x86_64 diff --git a/README.md b/README.md index e05c932d1e..7fc943b202 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ cd CacheLib Re-running `./contrib/build.sh` will update CacheLib and its dependencies to their latest versions and rebuild them. -See [build](https://cachelib.org/docs/installation/installation) for more details about +See [build](https://cachelib.org/docs/installation/) for more details about the building and installation process. @@ -77,3 +77,18 @@ https://www.facebook.com/whitehat Facebook's security team will triage your report and determine whether or not is it eligible for a bounty under our program. + + +## Build status + +Clicking on a badge will show you the recent builds for that OS. If your target OS's build is failing, you may wish to check recent issues and PRs for known workarounds. + +- [![CentOS 8.1](https://github.com/facebook/cachelib/actions/workflows/build-cachelib-centos-8-1.yml/badge.svg?event=schedule)](https://github.com/facebook/cachelib/actions/workflows/build-cachelib-centos-8-1.yml?query=event%3Aschedule) +- [![CentOS 8.5](https://github.com/facebook/cachelib/actions/workflows/build-cachelib-centos-8-5.yml/badge.svg?event=schedule)](https://github.com/facebook/cachelib/actions/workflows/build-cachelib-centos-8-5.yml?query=event%3Aschedule) +- [![Debian 10](https://github.com/facebook/cachelib/actions/workflows/build-cachelib-debian-10.yml/badge.svg?event=schedule)](https://github.com/facebook/cachelib/actions/workflows/build-cachelib-debian-10.yml?query=event%3Aschedule) +- [![Fedora 36](https://github.com/facebook/cachelib/actions/workflows/build-cachelib-fedora-36.yml/badge.svg?event=schedule)](https://github.com/facebook/cachelib/actions/workflows/build-cachelib-fedora-36.yml?query=event%3Aschedule) +- [![Rocky Linux 8](https://github.com/facebook/cachelib/actions/workflows/build-cachelib-rockylinux-8.yml/badge.svg?event=schedule)](https://github.com/facebook/cachelib/actions/workflows/build-cachelib-rockylinux-8.yml?query=event%3Aschedule) +- [![Rocky Linux 9](https://github.com/facebook/cachelib/actions/workflows/build-cachelib-rockylinux-9.yml/badge.svg?event=schedule)](https://github.com/facebook/cachelib/actions/workflows/build-cachelib-rockylinux-9.yml?query=event%3Aschedule) +- [![Ubuntu 18](https://github.com/facebook/cachelib/actions/workflows/build-cachelib-ubuntu-18.yml/badge.svg?event=schedule)](https://github.com/facebook/cachelib/actions/workflows/build-cachelib-ubuntu-18.yml?query=event%3Aschedule) +- [![Ubuntu 20](https://github.com/facebook/cachelib/actions/workflows/build-cachelib-ubuntu-20.yml/badge.svg?event=schedule)](https://github.com/facebook/cachelib/actions/workflows/build-cachelib-ubuntu-20.yml?query=event%3Aschedule) +- [![Ubuntu 22](https://github.com/facebook/cachelib/actions/workflows/build-cachelib-ubuntu-22.yml/badge.svg?event=schedule)](https://github.com/facebook/cachelib/actions/workflows/build-cachelib-ubuntu-22.yml?query=event%3Aschedule) diff --git a/cachelib/allocator/BackgroundMover-inl.h b/cachelib/allocator/BackgroundMover-inl.h new file mode 100644 index 0000000000..c0a827d6ec --- /dev/null +++ b/cachelib/allocator/BackgroundMover-inl.h @@ -0,0 +1,112 @@ +/* + * Copyright (c) Intel and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace facebook { +namespace cachelib { + +template +BackgroundMover::BackgroundMover( + Cache& cache, + std::shared_ptr strategy, + MoverDir direction) + : cache_(cache), strategy_(strategy), direction_(direction) { + if (direction_ == MoverDir::Evict) { + moverFunc = BackgroundMoverAPIWrapper::traverseAndEvictItems; + + } else if (direction_ == MoverDir::Promote) { + moverFunc = BackgroundMoverAPIWrapper::traverseAndPromoteItems; + } +} + +template +BackgroundMover::~BackgroundMover() { + stop(std::chrono::seconds(0)); +} + +template +void BackgroundMover::work() { + try { + checkAndRun(); + } catch (const std::exception& ex) { + XLOGF(ERR, "BackgroundMover interrupted due to exception: {}", ex.what()); + } +} + +template +void BackgroundMover::setAssignedMemory( + std::vector&& assignedMemory) { + XLOG(INFO, "Class assigned to background worker:"); + for (auto [pid, cid] : assignedMemory) { + XLOGF(INFO, "Pid: {}, Cid: {}", pid, cid); + } + + mutex.lock_combine([this, &assignedMemory] { + this->assignedMemory_ = std::move(assignedMemory); + }); +} + +// Look for classes that exceed the target memory capacity +// and return those for eviction +template +void BackgroundMover::checkAndRun() { + auto assignedMemory = mutex.lock_combine([this] { return assignedMemory_; }); + + unsigned int moves = 0; + std::set classes{}; + auto batches = strategy_->calculateBatchSizes(cache_, assignedMemory); + + for (size_t i = 0; i < batches.size(); i++) { + const auto [pid, cid] = assignedMemory[i]; + const auto batch = batches[i]; + + classes.insert(cid); + const auto& mpStats = cache_.getPool(pid).getStats(); + + if (!batch) { + continue; + } + + // try moving BATCH items from the class in order to reach free target + auto moved = moverFunc(cache_, pid, cid, batch); + moves += moved; + moves_per_class_[pid][cid] += moved; + totalBytesMoved.add(moved * mpStats.acStats.at(cid).allocSize); + } + + numTraversals.inc(); + numMovedItems.add(moves); + totalClasses.add(classes.size()); +} + +template +BackgroundMoverStats BackgroundMover::getStats() const noexcept { + BackgroundMoverStats stats; + stats.numMovedItems = numMovedItems.get(); + stats.runCount = numTraversals.get(); + stats.totalBytesMoved = totalBytesMoved.get(); + stats.totalClasses = totalClasses.get(); + + return stats; +} + +template +std::map> +BackgroundMover::getClassStats() const noexcept { + return moves_per_class_; +} + +} // namespace cachelib +} // namespace facebook diff --git a/cachelib/allocator/BackgroundMover.h b/cachelib/allocator/BackgroundMover.h new file mode 100644 index 0000000000..309d88df2a --- /dev/null +++ b/cachelib/allocator/BackgroundMover.h @@ -0,0 +1,96 @@ +/* + * Copyright (c) Intel and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "cachelib/allocator/BackgroundMoverStrategy.h" +#include "cachelib/allocator/CacheStats.h" +#include "cachelib/common/AtomicCounter.h" +#include "cachelib/common/PeriodicWorker.h" + +namespace facebook { +namespace cachelib { + +// wrapper that exposes the private APIs of CacheType that are specifically +// needed for the cache api +template +struct BackgroundMoverAPIWrapper { + static size_t traverseAndEvictItems(C& cache, + unsigned int pid, + unsigned int cid, + size_t batch) { + return cache.traverseAndEvictItems(pid, cid, batch); + } + + static size_t traverseAndPromoteItems(C& cache, + unsigned int pid, + unsigned int cid, + size_t batch) { + return cache.traverseAndPromoteItems(pid, cid, batch); + } +}; + +enum class MoverDir { Evict = 0, Promote }; + +// Periodic worker that evicts items from tiers in batches +// The primary aim is to reduce insertion times for new items in the +// cache +template +class BackgroundMover : public PeriodicWorker { + public: + using Cache = CacheT; + // @param cache the cache interface + // @param strategy the stragey class that defines how objects are + // moved, + // (promoted vs. evicted and how much) + BackgroundMover(Cache& cache, + std::shared_ptr strategy, + MoverDir direction_); + + ~BackgroundMover() override; + + BackgroundMoverStats getStats() const noexcept; + std::map> getClassStats() const noexcept; + + void setAssignedMemory(std::vector&& assignedMemory); + + private: + std::map> moves_per_class_; + // cache allocator's interface for evicting + using Item = typename Cache::Item; + + Cache& cache_; + std::shared_ptr strategy_; + MoverDir direction_; + + std::function moverFunc; + + // implements the actual logic of running the background evictor + void work() override final; + void checkAndRun(); + + AtomicCounter numMovedItems{0}; + AtomicCounter numTraversals{0}; + AtomicCounter totalClasses{0}; + AtomicCounter totalBytesMoved{0}; + + std::vector assignedMemory_; + folly::DistributedMutex mutex; +}; +} // namespace cachelib +} // namespace facebook + +#include "cachelib/allocator/BackgroundMover-inl.h" diff --git a/cachelib/allocator/BackgroundMoverStrategy.h b/cachelib/allocator/BackgroundMoverStrategy.h new file mode 100644 index 0000000000..ca4ad2568c --- /dev/null +++ b/cachelib/allocator/BackgroundMoverStrategy.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "cachelib/allocator/Cache.h" + +namespace facebook { +namespace cachelib { + +struct MemoryDescriptorType { + MemoryDescriptorType(PoolId pid, ClassId cid) : pid_(pid), cid_(cid) {} + PoolId pid_; + ClassId cid_; +}; + +// Base class for background eviction strategy. +class BackgroundMoverStrategy { + public: + virtual std::vector calculateBatchSizes( + const CacheBase& cache, std::vector acVec) = 0; +}; + +} // namespace cachelib +} // namespace facebook diff --git a/cachelib/allocator/CMakeLists.txt b/cachelib/allocator/CMakeLists.txt index 78cfa7ca06..38567add06 100644 --- a/cachelib/allocator/CMakeLists.txt +++ b/cachelib/allocator/CMakeLists.txt @@ -35,6 +35,7 @@ add_library (cachelib_allocator CCacheManager.cpp ContainerTypes.cpp FreeMemStrategy.cpp + FreeThresholdStrategy.cpp HitsPerSlabStrategy.cpp LruTailAgeStrategy.cpp MarginalHitsOptimizeStrategy.cpp diff --git a/cachelib/allocator/CacheAllocator-inl.h b/cachelib/allocator/CacheAllocator-inl.h index 1d89593268..ffdac068fb 100644 --- a/cachelib/allocator/CacheAllocator-inl.h +++ b/cachelib/allocator/CacheAllocator-inl.h @@ -235,6 +235,18 @@ void CacheAllocator::initWorkers() { config_.poolOptimizeStrategy, config_.ccacheOptimizeStepSizePercent); } + + if (config_.backgroundEvictorEnabled()) { + startNewBackgroundEvictor(config_.backgroundEvictorInterval, + config_.backgroundEvictorStrategy, + config_.backgroundEvictorThreads); + } + + if (config_.backgroundPromoterEnabled()) { + startNewBackgroundPromoter(config_.backgroundPromoterInterval, + config_.backgroundPromoterStrategy, + config_.backgroundPromoterThreads); + } } template @@ -318,13 +330,30 @@ CacheAllocator::allocate(PoolId poolId, ttlSecs == 0 ? 0 : creationTime + ttlSecs); } +template +bool CacheAllocator::shouldWakeupBgEvictor(PoolId pid, + ClassId cid) { + return false; +} + +template +size_t CacheAllocator::backgroundWorkerId(PoolId pid, + ClassId cid, + size_t numWorkers) { + XDCHECK(numWorkers); + + // TODO: came up with some better sharding (use hashing?) + return (pid + cid) % numWorkers; +} + template typename CacheAllocator::WriteHandle CacheAllocator::allocateInternal(PoolId pid, typename Item::Key key, uint32_t size, uint32_t creationTime, - uint32_t expiryTime) { + uint32_t expiryTime, + bool fromBgThread) { util::LatencyTracker tracker{stats().allocateLatency_}; SCOPE_FAIL { stats_.invalidAllocs.inc(); }; @@ -338,6 +367,13 @@ CacheAllocator::allocateInternal(PoolId pid, (*stats_.allocAttempts)[pid][cid].inc(); void* memory = allocator_->allocate(pid, requiredSize); + + if (backgroundEvictor_.size() && !fromBgThread && + (memory == nullptr || shouldWakeupBgEvictor(pid, cid))) { + backgroundEvictor_[backgroundWorkerId(pid, cid, backgroundEvictor_.size())] + ->wakeUp(); + } + if (memory == nullptr) { memory = findEviction(pid, cid); } @@ -832,20 +868,21 @@ CacheAllocator::releaseBackToAllocator(Item& it, removeFromMMContainer(*head); - // If this chained item is marked as exclusive, we will not free it. - // We must capture the exclusive state before we do the decRef when + // If this chained item is marked as moving, we will not free it. + // We must capture the moving state before we do the decRef when // we know the item must still be valid - const bool wasExclusive = head->isExclusive(); + const bool wasMoving = head->isMoving(); + XDCHECK(!head->isMarkedForEviction()); // Decref and check if we were the last reference. Now if the item - // was marked exclusive, after decRef, it will be free to be released + // was marked moving, after decRef, it will be free to be released // by slab release thread const auto childRef = head->decRef(); - // If the item is already exclusive and we already decremented the + // If the item is already moving and we already decremented the // refcount, we don't need to free this item. We'll let the slab // release thread take care of that - if (!wasExclusive) { + if (!wasMoving) { if (childRef != 0) { throw std::runtime_error(folly::sformat( "chained item refcount is not zero. We cannot proceed! " @@ -853,7 +890,7 @@ CacheAllocator::releaseBackToAllocator(Item& it, childRef, head->toString())); } - // Item is not exclusive and refcount is 0, we can proceed to + // Item is not moving and refcount is 0, we can proceed to // free it or recylce the memory if (head == toRecycle) { XDCHECK(ReleaseRes::kReleased != res); @@ -881,9 +918,12 @@ CacheAllocator::releaseBackToAllocator(Item& it, } template -void CacheAllocator::incRef(Item& it) { - it.incRef(); - ++handleCount_.tlStats(); +bool CacheAllocator::incRef(Item& it) { + if (it.incRef()) { + ++handleCount_.tlStats(); + return true; + } + return false; } template @@ -903,8 +943,12 @@ CacheAllocator::acquire(Item* it) { SCOPE_FAIL { stats_.numRefcountOverflow.inc(); }; - incRef(*it); - return WriteHandle{it, *this}; + if (LIKELY(incRef(*it))) { + return WriteHandle{it, *this}; + } else { + // item is being evicted + return WriteHandle{}; + } } template @@ -1179,7 +1223,7 @@ bool CacheAllocator::moveChainedItem(ChainedItem& oldItem, // This item has been unlinked from its parent and we're the only // owner of it, so we're done here - if (!oldItem.isInMMContainer() || oldItem.isOnlyExclusive()) { + if (!oldItem.isInMMContainer() || oldItem.isOnlyMoving()) { return false; } @@ -1210,7 +1254,7 @@ bool CacheAllocator::moveChainedItem(ChainedItem& oldItem, // In case someone else had removed this chained item from its parent by now // So we check again to see if the it has been unlinked from its parent - if (!oldItem.isInMMContainer() || oldItem.isOnlyExclusive()) { + if (!oldItem.isInMMContainer() || oldItem.isOnlyMoving()) { return false; } @@ -1226,7 +1270,7 @@ bool CacheAllocator::moveChainedItem(ChainedItem& oldItem, // parent's chain and the MMContainer. auto oldItemHandle = replaceChainedItemLocked(oldItem, std::move(newItemHdl), *parentHandle); - XDCHECK(oldItemHandle->isExclusive()); + XDCHECK(oldItemHandle->isMoving()); XDCHECK(!oldItemHandle->isInMMContainer()); return true; @@ -1255,7 +1299,7 @@ CacheAllocator::findEviction(PoolId pid, ClassId cid) { : toRecycle; // make sure no other thead is evicting the item - if (candidate->getRefCount() != 0 || !candidate->markExclusive()) { + if (candidate->getRefCount() != 0 || !candidate->markMoving()) { ++itr; continue; } @@ -1270,11 +1314,11 @@ CacheAllocator::findEviction(PoolId pid, ClassId cid) { ? advanceIteratorAndTryEvictChainedItem(itr) : advanceIteratorAndTryEvictRegularItem(mmContainer, itr); evictionSuccessful = toReleaseHandle != nullptr; - // destroy toReleseHandle. The item won't be released to allocator - // since we marked it as exclusive. + // destroy toReleaseHandle. The item won't be released to allocator + // since we marked for eviction. } - const auto ref = candidate->unmarkExclusive(); + const auto ref = candidate->unmarkMoving(); if (ref == 0u) { // Invalidate iterator since later on we may use this mmContainer // again, which cannot be done unless we drop this iterator @@ -2030,6 +2074,19 @@ PoolId CacheAllocator::addPool( createMMContainers(pid, std::move(config)); setRebalanceStrategy(pid, std::move(rebalanceStrategy)); setResizeStrategy(pid, std::move(resizeStrategy)); + + if (backgroundEvictor_.size()) { + for (size_t id = 0; id < backgroundEvictor_.size(); id++) + backgroundEvictor_[id]->setAssignedMemory( + getAssignedMemoryToBgWorker(id, backgroundEvictor_.size())); + } + + if (backgroundPromoter_.size()) { + for (size_t id = 0; id < backgroundPromoter_.size(); id++) + backgroundPromoter_[id]->setAssignedMemory( + getAssignedMemoryToBgWorker(id, backgroundPromoter_.size())); + } + return pid; } @@ -2361,7 +2418,7 @@ void CacheAllocator::releaseSlabImpl( // Need to mark an item for release before proceeding // If we can't mark as moving, it means the item is already freed const bool isAlreadyFreed = - !markExclusiveForSlabRelease(releaseContext, alloc, throttler); + !markMovingForSlabRelease(releaseContext, alloc, throttler); if (isAlreadyFreed) { continue; } @@ -2406,8 +2463,8 @@ bool CacheAllocator::moveForSlabRelease( stats_.numMoveAttempts.inc(); // Nothing to move and the key is likely also bogus for chained items. - if (oldItem.isOnlyExclusive()) { - oldItem.unmarkExclusive(); + if (oldItem.isOnlyMoving()) { + oldItem.unmarkMoving(); const auto res = releaseBackToAllocator(oldItem, RemoveContext::kNormal, false); XDCHECK(res == ReleaseRes::kReleased); @@ -2446,7 +2503,7 @@ bool CacheAllocator::moveForSlabRelease( // that's identical to this one to replace it. Here we just need to wait // until all users have dropped the item handles before we can proceed. startTime = util::getCurrentTimeSec(); - while (!oldItem.isOnlyExclusive()) { + while (!oldItem.isOnlyMoving()) { throttleWith(throttler, [&] { XLOGF(WARN, "Spent {} seconds, slab release still waiting for refcount to " @@ -2500,8 +2557,8 @@ CacheAllocator::allocateNewItemForOldItem(const Item& oldItem) { return {}; } - // Set up the destination for the move. Since oldChainedItem would have - // the exclusive bit set, it won't be picked for eviction. + // Set up the destination for the move. Since oldChainedItem would be + // marked as moving, it won't be picked for eviction. auto newItemHdl = allocateChainedItemInternal(parentHandle, oldChainedItem.getSize()); if (!newItemHdl) { @@ -2526,7 +2583,8 @@ CacheAllocator::allocateNewItemForOldItem(const Item& oldItem) { oldItem.getKey(), oldItem.getSize(), oldItem.getCreationTime(), - oldItem.getExpiryTime()); + oldItem.getExpiryTime(), + false); if (!newItemHdl) { return {}; } @@ -2553,7 +2611,7 @@ bool CacheAllocator::tryMovingForSlabRelease( // item is still valid. const std::string parentKey = oldItem.asChainedItem().getParentItem(compressor_).getKey().str(); - if (oldItem.isOnlyExclusive()) { + if (oldItem.isOnlyMoving()) { // If chained item no longer has a refcount, its parent is already // being released, so we abort this try to moving. return false; @@ -2583,10 +2641,10 @@ void CacheAllocator::evictForSlabRelease( stats_.numEvictionAttempts.inc(); // if the item is already in a state where only the exclusive bit is set, - // nothing needs to be done. We simply need to unmark exclusive bit and free + // nothing needs to be done. We simply need to call unmarkMoving and free // the item. - if (item.isOnlyExclusive()) { - item.unmarkExclusive(); + if (item.isOnlyMoving()) { + item.unmarkMoving(); const auto res = releaseBackToAllocator(item, RemoveContext::kNormal, false); XDCHECK(ReleaseRes::kReleased == res); @@ -2617,7 +2675,7 @@ void CacheAllocator::evictForSlabRelease( stats_.numEvictionSuccesses.inc(); // we have the last handle. no longer need to hold on to the exclusive bit - item.unmarkExclusive(); + item.unmarkMoving(); // manually decrement the refcount to call releaseBackToAllocator const auto ref = decRef(*owningHandle); @@ -2629,7 +2687,7 @@ void CacheAllocator::evictForSlabRelease( } if (shutDownInProgress_) { - item.unmarkExclusive(); + item.unmarkMoving(); allocator_->abortSlabRelease(ctx); throw exception::SlabReleaseAborted( folly::sformat("Slab Release aborted while trying to evict" @@ -2775,9 +2833,9 @@ CacheAllocator::advanceIteratorAndTryEvictChainedItem( template typename CacheAllocator::WriteHandle CacheAllocator::evictNormalItemForSlabRelease(Item& item) { - XDCHECK(item.isExclusive()); + XDCHECK(item.isMoving()); - if (item.isOnlyExclusive()) { + if (item.isOnlyMoving()) { return WriteHandle{}; } @@ -2789,7 +2847,7 @@ CacheAllocator::evictNormalItemForSlabRelease(Item& item) { // We remove the item from both access and mm containers. It doesn't matter // if someone else calls remove on the item at this moment, the item cannot - // be freed as long as we have the exclusive bit set. + // be freed as long as it's marked for eviction. auto handle = accessContainer_->removeIf(item, std::move(predicate)); if (!handle) { @@ -2813,7 +2871,7 @@ CacheAllocator::evictNormalItemForSlabRelease(Item& item) { template typename CacheAllocator::WriteHandle CacheAllocator::evictChainedItemForSlabRelease(ChainedItem& child) { - XDCHECK(child.isExclusive()); + XDCHECK(child.isMoving()); // We have the child marked as moving, but dont know anything about the // state of the parent. Unlike the case of regular eviction where we are @@ -2835,7 +2893,7 @@ CacheAllocator::evictChainedItemForSlabRelease(ChainedItem& child) { // check if the child is still in mmContainer and the expected parent is // valid under the chained item lock. if (expectedParent.getKey() != parentKey || !child.isInMMContainer() || - child.isOnlyExclusive() || + child.isOnlyMoving() || &expectedParent != &child.getParentItem(compressor_) || !expectedParent.isAccessible() || !expectedParent.hasChainedItem()) { return {}; @@ -2890,14 +2948,14 @@ CacheAllocator::evictChainedItemForSlabRelease(ChainedItem& child) { // In case someone else had removed this chained item from its parent by now // So we check again to see if it has been unlinked from its parent - if (!child.isInMMContainer() || child.isOnlyExclusive()) { + if (!child.isInMMContainer() || child.isOnlyMoving()) { return {}; } // check after removing from the MMContainer that the parent is still not // being marked as moving. If parent is moving, it will release the child // item and we will wait for that. - if (parentHandle->isExclusive()) { + if (parentHandle->isMoving()) { return {}; } @@ -2930,7 +2988,7 @@ bool CacheAllocator::removeIfExpired(const ReadHandle& handle) { } template -bool CacheAllocator::markExclusiveForSlabRelease( +bool CacheAllocator::markMovingForSlabRelease( const SlabReleaseContext& ctx, void* alloc, util::Throttler& throttler) { // MemoryAllocator::processAllocForRelease will execute the callback // if the item is not already free. So there are three outcomes here: @@ -2949,7 +3007,7 @@ bool CacheAllocator::markExclusiveForSlabRelease( // Since this callback is executed, the item is not yet freed itemFreed = false; Item* item = static_cast(memory); - if (item->markExclusive()) { + if (item->markMoving()) { markedMoving = true; } }; @@ -3148,6 +3206,8 @@ bool CacheAllocator::stopWorkers(std::chrono::seconds timeout) { success &= stopPoolResizer(timeout); success &= stopMemMonitor(timeout); success &= stopReaper(timeout); + success &= stopBackgroundEvictor(timeout); + success &= stopBackgroundPromoter(timeout); return success; } @@ -3403,6 +3463,8 @@ GlobalCacheStats CacheAllocator::getGlobalCacheStats() const { ret.nvmUpTime = currTime - nvmCacheState_.getCreationTime(); ret.nvmCacheEnabled = nvmCache_ ? nvmCache_->isEnabled() : false; ret.reaperStats = getReaperStats(); + ret.evictionStats = getBackgroundMoverStats(MoverDir::Evict); + ret.promotionStats = getBackgroundMoverStats(MoverDir::Promote); ret.numActiveHandles = getNumActiveHandles(); ret.isNewRamCache = cacheCreationTime_ == cacheInstanceCreationTime_; @@ -3586,6 +3648,68 @@ bool CacheAllocator::startNewReaper( return true; } +template +auto CacheAllocator::getAssignedMemoryToBgWorker( + size_t evictorId, size_t numWorkers) { + std::vector asssignedMemory; + auto pools = filterCompactCachePools(allocator_->getPoolIds()); + for (const auto pid : pools) { + const auto& mpStats = getPool(pid).getStats(); + for (const auto cid : mpStats.classIds) { + if (backgroundWorkerId(pid, cid, numWorkers) == evictorId) { + asssignedMemory.emplace_back(pid, cid); + } + } + } + return asssignedMemory; +} + +template +bool CacheAllocator::startNewBackgroundEvictor( + std::chrono::milliseconds interval, + std::shared_ptr strategy, + size_t threads) { + XDCHECK(threads > 0); + backgroundEvictor_.resize(threads); + bool result = true; + + for (size_t i = 0; i < threads; i++) { + auto ret = startNewWorker("BackgroundEvictor" + std::to_string(i), + backgroundEvictor_[i], interval, strategy, + MoverDir::Evict); + result = result && ret; + + if (result) { + backgroundEvictor_[i]->setAssignedMemory( + getAssignedMemoryToBgWorker(i, backgroundEvictor_.size())); + } + } + return result; +} + +template +bool CacheAllocator::startNewBackgroundPromoter( + std::chrono::milliseconds interval, + std::shared_ptr strategy, + size_t threads) { + XDCHECK(threads > 0); + backgroundPromoter_.resize(threads); + bool result = true; + + for (size_t i = 0; i < threads; i++) { + auto ret = startNewWorker("BackgroundPromoter" + std::to_string(i), + backgroundPromoter_[i], interval, strategy, + MoverDir::Promote); + result = result && ret; + + if (result) { + backgroundPromoter_[i]->setAssignedMemory( + getAssignedMemoryToBgWorker(i, backgroundPromoter_.size())); + } + } + return result; +} + template bool CacheAllocator::stopPoolRebalancer( std::chrono::seconds timeout) { @@ -3613,6 +3737,29 @@ bool CacheAllocator::stopReaper(std::chrono::seconds timeout) { return stopWorker("Reaper", reaper_, timeout); } +template +bool CacheAllocator::stopBackgroundEvictor( + std::chrono::seconds timeout) { + bool result = true; + for (size_t i = 0; i < backgroundEvictor_.size(); i++) { + auto ret = stopWorker("BackgroundEvictor", backgroundEvictor_[i], timeout); + result = result && ret; + } + return result; +} + +template +bool CacheAllocator::stopBackgroundPromoter( + std::chrono::seconds timeout) { + bool result = true; + for (size_t i = 0; i < backgroundPromoter_.size(); i++) { + auto ret = + stopWorker("BackgroundPromoter", backgroundPromoter_[i], timeout); + result = result && ret; + } + return result; +} + template bool CacheAllocator::cleanupStrayShmSegments( const std::string& cacheDir, bool posix) { diff --git a/cachelib/allocator/CacheAllocator.h b/cachelib/allocator/CacheAllocator.h index ed0096390a..37b91af1ce 100644 --- a/cachelib/allocator/CacheAllocator.h +++ b/cachelib/allocator/CacheAllocator.h @@ -38,6 +38,7 @@ #include #pragma GCC diagnostic pop +#include "cachelib/allocator/BackgroundMover.h" #include "cachelib/allocator/CCacheManager.h" #include "cachelib/allocator/Cache.h" #include "cachelib/allocator/CacheAllocatorConfig.h" @@ -710,6 +711,11 @@ class CacheAllocator : public CacheBase { // @return the full usable size for this item uint32_t getUsableSize(const Item& item) const; + // gets the allocation class assigned to BG worker + auto getAssignedMemoryToBgWorker(size_t evictorId, size_t numWorkers); + bool shouldWakeupBgEvictor(PoolId pid, ClassId cid); + size_t backgroundWorkerId(PoolId pid, ClassId cid, size_t numWorkers); + // Get a random item from memory // This is useful for profiling and sampling cachelib managed memory // @@ -1054,6 +1060,15 @@ class CacheAllocator : public CacheBase { bool startNewReaper(std::chrono::milliseconds interval, util::Throttler::Config reaperThrottleConfig); + bool startNewBackgroundPromoter( + std::chrono::milliseconds interval, + std::shared_ptr strategy, + size_t threads); + bool startNewBackgroundEvictor( + std::chrono::milliseconds interval, + std::shared_ptr strategy, + size_t threads); + // Stop existing workers with a timeout bool stopPoolRebalancer(std::chrono::seconds timeout = std::chrono::seconds{ 0}); @@ -1062,6 +1077,10 @@ class CacheAllocator : public CacheBase { 0}); bool stopMemMonitor(std::chrono::seconds timeout = std::chrono::seconds{0}); bool stopReaper(std::chrono::seconds timeout = std::chrono::seconds{0}); + bool stopBackgroundEvictor( + std::chrono::seconds timeout = std::chrono::seconds{0}); + bool stopBackgroundPromoter( + std::chrono::seconds timeout = std::chrono::seconds{0}); // Set pool optimization to either true or false // @@ -1146,6 +1165,44 @@ class CacheAllocator : public CacheBase { return stats; } + // returns the background mover stats + BackgroundMoverStats getBackgroundMoverStats(MoverDir direction) const { + auto stats = BackgroundMoverStats{}; + if (direction == MoverDir::Evict) { + for (auto& bg : backgroundEvictor_) + stats += bg->getStats(); + } else if (direction == MoverDir::Promote) { + for (auto& bg : backgroundPromoter_) + stats += bg->getStats(); + } + return stats; + } + + std::map> getBackgroundMoverClassStats( + MoverDir direction) const { + std::map> stats; + + if (direction == MoverDir::Evict) { + for (auto& bg : backgroundEvictor_) { + for (auto& pid : bg->getClassStats()) { + for (auto& cid : pid.second) { + stats[pid.first][cid.first] += cid.second; + } + } + } + } else if (direction == MoverDir::Promote) { + for (auto& bg : backgroundPromoter_) { + for (auto& pid : bg->getClassStats()) { + for (auto& cid : pid.second) { + stats[pid.first][cid.first] += cid.second; + } + } + } + } + + return stats; + } + // return the LruType of an item typename MMType::LruType getItemLruType(const Item& item) const; @@ -1308,7 +1365,7 @@ class CacheAllocator : public CacheBase { private: // wrapper around Item's refcount and active handle tracking - FOLLY_ALWAYS_INLINE void incRef(Item& it); + FOLLY_ALWAYS_INLINE bool incRef(Item& it); FOLLY_ALWAYS_INLINE RefcountWithFlags::Value decRef(Item& it); // drops the refcount and if needed, frees the allocation back to the memory @@ -1407,6 +1464,7 @@ class CacheAllocator : public CacheBase { // @param creationTime Timestamp when this item was created // @param expiryTime set an expiry timestamp for the item (0 means no // expiration time). + // @param fromBgThread whether this is called from BG thread // // @return the handle for the item or an invalid handle(nullptr) if the // allocation failed. Allocation can fail if one such @@ -1420,7 +1478,8 @@ class CacheAllocator : public CacheBase { Key key, uint32_t size, uint32_t creationTime, - uint32_t expiryTime); + uint32_t expiryTime, + bool fromBgThread = false); // Allocate a chained item // @@ -1756,9 +1815,9 @@ class CacheAllocator : public CacheBase { // @return true when successfully marked as moving, // fasle when this item has already been freed - bool markExclusiveForSlabRelease(const SlabReleaseContext& ctx, - void* alloc, - util::Throttler& throttler); + bool markMovingForSlabRelease(const SlabReleaseContext& ctx, + void* alloc, + util::Throttler& throttler); // "Move" (by copying) the content in this item to another memory // location by invoking the move callback. @@ -1828,6 +1887,22 @@ class CacheAllocator : public CacheBase { stats().numReaperSkippedSlabs.add(slabsSkipped); } + // exposed for the background evictor to iterate through the memory and evict + // in batch. This should improve insertion path for tiered memory config + size_t traverseAndEvictItems(unsigned int pid, + unsigned int cid, + size_t batch) { + throw std::runtime_error("Not supported yet!"); + } + + // exposed for the background promoter to iterate through the memory and + // promote in batch. This should improve find latency + size_t traverseAndPromoteItems(unsigned int pid, + unsigned int cid, + size_t batch) { + throw std::runtime_error("Not supported yet!"); + } + // returns true if nvmcache is enabled and we should write this item to // nvmcache. bool shouldWriteToNvmCache(const Item& item); @@ -1937,7 +2012,7 @@ class CacheAllocator : public CacheBase { } static bool parentEvictForSlabReleasePredicate(const Item& item) { - return item.getRefCount() == 1 && !item.isExclusive(); + return item.getRefCount() == 1 && !item.isMoving(); } std::unique_ptr createDeserializer(); @@ -2056,6 +2131,10 @@ class CacheAllocator : public CacheBase { // free memory monitor std::unique_ptr memMonitor_; + // background evictor + std::vector>> backgroundEvictor_; + std::vector>> backgroundPromoter_; + // check whether a pool is a slabs pool std::array isCompactCachePool_{}; @@ -2100,6 +2179,7 @@ class CacheAllocator : public CacheBase { // Make this friend to give access to acquire and release friend ReadHandle; friend ReaperAPIWrapper; + friend BackgroundMoverAPIWrapper; friend class CacheAPIWrapperForNvm; friend class FbInternalRuntimeUpdateWrapper; friend class objcache2::ObjectCache; diff --git a/cachelib/allocator/CacheAllocatorConfig.h b/cachelib/allocator/CacheAllocatorConfig.h index ec44ff8467..31921ebe20 100644 --- a/cachelib/allocator/CacheAllocatorConfig.h +++ b/cachelib/allocator/CacheAllocatorConfig.h @@ -25,6 +25,7 @@ #include #include +#include "cachelib/allocator/BackgroundMoverStrategy.h" #include "cachelib/allocator/Cache.h" #include "cachelib/allocator/MM2Q.h" #include "cachelib/allocator/MemoryMonitor.h" @@ -263,6 +264,18 @@ class CacheAllocatorConfig { std::chrono::seconds ccacheInterval, uint32_t ccacheStepSizePercent); + // Enable the background evictor - scans a tier to look for objects + // to evict to the next tier + CacheAllocatorConfig& enableBackgroundEvictor( + std::shared_ptr backgroundMoverStrategy, + std::chrono::milliseconds regularInterval, + size_t threads); + + CacheAllocatorConfig& enableBackgroundPromoter( + std::shared_ptr backgroundMoverStrategy, + std::chrono::milliseconds regularInterval, + size_t threads); + // This enables an optimization for Pool rebalancing and resizing. // The rough idea is to ensure only the least useful items are evicted when // we move slab memory around. Come talk to Cache Library team if you think @@ -337,6 +350,17 @@ class CacheAllocatorConfig { poolOptimizeStrategy != nullptr; } + // @return whether background evictor thread is enabled + bool backgroundEvictorEnabled() const noexcept { + return backgroundEvictorInterval.count() > 0 && + backgroundEvictorStrategy != nullptr; + } + + bool backgroundPromoterEnabled() const noexcept { + return backgroundPromoterInterval.count() > 0 && + backgroundPromoterStrategy != nullptr; + } + // @return whether memory monitor is enabled bool memMonitoringEnabled() const noexcept { return memMonitorConfig.mode != MemoryMonitor::Disabled && @@ -449,6 +473,18 @@ class CacheAllocatorConfig { // make any progress for the below threshold std::chrono::milliseconds slabReleaseStuckThreshold{std::chrono::seconds(60)}; + // rebalance to avoid alloc fialures. + std::shared_ptr backgroundEvictorStrategy; + std::shared_ptr backgroundPromoterStrategy; + // time interval to sleep between runs of the background evictor + std::chrono::milliseconds backgroundEvictorInterval{ + std::chrono::milliseconds{1000}}; + std::chrono::milliseconds backgroundPromoterInterval{ + std::chrono::milliseconds{1000}}; + + size_t backgroundEvictorThreads{1}; + size_t backgroundPromoterThreads{1}; + // time interval to sleep between iterations of pool size optimization, // for regular pools and compact caches std::chrono::seconds regularPoolOptimizeInterval{0}; @@ -588,6 +624,9 @@ class CacheAllocatorConfig { // CacheAllocator::startCacheWorkers() bool delayCacheWorkersStart{false}; + uint64_t evictorThreads{1}; + uint64_t promoterThreads{1}; + friend CacheT; private: @@ -924,6 +963,28 @@ CacheAllocatorConfig& CacheAllocatorConfig::enablePoolRebalancing( return *this; } +template +CacheAllocatorConfig& CacheAllocatorConfig::enableBackgroundEvictor( + std::shared_ptr strategy, + std::chrono::milliseconds interval, + size_t evictorThreads) { + backgroundEvictorStrategy = strategy; + backgroundEvictorInterval = interval; + backgroundEvictorThreads = evictorThreads; + return *this; +} + +template +CacheAllocatorConfig& CacheAllocatorConfig::enableBackgroundPromoter( + std::shared_ptr strategy, + std::chrono::milliseconds interval, + size_t promoterThreads) { + backgroundPromoterStrategy = strategy; + backgroundPromoterInterval = interval; + backgroundPromoterThreads = promoterThreads; + return *this; +} + template CacheAllocatorConfig& CacheAllocatorConfig::enablePoolResizing( std::shared_ptr resizeStrategy, diff --git a/cachelib/allocator/CacheItem-inl.h b/cachelib/allocator/CacheItem-inl.h index f59fa9d599..bf77b43aa5 100644 --- a/cachelib/allocator/CacheItem-inl.h +++ b/cachelib/allocator/CacheItem-inl.h @@ -148,15 +148,16 @@ std::string CacheItem::toString() const { return folly::sformat( "item: " "memory={}:raw-ref={}:size={}:key={}:hex-key={}:" - "isInMMContainer={}:isAccessible={}:isExclusive={}:references={}:ctime=" + "isInMMContainer={}:isAccessible={}:isMarkedForEviction={}:" + "isMoving={}:references={}:ctime=" "{}:" "expTime={}:updateTime={}:isNvmClean={}:isNvmEvicted={}:hasChainedItem=" "{}", this, getRefCountAndFlagsRaw(), getSize(), folly::humanify(getKey().str()), folly::hexlify(getKey()), - isInMMContainer(), isAccessible(), isExclusive(), getRefCount(), - getCreationTime(), getExpiryTime(), getLastAccessTime(), isNvmClean(), - isNvmEvicted(), hasChainedItem()); + isInMMContainer(), isAccessible(), isMarkedForEviction(), isMoving(), + getRefCount(), getCreationTime(), getExpiryTime(), getLastAccessTime(), + isNvmClean(), isNvmEvicted(), hasChainedItem()); } } @@ -217,23 +218,43 @@ bool CacheItem::isInMMContainer() const noexcept { } template -bool CacheItem::markExclusive() noexcept { - return ref_.markExclusive(); +bool CacheItem::markForEviction() noexcept { + return ref_.markForEviction(); } template -RefcountWithFlags::Value CacheItem::unmarkExclusive() noexcept { - return ref_.unmarkExclusive(); +RefcountWithFlags::Value CacheItem::unmarkForEviction() noexcept { + return ref_.unmarkForEviction(); } template -bool CacheItem::isExclusive() const noexcept { - return ref_.isExclusive(); +bool CacheItem::isMarkedForEviction() const noexcept { + return ref_.isMarkedForEviction(); } template -bool CacheItem::isOnlyExclusive() const noexcept { - return ref_.isOnlyExclusive(); +bool CacheItem::markForEvictionWhenMoving() { + return ref_.markForEvictionWhenMoving(); +} + +template +bool CacheItem::markMoving() { + return ref_.markMoving(); +} + +template +RefcountWithFlags::Value CacheItem::unmarkMoving() noexcept { + return ref_.unmarkMoving(); +} + +template +bool CacheItem::isMoving() const noexcept { + return ref_.isMoving(); +} + +template +bool CacheItem::isOnlyMoving() const noexcept { + return ref_.isOnlyMoving(); } template @@ -335,7 +356,8 @@ bool CacheItem::updateExpiryTime(uint32_t expiryTimeSecs) noexcept { // check for moving to make sure we are not updating the expiry time while at // the same time re-allocating the item with the old state of the expiry time // in moveRegularItem(). See D6852328 - if (isExclusive() || !isInMMContainer() || isChainedItem()) { + if (isMoving() || isMarkedForEviction() || !isInMMContainer() || + isChainedItem()) { return false; } // attempt to atomically update the value of expiryTime @@ -451,12 +473,14 @@ std::string CacheChainedItem::toString() const { return folly::sformat( "chained item: " "memory={}:raw-ref={}:size={}:parent-compressed-ptr={}:" - "isInMMContainer={}:isAccessible={}:isExclusive={}:references={}:ctime={}" + "isInMMContainer={}:isAccessible={}:isMarkedForEviction={}:" + "isMoving={}:references={}:ctime={}" ":" "expTime={}:updateTime={}", this, Item::getRefCountAndFlagsRaw(), Item::getSize(), cPtr.getRaw(), - Item::isInMMContainer(), Item::isAccessible(), Item::isExclusive(), - Item::getRefCount(), Item::getCreationTime(), Item::getExpiryTime(), + Item::isInMMContainer(), Item::isAccessible(), + Item::isMarkedForEviction(), Item::isMoving(), Item::getRefCount(), + Item::getCreationTime(), Item::getExpiryTime(), Item::getLastAccessTime()); } diff --git a/cachelib/allocator/CacheItem.h b/cachelib/allocator/CacheItem.h index 06136db032..afee315cbb 100644 --- a/cachelib/allocator/CacheItem.h +++ b/cachelib/allocator/CacheItem.h @@ -305,12 +305,17 @@ class CACHELIB_PACKED_ATTR CacheItem { */ RefcountWithFlags::Value getRefCountAndFlagsRaw() const noexcept; - FOLLY_ALWAYS_INLINE void incRef() { - if (LIKELY(ref_.incRef())) { - return; + // Increments item's ref count + // + // @return true on success, failure if item is marked as exclusive + // @throw exception::RefcountOverflow on ref count overflow + FOLLY_ALWAYS_INLINE bool incRef() { + try { + return ref_.incRef(); + } catch (exception::RefcountOverflow& e) { + throw exception::RefcountOverflow( + folly::sformat("{} item: {}", e.what(), toString())); } - throw exception::RefcountOverflow( - folly::sformat("Refcount maxed out. item: {}", toString())); } FOLLY_ALWAYS_INLINE RefcountWithFlags::Value decRef() { @@ -344,23 +349,43 @@ class CACHELIB_PACKED_ATTR CacheItem { /** * The following two functions corresond to whether or not an item is - * currently in the process of being moved. This happens during a slab - * rebalance, eviction or resize operation. + * currently in the process of being evicted. * - * An item can only be marked exclusive when `isInMMContainer` returns true. + * An item can only be marked exclusive when `isInMMContainer` returns true + * and item is not already exclusive nor moving and the ref count is 0. * This operation is atomic. * - * User can also query if an item "isOnlyExclusive". This returns true only - * if the refcount is 0 and only the exclusive bit is set. - * - * Unmarking exclusive does not depend on `isInMMContainer`. + * Unmarking exclusive does not depend on `isInMMContainer` * Unmarking exclusive will also return the refcount at the moment of * unmarking. */ - bool markExclusive() noexcept; - RefcountWithFlags::Value unmarkExclusive() noexcept; - bool isExclusive() const noexcept; - bool isOnlyExclusive() const noexcept; + bool markForEviction() noexcept; + RefcountWithFlags::Value unmarkForEviction() noexcept; + bool isMarkedForEviction() const noexcept; + + /** + * The following functions correspond to whether or not an item is + * currently in the processed of being moved. When moving, ref count + * is always >= 1. + * + * An item can only be marked moving when `isInMMContainer` returns true + * and item is not already exclusive nor moving. + * + * User can also query if an item "isOnlyMoving". This returns true only + * if the refcount is one and only the exclusive bit is set. + * + * Unmarking moving does not depend on `isInMMContainer` + * Unmarking moving will also return the refcount at the moment of + * unmarking. + */ + bool markMoving(); + RefcountWithFlags::Value unmarkMoving() noexcept; + bool isMoving() const noexcept; + bool isOnlyMoving() const noexcept; + + /** This function attempts to mark item as exclusive. + * Can only be called on the item that is moving.*/ + bool markForEvictionWhenMoving(); /** * Item cannot be marked both chained allocation and diff --git a/cachelib/allocator/CacheStats.h b/cachelib/allocator/CacheStats.h index fb9955b805..365f0c23ca 100644 --- a/cachelib/allocator/CacheStats.h +++ b/cachelib/allocator/CacheStats.h @@ -289,6 +289,26 @@ struct ReaperStats { uint64_t avgTraversalTimeMs{0}; }; +// Mover Stats +struct BackgroundMoverStats { + // the number of items this worker moved by looking at pools/classes stats + uint64_t numMovedItems{0}; + // number of times we went executed the thread //TODO: is this def correct? + uint64_t runCount{0}; + // total number of classes + uint64_t totalClasses{0}; + // eviction size + uint64_t totalBytesMoved{0}; + + BackgroundMoverStats& operator+=(const BackgroundMoverStats& rhs) { + numMovedItems += rhs.numMovedItems; + runCount += rhs.runCount; + totalClasses += rhs.totalClasses; + totalBytesMoved += rhs.totalBytesMoved; + return *this; + } +}; + // CacheMetadata type to export struct CacheMetadata { // allocator_version @@ -309,6 +329,11 @@ struct Stats; // Stats that apply globally in cache and // the ones that are aggregated over all pools struct GlobalCacheStats { + // background eviction stats + BackgroundMoverStats evictionStats; + + BackgroundMoverStats promotionStats; + // number of calls to CacheAllocator::find uint64_t numCacheGets{0}; diff --git a/cachelib/allocator/FreeThresholdStrategy.cpp b/cachelib/allocator/FreeThresholdStrategy.cpp new file mode 100644 index 0000000000..c01938e812 --- /dev/null +++ b/cachelib/allocator/FreeThresholdStrategy.cpp @@ -0,0 +1,39 @@ +/* + * Copyright (c) Intel and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "cachelib/allocator/FreeThresholdStrategy.h" + +#include + +namespace facebook { +namespace cachelib { + +FreeThresholdStrategy::FreeThresholdStrategy(double lowEvictionAcWatermark, + double highEvictionAcWatermark, + uint64_t maxEvictionBatch, + uint64_t minEvictionBatch) + : lowEvictionAcWatermark(lowEvictionAcWatermark), + highEvictionAcWatermark(highEvictionAcWatermark), + maxEvictionBatch(maxEvictionBatch), + minEvictionBatch(minEvictionBatch) {} + +std::vector FreeThresholdStrategy::calculateBatchSizes( + const CacheBase& cache, std::vector acVec) { + return {}; +} + +} // namespace cachelib +} // namespace facebook diff --git a/cachelib/allocator/FreeThresholdStrategy.h b/cachelib/allocator/FreeThresholdStrategy.h new file mode 100644 index 0000000000..a845c1027f --- /dev/null +++ b/cachelib/allocator/FreeThresholdStrategy.h @@ -0,0 +1,47 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "cachelib/allocator/BackgroundMoverStrategy.h" +#include "cachelib/allocator/Cache.h" + +namespace facebook { +namespace cachelib { + +// Free threshold strategy for background promotion worker. +// This strategy tries to keep certain percent of memory free +// at all times. +class FreeThresholdStrategy : public BackgroundMoverStrategy { + public: + FreeThresholdStrategy(double lowEvictionAcWatermark, + double highEvictionAcWatermark, + uint64_t maxEvictionBatch, + uint64_t minEvictionBatch); + ~FreeThresholdStrategy() {} + + std::vector calculateBatchSizes( + const CacheBase& cache, std::vector acVecs); + + private: + double lowEvictionAcWatermark{2.0}; + double highEvictionAcWatermark{5.0}; + uint64_t maxEvictionBatch{40}; + uint64_t minEvictionBatch{5}; +}; + +} // namespace cachelib +} // namespace facebook diff --git a/cachelib/allocator/Handle.h b/cachelib/allocator/Handle.h index a125ace1b7..11d2bed2be 100644 --- a/cachelib/allocator/Handle.h +++ b/cachelib/allocator/Handle.h @@ -242,8 +242,6 @@ struct ReadHandleImpl { return hdl; } - bool isWriteHandle() const { return false; } - protected: // accessor. Calling getInternal() on handle with isReady() == false blocks // the thread until the handle is ready. @@ -571,8 +569,6 @@ struct WriteHandleImpl : public ReadHandleImpl { // creating this item handle. WriteHandleImpl clone() const { return WriteHandleImpl{ReadHandle::clone()}; } - bool isWriteHandle() const { return true; } - // Friends friend ReadHandle; // Only CacheAllocator and NvmCache can create non-default constructed handles diff --git a/cachelib/allocator/PromotionStrategy.h b/cachelib/allocator/PromotionStrategy.h new file mode 100644 index 0000000000..97d3f7d073 --- /dev/null +++ b/cachelib/allocator/PromotionStrategy.h @@ -0,0 +1,48 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "cachelib/allocator/BackgroundMoverStrategy.h" +#include "cachelib/allocator/Cache.h" + +namespace facebook { +namespace cachelib { + +// Strategy for background promotion worker. +class PromotionStrategy : public BackgroundMoverStrategy { + public: + PromotionStrategy(uint64_t promotionAcWatermark, + uint64_t maxPromotionBatch, + uint64_t minPromotionBatch) + : promotionAcWatermark(promotionAcWatermark), + maxPromotionBatch(maxPromotionBatch), + minPromotionBatch(minPromotionBatch) {} + ~PromotionStrategy() {} + + std::vector calculateBatchSizes( + const CacheBase& cache, std::vector acVec) { + return {}; + } + + private: + double promotionAcWatermark{4.0}; + uint64_t maxPromotionBatch{40}; + uint64_t minPromotionBatch{5}; +}; + +} // namespace cachelib +} // namespace facebook diff --git a/cachelib/allocator/Refcount.h b/cachelib/allocator/Refcount.h index c60dea34f1..107e10735e 100644 --- a/cachelib/allocator/Refcount.h +++ b/cachelib/allocator/Refcount.h @@ -132,32 +132,28 @@ class FOLLY_PACK_ATTR RefcountWithFlags { RefcountWithFlags& operator=(RefcountWithFlags&&) = delete; // Bumps up the reference count only if the new count will be strictly less - // than or equal to the maxCount. - // @return true if refcount is bumped. false otherwise. - FOLLY_ALWAYS_INLINE bool incRef() noexcept { - Value* const refPtr = &refCount_; - unsigned int nCASFailures = 0; - constexpr bool isWeak = false; - Value oldVal = __atomic_load_n(refPtr, __ATOMIC_RELAXED); - - while (true) { - const Value newCount = oldVal + static_cast(1); - if (UNLIKELY((oldVal & kAccessRefMask) == (kAccessRefMask))) { - return false; + // than or equal to the maxCount and the item is not exclusive + // @return true if refcount is bumped. false otherwise (if item is exclusive) + // @throw exception::RefcountOverflow if new count would be greater than + // maxCount + FOLLY_ALWAYS_INLINE bool incRef() { + auto predicate = [](const Value curValue) { + Value bitMask = getAdminRef(); + + const bool exlusiveBitIsSet = curValue & bitMask; + if (UNLIKELY((curValue & kAccessRefMask) == (kAccessRefMask))) { + throw exception::RefcountOverflow("Refcount maxed out."); } - if (__atomic_compare_exchange_n(refPtr, &oldVal, newCount, isWeak, - __ATOMIC_ACQ_REL, __ATOMIC_RELAXED)) { - return true; - } + // Check if the item is not marked for eviction + return !exlusiveBitIsSet || ((curValue & kAccessRefMask) != 0); + }; - if ((++nCASFailures % 4) == 0) { - // this pause takes up to 40 clock cycles on intel and the lock cmpxchgl - // above should take about 100 clock cycles. we pause once every 400 - // cycles or so if we are extremely unlucky. - folly::asm_volatile_pause(); - } - } + auto newValue = [](const Value curValue) { + return (curValue + static_cast(1)); + }; + + return atomicUpdateValue(predicate, newValue); } // Bumps down the reference count @@ -167,33 +163,38 @@ class FOLLY_PACK_ATTR RefcountWithFlags { // @throw RefcountUnderflow when we are trying to decremenet from 0 // refcount and have a refcount leak. FOLLY_ALWAYS_INLINE Value decRef() { - Value* const refPtr = &refCount_; - unsigned int nCASFailures = 0; - constexpr bool isWeak = false; - - Value oldVal = __atomic_load_n(refPtr, __ATOMIC_RELAXED); - while (true) { - const Value newCount = oldVal - static_cast(1); - if ((oldVal & kAccessRefMask) == 0) { + auto predicate = [](const Value curValue) { + if ((curValue & kAccessRefMask) == 0) { throw exception::RefcountUnderflow( "Trying to decRef with no refcount. RefCount Leak!"); } + return true; + }; - if (__atomic_compare_exchange_n(refPtr, &oldVal, newCount, isWeak, - __ATOMIC_ACQ_REL, __ATOMIC_RELAXED)) { - return newCount & kRefMask; - } - if ((++nCASFailures % 4) == 0) { - // this pause takes up to 40 clock cycles on intel and the lock cmpxchgl - // above should take about 100 clock cycles. we pause once every 400 - // cycles or so if we are extremely unlucky - folly::asm_volatile_pause(); - } - } + Value retValue; + auto newValue = [&retValue](const Value curValue) { + retValue = (curValue - static_cast(1)); + return retValue; + }; + + auto updated = atomicUpdateValue(predicate, newValue); + XDCHECK(updated); + + return retValue & kRefMask; } - // Return refcount excluding control bits and flags - Value getAccessRef() const noexcept { return getRaw() & kAccessRefMask; } + // Return refcount excluding moving refcount, control bits and flags. + Value getAccessRef() const noexcept { + auto raw = getRaw(); + auto accessRef = raw & kAccessRefMask; + + if ((raw & getAdminRef()) && accessRef >= 1) { + // if item is moving, ignore the extra ref + return accessRef - static_cast(1); + } else { + return accessRef; + } + } // Return access ref and the admin ref bits Value getRefWithAccessAndAdmin() const noexcept { @@ -246,65 +247,160 @@ class FOLLY_PACK_ATTR RefcountWithFlags { } /** - * The following four functions are used to track whether or not - * an item is currently in the process of being moved. This happens during a - * slab rebalance or resize operation or during eviction. + * The following two functions correspond to whether or not an item is + * currently in the process of being evicted. * - * An item can only be marked exclusive when `isInMMContainer` returns true - * and the item is not yet marked as exclusive. This operation is atomic. + * An item that is marked for eviction prevents from obtaining a handle to + * the item (incRef() will return false). This guarantees that eviction of + * marked item will always suceed. * - * User can also query if an item "isOnlyExclusive". This returns true only - * if the refcount is 0 and only the exclusive bit is set. + * An item can only be marked for eviction when `isInMMContainer` returns true + * and item does not have `kExclusive` bit set and access ref count is 0. + * This operation is atomic. * - * Unmarking exclusive does not depend on `isInMMContainer`. - * Unmarking exclusive will also return the refcount at the moment of - * unmarking. + * When item is marked for eviction, `kExclusive` bit is set and ref count is + * zero. + * + * Unmarking for eviction clears the `kExclusive` bit. `unamrkForEviction` + * does not depend on `isInMMContainer` nor `isAccessible` */ - bool markExclusive() noexcept { - Value bitMask = getAdminRef(); - Value conditionBitMask = getAdminRef(); + bool markForEviction() noexcept { + Value linkedBitMask = getAdminRef(); + Value exclusiveBitMask = getAdminRef(); - Value* const refPtr = &refCount_; - unsigned int nCASFailures = 0; - constexpr bool isWeak = false; - Value curValue = __atomic_load_n(refPtr, __ATOMIC_RELAXED); - while (true) { - const bool flagSet = curValue & conditionBitMask; - const bool alreadyExclusive = curValue & bitMask; - if (!flagSet || alreadyExclusive) { + auto predicate = [linkedBitMask, exclusiveBitMask](const Value curValue) { + const bool unlinked = !(curValue & linkedBitMask); + const bool alreadyExclusive = curValue & exclusiveBitMask; + + if (unlinked || alreadyExclusive) { return false; } - - const Value newValue = curValue | bitMask; - if (__atomic_compare_exchange_n(refPtr, &curValue, newValue, isWeak, - __ATOMIC_ACQ_REL, __ATOMIC_RELAXED)) { - XDCHECK(newValue & conditionBitMask); - return true; + if ((curValue & kAccessRefMask) != 0) { + return false; } - if ((++nCASFailures % 4) == 0) { - // this pause takes up to 40 clock cycles on intel and the lock cmpxchgl - // above should take about 100 clock cycles. we pause once every 400 - // cycles or so if we are extremely unlucky. - folly::asm_volatile_pause(); - } - } + return true; + }; + + auto newValue = [exclusiveBitMask](const Value curValue) { + return curValue | exclusiveBitMask; + }; + + return atomicUpdateValue(predicate, newValue); } - Value unmarkExclusive() noexcept { + + Value unmarkForEviction() noexcept { + XDCHECK(isMarkedForEviction()); Value bitMask = ~getAdminRef(); return __atomic_and_fetch(&refCount_, bitMask, __ATOMIC_ACQ_REL) & kRefMask; } - bool isExclusive() const noexcept { - return getRaw() & getAdminRef(); + + bool isMarkedForEviction() const noexcept { + auto raw = getRaw(); + return (raw & getAdminRef()) && ((raw & kAccessRefMask) == 0); + } + + /** + * The following functions correspond to whether or not an item is + * currently in the processed of being moved. + * + * A `moving` item cannot be recycled nor freed to the allocator. It has + * to be unmarked first. + * + * When moving, internal ref count is always >= 1 and `kExclusive` bit is set + * getRefCount does not return the extra ref (it may return 0). + * + * An item can only be marked moving when `isInMMContainer` returns true + * and does not have `kExclusive` bit set. + * + * User can also query if an item "isOnlyMoving". This returns true only + * if the refcount is one and only the exlusive bit is set. + * + * Unmarking clears `kExclusive` bit and decreses the interanl refCount by 1. + * `unmarkMoving` does does not depend on `isInMMContainer` + */ + bool markMoving() { + Value linkedBitMask = getAdminRef(); + Value exclusiveBitMask = getAdminRef(); + + auto predicate = [linkedBitMask, exclusiveBitMask](const Value curValue) { + const bool unlinked = !(curValue & linkedBitMask); + const bool alreadyExclusive = curValue & exclusiveBitMask; + + if (unlinked || alreadyExclusive) { + return false; + } + if (UNLIKELY((curValue & kAccessRefMask) == (kAccessRefMask))) { + throw exception::RefcountOverflow("Refcount maxed out."); + } + + return true; + }; + + auto newValue = [exclusiveBitMask](const Value curValue) { + // Set exclusive flag and make the ref count non-zero (to distinguish + // from exclusive case). This extra ref will not be reported to the + // user + return (curValue + static_cast(1)) | exclusiveBitMask; + }; + + return atomicUpdateValue(predicate, newValue); + } + + Value unmarkMoving() noexcept { + XDCHECK(isMoving()); + auto predicate = [](const Value curValue) { + XDCHECK((curValue & kAccessRefMask) != 0); + return true; + }; + + Value retValue; + auto newValue = [&retValue](const Value curValue) { + retValue = + (curValue - static_cast(1)) & ~getAdminRef(); + return retValue; + }; + + auto updated = atomicUpdateValue(predicate, newValue); + XDCHECK(updated); + + return retValue & kRefMask; + } + + bool isMoving() const noexcept { + auto raw = getRaw(); + return (raw & getAdminRef()) && ((raw & kAccessRefMask) != 0); + } + + /** + * This function attempts to mark item for eviction. + * Can only be called on the item that is moving. + * + * Returns true and marks the item for eviction only if item isOnlyMoving. + * Leaves the item marked as moving and returns false otherwise. + */ + bool markForEvictionWhenMoving() { + XDCHECK(isMoving()); + + auto predicate = [](const Value curValue) { + return (curValue & kAccessRefMask) == 1; + }; + + auto newValue = [](const Value curValue) { + XDCHECK((curValue & kAccessRefMask) == 1); + return (curValue - static_cast(1)); + }; + + return atomicUpdateValue(predicate, newValue); } - bool isOnlyExclusive() const noexcept { - // An item is only exclusive when its refcount is zero and only the - // exclusive bit among all the control bits is set. This indicates an item - // is exclusive to the current thread. No other thread is allowed to - // do anything with it. + + bool isOnlyMoving() const noexcept { + // An item is only moving when its refcount is one and only the exclusive + // bit among all the control bits is set. This indicates an item is already + // on its way out of cache. auto ref = getRefWithAccessAndAdmin(); - bool anyOtherBitSet = ref & ~getAdminRef(); - if (anyOtherBitSet) { + Value valueWithoutExclusiveBit = ref & ~getAdminRef(); + if (valueWithoutExclusiveBit != 1) { return false; } return ref & getAdminRef(); @@ -370,6 +466,39 @@ class FOLLY_PACK_ATTR RefcountWithFlags { } private: + /** + * Helper function to modify refCount_ atomically. + * + * If predicate(currentValue) is true, then it atomically assigns result + * of newValueF(currentValue) to refCount_ and returns true. Otherwise + * returns false and leaves refCount_ unmodified. + */ + template + bool atomicUpdateValue(P&& predicate, F&& newValueF) { + Value* const refPtr = &refCount_; + unsigned int nCASFailures = 0; + constexpr bool isWeak = false; + Value curValue = __atomic_load_n(refPtr, __ATOMIC_RELAXED); + while (true) { + if (!predicate(curValue)) { + return false; + } + + const Value newValue = newValueF(curValue); + if (__atomic_compare_exchange_n(refPtr, &curValue, newValue, isWeak, + __ATOMIC_ACQ_REL, __ATOMIC_RELAXED)) { + return true; + } + + if ((++nCASFailures % 4) == 0) { + // this pause takes up to 40 clock cycles on intel and the lock cmpxchgl + // above should take about 100 clock cycles. we pause once every 400 + // cycles or so if we are extremely unlucky. + folly::asm_volatile_pause(); + } + } + } + template static Value getFlag() noexcept { static_assert(flagBit >= kNumAccessRefBits + kNumAdminRefBits, diff --git a/cachelib/allocator/datastruct/serialize/objects.thrift b/cachelib/allocator/datastruct/serialize/objects.thrift index bd2c8b79bc..223b804e5b 100644 --- a/cachelib/allocator/datastruct/serialize/objects.thrift +++ b/cachelib/allocator/datastruct/serialize/objects.thrift @@ -22,17 +22,17 @@ namespace cpp2 facebook.cachelib.serialization // Saved state for an SList struct SListObject { - 2: required i64 size, - 3: required i64 compressedHead, // Pointer to the head element + 2: required i64 size; + 3: required i64 compressedHead; // Pointer to the head element // TODO(bwatling): remove the default value and clean up SList::SList() once // we can rely on 'compressedTail' always being valid. - 4: i64 compressedTail = -1, // Pointer to the tail element + 4: i64 compressedTail = -1; // Pointer to the tail element } struct DListObject { - 1: required i64 compressedHead, - 2: required i64 compressedTail, - 3: required i64 size, + 1: required i64 compressedHead; + 2: required i64 compressedTail; + 3: required i64 size; } struct MultiDListObject { diff --git a/cachelib/allocator/datastruct/tests/test_objects.thrift b/cachelib/allocator/datastruct/tests/test_objects.thrift index 51af098510..4cb58b1886 100644 --- a/cachelib/allocator/datastruct/tests/test_objects.thrift +++ b/cachelib/allocator/datastruct/tests/test_objects.thrift @@ -20,6 +20,6 @@ namespace cpp2 facebook.cachelib.test_serialization // testing warm rolls from the old format to the new format. // TODO(bwatling): remove this when 'compressedTail' is always present. struct SListObjectNoCompressedTail { - 2: required i64 size, - 3: required i64 compressedHead, // Pointer to the head element + 2: required i64 size; + 3: required i64 compressedHead; // Pointer to the head element } diff --git a/cachelib/allocator/nvmcache/tests/NvmCacheTests.cpp b/cachelib/allocator/nvmcache/tests/NvmCacheTests.cpp index 7355627fea..ec74c51980 100644 --- a/cachelib/allocator/nvmcache/tests/NvmCacheTests.cpp +++ b/cachelib/allocator/nvmcache/tests/NvmCacheTests.cpp @@ -245,7 +245,13 @@ TEST_F(NvmCacheTest, EvictToNvmGetCheckCtime) { ASSERT_NE(nullptr, it); cache_->insertOrReplace(it); keyToCtime.insert({key, it->getCreationTime()}); + // Avoid any nvm eviction being dropped due to the race with still + // outstanding remove operation for insertion + if (i % 100 == 0) { + nvm.flushNvmCache(); + } } + nvm.flushNvmCache(); const auto nEvictions = this->evictionCount() - evictBefore; ASSERT_LT(0, nEvictions); @@ -331,6 +337,11 @@ TEST_F(NvmCacheTest, Delete) { auto it = nvm.allocate(pid, key, 15 * 1024); ASSERT_NE(nullptr, it); nvm.insertOrReplace(it); + // Avoid any nvm eviction being dropped due to the race with still + // outstanding remove operation for insertion + if (i % 100 == 0) { + nvm.flushNvmCache(); + } } nvm.flushNvmCache(); @@ -533,6 +544,11 @@ TEST_F(NvmCacheTest, NvmEvicted) { auto it = nvm.allocate(pid, key, allocSize); ASSERT_NE(nullptr, it); nvm.insertOrReplace(it); + // Avoid any nvm eviction being dropped due to the race with still + // outstanding remove operation for insertion + if (i % 100 == 0) { + nvm.flushNvmCache(); + } } nvm.flushNvmCache(); diff --git a/cachelib/allocator/serialize/objects.thrift b/cachelib/allocator/serialize/objects.thrift index 8d30ee8d83..61297cdf1d 100644 --- a/cachelib/allocator/serialize/objects.thrift +++ b/cachelib/allocator/serialize/objects.thrift @@ -23,136 +23,136 @@ include "cachelib/allocator/datastruct/serialize/objects.thrift" // make sure to communicate that with our users. struct CacheAllocatorMetadata { - 1: required i64 allocatorVersion, // version of cache alloctor - 2: i64 cacheCreationTime = 0, // time when the cache was created. - 3: required i64 accessType = 0, // default chained alloc - 4: required i64 mmType = 0, // default LRU - 5: map> fragmentationSize, - 6: list compactCachePools, - 7: i64 numPermanentItems, - 8: i64 numChainedParentItems, - 9: i64 numChainedChildItems, - 10: i64 ramFormatVersion = 0, // format version of ram cache - 11: i64 numAbortedSlabReleases = 0, // number of times slab release is aborted + 1: required i64 allocatorVersion; // version of cache alloctor + 2: i64 cacheCreationTime = 0; // time when the cache was created. + 3: required i64 accessType = 0; // default chained alloc + 4: required i64 mmType = 0; // default LRU + 5: map> fragmentationSize; + 6: list compactCachePools; + 7: i64 numPermanentItems; + 8: i64 numChainedParentItems; + 9: i64 numChainedChildItems; + 10: i64 ramFormatVersion = 0; // format version of ram cache + 11: i64 numAbortedSlabReleases = 0; // number of times slab release is aborted } struct NvmCacheMetadata { - 1: i64 nvmFormatVersion = 0, - 2: i64 creationTime = 0, - 3: bool safeShutDown = false, - 4: bool encryptionEnabled = false, - 5: bool truncateAllocSize = false, + 1: i64 nvmFormatVersion = 0; + 2: i64 creationTime = 0; + 3: bool safeShutDown = false; + 4: bool encryptionEnabled = false; + 5: bool truncateAllocSize = false; } struct CompactCacheMetadataObject { - 1: required i64 keySize, - 2: required i64 valueSize, + 1: required i64 keySize; + 2: required i64 valueSize; } struct CompactCacheAllocatorObject { - 1: required list chunks, - 2: required CompactCacheMetadataObject ccMetadata, + 1: required list chunks; + 2: required CompactCacheMetadataObject ccMetadata; } struct CompactCacheAllocatorManagerObject { - 1: required map allocators, + 1: required map allocators; } struct MMLruConfig { - 1: required i32 lruRefreshTime, - 2: required bool updateOnWrite, - 3: required i32 lruInsertionPointSpec, - 4: bool updateOnRead = true, - 5: bool tryLockUpdate = false, - 6: double lruRefreshRatio = 0.0, + 1: required i32 lruRefreshTime; + 2: required bool updateOnWrite; + 3: required i32 lruInsertionPointSpec; + 4: bool updateOnRead = true; + 5: bool tryLockUpdate = false; + 6: double lruRefreshRatio = 0.0; } struct MMLruObject { - 1: required MMLruConfig config, + 1: required MMLruConfig config; // number of evictions for this MM object. - 5: i64 evictions = 0, + 5: i64 evictions = 0; - 6: required i64 insertionPoint, - 7: required i64 tailSize, - 8: required DListObject lru, - 9: required i64 compressedInsertionPoint, + 6: required i64 insertionPoint; + 7: required i64 tailSize; + 8: required DListObject lru; + 9: required i64 compressedInsertionPoint; } struct MMLruCollection { - 1: required map> pools, + 1: required map> pools; } struct MM2QConfig { - 1: required i32 lruRefreshTime, - 2: required bool updateOnWrite, - 3: required i32 hotSizePercent, - 4: required i32 coldSizePercent, - 5: bool updateOnRead = true, - 6: bool tryLockUpdate = false, - 7: bool rebalanceOnRecordAccess = true, - 8: double lruRefreshRatio = 0.0, + 1: required i32 lruRefreshTime; + 2: required bool updateOnWrite; + 3: required i32 hotSizePercent; + 4: required i32 coldSizePercent; + 5: bool updateOnRead = true; + 6: bool tryLockUpdate = false; + 7: bool rebalanceOnRecordAccess = true; + 8: double lruRefreshRatio = 0.0; } struct MM2QObject { - 1: required MM2QConfig config, - 13: bool tailTrackingEnabled = false, + 1: required MM2QConfig config; + 13: bool tailTrackingEnabled = false; // number of evictions for this MM object. - 11: i64 evictions = 0, + 11: i64 evictions = 0; // Warm, hot and cold lrus - 12: required MultiDListObject lrus, + 12: required MultiDListObject lrus; } struct MM2QCollection { - 1: required map> pools, + 1: required map> pools; } struct MMTinyLFUConfig { - 1: required i32 lruRefreshTime, - 2: required bool updateOnWrite, - 3: required i32 windowToCacheSizeRatio, - 4: required i32 tinySizePercent, - 5: bool updateOnRead = true, - 6: bool tryLockUpdate = false, - 7: double lruRefreshRatio = 0.0, + 1: required i32 lruRefreshTime; + 2: required bool updateOnWrite; + 3: required i32 windowToCacheSizeRatio; + 4: required i32 tinySizePercent; + 5: bool updateOnRead = true; + 6: bool tryLockUpdate = false; + 7: double lruRefreshRatio = 0.0; } struct MMTinyLFUObject { - 1: required MMTinyLFUConfig config, + 1: required MMTinyLFUConfig config; // number of evictions for this MM object. - 2: i64 evictions = 0, + 2: i64 evictions = 0; // Warm, hot and cold lrus - 3: required MultiDListObject lrus, + 3: required MultiDListObject lrus; } struct MMTinyLFUCollection { - 1: required map> pools, + 1: required map> pools; } struct ChainedHashTableObject { // fields in ChainedHashTable::Config - 1: required i32 bucketsPower, - 2: required i32 locksPower, - 3: i64 numKeys, + 1: required i32 bucketsPower; + 2: required i32 locksPower; + 3: i64 numKeys; // this magic id ensures on a warm roll, user cannot // start the cache with a different hash function - 4: i32 hasherMagicId = 0, + 4: i32 hasherMagicId = 0; } struct MMTTLBucketObject { - 4: i64 expirationTime, - 5: i64 creationTime, - 6: required DListObject dList, + 4: i64 expirationTime; + 5: i64 creationTime; + 6: required DListObject dList; } struct TTLBucketCollection { - 1: required map buckets, - 2: i64 minEpoch = 0, - 3: i64 maxTTL = 0, - 4: i64 interval = 0, + 1: required map buckets; + 2: i64 minEpoch = 0; + 3: i64 maxTTL = 0; + 4: i64 interval = 0; } diff --git a/cachelib/allocator/tests/BaseAllocatorTest.h b/cachelib/allocator/tests/BaseAllocatorTest.h index d684545cb9..aa9d38a857 100644 --- a/cachelib/allocator/tests/BaseAllocatorTest.h +++ b/cachelib/allocator/tests/BaseAllocatorTest.h @@ -713,35 +713,29 @@ class BaseAllocatorTest : public AllocatorTest { auto handle = alloc.find("key"); ASSERT_NE(handle, nullptr); ASSERT_TRUE(isConst(handle->getMemory())); - ASSERT_EQ(handle.isWriteHandle(), false); // read handle clone auto handle2 = handle.clone(); ASSERT_TRUE(isConst(handle2->getMemory())); - ASSERT_EQ(handle2.isWriteHandle(), false); // upgrade a read handle to a write handle auto handle3 = std::move(handle).toWriteHandle(); ASSERT_FALSE(isConst(handle3->getMemory())); - ASSERT_EQ(handle3.isWriteHandle(), true); } { auto handle = alloc.findToWrite("key"); ASSERT_NE(handle, nullptr); ASSERT_FALSE(isConst(handle->getMemory())); - ASSERT_EQ(handle.isWriteHandle(), true); // write handle clone auto handle2 = handle.clone(); ASSERT_FALSE(isConst(handle2->getMemory())); - ASSERT_EQ(handle2.isWriteHandle(), true); // downgrade a write handle to a read handle ReadHandle handle3 = handle.clone(); ASSERT_NE(handle3, nullptr); ASSERT_TRUE(isConst(handle3->getMemory())); - ASSERT_EQ(handle3.isWriteHandle(), false); } { @@ -752,7 +746,7 @@ class BaseAllocatorTest : public AllocatorTest { // This is like doing a "clone" and setting it into wait context waitContext->set(alloc.find("key")); auto handle2 = std::move(handle).toWriteHandle(); - ASSERT_EQ(handle2.isWriteHandle(), true); + ASSERT_FALSE(isConst(handle2->getMemory())); } } diff --git a/cachelib/allocator/tests/ItemTest.cpp b/cachelib/allocator/tests/ItemTest.cpp index b0f3a2fdec..70dd1277fe 100644 --- a/cachelib/allocator/tests/ItemTest.cpp +++ b/cachelib/allocator/tests/ItemTest.cpp @@ -83,10 +83,20 @@ TEST(ItemTest, ExpiryTime) { EXPECT_EQ(tenMins, item->getConfiguredTTL()); // Test that writes fail while the item is moving - item->markExclusive(); + result = item->markMoving(); + EXPECT_TRUE(result); + result = item->updateExpiryTime(0); + EXPECT_FALSE(result); + item->unmarkMoving(); + + // Test that writes fail while the item is marked for eviction + item->markAccessible(); + result = item->markForEviction(); + EXPECT_TRUE(result); result = item->updateExpiryTime(0); EXPECT_FALSE(result); - item->unmarkExclusive(); + item->unmarkForEviction(); + item->unmarkAccessible(); // Test that writes fail while the item is not in an MMContainer item->unmarkInMMContainer(); diff --git a/cachelib/allocator/tests/MMTypeTest.h b/cachelib/allocator/tests/MMTypeTest.h index d38f6ce2c1..dbc55677ea 100644 --- a/cachelib/allocator/tests/MMTypeTest.h +++ b/cachelib/allocator/tests/MMTypeTest.h @@ -147,7 +147,9 @@ class MMTypeTest : public testing::Test { void testRecordAccessBasic(Config c); void testSerializationBasic(Config c); void testIterate(std::vector>& nodes, Container& c); + void testIterateHot(std::vector>& nodes, Container& c); void testMatch(std::string expected, Container& c); + void testMatchHot(std::string expected, Container& c); size_t getListSize(const Container& c, typename MMType::LruType list); void verifyIterationVariants(Container& c); }; diff --git a/cachelib/allocator/tests/NvmTestUtils.h b/cachelib/allocator/tests/NvmTestUtils.h index 6d6242aadf..cad96c41d4 100644 --- a/cachelib/allocator/tests/NvmTestUtils.h +++ b/cachelib/allocator/tests/NvmTestUtils.h @@ -27,7 +27,7 @@ namespace utils { using NavyConfig = navy::NavyConfig; inline NavyConfig getNvmTestConfig(const std::string& cacheDir) { NavyConfig config{}; - config.setSimpleFile(cacheDir + "/navy", 100 * 1024ULL * 1024ULL); + config.setSimpleFile(cacheDir + "/navy", 200 * 1024ULL * 1024ULL); config.setDeviceMetadataSize(4 * 1024 * 1024); config.setBlockSize(1024); config.setNavyReqOrderingShards(10); diff --git a/cachelib/allocator/tests/RefCountTest.cpp b/cachelib/allocator/tests/RefCountTest.cpp index b355a48a8e..1f31894ddc 100644 --- a/cachelib/allocator/tests/RefCountTest.cpp +++ b/cachelib/allocator/tests/RefCountTest.cpp @@ -30,6 +30,7 @@ class RefCountTest : public AllocTestBase { public: static void testMultiThreaded(); static void testBasic(); + static void testMarkForEvictionAndMoving(); }; void RefCountTest::testMultiThreaded() { @@ -81,7 +82,7 @@ void RefCountTest::testBasic() { ASSERT_EQ(0, ref.getRaw()); ASSERT_FALSE(ref.isInMMContainer()); ASSERT_FALSE(ref.isAccessible()); - ASSERT_FALSE(ref.isExclusive()); + ASSERT_FALSE(ref.isMoving()); ASSERT_FALSE(ref.template isFlagSet()); ASSERT_FALSE(ref.template isFlagSet()); @@ -89,7 +90,7 @@ void RefCountTest::testBasic() { ref.markInMMContainer(); ASSERT_TRUE(ref.isInMMContainer()); ASSERT_FALSE(ref.isAccessible()); - ASSERT_FALSE(ref.isExclusive()); + ASSERT_FALSE(ref.isMoving()); ASSERT_EQ(0, ref.getAccessRef()); ASSERT_FALSE(ref.template isFlagSet()); ASSERT_FALSE(ref.template isFlagSet()); @@ -105,13 +106,13 @@ void RefCountTest::testBasic() { // Incrementing past the max will fail auto rawRef = ref.getRaw(); - ASSERT_FALSE(ref.incRef()); + ASSERT_THROW(ref.incRef(), std::overflow_error); ASSERT_EQ(rawRef, ref.getRaw()); // Bumping up access ref shouldn't affect admin ref and flags ASSERT_TRUE(ref.isInMMContainer()); ASSERT_FALSE(ref.isAccessible()); - ASSERT_FALSE(ref.isExclusive()); + ASSERT_FALSE(ref.isMoving()); ASSERT_EQ(RefcountWithFlags::kAccessRefMask, ref.getAccessRef()); ASSERT_TRUE(ref.template isFlagSet()); ASSERT_FALSE(ref.template isFlagSet()); @@ -128,7 +129,7 @@ void RefCountTest::testBasic() { // Bumping down access ref shouldn't affect admin ref and flags ASSERT_TRUE(ref.isInMMContainer()); ASSERT_FALSE(ref.isAccessible()); - ASSERT_FALSE(ref.isExclusive()); + ASSERT_FALSE(ref.isMoving()); ASSERT_EQ(0, ref.getAccessRef()); ASSERT_TRUE(ref.template isFlagSet()); ASSERT_FALSE(ref.template isFlagSet()); @@ -136,7 +137,7 @@ void RefCountTest::testBasic() { ref.template unSetFlag(); ASSERT_TRUE(ref.isInMMContainer()); ASSERT_FALSE(ref.isAccessible()); - ASSERT_FALSE(ref.isExclusive()); + ASSERT_FALSE(ref.isMoving()); ASSERT_EQ(0, ref.getAccessRef()); ASSERT_FALSE(ref.template isFlagSet()); ASSERT_FALSE(ref.template isFlagSet()); @@ -145,33 +146,104 @@ void RefCountTest::testBasic() { ASSERT_EQ(0, ref.getRaw()); ASSERT_FALSE(ref.isInMMContainer()); ASSERT_FALSE(ref.isAccessible()); - ASSERT_FALSE(ref.isExclusive()); + ASSERT_FALSE(ref.isMoving()); ASSERT_EQ(0, ref.getAccessRef()); ASSERT_FALSE(ref.template isFlagSet()); ASSERT_FALSE(ref.template isFlagSet()); // conditionally set flags - ASSERT_FALSE((ref.markExclusive())); + ASSERT_FALSE((ref.markMoving())); ref.markInMMContainer(); - ASSERT_TRUE((ref.markExclusive())); - ASSERT_FALSE((ref.isOnlyExclusive())); + // only first one succeeds + ASSERT_TRUE((ref.markMoving())); + ASSERT_FALSE((ref.markMoving())); ref.unmarkInMMContainer(); + ref.template setFlag(); - // Have no other admin refcount but with a flag still means "isOnlyExclusive" - ASSERT_TRUE((ref.isOnlyExclusive())); + // Have no other admin refcount but with a flag still means "isOnlyMoving" + ASSERT_TRUE((ref.isOnlyMoving())); - // Set some flags and verify that "isOnlyExclusive" does not care about flags + // Set some flags and verify that "isOnlyMoving" does not care about flags ref.markIsChainedItem(); ASSERT_TRUE(ref.isChainedItem()); - ASSERT_TRUE((ref.isOnlyExclusive())); + ASSERT_TRUE((ref.isOnlyMoving())); ref.unmarkIsChainedItem(); ASSERT_FALSE(ref.isChainedItem()); - ASSERT_TRUE((ref.isOnlyExclusive())); + ASSERT_TRUE((ref.isOnlyMoving())); +} + +void RefCountTest::testMarkForEvictionAndMoving() { + { + // cannot mark for eviction when not in MMContainer + RefcountWithFlags ref; + ASSERT_FALSE(ref.markForEviction()); + } + + { + // can mark for eviction when in MMContainer + // and unmarkForEviction return value contains admin bits + RefcountWithFlags ref; + ref.markInMMContainer(); + ASSERT_TRUE(ref.markForEviction()); + ASSERT_TRUE(ref.unmarkForEviction() > 0); + } + + { + // cannot mark for eviction when moving + RefcountWithFlags ref; + ref.markInMMContainer(); + + ASSERT_TRUE(ref.markMoving()); + ASSERT_FALSE(ref.markForEviction()); + + ref.unmarkInMMContainer(); + auto ret = ref.unmarkMoving(); + ASSERT_EQ(ret, 0); + } + + { + // cannot mark moving when marked for eviction + RefcountWithFlags ref; + ref.markInMMContainer(); + + ASSERT_TRUE(ref.markForEviction()); + ASSERT_FALSE(ref.markMoving()); + + ref.unmarkInMMContainer(); + auto ret = ref.unmarkForEviction(); + ASSERT_EQ(ret, 0); + } + + { + // can mark moving when ref count > 0 + RefcountWithFlags ref; + ref.markInMMContainer(); + + ref.incRef(); + + ASSERT_TRUE(ref.markMoving()); + + ref.unmarkInMMContainer(); + auto ret = ref.unmarkMoving(); + ASSERT_EQ(ret, 1); + } + + { + // cannot mark for eviction when ref count > 0 + RefcountWithFlags ref; + ref.markInMMContainer(); + + ref.incRef(); + ASSERT_FALSE(ref.markForEviction()); + } } } // namespace TEST_F(RefCountTest, MutliThreaded) { testMultiThreaded(); } TEST_F(RefCountTest, Basic) { testBasic(); } +TEST_F(RefCountTest, MarkForEvictionAndMoving) { + testMarkForEvictionAndMoving(); +} } // namespace tests } // namespace cachelib } // namespace facebook diff --git a/cachelib/benchmarks/DataTypeBench.thrift b/cachelib/benchmarks/DataTypeBench.thrift index a1f16f4b7e..94d53ade81 100644 --- a/cachelib/benchmarks/DataTypeBench.thrift +++ b/cachelib/benchmarks/DataTypeBench.thrift @@ -17,11 +17,9 @@ namespace cpp2 facebook.cachelib.datatypebench struct StdMap { - 1: required map m, + 1: required map m; } struct StdUnorderedMap { - 1: required map - (cpp.template = "std::unordered_map") - m, + 1: required map (cpp.template = "std::unordered_map") m; } diff --git a/cachelib/cachebench/cache/Cache-inl.h b/cachelib/cachebench/cache/Cache-inl.h index ed8bfd1b04..bebc4f1b13 100644 --- a/cachelib/cachebench/cache/Cache-inl.h +++ b/cachelib/cachebench/cache/Cache-inl.h @@ -624,13 +624,15 @@ Stats Cache::getStats() const { const auto rebalanceStats = cache_->getSlabReleaseStats(); const auto navyStats = cache_->getNvmCacheStatsMap().toMap(); - ret.numEvictions = aggregate.numEvictions(); - ret.numItems = aggregate.numItems(); - ret.evictAttempts = cacheStats.evictionAttempts; - ret.allocAttempts = cacheStats.allocAttempts; - ret.allocFailures = cacheStats.allocFailures; + ret.backgndEvicStats.nEvictedItems = cacheStats.evictionStats.numMovedItems; + ret.backgndEvicStats.nTraversals = cacheStats.evictionStats.runCount; + ret.backgndEvicStats.nClasses = cacheStats.evictionStats.totalClasses; + ret.backgndEvicStats.evictionSize = cacheStats.evictionStats.totalBytesMoved; + + ret.backgndPromoStats.nPromotedItems = + cacheStats.promotionStats.numMovedItems; + ret.backgndPromoStats.nTraversals = cacheStats.promotionStats.runCount; - ret.numCacheGets = cacheStats.numCacheGets; ret.numCacheGetMiss = cacheStats.numCacheGetMiss; ret.numCacheEvictions = cacheStats.numCacheEvictions; ret.numRamDestructorCalls = cacheStats.numRamDestructorCalls; @@ -677,6 +679,11 @@ Stats Cache::getStats() const { ret.nvmCounters = cache_->getNvmCacheStatsMap().toMap(); } + ret.backgroundEvictionClasses = + cache_->getBackgroundMoverClassStats(MoverDir::Evict); + ret.backgroundPromotionClasses = + cache_->getBackgroundMoverClassStats(MoverDir::Promote); + // nvm stats from navy if (!isRamOnly() && !navyStats.empty()) { auto lookup = [&navyStats](const std::string& key) { @@ -794,8 +801,17 @@ void Cache::setUint64ToItem(WriteHandle& handle, template void Cache::setStringItem(WriteHandle& handle, const std::string& str) { - auto ptr = reinterpret_cast(getMemory(handle)); - std::memcpy(ptr, str.data(), std::min(str.size(), getSize(handle))); + auto dataSize = getSize(handle); + if (dataSize < 1) + return; + + auto ptr = reinterpret_cast(getMemory(handle)); + std::strncpy(ptr, str.c_str(), dataSize); + + // Make sure the copied string ends with null char + if (str.size() + 1 > dataSize) { + ptr[dataSize - 1] = '\0'; + } } template diff --git a/cachelib/cachebench/cache/CacheStats.h b/cachelib/cachebench/cache/CacheStats.h index d6c9e53584..efc2658ebf 100644 --- a/cachelib/cachebench/cache/CacheStats.h +++ b/cachelib/cachebench/cache/CacheStats.h @@ -25,7 +25,33 @@ DECLARE_bool(report_api_latency); namespace facebook { namespace cachelib { namespace cachebench { + +struct BackgroundEvictionStats { + // the number of items this worker evicted by looking at pools/classes stats + uint64_t nEvictedItems{0}; + + // number of times we went executed the thread //TODO: is this def correct? + uint64_t nTraversals{0}; + + // number of classes + uint64_t nClasses{0}; + + // size of evicted items + uint64_t evictionSize{0}; +}; + +struct BackgroundPromotionStats { + // the number of items this worker evicted by looking at pools/classes stats + uint64_t nPromotedItems{0}; + + // number of times we went executed the thread //TODO: is this def correct? + uint64_t nTraversals{0}; +}; + struct Stats { + BackgroundEvictionStats backgndEvicStats; + BackgroundPromotionStats backgndPromoStats; + uint64_t numEvictions{0}; uint64_t numItems{0}; @@ -105,6 +131,9 @@ struct Stats { // cachebench. std::unordered_map nvmCounters; + std::map> backgroundEvictionClasses; + std::map> backgroundPromotionClasses; + // errors from the nvm engine. std::unordered_map nvmErrors; @@ -125,6 +154,14 @@ struct Stats { << std::endl; out << folly::sformat("RAM Evictions : {:,}", numEvictions) << std::endl; + auto foreachAC = [&](auto& map, auto cb) { + for (auto& pidStat : map) { + for (auto& cidStat : pidStat.second) { + cb(pidStat.first, cidStat.first, cidStat.second); + } + } + }; + for (auto pid = 0U; pid < poolUsageFraction.size(); pid++) { out << folly::sformat("Fraction of pool {:,} used : {:.2f}", pid, poolUsageFraction[pid]) @@ -161,6 +198,42 @@ struct Stats { } } + if (!backgroundEvictionClasses.empty() && + backgndEvicStats.nEvictedItems > 0) { + out << "== Class Background Eviction Counters Map ==" << std::endl; + foreachAC(backgroundEvictionClasses, + [&](auto pid, auto cid, auto evicted) { + out << folly::sformat("pid{:2} cid{:4} evicted: {:4}", pid, + cid, evicted) + << std::endl; + }); + + out << folly::sformat("Background Evicted Items : {:,}", + backgndEvicStats.nEvictedItems) + << std::endl; + out << folly::sformat("Background Evictor Traversals : {:,}", + backgndEvicStats.nTraversals) + << std::endl; + } + + if (!backgroundPromotionClasses.empty() && + backgndPromoStats.nPromotedItems > 0) { + out << "== Class Background Promotion Counters Map ==" << std::endl; + foreachAC(backgroundPromotionClasses, + [&](auto pid, auto cid, auto promoted) { + out << folly::sformat("pid{:2} cid{:4} promoted: {:4}", pid, + cid, promoted) + << std::endl; + }); + + out << folly::sformat("Background Promoted Items : {:,}", + backgndPromoStats.nPromotedItems) + << std::endl; + out << folly::sformat("Background Promoter Traversals : {:,}", + backgndPromoStats.nTraversals) + << std::endl; + } + if (numNvmGets > 0 || numNvmDeletes > 0 || numNvmPuts > 0) { const double ramHitRatio = invertPctFn(numCacheGetMiss, numCacheGets); const double nvmHitRatio = invertPctFn(numNvmGetMiss, numNvmGets); diff --git a/cachelib/cachebench/runner/AsyncCacheStressor.h b/cachelib/cachebench/runner/AsyncCacheStressor.h index 5b50db43b4..830b503795 100644 --- a/cachelib/cachebench/runner/AsyncCacheStressor.h +++ b/cachelib/cachebench/runner/AsyncCacheStressor.h @@ -287,6 +287,10 @@ class AsyncCacheStressor : public Stressor { ++stats.get; auto lock = chainedItemAcquireUniqueLock(*key); + // This was moved outside the lambda, as otherwise gcc-8.x crashes with an + // internal compiler error here (suspected regression in folly). + XDCHECK(req->sizeBegin + 1 != req->sizeEnd); + auto onReadyFn = [&, req, key, l = std::move(lock), pid](auto hdl) { WriteHandle wHdl; if (hdl == nullptr) { @@ -303,7 +307,6 @@ class AsyncCacheStressor : public Stressor { } else { wHdl = std::move(hdl).toWriteHandle(); } - XDCHECK(req->sizeBegin + 1 != req->sizeEnd); bool chainSuccessful = false; for (auto j = req->sizeBegin + 1; j != req->sizeEnd; j++) { ++stats.addChained; diff --git a/cachelib/cachebench/runner/CacheStressor.h b/cachelib/cachebench/runner/CacheStressor.h index 4ea54b1f14..d2433a734e 100644 --- a/cachelib/cachebench/runner/CacheStressor.h +++ b/cachelib/cachebench/runner/CacheStressor.h @@ -254,9 +254,7 @@ class CacheStressor : public Stressor { } if (!itemValue.empty()) { - // Add the null character to ensure this is a proper c string. - // TODO(T141356292): Clean this up to avoid allocating a new string - cache_->setStringItem(handle, itemValue + "\0"); + cache_->setStringItem(handle, itemValue); } else { cache_->setStringItem(handle, hardcodedString_); } diff --git a/cachelib/cachebench/util/NandWrites.cpp b/cachelib/cachebench/util/NandWrites.cpp index ae82aca65c..370ddfa2b6 100644 --- a/cachelib/cachebench/util/NandWrites.cpp +++ b/cachelib/cachebench/util/NandWrites.cpp @@ -400,6 +400,7 @@ uint64_t nandWriteBytes(const folly::StringPiece& deviceName, const folly::StringPiece&)>> vendorMap{{"samsung", samsungWriteBytes}, {"mz1lb960hbjr-", samsungWriteBytes}, + {"mzol23t8hcls-", samsungWriteBytes}, // The Samsung PM983a doesn't include Samsung in the model // number at this time, but it's a Samsung device. {"liteon", liteonWriteBytes}, diff --git a/cachelib/cachebench/util/tests/NandWritesTest.cpp b/cachelib/cachebench/util/tests/NandWritesTest.cpp index 0002e8a837..af09593f41 100644 --- a/cachelib/cachebench/util/tests/NandWritesTest.cpp +++ b/cachelib/cachebench/util/tests/NandWritesTest.cpp @@ -240,6 +240,96 @@ TEST_F(NandWritesTest, nandWriteBytes_handlesSamsungPM983aDevice) { EXPECT_EQ(nandWriteBytes("nvme1n1", kNvmePath, mockFactory_), 35061362294784); } +TEST_F(NandWritesTest, nandWriteBytes_handlesSamsungPM9A3Device) { + constexpr auto& kListOutput = R"EOF({ + "Devices" : [ + { + "DevicePath" : "/dev/nvme0n1", + "Firmware" : "P1FB007", + "Index" : 0, + "NameSpace" : 1, + "ModelNumber" : "MTFDHBA512TCK", + "ProductName" : "Non-Volatile memory controller: Micron Technology Inc Device 0x5410", + "SerialNumber" : " 21062E6B8061", + "UsedBytes" : 512110190592, + "MaximumLBA" : 1000215216, + "PhysicalSize" : 512110190592, + "SectorSize" : 512 + }, + { + "DevicePath" : "/dev/nvme1n1", + "Firmware" : "GDA82F2Q", + "Index" : 1, + "NameSpace" : 1, + "ModelNumber" : "MZOL23T8HCLS-00AFB", + "ProductName" : "Unknown device", + "SerialNumber" : "S5X9NG0T116005", + "UsedBytes" : 104910848, + "MaximumLBA" : 918149526, + "PhysicalSize" : 3760740458496, + "SectorSize" : 4096 + }, + { + "DevicePath" : "/dev/nvme2n1", + "Firmware" : "GDA82F2Q", + "Index" : 2, + "NameSpace" : 1, + "ModelNumber" : "MZOL23T8HCLS-00AFB", + "ProductName" : "Unknown device", + "SerialNumber" : "S5X9NG0T116027", + "UsedBytes" : 0, + "MaximumLBA" : 918149526, + "PhysicalSize" : 3760740458496, + "SectorSize" : 4096 + } + ] +})EOF"; + + constexpr auto& kSmartLogOutput = R"EOF( +[015:000] PhysicallyWrittenBytes : 241393664 +[031:016] Physically Read Bytes : 106217472 +[037:032] Bad NAND Block Count (Raw Value) : 0 +[039:038] Bad NAND Block Count (Normalized Value) : 100 +[047:040] Uncorrectable Read Error Count : 0 +[055:048] Soft ECC Error Count : 0 +[059:056] SSD End to end Correction Count (Detected Errors) : 0 +[063:060] SSD End to end Correction Count (Corrected Errors): 0 +[064:064] System Data Percentage Used : 0 +[068:065] User Data Erase Count (Min) : 0 +[072:069] User Data Erase Count (Max) : 1 +[080:073] Refresh Count : 0 +[086:081] Program Fail Count (Raw Value) : 0 +[088:087] Program Fail Count (Normalized Value) : 100 +[094:089] User Data Erase Fail Count (Raw Value) : 0 +[096:095] User Data Erase Fail Count (Normalized Value) : 100 +[102:097] System Area Erase Fail Count (Raw Value) : 0 +[104:103] System Area Erase Fail Count (Normalized value) : 100 +[105:105] Thermal Throttling Status : 0 +[106:106] Thermal Throttling Count : 0 +[108:107] PHY Error Count : 0 +[110:109] Bad DLLP Count : 0 +[112:111] Bad TLP Count : 0 +[114:113] Reserved : 0 +[118:115] Incomplete Shutdowns : 0 +[119:119] % Free Blocks : 96 +[121:120] PCIe Correctable Error Count (RTS) : 0 +[123:122] PCIe Correctable Error Count (RRS) : 0 +[131:124] XOR Recovery Count : 0 +[137:132] Bad System NAND block count (Raw Value) : 0 +[139:138] Bad System NAND block count (Normalized Value) : 100 +[141:140] Capacitor Health : 163 +[157:142] Endurance Estimate : 28862181 +[165:158] Security Version Number : 4294967296 +[167:166] Log Page Version : 1 +)EOF"; + + mockFactory_->expectedCommands( + {{{kNvmePath, "list", "-o", "json"}, kListOutput}, + {{kNvmePath, "samsung", "vs-smart-add-log", "/dev/nvme1n1"}, + kSmartLogOutput}}); + EXPECT_EQ(nandWriteBytes("nvme1n1", kNvmePath, mockFactory_), 241393664); +} + TEST_F(NandWritesTest, nandWriteBytes_handlesSeagateDevice) { constexpr auto& kListOutput = R"EOF({ "Devices" : [ diff --git a/cachelib/cachebench/workload/KVReplayGenerator.h b/cachelib/cachebench/workload/KVReplayGenerator.h index 4b12970081..a9124e2bd7 100644 --- a/cachelib/cachebench/workload/KVReplayGenerator.h +++ b/cachelib/cachebench/workload/KVReplayGenerator.h @@ -230,10 +230,10 @@ inline bool KVReplayGenerator::parseRequest(const std::string& line, // Set op const auto& op = fields[SampleFields::OP]; - // TODO only memcache optypes are supported - if (!op.compare("GET")) { + // TODO implement GET_LEASE and SET_LEASE emulations + if (!op.compare("GET") || !op.compare("GET_LEASE")) { req->req_.setOp(OpType::kGet); - } else if (!op.compare("SET")) { + } else if (!op.compare("SET") || !op.compare("SET_LEASE")) { req->req_.setOp(OpType::kSet); } else if (!op.compare("DELETE")) { req->req_.setOp(OpType::kDel); diff --git a/cachelib/cachebench/workload/tests/KVReplayGeneratorTest.cpp b/cachelib/cachebench/workload/tests/KVReplayGeneratorTest.cpp index 72a55a4020..16e4e52060 100644 --- a/cachelib/cachebench/workload/tests/KVReplayGeneratorTest.cpp +++ b/cachelib/cachebench/workload/tests/KVReplayGeneratorTest.cpp @@ -56,6 +56,18 @@ struct TraceEntry { size_t expKeySize = std::max(keySize_, reqKey.size()); expKeySize = std::min(expKeySize, 256); ASSERT_EQ(reqKey.size(), expKeySize); + ASSERT_EQ(req.req_.getOp(), getOpType()); + } + + OpType getOpType() { + if (!op_.compare("GET") || !op_.compare("GET_LEASE")) { + return OpType::kGet; + } else if (!op_.compare("SET") || !op_.compare("SET_LEASE")) { + return OpType::kSet; + } else if (!op_.compare("DELETE")) { + return OpType::kDel; + } + return OpType::kSize; } std::string key_; @@ -86,8 +98,11 @@ TEST(KVReplayGeneratorTest, BasicFormat) { // ,,,,, {7, "GET", 0, 2, std::nullopt, true}, {7, "GET", 0, 2, 50, true}, + {7, "GET_LEASE", 0, 2, 50, true}, {20, "SET", 100, 35, std::nullopt, true}, {20, "SET", 100, 35, 3600, true}, + {20, "SAT", 100, 35, 3600, false}, // invalid op name + {20, "SET_LEASE", 100, 35, 3600, true}, {7, "GET", 0, 0, std::nullopt, false}, // invalid op count {7, "GET", 0, 0, 600, false}, // invalid op count {1024, "SET", 100, 35, 300, true}, // key truncated diff --git a/cachelib/experimental/objcache/ObjectCachePersistence.thrift b/cachelib/experimental/objcache/ObjectCachePersistence.thrift index fcddb82ef5..e5acba4d54 100644 --- a/cachelib/experimental/objcache/ObjectCachePersistence.thrift +++ b/cachelib/experimental/objcache/ObjectCachePersistence.thrift @@ -17,9 +17,9 @@ namespace cpp2 facebook.cachelib.objcache.serialization struct Item { - 1: byte poolId - 2: i32 creationTime, - 3: i32 expiryTime, - 4: string key, - 5: string payload, + 1: byte poolId; + 2: i32 creationTime; + 3: i32 expiryTime; + 4: string key; + 5: string payload; } diff --git a/cachelib/experimental/objcache/tests/ThriftCustomAllocator.thrift b/cachelib/experimental/objcache/tests/ThriftCustomAllocator.thrift index 97f82d466a..b1610b3aad 100644 --- a/cachelib/experimental/objcache/tests/ThriftCustomAllocator.thrift +++ b/cachelib/experimental/objcache/tests/ThriftCustomAllocator.thrift @@ -24,22 +24,26 @@ struct UseSimpleCustomAllocator { // A template type like map needs to use "cpp.template" to specify a replacement template 1: map< + // A concrete type like string needs to use "cpp.type" to specify a replacement type + string ( + cpp.use_allocator, + cpp.type = "facebook::cachelib::objcache::test::TestString", + ), + string ( + cpp.use_allocator, + cpp.type = "facebook::cachelib::objcache::test::TestString", + ) + > ( + cpp.use_allocator, + cpp.template = "facebook::cachelib::objcache::test::TestMap", + ) m; - // A concrete type like string needs to use "cpp.type" to specify a replacement type - string - (cpp.use_allocator, - cpp.type = "facebook::cachelib::objcache::test::TestString"), - - string - (cpp.use_allocator, - cpp.type = "facebook::cachelib::objcache::test::TestString") - - > (cpp.use_allocator, - cpp.template = "facebook::cachelib::objcache::test::TestMap") m; - - // Native types or types that do not allocate memory do NOT need custom allocator - 2: i32 m2; -} (cpp.allocator="facebook::cachelib::objcache::test::ScopedTestAllocator", cpp.allocator_via="m") + // Native types or types that do not allocate memory do NOT need custom allocator + 2: i32 m2; +} ( + cpp.allocator = "facebook::cachelib::objcache::test::ScopedTestAllocator", + cpp.allocator_via = "m", +) // TODO: thrift allocator propagation behavior is broken. Right now, for the following // myObj1 = myObj2; // even if the allocator copy-assignment propagation is false, myObj2's @@ -50,17 +54,21 @@ struct UseSimpleCustomAllocator { union UnionWithCustomAllocator { 1: map< - i32, - string - (cpp.use_allocator, - cpp.type = "facebook::cachelib::objcache::test::TestString") - > (cpp.use_allocator, - cpp.template = "facebook::cachelib::objcache::test::TestMap")m1; - 2: string - (cpp.use_allocator, - cpp.type = "facebook::cachelib::objcache::test::TestString") m2; + i32, + string ( + cpp.use_allocator, + cpp.type = "facebook::cachelib::objcache::test::TestString", + ) + > ( + cpp.use_allocator, + cpp.template = "facebook::cachelib::objcache::test::TestMap", + ) m1; + 2: string ( + cpp.use_allocator, + cpp.type = "facebook::cachelib::objcache::test::TestString", + ) m2; 3: i32 m3; -} (cpp.allocator="facebook::cachelib::objcache::test::ScopedTestAllocator") +} (cpp.allocator = "facebook::cachelib::objcache::test::ScopedTestAllocator") // TODO: even though thrift union does not support allocator. We still need to // annotate it with allocator so it has a `get_allocator()` method so // that when deserializing it will be able to pass an allocator an inner @@ -89,11 +97,14 @@ union UnionWithCustomAllocator { // } struct UseTwoF14Maps { - 1: map - (cpp.use_allocator, - cpp.template = "facebook::cachelib::objcache::test::TestFollyF14FastMap") m1; - 2: map - (cpp.use_allocator, - cpp.template = "facebook::cachelib::objcache::test::TestFollyF14FastMap") m2; -} (cpp.allocator= - "facebook::cachelib::objcache::test::TestF14TemplateAllocator>") + 1: map ( + cpp.use_allocator, + cpp.template = "facebook::cachelib::objcache::test::TestFollyF14FastMap", + ) m1; + 2: map ( + cpp.use_allocator, + cpp.template = "facebook::cachelib::objcache::test::TestFollyF14FastMap", + ) m2; +} ( + cpp.allocator = "facebook::cachelib::objcache::test::TestF14TemplateAllocator>", +) diff --git a/cachelib/experimental/objcache2/ObjectCache-inl.h b/cachelib/experimental/objcache2/ObjectCache-inl.h index 345a27d528..9f1b91631d 100644 --- a/cachelib/experimental/objcache2/ObjectCache-inl.h +++ b/cachelib/experimental/objcache2/ObjectCache-inl.h @@ -128,8 +128,8 @@ std::shared_ptr ObjectCache::find(folly::StringPiece key) { succL1Lookups_.inc(); auto ptr = found->template getMemoryAs()->objectPtr; - // Just release the handle. Cache destorys object when all handles released. - auto deleter = [h = std::move(found)](const T*) {}; + // Use custom deleter + auto deleter = Deleter(std::move(found)); return std::shared_ptr(reinterpret_cast(ptr), std::move(deleter)); } @@ -146,19 +146,20 @@ std::shared_ptr ObjectCache::findToWrite( succL1Lookups_.inc(); auto ptr = found->template getMemoryAs()->objectPtr; - // Just release the handle. Cache destorys object when all handles released. - auto deleter = [h = std::move(found)](T*) {}; + // Use custom deleter + auto deleter = Deleter(std::move(found)); return std::shared_ptr(reinterpret_cast(ptr), std::move(deleter)); } template template -std::pair::AllocStatus, std::shared_ptr> +std::tuple::AllocStatus, + std::shared_ptr, + std::shared_ptr> ObjectCache::insertOrReplace(folly::StringPiece key, std::unique_ptr object, size_t objectSize, - uint32_t ttlSecs, - std::shared_ptr* replacedPtr) { + uint32_t ttlSecs) { if (config_.objectSizeTrackingEnabled && objectSize == 0) { throw std::invalid_argument( "Object size tracking is enabled but object size is set to be 0."); @@ -176,7 +177,8 @@ ObjectCache::insertOrReplace(folly::StringPiece key, allocateFromL1(key, ttlSecs, 0 /* use current time as creationTime */); if (!handle) { insertErrors_.inc(); - return {AllocStatus::kAllocError, std::shared_ptr(std::move(object))}; + return {AllocStatus::kAllocError, std::shared_ptr(std::move(object)), + nullptr}; } // We don't release the object here because insertOrReplace could throw when // the replaced item is out of refcount; in this case, the object isn't @@ -187,21 +189,17 @@ ObjectCache::insertOrReplace(folly::StringPiece key, auto replaced = this->l1Cache_->insertOrReplace(handle); + std::shared_ptr replacedPtr = nullptr; if (replaced) { replaces_.inc(); - if (replacedPtr) { - auto itemPtr = reinterpret_cast(replaced->getMemory()); - // Just release the handle. Cache destorys object when all handles - // released. - auto deleter = [h = std::move(replaced)](T*) {}; - *replacedPtr = std::shared_ptr( - reinterpret_cast(itemPtr->objectPtr), std::move(deleter)); - } + auto itemPtr = reinterpret_cast(replaced->getMemory()); + // Just release the handle. Cache destorys object when all handles + // released. + auto deleter = [h = std::move(replaced)](T*) {}; + replacedPtr = std::shared_ptr(reinterpret_cast(itemPtr->objectPtr), + std::move(deleter)); } - // Just release the handle. Cache destorys object when all handles released. - auto deleter = [h = std::move(handle)](T*) {}; - // update total object size if (config_.objectSizeTrackingEnabled) { totalObjectSizeBytes_.fetch_add(objectSize, std::memory_order_relaxed); @@ -209,7 +207,11 @@ ObjectCache::insertOrReplace(folly::StringPiece key, // Release the object as it has been successfully inserted to the cache. object.release(); - return {AllocStatus::kSuccess, std::shared_ptr(ptr, std::move(deleter))}; + + // Use custom deleter + auto deleter = Deleter(std::move(handle)); + return {AllocStatus::kSuccess, std::shared_ptr(ptr, std::move(deleter)), + replacedPtr}; } template @@ -254,8 +256,8 @@ ObjectCache::insert(folly::StringPiece key, object.release(); } - // Just release the handle. Cache destorys object when all handles released. - auto deleter = [h = std::move(handle)](T*) {}; + // Use custom deleter + auto deleter = Deleter(std::move(handle)); return {success ? AllocStatus::kSuccess : AllocStatus::kKeyAlreadyExists, std::shared_ptr(ptr, std::move(deleter))}; } diff --git a/cachelib/experimental/objcache2/ObjectCache.h b/cachelib/experimental/objcache2/ObjectCache.h index 85abac068e..1253213284 100644 --- a/cachelib/experimental/objcache2/ObjectCache.h +++ b/cachelib/experimental/objcache2/ObjectCache.h @@ -94,6 +94,43 @@ class ObjectCache : public ObjectCacheBase { // make constructor private, but constructable by std::make_unique struct InternalConstructor {}; + template + class Deleter { + public: + using ReadHandle = typename AllocatorT::ReadHandle; + using WriteHandle = typename AllocatorT::WriteHandle; + using Handle = std::variant; + + explicit Deleter(typename AllocatorT::ReadHandle&& hdl) + : hdl_(std::move(hdl)) {} + explicit Deleter(typename AllocatorT::WriteHandle&& hdl) + : hdl_(std::move(hdl)) {} + + void operator()(T*) { + // Just release the handle. + // Cache destorys object when all handles released. + std::holds_alternative(hdl_) + ? std::get(hdl_).reset() + : std::get(hdl_).reset(); + } + + WriteHandle& getWriteHandleRef() { + if (std::holds_alternative(hdl_)) { + hdl_ = std::move(std::get(hdl_)).toWriteHandle(); + } + return std::get(hdl_); + } + + ReadHandle& getReadHandleRef() { + return std::holds_alternative(hdl_) + ? std::get(hdl_) + : std::get(hdl_); + } + + private: + Handle hdl_; + }; + public: using ItemDestructor = std::function; using Key = KAllocation::Key; @@ -146,22 +183,20 @@ class ObjectCache : public ObjectCacheBase { // if objectSizeTracking is enabled, a non-zero value must // be passed. // @param ttlSecs object expiring seconds. - // @param replacedPtr a pointer to a shared_ptr, if it is not nullptr it will - // be assigned to the replaced object. // // @throw cachelib::exception::RefcountOverflow if the item we are replacing // is already out of refcounts. // @throw std::invalid_argument if objectSizeTracking is enabled but // objectSize is 0. - // @return a pair of allocation status and shared_ptr of newly inserted - // object. + // @return a tuple of allocation status, shared_ptr of newly inserted + // object and shared_ptr of old object that has been replaced (nullptr + // if no replacement happened) template - std::pair> insertOrReplace( - folly::StringPiece key, - std::unique_ptr object, - size_t objectSize = 0, - uint32_t ttlSecs = 0, - std::shared_ptr* replacedPtr = nullptr); + std::tuple, std::shared_ptr> + insertOrReplace(folly::StringPiece key, + std::unique_ptr object, + size_t objectSize = 0, + uint32_t ttlSecs = 0); // Insert the object into the cache with given key. If the key exists in the // cache, the new object won't be inserted. @@ -232,6 +267,63 @@ class ObjectCache : public ObjectCacheBase { : sizeController_->getCurrentEntriesLimit(); } + // Get the expiry timestamp of the object + // @param object object shared pointer returned from ObjectCache APIs + // + // @return the expiry timestamp in seconds of the object + // 0 if object is nullptr + template + uint32_t getExpiryTimeSec(const std::shared_ptr& object) const { + if (object == nullptr) { + return 0; + } + return getReadHandleRefInternal(object)->getExpiryTime(); + } + + // Get the configured TTL of the object + // @param object object shared pointer returned from ObjectCache APIs + // + // @return the configured TTL in seconds of the object + // 0 if object is nullptr + template + std::chrono::seconds getConfiguredTtl( + const std::shared_ptr& object) const { + if (object == nullptr) { + return std::chrono::seconds{0}; + } + return getReadHandleRefInternal(object)->getConfiguredTTL(); + } + + // Update the expiry timestamp of an object + // + // @param object object shared pointer returned from ObjectCache APIs + // @param expiryTimeSecs the expiryTime in seconds to update + // + // @return boolean indicating whether expiry time was successfully updated + template + bool updateExpiryTimeSec(std::shared_ptr& object, + uint32_t expiryTimeSecs) { + if (object == nullptr) { + return false; + } + return getWriteHandleRefInternal(object)->updateExpiryTime( + expiryTimeSecs); + } + + // Update expiry time to @ttl seconds from now. + // + // @param object object shared pointer returned from ObjectCache APIs + // @param ttl TTL in seconds (from now) + // + // @return boolean indicating whether TTL was successfully extended + template + bool extendTtl(std::shared_ptr& object, std::chrono::seconds ttl) { + if (object == nullptr) { + return false; + } + return getWriteHandleRefInternal(object)->extendTTL(ttl); + } + protected: // Serialize cache allocator config for exporting to Scuba std::map serializeConfigParams() const override; @@ -272,6 +364,28 @@ class ObjectCache : public ObjectCacheBase { bool stopSizeController(std::chrono::seconds timeout = std::chrono::seconds{ 0}); + // Get a ReadHandle reference from the object shared_ptr + template + typename AllocatorT::ReadHandle& getReadHandleRefInternal( + const std::shared_ptr& object) const { + auto* deleter = std::get_deleter>(object); + XDCHECK(deleter != nullptr); + auto& hdl = deleter->getReadHandleRef(); + XDCHECK(hdl != nullptr); + return hdl; + } + + // Get a WriteHandle reference from the object shared_ptr + template + typename AllocatorT::WriteHandle& getWriteHandleRefInternal( + std::shared_ptr& object) { + auto* deleter = std::get_deleter>(object); + XDCHECK(deleter != nullptr); + auto& hdl = deleter->getWriteHandleRef(); + XDCHECK(hdl != nullptr); + return hdl; + } + // Config passed to the cache. Config config_{}; diff --git a/cachelib/experimental/objcache2/persistence/Serialization.h b/cachelib/experimental/objcache2/persistence/Serialization.h index cccb414b45..4edad88e4b 100644 --- a/cachelib/experimental/objcache2/persistence/Serialization.h +++ b/cachelib/experimental/objcache2/persistence/Serialization.h @@ -68,9 +68,9 @@ struct ObjectDeserializer { Deserializer deserializer{reinterpret_cast(payload.begin()), reinterpret_cast(payload.end())}; auto ptr = std::make_unique(deserializer.deserialize()); - auto [allocStatus, _] = + auto res = objCache_.insertOrReplace(key, std::move(ptr), objectSize, ttlSecs); - return allocStatus == ObjectCache::AllocStatus::kSuccess; + return std::get<0>(res) == ObjectCache::AllocStatus::kSuccess; } // cache key of the object to be deserialized diff --git a/cachelib/experimental/objcache2/persistence/persistent_data.thrift b/cachelib/experimental/objcache2/persistence/persistent_data.thrift index 486e130a16..1d310693f5 100644 --- a/cachelib/experimental/objcache2/persistence/persistent_data.thrift +++ b/cachelib/experimental/objcache2/persistence/persistent_data.thrift @@ -17,12 +17,12 @@ namespace cpp2 facebook.cachelib.objcache2.persistence struct Item { - 1: string key, - 2: string payload, - 3: i32 objectSize, - 4: i32 expiryTime, + 1: string key; + 2: string payload; + 3: i32 objectSize; + 4: i32 expiryTime; } struct Metadata { - 1: i32 threadCount, + 1: i32 threadCount; } diff --git a/cachelib/experimental/objcache2/tests/ObjectCacheTest.cpp b/cachelib/experimental/objcache2/tests/ObjectCacheTest.cpp index 0bcc120de9..701654cd1c 100644 --- a/cachelib/experimental/objcache2/tests/ObjectCacheTest.cpp +++ b/cachelib/experimental/objcache2/tests/ObjectCacheTest.cpp @@ -206,12 +206,12 @@ class ObjectCacheTest : public ::testing::Test { foo->a = 1; foo->b = 2; foo->c = 3; - auto res = objcache->insertOrReplace("Foo", std::move(foo)); - EXPECT_EQ(ObjectCache::AllocStatus::kSuccess, res.first); - ASSERT_NE(nullptr, res.second); - EXPECT_EQ(1, res.second->a); - EXPECT_EQ(2, res.second->b); - EXPECT_EQ(3, res.second->c); + auto [allocRes, ptr, _] = objcache->insertOrReplace("Foo", std::move(foo)); + EXPECT_EQ(ObjectCache::AllocStatus::kSuccess, allocRes); + ASSERT_NE(nullptr, ptr); + EXPECT_EQ(1, ptr->a); + EXPECT_EQ(2, ptr->b); + EXPECT_EQ(3, ptr->c); auto found2 = objcache->template find("Foo"); ASSERT_NE(nullptr, found2); @@ -238,7 +238,7 @@ class ObjectCacheTest : public ::testing::Test { foo->b = 2; foo->c = 3; auto res1 = objcache->insertOrReplace("Foo", std::move(foo)); - EXPECT_EQ(ObjectCache::AllocStatus::kSuccess, res1.first); + EXPECT_EQ(ObjectCache::AllocStatus::kSuccess, std::get<0>(res1)); auto found1 = objcache->template find("Foo"); ASSERT_NE(nullptr, found1); @@ -251,7 +251,7 @@ class ObjectCacheTest : public ::testing::Test { foo2->e = 5; foo2->f = 6; auto res2 = objcache->insertOrReplace("Foo2", std::move(foo2)); - EXPECT_EQ(ObjectCache::AllocStatus::kSuccess, res2.first); + EXPECT_EQ(ObjectCache::AllocStatus::kSuccess, std::get<0>(res2)); auto found2 = objcache->template find("Foo2"); ASSERT_NE(nullptr, found2); @@ -272,7 +272,7 @@ class ObjectCacheTest : public ::testing::Test { foo4->b = 2; foo4->c = 3; auto res1 = objcache->insertOrReplace("Foo4", std::move(foo4)); - EXPECT_EQ(ObjectCache::AllocStatus::kSuccess, res1.first); + EXPECT_EQ(ObjectCache::AllocStatus::kSuccess, std::get<0>(res1)); auto found1 = objcache->template find("Foo4"); ASSERT_NE(nullptr, found1); @@ -285,7 +285,7 @@ class ObjectCacheTest : public ::testing::Test { foo5->e = 5; foo5->f = 6; auto res2 = objcache->insertOrReplace("Foo5", std::move(foo5)); - EXPECT_EQ(ObjectCache::AllocStatus::kSuccess, res2.first); + EXPECT_EQ(ObjectCache::AllocStatus::kSuccess, std::get<0>(res2)); auto found2 = objcache->template find("Foo5"); ASSERT_NE(nullptr, found2); @@ -385,11 +385,14 @@ class ObjectCacheTest : public ::testing::Test { foo1->a = 1; foo1->b = 2; foo1->c = 3; - std::shared_ptr replaced; - auto res = - objcache->insertOrReplace("Foo", std::move(foo1), 0, 0, &replaced); - EXPECT_EQ(ObjectCache::AllocStatus::kSuccess, res.first); - EXPECT_EQ(nullptr, replaced); + + auto [res1, ptr1, replaced1] = + objcache->insertOrReplace("Foo", std::move(foo1)); + EXPECT_EQ(ObjectCache::AllocStatus::kSuccess, res1); + EXPECT_EQ(1, ptr1->a); + EXPECT_EQ(2, ptr1->b); + EXPECT_EQ(3, ptr1->c); + EXPECT_EQ(nullptr, replaced1); auto found1 = objcache->template find("Foo"); ASSERT_NE(nullptr, found1); @@ -401,12 +404,16 @@ class ObjectCacheTest : public ::testing::Test { foo2->a = 10; foo2->b = 20; foo2->c = 30; - res = objcache->insertOrReplace("Foo", std::move(foo2), 0, 0, &replaced); - EXPECT_EQ(ObjectCache::AllocStatus::kSuccess, res.first); - ASSERT_NE(nullptr, replaced); - EXPECT_EQ(1, replaced->a); - EXPECT_EQ(2, replaced->b); - EXPECT_EQ(3, replaced->c); + auto [res2, ptr2, replaced2] = + objcache->insertOrReplace("Foo", std::move(foo2)); + EXPECT_EQ(ObjectCache::AllocStatus::kSuccess, res2); + EXPECT_EQ(10, ptr2->a); + EXPECT_EQ(20, ptr2->b); + EXPECT_EQ(30, ptr2->c); + ASSERT_NE(nullptr, replaced2); + EXPECT_EQ(1, replaced2->a); + EXPECT_EQ(2, replaced2->b); + EXPECT_EQ(3, replaced2->c); auto found2 = objcache->template find("Foo"); ASSERT_NE(nullptr, found2); @@ -497,7 +504,7 @@ class ObjectCacheTest : public ::testing::Test { // replace foo1 with foo2 { auto res = objcache->insertOrReplace("Foo", std::move(foo2), foo2Size); - ASSERT_EQ(ObjectCache::AllocStatus::kSuccess, res.first); + ASSERT_EQ(ObjectCache::AllocStatus::kSuccess, std::get<0>(res)); auto found = objcache->template find("Foo"); ASSERT_NE(nullptr, found); @@ -879,6 +886,69 @@ class ObjectCacheTest : public ::testing::Test { } } + void testGetTtl() { + const uint32_t ttlSecs = 600; + + ObjectCacheConfig config; + config.setCacheName("test").setCacheCapacity(10'000).setItemDestructor( + [&](ObjectCacheDestructorData data) { data.deleteObject(); }); + auto objcache = ObjectCache::create(config); + + auto before = util::getCurrentTimeSec(); + std::this_thread::sleep_for(std::chrono::seconds{3}); + objcache->insertOrReplace("Foo", std::make_unique(), 0 /*object size*/, + ttlSecs); + + // lookup via find API + auto found1 = objcache->template find("Foo"); + ASSERT_NE(nullptr, found1); + + // get TTL info + EXPECT_EQ(ttlSecs, objcache->getConfiguredTtl(found1).count()); + EXPECT_LE(before + ttlSecs, objcache->getExpiryTimeSec(found1)); + + // lookup via findToWrite API + auto found2 = objcache->template findToWrite("Foo"); + ASSERT_NE(nullptr, found2); + + // get TTL info + EXPECT_EQ(ttlSecs, objcache->getConfiguredTtl(found2).count()); + EXPECT_LE(before + ttlSecs, objcache->getExpiryTimeSec(found2)); + } + + void testUpdateTtl() { + const uint32_t ttlSecs = 600; + + ObjectCacheConfig config; + config.setCacheName("test").setCacheCapacity(10'000).setItemDestructor( + [&](ObjectCacheDestructorData data) { data.deleteObject(); }); + auto objcache = ObjectCache::create(config); + + auto insertionTime = util::getCurrentTimeSec(); + objcache->insertOrReplace("Foo", std::make_unique(), 0 /*object size*/, + ttlSecs); + + auto found = objcache->template find("Foo"); + ASSERT_NE(nullptr, found); + + // get TTL info + EXPECT_EQ(ttlSecs, objcache->getConfiguredTtl(found).count()); + EXPECT_LE(insertionTime + ttlSecs, objcache->getExpiryTimeSec(found)); + + // update expiry time + auto currExpTime = objcache->getExpiryTimeSec(found); + EXPECT_TRUE(objcache->updateExpiryTimeSec(found, currExpTime + ttlSecs)); + EXPECT_EQ(2 * ttlSecs, objcache->getConfiguredTtl(found).count()); + EXPECT_EQ(currExpTime + ttlSecs, objcache->getExpiryTimeSec(found)); + + // extend TTL + auto now = util::getCurrentTimeSec(); + std::this_thread::sleep_for(std::chrono::seconds{3}); + EXPECT_TRUE(objcache->extendTtl(found, std::chrono::seconds(3 * ttlSecs))); + EXPECT_LE(now + ttlSecs, objcache->getExpiryTimeSec(found)); + EXPECT_LE(3 * ttlSecs, objcache->getConfiguredTtl(found).count()); + } + void testMultithreadReplace() { // Sanity test to see if insertOrReplace across multiple // threads are safe. @@ -1072,6 +1142,32 @@ class ObjectCacheTest : public ::testing::Test { fs[i].join(); } } + + void testMultithreadUpdateTtl() { + // Sanity test to see if update TTL across multiple + // threads is safe. + ObjectCacheConfig config; + config.setCacheName("test").setCacheCapacity(10'000).setItemDestructor( + [&](ObjectCacheDestructorData data) { data.deleteObject(); }); + auto objcache = ObjectCache::create(config); + objcache->insertOrReplace("key", std::make_unique(), 0, 60); + + auto runUpdateTtlOps = [&] { + for (int i = 0; i < 2000; i++) { + auto found = objcache->template find("key"); + auto configuredTtlSecs = objcache->getConfiguredTtl(found).count(); + objcache->extendTtl(found, std::chrono::seconds{configuredTtlSecs}); + } + }; + + std::vector ts; + for (int i = 0; i < 10; i++) { + ts.push_back(std::thread{runUpdateTtlOps}); + } + for (int i = 0; i < 10; i++) { + ts[i].join(); + } + } }; using AllocatorTypes = ::testing::TypestestPersistenceHighLoad(); } +TYPED_TEST(ObjectCacheTest, GetTtl) { this->testGetTtl(); } +TYPED_TEST(ObjectCacheTest, UpdateTtl) { this->testUpdateTtl(); } + TYPED_TEST(ObjectCacheTest, MultithreadReplace) { this->testMultithreadReplace(); } @@ -1128,6 +1227,9 @@ TYPED_TEST(ObjectCacheTest, MultithreadFindAndEviction) { TYPED_TEST(ObjectCacheTest, MultithreadFindAndReplaceWith10Shards) { this->testMultithreadFindAndReplaceWith10Shards(); } +TYPED_TEST(ObjectCacheTest, MultithreadUpdateTtl) { + this->testMultithreadUpdateTtl(); +} using ObjectCache = ObjectCache; TEST(ObjectCacheTest, LruEviction) { diff --git a/cachelib/experimental/objcache2/tests/test_object.thrift b/cachelib/experimental/objcache2/tests/test_object.thrift index bf6ad07432..a448014cd9 100644 --- a/cachelib/experimental/objcache2/tests/test_object.thrift +++ b/cachelib/experimental/objcache2/tests/test_object.thrift @@ -17,13 +17,13 @@ namespace cpp2 facebook.cachelib.objcache2.test struct ThriftFoo { - 1: i32 a; - 2: i32 b; - 3: i32 c; + 1: i32 a; + 2: i32 b; + 3: i32 c; } struct ThriftFoo2 { - 1: i32 d; - 2: i32 e; - 3: i32 f; + 1: i32 d; + 2: i32 e; + 3: i32 f; } diff --git a/cachelib/external/fbthrift b/cachelib/external/fbthrift index 33a9fbc258..cbc3de581f 160000 --- a/cachelib/external/fbthrift +++ b/cachelib/external/fbthrift @@ -1 +1 @@ -Subproject commit 33a9fbc258f21818f20ea03a55c979014882e84d +Subproject commit cbc3de581fdf36ba474b0c135b9e785e504f1c1e diff --git a/cachelib/external/fizz b/cachelib/external/fizz index 9198ca6e7d..287625bd66 160000 --- a/cachelib/external/fizz +++ b/cachelib/external/fizz @@ -1 +1 @@ -Subproject commit 9198ca6e7daa50fae6b8413d745b4faaf97dfd10 +Subproject commit 287625bd6676b812e75ad0b088a61f72b4c9e681 diff --git a/cachelib/external/folly b/cachelib/external/folly index 128cfac6ac..ce2b95715d 160000 --- a/cachelib/external/folly +++ b/cachelib/external/folly @@ -1 +1 @@ -Subproject commit 128cfac6ac3d69825bad2af852fced3f63d87411 +Subproject commit ce2b95715de229fcb51bd97410469a3ad4d2bfb2 diff --git a/cachelib/external/wangle b/cachelib/external/wangle index 6bc77c8d46..44690e7894 160000 --- a/cachelib/external/wangle +++ b/cachelib/external/wangle @@ -1 +1 @@ -Subproject commit 6bc77c8d46b5ef68d77e921bb1e3d1e576adb8fe +Subproject commit 44690e7894842a7127245837b69627d4b964aabd diff --git a/cachelib/navy/block_cache/BlockCache.cpp b/cachelib/navy/block_cache/BlockCache.cpp index f34605b68b..84dadd13e7 100644 --- a/cachelib/navy/block_cache/BlockCache.cpp +++ b/cachelib/navy/block_cache/BlockCache.cpp @@ -723,9 +723,11 @@ void BlockCache::getCounters(const CounterVisitor& visitor) const { reclaimValueChecksumErrorCount_.get(), CounterVisitor::CounterType::RATE); visitor("navy_bc_cleanup_entry_header_checksum_errors", - cleanupEntryHeaderChecksumErrorCount_.get()); + cleanupEntryHeaderChecksumErrorCount_.get(), + CounterVisitor::CounterType::RATE); visitor("navy_bc_cleanup_value_checksum_errors", - cleanupValueChecksumErrorCount_.get()); + cleanupValueChecksumErrorCount_.get(), + CounterVisitor::CounterType::RATE); visitor("navy_bc_succ_lookups", succLookupCount_.get(), CounterVisitor::CounterType::RATE); visitor("navy_bc_removes", removeCount_.get(), @@ -750,7 +752,8 @@ void BlockCache::getCounters(const CounterVisitor& visitor) const { visitor("navy_bc_reinsertion_errors", reinsertionErrorCount_.get(), CounterVisitor::CounterType::RATE); visitor("navy_bc_lookup_for_item_destructor_errors", - lookupForItemDestructorErrorCount_.get()); + lookupForItemDestructorErrorCount_.get(), + CounterVisitor::CounterType::RATE); visitor("navy_bc_remove_attempt_collisions", removeAttemptCollisions_.get(), CounterVisitor::CounterType::RATE); // Allocator visits region manager diff --git a/cachelib/navy/driver/Driver.cpp b/cachelib/navy/driver/Driver.cpp index 1615d1cc48..29215cc161 100644 --- a/cachelib/navy/driver/Driver.cpp +++ b/cachelib/navy/driver/Driver.cpp @@ -273,8 +273,10 @@ void Driver::getCounters(const CounterVisitor& visitor) const { CounterVisitor::CounterType::RATE); visitor("navy_rejected_bytes", rejectedBytes_.get(), CounterVisitor::CounterType::RATE); - visitor("navy_accepted_bytes", acceptedBytes_.get()); - visitor("navy_accepted", acceptedCount_.get()); + visitor("navy_accepted_bytes", acceptedBytes_.get(), + CounterVisitor::CounterType::RATE); + visitor("navy_accepted", acceptedCount_.get(), + CounterVisitor::CounterType::RATE); visitor("navy_parcel_memory", parcelMemory_.get()); visitor("navy_concurrent_inserts", concurrentInserts_.get()); diff --git a/cachelib/navy/serialization/objects.thrift b/cachelib/navy/serialization/objects.thrift index 32be4bd17e..887c90c6ba 100644 --- a/cachelib/navy/serialization/objects.thrift +++ b/cachelib/navy/serialization/objects.thrift @@ -17,78 +17,78 @@ namespace cpp2 facebook.cachelib.navy.serialization struct IndexEntry { - 1: required i32 key = 0, - 2: required i32 address = 0, - 3: i16 sizeHint = 0, - 4: byte totalHits = 0, - 5: byte currentHits = 0, + 1: required i32 key = 0; + 2: required i32 address = 0; + 3: i16 sizeHint = 0; + 4: byte totalHits = 0; + 5: byte currentHits = 0; } struct IndexBucket { - 1: required i32 bucketId = 0, - 2: required list entries, + 1: required i32 bucketId = 0; + 2: required list entries; } struct Region { - 1: required i32 regionId = 0, - 2: required i32 lastEntryEndOffset = 0, - 3: required i32 classId = 0, - 4: required i32 numItems = 0, - 5: required bool pinned = false, - 6: i32 priority = 0, + 1: required i32 regionId = 0; + 2: required i32 lastEntryEndOffset = 0; + 3: required i32 classId = 0; + 4: required i32 numItems = 0; + 5: required bool pinned = false; + 6: i32 priority = 0; } struct RegionData { - 1: required list regions, - 2: required i32 regionSize = 0, + 1: required list regions; + 2: required i32 regionSize = 0; } struct FifoPolicyNodeData { - 1: required i32 idx, - 2: required i64 trackTime, + 1: required i32 idx; + 2: required i64 trackTime; } -struct FifoPolicyData{ - 1: required list queue, +struct FifoPolicyData { + 1: required list queue; } struct AccessStats { - 1: byte totalHits = 0, - 2: byte currHits = 0, - 3: byte numReinsertions = 0, + 1: byte totalHits = 0; + 2: byte currHits = 0; + 3: byte numReinsertions = 0; } struct AccessStatsPair { - 1: i64 key, - 2: AccessStats stats, + 1: i64 key; + 2: AccessStats stats; } struct AccessTracker { - 1: map deprecated_data, - 2: list data, + 1: map deprecated_data; + 2: list data; } struct BlockCacheConfig { - 1: required i64 version = 0, - 2: required i64 cacheBaseOffset = 0, - 3: required i64 cacheSize = 0, - 4: required i32 allocAlignSize = 0, - 5: required set deprecated_sizeClasses, - 6: required bool checksum = false, - 7: map deprecated_sizeDist, - 8: i64 holeCount = 0, - 9: i64 holeSizeTotal = 0, - 10: bool reinsertionPolicyEnabled = false, - 11: i64 usedSizeBytes = 0, + 1: required i64 version = 0; + 2: required i64 cacheBaseOffset = 0; + 3: required i64 cacheSize = 0; + 4: required i32 allocAlignSize = 0; + 5: required set deprecated_sizeClasses; + 6: required bool checksum = false; + 7: map deprecated_sizeDist; + 8: i64 holeCount = 0; + 9: i64 holeSizeTotal = 0; + 10: bool reinsertionPolicyEnabled = false; + 11: i64 usedSizeBytes = 0; } struct BigHashPersistentData { - 1: required i32 version = 0, - 2: required i64 generationTime = 0, - 3: required i64 itemCount = 0, - 4: required i64 bucketSize = 0, - 5: required i64 cacheBaseOffset = 0, - 6: required i64 numBuckets = 0, - 7: map deprecated_sizeDist, - 8: i64 usedSizeBytes = 0, + 1: required i32 version = 0; + 2: required i64 generationTime = 0; + 3: required i64 itemCount = 0; + 4: required i64 bucketSize = 0; + 5: required i64 cacheBaseOffset = 0; + 6: required i64 numBuckets = 0; + 7: map deprecated_sizeDist; + 8: i64 usedSizeBytes = 0; } diff --git a/cachelib/persistence/tests/PersistenceCache.h b/cachelib/persistence/tests/PersistenceCache.h index 5400b4d4ea..1db5b5fc8a 100644 --- a/cachelib/persistence/tests/PersistenceCache.h +++ b/cachelib/persistence/tests/PersistenceCache.h @@ -213,7 +213,7 @@ class PersistenceCache { public: const uint32_t kNumKeys = 1024 * 1024; // 1 million const size_t kCacheSize = 100 * kNumKeys; // 100MB - const size_t kCapacity = 4 * kCacheSize; // 400MB + const size_t kCapacity = 5 * kCacheSize; // 500MB std::unique_ptr buffer_; std::string cacheDir_; diff --git a/cachelib/shm/shm.thrift b/cachelib/shm/shm.thrift index 0372d7c8f7..7022947224 100644 --- a/cachelib/shm/shm.thrift +++ b/cachelib/shm/shm.thrift @@ -17,6 +17,6 @@ namespace cpp2 facebook.cachelib.serialization struct ShmManagerObject { - 1: required byte shmVal, - 3: required map nameToKeyMap, + 1: required byte shmVal; + 3: required map nameToKeyMap; } diff --git a/contrib/build-package.sh b/contrib/build-package.sh index ff487967cb..755933bd44 100755 --- a/contrib/build-package.sh +++ b/contrib/build-package.sh @@ -102,11 +102,12 @@ test "$#" -eq 0 \ && die "missing dependancy name to build. See -h for help" ###################################### -## Check which dependecy was requested +## Check which dependency was requested ###################################### external_git_clone= external_git_branch= +# external_git_tag can also be used for commit hashes external_git_tag= update_submodules= cmake_custom_params= @@ -160,6 +161,7 @@ case "$1" in REPODIR=cachelib/external/$NAME SRCDIR=$REPODIR external_git_clone=yes + external_git_tag="8.0.1" cmake_custom_params="-DBUILD_SHARED_LIBS=ON" if test "$build_tests" = "yes" ; then cmake_custom_params="$cmake_custom_params -DFMT_TEST=YES" @@ -174,7 +176,10 @@ case "$1" in REPODIR=cachelib/external/$NAME SRCDIR=$REPODIR/build/cmake external_git_clone=yes - external_git_branch=release + # Previously, we pinned to release branch. v1.5.4 needed + # CMake >= 3.18, later reverted. While waiting for v1.5.5, + # pin to the fix: https://github.com/facebook/zstd/pull/3510 + external_git_tag=8420502e if test "$build_tests" = "yes" ; then cmake_custom_params="-DZSTD_BUILD_TESTS=ON" else diff --git a/contrib/prerequisites-arch.sh b/contrib/prerequisites-arch.sh index 85a8656f7b..249f6c8082 100755 --- a/contrib/prerequisites-arch.sh +++ b/contrib/prerequisites-arch.sh @@ -19,4 +19,5 @@ sudo pacman -S --needed --noconfirm cmake \ boost \ double-conversion \ libdwarf \ + numactl \ libsodium diff --git a/contrib/prerequisites-fedora32.sh b/contrib/prerequisites-fedora32.sh index 235d6c1a8a..942cac0470 100755 --- a/contrib/prerequisites-fedora32.sh +++ b/contrib/prerequisites-fedora32.sh @@ -21,6 +21,7 @@ sudo dnf -y install bison flex patch bzip2 cmake \ zlib-devel lz4-devel xz-devel bzip2-devel \ jemalloc-devel snappy-devel libsodium-devel libdwarf-devel libaio-devel \ gmock-devel gflags-devel gtest gtest-devel \ + numactl-devel \ fmt fmt-devel # DO NOT INSTALL glog-devel - need to build from source for the glog-*.cmake files diff --git a/contrib/prerequisites-fedora34.sh b/contrib/prerequisites-fedora34.sh index 7e45c8740d..c7182cc513 100755 --- a/contrib/prerequisites-fedora34.sh +++ b/contrib/prerequisites-fedora34.sh @@ -19,4 +19,5 @@ sudo dnf -y install bison flex patch bzip2 cmake \ double-conversion double-conversion-devel make g++ \ boost-devel libevent-devel openssl-devel libunwind-devel \ zlib-devel lz4-devel xz-devel bzip2-devel \ - jemalloc-devel snappy-devel libsodium-devel libdwarf-devel libaio-devel + jemalloc-devel snappy-devel libsodium-devel libdwarf-devel libaio-devel \ + numactl-devel diff --git a/contrib/prerequisites-rocky9.sh b/contrib/prerequisites-rocky9.sh index bec5b82011..06720aba2e 100755 --- a/contrib/prerequisites-rocky9.sh +++ b/contrib/prerequisites-rocky9.sh @@ -38,7 +38,8 @@ sudo dnf install -y \ jemalloc-devel \ libsodium-devel \ libaio-devel \ - binutils-devel + binutils-devel \ + numactl-devel sudo dnf install -y \ diff --git a/website/docs/Cache_Library_Architecture_Guide/Overview_A_random_walk_down_the_Cache_Library.md b/website/docs/Cache_Library_Architecture_Guide/Overview_A_random_walk_down_the_Cache_Library.md index f13fc13222..cf4e750c45 100644 --- a/website/docs/Cache_Library_Architecture_Guide/Overview_A_random_walk_down_the_Cache_Library.md +++ b/website/docs/Cache_Library_Architecture_Guide/Overview_A_random_walk_down_the_Cache_Library.md @@ -107,7 +107,7 @@ There will be a section discussing each of the bullets below. * For regular cache: Find the item in the chained hash map. From the item, get the slab it lives on and form the slab, identify the allocation class and promote the item on that particular LRU queue. Increment the refcount and return the item handle. ## Flash overview -Flash is organized in a similar way: there is a cache for smaller items (BigHash) and for larger item (Block Cache). Unlike DRAM, the client does not get to choose where the item goes. It's done automatically thresholding the size. Together, this constitutes [Navy](/docs/Cache_Library_Architecture_Guide/Navy_Architecture_Overview ) -- the flash cache engine of CacheLib. +Flash is organized in a similar way: there is a cache for smaller items (BigHash) and for larger item (Block Cache). Unlike DRAM, the client does not get to choose where the item goes. It's done automatically thresholding the size. Together, this constitutes [Navy](/docs/Cache_Library_Architecture_Guide/Navy_Overview ) -- the flash cache engine of CacheLib. * "block device" refers to devices that's read/write happen with a fixed size block (if it helps, substitute the word "page" here). It means you can't write with precision of bytes but have to incur overhead if you don't write an entire block. diff --git a/website/docs/Cache_Library_User_Guides/Cachebench_FB_HW_eval.md b/website/docs/Cache_Library_User_Guides/Cachebench_FB_HW_eval.md index ce99649133..d17b7ac522 100644 --- a/website/docs/Cache_Library_User_Guides/Cachebench_FB_HW_eval.md +++ b/website/docs/Cache_Library_User_Guides/Cachebench_FB_HW_eval.md @@ -17,10 +17,11 @@ sufficient free memory (50+GB) and SSD capacity (1TB). * SSD Capacity: 100GB or more available capacity * Internet connection capable of accessing github.com and installing packages -## Set up the SSD devices using mdraid +## Set up the SSD devices -To gather SSD performance metrics, the SSD must be setup first. An example -below sets up a raid device to handle two ssds being used by CacheBench. +To gather SSD performance metrics, the SSD must be setup first. Cachebench (and CacheLib) supports using various types of devices for NVM cache including a raw block device or a regular file. When one wants to use multiple SSDs as NVM cache, the CacheLib also provides a native support for RAID0 (i.e., striping). + +Optionally, as an example, an user can setup and use md devices as follows. In this example, the md device is created from two ssd devices to be used as a raw block device in CacheBench. ```sh mdadm --create /dev/md0 --force --raid-devices=2 --level=0 --chunk=256 /dev/nvme1n1 /dev/nvme2n1 @@ -142,7 +143,7 @@ mdadm --create /dev/md0 --force --raid-devices=2 --level=0 --chunk=256 /dev/nvme make install ``` -See [build and installation](/docs/installation/installation) for further details. +See [build and installation](/docs/installation) for further details. ## Running the benchmark for SSD perf testing @@ -196,7 +197,6 @@ For a full list of options that can be configured, see [configuring cachebench]( using the `--progress` and specifying a duration in seconds. If `--progress-stats-file` is also specified, on every progress interval, `cachebench` would log the internal stats to the specified file. - ## Running cachebench with the trace workload Meta is sharing anonymized traces captured from large scale production cache services. These traces are licensed under the same license as CacheLib. They are meant to help academic and industry researchers to optimize for our caching workloads. One can freely download it from our AWS S3 bucket and run the CacheBench to replay the trace with varying configuration as follows. diff --git a/website/docs/Cache_Library_User_Guides/Cachebench_Overview.md b/website/docs/Cache_Library_User_Guides/Cachebench_Overview.md index eb72646542..8c878e1be6 100644 --- a/website/docs/Cache_Library_User_Guides/Cachebench_Overview.md +++ b/website/docs/Cache_Library_User_Guides/Cachebench_Overview.md @@ -53,6 +53,6 @@ developer's need. The following are few examples. ## Building cachebench -Follow instructions in [Installation](/docs/installation/installation) to build +Follow instructions in [Installation](/docs/installation) to build cachebench. This should install cachebench in your local machine under ```opt/cachelib/bin/cachebench``` diff --git a/website/docs/Cache_Library_User_Guides/eviction_policy.md b/website/docs/Cache_Library_User_Guides/eviction_policy.md index 356a3cb198..cce947619a 100644 --- a/website/docs/Cache_Library_User_Guides/eviction_policy.md +++ b/website/docs/Cache_Library_User_Guides/eviction_policy.md @@ -23,7 +23,7 @@ The second modification is promotion delay. Normally every access item is moved How often does cachelib refresh a previously accessed item. By default this is 60 seconds. * `updateOnWrite`/`updateOnRead` -Specifies if a LRU promotion happens on read or write or both. As a rule of thumb, for most services that care primarily about read performance, turn on `updateOnRead`. However, if your service cares a lot about retention time of items that are recently written, then turn on `updateOnWrite` as well. +Specifies if a LRU promotion happens on read or write or both. As a rule of thumb, for most services that care primarily about read performance, turn on `updateOnRead`. However, if your service cares a lot about retention time of items that are recently written, then turn on `updateOnWrite` as well. By default, `updateOnRead = true` and `updateOnWrite = false`. * `ipSpec` This essentially turns the LRU into a two-segmented LRU. Setting this to `1` means every new insertion will be inserted 1/2 from the end of the LRU, `2` means 1/4 from the end of the LRU, and so on. diff --git a/website/docs/installation/testing.md b/website/docs/installation/testing.md index 02b2cb747c..d8730127b4 100644 --- a/website/docs/installation/testing.md +++ b/website/docs/installation/testing.md @@ -11,7 +11,7 @@ of the cache infrastructure. ## Building CacheLib Unit Tests To build the cachelib unit tests, use one of the following commands -(see [installation](docs/installation/installation) instructions for more details): +(see [installation](/docs/installation) instructions for more details): 1. Use `./contrib/build.sh` script with the `-T` option. 2. Use `./contrib/build-package.sh -t cachelib` (with the `-t` option) @@ -42,7 +42,7 @@ Running a single unit test binary: ```sh $ cd opt/cachelib/tests -$ ./allocator-test-ItemTest +$ ./allocator-test-ItemTest [==========] Running 6 tests from 1 test suite. [----------] Global test environment set-up. [----------] 6 tests from ItemTest diff --git a/website/package.json b/website/package.json index 8c58fda9a2..ac9801eeeb 100644 --- a/website/package.json +++ b/website/package.json @@ -43,7 +43,8 @@ "ansi-html": "0.0.8", "ua-parser-js": "^1.0.33", "eta": "^2.0.0", - "http-cache-semantics": "^4.1.1" + "http-cache-semantics": "^4.1.1", + "@braintree/sanitize-url": "^6.0.1" }, "browserslist": { "production": [ diff --git a/website/src/pages/index.js b/website/src/pages/index.js index 151ec3f3fb..5886c079f9 100644 --- a/website/src/pages/index.js +++ b/website/src/pages/index.js @@ -117,7 +117,7 @@ function Home() { 'button button--secondary button--lg', styles.getStarted, )} - to={ useBaseUrl('docs/installation/installation') }> + to={ useBaseUrl('docs/installation') }> Get Started diff --git a/website/yarn.lock b/website/yarn.lock index 19f12eb0d5..51e55da8c7 100644 --- a/website/yarn.lock +++ b/website/yarn.lock @@ -1809,10 +1809,10 @@ "@babel/helper-validator-identifier" "^7.18.6" to-fast-properties "^2.0.0" -"@braintree/sanitize-url@^6.0.0": - version "6.0.0" - resolved "https://registry.yarnpkg.com/@braintree/sanitize-url/-/sanitize-url-6.0.0.tgz#fe364f025ba74f6de6c837a84ef44bdb1d61e68f" - integrity sha512-mgmE7XBYY/21erpzhexk4Cj1cyTQ9LzvnTxtzM17BJ7ERMNE6W72mQRo0I1Ud8eFJ+RVVIcBNhLFZ3GX4XFz5w== +"@braintree/sanitize-url@^6.0.0", "@braintree/sanitize-url@^6.0.1": + version "6.0.2" + resolved "https://registry.yarnpkg.com/@braintree/sanitize-url/-/sanitize-url-6.0.2.tgz#6110f918d273fe2af8ea1c4398a88774bb9fc12f" + integrity sha512-Tbsj02wXCbqGmzdnXNk0SOF19ChhRU70BsroIi4Pm6Ehp56in6vch94mfbdQ17DozxkL3BAVjbZ4Qc1a0HFRAg== "@colors/colors@1.5.0": version "1.5.0"