diff --git a/.clang-format b/.clang-format new file mode 100644 index 0000000000..ac12399b46 --- /dev/null +++ b/.clang-format @@ -0,0 +1,45 @@ +--- +AccessModifierOffset: -1 +AlignEscapedNewlinesLeft: true +AlignTrailingComments: true +AllowAllParametersOfDeclarationOnNextLine: true +AllowShortIfStatementsOnASingleLine: false +AllowShortLoopsOnASingleLine: false +AlwaysBreakBeforeMultilineStrings: true +AlwaysBreakTemplateDeclarations: true +BasedOnStyle: Google +BinPackParameters: false +BreakBeforeBinaryOperators: false +BreakBeforeBraces: Attach +BreakConstructorInitializersBeforeComma: false +ColumnLimit: 80 +ConstructorInitializerAllOnOneLineOrOnePerLine: true +ConstructorInitializerIndentWidth: 4 +ContinuationIndentWidth: 4 +Cpp11BracedListStyle: true +DerivePointerAlignment: false +ExperimentalAutoDetectBinPacking: true +IndentCaseLabels: false +IndentFunctionDeclarationAfterType: false +IndentWidth: 2 +MaxEmptyLinesToKeep: 1 +NamespaceIndentation: None +ObjCSpaceBeforeProtocolList: false +PenaltyBreakBeforeFirstCallParameter: 10 +PenaltyBreakComment: 60 +PenaltyBreakFirstLessLess: 20 +PenaltyBreakString: 1000 +PenaltyExcessCharacter: 1000000 +PenaltyReturnTypeOnItsOwnLine: 200 +PointerAlignment: Left +SpaceAfterControlStatementKeyword: true +SpaceBeforeAssignmentOperators: true +SpaceInEmptyParentheses: false +SpacesBeforeTrailingComments: 1 +SpacesInAngles: false +SpacesInCStyleCastParentheses: false +SpacesInParentheses: false +Standard: Cpp11 +TabWidth: 8 +UseTab: Never +... diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md deleted file mode 100644 index 670f752033..0000000000 --- a/.github/ISSUE_TEMPLATE.md +++ /dev/null @@ -1,17 +0,0 @@ -# Summary of Issue or Feature request - -## Goal -Increase flash space efficiency or build is not handling X - -## Context -This is related to project Bar. It addresses a key gap in its design. - -## Suggested Approach -We will add an additional component between X and Y. We will introduce a new API called Hoop. - -## Success Metrics - -No regression on cpu -No regression on read and write latency to flash device -Increase space density by 20% -Added a new metrics here diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000000..dd84ea7824 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,38 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: '' +assignees: '' + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Steps to reproduce the behavior: +1. Go to '...' +2. Click on '....' +3. Scroll down to '....' +4. See error + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Screenshots** +If applicable, add screenshots to help explain your problem. + +**Desktop (please complete the following information):** + - OS: [e.g. iOS] + - Browser [e.g. chrome, safari] + - Version [e.g. 22] + +**Smartphone (please complete the following information):** + - Device: [e.g. iPhone6] + - OS: [e.g. iOS8.1] + - Browser [e.g. stock browser, safari] + - Version [e.g. 22] + +**Additional context** +Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000000..bbcbbe7d61 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,20 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: '' +assignees: '' + +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context or screenshots about the feature request here. diff --git a/.github/ISSUE_TEMPLATE/general-question.md b/.github/ISSUE_TEMPLATE/general-question.md new file mode 100644 index 0000000000..49e480cb96 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/general-question.md @@ -0,0 +1,10 @@ +--- +name: General Question +about: General Questions about CacheLib usage, compilation, and anything else +title: '' +labels: '' +assignees: '' + +--- + + diff --git a/.github/workflows/build-cachelib-centos.yml b/.github/workflows/build-cachelib-centos.yml deleted file mode 100644 index 3b071a186a..0000000000 --- a/.github/workflows/build-cachelib-centos.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: build-cachelib-centos-latest -on: - schedule: - - cron: '30 5 * * 1,4' -jobs: - build-cachelib-centos8-latest: - name: "CentOS/latest - Build CacheLib with all dependencies" - runs-on: ubuntu-latest - # Docker container image name - container: "centos:latest" - steps: - - name: "update packages" - run: dnf upgrade -y - - name: "install sudo,git" - run: dnf install -y sudo git cmake gcc - - name: "System Information" - run: | - echo === uname === - uname -a - echo === /etc/os-release === - cat /etc/os-release - echo === df -hl === - df -hl - echo === free -h === - free -h - echo === top === - top -b -n1 -1 -Eg || timeout 1 top -b -n1 - echo === env === - env - echo === gcc -v === - gcc -v - - name: "checkout sources" - uses: actions/checkout@v2 - - name: "build CacheLib using build script" - run: ./contrib/build.sh -j -v -T diff --git a/.github/workflows/build-cachelib-docker.yml b/.github/workflows/build-cachelib-docker.yml new file mode 100644 index 0000000000..b6d872d5ca --- /dev/null +++ b/.github/workflows/build-cachelib-docker.yml @@ -0,0 +1,50 @@ +name: build-cachelib-docker +on: + push: + pull_request: + workflow_dispatch: + +jobs: + build-cachelib-docker: + name: "CentOS/latest - Build CacheLib with all dependencies" + runs-on: ubuntu-latest + env: + REPO: cachelib + GITHUB_REPO: pmem/CacheLib + CONTAINER_REG: ghcr.io/pmem/cachelib + CONTAINER_REG_USER: ${{ secrets.GH_CR_USER }} + CONTAINER_REG_PASS: ${{ secrets.GH_CR_PAT }} + FORCE_IMAGE_ACTION: ${{ secrets.FORCE_IMAGE_ACTION }} + HOST_WORKDIR: ${{ github.workspace }} + WORKDIR: docker + IMG_VER: main + strategy: + matrix: + CONFIG: ["OS=centos OS_VER=8streams PUSH_IMAGE=1"] + steps: + - name: "System Information" + run: | + echo === uname === + uname -a + echo === /etc/os-release === + cat /etc/os-release + echo === df -hl === + df -hl + echo === free -h === + free -h + echo === top === + top -b -n1 -1 -Eg || timeout 1 top -b -n1 + echo === env === + env + echo === gcc -v === + gcc -v + - name: "checkout sources" + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Pull the image or rebuild and push it + run: cd $WORKDIR && ${{ matrix.CONFIG }} ./pull-or-rebuild-image.sh $FORCE_IMAGE_ACTION + + - name: Run the build + run: cd $WORKDIR && ${{ matrix.CONFIG }} ./build.sh diff --git a/.github/workflows/build-cachelib.yml b/.github/workflows/build-cachelib.yml deleted file mode 100644 index 15161c40e0..0000000000 --- a/.github/workflows/build-cachelib.yml +++ /dev/null @@ -1,147 +0,0 @@ -# NOTES: -# 1. While Github-Actions enables cache of dependencies, -# Facebook's projects (folly,fizz,wangle,fbthrift) -# are fast-moving targets - so we always checkout the latest version -# (as opposed to using gitactions cache, which is recommended in the -# documentation). -# -# 2. Using docker containers to build on CentOS and Debian, -# Specifically CentOS v8.1.1911 as that -# version is closest to Facebook's internal dev machines. -# -# 3. When using docker containers we install 'sudo', -# as the docker images are typically very minimal and without -# 'sudo', while the ./contrib/ scripts use sudo. -# -# 4. When using the docker containers we install 'git' -# BEFORE getting the CacheLib source code (with the 'checkout' action). -# Otherwise, the 'checkout@v2' action script falls back to downloading -# the git repository files only, without the ".git" directory. -# We need the ".git" directory to updating the git-submodules -# (folly/wangle/fizz/fbthrift). See: -# https://github.com/actions/checkout/issues/126#issuecomment-570288731 -# -# 5. To reduce less-critical (and yet frequent) rebuilds, the jobs -# check the author of the commit, and SKIP the build if -# the author is "svcscm". These commits are automatic updates -# for the folly/fbthrift git-submodules, and can happen several times a day. -# While there is a possiblity that updating the git-submodules breaks -# CacheLib, it is less likely, and will be detected once an actual -# code change commit triggers a full build. -# e.g. https://github.com/facebookincubator/CacheLib/commit/9372a82190dd71a6e2bcb668828cfed9d1bd25c1 -# -# 6. The 'if' condition checking the author name of the commit (see #5 above) -# uses github actions metadata variable: -# 'github.event.head_commit.author.name' -# GitHub have changed in the past the metadata structure and broke -# such conditions. If you need to debug the metadata values, -# see the "dummy-show-github-event" job below. -# E.g. https://github.blog/changelog/2019-10-16-changes-in-github-actions-push-event-payload/ -# As of Jan-2021, the output is: -# { -# "author": { -# "email": "mimi@moo.moo", -# "name": "mimi" -# }, -# "committer": { -# "email": "assafgordon@gmail.com", -# "name": "Assaf Gordon", -# "username": "agordon" -# }, -# "distinct": true, -# "id": "6c3aab0970f4a07cc2af7658756a6ef9d82f3276", -# "message": "gitactions: test", -# "timestamp": "2021-01-26T11:11:57-07:00", -# "tree_id": "741cd1cb802df84362a51e5d01f28788845d08b7", -# "url": "https://github.com/agordon/CacheLib/commit/6c3aab0970f4a07cc2af7658756a6ef9d82f3276" -# } -# -# 7. When checking the commit's author name, we use '...author.name', -# NOT '...author.username' - because the 'svcscm' author does not -# have a github username (see the 'mimi' example above). -# - -name: build-cachelib -on: [push] -jobs: - dummy-show-github-event: - name: "Show GitHub Action event.head_commit variable" - runs-on: ubuntu-latest - steps: - - name: "GitHub Variable Content" - env: - CONTENT: ${{ toJSON(github.event.head_commit) }} - run: echo "$CONTENT" - - - build-cachelib-centos8-1-1911: - if: "!contains(github.event.head_commit.author.name, 'svcscm')" - name: "CentOS/8.1.1911 - Build CacheLib with all dependencies" - runs-on: ubuntu-latest - # Docker container image name - container: "centos:8.1.1911" - steps: - - name: "update packages" - # stock centos has a problem with CMAKE, fails with: - # "cmake: symbol lookup error: cmake: undefined symbol: archive_write_add_filter_zstd" - # updating solves it - run: dnf update -y - - name: "install sudo,git" - run: dnf install -y sudo git cmake gcc - - name: "System Information" - run: | - echo === uname === - uname -a - echo === /etc/os-release === - cat /etc/os-release - echo === df -hl === - df -hl - echo === free -h === - free -h - echo === top === - top -b -n1 -1 -Eg || timeout 1 top -b -n1 - echo === env === - env - echo === gcc -v === - gcc -v - - name: "checkout sources" - uses: actions/checkout@v2 - - name: "Install Prerequisites" - run: ./contrib/build.sh -S -B - - name: "Test: update-submodules" - run: ./contrib/update-submodules.sh - - name: "Install dependency: zstd" - run: ./contrib/build-package.sh -j -v -i zstd - - name: "Install dependency: googleflags" - run: ./contrib/build-package.sh -j -v -i googleflags - - name: "Install dependency: googlelog" - run: ./contrib/build-package.sh -j -v -i googlelog - - name: "Install dependency: googletest" - run: ./contrib/build-package.sh -j -v -i googletest - - name: "Install dependency: sparsemap" - run: ./contrib/build-package.sh -j -v -i sparsemap - - name: "Install dependency: fmt" - run: ./contrib/build-package.sh -j -v -i fmt - - name: "Install dependency: folly" - run: ./contrib/build-package.sh -j -v -i folly - - name: "Install dependency: fizz" - run: ./contrib/build-package.sh -j -v -i fizz - - name: "Install dependency: wangle" - run: ./contrib/build-package.sh -j -v -i wangle - - name: "Install dependency: fbthrift" - run: ./contrib/build-package.sh -j -v -i fbthrift - - name: "build CacheLib" - # Build cachelib in debug mode (-d) and with all tests (-t) - run: ./contrib/build-package.sh -j -v -i -d -t cachelib - - uses: actions/upload-artifact@v2 - if: failure() - with: - name: cachelib-cmake-logs - path: | - build-cachelib/CMakeFiles/*.log - build-cachelib/CMakeCache.txt - build-cachelib/Makefile - build-cachelib/**/Makefile - if-no-files-found: warn - retention-days: 1 - diff --git a/.github/workflows/clang-format-check.yml b/.github/workflows/clang-format-check.yml new file mode 100644 index 0000000000..4b4897b610 --- /dev/null +++ b/.github/workflows/clang-format-check.yml @@ -0,0 +1,19 @@ +# From: https://github.com/marketplace/actions/clang-format-check#multiple-paths +name: clang-format Check +on: [pull_request] +jobs: + formatting-check: + name: Formatting Check + runs-on: ubuntu-latest + strategy: + matrix: + path: + - 'cachelib' + - 'examples' + steps: + - uses: actions/checkout@v2 + - name: Run clang-format style check for C/C++ programs. + uses: jidicula/clang-format-action@v4.6.2 + with: + clang-format-version: '13' + check-path: ${{ matrix.path }} diff --git a/.packit.yaml b/.packit.yaml new file mode 100644 index 0000000000..bea307d9d0 --- /dev/null +++ b/.packit.yaml @@ -0,0 +1,25 @@ +# See the documentation for more information: +# https://packit.dev/docs/configuration + +specfile_path: cachelib.spec + +upstream_package_name: CacheLib +downstream_package_name: cachelib + +actions: + fix-spec-file: + - bash -c "sed -i cachelib.spec -e \"s/%global commit.*/%global commit $(git rev-parse HEAD)/\"" + - bash -c "sed -i cachelib.spec -e \"s/%global date.*/%global date $(git show -s --date=format:'%Y%m%d' --format=%cd)/\"" + create-archive: + - bash -c "COMMIT=$(git rev-parse HEAD); curl -ORL https://github.com/facebook/CacheLib/archive/${COMMIT}/cachelib-${COMMIT}.tar.gz; echo cachelib-${COMMIT}.tar.gz" + post-upstream-clone: "bash -c \"rm -rf cachelib-dist-git; git clone -b packit https://pagure.io/meta/cachelib.git cachelib-dist-git && mv cachelib-dist-git/cachelib*.{spec,patch} .\"" + +jobs: +- job: copr_build + trigger: pull_request + metadata: + targets: + - fedora-rawhide-aarch64 + - fedora-rawhide-x86_64 + - fedora-35-aarch64 + - fedora-35-x86_64 diff --git a/CHANGELOG.md b/CHANGELOG.md index 3058251a48..ebe779f258 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,8 @@ # Changelog -## V14 +## V16 -Inception version when Cachelib goes open source. +This version is incompatible with versions below 15. Downgrading from this version directly to a version below 15 will require the cache to be dropped. If you need to downgrade from this version, please make sure you downgrade to version 15 first to avoid dropping the cache. ## V15 @@ -13,3 +13,7 @@ Updating to this version may cause compliation error because: 1. CacheAllocator::allocatePermanent_deprecated. Updating to this version will not require dropping the cache. + +## V14 + +Inception version when Cachelib goes open source. diff --git a/cachelib/CMakeLists.txt b/cachelib/CMakeLists.txt index 3ff0b01ef0..f666025093 100644 --- a/cachelib/CMakeLists.txt +++ b/cachelib/CMakeLists.txt @@ -17,21 +17,25 @@ # refer to the root source directory of the project as ${HELLO_SOURCE_DIR} and # to the root binary directory of the project as ${HELLO_BINARY_DIR}. -cmake_minimum_required (VERSION 3.10) +cmake_minimum_required (VERSION 3.12) ## TODO: get version from variable project (CacheLib VERSION 0.1) #configure_file(cachelib/cachelib_config.h.in cachelib_config.h) -set(CACHELIB_MAJOR_VERSION 0) +if (NOT DEFINED CACHELIB_MAJOR_VERSION) + set(CACHELIB_MAJOR_VERSION 0) +endif () set(CACHELIB_MINOR_VERSION 1) set(CACHELIB_PATCH_VERSION 0) set(CACHELIB_VERSION ${CACHELIB_MAJOR_VERSION}.${CACHELIB_MINOR_VERSION}.${CACHELIB_PATCH_VERSION}) set(PACKAGE_NAME "cachelib") -set(PACKAGE_VERSION "${CACHELIB_VERSION}") +if (NOT DEFINED PACKAGE_VERSION) + set(PACKAGE_VERSION "${CACHELIB_VERSION}") +endif () set(PACKAGE_STRING "${PACKAGE_NAME} ${PACKAGE_VERSION}") set(PACKAGE_TARNAME "${PACKAGE_NAME}-${PACKAGE_VERSION}") set(PACKAGE_BUGREPORT "https://github.com/facebook/TBD") @@ -361,6 +365,20 @@ install(EXPORT cachelib-exports #NAMESPACE cachelib:: DESTINATION ${CMAKE_INSTALL_DIR}) +if (BUILD_SHARED_LIBS) + set_target_properties( + cachelib_allocator + cachelib_cachebench + cachelib_common + cachelib_datatype + cachelib_navy + cachelib_shm + PROPERTIES + SOVERSION ${CACHELIB_MAJOR_VERSION} + VERSION ${PACKAGE_VERSION} + ) +endif () + if (BUILD_TESTS) get_property(TEST_BINARIES GLOBAL PROPERTY TEST_BINARIES) #message(STATUS "=== Test binaries : ${TEST_BINARIES} ===") diff --git a/cachelib/allocator/CCacheAllocator.cpp b/cachelib/allocator/CCacheAllocator.cpp index 6f0bab6727..cff4bded4b 100644 --- a/cachelib/allocator/CCacheAllocator.cpp +++ b/cachelib/allocator/CCacheAllocator.cpp @@ -30,12 +30,12 @@ CCacheAllocator::CCacheAllocator(MemoryAllocator& allocator, PoolId poolId) CCacheAllocator::CCacheAllocator(MemoryAllocator& allocator, PoolId poolId, const SerializationType& object) - : CCacheAllocatorBase(*object.ccMetadata_ref()), + : CCacheAllocatorBase(*object.ccMetadata()), allocator_(allocator), poolId_(poolId), currentChunksIndex_(0) { auto& currentChunks = chunks_[currentChunksIndex_]; - for (auto chunk : *object.chunks_ref()) { + for (auto chunk : *object.chunks()) { currentChunks.push_back(allocator_.unCompress(CompressedPtr(chunk))); } } @@ -93,11 +93,11 @@ size_t CCacheAllocator::resize() { CCacheAllocator::SerializationType CCacheAllocator::saveState() { CCacheAllocator::SerializationType object; - *object.ccMetadata_ref() = ccType_.saveState(); + *object.ccMetadata() = ccType_.saveState(); std::lock_guard guard(resizeLock_); for (auto chunk : getCurrentChunks()) { - object.chunks_ref()->push_back(allocator_.compress(chunk).saveState()); + object.chunks()->push_back(allocator_.compress(chunk).saveState()); } return object; } diff --git a/cachelib/allocator/CCacheManager.cpp b/cachelib/allocator/CCacheManager.cpp index d9d1d6db7a..6750139ff0 100644 --- a/cachelib/allocator/CCacheManager.cpp +++ b/cachelib/allocator/CCacheManager.cpp @@ -24,7 +24,7 @@ CCacheManager::CCacheManager(const SerializationType& object, : memoryAllocator_(memoryAllocator) { std::lock_guard guard(lock_); - for (const auto& allocator : *object.allocators_ref()) { + for (const auto& allocator : *object.allocators()) { auto id = memoryAllocator_.getPoolId(allocator.first); allocators_.emplace( std::piecewise_construct, @@ -81,8 +81,7 @@ CCacheManager::SerializationType CCacheManager::saveState() { SerializationType object; for (auto& allocator : allocators_) { - object.allocators_ref()->emplace(allocator.first, - allocator.second.saveState()); + object.allocators()->emplace(allocator.first, allocator.second.saveState()); } return object; } diff --git a/cachelib/allocator/CMakeLists.txt b/cachelib/allocator/CMakeLists.txt index fc5d8610d8..059b76b6ce 100644 --- a/cachelib/allocator/CMakeLists.txt +++ b/cachelib/allocator/CMakeLists.txt @@ -65,12 +65,19 @@ target_link_libraries(cachelib_allocator PUBLIC cachelib_shm ) +if ((CMAKE_SYSTEM_NAME STREQUAL Linux) AND + (CMAKE_SYSTEM_PROCESSOR STREQUAL x86_64)) +else() + target_compile_definitions(cachelib_allocator PRIVATE SKIP_SIZE_VERIFY) +endif() + + install(TARGETS cachelib_allocator EXPORT cachelib-exports DESTINATION ${LIB_INSTALL_DIR} ) if (BUILD_TESTS) - add_library (allocator_test_support + add_library (allocator_test_support OBJECT ${DATASTRUCT_TESTS_THRIFT_FILES} ./nvmcache/tests/NvmTestBase.cpp ./memory/tests/TestBase.cpp @@ -78,6 +85,7 @@ if (BUILD_TESTS) add_dependencies(allocator_test_support thrift_generated_files) target_link_libraries (allocator_test_support PUBLIC cachelib_allocator + common_test_utils glog::glog gflags GTest::gtest diff --git a/cachelib/allocator/Cache.h b/cachelib/allocator/Cache.h index 02fd706588..3c3fd5cbe5 100644 --- a/cachelib/allocator/Cache.h +++ b/cachelib/allocator/Cache.h @@ -49,9 +49,27 @@ struct SimplePoolOptimizeStrategy; // to differentiate between the access modes and do appropriate action. enum class AccessMode { kRead, kWrite }; -// enum value to indicate if the removal from the MMContainer was an eviction -// or not. +// used by RemoveCB, indicating if the removal from the MMContainer was an +// eviction or not. enum class RemoveContext { kEviction, kNormal }; +// used by ItemDestructor, indicating how the item is destructed +enum class DestructorContext { + // item was in dram and evicted from dram. it could have + // been present in nvm as well. + kEvictedFromRAM, + + // item was only in nvm and evicted from nvm + kEvictedFromNVM, + + // item was present in dram and removed by user calling + // remove()/insertOrReplace, or removed due to expired. + // it could have been present in nvm as well. + kRemovedFromRAM, + + // item was present only in nvm and removed by user calling + // remove()/insertOrReplace. + kRemovedFromNVM +}; // A base class of cache exposing members and status agnostic of template type. class CacheBase { @@ -169,6 +187,10 @@ class CacheBase { // pool id virtual const ICompactCache& getCompactCache(PoolId pid) const = 0; + // return object cache stats + virtual void getObjectCacheCounters( + std::function) const {} + protected: // move bytes from one pool to another. The source pool should be at least // _bytes_ in size. diff --git a/cachelib/allocator/CacheAllocator-inl.h b/cachelib/allocator/CacheAllocator-inl.h index 15cfee7432..9be5e4ca0c 100644 --- a/cachelib/allocator/CacheAllocator-inl.h +++ b/cachelib/allocator/CacheAllocator-inl.h @@ -16,9 +16,7 @@ #pragma once -#include "cachelib/allocator/CacheVersion.h" -#include "cachelib/common/Utils.h" - +#include "cachelib/common/Hash.h" namespace facebook { namespace cachelib { @@ -39,15 +37,21 @@ CacheAllocator::CacheAllocator(Config config) accessContainer_(std::make_unique( config_.accessConfig, compressor_, - [this](Item* it) -> ItemHandle { return acquire(it); })), + [this](Item* it) -> WriteHandle { return acquire(it); })), chainedItemAccessContainer_(std::make_unique( config_.chainedItemAccessConfig, compressor_, - [this](Item* it) -> ItemHandle { return acquire(it); })), + [this](Item* it) -> WriteHandle { return acquire(it); })), chainedItemLocks_(config_.chainedItemsLockPower, std::make_shared()), cacheCreationTime_{util::getCurrentTimeSec()}, - nvmCacheState_{config_.cacheDir, config_.isNvmCacheEncryptionEnabled(), + // Keep cacheInstanceCreationTime_ in sync with cacheCreationTime_ as + // both are current time + cacheInstanceCreationTime_{cacheCreationTime_}, + // Pass in cacheInstnaceCreationTime_ as the current time to keep + // nvmCacheState's current time in sync + nvmCacheState_{cacheInstanceCreationTime_, config_.cacheDir, + config_.isNvmCacheEncryptionEnabled(), config_.isNvmCacheTruncateAllocSizeEnabled()} { initCommon(false); } @@ -68,10 +72,12 @@ CacheAllocator::CacheAllocator(SharedMemNewT, Config config) AccessContainer::getRequiredSize( config_.accessConfig.getNumBuckets()), nullptr, - ShmSegmentOpts(config_.accessConfig.getPageSize())) + ShmSegmentOpts(config_.accessConfig.getPageSize(), + false, + config_.usePosixShm)) .addr, compressor_, - [this](Item* it) -> ItemHandle { return acquire(it); })), + [this](Item* it) -> WriteHandle { return acquire(it); })), chainedItemAccessContainer_(std::make_unique( config_.chainedItemAccessConfig, shmManager_ @@ -79,17 +85,26 @@ CacheAllocator::CacheAllocator(SharedMemNewT, Config config) AccessContainer::getRequiredSize( config_.chainedItemAccessConfig.getNumBuckets()), nullptr, - ShmSegmentOpts(config_.accessConfig.getPageSize())) + ShmSegmentOpts(config_.accessConfig.getPageSize(), + false, + config_.usePosixShm)) .addr, compressor_, - [this](Item* it) -> ItemHandle { return acquire(it); })), + [this](Item* it) -> WriteHandle { return acquire(it); })), chainedItemLocks_(config_.chainedItemsLockPower, std::make_shared()), cacheCreationTime_{util::getCurrentTimeSec()}, - nvmCacheState_{config_.cacheDir, config_.isNvmCacheEncryptionEnabled(), + // Keep cacheInstanceCreationTime_ in sync with cacheCreationTime_ as + // both are current time + cacheInstanceCreationTime_{cacheCreationTime_}, + // Pass in cacheInstnaceCreationTime_ as the current time to keep + // nvmCacheState's current time in sync + nvmCacheState_{cacheInstanceCreationTime_, config_.cacheDir, + config_.isNvmCacheEncryptionEnabled(), config_.isNvmCacheTruncateAllocSizeEnabled()} { initCommon(false); - shmManager_->removeShm(detail::kShmInfoName); + shmManager_->removeShm(detail::kShmInfoName, + PosixSysVSegmentOpts(config_.usePosixShm)); } template @@ -107,21 +122,31 @@ CacheAllocator::CacheAllocator(SharedMemAttachT, Config config) accessContainer_(std::make_unique( deserializer_->deserialize(), config_.accessConfig, - shmManager_->attachShm(detail::kShmHashTableName), + shmManager_->attachShm( + detail::kShmHashTableName, + nullptr, + ShmSegmentOpts(PageSizeT::NORMAL, false, config_.usePosixShm)), compressor_, - [this](Item* it) -> ItemHandle { return acquire(it); })), + [this](Item* it) -> WriteHandle { return acquire(it); })), chainedItemAccessContainer_(std::make_unique( deserializer_->deserialize(), config_.chainedItemAccessConfig, - shmManager_->attachShm(detail::kShmChainedItemHashTableName), + shmManager_->attachShm( + detail::kShmChainedItemHashTableName, + nullptr, + ShmSegmentOpts(PageSizeT::NORMAL, false, config_.usePosixShm)), compressor_, - [this](Item* it) -> ItemHandle { return acquire(it); })), + [this](Item* it) -> WriteHandle { return acquire(it); })), chainedItemLocks_(config_.chainedItemsLockPower, std::make_shared()), - cacheCreationTime_{*metadata_.cacheCreationTime_ref()}, - nvmCacheState_{config_.cacheDir, config_.isNvmCacheEncryptionEnabled(), + cacheCreationTime_{static_cast(*metadata_.cacheCreationTime())}, + cacheInstanceCreationTime_{util::getCurrentTimeSec()}, + // Pass in cacheInstnaceCreationTime_ as the current time to keep + // nvmCacheState's current time in sync + nvmCacheState_{cacheInstanceCreationTime_, config_.cacheDir, + config_.isNvmCacheEncryptionEnabled(), config_.isNvmCacheTruncateAllocSizeEnabled()} { - for (auto pid : *metadata_.compactCachePools_ref()) { + for (auto pid : *metadata_.compactCachePools()) { isCompactCachePool_[pid] = true; } @@ -130,7 +155,8 @@ CacheAllocator::CacheAllocator(SharedMemAttachT, Config config) // We will create a new info shm segment on shutDown(). If we don't remove // this info shm segment here and the new info shm segment's size is larger // than this one, creating new one will fail. - shmManager_->removeShm(detail::kShmInfoName); + shmManager_->removeShm(detail::kShmInfoName, + PosixSysVSegmentOpts(config_.usePosixShm)); } template @@ -148,6 +174,7 @@ std::unique_ptr CacheAllocator::createNewMemoryAllocator() { ShmSegmentOpts opts; opts.alignment = sizeof(Slab); + opts.typeOpts = PosixSysVSegmentOpts(config_.usePosixShm); return std::make_unique( getAllocatorConfig(config_), shmManager_ @@ -162,6 +189,7 @@ std::unique_ptr CacheAllocator::restoreMemoryAllocator() { ShmSegmentOpts opts; opts.alignment = sizeof(Slab); + opts.typeOpts = PosixSysVSegmentOpts(config_.usePosixShm); return std::make_unique( deserializer_->deserialize(), shmManager_ @@ -199,7 +227,10 @@ void CacheAllocator::initCommon(bool dramCacheAttached) { } initStats(); initNvmCache(dramCacheAttached); - initWorkers(); + + if (!config_.delayCacheWorkersStart) { + initWorkers(); + } } template @@ -219,7 +250,8 @@ void CacheAllocator::initNvmCache(bool dramCacheAttached) { nvmCacheState_.markTruncated(); } - nvmCache_ = std::make_unique(*this, *config_.nvmConfig, truncate); + nvmCache_ = std::make_unique(*this, *config_.nvmConfig, truncate, + config_.itemDestructor); if (!config_.cacheDir.empty()) { nvmCacheState_.clearPrevState(); } @@ -227,19 +259,19 @@ void CacheAllocator::initNvmCache(bool dramCacheAttached) { template void CacheAllocator::initWorkers() { - if (config_.poolResizingEnabled()) { + if (config_.poolResizingEnabled() && !poolResizer_) { startNewPoolResizer(config_.poolResizeInterval, config_.poolResizeSlabsPerIter, config_.poolResizeStrategy); } - if (config_.poolRebalancingEnabled()) { + if (config_.poolRebalancingEnabled() && !poolRebalancer_) { startNewPoolRebalancer(config_.poolRebalanceInterval, config_.defaultPoolRebalanceStrategy, config_.poolRebalancerFreeAllocThreshold); } - if (config_.memMonitoringEnabled()) { + if (config_.memMonitoringEnabled() && !memMonitor_) { if (!isOnShm_) { throw std::invalid_argument( "Memory monitoring is not supported for cache on heap. It is " @@ -251,11 +283,11 @@ void CacheAllocator::initWorkers() { config_.poolAdviseStrategy); } - if (config_.itemsReaperEnabled()) { + if (config_.itemsReaperEnabled() && !reaper_) { startNewReaper(config_.reaperInterval, config_.reaperConfig); } - if (config_.poolOptimizerEnabled()) { + if (config_.poolOptimizerEnabled() && !poolOptimizer_) { startNewPoolOptimizer(config_.regularPoolOptimizeInterval, config_.compactCacheOptimizeInterval, config_.poolOptimizeStrategy, @@ -265,14 +297,16 @@ void CacheAllocator::initWorkers() { template std::unique_ptr CacheAllocator::createDeserializer() { - auto infoAddr = shmManager_->attachShm(detail::kShmInfoName); + auto infoAddr = shmManager_->attachShm( + detail::kShmInfoName, nullptr, + ShmSegmentOpts(PageSizeT::NORMAL, false, config_.usePosixShm)); return std::make_unique( reinterpret_cast(infoAddr.addr), reinterpret_cast(infoAddr.addr) + infoAddr.size); } template -typename CacheAllocator::ItemHandle +typename CacheAllocator::WriteHandle CacheAllocator::allocate(PoolId poolId, typename Item::Key key, uint32_t size, @@ -286,7 +320,7 @@ CacheAllocator::allocate(PoolId poolId, } template -typename CacheAllocator::ItemHandle +typename CacheAllocator::WriteHandle CacheAllocator::allocateInternal(PoolId pid, typename Item::Key key, uint32_t size, @@ -305,11 +339,11 @@ CacheAllocator::allocateInternal(PoolId pid, (*stats_.allocAttempts)[pid][cid].inc(); void* memory = allocator_->allocate(pid, requiredSize); - if (memory == nullptr && !config_.disableEviction) { + if (memory == nullptr && !config_.isEvictionDisabled()) { memory = findEviction(pid, cid); } - ItemHandle handle; + WriteHandle handle; if (memory != nullptr) { // At this point, we have a valid memory allocation that is ready for use. // Ensure that when we abort from here under any circumstances, we free up @@ -346,8 +380,8 @@ CacheAllocator::allocateInternal(PoolId pid, } template -typename CacheAllocator::ItemHandle -CacheAllocator::allocateChainedItem(const ItemHandle& parent, +typename CacheAllocator::WriteHandle +CacheAllocator::allocateChainedItem(const ReadHandle& parent, uint32_t size) { if (!parent) { throw std::invalid_argument( @@ -365,9 +399,9 @@ CacheAllocator::allocateChainedItem(const ItemHandle& parent, } template -typename CacheAllocator::ItemHandle +typename CacheAllocator::WriteHandle CacheAllocator::allocateChainedItemInternal( - const ItemHandle& parent, uint32_t size) { + const ReadHandle& parent, uint32_t size) { util::LatencyTracker tracker{stats().allocateLatency_}; SCOPE_FAIL { stats_.invalidAllocs.inc(); }; @@ -386,13 +420,14 @@ CacheAllocator::allocateChainedItemInternal( } if (memory == nullptr) { (*stats_.allocFailures)[pid][cid].inc(); - return ItemHandle{}; + return WriteHandle{}; } SCOPE_FAIL { allocator_->free(memory); }; - auto child = acquire(new (memory) ChainedItem( - compressor_.compress(parent.get()), size, util::getCurrentTimeSec())); + auto child = acquire( + new (memory) ChainedItem(compressor_.compress(parent.getInternal()), size, + util::getCurrentTimeSec())); if (child) { child.markNascent(); @@ -404,8 +439,8 @@ CacheAllocator::allocateChainedItemInternal( } template -void CacheAllocator::addChainedItem(ItemHandle& parent, - ItemHandle child) { +void CacheAllocator::addChainedItem(WriteHandle& parent, + WriteHandle child) { if (!parent || !child || !child->isChainedItem()) { throw std::invalid_argument( folly::sformat("Invalid parent or child. parent: {}, child: {}", @@ -449,14 +484,14 @@ void CacheAllocator::addChainedItem(ItemHandle& parent, } template -typename CacheAllocator::ItemHandle -CacheAllocator::popChainedItem(ItemHandle& parent) { +typename CacheAllocator::WriteHandle +CacheAllocator::popChainedItem(WriteHandle& parent) { if (!parent || !parent->hasChainedItem()) { throw std::invalid_argument(folly::sformat( "Invalid parent {}", parent ? parent->toString() : nullptr)); } - ItemHandle head; + WriteHandle head; { // scope of chained item lock. auto l = chainedItemLocks_.lockExclusive(parent->getKey()); @@ -503,8 +538,8 @@ CacheAllocator::getParentKey(const Item& chainedItem) { } template -void CacheAllocator::transferChainLocked(ItemHandle& parent, - ItemHandle& newParent) { +void CacheAllocator::transferChainLocked(WriteHandle& parent, + WriteHandle& newParent) { // parent must be in a state to not have concurrent readers. Eviction code // paths rely on holding the last item handle. Since we hold on to an item // handle here, the chain will not be touched by any eviction code path. @@ -546,7 +581,7 @@ void CacheAllocator::transferChainLocked(ItemHandle& parent, template void CacheAllocator::transferChainAndReplace( - ItemHandle& parent, ItemHandle& newParent) { + WriteHandle& parent, WriteHandle& newParent) { if (!parent || !newParent) { throw std::invalid_argument("invalid parent or new parent"); } @@ -592,9 +627,9 @@ bool CacheAllocator::replaceIfAccessible(Item& oldItem, } template -typename CacheAllocator::ItemHandle +typename CacheAllocator::WriteHandle CacheAllocator::replaceChainedItem(Item& oldItem, - ItemHandle newItemHandle, + WriteHandle newItemHandle, Item& parent) { if (!newItemHandle) { throw std::invalid_argument("Empty handle for newItem"); @@ -619,9 +654,9 @@ CacheAllocator::replaceChainedItem(Item& oldItem, } template -typename CacheAllocator::ItemHandle +typename CacheAllocator::WriteHandle CacheAllocator::replaceChainedItemLocked(Item& oldItem, - ItemHandle newItemHdl, + WriteHandle newItemHdl, const Item& parent) { XDCHECK(newItemHdl != nullptr); XDCHECK_GE(1u, oldItem.getRefCount()); @@ -734,6 +769,22 @@ CacheAllocator::releaseBackToAllocator(Item& it, config_.removeCb(RemoveCbData{ctx, it, viewAsChainedAllocsRange(it)}); } + // only skip destructor for evicted items that are either in the queue to put + // into nvm or already in nvm + if (!nascent && config_.itemDestructor && + (ctx != RemoveContext::kEviction || !it.isNvmClean() || + it.isNvmEvicted())) { + try { + config_.itemDestructor(DestructorData{ + ctx, it, viewAsChainedAllocsRange(it), allocInfo.poolId}); + stats().numRamDestructorCalls.inc(); + } catch (const std::exception& e) { + stats().numDestructorExceptions.inc(); + XLOG_EVERY_N(INFO, 100) + << "Catch exception from user's item destructor: " << e.what(); + } + } + // If no `toRecycle` is set, then the result is kReleased // Because this function cannot fail to release "it" ReleaseRes res = @@ -835,16 +886,16 @@ RefcountWithFlags::Value CacheAllocator::decRef(Item& it) { } template -typename CacheAllocator::ItemHandle +typename CacheAllocator::WriteHandle CacheAllocator::acquire(Item* it) { if (UNLIKELY(!it)) { - return ItemHandle{}; + return WriteHandle{}; } SCOPE_FAIL { stats_.numRefcountOverflow.inc(); }; incRef(*it); - return ItemHandle{it, *this}; + return WriteHandle{it, *this}; } template @@ -928,12 +979,12 @@ void CacheAllocator::insertInMMContainer(Item& item) { */ template -bool CacheAllocator::insert(const ItemHandle& handle) { +bool CacheAllocator::insert(const WriteHandle& handle) { return insertImpl(handle, AllocatorApiEvent::INSERT); } template -bool CacheAllocator::insertImpl(const ItemHandle& handle, +bool CacheAllocator::insertImpl(const WriteHandle& handle, AllocatorApiEvent event) { XDCHECK(handle); XDCHECK(event == AllocatorApiEvent::INSERT || @@ -969,18 +1020,30 @@ bool CacheAllocator::insertImpl(const ItemHandle& handle, } template -typename CacheAllocator::ItemHandle -CacheAllocator::insertOrReplace(const ItemHandle& handle) { +typename CacheAllocator::WriteHandle +CacheAllocator::insertOrReplace(const WriteHandle& handle) { XDCHECK(handle); if (handle->isAccessible()) { throw std::invalid_argument("Handle is already accessible"); } + HashedKey hk{handle->getKey()}; + insertInMMContainer(*(handle.getInternal())); - ItemHandle replaced; + WriteHandle replaced; try { + auto lock = nvmCache_ ? nvmCache_->getItemDestructorLock(hk) + : std::unique_lock(); + replaced = accessContainer_->insertOrReplace(*(handle.getInternal())); - } catch (const exception::RefcountOverflow&) { + + if (replaced && replaced->isNvmClean() && !replaced->isNvmEvicted()) { + // item is to be replaced and the destructor will be executed + // upon memory released, mark it in nvm to avoid destructor + // executed from nvm + nvmCache_->markNvmItemRemovedLocked(hk); + } + } catch (const std::exception&) { removeFromMMContainer(*(handle.getInternal())); if (auto eventTracker = getEventTracker()) { eventTracker->record(AllocatorApiEvent::INSERT_OR_REPLACE, @@ -1001,8 +1064,7 @@ CacheAllocator::insertOrReplace(const ItemHandle& handle) { // We can avoid nvm delete only if we have non nvm clean item in cache. // In all other cases we must enqueue delete. if (!replaced || replaced->isNvmClean()) { - nvmCache_->remove(handle->getKey(), - nvmCache_->createDeleteTombStone(handle->getKey())); + nvmCache_->remove(hk, nvmCache_->createDeleteTombStone(hk)); } } @@ -1022,7 +1084,7 @@ CacheAllocator::insertOrReplace(const ItemHandle& handle) { template bool CacheAllocator::moveRegularItem(Item& oldItem, - ItemHandle& newItemHdl) { + WriteHandle& newItemHdl) { XDCHECK(config_.moveCb); util::LatencyTracker tracker{stats_.moveRegularLatency_}; @@ -1055,7 +1117,7 @@ bool CacheAllocator::moveRegularItem(Item& oldItem, // there is no point to replace it since it had already been removed // or in the process of being removed. If the item is in cache but the // refcount is non-zero, it means user could be attempting to remove - // this item through an API such as remove(ItemHandle). In this case, + // this item through an API such as remove(itemHandle). In this case, // it is unsafe to replace the old item with a new one, so we should // also abort. if (!accessContainer_->replaceIf(oldItem, *newItemHdl, itemMovingPredicate)) { @@ -1101,7 +1163,7 @@ bool CacheAllocator::moveRegularItem(Item& oldItem, template bool CacheAllocator::moveChainedItem(ChainedItem& oldItem, - ItemHandle& newItemHdl) { + WriteHandle& newItemHdl) { XDCHECK(config_.moveCb); util::LatencyTracker tracker{stats_.moveChainedLatency_}; @@ -1142,7 +1204,7 @@ bool CacheAllocator::moveChainedItem(ChainedItem& oldItem, return false; } - auto parentPtr = parentHandle.get(); + auto parentPtr = parentHandle.getInternal(); XDCHECK_EQ(reinterpret_cast(parentPtr), reinterpret_cast(&oldItem.getParentItem(compressor_))); @@ -1275,7 +1337,7 @@ bool CacheAllocator::shouldWriteToNvmCacheExclusive( } template -typename CacheAllocator::ItemHandle +typename CacheAllocator::WriteHandle CacheAllocator::advanceIteratorAndTryEvictRegularItem( MMContainer& mmContainer, EvictionIterator& itr) { // we should flush this to nvmcache if it is not already present in nvmcache @@ -1290,7 +1352,7 @@ CacheAllocator::advanceIteratorAndTryEvictRegularItem( if (evictToNvmCache && !token.isValid()) { ++itr; stats_.evictFailConcurrentFill.inc(); - return ItemHandle{}; + return WriteHandle{}; } // If there are other accessors, we should abort. Acquire a handle here since @@ -1317,7 +1379,7 @@ CacheAllocator::advanceIteratorAndTryEvictRegularItem( // is set. Iterator was already advance by the remove call above. if (evictHandle->isMoving()) { stats_.evictFailMove.inc(); - return ItemHandle{}; + return WriteHandle{}; } // Invalidate iterator since later on if we are not evicting this @@ -1341,7 +1403,7 @@ CacheAllocator::advanceIteratorAndTryEvictRegularItem( } template -typename CacheAllocator::ItemHandle +typename CacheAllocator::WriteHandle CacheAllocator::advanceIteratorAndTryEvictChainedItem( EvictionIterator& itr) { XDCHECK(itr->isChainedItem()); @@ -1363,7 +1425,7 @@ CacheAllocator::advanceIteratorAndTryEvictChainedItem( // if token is invalid, return. iterator is already advanced. if (evictToNvmCache && !token.isValid()) { stats_.evictFailConcurrentFill.inc(); - return ItemHandle{}; + return WriteHandle{}; } // check if the parent exists in the hashtable and refcount is drained. @@ -1399,7 +1461,7 @@ CacheAllocator::advanceIteratorAndTryEvictChainedItem( // here since moving bit is set. if (parentHandle->isMoving()) { stats_.evictFailParentMove.inc(); - return ItemHandle{}; + return WriteHandle{}; } if (evictToNvmCache && shouldWriteToNvmCacheExclusive(*parentHandle)) { @@ -1458,14 +1520,15 @@ CacheAllocator::remove(typename Item::Key key) { // flight after removing from the hashtable. // stats_.numCacheRemoves.inc(); + HashedKey hk{key}; using Guard = typename NvmCacheT::DeleteTombStoneGuard; - auto tombStone = nvmCache_ ? nvmCache_->createDeleteTombStone(key) : Guard{}; + auto tombStone = nvmCache_ ? nvmCache_->createDeleteTombStone(hk) : Guard{}; auto handle = findInternal(key); if (!handle) { if (nvmCache_) { - nvmCache_->remove(key, std::move(tombStone)); + nvmCache_->remove(hk, std::move(tombStone)); } if (auto eventTracker = getEventTracker()) { eventTracker->record(AllocatorApiEvent::REMOVE, key, @@ -1474,13 +1537,13 @@ CacheAllocator::remove(typename Item::Key key) { return RemoveRes::kNotFoundInRam; } - return removeImpl(*handle, std::move(tombStone)); + return removeImpl(hk, *handle, std::move(tombStone)); } template bool CacheAllocator::removeFromRamForTesting( typename Item::Key key) { - return removeImpl(*findInternal(key), DeleteTombStoneGuard{}, + return removeImpl(HashedKey{key}, *findInternal(key), DeleteTombStoneGuard{}, false /* removeFromNvm */) == RemoveRes::kSuccess; } @@ -1488,7 +1551,8 @@ template void CacheAllocator::removeFromNvmForTesting( typename Item::Key key) { if (nvmCache_) { - nvmCache_->remove(key, nvmCache_->createDeleteTombStone(key)); + HashedKey hk{key}; + nvmCache_->remove(hk, nvmCache_->createDeleteTombStone(hk)); } } @@ -1521,44 +1585,60 @@ CacheAllocator::remove(AccessIterator& it) { AllocatorApiResult::REMOVED, it->getSize(), it->getConfiguredTTL().count()); } - auto tombstone = nvmCache_ ? nvmCache_->createDeleteTombStone(it->getKey()) - : DeleteTombStoneGuard{}; - return removeImpl(*it, std::move(tombstone)); + HashedKey hk{it->getKey()}; + auto tombstone = + nvmCache_ ? nvmCache_->createDeleteTombStone(hk) : DeleteTombStoneGuard{}; + return removeImpl(hk, *it, std::move(tombstone)); } template typename CacheAllocator::RemoveRes -CacheAllocator::remove(const ItemHandle& it) { +CacheAllocator::remove(const ReadHandle& it) { stats_.numCacheRemoves.inc(); if (!it) { throw std::invalid_argument("Trying to remove a null item handle"); } - auto tombstone = nvmCache_ ? nvmCache_->createDeleteTombStone(it->getKey()) - : DeleteTombStoneGuard{}; - return removeImpl(*(it.getInternal()), std::move(tombstone)); + HashedKey hk{it->getKey()}; + auto tombstone = + nvmCache_ ? nvmCache_->createDeleteTombStone(hk) : DeleteTombStoneGuard{}; + return removeImpl(hk, *(it.getInternal()), std::move(tombstone)); } template typename CacheAllocator::RemoveRes -CacheAllocator::removeImpl(Item& item, +CacheAllocator::removeImpl(HashedKey hk, + Item& item, DeleteTombStoneGuard tombstone, bool removeFromNvm, bool recordApiEvent) { - // Enqueue delete to nvmCache if we know from the item that it was pulled in - // from NVM. If the item was not pulled in from NVM, it is not possible to - // have it be written to NVM. - if (nvmCache_ && removeFromNvm && item.isNvmClean()) { - XDCHECK(tombstone); - nvmCache_->remove(item.getKey(), std::move(tombstone)); - } + bool success = false; + { + auto lock = nvmCache_ ? nvmCache_->getItemDestructorLock(hk) + : std::unique_lock(); + + success = accessContainer_->remove(item); - const bool success = accessContainer_->remove(item); + if (removeFromNvm && success && item.isNvmClean() && !item.isNvmEvicted()) { + // item is to be removed and the destructor will be executed + // upon memory released, mark it in nvm to avoid destructor + // executed from nvm + nvmCache_->markNvmItemRemovedLocked(hk); + } + } XDCHECK(!item.isAccessible()); // remove it from the mm container. this will be no-op if it is already // removed. removeFromMMContainer(item); + // Enqueue delete to nvmCache if we know from the item that it was pulled in + // from NVM. If the item was not pulled in from NVM, it is not possible to + // have it be written to NVM. + if (removeFromNvm && item.isNvmClean()) { + XDCHECK(tombstone); + nvmCache_->remove(hk, std::move(tombstone)); + } + auto eventTracker = getEventTracker(); if (recordApiEvent && eventTracker) { const auto result = @@ -1579,9 +1659,17 @@ CacheAllocator::removeImpl(Item& item, template void CacheAllocator::invalidateNvm(Item& item) { if (nvmCache_ != nullptr && item.isAccessible() && item.isNvmClean()) { - item.unmarkNvmClean(); - nvmCache_->remove(item.getKey(), - nvmCache_->createDeleteTombStone(item.getKey())); + HashedKey hk{item.getKey()}; + { + auto lock = nvmCache_->getItemDestructorLock(hk); + if (!item.isNvmEvicted() && item.isNvmClean() && item.isAccessible()) { + // item is being updated and invalidated in nvm. Mark the item to avoid + // destructor to be executed from nvm + nvmCache_->markNvmItemRemovedLocked(hk); + } + item.unmarkNvmClean(); + } + nvmCache_->remove(hk, nvmCache_->createDeleteTombStone(hk)); } } @@ -1603,17 +1691,16 @@ CacheAllocator::getMMContainer(PoolId pid, } template -typename CacheAllocator::ItemHandle +typename CacheAllocator::ReadHandle CacheAllocator::peek(typename Item::Key key) { - auto handle = findInternal(key); - return handle; + return findInternal(key); } template -std::pair::ItemHandle, - typename CacheAllocator::ItemHandle> +std::pair::ReadHandle, + typename CacheAllocator::ReadHandle> CacheAllocator::inspectCache(typename Item::Key key) { - std::pair res; + std::pair res; res.first = findInternal(key); res.second = nvmCache_ ? nvmCache_->peek(key) : nullptr; return res; @@ -1623,9 +1710,9 @@ CacheAllocator::inspectCache(typename Item::Key key) { // CacheAllocator. Hence the sprinkling of UNLIKELY/LIKELY to tell the // compiler which executions we don't want to optimize on. template -typename CacheAllocator::ItemHandle -CacheAllocator::findFastImpl(typename Item::Key key, - AccessMode mode) { +typename CacheAllocator::WriteHandle +CacheAllocator::findFastInternal(typename Item::Key key, + AccessMode mode) { auto handle = findInternal(key); stats_.numCacheGets.inc(); @@ -1639,9 +1726,10 @@ CacheAllocator::findFastImpl(typename Item::Key key, } template -typename CacheAllocator::ItemHandle -CacheAllocator::findFast(typename Item::Key key, AccessMode mode) { - auto handle = findFastImpl(key, mode); +typename CacheAllocator::WriteHandle +CacheAllocator::findFastImpl(typename Item::Key key, + AccessMode mode) { + auto handle = findFastInternal(key, mode); auto eventTracker = getEventTracker(); if (UNLIKELY(eventTracker != nullptr)) { if (handle) { @@ -1658,9 +1746,29 @@ CacheAllocator::findFast(typename Item::Key key, AccessMode mode) { } template -typename CacheAllocator::ItemHandle -CacheAllocator::find(typename Item::Key key, AccessMode mode) { - auto handle = findFastImpl(key, mode); +typename CacheAllocator::ReadHandle +CacheAllocator::findFast(typename Item::Key key) { + return findFastImpl(key, AccessMode::kRead); +} + +template +typename CacheAllocator::WriteHandle +CacheAllocator::findFastToWrite(typename Item::Key key, + bool doNvmInvalidation) { + auto handle = findFastImpl(key, AccessMode::kWrite); + if (handle == nullptr) { + return nullptr; + } + if (doNvmInvalidation) { + invalidateNvm(*handle); + } + return handle; +} + +template +typename CacheAllocator::WriteHandle +CacheAllocator::findImpl(typename Item::Key key, AccessMode mode) { + auto handle = findFastInternal(key, mode); if (handle) { if (UNLIKELY(handle->isExpired())) { @@ -1672,7 +1780,7 @@ CacheAllocator::find(typename Item::Key key, AccessMode mode) { eventTracker->record(AllocatorApiEvent::FIND, key, AllocatorApiResult::NOT_FOUND); } - ItemHandle ret; + WriteHandle ret; ret.markExpired(); return ret; } @@ -1689,7 +1797,7 @@ CacheAllocator::find(typename Item::Key key, AccessMode mode) { auto eventResult = AllocatorApiResult::NOT_FOUND; if (nvmCache_) { - handle = nvmCache_->find(key); + handle = nvmCache_->find(HashedKey{key}); eventResult = AllocatorApiResult::NOT_FOUND_IN_MEMORY; } @@ -1702,16 +1810,38 @@ CacheAllocator::find(typename Item::Key key, AccessMode mode) { } template -void CacheAllocator::markUseful(const ItemHandle& handle, +typename CacheAllocator::WriteHandle +CacheAllocator::findToWrite(typename Item::Key key, + bool doNvmInvalidation) { + auto handle = findImpl(key, AccessMode::kWrite); + if (handle == nullptr) { + return nullptr; + } + if (doNvmInvalidation) { + invalidateNvm(*handle); + } + return handle; +} + +template +typename CacheAllocator::ReadHandle +CacheAllocator::find(typename Item::Key key) { + return findImpl(key, AccessMode::kRead); +} + +template +void CacheAllocator::markUseful(const ReadHandle& handle, AccessMode mode) { if (!handle) { return; } auto& item = *(handle.getInternal()); - recordAccessInMMContainer(item, mode); + bool recorded = recordAccessInMMContainer(item, mode); - if (LIKELY(!item.hasChainedItem())) { + // if parent is not recorded, skip children as well when the config is set + if (LIKELY(!item.hasChainedItem() || + (!recorded && config_.isSkipPromoteChildrenWhenParentFailed()))) { return; } @@ -1721,7 +1851,7 @@ void CacheAllocator::markUseful(const ItemHandle& handle, } template -void CacheAllocator::recordAccessInMMContainer(Item& item, +bool CacheAllocator::recordAccessInMMContainer(Item& item, AccessMode mode) { const auto allocInfo = allocator_->getAllocInfo(static_cast(&item)); @@ -1733,7 +1863,7 @@ void CacheAllocator::recordAccessInMMContainer(Item& item, } auto& mmContainer = getMMContainer(allocInfo.poolId, allocInfo.classId); - mmContainer.recordAccess(item, mode); + return mmContainer.recordAccess(item, mode); } template @@ -1746,20 +1876,20 @@ uint32_t CacheAllocator::getUsableSize(const Item& item) const { } template -typename CacheAllocator::ItemHandle +typename CacheAllocator::ReadHandle CacheAllocator::getSampleItem() { const auto* item = reinterpret_cast(allocator_->getRandomAlloc()); if (!item) { - return ItemHandle{}; + return ReadHandle{}; } - auto handle = findInternal(item->getKey()); + ReadHandle handle = findInternal(item->getKey()); // Check that item returned is the same that was sampled if (handle.get() == item) { return handle; } - return ItemHandle{}; + return ReadHandle{}; } template @@ -1790,12 +1920,13 @@ std::vector CacheAllocator::dumpEvictionIterator( } template -folly::IOBuf CacheAllocator::convertToIOBuf(ItemHandle handle) { +template +folly::IOBuf CacheAllocator::convertToIOBufT(Handle& handle) { if (!handle) { throw std::invalid_argument("null item handle for converting to IOBUf"); } - Item* item = handle.get(); + Item* item = handle.getInternal(); const uint32_t dataOffset = item->getOffsetForMemory(); using ConvertChainedItem = std::function( @@ -1804,10 +1935,10 @@ folly::IOBuf CacheAllocator::convertToIOBuf(ItemHandle handle) { ConvertChainedItem converter; // based on current refcount and threshold from config - // determine to use a new ItemHandle for each chain items - // or use shared ItemHandle for all chain items + // determine to use a new Item Handle for each chain items + // or use shared Item Handle for all chain items if (item->getRefCount() > config_.thresholdForConvertingToIOBuf) { - auto sharedHdl = std::make_shared(std::move(handle)); + auto sharedHdl = std::make_shared(std::move(handle)); iobuf = folly::IOBuf{ folly::IOBuf::TAKE_OWNERSHIP, item, @@ -1818,10 +1949,10 @@ folly::IOBuf CacheAllocator::convertToIOBuf(ItemHandle handle) { dataOffset + item->getSize(), [](void*, void* userData) { - auto* hdl = reinterpret_cast*>(userData); + auto* hdl = reinterpret_cast*>(userData); delete hdl; } /* freeFunc */, - new std::shared_ptr{sharedHdl} /* userData for freeFunc */}; + new std::shared_ptr{sharedHdl} /* userData for freeFunc */}; if (item->hasChainedItem()) { converter = [sharedHdl](Item*, ChainedItem& chainedItem) { @@ -1836,31 +1967,30 @@ folly::IOBuf CacheAllocator::convertToIOBuf(ItemHandle handle) { chainedItemDataOffset + chainedItem.getSize(), [](void*, void* userData) { - auto* hdl = - reinterpret_cast*>(userData); + auto* hdl = reinterpret_cast*>(userData); delete hdl; } /* freeFunc */, - new std::shared_ptr{ - sharedHdl} /* userData for freeFunc */); + new std::shared_ptr{sharedHdl} /* userData for freeFunc */); }; } } else { - iobuf = - folly::IOBuf{folly::IOBuf::TAKE_OWNERSHIP, item, + // following IOBuf will take the item's ownership and trigger freeFunc to + // release the reference count. + handle.release(); + iobuf = folly::IOBuf{folly::IOBuf::TAKE_OWNERSHIP, item, - // Since we'll be moving the IOBuf data pointer forward - // by dataOffset, we need to adjust the IOBuf length - // accordingly - dataOffset + item->getSize(), + // Since we'll be moving the IOBuf data pointer forward + // by dataOffset, we need to adjust the IOBuf length + // accordingly + dataOffset + item->getSize(), - [](void* buf, void* userData) { - ItemHandle{reinterpret_cast(buf), + [](void* buf, void* userData) { + Handle{reinterpret_cast(buf), *reinterpret_cast(userData)} - .reset(); - } /* freeFunc */, - this /* userData for freeFunc */}; - handle.release(); + .reset(); + } /* freeFunc */, + this /* userData for freeFunc */}; if (item->hasChainedItem()) { converter = [this](Item* parentItem, ChainedItem& chainedItem) { @@ -1889,7 +2019,7 @@ folly::IOBuf CacheAllocator::convertToIOBuf(ItemHandle handle) { auto* cache = reinterpret_cast(userData); auto* child = reinterpret_cast(buf); auto* parent = &child->getParentItem(cache->compressor_); - ItemHandle{parent, *cache}.reset(); + Handle{parent, *cache}.reset(); } /* freeFunc */, this /* userData for freeFunc */); }; @@ -2011,7 +2141,6 @@ void CacheAllocator::overridePoolConfig(PoolId pid, .getAllocsPerSlab() : 0); DCHECK_NOTNULL(mmContainers_[pid][cid].get()); - mmContainers_[pid][cid]->setConfig(mmConfig); } } @@ -2312,7 +2441,7 @@ bool CacheAllocator::moveForSlabRelease( bool isMoved = false; auto startTime = util::getCurrentTimeSec(); - ItemHandle newItemHdl = allocateNewItemForOldItem(oldItem); + WriteHandle newItemHdl = allocateNewItemForOldItem(oldItem); for (unsigned int itemMovingAttempts = 0; itemMovingAttempts < config_.movingTries; @@ -2379,10 +2508,10 @@ bool CacheAllocator::moveForSlabRelease( } template -typename CacheAllocator::ItemHandle +typename CacheAllocator::ReadHandle CacheAllocator::validateAndGetParentHandleForChainedMoveLocked( const ChainedItem& item, const Key& parentKey) { - ItemHandle parentHandle{}; + ReadHandle parentHandle{}; try { parentHandle = findInternal(parentKey); // If the parent is not the same as the parent of the chained item, @@ -2399,7 +2528,7 @@ CacheAllocator::validateAndGetParentHandleForChainedMoveLocked( } template -typename CacheAllocator::ItemHandle +typename CacheAllocator::WriteHandle CacheAllocator::allocateNewItemForOldItem(const Item& oldItem) { if (oldItem.isChainedItem()) { const auto& oldChainedItem = oldItem.asChainedItem(); @@ -2423,7 +2552,7 @@ CacheAllocator::allocateNewItemForOldItem(const Item& oldItem) { } XDCHECK_EQ(newItemHdl->getSize(), oldChainedItem.getSize()); - auto parentPtr = parentHandle.get(); + auto parentPtr = parentHandle.getInternal(); XDCHECK_EQ(reinterpret_cast(parentPtr), reinterpret_cast( &oldChainedItem.getParentItem(compressor_))); @@ -2454,7 +2583,7 @@ CacheAllocator::allocateNewItemForOldItem(const Item& oldItem) { template bool CacheAllocator::tryMovingForSlabRelease( - Item& oldItem, ItemHandle& newItemHdl) { + Item& oldItem, WriteHandle& newItemHdl) { // By holding onto a user-level synchronization object, we ensure moving // a regular item or chained item is synchronized with any potential // user-side mutation. @@ -2492,7 +2621,7 @@ bool CacheAllocator::tryMovingForSlabRelease( template void CacheAllocator::evictForSlabRelease( const SlabReleaseContext& ctx, Item& item, util::Throttler& throttler) { - XDCHECK(!config_.disableEviction); + XDCHECK(!config_.isEvictionDisabled()); auto startTime = util::getCurrentTimeSec(); while (true) { @@ -2571,12 +2700,12 @@ void CacheAllocator::evictForSlabRelease( } template -typename CacheAllocator::ItemHandle +typename CacheAllocator::WriteHandle CacheAllocator::evictNormalItemForSlabRelease(Item& item) { XDCHECK(item.isMoving()); if (item.isOnlyMoving()) { - return ItemHandle{}; + return WriteHandle{}; } auto predicate = [](const Item& it) { return it.getRefCount() == 0; }; @@ -2609,7 +2738,7 @@ CacheAllocator::evictNormalItemForSlabRelease(Item& item) { } template -typename CacheAllocator::ItemHandle +typename CacheAllocator::WriteHandle CacheAllocator::evictChainedItemForSlabRelease(ChainedItem& child) { XDCHECK(child.isMoving()); @@ -2710,7 +2839,7 @@ CacheAllocator::evictChainedItemForSlabRelease(ChainedItem& child) { } template -bool CacheAllocator::removeIfExpired(const ItemHandle& handle) { +bool CacheAllocator::removeIfExpired(const ReadHandle& handle) { if (!handle) { return false; } @@ -2870,7 +2999,6 @@ typename CacheTrait::MMType::LruType CacheAllocator::getItemLruType( // --------------------------------- // | accessContainer_ | // | mmContainers_ | -// | emptyMMContainers | // | compactCacheManager_ | // | allocator_ | // | metadata_ | @@ -2882,31 +3010,31 @@ folly::IOBufQueue CacheAllocator::saveStateToIOBuf() { "There are still slabs being released at the moment"); } - *metadata_.allocatorVersion_ref() = kCachelibVersion; - *metadata_.ramFormatVersion_ref() = kCacheRamFormatVersion; - *metadata_.cacheCreationTime_ref() = static_cast(cacheCreationTime_); - *metadata_.mmType_ref() = MMType::kId; - *metadata_.accessType_ref() = AccessType::kId; + *metadata_.allocatorVersion() = kCachelibVersion; + *metadata_.ramFormatVersion() = kCacheRamFormatVersion; + *metadata_.cacheCreationTime() = static_cast(cacheCreationTime_); + *metadata_.mmType() = MMType::kId; + *metadata_.accessType() = AccessType::kId; - metadata_.compactCachePools_ref()->clear(); + metadata_.compactCachePools()->clear(); const auto pools = getPoolIds(); { folly::SharedMutex::ReadHolder lock(compactCachePoolsLock_); for (PoolId pid : pools) { for (unsigned int cid = 0; cid < (*stats_.fragmentationSize)[pid].size(); ++cid) { - metadata_.fragmentationSize_ref()[pid][static_cast(cid)] = + metadata_.fragmentationSize()[pid][static_cast(cid)] = (*stats_.fragmentationSize)[pid][cid].get(); } if (isCompactCachePool_[pid]) { - metadata_.compactCachePools_ref()->push_back(pid); + metadata_.compactCachePools()->push_back(pid); } } } - *metadata_.numChainedParentItems_ref() = stats_.numChainedParentItems.get(); - *metadata_.numChainedChildItems_ref() = stats_.numChainedChildItems.get(); - *metadata_.numAbortedSlabReleases_ref() = stats_.numAbortedSlabReleases.get(); + *metadata_.numChainedParentItems() = stats_.numChainedParentItems.get(); + *metadata_.numChainedChildItems() = stats_.numChainedChildItems.get(); + *metadata_.numAbortedSlabReleases() = stats_.numAbortedSlabReleases.get(); auto serializeMMContainers = [](MMContainers& mmContainers) { MMSerializationTypeContainer state; @@ -2922,13 +3050,6 @@ folly::IOBufQueue CacheAllocator::saveStateToIOBuf() { MMSerializationTypeContainer mmContainersState = serializeMMContainers(mmContainers_); - // On version 15, persist the empty unevictable mmcontainer. - // So that version <= 14 can still load a metadata saved by version 15. - // TODO: Remove this on version 16. - MMContainers dummyMMContainers = createEmptyMMContainers(); - MMSerializationTypeContainer unevictableMMContainersState = - serializeMMContainers(dummyMMContainers); - AccessSerializationType accessContainerState = accessContainer_->saveState(); MemoryAllocator::SerializationType allocatorState = allocator_->saveState(); CCacheManager::SerializationType ccState = compactCacheManager_->saveState(); @@ -2943,7 +3064,6 @@ folly::IOBufQueue CacheAllocator::saveStateToIOBuf() { Serializer::serializeToIOBufQueue(queue, allocatorState); Serializer::serializeToIOBufQueue(queue, ccState); Serializer::serializeToIOBufQueue(queue, mmContainersState); - Serializer::serializeToIOBufQueue(queue, unevictableMMContainersState); Serializer::serializeToIOBufQueue(queue, accessContainerState); Serializer::serializeToIOBufQueue(queue, chainedItemAccessContainerState); return queue; @@ -3041,8 +3161,13 @@ void CacheAllocator::saveRamCache() { std::unique_ptr ioBuf = serializedBuf.move(); ioBuf->coalesce(); + ShmSegmentOpts opts; + opts.typeOpts = PosixSysVSegmentOpts(config_.usePosixShm); + void* infoAddr = - shmManager_->createShm(detail::kShmInfoName, ioBuf->length()).addr; + shmManager_ + ->createShm(detail::kShmInfoName, ioBuf->length(), nullptr, opts) + .addr; Serializer serializer(reinterpret_cast(infoAddr), reinterpret_cast(infoAddr) + ioBuf->length()); serializer.writeToBuffer(std::move(ioBuf)); @@ -3075,8 +3200,8 @@ CacheAllocator::deserializeMMContainers( } } // We need to drop the unevictableMMContainer in the desierializer. - // TODO: remove this when all use case are later than version 15. - if (metadata_.allocatorVersion_ref() <= 15) { + // TODO: remove this at version 17. + if (metadata_.allocatorVersion() <= 15) { deserializer.deserialize(); } return mmContainers; @@ -3107,22 +3232,22 @@ CacheAllocator::deserializeCacheAllocatorMetadata( // TODO: // Once everyone is on v8 or later, remove the outter if. if (kCachelibVersion > 8) { - if (*meta.ramFormatVersion_ref() != kCacheRamFormatVersion) { + if (*meta.ramFormatVersion() != kCacheRamFormatVersion) { throw std::runtime_error( folly::sformat("Expected cache ram format version {}. But found {}.", - kCacheRamFormatVersion, *meta.ramFormatVersion_ref())); + kCacheRamFormatVersion, *meta.ramFormatVersion())); } } - if (*meta.accessType_ref() != AccessType::kId) { + if (*meta.accessType() != AccessType::kId) { throw std::invalid_argument( - folly::sformat("Expected {}, got {} for AccessType", - *meta.accessType_ref(), AccessType::kId)); + folly::sformat("Expected {}, got {} for AccessType", *meta.accessType(), + AccessType::kId)); } - if (*meta.mmType_ref() != MMType::kId) { - throw std::invalid_argument(folly::sformat( - "Expected {}, got {} for MMType", *meta.mmType_ref(), MMType::kId)); + if (*meta.mmType() != MMType::kId) { + throw std::invalid_argument(folly::sformat("Expected {}, got {} for MMType", + *meta.mmType(), MMType::kId)); } return meta; } @@ -3153,7 +3278,7 @@ void CacheAllocator::initStats() { stats_.init(); // deserialize the fragmentation size of each thread. - for (const auto& pid : *metadata_.fragmentationSize_ref()) { + for (const auto& pid : *metadata_.fragmentationSize()) { for (const auto& cid : pid.second) { (*stats_.fragmentationSize)[pid.first][cid.first].set( static_cast(cid.second)); @@ -3161,10 +3286,10 @@ void CacheAllocator::initStats() { } // deserialize item counter stats - stats_.numChainedParentItems.set(*metadata_.numChainedParentItems_ref()); - stats_.numChainedChildItems.set(*metadata_.numChainedChildItems_ref()); + stats_.numChainedParentItems.set(*metadata_.numChainedParentItems()); + stats_.numChainedChildItems.set(*metadata_.numChainedChildItems()); stats_.numAbortedSlabReleases.set( - static_cast(*metadata_.numAbortedSlabReleases_ref())); + static_cast(*metadata_.numAbortedSlabReleases())); } template @@ -3185,7 +3310,7 @@ void CacheAllocator::forEachChainedItem( } template -typename CacheAllocator::ItemHandle +typename CacheAllocator::WriteHandle CacheAllocator::findChainedItem(const Item& parent) const { const auto cPtr = compressor_.compress(&parent); return chainedItemAccessContainer_->find( @@ -3193,8 +3318,9 @@ CacheAllocator::findChainedItem(const Item& parent) const { } template -typename CacheAllocator::ChainedAllocs -CacheAllocator::viewAsChainedAllocs(const ItemHandle& parent) { +template +CacheChainedAllocs, Handle, Iter> +CacheAllocator::viewAsChainedAllocsT(const Handle& parent) { XDCHECK(parent); auto handle = parent.clone(); if (!handle) { @@ -3210,7 +3336,8 @@ CacheAllocator::viewAsChainedAllocs(const ItemHandle& parent) { auto l = chainedItemLocks_.lockShared(handle->getKey()); auto head = findChainedItem(*handle); - return ChainedAllocs{std::move(l), std::move(handle), *head, compressor_}; + return CacheChainedAllocs, Handle, Iter>{ + std::move(l), std::move(handle), *head, compressor_}; } template @@ -3221,12 +3348,17 @@ GlobalCacheStats CacheAllocator::getGlobalCacheStats() const { ret.numItems = accessContainer_->getStats().numKeys; const uint64_t currTime = util::getCurrentTimeSec(); + ret.cacheInstanceUpTime = currTime - cacheInstanceCreationTime_; ret.ramUpTime = currTime - cacheCreationTime_; ret.nvmUpTime = currTime - nvmCacheState_.getCreationTime(); ret.nvmCacheEnabled = nvmCache_ ? nvmCache_->isEnabled() : false; ret.reaperStats = getReaperStats(); ret.numActiveHandles = getNumActiveHandles(); + ret.isNewRamCache = cacheCreationTime_ == cacheInstanceCreationTime_; + ret.isNewNvmCache = + nvmCacheState_.getCreationTime() == cacheInstanceCreationTime_; + return ret; } @@ -3251,8 +3383,8 @@ CacheMemoryStats CacheAllocator::getCacheMemoryStats() const { memMonitor_ ? memMonitor_->getMaxAdvisePct() : 0, allocator_->getUnreservedMemorySize(), nvmCache_ ? nvmCache_->getSize() : 0, - memMonitor_ ? memMonitor_->getMemAvailableSize() : 0, - memMonitor_ ? memMonitor_->getMemRssSize() : 0}; + util::getMemAvailable(), + util::getRSSBytes()}; } template @@ -3267,6 +3399,11 @@ bool CacheAllocator::autoResizeEnabledForPool(PoolId pid) const { } } +template +void CacheAllocator::startCacheWorkers() { + initWorkers(); +} + template template bool CacheAllocator::stopWorker(folly::StringPiece name, @@ -3385,8 +3522,8 @@ bool CacheAllocator::stopReaper(std::chrono::seconds timeout) { } template -bool CacheAllocator::cleanupStrayShmSegments( - const std::string& cacheDir, bool posix) { +bool CacheAllocator< + CacheTrait>::cleanupStrayShmSegments(const std::string& cacheDir, bool posix /*TODO(SHM_FILE): const std::vector& config */) { if (util::getStatIfExists(cacheDir, nullptr) && util::isDir(cacheDir)) { try { // cache dir exists. clean up only if there are no other processes @@ -3405,12 +3542,23 @@ bool CacheAllocator::cleanupStrayShmSegments( ShmManager::removeByName(cacheDir, detail::kShmHashTableName, posix); ShmManager::removeByName(cacheDir, detail::kShmChainedItemHashTableName, posix); + + // TODO(SHM_FILE): try to nuke segments of differente types (which require + // extra info) + // for (auto &tier : config) { + // ShmManager::removeByName(cacheDir, tierShmName, + // config_.memoryTiers[i].opts); + // } } return true; } template -uintptr_t CacheAllocator::getItemPtrAsOffset(const void* ptr) { +uint64_t CacheAllocator::getItemPtrAsOffset(const void* ptr) { + // Return unt64_t instead of uintptr_t to accommodate platforms where + // the two differ (e.g. Mac OS 12) - causing templating instantiation + // errors downstream. + // if this succeeeds, the address is valid within the cache. allocator_->getAllocInfo(ptr); @@ -3420,8 +3568,8 @@ uintptr_t CacheAllocator::getItemPtrAsOffset(const void* ptr) { const auto& shm = shmManager_->getShmByName(detail::kShmCacheName); - return reinterpret_cast(ptr) - - reinterpret_cast(shm.getCurrentMapping().addr); + return reinterpret_cast(ptr) - + reinterpret_cast(shm.getCurrentMapping().addr); } template diff --git a/cachelib/allocator/CacheAllocator.h b/cachelib/allocator/CacheAllocator.h index a065ff208f..cfbf7b74f5 100644 --- a/cachelib/allocator/CacheAllocator.h +++ b/cachelib/allocator/CacheAllocator.h @@ -21,9 +21,11 @@ #include #include #include +#include #include #include +#include #include #include #include @@ -81,6 +83,17 @@ namespace cachelib { template class FbInternalRuntimeUpdateWrapper; +template +class ReadOnlyMap; + +namespace objcache2 { +template +class ObjectCache; + +template +class ObjectCacheBase; +} // namespace objcache2 + namespace cachebench { template class Cache; @@ -99,6 +112,12 @@ class AllocatorHitStatsTest; template class AllocatorResizeTest; +template +class FixedSizeArrayTest; + +template +class MapTest; + class NvmCacheTest; template @@ -110,6 +129,20 @@ class CacheAllocatorTestWrapper; class PersistenceCache; } // namespace tests +namespace objcache { +template +class ObjectCache; +namespace test { +#define GET_CLASS_NAME(test_case_name, test_name) \ + test_case_name##_##test_name##_Test + +#define GET_DECORATED_CLASS_NAME(namespace, test_case_name, test_name) \ + namespace ::GET_CLASS_NAME(test_case_name, test_name) + +class GET_CLASS_NAME(ObjectCache, ObjectHandleInvalid); +} // namespace test +} // namespace objcache + // CacheAllocator can provide an interface to make Keyed Allocations(Item) and // takes two templated types that control how the allocation is // maintained(MMType aka MemoryManagementType) and accessed(AccessType). The @@ -151,10 +184,29 @@ class CacheAllocator : public CacheBase { using Item = CacheItem; using ChainedItem = typename Item::ChainedItem; + // the holder for the item when we hand it to the caller. This ensures + // that the reference count is maintained when the caller is done with the + // item. The ReadHandle/WriteHandle provides a getMemory() and getKey() + // interface. The caller is free to use the result of these two as long as the + // handle is active/alive. Using the result of the above interfaces after + // destroying the ReadHandle/WriteHandle is UB. The ReadHandle/WriteHandle + // safely wraps a pointer to the "const Item"/"Item". + using ReadHandle = typename Item::ReadHandle; + using WriteHandle = typename Item::WriteHandle; + using ItemHandle = WriteHandle; + template > + using TypedHandle = TypedHandleImpl; + // TODO (sathya) some types take CacheT and some take CacheTrait. need to // clean this up and come up with a consistent policy that is intuitive. - using ChainedAllocs = CacheChainedAllocs; - using ChainedItemIter = CacheChainedItemIterator; + using ChainedItemIter = CacheChainedItemIterator; + using WritableChainedItemIter = CacheChainedItemIterator; + using ChainedAllocs = CacheChainedAllocs; + using WritableChainedAllocs = + CacheChainedAllocs; + using Key = typename Item::Key; using PoolIds = std::set; @@ -171,6 +223,49 @@ class CacheAllocator : public CacheBase { // Iterator range pointing to chained allocs associated with @item folly::Range chainedAllocs; }; + struct DestructorData { + DestructorData(DestructorContext ctx, + Item& it, + folly::Range iter, + PoolId id) + : context(ctx), item(it), chainedAllocs(iter), pool(id) {} + + // helps to convert RemoveContext to DestructorContext, + // the context for RemoveCB is re-used to create DestructorData, + // this can be removed if RemoveCB is dropped. + DestructorData(RemoveContext ctx, + Item& it, + folly::Range iter, + PoolId id) + : item(it), chainedAllocs(iter), pool(id) { + if (ctx == RemoveContext::kEviction) { + context = DestructorContext::kEvictedFromRAM; + } else { + context = DestructorContext::kRemovedFromRAM; + } + } + + // remove or eviction + DestructorContext context; + + // item about to be freed back to allocator + // when the item is evicted/removed from NVM, the item is created on the + // heap, functions (e.g. CacheAllocator::getAllocInfo) that assumes item is + // located in cache slab doesn't work in such case. + // chained items must be iterated though @chainedAllocs. + // Other APIs used to access chained items are not compatible and should not + // be used. + Item& item; + + // Iterator range pointing to chained allocs associated with @item + // when chained items are evicted/removed from NVM, items are created on the + // heap, functions (e.g. CacheAllocator::getAllocInfo) that assumes items + // are located in cache slab doesn't work in such case. + folly::Range chainedAllocs; + + // the pool that this item is/was + PoolId pool; + }; // call back to execute when moving an item, this could be a simple memcpy // or something more complex. @@ -180,20 +275,14 @@ class CacheAllocator : public CacheBase { std::function; // call back type that is executed when the cache item is removed - // (evicted / freed) + // (evicted / freed) from RAM, only items inserted into cache (not nascent) + // successfully are tracked using RemoveCb = std::function; - // the holder for the item when we hand it to the caller. This ensures - // that the reference count is maintained when the caller is done with the - // item. The ItemHandle provides a getMemory() and getKey() interface. The - // caller is free to use the result of these two as long as the handle is - // active/alive. Using the result of the above interfaces after destroying - // the ItemHandle is UB. The ItemHandle safely wraps a pointer to the Item. - using ItemHandle = typename Item::Handle; - template > - using TypedHandle = TypedHandleImpl; + // the destructor being executed when the item is removed from cache (both RAM + // and NVM), only items inserted into cache (not nascent) successfully are + // tracked. + using ItemDestructor = std::function; using NvmCacheT = NvmCache; using NvmCacheConfig = typename NvmCacheT::Config; @@ -271,11 +360,11 @@ class CacheAllocator : public CacheBase { // @throw std::invalid_argument if the poolId is invalid or the size // requested is invalid or if the key is invalid(key.size() == 0 or // key.size() > 255) - ItemHandle allocate(PoolId id, - Key key, - uint32_t size, - uint32_t ttlSecs = 0, - uint32_t creationTime = 0); + WriteHandle allocate(PoolId id, + Key key, + uint32_t size, + uint32_t ttlSecs = 0, + uint32_t creationTime = 0); // Allocate a chained item // @@ -291,7 +380,7 @@ class CacheAllocator : public CacheBase { // @return handle to the chained allocation // @throw std::invalid_argument if the size requested is invalid or // if the item is invalid - ItemHandle allocateChainedItem(const ItemHandle& parent, uint32_t size); + WriteHandle allocateChainedItem(const ReadHandle& parent, uint32_t size); // Link a chained item to a parent item and mark this parent handle as having // chained allocations. @@ -302,7 +391,7 @@ class CacheAllocator : public CacheBase { // @param child chained item that will be linked to the parent // // @throw std::invalid_argument if parent is nullptr - void addChainedItem(ItemHandle& parent, ItemHandle child); + void addChainedItem(WriteHandle& parent, WriteHandle child); // Pop the first chained item assocaited with this parent and unmark this // parent handle as having chained allocations. @@ -313,7 +402,7 @@ class CacheAllocator : public CacheBase { // // @return ChainedItem head if there exists one // nullptr otherwise - ItemHandle popChainedItem(ItemHandle& parent); + WriteHandle popChainedItem(WriteHandle& parent); // Return the key to the parent item. // @@ -339,9 +428,9 @@ class CacheAllocator : public CacheBase { // @return handle to the oldItem on return. // // @throw std::invalid_argument if any of the pre-conditions fails - ItemHandle replaceChainedItem(Item& oldItem, - ItemHandle newItem, - Item& parent); + WriteHandle replaceChainedItem(Item& oldItem, + WriteHandle newItem, + Item& parent); // Transfers the ownership of the chain from the current parent to the new // parent and inserts the new parent into the cache. Parent will be unmarked @@ -362,7 +451,7 @@ class CacheAllocator : public CacheBase { // @throw std::invalid_argument if the parent does not have chained item or // incorrect state of chained item or if any of the pre-conditions // are not met - void transferChainAndReplace(ItemHandle& parent, ItemHandle& newParent); + void transferChainAndReplace(WriteHandle& parent, WriteHandle& newParent); // Inserts the allocated handle into the AccessContainer, making it // accessible for everyone. This needs to be the handle that the caller @@ -375,7 +464,7 @@ class CacheAllocator : public CacheBase { // and is now accessible to everyone. False if there was an error. // // @throw std::invalid_argument if the handle is already accessible. - bool insert(const ItemHandle& handle); + bool insert(const WriteHandle& handle); // Replaces the allocated handle into the AccessContainer, making it // accessible for everyone. If an existing handle is already in the @@ -389,28 +478,48 @@ class CacheAllocator : public CacheBase { // @throw cachelib::exception::RefcountOverflow if the item we are replacing // is already out of refcounts. // @return handle to the old item that had been replaced - ItemHandle insertOrReplace(const ItemHandle& handle); + WriteHandle insertOrReplace(const WriteHandle& handle); // look up an item by its key across the nvm cache as well if enabled. // - // @param key the key for lookup - // @param mode the mode of access for the lookup. defaults to - // AccessMode::kRead + // @param key the key for lookup // - // @return the handle for the item or a handle to nullptr if the key does - // not exist. - ItemHandle find(Key key, AccessMode mode = AccessMode::kRead); + // @return the read handle for the item or a handle to nullptr if the + // key does not exist. + ReadHandle find(Key key); + + // look up an item by its key across the nvm cache as well if enabled. Users + // should call this API only when they are going to mutate the item data. + // + // @param key the key for lookup + // @param isNvmInvalidate whether to do nvm invalidation; + // defaults to be true + // + // @return the write handle for the item or a handle to nullptr if the + // key does not exist. + WriteHandle findToWrite(Key key, bool doNvmInvalidation = true); // look up an item by its key. This ignores the nvm cache and only does RAM // lookup. // // @param key the key for lookup - // @param mode the mode of access for the lookup. defaults to - // AccessMode::kRead // - // @return the handle for the item or a handle to nullptr if the key does - // not exist. - FOLLY_ALWAYS_INLINE ItemHandle findFast(Key key, AccessMode mode); + // @return the read handle for the item or a handle to nullptr if the key + // does not exist. + FOLLY_ALWAYS_INLINE ReadHandle findFast(Key key); + + // look up an item by its key. This ignores the nvm cache and only does RAM + // lookup. Users should call this API only when they are going to mutate the + // item data. + // + // @param key the key for lookup + // @param isNvmInvalidate whether to do nvm invalidation; + // defaults to be true + // + // @return the write handle for the item or a handle to nullptr if the + // key does not exist. + FOLLY_ALWAYS_INLINE WriteHandle + findFastToWrite(Key key, bool doNvmInvalidation = true); // look up an item by its key. This ignores the nvm cache and only does RAM // lookup. This API does not update the stats related to cache gets and misses @@ -419,7 +528,7 @@ class CacheAllocator : public CacheBase { // @param key the key for lookup // @return the handle for the item or a handle to nullptr if the key does // not exist. - FOLLY_ALWAYS_INLINE ItemHandle peek(Key key); + FOLLY_ALWAYS_INLINE ReadHandle peek(Key key); // Mark an item that was fetched through peek as useful. This is useful when // users want to look into the cache and only mark items as useful when they @@ -428,7 +537,7 @@ class CacheAllocator : public CacheBase { // @param handle the item handle // @param mode the mode of access for the lookup. defaults to // AccessMode::kRead - void markUseful(const ItemHandle& handle, AccessMode mode); + void markUseful(const ReadHandle& handle, AccessMode mode); using AccessIterator = typename AccessContainer::Iterator; // Iterator interface for the cache. It guarantees that all keys that were @@ -456,7 +565,7 @@ class CacheAllocator : public CacheBase { // removes the allocation corresponding to the key, if present in the hash // table. The key will not be accessible through find() after this returns // success. The allocation for the key will be recycled once all active - // ItemHandles are released. + // Item handles are released. // // @param key the key for the allocation. // @return kSuccess if the key exists and was successfully removed. @@ -478,28 +587,47 @@ class CacheAllocator : public CacheBase { // removes the allocation corresponding to the handle. The allocation will // be freed when all the existing handles are released. // - // @param it item handle + // @param it item read handle // // @return kSuccess if the item exists and was successfully removed. // kNotFoundInRam otherwise // // @throw std::invalid_argument if item handle is null - RemoveRes remove(const ItemHandle& it); + RemoveRes remove(const ReadHandle& it); + + // view a read-only parent item as a chain of allocations if it has chained + // alloc. The returned chained-alloc is good to iterate upon, but will block + // any concurrent addChainedItem or popChainedItem for the same key until the + // ChainedAllocs object is released. This is ideal for use cases which do + // very brief operations on the chain of allocations. + // + // The ordering of the iteration for the chain is LIFO. Check + // CacheChainedAllocs.h for the API and usage. + // + // @param parent the parent allocation of the chain from a ReadHandle. + // @return read-only chained alloc view of the parent + // + // @throw std::invalid_argument if the parent does not have chained allocs + ChainedAllocs viewAsChainedAllocs(const ReadHandle& parent) { + return viewAsChainedAllocsT(parent); + } - // view a parent item as a chain of allocations if it has chained alloc. - // The returned chained-alloc is good to iterate upon, but will block any - // concurrent addChainedItem or popChainedItem for the same key until the + // view a writable parent item as a chain of allocations if it has chained + // alloc. The returned chained-alloc is good to iterate upon, but will block + // any concurrent addChainedItem or popChainedItem for the same key until the // ChainedAllocs object is released. This is ideal for use cases which do // very brief operations on the chain of allocations. // // The ordering of the iteration for the chain is LIFO. Check // CacheChainedAllocs.h for the API and usage. // - // @param parent the parent allocation of the chain. - // @return chained alloc view of the paren + // @param parent the parent allocation of the chain from a WriteHandle. + // @return writable chained alloc view of the parent // // @throw std::invalid_argument if the parent does not have chained allocs - ChainedAllocs viewAsChainedAllocs(const ItemHandle& parent); + WritableChainedAllocs viewAsWritableChainedAllocs(const WriteHandle& parent) { + return viewAsChainedAllocsT(parent); + } // Returns the full usable size for this item // This can be bigger than item.getSize() @@ -512,30 +640,50 @@ class CacheAllocator : public CacheBase { // Get a random item from memory // This is useful for profiling and sampling cachelib managed memory // - // @return ItemHandle if an valid item is found + // @return ReadHandle if an valid item is found // // nullptr if the randomly chosen memory does not belong // to an valid item - ItemHandle getSampleItem(); + ReadHandle getSampleItem(); - // TODO: When Read/Write Handles are ready, change this to allow - // const-only access to data manged by iobuf and offer a - // convertToWritableIOBuf() API. - // - // Convert an item handle to an IOBuf. The returned IOBuf gives a + // Convert a Read Handle to an IOBuf. The returned IOBuf gives a // read-only view to the user. The item's ownership is retained by // the IOBuf until its destruction. // - // When the item handle has one or more chained items attached to it, + // When the read handle has one or more chained items attached to it, + // user will also get a series of IOBufs (first of which is the Parent). + // + // **WARNING**: folly::IOBuf allows mutation to a cachelib item even when the + // item is read-only. User is responsible to ensure no mutation occurs (i.e. + // only const functions are called). If mutation is required, please use + // `convertToIOBufForWrite`. + // + // @param handle read handle that will transfer its ownership to an IOBuf + // + // @return an IOBuf that contains the value of the item. + // This IOBuf acts as a Read Handle, on destruction, it will + // properly decrement the refcount (to release the item). + // @throw std::invalid_argument if ReadHandle is nullptr + folly::IOBuf convertToIOBuf(ReadHandle handle) { + return convertToIOBufT(handle); + } + + // Convert a Write Handle to an IOBuf. The returned IOBuf gives a + // writable view to the user. The item's ownership is retained by + // the IOBuf until its destruction. + // + // When the write handle has one or more chained items attached to it, // user will also get a series of IOBufs (first of which is the Parent). // - // @param handle item handle that will transfer its ownership to an IOBuf + // @param handle write handle that will transfer its ownership to an IOBuf // // @return an IOBuf that contains the value of the item. - // This IOBuf acts as an Item Handle, on destruction, it will + // This IOBuf acts as a Write Handle, on destruction, it will // properly decrement the refcount (to release the item). - // @throw std::invalid_argument if ItemHandle is nullptr - folly::IOBuf convertToIOBuf(ItemHandle handle); + // @throw std::invalid_argument if WriteHandle is nullptr + folly::IOBuf convertToIOBufForWrite(WriteHandle handle) { + return convertToIOBufT(handle); + } // TODO: When Read/Write Handles are ready, change this to allow // const-only access to data manged by iobuf and offer a @@ -737,6 +885,11 @@ class CacheAllocator : public CacheBase { // kSavedOnlyDRAM and kSavedOnlyNvmCache - partial content saved ShutDownStatus shutDown(); + // No-op for workers that are already running. Typically user uses this in + // conjunction with `config.delayWorkerStart()` to avoid initialization + // ordering issues with user callback for cachelib's workers. + void startCacheWorkers(); + // Functions that stop existing ones (if any) and create new workers // start pool rebalancer @@ -909,6 +1062,11 @@ class CacheAllocator : public CacheBase { return accessContainer_->getStats(); } + // Get the total number of keys inserted into the access container + uint64_t getAccessContainerNumKeys() const { + return accessContainer_->getNumKeys(); + } + // returns the reaper stats ReaperStats getReaperStats() const { auto stats = reaper_ ? reaper_->getStats() : ReaperStats{}; @@ -975,9 +1133,9 @@ class CacheAllocator : public CacheBase { // Inspects the cache without changing its state. // // @param key for the cache item - // @return std::pair the first represents the state + // @return std::pair the first represents the state // in the RAM and the second is a copy of the state in NVM - std::pair inspectCache(Key key); + std::pair inspectCache(Key key); // blocks until the inflight operations are flushed to nvmcache. Used for // benchmarking when we want to load up the cache first with some data and @@ -1026,6 +1184,10 @@ class CacheAllocator : public CacheBase { } } + // Mark the item as dirty and enqueue for deletion from nvmcache + // @param item item to invalidate. + void invalidateNvm(Item& item); + // Attempts to clean up left-over shared memory from preivous instance of // cachelib cache for the cache directory. If there are other processes // using the same directory, we don't touch it. If the directory is not @@ -1035,7 +1197,9 @@ class CacheAllocator : public CacheBase { // returns true if there was no error in trying to cleanup the segment // because another process was attached. False if the user tried to clean up // and the cache was actually attached. - static bool cleanupStrayShmSegments(const std::string& cacheDir, bool posix); + static bool cleanupStrayShmSegments( + const std::string& cacheDir, bool posix + /*TODO: const std::vector& config = {} */); // gives a relative offset to a pointer within the cache. uint64_t getItemPtrAsOffset(const void* ptr); @@ -1116,18 +1280,18 @@ class CacheAllocator : public CacheBase { // acquires an handle on the item. returns an empty handle if it is null. // @param it pointer to an item - // @return ItemHandle return a handle to this item + // @return WriteHandle return a handle to this item // @throw std::overflow_error is the maximum item refcount is execeeded by // creating this item handle. - ItemHandle acquire(Item* it); + WriteHandle acquire(Item* it); // creates an item handle with wait context. - ItemHandle createNvmCacheFillHandle() { return ItemHandle{*this}; } + WriteHandle createNvmCacheFillHandle() { return WriteHandle{*this}; } // acquires the wait context for the handle. This is used by NvmCache to // maintain a list of waiters - std::shared_ptr> getWaitContext( - ItemHandle& hdl) const { + std::shared_ptr> getWaitContext( + ReadHandle& hdl) const { return hdl.getItemWaitContext(); } @@ -1177,11 +1341,11 @@ class CacheAllocator : public CacheBase { // @throw std::invalid_argument if the poolId is invalid or the size // requested is invalid or if the key is invalid(key.size() == 0 or // key.size() > 255) - ItemHandle allocateInternal(PoolId id, - Key key, - uint32_t size, - uint32_t creationTime, - uint32_t expiryTime); + WriteHandle allocateInternal(PoolId id, + Key key, + uint32_t size, + uint32_t creationTime, + uint32_t expiryTime); // Allocate a chained item // @@ -1197,8 +1361,8 @@ class CacheAllocator : public CacheBase { // @return handle to the chained allocation // @throw std::invalid_argument if the size requested is invalid or // if the item is invalid - ItemHandle allocateChainedItemInternal(const ItemHandle& parent, - uint32_t size); + WriteHandle allocateChainedItemInternal(const ReadHandle& parent, + uint32_t size); // Given an item and its parentKey, validate that the parentKey // corresponds to an item that's the parent of the supplied item. @@ -1207,19 +1371,19 @@ class CacheAllocator : public CacheBase { // @param parentKey key of the item's parent // // @return handle to the parent item if the validations pass - // otherwise, an empty ItemHandle is returned. + // otherwise, an empty Handle is returned. // - ItemHandle validateAndGetParentHandleForChainedMoveLocked( + ReadHandle validateAndGetParentHandleForChainedMoveLocked( const ChainedItem& item, const Key& parentKey); // Given an existing item, allocate a new one for the // existing one to later be moved into. // - // @param oldItem handle to item we want to allocate a new item for + // @param oldItem the item we want to allocate a new item for // // @return handle to the newly allocated item // - ItemHandle allocateNewItemForOldItem(const Item& oldItem); + WriteHandle allocateNewItemForOldItem(const Item& oldItem); // internal helper that grabs a refcounted handle to the item. This does // not record the access to reflect in the mmContainer. @@ -1230,7 +1394,7 @@ class CacheAllocator : public CacheBase { // // @throw std::overflow_error is the maximum item refcount is execeeded by // creating this item handle. - ItemHandle findInternal(Key key) { + WriteHandle findInternal(Key key) { // Note: this method can not be const because we need a non-const // reference to create the ItemReleaser. return accessContainer_->find(key); @@ -1240,12 +1404,33 @@ class CacheAllocator : public CacheBase { // lookup. // // @param key the key for lookup - // @param mode the mode of access for the lookup. defaults to - // AccessMode::kRead + // @param mode the mode of access for the lookup. + // AccessMode::kRead or AccessMode::kWrite + // + // @return the handle for the item or a handle to nullptr if the key does + // not exist. + FOLLY_ALWAYS_INLINE WriteHandle findFastInternal(Key key, AccessMode mode); + + // look up an item by its key across the nvm cache as well if enabled. + // + // @param key the key for lookup + // @param mode the mode of access for the lookup. + // AccessMode::kRead or AccessMode::kWrite + // + // @return the handle for the item or a handle to nullptr if the key does + // not exist. + FOLLY_ALWAYS_INLINE WriteHandle findImpl(Key key, AccessMode mode); + + // look up an item by its key. This ignores the nvm cache and only does RAM + // lookup. + // + // @param key the key for lookup + // @param mode the mode of access for the lookup. + // AccessMode::kRead or AccessMode::kWrite // // @return the handle for the item or a handle to nullptr if the key does // not exist. - FOLLY_ALWAYS_INLINE ItemHandle findFastImpl(Key key, AccessMode mode); + FOLLY_ALWAYS_INLINE WriteHandle findFastImpl(Key key, AccessMode mode); // Moves a regular item to a different slab. This should only be used during // slab release after the item's moving bit has been set. The user supplied @@ -1257,7 +1442,18 @@ class CacheAllocator : public CacheBase { // // @return true If the move was completed, and the containers were updated // successfully. - bool moveRegularItem(Item& oldItem, ItemHandle& newItemHdl); + bool moveRegularItem(Item& oldItem, WriteHandle& newItemHdl); + + // template class for viewAsChainedAllocs that takes either ReadHandle or + // WriteHandle + template + CacheChainedAllocs viewAsChainedAllocsT( + const Handle& parent); + + // template class for convertToIOBuf that takes either ReadHandle or + // WriteHandle + template + folly::IOBuf convertToIOBufT(Handle& handle); // Moves a chained item to a different slab. This should only be used during // slab release after the item's moving bit has been set. The user supplied @@ -1273,7 +1469,7 @@ class CacheAllocator : public CacheBase { // // @return true If the move was completed, and the containers were updated // successfully. - bool moveChainedItem(ChainedItem& oldItem, ItemHandle& newItemHdl); + bool moveChainedItem(ChainedItem& oldItem, WriteHandle& newItemHdl); // Transfers the chain ownership from parent to newParent. Parent // will be unmarked as having chained allocations. Parent will not be null @@ -1290,7 +1486,7 @@ class CacheAllocator : public CacheBase { // @param newParent the new parent for the chain // // @throw if any of the conditions for parent or newParent are not met. - void transferChainLocked(ItemHandle& parent, ItemHandle& newParent); + void transferChainLocked(WriteHandle& parent, WriteHandle& newParent); // replace a chained item in the existing chain. This needs to be called // with the chained item lock held exclusive @@ -1300,13 +1496,9 @@ class CacheAllocator : public CacheBase { // @param parent the parent for the chain // // @return handle to the oldItem - ItemHandle replaceChainedItemLocked(Item& oldItem, - ItemHandle newItemHdl, - const Item& parent); - - // Mark the item as dirty and enqueue for deletion from nvmcache - // @param hdl item to invalidate. - void invalidateNvm(Item& item); + WriteHandle replaceChainedItemLocked(Item& oldItem, + WriteHandle newItemHdl, + const Item& parent); // Insert an item into MM container. The caller must hold a valid handle for // the item. @@ -1361,24 +1553,26 @@ class CacheAllocator : public CacheBase { // @param event AllocatorApiEvent that corresponds to the current operation. // supported events are INSERT, corresponding to the client // insert call, and INSERT_FROM_NVM, cooresponding to the insert - // call that happens when an item is promoted from NVM storate + // call that happens when an item is promoted from NVM storage // to memory. // // @return true if the handle was successfully inserted into the hashtable // and is now accessible to everyone. False if there was an error. // // @throw std::invalid_argument if the handle is already accessible or invalid - bool insertImpl(const ItemHandle& handle, AllocatorApiEvent event); + bool insertImpl(const WriteHandle& handle, AllocatorApiEvent event); // Removes an item from the access container and MM container. // + // @param hk the hashed key for the item // @param it Item to remove // @param tombstone A tombstone for nvm::remove job created by // nvm::createDeleteTombStone, can be empty if nvm is // not enable, or removeFromNvm is false // @param removeFromNvm if true clear key from nvm // @param recordApiEvent should we record API event for this operation. - RemoveRes removeImpl(Item& it, + RemoveRes removeImpl(HashedKey hk, + Item& it, DeleteTombStoneGuard tombstone, bool removeFromNvm = true, bool recordApiEvent = true); @@ -1400,8 +1594,8 @@ class CacheAllocator : public CacheBase { // // @return valid handle to regular item on success. This will be the last // handle to the item. On failure an empty handle. - ItemHandle advanceIteratorAndTryEvictRegularItem(MMContainer& mmContainer, - EvictionIterator& itr); + WriteHandle advanceIteratorAndTryEvictRegularItem(MMContainer& mmContainer, + EvictionIterator& itr); // Advance the current iterator and try to evict a chained item // Iterator may also be reset during the course of this function @@ -1410,7 +1604,7 @@ class CacheAllocator : public CacheBase { // // @return valid handle to the parent item on success. This will be the last // handle to the item - ItemHandle advanceIteratorAndTryEvictChainedItem(EvictionIterator& itr); + WriteHandle advanceIteratorAndTryEvictChainedItem(EvictionIterator& itr); // Deserializer CacheAllocatorMetadata and verify the version // @@ -1521,7 +1715,7 @@ class CacheAllocator : public CacheBase { // // @return true if the item has been moved // false if we have exhausted moving attempts - bool tryMovingForSlabRelease(Item& item, ItemHandle& newItemHdl); + bool tryMovingForSlabRelease(Item& item, WriteHandle& newItemHdl); // Evict an item from access and mm containers and // ensure it is safe for freeing. @@ -1537,19 +1731,19 @@ class CacheAllocator : public CacheBase { // // @return last handle for corresponding to item on success. empty handle on // failure. caller can retry if needed. - ItemHandle evictNormalItemForSlabRelease(Item& item); + WriteHandle evictNormalItemForSlabRelease(Item& item); // Helper function to evict a child item for slab release // As a side effect, the parent item is also evicted // // @return last handle to the parent item of the child on success. empty // handle on failure. caller can retry. - ItemHandle evictChainedItemForSlabRelease(ChainedItem& item); + WriteHandle evictChainedItemForSlabRelease(ChainedItem& item); // Helper function to remove a item if expired. // // @return true if it item expire and removed successfully. - bool removeIfExpired(const ItemHandle& handle); + bool removeIfExpired(const ReadHandle& handle); // exposed for the Reaper to iterate through the memory and find items to // reap under the super charged mode. This is faster if there are lots of @@ -1563,7 +1757,8 @@ class CacheAllocator : public CacheBase { // primitives. So we consciously exempt ourselves here from TSAN data race // detection. folly::annotate_ignore_thread_sanitizer_guard g(__FILE__, __LINE__); - allocator_->forEachAllocation(std::forward(f)); + auto slabsSkipped = allocator_->forEachAllocation(std::forward(f)); + stats().numSkippedSlabReleases.add(slabsSkipped); } // returns true if nvmcache is enabled and we should write this item to @@ -1681,16 +1876,17 @@ class CacheAllocator : public CacheBase { // @param item Record the item has been accessed in its mmContainer // @param mode the mode of access // @param stats stats object to avoid a thread local lookup. - void recordAccessInMMContainer(Item& item, AccessMode mode); + // @return true if successfully recorded in MMContainer + bool recordAccessInMMContainer(Item& item, AccessMode mode); - ItemHandle findChainedItem(const Item& parent) const; + WriteHandle findChainedItem(const Item& parent) const; // Get the thread local version of the Stats detail::Stats& stats() const noexcept { return stats_; } void initStats(); - // return an iterator to the item's chained allocations. The order of + // return a read-only iterator to the item's chained allocations. The order of // iteration on the item will be LIFO of the addChainedItem calls. folly::Range viewAsChainedAllocsRange( const Item& parent) const; @@ -1763,6 +1959,7 @@ class CacheAllocator : public CacheBase { std::unique_ptr chainedItemAccessContainer_{nullptr}; friend ChainedAllocs; + friend WritableChainedAllocs; // ensure any modification to a chain of chained items are synchronized using ChainedItemLock = facebook::cachelib::SharedMutexBuckets; ChainedItemLock chainedItemLocks_; @@ -1794,7 +1991,13 @@ class CacheAllocator : public CacheBase { mutable std::mutex workersMutex_; // time when the ram cache was first created - const time_t cacheCreationTime_{0}; + const uint32_t cacheCreationTime_{0}; + + // time when CacheAllocator structure is created. Whenever a process restarts + // and even if cache content is persisted, this will be reset. It's similar + // to process uptime. (But alternatively if user explicitly shuts down and + // re-attach cache, this will be reset as well) + const uint32_t cacheInstanceCreationTime_{0}; // thread local accumulation of handle counts mutable util::FastStats handleCount_{}; @@ -1818,10 +2021,14 @@ class CacheAllocator : public CacheBase { // END private members // Make this friend to give access to acquire and release - friend ItemHandle; + friend ReadHandle; friend ReaperAPIWrapper; friend class CacheAPIWrapperForNvm; friend class FbInternalRuntimeUpdateWrapper; + friend class objcache2::ObjectCache; + friend class objcache2::ObjectCacheBase; + template + friend class ReadOnlyMap; // tests friend class facebook::cachelib::tests::NvmCacheTest; @@ -1836,11 +2043,26 @@ class CacheAllocator : public CacheBase { friend class facebook::cachelib::tests::NvmAdmissionPolicyTest; friend class facebook::cachelib::tests::CacheAllocatorTestWrapper; friend class facebook::cachelib::tests::PersistenceCache; + template + friend class facebook::cachelib::tests::FixedSizeArrayTest; + template + friend class facebook::cachelib::tests::MapTest; // benchmarks template friend class facebook::cachelib::cachebench::Cache; friend class facebook::cachelib::cachebench::tests::CacheTest; + friend void lookupCachelibBufferManagerOnly(); + friend void lookupCachelibMap(); + friend void benchCachelibMap(); + friend void benchCachelibRangeMap(); + + // objectCache + template + friend class facebook::cachelib::objcache::ObjectCache; + friend class GET_DECORATED_CLASS_NAME(objcache::test, + ObjectCache, + ObjectHandleInvalid); }; } // namespace cachelib } // namespace facebook diff --git a/cachelib/allocator/CacheAllocatorConfig.h b/cachelib/allocator/CacheAllocatorConfig.h index 1207036a95..2500c41e7e 100644 --- a/cachelib/allocator/CacheAllocatorConfig.h +++ b/cachelib/allocator/CacheAllocatorConfig.h @@ -44,6 +44,7 @@ class CacheAllocatorConfig { using AccessConfig = typename CacheT::AccessConfig; using ChainedItemMovingSync = typename CacheT::ChainedItemMovingSync; using RemoveCb = typename CacheT::RemoveCb; + using ItemDestructor = typename CacheT::ItemDestructor; using NvmCacheEncodeCb = typename CacheT::NvmCacheT::EncodeCB; using NvmCacheDecodeCb = typename CacheT::NvmCacheT::DecodeCB; using NvmCacheDeviceEncryptor = typename CacheT::NvmCacheT::DeviceEncryptor; @@ -81,9 +82,13 @@ class CacheAllocatorConfig { CacheAllocatorConfig& setAccessConfig(size_t numEntries); // RemoveCallback is invoked for each item that is evicted or removed - // explicitly + // explicitly from RAM CacheAllocatorConfig& setRemoveCallback(RemoveCb cb); + // ItemDestructor is invoked for each item that is evicted or removed + // explicitly from cache (both RAM and NVM) + CacheAllocatorConfig& setItemDestructor(ItemDestructor destructor); + // Config for NvmCache. If enabled, cachelib will also make use of flash. CacheAllocatorConfig& enableNvmCache(NvmCacheConfig config); @@ -287,6 +292,25 @@ class CacheAllocatorConfig { // smaller than this will always be rejected by NvmAdmissionPolicy. CacheAllocatorConfig& setNvmAdmissionMinTTL(uint64_t ttl); + // Skip promote children items in chained when parent fail to promote + CacheAllocatorConfig& setSkipPromoteChildrenWhenParentFailed(); + + // (deprecated) Disable cache eviction. + // Please do not create new callers. CacheLib will stop supporting disabled + // eviction. + [[deprecated]] CacheAllocatorConfig& deprecated_disableEviction(); + + bool isEvictionDisabled() const noexcept { return disableEviction; } + + // We will delay worker start until user explicitly calls + // CacheAllocator::startCacheWorkers() + CacheAllocatorConfig& setDelayCacheWorkersStart(); + + // skip promote children items in chained when parent fail to promote + bool isSkipPromoteChildrenWhenParentFailed() const noexcept { + return skipPromoteChildrenWhenParentFailed; + } + // @return whether compact cache is enabled bool isCompactCacheEnabled() const noexcept { return enableZeroedSlabAllocs; } @@ -448,11 +472,6 @@ class CacheAllocatorConfig { // ABOVE are the config for various cache workers // - // if turned on, cache allocator will not evict any item when the - // system is out of memory. The user must free previously allocated - // items to make more room. - bool disableEviction = false; - // the number of tries to search for an item to evict // 0 means it's infinite unsigned int evictionSearchTries{50}; @@ -486,9 +505,14 @@ class CacheAllocatorConfig { // for all normal items AccessConfig accessConfig{}; - // user defined callback invoked when an item is being evicted or freed + // user defined callback invoked when an item is being evicted or freed from + // RAM RemoveCb removeCb{}; + // user defined item destructor invoked when an item is being + // evicted or freed from cache (both RAM and NVM) + ItemDestructor itemDestructor{}; + // user defined call back to move the item. This is executed while holding // the user provided movingSync. For items without chained allocations, // there is no specific need for explicit movingSync and user can skip @@ -541,6 +565,13 @@ class CacheAllocatorConfig { // cache. uint64_t nvmAdmissionMinTTL{0}; + // Skip promote children items in chained when parent fail to promote + bool skipPromoteChildrenWhenParentFailed{false}; + + // If true, we will delay worker start until user explicitly calls + // CacheAllocator::startCacheWorkers() + bool delayCacheWorkersStart{false}; + friend CacheT; private: @@ -551,6 +582,11 @@ class CacheAllocatorConfig { std::string stringifyAddr(const void* addr) const; std::string stringifyRebalanceStrategy( const std::shared_ptr& strategy) const; + + // if turned on, cache allocator will not evict any item when the + // system is out of memory. The user must free previously allocated + // items to make more room. + bool disableEviction = false; }; template @@ -613,6 +649,13 @@ CacheAllocatorConfig& CacheAllocatorConfig::setRemoveCallback( return *this; } +template +CacheAllocatorConfig& CacheAllocatorConfig::setItemDestructor( + ItemDestructor destructor) { + itemDestructor = std::move(destructor); + return *this; +} + template CacheAllocatorConfig& CacheAllocatorConfig::enableRejectFirstAPForNvm( uint64_t numEntries, @@ -916,6 +959,25 @@ CacheAllocatorConfig& CacheAllocatorConfig::setNvmAdmissionMinTTL( return *this; } +template +CacheAllocatorConfig& +CacheAllocatorConfig::setSkipPromoteChildrenWhenParentFailed() { + skipPromoteChildrenWhenParentFailed = true; + return *this; +} + +template +CacheAllocatorConfig& CacheAllocatorConfig::deprecated_disableEviction() { + disableEviction = true; + return *this; +} + +template +CacheAllocatorConfig& CacheAllocatorConfig::setDelayCacheWorkersStart() { + delayCacheWorkersStart = true; + return *this; +} + template const CacheAllocatorConfig& CacheAllocatorConfig::validate() const { // we can track tail hits only if MMType is MM2Q @@ -924,11 +986,7 @@ const CacheAllocatorConfig& CacheAllocatorConfig::validate() const { "Tail hits tracking cannot be enabled on MMTypes except MM2Q."); } - // The first part determines max number of "slots" we can address using - // CompressedPtr; - // The second part specifies the minimal allocation size for each slot. - // Multiplied, they inform us the maximal addressable space for cache. - size_t maxCacheSize = (1ul << CompressedPtr::kNumBits) * Slab::kMinAllocSize; + size_t maxCacheSize = CompressedPtr::getMaxAddressableSize(); // Configured cache size should not exceed the maximal addressable space for // cache. if (size > maxCacheSize) { @@ -937,6 +995,12 @@ const CacheAllocatorConfig& CacheAllocatorConfig::validate() const { size, maxCacheSize)); } + + // we don't allow user to enable both RemoveCB and ItemDestructor + if (removeCb && itemDestructor) { + throw std::invalid_argument( + "It's not allowed to enable both RemoveCB and ItemDestructor."); + } return *this; } diff --git a/cachelib/allocator/CacheChainedItemIterator.h b/cachelib/allocator/CacheChainedItemIterator.h index 741e7511ef..2d55d5eebd 100644 --- a/cachelib/allocator/CacheChainedItemIterator.h +++ b/cachelib/allocator/CacheChainedItemIterator.h @@ -16,6 +16,8 @@ #pragma once +#include + #include #include "cachelib/common/Iterators.h" @@ -26,21 +28,28 @@ namespace cachelib { namespace tests { template class BaseAllocatorTest; -} +} // namespace tests // Class to iterate through chained items in the special case that the caller // has the item but no itemhandle (e.g. during release) -template +template class CacheChainedItemIterator - : public detail::IteratorFacade, - typename Cache::Item, + : public detail::IteratorFacade, + ItemT, std::forward_iterator_tag> { public: - using Item = typename Cache::Item; - + using Item = ItemT; CacheChainedItemIterator() = default; - Item& dereference() const { return *curr_; } + Item& dereference() const { + if (curr_) { + return *curr_; + } + if (curIOBuf_) { + return *reinterpret_cast(curIOBuf_->writableData()); + } + throw std::runtime_error("no item to dereference"); + } // advance the iterator. // Do nothing if uninitizliaed. @@ -48,10 +57,16 @@ class CacheChainedItemIterator if (curr_) { curr_ = curr_->asChainedItem().getNext(*compressor_); } + if (curIOBuf_) { + curIOBuf_ = curIOBuf_->next(); + } } - bool equal(const CacheChainedItemIterator& other) const { - return curr_ == other.curr_; + bool equal(const CacheChainedItemIterator& other) const { + if (curr_ || other.curr_) { + return curr_ == other.curr_; + } + return curIOBuf_ == other.curIOBuf_; } private: @@ -68,14 +83,34 @@ class CacheChainedItemIterator } } + // only NvmCacheT can create with this constructor + // this is used to construct chained item for ItemDestructor + // with DipperItem on Navy, Item is allocated at heap (as IOBuf) + // instead of in allocator memory pool. + explicit CacheChainedItemIterator(folly::IOBuf* iobuf) : curIOBuf_(iobuf) { + // If @item is not nullptr, check that it is a chained item or parent item + // sine IOBuf chains is a circle, so we need to let the parent be the end + // iterator + if (curIOBuf_ && !dereference().isChainedItem() && + !dereference().hasChainedItem()) { + throw std::invalid_argument( + "Cannot initialize ChainedAllocIterator, Item is not a ChainedItem"); + } + } + // Current iterator position in chain Item* curr_{nullptr}; + // Removed/evicted from NVM + folly::IOBuf* curIOBuf_{nullptr}; + // Pointer compressor to traverse the chain. const PtrCompressor* compressor_{nullptr}; friend Cache; + friend typename Cache::NvmCacheT; friend typename Cache::ChainedAllocs; + friend typename Cache::WritableChainedAllocs; // For testing template diff --git a/cachelib/allocator/CacheItem-inl.h b/cachelib/allocator/CacheItem-inl.h index db6e1cea7d..a1c2456af5 100644 --- a/cachelib/allocator/CacheItem-inl.h +++ b/cachelib/allocator/CacheItem-inl.h @@ -55,18 +55,17 @@ const typename CacheItem::Key CacheItem::getKey() } template -void* CacheItem::getMemory() const noexcept { - if (isChainedItem()) { - return asChainedItem().getMemory(); - } else { - return alloc_.getMemory(); - } +const void* CacheItem::getMemory() const noexcept { + return getMemoryInternal(); +} + +template +void* CacheItem::getMemory() noexcept { + return getMemoryInternal(); } template -void* CacheItem::getWritableMemory() const { - // TODO : check AccessMode, throw exception if not writable - // TODO : add nvm invalidation logic +void* CacheItem::getMemoryInternal() const noexcept { if (isChainedItem()) { return asChainedItem().getMemory(); } else { diff --git a/cachelib/allocator/CacheItem.h b/cachelib/allocator/CacheItem.h index dd8d9e0581..87c8b8a19e 100644 --- a/cachelib/allocator/CacheItem.h +++ b/cachelib/allocator/CacheItem.h @@ -65,7 +65,7 @@ class ChainedItemPayload; template class NvmCache; -template +template class CacheChainedAllocs; template @@ -129,7 +129,9 @@ class CACHELIB_PACKED_ATTR CacheItem { * the item is freed when it is not linked to access/mm containers * and its refcount drops to 0. */ - using Handle = detail::HandleImpl; + using ReadHandle = detail::ReadHandleImpl; + using WriteHandle = detail::WriteHandleImpl; + using Handle = WriteHandle; using HandleMaker = std::function; /** @@ -162,27 +164,23 @@ class CACHELIB_PACKED_ATTR CacheItem { const Key getKey() const noexcept; // Readonly memory for this allocation. - // TODO: switch the return type to 'const void*' once all the callsites - // are modified to use getMemory() and getWritableMemory() correctly - void* getMemory() const noexcept; + const void* getMemory() const noexcept; // Writable memory for this allocation. The caller is free to do whatever he - // wants with it and needs to ensure thread sage for access into this + // wants with it and needs to ensure thread safety for access into this // piece of memory. - void* getWritableMemory() const; + void* getMemory() noexcept; // Cast item's readonly memory to a readonly user type - // TODO: switch the return type to 'const T*' once all the callsites - // are modified to use getMemory() and getWritableMemory() correctly template - T* getMemoryAs() const noexcept { - return reinterpret_cast(getMemory()); + const T* getMemoryAs() const noexcept { + return reinterpret_cast(getMemory()); } // Cast item's writable memory to a writable user type template - T* getWritableMemoryAs() noexcept { - return reinterpret_cast(getWritableMemory()); + T* getMemoryAs() noexcept { + return reinterpret_cast(getMemory()); } // This is the size of the memory allocation requested by the user. @@ -242,20 +240,27 @@ class CACHELIB_PACKED_ATTR CacheItem { /** * Function to set the timestamp for when to expire an item - * Employs a best-effort approach to update the expiryTime. Item's expiry - * time can only be updated when the item is a regular item and is part of - * the cache and not in the moving state. + * + * This API will only succeed when an item is a regular item, and user + * has already inserted it into the cache (via @insert or @insertOrReplace). + * In addition, the item cannot be in a "moving" state. * * @param expiryTime the expiryTime value to update to * * @return boolean indicating whether expiry time was successfully updated + * false when item is not linked in cache, or in moving state, or a + * chained item */ bool updateExpiryTime(uint32_t expiryTimeSecs) noexcept; // Same as @updateExpiryTime, but sets expiry time to @ttl seconds from now. + // It has the same restrictions as @updateExpiryTime. An item must be a + // regular item and is part of the cache and NOT in the moving state. // // @param ttl TTL (from now) - // @return Boolean indicating whether expiry time was successfully updated. + // @return boolean indicating whether expiry time was successfully updated + // false when item is not linked in cache, or in moving state, or a + // chained item bool extendTTL(std::chrono::seconds ttl) noexcept; // Return the refcount of an item @@ -284,6 +289,8 @@ class CACHELIB_PACKED_ATTR CacheItem { // size does not match with the current key void changeKey(Key key); + void* getMemoryInternal() const noexcept; + /** * CacheItem's refcount contain admin references, access referneces, and * flags, refer to Refcount.h for details. @@ -418,8 +425,10 @@ class CACHELIB_PACKED_ATTR CacheItem { friend AccessContainer; friend MMContainer; friend NvmCacheT; - friend CacheChainedAllocs>; - friend CacheChainedItemIterator>; + template + friend class CacheChainedAllocs; + template + friend class CacheChainedItemIterator; friend class facebook::cachelib::tests::CacheAllocatorTestWrapper; template friend class Map; @@ -460,8 +469,8 @@ class CACHELIB_PACKED_ATTR CacheItem { // | a | | y | | // | t | | l | | // | i | | o | | -// | o | | d | | -// | n | | | | +// | o | | a | | +// | n | | d | | // | --------------------- | template class CACHELIB_PACKED_ATTR CacheChainedItem : public CacheItem { @@ -538,8 +547,11 @@ class CACHELIB_PACKED_ATTR CacheChainedItem : public CacheItem { friend Payload; friend CacheAllocator; - friend CacheChainedAllocs>; - friend CacheChainedItemIterator>; + template + friend class CacheChainedAllocs; + template + friend class CacheChainedItemIterator; + friend NvmCache>; template friend class facebook::cachelib::tests::BaseAllocatorTest; FRIEND_TEST(ItemTest, ChainedItemConstruction); diff --git a/cachelib/allocator/CacheStats.cpp b/cachelib/allocator/CacheStats.cpp index 1d49f25865..5b4c855f02 100644 --- a/cachelib/allocator/CacheStats.cpp +++ b/cachelib/allocator/CacheStats.cpp @@ -48,18 +48,24 @@ template struct SizeVerify {}; void Stats::populateGlobalCacheStats(GlobalCacheStats& ret) const { - SizeVerify a = SizeVerify<15600>{}; +#ifndef SKIP_SIZE_VERIFY + SizeVerify a = SizeVerify<16160>{}; std::ignore = a; +#endif ret.numCacheGets = numCacheGets.get(); ret.numCacheGetMiss = numCacheGetMiss.get(); ret.numCacheGetExpiries = numCacheGetExpiries.get(); ret.numCacheRemoves = numCacheRemoves.get(); ret.numCacheRemoveRamHits = numCacheRemoveRamHits.get(); + ret.numRamDestructorCalls = numRamDestructorCalls.get(); + ret.numDestructorExceptions = numDestructorExceptions.get(); ret.numNvmGets = numNvmGets.get(); ret.numNvmGetMiss = numNvmGetMiss.get(); ret.numNvmGetMissFast = numNvmGetMissFast.get(); ret.numNvmGetMissExpired = numNvmGetMissExpired.get(); + ret.numNvmGetMissDueToInflightRemove = numNvmGetMissDueToInflightRemove.get(); + ret.numNvmGetMissErrs = numNvmGetMissErrs.get(); ret.numNvmGetCoalesced = numNvmGetCoalesced.get(); ret.numNvmPuts = numNvmPuts.get(); ret.numNvmDeletes = numNvmDeletes.get(); @@ -71,6 +77,8 @@ void Stats::populateGlobalCacheStats(GlobalCacheStats& ret) const { ret.numNvmAbortedPutOnInflightGet = numNvmAbortedPutOnInflightGet.get(); ret.numNvmCleanEvict = numNvmCleanEvict.get(); ret.numNvmCleanDoubleEvict = numNvmCleanDoubleEvict.get(); + ret.numNvmDestructorCalls = numNvmDestructorCalls.get(); + ret.numNvmDestructorRefcountOverflow = numNvmDestructorRefcountOverflow.get(); ret.numNvmExpiredEvict = numNvmExpiredEvict.get(); ret.numNvmPutFromClean = numNvmPutFromClean.get(); ret.numNvmEvictions = numNvmEvictions.get(); @@ -85,6 +93,8 @@ void Stats::populateGlobalCacheStats(GlobalCacheStats& ret) const { ret.numChainedParentItems = numChainedParentItems.get(); ret.numChainedChildItems = numChainedChildItems.get(); ret.numNvmAllocAttempts = numNvmAllocAttempts.get(); + ret.numNvmAllocForItemDestructor = numNvmAllocForItemDestructor.get(); + ret.numNvmItemDestructorAllocErrors = numNvmItemDestructorAllocErrors.get(); ret.allocateLatencyNs = this->allocateLatency_.estimate(); ret.moveChainedLatencyNs = this->moveChainedLatency_.estimate(); @@ -124,6 +134,7 @@ void Stats::populateGlobalCacheStats(GlobalCacheStats& ret) const { ret.numEvictionFailureFromMoving = evictFailMove.get(); ret.numEvictionFailureFromParentMoving = evictFailParentMove.get(); ret.numAbortedSlabReleases = numAbortedSlabReleases.get(); + ret.numSkippedSlabReleases = numSkippedSlabReleases.get(); } } // namespace detail @@ -178,9 +189,6 @@ PoolStats& PoolStats::operator+=(const PoolStats& other) { d.oldestTimeSec = s.oldestTimeSec; } - d.numLockByInserts += s.numLockByInserts; - d.numLockByRecordAccesses += s.numLockByRecordAccesses; - d.numLockByRemoves += s.numLockByRemoves; d.numHotAccesses += s.numHotAccesses; d.numColdAccesses += s.numColdAccesses; d.numWarmAccesses += s.numWarmAccesses; diff --git a/cachelib/allocator/CacheStats.h b/cachelib/allocator/CacheStats.h index 52d4e3ce43..7d458d7a6a 100644 --- a/cachelib/allocator/CacheStats.h +++ b/cachelib/allocator/CacheStats.h @@ -86,15 +86,6 @@ struct MMContainerStat { // the container. uint64_t oldestTimeSec; - // number of lock hits by inserts into the LRU - uint64_t numLockByInserts; - - // number of lock hits by recordAccess - uint64_t numLockByRecordAccesses; - - // number of lock hits by removes - uint64_t numLockByRemoves; - // refresh time for LRU uint64_t lruRefreshTime; @@ -331,12 +322,21 @@ struct GlobalCacheStats { // number of remove calls that resulted in a ram hit uint64_t numCacheRemoveRamHits{0}; + // number of item destructor calls from ram + uint64_t numRamDestructorCalls{0}; + // number of nvm gets uint64_t numNvmGets{0}; // number of nvm misses uint64_t numNvmGetMiss{0}; + // number of nvm isses due to internal errors + uint64_t numNvmGetMissErrs{0}; + + // number of nvm misses due to inflight remove on the same key + uint64_t numNvmGetMissDueToInflightRemove{0}; + // number of nvm misses that happened synchronously uint64_t numNvmGetMissFast{0}; @@ -386,12 +386,27 @@ struct GlobalCacheStats { // number of evictions that were already expired uint64_t numNvmExpiredEvict{0}; + // number of item destructor calls from nvm + uint64_t numNvmDestructorCalls{0}; + + // number of RefcountOverflow happens causing item destructor + // being skipped in nvm + uint64_t numNvmDestructorRefcountOverflow{0}; + // number of puts to nvm of a clean item in RAM due to nvm eviction. uint64_t numNvmPutFromClean{0}; // attempts made from nvm cache to allocate an item for promotion uint64_t numNvmAllocAttempts{0}; + // attempts made from nvm cache to allocate an item for its destructor + uint64_t numNvmAllocForItemDestructor{0}; + // heap allocate errors for item destrutor + uint64_t numNvmItemDestructorAllocErrors{0}; + + // size of itemRemoved_ hash set in nvm + uint64_t numNvmItemRemovedSetSize{0}; + // number of attempts to allocate an item uint64_t allocAttempts{0}; @@ -410,6 +425,9 @@ struct GlobalCacheStats { // number of refcount overflows uint64_t numRefcountOverflow{0}; + // number of exception occurred inside item destructor + uint64_t numDestructorExceptions{0}; + // number of allocated and CHAINED items that are parents (i.e., // consisting of at least one chained child) uint64_t numChainedChildItems{0}; @@ -440,12 +458,26 @@ struct GlobalCacheStats { util::PercentileStats::Estimates nvmEvictionSecondsToExpiry{}; util::PercentileStats::Estimates nvmPutSize{}; + // time when CacheAllocator structure is created. Whenever a process restarts + // and even if cache content is persisted, this will be reset. It's similar + // to process uptime. (But alternatively if user explicitly shuts down and + // re-attach cache, this will be reset as well) + uint64_t cacheInstanceUpTime{0}; + // time since the ram cache was created in seconds uint64_t ramUpTime{0}; // time since the nvm cache was created in seconds uint64_t nvmUpTime{0}; + // If true, it means ram cache is brand new, or it was not restored from a + // previous cache instance + bool isNewRamCache{false}; + + // If true, it means nvm cache is brand new, or it was not restored from a + // previous cache instance + bool isNewNvmCache{false}; + // if nvmcache is currently active and serving gets bool nvmCacheEnabled; @@ -463,6 +495,9 @@ struct GlobalCacheStats { // Number of times slab release was aborted due to shutdown uint64_t numAbortedSlabReleases{0}; + // Number of times slab was skipped when reaper runs + uint64_t numSkippedSlabReleases{0}; + // current active handles outstanding. This stat should // not go to negative. If it's negative, it means we have // leaked handles (or some sort of accounting bug internally) diff --git a/cachelib/allocator/CacheStatsInternal.h b/cachelib/allocator/CacheStatsInternal.h index d56c854e49..0d910c4918 100644 --- a/cachelib/allocator/CacheStatsInternal.h +++ b/cachelib/allocator/CacheStatsInternal.h @@ -51,6 +51,9 @@ struct Stats { // number of remove calls that resulted in a ram hit TLCounter numCacheRemoveRamHits{0}; + // number of item destructor calls from ram + TLCounter numRamDestructorCalls{0}; + // number of nvm gets TLCounter numNvmGets{0}; @@ -60,6 +63,12 @@ struct Stats { // number of nvm misses TLCounter numNvmGetMiss{0}; + // number of nvm isses due to internal errors + TLCounter numNvmGetMissErrs{0}; + + // number of nvm misses due to inflight remove on the same key + TLCounter numNvmGetMissDueToInflightRemove{0}; + // number of nvm gets that are expired TLCounter numNvmGetMissExpired{0}; @@ -106,6 +115,13 @@ struct Stats { // number of evictions that were already expired AtomicCounter numNvmExpiredEvict{0}; + // number of item destructor calls from nvm + AtomicCounter numNvmDestructorCalls{0}; + + // number of RefcountOverflow happens causing item destructor + // being skipped in nvm + AtomicCounter numNvmDestructorRefcountOverflow{0}; + // number of entries that were clean in RAM, but evicted and rewritten to // nvmcache because the nvmcache version was evicted AtomicCounter numNvmPutFromClean{0}; @@ -122,6 +138,14 @@ struct Stats { // attempts made from nvm cache to allocate an item for promotion TLCounter numNvmAllocAttempts{0}; + // attempts made from nvm cache to allocate an item for its destructor + TLCounter numNvmAllocForItemDestructor{0}; + // heap allocate errors for item destrutor + TLCounter numNvmItemDestructorAllocErrors{0}; + + // the number of allocated items that are permanent + TLCounter numPermanentItems{0}; + // the number of allocated and CHAINED items that are parents (i.e., // consisting of at least one chained child) TLCounter numChainedParentItems{0}; @@ -140,6 +164,9 @@ struct Stats { // being thrown AtomicCounter numRefcountOverflow{0}; + // number of exception occurred inside item destructor + AtomicCounter numDestructorExceptions{0}; + // The number of slabs being released right now. // This must be zero when `saveState()` is called. AtomicCounter numActiveSlabReleases{0}; @@ -148,6 +175,7 @@ struct Stats { AtomicCounter numReleasedForResize{0}; AtomicCounter numReleasedForAdvise{0}; AtomicCounter numAbortedSlabReleases{0}; + AtomicCounter numSkippedSlabReleases{0}; // allocations with invalid parameters AtomicCounter invalidAllocs{0}; diff --git a/cachelib/allocator/CacheVersion.h b/cachelib/allocator/CacheVersion.h index 033504e693..0189301d44 100644 --- a/cachelib/allocator/CacheVersion.h +++ b/cachelib/allocator/CacheVersion.h @@ -28,7 +28,7 @@ namespace cachelib { // then you only need to bump this version. // I.e. you're rolling out a new feature that is cache compatible with previous // Cachelib instances. -constexpr uint64_t kCachelibVersion = 15; +constexpr uint64_t kCachelibVersion = 16; // Updating this version will cause RAM cache to be dropped for all // cachelib users!!! Proceed with care!! You must coordinate with diff --git a/cachelib/allocator/ChainedAllocs.h b/cachelib/allocator/ChainedAllocs.h index f0f062260a..fdf3ae1dcd 100644 --- a/cachelib/allocator/ChainedAllocs.h +++ b/cachelib/allocator/ChainedAllocs.h @@ -23,17 +23,16 @@ namespace cachelib { // index. The chain is traversed in the LIFO order. The caller needs to ensure // that there are no concurrent addChainedItem or popChainedItem while this // happens. -template +template class CacheChainedAllocs { public: using Item = typename Cache::Item; - using Iter = typename Cache::ChainedItemIter; + using ChainedItem = typename Iter::Item; CacheChainedAllocs(CacheChainedAllocs&&) = default; CacheChainedAllocs& operator=(CacheChainedAllocs&&) = default; // return the parent of the chain. - Item& getParentItem() noexcept { return *parent_; } const Item& getParentItem() const noexcept { return *parent_; } // iterate and compute the length of the chain. This is O(N) computation. // @@ -45,7 +44,7 @@ class CacheChainedAllocs { // return the nTh in the chain from the beginning. n = 0 is the first in the // chain and last inserted. - Item* getNthInChain(size_t n) { + ChainedItem* getNthInChain(size_t n) { size_t i = 0; for (auto& c : getChain()) { if (i++ == n) { @@ -64,7 +63,6 @@ class CacheChainedAllocs { using LockType = typename Cache::ChainedItemLock; using ReadLockHolder = typename LockType::ReadLockHolder; using PtrCompressor = typename Item::PtrCompressor; - using ItemHandle = typename Cache::ItemHandle; CacheChainedAllocs(const CacheChainedAllocs&) = delete; CacheChainedAllocs& operator=(const CacheChainedAllocs&) = delete; @@ -76,7 +74,7 @@ class CacheChainedAllocs { // @param head beginning of the chain of the allocations // @param c pointer compressor to traverse the chain CacheChainedAllocs(ReadLockHolder l, - ItemHandle parent, + Handle parent, Item& head, const PtrCompressor& c) : lock_(std::move(l)), @@ -97,7 +95,7 @@ class CacheChainedAllocs { // handle to the parent item. holding this ensures that remaining of the // chain is not evicted. - ItemHandle parent_; + Handle parent_; // verify this would not cause issues with the moving slab release logic. // Evicting logic is fine since it looks for the parent's refcount diff --git a/cachelib/allocator/ChainedHashTable-inl.h b/cachelib/allocator/ChainedHashTable-inl.h index 1c0453855c..edd112dcb9 100644 --- a/cachelib/allocator/ChainedHashTable-inl.h +++ b/cachelib/allocator/ChainedHashTable-inl.h @@ -241,12 +241,12 @@ ChainedHashTable::Container::Container( ht_{config_.getNumBuckets(), memStart, compressor, config_.getHasher(), false /* resetMem */}, locks_{config_.getLocksPower(), config_.getHasher()}, - numKeys_(*object.numKeys_ref()) { + numKeys_(*object.numKeys()) { if (config_.getBucketsPower() != - static_cast(*object.bucketsPower_ref())) { + static_cast(*object.bucketsPower())) { throw std::invalid_argument(folly::sformat( "Hashtable bucket power not compatible. old = {}, new = {}", - *object.bucketsPower_ref(), + *object.bucketsPower(), config.getBucketsPower())); } @@ -260,11 +260,11 @@ ChainedHashTable::Container::Container( // checking hasher magic id not equal to 0 is to ensure it'll be // a warm roll going from a cachelib without hasher magic id to // one with a magic id - if (*object.hasherMagicId_ref() != 0 && - *object.hasherMagicId_ref() != config_.getHasher()->getMagicId()) { + if (*object.hasherMagicId() != 0 && + *object.hasherMagicId() != config_.getHasher()->getMagicId()) { throw std::invalid_argument(folly::sformat( "Hash object's ID mismatch. expected = {}, actual = {}", - *object.hasherMagicId_ref(), config_.getHasher()->getMagicId())); + *object.hasherMagicId(), config_.getHasher()->getMagicId())); } } @@ -476,10 +476,10 @@ ChainedHashTable::Container::saveState() const { } serialization::ChainedHashTableObject object; - *object.bucketsPower_ref() = config_.getBucketsPower(); - *object.locksPower_ref() = config_.getLocksPower(); - *object.numKeys_ref() = numKeys_; - *object.hasherMagicId_ref() = config_.getHasher()->getMagicId(); + *object.bucketsPower() = config_.getBucketsPower(); + *object.locksPower() = config_.getLocksPower(); + *object.numKeys() = numKeys_; + *object.hasherMagicId() = config_.getHasher()->getMagicId(); return object; } diff --git a/cachelib/allocator/ChainedHashTable.h b/cachelib/allocator/ChainedHashTable.h index f58b435455..f3c87bfa7a 100644 --- a/cachelib/allocator/ChainedHashTable.h +++ b/cachelib/allocator/ChainedHashTable.h @@ -272,13 +272,19 @@ class ChainedHashTable { } // Estimate bucketsPower and LocksPower based on cache entries. - void sizeBucketsPowerAndLocksPower(size_t cacheEntries) noexcept { + void sizeBucketsPowerAndLocksPower(size_t cacheEntries) { // The percentage of used buckets vs unused buckets is measured by a load // factor. For optimal performance, the load factor should not be more // than 60%. bucketsPower_ = static_cast(ceil(log2(cacheEntries * 1.6 /* load factor */))); + if (bucketsPower_ > kMaxBucketPower) { + throw std::invalid_argument(folly::sformat( + "Invalid arguments to the config constructor cacheEntries = {}", + cacheEntries)); + } + // 1 lock per 1000 buckets. locksPower_ = std::max(1, bucketsPower_ - 10); } @@ -550,6 +556,7 @@ class ChainedHashTable { return !(*this == other); } + // TODO(jiayueb): change to return ReadHandle after fixing all the breaks const Handle& asHandle() { return curr(); } // reset the Iterator to begin of container @@ -629,7 +636,9 @@ class ChainedHashTable { Stats getStats() const noexcept { return {numKeys_, ht_.getNumBuckets()}; } // Get the total number of keys inserted into the hash table - uint64_t getNumKeys() const noexcept { return numKeys_; } + uint64_t getNumKeys() const noexcept { + return numKeys_.load(std::memory_order_relaxed); + } private: using Hashtable = Impl; diff --git a/cachelib/allocator/Handle.h b/cachelib/allocator/Handle.h index f253b963de..aa3c633fde 100644 --- a/cachelib/allocator/Handle.h +++ b/cachelib/allocator/Handle.h @@ -43,18 +43,21 @@ enum class HandleFlags : uint8_t { kWentToNvm = 1 << 2, }; +template +struct WriteHandleImpl; + // RAII class that manages cache item pointer lifetime. These handles // can only be created by a CacheAllocator and upon destruction the handle // takes care of releasing the item to the correct cache allocator instance. // Handle must be destroyed *before* the instance of the CacheAllocator // gets destroyed. template -struct HandleImpl { +struct ReadHandleImpl { using Item = T; using CacheT = typename T::CacheT; - HandleImpl() = default; - HandleImpl(std::nullptr_t) {} + ReadHandleImpl() = default; + /*implicit*/ ReadHandleImpl(std::nullptr_t) {} // reset the handle by releasing the item it holds. void reset() noexcept { @@ -87,58 +90,64 @@ struct HandleImpl { return ret; } - ~HandleImpl() noexcept { reset(); } + ~ReadHandleImpl() noexcept { reset(); } - HandleImpl(const HandleImpl&) = delete; - HandleImpl& operator=(const HandleImpl&) = delete; + ReadHandleImpl(const ReadHandleImpl&) = delete; + ReadHandleImpl& operator=(const ReadHandleImpl&) = delete; - FOLLY_ALWAYS_INLINE HandleImpl(HandleImpl&& other) noexcept + FOLLY_ALWAYS_INLINE ReadHandleImpl(ReadHandleImpl&& other) noexcept : alloc_(other.alloc_), it_(other.releaseItem()), waitContext_(std::move(other.waitContext_)), flags_(other.getFlags()) {} - FOLLY_ALWAYS_INLINE HandleImpl& operator=(HandleImpl&& other) noexcept { + FOLLY_ALWAYS_INLINE ReadHandleImpl& operator=( + ReadHandleImpl&& other) noexcept { if (this != &other) { - this->~HandleImpl(); - new (this) HandleImpl(std::move(other)); + this->~ReadHandleImpl(); + new (this) ReadHandleImpl(std::move(other)); } return *this; } // == and != operators for comparison with Item* - friend bool operator==(const HandleImpl& a, const Item* it) noexcept { + friend bool operator==(const ReadHandleImpl& a, const Item* it) noexcept { return a.get() == it; } - friend bool operator==(const Item* it, const HandleImpl& a) noexcept { + friend bool operator==(const Item* it, const ReadHandleImpl& a) noexcept { return a == it; } - friend bool operator!=(const HandleImpl& a, const Item* it) noexcept { + friend bool operator!=(const ReadHandleImpl& a, const Item* it) noexcept { return !(a == it); } - friend bool operator!=(const Item* it, const HandleImpl& a) noexcept { + friend bool operator!=(const Item* it, const ReadHandleImpl& a) noexcept { return !(a == it); } // == and != operators for comparison with nullptr - friend bool operator==(const HandleImpl& a, std::nullptr_t) noexcept { + friend bool operator==(const ReadHandleImpl& a, std::nullptr_t) noexcept { return a.get() == nullptr; } - friend bool operator==(std::nullptr_t nullp, const HandleImpl& a) noexcept { + friend bool operator==(std::nullptr_t nullp, + const ReadHandleImpl& a) noexcept { return a == nullp; } - friend bool operator!=(const HandleImpl& a, std::nullptr_t nullp) noexcept { + friend bool operator!=(const ReadHandleImpl& a, + std::nullptr_t nullp) noexcept { return !(a == nullp); } - friend bool operator!=(std::nullptr_t nullp, const HandleImpl& a) noexcept { + friend bool operator!=(std::nullptr_t nullp, + const ReadHandleImpl& a) noexcept { return !(a == nullp); } // == and != operator - friend bool operator==(const HandleImpl& a, const HandleImpl& b) noexcept { + friend bool operator==(const ReadHandleImpl& a, + const ReadHandleImpl& b) noexcept { return a.get() == b.get(); } - friend bool operator!=(const HandleImpl& a, const HandleImpl& b) noexcept { + friend bool operator!=(const ReadHandleImpl& a, + const ReadHandleImpl& b) noexcept { return !(a == b); } @@ -147,23 +156,23 @@ struct HandleImpl { return get() != nullptr; } - // accessors. Calling get on handle with isReady() == false blocks the thread - // until the handle is ready. - FOLLY_ALWAYS_INLINE const Item* operator->() const noexcept { return get(); } - FOLLY_ALWAYS_INLINE Item* operator->() noexcept { return get(); } - FOLLY_ALWAYS_INLINE const Item& operator*() const noexcept { return *get(); } - FOLLY_ALWAYS_INLINE Item& operator*() noexcept { return *get(); } + // Accessors always return a const item. + FOLLY_ALWAYS_INLINE const Item* operator->() const noexcept { + return getInternal(); + } + FOLLY_ALWAYS_INLINE const Item& operator*() const noexcept { + return *getInternal(); + } FOLLY_ALWAYS_INLINE const Item* get() const noexcept { return getInternal(); } - FOLLY_ALWAYS_INLINE Item* get() noexcept { return getInternal(); } // Convert to semi future. - folly::SemiFuture toSemiFuture() && { + folly::SemiFuture toSemiFuture() && { if (isReady()) { - return folly::makeSemiFuture(std::forward(*this)); + return folly::makeSemiFuture(std::forward(*this)); } - folly::Promise promise; + folly::Promise promise; auto semiFuture = promise.getSemiFuture(); - auto cb = onReady([p = std::move(promise)](HandleImpl handle) mutable { + auto cb = onReady([p = std::move(promise)](ReadHandleImpl handle) mutable { p.setValue(std::move(handle)); }); if (cb) { @@ -172,7 +181,7 @@ struct HandleImpl { cb(std::move(*this)); return semiFuture; } else { - return std::move(semiFuture).deferValue([](HandleImpl handle) { + return std::move(semiFuture).deferValue([](ReadHandleImpl handle) { if (handle) { // Increment one refcount on user thread since we transferred a handle // from a cachelib internal thread. @@ -183,7 +192,14 @@ struct HandleImpl { } } - using ReadyCallback = folly::Function; + WriteHandleImpl toWriteHandle() && { + XDCHECK_NE(alloc_, nullptr); + XDCHECK_NE(getInternal(), nullptr); + alloc_->invalidateNvm(*getInternal()); + return WriteHandleImpl{std::move(*this)}; + } + + using ReadyCallback = folly::Function; // Return true iff item handle is ready to use. // Empty handles are considered ready with it_ == nullptr. @@ -217,12 +233,26 @@ struct HandleImpl { // @return HandleImpl return a handle to this item // @throw std::overflow_error is the maximum item refcount is execeeded by // creating this item handle. - HandleImpl clone() { return cloneInternal(); } + ReadHandleImpl clone() const { + ReadHandleImpl hdl{}; + if (alloc_) { + hdl = alloc_->acquire(getInternal()); + } + hdl.cloneFlags(*this); + return hdl; + } - const HandleImpl clone() const { return cloneInternal(); } + bool isWriteHandle() const { return false; } + + protected: + // accessor. Calling getInternal() on handle with isReady() == false blocks + // the thread until the handle is ready. + FOLLY_ALWAYS_INLINE Item* getInternal() const noexcept { + return waitContext_ ? waitContext_->get() : it_; + } private: - struct ItemWaitContext : public WaitContext { + struct ItemWaitContext : public WaitContext { explicit ItemWaitContext(CacheT& alloc) : alloc_(alloc) {} // @return managed item pointer @@ -286,12 +316,12 @@ struct HandleImpl { // In addition, we will be bumping the handle count by 1, when SemiFuture // is evaluated (via defer callback). This is because we have cloned // an item handle to be passed to the SemiFuture. - void set(HandleImpl hdl) override { + void set(ReadHandleImpl hdl) override { XDCHECK(!isReady()); SCOPE_EXIT { hdl.release(); }; flags_ = hdl.getFlags(); - auto it = hdl.get(); + auto it = hdl.getInternal(); it_.store(it, std::memory_order_release); // Handles are fulfilled by threads different from the owners. Adjust // the refcount tracking accordingly. use the local copy to not make @@ -307,11 +337,11 @@ struct HandleImpl { // to 0 on this thread. In the user thread, they must increment by // 1. It is done automatically if the user converted their ItemHandle // to a SemiFuture via toSemiFuture(). - auto itemHandle = hdl.clone(); - if (itemHandle) { + auto readHandle = hdl.clone(); + if (readHandle) { alloc_.adjustHandleCountForThread_private(-1); } - onReadyCallback_(std::move(itemHandle)); + onReadyCallback_(std::move(readHandle)); } } baton_.post(); @@ -415,19 +445,6 @@ struct HandleImpl { return waitContext_; } - FOLLY_ALWAYS_INLINE Item* getInternal() const noexcept { - return waitContext_ ? waitContext_->get() : it_; - } - - HandleImpl cloneInternal() const { - HandleImpl hdl{}; - if (alloc_) { - hdl = alloc_->acquire(getInternal()); - } - hdl.cloneFlags(*this); - return hdl; - } - // Internal book keeping to track handles that correspond to items that are // not present in cache. This state is mutated, but does not affect the user // visible meaning of the item handle(public API). Hence this is const. @@ -452,7 +469,7 @@ struct HandleImpl { uint8_t getFlags() const { return waitContext_ ? waitContext_->getFlags() : flags_; } - void cloneFlags(const HandleImpl& other) { flags_ = other.getFlags(); } + void cloneFlags(const ReadHandleImpl& other) { flags_ = other.getFlags(); } Item* releaseItem() noexcept { return std::exchange(it_, nullptr); } @@ -463,13 +480,13 @@ struct HandleImpl { } // Handle which has the item already - FOLLY_ALWAYS_INLINE HandleImpl(Item* it, CacheT& alloc) noexcept + FOLLY_ALWAYS_INLINE ReadHandleImpl(Item* it, CacheT& alloc) noexcept : alloc_(&alloc), it_(it) {} // handle that has a wait context allocated. Used for async handles // In this case, the it_ will be filled in asynchronously and mulitple // ItemHandles can wait on the one underlying handle - explicit HandleImpl(CacheT& alloc) noexcept + explicit ReadHandleImpl(CacheT& alloc) noexcept : alloc_(&alloc), it_(nullptr), waitContext_(std::make_shared(alloc)) {} @@ -528,8 +545,80 @@ struct HandleImpl { FRIEND_TEST(ItemHandleTest, onReadyWithNoWaitContext); }; +// WriteHandleImpl is a sub class of ReadHandleImpl to function as a mutable +// handle. User is able to obtain a mutable item from a "write handle". +template +struct WriteHandleImpl : public ReadHandleImpl { + using Item = T; + using CacheT = typename T::CacheT; + using ReadHandle = ReadHandleImpl; + using ReadHandle::ReadHandle; // inherit constructors + + // TODO(jiayueb): remove this constructor after we finish R/W handle + // migration. In the end, WriteHandle should only be obtained via + // CacheAllocator APIs like findToWrite(). + explicit WriteHandleImpl(ReadHandle&& readHandle) + : ReadHandle(std::move(readHandle)) {} + + // Accessors always return a non-const item. + FOLLY_ALWAYS_INLINE Item* operator->() const noexcept { + return ReadHandle::getInternal(); + } + FOLLY_ALWAYS_INLINE Item& operator*() const noexcept { + return *ReadHandle::getInternal(); + } + FOLLY_ALWAYS_INLINE Item* get() const noexcept { + return ReadHandle::getInternal(); + } + + // Clones write handle. returns an empty handle if it is null. + // @return WriteHandleImpl return a handle to this item + // @throw std::overflow_error is the maximum item refcount is execeeded by + // creating this item handle. + WriteHandleImpl clone() const { return WriteHandleImpl{ReadHandle::clone()}; } + + bool isWriteHandle() const { return true; } + + // Friends + // Only CacheAllocator and NvmCache can create non-default constructed handles + friend CacheT; + friend typename CacheT::NvmCacheT; + + // Object-cache's c++ allocator will need to create a zero refcount handle in + // order to access CacheAllocator API. Search for this function for details. + template + friend ItemHandle2* objcacheInitializeZeroRefcountHandle(void* handleStorage, + Item2* it, + Cache2& alloc); + + // A handle is marked as nascent when it was not yet inserted into the cache. + // However, user can override it by marking an item as "not nascent" even if + // it's not inserted into the cache. Unmarking it means a not-yet-inserted + // item will still be processed by RemoveCallback if user frees it. Today, + // the only user who can do this is Cachelib's ObjectCache API to ensure the + // correct RAII behavior for an object. + template + friend void objcacheUnmarkNascent(const ItemHandle2& hdl); + + // Object-cache's c++ allocator needs to access CacheAllocator directly from + // an item handle in order to access CacheAllocator APIs. + template + friend typename ItemHandle2::CacheT& objcacheGetCache(const ItemHandle2& hdl); + + // Following methods are only used in tests where we need to access private + // methods in ItemHandle + template + friend T1 createHandleWithWaitContextForTest(T2&); + template + friend std::shared_ptr getWaitContextForTest( + T1&); + FRIEND_TEST(ItemHandleTest, WaitContext_readycb); + FRIEND_TEST(ItemHandleTest, WaitContext_ready_immediate); + FRIEND_TEST(ItemHandleTest, onReadyWithNoWaitContext); +}; + template -std::ostream& operator<<(std::ostream& os, const HandleImpl& it) { +std::ostream& operator<<(std::ostream& os, const ReadHandleImpl& it) { if (it) { os << it->toString(); } else { diff --git a/cachelib/allocator/MM2Q-inl.h b/cachelib/allocator/MM2Q-inl.h index 39fa573b61..998a80daf6 100644 --- a/cachelib/allocator/MM2Q-inl.h +++ b/cachelib/allocator/MM2Q-inl.h @@ -21,9 +21,9 @@ namespace cachelib { template T::*HookPtr> MM2Q::Container::Container(const serialization::MM2QObject& object, PtrCompressor compressor) - : lru_(*object.lrus_ref(), compressor), - tailTrackingEnabled_(*object.tailTrackingEnabled_ref()), - config_(*object.config_ref()) { + : lru_(*object.lrus(), compressor), + tailTrackingEnabled_(*object.tailTrackingEnabled()), + config_(*object.config()) { lruRefreshTime_ = config_.lruRefreshTime; nextReconfigureTime_ = config_.mmReconfigureIntervalSecs.count() == 0 ? std::numeric_limits