[Upstreaming] Added memory mapped file support

cachelib/allocator/CacheAllocator-inl.h

@@ -66,7 +66,9 @@ CacheAllocator<CacheTrait>::CacheAllocator(SharedMemNewT, Config config)
                           AccessContainer::getRequiredSize(
                               config_.accessConfig.getNumBuckets()),
                           nullptr,
-                          ShmSegmentOpts(config_.accessConfig.getPageSize()))
+                          ShmSegmentOpts(config_.accessConfig.getPageSize(),
+                                         false,
+                                         config_.usePosixShm))
               .addr,
           compressor_,
           [this](Item* it) -> ItemHandle { return acquire(it); })),
@@ -77,7 +79,9 @@ CacheAllocator<CacheTrait>::CacheAllocator(SharedMemNewT, Config config)
                           AccessContainer::getRequiredSize(
                               config_.chainedItemAccessConfig.getNumBuckets()),
                           nullptr,
-                          ShmSegmentOpts(config_.accessConfig.getPageSize()))
+                          ShmSegmentOpts(config_.accessConfig.getPageSize(),
+                                         false,
+                                         config_.usePosixShm))
               .addr,
           compressor_,
           [this](Item* it) -> ItemHandle { return acquire(it); })),
@@ -87,7 +91,8 @@ CacheAllocator<CacheTrait>::CacheAllocator(SharedMemNewT, Config config)
       nvmCacheState_{config_.cacheDir, config_.isNvmCacheEncryptionEnabled(),
                      config_.isNvmCacheTruncateAllocSizeEnabled()} {
   initCommon(false);
-  shmManager_->removeShm(detail::kShmInfoName);
+  shmManager_->removeShm(detail::kShmInfoName,
+                         PosixSysVSegmentOpts(config_.usePosixShm));
 }
 
 template <typename CacheTrait>
@@ -105,13 +110,19 @@ CacheAllocator<CacheTrait>::CacheAllocator(SharedMemAttachT, Config config)
       accessContainer_(std::make_unique<AccessContainer>(
           deserializer_->deserialize<AccessSerializationType>(),
           config_.accessConfig,
-          shmManager_->attachShm(detail::kShmHashTableName),
+          shmManager_->attachShm(
+              detail::kShmHashTableName,
+              nullptr,
+              ShmSegmentOpts(PageSizeT::NORMAL, false, config_.usePosixShm)),
           compressor_,
           [this](Item* it) -> ItemHandle { return acquire(it); })),
       chainedItemAccessContainer_(std::make_unique<AccessContainer>(
           deserializer_->deserialize<AccessSerializationType>(),
           config_.chainedItemAccessConfig,
-          shmManager_->attachShm(detail::kShmChainedItemHashTableName),
+          shmManager_->attachShm(
+              detail::kShmChainedItemHashTableName,
+              nullptr,
+              ShmSegmentOpts(PageSizeT::NORMAL, false, config_.usePosixShm)),
           compressor_,
           [this](Item* it) -> ItemHandle { return acquire(it); })),
       chainedItemLocks_(config_.chainedItemsLockPower,
@@ -128,7 +139,8 @@ CacheAllocator<CacheTrait>::CacheAllocator(SharedMemAttachT, Config config)
   // We will create a new info shm segment on shutDown(). If we don't remove
   // this info shm segment here and the new info shm segment's size is larger
   // than this one, creating new one will fail.
-  shmManager_->removeShm(detail::kShmInfoName);
+  shmManager_->removeShm(detail::kShmInfoName,
+                         PosixSysVSegmentOpts(config_.usePosixShm));
 }
 
 template <typename CacheTrait>
@@ -146,6 +158,7 @@ std::unique_ptr<MemoryAllocator>
 CacheAllocator<CacheTrait>::createNewMemoryAllocator() {
   ShmSegmentOpts opts;
   opts.alignment = sizeof(Slab);
+  opts.typeOpts = PosixSysVSegmentOpts(config_.usePosixShm);
   return std::make_unique<MemoryAllocator>(
       getAllocatorConfig(config_),
       shmManager_
@@ -160,6 +173,7 @@ std::unique_ptr<MemoryAllocator>
 CacheAllocator<CacheTrait>::restoreMemoryAllocator() {
   ShmSegmentOpts opts;
   opts.alignment = sizeof(Slab);
+  opts.typeOpts = PosixSysVSegmentOpts(config_.usePosixShm);
   return std::make_unique<MemoryAllocator>(
       deserializer_->deserialize<MemoryAllocator::SerializationType>(),
       shmManager_
@@ -267,7 +281,9 @@ void CacheAllocator<CacheTrait>::initWorkers() {
 
 template <typename CacheTrait>
 std::unique_ptr<Deserializer> CacheAllocator<CacheTrait>::createDeserializer() {
-  auto infoAddr = shmManager_->attachShm(detail::kShmInfoName);
+  auto infoAddr = shmManager_->attachShm(
+      detail::kShmInfoName, nullptr,
+      ShmSegmentOpts(PageSizeT::NORMAL, false, config_.usePosixShm));
   return std::make_unique<Deserializer>(
       reinterpret_cast<uint8_t*>(infoAddr.addr),
       reinterpret_cast<uint8_t*>(infoAddr.addr) + infoAddr.size);
@@ -3129,8 +3145,13 @@ void CacheAllocator<CacheTrait>::saveRamCache() {
   std::unique_ptr<folly::IOBuf> ioBuf = serializedBuf.move();
   ioBuf->coalesce();
 
+  ShmSegmentOpts opts;
+  opts.typeOpts = PosixSysVSegmentOpts(config_.usePosixShm);
+
   void* infoAddr =
-      shmManager_->createShm(detail::kShmInfoName, ioBuf->length()).addr;
+      shmManager_
+          ->createShm(detail::kShmInfoName, ioBuf->length(), nullptr, opts)
+          .addr;
   Serializer serializer(reinterpret_cast<uint8_t*>(infoAddr),
                         reinterpret_cast<uint8_t*>(infoAddr) + ioBuf->length());
   serializer.writeToBuffer(std::move(ioBuf));
@@ -3480,8 +3501,8 @@ bool CacheAllocator<CacheTrait>::stopReaper(std::chrono::seconds timeout) {
 }
 
 template <typename CacheTrait>
-bool CacheAllocator<CacheTrait>::cleanupStrayShmSegments(
-    const std::string& cacheDir, bool posix) {
+bool CacheAllocator<
+    CacheTrait>::cleanupStrayShmSegments(const std::string& cacheDir, bool posix /*TODO(SHM_FILE): const std::vector<CacheMemoryTierConfig>& config */) {
   if (util::getStatIfExists(cacheDir, nullptr) && util::isDir(cacheDir)) {
     try {
       // cache dir exists. clean up only if there are no other processes
@@ -3500,6 +3521,13 @@ bool CacheAllocator<CacheTrait>::cleanupStrayShmSegments(
     ShmManager::removeByName(cacheDir, detail::kShmHashTableName, posix);
     ShmManager::removeByName(cacheDir, detail::kShmChainedItemHashTableName,
                              posix);
+
+    // TODO(SHM_FILE): try to nuke segments of differente types (which require
+    // extra info)
+    // for (auto &tier : config) {
+    //   ShmManager::removeByName(cacheDir, tierShmName,
+    //   config_.memoryTiers[i].opts);
+    // }
   }
   return true;
 }

cachelib/allocator/CacheAllocator.h

@@ -1191,7 +1191,9 @@ class CacheAllocator : public CacheBase {
   // returns true if there was no error in trying to cleanup the segment
   // because another process was attached. False if the user tried to clean up
   // and the cache was actually attached.
-  static bool cleanupStrayShmSegments(const std::string& cacheDir, bool posix);
+  static bool cleanupStrayShmSegments(
+      const std::string& cacheDir, bool posix
+      /*TODO: const std::vector<CacheMemoryTierConfig>& config = {} */);
 
   // gives a relative offset to a pointer within the cache.
   uint64_t getItemPtrAsOffset(const void* ptr);

cachelib/allocator/TempShmMapping.cpp

@@ -34,7 +34,8 @@ TempShmMapping::TempShmMapping(size_t size)
 TempShmMapping::~TempShmMapping() {
   try {
     if (addr_) {
-      shmManager_->removeShm(detail::kTempShmCacheName.str());
+      shmManager_->removeShm(detail::kTempShmCacheName.str(),
+                             PosixSysVSegmentOpts(false /* posix */));
     }
     if (shmManager_) {
       shmManager_.reset();
@@ -77,7 +78,8 @@ void* TempShmMapping::createShmMapping(ShmManager& shmManager,
     return shmAddr;
   } catch (...) {
     if (shmAddr) {
-      shmManager.removeShm(detail::kTempShmCacheName.str());
+      shmManager.removeShm(detail::kTempShmCacheName.str(),
+                           PosixSysVSegmentOpts(false /* posix */));
     } else {
       munmap(addr, size);
     }

cachelib/allocator/memory/tests/SlabAllocatorTest.cpp

@@ -584,7 +584,7 @@ TEST_F(SlabAllocatorTest, AdviseRelease) {
   shmName += std::to_string(::getpid());
   shmManager.createShm(shmName, allocSize, memory);
 
-  SCOPE_EXIT { shmManager.removeShm(shmName); };
+  SCOPE_EXIT { shmManager.removeShm(shmName, PosixSysVSegmentOpts(false)); };
 
   memory = util::align(Slab::kSize, size, memory, allocSize);
 
@@ -714,7 +714,7 @@ TEST_F(SlabAllocatorTest, AdviseSaveRestore) {
   ShmManager shmManager(cacheDir, false /* posix */);
   shmManager.createShm(shmName, allocSize, memory);
 
-  SCOPE_EXIT { shmManager.removeShm(shmName); };
+  SCOPE_EXIT { shmManager.removeShm(shmName, PosixSysVSegmentOpts(false)); };
 
   {
     SlabAllocator s(memory, size, config);

cachelib/shm/CMakeLists.txt

@@ -16,6 +16,7 @@ add_thrift_file(SHM shm.thrift frozen2)
 
 add_library (cachelib_shm
   ${SHM_THRIFT_FILES}
+  FileShmSegment.cpp
   PosixShmSegment.cpp
   ShmCommon.cpp
   ShmManager.cpp

cachelib/shm/FileShmSegment.cpp

@@ -0,0 +1,199 @@
+/*
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "cachelib/shm/FileShmSegment.h"
+
+#include <fcntl.h>
+#include <folly/logging/xlog.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+
+#include "cachelib/common/Utils.h"
+
+namespace facebook {
+namespace cachelib {
+
+FileShmSegment::FileShmSegment(ShmAttachT,
+                               const std::string& name,
+                               ShmSegmentOpts opts)
+    : ShmBase(std::move(opts), name), fd_(getExisting(getPath(), opts_)) {
+  XDCHECK_NE(fd_, kInvalidFD);
+  markActive();
+  createReferenceMapping();
+}
+
+FileShmSegment::FileShmSegment(ShmNewT,
+                               const std::string& name,
+                               size_t size,
+                               ShmSegmentOpts opts)
+    : ShmBase(std::move(opts), name), fd_(createNewSegment(getPath())) {
+  markActive();
+  resize(size);
+  XDCHECK(isActive());
+  XDCHECK_NE(fd_, kInvalidFD);
+  // this ensures that the segment lives while the object lives.
+  createReferenceMapping();
+}
+
+FileShmSegment::~FileShmSegment() {
+  try {
+    // delete the reference mapping so the segment can be deleted if its
+    // marked to be.
+    deleteReferenceMapping();
+  } catch (const std::system_error& e) {
+  }
+
+  // need to close the fd without throwing any exceptions. so we call close
+  // directly.
+  if (fd_ != kInvalidFD) {
+    const int ret = close(fd_);
+    if (ret != 0) {
+      XDCHECK_NE(errno, EIO);
+      XDCHECK_NE(errno, EINTR);
+      XDCHECK_EQ(errno, EBADF);
+      XDCHECK(!errno);
+    }
+  }
+}
+
+int FileShmSegment::createNewSegment(const std::string& name) {
+  constexpr static int createFlags = O_RDWR | O_CREAT | O_EXCL;
+  detail::open_func_t open_func = std::bind(open, name.c_str(), createFlags);
+  return detail::openImpl(open_func, createFlags);
+}
+
+int FileShmSegment::getExisting(const std::string& name,
+                                const ShmSegmentOpts& opts) {
+  int flags = opts.readOnly ? O_RDONLY : O_RDWR;
+  detail::open_func_t open_func = std::bind(open, name.c_str(), flags);
+  return detail::openImpl(open_func, flags);
+}
+
+void FileShmSegment::markForRemoval() {
+  if (isActive()) {
+    // we still have the fd open. so we can use it to perform ftruncate
+    // even after marking for removal through unlink. The fd does not get
+    // recycled until we actually destroy this object.
+    removeByPath(getPath());
+    markForRemove();
+  } else {
+    XDCHECK(false);
+  }
+}
+
+bool FileShmSegment::removeByPath(const std::string& path) {
+  try {
+    detail::unlink_func_t unlink_func = std::bind(unlink, path.c_str());
+    detail::unlinkImpl(unlink_func);
+    return true;
+  } catch (const std::system_error& e) {
+    // unlink is opaque unlike sys-V api where its through the shmid. Hence
+    // if someone has already unlinked it for us, we just let it pass.
+    if (e.code().value() != ENOENT) {
+      throw;
+    }
+    return false;
+  }
+}
+
+std::string FileShmSegment::getPath() const {
+  return std::get<FileShmSegmentOpts>(opts_.typeOpts).path;
+}
+
+size_t FileShmSegment::getSize() const {
+  if (isActive() || isMarkedForRemoval()) {
+    stat_t buf = {};
+    detail::fstatImpl(fd_, &buf);
+    return buf.st_size;
+  } else {
+    throw std::runtime_error(folly::sformat(
+        "Trying to get size of segment with name {} in an invalid state",
+        getName()));
+  }
+  return 0;
+}
+
+void FileShmSegment::resize(size_t size) const {
+  size = detail::getPageAlignedSize(size, opts_.pageSize);
+  XDCHECK(isActive() || isMarkedForRemoval());
+  if (isActive() || isMarkedForRemoval()) {
+    XDCHECK_NE(fd_, kInvalidFD);
+    detail::ftruncateImpl(fd_, size);
+  } else {
+    throw std::runtime_error(folly::sformat(
+        "Trying to resize segment with name {} in an invalid state",
+        getName()));
+  }
+}
+
+void* FileShmSegment::mapAddress(void* addr) const {
+  size_t size = getSize();
+  if (!detail::isPageAlignedSize(size, opts_.pageSize) ||
+      !detail::isPageAlignedAddr(addr, opts_.pageSize)) {
+    util::throwSystemError(EINVAL, "Address/size not aligned");
+  }
+
+#ifndef MAP_HUGE_2MB
+#define MAP_HUGE_2MB (21 << MAP_HUGE_SHIFT)
+#endif
+
+#ifndef MAP_HUGE_1GB
+#define MAP_HUGE_1GB (30 << MAP_HUGE_SHIFT)
+#endif
+
+  int flags = MAP_SHARED;
+  if (opts_.pageSize == PageSizeT::TWO_MB) {
+    flags |= MAP_HUGETLB | MAP_HUGE_2MB;
+  } else if (opts_.pageSize == PageSizeT::ONE_GB) {
+    flags |= MAP_HUGETLB | MAP_HUGE_1GB;
+  }
+  // If users pass in an address, they must make sure that address is unused.
+  if (addr != nullptr) {
+    flags |= MAP_FIXED;
+  }
+
+  const int prot = opts_.readOnly ? PROT_READ : PROT_WRITE | PROT_READ;
+
+  void* retAddr = detail::mmapImpl(addr, size, prot, flags, fd_, 0);
+  // if there was hint for mapping, then fail if we cannot respect this
+  // because we want to be specific about mapping to exactly that address.
+  if (retAddr != nullptr && addr != nullptr && retAddr != addr) {
+    util::throwSystemError(EINVAL, "Address already mapped");
+  }
+  XDCHECK(retAddr == addr || addr == nullptr);
+  return retAddr;
+}
+
+void FileShmSegment::unMap(void* addr) const {
+  detail::munmapImpl(addr, getSize());
+}
+
+void FileShmSegment::createReferenceMapping() {
+  // create a mapping that lasts the life of this object. mprotect it to
+  // ensure there are no actual accesses.
+  referenceMapping_ = detail::mmapImpl(
+      nullptr, detail::getPageSize(), PROT_NONE, MAP_SHARED, fd_, 0);
+  XDCHECK(referenceMapping_ != nullptr);
+}
+
+void FileShmSegment::deleteReferenceMapping() const {
+  if (referenceMapping_ != nullptr) {
+    detail::munmapImpl(referenceMapping_, detail::getPageSize());
+  }
+}
+} // namespace cachelib
+} // namespace facebook