CRC / Copy offloading to Intel Data Streaming Accelerator

cachelib/CMakeLists.txt

@@ -44,6 +44,9 @@ set(CMAKE_POSITION_INDEPENDENT_CODE ON)
 
 option(BUILD_TESTS "If enabled, compile the tests." ON)
 
+option(BUILD_WITH_DTO
+  "If enabled, build with the DTO library for Intel DSA offload support." OFF)
+
 
 set(BIN_INSTALL_DIR bin CACHE STRING
     "The subdirectory where binaries should be installed")
@@ -312,6 +315,56 @@ endfunction()
 add_thrift_file(OBJECT_CACHE_PERSISTENCE
                 object_cache/persistence/persistent_data.thrift json)
 
+if (BUILD_WITH_DTO)
+  # DSA offload runtime (DTO). Resolved here so every cachelib library can
+  # use it: DTO::dto is linked PUBLIC into cachelib_common (see
+  # common/CMakeLists.txt), the base library of every other cachelib target,
+  # which propagates the headers, the link dependency, and the
+  # CACHELIB_BUILD_WITH_DTO compile definition to the whole tree and to
+  # consumers of the installed package.
+  #
+  # The DSA checksum offload code requires a DTO with the caller-allocated
+  # async submit/poll API and the raw-CRC32C convention, which upstream
+  # intel/DTO does not have — so when no installed DTO cmake package is
+  # found, fetch and build the pinned revision in-tree rather than linking
+  # whatever libdto happens to be on the system.
+  set(CACHELIB_DTO_GIT_REPOSITORY "https://github.com/byrnedj/DTO.git"
+      CACHE STRING "Git repository for DTO when no installed copy is found")
+  set(CACHELIB_DTO_GIT_TAG "64b01f8d06c939d8a70066e0b643cb97ffced060"
+      CACHE STRING "DTO revision to fetch when no installed copy is found")
+
+  find_package(DTO CONFIG QUIET)
+  if (DTO_FOUND)
+    message(STATUS "Using installed DTO package: ${DTO_DIR}")
+  else()
+    if (CMAKE_VERSION VERSION_LESS 3.14)
+      message(FATAL_ERROR
+        "BUILD_WITH_DTO needs an installed DTO cmake package or "
+        "CMake >= 3.14 to fetch one. Install DTO from "
+        "${CACHELIB_DTO_GIT_REPOSITORY} at ${CACHELIB_DTO_GIT_TAG}, "
+        "or upgrade CMake.")
+    endif()
+    # DTO links against libaccel-config and libnuma; check for them here so
+    # a missing system package fails at configure time with a clear message
+    # instead of at link time inside the fetched project.
+    find_library(ACCEL_CONFIG_LIBRARY accel-config)
+    find_library(NUMA_LIBRARY numa)
+    if (NOT ACCEL_CONFIG_LIBRARY OR NOT NUMA_LIBRARY)
+      message(FATAL_ERROR
+        "BUILD_WITH_DTO requires the libaccel-config and libnuma "
+        "development packages (e.g. apt install libaccel-config-dev "
+        "libnuma-dev)")
+    endif()
+    message(STATUS "DTO not installed; fetching "
+      "${CACHELIB_DTO_GIT_REPOSITORY} @ ${CACHELIB_DTO_GIT_TAG}")
+    include(FetchContent)
+    FetchContent_Declare(dto
+      GIT_REPOSITORY "${CACHELIB_DTO_GIT_REPOSITORY}"
+      GIT_TAG "${CACHELIB_DTO_GIT_TAG}")
+    FetchContent_MakeAvailable(dto)
+  endif()
+endif()
+
 add_subdirectory (common)
 add_subdirectory (shm)
 add_subdirectory (navy)

cachelib/allocator/nvmcache/NavyConfig.cpp

@@ -321,6 +321,10 @@ std::map<std::string, std::string> EnginesConfig::serialize() const {
       folly::to<std::string>(blockCache().getNumInMemBuffers());
   configMap["navyConfig::blockCacheDataChecksum"] =
       blockCache().getDataChecksum() ? "true" : "false";
+  configMap["navyConfig::blockCacheChecksumOffload"] =
+      blockCache().getChecksumOffload() ? "true" : "false";
+  configMap["navyConfig::blockCacheChecksumOffloadMinSize"] =
+      folly::to<std::string>(blockCache().getChecksumOffloadMinSize());
   configMap["navyConfig::blockCacheSegmentedFifoSegmentRatio"] =
       folly::join(",", blockCache().getSFifoSegmentRatio());
 
@@ -333,6 +337,8 @@ std::map<std::string, std::string> EnginesConfig::serialize() const {
       folly::to<std::string>(bigHash().getBucketBfSize());
   configMap["navyConfig::bigHashSmallItemMaxSize"] =
       folly::to<std::string>(bigHash().getSmallItemMaxSize());
+  configMap["navyConfig::bigHashChecksumOffload"] =
+      bigHash().getChecksumOffload() ? "true" : "false";
   return configMap;
 }
 

cachelib/allocator/nvmcache/NavyConfig.h

@@ -535,6 +535,18 @@ class BlockCacheConfig {
     return *this;
   }
 
+  // Offload data checksumming (fused with the value copy on the write path)
+  // to Intel DSA via the DTO library. Requires data checksum to be enabled
+  // and CacheLib built with BUILD_WITH_DTO. @minSize is the minimum value
+  // size to use the offloaded path; smaller values are checksummed in
+  // software to avoid accelerator submission overhead.
+  BlockCacheConfig& setChecksumOffload(bool checksumOffload,
+                                       uint32_t minSize = 4096) noexcept {
+    checksumOffload_ = checksumOffload;
+    checksumOffloadMinSize_ = minSize;
+    return *this;
+  }
+
   BlockCacheConfig& setPreciseRemove(bool preciseRemove) noexcept {
     preciseRemove_ = preciseRemove;
     return *this;
@@ -616,6 +628,10 @@ class BlockCacheConfig {
 
   bool getDataChecksum() const { return dataChecksum_; }
 
+  bool getChecksumOffload() const { return checksumOffload_; }
+
+  uint32_t getChecksumOffloadMinSize() const { return checksumOffloadMinSize_; }
+
   uint64_t getSize() const { return size_; }
 
   bool isRegionManagerFlushAsync() const { return regionManagerFlushAsync_; }
@@ -659,6 +675,10 @@ class BlockCacheConfig {
   uint32_t regionSize_{16 * 1024 * 1024};
   // Whether enabling data checksum for Navy BlockCache.
   bool dataChecksum_{true};
+  // Whether to offload data checksumming to Intel DSA (fused with the value
+  // copy on the write path), and the minimum value size to do so.
+  bool checksumOffload_{false};
+  uint32_t checksumOffloadMinSize_{4096};
   // Whether to remove an item by checking the key (true) or only the hash value
   // (false).
   bool preciseRemove_{false};
@@ -738,6 +758,14 @@ class BigHashConfig {
     return *this;
   }
 
+  // Offload bucket checksumming to Intel DSA via the DTO library. Most
+  // beneficial with large buckets (16KB+). Requires CacheLib built with
+  // BUILD_WITH_DTO.
+  BigHashConfig& setChecksumOffload(bool checksumOffload) noexcept {
+    checksumOffload_ = checksumOffload;
+    return *this;
+  }
+
   bool isBloomFilterEnabled() const { return bucketBfSize_ > 0; }
 
   unsigned int getSizePct() const { return sizePct_; }
@@ -750,6 +778,8 @@ class BigHashConfig {
 
   uint8_t getNumMutexesPower() const { return numMutexesPower_; }
 
+  bool getChecksumOffload() const { return checksumOffload_; }
+
  private:
   // Percentage of how much of the device out of all is given to BigHash
   // engine in Navy, e.g. 50.
@@ -765,6 +795,8 @@ class BigHashConfig {
   uint64_t smallItemMaxSize_{};
   // numMutexes = 1 << numMutexesPower_.
   uint8_t numMutexesPower_{14};
+  // Whether to offload bucket checksumming to Intel DSA.
+  bool checksumOffload_{false};
 };
 
 // Config for a pair of small,large engines.

cachelib/allocator/nvmcache/NavySetup.cpp

@@ -108,6 +108,8 @@ uint64_t setupBigHash(const navy::BigHashConfig& bigHashConfig,
   // Set number of mutexes from config
   bigHash->setNumMutexesPower(bigHashConfig.getNumMutexesPower());
 
+  bigHash->setChecksumOffload(bigHashConfig.getChecksumOffload());
+
   proto.setBigHash(std::move(bigHash), bigHashConfig.getSmallItemMaxSize());
 
   if (bigHashCacheOffset <= bigHashStartOffsetLimit) {
@@ -164,6 +166,8 @@ uint64_t setupBlockCache(const navy::BlockCacheConfig& blockCacheConfig,
   auto blockCache = cachelib::navy::createBlockCacheProto();
   blockCache->setLayout(blockCacheOffset, blockCacheSize, regionSize);
   blockCache->setChecksum(blockCacheConfig.getDataChecksum());
+  blockCache->setChecksumOffload(blockCacheConfig.getChecksumOffload(),
+                                 blockCacheConfig.getChecksumOffloadMinSize());
 
   // set eviction policy
   auto segmentRatio = blockCacheConfig.getSFifoSegmentRatio();

cachelib/cachebench/cache/Cache.h

@@ -26,7 +26,9 @@
 #include <sys/stat.h>
 #include <sys/types.h>
 
+#include <algorithm>
 #include <atomic>
+#include <cstring>
 #include <iostream>
 
 #include "cachelib/allocator/CacheAllocator.h"
@@ -678,6 +680,8 @@ Cache<Allocator>::Cache(const CacheConfig& config,
     // configure BlockCache
     auto& bcConfig = nvmConfig.navyConfig.blockCache()
                          .setDataChecksum(config_.navyDataChecksum)
+                         .setChecksumOffload(config_.navyChecksumOffload,
+                                             config_.navyChecksumOffloadMinSize)
                          .setCleanRegions(config_.navyCleanRegions,
                                           config_.navyCleanRegionThreads)
                          .setRegionSize(config_.navyRegionSizeMB * MB);
@@ -716,7 +720,8 @@ Cache<Allocator>::Cache(const CacheConfig& config,
           .setSizePctAndMaxItemSize(config_.navyBigHashSizePct,
                                     config_.navySmallItemMaxSize)
           .setBucketSize(config_.navyBigHashBucketSize)
-          .setBucketBfSize(config_.navyBloomFilterPerBucketSize);
+          .setBucketBfSize(config_.navyBloomFilterPerBucketSize)
+          .setChecksumOffload(config_.navyBigHashChecksumOffload);
     }
 
     nvmConfig.navyConfig.setMaxParcelMemoryMB(config_.navyParcelMemoryMB);
@@ -1408,11 +1413,18 @@ void Cache<Allocator>::setStringItem(WriteHandle& handle,
   }
 
   auto ptr = reinterpret_cast<char*>(getMemory(handle));
-  std::strncpy(ptr, str.c_str(), dataSize);
+  // memcpy/memset instead of strncpy: strncpy scans for the terminator byte
+  // by byte and is not interposed by DTO, while memcpy/memset of large
+  // values can be offloaded to DSA. Like strncpy, write exactly dataSize
+  // bytes: the string (truncated if needed), then a zero-filled tail.
+  const size_t copyLen = std::min<size_t>(str.size() + 1, dataSize);
+  std::memcpy(ptr, str.c_str(), copyLen);
 
   // Make sure the copied string ends with null char
   if (str.size() + 1 > dataSize) {
     ptr[dataSize - 1] = '\0';
+  } else if (copyLen < dataSize) {
+    std::memset(ptr + copyLen, 0, dataSize - copyLen);
   }
 }
 

cachelib/cachebench/cache/components/FlashComponent.cpp

@@ -137,6 +137,8 @@ std::unique_ptr<CacheComponent> createFlashCacheComponent(
   // BlockCacheConfig::setCleanRegions() in NavyConfig.cpp
   bcConfig.numInMemBuffers = 2 * config.navyCleanRegions;
   bcConfig.checksum = config.navyDataChecksum;
+  bcConfig.checksumOffload = config.navyChecksumOffload;
+  bcConfig.checksumOffloadMinSize = config.navyChecksumOffloadMinSize;
   // Note: LRU is not yet supported
   if (config.navySegmentedFifoSegmentRatio.empty() ||
       config.navySegmentedFifoSegmentRatio.size() == 1) {

cachelib/cachebench/util/CacheConfig.cpp

@@ -84,6 +84,9 @@ CacheConfig::CacheConfig(const folly::dynamic& configJson) {
   JSONSetVal(configJson, navyAdmissionWriteRateMB);
   JSONSetVal(configJson, navyMaxConcurrentInserts);
   JSONSetVal(configJson, navyDataChecksum);
+  JSONSetVal(configJson, navyChecksumOffload);
+  JSONSetVal(configJson, navyChecksumOffloadMinSize);
+  JSONSetVal(configJson, navyBigHashChecksumOffload);
   JSONSetVal(configJson, truncateItemToOriginalAllocSizeInNvm);
   JSONSetVal(configJson, navyEncryption);
   JSONSetVal(configJson, deviceMaxWriteSize);
@@ -121,7 +124,7 @@ CacheConfig::CacheConfig(const folly::dynamic& configJson) {
   // if you added new fields to the configuration, update the JSONSetVal
   // to make them available for the json configs and increment the size
   // below
-  checkCorrectSize<CacheConfig, 864>();
+  checkCorrectSize<CacheConfig, 872>();
 
   if (numPools != poolSizes.size()) {
     throw std::invalid_argument(fmt::format(

cachelib/cachebench/util/CacheConfig.h

@@ -233,6 +233,17 @@ struct CacheConfig : public JSONConfig {
   // default
   bool navyDataChecksum{true};
 
+  // offloads navy BlockCache data checksumming (fused with the value copy on
+  // the write path) to Intel DSA via the DTO library, for values of at least
+  // navyChecksumOffloadMinSize bytes. Requires navyDataChecksum and a build
+  // with BUILD_WITH_DTO.
+  bool navyChecksumOffload{false};
+  uint32_t navyChecksumOffloadMinSize{4096};
+
+  // offloads navy BigHash bucket checksumming to Intel DSA. Most beneficial
+  // with large buckets (navyBigHashBucketSize of 16KB+).
+  bool navyBigHashChecksumOffload{false};
+
   // by default, only store the size requested by the user into nvm cache
   bool truncateItemToOriginalAllocSizeInNvm = false;
 

cachelib/cmake/cachelib-config.cmake.in

@@ -35,6 +35,13 @@ find_dependency(fizz )
 find_dependency(wangle)
 find_dependency(FBThrift)
 
+# When cachelib was built with DSA checksum offload, cachelib_navy links the
+# DTO runtime; resolve its imported target for consumers. A DTO fetched and
+# built in-tree installs its package into this same prefix.
+if (@BUILD_WITH_DTO@)
+  find_dependency(DTO CONFIG)
+endif()
+
 if (NOT TARGET cachelib)
 	include("${CACHELIB_CMAKE_DIR}/cachelib-targets.cmake")
 endif()

cachelib/common/CMakeLists.txt

@@ -51,6 +51,15 @@ target_link_libraries(cachelib_common PUBLIC
   ${XXHASH_LIBRARY}
 )
 
+if (BUILD_WITH_DTO)
+  # PUBLIC so the DTO headers/link dependency and the compile definition
+  # propagate to every cachelib library and to installed-package consumers.
+  # DTO itself is resolved (installed package or pinned fetch) in the
+  # top-level CMakeLists.txt.
+  target_link_libraries(cachelib_common PUBLIC DTO::dto)
+  target_compile_definitions(cachelib_common PUBLIC CACHELIB_BUILD_WITH_DTO)
+endif()
+
 install(TARGETS cachelib_common
         EXPORT cachelib-exports
         DESTINATION ${LIB_INSTALL_DIR} )

cachelib/navy/CMakeLists.txt

@@ -32,6 +32,7 @@ add_library (cachelib_navy
   block_cache/RegionManager.cpp
   block_cache/SparseMapIndex.cpp
   common/Buffer.cpp
+  common/ChecksumOffload.cpp
   common/Device.cpp
   common/FdpNvme.cpp
   common/Hash.cpp
@@ -65,6 +66,10 @@ target_link_libraries(cachelib_navy PUBLIC
   GTest::gmock
   )
 
+# DTO (DSA offload runtime) is resolved in the top-level CMakeLists.txt and
+# linked PUBLIC into cachelib_common, so cachelib_navy inherits DTO::dto and
+# the CACHELIB_BUILD_WITH_DTO definition transitively.
+
 install(TARGETS cachelib_navy
         EXPORT cachelib-exports
         DESTINATION ${LIB_INSTALL_DIR} )
@@ -96,6 +101,7 @@ if (BUILD_TESTS)
   endfunction()
 
   add_source_test (common/tests/BufferTest.cpp)
+  add_source_test (common/tests/ChecksumOffloadTest.cpp)
   add_source_test (common/tests/HashTest.cpp)
   add_source_test (common/tests/UtilsTest.cpp)
   add_source_test (bighash/tests/BucketStorageTest.cpp)