From 3153247b9b3e3ab55dff359fedcf36679db28c44 Mon Sep 17 00:00:00 2001 From: Daniel Byrne Date: Wed, 8 Jul 2026 16:51:33 +0000 Subject: [PATCH 1/3] checksum offloading via DTO --- cachelib/CMakeLists.txt | 3 ++ cachelib/allocator/nvmcache/NavyConfig.cpp | 6 +++ cachelib/allocator/nvmcache/NavyConfig.h | 32 +++++++++++ cachelib/allocator/nvmcache/NavySetup.cpp | 4 ++ cachelib/cachebench/cache/Cache.h | 5 +- .../cache/components/FlashComponent.cpp | 2 + cachelib/cachebench/util/CacheConfig.cpp | 5 +- cachelib/cachebench/util/CacheConfig.h | 11 ++++ cachelib/navy/CMakeLists.txt | 10 ++++ cachelib/navy/Factory.cpp | 41 ++++++++++++++ cachelib/navy/Factory.h | 9 ++++ cachelib/navy/bighash/BigHash.cpp | 53 ++++++++++++------- cachelib/navy/bighash/BigHash.h | 22 +++++++- cachelib/navy/bighash/Bucket.cpp | 8 +++ cachelib/navy/bighash/Bucket.h | 7 +++ cachelib/navy/bighash/tests/BigHashTest.cpp | 24 +++++++++ cachelib/navy/block_cache/BlockCache.cpp | 49 ++++++++++++----- cachelib/navy/block_cache/BlockCache.h | 17 +++++- .../navy/block_cache/tests/BlockCacheTest.cpp | 45 ++++++++++++++++ cachelib/navy/common/Hash.cpp | 7 ++- cachelib/navy/common/Hash.h | 3 +- cachelib/navy/common/tests/HashTest.cpp | 29 ++++++++++ 22 files changed, 353 insertions(+), 39 deletions(-) diff --git a/cachelib/CMakeLists.txt b/cachelib/CMakeLists.txt index d8f78edcbd..c33e70d624 100644 --- a/cachelib/CMakeLists.txt +++ b/cachelib/CMakeLists.txt @@ -44,6 +44,9 @@ set(CMAKE_POSITION_INDEPENDENT_CODE ON) option(BUILD_TESTS "If enabled, compile the tests." ON) +option(BUILD_WITH_DTO + "If enabled, build with the DTO library for Intel DSA offload support." OFF) + set(BIN_INSTALL_DIR bin CACHE STRING "The subdirectory where binaries should be installed") diff --git a/cachelib/allocator/nvmcache/NavyConfig.cpp b/cachelib/allocator/nvmcache/NavyConfig.cpp index dcb53d6de1..d4a54994a9 100644 --- a/cachelib/allocator/nvmcache/NavyConfig.cpp +++ b/cachelib/allocator/nvmcache/NavyConfig.cpp @@ -321,6 +321,10 @@ std::map EnginesConfig::serialize() const { folly::to(blockCache().getNumInMemBuffers()); configMap["navyConfig::blockCacheDataChecksum"] = blockCache().getDataChecksum() ? "true" : "false"; + configMap["navyConfig::blockCacheChecksumOffload"] = + blockCache().getChecksumOffload() ? "true" : "false"; + configMap["navyConfig::blockCacheChecksumOffloadMinSize"] = + folly::to(blockCache().getChecksumOffloadMinSize()); configMap["navyConfig::blockCacheSegmentedFifoSegmentRatio"] = folly::join(",", blockCache().getSFifoSegmentRatio()); @@ -333,6 +337,8 @@ std::map EnginesConfig::serialize() const { folly::to(bigHash().getBucketBfSize()); configMap["navyConfig::bigHashSmallItemMaxSize"] = folly::to(bigHash().getSmallItemMaxSize()); + configMap["navyConfig::bigHashChecksumOffload"] = + bigHash().getChecksumOffload() ? "true" : "false"; return configMap; } diff --git a/cachelib/allocator/nvmcache/NavyConfig.h b/cachelib/allocator/nvmcache/NavyConfig.h index d68d85d8b6..a1be8a9c6c 100644 --- a/cachelib/allocator/nvmcache/NavyConfig.h +++ b/cachelib/allocator/nvmcache/NavyConfig.h @@ -535,6 +535,18 @@ class BlockCacheConfig { return *this; } + // Offload data checksumming (fused with the value copy on the write path) + // to Intel DSA via the DTO library. Requires data checksum to be enabled + // and CacheLib built with BUILD_WITH_DTO. @minSize is the minimum value + // size to use the offloaded path; smaller values are checksummed in + // software to avoid accelerator submission overhead. + BlockCacheConfig& setChecksumOffload(bool checksumOffload, + uint32_t minSize = 4096) noexcept { + checksumOffload_ = checksumOffload; + checksumOffloadMinSize_ = minSize; + return *this; + } + BlockCacheConfig& setPreciseRemove(bool preciseRemove) noexcept { preciseRemove_ = preciseRemove; return *this; @@ -616,6 +628,10 @@ class BlockCacheConfig { bool getDataChecksum() const { return dataChecksum_; } + bool getChecksumOffload() const { return checksumOffload_; } + + uint32_t getChecksumOffloadMinSize() const { return checksumOffloadMinSize_; } + uint64_t getSize() const { return size_; } bool isRegionManagerFlushAsync() const { return regionManagerFlushAsync_; } @@ -659,6 +675,10 @@ class BlockCacheConfig { uint32_t regionSize_{16 * 1024 * 1024}; // Whether enabling data checksum for Navy BlockCache. bool dataChecksum_{true}; + // Whether to offload data checksumming to Intel DSA (fused with the value + // copy on the write path), and the minimum value size to do so. + bool checksumOffload_{false}; + uint32_t checksumOffloadMinSize_{4096}; // Whether to remove an item by checking the key (true) or only the hash value // (false). bool preciseRemove_{false}; @@ -738,6 +758,14 @@ class BigHashConfig { return *this; } + // Offload bucket checksumming to Intel DSA via the DTO library. Most + // beneficial with large buckets (16KB+). Requires CacheLib built with + // BUILD_WITH_DTO. + BigHashConfig& setChecksumOffload(bool checksumOffload) noexcept { + checksumOffload_ = checksumOffload; + return *this; + } + bool isBloomFilterEnabled() const { return bucketBfSize_ > 0; } unsigned int getSizePct() const { return sizePct_; } @@ -750,6 +778,8 @@ class BigHashConfig { uint8_t getNumMutexesPower() const { return numMutexesPower_; } + bool getChecksumOffload() const { return checksumOffload_; } + private: // Percentage of how much of the device out of all is given to BigHash // engine in Navy, e.g. 50. @@ -765,6 +795,8 @@ class BigHashConfig { uint64_t smallItemMaxSize_{}; // numMutexes = 1 << numMutexesPower_. uint8_t numMutexesPower_{14}; + // Whether to offload bucket checksumming to Intel DSA. + bool checksumOffload_{false}; }; // Config for a pair of small,large engines. diff --git a/cachelib/allocator/nvmcache/NavySetup.cpp b/cachelib/allocator/nvmcache/NavySetup.cpp index 29937071db..b04365a53c 100644 --- a/cachelib/allocator/nvmcache/NavySetup.cpp +++ b/cachelib/allocator/nvmcache/NavySetup.cpp @@ -108,6 +108,8 @@ uint64_t setupBigHash(const navy::BigHashConfig& bigHashConfig, // Set number of mutexes from config bigHash->setNumMutexesPower(bigHashConfig.getNumMutexesPower()); + bigHash->setChecksumOffload(bigHashConfig.getChecksumOffload()); + proto.setBigHash(std::move(bigHash), bigHashConfig.getSmallItemMaxSize()); if (bigHashCacheOffset <= bigHashStartOffsetLimit) { @@ -164,6 +166,8 @@ uint64_t setupBlockCache(const navy::BlockCacheConfig& blockCacheConfig, auto blockCache = cachelib::navy::createBlockCacheProto(); blockCache->setLayout(blockCacheOffset, blockCacheSize, regionSize); blockCache->setChecksum(blockCacheConfig.getDataChecksum()); + blockCache->setChecksumOffload(blockCacheConfig.getChecksumOffload(), + blockCacheConfig.getChecksumOffloadMinSize()); // set eviction policy auto segmentRatio = blockCacheConfig.getSFifoSegmentRatio(); diff --git a/cachelib/cachebench/cache/Cache.h b/cachelib/cachebench/cache/Cache.h index 0dfdc86d66..0e7e5d96fa 100644 --- a/cachelib/cachebench/cache/Cache.h +++ b/cachelib/cachebench/cache/Cache.h @@ -678,6 +678,8 @@ Cache::Cache(const CacheConfig& config, // configure BlockCache auto& bcConfig = nvmConfig.navyConfig.blockCache() .setDataChecksum(config_.navyDataChecksum) + .setChecksumOffload(config_.navyChecksumOffload, + config_.navyChecksumOffloadMinSize) .setCleanRegions(config_.navyCleanRegions, config_.navyCleanRegionThreads) .setRegionSize(config_.navyRegionSizeMB * MB); @@ -716,7 +718,8 @@ Cache::Cache(const CacheConfig& config, .setSizePctAndMaxItemSize(config_.navyBigHashSizePct, config_.navySmallItemMaxSize) .setBucketSize(config_.navyBigHashBucketSize) - .setBucketBfSize(config_.navyBloomFilterPerBucketSize); + .setBucketBfSize(config_.navyBloomFilterPerBucketSize) + .setChecksumOffload(config_.navyBigHashChecksumOffload); } nvmConfig.navyConfig.setMaxParcelMemoryMB(config_.navyParcelMemoryMB); diff --git a/cachelib/cachebench/cache/components/FlashComponent.cpp b/cachelib/cachebench/cache/components/FlashComponent.cpp index 48c336d2c7..0c4039361b 100644 --- a/cachelib/cachebench/cache/components/FlashComponent.cpp +++ b/cachelib/cachebench/cache/components/FlashComponent.cpp @@ -137,6 +137,8 @@ std::unique_ptr createFlashCacheComponent( // BlockCacheConfig::setCleanRegions() in NavyConfig.cpp bcConfig.numInMemBuffers = 2 * config.navyCleanRegions; bcConfig.checksum = config.navyDataChecksum; + bcConfig.checksumOffload = config.navyChecksumOffload; + bcConfig.checksumOffloadMinSize = config.navyChecksumOffloadMinSize; // Note: LRU is not yet supported if (config.navySegmentedFifoSegmentRatio.empty() || config.navySegmentedFifoSegmentRatio.size() == 1) { diff --git a/cachelib/cachebench/util/CacheConfig.cpp b/cachelib/cachebench/util/CacheConfig.cpp index 6188e69528..dc41c9bb5e 100644 --- a/cachelib/cachebench/util/CacheConfig.cpp +++ b/cachelib/cachebench/util/CacheConfig.cpp @@ -84,6 +84,9 @@ CacheConfig::CacheConfig(const folly::dynamic& configJson) { JSONSetVal(configJson, navyAdmissionWriteRateMB); JSONSetVal(configJson, navyMaxConcurrentInserts); JSONSetVal(configJson, navyDataChecksum); + JSONSetVal(configJson, navyChecksumOffload); + JSONSetVal(configJson, navyChecksumOffloadMinSize); + JSONSetVal(configJson, navyBigHashChecksumOffload); JSONSetVal(configJson, truncateItemToOriginalAllocSizeInNvm); JSONSetVal(configJson, navyEncryption); JSONSetVal(configJson, deviceMaxWriteSize); @@ -121,7 +124,7 @@ CacheConfig::CacheConfig(const folly::dynamic& configJson) { // if you added new fields to the configuration, update the JSONSetVal // to make them available for the json configs and increment the size // below - checkCorrectSize(); + checkCorrectSize(); if (numPools != poolSizes.size()) { throw std::invalid_argument(fmt::format( diff --git a/cachelib/cachebench/util/CacheConfig.h b/cachelib/cachebench/util/CacheConfig.h index 1f8387c32a..092e612111 100644 --- a/cachelib/cachebench/util/CacheConfig.h +++ b/cachelib/cachebench/util/CacheConfig.h @@ -233,6 +233,17 @@ struct CacheConfig : public JSONConfig { // default bool navyDataChecksum{true}; + // offloads navy BlockCache data checksumming (fused with the value copy on + // the write path) to Intel DSA via the DTO library, for values of at least + // navyChecksumOffloadMinSize bytes. Requires navyDataChecksum and a build + // with BUILD_WITH_DTO. + bool navyChecksumOffload{false}; + uint32_t navyChecksumOffloadMinSize{4096}; + + // offloads navy BigHash bucket checksumming to Intel DSA. Most beneficial + // with large buckets (navyBigHashBucketSize of 16KB+). + bool navyBigHashChecksumOffload{false}; + // by default, only store the size requested by the user into nvm cache bool truncateItemToOriginalAllocSizeInNvm = false; diff --git a/cachelib/navy/CMakeLists.txt b/cachelib/navy/CMakeLists.txt index e4d9af3eec..031864b087 100644 --- a/cachelib/navy/CMakeLists.txt +++ b/cachelib/navy/CMakeLists.txt @@ -32,6 +32,7 @@ add_library (cachelib_navy block_cache/RegionManager.cpp block_cache/SparseMapIndex.cpp common/Buffer.cpp + common/ChecksumOffload.cpp common/Device.cpp common/FdpNvme.cpp common/Hash.cpp @@ -65,6 +66,14 @@ target_link_libraries(cachelib_navy PUBLIC GTest::gmock ) +if (BUILD_WITH_DTO) + find_path(DTO_INCLUDE_DIR NAMES dto.h REQUIRED) + find_library(DTO_LIBRARY NAMES dto REQUIRED) + target_include_directories(cachelib_navy PRIVATE ${DTO_INCLUDE_DIR}) + target_compile_definitions(cachelib_navy PRIVATE CACHELIB_BUILD_WITH_DTO) + target_link_libraries(cachelib_navy PUBLIC ${DTO_LIBRARY} accel-config) +endif() + install(TARGETS cachelib_navy EXPORT cachelib-exports DESTINATION ${LIB_INSTALL_DIR} ) @@ -96,6 +105,7 @@ if (BUILD_TESTS) endfunction() add_source_test (common/tests/BufferTest.cpp) + add_source_test (common/tests/ChecksumOffloadTest.cpp) add_source_test (common/tests/HashTest.cpp) add_source_test (common/tests/UtilsTest.cpp) add_source_test (bighash/tests/BucketStorageTest.cpp) diff --git a/cachelib/navy/Factory.cpp b/cachelib/navy/Factory.cpp index e6a2632eba..3a2ceef078 100644 --- a/cachelib/navy/Factory.cpp +++ b/cachelib/navy/Factory.cpp @@ -18,6 +18,7 @@ #include #include +#include #include @@ -27,6 +28,7 @@ #include "cachelib/navy/block_cache/BlockCache.h" #include "cachelib/navy/block_cache/FifoPolicy.h" #include "cachelib/navy/block_cache/LruPolicy.h" +#include "cachelib/navy/common/ChecksumOffload.h" #include "cachelib/navy/common/Device.h" #include "cachelib/navy/driver/Driver.h" @@ -37,6 +39,30 @@ namespace facebook::cachelib::navy { namespace { +// Returns true if checksum offload may be enabled. Logs and returns false +// (callers fall back to software checksums) when built without DTO support +// or when the runtime DSA-vs-CPU checksum parity self-check fails. The +// self-check protects against a mixed deployment where accelerator-computed +// checksums would not verify against CPU-computed ones. +bool verifyChecksumOffload(folly::StringPiece engine) { + if (!checksumOffloadSupported()) { + XLOGF(WARN, + "{}: checksum offload requested, but this binary was built without " + "DTO support. Falling back to software checksums.", + engine); + return false; + } + if (!checksumOffloadSelfCheck()) { + XLOGF(ERR, + "{}: DSA checksum offload self-check failed (accelerator checksum " + "does not match CPU checksum on this machine). Falling back to " + "software checksums.", + engine); + return false; + } + return true; +} + class BlockCacheProtoImpl final : public BlockCacheProto { public: BlockCacheProtoImpl() = default; @@ -56,6 +82,11 @@ class BlockCacheProtoImpl final : public BlockCacheProto { void setChecksum(bool enable) override { config_.checksum = enable; } + void setChecksumOffload(bool enable, uint32_t minSize) override { + config_.checksumOffload = enable; + config_.checksumOffloadMinSize = minSize; + } + void setLruEvictionPolicy() override { if (!(config_.cacheSize > 0 && config_.regionSize > 0)) { throw std::logic_error("layout is not set"); @@ -155,6 +186,9 @@ class BlockCacheProtoImpl final : public BlockCacheProto { DestructorCallback cb) && { config_.checkExpired = std::move(checkExpired); config_.destructorCb = std::move(cb); + if (config_.checksumOffload && !verifyChecksumOffload("BlockCache")) { + config_.checksumOffload = false; + } return std::make_unique(std::move(config_)); } @@ -189,6 +223,10 @@ class BigHashProtoImpl final : public BigHashProto { config_.numMutexesPower = numMutexesPower; } + void setChecksumOffload(bool enable) override { + config_.checksumOffload = enable; + } + void setDevice(Device* device) { config_.device = device; } void setDestructorCb(DestructorCallback cb) { @@ -197,6 +235,9 @@ class BigHashProtoImpl final : public BigHashProto { std::unique_ptr create(ExpiredCheck checkExpired) && { config_.checkExpired = std::move(checkExpired); + if (config_.checksumOffload && !verifyChecksumOffload("BigHash")) { + config_.checksumOffload = false; + } if (bloomFilterEnabled_) { if (config_.bucketSize == 0) { throw std::invalid_argument{"invalid bucket size"}; diff --git a/cachelib/navy/Factory.h b/cachelib/navy/Factory.h index 0d487c8b8c..2899fe9073 100644 --- a/cachelib/navy/Factory.h +++ b/cachelib/navy/Factory.h @@ -50,6 +50,11 @@ class BlockCacheProto { // Enable data checksumming (default: disabled) virtual void setChecksum(bool enable) = 0; + // (Optional) Offload data checksumming to Intel DSA (fused with the value + // copy on the write path) for values of at least @minSize bytes. Requires + // checksumming enabled and a build with DTO support. + virtual void setChecksumOffload(bool enable, uint32_t minSize) = 0; + // set*EvictionPolicy function family: sets eviction policy. Supports LRU, // LRU with deferred insert and FIFO. Must set up one of them. @@ -135,6 +140,10 @@ class BigHashProto { // (Optional) Set number of mutexes for bucket locking as power of 2. // numMutexes = 1 << numMutexesPower. Default: 14 (16K mutexes) virtual void setNumMutexesPower(uint8_t numMutexesPower) = 0; + + // (Optional) Offload bucket checksumming to Intel DSA. Requires a build + // with DTO support. + virtual void setChecksumOffload(bool enable) = 0; }; class EnginePairProto { diff --git a/cachelib/navy/bighash/BigHash.cpp b/cachelib/navy/bighash/BigHash.cpp index 8bc927d27d..504e705d04 100644 --- a/cachelib/navy/bighash/BigHash.cpp +++ b/cachelib/navy/bighash/BigHash.cpp @@ -84,6 +84,7 @@ BigHash::BigHash(Config&& config, ValidConfigTag) } }}, bucketSize_{config.bucketSize}, + checksumOffload_{config.checksumOffload}, cacheBaseOffset_{config.cacheBaseOffset}, numBuckets_{config.numBuckets()}, bloomFilter_{std::move(config.bloomFilter)}, @@ -336,16 +337,17 @@ Status BigHash::insert(HashedKey hk, bucket->insert(hk, value, checkExpired_, cb); newRemainingBytes = bucket->remainingBytes(); - // rebuild / fix the bloom filter before we move the buffer to do the - // actual write - if (removed + evicted == 0) { - // In case nothing was removed or evicted, we can just add - bfSet(bid, hk.keyHash()); - } else { - bfRebuild(bid, bucket); - } - - const auto res = writeBucket(bid, std::move(buffer)); + // The bloom filter update runs as overlap work: concurrently with the + // bucket checksum when offloaded to DSA, right before it otherwise. In + // both cases it completes before the bucket is written to the device. + const auto res = writeBucket(bid, std::move(buffer), [&] { + if (removed + evicted == 0) { + // In case nothing was removed or evicted, we can just add + bfSet(bid, hk.keyHash()); + } else { + bfRebuild(bid, bucket); + } + }); if (!res) { bfClear(bid); ioErrorCount_.inc(); @@ -478,11 +480,11 @@ Status BigHash::remove(HashedKey hk) { } newRemainingBytes = bucket->remainingBytes(); - // We compute bloom filter before writing the bucket because when encryption - // is enabled, we will "move" the bucket content into writeBucket(). - bfRebuild(bid, bucket); - - const auto res = writeBucket(bid, std::move(buffer)); + // The bloom filter rebuild runs as overlap work: concurrently with the + // bucket checksum when offloaded to DSA, right before it otherwise. In + // both cases it completes before the bucket is written to the device. + const auto res = + writeBucket(bid, std::move(buffer), [&] { bfRebuild(bid, bucket); }); if (!res) { bfClear(bid); ioErrorCount_.inc(); @@ -572,9 +574,12 @@ Buffer BigHash::readBucket(BucketId bid) { auto* bucket = reinterpret_cast(buffer.data()); - auto checksumCheck = [](auto* b, auto bufferView) { - const bool checksumSuccess = - Bucket::computeChecksum(bufferView) == b->getChecksum(); + auto checksumCheck = [this](auto* b, auto bufferView) { + const uint32_t computed = checksumOffload_ + ? Bucket::computeChecksumOffloaded( + bufferView, [] {}) + : Bucket::computeChecksum(bufferView); + const bool checksumSuccess = computed == b->getChecksum(); // TODO (T93631284) we only read a bucket if the bloom filter indicates that // the bucket could have the element. Hence, if check sum errors out and // bloom filter is enabled, we could record the checksum error. However, @@ -598,9 +603,17 @@ Buffer BigHash::readBucket(BucketId bid) { return buffer; } -bool BigHash::writeBucket(BucketId bid, Buffer buffer) { +bool BigHash::writeBucket(BucketId bid, + Buffer buffer, + folly::FunctionRef overlap) { auto* bucket = reinterpret_cast(buffer.data()); - bucket->setChecksum(Bucket::computeChecksum(buffer.view())); + if (checksumOffload_) { + bucket->setChecksum( + Bucket::computeChecksumOffloaded(buffer.view(), overlap)); + } else { + overlap(); + bucket->setChecksum(Bucket::computeChecksum(buffer.view())); + } const bool res = device_.write(getBucketOffset(bid), std::move(buffer), placementHandle_); if (!res) { diff --git a/cachelib/navy/bighash/BigHash.h b/cachelib/navy/bighash/BigHash.h index 90a0de209d..0bf58d530b 100644 --- a/cachelib/navy/bighash/BigHash.h +++ b/cachelib/navy/bighash/BigHash.h @@ -16,6 +16,7 @@ #pragma once +#include #include #include @@ -66,6 +67,12 @@ class BigHash final : public Engine, folly::NonCopyableNonMovable { struct Config { uint32_t bucketSize{4 * 1024}; + // Offload bucket checksumming to Intel DSA via the DTO library when + // available. Most beneficial with large buckets (16KB+), where the bloom + // filter rebuild overlaps with the accelerator computing the checksum. + // No effect when built without DTO support. + bool checksumOffload{false}; + // The range of device that BigHash will access is guaranteed to be // within [baseOffset, baseOffset + cacheSize) uint64_t cacheBaseOffset{}; @@ -184,7 +191,13 @@ class BigHash final : public Engine, folly::NonCopyableNonMovable { BigHash(Config&& config, ValidConfigTag); Buffer readBucket(BucketId bid); - bool writeBucket(BucketId bid, Buffer buffer); + + // Checksums the bucket and writes it to the device. @overlap is CPU work + // (e.g. bloom filter maintenance) run while the checksum is computed by + // DSA when checksum offload is enabled, or immediately before the software + // checksum otherwise. It may read the bucket but must not modify it. + bool writeBucket( + BucketId bid, Buffer buffer, folly::FunctionRef overlap = [] {}); // The corresponding r/w bucket lock must be held during the entire // duration of the read and write operations. For example, during write, @@ -220,11 +233,16 @@ class BigHash final : public Engine, folly::NonCopyableNonMovable { const size_t numMutexes_; // Serialization format version. Never 0. Versions < 10 reserved for testing. - static constexpr uint32_t kFormatVersion = 10; + // Version 11: navy::checksum switched from CRC-32 (IEEE) to CRC-32C + // (Castagnoli) for DSA offload compatibility. Buckets written by prior + // versions would fail checksum verification. + static constexpr uint32_t kFormatVersion = 11; const ExpiredCheck checkExpired_{}; const DestructorCallback destructorCb_{}; const uint64_t bucketSize_{}; + // Whether to offload bucket checksumming to DSA. See Config::checksumOffload. + const bool checksumOffload_{}; const uint64_t cacheBaseOffset_{}; const uint64_t numBuckets_{}; std::unique_ptr bloomFilter_; diff --git a/cachelib/navy/bighash/Bucket.cpp b/cachelib/navy/bighash/Bucket.cpp index 0138396bcb..1927419be6 100644 --- a/cachelib/navy/bighash/Bucket.cpp +++ b/cachelib/navy/bighash/Bucket.cpp @@ -18,6 +18,7 @@ #include +#include "cachelib/navy/common/ChecksumOffload.h" #include "cachelib/navy/common/Hash.h" namespace facebook::cachelib::navy { @@ -53,6 +54,13 @@ uint32_t Bucket::computeChecksum(BufferView view) { return navy::checksum(data); } +uint32_t Bucket::computeChecksumOffloaded(BufferView view, + folly::FunctionRef overlap) { + constexpr auto kChecksumStart = sizeof(checksum_); + auto data = view.slice(kChecksumStart, view.size() - kChecksumStart); + return checksumWithOverlap(data, overlap); +} + Bucket& Bucket::initNew(MutableBufferView view, uint64_t generationTime) { return *new (view.data()) Bucket(generationTime, view.size() - sizeof(Bucket)); diff --git a/cachelib/navy/bighash/Bucket.h b/cachelib/navy/bighash/Bucket.h index eda5ced271..21c6ca565c 100644 --- a/cachelib/navy/bighash/Bucket.h +++ b/cachelib/navy/bighash/Bucket.h @@ -16,6 +16,7 @@ #pragma once +#include #include #include "cachelib/navy/bighash/BucketStorage.h" @@ -70,6 +71,12 @@ class FOLLY_PACK_ATTR Bucket { // User will pass in a view that contains the memory that is a Bucket static uint32_t computeChecksum(BufferView view); + // Same checksum, but computed by DSA (when built with DTO support) with + // @overlap CPU work running while the accelerator operates. @overlap may + // read the bucket but must not modify it. + static uint32_t computeChecksumOffloaded(BufferView view, + folly::FunctionRef overlap); + // Initialize a brand new Bucket given a piece of memory in the case // that the existing bucket is invalid. (I.e. checksum or generation // and generation time for. diff --git a/cachelib/navy/bighash/tests/BigHashTest.cpp b/cachelib/navy/bighash/tests/BigHashTest.cpp index a2730c42ad..f61f8a8ffb 100644 --- a/cachelib/navy/bighash/tests/BigHashTest.cpp +++ b/cachelib/navy/bighash/tests/BigHashTest.cpp @@ -79,6 +79,30 @@ TEST(BigHash, InsertAndRemove) { EXPECT_EQ(Status::NotFound, bh.remove(makeHK("key"))); } +// Exercises the offloaded bucket-checksum path (DSA when built with DTO +// support, software otherwise), including the bloom-filter-rebuild overlap +// in insert/remove and offloaded verification in readBucket. +TEST(BigHash, InsertRemoveChecksumOffload) { + BigHash::Config config; + setLayout(config, 128, 2); + config.checksumOffload = true; + auto device = std::make_unique>(config.cacheSize, 128); + config.device = device.get(); + + BigHash bh(std::move(config)); + + Buffer value; + uint32_t lat = 0; + EXPECT_EQ(Status::Ok, + bh.insert(makeHK("key"), makeView("12345"), 0 /* poolId */, + 0 /* expiryTime */)); + EXPECT_EQ(Status::Ok, bh.lookup(makeHK("key"), value, lat)); + EXPECT_EQ(makeView("12345"), value.view()); + + EXPECT_EQ(Status::Ok, bh.remove(makeHK("key"))); + EXPECT_EQ(Status::NotFound, bh.lookup(makeHK("key"), value, lat)); +} + // without bloom filters, could exist always returns true. TEST(BigHash, CouldExistWithoutBF) { BigHash::Config config; diff --git a/cachelib/navy/block_cache/BlockCache.cpp b/cachelib/navy/block_cache/BlockCache.cpp index 96db988cbc..0a98bc0373 100644 --- a/cachelib/navy/block_cache/BlockCache.cpp +++ b/cachelib/navy/block_cache/BlockCache.cpp @@ -31,6 +31,7 @@ #include "cachelib/navy/block_cache/HitsReinsertionPolicy.h" #include "cachelib/navy/block_cache/PercentageReinsertionPolicy.h" #include "cachelib/navy/block_cache/SparseMapIndex.h" +#include "cachelib/navy/common/ChecksumOffload.h" #include "cachelib/navy/common/Hash.h" #include "cachelib/navy/common/Types.h" #include "folly/Range.h" @@ -103,6 +104,11 @@ BlockCache::Config& BlockCache::Config::validate() { } } + if (checksumOffload && !checksum) { + throw std::invalid_argument( + "checksum offload requires data checksum to be enabled"); + } + reinsertionConfig.validate(); indexConfig.validate(); @@ -220,6 +226,8 @@ BlockCache::BlockCache(Config&& config, ValidConfigTag) checkExpired_{std::move(config.checkExpired)}, destructorCb_{std::move(config.destructorCb)}, checksumData_{config.checksum}, + checksumOffload_{config.checksumOffload}, + checksumOffloadMinSize_{config.checksumOffloadMinSize}, device_{*config.device}, allocAlignSize_{calcAllocAlignSize()}, readBufferSize_{config.readBufferSize < kDefReadBufferSize @@ -1331,20 +1339,37 @@ void BlockCache::writeEntry(MutableBufferView buffer, // Copy descriptor and the key to the end uint8_t* dest = buffer.data() + buffer.size() - sizeof(EntryDesc); - auto desc = new (dest) EntryDesc(static_cast(keySize), - static_cast(value.size()), keyHash, - lastAccessTimeSecs); - - if (checksumData_) { - desc->cs = checksum(value); - } + EntryDesc* desc = nullptr; + + // Constructs the entry descriptor (which computes its own header checksum + // over the fields preceding csSelf) and copies the key in front of it. The + // value area at the head of the buffer is not touched, so this can overlap + // with the value copy. + auto writeDescAndKey = [&] { + desc = new (dest) EntryDesc(static_cast(keySize), + static_cast(value.size()), keyHash, + lastAccessTimeSecs); + if (!combinedEntry) { + // Key info will be inside the value itself for combined entry block + std::memcpy(dest - keySize, hk->key().data(), keySize); + logicalWriteSize = keySize + value.size(); + } + }; - if (!combinedEntry) { - // Key info will be inside the value itself for combined entry block - std::memcpy(dest - keySize, hk->key().data(), keySize); - logicalWriteSize = keySize + value.size(); + if (checksumData_ && checksumOffload_ && + value.size() >= checksumOffloadMinSize_) { + // Fused copy + checksum on DSA; the descriptor and key are written by + // the CPU while the accelerator processes the value. The value checksum + // is not covered by csSelf, so it can be filled in afterwards. + const uint32_t cs = copyAndChecksum(buffer.data(), value, writeDescAndKey); + desc->cs = cs; + } else { + writeDescAndKey(); + if (checksumData_) { + desc->cs = checksum(value); + } + std::memcpy(buffer.data(), value.data(), value.size()); } - std::memcpy(buffer.data(), value.data(), value.size()); logicalWrittenCount_.add(logicalWriteSize); } diff --git a/cachelib/navy/block_cache/BlockCache.h b/cachelib/navy/block_cache/BlockCache.h index 51124fd581..73f2165d05 100644 --- a/cachelib/navy/block_cache/BlockCache.h +++ b/cachelib/navy/block_cache/BlockCache.h @@ -56,6 +56,13 @@ class BlockCache final : public Engine { DestructorCallback destructorCb; // Checksum data read/written bool checksum{}; + // Offload value checksumming (fused with the value copy on the write + // path) to Intel DSA via the DTO library when available. Requires + // checksum to be enabled. No effect when built without DTO support. + bool checksumOffload{false}; + // Minimum value size to use the offloaded (fused) path; smaller values + // use software copy+checksum to avoid accelerator submission overhead. + uint32_t checksumOffloadMinSize{4096}; // Base offset and size (in bytes) of cache on the device uint64_t cacheBaseOffset{}; uint64_t cacheSize{}; @@ -286,7 +293,10 @@ class BlockCache final : public Engine { private: // Serialization format version. Never 0. Versions < 10 reserved for testing. - static constexpr uint32_t kFormatVersion = 13; + // Version 14: navy::checksum switched from CRC-32 (IEEE) to CRC-32C + // (Castagnoli) for DSA offload compatibility. Entries written by prior + // versions would fail checksum verification. + static constexpr uint32_t kFormatVersion = 14; // This should be at least the nextTwoPow(sizeof(EntryDesc)). static constexpr uint32_t kDefReadBufferSize = 4096; // Default priority for an item inserted into block cache @@ -535,6 +545,11 @@ class BlockCache final : public Engine { const ExpiredCheck checkExpired_; const DestructorCallback destructorCb_; const bool checksumData_{}; + + // Whether to offload value checksum+copy to DSA on the write path, and the + // minimum value size for which to do so. See Config::checksumOffload. + const bool checksumOffload_{}; + const uint32_t checksumOffloadMinSize_{}; // reference to the under-lying device. const Device& device_; // alloc alignment size indicates the granularity of entry sizes on device. diff --git a/cachelib/navy/block_cache/tests/BlockCacheTest.cpp b/cachelib/navy/block_cache/tests/BlockCacheTest.cpp index afdd2405a2..c3cb7b093c 100644 --- a/cachelib/navy/block_cache/tests/BlockCacheTest.cpp +++ b/cachelib/navy/block_cache/tests/BlockCacheTest.cpp @@ -244,6 +244,51 @@ TEST(BlockCache, InsertLookup) { EXPECT_EQ(0, hits[3]); } +// Exercises the fused copy+checksum write path (DSA-offloaded when built +// with DTO support, software otherwise). minSize of 0 forces every insert +// through the fused path; values large and small verify both roundtrip and +// checksum verification on lookup. +TEST(BlockCache, InsertLookupChecksumOffload) { + std::vector log; + + std::vector hits(4); + auto policy = std::make_unique>(&hits); + auto device = createMemoryDevice(kDeviceSize, nullptr /* encryption */); + auto ex = makeJobScheduler(); + auto config = makeConfig(std::move(policy), *device); + config.checksum = true; + config.checksumOffload = true; + config.checksumOffloadMinSize = 0; + auto engine = makeEngine(std::move(config)); + auto driver = makeDriver(std::move(engine), std::move(ex)); + + BufferGen bg; + for (size_t i = 0; i < 16; i++) { + // Mix of sizes: small and near-region-slot-size values + CacheEntry e{bg.gen(8), bg.gen(i % 2 == 0 ? 800 : 3000)}; + EXPECT_EQ(Status::Ok, + driver->insertAsync(e.key(), e.value(), nullptr, 0 /* poolId */, + 0 /* expiryTime */)); + log.push_back(std::move(e)); + } + driver->flush(); + + for (auto& e : log) { + Buffer value; + uint32_t lat = 0; + EXPECT_EQ(Status::Ok, driver->lookup(e.key(), value, lat)); + EXPECT_EQ(e.value(), value.view()); + } + + // No checksum errors must have been recorded + driver->getCounters({[](folly::StringPiece name, double count, + CounterVisitor::CounterType) { + if (name.contains("checksum_error")) { + EXPECT_EQ(0, count) << name; + } + }}); +} + TEST(BlockCache, LookupReturnsLastAccessTime) { auto hits = std::make_unique>(4); auto policy = std::make_unique>(hits.get()); diff --git a/cachelib/navy/common/Hash.cpp b/cachelib/navy/common/Hash.cpp index 50ef925e46..e18c2bf4fc 100644 --- a/cachelib/navy/common/Hash.cpp +++ b/cachelib/navy/common/Hash.cpp @@ -24,6 +24,11 @@ uint64_t hashBuffer(BufferView key, uint64_t seed) { } uint32_t checksum(BufferView data, uint32_t startingChecksum) { - return folly::crc32(data.data(), data.size(), startingChecksum); + // CRC-32C (Castagnoli) with raw seed semantics (default 0, no final + // inversion). This matches the value produced by chained SSE4.2 + // _mm_crc32_* instructions and by Intel DSA's CRC generation as used by + // the DTO library, so checksums can be offloaded to DSA and verified on + // CPU (and vice versa) interchangeably. + return folly::crc32c(data.data(), data.size(), startingChecksum); } } // namespace facebook::cachelib::navy diff --git a/cachelib/navy/common/Hash.h b/cachelib/navy/common/Hash.h index 7eed7464f8..d1c0486208 100644 --- a/cachelib/navy/common/Hash.h +++ b/cachelib/navy/common/Hash.h @@ -27,7 +27,8 @@ namespace navy { // Default hash function uint64_t hashBuffer(BufferView key, uint64_t seed = 0); -// Default checksumming function +// Default checksumming function: CRC-32C (Castagnoli) with raw seed +// semantics (no final inversion), DSA-offload compatible. uint32_t checksum(BufferView data, uint32_t startingChecksum = 0); // Convenience utils to convert a piece of buffer to a hashed key diff --git a/cachelib/navy/common/tests/HashTest.cpp b/cachelib/navy/common/tests/HashTest.cpp index d69a22bd78..7bc643b25c 100644 --- a/cachelib/navy/common/tests/HashTest.cpp +++ b/cachelib/navy/common/tests/HashTest.cpp @@ -19,6 +19,35 @@ #include "cachelib/navy/common/Hash.h" namespace facebook::cachelib::navy::tests { +// navy::checksum must compute raw CRC-32C (Castagnoli, reflected poly +// 0x82F63B78), seed 0, no final inversion. This is the value produced by +// chained SSE4.2 _mm_crc32_* instructions and by Intel DSA CRC generation +// (as used by the DTO library). If this test fails, DSA-offloaded checksums +// would not verify against CPU-computed ones (and vice versa), and on-disk +// format versions must be revisited. +TEST(Hash, ChecksumIsRawCrc32c) { + auto cs = [](folly::StringPiece s, uint32_t seed = 0) { + return checksum( + BufferView{s.size(), reinterpret_cast(s.data())}, + seed); + }; + EXPECT_EQ(0x00000000u, cs("")); + EXPECT_EQ(0x93AD1061u, cs("a")); + EXPECT_EQ(0x58E3FA20u, cs("123456789")); + EXPECT_EQ(0xB3FEE25Eu, cs("The quick brown fox jumps over the lazy dog")); + + uint8_t bytes[256]; + for (size_t i = 0; i < sizeof(bytes); i++) { + bytes[i] = static_cast(i); + } + EXPECT_EQ(0x2436A9DBu, checksum(BufferView{sizeof(bytes), bytes})); + + // Chaining with a starting checksum must equal a single-shot computation. + folly::StringPiece full{"123456789"}; + auto part1 = cs("1234"); + EXPECT_EQ(cs(full), cs("56789", part1)); +} + TEST(Hash, HashedKeyCollision) { HashedKey hk1{"key 1"}; HashedKey hk2{"key 2"}; From 695f77420c29e1d368410cc42f2a694f4ed0b993 Mon Sep 17 00:00:00 2001 From: Daniel Byrne Date: Wed, 15 Jul 2026 16:39:33 +0000 Subject: [PATCH 2/3] small fixes for CRC generation --- cachelib/navy/block_cache/BlockCache.cpp | 142 +++++++++++++---------- cachelib/navy/block_cache/BlockCache.h | 5 + cachelib/navy/common/Device.cpp | 9 ++ cachelib/navy/common/Hash.cpp | 44 +++++++ 4 files changed, 139 insertions(+), 61 deletions(-) diff --git a/cachelib/navy/block_cache/BlockCache.cpp b/cachelib/navy/block_cache/BlockCache.cpp index 0a98bc0373..46970d8c8a 100644 --- a/cachelib/navy/block_cache/BlockCache.cpp +++ b/cachelib/navy/block_cache/BlockCache.cpp @@ -281,6 +281,15 @@ std::shared_ptr BlockCache::makeReinsertionPolicy( return reinsertionConfig.getCustomPolicy(*index_); } +uint32_t BlockCache::valueChecksum(BufferView value) const { + if (checksumOffload_ && value.size() >= checksumOffloadMinSize_) { + // While DSA computes the checksum, the calling fiber yields (or the + // thread pause-polls), freeing the reader for other requests. + return checksumWithOverlap(value, [] {}); + } + return checksum(value); +} + uint32_t BlockCache::serializedSize(uint32_t keySize, uint32_t valueSize) const { uint32_t size = sizeof(EntryDesc) + keySize + valueSize; @@ -564,21 +573,24 @@ std::pair BlockCache::getRandomAlloc(Buffer& value) { } BufferView valueView{desc.valueSize, entryEnd - entrySize}; - if (checksumData_ && desc.cs != checksum(valueView)) { - XLOGF(ERR, - "Item value checksum mismatch in getRandomAlloc(). Region {} is " - "likely corrupted. Expected: {}, Actual: {}, Offset: {}, " - "Physical-offset: {}, Value-size: {}, Payload (hex): {}", - rid.index(), - desc.cs, - checksum(valueView), - addrEnd.offset() - entrySize, - regionManager_.physicalOffset(addrEnd) - entrySize, - desc.valueSize, - // call folly::unhexlify to convert it back to binary data - folly::hexlify( - folly::ByteRange(valueView.data(), valueView.dataEnd()))); - break; + if (checksumData_) { + const uint32_t valueCs = valueChecksum(valueView); + if (desc.cs != valueCs) { + XLOGF(ERR, + "Item value checksum mismatch in getRandomAlloc(). Region {} is " + "likely corrupted. Expected: {}, Actual: {}, Offset: {}, " + "Physical-offset: {}, Value-size: {}, Payload (hex): {}", + rid.index(), + desc.cs, + valueCs, + addrEnd.offset() - entrySize, + regionManager_.physicalOffset(addrEnd) - entrySize, + desc.valueSize, + // call folly::unhexlify to convert it back to binary data + folly::hexlify( + folly::ByteRange(valueView.data(), valueView.dataEnd()))); + break; + } } // confirm that the chosen NvmItem is still being mapped with the key @@ -777,7 +789,7 @@ void BlockCache::onRegionCleanup(RegionId rid, BufferView buffer) { HashedKey hk = makeHK(entryEnd - sizeof(EntryDesc) - desc.keySize, desc.keySize); BufferView value{desc.valueSize, entryEnd - entrySize}; - if (checksumData_ && desc.cs != checksum(value)) { + if (checksumData_ && desc.cs != valueChecksum(value)) { // We do not need to abort here since the EntryDesc checksum was good, so // we can safely proceed to read the next entry. cleanupValueChecksumErrorCount_.inc(); @@ -1029,20 +1041,22 @@ Status BlockCache::onReadCombinedEntryBlock(uint32_t address, } // Check the checksum for the value - if (checksumData_ && entryDesc.cs != checksum(BufferView(entryDesc.valueSize, - buffer.data()))) { - XLOGF(ERR, - "Item value checksum mismatch in onReadCombinedEntryBlock(). " - "Region {} is likely corrupted. Expected: {}, Actual: {}, Offset: " - "{}, Physical-offset: {}, " - "Value-size: {} Payload (hex): {}", - addrEnd.rid().index(), entryDesc.cs, - checksum(BufferView(entryDesc.valueSize, buffer.data())), - addrEnd.offset() - size, - regionManager_.physicalOffset(addrEnd) - size, entryDesc.valueSize, - folly::hexlify(folly::ByteRange( - buffer.data(), buffer.data() + entryDesc.valueSize))); - return Status::ChecksumError; + if (checksumData_) { + const uint32_t cebValueCs = + valueChecksum(BufferView(entryDesc.valueSize, buffer.data())); + if (entryDesc.cs != cebValueCs) { + XLOGF(ERR, + "Item value checksum mismatch in onReadCombinedEntryBlock(). " + "Region {} is likely corrupted. Expected: {}, Actual: {}, Offset: " + "{}, Physical-offset: {}, " + "Value-size: {} Payload (hex): {}", + addrEnd.rid().index(), entryDesc.cs, cebValueCs, + addrEnd.offset() - size, + regionManager_.physicalOffset(addrEnd) - size, entryDesc.valueSize, + folly::hexlify(folly::ByteRange( + buffer.data(), buffer.data() + entryDesc.valueSize))); + return Status::ChecksumError; + } } // shrink and set it as CEB's buffer @@ -1194,23 +1208,26 @@ AllocatorApiResult BlockCache::reinsertOrRemoveItem( } // Validate checksum as we want to reinsert the item - if (checksumData_ && entryDesc.cs != checksum(value)) { - // We do not need to abort here since the EntryDesc checksum was good, so - // we can safely proceed to read the next entry. - XLOGF(ERR, - "Item value checksum mismatch in reinsertOrRemoveItem(). " - "Item is likely corrupted. Item will not be reinserted. " - "We will continue evaluating remaining items in the region." - "Expected: {}, Actual: {}, Value-size: {}, Payload (hex): {}", - entryDesc.cs, - checksum(value), - entryDesc.valueSize, - folly::hexlify(folly::ByteRange(value.data(), value.dataEnd()))); - reclaimValueChecksumErrorCount_.inc(); - removeItem(false); - recordEvent(hk.key(), AllocatorApiEvent::NVM_REINSERT, - AllocatorApiResult::CORRUPTED, entrySize); - return AllocatorApiResult::CORRUPTED; + if (checksumData_) { + const uint32_t reinsertValueCs = valueChecksum(value); + if (entryDesc.cs != reinsertValueCs) { + // We do not need to abort here since the EntryDesc checksum was good, so + // we can safely proceed to read the next entry. + XLOGF(ERR, + "Item value checksum mismatch in reinsertOrRemoveItem(). " + "Item is likely corrupted. Item will not be reinserted. " + "We will continue evaluating remaining items in the region." + "Expected: {}, Actual: {}, Value-size: {}, Payload (hex): {}", + entryDesc.cs, + reinsertValueCs, + entryDesc.valueSize, + folly::hexlify(folly::ByteRange(value.data(), value.dataEnd()))); + reclaimValueChecksumErrorCount_.inc(); + removeItem(false); + recordEvent(hk.key(), AllocatorApiEvent::NVM_REINSERT, + AllocatorApiResult::CORRUPTED, entrySize); + return AllocatorApiResult::CORRUPTED; + } } // Priority of an re-inserted item is determined by its past accesses @@ -1453,20 +1470,23 @@ void BlockCache::readEntry(RegionDescriptor& regionDesc, ld.valueSize_ = desc.valueSize; ld.lastAccessTimeSecs_ = desc.lastAccessTimeSecs; auto slice = ld.buffer_.view().slice(0, desc.valueSize); - if (checksumData_ && desc.cs != checksum(slice)) { - XLOG_N_PER_MS(ERR, 10, 10'000) << fmt::format( - "Item value checksum mismatch in readEntry() looking up key {} in " - "Region {}. Expected: {}, Actual: {}, Offset: {}, Physical-offset: {}, " - "Value-size: {} Payload (hex): {}", - key, addrEnd.rid().index(), desc.cs, checksum(slice), - addrEnd.offset() - size, regionManager_.physicalOffset(addrEnd) - size, - slice.size(), - folly::hexlify( - folly::ByteRange(slice.data(), slice.data() + slice.size()))); - ld.buffer_.reset(); - lookupValueChecksumErrorCount_.inc(); - ld.status_ = Status::ChecksumError; - return; + if (checksumData_) { + const uint32_t sliceCs = valueChecksum(slice); + if (desc.cs != sliceCs) { + XLOG_N_PER_MS(ERR, 10, 10'000) << fmt::format( + "Item value checksum mismatch in readEntry() looking up key {} in " + "Region {}. Expected: {}, Actual: {}, Offset: {}, Physical-offset: " + "{}, Value-size: {} Payload (hex): {}", + key, addrEnd.rid().index(), desc.cs, sliceCs, + addrEnd.offset() - size, + regionManager_.physicalOffset(addrEnd) - size, slice.size(), + folly::hexlify( + folly::ByteRange(slice.data(), slice.data() + slice.size()))); + ld.buffer_.reset(); + lookupValueChecksumErrorCount_.inc(); + ld.status_ = Status::ChecksumError; + return; + } } ld.status_ = Status::Ok; } diff --git a/cachelib/navy/block_cache/BlockCache.h b/cachelib/navy/block_cache/BlockCache.h index 73f2165d05..65d3bb7f69 100644 --- a/cachelib/navy/block_cache/BlockCache.h +++ b/cachelib/navy/block_cache/BlockCache.h @@ -550,6 +550,11 @@ class BlockCache final : public Engine { // minimum value size for which to do so. See Config::checksumOffload. const bool checksumOffload_{}; const uint32_t checksumOffloadMinSize_{}; + + // Computes the checksum of @value for verification (lookup, reclaim and + // cleanup paths), offloading to DSA when checksum offload is enabled and + // the value meets the size gate. + uint32_t valueChecksum(BufferView value) const; // reference to the under-lying device. const Device& device_; // alloc alignment size indicates the granularity of entry sizes on device. diff --git a/cachelib/navy/common/Device.cpp b/cachelib/navy/common/Device.cpp index ee84808fb9..b79f68aa11 100644 --- a/cachelib/navy/common/Device.cpp +++ b/cachelib/navy/common/Device.cpp @@ -1116,6 +1116,15 @@ IoContext* FileDevice::getIoContext() { asyncBase = std::make_unique(qDepthPerContext_, pollMode); } + if (!asyncBase) { + // Reachable when io_uring is requested but this binary was built + // without liburing (CACHELIB_IOURING_DISABLE). Fail loudly instead of + // crashing on a null deref in AsyncIoContext. + throw std::runtime_error( + "async io requested but no AsyncBase backend is available in this " + "build (io_uring support compiled out?)"); + } + auto idx = incrementalIdx_++; tlContext_.reset(new AsyncIoContext(std::move(asyncBase), idx, evb, qDepthPerContext_, useIoUring, diff --git a/cachelib/navy/common/Hash.cpp b/cachelib/navy/common/Hash.cpp index e18c2bf4fc..67003c2a33 100644 --- a/cachelib/navy/common/Hash.cpp +++ b/cachelib/navy/common/Hash.cpp @@ -18,17 +18,61 @@ #include +#if defined(__x86_64__) +#include +#endif + namespace facebook::cachelib::navy { uint64_t hashBuffer(BufferView key, uint64_t seed) { return folly::hash::SpookyHashV2::Hash64(key.data(), key.size(), seed); } +namespace { +#if defined(__x86_64__) +// Hardware CRC-32C via the SSE4.2 crc32 instruction, compiled with a target +// attribute and dispatched at runtime, so it is used even when folly (whose +// crc32c_hw is gated on compile-time SSE4.2 flags) was built without SIMD +// flags and would silently fall back to a ~20x slower table implementation. +__attribute__((target("sse4.2"))) uint32_t crc32cHardware(const uint8_t* data, + size_t n, + uint32_t crc) { + while (n >= 8) { + uint64_t v; + memcpy(&v, data, 8); + crc = static_cast(_mm_crc32_u64(crc, v)); + data += 8; + n -= 8; + } + if (n >= 4) { + uint32_t v; + memcpy(&v, data, 4); + crc = _mm_crc32_u32(crc, v); + data += 4; + n -= 4; + } + while (n > 0) { + crc = _mm_crc32_u8(crc, *data++); + n--; + } + return crc; +} + +bool hasSse42() { return __builtin_cpu_supports("sse4.2"); } +#endif +} // namespace + uint32_t checksum(BufferView data, uint32_t startingChecksum) { // CRC-32C (Castagnoli) with raw seed semantics (default 0, no final // inversion). This matches the value produced by chained SSE4.2 // _mm_crc32_* instructions and by Intel DSA's CRC generation as used by // the DTO library, so checksums can be offloaded to DSA and verified on // CPU (and vice versa) interchangeably. +#if defined(__x86_64__) + static const bool kUseHw = hasSse42(); + if (kUseHw) { + return crc32cHardware(data.data(), data.size(), startingChecksum); + } +#endif return folly::crc32c(data.data(), data.size(), startingChecksum); } } // namespace facebook::cachelib::navy From 7874531b9619525543940ef5172fefcd443e8e16 Mon Sep 17 00:00:00 2001 From: Daniel Byrne Date: Wed, 15 Jul 2026 18:22:24 +0000 Subject: [PATCH 3/3] forget additional files --- cachelib/CMakeLists.txt | 50 ++++ cachelib/cachebench/cache/Cache.h | 11 +- cachelib/cmake/cachelib-config.cmake.in | 7 + cachelib/common/CMakeLists.txt | 9 + cachelib/navy/CMakeLists.txt | 10 +- cachelib/navy/common/ChecksumOffload.cpp | 200 ++++++++++++++ cachelib/navy/common/ChecksumOffload.h | 115 ++++++++ .../navy/common/tests/ChecksumOffloadTest.cpp | 176 ++++++++++++ run_dsa_cachebench.sh | 255 ++++++++++++++++++ 9 files changed, 825 insertions(+), 8 deletions(-) create mode 100644 cachelib/navy/common/ChecksumOffload.cpp create mode 100644 cachelib/navy/common/ChecksumOffload.h create mode 100644 cachelib/navy/common/tests/ChecksumOffloadTest.cpp create mode 100755 run_dsa_cachebench.sh diff --git a/cachelib/CMakeLists.txt b/cachelib/CMakeLists.txt index c33e70d624..844eb7cd51 100644 --- a/cachelib/CMakeLists.txt +++ b/cachelib/CMakeLists.txt @@ -315,6 +315,56 @@ endfunction() add_thrift_file(OBJECT_CACHE_PERSISTENCE object_cache/persistence/persistent_data.thrift json) +if (BUILD_WITH_DTO) + # DSA offload runtime (DTO). Resolved here so every cachelib library can + # use it: DTO::dto is linked PUBLIC into cachelib_common (see + # common/CMakeLists.txt), the base library of every other cachelib target, + # which propagates the headers, the link dependency, and the + # CACHELIB_BUILD_WITH_DTO compile definition to the whole tree and to + # consumers of the installed package. + # + # The DSA checksum offload code requires a DTO with the caller-allocated + # async submit/poll API and the raw-CRC32C convention, which upstream + # intel/DTO does not have — so when no installed DTO cmake package is + # found, fetch and build the pinned revision in-tree rather than linking + # whatever libdto happens to be on the system. + set(CACHELIB_DTO_GIT_REPOSITORY "https://github.com/byrnedj/DTO.git" + CACHE STRING "Git repository for DTO when no installed copy is found") + set(CACHELIB_DTO_GIT_TAG "64b01f8d06c939d8a70066e0b643cb97ffced060" + CACHE STRING "DTO revision to fetch when no installed copy is found") + + find_package(DTO CONFIG QUIET) + if (DTO_FOUND) + message(STATUS "Using installed DTO package: ${DTO_DIR}") + else() + if (CMAKE_VERSION VERSION_LESS 3.14) + message(FATAL_ERROR + "BUILD_WITH_DTO needs an installed DTO cmake package or " + "CMake >= 3.14 to fetch one. Install DTO from " + "${CACHELIB_DTO_GIT_REPOSITORY} at ${CACHELIB_DTO_GIT_TAG}, " + "or upgrade CMake.") + endif() + # DTO links against libaccel-config and libnuma; check for them here so + # a missing system package fails at configure time with a clear message + # instead of at link time inside the fetched project. + find_library(ACCEL_CONFIG_LIBRARY accel-config) + find_library(NUMA_LIBRARY numa) + if (NOT ACCEL_CONFIG_LIBRARY OR NOT NUMA_LIBRARY) + message(FATAL_ERROR + "BUILD_WITH_DTO requires the libaccel-config and libnuma " + "development packages (e.g. apt install libaccel-config-dev " + "libnuma-dev)") + endif() + message(STATUS "DTO not installed; fetching " + "${CACHELIB_DTO_GIT_REPOSITORY} @ ${CACHELIB_DTO_GIT_TAG}") + include(FetchContent) + FetchContent_Declare(dto + GIT_REPOSITORY "${CACHELIB_DTO_GIT_REPOSITORY}" + GIT_TAG "${CACHELIB_DTO_GIT_TAG}") + FetchContent_MakeAvailable(dto) + endif() +endif() + add_subdirectory (common) add_subdirectory (shm) add_subdirectory (navy) diff --git a/cachelib/cachebench/cache/Cache.h b/cachelib/cachebench/cache/Cache.h index 0e7e5d96fa..f6e62127fe 100644 --- a/cachelib/cachebench/cache/Cache.h +++ b/cachelib/cachebench/cache/Cache.h @@ -26,7 +26,9 @@ #include #include +#include #include +#include #include #include "cachelib/allocator/CacheAllocator.h" @@ -1411,11 +1413,18 @@ void Cache::setStringItem(WriteHandle& handle, } auto ptr = reinterpret_cast(getMemory(handle)); - std::strncpy(ptr, str.c_str(), dataSize); + // memcpy/memset instead of strncpy: strncpy scans for the terminator byte + // by byte and is not interposed by DTO, while memcpy/memset of large + // values can be offloaded to DSA. Like strncpy, write exactly dataSize + // bytes: the string (truncated if needed), then a zero-filled tail. + const size_t copyLen = std::min(str.size() + 1, dataSize); + std::memcpy(ptr, str.c_str(), copyLen); // Make sure the copied string ends with null char if (str.size() + 1 > dataSize) { ptr[dataSize - 1] = '\0'; + } else if (copyLen < dataSize) { + std::memset(ptr + copyLen, 0, dataSize - copyLen); } } diff --git a/cachelib/cmake/cachelib-config.cmake.in b/cachelib/cmake/cachelib-config.cmake.in index 6e850f5561..a6a6475ce1 100644 --- a/cachelib/cmake/cachelib-config.cmake.in +++ b/cachelib/cmake/cachelib-config.cmake.in @@ -35,6 +35,13 @@ find_dependency(fizz ) find_dependency(wangle) find_dependency(FBThrift) +# When cachelib was built with DSA checksum offload, cachelib_navy links the +# DTO runtime; resolve its imported target for consumers. A DTO fetched and +# built in-tree installs its package into this same prefix. +if (@BUILD_WITH_DTO@) + find_dependency(DTO CONFIG) +endif() + if (NOT TARGET cachelib) include("${CACHELIB_CMAKE_DIR}/cachelib-targets.cmake") endif() diff --git a/cachelib/common/CMakeLists.txt b/cachelib/common/CMakeLists.txt index 308f5269b9..79e24dd04a 100644 --- a/cachelib/common/CMakeLists.txt +++ b/cachelib/common/CMakeLists.txt @@ -51,6 +51,15 @@ target_link_libraries(cachelib_common PUBLIC ${XXHASH_LIBRARY} ) +if (BUILD_WITH_DTO) + # PUBLIC so the DTO headers/link dependency and the compile definition + # propagate to every cachelib library and to installed-package consumers. + # DTO itself is resolved (installed package or pinned fetch) in the + # top-level CMakeLists.txt. + target_link_libraries(cachelib_common PUBLIC DTO::dto) + target_compile_definitions(cachelib_common PUBLIC CACHELIB_BUILD_WITH_DTO) +endif() + install(TARGETS cachelib_common EXPORT cachelib-exports DESTINATION ${LIB_INSTALL_DIR} ) diff --git a/cachelib/navy/CMakeLists.txt b/cachelib/navy/CMakeLists.txt index 031864b087..ec85134f00 100644 --- a/cachelib/navy/CMakeLists.txt +++ b/cachelib/navy/CMakeLists.txt @@ -66,13 +66,9 @@ target_link_libraries(cachelib_navy PUBLIC GTest::gmock ) -if (BUILD_WITH_DTO) - find_path(DTO_INCLUDE_DIR NAMES dto.h REQUIRED) - find_library(DTO_LIBRARY NAMES dto REQUIRED) - target_include_directories(cachelib_navy PRIVATE ${DTO_INCLUDE_DIR}) - target_compile_definitions(cachelib_navy PRIVATE CACHELIB_BUILD_WITH_DTO) - target_link_libraries(cachelib_navy PUBLIC ${DTO_LIBRARY} accel-config) -endif() +# DTO (DSA offload runtime) is resolved in the top-level CMakeLists.txt and +# linked PUBLIC into cachelib_common, so cachelib_navy inherits DTO::dto and +# the CACHELIB_BUILD_WITH_DTO definition transitively. install(TARGETS cachelib_navy EXPORT cachelib-exports diff --git a/cachelib/navy/common/ChecksumOffload.cpp b/cachelib/navy/common/ChecksumOffload.cpp new file mode 100644 index 0000000000..035938ea6d --- /dev/null +++ b/cachelib/navy/common/ChecksumOffload.cpp @@ -0,0 +1,200 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "cachelib/navy/common/ChecksumOffload.h" + +#include +#include +#include + +#include +#include +#include + +#include "cachelib/navy/common/Hash.h" + +#ifdef CACHELIB_BUILD_WITH_DTO +#include +#endif + +namespace facebook { +namespace cachelib { +namespace navy { + +#ifdef CACHELIB_BUILD_WITH_DTO + +static_assert(sizeof(dto_async_op) <= 192, + "AsyncChecksumOp::opStorage_ too small for dto_async_op"); +static_assert(alignof(dto_async_op) <= 64, + "AsyncChecksumOp::opStorage_ under-aligned for dto_async_op"); + +bool checksumOffloadSupported() { return true; } + +AsyncChecksumOp::~AsyncChecksumOp() { + // A submitted DSA operation writes to opStorage_ (completion record) and, + // for copies, to dest_. It must be drained before this object dies. + if (state_ == State::kDsaPending) { + wait(); + } +} + +void AsyncChecksumOp::submitImpl(uint8_t* dest, + BufferView src, + bool cacheControl) { + XDCHECK(state_ == State::kIdle); + dest_ = dest; + src_ = src; + copy_ = dest != nullptr; + + auto* op = reinterpret_cast(opStorage_); + const int rc = copy_ ? dto_submit_memcpy_crc(op, dest, src.data(), + src.size(), cacheControl) + : dto_submit_crc(op, src.data(), src.size()); + if (rc == DTO_ASYNC_SUBMITTED) { + state_ = State::kDsaPending; + return; + } + // DTO_ASYNC_FALLBACK: nothing submitted or copied; run on CPU now. + if (copy_) { + std::memcpy(dest, src.data(), src.size()); + } + crc_ = checksum(src); + state_ = State::kCpuDone; +} + +uint32_t AsyncChecksumOp::wait() { + XDCHECK(state_ != State::kIdle); + if (state_ == State::kCpuDone) { + state_ = State::kIdle; + return crc_; + } + + auto* op = reinterpret_cast(opStorage_); + // Yielding suspends this fiber and lets other request fibers run on the + // same NavyThread while the accelerator works; that is the actual + // "free the writer thread" mechanism. Yield ONLY when another fiber is + // ready to run: with a lone fiber, yield() returns immediately and the + // loop would busy-spin through the FiberManager at full rate for the + // whole accelerator operation, which is far more expensive than a pause + // poll. Outside fiber context (plain thread-pool schedulers, tests) this + // reduces to a pause-poll. + auto* fm = folly::fibers::onFiber() + ? folly::fibers::FiberManager::getFiberManagerUnsafe() + : nullptr; + int rc; + while ((rc = dto_async_poll(op)) == DTO_ASYNC_PENDING) { + if (fm && fm->hasReadyTasks()) { + folly::fibers::yield(); + } else { + folly::asm_volatile_pause(); + } + } + state_ = State::kIdle; + if (rc == DTO_ASYNC_DONE) { + return static_cast(dto_async_crc_val(op)); + } + // Accelerator failure: destination contents are unspecified, so redo the + // whole operation on the CPU. dest_ is not yet visible to readers per the + // submit contract, so overwriting is safe. + if (copy_) { + std::memcpy(dest_, src_.data(), src_.size()); + } + return checksum(src_); +} + +#else // !CACHELIB_BUILD_WITH_DTO + +bool checksumOffloadSupported() { return false; } + +AsyncChecksumOp::~AsyncChecksumOp() = default; + +void AsyncChecksumOp::submitImpl(uint8_t* dest, + BufferView src, + bool /* cacheControl */) { + XDCHECK(state_ == State::kIdle); + if (dest != nullptr) { + std::memcpy(dest, src.data(), src.size()); + } + crc_ = checksum(src); + state_ = State::kCpuDone; +} + +uint32_t AsyncChecksumOp::wait() { + XDCHECK(state_ == State::kCpuDone); + state_ = State::kIdle; + return crc_; +} + +#endif // CACHELIB_BUILD_WITH_DTO + +void AsyncChecksumOp::submitCopyAndChecksum(uint8_t* dest, + BufferView src, + bool cacheControl) { + XDCHECK(dest); + submitImpl(dest, src, cacheControl); +} + +void AsyncChecksumOp::submitChecksum(BufferView src) { + submitImpl(nullptr, src, false); +} + +uint32_t copyAndChecksum(uint8_t* dest, + BufferView src, + folly::FunctionRef overlap) { + AsyncChecksumOp op; + // Cache control: destinations of fused copies (write buffers) are read + // again shortly, by lookups served from in-memory buffers and by the + // device flush path. + op.submitCopyAndChecksum(dest, src, true /* cacheControl */); + overlap(); + return op.wait(); +} + +uint32_t checksumWithOverlap(BufferView src, + folly::FunctionRef overlap) { + AsyncChecksumOp op; + op.submitChecksum(src); + overlap(); + return op.wait(); +} + +bool checksumOffloadSelfCheck() { + if (!checksumOffloadSupported()) { + return false; + } + // Exercise both operations on a buffer large enough to exceed DTO's + // minimum-size gates (DTO_CRC_MIN_BYTES / DTO_MIN_BYTES) so the DSA path + // actually runs, and verify parity with navy::checksum() plus copy + // fidelity. If DSA is unavailable, the CPU fallback must also match. + constexpr size_t kSize = 1024 * 1024; + std::vector src(kSize); + std::vector dst(kSize, 0); + std::mt19937 gen{12345}; + for (auto& b : src) { + b = static_cast(gen()); + } + + const BufferView view{src.size(), src.data()}; + const uint32_t sw = checksum(view); + const uint32_t viaCrc = checksumWithOverlap(view, [] {}); + const uint32_t viaCopy = copyAndChecksum(dst.data(), view, [] {}); + return viaCrc == sw && viaCopy == sw && + std::memcmp(dst.data(), src.data(), kSize) == 0; +} + +} // namespace navy +} // namespace cachelib +} // namespace facebook diff --git a/cachelib/navy/common/ChecksumOffload.h b/cachelib/navy/common/ChecksumOffload.h new file mode 100644 index 0000000000..5bacb12c2b --- /dev/null +++ b/cachelib/navy/common/ChecksumOffload.h @@ -0,0 +1,115 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#include "cachelib/navy/common/Buffer.h" + +namespace facebook { +namespace cachelib { +namespace navy { + +// Checksum (and fused copy+checksum) helpers that can offload to Intel DSA +// via the DTO library when built with CACHELIB_BUILD_WITH_DTO. Without DTO +// support (or when DSA is unavailable at runtime), they run equivalent +// software implementations. In all cases the returned checksum is +// navy::checksum() (raw CRC-32C, seed 0) of @src, so offloaded and +// CPU-computed checksums verify against each other interchangeably. + +// A single asynchronous checksum or fused copy+checksum operation. +// submit*() enqueues the operation on DSA and returns immediately (or runs +// it synchronously on the CPU when offload is unavailable); wait() completes +// it. While the accelerator works, wait() yields the calling fiber (when +// running on one, e.g. on a NavyThread) so other requests can execute on +// this thread — this is what makes the offload truly asynchronous. Off +// fibers it polls with a CPU pause. +// +// The object is not copyable or movable: the device holds a pointer into +// its storage while the operation is in flight. Each submit must be paired +// with exactly one wait() before reuse or destruction. +class AsyncChecksumOp { + public: + AsyncChecksumOp() = default; + AsyncChecksumOp(const AsyncChecksumOp&) = delete; + AsyncChecksumOp& operator=(const AsyncChecksumOp&) = delete; + ~AsyncChecksumOp(); + + // Copies @src to @dest and computes checksum(src) as one fused DSA + // operation. @dest must not overlap @src and must not be accessed until + // wait() returns: on a (rare) accelerator failure the copy is redone on + // the CPU, so intermediate destination contents are unspecified. + // @cacheControl directs the DSA copy output toward the CPU cache; use it + // when the destination will be read again soon (e.g. an in-memory region + // buffer that serves lookups and is flushed to the device shortly after). + void submitCopyAndChecksum(uint8_t* dest, BufferView src, bool cacheControl); + + // Computes checksum(src). @src may be read concurrently (e.g. to rebuild + // a bloom filter) but must not be modified until wait() returns. + void submitChecksum(BufferView src); + + // Completes the submitted operation and returns checksum(src). + uint32_t wait(); + + private: + enum class State : uint8_t { kIdle, kCpuDone, kDsaPending }; + + void submitImpl(uint8_t* dest, BufferView src, bool cacheControl); + + State state_{State::kIdle}; + bool copy_{false}; + uint32_t crc_{0}; + uint8_t* dest_{nullptr}; + BufferView src_; + // Opaque storage for the DTO async operation (descriptor + completion + // record); sized/aligned to hold dto_async_op without exposing dto.h here. + alignas(64) unsigned char opStorage_[192]; +}; + +// Convenience wrappers over AsyncChecksumOp. The @overlap callback is +// invoked exactly once after submission. Use it for CPU work that can +// proceed while the accelerator operates. It may read the source range but +// must not write it, and must not access the destination range at all. +// Note: when the operation falls back to software (non-DTO build, or DSA +// submission failure), the copy+checksum completes synchronously inside the +// submit step, so @overlap runs after the operation rather than overlapping +// it — the callback's constraints above still apply either way. + +// Returns true if this binary was built with DTO/DSA support. +bool checksumOffloadSupported(); + +// Verifies at runtime that the DTO/DSA checksum matches navy::checksum() and +// that the fused copy is faithful. Returns true iff offload is usable and +// consistent; returns false when built without DTO support. Callers should +// enable offload only if this returns true. +bool checksumOffloadSelfCheck(); + +// Copies @src to @dest and returns the checksum of @src, as a single fused +// DSA "Memory Copy with CRC Generation" operation when available. The copy +// lands toward the CPU cache (cache control) since such destinations are +// typically read again soon. +uint32_t copyAndChecksum(uint8_t* dest, + BufferView src, + folly::FunctionRef overlap); + +// Returns the checksum of @src, computed by DSA when available. +uint32_t checksumWithOverlap(BufferView src, + folly::FunctionRef overlap); + +} // namespace navy +} // namespace cachelib +} // namespace facebook diff --git a/cachelib/navy/common/tests/ChecksumOffloadTest.cpp b/cachelib/navy/common/tests/ChecksumOffloadTest.cpp new file mode 100644 index 0000000000..bae5482fe1 --- /dev/null +++ b/cachelib/navy/common/tests/ChecksumOffloadTest.cpp @@ -0,0 +1,176 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +#include +#include +#include + +#include "cachelib/navy/common/ChecksumOffload.h" +#include "cachelib/navy/common/Hash.h" + +namespace facebook::cachelib::navy::tests { +namespace { +std::vector randomBytes(size_t size, uint32_t seed) { + std::mt19937 gen{seed}; + std::vector bytes(size); + for (auto& b : bytes) { + b = static_cast(gen()); + } + return bytes; +} +} // namespace + +// Sizes spanning below and above typical offload gates (DTO_MIN_BYTES and +// navy-level thresholds), so both the DSA path and internal CPU fallbacks +// are exercised when built with DTO support. +const size_t kSizes[] = {1, 13, 175, 4096, 16 * 1024, 64 * 1024, 1024 * 1024}; + +TEST(ChecksumOffload, CopyAndChecksumMatchesSoftware) { + uint32_t seed = 1; + for (auto size : kSizes) { + auto src = randomBytes(size, seed++); + std::vector dst(size, 0xAA); + const BufferView view{src.size(), src.data()}; + + const uint32_t cs = copyAndChecksum(dst.data(), view, [] {}); + EXPECT_EQ(checksum(view), cs) << "size=" << size; + EXPECT_EQ(0, std::memcmp(src.data(), dst.data(), size)) << "size=" << size; + } +} + +TEST(ChecksumOffload, ChecksumWithOverlapMatchesSoftware) { + uint32_t seed = 100; + for (auto size : kSizes) { + auto src = randomBytes(size, seed++); + const BufferView view{src.size(), src.data()}; + EXPECT_EQ(checksum(view), checksumWithOverlap(view, [] {})) + << "size=" << size; + } +} + +TEST(ChecksumOffload, OverlapRunsExactlyOnce) { + for (auto size : {175UL, 1024UL * 1024UL}) { + auto src = randomBytes(size, 7); + std::vector dst(size, 0); + const BufferView view{src.size(), src.data()}; + + int copyOverlapRuns = 0; + copyAndChecksum(dst.data(), view, [&] { copyOverlapRuns++; }); + EXPECT_EQ(1, copyOverlapRuns) << "size=" << size; + + int crcOverlapRuns = 0; + checksumWithOverlap(view, [&] { crcOverlapRuns++; }); + EXPECT_EQ(1, crcOverlapRuns) << "size=" << size; + } +} + +// The overlap callback is allowed to read the source range concurrently with +// the operation (e.g. BigHash rebuilds its bloom filter from the bucket while +// DSA checksums it). +TEST(ChecksumOffload, OverlapMayReadSource) { + const size_t size = 64 * 1024; + auto src = randomBytes(size, 11); + const BufferView view{src.size(), src.data()}; + + uint64_t sum = 0; + const uint32_t cs = checksumWithOverlap(view, [&] { + for (auto b : src) { + sum += b; + } + }); + EXPECT_EQ(checksum(view), cs); + EXPECT_NE(0ULL, sum); +} + +// Multiple async operations in flight from one thread, out-of-order waits. +TEST(ChecksumOffload, AsyncOpsInFlight) { + const size_t size = 64 * 1024; + auto src1 = randomBytes(size, 21); + auto src2 = randomBytes(size, 22); + std::vector dst1(size, 0); + const BufferView v1{src1.size(), src1.data()}; + const BufferView v2{src2.size(), src2.data()}; + + AsyncChecksumOp op1; + AsyncChecksumOp op2; + op1.submitCopyAndChecksum(dst1.data(), v1, true /* cacheControl */); + op2.submitChecksum(v2); + + // Complete in reverse submission order. + EXPECT_EQ(checksum(v2), op2.wait()); + EXPECT_EQ(checksum(v1), op1.wait()); + EXPECT_EQ(0, std::memcmp(src1.data(), dst1.data(), size)); + + // An op object is reusable after wait(). + op1.submitChecksum(v2); + EXPECT_EQ(checksum(v2), op1.wait()); +} + +// wait() on a fiber takes the yield path: other fibers run on the thread +// while the operation is in flight. +TEST(ChecksumOffload, AsyncOpOnFiber) { + const size_t size = 1024 * 1024; + auto src = randomBytes(size, 27); + std::vector dst(size, 0); + const BufferView view{src.size(), src.data()}; + + folly::EventBase evb; + auto& fm = folly::fibers::getFiberManager(evb); + uint32_t got = 0; + bool otherFiberRan = false; + fm.addTask([&] { + AsyncChecksumOp op; + op.submitCopyAndChecksum(dst.data(), view, true /* cacheControl */); + got = op.wait(); + }); + fm.addTask([&] { otherFiberRan = true; }); + evb.loop(); + + EXPECT_EQ(checksum(view), got); + EXPECT_TRUE(otherFiberRan); + EXPECT_EQ(0, std::memcmp(src.data(), dst.data(), size)); +} + +// Destructor drains an in-flight op (does not leave the device writing to +// freed stack memory). +TEST(ChecksumOffload, AsyncOpDrainOnDestroy) { + const size_t size = 256 * 1024; + auto src = randomBytes(size, 33); + std::vector dst(size, 0); + { + AsyncChecksumOp op; + op.submitCopyAndChecksum(dst.data(), BufferView{src.size(), src.data()}, + false /* cacheControl */); + // no wait(): destructor must drain + } + EXPECT_EQ(0, std::memcmp(src.data(), dst.data(), size)); +} + +TEST(ChecksumOffload, SelfCheck) { + if (checksumOffloadSupported()) { + // Built with DTO: DSA (or DTO's internal CPU fallback) must agree with + // navy::checksum. A failure here means offloaded checksums would not + // verify against CPU-computed ones on this machine. + EXPECT_TRUE(checksumOffloadSelfCheck()); + } else { + EXPECT_FALSE(checksumOffloadSelfCheck()); + } +} +} // namespace facebook::cachelib::navy::tests diff --git a/run_dsa_cachebench.sh b/run_dsa_cachebench.sh new file mode 100755 index 0000000000..b8d24922a8 --- /dev/null +++ b/run_dsa_cachebench.sh @@ -0,0 +1,255 @@ +#!/bin/bash +# Build CacheLib with DTO, configure DSA work queues, and run cachebench +# pinned to socket 0 with DSA checksum offload and (optionally) DTO's +# transparent memcpy/memset interception. +# +# Usage: ./run_dsa_cachebench.sh [options] +# --workload cdn|bigcache +# cdn = synthetic CDN hit-ratio config (default) +# bigcache = production BigCache trace replay +# (~/bigcache_trace_sea1c01_20250414_20250421.json, +# scaled exactly as the offload study: 400GB Navy, +# 16GB parcel, 500k ops/thread, fiber scheduler) +# --config PATH base cachebench config (overrides the workload default) +# --offload on|off|none +# on = Navy data checksums on, computed by DSA +# off = Navy data checksums on, computed in software +# none = base config's own checksum setting +# (cdn: no Navy section added; bigcache: checksums +# off = the study's no-checksum baseline A) +# --intercept on|off transparent DTO interception of large memcpy/memset. +# off (default) sets DTO_MIN_BYTES=1GB so only the explicit +# checksum-offload API uses DSA; on leaves DTO's own +# size threshold (32KB) in effect. +# --fibers on|off Navy fiber scheduler (NavyRequestScheduler, libaio, +# study settings: maxNumReads 1024 / maxNumWrites 1200 / +# qDepth 32). Default off; pass --fibers on to enable. +# --nvm-size-mb N Navy cache size (default: cdn 20480, bigcache 409600) +# --num-ops N override test_config.numOps (per stressor thread; +# bigcache default 500000 = the study's 12M-op replay) +# --devices "0 2 4 6" DSA device ids to enable (default "0 2 4 6") +# --out DIR output/work directory (default ./dsa_bench_out) +# --skip-build do not (re)build cachebench +# --skip-dsa do not reconfigure DSA work queues +set -euo pipefail + +REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +BUILD_DIR="$REPO/build-cachelib" +PREFIX="$REPO/opt/cachelib" +DTO_DIR="${DTO_DIR:-$HOME/DTO}" + +WORKLOAD=cdn +CONFIG="" +OFFLOAD=on +INTERCEPT=off +FIBERS=off +NVM_SIZE_MB="" +NUM_OPS="" +DEVICES="0 2 4 6" +OUT="$PWD/dsa_bench_out" +SKIP_BUILD=0 +SKIP_DSA=0 + +while [ $# -gt 0 ]; do + case "$1" in + --workload) WORKLOAD="$2"; shift 2 ;; + --config) CONFIG="$2"; shift 2 ;; + --offload) OFFLOAD="$2"; shift 2 ;; + --intercept) INTERCEPT="$2"; shift 2 ;; + --fibers) FIBERS="$2"; shift 2 ;; + --nvm-size-mb) NVM_SIZE_MB="$2"; shift 2 ;; + --num-ops) NUM_OPS="$2"; shift 2 ;; + --devices) DEVICES="$2"; shift 2 ;; + --out) OUT="$2"; shift 2 ;; + --skip-build) SKIP_BUILD=1; shift ;; + --skip-dsa) SKIP_DSA=1; shift ;; + -h|--help) sed -n '2,33p' "$0"; exit 0 ;; + *) echo "unknown option: $1" >&2; exit 1 ;; + esac +done + +case "$WORKLOAD" in + cdn) + [ -n "$CONFIG" ] || CONFIG="$REPO/cachelib/cachebench/test_configs/hit_ratio/cdn/config.json" + [ -n "$NVM_SIZE_MB" ] || NVM_SIZE_MB=20480 + ;; + bigcache) + [ -n "$CONFIG" ] || CONFIG="$HOME/bigcache_trace_sea1c01_20250414_20250421.json" + [ -n "$NVM_SIZE_MB" ] || NVM_SIZE_MB=409600 + [ -n "$NUM_OPS" ] || NUM_OPS=500000 + ;; + *) echo "--workload must be cdn|bigcache" >&2; exit 1 ;; +esac +case "$OFFLOAD" in on|off|none) ;; *) echo "--offload must be on|off|none" >&2; exit 1 ;; esac +case "$INTERCEPT" in on|off) ;; *) echo "--intercept must be on|off" >&2; exit 1 ;; esac +case "$FIBERS" in on|off) ;; *) echo "--fibers must be on|off" >&2; exit 1 ;; esac +mkdir -p "$OUT" + +export LD_LIBRARY_PATH="$PREFIX/lib:/usr/local/lib:${LD_LIBRARY_PATH:-}" + +# ---------------------------------------------------------------- build ---- +if [ "$SKIP_BUILD" = 0 ]; then + echo "== Building cachebench (BUILD_WITH_DTO=ON) ==" + if [ ! -f "$BUILD_DIR/CMakeCache.txt" ]; then + mkdir -p "$BUILD_DIR" + export CMAKE_PREFIX_PATH="$PREFIX/lib/cmake:$PREFIX:/usr/local${CMAKE_PREFIX_PATH:+:$CMAKE_PREFIX_PATH}" + # CMAKE_DISABLE_FIND_PACKAGE_uring: cachelib enables its io_uring path + # whenever liburing exists on the system, but that requires a folly + # built with liburing; disable the probe so the two cannot disagree. + cmake -S "$REPO/cachelib" -B "$BUILD_DIR" \ + -DCMAKE_INSTALL_PREFIX="$PREFIX" \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DBUILD_TESTS=ON \ + -DBUILD_WITH_DTO=ON \ + -DCMAKE_DISABLE_FIND_PACKAGE_uring=ON + fi + cmake --build "$BUILD_DIR" --target cachebench -j "$(nproc)" +fi +CACHEBENCH="$BUILD_DIR/cachebench/cachebench" +[ -x "$CACHEBENCH" ] || { echo "cachebench not found at $CACHEBENCH" >&2; exit 1; } + +# ------------------------------------------------------------ DSA setup ---- +if [ "$SKIP_DSA" = 0 ]; then + echo "== Configuring DSA shared work queues (devices: $DEVICES) ==" + for d in $DEVICES; do + sudo "$DTO_DIR/accelConfig.sh" "$d" yes 0 4 >/dev/null + done + sudo chmod 0666 /dev/dsa/wq*.0 +fi +ls /dev/dsa/wq*.0 >/dev/null 2>&1 || { echo "no enabled DSA WQs under /dev/dsa" >&2; exit 1; } +echo "DSA WQs: $(ls /dev/dsa/)" + +# ------------------------------------------------------- derive config ----- +RUN_CONFIG="$OUT/config_${WORKLOAD}_offload_${OFFLOAD}.json" +CONFIG="$CONFIG" RUN_CONFIG="$RUN_CONFIG" OFFLOAD="$OFFLOAD" \ +NVM_SIZE_MB="$NVM_SIZE_MB" NUM_OPS="$NUM_OPS" OUT="$OUT" FIBERS="$FIBERS" \ +python3 - <<'EOF' +import json, os, shutil, sys +cfgPath = os.environ["CONFIG"] +cfg = json.load(open(cfgPath)) +base = os.path.dirname(os.path.abspath(cfgPath)) +tc = cfg["test_config"] +cc = cfg["cache_config"] +offload = os.environ["OFFLOAD"] +isTraceReplay = "replay" in tc.get("generator", "") + +# cachebench resolves distribution files by prepending the config file's +# directory (even to absolute paths), so copy them next to the derived +# config and reference them by basename. +for k in ("popDistFile", "valSizeDistFile"): + if k in tc: + src = tc[k] if os.path.isabs(tc[k]) else os.path.join(base, tc[k]) + dst = os.path.join(os.environ["OUT"], os.path.basename(src)) + if os.path.abspath(src) != os.path.abspath(dst): + shutil.copyfile(src, dst) + tc[k] = os.path.basename(src) +# the (multi-GB) trace file is opened directly: absolutize, don't copy +if "traceFileName" in tc and not os.path.isabs(tc["traceFileName"]): + tc["traceFileName"] = os.path.join(base, tc["traceFileName"]) +if "traceFileName" in tc and not os.path.exists(tc["traceFileName"]): + sys.exit(f"trace file not found: {tc['traceFileName']}") + +if os.environ["NUM_OPS"]: + tc["numOps"] = int(os.environ["NUM_OPS"]) + +if isTraceReplay: + # Trace replay (BigCache): the base config already has a Navy section + # sized for its original production host; rescale it to this machine + # exactly as the offload study did. + cc["nvmCacheSizeMB"] = int(os.environ["NVM_SIZE_MB"]) + cc["nvmCachePaths"] = [os.path.join(os.environ["OUT"], "navy_cache_file")] + cc["navyParcelMemoryMB"] = 16384 + if offload != "none": + cc["navyDataChecksum"] = True + cc["navyChecksumOffload"] = offload == "on" + cc["navyChecksumOffloadMinSize"] = 4096 +elif offload != "none": + # Synthetic workloads (CDN): the base config is DRAM-only; add the + # hybrid Navy tier the offload applies to. + cc["nvmCacheSizeMB"] = int(os.environ["NVM_SIZE_MB"]) + cc["nvmCachePaths"] = [os.path.join(os.environ["OUT"], "navy_cache_file")] + cc["navyReaderThreads"] = 32 + cc["navyWriterThreads"] = 32 + cc["navyDataChecksum"] = True + cc["navyChecksumOffload"] = offload == "on" + cc["navyChecksumOffloadMinSize"] = 4096 + +if os.environ["FIBERS"] == "on": + # NavyRequestScheduler (fibers, libaio) with the study's settings; + # required for the offload's yield-during-DSA-wait to overlap work. + cc["navyMaxNumReads"] = 1024 + cc["navyMaxNumWrites"] = 1200 + cc["navyQDepth"] = 32 + cc["navyEnableIoUring"] = False + +json.dump(cfg, open(os.environ["RUN_CONFIG"], "w"), indent=2) +print(f"wrote {os.environ['RUN_CONFIG']}") +EOF + +# ---------------------------------------------------------------- run ------ +export DTO_USESTDC_CALLS=0 +export DTO_CRC_MIN_BYTES=4096 +export DTO_WAIT_METHOD="${DTO_WAIT_METHOD:-busypoll}" +export DTO_COLLECT_STATS=1 +if [ "$INTERCEPT" = off ]; then + export DTO_MIN_BYTES=1073741824 # 1GB: transparent interception off +else + unset DTO_MIN_BYTES # DTO's own threshold decides +fi + +LOG="$OUT/cachebench_${WORKLOAD}_offload_${OFFLOAD}_intercept_${INTERCEPT}.log" +echo "== Running cachebench on socket 0 (workload=$WORKLOAD offload=$OFFLOAD intercept=$INTERCEPT fibers=$FIBERS) ==" +echo " log: $LOG" + +# The fiber scheduler has a known ~1-in-4 startup hang (frozen at +# NavyRequestDispatcher startup). With --progress 60 a healthy run writes to +# its log at least once a minute, so 150s of log silence while the process +# lives means hung: kill and retry (up to 3 attempts). +attempt_run() { + rm -f "$OUT/navy_cache_file" + /usr/bin/time -v numactl -N 0 \ + "$CACHEBENCH" --json_test_config "$RUN_CONFIG" --progress 60 \ + > "$LOG" 2>&1 & + local tpid=$! + local last=-1 silent=0 + while kill -0 "$tpid" 2>/dev/null; do + sleep 30 + local sz + sz=$(stat -c %s "$LOG" 2>/dev/null || echo 0) + if [ "$sz" = "$last" ]; then + silent=$((silent + 1)) + if [ "$silent" -ge 5 ]; then + pkill -9 -P "$tpid" 2>/dev/null + kill -9 "$tpid" 2>/dev/null + wait "$tpid" 2>/dev/null + return 99 + fi + else + silent=0 + fi + last=$sz + done + wait "$tpid" +} + +set +e +rc=99 +for attempt in 1 2 3; do + attempt_run + rc=$? + [ "$rc" -ne 99 ] && break + echo "WATCHDOG: run frozen for 150s+ (startup hang?), retrying (attempt $attempt of 3)" +done +set -e +rm -f "$OUT/navy_cache_file" + +# ------------------------------------------------------------- report ------ +echo "exit=$rc" +grep -E "Total Ops|get |set |NVM Gets|NVM Puts" "$LOG" | head -6 || true +echo "checksum error lines: $(grep -ciE 'checksum.*(error|mismatch)' "$LOG" || true)" +grep -E "User time|System time|Elapsed \(wall|Maximum resident" "$LOG" || true +if grep -q "Number of Memory Operations" "$LOG"; then + echo "-- DTO op counts (see full table in the log) --" + grep -A20 "Number of Memory Operations" "$LOG" | head -24 +fi +exit $rc