async IO support for RAID0 and refactoring around Device implementations

Summary:
This change supports async IO for the RAID0 devices (i.e., multiple device files). To do so, this
change introduces the concept of IOOp and IOReq concepts where a single IOReq can be split into
multiple IOOp for its operation. This change also performs the refactoring around FileDevice and
RAID0Device such that FileDevice can support all the combinations of single device vs multiple
devices and async vs sync IOs.

This change disables static listing for //cachelib/navy/common/tests:tests because it is not
supported for value parameterized tests.

Reviewed By: therealgymmy

Differential Revision: D47459535

fbshipit-source-id: d272c4052d913c71b4c986ded516c5a471421492

Author: Jaesoo Lee

cachelib/allocator/nvmcache/NavyConfig.cpp

@@ -64,14 +64,13 @@ RandomAPConfig& RandomAPConfig::setAdmProbability(double admProbability) {
 }
 
 // device settings
-void NavyConfig::setIoThreads(unsigned int numIoThreads,
-                              unsigned int qDepthPerThread) {
-  if (usesRaidFiles()) {
+void NavyConfig::enableAsyncIo(unsigned int qDepth, bool enableIoUring) {
+  if (qDepth == 0) {
     throw std::invalid_argument(
-        "AsyncDevice is not yet supported for RAID files");
+        folly::sformat("qdepth {} should be >=1 to use async IO", qDepth));
   }
-  numIoThreads_ = numIoThreads;
-  qDepthPerThread_ = qDepthPerThread;
+  ioEngine_ = enableIoUring ? IoEngine::IoUring : IoEngine::LibAio;
+  qDepth_ = qDepth;
 }
 
 void NavyConfig::setSimpleFile(const std::string& fileName,
@@ -220,6 +219,8 @@ std::map<std::string, std::string> NavyConfig::serialize() const {
   configMap["navyConfig::truncateFile"] = truncateFile_ ? "true" : "false";
   configMap["navyConfig::deviceMaxWriteSize"] =
       folly::to<std::string>(deviceMaxWriteSize_);
+  configMap["navyConfig::ioEngine"] = getIoEngineName(ioEngine_).str();
+  configMap["navyConfig::QDepth"] = folly::to<std::string>(qDepth_);
 
   // Job scheduler settings
   configMap["navyConfig::readerThreads"] =
@@ -229,10 +230,6 @@ std::map<std::string, std::string> NavyConfig::serialize() const {
   configMap["navyConfig::navyReqOrderingShards"] =
       folly::to<std::string>(navyReqOrderingShards_);
 
-  configMap["navyConfig::numIoThreads"] = folly::to<std::string>(numIoThreads_);
-  configMap["navyConfig::QDepthPerThread"] =
-      folly::to<std::string>(qDepthPerThread_);
-  configMap["navyConfig::enableIoUring"] = enableIoUring_ ? "true" : "false";
   // Other settings
   configMap["navyConfig::maxConcurrentInserts"] =
       folly::to<std::string>(maxConcurrentInserts_);

cachelib/allocator/nvmcache/NavyConfig.h

@@ -440,6 +440,21 @@ class EnginesConfig {
   BigHashConfig bigHashConfig_;
 };
 
+enum class IoEngine : uint8_t { IoUring, LibAio, Sync };
+
+inline const folly::StringPiece getIoEngineName(IoEngine e) {
+  switch (e) {
+  case IoEngine::IoUring:
+    return "io_uring";
+  case IoEngine::LibAio:
+    return "libaio";
+  case IoEngine::Sync:
+    return "sync";
+  }
+  XDCHECK(false);
+  return "invalid";
+}
+
 /**
  * NavyConfig provides APIs for users to set up Navy related settings for
  * NvmCache.
@@ -478,16 +493,15 @@ class NavyConfig {
   const RandomAPConfig& randomAdmPolicy() const { return randomAPConfig_; }
 
   // ============ Device settings =============
-  bool getEnableIoUring() const { return enableIoUring_; }
-  unsigned int getNumIoThreads() const { return numIoThreads_; }
-  unsigned int getQDepthPerThread() const { return qDepthPerThread_; }
   uint64_t getBlockSize() const { return blockSize_; }
   const std::string& getFileName() const;
   const std::vector<std::string>& getRaidPaths() const;
   uint64_t getDeviceMetadataSize() const { return deviceMetadataSize_; }
   uint64_t getFileSize() const { return fileSize_; }
   bool getTruncateFile() const { return truncateFile_; }
   uint32_t getDeviceMaxWriteSize() const { return deviceMaxWriteSize_; }
+  IoEngine getIoEngine() const { return ioEngine_; }
+  unsigned int getQDepth() const { return qDepth_; }
 
   // Return a const BlockCacheConfig to read values of its parameters.
   const BigHashConfig& bigHash() const {
@@ -523,16 +537,6 @@ class NavyConfig {
   RandomAPConfig& enableRandomAdmPolicy();
 
   // ============ Device settings =============
-  // Set the number of IO threads and queue depth per thread.
-  // If set, AsyncDevice instead of Device will be created with given
-  // configurations. For now, AsyncDevice is not supported for RAID.
-  void setIoThreads(unsigned int numIoThreads, unsigned int qDepthPerThread);
-
-  // Enable io_uring engine; applicable for AsyncDevice only
-  void setEnableIoUring(bool enableIoUring) noexcept {
-    enableIoUring_ = enableIoUring;
-  }
-
   // Set the device block size, i.e., minimum unit of IO
   void setBlockSize(uint64_t blockSize) noexcept { blockSize_ = blockSize; }
   // Set the parameters for a simple file.
@@ -557,6 +561,9 @@ class NavyConfig {
     deviceMaxWriteSize_ = deviceMaxWriteSize;
   }
 
+  // Enable AsyncIo
+  void enableAsyncIo(unsigned int qDepth, bool enableIoUring);
+
   // ============ BlockCache settings =============
   // Return BlockCacheConfig for configuration.
   BlockCacheConfig& blockCache() noexcept {
@@ -627,12 +634,12 @@ class NavyConfig {
   // This is only used when in-mem buffer is enabled.
   uint32_t deviceMaxWriteSize_{};
 
-  // Number of IO threads for AsyncDevice.
-  unsigned int numIoThreads_{0};
-  // Number of queue depth per thread for AsyncDevice.
-  unsigned int qDepthPerThread_{64};
-  // Enable io_uring
-  bool enableIoUring_{true};
+  // IoEngine type used for IO
+  IoEngine ioEngine_{IoEngine::Sync};
+
+  // Number of queue depth per thread for async IO.
+  // 0 for Sync io engine and >1 for libaio and io_uring
+  unsigned int qDepth_{0};
 
   // ============ Engines settings =============
   // Currently we support one pair of engines.

cachelib/allocator/nvmcache/NavySetup.cpp

@@ -312,37 +312,28 @@ std::unique_ptr<cachelib::navy::Device> createDevice(
     std::shared_ptr<navy::DeviceEncryptor> encryptor) {
   auto blockSize = config.getBlockSize();
   auto maxDeviceWriteSize = config.getDeviceMaxWriteSize();
-
-  if (config.usesRaidFiles()) {
-    auto stripeSize = getRegionSize(config);
-    return cachelib::navy::createRAIDDevice(
-        config.getRaidPaths(),
-        alignDown(config.getFileSize(), stripeSize),
-        config.getTruncateFile(),
-        blockSize,
-        stripeSize,
-        std::move(encryptor),
-        maxDeviceWriteSize > 0 ? alignDown(maxDeviceWriteSize, blockSize) : 0);
-  } else if (config.usesSimpleFile()) {
-    if (config.getNumIoThreads() > 0) {
-      return cachelib::navy::createAsyncFileDevice(
-          config.getFileName(),
-          config.getFileSize(),
-          config.getTruncateFile(),
-          blockSize,
-          config.getNumIoThreads(),
-          config.getQDepthPerThread(),
-          std::move(encryptor),
-          maxDeviceWriteSize > 0 ? alignDown(maxDeviceWriteSize, blockSize) : 0,
-          config.getEnableIoUring());
+  if (config.usesRaidFiles() || config.usesSimpleFile()) {
+    auto stripeSize = 0;
+    auto fileSize = config.getFileSize();
+    std::vector<std::string> filePaths;
+    if (config.usesSimpleFile()) {
+      filePaths.emplace_back(config.getFileName());
+    } else {
+      stripeSize = getRegionSize(config);
+      filePaths = config.getRaidPaths();
+      fileSize = alignDown(fileSize, stripeSize);
     }
+
     return cachelib::navy::createFileDevice(
-        config.getFileName(),
-        config.getFileSize(),
+        filePaths,
+        fileSize,
         config.getTruncateFile(),
         blockSize,
-        std::move(encryptor),
-        maxDeviceWriteSize > 0 ? alignDown(maxDeviceWriteSize, blockSize) : 0);
+        stripeSize,
+        maxDeviceWriteSize > 0 ? alignDown(maxDeviceWriteSize, blockSize) : 0,
+        config.getIoEngine(),
+        config.getQDepth(),
+        std::move(encryptor));
   } else {
     return cachelib::navy::createMemoryDevice(config.getFileSize(),
                                               std::move(encryptor), blockSize);

cachelib/allocator/nvmcache/tests/NavyConfigTest.cpp

@@ -43,6 +43,8 @@ const uint64_t deviceMetadataSize = 1024 * 1024 * 1024;
 const uint64_t fileSize = 10 * 1024 * 1024;
 const bool truncateFile = false;
 const uint32_t deviceMaxWriteSize = 4 * 1024 * 1024;
+const navy::IoEngine ioEngine = navy::IoEngine::IoUring;
+const unsigned int qDepth = 64;
 
 // BlockCache settings
 const uint32_t blockCacheRegionSize = 16 * 1024 * 1024;
@@ -87,6 +89,7 @@ void setDeviceTestSettings(NavyConfig& config) {
   config.setRaidFiles(raidPaths, fileSize, truncateFile);
   config.setDeviceMetadataSize(deviceMetadataSize);
   config.setDeviceMaxWriteSize(deviceMaxWriteSize);
+  config.enableAsyncIo(qDepth, ioEngine == navy::IoEngine::IoUring);
 }
 
 void setBlockCacheTestSettings(NavyConfig& config) {
@@ -183,9 +186,8 @@ TEST(NavyConfigTest, Serialization) {
   expectedConfigMap["navyConfig::fileSize"] = "10485760";
   expectedConfigMap["navyConfig::truncateFile"] = "false";
   expectedConfigMap["navyConfig::deviceMaxWriteSize"] = "4194304";
-  expectedConfigMap["navyConfig::numIoThreads"] = "0";
-  expectedConfigMap["navyConfig::QDepthPerThread"] = "64";
-  expectedConfigMap["navyConfig::enableIoUring"] = "true";
+  expectedConfigMap["navyConfig::ioEngine"] = "io_uring";
+  expectedConfigMap["navyConfig::QDepth"] = "64";
 
   expectedConfigMap["navyConfig::blockCacheLru"] = "false";
   expectedConfigMap["navyConfig::blockCacheRegionSize"] = "16777216";
@@ -245,30 +247,47 @@ TEST(NavyConfigTest, AdmissionPolicy) {
 }
 
 TEST(NavyConfigTest, Device) {
-  NavyConfig config1{};
-  config1.setBlockSize(blockSize);
-  config1.setDeviceMetadataSize(deviceMetadataSize);
-  EXPECT_EQ(config1.getBlockSize(), blockSize);
-  EXPECT_EQ(config1.getDeviceMetadataSize(), deviceMetadataSize);
-
-  // set simple file
-  config1.setSimpleFile(fileName, fileSize, truncateFile);
-  EXPECT_EQ(config1.getFileName(), fileName);
-  EXPECT_EQ(config1.getFileSize(), fileSize);
-  EXPECT_EQ(config1.getTruncateFile(), truncateFile);
-  EXPECT_THROW(config1.setRaidFiles(raidPaths, fileSize, truncateFile),
-               std::invalid_argument);
-
-  // set RAID files
-  NavyConfig config2{};
-  EXPECT_THROW(config2.setRaidFiles(raidPathsInvalid, fileSize, truncateFile),
-               std::invalid_argument);
-  config2.setRaidFiles(raidPaths, fileSize, truncateFile);
-  EXPECT_EQ(config2.getRaidPaths(), raidPaths);
-  EXPECT_EQ(config2.getFileSize(), fileSize);
-  EXPECT_EQ(config1.getTruncateFile(), truncateFile);
-  EXPECT_THROW(config2.setSimpleFile(fileName, fileSize, truncateFile),
-               std::invalid_argument);
+  {
+    NavyConfig config{};
+    config.setBlockSize(blockSize);
+    config.setDeviceMetadataSize(deviceMetadataSize);
+    EXPECT_EQ(config.getBlockSize(), blockSize);
+    EXPECT_EQ(config.getDeviceMetadataSize(), deviceMetadataSize);
+
+    // set simple file
+    config.setSimpleFile(fileName, fileSize, truncateFile);
+    EXPECT_EQ(config.getFileName(), fileName);
+    EXPECT_EQ(config.getFileSize(), fileSize);
+    EXPECT_EQ(config.getTruncateFile(), truncateFile);
+    EXPECT_THROW(config.setRaidFiles(raidPaths, fileSize, truncateFile),
+                 std::invalid_argument);
+  }
+
+  {
+    // set RAID files
+    NavyConfig config{};
+    EXPECT_THROW(config.setRaidFiles(raidPathsInvalid, fileSize, truncateFile),
+                 std::invalid_argument);
+    config.setRaidFiles(raidPaths, fileSize, truncateFile);
+    EXPECT_EQ(config.getRaidPaths(), raidPaths);
+    EXPECT_EQ(config.getFileSize(), fileSize);
+    EXPECT_EQ(config.getTruncateFile(), truncateFile);
+    EXPECT_THROW(config.setSimpleFile(fileName, fileSize, truncateFile),
+                 std::invalid_argument);
+  }
+  {
+    // set io engines
+    NavyConfig config{};
+    EXPECT_EQ(config.getIoEngine(), navy::IoEngine::Sync);
+    EXPECT_EQ(config.getQDepth(), 0);
+    EXPECT_THROW(config.enableAsyncIo(0, false), std::invalid_argument);
+    EXPECT_THROW(config.enableAsyncIo(0, true), std::invalid_argument);
+    config.enableAsyncIo(1, false);
+    EXPECT_EQ(config.getIoEngine(), navy::IoEngine::LibAio);
+    config.enableAsyncIo(64, true);
+    EXPECT_EQ(config.getIoEngine(), navy::IoEngine::IoUring);
+    EXPECT_EQ(config.getQDepth(), 64);
+  }
 }
 
 TEST(NavyConfigTest, BlockCache) {

cachelib/allocator/nvmcache/tests/NavySetupTest.cpp

@@ -233,6 +233,7 @@ TEST(NavySetupTest, EnginesSetup) {
     // 20MB block cache 0.
     cfg.blockCache().setSize(20 * 1024 * 1024);
     navy::EnginesConfig pair1;
+    pair1.blockCache().setRegionSize(4 * 1024 * 1024);
     // Second bighash: 5 pct, 640 threshold.
     pair1.bigHash().setSizePctAndMaxItemSize(5, 640);
     cfg.addEnginePair(std::move(pair1));
@@ -252,6 +253,7 @@ TEST(NavySetupTest, EnginesSetup) {
     // 20MB block cache 0.
     cfg.blockCache().setSize(20 * 1024 * 1024);
     navy::EnginesConfig pair1;
+    pair1.blockCache().setRegionSize(4 * 1024 * 1024);
     // Second bighash: 5 pct, 640 threshold.
     pair1.bigHash().setSizePctAndMaxItemSize(5, 640);
     cfg.addEnginePair(std::move(pair1));
@@ -269,6 +271,7 @@ TEST(NavySetupTest, EnginesSetup) {
     // 20MB block cache 0.
     cfg.blockCache().setSize(20 * 1024 * 1024);
     navy::EnginesConfig pair1;
+    pair1.blockCache().setRegionSize(4 * 1024 * 1024);
     // Second bighash: 5 pct, 640 threshold.
     pair1.bigHash().setSizePctAndMaxItemSize(5, 640);
     cfg.addEnginePair(std::move(pair1));
@@ -284,6 +287,7 @@ TEST(NavySetupTest, EnginesSetup) {
     // 20MB block cache 0.
     cfg.blockCache().setSize(20 * 1024 * 1024);
     navy::EnginesConfig pair1;
+    pair1.blockCache().setRegionSize(4 * 1024 * 1024);
     // Second bighash: 5 pct, 640 threshold.
     pair1.bigHash().setSizePctAndMaxItemSize(5, 640);
     // Will throw. Set size for the second block cache.

cachelib/cachebench/cache/Cache-inl.h

@@ -212,10 +212,12 @@ Cache<Allocator>::Cache(const CacheConfig& config,
 
     nvmConfig.navyConfig.setMaxParcelMemoryMB(config_.navyParcelMemoryMB);
 
-    if (config_.navyNumIoThreads > 0) {
-      nvmConfig.navyConfig.setIoThreads(config_.navyNumIoThreads,
-                                        config_.navyQDepthPerThread);
+    // Enable async IO if qdepth is greater than 0
+    if (config_.navyQDepth > 0) {
+      nvmConfig.navyConfig.enableAsyncIo(config_.navyQDepth,
+                                         config_.navyEnableIoUring);
     }
+
     nvmConfig.navyConfig.setReaderAndWriterThreads(config_.navyReaderThreads,
                                                    config_.navyWriterThreads);
 

cachelib/cachebench/util/CacheConfig.cpp

@@ -74,8 +74,8 @@ CacheConfig::CacheConfig(const folly::dynamic& configJson) {
   JSONSetVal(configJson, navyProbabilityReinsertionThreshold);
   JSONSetVal(configJson, navyReaderThreads);
   JSONSetVal(configJson, navyWriterThreads);
-  JSONSetVal(configJson, navyNumIoThreads);
-  JSONSetVal(configJson, navyQDepthPerThread);
+  JSONSetVal(configJson, navyQDepth);
+  JSONSetVal(configJson, navyEnableIoUring);
   JSONSetVal(configJson, navyCleanRegions);
   JSONSetVal(configJson, navyAdmissionWriteRateMB);
   JSONSetVal(configJson, navyMaxConcurrentInserts);

cachelib/cachebench/util/CacheConfig.h

@@ -184,9 +184,10 @@ struct CacheConfig : public JSONConfig {
   // number of asynchronous worker thread for navy write operation,
   uint32_t navyWriterThreads{32};
 
-  uint32_t navyNumIoThreads{0};
-
-  uint32_t navyQDepthPerThread{1024};
+  // QDepth >= 1 for async IO
+  uint32_t navyQDepth{0};
+  // Use either io_uring or libaio for async IO
+  bool navyEnableIoUring{true};
 
   // buffer of clean regions to be maintained free to ensure writes
   // into navy don't queue behind a reclaim of region.