AsyncDevice: add support for libaio

Summary:
This change adds the support for libaio. As such, user can select which aio implementation to use
via a configuration option NavyConfig::enableIoUring_ between io_uring and libaio.

Reviewed By: therealgymmy

Differential Revision: D47306900

fbshipit-source-id: f09555213a3d39e8e666486f30b2e87f0c093188

Author: Jaesoo Lee

cachelib/allocator/nvmcache/NavyConfig.cpp

@@ -232,6 +232,7 @@ std::map<std::string, std::string> NavyConfig::serialize() const {
   configMap["navyConfig::numIoThreads"] = folly::to<std::string>(numIoThreads_);
   configMap["navyConfig::QDepthPerThread"] =
       folly::to<std::string>(qDepthPerThread_);
+  configMap["navyConfig::enableIoUring"] = enableIoUring_ ? "true" : "false";
   // Other settings
   configMap["navyConfig::maxConcurrentInserts"] =
       folly::to<std::string>(maxConcurrentInserts_);

cachelib/allocator/nvmcache/NavyConfig.h

@@ -478,6 +478,7 @@ class NavyConfig {
   const RandomAPConfig& randomAdmPolicy() const { return randomAPConfig_; }
 
   // ============ Device settings =============
+  bool getEnableIoUring() const { return enableIoUring_; }
   unsigned int getNumIoThreads() const { return numIoThreads_; }
   unsigned int getQDepthPerThread() const { return qDepthPerThread_; }
   uint64_t getBlockSize() const { return blockSize_; }
@@ -527,6 +528,11 @@ class NavyConfig {
   // configurations. For now, AsyncDevice is not supported for RAID.
   void setIoThreads(unsigned int numIoThreads, unsigned int qDepthPerThread);
 
+  // Enable io_uring engine; applicable for AsyncDevice only
+  void setEnableIoUring(bool enableIoUring) noexcept {
+    enableIoUring_ = enableIoUring;
+  }
+
   // Set the device block size, i.e., minimum unit of IO
   void setBlockSize(uint64_t blockSize) noexcept { blockSize_ = blockSize; }
   // Set the parameters for a simple file.
@@ -621,6 +627,13 @@ class NavyConfig {
   // This is only used when in-mem buffer is enabled.
   uint32_t deviceMaxWriteSize_{};
 
+  // Number of IO threads for AsyncDevice.
+  unsigned int numIoThreads_{0};
+  // Number of queue depth per thread for AsyncDevice.
+  unsigned int qDepthPerThread_{64};
+  // Enable io_uring
+  bool enableIoUring_{true};
+
   // ============ Engines settings =============
   // Currently we support one pair of engines.
   std::vector<EnginesConfig> enginesConfigs_{1};
@@ -637,10 +650,6 @@ class NavyConfig {
   // This value needs to be non-zero.
   uint64_t navyReqOrderingShards_{20};
 
-  // Number of IO threads for AsyncDevice.
-  unsigned int numIoThreads_{0};
-  // Number of queue depth per thread for AsyncDevice.
-  unsigned int qDepthPerThread_{64};
   // ============ Other settings =============
   // Maximum number of concurrent inserts we allow globally for Navy.
   // 0 means unlimited.

cachelib/allocator/nvmcache/NavySetup.cpp

@@ -333,8 +333,8 @@ std::unique_ptr<cachelib::navy::Device> createDevice(
           config.getNumIoThreads(),
           config.getQDepthPerThread(),
           std::move(encryptor),
-          maxDeviceWriteSize > 0 ? alignDown(maxDeviceWriteSize, blockSize)
-                                 : 0);
+          maxDeviceWriteSize > 0 ? alignDown(maxDeviceWriteSize, blockSize) : 0,
+          config.getEnableIoUring());
     }
     return cachelib::navy::createFileDevice(
         config.getFileName(),

cachelib/allocator/nvmcache/tests/NavyConfigTest.cpp

@@ -185,6 +185,7 @@ TEST(NavyConfigTest, Serialization) {
   expectedConfigMap["navyConfig::deviceMaxWriteSize"] = "4194304";
   expectedConfigMap["navyConfig::numIoThreads"] = "0";
   expectedConfigMap["navyConfig::QDepthPerThread"] = "64";
+  expectedConfigMap["navyConfig::enableIoUring"] = "true";
 
   expectedConfigMap["navyConfig::blockCacheLru"] = "false";
   expectedConfigMap["navyConfig::blockCacheRegionSize"] = "16777216";

cachelib/navy/Factory.cpp

@@ -503,17 +503,18 @@ std::unique_ptr<Device> createAsyncFileDevice(
     uint32_t numIoThreads,
     uint32_t qDepthPerThread,
     std::shared_ptr<DeviceEncryptor> encryptor,
-    uint32_t maxDeviceWriteSize) {
+    uint32_t maxDeviceWriteSize,
+    bool enableIoUring) {
   folly::File f;
   try {
     f = openCacheFile(fileName, singleFileSize, truncateFile);
   } catch (const std::exception& e) {
     XLOG(ERR) << "Exception in openCacheFile: " << e.what();
     throw;
   }
-  return createAsyncIoFileDevice(std::move(f), singleFileSize, blockSize,
-                                 numIoThreads, qDepthPerThread,
-                                 std::move(encryptor), maxDeviceWriteSize);
+  return createAsyncIoFileDevice(
+      std::move(f), singleFileSize, blockSize, numIoThreads, qDepthPerThread,
+      std::move(encryptor), maxDeviceWriteSize, enableIoUring);
 }
 
 } // namespace navy

cachelib/navy/Factory.h

@@ -247,7 +247,8 @@ std::unique_ptr<Device> createAsyncFileDevice(
     uint32_t numIoThreads,
     uint32_t qDepthPerThread,
     std::shared_ptr<DeviceEncryptor> encryptor,
-    uint32_t maxDeviceWriteSize);
+    uint32_t maxDeviceWriteSize,
+    bool enableIoUring);
 
 } // namespace navy
 } // namespace cachelib