adding support for multiple trace files in test_config

Summary:
This change adds the support for replaying the multiple trace files. The user can now specify
multiple trace files to be replayed as an array 'traceFileNames' for the 'test_config'. Previous
'traceFileName' is also supported for backward compatibility, but only one of them can be specified.

This change also adds the 'ignoreOpCount' option (default is false) which allows to ignore the
op_count column and replay each operation only once.

This change also performs several refactorings; (1) made the logic reading trace files reusable
(i.e., new TraceFileStream class) and (2) deprecated AmplifiedReplayGenerator and made the
ReplayGenerator supports both single and multi-threaded trace replays.

Finally, this change implements a new unit test to test the parse logic.

Reviewed By: jcipar

Differential Revision: D41173006

fbshipit-source-id: 7630ad6dee6cbda2702b98cddb860ebf6d01b1fa

Author: Jaesoo Lee

cachelib/cachebench/runner/Stressor.cpp

@@ -21,7 +21,6 @@
 #include "cachelib/cachebench/runner/CacheStressor.h"
 #include "cachelib/cachebench/runner/FastShutdown.h"
 #include "cachelib/cachebench/runner/IntegrationStressor.h"
-#include "cachelib/cachebench/workload/AmplifiedReplayGenerator.h"
 #include "cachelib/cachebench/workload/OnlineGenerator.h"
 #include "cachelib/cachebench/workload/PieceWiseReplayGenerator.h"
 #include "cachelib/cachebench/workload/ReplayGenerator.h"
@@ -142,11 +141,7 @@ std::unique_ptr<GeneratorBase> makeGenerator(const StressorConfig& config) {
   if (config.generator == "piecewise-replay") {
     return std::make_unique<PieceWiseReplayGenerator>(config);
   } else if (config.generator == "replay") {
-    if (config.numThreads > 1) {
-      return std::make_unique<AmplifiedReplayGenerator>(config);
-    } else {
-      return std::make_unique<ReplayGenerator>(config);
-    }
+    return std::make_unique<ReplayGenerator>(config);
   } else if (config.generator.empty() || config.generator == "workload") {
     // TODO: Remove the empty() check once we label workload-based configs
     // properly

cachelib/cachebench/util/Config.cpp

@@ -31,6 +31,7 @@ StressorConfig::StressorConfig(const folly::dynamic& configJson) {
 
   JSONSetVal(configJson, enableLookaside);
   JSONSetVal(configJson, onlySetIfMiss);
+  JSONSetVal(configJson, ignoreOpCount);
   JSONSetVal(configJson, populateItem);
   JSONSetVal(configJson, samplingIntervalMs);
 
@@ -53,6 +54,7 @@ StressorConfig::StressorConfig(const folly::dynamic& configJson) {
   JSONSetVal(configJson, maxInconsistencyCount);
 
   JSONSetVal(configJson, traceFileName);
+  JSONSetVal(configJson, traceFileNames);
   JSONSetVal(configJson, configPath);
 
   JSONSetVal(configJson, cachePieceSize);
@@ -78,10 +80,15 @@ StressorConfig::StressorConfig(const folly::dynamic& configJson) {
         ReplayGeneratorConfig{configJson["replayGeneratorConfig"]};
   }
 
+  if (!traceFileName.empty() && !traceFileNames.empty()) {
+    throw std::invalid_argument(
+        folly::sformat("set only one of traceFileName or traceFileNames"));
+  }
+
   // If you added new fields to the configuration, update the JSONSetVal
   // to make them available for the json configs and increment the size
   // below
-  checkCorrectSize<StressorConfig, 464>();
+  checkCorrectSize<StressorConfig, 488>();
 }
 
 bool StressorConfig::usesChainedItems() const {

cachelib/cachebench/util/Config.h

@@ -181,6 +181,9 @@ struct StressorConfig : public JSONConfig {
   // follow get misses with a set
   bool enableLookaside{false};
 
+  // ignore opCount and does not repeat operations
+  bool ignoreOpCount{false};
+
   // only set a key in the cache if the key already doesn't exist
   // this is useful for replaying traces with both get and set, and also
   // for manually configured synthetic workloads.
@@ -233,6 +236,7 @@ struct StressorConfig : public JSONConfig {
   // Supported formats include specifying an absolute filename and filename
   // relative to the configPath
   std::string traceFileName{};
+  std::vector<std::string> traceFileNames{};
 
   // location of the path for the files referenced inside the json. If not
   // specified, it defaults to the path of the json file being parsed.

cachelib/cachebench/workload/AmplifiedReplayGenerator.h

@@ -80,14 +80,9 @@ void PieceWiseReplayGenerator::getReqFromTrace() {
   auto totalFieldCount =
       partialFieldCount + config_.replayGeneratorConfig.numExtraFields;
   while (true) {
-    if (!std::getline(infile_, line)) {
-      if (repeatTraceReplay_) {
-        XLOG_EVERY_MS(
-            INFO, 100'000,
-            "Reached the end of trace file. Restarting from beginning.");
-        resetTraceFileToBeginning();
-        continue;
-      }
+    try {
+      traceStream_.getline(line);
+    } catch (const cachelib::cachebench::EndOfTrace& e) {
       isEndOfFile_.store(true, std::memory_order_relaxed);
       break;
     }

cachelib/cachebench/workload/PieceWiseReplayGenerator.cpp

@@ -32,6 +32,7 @@ class PieceWiseReplayGenerator : public ReplayGeneratorBase {
  public:
   explicit PieceWiseReplayGenerator(const StressorConfig& config)
       : ReplayGeneratorBase(config),
+        traceStream_(config, 0),
         pieceCacheAdapter_(config.maxCachePieces,
                            config.replayGeneratorConfig.numAggregationFields,
                            config.replayGeneratorConfig.statsPerAggField),
@@ -137,6 +138,8 @@ class PieceWiseReplayGenerator : public ReplayGeneratorBase {
     TOTAL_DEFINED_FIELDS = 11
   };
 
+  TraceFileStream traceStream_;
+
   PieceWiseCacheAdapter pieceCacheAdapter_;
 
   const ReplayGeneratorConfig::SerializeMode mode_{