Add a test script to generate many Protobuf configuration files. (#16)

* run_test_batch script attempt for 1

* add options & fixes

* add options for test script

* fix lint

* fix lint isort

Author: vvbae

options.cpp

@@ -16,6 +16,40 @@ const char *RtSmithOptions::getIncludePath() {
 }
 
 RtSmithOptions::RtSmithOptions()
-    : AbstractP4cToolOptions("Remove control-plane dead code from a P4 program.") {}
+    : AbstractP4cToolOptions("Remove control-plane dead code from a P4 program.") {
+    registerOption(
+        "--print-to-stdout", nullptr,
+        [this](const char *) {
+            printToStdout_ = true;
+            return true;
+        },
+        "Whether to write the generated config to a file or to stdout.");
+
+    registerOption(
+        "--output-dir", "outputDir",
+        [this](const char *arg) {
+            outputDir_ = std::filesystem::path(arg);
+            return true;
+        },
+        "The path to the output file of the config file(s).");
+
+    registerOption(
+        "--generate-config", "filePath",
+        [this](const char *arg) {
+            configFilePath = arg;
+            if (configFilePath.value().extension() != ".txtpb") {
+                ::error("%1% must have a .txtpb extension.", configFilePath.value().c_str());
+                return false;
+            }
+            return true;
+        },
+        "Write the generated config to the specified .txtpb file.");
+}
+
+std::optional<std::filesystem::path> RtSmithOptions::getOutputDir() const { return outputDir_; }
+
+bool RtSmithOptions::printToStdout() const { return printToStdout_; }
+
+std::optional<std::string> RtSmithOptions::getConfigFilePath() const { return configFilePath; }
 
 }  // namespace P4Tools

options.h

@@ -1,6 +1,9 @@
 #ifndef BACKENDS_P4TOOLS_MODULES_P4RTSMITH_OPTIONS_H_
 #define BACKENDS_P4TOOLS_MODULES_P4RTSMITH_OPTIONS_H_
 
+#include <filesystem>
+#include <optional>
+
 #include "backends/p4tools/common/options.h"
 
 namespace P4Tools {
@@ -23,8 +26,26 @@ class RtSmithOptions : public AbstractP4cToolOptions {
 
     const char *getIncludePath() override;
 
+    /// @returns true when the --print-to-stdout option has been set.
+    [[nodiscard]] bool printToStdout() const;
+
+    /// @returns the path set with --output-dir.
+    [[nodiscard]] std::optional<std::filesystem::path> getOutputDir() const;
+
+    /// @returns the path set with --generate-config.
+    [[nodiscard]] std::optional<std::string> getConfigFilePath() const;
+
  private:
     RtSmithOptions();
+
+    // Write the generated config to the specified file.
+    std::optional<std::filesystem::path> configFilePath = std::nullopt;
+
+    /// The path to the output file of the config file.
+    std::optional<std::filesystem::path> outputDir_ = std::nullopt;
+
+    /// Whether to write the generated config to a file or to stdout.
+    bool printToStdout_ = false;
 };
 
 }  // namespace P4Tools

rtsmith.cpp

@@ -3,13 +3,16 @@
 #include <google/protobuf/text_format.h>
 
 #include <cstdlib>
+#include <filesystem>
 
 #include "backends/p4tools/common/compiler/context.h"
 #include "backends/p4tools/common/lib/logging.h"
 #include "backends/p4tools/modules/p4rtsmith/core/target.h"
 #include "backends/p4tools/modules/p4rtsmith/core/util.h"
 #include "backends/p4tools/modules/p4rtsmith/register.h"
+#include "control-plane/p4RuntimeSerializer.h"
 #include "lib/error.h"
+#include "lib/nullstream.h"
 
 namespace P4Tools::RTSmith {
 
@@ -40,17 +43,59 @@ int RtSmith::mainImpl(const CompilerResult &compilerResult) {
     printInfo("Inferred API:\n%1%", p4RuntimeApi.p4Info->DebugString());
 
     auto &fuzzer = RtSmithTarget::getFuzzer(*programInfo);
+    const auto &rtsmithOptions = RtSmithOptions::get();
 
     auto initialConfig = fuzzer.produceInitialConfig();
-    printInfo("Generated initial configuration:");
-    for (const auto &writeRequest : initialConfig) {
-        printInfo("%1%", writeRequest.DebugString());
+    auto timeSeriesUpdates = fuzzer.produceUpdateTimeSeries();
+
+    if (rtsmithOptions.getConfigFilePath().has_value() &&
+        rtsmithOptions.getOutputDir().has_value()) {
+        auto dirPath = rtsmithOptions.getOutputDir().value();
+        auto fileName = rtsmithOptions.getConfigFilePath().value();
+
+        if (!(std::filesystem::exists(dirPath) && std::filesystem::is_directory(dirPath))) {
+            if (!std::filesystem::create_directory(dirPath)) {
+                ::error("P4RuntimeSmith: Failed to create output directory. Exiting");
+                return EXIT_FAILURE;
+            }
+        }
+
+        auto fullFilePath = (dirPath / fileName).generic_string();
+        auto *outputFile = openFile(fullFilePath, true);
+        if (outputFile == nullptr) {
+            ::error("P4RuntimeSmith: Config file path doesn't exist. Exiting");
+            return EXIT_FAILURE;
+        }
+
+        for (const auto &writeRequest : initialConfig) {
+            std::string output;
+            google::protobuf::TextFormat::Printer textPrinter;
+            textPrinter.SetExpandAny(true);
+            if (!textPrinter.PrintToString(writeRequest, &output)) {
+                ::error(ErrorType::ERR_IO, "Failed to serialize protobuf message to text");
+                return false;
+            }
+
+            *outputFile << output;
+            if (!outputFile->good()) {
+                ::error(ErrorType::ERR_IO, "Failed to write text protobuf message to the output");
+                return false;
+            }
+
+            outputFile->flush();
+        }
     }
 
-    auto timeSeriesUpdates = fuzzer.produceUpdateTimeSeries();
-    printInfo("Time series updates:");
-    for (const auto &[time, writeRequest] : timeSeriesUpdates) {
-        printInfo("Time %1%:\n%2%", writeRequest.DebugString());
+    if (rtsmithOptions.printToStdout()) {
+        printInfo("Generated initial configuration:");
+        for (const auto &writeRequest : initialConfig) {
+            printInfo("%1%", writeRequest.DebugString());
+        }
+
+        printInfo("Time series updates:");
+        for (const auto &[time, writeRequest] : timeSeriesUpdates) {
+            printInfo("Time %1%:\n%2%", writeRequest.DebugString());
+        }
     }
 
     return ::errorCount() == 0 ? EXIT_SUCCESS : EXIT_FAILURE;

targets/bmv2/test/run_test_batch.py

@@ -0,0 +1,123 @@
+import argparse
+import logging
+import os
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+from typing import Any, Optional
+
+PARSER = argparse.ArgumentParser()
+PARSER.add_argument(
+    "-td",
+    "--testdir",
+    dest="testdir",
+    help="The location of the test directory.",
+)
+PARSER.add_argument("p4_file", help="The absolute path for the p4 file to process.")
+PARSER.add_argument(
+    "p4rtsmith",
+    help="The absolute path for the p4rtsmith bin file for this test.",
+)
+PARSER.add_argument(
+    "-s",
+    "--seed",
+    default=1,
+    dest="seed",
+    help="Random seed for generating configs.",
+)
+
+
+# Parse options and process argv
+ARGS, ARGV = PARSER.parse_known_args()
+
+# Append the root directory to the import path.
+FILE_DIR = Path(__file__).resolve().parent
+
+
+class Options:
+    """Options for this testing script. Usually correspond to command line inputs."""
+
+    # File that is being compiled.
+    p4_file: Path = Path(".")
+    # The path for the p4rtsmith bin file for this test.
+    p4rtsmith: Path = Path(".")
+    # Actual location of the test framework.
+    testdir: Path = Path(".")
+    # Random seed for generating configs.
+    seed: int = 1
+
+
+def generate_config(p4rtsmith_path, seed, p4_program_path, testdir, config_file_path):
+    command = f"{p4rtsmith_path} --target bmv2 --arch v1model --seed {seed} --output-dir {testdir} --generate-config {config_file_path} {p4_program_path}"
+    subprocess.run(command, shell=True)
+
+
+def run_test(run_test_script, p4_program_path, config_file_path):
+    command = f"sudo -E {run_test_script} .. {p4_program_path} -tf {config_file_path}"
+    subprocess.run(command, shell=True)
+
+
+def find_p4c_dir():
+    current_dir = Path(__file__).resolve()
+
+    while True:
+        p4c_build_dir = current_dir / "p4c"
+        if p4c_build_dir.exists():
+            return p4c_build_dir
+
+        current_dir = current_dir.parent
+
+        if current_dir == current_dir.parent:
+            raise RuntimeError("p4c/build directory not found.")
+
+
+def run_tests(options: Options) -> int:
+    config_file_path = "initial_config.txtpb"
+
+    seed = options.seed
+    testdir = options.testdir
+    p4rtsmith_path = options.p4rtsmith
+    run_test_script = FILE_DIR / "run-bmv2-proto-test.py"
+    p4_program_path = options.p4_file
+
+    generate_config(p4rtsmith_path, seed, p4_program_path, testdir, config_file_path)
+    run_test(run_test_script, p4_program_path, testdir / config_file_path)
+
+
+def create_options(test_args: Any) -> Optional[Options]:
+    """Parse the input arguments and create a processed options object."""
+    options = Options()
+    options.p4_file = Path(test_args.p4_file).absolute()
+    options.p4rtsmith = Path(test_args.p4rtsmith).absolute()
+
+    p4c_dir = find_p4c_dir()
+    p4c_build_dir = p4c_dir / "build"
+    os.chdir(p4c_build_dir)
+
+    testdir = test_args.testdir
+    if not testdir:
+        testdir = tempfile.mkdtemp(dir=Path(".").absolute())
+        os.chmod(testdir, 0o755)
+    options.testdir = Path(testdir)
+    options.seed = test_args.seed
+
+    # Configure logging.
+    logging.basicConfig(
+        filename=options.testdir.joinpath("test.log"),
+        format="%(levelname)s: %(message)s",
+        level=getattr(logging, "WARNING"),
+        filemode="w",
+    )
+    stderr_log = logging.StreamHandler()
+    stderr_log.setFormatter(logging.Formatter("%(levelname)s: %(message)s"))
+    logging.getLogger().addHandler(stderr_log)
+    return options
+
+
+if __name__ == "__main__":
+    test_options = create_options(ARGS)
+    if not test_options:
+        sys.exit()
+
+    run_tests(test_options)