feat: divide tests into error/regression regions

Samat Gaynutdinov · antipeon · commit 84ea26369a97 · 2022-02-07T10:45:37.000+03:00
diff --git a/server/src/Tests.cpp b/server/src/Tests.cpp
@@ -15,15 +15,23 @@ using namespace types;
 static const string INT64_MIN_STRING =
     std::to_string(std::numeric_limits<int64_t>::min());
 
-const string Tests::DEFAULT_SCOPE_NAME = "regression";
-const string Tests::ERROR_SCOPE_NAME = "error";
+const string Tests::DEFAULT_SUITE_NAME = "regression";
+const string Tests::ERROR_SUITE_NAME = "error";
 
 const Tests::MethodParam &tests::Tests::getStdinMethodParam() {
     static const Tests::MethodParam stdinMethodParam =
         MethodParam(types::Type::CStringType(), types::Type::getStdinParamName(), std::nullopt);
     return stdinMethodParam;
 }
 
+Tests::MethodDescription::MethodDescription() : suiteTestCases{
+    {Tests::DEFAULT_SUITE_NAME, std::vector<MethodTestCase>()},
+    {Tests::ERROR_SUITE_NAME, std::vector<MethodTestCase>()}
+}, codeText{
+    {Tests::DEFAULT_SUITE_NAME, std::string()},
+    {Tests::ERROR_SUITE_NAME, std::string()}
+} {}
+
 static string makeDecimalConstant(string value, const string &typeName) {
     if (typeName == "long") {
         if (value == INT64_MIN_STRING) {
@@ -516,13 +524,13 @@ void KTestObjectParser::parseKTest(const MethodKtests &batch,
     }
 }
 
-static string getScopeName(const UTBotKTest::Status &status,
+static string getSuiteName(const UTBotKTest::Status &status,
                            const shared_ptr<LineInfo> lineInfo) {
     bool forAssert = lineInfo != nullptr && lineInfo->forAssert;
     if (status == UTBotKTest::Status::FAILED || forAssert) {
-        return Tests::ERROR_SCOPE_NAME;
+        return Tests::ERROR_SUITE_NAME;
     }
-    return Tests::DEFAULT_SCOPE_NAME;
+    return Tests::DEFAULT_SUITE_NAME;
 }
 
 int KTestObjectParser::findFieldIndex(const StructInfo &structInfo, unsigned int offset) {
@@ -671,8 +679,8 @@ void KTestObjectParser::parseTestCases(const UTBotKTestList &cases,
     for (const auto &case_ : cases) {
         std::stringstream traceStream;
         traceStream << "Test case #" << (++caseCounter) << ":\n";
-        string scopeName = getScopeName(case_.status, lineInfo);
-        Tests::MethodTestCase testCase{ scopeName };
+        string suiteName = getSuiteName(case_.status, lineInfo);
+        Tests::MethodTestCase testCase{ suiteName };
         vector<Tests::TestCaseParamValue> paramValues;
 
         Tests::TestCaseDescription testCaseDescription;
@@ -734,6 +742,7 @@ void KTestObjectParser::parseTestCases(const UTBotKTestList &cases,
         assignTypeStubVar(testCase, methodDescription);
 
         methodDescription.testCases.push_back(testCase);
+        methodDescription.suiteTestCases[testCase.suiteName].push_back(testCase);
     }
 }
 
@@ -1124,6 +1133,6 @@ bool isUnnamed(char *name) {
 }
 
 bool Tests::MethodTestCase::isError() const {
-    return scopeName == ERROR_SCOPE_NAME;
+    return suiteName == ERROR_SUITE_NAME;
 }
 }
diff --git a/server/src/Tests.h b/server/src/Tests.h
@@ -338,7 +338,7 @@ namespace tests {
         };
 
         struct TestCaseDescription {
-            string scopeName;
+            string suiteName;
 
             vector<TestCaseParamValue> globalPreValues;
             vector<TestCaseParamValue> globalPostValues;
@@ -361,7 +361,7 @@ namespace tests {
         };
 
         struct MethodTestCase {
-            string scopeName;
+            string suiteName;
 
             vector<TestCaseParamValue> globalPreValues;
             vector<TestCaseParamValue> globalPostValues;
@@ -396,7 +396,8 @@ namespace tests {
         struct MethodDescription {
             std::optional<MethodParam> classObj;
             std::string name;
-            std::string code;
+            typedef std::unordered_map<string, string> SuiteNameToCodeTextMap;
+            SuiteNameToCodeTextMap codeText;
             std::string paramsString;
 
             types::Type returnType;
@@ -411,9 +412,13 @@ namespace tests {
             typedef std::unordered_map<string, std::shared_ptr<types::FunctionInfo>> FPointerMap;
             FPointerMap functionPointers;
             vector<MethodTestCase> testCases;
+            typedef std::unordered_map<string, vector<MethodTestCase>> SuiteNameToTestCasesMap;
+            SuiteNameToTestCasesMap suiteTestCases;
 
             bool operator==(const MethodDescription &other) const;
 
+            MethodDescription();
+
             [[nodiscard]] vector<types::Type> getParamTypes() const {
                 return CollectionUtils::transform(params, [](auto const& param) {
                     return param.type;
@@ -484,8 +489,8 @@ namespace tests {
 
         using MethodsMap = tsl::ordered_map<std::string, MethodDescription>;
 
-        static const string DEFAULT_SCOPE_NAME;
-        static const string ERROR_SCOPE_NAME;
+        static const string DEFAULT_SUITE_NAME;
+        static const string ERROR_SUITE_NAME;
         static const MethodParam &getStdinMethodParam();
 
         fs::path sourceFilePath;
diff --git a/server/src/printers/TestsPrinter.cpp b/server/src/printers/TestsPrinter.cpp
@@ -55,14 +55,26 @@ void TestsPrinter::joinToFinalCode(Tests &tests, const fs::path& generatedHeader
     }
     writeStubsForStructureFields(tests);
     ss << NL;
-    for (const auto &[methodName, methodStub] : tests.methods) {
-        if (methodStub.code.empty()) {
+    printSuite(Tests::DEFAULT_SUITE_NAME, tests.methods);
+    printSuite(Tests::ERROR_SUITE_NAME, tests.methods);
+    ss << RB();
+    tests.code = ss.str();
+}
+
+void TestsPrinter::printSuite(const string &suiteName, const Tests::MethodsMap &methods) {
+    if (std::all_of(methods.begin(), methods.end(), [&suiteName](const auto& method) {
+        return method.second.codeText.at(suiteName).empty();
+    })) {
+        return;
+    }
+    ss << "#pragma region " << suiteName << NL;
+    for (const auto &[methodName, methodStub] : methods) {
+        if (methodStub.codeText.at(suiteName).empty()) {
             continue;
         }
-        ss << methodStub.code;
+        ss << methodStub.codeText.at(suiteName);
     }
-    ss << RB();
-    tests.code = ss.str();
+    ss << "#pragma endregion" << NL;
 }
 
 void TestsPrinter::genCode(Tests::MethodDescription &methodDescription,
@@ -78,17 +90,42 @@ void TestsPrinter::genCode(Tests::MethodDescription &methodDescription,
 
     writeStubsForFunctionParams(typesHandler, methodDescription, false);
     writeExternForSymbolicStubs(methodDescription);
-    for (auto &testCase : methodDescription.testCases) {
+
+    genCodeBySuiteName(Tests::DEFAULT_SUITE_NAME,
+                       methodDescription,
+                       predicateInfo,
+                       verbose,
+                       testNum);
+    resetStream();
+    genCodeBySuiteName(Tests::ERROR_SUITE_NAME,
+                       methodDescription,
+                       predicateInfo,
+                       verbose,
+                       testNum);
+    resetStream();
+}
+
+void TestsPrinter::genCodeBySuiteName(const string &targetSuiteName,
+                                      Tests::MethodDescription &methodDescription,
+                                      const std::optional<LineInfo::PredicateInfo>& predicateInfo,
+                                      bool verbose,
+                                      int& testNum) {
+    const auto& testCases = methodDescription.suiteTestCases[targetSuiteName];
+    if (testCases.empty()) {
+        return;
+    }
+    for (auto &testCase : testCases) {
         testNum++;
-        testHeader(testCase.scopeName, methodDescription, testNum);
+        testHeader(testCase.suiteName, methodDescription, testNum);
         redirectStdin(methodDescription, testCase, verbose);
         if (verbose) {
             genVerboseTestCase(methodDescription, testCase, predicateInfo);
         } else {
             genParametrizedTestCase(methodDescription, testCase, predicateInfo);
         }
     }
-    methodDescription.code = ss.str();
+
+    methodDescription.codeText[targetSuiteName] = ss.str();
 }
 
 void TestsPrinter::genVerboseTestCase(const Tests::MethodDescription &methodDescription,
diff --git a/server/src/printers/TestsPrinter.h b/server/src/printers/TestsPrinter.h
@@ -146,6 +146,14 @@ namespace printer {
                                 const Tests::MethodTestCase &testCase);
 
         static Tests::MethodParam getValueParam(const Tests::MethodParam &param);
+
+        void genCodeBySuiteName(const string &targetSuiteName,
+                                Tests::MethodDescription &methodDescription,
+                                const std::optional<LineInfo::PredicateInfo>& predicateInfo,
+                                bool verbose,
+                                int& testNum);
+
+        void printSuite(const string &suiteName, const Tests::MethodsMap &methods);
     };
 }
 #endif // UNITTESTBOT_TESTSPRINTER_H
diff --git a/server/test/framework/Server_Tests.cpp b/server/test/framework/Server_Tests.cpp
@@ -828,6 +828,16 @@ namespace {
         checkFloatingPointPlain_C(testGen);
     }
 
+    TEST_F(Server_Test, Correct_CodeText_For_Regression) {
+        auto [testGen, status] = performFeatureFileTestsRequest(floating_point_plain_c);
+        const string code = testGen.tests.begin()->second.code;
+        const string beginRegressionRegion = "#pragma region " + Tests::DEFAULT_SUITE_NAME + NL;
+        const string endRegion = std::string("#pragma endregion") + NL;
+        const string beginErrorRegion = "#pragma region " + Tests::ERROR_SUITE_NAME + NL;
+        ASSERT_TRUE(code.find(beginRegressionRegion) != std::string::npos) << "No regression begin region";
+        ASSERT_TRUE(code.find(endRegion) != std::string::npos) << "No regression end region";
+    }
+
     TEST_F(Server_Test, Linkage) {
         auto [testGen, status] = performFeatureFileTestsRequest(linkage_c);
         ASSERT_TRUE(status.ok()) << status.error_message();
diff --git a/server/test/framework/Syntax_Tests.cpp b/server/test/framework/Syntax_Tests.cpp
@@ -1295,6 +1295,17 @@ namespace {
         );
     }
 
+    TEST_F(Syntax_Test, Correct_CodeText_For_Regression_And_Error) {
+        auto [testGen, status] = createTestForFunction(linked_list_c, 3);
+        const string code = testGen.tests.begin()->second.code;
+        const string beginRegressionRegion = "#pragma region " + Tests::DEFAULT_SUITE_NAME + NL;
+        const string endRegion = std::string("#pragma endregion") + NL;
+        const string beginErrorRegion = "#pragma region " + Tests::ERROR_SUITE_NAME + NL;
+        ASSERT_TRUE(code.find(beginRegressionRegion) != std::string::npos) << "No regression begin region";
+        ASSERT_TRUE(code.find(endRegion) != std::string::npos) << "No regression end region";
+        ASSERT_TRUE(code.find(beginErrorRegion) != std::string::npos) << "No error begin region";
+    }
+
     TEST_F(Syntax_Test, Function_Pointers_StructFieldParam) {
         auto [testGen, status] = createTestForFunction(functions_as_params_c, 52);
 
