MQE-2494: Missing steps in Allure logs for certain test failures

jilu1 · jilu1 · commit f80e2db7078f · 2021-02-08T12:15:41.000-06:00
diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Util/Sorter/ParallelGroupSorterTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Util/Sorter/ParallelGroupSorterTest.php
@@ -101,8 +101,8 @@ public function testSortWithSuites()
         $this->assertCount(5, $actualResult);
 
         $expectedResults =  [
-            1 => ['mockSuite1_0'],
-            2 => ['mockSuite1_1'],
+            1 => ['mockSuite1_0_G'],
+            2 => ['mockSuite1_1_G'],
             3 => ['test3'],
             4 => ['test2','test5', 'test4'],
             5 => ['test1'],
diff --git a/dev/tests/verification/Tests/SuiteGenerationTest.php b/dev/tests/verification/Tests/SuiteGenerationTest.php
@@ -118,10 +118,10 @@ public function testSuiteGenerationParallel()
         $groupName = 'functionalSuite1';
 
         $expectedGroups = [
-            'functionalSuite1_0',
-            'functionalSuite1_1',
-            'functionalSuite1_2',
-            'functionalSuite1_3'
+            'functionalSuite1_0_G',
+            'functionalSuite1_1_G',
+            'functionalSuite1_2_G',
+            'functionalSuite1_3_G'
         ];
 
         $expectedContents = [
diff --git a/src/Magento/FunctionalTestingFramework/Allure/Adapter/MagentoAllureAdapter.php b/src/Magento/FunctionalTestingFramework/Allure/Adapter/MagentoAllureAdapter.php
@@ -5,23 +5,30 @@
  */
 namespace Magento\FunctionalTestingFramework\Allure\Adapter;
 
+use Codeception\Codecept;
+use Codeception\Test\Cest;
 use Codeception\Step\Comment;
 use Magento\FunctionalTestingFramework\Suite\Handlers\SuiteObjectHandler;
 use Magento\FunctionalTestingFramework\Test\Objects\ActionGroupObject;
 use Magento\FunctionalTestingFramework\Test\Objects\ActionObject;
 use Magento\FunctionalTestingFramework\Util\TestGenerator;
+use Yandex\Allure\Adapter\Model\Failure;
+use Yandex\Allure\Adapter\Model\Provider;
 use Yandex\Allure\Adapter\Model\Status;
 use Yandex\Allure\Adapter\Model\Step;
+use Yandex\Allure\Adapter\Allure;
 use Yandex\Allure\Codeception\AllureCodeception;
 use Yandex\Allure\Adapter\Event\StepStartedEvent;
 use Yandex\Allure\Adapter\Event\StepFinishedEvent;
 use Yandex\Allure\Adapter\Event\StepFailedEvent;
 use Yandex\Allure\Adapter\Event\TestCaseFailedEvent;
 use Yandex\Allure\Adapter\Event\TestCaseFinishedEvent;
 use Yandex\Allure\Adapter\Event\TestCaseBrokenEvent;
+use Yandex\Allure\Adapter\Event\AddAttachmentEvent;
 use Codeception\Event\FailEvent;
 use Codeception\Event\SuiteEvent;
 use Codeception\Event\StepEvent;
+use Codeception\Event\TestEvent;
 
 /**
  * Class MagentoAllureAdapter
@@ -114,6 +121,7 @@ private function sanitizeGroupName($group)
         // if we can't find this group in the generated suites we have to assume that the group was split for generation
         $groupNameSplit = explode("_", $group);
         array_pop($groupNameSplit);
+        array_pop($groupNameSplit);
         $originalName = implode("_", $groupNameSplit);
 
         // confirm our original name is one of the existing suite names otherwise just return the original group name
@@ -245,18 +253,25 @@ public function testError(FailEvent $failEvent)
 
     /**
      * Override of parent method, polls stepStorage for testcase and formats it according to actionGroup nesting.
-     *
+     * @param TestEvent $testEvent
+     * @throws \Yandex\Allure\Adapter\AllureException
      * @return void
      */
-    public function testEnd()
+    public function testEnd(TestEvent $testEvent)
     {
+        $test = $this->getLifecycle()->getTestCaseStorage()->get();
+        // update testClass label to consolidate re-try reporting
+        $this->formatAllureTestClassName($test);
+        // Peek top of testCaseStorage to check of failure
+        $testFailed = $test->getFailure();
         // Pops top of stepStorage, need to add it back in after processing
         $rootStep = $this->getLifecycle()->getStepStorage()->pollLast();
         $formattedSteps = [];
         $actionGroupStepContainer = null;
 
         $actionGroupStepKey = null;
         foreach ($rootStep->getSteps() as $step) {
+            $this->removeAttachments($step, $testFailed);
             $stepKey = str_replace($actionGroupStepKey, '', $step->getName());
             if ($stepKey !== '[]' && $stepKey !== null) {
                 $step->setName($stepKey);
@@ -309,9 +324,28 @@ function () use ($rootStep, $formattedSteps) {
 
         $this->getLifecycle()->getStepStorage()->put($rootStep);
 
+        $this->addAttachmentEvent($testEvent);
+
         $this->getLifecycle()->fire(new TestCaseFinishedEvent());
     }
 
+    /**
+     * Fire add attachment event
+     * @param TestEvent $testEvent
+     * @throws \Yandex\Allure\Adapter\AllureException
+     * @return void
+     */
+    private function addAttachmentEvent(TestEvent $testEvent)
+    {
+        // attachments supported since Codeception 3.0
+        if (version_compare(Codecept::VERSION, '3.0.0') > -1 && $testEvent->getTest() instanceof Cest) {
+            $artifacts = $testEvent->getTest()->getMetadata()->getReports();
+            foreach ($artifacts as $name => $artifact) {
+                Allure::lifecycle()->fire(new AddAttachmentEvent($artifact, $name, null));
+            }
+        }
+    }
+
     /**
      * Reads action group stepKey from step.
      *
@@ -354,4 +388,60 @@ private function retrieveStepKey($stepLine)
 
         return $stepKey;
     }
+
+    /**
+     * Removes attachments from step depending on MFTF configuration
+     * @param Step    $step
+     * @param Failure $testFailed
+     * @return void
+     */
+    private function removeAttachments($step, $testFailed)
+    {
+        //Remove Attachments if verbose flag is not true AND test did not fail
+        if (getenv('VERBOSE_ARTIFACTS') !== "true" && $testFailed === null) {
+            foreach ($step->getAttachments() as $index => $attachment) {
+                $step->removeAttachment($index);
+                unlink(Provider::getOutputDirectory() . DIRECTORY_SEPARATOR . $attachment->getSource());
+            }
+        }
+    }
+
+    /**
+     * Format testClass label to consolidate re-try reporting for groups split for parallel execution
+     * @param TestCase $test
+     * @return void
+     */
+    private function formatAllureTestClassName($test)
+    {
+        if ($this->getGroup() !== null) {
+            foreach ($test->getLabels() as $name => $label) {
+                if ($label->getName() == 'testClass') {
+                    $originalTestClass = $this->sanitizeTestClassLabel($label->getValue());
+                    call_user_func(\Closure::bind(
+                        function () use ($label, $originalTestClass) {
+                            $label->value = $originalTestClass;
+                        },
+                        null,
+                        $label
+                    ));
+                    break;
+                }
+            }
+        }
+    }
+
+    /**
+     * Function which sanitizes testClass label for split group runs
+     * @param string $testClass
+     * @return string
+     */
+    private function sanitizeTestClassLabel($testClass)
+    {
+        $originalTestClass = $testClass;
+        $originalGroupName = $this->sanitizeGroupName($this->getGroup());
+        if ($originalGroupName !== $this->getGroup()) {
+            $originalTestClass = str_replace($this->getGroup(), $originalGroupName, $testClass);
+        }
+        return $originalTestClass;
+    }
 }
diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php
@@ -168,11 +168,7 @@ private function getFailedTestList()
                     if ($suiteName == self::DEFAULT_TEST_GROUP) {
                         array_push($failedTestDetails['tests'], $testName);
                     } else {
-                        // Trim potential suite_parallel_0 to suite_parallel
-                        $suiteNameArray = explode("_", $suiteName);
-                        if (is_numeric(array_pop($suiteNameArray))) {
-                            $suiteName = implode("_", $suiteNameArray);
-                        }
+                        $suiteName = $this->sanitizeSuiteName($suiteName);
                         $failedTestDetails['suites'] = array_merge_recursive(
                             $failedTestDetails['suites'],
                             [$suiteName => [$testName]]
@@ -194,6 +190,23 @@ private function getFailedTestList()
         return $testConfigurationJson;
     }
 
+    /**
+     * Trim potential suite_parallel_0_G to suite_parallel
+     *
+     * @param string $suiteName
+     * @return string
+     */
+    private function sanitizeSuiteName($suiteName)
+    {
+        $suiteNameArray = explode("_", $suiteName);
+        if (array_pop($suiteNameArray) == 'G') {
+            if (is_numeric(array_pop($suiteNameArray))) {
+                $suiteName = implode("_", $suiteNameArray);
+            }
+        }
+        return $suiteName;
+    }
+
     /**
      * Returns an array of run commands read from the manifest file created post generation
      *
diff --git a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php
@@ -860,8 +860,7 @@ public function saveScreenshot()
      */
     public function amOnPage($page)
     {
-        (0 === strpos($page, 'http')) ? parent::amOnUrl($page) : parent::amOnPage($page);
-
+        parent::amOnPage($page);
         $this->waitForPageLoad();
     }
 
diff --git a/src/Magento/FunctionalTestingFramework/Util/Sorter/ParallelGroupSorter.php b/src/Magento/FunctionalTestingFramework/Util/Sorter/ParallelGroupSorter.php
@@ -232,7 +232,7 @@ private function getSuiteToSize($suiteNamesToTests)
      *
      * E.g.
      * Input {suitename = 'sample', tests = ['test1' => 100,'test2' => 150, 'test3' => 300], linelimit = 275}
-     * Result { ['sample_01' => ['test3' => 300], 'sample_02' => ['test2' => 150, 'test1' => 100]] }
+     * Result { ['sample_01_G' => ['test3' => 300], 'sample_02_G' => ['test2' => 150, 'test1' => 100]] }
      *
      * @param string  $suiteName
      * @param array   $tests
@@ -252,8 +252,8 @@ private function splitTestSuite($suiteName, $tests, $maxTime)
             }
 
             $group = $this->createTestGroup($maxTime, $test, $size, $availableTests);
-            $splitSuites["{$suiteName}_${splitCount}"] = $group;
-            $this->addSuiteToConfig($suiteName, "{$suiteName}_${splitCount}", $group);
+            $splitSuites["{$suiteName}_${splitCount}_G"] = $group;
+            $this->addSuiteToConfig($suiteName, "{$suiteName}_${splitCount}_G", $group);
 
             $availableTests = array_diff_key($availableTests, $group);
             $splitCount++;

Original file line number	Diff line number	Diff line change
`@@ -860,8 +860,7 @@ public function saveScreenshot()`
`860`	`860`	`*/`
`861`	`861`	`public function amOnPage($page)`
`862`	`862`	`{`
`863`		`- (0 === strpos($page, 'http')) ? parent::amOnUrl($page) : parent::amOnPage($page);`
`864`		`-`
	`863`	`+ parent::amOnPage($page);`
`865`	`864`	`$this->waitForPageLoad();`
`866`	`865`	`}`
`867`	`866`
Original file line number	Diff line number	Diff line change
`@@ -232,7 +232,7 @@ private function getSuiteToSize($suiteNamesToTests)`
`232`	`232`	`*`
`233`	`233`	`* E.g.`
`234`	`234`	`* Input {suitename = 'sample', tests = ['test1' => 100,'test2' => 150, 'test3' => 300], linelimit = 275}`
`235`		`- * Result { ['sample_01' => ['test3' => 300], 'sample_02' => ['test2' => 150, 'test1' => 100]] }`
	`235`	`+ * Result { ['sample_01_G' => ['test3' => 300], 'sample_02_G' => ['test2' => 150, 'test1' => 100]] }`
`236`	`236`	`*`
`237`	`237`	`* @param string $suiteName`
`238`	`238`	`* @param array $tests`
`@@ -252,8 +252,8 @@ private function splitTestSuite($suiteName, $tests, $maxTime)`
`252`	`252`	`}`
`253`	`253`
`254`	`254`	`$group = $this->createTestGroup($maxTime, $test, $size, $availableTests);`
`255`		`- $splitSuites["{$suiteName}_${splitCount}"] = $group;`
`256`		`- $this->addSuiteToConfig($suiteName, "{$suiteName}_${splitCount}", $group);`
	`255`	`+ $splitSuites["{$suiteName}_${splitCount}_G"] = $group;`
	`256`	`+ $this->addSuiteToConfig($suiteName, "{$suiteName}_${splitCount}_G", $group);`
`257`	`257`
`258`	`258`	`$availableTests = array_diff_key($availableTests, $group);`
`259`	`259`	`$splitCount++;`