MQE-1372: If Suite before fails build must be RED

- Added testIncomplete override to allure adapter, will mark incomplete tests as failed
- Lifted runFailed functionality from extension into TestContextExtension to be able to write incomplete tests to the failed fail
- Removed runFailed exception from etc version of Codeception.yml

Author: KevinBKozan

dev/tests/verification/Resources/functionalSuiteHooks.txt

@@ -30,7 +30,7 @@ class functionalSuiteHooks extends \Codeception\GroupObject
 
         if ($this->preconditionFailure != null) {
             //if our preconditions fail, we need to mark all the tests as incomplete.
-            $e->getTest()->getMetadata()->setIncomplete($this->preconditionFailure);
+            $e->getTest()->getMetadata()->setIncomplete("SUITE PRECONDITION FAILED:" . PHP_EOL . $this->preconditionFailure);
         }
     }
 

etc/config/codeception.dist.yml

@@ -12,7 +12,6 @@ settings:
     memory_limit: 1024M
 extensions:
     enabled:
-        - Codeception\Extension\RunFailed
         - Magento\FunctionalTestingFramework\Extension\TestContextExtension
         - Magento\FunctionalTestingFramework\Allure\Adapter\MagentoAllureAdapter
     config:

src/Magento/FunctionalTestingFramework/Allure/Adapter/MagentoAllureAdapter.php

@@ -10,6 +10,8 @@
 use Yandex\Allure\Adapter\Event\StepStartedEvent;
 use Yandex\Allure\Adapter\Event\StepFinishedEvent;
 use Yandex\Allure\Adapter\Event\StepFailedEvent;
+use Yandex\Allure\Adapter\Event\TestCaseFailedEvent;
+use Codeception\Event\FailEvent;
 use Codeception\Event\SuiteEvent;
 use Codeception\Event\StepEvent;
 
@@ -132,4 +134,18 @@ public function stepAfter(StepEvent $stepEvent = null)
         }
         $this->getLifecycle()->fire(new StepFinishedEvent());
     }
+
+    /**
+     * Override of parent method, fires a TestCaseFailedEvent if a test is marked as incomplete.
+     *
+     * @param FailEvent $failEvent
+     * @return void
+     */
+    public function testIncomplete(FailEvent $failEvent)
+    {
+        $event = new TestCaseFailedEvent();
+        $e = $failEvent->getFail();
+        $message = $e->getMessage();
+        $this->getLifecycle()->fire($event->withException($e)->withMessage($message));
+    }
 }

src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php

@@ -16,6 +16,7 @@
 class TestContextExtension extends BaseExtension
 {
     const TEST_PHASE_AFTER = "_after";
+    const TEST_FAILED_FILE = 'failed';
 
     /**
      * Codeception Events Mapping to methods
@@ -35,7 +36,8 @@ public function _initialize()
             Events::TEST_START => 'testStart',
             Events::TEST_FAIL => 'testFail',
             Events::STEP_AFTER => 'afterStep',
-            Events::TEST_END => 'testEnd'
+            Events::TEST_END => 'testEnd',
+            Events::RESULT_PRINT_AFTER => 'saveFailed'
         ];
         self::$events = array_merge(parent::$events, $events);
         parent::_initialize();
@@ -170,4 +172,52 @@ public function afterStep(\Codeception\Event\StepEvent $e)
     {
         ErrorLogger::getInstance()->logErrors($this->getDriver(), $e);
     }
+
+    /**
+     * Saves failed tests from last codecept run command into a file in _output directory
+     * Removes file if there were no failures in last run command
+     * @param \Codeception\Event\PrintResultEvent $e
+     * @return void
+     */
+    public function saveFailed(\Codeception\Event\PrintResultEvent $e)
+    {
+        $file = $this->getLogDir() . self::TEST_FAILED_FILE;
+        $result = $e->getResult();
+        $output = [];
+
+        // Remove previous file regardless if we're writing a new file
+        if (is_file($file)) {
+            unlink($file);
+        }
+
+        foreach ($result->failures() as $fail) {
+            $output[] = $this->localizePath(\Codeception\Test\Descriptor::getTestFullName($fail->failedTest()));
+        }
+        foreach ($result->errors() as $fail) {
+            $output[] = $this->localizePath(\Codeception\Test\Descriptor::getTestFullName($fail->failedTest()));
+        }
+        foreach ($result->notImplemented() as $fail) {
+            $output[] = $this->localizePath(\Codeception\Test\Descriptor::getTestFullName($fail->failedTest()));
+        }
+
+        if (empty($output)) {
+            return;
+        }
+
+        file_put_contents($file, implode("\n", $output));
+    }
+
+    /**
+     * Returns localized path to string, for writing failed file.
+     * @param string $path
+     * @return string
+     */
+    protected function localizePath($path)
+    {
+        $root = realpath($this->getRootDir()) . DIRECTORY_SEPARATOR;
+        if (substr($path, 0, strlen($root)) == $root) {
+            return substr($path, strlen($root));
+        }
+        return $path;
+    }
 }

src/Magento/FunctionalTestingFramework/Suite/views/SuiteClass.mustache

@@ -31,7 +31,7 @@ class {{suiteName}} extends \Codeception\GroupObject
 
         if ($this->preconditionFailure != null) {
             //if our preconditions fail, we need to mark all the tests as incomplete.
-            $e->getTest()->getMetadata()->setIncomplete($this->preconditionFailure);
+            $e->getTest()->getMetadata()->setIncomplete("SUITE PRECONDITION FAILED:" . PHP_EOL . $this->preconditionFailure);
         }
     }