Merge pull request #21 from magento-commerce/MQE-2466

MQE-2466: Test generation error in a split suite group (--config=para…

Author: soumyau

dev/tests/unit/Magento/FunctionalTestFramework/Suite/SuiteGeneratorTest.php

@@ -226,6 +226,57 @@ public function testNonExistentSuiteTestPair()
         $this->assertArrayHasKey('Suite3', $suiteErrors);
     }
 
+    /**
+     * Tests generating split suites for parallel test generation
+     */
+    public function testGenerateSplitSuiteFromTest()
+    {
+        $suiteDataArrayBuilder = new SuiteDataArrayBuilder();
+        $mockSuiteData = $suiteDataArrayBuilder
+            ->withName('mockSuite')
+            ->includeGroups(['group1'])
+            ->build();
+        $testDataArrayBuilder = new TestDataArrayBuilder();
+        $mockSimpleTest1 = $testDataArrayBuilder
+            ->withName('simpleTest1')
+            ->withAnnotations(['group' => [['value' => 'group1']]])
+            ->withTestReference("NonExistantTest")
+            ->withTestActions()
+            ->build();
+        $mockSimpleTest2 = $testDataArrayBuilder
+            ->withName('simpleTest2')
+            ->withAnnotations(['group' => [['value' => 'group1']]])
+            ->withTestActions()
+            ->build();
+        $mockSimpleTest3 = $testDataArrayBuilder
+            ->withName('simpleTest3')
+            ->withAnnotations(['group' => [['value' => 'group1']]])
+            ->withTestActions()
+            ->build();
+        $mockTestData = array_merge($mockSimpleTest1, $mockSimpleTest2, $mockSimpleTest3);
+        $this->setMockTestAndSuiteParserOutput($mockTestData, $mockSuiteData);
+
+        // Make manifest for split suites
+        $suiteConfig = [
+            'mockSuite' => [
+                'mockSuite_0_G' => ['simpleTest1', 'simpleTest2'],
+                'mockSuite_1_G' => ['simpleTest3'],
+            ],
+        ];
+        $manifest = TestManifestFactory::makeManifest('default', $suiteConfig);
+
+        // parse and generate suite object with mocked data and manifest
+        $mockSuiteGenerator = SuiteGenerator::getInstance();
+        $mockSuiteGenerator->generateAllSuites($manifest);
+
+        // assert last split suite group generated
+        TestLoggingUtil::getInstance()->validateMockLogStatement(
+            'info',
+            "suite generated",
+            ['suite' => 'mockSuite_1_G', 'relative_path' => "_generated" . DIRECTORY_SEPARATOR . "mockSuite_1_G"]
+        );
+    }
+
     /**
      * Function used to set mock for parser return and force init method to run between tests.
      *

src/Magento/FunctionalTestingFramework/Suite/SuiteGenerator.php

@@ -265,7 +265,16 @@ private function validateTestsReferencedInSuite($suiteName, $testsReferenced, $o
     private function generateSplitSuiteFromTest($suiteName, $suiteContent)
     {
         foreach ($suiteContent as $suiteSplitName => $tests) {
-            $this->generateSuiteFromTest($suiteSplitName, $tests, $suiteName);
+            try {
+                $this->generateSuiteFromTest($suiteSplitName, $tests, $suiteName);
+            } catch (FastFailException $e) {
+                    throw $e;
+            } catch (\Exception $e) {
+                // There are suites that include tests that reference tests from other Magento editions
+                // To keep backward compatibility, we will catch such exceptions with no error.
+                // This might inevitably hide some suite errors that are resulted by tests with broken references
+                //TODO MQE-2484
+            }
         }
     }