Merge remote-tracking branch 'origin/develop' into admin-token-refresh

danmooney2 · danmooney2 · commit 9d86d8ffbd93 · 2021-02-04T12:00:57.000-06:00
diff --git a/composer.json b/composer.json
@@ -22,7 +22,7 @@
         "codeception/module-sequence": "^1.0",
         "codeception/module-webdriver": "^1.0",
         "composer/composer": "^1.9",
-        "csharpru/vault-php": "~3.5.3",
+        "csharpru/vault-php": "^4.1.0",
         "csharpru/vault-php-guzzle6-transport": "^2.0",
         "hoa/console": "~3.0",
         "monolog/monolog": "^1.17",
@@ -35,7 +35,8 @@
         "symfony/mime": "^5.0",
         "symfony/process": "^4.4",
         "vlucas/phpdotenv": "^2.4",
-        "weew/helpers-array": "^1.3"
+        "weew/helpers-array": "^1.3",
+        "nikic/php-parser": "~4.4.0"
     },
     "require-dev": {
         "brainmaestro/composer-git-hooks": "^2.3.1",
@@ -47,7 +48,7 @@
         "phpmd/phpmd": "^2.8.0",
         "phpunit/phpunit": "^9.0",
         "rregeer/phpunit-coverage-check": "^0.1.4",
-        "sebastian/phpcpd": "~5.0.0",
+        "sebastian/phpcpd": "~6.0.0",
         "squizlabs/php_codesniffer": "~3.5.4",
         "symfony/stopwatch": "~3.4.6"
     },
diff --git a/composer.lock b/composer.lock
diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Suite/SuiteGeneratorTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Suite/SuiteGeneratorTest.php
@@ -226,6 +226,57 @@ public function testNonExistentSuiteTestPair()
         $this->assertArrayHasKey('Suite3', $suiteErrors);
     }
 
+    /**
+     * Tests generating split suites for parallel test generation
+     */
+    public function testGenerateSplitSuiteFromTest()
+    {
+        $suiteDataArrayBuilder = new SuiteDataArrayBuilder();
+        $mockSuiteData = $suiteDataArrayBuilder
+            ->withName('mockSuite')
+            ->includeGroups(['group1'])
+            ->build();
+        $testDataArrayBuilder = new TestDataArrayBuilder();
+        $mockSimpleTest1 = $testDataArrayBuilder
+            ->withName('simpleTest1')
+            ->withAnnotations(['group' => [['value' => 'group1']]])
+            ->withTestReference("NonExistantTest")
+            ->withTestActions()
+            ->build();
+        $mockSimpleTest2 = $testDataArrayBuilder
+            ->withName('simpleTest2')
+            ->withAnnotations(['group' => [['value' => 'group1']]])
+            ->withTestActions()
+            ->build();
+        $mockSimpleTest3 = $testDataArrayBuilder
+            ->withName('simpleTest3')
+            ->withAnnotations(['group' => [['value' => 'group1']]])
+            ->withTestActions()
+            ->build();
+        $mockTestData = array_merge($mockSimpleTest1, $mockSimpleTest2, $mockSimpleTest3);
+        $this->setMockTestAndSuiteParserOutput($mockTestData, $mockSuiteData);
+
+        // Make manifest for split suites
+        $suiteConfig = [
+            'mockSuite' => [
+                'mockSuite_0_G' => ['simpleTest1', 'simpleTest2'],
+                'mockSuite_1_G' => ['simpleTest3'],
+            ],
+        ];
+        $manifest = TestManifestFactory::makeManifest('default', $suiteConfig);
+
+        // parse and generate suite object with mocked data and manifest
+        $mockSuiteGenerator = SuiteGenerator::getInstance();
+        $mockSuiteGenerator->generateAllSuites($manifest);
+
+        // assert last split suite group generated
+        TestLoggingUtil::getInstance()->validateMockLogStatement(
+            'info',
+            "suite generated",
+            ['suite' => 'mockSuite_1_G', 'relative_path' => "_generated" . DIRECTORY_SEPARATOR . "mockSuite_1_G"]
+        );
+    }
+
     /**
      * Function used to set mock for parser return and force init method to run between tests.
      *
diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Test/Util/ObjectExtensionUtilTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Test/Util/ObjectExtensionUtilTest.php
@@ -346,6 +346,41 @@ public function testExtendingExtendedActionGroup()
         }
     }
 
+    /**
+     * Tests generating a test that extends a skipped parent test
+     *
+     * @throws \Exception
+     */
+    public function testExtendedTestSkippedParent()
+    {
+        $testDataArrayBuilder = new TestDataArrayBuilder();
+        $mockParentTest = $testDataArrayBuilder
+            ->withName('baseTest')
+            ->withAnnotations([
+                'skip' => ['nodeName' => 'skip', 'issueId' => [['nodeName' => 'issueId', 'value' => 'someIssue']]]
+            ])
+            ->build();
+
+        $testDataArrayBuilder->reset();
+        $mockExtendedTest = $testDataArrayBuilder
+            ->withName('extendTest')
+            ->withTestReference("baseTest")
+            ->build();
+
+        $mockTestData = array_merge($mockParentTest, $mockExtendedTest);
+        $this->setMockTestOutput($mockTestData);
+
+        // parse and generate test object with mocked data
+        TestObjectHandler::getInstance()->getObject('extendTest');
+
+        // validate log statement
+        TestLoggingUtil::getInstance()->validateMockLogStatement(
+            'debug',
+            "extendTest is skipped due to ParentTestIsSkipped",
+            []
+        );
+    }
+
     /**
      * Function used to set mock for parser return and force init method to run between tests.
      *
diff --git a/dev/tests/unit/Util/TestDataArrayBuilder.php b/dev/tests/unit/Util/TestDataArrayBuilder.php
@@ -232,6 +232,28 @@ public function withTestReference($reference = null)
         return $this;
     }
 
+    /**
+     * Reset data array builder
+     *
+     * @return void
+     */
+    public function reset()
+    {
+        // reset
+        $this->testName = 'testTest';
+        $this->filename = null;
+        $this->testActionBeforeName = 'testActionBefore';
+        $this->testActionAfterName = 'testActionAfter';
+        $this->testActionFailedName = 'testActionFailed';
+        $this->testActionType = 'testAction';
+        $this->annotations = [];
+        $this->beforeHook = [];
+        $this->afterHook = [];
+        $this->failedHook = [];
+        $this->testReference = null;
+        $this->testActions = [];
+    }
+
     /**
      * Output the resulting test data array based on parameters set in the object
      *
diff --git a/dev/tests/verification/Resources/ExtendingSkippedTest.txt b/dev/tests/verification/Resources/ExtendingSkippedTest.txt
@@ -19,33 +19,6 @@ use Yandex\Allure\Adapter\Annotation\TestCaseId;
  */
 class ExtendingSkippedTestCest
 {
-	/**
-	  * @param AcceptanceTester $I
-	  * @throws \Exception
-	  */
-	public function _before(AcceptanceTester $I)
-	{
-		$I->amOnPage("/beforeUrl"); // stepKey: beforeAmOnPageKey
-	}
-
-	/**
-	  * @param AcceptanceTester $I
-	  * @throws \Exception
-	  */
-	public function _after(AcceptanceTester $I)
-	{
-		$I->amOnPage("/afterUrl"); // stepKey: afterAmOnPageKey
-	}
-
-	/**
-	  * @param AcceptanceTester $I
-	  * @throws \Exception
-	  */
-	public function _failed(AcceptanceTester $I)
-	{
-		$I->saveScreenshot(); // stepKey: saveScreenshot
-	}
-
 	/**
 	 * @Severity(level = SeverityLevel::CRITICAL)
 	 * @Features({"TestModule"})
@@ -55,9 +28,8 @@ class ExtendingSkippedTestCest
 	 * @return void
 	 * @throws \Exception
 	 */
-	public function ExtendingSkippedTest(AcceptanceTester $I)
+	public function ExtendingSkippedTest(AcceptanceTester $I, \Codeception\Scenario $scenario)
 	{
-		$I->comment("text");
-		$I->comment("child");
+		$scenario->skip("This test is skipped due to the following issues:\nParentTestIsSkipped");
 	}
 }
diff --git a/docs/extending.md b/docs/extending.md
@@ -15,6 +15,11 @@ Specify needed variations for a parent object and produce a copy of the original
 Unlike merging, the parent test (or action group) will still exist after the test generation.
 </div>
 
+<div class="bs-callout-warning" markdown="1">
+<br>
+Note: The extended test will be skipped if the parent test is skipped.
+</div>
+
 ## Extending tests
 
 ### Update a test step
diff --git a/src/Magento/FunctionalTestingFramework/Console/BuildProjectCommand.php b/src/Magento/FunctionalTestingFramework/Console/BuildProjectCommand.php
@@ -67,7 +67,7 @@ protected function configure()
      * @param OutputInterface $output
      * @return void
      * @throws \Exception
-     * @SuppressWarnings(PHPMD.UnusedLocalVariable)
+     * @SuppressWarnings(PHPMD.UnusedFormalParameter)
      */
     protected function execute(InputInterface $input, OutputInterface $output)
     {
diff --git a/src/Magento/FunctionalTestingFramework/Console/RunManifestCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunManifestCommand.php
@@ -112,7 +112,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int
      * @return void
      * @throws \Exception
      *
-     * @SuppressWarnings(PHPMD.UnusedLocalVariable) Need this because of the unused $type variable in the closure
+     * @SuppressWarnings(PHPMD.UnusedFormalParameter) Need this because of the unused $type variable in the closure
      */
     private function runManifestLine($manifestLine, $output, $exit = false)
     {
diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php
@@ -215,7 +215,7 @@ private function runTestsInSuite(array $suitesConfig, OutputInterface $output)
      * @param OutputInterface $output
      * @return integer
      *
-     * @SuppressWarnings(PHPMD.UnusedLocalVariable)
+     * @SuppressWarnings(PHPMD.UnusedFormalParameter)
      */
     private function executeTestCommand(string $command, OutputInterface $output)
     {
diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php
@@ -52,7 +52,7 @@ protected function configure()
      * @return integer
      * @throws \Exception
      *
-     * @SuppressWarnings(PHPMD.UnusedLocalVariable)
+     * @SuppressWarnings(PHPMD.UnusedFormalParameter)
      * @SuppressWarnings(PHPMD.CyclomaticComplexity)
      * @SuppressWarnings(PHPMD.NPathComplexity)
      */
diff --git a/src/Magento/FunctionalTestingFramework/Suite/SuiteGenerator.php b/src/Magento/FunctionalTestingFramework/Suite/SuiteGenerator.php
@@ -265,7 +265,16 @@ private function validateTestsReferencedInSuite($suiteName, $testsReferenced, $o
     private function generateSplitSuiteFromTest($suiteName, $suiteContent)
     {
         foreach ($suiteContent as $suiteSplitName => $tests) {
-            $this->generateSuiteFromTest($suiteSplitName, $tests, $suiteName);
+            try {
+                $this->generateSuiteFromTest($suiteSplitName, $tests, $suiteName);
+            } catch (FastFailException $e) {
+                    throw $e;
+            } catch (\Exception $e) {
+                // There are suites that include tests that reference tests from other Magento editions
+                // To keep backward compatibility, we will catch such exceptions with no error.
+                // This might inevitably hide some suite errors that are resulted by tests with broken references
+                //TODO MQE-2484
+            }
         }
     }
 
diff --git a/src/Magento/FunctionalTestingFramework/Test/Util/ObjectExtensionUtil.php b/src/Magento/FunctionalTestingFramework/Test/Util/ObjectExtensionUtil.php
@@ -42,13 +42,13 @@ public function extendTest($testObject)
         try {
             $parentTest = TestObjectHandler::getInstance()->getObject($testObject->getParentName());
         } catch (TestReferenceException $error) {
+            $skippedTest = $this->skipTest($testObject, 'ParentTestDoesNotExist');
             if (MftfApplicationConfig::getConfig()->verboseEnabled()) {
                 LoggingUtil::getInstance()->getLogger(ObjectExtensionUtil::class)->debug(
                     "parent test not defined. test will be skipped",
                     ["parent" => $testObject->getParentName(), "test" => $testObject->getName()]
                 );
             }
-            $skippedTest = $this->skipTest($testObject);
             return $skippedTest;
         }
 
@@ -64,6 +64,11 @@ public function extendTest($testObject)
                 ->debug("extending test", ["parent" => $parentTest->getName(), "test" => $testObject->getName()]);
         }
 
+        // Skip test if parent is skipped
+        if ($parentTest->isSkipped()) {
+            return $this->skipTest($testObject);
+        }
+
         // Get steps for both the parent and the child tests
         $referencedTestSteps = $parentTest->getUnresolvedSteps();
         $newSteps = $this->processRemoveActions(array_merge($referencedTestSteps, $testObject->getUnresolvedSteps()));
@@ -207,17 +212,19 @@ private function processRemoveActions($actions)
      * This method returns a skipped form of the Test Object
      *
      * @param TestObject $testObject
+     * @param string     $skipReason
      * @return TestObject
      */
-    public function skipTest($testObject)
+    public function skipTest($testObject, $skipReason = null)
     {
         $annotations = $testObject->getAnnotations();
+        $skipReason = $skipReason ?? 'ParentTestIsSkipped';
 
         // Add skip to the group array if it doesn't already exist
         if (array_key_exists('skip', $annotations)) {
             return $testObject;
         } elseif (!array_key_exists('skip', $annotations)) {
-            $annotations['skip'] = ['issueId' => "ParentTestDoesNotExist"];
+            $annotations['skip'] = ['issueId' => $skipReason];
         }
 
         $skippedTest = new TestObject(
@@ -229,6 +236,12 @@ public function skipTest($testObject)
             $testObject->getParentName()
         );
 
+        if (MftfApplicationConfig::getConfig()->verboseEnabled()) {
+            LoggingUtil::getInstance()->getLogger(ObjectExtensionUtil::class)->debug(
+                "{$testObject->getName()} is skipped due to {$skipReason}"
+            );
+        }
+
         return $skippedTest;
     }
 }

Original file line number	Diff line number	Diff line change
`@@ -67,7 +67,7 @@ protected function configure()`
`67`	`67`	`* @param OutputInterface $output`
`68`	`68`	`* @return void`
`69`	`69`	`* @throws \Exception`
`70`		`- * @SuppressWarnings(PHPMD.UnusedLocalVariable)`
	`70`	`+ * @SuppressWarnings(PHPMD.UnusedFormalParameter)`
`71`	`71`	`*/`
`72`	`72`	`protected function execute(InputInterface $input, OutputInterface $output)`
`73`	`73`	`{`
Original file line number	Diff line number	Diff line change
`@@ -112,7 +112,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int`
`112`	`112`	`* @return void`
`113`	`113`	`* @throws \Exception`
`114`	`114`	`*`
`115`		`- * @SuppressWarnings(PHPMD.UnusedLocalVariable) Need this because of the unused $type variable in the closure`
	`115`	`+ * @SuppressWarnings(PHPMD.UnusedFormalParameter) Need this because of the unused $type variable in the closure`
`116`	`116`	`*/`
`117`	`117`	`private function runManifestLine($manifestLine, $output, $exit = false)`
`118`	`118`	`{`
Original file line number	Diff line number	Diff line change
`@@ -215,7 +215,7 @@ private function runTestsInSuite(array $suitesConfig, OutputInterface $output)`
`215`	`215`	`* @param OutputInterface $output`
`216`	`216`	`* @return integer`
`217`	`217`	`*`
`218`		`- * @SuppressWarnings(PHPMD.UnusedLocalVariable)`
	`218`	`+ * @SuppressWarnings(PHPMD.UnusedFormalParameter)`
`219`	`219`	`*/`
`220`	`220`	`private function executeTestCommand(string $command, OutputInterface $output)`
`221`	`221`	`{`