Skip to content

Commit 3a95153

Browse files
jilu1jilu1
committed
MQE-2495: config parallel by number of groups
1 parent 58e8110 commit 3a95153

File tree

3 files changed

+256
-24
lines changed

3 files changed

+256
-24
lines changed

‎dev/tests/unit/Magento/FunctionalTestFramework/Util/Sorter/ParallelGroupSorterTest.php‎

Lines changed: 149 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ class ParallelGroupSorterTest extends MagentoTestCase
1818
/**
1919
* Test a basic sort of available tests based on size
2020
*/
21-
public function testBasicTestGroupSort()
21+
public function testBasicTestGroupSplitByTime()
2222
{
2323
$sampleTestArray = [
2424
'test1' => 100,
@@ -55,7 +55,7 @@ public function testBasicTestGroupSort()
5555
/**
5656
* Test a sort of both tests and a suite which is larger than the given line limitation
5757
*/
58-
public function testSortWithSuites()
58+
public function testTestsAndSuitesSplitByTime()
5959
{
6060
// mock tests for test object handler.
6161
$numberOfCalls = 0;
@@ -112,4 +112,151 @@ public function testSortWithSuites()
112112
$this->assertEquals($expectedResults[$groupNum], array_keys($group));
113113
}
114114
}
115+
116+
/**
117+
* Test a basic sort of available tests based on size
118+
*/
119+
public function testBasicTestGroupSplitByGroup()
120+
{
121+
$sampleTestArray = [
122+
'test1' => 100,
123+
'test2' => 300,
124+
'test3' => 50,
125+
'test4' => 60,
126+
'test5' => 25,
127+
'test6' => 125,
128+
'test7' => 250,
129+
'test8' => 1,
130+
'test9' => 80,
131+
'test10' => 25,
132+
'test11' => 89,
133+
'test12' => 69,
134+
'test13' => 23,
135+
'test14' => 15,
136+
'test15' => 25,
137+
'test16' => 71,
138+
'test17' => 67,
139+
'test18' => 34,
140+
'test19' => 45,
141+
'test20' => 58,
142+
'test21' => 9,
143+
];
144+
145+
$expectedResult = [
146+
0 => ['test2', 'test8'],
147+
1 => ['test11', 'test9', 'test17', 'test19', 'test13'],
148+
2 => ['test7', 'test18', 'test14', 'test21'],
149+
3 => ['test6', 'test12', 'test20', 'test5', 'test10'],
150+
4 => ['test1', 'test16', 'test4', 'test3', 'test15']
151+
];
152+
153+
$testSorter = new ParallelGroupSorter();
154+
$actualResult = $testSorter->getTestsGroupedByFixedGroupCount([], $sampleTestArray, 5);
155+
156+
$this->assertCount(5, $actualResult);
157+
158+
foreach ($actualResult as $gropuNumber => $actualTests) {
159+
$expectedTests = $expectedResult[$gropuNumber];
160+
$this->assertEquals($expectedTests, array_keys($actualTests));
161+
}
162+
}
163+
164+
/**
165+
* Test a sort of both tests and a suite which is larger than the given line limitation
166+
*/
167+
public function testTestsAndSuitesSplitByGroup()
168+
{
169+
// mock tests for test object handler.
170+
$numberOfCalls = 0;
171+
$mockTest1 = AspectMock::double(
172+
TestObject::class,
173+
['getEstimatedDuration' => function () use (&$numberOfCalls) {
174+
$actionCount = [300, 275];
175+
$result = $actionCount[$numberOfCalls];
176+
$numberOfCalls++;
177+
178+
return $result;
179+
}]
180+
)->make();
181+
182+
$mockHandler = AspectMock::double(
183+
TestObjectHandler::class,
184+
['getObject' => function () use ($mockTest1) {
185+
return $mockTest1;
186+
}]
187+
)->make();
188+
189+
AspectMock::double(TestObjectHandler::class, ['getInstance' => $mockHandler])->make();
190+
191+
// create test to size array
192+
$sampleTestArray = [
193+
'test1' => 1,
194+
'test2' => 125,
195+
'test3' => 35,
196+
'test4' => 111,
197+
'test5' => 43,
198+
'test6' => 321,
199+
'test7' => 260,
200+
'test8' => 5,
201+
'test9' => 189,
202+
'test10' => 246,
203+
'test11' => 98,
204+
'test12' => 96,
205+
'test13' => 232,
206+
'test14' => 51,
207+
'test15' => 52,
208+
'test16' => 127,
209+
'test17' => 76,
210+
'test18' => 43,
211+
'test19' => 154,
212+
'test20' => 85,
213+
'test21' => 219,
214+
'test22' => 87,
215+
'test23' => 65,
216+
'test24' => 216,
217+
'test25' => 271,
218+
'test26' => 99,
219+
'test27' => 102,
220+
'test28' => 179,
221+
'test29' => 243,
222+
'test30' => 93,
223+
'test31' => 330,
224+
'test32' => 85,
225+
'test33' => 291,
226+
];
227+
228+
// create mock suite references
229+
$sampleSuiteArray = [
230+
'mockSuite1' => ['mockTest1', 'mockTest2']
231+
];
232+
233+
// perform sort
234+
$testSorter = new ParallelGroupSorter();
235+
$actualResult = $testSorter->getTestsGroupedByFixedGroupCount($sampleSuiteArray, $sampleTestArray, 15);
236+
237+
// verify the resulting groups
238+
$this->assertCount(15, $actualResult);
239+
240+
$expectedResults = [
241+
0 => ['test31', 'test8', 'test1'],
242+
1 => ['test6', 'test5'],
243+
2 => ['test33', 'test17'],
244+
3 => ['test25', 'test32'],
245+
4 => ['test7', 'test22'],
246+
5 => ['test10', 'test30'],
247+
6 => ['test29', 'test12'],
248+
7 => ['test13', 'test11', 'test3'],
249+
8 => ['test21', 'test26', 'test14'],
250+
9 => ['test24', 'test27', 'test18'],
251+
10 => ['test9', 'test4', 'test23'],
252+
11 => ['test28', 'test2', 'test15'],
253+
12 => ['test19', 'test16', 'test20'],
254+
13 => ['mockSuite1_0_G'],
255+
14 => ['mockSuite1_1_G'],
256+
];
257+
258+
foreach ($actualResult as $groupNum => $group) {
259+
$this->assertEquals($expectedResults[$groupNum], array_keys($group));
260+
}
261+
}
115262
}

‎dev/tests/verification/Tests/SuiteGenerationTest.php‎

Lines changed: 53 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
use Magento\FunctionalTestingFramework\Util\Filesystem\DirSetupUtil;
1212
use Magento\FunctionalTestingFramework\Util\Manifest\DefaultTestManifest;
1313
use Magento\FunctionalTestingFramework\Util\Manifest\ParallelByTimeTestManifest;
14+
use Magento\FunctionalTestingFramework\Util\Manifest\ParallelByGroupTestManifest;
1415
use Magento\FunctionalTestingFramework\Util\Manifest\TestManifestFactory;
1516
use Magento\FunctionalTestingFramework\Util\Path\FilePathFormatter;
1617
use Symfony\Component\Yaml\Yaml;
@@ -107,7 +108,7 @@ public function testSuiteGeneration1()
107108
/**
108109
* Test generation of parallel suite groups
109110
*/
110-
public function testSuiteGenerationParallel()
111+
public function testSuiteGenerationParallelByTime()
111112
{
112113
$groupName = 'functionalSuite1';
113114

@@ -157,6 +158,57 @@ public function testSuiteGenerationParallel()
157158
}
158159
}
159160

161+
/**
162+
* Test generation of parallel suite groups
163+
*/
164+
public function testSuiteGenerationParallelByGroup()
165+
{
166+
$groupName = 'functionalSuite1';
167+
168+
$expectedGroups = [
169+
'functionalSuite1_0_G',
170+
'functionalSuite1_1_G',
171+
];
172+
173+
$expectedContents = SuiteTestReferences::$data[$groupName];
174+
175+
//createParallelManifest
176+
/** @var ParallelByGroupTestManifest $parallelManifest */
177+
$parallelManifest = TestManifestFactory::makeManifest("parallelByGroup", ["functionalSuite1" => []]);
178+
179+
// Generate the Suite
180+
$parallelManifest->createTestGroups(2);
181+
SuiteGenerator::getInstance()->generateAllSuites($parallelManifest);
182+
183+
// Validate log message (for final group) and add group name for later deletion
184+
$expectedGroup = $expectedGroups[count($expectedGroups)-1] ;
185+
TestLoggingUtil::getInstance()->validateMockLogStatement(
186+
'info',
187+
"suite generated",
188+
['suite' => $expectedGroup, 'relative_path' => "_generated" . DIRECTORY_SEPARATOR . $expectedGroup]
189+
);
190+
191+
self::$TEST_GROUPS[] = $groupName;
192+
193+
// Validate Yaml file updated
194+
$yml = Yaml::parse(file_get_contents(self::CONFIG_YML_FILE));
195+
$this->assertEquals(array_intersect($expectedGroups, array_keys($yml['groups'])), $expectedGroups);
196+
197+
foreach ($expectedGroups as $expectedFolder) {
198+
$suiteResultBaseDir = self::GENERATE_RESULT_DIR .
199+
DIRECTORY_SEPARATOR .
200+
$expectedFolder .
201+
DIRECTORY_SEPARATOR;
202+
203+
// Validate tests have been generated
204+
$dirContents = array_diff(scandir($suiteResultBaseDir), ['..', '.']);
205+
206+
//Validate two test has been added to each group since lines are set to 1
207+
$this->assertEquals(2, count($dirContents));
208+
$this->assertContains(array_values($dirContents)[0], $expectedContents);
209+
}
210+
}
211+
160212
/**
161213
* Test hook groups generated during suite generation
162214
*/

‎src/Magento/FunctionalTestingFramework/Util/Sorter/ParallelGroupSorter.php‎

Lines changed: 54 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -87,19 +87,48 @@ public function getTestsGroupedBySize($suiteConfiguration, $testNameToSize, $tim
8787
public function getTestsGroupedByFixedGroupCount($suiteConfiguration, $testNameToSize, $groupTotal)
8888
{
8989
$suiteNameToTestSize = $this->getSuiteNameToTestSize($suiteConfiguration);
90-
$suiteNameToSize = $this->getSuiteToSize($suiteNameToTestSize);
9190

9291
$minRequiredGroupCount = count($suiteNameToTestSize);
9392
if (!empty($testNameToSize)) {
9493
$minRequiredGroupCount += 1;
9594
}
96-
9795
if ($groupTotal < $minRequiredGroupCount) {
9896
throw new FastFailException(
9997
"Invalid parameter 'groupTotal': must be greater than {$minRequiredGroupCount}\n"
10098
);
10199
}
102100

101+
if (empty($suiteConfiguration)) {
102+
return $this->splitTestsIntoGroups($testNameToSize, $groupTotal);
103+
}
104+
105+
// Calculate suite group totals
106+
$suiteNameToSize = $this->getSuiteToSize($suiteNameToTestSize);
107+
$suiteNameToGroupCount = $this->getSuiteGroupCounts($suiteNameToSize, $testNameToSize, $groupTotal);
108+
109+
// Calculate test group total
110+
$testGroupTotal = $groupTotal - array_sum($suiteNameToGroupCount);
111+
112+
// Split tests and suites
113+
$testGroups = $this->splitTestsIntoGroups($testNameToSize, $testGroupTotal);
114+
$testGroups = array_merge(
115+
$testGroups,
116+
$this->splitSuitesIntoGroups($suiteNameToTestSize, $suiteNameToGroupCount)
117+
);
118+
119+
return $testGroups;
120+
}
121+
122+
/**
123+
* Return suite's group counts from a group total
124+
*
125+
* @param $suiteNameToSize
126+
* @param $testNameToSize
127+
* @param $groupTotal
128+
* @return array
129+
*/
130+
private function getSuiteGroupCounts($suiteNameToSize, $testNameToSize, $groupTotal)
131+
{
103132
// Calculate the minimum possible group time
104133
$minGroupTime = ceil((array_sum($testNameToSize) + array_sum($suiteNameToSize)) / $groupTotal);
105134

@@ -110,24 +139,37 @@ public function getTestsGroupedByFixedGroupCount($suiteConfiguration, $testNameT
110139
$ceilSuiteGroupNumber = ceil($maxSuiteTime / $minGroupTime);
111140
$ceilSuiteGroupTime = max(ceil($maxSuiteTime / $ceilSuiteGroupNumber), $minGroupTime);
112141
$floorSuiteGroupNumber = floor($maxSuiteTime / $minGroupTime);
113-
if ($floorSuiteGroupNumber != 0.0) {
142+
if ($floorSuiteGroupNumber != 0) {
114143
$floorSuiteGroupTime = max(ceil($maxSuiteTime / $floorSuiteGroupNumber), $minGroupTime);
115144
}
116145

117146
// Calculate test group time for ceiling
118-
$ceilSuiteNameToGroupCount = $this->getSuiteNameToGroupCount($suiteNameToSize, $ceilSuiteGroupTime);
147+
$ceilSuiteNameToGroupCount = $this->getSuiteGroupCountFromGroupTime($suiteNameToSize, $ceilSuiteGroupTime);
119148
$ceilSuiteGroupTotal = array_sum($ceilSuiteNameToGroupCount);
120149
$ceilTestGroupTotal = $groupTotal - $ceilSuiteGroupTotal;
121-
$ceilTestGroupTime = ceil(array_sum($testNameToSize) / $ceilTestGroupTotal);
150+
151+
if ($ceilTestGroupTotal == 0) {
152+
$ceilTestGroupTime = 0;
153+
} else {
154+
$ceilTestGroupTime = ceil(array_sum($testNameToSize) / $ceilTestGroupTotal);
155+
}
156+
122157
// Set suite group total to ceiling
123158
$suiteNameToGroupCount = $ceilSuiteNameToGroupCount;
124159

125160
if (isset($floorSuiteGroupTime) && $ceilSuiteGroupTime != $floorSuiteGroupTime) {
126161
// Calculate test group time for floor
127-
$floorSuiteNameToGroupCount = $this->getSuiteNameToGroupCount($suiteNameToSize, $floorSuiteGroupTime);
162+
$floorSuiteNameToGroupCount = $this->getSuiteGroupCountFromGroupTime(
163+
$suiteNameToSize,
164+
$floorSuiteGroupTime
165+
);
128166
$floorSuiteGroupTotal = array_sum($floorSuiteNameToGroupCount);
129167
$floorTestGroupTotal = $groupTotal - $floorSuiteGroupTotal;
130-
$floorTestGroupTime = ceil(array_sum($testNameToSize) / $floorTestGroupTotal);
168+
if ($floorTestGroupTotal == 0) {
169+
$floorTestGroupTime = 0;
170+
} else {
171+
$floorTestGroupTime = ceil(array_sum($testNameToSize) / $floorTestGroupTotal);
172+
}
131173

132174
// Choose the closer value between test group time and suite group time
133175
$ceilDiff = abs($ceilTestGroupTime - $ceilSuiteGroupTime);
@@ -138,17 +180,7 @@ public function getTestsGroupedByFixedGroupCount($suiteConfiguration, $testNameT
138180
}
139181
}
140182

141-
// Calculate test group total
142-
$testGroupTotal = $groupTotal - array_sum($suiteNameToGroupCount);
143-
144-
// Split tests and suites
145-
$testGroups = $this->splitTestsIntoGroups($testNameToSize, $testGroupTotal);
146-
$testGroups = array_merge(
147-
$testGroups,
148-
$this->splitSuitesIntoGroups($suiteNameToTestSize, $suiteNameToGroupCount)
149-
);
150-
151-
return $testGroups;
183+
return $suiteNameToGroupCount;
152184
}
153185

154186
/**
@@ -158,7 +190,7 @@ public function getTestsGroupedByFixedGroupCount($suiteConfiguration, $testNameT
158190
* @param integer $time
159191
* @return array
160192
*/
161-
private function getSuiteNameToGroupCount($suites, $time)
193+
private function getSuiteGroupCountFromGroupTime($suites, $time)
162194
{
163195
$suiteNameToGroupCount = [];
164196
foreach ($suites as $suiteName => $suiteTime) {
@@ -190,9 +222,10 @@ private function splitTestsIntoGroups($tests, $groupCnt)
190222
}
191223
asort($sums);
192224
// Always add the next test to the group with the smallest sum
193-
$groups[array_keys($sums)[0]][$test] = $size;
225+
$groups[array_key_first($sums)][$test] = $size;
194226
}
195-
return $groups;
227+
// Filter empty array
228+
return array_filter($groups);
196229
}
197230

198231
/**

0 commit comments

Comments
 (0)