Skip to content

Commit fe1785b

Browse files
shahmishalshahmishal
authored
Merge pull request #599 from ahoppen/pr/stress-tester-other-xfails
Manage the project XFails under a different job type for the stress tester
2 parents 0bf5fc0 + fed7a75 commit fe1785b

File tree

5 files changed

+65
-36
lines changed

5 files changed

+65
-36
lines changed

‎build_incremental.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ def main():
4444
args.swiftc,
4545
args.swift_version,
4646
args.swift_branch,
47+
args.job_type,
4748
args.sandbox_profile_xcodebuild,
4849
args.sandbox_profile_package,
4950
args.add_swift_flags,

‎project_future.py‎

Lines changed: 17 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -414,8 +414,8 @@ def dispatch(root_path, repo, action, swiftc, swift_version,
414414
raise common.Unimplemented("Unknown action: %s" % action['action'])
415415

416416

417-
def is_xfailed(xfail_args, compatible_version, platform, swift_branch, build_config):
418-
"""Return whether the specified swift version/platform/branch/configuration is xfailed."""
417+
def is_xfailed(xfail_args, compatible_version, platform, swift_branch, build_config, job_type):
418+
"""Return whether the specified swift version/platform/branch/configuration/job is xfailed."""
419419
if isinstance(xfail_args, dict):
420420
xfail_args = [xfail_args]
421421

@@ -427,7 +427,8 @@ def matches(spec):
427427
current = {
428428
'compatibility': compatible_version,
429429
'branch': swift_branch,
430-
'platform': platform
430+
'platform': platform,
431+
'job': job_type,
431432
}
432433
if 'configuration' in spec:
433434
if build_config is None:
@@ -583,6 +584,9 @@ def add_arguments(parser):
583584
parser.add_argument("--report-time-path",
584585
help='export time for building each xcode build target to the specified json file',
585586
type=os.path.abspath)
587+
parser.add_argument("--job-type",
588+
help="The type of job to run. This influences which projects are XFailed, for example the stress tester tracks its XFails under a different job type. Defaults to 'source-compat'.",
589+
default='source-compat')
586590

587591
def add_minimal_arguments(parser):
588592
"""Add common arguments to parser."""
@@ -942,7 +946,7 @@ def output_fd(self, subtarget):
942946

943947

944948
class ActionBuilder(Factory):
945-
def __init__(self, swiftc, swift_version, swift_branch,
949+
def __init__(self, swiftc, swift_version, swift_branch, job_type,
946950
sandbox_profile_xcodebuild,
947951
sandbox_profile_package,
948952
added_swift_flags,
@@ -968,6 +972,7 @@ def __init__(self, swiftc, swift_version, swift_branch,
968972
self.build_config = build_config
969973
self.strip_resource_phases = strip_resource_phases
970974
self.time_reporter = time_reporter
975+
self.job_type = job_type
971976
self.init()
972977

973978
def init(self):
@@ -1055,7 +1060,7 @@ def succeeded(self, identifier):
10551060

10561061
class CompatActionBuilder(ActionBuilder):
10571062
def __init__(self,
1058-
swiftc, swift_version, swift_branch,
1063+
swiftc, swift_version, swift_branch, job_type,
10591064
sandbox_profile_xcodebuild,
10601065
sandbox_profile_package,
10611066
added_swift_flags,
@@ -1067,7 +1072,7 @@ def __init__(self,
10671072
time_reporter,
10681073
action, version, project):
10691074
super(CompatActionBuilder, self).__init__(
1070-
swiftc, swift_version, swift_branch,
1075+
swiftc, swift_version, swift_branch, job_type,
10711076
sandbox_profile_xcodebuild,
10721077
sandbox_profile_package,
10731078
added_swift_flags,
@@ -1136,7 +1141,8 @@ def failed(self, identifier, error):
11361141
self.version['version'],
11371142
self.current_platform,
11381143
self.swift_branch,
1139-
build_config)
1144+
build_config,
1145+
self.job_type)
11401146
if bug_identifier:
11411147
error_str = 'XFAIL: {bug}, {project}, {compatibility}, {commit}, {action_target}'.format(
11421148
bug=bug_identifier,
@@ -1170,7 +1176,8 @@ def succeeded(self, identifier):
11701176
self.version['version'],
11711177
self.current_platform,
11721178
self.swift_branch,
1173-
build_config)
1179+
build_config,
1180+
self.job_type)
11741181
if bug_identifier:
11751182
error_str = 'UPASS: {bug}, {project}, {compatibility}, {commit}, {action_target}'.format(
11761183
bug=bug_identifier,
@@ -1243,15 +1250,15 @@ def have_same_trees(full, incr, d):
12431250

12441251
class IncrementalActionBuilder(ActionBuilder):
12451252

1246-
def __init__(self, swiftc, swift_version, swift_branch,
1253+
def __init__(self, swiftc, swift_version, swift_branch, job_type,
12471254
sandbox_profile_xcodebuild,
12481255
sandbox_profile_package,
12491256
added_swift_flags, build_config,
12501257
strip_resource_phases,
12511258
time_reporter,
12521259
project, action):
12531260
super(IncrementalActionBuilder,
1254-
self).__init__(swiftc, swift_version, swift_branch,
1261+
self).__init__(swiftc, swift_version, swift_branch, job_type,
12551262
sandbox_profile_xcodebuild,
12561263
sandbox_profile_package,
12571264
added_swift_flags,

‎projects.json‎

Lines changed: 42 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,8 @@
2626
{
2727
"issue": "https://bugs.swift.org/browse/SR-15116",
2828
"compatibility": ["5.1"],
29-
"branch": ["main", "release/5.5"]
29+
"branch": ["main", "release/5.5"],
30+
"job": ["source-compat"]
3031
}
3132
]
3233
}
@@ -680,12 +681,14 @@
680681
{
681682
"issue": "https://bugs.swift.org/browse/SR-12736",
682683
"compatibility": "5.3",
683-
"branch": ["main", "release/5.5", "release/5.4"]
684+
"branch": ["main", "release/5.5", "release/5.4"],
685+
"job": ["source-compat"]
684686
},
685687
{
686688
"issue": "https://bugs.swift.org/browse/SR-14659",
687689
"compatibility": "5.3",
688-
"branch": ["main", "release/5.5", "release/5.4"]
690+
"branch": ["main", "release/5.5", "release/5.4"],
691+
"job": ["source-compat"]
689692
}
690693
]
691694
},
@@ -816,7 +819,8 @@
816819
{
817820
"issue": "https://bugs.swift.org/browse/SR-15106",
818821
"compatibility": ["4.0"],
819-
"branch": ["main", "release/5.5"]
822+
"branch": ["main", "release/5.5"],
823+
"job": ["source-compat"]
820824
}
821825
]
822826
}
@@ -1236,7 +1240,8 @@
12361240
{
12371241
"issue": "rdar://problem/75437284",
12381242
"compatibility": ["5.1"],
1239-
"branch": ["main", "release/5.5"]
1243+
"branch": ["main", "release/5.5"],
1244+
"job": ["source-compat"]
12401245
}
12411246
]
12421247
},
@@ -1250,12 +1255,14 @@
12501255
{
12511256
"issue": "rdar://problem/75437284",
12521257
"compatibility": ["5.1"],
1253-
"branch": ["main", "release/5.5"]
1258+
"branch": ["main", "release/5.5"],
1259+
"job": ["source-compat"]
12541260
},
12551261
{
12561262
"issue": "https://bugs.swift.org/browse/SR-15107",
12571263
"compatibility": ["4.0"],
1258-
"branch": ["main", "release/5.5"]
1264+
"branch": ["main", "release/5.5"],
1265+
"job": ["source-compat"]
12591266
}
12601267
]
12611268
}
@@ -1287,7 +1294,8 @@
12871294
{
12881295
"issue": "https://bugs.swift.org/browse/SR-15109",
12891296
"compatibility": ["5.1"],
1290-
"branch": ["main", "release/5.5"]
1297+
"branch": ["main", "release/5.5"],
1298+
"job": ["source-compat"]
12911299
}
12921300
]
12931301
},
@@ -1641,7 +1649,8 @@
16411649
{
16421650
"issue": "https://bugs.swift.org/browse/SR-15116",
16431651
"compatibility": ["5.1"],
1644-
"branch": ["main", "release/5.5"]
1652+
"branch": ["main", "release/5.5"],
1653+
"job": ["source-compat"]
16451654
}
16461655
]
16471656
}
@@ -2184,7 +2193,8 @@
21842193
"issue": "rdar://problem/68590422",
21852194
"compatibility": ["4.0", "5.1"],
21862195
"branch": ["release/5.4"],
2187-
"configuration": "debug"
2196+
"configuration": "debug",
2197+
"job": ["source-compat"]
21882198
}
21892199
]
21902200
},
@@ -2199,7 +2209,8 @@
21992209
"issue": "rdar://problem/68590422",
22002210
"compatibility": ["4.0", "5.1"],
22012211
"branch": ["release/5.4"],
2202-
"configuration": "debug"
2212+
"configuration": "debug",
2213+
"job": ["source-compat"]
22032214
}
22042215
]
22052216
},
@@ -2246,7 +2257,8 @@
22462257
{
22472258
"issue": "https://bugs.swift.org/browse/SR-15109",
22482259
"compatibility": ["5.1"],
2249-
"branch": ["main", "release/5.5"]
2260+
"branch": ["main", "release/5.5"],
2261+
"job": ["source-compat"]
22502262
}
22512263
]
22522264
},
@@ -2420,7 +2432,8 @@
24202432
{
24212433
"issue": "https://bugs.swift.org/browse/SR-15232",
24222434
"compatibility": ["4.2", "5.3"],
2423-
"branch": ["main", "release/5.5"]
2435+
"branch": ["main", "release/5.5"],
2436+
"job": ["source-compat"]
24242437
}
24252438
],
24262439
"configuration": "release"
@@ -2485,7 +2498,8 @@
24852498
"xfail": {
24862499
"issue": "https://bugs.swift.org/browse/SR-11141",
24872500
"compatibility": "5.1",
2488-
"branch": ["main", "release/5.3", "release/5.4", "release/5.5"]
2501+
"branch": ["main", "release/5.3", "release/5.4", "release/5.5"],
2502+
"job": ["source-compat"]
24892503
}
24902504
},
24912505
{
@@ -2549,7 +2563,8 @@
25492563
{
25502564
"issue": "https://bugs.swift.org/browse/SR-14655;rdar://81276100",
25512565
"compatibility": ["5.1"],
2552-
"branch": ["main", "release/5.5"]
2566+
"branch": ["main", "release/5.5"],
2567+
"job": ["source-compat"]
25532568
}
25542569
]
25552570
},
@@ -2563,7 +2578,8 @@
25632578
{
25642579
"issue": "https://bugs.swift.org/browse/SR-14655;rdar://81276100",
25652580
"compatibility": ["5.1"],
2566-
"branch": ["main", "release/5.5"]
2581+
"branch": ["main", "release/5.5"],
2582+
"job": ["source-compat"]
25672583
}
25682584
]
25692585
},
@@ -2577,7 +2593,8 @@
25772593
{
25782594
"issue": "https://bugs.swift.org/browse/SR-14655;rdar://81276100",
25792595
"compatibility": ["5.1"],
2580-
"branch": ["main", "release/5.5"]
2596+
"branch": ["main", "release/5.5"],
2597+
"job": ["source-compat"]
25812598
}
25822599
]
25832600
}
@@ -2611,7 +2628,8 @@
26112628
{
26122629
"issue": "https://bugs.swift.org/browse/SR-13190",
26132630
"compatibility": ["4.0", "5.1"],
2614-
"branch": ["main", "release/5.4", "release/5.5"]
2631+
"branch": ["main", "release/5.4", "release/5.5"],
2632+
"job": ["source-compat"]
26152633
}
26162634
]
26172635
},
@@ -3128,7 +3146,8 @@
31283146
{
31293147
"issue": "rdar://81180448",
31303148
"compatibility": ["4.2", "5.0", "5.1"],
3131-
"branch": ["main", "release/5.5"]
3149+
"branch": ["main", "release/5.5"],
3150+
"job": ["source-compat"]
31323151
}
31333152
]
31343153
},
@@ -3960,7 +3979,8 @@
39603979
{
39613980
"issue": "https://bugs.swift.org/browse/SR-15105",
39623981
"compatibility": ["5.1"],
3963-
"branch": ["main", "release/5.5"]
3982+
"branch": ["main", "release/5.5"],
3983+
"job": ["source-compat"]
39643984
}
39653985
]
39663986
},
@@ -4304,7 +4324,8 @@
43044324
"issue": "https://bugs.swift.org/browse/SR-15105",
43054325
"compatibility": ["5.1"],
43064326
"branch": ["main", "release/5.5"],
4307-
"configuration": "debug"
4327+
"configuration": "debug",
4328+
"job": ["source-compat"]
43084329
}
43094330
]
43104331
},

‎run_sk_stress_test‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -258,6 +258,7 @@ class StressTesterRunner(object):
258258
'--verbose',
259259
'--swiftc', self.wrapper,
260260
'--swift-branch', self.swift_branch,
261+
'--job-type', 'stress-tester',
261262
'--default-timeout', str(-1),
262263
'--only-latest-versions',
263264
# archs override is set to arm64 for generic/iOS actions that would otherwise invoke the stress tester for both arm64 and armv7
@@ -284,7 +285,7 @@ class StressTesterRunner(object):
284285

285286
def _process_output(self, results_path, xfails_path):
286287
if not os.path.isfile(results_path):
287-
return True
288+
return not self.compat_runner_failed
288289

289290
with open(results_path, 'rb') as results_file:
290291
results = json.load(results_file, encoding='utf-8')
@@ -305,7 +306,7 @@ class StressTesterRunner(object):
305306
num_xfail_issues = len(results['expectedIssues'])
306307
unmatched = results['unmatchedExpectedIssues']
307308

308-
success = num_failures == 0 and len(unmatched) == 0
309+
success = self.compat_runner_failed == False and num_failures == 0 and len(unmatched) == 0
309310

310311
if num_xfails > 0:
311312
print('Expected stress tester issues:')
@@ -336,9 +337,7 @@ class StressTesterRunner(object):
336337
print('SourceKit Stress Tester summary:')
337338

338339
print(' {} underlying source compatibility build'.format('failed' if self.compat_runner_failed else 'passed'))
339-
if self.compat_runner_failed:
340-
print(' > treat this as a source compatibility failure')
341-
340+
342341
print(' {} XFails not processed'.format(len(xfails_not_processed)))
343342
if num_failures > 0:
344343
print(' > see "XFails not processed" how to handle them. This is an info, not an error')

‎runner.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@ def main():
6565
args.swiftc,
6666
args.swift_version,
6767
args.swift_branch,
68+
args.job_type,
6869
args.sandbox_profile_xcodebuild,
6970
args.sandbox_profile_package,
7071
swift_flags,

0 commit comments

Comments
 (0)