diff --git a/Sources/Testing/ABI/EntryPoints/EntryPoint.swift b/Sources/Testing/ABI/EntryPoints/EntryPoint.swift index f4f1a751c..0b824394c 100644 --- a/Sources/Testing/ABI/EntryPoints/EntryPoint.swift +++ b/Sources/Testing/ABI/EntryPoints/EntryPoint.swift @@ -210,6 +210,9 @@ public struct __CommandLineArguments_v0: Sendable { /// The value of the `--parallel` or `--no-parallel` argument. public var parallel: Bool? + /// The maximum number of test tasks to run in parallel. + public var experimentalMaximumParallelizationWidth: Int? + /// The value of the `--symbolicate-backtraces` argument. public var symbolicateBacktraces: String? @@ -336,6 +339,7 @@ extension __CommandLineArguments_v0: Codable { enum CodingKeys: String, CodingKey { case listTests case parallel + case experimentalMaximumParallelizationWidth case symbolicateBacktraces case verbose case veryVerbose @@ -485,6 +489,11 @@ func parseCommandLineArguments(from args: [String]) throws -> __CommandLineArgum if args.contains("--no-parallel") { result.parallel = false } + if let maximumParallelizationWidth = args.argumentValue(forLabel: "--experimental-maximum-parallelization-width").flatMap(Int.init) + ?? Environment.variable(named: "SWT_EXPERIMENTAL_MAXIMUM_PARALLELIZATION_WIDTH").flatMap(Int.init) { + // TODO: decide if we want to repurpose --num-workers for this use case? + result.experimentalMaximumParallelizationWidth = maximumParallelizationWidth + } // Whether or not to symbolicate backtraces in the event stream. if let symbolicateBacktraces = args.argumentValue(forLabel: "--symbolicate-backtraces") { @@ -546,6 +555,10 @@ public func configurationForEntryPoint(from args: __CommandLineArguments_v0) thr // Parallelization (on by default) configuration.isParallelizationEnabled = args.parallel ?? true + if let maximumParallelizationWidth = args.experimentalMaximumParallelizationWidth { + try! FileHandle.stderr.write("MAX WIDTH: \(maximumParallelizationWidth)\n") + configuration.maximumParallelizationWidth = maximumParallelizationWidth + } // Whether or not to symbolicate backtraces in the event stream. if let symbolicateBacktraces = args.symbolicateBacktraces { diff --git a/Sources/Testing/CMakeLists.txt b/Sources/Testing/CMakeLists.txt index bebf05eb9..ebcfe4e21 100644 --- a/Sources/Testing/CMakeLists.txt +++ b/Sources/Testing/CMakeLists.txt @@ -90,6 +90,7 @@ add_library(Testing Support/Graph.swift Support/JSON.swift Support/Locked.swift + Support/Serializer.swift Support/VersionNumber.swift Support/Versions.swift Discovery+Macro.swift diff --git a/Sources/Testing/Running/Configuration.swift b/Sources/Testing/Running/Configuration.swift index 12b8827de..61dc2dd37 100644 --- a/Sources/Testing/Running/Configuration.swift +++ b/Sources/Testing/Running/Configuration.swift @@ -8,6 +8,8 @@ // See https://swift.org/CONTRIBUTORS.txt for Swift project authors // +private import _TestingInternals + /// A type containing settings for preparing and running tests. @_spi(ForToolsIntegrationOnly) public struct Configuration: Sendable { @@ -20,6 +22,18 @@ public struct Configuration: Sendable { /// Whether or not to parallelize the execution of tests and test cases. public var isParallelizationEnabled: Bool = true + /// The maximum width of parallelization. + /// + /// The value of this property determines how many tests (or rather, test + /// cases) will run in parallel. The default value of this property is equal + /// to twice the number of CPU cores reported by the operating system, or + /// `Int.max` if that value is not available. + /// + /// If the value of ``isParallelizationEnabled`` is `false`, this property has + /// no effect. + @_spi(Experimental) + public var maximumParallelizationWidth: Int = cpuCoreCount.map { max(1, $0) * 2 } ?? .max + /// How to symbolicate backtraces captured during a test run. /// /// If the value of this property is not `nil`, symbolication will be diff --git a/Sources/Testing/Running/Runner.swift b/Sources/Testing/Running/Runner.swift index 1cedf6182..f67d2446f 100644 --- a/Sources/Testing/Running/Runner.swift +++ b/Sources/Testing/Running/Runner.swift @@ -66,6 +66,12 @@ extension Runner { .current ?? .init() } + /// Context to apply to a test run. + struct Context: Sendable { + /// A serializer used to reduce parallelism among test cases. + var testCaseSerializer: Serializer + } + /// Apply the custom scope for any test scope providers of the traits /// associated with a specified test by calling their /// ``TestScoping/provideScope(for:testCase:performing:)`` function. @@ -179,6 +185,7 @@ extension Runner { /// /// - Parameters: /// - stepGraph: The subgraph whose root value, a step, is to be run. + /// - context: Context for the test run. /// /// - Throws: Whatever is thrown from the test body. Thrown errors are /// normally reported as test failures. @@ -193,7 +200,7 @@ extension Runner { /// ## See Also /// /// - ``Runner/run()`` - private static func _runStep(atRootOf stepGraph: Graph) async throws { + private static func _runStep(atRootOf stepGraph: Graph, context: Context) async throws { // Whether to send a `.testEnded` event at the end of running this step. // Some steps' actions may not require a final event to be sent — for // example, a skip event only sends `.testSkipped`. @@ -250,18 +257,18 @@ extension Runner { try await _applyScopingTraits(for: step.test, testCase: nil) { // Run the test function at this step (if one is present.) if let testCases = step.test.testCases { - await _runTestCases(testCases, within: step) + await _runTestCases(testCases, within: step, context: context) } // Run the children of this test (i.e. the tests in this suite.) - try await _runChildren(of: stepGraph) + try await _runChildren(of: stepGraph, context: context) } } } } else { // There is no test at this node in the graph, so just skip down to the // child nodes. - try await _runChildren(of: stepGraph) + try await _runChildren(of: stepGraph, context: context) } } @@ -286,10 +293,11 @@ extension Runner { /// - Parameters: /// - stepGraph: The subgraph whose root value, a step, will be used to /// find children to run. + /// - context: Context for the test run. /// /// - Throws: Whatever is thrown from the test body. Thrown errors are /// normally reported as test failures. - private static func _runChildren(of stepGraph: Graph) async throws { + private static func _runChildren(of stepGraph: Graph, context: Context) async throws { let childGraphs = if _configuration.isParallelizationEnabled { // Explicitly shuffle the steps to help detect accidental dependencies // between tests due to their ordering. @@ -331,7 +339,7 @@ extension Runner { // Run the child nodes. try await _forEach(in: childGraphs.lazy.map(\.value), namingTasksWith: taskNamer) { childGraph in - try await _runStep(atRootOf: childGraph) + try await _runStep(atRootOf: childGraph, context: context) } } @@ -340,12 +348,15 @@ extension Runner { /// - Parameters: /// - testCases: The test cases to be run. /// - step: The runner plan step associated with this test case. + /// - context: Context for the test run. /// /// If parallelization is supported and enabled, the generated test cases will /// be run in parallel using a task group. - private static func _runTestCases(_ testCases: some Sequence, within step: Plan.Step) async { + private static func _runTestCases(_ testCases: some Sequence, within step: Plan.Step, context: Context) async { + let configuration = _configuration + // Apply the configuration's test case filter. - let testCaseFilter = _configuration.testCaseFilter + let testCaseFilter = configuration.testCaseFilter let testCases = testCases.lazy.filter { testCase in testCaseFilter(testCase, step.test) } @@ -359,7 +370,11 @@ extension Runner { } await _forEach(in: testCases.enumerated(), namingTasksWith: taskNamer) { _, testCase in - await _runTestCase(testCase, within: step) + if configuration.isParallelizationEnabled { + await context.testCaseSerializer.run { await _runTestCase(testCase, within: step, context: context) } + } else { + await _runTestCase(testCase, within: step, context: context) + } } } @@ -368,10 +383,11 @@ extension Runner { /// - Parameters: /// - testCase: The test case to run. /// - step: The runner plan step associated with this test case. + /// - context: Context for the test run. /// /// This function sets ``Test/Case/current``, then invokes the test case's /// body closure. - private static func _runTestCase(_ testCase: Test.Case, within step: Plan.Step) async { + private static func _runTestCase(_ testCase: Test.Case, within step: Plan.Step, context: Context) async { let configuration = _configuration Event.post(.testCaseStarted, for: (step.test, testCase), configuration: configuration) @@ -428,6 +444,14 @@ extension Runner { eventHandler(event, context) } + // Context to pass into the test run. We intentionally don't pass the Runner + // itself (implicitly as `self` nor as an argument) because we don't want to + // accidentally depend on e.g. the `configuration` property rather than the + // current configuration. + let context = Context( + testCaseSerializer: Serializer(maximumWidth: runner.configuration.maximumParallelizationWidth) + ) + await Configuration.withCurrent(runner.configuration) { // Post an event for every test in the test plan being run. These events // are turned into JSON objects if JSON output is enabled. @@ -454,7 +478,7 @@ extension Runner { taskAction = "running iteration #\(iterationIndex + 1)" } _ = taskGroup.addTaskUnlessCancelled(name: decorateTaskName("test run", withAction: taskAction)) { - try? await _runStep(atRootOf: runner.plan.stepGraph) + try? await _runStep(atRootOf: runner.plan.stepGraph, context: context) } await taskGroup.waitForAll() } diff --git a/Sources/Testing/Support/Serializer.swift b/Sources/Testing/Support/Serializer.swift new file mode 100644 index 000000000..80ed187b0 --- /dev/null +++ b/Sources/Testing/Support/Serializer.swift @@ -0,0 +1,86 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024–2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +private import _TestingInternals + +/// The number of CPU cores on the current system, or `nil` if that +/// information is not available. +var cpuCoreCount: Int? { +#if SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(OpenBSD) || os(Android) + return Int(sysconf(Int32(_SC_NPROCESSORS_CONF))) +#elseif os(Windows) + var siInfo = SYSTEM_INFO() + GetSystemInfo(&siInfo) + return Int(siInfo.dwNumberOfProcessors) +#elseif os(WASI) + return 1 +#else +#warning("Platform-specific implementation missing: CPU core count unavailable") + return nil +#endif +} + +/// A type whose instances can run a series of work items in strict order. +/// +/// When a work item is scheduled on an instance of this type, it runs after any +/// previously-scheduled work items. If it suspends, subsequently-scheduled work +/// items do not start running; they must wait until the suspended work item +/// either returns or throws an error. +/// +/// This type is not part of the public interface of the testing library. +final actor Serializer { + /// The maximum number of work items that may run concurrently. + nonisolated let maximumWidth: Int + + /// The number of scheduled work items, including any currently running. + private var _currentWidth = 0 + + /// Continuations for any scheduled work items that haven't started yet. + private var _continuations = [CheckedContinuation]() + + init(maximumWidth: Int = 1) { + self.maximumWidth = maximumWidth + } + + /// Run a work item serially after any previously-scheduled work items. + /// + /// - Parameters: + /// - workItem: A closure to run. + /// + /// - Returns: Whatever is returned from `workItem`. + /// + /// - Throws: Whatever is thrown by `workItem`. + func run(_ workItem: @isolated(any) @Sendable () async throws -> R) async rethrows -> R where R: Sendable { + _currentWidth += 1 + defer { + // Resume the next scheduled closure. + if !_continuations.isEmpty { + let continuation = _continuations.removeFirst() + continuation.resume() + } + + _currentWidth -= 1 + } + + await withCheckedContinuation { continuation in + if _currentWidth <= maximumWidth { + // Nothing else was scheduled, so we can resume immediately. + continuation.resume() + } else { + // Something was scheduled, so add the continuation to the + // list. When it resumes, we can run. + _continuations.append(continuation) + } + } + + return try await workItem() + } +} +