Add support for parallel execution of test methods to JUnit Vintage engine (#4242)

New configuration parameters allow configuring whether test classes, 
methods, or both should be executed in parallel.

Resolves #4238.

---------

Co-authored-by: Marc Philipp <mail@marcphilipp.de>

Author: YongGoose

documentation/src/docs/asciidoc/release-notes/release-notes-5.12.0-RC2.adoc

@@ -1,7 +1,7 @@
 [[release-notes-5.12.0-RC2]]
 == 5.12.0-RC2
 
-*Date of Release:* ❓
+*Date of Release:* February 14, 2025
 
 *Scope:* Minor enhancements since JUnit 5.12.0-RC1.
 
@@ -64,4 +64,6 @@ repository on GitHub.
 [[release-notes-5.12.0-RC2-junit-vintage-new-features-and-improvements]]
 ==== New Features and Improvements
 
-* ❓
+* Support for executing test methods in parallel. Please refer to the
+  <<../user-guide/index.adoc#migrating-from-junit4-parallel-execution, User Guide>> for
+  more information.

documentation/src/docs/asciidoc/user-guide/migration-from-junit4.adoc

@@ -39,13 +39,20 @@ discovered tests before executing them (see <<running-tests>> for details).
 [[migrating-from-junit4-parallel-execution]]
 === Parallel Execution
 
-The JUnit Vintage test engine supports parallel execution of top-level test classes,
+The JUnit Vintage test engine supports parallel execution of top-level test classes and test methods,
 allowing existing JUnit 3 and JUnit 4 tests to benefit from improved performance through
 concurrent test execution. It can be enabled and configured using the following
 <<running-tests-config-params, configuration parameters>>:
 
 `junit.vintage.execution.parallel.enabled=true|false`::
-  Enable/disable parallel execution (defaults to `false`).
+  Enable/disable parallel execution (defaults to `false`). Requires opt-in for `classes`
+  or `methods` to be executed in parallel using the configuration parameters below.
+
+`junit.vintage.execution.parallel.classes=true|false`::
+  Enable/disable parallel execution of test classes (defaults to `false`).
+
+`junit.vintage.execution.parallel.methods=true|false`::
+  Enable/disable parallel execution of test methods (defaults to `false`).
 
 `junit.vintage.execution.parallel.pool-size=<number>`::
   Specifies the size of the thread pool to be used for parallel execution. By default, the
@@ -56,6 +63,8 @@ Example configuration in `junit-platform.properties`:
 [source,properties]
 ----
 junit.vintage.execution.parallel.enabled=true
+junit.vintage.execution.parallel.classes=true
+junit.vintage.execution.parallel.methods=true
 junit.vintage.execution.parallel.pool-size=4
 ----
 

junit-vintage-engine/src/main/java/org/junit/vintage/engine/Constants.java

@@ -46,6 +46,28 @@ public final class Constants {
 	@API(status = EXPERIMENTAL, since = "5.12")
 	public static final String PARALLEL_POOL_SIZE = "junit.vintage.execution.parallel.pool-size";
 
+	/**
+	 * Indicates whether parallel execution is enabled for test classes in the JUnit Vintage engine.
+	 *
+	 * <p>Set this property to {@code true} to enable parallel execution of test classes.
+	 * Defaults to {@code false}.
+	 *
+	 * @since 5.12
+	 */
+	@API(status = EXPERIMENTAL, since = "5.12")
+	public static final String PARALLEL_CLASS_EXECUTION = "junit.vintage.execution.parallel.classes";
+
+	/**
+	 * Indicates whether parallel execution is enabled for test methods in the JUnit Vintage engine.
+	 *
+	 * <p>Set this property to {@code true} to enable parallel execution of test methods.
+	 * Defaults to {@code false}.
+	 *
+	 * @since 5.12
+	 */
+	@API(status = EXPERIMENTAL, since = "5.12")
+	public static final String PARALLEL_METHOD_EXECUTION = "junit.vintage.execution.parallel.methods";
+
 	private Constants() {
 		/* no-op */
 	}

junit-vintage-engine/src/main/java/org/junit/vintage/engine/VintageTestEngine.java

@@ -12,34 +12,20 @@
 
 import static org.apiguardian.api.API.Status.INTERNAL;
 import static org.junit.platform.engine.TestExecutionResult.successful;
-import static org.junit.vintage.engine.Constants.PARALLEL_EXECUTION_ENABLED;
-import static org.junit.vintage.engine.Constants.PARALLEL_POOL_SIZE;
 import static org.junit.vintage.engine.descriptor.VintageTestDescriptor.ENGINE_ID;
 
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
 import java.util.Optional;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
 
 import org.apiguardian.api.API;
-import org.junit.platform.commons.logging.Logger;
-import org.junit.platform.commons.logging.LoggerFactory;
-import org.junit.platform.commons.util.ExceptionUtils;
 import org.junit.platform.engine.EngineDiscoveryRequest;
 import org.junit.platform.engine.EngineExecutionListener;
 import org.junit.platform.engine.ExecutionRequest;
 import org.junit.platform.engine.TestDescriptor;
 import org.junit.platform.engine.TestEngine;
 import org.junit.platform.engine.UniqueId;
-import org.junit.vintage.engine.descriptor.RunnerTestDescriptor;
 import org.junit.vintage.engine.descriptor.VintageEngineDescriptor;
 import org.junit.vintage.engine.discovery.VintageDiscoverer;
-import org.junit.vintage.engine.execution.RunnerExecutor;
+import org.junit.vintage.engine.execution.VintageExecutor;
 
 /**
  * The JUnit Vintage {@link TestEngine}.
@@ -49,11 +35,6 @@
 @API(status = INTERNAL, since = "4.12")
 public final class VintageTestEngine implements TestEngine {
 
-	private static final Logger logger = LoggerFactory.getLogger(VintageTestEngine.class);
-
-	private static final int DEFAULT_THREAD_POOL_SIZE = Runtime.getRuntime().availableProcessors();
-	private static final int SHUTDOWN_TIMEOUT_SECONDS = 30;
-
 	@Override
 	public String getId() {
 		return ENGINE_ID;
@@ -86,96 +67,7 @@ public void execute(ExecutionRequest request) {
 		EngineExecutionListener engineExecutionListener = request.getEngineExecutionListener();
 		VintageEngineDescriptor engineDescriptor = (VintageEngineDescriptor) request.getRootTestDescriptor();
 		engineExecutionListener.executionStarted(engineDescriptor);
-		executeAllChildren(engineDescriptor, engineExecutionListener, request);
+		new VintageExecutor(engineDescriptor, engineExecutionListener, request).executeAllChildren();
 		engineExecutionListener.executionFinished(engineDescriptor, successful());
 	}
-
-	private void executeAllChildren(VintageEngineDescriptor engineDescriptor,
-			EngineExecutionListener engineExecutionListener, ExecutionRequest request) {
-		boolean parallelExecutionEnabled = getParallelExecutionEnabled(request);
-
-		if (parallelExecutionEnabled) {
-			if (executeInParallel(engineDescriptor, engineExecutionListener, request)) {
-				Thread.currentThread().interrupt();
-			}
-		}
-		else {
-			executeSequentially(engineDescriptor, engineExecutionListener);
-		}
-	}
-
-	private boolean executeInParallel(VintageEngineDescriptor engineDescriptor,
-			EngineExecutionListener engineExecutionListener, ExecutionRequest request) {
-		ExecutorService executorService = Executors.newFixedThreadPool(getThreadPoolSize(request));
-		RunnerExecutor runnerExecutor = new RunnerExecutor(engineExecutionListener);
-
-		List<CompletableFuture<Void>> futures = new ArrayList<>();
-		for (Iterator<TestDescriptor> iterator = engineDescriptor.getModifiableChildren().iterator(); iterator.hasNext();) {
-			TestDescriptor descriptor = iterator.next();
-			CompletableFuture<Void> future = CompletableFuture.runAsync(() -> {
-				runnerExecutor.execute((RunnerTestDescriptor) descriptor);
-			}, executorService);
-
-			futures.add(future);
-			iterator.remove();
-		}
-
-		CompletableFuture<Void> allOf = CompletableFuture.allOf(futures.toArray(new CompletableFuture<?>[0]));
-		boolean wasInterrupted = false;
-		try {
-			allOf.get();
-		}
-		catch (InterruptedException e) {
-			logger.warn(e, () -> "Interruption while waiting for parallel test execution to finish");
-			wasInterrupted = true;
-		}
-		catch (ExecutionException e) {
-			throw ExceptionUtils.throwAsUncheckedException(e.getCause());
-		}
-		finally {
-			shutdownExecutorService(executorService);
-		}
-		return wasInterrupted;
-	}
-
-	private void shutdownExecutorService(ExecutorService executorService) {
-		try {
-			executorService.shutdown();
-			if (!executorService.awaitTermination(SHUTDOWN_TIMEOUT_SECONDS, TimeUnit.SECONDS)) {
-				logger.warn(() -> "Executor service did not terminate within the specified timeout");
-				executorService.shutdownNow();
-			}
-		}
-		catch (InterruptedException e) {
-			logger.warn(e, () -> "Interruption while waiting for executor service to shut down");
-			Thread.currentThread().interrupt();
-		}
-	}
-
-	private void executeSequentially(VintageEngineDescriptor engineDescriptor,
-			EngineExecutionListener engineExecutionListener) {
-		RunnerExecutor runnerExecutor = new RunnerExecutor(engineExecutionListener);
-		for (Iterator<TestDescriptor> iterator = engineDescriptor.getModifiableChildren().iterator(); iterator.hasNext();) {
-			runnerExecutor.execute((RunnerTestDescriptor) iterator.next());
-			iterator.remove();
-		}
-	}
-
-	private boolean getParallelExecutionEnabled(ExecutionRequest request) {
-		return request.getConfigurationParameters().getBoolean(PARALLEL_EXECUTION_ENABLED).orElse(false);
-	}
-
-	private int getThreadPoolSize(ExecutionRequest request) {
-		Optional<String> poolSize = request.getConfigurationParameters().get(PARALLEL_POOL_SIZE);
-		if (poolSize.isPresent()) {
-			try {
-				return Integer.parseInt(poolSize.get());
-			}
-			catch (NumberFormatException e) {
-				logger.warn(() -> "Invalid value for parallel pool size: " + poolSize.get());
-			}
-		}
-		return DEFAULT_THREAD_POOL_SIZE;
-	}
-
 }

junit-vintage-engine/src/main/java/org/junit/vintage/engine/descriptor/RunnerTestDescriptor.java

@@ -18,6 +18,11 @@
 import java.util.List;
 import java.util.Optional;
 import java.util.Set;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.Consumer;
 
 import org.apiguardian.api.API;
@@ -26,12 +31,16 @@
 import org.junit.platform.commons.logging.LoggerFactory;
 import org.junit.platform.engine.UniqueId;
 import org.junit.platform.engine.support.descriptor.ClassSource;
+import org.junit.platform.engine.support.hierarchical.OpenTest4JAwareThrowableCollector;
+import org.junit.platform.engine.support.hierarchical.ThrowableCollector;
 import org.junit.runner.Description;
 import org.junit.runner.Request;
 import org.junit.runner.Runner;
 import org.junit.runner.manipulation.Filter;
 import org.junit.runner.manipulation.Filterable;
 import org.junit.runner.manipulation.NoTestsRemainException;
+import org.junit.runners.ParentRunner;
+import org.junit.runners.model.RunnerScheduler;
 
 /**
  * @since 4.12
@@ -161,6 +170,47 @@ public boolean isIgnored() {
 		return ignored;
 	}
 
+	public void setExecutorService(ExecutorService executorService) {
+		Runner runner = getRunnerToReport();
+		if (runner instanceof ParentRunner) {
+			((ParentRunner<?>) runner).setScheduler(new RunnerScheduler() {
+
+				private final List<Future<?>> futures = new CopyOnWriteArrayList<>();
+
+				@Override
+				public void schedule(Runnable childStatement) {
+					futures.add(executorService.submit(childStatement));
+				}
+
+				@Override
+				public void finished() {
+					ThrowableCollector collector = new OpenTest4JAwareThrowableCollector();
+					AtomicBoolean wasInterrupted = new AtomicBoolean(false);
+					for (Future<?> future : futures) {
+						collector.execute(() -> {
+							// We're calling `Future.get()` individually to allow for work stealing
+							// in case `ExecutorService` is a `ForkJoinPool`
+							try {
+								future.get();
+							}
+							catch (ExecutionException e) {
+								throw e.getCause();
+							}
+							catch (InterruptedException e) {
+								wasInterrupted.set(true);
+							}
+						});
+					}
+					collector.assertEmpty();
+					if (wasInterrupted.get()) {
+						logger.warn(() -> "Interrupted while waiting for runner to finish");
+						Thread.currentThread().interrupt();
+					}
+				}
+			});
+		}
+	}
+
 	private static class ExcludeDescriptionFilter extends Filter {
 
 		private final Description description;