From 289bfb8592747c45fb4b5dce1ca99ce9de9ab830 Mon Sep 17 00:00:00 2001 From: nilsreichardt Date: Sat, 24 May 2025 17:52:17 +0200 Subject: [PATCH 1/4] Add `--suite-load-timeout` option --- pkgs/test/test/utils.dart | 4 ++++ .../lib/src/runner/configuration.dart | 10 +++++++++ .../lib/src/runner/configuration/args.dart | 5 +++++ pkgs/test_core/lib/src/runner/load_suite.dart | 14 +++--------- pkgs/test_core/lib/src/runner/suite.dart | 22 ++++++++++++++++--- 5 files changed, 41 insertions(+), 14 deletions(-) diff --git a/pkgs/test/test/utils.dart b/pkgs/test/test/utils.dart index 23d5b7e14..229167150 100644 --- a/pkgs/test/test/utils.dart +++ b/pkgs/test/test/utils.dart @@ -157,6 +157,7 @@ SuiteConfiguration suiteConfiguration( Map? tags, Map? onPlatform, bool? ignoreTimeouts, + Timeout? suiteLoadTimeout, // Test-level configuration Timeout? timeout, @@ -179,6 +180,7 @@ SuiteConfiguration suiteConfiguration( tags: tags, onPlatform: onPlatform, ignoreTimeouts: ignoreTimeouts, + suiteLoadTimeout: suiteLoadTimeout, timeout: timeout, verboseTrace: verboseTrace, chainStackTraces: chainStackTraces, @@ -212,6 +214,7 @@ Configuration configuration( Map? defineRuntimes, bool? noRetry, bool? ignoreTimeouts, + Timeout? suiteLoadTimeout, // Suite-level configuration bool? allowDuplicateTestNames, @@ -262,6 +265,7 @@ Configuration configuration( defineRuntimes: defineRuntimes, noRetry: noRetry, ignoreTimeouts: ignoreTimeouts, + suiteLoadTimeout: suiteLoadTimeout, allowDuplicateTestNames: allowDuplicateTestNames, allowTestRandomization: allowTestRandomization, jsTrace: jsTrace, diff --git a/pkgs/test_core/lib/src/runner/configuration.dart b/pkgs/test_core/lib/src/runner/configuration.dart index 8ba4aa810..b69ddacd4 100644 --- a/pkgs/test_core/lib/src/runner/configuration.dart +++ b/pkgs/test_core/lib/src/runner/configuration.dart @@ -287,6 +287,7 @@ class Configuration { required Map? tags, required Map? onPlatform, required bool? ignoreTimeouts, + required Timeout? suiteLoadTimeout, // Test-level configuration required Timeout? timeout, @@ -338,6 +339,7 @@ class Configuration { tags: tags, onPlatform: onPlatform, ignoreTimeouts: ignoreTimeouts, + suiteLoadTimeout: suiteLoadTimeout, // Test-level configuration timeout: timeout, @@ -396,6 +398,7 @@ class Configuration { Map? tags, Map? onPlatform, bool? ignoreTimeouts, + Timeout? suiteLoadTimeout, // Test-level configuration Timeout? timeout, @@ -445,6 +448,7 @@ class Configuration { tags: tags, onPlatform: onPlatform, ignoreTimeouts: ignoreTimeouts, + suiteLoadTimeout: suiteLoadTimeout, timeout: timeout, verboseTrace: verboseTrace, chainStackTraces: chainStackTraces, @@ -498,6 +502,7 @@ class Configuration { testRandomizeOrderingSeed: null, stopOnFirstFailure: null, ignoreTimeouts: null, + suiteLoadTimeout: null, allowDuplicateTestNames: null, allowTestRandomization: null, runSkipped: null, @@ -576,6 +581,7 @@ class Configuration { tags: null, onPlatform: null, ignoreTimeouts: null, + suiteLoadTimeout: null, timeout: null, verboseTrace: null, chainStackTraces: null, @@ -639,6 +645,7 @@ class Configuration { tags: null, onPlatform: null, ignoreTimeouts: null, + suiteLoadTimeout: null, timeout: null, verboseTrace: null, chainStackTraces: null, @@ -700,6 +707,7 @@ class Configuration { tags: null, onPlatform: null, ignoreTimeouts: null, + suiteLoadTimeout: null, timeout: null, verboseTrace: null, chainStackTraces: null, @@ -983,6 +991,7 @@ class Configuration { bool? noRetry, int? testRandomizeOrderingSeed, bool? ignoreTimeouts, + Timeout? suiteLoadTimeout, // Suite-level configuration bool? allowDuplicateTestNames, @@ -1051,6 +1060,7 @@ class Configuration { testOn: testOn, addTags: addTags, ignoreTimeouts: ignoreTimeouts, + suiteLoadTimeout: suiteLoadTimeout, )); return config._resolvePresets(); } diff --git a/pkgs/test_core/lib/src/runner/configuration/args.dart b/pkgs/test_core/lib/src/runner/configuration/args.dart index e981bedc3..a961a089f 100644 --- a/pkgs/test_core/lib/src/runner/configuration/args.dart +++ b/pkgs/test_core/lib/src/runner/configuration/args.dart @@ -100,6 +100,10 @@ final ArgParser _parser = (() { parser.addOption('timeout', help: 'The default test timeout. For example: 15s, 2x, none', defaultsTo: '30s'); + parser.addOption('suite-load-timeout', + help: 'The timeout for loading a test suite. Loading the test suite ' + 'includes compiling the test suite. For example: 15s, 2x, none', + defaultsTo: '12m'); parser.addFlag('ignore-timeouts', help: 'Ignore all timeouts (useful if debugging)', negatable: false); parser.addFlag('pause-after-load', @@ -333,6 +337,7 @@ class _Parser { shardIndex: shardIndex, totalShards: totalShards, timeout: _parseOption('timeout', Timeout.parse), + suiteLoadTimeout: _parseOption('suite-load-timeout', Timeout.parse), globalPatterns: patterns, compilerSelections: compilerSelections, runtimes: runtimes, diff --git a/pkgs/test_core/lib/src/runner/load_suite.dart b/pkgs/test_core/lib/src/runner/load_suite.dart index 633739afc..ec795a4e1 100644 --- a/pkgs/test_core/lib/src/runner/load_suite.dart +++ b/pkgs/test_core/lib/src/runner/load_suite.dart @@ -5,7 +5,6 @@ import 'dart:async'; import 'package:stack_trace/stack_trace.dart'; -import 'package:test_api/scaffolding.dart' show Timeout; import 'package:test_api/src/backend/group.dart'; // ignore: implementation_imports import 'package:test_api/src/backend/invoker.dart'; // ignore: implementation_imports import 'package:test_api/src/backend/metadata.dart'; // ignore: implementation_imports @@ -20,14 +19,6 @@ import 'plugin/environment.dart'; import 'runner_suite.dart'; import 'suite.dart'; -/// The timeout for loading a test suite. -/// -/// We want this to be long enough that even a very large application being -/// compiled with dart2js doesn't trigger it, but short enough that it fires -/// before the host kills it. For example, Google's Forge service has a -/// 15-minute timeout. -final _timeout = const Duration(minutes: 12); - /// A [Suite] emitted by a [Loader] that provides a test-like interface for /// loading a test file. /// @@ -150,8 +141,9 @@ class LoadSuite extends Suite implements RunnerSuite { void Function() body, this._suiteAndZone, {required bool ignoreTimeouts, String? path}) : super( - Group.root( - [LocalTest(name, Metadata(timeout: Timeout(_timeout)), body)]), + Group.root([ + LocalTest(name, Metadata(timeout: config.suiteLoadTimeout), body) + ]), platform, path: path, ignoreTimeouts: ignoreTimeouts); diff --git a/pkgs/test_core/lib/src/runner/suite.dart b/pkgs/test_core/lib/src/runner/suite.dart index b5ee0bea5..7d96db7be 100644 --- a/pkgs/test_core/lib/src/runner/suite.dart +++ b/pkgs/test_core/lib/src/runner/suite.dart @@ -57,7 +57,8 @@ final class SuiteConfiguration { tags: null, onPlatform: null, metadata: null, - ignoreTimeouts: null); + ignoreTimeouts: null, + suiteLoadTimeout: null); /// Whether or not duplicate test (or group) names are allowed within the same /// test suite. @@ -149,6 +150,11 @@ final class SuiteConfiguration { final bool? _ignoreTimeouts; bool get ignoreTimeouts => _ignoreTimeouts ?? false; + /// The timeout for loading a test suite. + final Timeout? _suiteLoadTimeout; + Timeout get suiteLoadTimeout => + _suiteLoadTimeout ?? const Timeout(Duration(minutes: 12)); + factory SuiteConfiguration( {required bool? allowDuplicateTestNames, required bool? allowTestRandomization, @@ -161,7 +167,7 @@ final class SuiteConfiguration { required Map? tags, required Map? onPlatform, required bool? ignoreTimeouts, - + required Timeout? suiteLoadTimeout, // Test-level configuration required Timeout? timeout, required bool? verboseTrace, @@ -184,6 +190,7 @@ final class SuiteConfiguration { tags: tags, onPlatform: onPlatform, ignoreTimeouts: ignoreTimeouts, + suiteLoadTimeout: suiteLoadTimeout, metadata: Metadata( timeout: timeout, verboseTrace: verboseTrace, @@ -212,6 +219,7 @@ final class SuiteConfiguration { Map? tags, Map? onPlatform, bool? ignoreTimeouts, + Timeout? suiteLoadTimeout, // Test-level configuration Timeout? timeout, @@ -234,6 +242,7 @@ final class SuiteConfiguration { tags: tags, onPlatform: onPlatform, ignoreTimeouts: ignoreTimeouts, + suiteLoadTimeout: suiteLoadTimeout, timeout: timeout, verboseTrace: verboseTrace, chainStackTraces: chainStackTraces, @@ -273,6 +282,7 @@ final class SuiteConfiguration { required Map? onPlatform, required Metadata? metadata, required bool? ignoreTimeouts, + required Timeout? suiteLoadTimeout, }) : _allowDuplicateTestNames = allowDuplicateTestNames, _allowTestRandomization = allowTestRandomization, _jsTrace = jsTrace, @@ -283,6 +293,7 @@ final class SuiteConfiguration { tags = _map(tags), onPlatform = _map(onPlatform), _ignoreTimeouts = ignoreTimeouts, + _suiteLoadTimeout = suiteLoadTimeout, _metadata = metadata ?? Metadata.empty; /// Creates a new [SuiteConfiguration] that takes its configuration from @@ -304,6 +315,7 @@ final class SuiteConfiguration { runtimes: null, compilerSelections: null, ignoreTimeouts: null, + suiteLoadTimeout: null, ); /// Returns an unmodifiable copy of [input]. @@ -348,7 +360,8 @@ final class SuiteConfiguration { tags: _mergeConfigMaps(tags, other.tags), onPlatform: _mergeConfigMaps(onPlatform, other.onPlatform), ignoreTimeouts: other._ignoreTimeouts ?? _ignoreTimeouts, - metadata: metadata.merge(other.metadata)); + metadata: metadata.merge(other.metadata), + suiteLoadTimeout: other._suiteLoadTimeout ?? _suiteLoadTimeout); return config._resolveTags(); } @@ -368,6 +381,7 @@ final class SuiteConfiguration { Map? tags, Map? onPlatform, bool? ignoreTimeouts, + Timeout? suiteLoadTimeout, // Test-level configuration Timeout? timeout, @@ -393,6 +407,7 @@ final class SuiteConfiguration { tags: tags ?? this.tags, onPlatform: onPlatform ?? this.onPlatform, ignoreTimeouts: ignoreTimeouts ?? _ignoreTimeouts, + suiteLoadTimeout: suiteLoadTimeout ?? _suiteLoadTimeout, metadata: _metadata.change( timeout: timeout, verboseTrace: verboseTrace, @@ -428,6 +443,7 @@ final class SuiteConfiguration { tags: tags, onPlatform: onPlatform, ignoreTimeouts: _ignoreTimeouts, + suiteLoadTimeout: suiteLoadTimeout, metadata: _metadata); } From 99346e1802eec760a6d6c4e8caad1502dbb93c06 Mon Sep 17 00:00:00 2001 From: Nate Bosch Date: Tue, 23 Sep 2025 01:57:28 +0000 Subject: [PATCH 2/4] Fix bad merge --- pkgs/test/test/utils.dart | 21 +++++++++++++++++++ .../lib/src/runner/configuration.dart | 16 ++++++++++++++ .../lib/src/runner/configuration/args.dart | 18 +++++++++++++++- 3 files changed, 54 insertions(+), 1 deletion(-) diff --git a/pkgs/test/test/utils.dart b/pkgs/test/test/utils.dart index 7bff6ba62..8787a042c 100644 --- a/pkgs/test/test/utils.dart +++ b/pkgs/test/test/utils.dart @@ -226,6 +226,8 @@ Configuration configuration({ String? reporter, Map? fileReporters, String? coverage, + String? coverageLcov, + bool? branchCoverage, int? concurrency, int? shardIndex, int? totalShards, @@ -240,6 +242,23 @@ Configuration configuration({ bool? noRetry, bool? ignoreTimeouts, Timeout? suiteLoadTimeout, + + // Suite-level configuration + bool? allowDuplicateTestNames, + bool? allowTestRandomization, + bool? jsTrace, + bool? runSkipped, + Iterable? dart2jsArgs, + String? precompiledPath, + Iterable? globalPatterns, + Iterable? compilerSelections, + Iterable? runtimes, + BooleanSelector? includeTags, + BooleanSelector? excludeTags, + Map? tags, + Map? onPlatform, + int? testRandomizeOrderingSeed, + // Test-level configuration Timeout? timeout, bool? verboseTrace, @@ -260,6 +279,8 @@ Configuration configuration({ reporter: reporter, fileReporters: fileReporters, coverage: coverage, + coverageLcov: coverageLcov, + branchCoverage: branchCoverage, concurrency: concurrency, shardIndex: shardIndex, totalShards: totalShards, diff --git a/pkgs/test_core/lib/src/runner/configuration.dart b/pkgs/test_core/lib/src/runner/configuration.dart index 6bb646901..5ed2653c1 100644 --- a/pkgs/test_core/lib/src/runner/configuration.dart +++ b/pkgs/test_core/lib/src/runner/configuration.dart @@ -327,6 +327,8 @@ class Configuration { reporter: reporter, fileReporters: fileReporters, coverage: coverage, + coverageLcov: coverageLcov, + branchCoverage: branchCoverage, concurrency: concurrency, shardIndex: shardIndex, totalShards: totalShards, @@ -441,6 +443,8 @@ class Configuration { reporter: reporter, fileReporters: fileReporters, coverage: coverage, + coverageLcov: coverageLcov, + branchCoverage: branchCoverage, concurrency: concurrency, shardIndex: shardIndex, totalShards: totalShards, @@ -510,6 +514,8 @@ class Configuration { reporter: null, fileReporters: null, coverage: null, + coverageLcov: null, + branchCoverage: null, concurrency: null, shardIndex: null, totalShards: null, @@ -575,6 +581,8 @@ class Configuration { reporter: null, fileReporters: null, coverage: null, + coverageLcov: null, + branchCoverage: null, concurrency: null, shardIndex: null, totalShards: null, @@ -643,6 +651,8 @@ class Configuration { color: null, configurationPath: null, coverage: null, + coverageLcov: null, + branchCoverage: null, shardIndex: null, totalShards: null, testSelections: null, @@ -705,6 +715,8 @@ class Configuration { reporter: null, fileReporters: null, coverage: null, + coverageLcov: null, + branchCoverage: null, concurrency: null, shardIndex: null, totalShards: null, @@ -1026,6 +1038,8 @@ class Configuration { String? reporter, Map? fileReporters, String? coverage, + String? coverageLcov, + bool? branchCoverage, int? concurrency, int? shardIndex, int? totalShards, @@ -1075,6 +1089,8 @@ class Configuration { reporter: reporter ?? _reporter, fileReporters: fileReporters ?? this.fileReporters, coverage: coverage ?? this.coverage, + coverageLcov: coverageLcov ?? this.coverageLcov, + branchCoverage: branchCoverage ?? _branchCoverage, concurrency: concurrency ?? _concurrency, shardIndex: shardIndex ?? this.shardIndex, totalShards: totalShards ?? this.totalShards, diff --git a/pkgs/test_core/lib/src/runner/configuration/args.dart b/pkgs/test_core/lib/src/runner/configuration/args.dart index bc1ad0380..1d672cf70 100644 --- a/pkgs/test_core/lib/src/runner/configuration/args.dart +++ b/pkgs/test_core/lib/src/runner/configuration/args.dart @@ -173,6 +173,20 @@ final ArgParser _parser = 'Implies --debug.', valueHelp: 'directory', ); + parser.addOption( + 'coverage-path', + help: + 'Gather coverage and output an lcov report to the specified file.\n' + 'Implies --debug.', + valueHelp: 'file', + ); + parser.addFlag( + 'branch-coverage', + help: + 'Include branch coverage information in the coverage report.\n' + 'Must be paired with --coverage or --coverage-path.', + negatable: false, + ); parser.addFlag( 'chain-stack-traces', help: @@ -452,7 +466,9 @@ class _Parser { precompiledPath: _ifParsed('precompiled'), reporter: reporter, fileReporters: _parseFileReporterOption(), - coverage: _ifParsed('coverage'), + coverage: coverageDir, + coverageLcov: coverageLcov, + branchCoverage: branchCoverage, concurrency: _parseOption('concurrency', int.parse), shardIndex: shardIndex, totalShards: totalShards, From b0f4a704c3ee19db388cd7511ecb94dcd4fc0fe3 Mon Sep 17 00:00:00 2001 From: Nate Bosch Date: Tue, 23 Sep 2025 02:14:46 +0000 Subject: [PATCH 3/4] Add config file support and a test --- .../runner/configuration/top_level_test.dart | 28 +++++++++++++++++++ .../lib/src/runner/configuration.dart | 3 +- .../lib/src/runner/configuration/load.dart | 2 ++ 3 files changed, 32 insertions(+), 1 deletion(-) diff --git a/pkgs/test/test/runner/configuration/top_level_test.dart b/pkgs/test/test/runner/configuration/top_level_test.dart index 815d01f6d..833dd50e8 100644 --- a/pkgs/test/test/runner/configuration/top_level_test.dart +++ b/pkgs/test/test/runner/configuration/top_level_test.dart @@ -448,6 +448,34 @@ void main() { await test.shouldExit(1); }); + test('uses the specified suite load timeout', () async { + await d + .file('dart_test.yaml', jsonEncode({'suite_load_timeout': '1s'})) + .create(); + + await d.file('test.dart', ''' + import 'dart:async'; + + import 'package:test/test.dart'; + + Future main() async { + await Future.delayed(Duration(seconds: 2)); + test('success', () {}); + } + ''').create(); + + var test = await runTest(['test.dart']); + expect( + test.stdout, + containsInOrder([ + 'loading test.dart [E]', + 'Test timed out after 1 seconds.', + '-1: Some tests failed.', + ]), + ); + await test.shouldExit(1); + }); + test('runs on the specified platforms', () async { await d .file( diff --git a/pkgs/test_core/lib/src/runner/configuration.dart b/pkgs/test_core/lib/src/runner/configuration.dart index 5ed2653c1..a1eb76690 100644 --- a/pkgs/test_core/lib/src/runner/configuration.dart +++ b/pkgs/test_core/lib/src/runner/configuration.dart @@ -493,6 +493,7 @@ class Configuration { required bool? verboseTrace, required bool? jsTrace, required Timeout? timeout, + required Timeout? suiteLoadTimeout, required Map? presets, required bool? chainStackTraces, required Iterable? foldTraceExcept, @@ -502,6 +503,7 @@ class Configuration { foldTraceOnly: foldTraceOnly, jsTrace: jsTrace, timeout: timeout, + suiteLoadTimeout: suiteLoadTimeout, verboseTrace: verboseTrace, chainStackTraces: chainStackTraces, help: null, @@ -529,7 +531,6 @@ class Configuration { testRandomizeOrderingSeed: null, stopOnFirstFailure: null, ignoreTimeouts: null, - suiteLoadTimeout: null, allowDuplicateTestNames: null, allowTestRandomization: null, runSkipped: null, diff --git a/pkgs/test_core/lib/src/runner/configuration/load.dart b/pkgs/test_core/lib/src/runner/configuration/load.dart index fc74fa008..31a4deec0 100644 --- a/pkgs/test_core/lib/src/runner/configuration/load.dart +++ b/pkgs/test_core/lib/src/runner/configuration/load.dart @@ -134,6 +134,7 @@ class _ConfigurationLoader { var jsTrace = _getBool('js_trace'); var timeout = _parseValue('timeout', Timeout.parse); + var suiteLoadTimeout = _parseValue('suite_load_timeout', Timeout.parse); var onPlatform = _getMap( 'on_platform', @@ -182,6 +183,7 @@ class _ConfigurationLoader { verboseTrace: verboseTrace, jsTrace: jsTrace, timeout: timeout, + suiteLoadTimeout: suiteLoadTimeout, presets: presets, chainStackTraces: chainStackTraces, foldTraceExcept: foldStackFrames['except'], From 189e5f3cfb518d1be54bb3102f93ebb4997da971 Mon Sep 17 00:00:00 2001 From: Nate Bosch Date: Tue, 23 Sep 2025 02:38:24 +0000 Subject: [PATCH 4/4] Change default to no timeout. Add changelog --- pkgs/test/CHANGELOG.md | 7 +++++-- pkgs/test_core/lib/src/runner/configuration/args.dart | 4 ++-- pkgs/test_core/lib/src/runner/suite.dart | 3 +-- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/pkgs/test/CHANGELOG.md b/pkgs/test/CHANGELOG.md index 2a9474861..8c9fce2f0 100644 --- a/pkgs/test/CHANGELOG.md +++ b/pkgs/test/CHANGELOG.md @@ -1,9 +1,12 @@ ## 1.27.0-wip -* Restrict to latest version of analyzer package. -* Require Dart 3.7 * Add `--coverage-path` and `--branch-coverage` options to `dart test`. +* Remove the default 12 minute timeout to compile and load test suites. +* Add a `--suite-load-timeout` argument to allow configuring a timeout for + compiling and loading individual test suites. * Serve dart2wasm source map files. +* Restrict to latest version of analyzer package. +* Require Dart 3.7 ## 1.26.3 diff --git a/pkgs/test_core/lib/src/runner/configuration/args.dart b/pkgs/test_core/lib/src/runner/configuration/args.dart index 1d672cf70..84582aa32 100644 --- a/pkgs/test_core/lib/src/runner/configuration/args.dart +++ b/pkgs/test_core/lib/src/runner/configuration/args.dart @@ -145,8 +145,8 @@ final ArgParser _parser = 'suite-load-timeout', help: 'The timeout for loading a test suite. Loading the test suite ' - 'includes compiling the test suite. For example: 15s, 2x, none', - defaultsTo: '12m', + 'includes compiling the test suite. For example: 15s, none', + defaultsTo: 'none', ); parser.addFlag( 'ignore-timeouts', diff --git a/pkgs/test_core/lib/src/runner/suite.dart b/pkgs/test_core/lib/src/runner/suite.dart index c3a454700..b753bed3f 100644 --- a/pkgs/test_core/lib/src/runner/suite.dart +++ b/pkgs/test_core/lib/src/runner/suite.dart @@ -156,8 +156,7 @@ final class SuiteConfiguration { /// The timeout for loading a test suite. final Timeout? _suiteLoadTimeout; - Timeout get suiteLoadTimeout => - _suiteLoadTimeout ?? const Timeout(Duration(minutes: 12)); + Timeout get suiteLoadTimeout => _suiteLoadTimeout ?? Timeout.none; factory SuiteConfiguration({ required bool? allowDuplicateTestNames,