Add --coverage-path and --branch-coverage options (#2517)

Author: liamappelbe

pkgs/test/CHANGELOG.md

@@ -1,7 +1,8 @@
-## 1.26.4-wip
+## 1.27.0-wip
 
 * Restrict to latest version of analyzer package.
 * Require Dart 3.7
+* Add `--coverage-path` and `--branch-coverage` options to `dart test`.
 
 ## 1.26.3
 

pkgs/test/README.md

@@ -257,36 +257,25 @@ The available options for the `--reporter` flag are:
 
 ### Collecting Code Coverage
 
-To collect code coverage, you can run tests with the `--coverage <directory>`
-argument. The directory specified can be an absolute or relative path.
-If a directory does not exist at the path specified, a directory will be
-created. If a directory does exist, files may be overwritten with the latest
-coverage data, if they conflict.
-
-This option will enable code coverage collection on a suite-by-suite basis,
-and the resulting coverage files will be outputted in the directory specified.
-The files can then be formatted using the `package:coverage`
-`format_coverage` executable.
+To collect code coverage, you can run tests with the `--coverage-path <file>`
+argument, which outputs a LCOV report to the given file path. The file
+specified can be an absolute or relative path. If a directory does not exist at
+the path specified, a directory will be created. If a directory does exist,
+files may be overwritten with the latest coverage data, if they conflict.
 
 Coverage gathering is currently only implemented for tests run on the Dart VM or
 Chrome.
 
-Here's an example of how to run tests and format the collected coverage to LCOV:
+Here's an example of how to run tests and collect coverage:
 
 ```shell
-## Run Dart tests and output them at directory `./coverage`:
-dart run test --coverage=./coverage
-
-## Activate package `coverage` (if needed):
-dart pub global activate coverage
-
-## Format collected coverage to LCOV (only for directory "lib")
-dart pub global run coverage:format_coverage --packages=.dart_tool/package_config.json --report-on=lib --lcov -o ./coverage/lcov.info -i ./coverage
+## Run Dart tests and output coverage info to `./coverage/lcov.info`:
+dart run test --coverage-path=./coverage/lcov.info
 
-## Generate LCOV report:
+## Generate a human readable report:
 genhtml -o ./coverage/report ./coverage/lcov.info
 
-## Open the HTML coverage report:
+## Open the coverage report:
 open ./coverage/report/index.html
 ```
 

pkgs/test/pubspec.yaml

@@ -1,5 +1,5 @@
 name: test
-version: 1.26.4-wip
+version: 1.27.0-wip
 description: >-
   A full featured library for writing and running Dart tests across platforms.
 repository: https://github.com/dart-lang/test/tree/master/pkgs/test
@@ -14,7 +14,7 @@ dependencies:
   async: ^2.5.0
   boolean_selector: ^2.1.0
   collection: ^1.15.0
-  coverage: ^1.0.1
+  coverage: ^1.15.0
   http_multi_server: ^3.0.0
   io: ^1.0.0
   js: '>=0.6.4 <0.8.0'

pkgs/test/test/io.dart

@@ -21,14 +21,6 @@ final Future<String> packageDir = Isolate.resolvePackageUri(
   return dir;
 });
 
-/// The path to the `pub` executable in the current Dart SDK.
-final _pubPath = p.absolute(
-  p.join(
-    p.dirname(Platform.resolvedExecutable),
-    Platform.isWindows ? 'pub.bat' : 'pub',
-  ),
-);
-
 /// The platform-specific message emitted when a nonexistent file is loaded.
 final String noSuchFileMessage =
     Platform.isWindows
@@ -174,8 +166,8 @@ Future<TestProcess> runPub(
   Map<String, String>? environment,
 }) {
   return TestProcess.start(
-    _pubPath,
-    args,
+    p.absolute(Platform.resolvedExecutable),
+    ['pub', ...args],
     workingDirectory: d.sandbox,
     environment: environment,
     description: 'pub ${args.first}',

pkgs/test/test/runner/coverage_test.dart

@@ -8,6 +8,7 @@ library;
 import 'dart:convert';
 import 'dart:io';
 
+import 'package:coverage/coverage.dart';
 import 'package:path/path.dart' as p;
 import 'package:test/test.dart';
 import 'package:test_descriptor/test_descriptor.dart' as d;
@@ -20,55 +21,136 @@ void main() {
 
   group('with the --coverage flag,', () {
     late Directory coverageDirectory;
+    late d.DirectoryDescriptor packageDirectory;
 
-    Future<void> validateCoverage(TestProcess test, String coveragePath) async {
+    Future<void> validateTest(TestProcess test) async {
       expect(test.stdout, emitsThrough(contains('+1: All tests passed!')));
       await test.shouldExit(0);
+    }
+
+    Future<Map<String, HitMap>> validateCoverage(
+      TestProcess test,
+      String coveragePath,
+    ) async {
+      await validateTest(test);
 
       final coverageFile = File(p.join(coverageDirectory.path, coveragePath));
       final coverage = await coverageFile.readAsString();
-      final jsonCoverage = json.decode(coverage);
-      expect(jsonCoverage['coverage'], isNotEmpty);
+      final jsonCoverage = json.decode(coverage)['coverage'] as List<dynamic>;
+      expect(jsonCoverage, isNotEmpty);
+
+      return HitMap.parseJson(jsonCoverage.cast<Map<String, dynamic>>());
     }
 
     setUp(() async {
-      await d.file('test.dart', '''
-        import 'package:test/test.dart';
-
-        void main() {
-          test("test 1", () {
-            expect(true, isTrue);
-          });
-        }
-      ''').create();
-
       coverageDirectory = await Directory.systemTemp.createTemp(
         'test_coverage',
       );
+
+      packageDirectory = d.dir(d.sandbox, [
+        d.dir('lib', [
+          d.file('calculate.dart', '''
+            int calculate(int x) {
+              if (x % 2 == 0) {
+                return x * 2;
+              } else {
+                return x * 3;
+              }
+            }
+          '''),
+        ]),
+        d.dir('test', [
+          d.file('test.dart', '''
+            import 'package:fake_package/calculate.dart';
+            import 'package:test/test.dart';
+
+            void main() {
+              test('test 1', () {
+                expect(calculate(6), 12);
+              });
+            }
+          '''),
+        ]),
+        d.file('pubspec.yaml', '''
+name: fake_package
+version: 1.0.0
+environment:
+  sdk: ^3.5.0
+dev_dependencies:
+  test: ^1.26.2
+        '''),
+      ]);
+      await packageDirectory.create();
     });
 
     tearDown(() async {
       await coverageDirectory.delete(recursive: true);
     });
 
     test('gathers coverage for VM tests', () async {
+      await (await runPub(['get'])).shouldExit(0);
       var test = await runTest([
         '--coverage',
         coverageDirectory.path,
-        'test.dart',
-      ]);
-      await validateCoverage(test, 'test.dart.vm.json');
+        'test/test.dart',
+      ], packageConfig: p.join(d.sandbox, '.dart_tool/package_config.json'));
+      final coverage = await validateCoverage(test, 'test/test.dart.vm.json');
+      final hitmap = coverage['package:fake_package/calculate.dart']!;
+      expect(hitmap.lineHits, {1: 1, 2: 2, 3: 1, 5: 0});
+      expect(hitmap.funcHits, isNull);
+      expect(hitmap.branchHits, isNull);
+    });
+
+    test('gathers branch coverage for VM tests', () async {
+      await (await runPub(['get'])).shouldExit(0);
+      var test = await runTest(
+        [
+          '--coverage',
+          coverageDirectory.path,
+          '--branch-coverage',
+          'test/test.dart',
+        ],
+        vmArgs: ['--branch-coverage'],
+        packageConfig: p.join(d.sandbox, '.dart_tool/package_config.json'),
+      );
+      final coverage = await validateCoverage(test, 'test/test.dart.vm.json');
+      final hitmap = coverage['package:fake_package/calculate.dart']!;
+      expect(hitmap.lineHits, {1: 1, 2: 2, 3: 1, 5: 0});
+      expect(hitmap.funcHits, isNull);
+      expect(hitmap.branchHits, {1: 1, 2: 1, 4: 0});
+    });
+
+    test('gathers lcov coverage for VM tests', () async {
+      await (await runPub(['get'])).shouldExit(0);
+      final lcovFile = p.join(coverageDirectory.path, 'lcov.info');
+      var test = await runTest([
+        '--coverage-path',
+        lcovFile,
+        'test/test.dart',
+      ], packageConfig: p.join(d.sandbox, '.dart_tool/package_config.json'));
+      await validateTest(test);
+      expect(File(lcovFile).readAsStringSync(), '''
+SF:${p.join(d.sandbox, 'lib', 'calculate.dart')}
+DA:1,1
+DA:2,2
+DA:3,1
+DA:5,0
+LF:4
+LH:3
+end_of_record
+''');
     });
 
     test('gathers coverage for Chrome tests', () async {
+      await (await runPub(['get'])).shouldExit(0);
       var test = await runTest([
         '--coverage',
         coverageDirectory.path,
-        'test.dart',
+        'test/test.dart',
         '-p',
         'chrome',
-      ]);
-      await validateCoverage(test, 'test.dart.chrome.json');
+      ], packageConfig: p.join(d.sandbox, '.dart_tool/package_config.json'));
+      await validateCoverage(test, 'test/test.dart.chrome.json');
     });
 
     test(

pkgs/test/test/runner/runner_test.dart

@@ -92,6 +92,10 @@ $_runtimeCompilers
     --debug                           Run the VM and Chrome tests in debug mode.
     --coverage=<directory>            Gather coverage and output it to the specified directory.
                                       Implies --debug.
+    --coverage-path=<file>            Gather coverage and output an lcov report to the specified file.
+                                      Implies --debug.
+    --branch-coverage                 Include branch coverage information in the coverage report.
+                                      Must be paired with --coverage or --coverage-path.
     --[no-]chain-stack-traces         Use chained stack traces to provide greater exception details
                                       especially for asynchronous code. It may be useful to disable
                                       to provide improved test performance but at the cost of

pkgs/test/test/utils.dart

@@ -138,6 +138,7 @@ Engine declareEngine(
   void Function() body, {
   bool runSkipped = false,
   String? coverage,
+  String? coverageLcov,
   bool stopOnFirstFailure = false,
 }) {
   var declarer = Declarer()..declare(body);
@@ -151,6 +152,7 @@ Engine declareEngine(
       ),
     ],
     coverage: coverage,
+    coverageLcov: coverageLcov,
     stopOnFirstFailure: stopOnFirstFailure,
   );
 }
@@ -222,6 +224,8 @@ Configuration configuration({
   String? reporter,
   Map<String, String>? fileReporters,
   String? coverage,
+  String? coverageLcov,
+  bool? branchCoverage,
   int? concurrency,
   int? shardIndex,
   int? totalShards,
@@ -272,6 +276,8 @@ Configuration configuration({
   reporter: reporter,
   fileReporters: fileReporters,
   coverage: coverage,
+  coverageLcov: coverageLcov,
+  branchCoverage: branchCoverage,
   concurrency: concurrency,
   shardIndex: shardIndex,
   totalShards: totalShards,

pkgs/test_core/CHANGELOG.md

@@ -2,6 +2,7 @@
 
 * Restrict to latest version of analyzer package.
 * Require Dart 3.7
+* Add `--coverage-path` and `--branch-coverage` options to `dart test`.
 
 ## 0.6.12
 

pkgs/test_core/lib/src/runner.dart

@@ -77,6 +77,7 @@ class Runner {
     var engine = Engine(
       concurrency: config.concurrency,
       coverage: config.coverage,
+      coverageLcov: config.coverageLcov,
       testRandomizeOrderingSeed: config.testRandomizeOrderingSeed,
       stopOnFirstFailure: config.stopOnFirstFailure,
     );