Report coverage as package reporter

Author: csweichel

pkg/leeway/build.go

@@ -16,6 +16,7 @@ import (
 	"os/exec"
 	"path/filepath"
 	"regexp"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -765,6 +766,18 @@ func (p *Package) build(buildctx *buildContext) (err error) {
 		}
 	}
 
+	if bld.TestCoverage != nil {
+		coverage, funcsWithoutTest, funcsWithTest, err := bld.TestCoverage()
+		if err != nil {
+			return err
+		}
+
+		pkgRep.TestCoverageAvailable = true
+		pkgRep.TestCoveragePercentage = coverage
+		pkgRep.FunctionsWithoutTest = funcsWithoutTest
+		pkgRep.FunctionsWithTest = funcsWithTest
+	}
+
 	err = executeCommandsForPackage(buildctx, p, builddir, bld.Commands[PackageBuildPhasePackage])
 	if err != nil {
 		return err
@@ -846,8 +859,17 @@ type packageBuild struct {
 	// If Subjects is not nil it's used to compute the provenance subjects of the
 	// package build. This field takes precedence over PostBuild
 	Subjects func() ([]in_toto.Subject, error)
+
+	// If TestCoverage is not nil it's used to compute the test coverage of the package build.
+	// This function is expected to return a value between 0 and 100.
+	// If the package build does not have any tests, this function must return 0.
+	// If the package build has tests but the test coverage cannot be computed, this function must return an error.
+	// This function is guaranteed to be called after the test phase has finished.
+	TestCoverage testCoverageFunc
 }
 
+type testCoverageFunc func() (coverage, funcsWithoutTest, funcsWithTest int, err error)
+
 const (
 	getYarnLockScript = `#!/bin/bash
 set -Eeuo pipefail
@@ -1236,10 +1258,14 @@ func (p *Package) buildGo(buildctx *buildContext, wd, result string) (res *packa
 			commands[PackageBuildPhaseLint] = append(commands[PackageBuildPhaseLint], cfg.LintCommand)
 		}
 	}
+	var reportCoverage testCoverageFunc
 	if !cfg.DontTest && !buildctx.DontTest {
 		testCommand := []string{goCommand, "test", "-v"}
 		if buildctx.buildOptions.CoverageOutputPath != "" {
 			testCommand = append(testCommand, fmt.Sprintf("-coverprofile=%v", codecovComponentName(p.FullName())))
+		} else {
+			testCommand = append(testCommand, "-coverprofile=testcoverage.out")
+			reportCoverage = collectGoTestCoverage(filepath.Join(wd, "testcoverage.out"), p.FullName())
 		}
 		testCommand = append(testCommand, "./...")
 
@@ -1269,10 +1295,61 @@ func (p *Package) buildGo(buildctx *buildContext, wd, result string) (res *packa
 	}
 
 	return &packageBuild{
-		Commands: commands,
+		Commands:     commands,
+		TestCoverage: reportCoverage,
 	}, nil
 }
 
+func collectGoTestCoverage(covfile, fullName string) testCoverageFunc {
+	return func() (coverage, funcsWithoutTest, funcsWithTest int, err error) {
+		// We need to collect the coverage for all packages in the module.
+		// To that end we load the coverage file.
+		// The coverage file contains the coverage for all packages in the module.
+
+		cmd := exec.Command("go", "tool", "cover", "-func", covfile)
+		out, err := cmd.CombinedOutput()
+		if err != nil {
+			err = xerrors.Errorf("cannot collect test coverage: %w: %s", err, string(out))
+			return
+		}
+
+		coverage, funcsWithoutTest, funcsWithTest, err = parseGoCoverOutput(string(out))
+		return
+	}
+}
+
+func parseGoCoverOutput(input string) (coverage, funcsWithoutTest, funcsWithTest int, err error) {
+	// The output of the coverage tool looks like this:
+	// github.com/gitpod-io/gitpod/content_ws/pkg/contentws/contentws.go:33:	New		100.0%
+	lines := strings.Split(input, "\n")
+
+	for _, line := range lines {
+		fields := strings.Fields(line)
+		if len(fields) < 3 {
+			continue
+		}
+		perc := strings.Trim(strings.TrimSpace(fields[2]), "%")
+		percF, err := strconv.ParseFloat(perc, 32)
+		if err != nil {
+			log.Warnf("cannot parse coverage percentage for line %s: %v", line, err)
+			continue
+		}
+		intCov := int(percF)
+		coverage += intCov
+		if intCov == 0 {
+			funcsWithoutTest++
+		} else {
+			funcsWithTest++
+		}
+	}
+
+	total := (funcsWithoutTest + funcsWithTest)
+	if total != 0 {
+		coverage = coverage / total
+	}
+	return
+}
+
 // buildDocker implements the build process for Docker packages.
 // If you change anything in this process that's not backwards compatible, make sure you increment buildProcessVersions accordingly.
 func (p *Package) buildDocker(buildctx *buildContext, wd, result string) (res *packageBuild, err error) {

pkg/leeway/build_internal_test.go

@@ -0,0 +1,55 @@
+package leeway
+
+import (
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+)
+
+func TestParseGoCoverOutput(t *testing.T) {
+	type Expectation struct {
+		Error            string
+		Coverage         int
+		FuncsWithoutTest int
+		FuncsWithTest    int
+	}
+	tests := []struct {
+		Name        string
+		Input       string
+		Expectation Expectation
+	}{
+		{
+			Name: "empty",
+		},
+		{
+			Name: "valid",
+			Input: `github.com/gitpod-io/leeway/store.go:165:                    Get                             100.0%
+			github.com/gitpod-io/leeway/store.go:173:                    Set                             100.0%
+			github.com/gitpod-io/leeway/store.go:178:                    Delete                          100.0%
+			github.com/gitpod-io/leeway/store.go:183:                    Scan                            80.0%
+			github.com/gitpod-io/leeway/store.go:194:                    Close                           0.0%
+			github.com/gitpod-io/leeway/store.go:206:                    Upsert                          0.0%`,
+			Expectation: Expectation{
+				Coverage:         63,
+				FuncsWithoutTest: 2,
+				FuncsWithTest:    4,
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.Name, func(t *testing.T) {
+			var act Expectation
+
+			var err error
+			act.Coverage, act.FuncsWithoutTest, act.FuncsWithTest, err = parseGoCoverOutput(test.Input)
+			if err != nil {
+				act.Error = err.Error()
+			}
+
+			if diff := cmp.Diff(test.Expectation, act); diff != "" {
+				t.Errorf("parseGoCoverOutput() mismatch (-want +got):\n%s", diff)
+			}
+		})
+	}
+}

pkg/leeway/reporter.go

@@ -54,6 +54,11 @@ type PackageBuildReport struct {
 
 	Phases []PackageBuildPhase
 	Error  error
+
+	TestCoverageAvailable  bool
+	TestCoveragePercentage int
+	FunctionsWithoutTest   int
+	FunctionsWithTest      int
 }
 
 // PhaseDuration returns the time it took to execute the phases commands
@@ -210,7 +215,11 @@ func (r *ConsoleReporter) PackageBuildFinished(pkg *Package, rep *PackageBuildRe
 	delete(r.times, nme)
 	r.mu.Unlock()
 
-	msg := color.Sprintf("<green>package build succeded</> <gray>(%.2fs)</>\n", dur.Seconds())
+	var coverage string
+	if rep.TestCoverageAvailable {
+		coverage = color.Sprintf("<fg=yellow>test coverage: %d%%</> <gray>(%d of %d functions have tests)</>\n", rep.TestCoveragePercentage, rep.FunctionsWithTest, rep.FunctionsWithTest+rep.FunctionsWithoutTest)
+	}
+	msg := color.Sprintf("%s<green>package build succeded</> <gray>(%.2fs)</>\n", coverage, dur.Seconds())
 	if rep.Error != nil {
 		msg = color.Sprintf("<red>package build failed while %sing</>\n<white>Reason:</> %s\n", rep.LastPhase(), rep.Error)
 	}
@@ -571,6 +580,11 @@ func (sr *SegmentReporter) PackageBuildFinished(pkg *Package, rep *PackageBuildR
 		"lastPhase":  rep.LastPhase(),
 		"durationMS": rep.TotalTime().Milliseconds(),
 	}
+	if rep.TestCoverageAvailable {
+		props["testCoverage"] = rep.TestCoveragePercentage
+		props["functionsWithoutTest"] = rep.FunctionsWithoutTest
+		props["functionsWithTest"] = rep.FunctionsWithTest
+	}
 	addPackageToSegmentEventProps(props, pkg)
 	sr.track("package_build_finished", props)
 }