diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 6c3ea763..d2af28dd 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -164,11 +164,34 @@ jobs: name: Check if docs are up-to-date run: ./.github/workflows/check-docs.sh - - name: Run tests + name: Run cmd tests run: | set -o pipefail go test -v ./cmd/... | sed ''/PASS/s//$(printf "\033[32mPASS\033[0m")/'' | sed ''/FAIL/s//$(printf "\033[31mFAIL\033[0m")/'' + - + name: Run internal tests + run: | + set -o pipefail go test -v ./internal/... | sed ''/PASS/s//$(printf "\033[32mPASS\033[0m")/'' | sed ''/FAIL/s//$(printf "\033[31mFAIL\033[0m")/'' + - + name: Run pkg tests + run: | + set -o pipefail go test -v ./pkg/... | sed ''/PASS/s//$(printf "\033[32mPASS\033[0m")/'' | sed ''/FAIL/s//$(printf "\033[31mFAIL\033[0m")/'' - go test -timeout 30m -v ./test/tasks/... -always-keep-tmp-workspaces | sed ''/PASS/s//$(printf "\033[32mPASS\033[0m")/'' | sed ''/FAIL/s//$(printf "\033[31mFAIL\033[0m")/'' + - + name: Run tasks tests + run: | + set -o pipefail + go test -json -v -parallel 4 -timeout 30m ./test/tasks/... -always-keep-tmp-workspaces | go run cmd/disentangle-output/main.go -test-result-dir=./test/testdata/test-results | sed ''/PASS/s//$(printf "\033[32mPASS\033[0m")/'' | sed ''/FAIL/s//$(printf "\033[31mFAIL\033[0m")/'' + - + name: Run e2e tests + run: | + set -o pipefail go test -timeout 10m -v ./test/e2e/... | sed ''/PASS/s//$(printf "\033[32mPASS\033[0m")/'' | sed ''/FAIL/s//$(printf "\033[31mFAIL\033[0m")/'' + - + name: Test Logs + if: ${{ always() }} + uses: actions/upload-artifact@v2 + with: + name: Test Logs + path: test/testdata/test-results/ diff --git a/.gitignore b/.gitignore index d16d6304..acd41fa9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ .vscode/ .idea/ test/testdata/workspaces/workspace-* +test/testdata/test-results ods.external.yaml artifact-download-linux-amd64 artifact-download-darwin-amd64 diff --git a/Makefile b/Makefile index 41ef175d..1d9731a9 100644 --- a/Makefile +++ b/Makefile @@ -79,7 +79,7 @@ test-pkg: ## Run testsuite of public packages. .PHONY: test-pkg test-tasks: ## Run testsuite of Tekton tasks. - go test -v -count=1 -timeout $${ODS_TESTTIMEOUT:-30m} ./test/tasks/... + go test -json -v -count=1 -timeout $${ODS_TESTTIMEOUT:-30m} ./test/tasks/... | go run cmd/disentangle-output/main.go -test-result-dir=./test/testdata/test-results .PHONY: test-tasks test-e2e: ## Run testsuite of end-to-end pipeline run. diff --git a/cmd/disentangle-output/main.go b/cmd/disentangle-output/main.go new file mode 100644 index 00000000..3a5b9dd1 --- /dev/null +++ b/cmd/disentangle-output/main.go @@ -0,0 +1,24 @@ +package main + +import ( + "flag" + "log" + "os" + + "github.com/opendevstack/pipeline/internal/testoutput" +) + +type options struct { + testResultDir string +} + +func main() { + opts := options{} + flag.StringVar(&opts.testResultDir, "test-result-dir", os.Getenv("TEST_RESULT_DIR"), "test-result-dir") + flag.Parse() + + err := testoutput.DisentangleTestOutputs(opts.testResultDir, os.Stdin, os.Stdout) + if err != nil { + log.Fatalf("could not disentangle test output: %v", err) + } +} diff --git a/internal/testoutput/disentangle.go b/internal/testoutput/disentangle.go new file mode 100644 index 00000000..234ab4da --- /dev/null +++ b/internal/testoutput/disentangle.go @@ -0,0 +1,77 @@ +package testoutput + +import ( + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "time" +) + +// TestEvent taken from `go doc test2json`, unfortunately, this struct is not +// exposed from any core go packages so we have to define it ourselves +type TestEvent struct { + Time time.Time `json:",omitempty"` + Action string + Package string `json:",omitempty"` + Test string `json:",omitempty"` + Elapsed float64 `json:",omitempty"` + Output string `json:",omitempty"` +} + +// DisentangleTestOutputs separates test outputs from a stream of interleaved +// test events into separate files per individual testcase +func DisentangleTestOutputs(testResultsPath string, in *os.File, out *os.File) error { + // holds a mapping from test to the file handles the output is redirected to + var fileHandles = make(map[string]*os.File) + decoder := json.NewDecoder(in) + + for { + var event TestEvent + if err := decoder.Decode(&event); err == io.EOF { + break + } else if err != nil { + return err + } + + // switch over fixed set of actions (c.f. `go doc test2json`) + switch event.Action { + case "run": + filePath := filepath.Join(testResultsPath, event.Test+".log") + dir := filepath.Dir(filePath) + err := os.MkdirAll(dir, 0777) + if err != nil { + return fmt.Errorf("failed to create directory: %w", err) + } + + f, err := os.OpenFile(filePath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) + if err != nil { + return fmt.Errorf("failed to open output file: %w", err) + } + fileHandles[event.Test] = f + case "pause", "cont", "skip": // do nothing + case "pass", "fail": + f, ok := fileHandles[event.Test] + if ok { + err := f.Close() + if err != nil { + return fmt.Errorf("could not close file handle: %w", err) + } + } + case "bench", "output": + // 1. redirect to file + f, ok := fileHandles[event.Test] + if ok { + if _, err := fmt.Fprint(f, event.Output); err != nil { + return fmt.Errorf("failed to write to output file: %w", err) + } + } + // 2. echo the output with the json stripped off to `out` + if _, err := fmt.Fprint(out, event.Output); err != nil { + return fmt.Errorf("failed to write to output stream: %w", err) + } + } + } + return nil +} diff --git a/pkg/tasktesting/logs.go b/pkg/tasktesting/logs.go index 6a6e4ece..9b5d19d5 100644 --- a/pkg/tasktesting/logs.go +++ b/pkg/tasktesting/logs.go @@ -4,12 +4,11 @@ import ( "bufio" "context" "fmt" - "log" - corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/kubernetes" + "testing" ) // getEventsAndLogsOfPod streams events of the pod until all containers are ready, @@ -20,7 +19,8 @@ func getEventsAndLogsOfPod( c kubernetes.Interface, pod *corev1.Pod, collectedLogsChan chan []byte, - errs chan error) { + errs chan error, + t *testing.T) { quitEvents := make(chan bool) podName := pod.Name podNamespace := pod.Namespace @@ -36,9 +36,9 @@ func getEventsAndLogsOfPod( watchingEvents := true for _, container := range pod.Spec.Containers { - err := streamContainerLogs(ctx, c, podNamespace, podName, container.Name, collectedLogsChan) + err := streamContainerLogs(ctx, c, podNamespace, podName, container.Name, collectedLogsChan, t) if err != nil { - fmt.Printf("failure while getting container logs: %s", err) + t.Logf("failure while getting container logs: %s", err) errs <- err return } @@ -52,8 +52,8 @@ func getEventsAndLogsOfPod( func streamContainerLogs( ctx context.Context, c kubernetes.Interface, - podNamespace, podName, containerName string, collectedLogsChan chan []byte) error { - log.Printf("Waiting for container %s from pod %s to be ready...\n", containerName, podName) + podNamespace, podName, containerName string, collectedLogsChan chan []byte, t *testing.T) error { + t.Logf("Waiting for container %s from pod %s to be ready...\n", containerName, podName) w, err := c.CoreV1().Pods(podNamespace).Watch(ctx, metav1.SingleObject(metav1.ObjectMeta{ Name: podName, @@ -67,7 +67,7 @@ func streamContainerLogs( ev := <-w.ResultChan() if cs, ok := containerFromEvent(ev, podName, containerName); ok { if cs.State.Running != nil { - log.Printf("---------------------- Logs from %s -------------------------\n", containerName) + t.Logf("---------------------- Logs from %s -------------------------\n", containerName) // Set up log stream using a new ctx so that it's not cancelled // when the task is done before all logs have been read. ls, err := c.CoreV1().Pods(podNamespace).GetLogs(podName, &corev1.PodLogOptions{ @@ -83,11 +83,11 @@ func streamContainerLogs( select { case <-ctx.Done(): collectedLogsChan <- reader.Bytes() - fmt.Println(reader.Text()) + t.Log(reader.Text()) return nil default: collectedLogsChan <- reader.Bytes() - fmt.Println(reader.Text()) + t.Log(reader.Text()) } } return reader.Err() diff --git a/pkg/tasktesting/run.go b/pkg/tasktesting/run.go index 41b5ac37..1a71d1fd 100644 --- a/pkg/tasktesting/run.go +++ b/pkg/tasktesting/run.go @@ -190,6 +190,7 @@ func WatchTaskRunUntilDone(t *testing.T, testOpts TestOpts, tr *tekton.TaskRun) pod, collectedLogsChan, errs, + t, ) } diff --git a/test/tasks/common_test.go b/test/tasks/common_test.go index a7922151..fff6b294 100644 --- a/test/tasks/common_test.go +++ b/test/tasks/common_test.go @@ -131,28 +131,33 @@ func runTaskTestCases(t *testing.T, taskName string, requiredServices []tasktest }, ) + t.Run(taskName, func(t *testing.T) { + for name, tc := range testCases { + tc := tc + name := name + t.Run(name, func(t *testing.T) { + t.Parallel() + tn := taskName + if tc.TaskVariant != "" { + tn = fmt.Sprintf("%s-%s", taskName, tc.TaskVariant) + } + if tc.Timeout == 0 { + tc.Timeout = 5 * time.Minute + } + tasktesting.Run(t, tc, tasktesting.TestOpts{ + TaskKindRef: taskKindRef, + TaskName: tn, + Clients: c, + Namespace: ns, + Timeout: tc.Timeout, + AlwaysKeepTmpWorkspaces: *alwaysKeepTmpWorkspacesFlag, + }) + }) + } + }) + tasktesting.CleanupOnInterrupt(func() { tasktesting.TearDown(t, c, ns) }, t.Logf) defer tasktesting.TearDown(t, c, ns) - - for name, tc := range testCases { - t.Run(name, func(t *testing.T) { - tn := taskName - if tc.TaskVariant != "" { - tn = fmt.Sprintf("%s-%s", taskName, tc.TaskVariant) - } - if tc.Timeout == 0 { - tc.Timeout = 5 * time.Minute - } - tasktesting.Run(t, tc, tasktesting.TestOpts{ - TaskKindRef: taskKindRef, - TaskName: tn, - Clients: c, - Namespace: ns, - Timeout: tc.Timeout, - AlwaysKeepTmpWorkspaces: *alwaysKeepTmpWorkspacesFlag, - }) - }) - } } func checkSonarQualityGate(t *testing.T, c *kclient.Clientset, namespace, sonarProject string, qualityGateFlag bool, wantQualityGateStatus string) { diff --git a/test/tasks/ods-build-typescript_test.go b/test/tasks/ods-build-typescript_test.go index b8d08ec8..873ce0b9 100644 --- a/test/tasks/ods-build-typescript_test.go +++ b/test/tasks/ods-build-typescript_test.go @@ -21,6 +21,7 @@ func TestTaskODSBuildTypescript(t *testing.T) { }, map[string]tasktesting.TestCase{ "build typescript app": { + Timeout: 15 * time.Minute, WorkspaceDirMapping: map[string]string{"source": "typescript-sample-app"}, PreRunFunc: func(t *testing.T, ctxt *tasktesting.TaskRunContext) { wsDir := ctxt.Workspaces["source"] @@ -48,6 +49,7 @@ func TestTaskODSBuildTypescript(t *testing.T) { }, }, "build typescript app in subdirectory": { + Timeout: 15 * time.Minute, WorkspaceDirMapping: map[string]string{"source": "hello-world-app"}, PreRunFunc: func(t *testing.T, ctxt *tasktesting.TaskRunContext) { wsDir := ctxt.Workspaces["source"] @@ -82,6 +84,7 @@ func TestTaskODSBuildTypescript(t *testing.T) { }, }, "fail linting typescript app and generate lint report": { + Timeout: 15 * time.Minute, WorkspaceDirMapping: map[string]string{"source": "typescript-sample-app-lint-error"}, PreRunFunc: func(t *testing.T, ctxt *tasktesting.TaskRunContext) { wsDir := ctxt.Workspaces["source"] @@ -111,7 +114,7 @@ func TestTaskODSBuildTypescript(t *testing.T) { WantSetupFail: true, }, "build backend typescript app": { - Timeout: 10 * time.Minute, + Timeout: 20 * time.Minute, WorkspaceDirMapping: map[string]string{"source": "typescript-sample-app"}, PreRunFunc: func(t *testing.T, ctxt *tasktesting.TaskRunContext) { wsDir := ctxt.Workspaces["source"] @@ -137,6 +140,7 @@ func TestTaskODSBuildTypescript(t *testing.T) { }, }, "build typescript app with custom build directory": { + Timeout: 15 * time.Minute, WorkspaceDirMapping: map[string]string{"source": "typescript-sample-app-build-dir"}, PreRunFunc: func(t *testing.T, ctxt *tasktesting.TaskRunContext) { wsDir := ctxt.Workspaces["source"]