Merge pull request #722 from opendevstack/feature/pkg-test

Refactor to ease development of tasks in external Git repositories

Author: Michael Sauter

.github/workflows/check-docs.sh

@@ -2,9 +2,9 @@
 set -eu
 
 make docs
-if ! git diff --quiet docs tasks; then
-    echo "Docs / tasks are not up-to-date! Run 'make docs' to update."
+if ! git diff --quiet docs; then
+    echo "Docs are not up-to-date! Run 'make docs' to update."
     exit 1
 else
-    echo "Docs /tasks are up-to-date."
+    echo "Docs are up-to-date."
 fi

.github/workflows/main.yaml

@@ -11,139 +11,44 @@ env:
   IMAGE_BASE: ${{ github.repository }}
 
 jobs:
-  build-images:
-    name: Build ODS images
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: true
-      matrix:
-        image: ["aqua-scan", "finish", "go-toolset", "gradle-toolset", "helm", "node16-npm-toolset", "node18-npm-toolset", "package-image", "pipeline-manager", "python-toolset", "sonar", "start"]
-    steps:
-      -
-        name: Checkout
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-      -
-        name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-        with:
-          driver-opts: |
-            image=moby/buildkit:master
-            network=host
-      -
-        name: Build image
-        uses: docker/build-push-action@v3
-        with:
-          context: .
-          push: false
-          file: build/package/Dockerfile.${{ matrix.image }}
-          tags: localhost:5000/ods/ods-${{ matrix.image }}:latest
-          outputs: type=docker,dest=/tmp/image-ods-${{ matrix.image }}.tar
-      -
-        name: Upload artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: buildx-image-ods-${{ matrix.image }}
-          path: /tmp/image-ods-${{ matrix.image }}.tar
-          retention-days: 1
-
   pipeline-tests:
     name: Tests
     runs-on: ubuntu-latest
-    needs: build-images
-    env:
-      IMAGES: aqua-scan finish go-toolset gradle-toolset helm node16-npm-toolset node18-npm-toolset package-image pipeline-manager python-toolset sonar start
     steps:
-      -
-        name: Download image artifacts
-        uses: actions/download-artifact@v3
-        with:
-          path: /tmp
       -
         name: Checkout
         uses: actions/checkout@v3
         with:
           fetch-depth: 0
       -
-        name: Setup KinD cluster with internal registry
-        working-directory: scripts
-        run: ./kind-with-registry.sh
-      -
-        name: Push images to local registry
-        run: |
-          images=(${{ env.IMAGES }})
-          for image in ${images[*]}
-          do
-            echo "::group::Push ods-$image to local registry"
-            docker load --input /tmp/buildx-image-ods-$image/image-ods-$image.tar
-            docker push localhost:5000/ods/ods-$image:latest
-            if [[ "${{ github.event_name }}" == 'pull_request' ]]
-            then
-              docker rmi localhost:5000/ods/ods-$image:latest
-            fi
-            echo "::endgroup::"
-          done
-      -
-        name: Delete image tarballs
-        run: |
-          rm -rf /tmp/buildx-image-*
-      -
-        name: Setup kubectl
-        uses: azure/setup-kubectl@v3
-        id: install
-      -
-        name: Install Tekton Core Components
-        run: make install-tekton-pipelines
-      -
-        name: Show disk space
-        run: df -h
-      -
-        name: Spin up Bitbucket container
-        run: make run-bitbucket
-      -
-        name: Spin up Nexus container
-        run: make run-nexus
-      -
-        name: Spin up SonarQube container
-        run: make run-sonarqube
-      -
-        name: Show disk space
-        run: df -h
-      -
-        name: Setup Go 1.19
-        uses: actions/setup-go@v3
+        name: Setup Go
+        uses: actions/setup-go@v4
         with:
-          go-version: '1.19'
+          go-version: '1.21'
       -
         name: Check if docs are up-to-date
         run: ./.github/workflows/check-docs.sh
       -
         name: Run tests
         run: |
-          set -o pipefail
-          go test -v ./cmd/... | sed ''/PASS/s//$(printf "\033[32mPASS\033[0m")/'' | sed ''/FAIL/s//$(printf "\033[31mFAIL\033[0m")/''
-          go test -v ./internal/... | sed ''/PASS/s//$(printf "\033[32mPASS\033[0m")/'' | sed ''/FAIL/s//$(printf "\033[31mFAIL\033[0m")/''
-          go test -v ./pkg/... | sed ''/PASS/s//$(printf "\033[32mPASS\033[0m")/'' | sed ''/FAIL/s//$(printf "\033[31mFAIL\033[0m")/''
-          go test -timeout 45m -v ./test/tasks/... -always-keep-tmp-workspaces | sed ''/PASS/s//$(printf "\033[32mPASS\033[0m")/'' | sed ''/FAIL/s//$(printf "\033[31mFAIL\033[0m")/''
-          go test -timeout 10m -v ./test/e2e/... | sed ''/PASS/s//$(printf "\033[32mPASS\033[0m")/'' | sed ''/FAIL/s//$(printf "\033[31mFAIL\033[0m")/''
-      -
-        name: Log into ghcr.io
-        if: ${{ github.event_name != 'pull_request' }}
-        uses: docker/login-action@v1
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-      -
-        name: Push images to ghcr.io
-        if: ${{ github.event_name != 'pull_request' }}
-        run: |
-          images=(${{ env.IMAGES }})
-          for image in ${images[*]}
-          do
-            echo "::group::Push ods-$image to ghcr.io"
-            docker tag localhost:5000/ods/ods-$image:latest ghcr.io/${{ env.IMAGE_BASE }}/ods-$image:latest
-            docker push ghcr.io/${{ env.IMAGE_BASE }}/ods-$image:latest
-            echo "::endgroup::"
-          done
+          make test
+      # -
+      #   name: Log into ghcr.io
+      #   if: ${{ github.event_name != 'pull_request' }}
+      #   uses: docker/login-action@v1
+      #   with:
+      #     registry: ghcr.io
+      #     username: ${{ github.actor }}
+      #     password: ${{ secrets.GITHUB_TOKEN }}
+      # -
+      #   name: Push images to ghcr.io
+      #   if: ${{ github.event_name != 'pull_request' }}
+      #   run: |
+      #     images=(${{ env.IMAGES }})
+      #     for image in ${images[*]}
+      #     do
+      #       echo "::group::Push ods-$image to ghcr.io"
+      #       docker tag localhost:5000/ods/ods-$image:latest ghcr.io/${{ env.IMAGE_BASE }}/ods-$image:latest
+      #       docker push ghcr.io/${{ env.IMAGE_BASE }}/ods-$image:latest
+      #       echo "::endgroup::"
+      #     done

CHANGELOG.md

@@ -13,18 +13,13 @@ listed in the changelog.
 ### Fixed
 
 - ods.yaml branch trigger patterns must be lowercase ([#713](https://github.com/opendevstack/ods-pipeline/issues/713))
-- Go module name was incorrectly set to `github.com/opendevstack/pipeline`
-
-- sonar-scanner invocations stderr not captured ([#719](https://github.com/opendevstack/ods-pipeline/issues/719))
-
-- sonar-scanner does not start properly: java is lacking tzdb.dat ([#723](https://github.com/opendevstack/ods-pipeline/issues/723))
 
-- update sonar-scanner and cnes-report ([#725](https://github.com/opendevstack/ods-pipeline/issues/725))
-
-- SonarQube doesn't scan FE-related code ([#716](https://github.com/opendevstack/ods-pipeline/issues/716))
+- Go module name was incorrectly set to `github.com/opendevstack/pipeline`
 
 ### Changed
 
+- Move pipeline tasks to separate repositories. This is a huge change with many implications. Instead of providing build, package and deploy taks as part of the `ods-pipeline` repository, the tasks are no provided by separate repositories, such as `ods-pipeline-go`, `ods-pipeline-sonar`, `ods-pipeline-image`, `ods-pipeline-helm` and so on. The only tasks that are provided by `ods-pipeline` are the start and finish tasks automatically injected into each pipeline. This change allows to have a different lifecycle for each task (or set of tasks). It also benefits maintenance greatly: running the tests for this repository is much faster now (around 10 minutes compared to 35+ minutes earlier). This repository facilitates task creation, maintenance and testing by providing a few Go packages that can be used by task repositories such as `ods-pipeline-helm`. For more information, see [#722](https://github.com/opendevstack/ods-pipeline/pull/722).
+
 - Build tasks streamlining and avoidance of file copies (#678 fixed by [#710](https://github.com/opendevstack/ods-pipeline/pull/710)). This is an incompatible change. Build tasks were adjusted to (mostly) no longer copy build files in a dedicated location. Instead one should adjust the Dockerfile (or other downstream tasks) to directly consume the build outputs from their natural locations. In addition build task skipping now supports parameter `build-extra-inputs`. The package-image task `dockerfile` and `docker-dir` parameters have been changed to assume that the docker context and file are at the repository root. See the PR for further information and the issue for more context.
 
 ## [0.13.2] - 2023-07-18

Makefile

@@ -15,7 +15,7 @@ help: ## Show this help screen.
 	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z0-9_-]+:.*?##/ { printf "  \033[36m%-25s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
 .PHONY: help
 
-check-system: ## Check if system meets prerequisites.
+check-system: ## Check if system meets prerequisites for development.
 	cd scripts && ./check-system.sh
 .PHONY: check-system
 
@@ -36,16 +36,26 @@ lint-shell: ## Run shellcheck.
 	shellcheck scripts/*.sh build/package/scripts/* deploy/*.sh
 .PHONY: lint-shell
 
-##@ Building
-
-tasks: ## Render tasks.
-	go run cmd/tasks/main.go
-.PHONY: tasks
-
-docs: tasks ## Render documentation for tasks.
-	go run cmd/docs/main.go
+docs: ## Render documentation for tasks.
+	renderedStartTask=$(shell mktemp); \
+	helm template ods-pipeline deploy/chart --show-only=templates/task-start.yaml > $$renderedStartTask; \
+	go run github.com/opendevstack/ods-pipeline/cmd/taskdoc \
+		-task $$renderedStartTask \
+		-description build/docs/task-start.adoc \
+		-destination docs/task-start.adoc; \
+	rm $$renderedStartTask
+
+	renderedFinishTask=$(shell mktemp); \
+	helm template ods-pipeline deploy/chart --show-only=templates/task-finish.yaml > $$renderedFinishTask; \
+	go run github.com/opendevstack/ods-pipeline/cmd/taskdoc \
+		-task $$renderedFinishTask \
+		-description build/docs/task-finish.adoc \
+		-destination docs/task-finish.adoc; \
+	rm $$renderedFinishTask
 .PHONY: docs
 
+##@ Building
+
 build-artifact-download: build-artifact-download-linux build-artifact-download-darwin-amd64 build-artifact-download-darwin-arm64 build-artifact-download-windows ## Build artifact-download binary for each supported OS/arch.
 .PHONY: build-artifact-download
 
@@ -67,7 +77,7 @@ build-artifact-download-windows: ## Build artifact-download Windows binary.
 
 ##@ Testing
 
-test: test-cmd test-internal test-pkg test-tasks test-e2e ## Run complete testsuite.
+test: test-cmd test-internal test-pkg test-e2e ## Run complete testsuite.
 .PHONY: test
 
 test-cmd: ## Run testsuite of cmd packages.
@@ -82,69 +92,7 @@ test-pkg: ## Run testsuite of public packages.
 	go test -cover ./pkg/...
 .PHONY: test-pkg
 
-test-tasks: ## Run testsuite of Tekton tasks.
-	go test -v -count=1 -timeout $${ODS_TESTTIMEOUT:-30m} ./test/tasks/...
-.PHONY: test-tasks
-
-test-e2e: ## Run testsuite of end-to-end pipeline run.
-	go test -v -count=1 -timeout $${ODS_TESTTIMEOUT:-10m} ./test/e2e/...
+test-e2e: ## Run testsuite of tasks and full pipeline run.
+	go test -v -count=1 -timeout 20m -skip ^TestPipelineRun  ./test/e2e/...
+	go test -v -count=1 -timeout 10m -run ^TestPipelineRun ./test/e2e/...
 .PHONY: test-e2e
-
-clear-tmp-workspaces: ## Clear temporary workspaces created in testruns.
-	chmod -R u+w test/testdata/workspaces/workspace-*
-	rm -rf test/testdata/workspaces/workspace-*
-.PHONY: clear-tmp-workspaces
-
-##@ KinD (local development environment)
-
-prepare-local-env: create-kind-with-registry build-and-push-images install-tekton-pipelines run-bitbucket run-nexus run-sonarqube ## Prepare local environment from scratch.
-.PHONY: prepare-local-env
-
-create-kind-with-registry: ## Create KinD cluster with local registry.
-	cd scripts && ./kind-with-registry.sh
-.PHONY: create-kind-with-registry
-
-install-tekton-pipelines: ## Install Tekton pipelines in KinD cluster.
-	cd scripts && ./install-tekton-pipelines.sh
-.PHONY: install-tekton-pipelines
-
-build-and-push-images: ## Build and push images to local registry.
-		cd scripts && ./build-and-push-images.sh
-.PHONY: build-and-push-images
-
-run-bitbucket: ## Run Bitbucket server (using timebomb license, in "kind" network).
-	cd scripts && ./run-bitbucket.sh
-.PHONY: run-bitbucket
-
-restart-bitbucket: ## Restart Bitbucket server (re-activating timebomb license).
-	cd scripts && ./restart-bitbucket.sh
-.PHONY: restart-bitbucket
-
-run-nexus: ## Run Nexus server (in "kind" network).
-	cd scripts && ./run-nexus.sh
-.PHONY: run-nexus
-
-run-sonarqube: ## Run SonarQube server (in "kind" network).
-	cd scripts && ./run-sonarqube.sh
-.PHONY: run-sonarqube
-
-recreate-kind-cluster: ## Recreate KinD cluster including Tekton tasks.
-	cd scripts && ./kind-with-registry.sh --recreate
-	cd scripts && ./install-tekton-pipelines.sh
-.PHONY: recreate-kind-cluster
-
-stop-local-env: ## Stop local environment.
-	cd scripts && ./stop-local-env.sh
-.PHONY: stop-local-env
-
-start-local-env: ## Restart stopped local environment.
-	cd scripts && ./start-local-env.sh
-.PHONY: start-local-env
-
-deploy: ## Install ODS pipeline resources in namespace.
-ifeq ($(strip $(namespace)),)
-	@echo "Argument 'namespace' is required, e.g. make deploy namespace=foo-cd"
-	@exit 1
-endif
-	cd scripts && ./install-inside-kind.sh -n $(namespace)
-.PHONY: deploy

README.md

@@ -21,10 +21,8 @@ ODS Pipeline is well suited for regulated development (e.g. medical device softw
 
 ### Technical Reference
 * [Repository configuration (ods.yaml)](/docs/ods-configuration.adoc)
-* Plumbing tasks: [ods-start](/docs/tasks/ods-start.adoc), [ods-finish](/docs/tasks/ods-finish.adoc)
-* Build tasks: [ods-build-go](/docs/tasks/ods-build-go.adoc), [ods-build-gradle](/docs/tasks/ods-build-gradle.adoc), [ods-build-npm](/docs/tasks/ods-build-npm.adoc), [ods-build-python](/docs/tasks/ods-build-python.adoc)
-* Package tasks: [ods-package-image](/docs/tasks/ods-package-image.adoc)
-* Deploy tasks: [ods-deploy-helm](/docs/tasks/ods-deploy-helm.adoc)
+* [Start task](/docs/task-start.adoc)
+* [Finish task](/docs/task-finish.adoc)
 
 ### How-To Guides
 * [Working with secrets in Helm](/docs/helm-secrets.adoc)