From e66450ec5a989a3f9eca443e1b77350d58a443b3 Mon Sep 17 00:00:00 2001 From: Mladen Rusev Date: Tue, 30 Sep 2025 11:48:13 +0300 Subject: [PATCH 01/30] try code coverage script --- .github/workflows/tests.yaml | 17 +++++++++ hack/e2e/test.sh | 60 +++++++++++++++++++++++++++++++ hack/e2e/values.coverage-pvc.yaml | 8 +++++ make/00_mod.mk | 7 ++++ 4 files changed, 92 insertions(+) create mode 100644 hack/e2e/values.coverage-pvc.yaml diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index c80904cf..dfdd916f 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -64,6 +64,12 @@ jobs: - run: make -j test-unit test-helm + - name: Upload test artifacts + uses: actions/upload-artifact@v4 + with: + name: unit-artifacts + path: _bin/artifacts + test-e2e: if: contains(github.event.pull_request.labels.*.name, 'test-e2e') runs-on: ubuntu-latest @@ -125,6 +131,11 @@ jobs: CLOUDSDK_COMPUTE_ZONE: europe-west1-b CLUSTER_NAME: ${{ steps.timestamp.outputs.cluster_name }} + - name: Setup upterm session + uses: owenthereal/action-upterm@v1 + with: + limit-access-to-actor: true + - name: Delete GKE Cluster # 'always()' - Run this step regardless of success or failure. # '!contains(...)' - AND only run if the list of PR labels DOES NOT contain 'keep-e2e-cluster'. @@ -136,3 +147,9 @@ jobs: --project=machineidentitysecurity-jsci-e \ --zone=europe-west1-b \ --quiet + + - name: Upload test artifacts + uses: actions/upload-artifact@v4 + with: + name: e2e-artifacts + path: _bin/artifacts \ No newline at end of file diff --git a/hack/e2e/test.sh b/hack/e2e/test.sh index dbf2195d..61056722 100755 --- a/hack/e2e/test.sh +++ b/hack/e2e/test.sh @@ -83,6 +83,19 @@ if ! gcloud container clusters get-credentials "${CLUSTER_NAME}"; then fi kubectl create ns venafi || true +kubectl apply -n venafi -f - < Date: Tue, 30 Sep 2025 12:12:52 +0300 Subject: [PATCH 02/30] move upterm job higher --- .github/workflows/tests.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index dfdd916f..6a362463 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -106,6 +106,11 @@ jobs: with: go-version: ${{ steps.go-version.outputs.result }} + - name: Setup upterm session + uses: owenthereal/action-upterm@v1 + with: + limit-access-to-actor: true + - name: Generate timestamp for cluster name id: timestamp # Give the step an ID to reference its output run: | @@ -131,11 +136,6 @@ jobs: CLOUDSDK_COMPUTE_ZONE: europe-west1-b CLUSTER_NAME: ${{ steps.timestamp.outputs.cluster_name }} - - name: Setup upterm session - uses: owenthereal/action-upterm@v1 - with: - limit-access-to-actor: true - - name: Delete GKE Cluster # 'always()' - Run this step regardless of success or failure. # '!contains(...)' - AND only run if the list of PR labels DOES NOT contain 'keep-e2e-cluster'. From 871995694886536c73d70a7ee8aff6c5ba1c2ce5 Mon Sep 17 00:00:00 2001 From: Mladen Rusev Date: Tue, 30 Sep 2025 12:32:34 +0300 Subject: [PATCH 03/30] move upterm job in always condition --- .github/workflows/tests.yaml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 6a362463..1179fd55 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -110,7 +110,7 @@ jobs: uses: owenthereal/action-upterm@v1 with: limit-access-to-actor: true - + - name: Generate timestamp for cluster name id: timestamp # Give the step an ID to reference its output run: | @@ -148,6 +148,13 @@ jobs: --zone=europe-west1-b \ --quiet +# TODO: REMOVE THIS DEBUGGING! + - name: Setup upterm session + if: always() + uses: owenthereal/action-upterm@v1 + with: + limit-access-to-actor: true + - name: Upload test artifacts uses: actions/upload-artifact@v4 with: From 554a4513708c1704f9cbc54f07a45a78dbb4e5b0 Mon Sep 17 00:00:00 2001 From: Mladen Rusev Date: Tue, 30 Sep 2025 12:41:42 +0300 Subject: [PATCH 04/30] remove early upterm job --- .github/workflows/tests.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 1179fd55..5bcedc6b 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -106,11 +106,6 @@ jobs: with: go-version: ${{ steps.go-version.outputs.result }} - - name: Setup upterm session - uses: owenthereal/action-upterm@v1 - with: - limit-access-to-actor: true - - name: Generate timestamp for cluster name id: timestamp # Give the step an ID to reference its output run: | From 94874d8cc13efe40fa386e9a5c6ee5c12680bd6c Mon Sep 17 00:00:00 2001 From: Mladen Rusev Date: Tue, 30 Sep 2025 14:05:22 +0300 Subject: [PATCH 05/30] instead of killing pod scale down deployment --- hack/e2e/test.sh | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/hack/e2e/test.sh b/hack/e2e/test.sh index 61056722..fe79e702 100755 --- a/hack/e2e/test.sh +++ b/hack/e2e/test.sh @@ -250,13 +250,10 @@ getCertificate() { # Wait 5 minutes for the certificate to appear. for ((i=0;;i++)); do if getCertificate; then exit 0; fi; sleep 30; done | timeout -v -- 5m cat -echo "Identifying the agent pod to terminate..." -export AGENT_POD_NAME=$(kubectl get pods -n venafi -l app.kubernetes.io/name=venafi-kubernetes-agent -o jsonpath="{.items[0].metadata.name}") - -echo "Gracefully deleting agent pod '${AGENT_POD_NAME}' to flush coverage to the PVC..." -kubectl delete pod -n venafi "${AGENT_POD_NAME}" --grace-period=30 +echo "Scaling down agent deployment to kill pod and flush coverage to the PVC..." +kubectl scale deployment venafi-kubernetes-agent -n venafi --replicas=0 echo "Waiting for agent pod to terminate..." -kubectl wait --for=delete pod/${AGENT_POD_NAME} -n venafi --timeout=90s +kubectl wait deployment venafi-kubernetes-agent -n venafi --for=condition=scaled --timeout=60s kubectl apply -n venafi -f - < Date: Tue, 30 Sep 2025 15:20:18 +0300 Subject: [PATCH 06/30] wait for pod killed, not deployment status --- hack/e2e/test.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/hack/e2e/test.sh b/hack/e2e/test.sh index fe79e702..b9cead27 100755 --- a/hack/e2e/test.sh +++ b/hack/e2e/test.sh @@ -250,10 +250,12 @@ getCertificate() { # Wait 5 minutes for the certificate to appear. for ((i=0;;i++)); do if getCertificate; then exit 0; fi; sleep 30; done | timeout -v -- 5m cat +export AGENT_POD_NAME=$(kubectl get pods -n venafi -l app.kubernetes.io/name=venafi-kubernetes-agent -o jsonpath="{.items[0].metadata.name}") echo "Scaling down agent deployment to kill pod and flush coverage to the PVC..." kubectl scale deployment venafi-kubernetes-agent -n venafi --replicas=0 -echo "Waiting for agent pod to terminate..." -kubectl wait deployment venafi-kubernetes-agent -n venafi --for=condition=scaled --timeout=60s +echo "Waiting for agent pod '${AGENT_POD_NAME}' to terminate as a result of the scale-down..." +kubectl wait --for=delete pod/${AGENT_POD_NAME} -n venafi --timeout=90s +echo "Starting helper pod to retrieve coverage files from the PVC..." kubectl apply -n venafi -f - < Date: Tue, 30 Sep 2025 15:39:57 +0300 Subject: [PATCH 07/30] provide the COVERAGE_HOST_PATH to the test.sh script --- make/02_mod.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/make/02_mod.mk b/make/02_mod.mk index 12760fa0..2bc5cba5 100644 --- a/make/02_mod.mk +++ b/make/02_mod.mk @@ -52,7 +52,7 @@ shared_generate_targets += generate-crds-venconn ## See `hack/e2e/test.sh` for the full test script. ## @category Testing test-e2e-gke: | $(NEEDS_HELM) $(NEEDS_STEP) $(NEEDS_VENCTL) - ./hack/e2e/test.sh + @COVERAGE_HOST_PATH="$(COVERAGE_HOST_PATH)" ./hack/e2e/test.sh .PHONY: test-helm ## Run `helm unittest`. From d2b81f8174eee8a9ed2854d217cb5197a8ba7dcf Mon Sep 17 00:00:00 2001 From: Mladen Rusev Date: Tue, 30 Sep 2025 18:06:46 +0300 Subject: [PATCH 08/30] TEMP DEBUG --- make/_shared/oci-build/01_mod.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/make/_shared/oci-build/01_mod.mk b/make/_shared/oci-build/01_mod.mk index 726ad13c..97e42730 100644 --- a/make/_shared/oci-build/01_mod.mk +++ b/make/_shared/oci-build/01_mod.mk @@ -28,7 +28,7 @@ $(ko_config_path_$1:$(CURDIR)/%=%): | $(NEEDS_YQ) $(bin_dir)/scratch/image $(YQ) '.builds[0].ldflags[0] = "-s"' | \ $(YQ) '.builds[0].ldflags[1] = "-w"' | \ $(YQ) '.builds[0].ldflags[2] = "{{.Env.LDFLAGS}}"' | \ - $(YQ) '.builds[0].flags[0] = "$(go_$1_flags)"' | \ + $(YQ) '.builds[0].flags[0] = "-cover"' | \ $(YQ) '.builds[0].linux_capabilities = "$(oci_$1_linux_capabilities)"' \ > $(CURDIR)/$(oci_layout_path_$1).ko_config.yaml From 9080de71bb3c9ba17fb9a3824848aa7d86e3211f Mon Sep 17 00:00:00 2001 From: Mladen Rusev Date: Tue, 30 Sep 2025 18:42:46 +0300 Subject: [PATCH 09/30] add a try to jq --- hack/e2e/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/e2e/test.sh b/hack/e2e/test.sh index b9cead27..a437e760 100755 --- a/hack/e2e/test.sh +++ b/hack/e2e/test.sh @@ -216,7 +216,7 @@ set +o pipefail kubectl logs deployments/venafi-kubernetes-agent \ --follow \ --namespace venafi \ - | timeout 60 jq 'if .msg | test("Data sent successfully") then . | halt_error(0) end' + | timeout 60 jq 'try (if .msg | test("Data sent successfully") then . | halt_error(0) end) catch .' set -o pipefail # Create a unique TLS Secret and wait for it to appear in the Venafi certificate From 486403fc186a2a99e43bd5d9e0ff271ea97ea920 Mon Sep 17 00:00:00 2001 From: Mladen Rusev Date: Tue, 30 Sep 2025 20:33:59 +0300 Subject: [PATCH 10/30] explicit set path --- hack/e2e/test.sh | 10 ++++++---- make/02_mod.mk | 3 ++- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/hack/e2e/test.sh b/hack/e2e/test.sh index a437e760..8a95a1d6 100755 --- a/hack/e2e/test.sh +++ b/hack/e2e/test.sh @@ -216,7 +216,7 @@ set +o pipefail kubectl logs deployments/venafi-kubernetes-agent \ --follow \ --namespace venafi \ - | timeout 60 jq 'try (if .msg | test("Data sent successfully") then . | halt_error(0) end) catch .' + | timeout 60 jq 'if .msg | test("Data sent successfully") then . | halt_error(0) end' set -o pipefail # Create a unique TLS Secret and wait for it to appear in the Venafi certificate @@ -280,12 +280,14 @@ echo "Waiting for the helper pod to be ready..." kubectl wait --for=condition=Ready pod/coverage-helper-pod -n venafi --timeout=2m echo "Copying coverage files from the helper pod..." -mkdir -p $COVERAGE_HOST_PATH +mkdir -p /home/runner/work/jetstack-secure/jetstack-secure/_bin/artifacts +#mkdir -p $COVERAGE_HOST_PATH # We copy from the helper pod's mount path. -kubectl cp -n venafi "coverage-helper-pod:/coverage-data/." $COVERAGE_HOST_PATH +kubectl cp -n venafi "coverage-helper-pod:/coverage-data/." /home/runner/work/jetstack-secure/jetstack-secure/_bin/artifacts echo "Coverage files retrieved. Listing contents:" -ls -la $COVERAGE_HOST_PATH +#ls -la $COVERAGE_HOST_PATH +ls -la /home/runner/work/jetstack-secure/jetstack-secure/_bin/artifacts # --- MANDATORY CLEANUP --- #echo "Cleaning up helper pod and PersistentVolumeClaim..." diff --git a/make/02_mod.mk b/make/02_mod.mk index 2bc5cba5..964875d4 100644 --- a/make/02_mod.mk +++ b/make/02_mod.mk @@ -52,7 +52,8 @@ shared_generate_targets += generate-crds-venconn ## See `hack/e2e/test.sh` for the full test script. ## @category Testing test-e2e-gke: | $(NEEDS_HELM) $(NEEDS_STEP) $(NEEDS_VENCTL) - @COVERAGE_HOST_PATH="$(COVERAGE_HOST_PATH)" ./hack/e2e/test.sh + #COVERAGE_HOST_PATH="$(COVERAGE_HOST_PATH)" ./hack/e2e/test.sh + ./hack/e2e/test.sh .PHONY: test-helm ## Run `helm unittest`. From c304321622c0f6deb5a7d1019e5ba8da447b205c Mon Sep 17 00:00:00 2001 From: Mladen Rusev Date: Wed, 1 Oct 2025 15:32:47 +0300 Subject: [PATCH 11/30] retur to using COVERAGE_HOST_PATH; add podSecurityContext --- hack/e2e/test.sh | 16 +++------------- hack/e2e/values.coverage-pvc.yaml | 3 ++- make/02_mod.mk | 4 ++-- 3 files changed, 7 insertions(+), 16 deletions(-) diff --git a/hack/e2e/test.sh b/hack/e2e/test.sh index 8a95a1d6..8c8620a0 100755 --- a/hack/e2e/test.sh +++ b/hack/e2e/test.sh @@ -280,17 +280,7 @@ echo "Waiting for the helper pod to be ready..." kubectl wait --for=condition=Ready pod/coverage-helper-pod -n venafi --timeout=2m echo "Copying coverage files from the helper pod..." -mkdir -p /home/runner/work/jetstack-secure/jetstack-secure/_bin/artifacts -#mkdir -p $COVERAGE_HOST_PATH -# We copy from the helper pod's mount path. -kubectl cp -n venafi "coverage-helper-pod:/coverage-data/." /home/runner/work/jetstack-secure/jetstack-secure/_bin/artifacts - +mkdir -p $COVERAGE_HOST_PATH +kubectl cp -n venafi "coverage-helper-pod:/coverage-data/." $COVERAGE_HOST_PATH echo "Coverage files retrieved. Listing contents:" -#ls -la $COVERAGE_HOST_PATH -ls -la /home/runner/work/jetstack-secure/jetstack-secure/_bin/artifacts - -# --- MANDATORY CLEANUP --- -#echo "Cleaning up helper pod and PersistentVolumeClaim..." -#kubectl delete pod coverage-helper-pod -n venafi -#kubectl delete pvc coverage-pvc -n venafi -#echo "Cleanup complete." \ No newline at end of file +ls -la $COVERAGE_HOST_PATH \ No newline at end of file diff --git a/hack/e2e/values.coverage-pvc.yaml b/hack/e2e/values.coverage-pvc.yaml index 0ae6dbe2..70e6519f 100644 --- a/hack/e2e/values.coverage-pvc.yaml +++ b/hack/e2e/values.coverage-pvc.yaml @@ -1,8 +1,9 @@ +podSecurityContext: + fsGroup: 2000 volumes: - name: coverage-storage persistentVolumeClaim: claimName: coverage-pvc - volumeMounts: - name: coverage-storage mountPath: /coverage \ No newline at end of file diff --git a/make/02_mod.mk b/make/02_mod.mk index 964875d4..3b58bdd0 100644 --- a/make/02_mod.mk +++ b/make/02_mod.mk @@ -52,8 +52,8 @@ shared_generate_targets += generate-crds-venconn ## See `hack/e2e/test.sh` for the full test script. ## @category Testing test-e2e-gke: | $(NEEDS_HELM) $(NEEDS_STEP) $(NEEDS_VENCTL) - #COVERAGE_HOST_PATH="$(COVERAGE_HOST_PATH)" ./hack/e2e/test.sh - ./hack/e2e/test.sh + COVERAGE_HOST_PATH="$(COVERAGE_HOST_PATH)" ./hack/e2e/test.sh + #./hack/e2e/test.sh .PHONY: test-helm ## Run `helm unittest`. From 2366d0194a22d49121f855cfd0dde843b8ed6477 Mon Sep 17 00:00:00 2001 From: Mladen Rusev Date: Wed, 1 Oct 2025 16:00:49 +0300 Subject: [PATCH 12/30] try sending sigquit signal --- hack/e2e/test.sh | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/hack/e2e/test.sh b/hack/e2e/test.sh index 8c8620a0..16dea180 100755 --- a/hack/e2e/test.sh +++ b/hack/e2e/test.sh @@ -251,7 +251,26 @@ getCertificate() { for ((i=0;;i++)); do if getCertificate; then exit 0; fi; sleep 30; done | timeout -v -- 5m cat export AGENT_POD_NAME=$(kubectl get pods -n venafi -l app.kubernetes.io/name=venafi-kubernetes-agent -o jsonpath="{.items[0].metadata.name}") -echo "Scaling down agent deployment to kill pod and flush coverage to the PVC..." + +echo "Sending SIGQUIT to agent pod '${AGENT_POD_NAME}' to trigger graceful shutdown and flush coverage..." +# Use kubectl debug to attach a busybox container to the running pod. +# --target specifies the container to share the process space with. +# --share-processes allows our new container to see and signal the agent process. +# We then run 'kill -s QUIT 1' to signal PID 1 (the agent) to quit gracefully. +kubectl debug -q -n venafi "${AGENT_POD_NAME}" \ + --image=busybox:1.36 \ + --target=venafi-kubernetes-agent \ + --share-processes \ + -- sh -c 'kill -s QUIT 1' + +echo "Waiting for agent pod '${AGENT_POD_NAME}' to terminate gracefully..." +# The pod will now terminate because its main process is exiting. +# We wait for Kubernetes to recognize this and delete the pod object. +kubectl wait --for=delete pod/${AGENT_POD_NAME} -n venafi --timeout=90s + +echo "Scaling down deployment to prevent pod from restarting..." +# Now that the pod is gone and coverage is flushed, we scale the deployment +# to ensure the ReplicaSet controller doesn't create a new one. kubectl scale deployment venafi-kubernetes-agent -n venafi --replicas=0 echo "Waiting for agent pod '${AGENT_POD_NAME}' to terminate as a result of the scale-down..." kubectl wait --for=delete pod/${AGENT_POD_NAME} -n venafi --timeout=90s From 334914eb19d2c8d7890ff5eefe3c5d7812930e87 Mon Sep 17 00:00:00 2001 From: Mladen Rusev Date: Wed, 1 Oct 2025 16:37:13 +0300 Subject: [PATCH 13/30] dont kill the agent --- hack/e2e/test.sh | 106 +++++++++++++++++++++++------------------------ 1 file changed, 53 insertions(+), 53 deletions(-) diff --git a/hack/e2e/test.sh b/hack/e2e/test.sh index 16dea180..bb3aec62 100755 --- a/hack/e2e/test.sh +++ b/hack/e2e/test.sh @@ -250,56 +250,56 @@ getCertificate() { # Wait 5 minutes for the certificate to appear. for ((i=0;;i++)); do if getCertificate; then exit 0; fi; sleep 30; done | timeout -v -- 5m cat -export AGENT_POD_NAME=$(kubectl get pods -n venafi -l app.kubernetes.io/name=venafi-kubernetes-agent -o jsonpath="{.items[0].metadata.name}") - -echo "Sending SIGQUIT to agent pod '${AGENT_POD_NAME}' to trigger graceful shutdown and flush coverage..." -# Use kubectl debug to attach a busybox container to the running pod. -# --target specifies the container to share the process space with. -# --share-processes allows our new container to see and signal the agent process. -# We then run 'kill -s QUIT 1' to signal PID 1 (the agent) to quit gracefully. -kubectl debug -q -n venafi "${AGENT_POD_NAME}" \ - --image=busybox:1.36 \ - --target=venafi-kubernetes-agent \ - --share-processes \ - -- sh -c 'kill -s QUIT 1' - -echo "Waiting for agent pod '${AGENT_POD_NAME}' to terminate gracefully..." -# The pod will now terminate because its main process is exiting. -# We wait for Kubernetes to recognize this and delete the pod object. -kubectl wait --for=delete pod/${AGENT_POD_NAME} -n venafi --timeout=90s - -echo "Scaling down deployment to prevent pod from restarting..." -# Now that the pod is gone and coverage is flushed, we scale the deployment -# to ensure the ReplicaSet controller doesn't create a new one. -kubectl scale deployment venafi-kubernetes-agent -n venafi --replicas=0 -echo "Waiting for agent pod '${AGENT_POD_NAME}' to terminate as a result of the scale-down..." -kubectl wait --for=delete pod/${AGENT_POD_NAME} -n venafi --timeout=90s -echo "Starting helper pod to retrieve coverage files from the PVC..." - -kubectl apply -n venafi -f - < Date: Wed, 1 Oct 2025 18:19:10 +0300 Subject: [PATCH 14/30] add shareProcessNamespace=true to the agent --- hack/e2e/test.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/hack/e2e/test.sh b/hack/e2e/test.sh index bb3aec62..f8087c45 100755 --- a/hack/e2e/test.sh +++ b/hack/e2e/test.sh @@ -134,6 +134,7 @@ venctl components kubernetes apply \ --venafi-enhanced-issuer \ --approver-policy-enterprise \ --venafi-kubernetes-agent \ + --venafi-kubernetes-agent-set "shareProcessNamespace=true" --venafi-kubernetes-agent-version "${RELEASE_HELM_CHART_VERSION}" \ --venafi-kubernetes-agent-values-files "${script_dir}/values.venafi-kubernetes-agent.yaml" \ --venafi-kubernetes-agent-values-files "${script_dir}/values.coverage-pvc.yaml" \ From e303a34fdaaf5b39d750c11a65a10ba31577d570 Mon Sep 17 00:00:00 2001 From: Mladen Rusev Date: Wed, 1 Oct 2025 18:32:42 +0300 Subject: [PATCH 15/30] fixx err no flag --- hack/e2e/test.sh | 1 - hack/e2e/values.coverage-pvc.yaml | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hack/e2e/test.sh b/hack/e2e/test.sh index f8087c45..bb3aec62 100755 --- a/hack/e2e/test.sh +++ b/hack/e2e/test.sh @@ -134,7 +134,6 @@ venctl components kubernetes apply \ --venafi-enhanced-issuer \ --approver-policy-enterprise \ --venafi-kubernetes-agent \ - --venafi-kubernetes-agent-set "shareProcessNamespace=true" --venafi-kubernetes-agent-version "${RELEASE_HELM_CHART_VERSION}" \ --venafi-kubernetes-agent-values-files "${script_dir}/values.venafi-kubernetes-agent.yaml" \ --venafi-kubernetes-agent-values-files "${script_dir}/values.coverage-pvc.yaml" \ diff --git a/hack/e2e/values.coverage-pvc.yaml b/hack/e2e/values.coverage-pvc.yaml index 70e6519f..135cbdb5 100644 --- a/hack/e2e/values.coverage-pvc.yaml +++ b/hack/e2e/values.coverage-pvc.yaml @@ -6,4 +6,5 @@ volumes: claimName: coverage-pvc volumeMounts: - name: coverage-storage - mountPath: /coverage \ No newline at end of file + mountPath: /coverage +shareProcessNamespace: true \ No newline at end of file From 964f548ec0c0b563352fbf6351f6a2c3ae072346 Mon Sep 17 00:00:00 2001 From: Mladen Rusev Date: Thu, 2 Oct 2025 12:33:41 +0300 Subject: [PATCH 16/30] remove shareProcessNamespace --- hack/e2e/values.coverage-pvc.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/hack/e2e/values.coverage-pvc.yaml b/hack/e2e/values.coverage-pvc.yaml index 135cbdb5..70e6519f 100644 --- a/hack/e2e/values.coverage-pvc.yaml +++ b/hack/e2e/values.coverage-pvc.yaml @@ -6,5 +6,4 @@ volumes: claimName: coverage-pvc volumeMounts: - name: coverage-storage - mountPath: /coverage -shareProcessNamespace: true \ No newline at end of file + mountPath: /coverage \ No newline at end of file From 8c1012a983e3fdbee672003036686ee281013d42 Mon Sep 17 00:00:00 2001 From: Mladen Rusev Date: Thu, 2 Oct 2025 17:16:04 +0300 Subject: [PATCH 17/30] remove revert to original test.sh; create new test_ci.sh; add mounts to values; add kind cluster setup --- hack/e2e/test.sh | 70 ------ hack/e2e/test_ci.sh | 230 +++++++++++++++++++ hack/e2e/values.coverage-pvc.yaml | 9 - hack/e2e/values.venafi-kubernetes-agent.yaml | 9 + make/02_mod.mk | 13 +- make/config/kind/cluster.yaml | 20 ++ 6 files changed, 269 insertions(+), 82 deletions(-) create mode 100755 hack/e2e/test_ci.sh delete mode 100644 hack/e2e/values.coverage-pvc.yaml create mode 100644 make/config/kind/cluster.yaml diff --git a/hack/e2e/test.sh b/hack/e2e/test.sh index bb3aec62..dbf2195d 100755 --- a/hack/e2e/test.sh +++ b/hack/e2e/test.sh @@ -83,19 +83,6 @@ if ! gcloud container clusters get-credentials "${CLUSTER_NAME}"; then fi kubectl create ns venafi || true -kubectl apply -n venafi -f - </dev/null && pwd) +root_dir=$(cd "${script_dir}/../.." && pwd) +export TERM=dumb + +# Your Venafi Cloud API key. +: ${VEN_API_KEY?} +# Separate API Key for getting a pull secret. +: ${VEN_API_KEY_PULL?} +# The Venafi Cloud zone. +: ${VEN_ZONE?} +# The hostname of the Venafi API server (e.g., api.venafi.cloud). +: ${VEN_API_HOST?} +# The region of the Venafi API server (e.g., "us" or "eu"). +: ${VEN_VCP_REGION?} +# The base URL of the OCI registry (e.g., ttl.sh/some-random-uuid). +: ${OCI_BASE?} + +REMOTE_AGENT_IMAGE="${OCI_BASE}/venafi-kubernetes-agent-e2e" + +cd "${script_dir}" + +# Build and PUSH agent image and Helm chart to the anonymous registry +echo ">>> Building and pushing agent to '${REMOTE_AGENT_IMAGE}'..." +pushd "${root_dir}" +> release.env +make release \ + OCI_SIGN_ON_PUSH=false \ + oci_platforms=linux/amd64 \ + oci_preflight_image_name=${REMOTE_AGENT_IMAGE} \ + helm_chart_image_name=$OCI_BASE/charts/venafi-kubernetes-agent \ + GITHUB_OUTPUT=release.env +source release.env +popd + +AGENT_IMAGE_WITH_TAG="${REMOTE_AGENT_IMAGE}:${RELEASE_HELM_CHART_VERSION}" +echo ">>> Successfully pushed image: ${AGENT_IMAGE_WITH_TAG}" + +kubectl create ns venafi || true + +# Create pull secret for Venafi's OCI registry if it doesn't exist. +if ! kubectl get secret venafi-image-pull-secret -n venafi; then + echo ">>> Creating Venafi OCI registry pull secret..." + venctl iam service-accounts registry create \ + --api-key $VEN_API_KEY_PULL \ + --no-prompts \ + --owning-team "$(curl --fail-with-body -sS "https://${VEN_API_HOST}/v1/teams" -H "tppl-api-key: ${VEN_API_KEY_PULL}" | jq '.teams[0].id' -r)" \ + --name "venafi-kubernetes-agent-e2e-registry-${RANDOM}" \ + --scopes enterprise-cert-manager,enterprise-venafi-issuer,enterprise-approver-policy \ + | jq '{ + "apiVersion": "v1", + "kind": "Secret", + "metadata": { + "name": "venafi-image-pull-secret" + }, + "type": "kubernetes.io/dockerconfigjson", + "stringData": { + ".dockerconfigjson": { + "auths": { + "\(.oci_registry)": { + "username": .username, + "password": .password + } + } + } | tostring + } + }' \ + | kubectl create -n venafi -f - +fi + +echo ">>> Generating temporary Helm values for the custom agent image..." +cat < /tmp/agent-image-values.yaml +image: + repository: ${REMOTE_AGENT_IMAGE} + tag: ${RELEASE_HELM_CHART_VERSION} + pullPolicy: IfNotPresent +EOF + +echo ">>> Applying Venafi components to the cluster..." +export VENAFI_KUBERNETES_AGENT_CLIENT_ID="not-used-but-required-by-venctl" +venctl components kubernetes apply \ + --region $VEN_VCP_REGION \ + --cert-manager \ + --venafi-enhanced-issuer \ + --approver-policy-enterprise \ + --venafi-kubernetes-agent \ + --venafi-kubernetes-agent-version "${RELEASE_HELM_CHART_VERSION}" \ + --venafi-kubernetes-agent-values-files "${script_dir}/values.venafi-kubernetes-agent.yaml" \ + --venafi-kubernetes-agent-values-files "/tmp/agent-image-values.yaml" \ + --venafi-kubernetes-agent-custom-chart-repository "oci://${OCI_BASE}/charts" + +kubectl apply -n venafi -f venafi-components.yaml + +# Configure Workload Identity Federation with Venafi Cloud +echo ">>> Configuring Workload Identity Federation..." +subject="system:serviceaccount:venafi:venafi-components" +audience="https://${VEN_API_HOST}" +issuerURL=$(kubectl get --raw /.well-known/openid-configuration | jq -r '.issuer') +openidDiscoveryURL="${issuerURL}/.well-known/openid-configuration" +jwksURI=$(curl --fail-with-body -sSL ${openidDiscoveryURL} | jq -r '.jwks_uri') + +# Create the Venafi agent service account if one does not already exist +echo ">>> Ensuring Venafi Cloud service account exists for the agent..." +while true; do + tenantID=$(curl --fail-with-body -sSL -H "tppl-api-key: $VEN_API_KEY" https://${VEN_API_HOST}/v1/serviceaccounts \ + | jq -r '.[] | select(.issuerURL==$issuerURL and .subject == $subject) | .companyId' \ + --arg issuerURL "${issuerURL}" \ + --arg subject "${subject}") + + if [[ "${tenantID}" != "" ]]; then + echo "Service account already exists." + break + fi + + echo "Service account not found, creating it..." + jq -n '{ + "name": "venafi-kubernetes-agent-e2e-agent-\($random)", + "authenticationType": "rsaKeyFederated", + "scopes": ["kubernetes-discovery-federated", "certificate-issuance"], + "subject": $subject, + "audience": $audience, + "issuerURL": $issuerURL, + "jwksURI": $jwksURI, + "owner": $owningTeamID + }' \ + --arg random "${RANDOM}" \ + --arg subject "${subject}" \ + --arg audience "${audience}" \ + --arg issuerURL "${issuerURL}" \ + --arg jwksURI "${jwksURI}" \ + --arg owningTeamID "$(curl --fail-with-body -sS "https://${VEN_API_HOST}/v1/teams" -H "tppl-api-key: $VEN_API_KEY" | jq '.teams[0].id' -r)" \ + | curl "https://${VEN_API_HOST}/v1/serviceaccounts" \ + -H "tppl-api-key: $VEN_API_KEY" \ + --fail-with-body \ + -sSL --json @- +done + +# Create the VenafiConnection resource +echo ">>> Applying VenafiConnection resource..." +kubectl apply -n venafi -f - <>> Testing certificate issuance..." +envsubst >> Waiting for agent log message confirming successful data upload..." +set +o pipefail +kubectl logs deployments/venafi-kubernetes-agent \ + --follow \ + --namespace venafi \ + | timeout 60s jq 'if .msg | test("Data sent successfully") then . | halt_error(0) end' +set -o pipefail + +# Create a unique TLS secret and verify its discovery by the agent +echo ">>> Testing discovery of a manually created TLS secret..." +commonname="venafi-kubernetes-agent-e2e.$(uuidgen | tr '[:upper:]' '[:lower:]')" +openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout /tmp/tls.key -out /tmp/tls.crt -subj "/CN=$commonname" +kubectl create secret tls "$commonname" --cert=/tmp/tls.crt --key=/tmp/tls.key -o yaml --dry-run=client | kubectl apply -f - + +getCertificate() { + jq -n '{ + "expression": { + "field": "subjectCN", + "operator": "MATCH", + "value": $commonname + }, + "ordering": { + "orders": [ + { "direction": "DESC", "field": "certificatInstanceModificationDate" } + ] + }, + "paging": { "pageNumber": 0, "pageSize": 10 } + }' --arg commonname "${commonname}" \ + | curl "https://${VEN_API_HOST}/outagedetection/v1/certificatesearch?excludeSupersededInstances=true&ownershipTree=true" \ + -fsSL \ + -H "tppl-api-key: $VEN_API_KEY" \ + --json @- \ + | jq 'if .count == 0 then . | halt_error(1) end' +} + +# Wait up to 5 minutes for the certificate to appear in the Venafi inventory +echo ">>> Waiting for certificate '${commonname}' to appear in Venafi Cloud inventory..." +for ((i=0;;i++)); do + if getCertificate; then + echo "Successfully found certificate in Venafi Cloud." + exit 0; + fi; + echo "Certificate not found yet, retrying in 30 seconds..." + sleep 30; +done | timeout -v -- 5m cat + +echo "!!! Test Failed: Timed out waiting for certificate to appear in Venafi Cloud." +exit 1 \ No newline at end of file diff --git a/hack/e2e/values.coverage-pvc.yaml b/hack/e2e/values.coverage-pvc.yaml deleted file mode 100644 index 70e6519f..00000000 --- a/hack/e2e/values.coverage-pvc.yaml +++ /dev/null @@ -1,9 +0,0 @@ -podSecurityContext: - fsGroup: 2000 -volumes: - - name: coverage-storage - persistentVolumeClaim: - claimName: coverage-pvc -volumeMounts: - - name: coverage-storage - mountPath: /coverage \ No newline at end of file diff --git a/hack/e2e/values.venafi-kubernetes-agent.yaml b/hack/e2e/values.venafi-kubernetes-agent.yaml index 0e5c2120..c268c270 100644 --- a/hack/e2e/values.venafi-kubernetes-agent.yaml +++ b/hack/e2e/values.venafi-kubernetes-agent.yaml @@ -11,3 +11,12 @@ authentication: extraArgs: - --logging-format=json - --log-level=4 + +volumeMounts: + - name: coverage-volume + mountPath: /coverage +volumes: + - name: coverage-volume + hostPath: + path: /coverage + type: DirectoryOrCreate diff --git a/make/02_mod.mk b/make/02_mod.mk index 3b58bdd0..e8a1d02f 100644 --- a/make/02_mod.mk +++ b/make/02_mod.mk @@ -1,6 +1,14 @@ include make/test-unit.mk include make/ark/02_mod.mk +$(kind_cluster_config): make/config/kind/cluster.yaml | $(bin_dir)/scratch + @echo "--- COVERAGE_HOST_PATH is $(COVERAGE_HOST_PATH) ---" + mkdir -p $(COVERAGE_HOST_PATH) + @cat $< | \ + sed -e 's|{{KIND_IMAGES}}|$(CURDIR)/$(images_tar_dir)|g' | \ + sed -e 's|{{COVERAGE_HOST_PATH}}|$(COVERAGE_HOST_PATH)|g' \ + > $@ + GITHUB_OUTPUT ?= /dev/stderr .PHONY: release ## Publish all release artifacts (image + helm chart) @@ -51,9 +59,8 @@ shared_generate_targets += generate-crds-venconn ## Wait for it to log a message indicating successful data upload. ## See `hack/e2e/test.sh` for the full test script. ## @category Testing -test-e2e-gke: | $(NEEDS_HELM) $(NEEDS_STEP) $(NEEDS_VENCTL) - COVERAGE_HOST_PATH="$(COVERAGE_HOST_PATH)" ./hack/e2e/test.sh - #./hack/e2e/test.sh +test-e2e-gke: | kind-cluster $(NEEDS_HELM) $(NEEDS_STEP) $(NEEDS_VENCTL) + COVERAGE_HOST_PATH="$(COVERAGE_HOST_PATH)" ./hack/e2e/test_ci.sh .PHONY: test-helm ## Run `helm unittest`. diff --git a/make/config/kind/cluster.yaml b/make/config/kind/cluster.yaml new file mode 100644 index 00000000..e5fd1ac8 --- /dev/null +++ b/make/config/kind/cluster.yaml @@ -0,0 +1,20 @@ +apiVersion: kind.x-k8s.io/v1alpha4 +kind: Cluster +kubeadmConfigPatches: + - | + kind: ClusterConfiguration + metadata: + name: config + etcd: + local: + extraArgs: + unsafe-no-fsync: "true" + networking: + serviceSubnet: 10.0.0.0/16 +nodes: + - role: control-plane + extraMounts: + - hostPath: {{KIND_IMAGES}} + containerPath: /mounted_images + - hostPath: {{COVERAGE_HOST_PATH}} + containerPath: /coverage From e6fc461638fa4f16f76940aa1f23ca9e37f38fe3 Mon Sep 17 00:00:00 2001 From: Mladen Rusev Date: Fri, 3 Oct 2025 13:25:21 +0300 Subject: [PATCH 18/30] try back the GKE script --- hack/e2e/test.sh | 69 ++++++++++++++++++++ hack/e2e/values.venafi-kubernetes-agent.yaml | 15 +++-- 2 files changed, 77 insertions(+), 7 deletions(-) diff --git a/hack/e2e/test.sh b/hack/e2e/test.sh index dbf2195d..7efc7044 100755 --- a/hack/e2e/test.sh +++ b/hack/e2e/test.sh @@ -83,6 +83,19 @@ if ! gcloud container clusters get-credentials "${CLUSTER_NAME}"; then fi kubectl create ns venafi || true +kubectl apply -n venafi -f - < Date: Fri, 3 Oct 2025 13:30:54 +0300 Subject: [PATCH 19/30] change the make target --- make/02_mod.mk | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/make/02_mod.mk b/make/02_mod.mk index e8a1d02f..0d9b4844 100644 --- a/make/02_mod.mk +++ b/make/02_mod.mk @@ -60,7 +60,8 @@ shared_generate_targets += generate-crds-venconn ## See `hack/e2e/test.sh` for the full test script. ## @category Testing test-e2e-gke: | kind-cluster $(NEEDS_HELM) $(NEEDS_STEP) $(NEEDS_VENCTL) - COVERAGE_HOST_PATH="$(COVERAGE_HOST_PATH)" ./hack/e2e/test_ci.sh + #COVERAGE_HOST_PATH="$(COVERAGE_HOST_PATH)" ./hack/e2e/test_ci.sh + COVERAGE_HOST_PATH="$(COVERAGE_HOST_PATH)" ./hack/e2e/test.sh .PHONY: test-helm ## Run `helm unittest`. From 8cbb3966ff4e810990d2e0ada94a9a0e2f4643d4 Mon Sep 17 00:00:00 2001 From: Mladen Rusev Date: Fri, 3 Oct 2025 14:08:42 +0300 Subject: [PATCH 20/30] start ci run --- hack/e2e/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/e2e/test.sh b/hack/e2e/test.sh index 7efc7044..37e969c6 100755 --- a/hack/e2e/test.sh +++ b/hack/e2e/test.sh @@ -203,7 +203,7 @@ spec: - vcpOAuth: tenantID: ${tenantID} EOF - + envsubst Date: Fri, 3 Oct 2025 14:26:02 +0300 Subject: [PATCH 21/30] try func to pull files --- coverage_dummy.go | 7 +++++++ coverage_server.go | 40 ++++++++++++++++++++++++++++++++++++++++ main.go | 1 + 3 files changed, 48 insertions(+) create mode 100644 coverage_dummy.go create mode 100644 coverage_server.go diff --git a/coverage_dummy.go b/coverage_dummy.go new file mode 100644 index 00000000..4b66c15c --- /dev/null +++ b/coverage_dummy.go @@ -0,0 +1,7 @@ +//go:build !coverage + +package main + +func startCoverageServer() { + +} diff --git a/coverage_server.go b/coverage_server.go new file mode 100644 index 00000000..44c22aeb --- /dev/null +++ b/coverage_server.go @@ -0,0 +1,40 @@ +//go:build coverage + +package main + +import ( + "log" + "net/http" + "runtime/coverage" +) + +func startCoverageServer() { + log.Println("Coverage build detected. Starting private admin server on localhost:8081...") + adminMux := http.NewServeMux() + + adminMux.HandleFunc("/_debug/coverage/download", func(w http.ResponseWriter, r *http.Request) { + log.Println("Received request to download coverage counter data...") + w.Header().Set("Content-Type", "application/octet-stream") + w.Header().Set("Content-Disposition", `attachment; filename="coverage.out"`) + + if err := coverage.WriteCounters(w); err != nil { + log.Printf("Error writing coverage counters to response: %v", err) + } + }) + + adminMux.HandleFunc("/_debug/coverage/meta/download", func(w http.ResponseWriter, r *http.Request) { + log.Println("Received request to download coverage metadata...") + w.Header().Set("Content-Type", "application/octet-stream") + w.Header().Set("Content-Disposition", `attachment; filename="coverage.meta"`) + + if err := coverage.WriteMeta(w); err != nil { + log.Printf("Error writing coverage metadata to response: %v", err) + } + }) + + go func() { + if err := http.ListenAndServe("localhost:8081", adminMux); err != nil { + log.Printf("Admin server failed: %v", err) + } + }() +} diff --git a/main.go b/main.go index 18f19838..057f15d3 100644 --- a/main.go +++ b/main.go @@ -3,5 +3,6 @@ package main import "github.com/jetstack/preflight/cmd" func main() { + startCoverageServer() cmd.Execute() } From 018d7ddacf0d2d224b954b5371ce60fe002bd85d Mon Sep 17 00:00:00 2001 From: Mladen Rusev Date: Fri, 3 Oct 2025 14:43:57 +0300 Subject: [PATCH 22/30] change the port to avoid clash with healthz --- coverage_server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coverage_server.go b/coverage_server.go index 44c22aeb..1189f08e 100644 --- a/coverage_server.go +++ b/coverage_server.go @@ -33,7 +33,7 @@ func startCoverageServer() { }) go func() { - if err := http.ListenAndServe("localhost:8081", adminMux); err != nil { + if err := http.ListenAndServe("localhost:8089", adminMux); err != nil { log.Printf("Admin server failed: %v", err) } }() From 87f4a1f1e1df8a091a4e7fbeb192d27794087ee9 Mon Sep 17 00:00:00 2001 From: Mladen Rusev Date: Fri, 3 Oct 2025 15:13:36 +0300 Subject: [PATCH 23/30] uncondintional build of coverage server --- coverage_dummy.go | 7 ------- coverage_server.go | 2 -- 2 files changed, 9 deletions(-) delete mode 100644 coverage_dummy.go diff --git a/coverage_dummy.go b/coverage_dummy.go deleted file mode 100644 index 4b66c15c..00000000 --- a/coverage_dummy.go +++ /dev/null @@ -1,7 +0,0 @@ -//go:build !coverage - -package main - -func startCoverageServer() { - -} diff --git a/coverage_server.go b/coverage_server.go index 1189f08e..a2c70940 100644 --- a/coverage_server.go +++ b/coverage_server.go @@ -1,5 +1,3 @@ -//go:build coverage - package main import ( From 76a7b994420dce27b9f367fc664f55faa90364d1 Mon Sep 17 00:00:00 2001 From: Mladen Rusev Date: Fri, 3 Oct 2025 15:30:09 +0300 Subject: [PATCH 24/30] jsonify logging --- coverage_server.go | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/coverage_server.go b/coverage_server.go index a2c70940..b69b932b 100644 --- a/coverage_server.go +++ b/coverage_server.go @@ -1,38 +1,55 @@ package main import ( + "encoding/json" // Import to safely escape error strings + "fmt" // Import to format strings "log" "net/http" "runtime/coverage" ) func startCoverageServer() { - log.Println("Coverage build detected. Starting private admin server on localhost:8081...") adminMux := http.NewServeMux() adminMux.HandleFunc("/_debug/coverage/download", func(w http.ResponseWriter, r *http.Request) { - log.Println("Received request to download coverage counter data...") + // Simple info log as a JSON string + log.Println(`{"level":"info","message":"Received request to download coverage counter data"}`) + w.Header().Set("Content-Type", "application/octet-stream") w.Header().Set("Content-Disposition", `attachment; filename="coverage.out"`) if err := coverage.WriteCounters(w); err != nil { - log.Printf("Error writing coverage counters to response: %v", err) + // Safely marshal the error to escape quotes and special characters + escapedError, _ := json.Marshal(err.Error()) + + // Construct the final JSON string and log it + log.Println( + fmt.Sprintf(`{"level":"error","message":"Error writing coverage counters to response","error":%s}`, string(escapedError)), + ) } }) adminMux.HandleFunc("/_debug/coverage/meta/download", func(w http.ResponseWriter, r *http.Request) { - log.Println("Received request to download coverage metadata...") + log.Println(`{"level":"info","message":"Received request to download coverage metadata"}`) + w.Header().Set("Content-Type", "application/octet-stream") w.Header().Set("Content-Disposition", `attachment; filename="coverage.meta"`) if err := coverage.WriteMeta(w); err != nil { - log.Printf("Error writing coverage metadata to response: %v", err) + escapedError, _ := json.Marshal(err.Error()) + log.Println( + fmt.Sprintf(`{"level":"error","message":"Error writing coverage metadata to response","error":%s}`, string(escapedError)), + ) } }) go func() { + log.Println(`{"level":"info","message":"Starting private admin server","address":"localhost:8089"}`) if err := http.ListenAndServe("localhost:8089", adminMux); err != nil { - log.Printf("Admin server failed: %v", err) + escapedError, _ := json.Marshal(err.Error()) + log.Println( + fmt.Sprintf(`{"level":"error","message":"Admin server failed","error":%s}`, string(escapedError)), + ) } }() } From 4d90fa8740568b3b9bdca2ff4858022b6a3959d8 Mon Sep 17 00:00:00 2001 From: Mladen Rusev Date: Fri, 3 Oct 2025 15:46:13 +0300 Subject: [PATCH 25/30] remove initial logging --- coverage_server.go | 29 ++++++----------------------- 1 file changed, 6 insertions(+), 23 deletions(-) diff --git a/coverage_server.go b/coverage_server.go index b69b932b..2c86b69b 100644 --- a/coverage_server.go +++ b/coverage_server.go @@ -1,55 +1,38 @@ package main import ( - "encoding/json" // Import to safely escape error strings - "fmt" // Import to format strings "log" "net/http" "runtime/coverage" ) func startCoverageServer() { + //log.Println("Coverage build detected. Starting private admin server on localhost:8081...") adminMux := http.NewServeMux() adminMux.HandleFunc("/_debug/coverage/download", func(w http.ResponseWriter, r *http.Request) { - // Simple info log as a JSON string - log.Println(`{"level":"info","message":"Received request to download coverage counter data"}`) - + log.Println("{\n \"message\": \"Received request to download coverage counter data...\"\n}") w.Header().Set("Content-Type", "application/octet-stream") w.Header().Set("Content-Disposition", `attachment; filename="coverage.out"`) if err := coverage.WriteCounters(w); err != nil { - // Safely marshal the error to escape quotes and special characters - escapedError, _ := json.Marshal(err.Error()) - - // Construct the final JSON string and log it - log.Println( - fmt.Sprintf(`{"level":"error","message":"Error writing coverage counters to response","error":%s}`, string(escapedError)), - ) + log.Printf("Error writing coverage counters to response: %v", err) } }) adminMux.HandleFunc("/_debug/coverage/meta/download", func(w http.ResponseWriter, r *http.Request) { - log.Println(`{"level":"info","message":"Received request to download coverage metadata"}`) - + log.Println("Received request to download coverage metadata...") w.Header().Set("Content-Type", "application/octet-stream") w.Header().Set("Content-Disposition", `attachment; filename="coverage.meta"`) if err := coverage.WriteMeta(w); err != nil { - escapedError, _ := json.Marshal(err.Error()) - log.Println( - fmt.Sprintf(`{"level":"error","message":"Error writing coverage metadata to response","error":%s}`, string(escapedError)), - ) + log.Printf("Error writing coverage metadata to response: %v", err) } }) go func() { - log.Println(`{"level":"info","message":"Starting private admin server","address":"localhost:8089"}`) if err := http.ListenAndServe("localhost:8089", adminMux); err != nil { - escapedError, _ := json.Marshal(err.Error()) - log.Println( - fmt.Sprintf(`{"level":"error","message":"Admin server failed","error":%s}`, string(escapedError)), - ) + log.Printf("Admin server failed: %v", err) } }() } From e38f94cdec3dd1816c40fd625ea619f02d94e6d3 Mon Sep 17 00:00:00 2001 From: Mladen Rusev Date: Fri, 3 Oct 2025 16:16:30 +0300 Subject: [PATCH 26/30] refactored script ver --- hack/e2e/test.sh | 78 +++++++++++++++--------------------------------- 1 file changed, 24 insertions(+), 54 deletions(-) diff --git a/hack/e2e/test.sh b/hack/e2e/test.sh index 37e969c6..90aff7f2 100755 --- a/hack/e2e/test.sh +++ b/hack/e2e/test.sh @@ -141,7 +141,7 @@ venctl components kubernetes apply \ kubectl apply -n venafi -f venafi-components.yaml kubectl set env deployments/venafi-kubernetes-agent -n venafi GOCOVERDIR=/coverage -kubectl rollout status deployment/venafi-kubernetes-agent -n venafi --timeout=2m +kubectl rollout status deployment/venafi-kubernetes-agent -n venafi --timeout=5m subject="system:serviceaccount:venafi:venafi-components" audience="https://${VEN_API_HOST}" @@ -249,56 +249,26 @@ getCertificate() { # Wait 5 minutes for the certificate to appear. for ((i=0;;i++)); do if getCertificate; then exit 0; fi; sleep 30; done | timeout -v -- 5m cat -#export AGENT_POD_NAME=$(kubectl get pods -n venafi -l app.kubernetes.io/name=venafi-kubernetes-agent -o jsonpath="{.items[0].metadata.name}") -# -#echo "Sending SIGQUIT to agent pod '${AGENT_POD_NAME}' to trigger graceful shutdown and flush coverage..." -## Use kubectl debug to attach a busybox container to the running pod. -## --target specifies the container to share the process space with. -## --share-processes allows our new container to see and signal the agent process. -## We then run 'kill -s QUIT 1' to signal PID 1 (the agent) to quit gracefully. -#kubectl debug -q -n venafi "${AGENT_POD_NAME}" \ -# --image=busybox:1.36 \ -# --target=venafi-kubernetes-agent \ -# --share-processes \ -# -- sh -c 'kill -s QUIT 1' -# -#echo "Waiting for agent pod '${AGENT_POD_NAME}' to terminate gracefully..." -## The pod will now terminate because its main process is exiting. -## We wait for Kubernetes to recognize this and delete the pod object. -#kubectl wait --for=delete pod/${AGENT_POD_NAME} -n venafi --timeout=90s -# -#echo "Scaling down deployment to prevent pod from restarting..." -## Now that the pod is gone and coverage is flushed, we scale the deployment -## to ensure the ReplicaSet controller doesn't create a new one. -#kubectl scale deployment venafi-kubernetes-agent -n venafi --replicas=0 -#echo "Waiting for agent pod '${AGENT_POD_NAME}' to terminate as a result of the scale-down..." -#kubectl wait --for=delete pod/${AGENT_POD_NAME} -n venafi --timeout=90s -#echo "Starting helper pod to retrieve coverage files from the PVC..." -# -#kubectl apply -n venafi -f - < Date: Fri, 3 Oct 2025 16:49:03 +0300 Subject: [PATCH 27/30] add intermediate buffer --- coverage_server.go | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/coverage_server.go b/coverage_server.go index 2c86b69b..04df2ae7 100644 --- a/coverage_server.go +++ b/coverage_server.go @@ -1,22 +1,42 @@ package main import ( + "bytes" "log" "net/http" "runtime/coverage" ) func startCoverageServer() { - //log.Println("Coverage build detected. Starting private admin server on localhost:8081...") adminMux := http.NewServeMux() adminMux.HandleFunc("/_debug/coverage/download", func(w http.ResponseWriter, r *http.Request) { log.Println("{\n \"message\": \"Received request to download coverage counter data...\"\n}") + + var buffer bytes.Buffer + + // Attempt to write the coverage counters to the buffer. + if err := coverage.WriteCounters(&buffer); err != nil { + log.Printf("Error writing coverage counters to buffer: %v", err) + // Inform the client that an internal error occurred. + http.Error(w, "Failed to generate coverage report", http.StatusInternalServerError) + return + } + + // Check if any data was written to the buffer. + if buffer.Len() == 0 { + log.Println("Coverage data is empty. No counters were written.") + } else { + log.Printf("Successfully wrote %d bytes of coverage data to the buffer.", buffer.Len()) + } + + // If successful, proceed to write the buffer's content to the actual HTTP response. w.Header().Set("Content-Type", "application/octet-stream") w.Header().Set("Content-Disposition", `attachment; filename="coverage.out"`) - if err := coverage.WriteCounters(w); err != nil { - log.Printf("Error writing coverage counters to response: %v", err) + // Write the captured coverage data from the buffer to the response writer. + if _, err := w.Write(buffer.Bytes()); err != nil { + log.Printf("Error writing coverage data from buffer to response: %v", err) } }) From 645ce631b5f09f48fd59bc15127e1c7e3c712bab Mon Sep 17 00:00:00 2001 From: Mladen Rusev Date: Fri, 3 Oct 2025 17:09:38 +0300 Subject: [PATCH 28/30] set the covermode to atomic --- make/_shared/oci-build/01_mod.mk | 1 + 1 file changed, 1 insertion(+) diff --git a/make/_shared/oci-build/01_mod.mk b/make/_shared/oci-build/01_mod.mk index 97e42730..93632387 100644 --- a/make/_shared/oci-build/01_mod.mk +++ b/make/_shared/oci-build/01_mod.mk @@ -29,6 +29,7 @@ $(ko_config_path_$1:$(CURDIR)/%=%): | $(NEEDS_YQ) $(bin_dir)/scratch/image $(YQ) '.builds[0].ldflags[1] = "-w"' | \ $(YQ) '.builds[0].ldflags[2] = "{{.Env.LDFLAGS}}"' | \ $(YQ) '.builds[0].flags[0] = "-cover"' | \ + $(YQ) '.builds[0].flags[1] = "-covermode=atomic"' | \ $(YQ) '.builds[0].linux_capabilities = "$(oci_$1_linux_capabilities)"' \ > $(CURDIR)/$(oci_layout_path_$1).ko_config.yaml From 49335c660c8ef3fda88a36840f0d624e53b3b842 Mon Sep 17 00:00:00 2001 From: Mladen Rusev Date: Fri, 3 Oct 2025 20:36:15 +0300 Subject: [PATCH 29/30] add intermediate buffer to meta file download --- coverage_server.go | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/coverage_server.go b/coverage_server.go index 04df2ae7..a59c7b0b 100644 --- a/coverage_server.go +++ b/coverage_server.go @@ -11,8 +11,6 @@ func startCoverageServer() { adminMux := http.NewServeMux() adminMux.HandleFunc("/_debug/coverage/download", func(w http.ResponseWriter, r *http.Request) { - log.Println("{\n \"message\": \"Received request to download coverage counter data...\"\n}") - var buffer bytes.Buffer // Attempt to write the coverage counters to the buffer. @@ -42,11 +40,25 @@ func startCoverageServer() { adminMux.HandleFunc("/_debug/coverage/meta/download", func(w http.ResponseWriter, r *http.Request) { log.Println("Received request to download coverage metadata...") + + var buffer bytes.Buffer + if err := coverage.WriteMeta(&buffer); err != nil { + log.Printf("Error writing coverage meta to buffer: %v", err) + // Inform the client that an internal error occurred. + http.Error(w, "Failed to generate coverage meta", http.StatusInternalServerError) + return + } + // Check if any data was written to the buffer. + if buffer.Len() == 0 { + log.Println("Coverage meta is empty.") + } else { + log.Printf("Successfully wrote %d bytes of coverage meta to the buffer.", buffer.Len()) + } + w.Header().Set("Content-Type", "application/octet-stream") w.Header().Set("Content-Disposition", `attachment; filename="coverage.meta"`) - - if err := coverage.WriteMeta(w); err != nil { - log.Printf("Error writing coverage metadata to response: %v", err) + if _, err := w.Write(buffer.Bytes()); err != nil { + log.Printf("Error writing coverage meta from buffer to response: %v", err) } }) From 6572f1ecae740b7d282c2b46f6039773e92edc2f Mon Sep 17 00:00:00 2001 From: Mladen Rusev Date: Fri, 3 Oct 2025 20:59:45 +0300 Subject: [PATCH 30/30] try an older version of script after resolving -covermode problem --- hack/e2e/test.sh | 72 ++++++++++++++++++++++++++++++++++-------------- 1 file changed, 51 insertions(+), 21 deletions(-) diff --git a/hack/e2e/test.sh b/hack/e2e/test.sh index 90aff7f2..36bf0146 100755 --- a/hack/e2e/test.sh +++ b/hack/e2e/test.sh @@ -141,7 +141,7 @@ venctl components kubernetes apply \ kubectl apply -n venafi -f venafi-components.yaml kubectl set env deployments/venafi-kubernetes-agent -n venafi GOCOVERDIR=/coverage -kubectl rollout status deployment/venafi-kubernetes-agent -n venafi --timeout=5m +kubectl rollout status deployment/venafi-kubernetes-agent -n venafi --timeout=2m subject="system:serviceaccount:venafi:venafi-components" audience="https://${VEN_API_HOST}" @@ -203,7 +203,7 @@ spec: - vcpOAuth: tenantID: ${tenantID} EOF - + envsubst