BUILD: ci: speed up ci with parallel start e2e stages

Author: oktalz

.gitlab-ci.yml

@@ -2,6 +2,7 @@ stages:
   - diff
   - lint
   - unit-tests
+  - build
   - e2e-tests
 variables:
   KUBECTL: v1.19.3
@@ -25,6 +26,7 @@ diff:
     - branches
 tidy:
   stage: lint
+  needs: []
   image:
     name: $CI_REGISTRY_GO/golang:1.19
     entrypoint: [""]
@@ -38,6 +40,7 @@ tidy:
     - branches
 golangci_lint:
   stage: lint
+  needs: []
   image:
     name: $CI_REGISTRY_GO/lint:1.33
     entrypoint: [""]
@@ -50,6 +53,7 @@ golangci_lint:
     - branches
 lint-commit-msg:
   stage: lint
+  needs: []
   image:
     name: $CI_REGISTRY_GO/check-commit:v2.1.0
     entrypoint: [""]
@@ -60,6 +64,7 @@ lint-commit-msg:
   only:
     - merge_requests
 unit-tests:
+  needs: ["diff", "tidy"]
   stage: unit-tests
   image:
     name: $CI_REGISTRY_GO/haproxy-alpine:2.5-go1.19
@@ -72,8 +77,9 @@ unit-tests:
   only:
     - merge_requests
     - branches
-.kind_deployment:
-  stage: e2e-tests
+docker-build:
+  stage: build
+  needs: []
   image: $CI_REGISTRY_GO/docker:stable-go1.19
   services:
     - name: $CI_REGISTRY_GO/docker:18.09.7-dind
@@ -82,13 +88,47 @@ unit-tests:
     - go
   before_script:
     - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY_GO
-  script:
     - go version
-    - docker pull $CI_REGISTRY_GO/alpine:3
+    - docker pull -q $CI_REGISTRY_GO/alpine:3
     - docker image tag $CI_REGISTRY_GO/alpine:3 alpine:3
-    - docker pull $CI_REGISTRY_GO/golang:1.19-alpine
+    - docker pull -q $CI_REGISTRY_GO/golang:1.19-alpine
     - docker image tag $CI_REGISTRY_GO/golang:1.19-alpine golang:1.19-alpine
-    - wget -q -o /dev/null -O &> /dev/null /usr/local/bin/kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND}/kind-linux-amd64
+    - sed -i "s~FROM golang~FROM $CI_REGISTRY_GO/golang~g" "build/Dockerfile"
+    - sed -i "s~FROM haproxytech/haproxy-alpine~FROM $CI_REGISTRY_GO/haproxy-alpine~g" "build/Dockerfile"
+  script:
+    - mkdir -p tar
+    - make build
+    - docker save -o tar/k8sIC.tar haproxytech/kubernetes-ingress:latest
+  artifacts:
+    when: on_success
+    paths:
+      - tar
+    expire_in: 15 minutes
+.kind_deployment:
+  stage: e2e-tests
+  needs: ["diff", "tidy", "docker-build"]
+  image: $CI_REGISTRY_GO/docker:stable-go1.19
+  services:
+    - name: $CI_REGISTRY_GO/docker:18.09.7-dind
+      alias: docker
+  tags:
+    - go
+  artifacts:
+    paths:
+      - tar
+    expire_in: 30 minutes
+  before_script:
+    - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY_GO
+    - docker load -i tar/k8sIC.tar
+    - go version
+    - go mod download &
+    #- docker pull -q $CI_REGISTRY_GO/alpine:3
+    #- docker image tag $CI_REGISTRY_GO/alpine:3 alpine:3
+    #- docker pull -q $CI_REGISTRY_GO/golang:1.19-alpine
+    #- docker image tag $CI_REGISTRY_GO/golang:1.19-alpine golang:1.19-alpine
+    - docker pull -q $CI_REGISTRY_GO/haproxytech/http-echo:latest
+    - docker image tag $CI_REGISTRY_GO/haproxytech/http-echo:latest haproxytech/http-echo:latest
+    - wget -nv -O /usr/local/bin/kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND}/kind-linux-amd64
     - chmod +x /usr/local/bin/kind
     - wget -q -o /dev/null -O &> /dev/null /usr/local/bin/kubectl https://storage.googleapis.com/kubernetes-release/release/${KUBECTL}/bin/linux/amd64/kubectl
     - chmod +x /usr/local/bin/kubectl
@@ -101,16 +141,20 @@ unit-tests:
     - sed -i -E -e "s/localhost|0\.0\.0\.0|127\.0\.0\.1/docker/g" "deploy/tests/e2e/client.go"
     - until kubectl get nodes -o wide;do sleep 2; done
     - kubectl get pods -A
-    - CI_ENV=gitlab deploy/tests/create.sh
-    - kubectl get pods -A
-    - KIND_URL=docker CGO_ENABLED=0 go test -v ./deploy/tests/e2e/... --tags=e2e_parallel
-    - KIND_URL=docker CGO_ENABLED=0 go test -v -p 1 ./deploy/tests/e2e/... --tags=e2e_sequential
   after_script:
     - kubectl get pods -A
     - kind delete cluster --name=dev
   only:
     - merge_requests
     - branches
+  parallel:
+    matrix:
+      - TEST_PART: [ "18#e2e_parallel", "1#e2e_sequential" ]
+        VAR_DUMMY: "YES"
+  script:
+    - CI_ENV=gitlab deploy/tests/create.sh
+    - kubectl get pods -A
+    - IFS='#' read -r -a array <<< $TEST_PART; KIND_URL=docker CGO_ENABLED=0 go test -v -p ${array[0]} ./deploy/tests/e2e/... --tags=${array[1]}
 .kind_deployment_schedules:
   only:
     - schedules

Makefile

@@ -7,6 +7,7 @@ test:
 
 .PHONY: e2e
 e2e:
+	go clean -testcache
 	go test ./... --tags=e2e_parallel
 	go test ./... -p 1 --tags=e2e_sequential
 

deploy/tests/create.sh

@@ -9,7 +9,12 @@ if [ -n "${CI_ENV}" ]; then
   echo "cluster was already created by $CI_ENV CI"
 
   echo "building image for ingress controller"
-  docker build --build-arg TARGETPLATFORM="linux/amd64" -t haproxytech/kubernetes-ingress -f build/Dockerfile .
+  if [ -n "${GITLAB_CI}" ]; then
+    echo "haproxytech/kubernetes-ingress image already available from previous stage"
+    #docker build --build-arg TARGETPLATFORM="linux/amd64" -t haproxytech/kubernetes-ingress -f build/Dockerfile .
+  else
+    docker build --build-arg TARGETPLATFORM="linux/amd64" -t haproxytech/kubernetes-ingress -f build/Dockerfile .
+  fi
 
   echo "loading image of ingress controller in kind"
   kind load docker-image haproxytech/kubernetes-ingress:latest  --name=$clustername
@@ -31,7 +36,11 @@ else
   kind load docker-image haproxytech/kubernetes-ingress:latest  --name=$clustername
 fi
 
-docker build --build-arg TARGETPLATFORM="linux/amd64" -t haproxytech/http-echo -f deploy/tests/images/http-echo/Dockerfile deploy/tests/images/http-echo
+if [ -n "${GITLAB_CI}" ]; then
+  echo "haproxytech/http-echo:latest pulled from CI registry"
+else
+  docker build --build-arg TARGETPLATFORM="linux/amd64" -t haproxytech/http-echo -f deploy/tests/images/http-echo/Dockerfile deploy/tests/images/http-echo
+fi
 echo "loading image http-echo in kind"
 kind load docker-image haproxytech/http-echo:latest  --name=$clustername