Merge pull request moby#5594 from tonistiigi/dockerfile-test-versioning

test: allow testing frontends from different version

Author: AkihiroSuda

.github/workflows/.test.yml

@@ -104,7 +104,6 @@ jobs:
       TESTFLAGS: "-v --parallel=6 --timeout=30m"
       GOTESTSUM_FORMAT: "standard-verbose"
       TEST_IMAGE_BUILD: "0"
-      TEST_IMAGE_ID: "buildkit-tests"
     strategy:
       fail-fast: false
       matrix:
@@ -161,7 +160,6 @@ jobs:
           targets: integration-tests
           set: |
             *.cache-from=type=gha,scope=${{ inputs.cache_scope }}
-            *.output=type=docker,name=${{ env.TEST_IMAGE_ID }}
         env:
           BUILDKITD_TAGS: ${{ matrix.tags }}
           TEST_COVERAGE: 1

docker-bake.hcl

@@ -51,6 +51,24 @@ variable "TEST_COVERAGE" {
   default = null
 }
 
+variable "TEST_IMAGE_NAME" {
+  default = "buildkit-tests"
+}
+
+variable "TEST_CONTEXT" {
+  default = "."
+  description = "Context for building the test image"
+}
+
+variable "TEST_BINARIES_CONTEXT" {
+  default = TEST_CONTEXT
+  description = "Context for building the buildkitd for test image"
+}
+
+variable "BUILDKIT_SYNTAX" {
+  default = null
+}
+
 function "bindir" {
   params = [defaultdir]
   result = DESTDIR != "" ? DESTDIR : "./bin/${defaultdir}"
@@ -117,12 +135,26 @@ target "integration-tests-base" {
   output = ["type=cacheonly"]
 }
 
+target "integration-tests-binaries" {
+  inherits = ["_common"]
+  target = "binaries"
+  context = TEST_BINARIES_CONTEXT
+}
+
 target "integration-tests" {
   inherits = ["integration-tests-base"]
   target = "integration-tests"
+  context = TEST_CONTEXT
+  contexts = TEST_CONTEXT != TEST_BINARIES_CONTEXT ? {
+    "binaries" = "target:integration-tests-binaries"
+  } : null
   args = {
     GOBUILDFLAGS = TEST_COVERAGE == "1" ? "-cover" : null
+    BUILDKIT_SYNTAX = BUILDKIT_SYNTAX
   }
+  output = [
+    "type=docker,name=${TEST_IMAGE_NAME}",
+  ]
 }
 
 group "validate" {

frontend/dockerfile/cmd/dockerfile-frontend/hack/release

@@ -104,6 +104,7 @@ case $TYP in
   buildxCmd build $platformFlag $cacheFromFlags $cacheToFlags $nocacheFilterFlag $(buildAttestFlags) \
     --build-arg "CHANNEL=$TAG" \
     --build-arg "BUILDTAGS=$buildTags" \
+    --build-arg "BUILDKIT_CONTEXT_KEEP_GIT_DIR=1" \
     --output "${outputFlag},name=$REPO:$pushTag" \
     --file "./frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile" \
     $currentcontext
@@ -125,6 +126,7 @@ case $TYP in
   buildxCmd build $platformFlag $cacheFromFlags $cacheToFlags $nocacheFilterFlag $(buildAttestFlags) \
     --build-arg "CHANNEL=$TAG" \
     --build-arg "BUILDTAGS=$buildTags" \
+    --build-arg "BUILDKIT_CONTEXT_KEEP_GIT_DIR=1" \
     --output "$outputFlag" \
     --file "./frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile" \
     $currentcontext

hack/images

@@ -112,5 +112,5 @@ if [[ "$RELEASE" = "true" ]] && [[ "$GITHUB_ACTIONS" = "true" ]]; then
   nocacheFilterFlag="--no-cache-filter=buildkit-export,gobuild-base,rootless"
 fi
 
-buildxCmd build --build-arg BUILDKIT_DEBUG $platformFlag $targetFlag $importCacheFlags $exportCacheFlags $tagFlags $outputFlag $nocacheFilterFlag $attestFlags \
+buildxCmd build --build-arg "BUILDKIT_CONTEXT_KEEP_GIT_DIR=1" --build-arg BUILDKIT_DEBUG $platformFlag $targetFlag $importCacheFlags $exportCacheFlags $tagFlags $outputFlag $nocacheFilterFlag $attestFlags \
   $currentcontext

hack/test

@@ -15,14 +15,15 @@ set -eu -o pipefail
 
 : "${TEST_COVERAGE=}"
 : "${TEST_IMAGE_BUILD=1}"
-: "${TEST_IMAGE_ID=buildkit-tests}"
+: "${TEST_IMAGE_NAME=buildkit-tests}"
 : "${TEST_INTEGRATION=}"
 : "${TEST_GATEWAY=}"
 : "${TEST_DOCKERFILE=}"
 : "${TEST_DOCKERD=}"
 : "${TEST_DOCKERD_BINARY=$(which dockerd)}"
 : "${TEST_REPORT_SUFFIX=}"
 : "${TEST_KEEP_CACHE=}"
+: "${TEST_SUITE_CONTEXT=}"
 : "${TESTFLAGS=}"
 
 : "${DOCKERFILE_RELEASES=}"
@@ -87,20 +88,15 @@ if [ "$TEST_COVERAGE" == "1" ]; then
   export GO_TEST_COVERPROFILE="/testreports/coverage-report$TEST_REPORT_SUFFIX.txt"
 fi
 
+if [ -n "$TEST_SUITE_CONTEXT" ]; then
+  export TEST_BINARIES_CONTEXT=$currentcontext
+  # FIXME: something breaks with the syntax when using the context
+  export BUILDKIT_SYNTAX="docker/dockerfile:1.10.0"
+  currentcontext=$TEST_SUITE_CONTEXT
+fi
+
 if [ "$TEST_IMAGE_BUILD" = "1" ]; then
-  buildxCmd build $cacheFromFlags \
-    --build-arg ALPINE_VERSION \
-    --build-arg GO_VERSION \
-    --build-arg BUILDKITD_TAGS \
-    --build-arg HTTP_PROXY \
-    --build-arg HTTPS_PROXY \
-    --build-arg NO_PROXY \
-    --build-arg GOBUILDFLAGS \
-    --build-arg VERIFYFLAGS \
-    --build-arg CGO_ENABLED \
-    --target "integration-tests" \
-    --output "type=docker,name=$TEST_IMAGE_ID" \
-    $currentcontext
+  TEST_CONTEXT=$currentcontext buildxCmd bake integration-tests
 fi
 
 cacheVolume="buildkit-test-cache"
@@ -131,7 +127,7 @@ if [ "$TEST_INTEGRATION" == 1 ]; then
   cid=$(dockerCmd create $baseCreateFlags \
     ${BUILDKIT_INTEGRATION_SNAPSHOTTER:+"-eBUILDKIT_INTEGRATION_SNAPSHOTTER"} \
     -e SKIP_INTEGRATION_TESTS \
-    $TEST_IMAGE_ID \
+    $TEST_IMAGE_NAME \
     gotestsumandcover $gotestsumArgs --packages="${TESTPKGS:-./...}" -- $gotestArgs ${TESTFLAGS:--v})
   if [ "$TEST_DOCKERD" = "1" ]; then
     dockerCmd cp "$TEST_DOCKERD_BINARY" $cid:/usr/bin/dockerd
@@ -141,8 +137,8 @@ fi
 
 if [ "$TEST_GATEWAY" == 1 ]; then
   # Build-test "github.com/moby/buildkit/frontend/gateway/client", which isn't otherwise built by CI
-  # It really only needs buildkit-base. We have integration-tests in $TEST_IMAGE_ID, which is a direct child of buildkit-base.
-  cid=$(dockerCmd create --rm --volumes-from=$cacheVolume --entrypoint="" $TEST_IMAGE_ID go build -v ./frontend/gateway/client)
+  # It really only needs buildkit-base. We have integration-tests in $TEST_IMAGE_NAME, which is a direct child of buildkit-base.
+  cid=$(dockerCmd create --rm --volumes-from=$cacheVolume --entrypoint="" $TEST_IMAGE_NAME go build -v ./frontend/gateway/client)
   dockerCmd start -a $cid
 fi
 
@@ -165,7 +161,7 @@ if [ "$TEST_DOCKERFILE" == 1 ]; then
     tarout=$(mktemp -t dockerfile-frontend.XXXXXXXXXX)
 
     buildxCmd build $cacheFromFlags \
-      --build-arg "BUILDTAGS=$buildtags" \
+      --build-arg "BUILDTAGS=$buildtags" --build-arg "BUILDKIT_CONTEXT_KEEP_GIT_DIR=1" \
       --file "./frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile" \
       --output "type=oci,dest=$tarout" \
       $currentcontext
@@ -176,7 +172,7 @@ if [ "$TEST_DOCKERFILE" == 1 ]; then
           -e BUILDKIT_WORKER_RANDOM \
           -e FRONTEND_GATEWAY_ONLY=local:/$release.tar \
           -e EXTERNAL_DF_FRONTEND=/dockerfile-frontend \
-          $TEST_IMAGE_ID \
+          $TEST_IMAGE_NAME \
           gotestsumandcover $gotestsumArgs --packages=./frontend/dockerfile -- $gotestArgs --count=1 -tags "$buildtags" ${TESTFLAGS:--v})
         dockerCmd cp $tarout $cid:/$release.tar
         if [ "$TEST_DOCKERD" = "1" ]; then

hack/util

@@ -36,12 +36,11 @@ buildAttestFlags() {
   fi
 }
 
-currentref=""
 currentcontext="."
 cacheFromFlags=""
 cacheToFlags=""
 if [ "$GITHUB_ACTIONS" = "true" ] && [ "$GITHUB_REPOSITORY" = "moby/buildkit" ]; then
-  currentref="https://github.com/$GITHUB_REPOSITORY.git#$GITHUB_REF"
+  currentcontext="https://github.com/$GITHUB_REPOSITORY.git#$GITHUB_REF"
   if [ -n "$CACHE_FROM" ]; then
     for cfrom in $CACHE_FROM; do
       if [[ $cfrom == *"type=gha"* ]]; then
@@ -69,9 +68,6 @@ if [ "$GITHUB_ACTIONS" = "true" ] && [ "$GITHUB_REPOSITORY" = "moby/buildkit" ];
     done
   fi
 fi
-if [ -n "$currentref" ]; then
-  currentcontext="--build-arg BUILDKIT_CONTEXT_KEEP_GIT_DIR=1 $currentref"
-fi
 if [ -n "$CONTEXT" ]; then
   currentcontext=$CONTEXT
 fi