feat: support starting from an already-built image (#296)

- Extracts 'magic directives' to constants.go.
- Adds 'magic image' file to signify envbuilder should skip the destructive 'build image' stage.
- Modifies logic for copying binary into built image: we now copy to build context and remove it after build finishes to avoid leaving around files owned by root:root. Also ensures files are created with consistent permissions.

Author: johnstcn

cmd/envbuilder/main.go

@@ -68,6 +68,7 @@ func envbuilderCmd() serpent.Command {
 				img, err := envbuilder.RunCacheProbe(inv.Context(), o)
 				if err != nil {
 					o.Logger(log.LevelError, "error: %s", err)
+					return err
 				}
 				digest, err := img.Digest()
 				if err != nil {

constants/constants.go

@@ -2,6 +2,7 @@ package constants
 
 import (
 	"errors"
+	"fmt"
 	"path/filepath"
 )
 
@@ -32,4 +33,32 @@ var (
 	// MagicFile is the location of the build context when
 	// using remote build mode.
 	MagicRemoteRepoDir = filepath.Join(MagicDir, "repo")
+
+	// MagicBinaryLocation is the expected location of the envbuilder binary
+	// inside a builder image.
+	MagicBinaryLocation = filepath.Join(MagicDir, "bin", "envbuilder")
+
+	// MagicImage is a file that is created in the image when
+	// envbuilder has already been run. This is used to skip
+	// the destructive initial build step when 'resuming' envbuilder
+	// from a previously built image.
+	MagicImage = filepath.Join(MagicDir, "image")
+
+	// MagicTempDir is a directory inside the build context inside which
+	// we place files referenced by MagicDirectives.
+	MagicTempDir = ".envbuilder.tmp"
+
+	// MagicDirectives are directives automatically appended to Dockerfiles
+	// when pushing the image. These directives allow the built image to be
+	// 're-used'.
+	MagicDirectives = fmt.Sprintf(`
+COPY --chmod=0755 %[1]s %[2]s
+COPY --chmod=0644 %[3]s %[4]s
+USER root
+WORKDIR /
+ENTRYPOINT [%[2]q]
+`,
+		".envbuilder.tmp/envbuilder", MagicBinaryLocation,
+		".envbuilder.tmp/image", MagicImage,
+	)
 )

envbuilder.go

@@ -27,6 +27,7 @@ import (
 	"github.com/coder/envbuilder/constants"
 	"github.com/coder/envbuilder/git"
 	"github.com/coder/envbuilder/options"
+	"github.com/go-git/go-billy/v5"
 
 	"github.com/GoogleContainerTools/kaniko/pkg/config"
 	"github.com/GoogleContainerTools/kaniko/pkg/creds"
@@ -311,26 +312,58 @@ func Run(ctx context.Context, opts options.Options) error {
 	}
 
 	// In order to allow 'resuming' envbuilder, embed the binary into the image
-	// if it is being pushed
+	// if it is being pushed.
+	// As these files will be owned by root, it is considerate to clean up
+	// after we're done!
+	cleanupBuildContext := func() {}
 	if opts.PushImage {
-		exePath, err := os.Executable()
-		if err != nil {
-			return xerrors.Errorf("get exe path: %w", err)
+		// Add exceptions in Kaniko's ignorelist for these magic files we add.
+		if err := util.AddAllowedPathToDefaultIgnoreList(opts.BinaryPath); err != nil {
+			return fmt.Errorf("add envbuilder binary to ignore list: %w", err)
+		}
+		if err := util.AddAllowedPathToDefaultIgnoreList(constants.MagicImage); err != nil {
+			return fmt.Errorf("add magic image file to ignore list: %w", err)
 		}
-		// Add an exception for the current running binary in kaniko ignore list
-		if err := util.AddAllowedPathToDefaultIgnoreList(exePath); err != nil {
-			return xerrors.Errorf("add exe path to ignore list: %w", err)
+		magicTempDir := filepath.Join(buildParams.BuildContext, constants.MagicTempDir)
+		if err := opts.Filesystem.MkdirAll(magicTempDir, 0o755); err != nil {
+			return fmt.Errorf("create magic temp dir in build context: %w", err)
 		}
+		// Add the magic directives that embed the binary into the built image.
+		buildParams.DockerfileContent += constants.MagicDirectives
 		// Copy the envbuilder binary into the build context.
-		buildParams.DockerfileContent += fmt.Sprintf(`
-COPY --chmod=0755 %s %s
-USER root
-WORKDIR /
-ENTRYPOINT [%q]`, exePath, exePath, exePath)
-		dst := filepath.Join(buildParams.BuildContext, exePath)
-		if err := copyFile(exePath, dst); err != nil {
-			return xerrors.Errorf("copy running binary to build context: %w", err)
+		// External callers will need to specify the path to the desired envbuilder binary.
+		envbuilderBinDest := filepath.Join(
+			magicTempDir,
+			filepath.Base(constants.MagicBinaryLocation),
+		)
+		// Also touch the magic file that signifies the image has been built!
+		magicImageDest := filepath.Join(
+			magicTempDir,
+			filepath.Base(constants.MagicImage),
+		)
+		// Clean up after build!
+		var cleanupOnce sync.Once
+		cleanupBuildContext = func() {
+			cleanupOnce.Do(func() {
+				for _, path := range []string{magicImageDest, envbuilderBinDest, magicTempDir} {
+					if err := opts.Filesystem.Remove(path); err != nil {
+						opts.Logger(log.LevelWarn, "failed to clean up magic temp dir from build context: %w", err)
+					}
+				}
+			})
+		}
+		defer cleanupBuildContext()
+
+		opts.Logger(log.LevelDebug, "copying envbuilder binary at %q to build context %q", opts.BinaryPath, envbuilderBinDest)
+		if err := copyFile(opts.Filesystem, opts.BinaryPath, envbuilderBinDest, 0o755); err != nil {
+			return fmt.Errorf("copy envbuilder binary to build context: %w", err)
+		}
+
+		opts.Logger(log.LevelDebug, "touching magic image file at %q in build context %q", magicImageDest, buildParams.BuildContext)
+		if err := touchFile(opts.Filesystem, magicImageDest, 0o755); err != nil {
+			return fmt.Errorf("touch magic image file in build context: %w", err)
 		}
+
 	}
 
 	// temp move of all ro mounts
@@ -354,8 +387,10 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath)
 	stderrWriter, closeStderr := log.Writer(opts.Logger)
 	defer closeStderr()
 	build := func() (v1.Image, error) {
-		_, err := opts.Filesystem.Stat(constants.MagicFile)
-		if err == nil && opts.SkipRebuild {
+		defer cleanupBuildContext()
+		_, alreadyBuiltErr := opts.Filesystem.Stat(constants.MagicFile)
+		_, isImageErr := opts.Filesystem.Stat(constants.MagicImage)
+		if (alreadyBuiltErr == nil && opts.SkipRebuild) || isImageErr == nil {
 			endStage := startStage("🏗️ Skipping build because of cache...")
 			imageRef, err := devcontainer.ImageFromDockerfile(buildParams.DockerfileContent)
 			if err != nil {
@@ -381,26 +416,7 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath)
 		if err := maybeDeleteFilesystem(opts.Logger, opts.ForceSafe); err != nil {
 			return nil, fmt.Errorf("delete filesystem: %w", err)
 		}
-		/*
-			stdoutReader, stdoutWriter := io.Pipe()
-			stderrReader, stderrWriter := io.Pipe()
-			defer stdoutReader.Close()
-			defer stdoutWriter.Close()
-			defer stderrReader.Close()
-			defer stderrWriter.Close()
-			go func() {
-				scanner := bufio.NewScanner(stdoutReader)
-				for scanner.Scan() {
-					opts.Logger(log.LevelInfo, "%s", scanner.Text())
-				}
-			}()
-			go func() {
-				scanner := bufio.NewScanner(stderrReader)
-				for scanner.Scan() {
-					opts.Logger(log.LevelInfo, "%s", scanner.Text())
-				}
-			}()
-		*/
+
 		cacheTTL := time.Hour * 24 * 7
 		if opts.CacheTTLDays != 0 {
 			cacheTTL = time.Hour * 24 * time.Duration(opts.CacheTTLDays)
@@ -1064,23 +1080,41 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error)
 	// We expect an image built and pushed by envbuilder to have the envbuilder
 	// binary present at a predefined path. In order to correctly replicate the
 	// build via executor.RunCacheProbe we need to have the *exact* copy of the
-	// envbuilder binary available used to build the image.
-	exePath := opts.BinaryPath
-	// Add an exception for the current running binary in kaniko ignore list
-	if err := util.AddAllowedPathToDefaultIgnoreList(exePath); err != nil {
-		return nil, xerrors.Errorf("add exe path to ignore list: %w", err)
-	}
+	// envbuilder binary available used to build the image and we also need to
+	// add the magic directives to the Dockerfile content.
+	buildParams.DockerfileContent += constants.MagicDirectives
+	magicTempDir := filepath.Join(buildParams.BuildContext, constants.MagicTempDir)
+	if err := opts.Filesystem.MkdirAll(magicTempDir, 0o755); err != nil {
+		return nil, fmt.Errorf("create magic temp dir in build context: %w", err)
+	}
+	envbuilderBinDest := filepath.Join(
+		magicTempDir,
+		filepath.Base(constants.MagicBinaryLocation),
+	)
+
 	// Copy the envbuilder binary into the build context.
-	buildParams.DockerfileContent += fmt.Sprintf(`
-COPY --chmod=0755 %s %s
-USER root
-WORKDIR /
-ENTRYPOINT [%q]`, exePath, exePath, exePath)
-	dst := filepath.Join(buildParams.BuildContext, exePath)
-	if err := copyFile(exePath, dst); err != nil {
+	opts.Logger(log.LevelDebug, "copying envbuilder binary at %q to build context %q", opts.BinaryPath, buildParams.BuildContext)
+	if err := copyFile(opts.Filesystem, opts.BinaryPath, envbuilderBinDest, 0o755); err != nil {
 		return nil, xerrors.Errorf("copy envbuilder binary to build context: %w", err)
 	}
 
+	// Also touch the magic file that signifies the image has been built!
+	magicImageDest := filepath.Join(
+		magicTempDir,
+		filepath.Base(constants.MagicImage),
+	)
+	if err := touchFile(opts.Filesystem, magicImageDest, 0o755); err != nil {
+		return nil, fmt.Errorf("touch magic image file in build context: %w", err)
+	}
+	defer func() {
+		// Clean up after we're done!
+		for _, path := range []string{magicImageDest, envbuilderBinDest, magicTempDir} {
+			if err := opts.Filesystem.Remove(path); err != nil {
+				opts.Logger(log.LevelWarn, "failed to clean up magic temp dir from build context: %w", err)
+			}
+		}
+	}()
+
 	stdoutWriter, closeStdout := log.Writer(opts.Logger)
 	defer closeStdout()
 	stderrWriter, closeStderr := log.Writer(opts.Logger)
@@ -1138,8 +1172,8 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath)
 		},
 		SrcContext: buildParams.BuildContext,
 
-		// For cached image utilization, produce reproducible builds.
-		Reproducible: opts.PushImage,
+		// When performing a cache probe, always perform reproducible snapshots.
+		Reproducible: true,
 	}
 
 	endStage := startStage("🏗️ Checking for cached image...")
@@ -1382,24 +1416,38 @@ func maybeDeleteFilesystem(logger log.Func, force bool) error {
 	return util.DeleteFilesystem()
 }
 
-func copyFile(src, dst string) error {
-	content, err := os.ReadFile(src)
+func copyFile(fs billy.Filesystem, src, dst string, mode fs.FileMode) error {
+	srcF, err := fs.Open(src)
 	if err != nil {
-		return fmt.Errorf("read file failed: %w", err)
+		return fmt.Errorf("open src file: %w", err)
 	}
+	defer srcF.Close()
 
-	err = os.MkdirAll(filepath.Dir(dst), 0o755)
+	err = fs.MkdirAll(filepath.Dir(dst), mode)
 	if err != nil {
-		return fmt.Errorf("mkdir all failed: %w", err)
+		return fmt.Errorf("create destination dir failed: %w", err)
 	}
 
-	err = os.WriteFile(dst, content, 0o644)
+	dstF, err := fs.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, mode)
 	if err != nil {
-		return fmt.Errorf("write file failed: %w", err)
+		return fmt.Errorf("open dest file for writing: %w", err)
+	}
+	defer dstF.Close()
+
+	if _, err := io.Copy(dstF, srcF); err != nil {
+		return fmt.Errorf("copy failed: %w", err)
 	}
 	return nil
 }
 
+func touchFile(fs billy.Filesystem, dst string, mode fs.FileMode) error {
+	f, err := fs.OpenFile(dst, os.O_RDWR|os.O_CREATE|os.O_TRUNC, mode)
+	if err != nil {
+		return xerrors.Errorf("failed to touch file: %w", err)
+	}
+	return f.Close()
+}
+
 func initDockerConfigJSON(dockerConfigBase64 string) (func() error, error) {
 	var cleanupOnce sync.Once
 	noop := func() error { return nil }

go.mod

@@ -4,7 +4,7 @@ go 1.22.4
 
 // There are a few options we need added to Kaniko!
 // See: https://github.com/GoogleContainerTools/kaniko/compare/main...coder:kaniko:main
-replace github.com/GoogleContainerTools/kaniko => github.com/coder/kaniko v0.0.0-20240717115058-0ba2908ca4d3
+replace github.com/GoogleContainerTools/kaniko => github.com/coder/kaniko v0.0.0-20240803153527-10d1800455b9
 
 // Required to import codersdk due to gvisor dependency.
 replace tailscale.com => github.com/coder/tailscale v1.1.1-0.20240702054557-aa558fbe5374

go.sum

@@ -171,8 +171,8 @@ github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoC
 github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI=
 github.com/coder/coder/v2 v2.10.1-0.20240704130443-c2d44d16a352 h1:L/EjCuZxs5tOcqqCaASj/nu65TRYEFcTt8qRQfHZXX0=
 github.com/coder/coder/v2 v2.10.1-0.20240704130443-c2d44d16a352/go.mod h1:P1KoQSgnKEAG6Mnd3YlGzAophty+yKA9VV48LpfNRvo=
-github.com/coder/kaniko v0.0.0-20240717115058-0ba2908ca4d3 h1:Q7L6cjKfw3DIyhKIcgCJEmgxnUTBajmMDrHxXvxgBZs=
-github.com/coder/kaniko v0.0.0-20240717115058-0ba2908ca4d3/go.mod h1:YMK7BlxerzLlMwihGxNWUaFoN9LXCij4P+w/8/fNlcM=
+github.com/coder/kaniko v0.0.0-20240803153527-10d1800455b9 h1:d01T5YbPN1yc1mXjIXG59YcQQoT/9idvqFErjWHfsZ4=
+github.com/coder/kaniko v0.0.0-20240803153527-10d1800455b9/go.mod h1:YMK7BlxerzLlMwihGxNWUaFoN9LXCij4P+w/8/fNlcM=
 github.com/coder/pretty v0.0.0-20230908205945-e89ba86370e0 h1:3A0ES21Ke+FxEM8CXx9n47SZOKOpgSE1bbJzlE4qPVs=
 github.com/coder/pretty v0.0.0-20230908205945-e89ba86370e0/go.mod h1:5UuS2Ts+nTToAMeOjNlnHFkPahrtDkmpydBen/3wgZc=
 github.com/coder/quartz v0.1.0 h1:cLL+0g5l7xTf6ordRnUMMiZtRE8Sq5LxpghS63vEXrQ=