Merge pull request docker#9563 from milas/e2e-env

e2e: isolate test command env from system env

Author: glours

pkg/e2e/compose_build_test.go

@@ -18,7 +18,6 @@ package e2e
 
 import (
 	"net/http"
-	"os"
 	"strings"
 	"testing"
 	"time"
@@ -81,11 +80,6 @@ func TestLocalComposeBuild(t *testing.T) {
 	})
 
 	t.Run("build failed with ssh default value", func(t *testing.T) {
-		//unset SSH_AUTH_SOCK to be sure we don't have a default value for the SSH Agent
-		defaultSSHAUTHSOCK := os.Getenv("SSH_AUTH_SOCK")
-		os.Unsetenv("SSH_AUTH_SOCK")                         //nolint:errcheck
-		defer os.Setenv("SSH_AUTH_SOCK", defaultSSHAUTHSOCK) //nolint:errcheck
-
 		res := c.RunDockerComposeCmdNoCheck(t, "--project-directory", "fixtures/build-test", "build", "--ssh", "")
 		res.Assert(t, icmd.Expected{
 			ExitCode: 1,

pkg/e2e/compose_environment_test.go

@@ -17,7 +17,6 @@
 package e2e
 
 import (
-	"os"
 	"strings"
 	"testing"
 
@@ -43,13 +42,11 @@ func TestEnvPriority(t *testing.T) {
 	// 4. Dockerfile
 	// 5. Variable is not defined
 	t.Run("compose file priority", func(t *testing.T) {
-		os.Setenv("WHEREAMI", "shell") //nolint:errcheck
-		defer os.Unsetenv("WHEREAMI")  //nolint:errcheck
-
-		res := c.RunDockerComposeCmd(t, "-f", "./fixtures/environment/env-priority/compose-with-env.yaml",
-			"--project-directory", projectDir, "--env-file", "./fixtures/environment/env-priority/.env.override",
-			"run", "--rm", "-e", "WHEREAMI", "env-compose-priority")
-
+		cmd := c.NewDockerComposeCmd(t, "-f", "./fixtures/environment/env-priority/compose-with-env.yaml",
+			"--project-directory", projectDir, "--env-file", "./fixtures/environment/env-priority/.env.override", "run",
+			"--rm", "-e", "WHEREAMI", "env-compose-priority")
+		cmd.Env = append(cmd.Env, "WHEREAMI=shell")
+		res := icmd.RunCmd(cmd)
 		assert.Equal(t, strings.TrimSpace(res.Stdout()), "Compose File")
 	})
 
@@ -60,12 +57,11 @@ func TestEnvPriority(t *testing.T) {
 	// 4. Dockerfile
 	// 5. Variable is not defined
 	t.Run("shell priority", func(t *testing.T) {
-		os.Setenv("WHEREAMI", "shell") //nolint:errcheck
-		defer os.Unsetenv("WHEREAMI")  //nolint:errcheck
-
-		res := c.RunDockerComposeCmd(t, "-f", "./fixtures/environment/env-priority/compose.yaml",
-			"--project-directory", projectDir, "--env-file", "./fixtures/environment/env-priority/.env.override",
-			"run", "--rm", "-e", "WHEREAMI", "env-compose-priority")
+		cmd := c.NewDockerComposeCmd(t, "-f", "./fixtures/environment/env-priority/compose.yaml", "--project-directory",
+			projectDir, "--env-file", "./fixtures/environment/env-priority/.env.override", "run", "--rm", "-e",
+			"WHEREAMI", "env-compose-priority")
+		cmd.Env = append(cmd.Env, "WHEREAMI=shell")
+		res := icmd.RunCmd(cmd)
 		assert.Equal(t, strings.TrimSpace(res.Stdout()), "shell")
 	})
 
@@ -137,11 +133,10 @@ func TestEnvInterpolation(t *testing.T) {
 	// 4. Dockerfile
 	// 5. Variable is not defined
 	t.Run("shell priority from run command", func(t *testing.T) {
-		os.Setenv("WHEREAMI", "shell") //nolint:errcheck
-		defer os.Unsetenv("WHEREAMI")  //nolint:errcheck
-		res := c.RunDockerComposeCmd(t, "-f", "./fixtures/environment/env-interpolation/compose.yaml",
+		cmd := c.NewDockerComposeCmd(t, "-f", "./fixtures/environment/env-interpolation/compose.yaml",
 			"--project-directory", projectDir, "config")
-
+		cmd.Env = append(cmd.Env, "WHEREAMI=shell")
+		res := icmd.RunCmd(cmd)
 		res.Assert(t, icmd.Expected{Out: `IMAGE: default_env:shell`})
 	})
 }

pkg/e2e/ddev_test.go

@@ -19,38 +19,52 @@ package e2e
 import (
 	"fmt"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"runtime"
 	"strings"
 	"testing"
 
+	"github.com/stretchr/testify/require"
 	"gotest.tools/v3/assert"
 )
 
 const ddevVersion = "v1.19.1"
 
 func TestComposeRunDdev(t *testing.T) {
 	if !composeStandaloneMode {
-		t.Skip("Not running on standalone mode.")
+		t.Skip("Not running in plugin mode - ddev only supports invoking standalone `docker-compose`")
 	}
 	if runtime.GOOS == "windows" {
 		t.Skip("Running on Windows. Skipping...")
 	}
-	_ = os.Setenv("DDEV_DEBUG", "true")
 
-	c := NewParallelCLI(t)
-	dir, err := os.MkdirTemp("", t.Name()+"-")
-	assert.NilError(t, err)
+	// ddev shells out to `docker` and `docker-compose` (standalone), so a
+	// temporary directory is created with symlinks to system Docker and the
+	// locally-built standalone Compose binary to use as PATH
+	requiredTools := []string{
+		findToolInPath(t, DockerExecutableName),
+		ComposeStandalonePath(t),
+		findToolInPath(t, "tar"),
+		findToolInPath(t, "gzip"),
+	}
+	pathDir := t.TempDir()
+	for _, tool := range requiredTools {
+		require.NoError(t, os.Symlink(tool, filepath.Join(pathDir, filepath.Base(tool))),
+			"Could not create symlink for %q", tool)
+	}
 
-	// ddev needs to be able to find mkcert to figure out where certs are.
-	_ = os.Setenv("PATH", fmt.Sprintf("%s:%s", os.Getenv("PATH"), dir))
+	c := NewCLI(t, WithEnv(
+		"DDEV_DEBUG=true",
+		fmt.Sprintf("PATH=%s", pathDir),
+	))
 
-	siteName := filepath.Base(dir)
+	ddevDir := t.TempDir()
+	siteName := filepath.Base(ddevDir)
 
 	t.Cleanup(func() {
-		_ = c.RunCmdInDir(t, dir, "./ddev", "delete", "-Oy")
-		_ = c.RunCmdInDir(t, dir, "./ddev", "poweroff")
-		_ = os.RemoveAll(dir)
+		_ = c.RunCmdInDir(t, ddevDir, "./ddev", "delete", "-Oy")
+		_ = c.RunCmdInDir(t, ddevDir, "./ddev", "poweroff")
 	})
 
 	osName := "linux"
@@ -59,28 +73,34 @@ func TestComposeRunDdev(t *testing.T) {
 	}
 
 	compressedFilename := fmt.Sprintf("ddev_%s-%s.%s.tar.gz", osName, runtime.GOARCH, ddevVersion)
-	c.RunCmdInDir(t, dir, "curl", "-LO",
-		fmt.Sprintf("https://github.com/drud/ddev/releases/download/%s/%s",
-			ddevVersion,
-			compressedFilename))
+	c.RunCmdInDir(t, ddevDir, "curl", "-LO", fmt.Sprintf("https://github.com/drud/ddev/releases/download/%s/%s",
+		ddevVersion,
+		compressedFilename))
 
-	c.RunCmdInDir(t, dir, "tar", "-xzf", compressedFilename)
+	c.RunCmdInDir(t, ddevDir, "tar", "-xzf", compressedFilename)
 
 	// Create a simple index.php we can test against.
-	c.RunCmdInDir(t, dir, "sh", "-c", "echo '<?php\nprint \"ddev is working\";' >index.php")
+	c.RunCmdInDir(t, ddevDir, "sh", "-c", "echo '<?php\nprint \"ddev is working\";' >index.php")
 
-	c.RunCmdInDir(t, dir, "./ddev", "config", "--auto")
-	c.RunCmdInDir(t, dir, "./ddev", "config", "global", "--use-docker-compose-from-path")
-	vRes := c.RunCmdInDir(t, dir, "./ddev", "version")
+	c.RunCmdInDir(t, ddevDir, "./ddev", "config", "--auto")
+	c.RunCmdInDir(t, ddevDir, "./ddev", "config", "global", "--use-docker-compose-from-path")
+	vRes := c.RunCmdInDir(t, ddevDir, "./ddev", "version")
 	out := vRes.Stdout()
 	fmt.Printf("ddev version: %s\n", out)
 
-	c.RunCmdInDir(t, dir, "./ddev", "poweroff")
+	c.RunCmdInDir(t, ddevDir, "./ddev", "poweroff")
 
-	c.RunCmdInDir(t, dir, "./ddev", "start", "-y")
+	c.RunCmdInDir(t, ddevDir, "./ddev", "start", "-y")
 
-	curlRes := c.RunCmdInDir(t, dir, "curl", "-sSL", fmt.Sprintf("http://%s.ddev.site", siteName))
+	curlRes := c.RunCmdInDir(t, ddevDir, "curl", "-sSL", fmt.Sprintf("http://%s.ddev.site", siteName))
 	out = curlRes.Stdout()
 	fmt.Println(out)
 	assert.Assert(t, strings.Contains(out, "ddev is working"), "Could not start project")
 }
+
+func findToolInPath(t testing.TB, name string) string {
+	t.Helper()
+	binPath, err := exec.LookPath(name)
+	require.NoError(t, err, "Could not find %q in path", name)
+	return binPath
+}

pkg/e2e/framework.go

@@ -30,8 +30,8 @@ import (
 	"time"
 
 	"github.com/pkg/errors"
+	"github.com/stretchr/testify/require"
 	"gotest.tools/v3/assert"
-	is "gotest.tools/v3/assert/cmp"
 	"gotest.tools/v3/icmd"
 	"gotest.tools/v3/poll"
 
@@ -59,19 +59,59 @@ func init() {
 
 // CLI is used to wrap the CLI for end to end testing
 type CLI struct {
+	// ConfigDir for Docker configuration (set as DOCKER_CONFIG)
 	ConfigDir string
+
+	// HomeDir for tools that look for user files (set as HOME)
+	HomeDir string
+
+	// env overrides to apply to every invoked command
+	//
+	// To populate, use WithEnv when creating a CLI instance.
+	env []string
 }
 
-// NewParallelCLI returns a configured CLI with t.Parallel() set
-func NewParallelCLI(t *testing.T) *CLI {
+// CLIOption to customize behavior for all commands for a CLI instance.
+type CLIOption func(c *CLI)
+
+// NewParallelCLI marks the parent test as parallel and returns a CLI instance
+// suitable for usage across child tests.
+func NewParallelCLI(t *testing.T, opts ...CLIOption) *CLI {
+	t.Helper()
 	t.Parallel()
-	return NewCLI(t)
+	return NewCLI(t, opts...)
 }
 
-// NewCLI returns a CLI to use for E2E tests
-func NewCLI(t testing.TB) *CLI {
-	d, err := ioutil.TempDir("", "")
-	assert.Check(t, is.Nil(err))
+// NewCLI creates a CLI instance for running E2E tests.
+func NewCLI(t testing.TB, opts ...CLIOption) *CLI {
+	t.Helper()
+
+	configDir := t.TempDir()
+	initializePlugins(t, configDir)
+
+	c := &CLI{
+		ConfigDir: configDir,
+		HomeDir:   t.TempDir(),
+	}
+
+	for _, opt := range opts {
+		opt(c)
+	}
+
+	return c
+}
+
+// WithEnv sets environment variables that will be passed to commands.
+func WithEnv(env ...string) CLIOption {
+	return func(c *CLI) {
+		c.env = append(c.env, env...)
+	}
+}
+
+// initializePlugins copies the necessary plugin files to the temporary config
+// directory for the test.
+func initializePlugins(t testing.TB, d string) {
+	t.Helper()
 
 	t.Cleanup(func() {
 		if t.Failed() {
@@ -101,8 +141,6 @@ func NewCLI(t testing.TB) *CLI {
 			panic(err)
 		}
 	}
-
-	return &CLI{ConfigDir: d}
 }
 
 func dirContents(dir string) []string {
@@ -154,28 +192,33 @@ func CopyFile(sourceFile string, destinationFile string) error {
 	return err
 }
 
+// BaseEnvironment provides the minimal environment variables used across all
+// Docker / Compose commands.
+func (c *CLI) BaseEnvironment() []string {
+	return []string{
+		"HOME=" + c.HomeDir,
+		"USER=" + os.Getenv("USER"),
+		"DOCKER_CONFIG=" + c.ConfigDir,
+		"KUBECONFIG=invalid",
+	}
+}
+
 // NewCmd creates a cmd object configured with the test environment set
 func (c *CLI) NewCmd(command string, args ...string) icmd.Cmd {
-	env := append(os.Environ(),
-		"DOCKER_CONFIG="+c.ConfigDir,
-		"KUBECONFIG=invalid",
-	)
 	return icmd.Cmd{
 		Command: append([]string{command}, args...),
-		Env:     env,
+		Env:     append(c.BaseEnvironment(), c.env...),
 	}
 }
 
 // NewCmdWithEnv creates a cmd object configured with the test environment set with additional env vars
 func (c *CLI) NewCmdWithEnv(envvars []string, command string, args ...string) icmd.Cmd {
-	env := append(os.Environ(),
-		append(envvars,
-			"DOCKER_CONFIG="+c.ConfigDir,
-			"KUBECONFIG=invalid")...,
-	)
+	// base env -> CLI overrides -> cmd overrides
+	cmdEnv := append(c.BaseEnvironment(), c.env...)
+	cmdEnv = append(cmdEnv, envvars...)
 	return icmd.Cmd{
 		Command: append([]string{command}, args...),
-		Env:     env,
+		Env:     cmdEnv,
 	}
 }
 
@@ -234,13 +277,34 @@ func (c *CLI) RunDockerComposeCmd(t testing.TB, args ...string) *icmd.Result {
 
 // RunDockerComposeCmdNoCheck runs a docker compose command, don't presume of any expectation and returns a result
 func (c *CLI) RunDockerComposeCmdNoCheck(t testing.TB, args ...string) *icmd.Result {
+	return icmd.RunCmd(c.NewDockerComposeCmd(t, args...))
+}
+
+// NewDockerComposeCmd creates a command object for Compose, either in plugin
+// or standalone mode (based on build tags).
+func (c *CLI) NewDockerComposeCmd(t testing.TB, args ...string) icmd.Cmd {
+	t.Helper()
 	if composeStandaloneMode {
-		composeBinary, err := findExecutable(DockerComposeExecutableName, []string{"../../bin", "../../../bin"})
-		assert.NilError(t, err)
-		return icmd.RunCmd(c.NewCmd(composeBinary, args...))
+		return c.NewCmd(ComposeStandalonePath(t), args...)
 	}
 	args = append([]string{"compose"}, args...)
-	return icmd.RunCmd(c.NewCmd(DockerExecutableName, args...))
+	return c.NewCmd(DockerExecutableName, args...)
+}
+
+// ComposeStandalonePath returns the path to the locally-built Compose
+// standalone binary from the repo.
+//
+// This function will fail the test immediately if invoked when not running
+// in standalone test mode.
+func ComposeStandalonePath(t testing.TB) string {
+	t.Helper()
+	if !composeStandaloneMode {
+		require.Fail(t, "Not running in standalone mode")
+	}
+	composeBinary, err := findExecutable(DockerComposeExecutableName, []string{"../../bin", "../../../bin"})
+	require.NoError(t, err, "Could not find standalone Compose binary (%q)",
+		DockerComposeExecutableName)
+	return composeBinary
 }
 
 // StdoutContains returns a predicate on command result expecting a string in stdout