e2e: robustness changes for ddev test

The most important change here is to ensure that the correct Compose
standalone binary is used by `ddev`. Since it invokes Compose itself,
we need to ensure that `PATH` is set appropriately such that it finds
the binary we want to test rather than something from the system.

As part of this, the rest of the environment has been isolated, which
should make the test more reliable, and avoids polluting `~/.ddev`
with test artifacts by using a tmpdir as `HOME` for the test instead
of the user's real home folder.

Signed-off-by: Milas Bowman <milas.bowman@docker.com>

Author: milas

pkg/e2e/ddev_test.go

@@ -19,38 +19,52 @@ package e2e
 import (
 	"fmt"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"runtime"
 	"strings"
 	"testing"
 
+	"github.com/stretchr/testify/require"
 	"gotest.tools/v3/assert"
 )
 
 const ddevVersion = "v1.19.1"
 
 func TestComposeRunDdev(t *testing.T) {
 	if !composeStandaloneMode {
-		t.Skip("Not running on standalone mode.")
+		t.Skip("Not running in plugin mode - ddev only supports invoking standalone `docker-compose`")
 	}
 	if runtime.GOOS == "windows" {
 		t.Skip("Running on Windows. Skipping...")
 	}
-	_ = os.Setenv("DDEV_DEBUG", "true")
 
-	c := NewParallelCLI(t)
-	dir, err := os.MkdirTemp("", t.Name()+"-")
-	assert.NilError(t, err)
-
-	// ddev needs to be able to find mkcert to figure out where certs are.
-	_ = os.Setenv("PATH", fmt.Sprintf("%s:%s", os.Getenv("PATH"), dir))
-
-	siteName := filepath.Base(dir)
+	// ddev shells out to `docker` and `docker-compose` (standalone), so a
+	// temporary directory is created with symlinks to system Docker and the
+	// locally-built standalone Compose binary to use as PATH
+	pathDir := t.TempDir()
+	dockerBin, err := exec.LookPath(DockerExecutableName)
+	require.NoError(t, err, "Could not find %q in path", DockerExecutableName)
+	require.NoError(t, os.Symlink(dockerBin, filepath.Join(pathDir, DockerExecutableName)),
+		"Could not create %q symlink", DockerExecutableName)
+
+	composeBin := ComposeStandalonePath(t)
+	require.NoError(t, os.Symlink(composeBin, filepath.Join(pathDir, DockerComposeExecutableName)),
+		"Could not create %q symlink", DockerComposeExecutableName)
+
+	c := NewCLI(t, WithEnv(
+		"DDEV_DEBUG=true",
+		fmt.Sprintf("HOME=%s", t.TempDir()),
+		fmt.Sprintf("USER=%s", os.Getenv("USER")),
+		fmt.Sprintf("PATH=%s", pathDir),
+	))
+
+	ddevDir := t.TempDir()
+	siteName := filepath.Base(ddevDir)
 
 	t.Cleanup(func() {
-		_ = c.RunCmdInDir(t, dir, "./ddev", "delete", "-Oy")
-		_ = c.RunCmdInDir(t, dir, "./ddev", "poweroff")
-		_ = os.RemoveAll(dir)
+		_ = c.RunCmdInDir(t, ddevDir, "./ddev", "delete", "-Oy")
+		_ = c.RunCmdInDir(t, ddevDir, "./ddev", "poweroff")
 	})
 
 	osName := "linux"
@@ -59,27 +73,26 @@ func TestComposeRunDdev(t *testing.T) {
 	}
 
 	compressedFilename := fmt.Sprintf("ddev_%s-%s.%s.tar.gz", osName, runtime.GOARCH, ddevVersion)
-	c.RunCmdInDir(t, dir, "curl", "-LO",
-		fmt.Sprintf("https://github.com/drud/ddev/releases/download/%s/%s",
-			ddevVersion,
-			compressedFilename))
+	c.RunCmdInDir(t, ddevDir, "curl", "-LO", fmt.Sprintf("https://github.com/drud/ddev/releases/download/%s/%s",
+		ddevVersion,
+		compressedFilename))
 
-	c.RunCmdInDir(t, dir, "tar", "-xzf", compressedFilename)
+	c.RunCmdInDir(t, ddevDir, "tar", "-xzf", compressedFilename)
 
 	// Create a simple index.php we can test against.
-	c.RunCmdInDir(t, dir, "sh", "-c", "echo '<?php\nprint \"ddev is working\";' >index.php")
+	c.RunCmdInDir(t, ddevDir, "sh", "-c", "echo '<?php\nprint \"ddev is working\";' >index.php")
 
-	c.RunCmdInDir(t, dir, "./ddev", "config", "--auto")
-	c.RunCmdInDir(t, dir, "./ddev", "config", "global", "--use-docker-compose-from-path")
-	vRes := c.RunCmdInDir(t, dir, "./ddev", "version")
+	c.RunCmdInDir(t, ddevDir, "./ddev", "config", "--auto")
+	c.RunCmdInDir(t, ddevDir, "./ddev", "config", "global", "--use-docker-compose-from-path")
+	vRes := c.RunCmdInDir(t, ddevDir, "./ddev", "version")
 	out := vRes.Stdout()
 	fmt.Printf("ddev version: %s\n", out)
 
-	c.RunCmdInDir(t, dir, "./ddev", "poweroff")
+	c.RunCmdInDir(t, ddevDir, "./ddev", "poweroff")
 
-	c.RunCmdInDir(t, dir, "./ddev", "start", "-y")
+	c.RunCmdInDir(t, ddevDir, "./ddev", "start", "-y")
 
-	curlRes := c.RunCmdInDir(t, dir, "curl", "-sSL", fmt.Sprintf("http://%s.ddev.site", siteName))
+	curlRes := c.RunCmdInDir(t, ddevDir, "curl", "-sSL", fmt.Sprintf("http://%s.ddev.site", siteName))
 	out = curlRes.Stdout()
 	fmt.Println(out)
 	assert.Assert(t, strings.Contains(out, "ddev is working"), "Could not start project")

pkg/e2e/framework.go

@@ -32,7 +32,6 @@ import (
 	"github.com/pkg/errors"
 	"github.com/stretchr/testify/require"
 	"gotest.tools/v3/assert"
-	is "gotest.tools/v3/assert/cmp"
 	"gotest.tools/v3/icmd"
 	"gotest.tools/v3/poll"
 
@@ -61,18 +60,50 @@ func init() {
 // CLI is used to wrap the CLI for end to end testing
 type CLI struct {
 	ConfigDir string
+
+	env []string
 }
 
-// NewParallelCLI returns a configured CLI with t.Parallel() set
-func NewParallelCLI(t *testing.T) *CLI {
+// CLIOption to customize behavior for all commands for a CLI instance.
+type CLIOption func(c *CLI)
+
+// NewParallelCLI marks the parent test as parallel and returns a CLI instance
+// suitable for usage across child tests.
+func NewParallelCLI(t *testing.T, opts ...CLIOption) *CLI {
+	t.Helper()
 	t.Parallel()
-	return NewCLI(t)
+	return NewCLI(t, opts...)
 }
 
-// NewCLI returns a CLI to use for E2E tests
-func NewCLI(t testing.TB) *CLI {
-	d, err := ioutil.TempDir("", "")
-	assert.Check(t, is.Nil(err))
+// NewCLI creates a CLI instance for running E2E tests.
+func NewCLI(t testing.TB, opts ...CLIOption) *CLI {
+	t.Helper()
+
+	configDir := t.TempDir()
+	initializePlugins(t, configDir)
+
+	c := &CLI{
+		ConfigDir: configDir,
+	}
+
+	for _, opt := range opts {
+		opt(c)
+	}
+
+	return c
+}
+
+// WithEnv sets environment variables that will be passed to commands.
+func WithEnv(env ...string) CLIOption {
+	return func(c *CLI) {
+		c.env = append(c.env, env...)
+	}
+}
+
+// initializePlugins copies the necessary plugin files to the temporary config
+// directory for the test.
+func initializePlugins(t testing.TB, d string) {
+	t.Helper()
 
 	t.Cleanup(func() {
 		if t.Failed() {
@@ -102,8 +133,6 @@ func NewCLI(t testing.TB) *CLI {
 			panic(err)
 		}
 	}
-
-	return &CLI{ConfigDir: d}
 }
 
 func dirContents(dir string) []string {
@@ -168,13 +197,15 @@ func (c *CLI) BaseEnvironment() []string {
 func (c *CLI) NewCmd(command string, args ...string) icmd.Cmd {
 	return icmd.Cmd{
 		Command: append([]string{command}, args...),
-		Env:     c.BaseEnvironment(),
+		Env:     append(c.BaseEnvironment(), c.env...),
 	}
 }
 
 // NewCmdWithEnv creates a cmd object configured with the test environment set with additional env vars
 func (c *CLI) NewCmdWithEnv(envvars []string, command string, args ...string) icmd.Cmd {
-	cmdEnv := append(c.BaseEnvironment(), envvars...)
+	// base env -> CLI overrides -> cmd overrides
+	cmdEnv := append(c.BaseEnvironment(), c.env...)
+	cmdEnv = append(cmdEnv, envvars...)
 	return icmd.Cmd{
 		Command: append([]string{command}, args...),
 		Env:     cmdEnv,