Skip to content

Commit d906505

Browse files
milasmilas
committed
e2e: fix subtests and block parallel unsafe tests
The big mechanical change here is to NOT store `t` as a field on the `CLI` object (which has been renamed as well to fix the odd capitalization). The way the tests are structured meant that the "subtests" were using the _parent_ `*testing.T` instance, which causes various oddities including confusing messages on failure about a child test causing a panic in the parent. Additionally, a few tests have been blocked from running in parallel because they are sharing `compose.yaml` fixtures and can fail as a result (e.g. due to a port conflict on the host). I'll fix these in follow-up PRs but want to focus on correctness for the tests before optimizing them. Signed-off-by: Milas Bowman <milas.bowman@docker.com>
1 parent 005fc25 commit d906505

22 files changed

+395
-358
lines changed

‎pkg/e2e/cancel_test.go‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -33,30 +33,33 @@ import (
3333
)
3434

3535
func TestComposeCancel(t *testing.T) {
36-
c := NewParallelE2eCLI(t, binDir)
36+
c := NewParallelCLI(t)
3737

3838
t.Run("metrics on cancel Compose build", func(t *testing.T) {
39-
c.RunDockerComposeCmd("ls")
39+
c.RunDockerComposeCmd(t, "ls")
4040
buildProjectPath := "fixtures/build-infinite/compose.yaml"
4141

4242
// require a separate groupID from the process running tests, in order to simulate ctrl+C from a terminal.
4343
// sending kill signal
4444
cmd, stdout, stderr, err := StartWithNewGroupID(c.NewDockerCmd("compose", "-f", buildProjectPath, "build", "--progress", "plain"))
4545
assert.NilError(t, err)
4646

47-
c.WaitForCondition(func() (bool, string) {
47+
c.WaitForCondition(t, func() (bool, string) {
4848
out := stdout.String()
4949
errors := stderr.String()
50-
return strings.Contains(out, "RUN sleep infinity"), fmt.Sprintf("'RUN sleep infinity' not found in : \n%s\nStderr: \n%s\n", out, errors)
50+
return strings.Contains(out,
51+
"RUN sleep infinity"), fmt.Sprintf("'RUN sleep infinity' not found in : \n%s\nStderr: \n%s\n", out,
52+
errors)
5153
}, 30*time.Second, 1*time.Second)
5254

5355
err = syscall.Kill(-cmd.Process.Pid, syscall.SIGINT) // simulate Ctrl-C : send signal to processGroup, children will have same groupId by default
5456

5557
assert.NilError(t, err)
56-
c.WaitForCondition(func() (bool, string) {
58+
c.WaitForCondition(t, func() (bool, string) {
5759
out := stdout.String()
5860
errors := stderr.String()
59-
return strings.Contains(out, "CANCELED"), fmt.Sprintf("'CANCELED' not found in : \n%s\nStderr: \n%s\n", out, errors)
61+
return strings.Contains(out, "CANCELED"), fmt.Sprintf("'CANCELED' not found in : \n%s\nStderr: \n%s\n", out,
62+
errors)
6063
}, 10*time.Second, 1*time.Second)
6164
})
6265
}

‎pkg/e2e/cascade_stop_test.go‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -23,28 +23,28 @@ import (
2323
)
2424

2525
func TestCascadeStop(t *testing.T) {
26-
c := NewParallelE2eCLI(t, binDir)
26+
c := NewParallelCLI(t)
2727

2828
const projectName = "e2e-cascade-stop"
2929

3030
t.Run("abort-on-container-exit", func(t *testing.T) {
31-
res := c.RunDockerOrExitError("compose", "-f", "./fixtures/cascade-stop-test/compose.yaml", "--project-name", projectName, "up", "--abort-on-container-exit")
31+
res := c.RunDockerOrExitError(t, "compose", "-f", "./fixtures/cascade-stop-test/compose.yaml", "--project-name", projectName, "up", "--abort-on-container-exit")
3232
res.Assert(t, icmd.Expected{ExitCode: 1, Out: `should_fail-1 exited with code 1`})
3333
res.Assert(t, icmd.Expected{ExitCode: 1, Out: `Aborting on container exit...`})
3434
})
3535

3636
t.Run("exit-code-from", func(t *testing.T) {
37-
res := c.RunDockerOrExitError("compose", "-f", "./fixtures/cascade-stop-test/compose.yaml", "--project-name", projectName, "up", "--exit-code-from=sleep")
37+
res := c.RunDockerOrExitError(t, "compose", "-f", "./fixtures/cascade-stop-test/compose.yaml", "--project-name", projectName, "up", "--exit-code-from=sleep")
3838
res.Assert(t, icmd.Expected{ExitCode: 137, Out: `should_fail-1 exited with code 1`})
3939
res.Assert(t, icmd.Expected{ExitCode: 137, Out: `Aborting on container exit...`})
4040
})
4141

4242
t.Run("exit-code-from unknown", func(t *testing.T) {
43-
res := c.RunDockerOrExitError("compose", "-f", "./fixtures/cascade-stop-test/compose.yaml", "--project-name", projectName, "up", "--exit-code-from=unknown")
43+
res := c.RunDockerOrExitError(t, "compose", "-f", "./fixtures/cascade-stop-test/compose.yaml", "--project-name", projectName, "up", "--exit-code-from=unknown")
4444
res.Assert(t, icmd.Expected{ExitCode: 1, Err: `no such service: unknown`})
4545
})
4646

4747
t.Run("down", func(t *testing.T) {
48-
_ = c.RunDockerComposeCmd("--project-name", projectName, "down")
48+
_ = c.RunDockerComposeCmd(t, "--project-name", projectName, "down")
4949
})
5050
}

‎pkg/e2e/compose_build_test.go‎

Lines changed: 46 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -28,55 +28,55 @@ import (
2828
)
2929

3030
func TestLocalComposeBuild(t *testing.T) {
31-
c := NewParallelE2eCLI(t, binDir)
31+
c := NewParallelCLI(t)
3232

3333
t.Run("build named and unnamed images", func(t *testing.T) {
3434
// ensure local test run does not reuse previously build image
35-
c.RunDockerOrExitError("rmi", "build-test_nginx")
36-
c.RunDockerOrExitError("rmi", "custom-nginx")
35+
c.RunDockerOrExitError(t, "rmi", "build-test_nginx")
36+
c.RunDockerOrExitError(t, "rmi", "custom-nginx")
3737

38-
res := c.RunDockerComposeCmd("--project-directory", "fixtures/build-test", "build")
38+
res := c.RunDockerComposeCmd(t, "--project-directory", "fixtures/build-test", "build")
3939

4040
res.Assert(t, icmd.Expected{Out: "COPY static /usr/share/nginx/html"})
41-
c.RunDockerCmd("image", "inspect", "build-test_nginx")
42-
c.RunDockerCmd("image", "inspect", "custom-nginx")
41+
c.RunDockerCmd(t, "image", "inspect", "build-test_nginx")
42+
c.RunDockerCmd(t, "image", "inspect", "custom-nginx")
4343
})
4444

4545
t.Run("build with build-arg", func(t *testing.T) {
4646
// ensure local test run does not reuse previously build image
47-
c.RunDockerOrExitError("rmi", "build-test_nginx")
48-
c.RunDockerOrExitError("rmi", "custom-nginx")
47+
c.RunDockerOrExitError(t, "rmi", "build-test_nginx")
48+
c.RunDockerOrExitError(t, "rmi", "custom-nginx")
4949

50-
c.RunDockerComposeCmd("--project-directory", "fixtures/build-test", "build", "--build-arg", "FOO=BAR")
50+
c.RunDockerComposeCmd(t, "--project-directory", "fixtures/build-test", "build", "--build-arg", "FOO=BAR")
5151

52-
res := c.RunDockerCmd("image", "inspect", "build-test_nginx")
52+
res := c.RunDockerCmd(t, "image", "inspect", "build-test_nginx")
5353
res.Assert(t, icmd.Expected{Out: `"FOO": "BAR"`})
5454
})
5555

5656
t.Run("build with build-arg set by env", func(t *testing.T) {
5757
// ensure local test run does not reuse previously build image
58-
c.RunDockerOrExitError("rmi", "build-test_nginx")
59-
c.RunDockerOrExitError("rmi", "custom-nginx")
58+
c.RunDockerOrExitError(t, "rmi", "build-test_nginx")
59+
c.RunDockerOrExitError(t, "rmi", "custom-nginx")
6060

6161
icmd.RunCmd(c.NewDockerCmd("compose", "--project-directory", "fixtures/build-test", "build", "--build-arg", "FOO"),
6262
func(cmd *icmd.Cmd) {
6363
cmd.Env = append(cmd.Env, "FOO=BAR")
6464
})
6565

66-
res := c.RunDockerCmd("image", "inspect", "build-test_nginx")
66+
res := c.RunDockerCmd(t, "image", "inspect", "build-test_nginx")
6767
res.Assert(t, icmd.Expected{Out: `"FOO": "BAR"`})
6868
})
6969

7070
t.Run("build with multiple build-args ", func(t *testing.T) {
7171
// ensure local test run does not reuse previously build image
72-
c.RunDockerOrExitError("rmi", "-f", "multi-args_multiargs")
72+
c.RunDockerOrExitError(t, "rmi", "-f", "multi-args_multiargs")
7373
cmd := c.NewDockerCmd("compose", "--project-directory", "fixtures/build-test/multi-args", "build")
7474

7575
icmd.RunCmd(cmd, func(cmd *icmd.Cmd) {
7676
cmd.Env = append(cmd.Env, "DOCKER_BUILDKIT=0")
7777
})
7878

79-
res := c.RunDockerCmd("image", "inspect", "multi-args_multiargs")
79+
res := c.RunDockerCmd(t, "image", "inspect", "multi-args_multiargs")
8080
res.Assert(t, icmd.Expected{Out: `"RESULT": "SUCCESS"`})
8181
})
8282

@@ -86,7 +86,7 @@ func TestLocalComposeBuild(t *testing.T) {
8686
os.Unsetenv("SSH_AUTH_SOCK") //nolint:errcheck
8787
defer os.Setenv("SSH_AUTH_SOCK", defaultSSHAUTHSOCK) //nolint:errcheck
8888

89-
res := c.RunDockerComposeCmdNoCheck("--project-directory", "fixtures/build-test", "build", "--ssh", "")
89+
res := c.RunDockerComposeCmdNoCheck(t, "--project-directory", "fixtures/build-test", "build", "--ssh", "")
9090
res.Assert(t, icmd.Expected{
9191
ExitCode: 1,
9292
Err: "invalid empty ssh agent socket: make sure SSH_AUTH_SOCK is set",
@@ -95,24 +95,24 @@ func TestLocalComposeBuild(t *testing.T) {
9595
})
9696

9797
t.Run("build succeed with ssh from Compose file", func(t *testing.T) {
98-
c.RunDockerOrExitError("rmi", "build-test-ssh")
98+
c.RunDockerOrExitError(t, "rmi", "build-test-ssh")
9999

100-
c.RunDockerComposeCmd("--project-directory", "fixtures/build-test/ssh", "build")
101-
c.RunDockerCmd("image", "inspect", "build-test-ssh")
100+
c.RunDockerComposeCmd(t, "--project-directory", "fixtures/build-test/ssh", "build")
101+
c.RunDockerCmd(t, "image", "inspect", "build-test-ssh")
102102
})
103103

104104
t.Run("build succeed with ssh from CLI", func(t *testing.T) {
105-
c.RunDockerOrExitError("rmi", "build-test-ssh")
105+
c.RunDockerOrExitError(t, "rmi", "build-test-ssh")
106106

107-
c.RunDockerComposeCmd("-f", "fixtures/build-test/ssh/compose-without-ssh.yaml", "--project-directory",
107+
c.RunDockerComposeCmd(t, "-f", "fixtures/build-test/ssh/compose-without-ssh.yaml", "--project-directory",
108108
"fixtures/build-test/ssh", "build", "--no-cache", "--ssh", "fake-ssh=./fixtures/build-test/ssh/fake_rsa")
109-
c.RunDockerCmd("image", "inspect", "build-test-ssh")
109+
c.RunDockerCmd(t, "image", "inspect", "build-test-ssh")
110110
})
111111

112112
t.Run("build failed with wrong ssh key id from CLI", func(t *testing.T) {
113-
c.RunDockerOrExitError("rmi", "build-test-ssh")
113+
c.RunDockerOrExitError(t, "rmi", "build-test-ssh")
114114

115-
res := c.RunDockerComposeCmdNoCheck("-f", "fixtures/build-test/ssh/compose-without-ssh.yaml",
115+
res := c.RunDockerComposeCmdNoCheck(t, "-f", "fixtures/build-test/ssh/compose-without-ssh.yaml",
116116
"--project-directory", "fixtures/build-test/ssh", "build", "--no-cache", "--ssh",
117117
"wrong-ssh=./fixtures/build-test/ssh/fake_rsa")
118118
res.Assert(t, icmd.Expected{
@@ -122,22 +122,22 @@ func TestLocalComposeBuild(t *testing.T) {
122122
})
123123

124124
t.Run("build succeed as part of up with ssh from Compose file", func(t *testing.T) {
125-
c.RunDockerOrExitError("rmi", "build-test-ssh")
125+
c.RunDockerOrExitError(t, "rmi", "build-test-ssh")
126126

127-
c.RunDockerComposeCmd("--project-directory", "fixtures/build-test/ssh", "up", "-d", "--build")
127+
c.RunDockerComposeCmd(t, "--project-directory", "fixtures/build-test/ssh", "up", "-d", "--build")
128128
t.Cleanup(func() {
129-
c.RunDockerComposeCmd("--project-directory", "fixtures/build-test/ssh", "down")
129+
c.RunDockerComposeCmd(t, "--project-directory", "fixtures/build-test/ssh", "down")
130130
})
131-
c.RunDockerCmd("image", "inspect", "build-test-ssh")
131+
c.RunDockerCmd(t, "image", "inspect", "build-test-ssh")
132132
})
133133

134134
t.Run("build as part of up", func(t *testing.T) {
135-
c.RunDockerOrExitError("rmi", "build-test_nginx")
136-
c.RunDockerOrExitError("rmi", "custom-nginx")
135+
c.RunDockerOrExitError(t, "rmi", "build-test_nginx")
136+
c.RunDockerOrExitError(t, "rmi", "custom-nginx")
137137

138-
res := c.RunDockerComposeCmd("--project-directory", "fixtures/build-test", "up", "-d")
138+
res := c.RunDockerComposeCmd(t, "--project-directory", "fixtures/build-test", "up", "-d")
139139
t.Cleanup(func() {
140-
c.RunDockerComposeCmd("--project-directory", "fixtures/build-test", "down")
140+
c.RunDockerComposeCmd(t, "--project-directory", "fixtures/build-test", "down")
141141
})
142142

143143
res.Assert(t, icmd.Expected{Out: "COPY static /usr/share/nginx/html"})
@@ -146,53 +146,53 @@ func TestLocalComposeBuild(t *testing.T) {
146146
output := HTTPGetWithRetry(t, "http://localhost:8070", http.StatusOK, 2*time.Second, 20*time.Second)
147147
assert.Assert(t, strings.Contains(output, "Hello from Nginx container"))
148148

149-
c.RunDockerCmd("image", "inspect", "build-test_nginx")
150-
c.RunDockerCmd("image", "inspect", "custom-nginx")
149+
c.RunDockerCmd(t, "image", "inspect", "build-test_nginx")
150+
c.RunDockerCmd(t, "image", "inspect", "custom-nginx")
151151
})
152152

153153
t.Run("no rebuild when up again", func(t *testing.T) {
154-
res := c.RunDockerComposeCmd("--project-directory", "fixtures/build-test", "up", "-d")
154+
res := c.RunDockerComposeCmd(t, "--project-directory", "fixtures/build-test", "up", "-d")
155155

156156
assert.Assert(t, !strings.Contains(res.Stdout(), "COPY static"), res.Stdout())
157157
})
158158

159159
t.Run("rebuild when up --build", func(t *testing.T) {
160-
res := c.RunDockerComposeCmd("--workdir", "fixtures/build-test", "up", "-d", "--build")
160+
res := c.RunDockerComposeCmd(t, "--workdir", "fixtures/build-test", "up", "-d", "--build")
161161

162162
res.Assert(t, icmd.Expected{Out: "COPY static /usr/share/nginx/html"})
163163
res.Assert(t, icmd.Expected{Out: "COPY static2 /usr/share/nginx/html"})
164164
})
165165

166166
t.Run("cleanup build project", func(t *testing.T) {
167-
c.RunDockerComposeCmd("--project-directory", "fixtures/build-test", "down")
168-
c.RunDockerCmd("rmi", "build-test_nginx")
169-
c.RunDockerCmd("rmi", "custom-nginx")
167+
c.RunDockerComposeCmd(t, "--project-directory", "fixtures/build-test", "down")
168+
c.RunDockerCmd(t, "rmi", "build-test_nginx")
169+
c.RunDockerCmd(t, "rmi", "custom-nginx")
170170
})
171171
}
172172

173173
func TestBuildSecrets(t *testing.T) {
174-
c := NewParallelE2eCLI(t, binDir)
174+
c := NewParallelCLI(t)
175175

176176
t.Run("build with secrets", func(t *testing.T) {
177177
// ensure local test run does not reuse previously build image
178-
c.RunDockerOrExitError("rmi", "build-test-secret")
178+
c.RunDockerOrExitError(t, "rmi", "build-test-secret")
179179

180-
res := c.RunDockerComposeCmd("--project-directory", "fixtures/build-test/secrets", "build")
180+
res := c.RunDockerComposeCmd(t, "--project-directory", "fixtures/build-test/secrets", "build")
181181
res.Assert(t, icmd.Success)
182182
})
183183
}
184184

185185
func TestBuildTags(t *testing.T) {
186-
c := NewParallelE2eCLI(t, binDir)
186+
c := NewParallelCLI(t)
187187

188188
t.Run("build with tags", func(t *testing.T) {
189189

190190
// ensure local test run does not reuse previously build image
191-
c.RunDockerOrExitError("rmi", "build-test-tags")
191+
c.RunDockerOrExitError(t, "rmi", "build-test-tags")
192192

193-
c.RunDockerComposeCmd("--project-directory", "./fixtures/build-test/tags", "build", "--no-cache")
193+
c.RunDockerComposeCmd(t, "--project-directory", "./fixtures/build-test/tags", "build", "--no-cache")
194194

195-
res := c.RunDockerCmd("image", "inspect", "build-test-tags")
195+
res := c.RunDockerCmd(t, "image", "inspect", "build-test-tags")
196196
expectedOutput := `"RepoTags": [
197197
"docker/build-test-tags:1.0.0",
198198
"build-test-tags:latest",

‎pkg/e2e/compose_down_test.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,12 +23,12 @@ import (
2323
)
2424

2525
func TestDown(t *testing.T) {
26-
c := NewParallelE2eCLI(t, binDir)
26+
c := NewParallelCLI(t)
2727

2828
const projectName = "e2e-down"
2929

3030
t.Run("no resource to remove", func(t *testing.T) {
31-
res := c.RunDockerOrExitError("compose", "--project-name", projectName, "down")
31+
res := c.RunDockerOrExitError(t, "compose", "--project-name", projectName, "down")
3232
res.Assert(t, icmd.Expected{ExitCode: 0, Err: `No resource found to remove for project "e2e-down"`})
3333
})
3434
}

0 commit comments

Comments
 (0)