Skip to content

Commit 340e5f8

Browse files
agneumagneum
committed
Merge branch '383-logical-dump-container-startup' into 'master'
fix(engine): update logical dump flow to start existing dump container (#383) Closes #383 See merge request postgres-ai/database-lab!544
2 parents 5818bd8 + 2950844 commit 340e5f8

File tree

8 files changed

+195
-124
lines changed

8 files changed

+195
-124
lines changed

‎engine/internal/embeddedui/embedded_ui.go‎

Lines changed: 26 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,6 @@ import (
1313

1414
"github.com/docker/docker/api/types"
1515
"github.com/docker/docker/api/types/container"
16-
"github.com/docker/docker/api/types/network"
1716
"github.com/docker/docker/client"
1817
"github.com/docker/go-connections/nat"
1918

@@ -95,54 +94,37 @@ func (ui *UIManager) Run(ctx context.Context) error {
9594
return fmt.Errorf("failed to prepare Docker image: %w", err)
9695
}
9796

98-
var containerID = ""
99-
100-
// try to fetch an existing UI container
101-
containerData, err := ui.docker.ContainerInspect(ctx, getEmbeddedUIName(ui.engProps.InstanceID))
102-
103-
if err == nil {
104-
containerID = containerData.ID
105-
}
106-
107-
if containerID == "" {
108-
embeddedUI, err := ui.docker.ContainerCreate(ctx,
109-
&container.Config{
110-
Labels: map[string]string{
111-
cont.DBLabSatelliteLabel: cont.DBLabEmbeddedUILabel,
112-
cont.DBLabInstanceIDLabel: ui.engProps.InstanceID,
113-
cont.DBLabEngineNameLabel: ui.engProps.ContainerName,
114-
},
115-
Image: ui.cfg.DockerImage,
116-
Env: []string{
117-
EnvEngineName + "=" + ui.engProps.ContainerName,
118-
EnvEnginePort + "=" + strconv.FormatUint(uint64(ui.engProps.EnginePort), 10),
119-
},
120-
Healthcheck: &container.HealthConfig{
121-
Interval: healthCheckInterval,
122-
Timeout: healthCheckTimeout,
123-
Retries: healthCheckRetries,
124-
},
97+
containerID, err := tools.CreateContainerIfMissing(ctx, ui.docker, getEmbeddedUIName(ui.engProps.InstanceID),
98+
&container.Config{
99+
Labels: map[string]string{
100+
cont.DBLabSatelliteLabel: cont.DBLabEmbeddedUILabel,
101+
cont.DBLabInstanceIDLabel: ui.engProps.InstanceID,
102+
cont.DBLabEngineNameLabel: ui.engProps.ContainerName,
103+
},
104+
Image: ui.cfg.DockerImage,
105+
Env: []string{
106+
EnvEngineName + "=" + ui.engProps.ContainerName,
107+
EnvEnginePort + "=" + strconv.FormatUint(uint64(ui.engProps.EnginePort), 10),
125108
},
126-
&container.HostConfig{
127-
PortBindings: map[nat.Port][]nat.PortBinding{
128-
"80/tcp": {
129-
{
130-
HostIP: ui.cfg.Host,
131-
HostPort: strconv.Itoa(ui.cfg.Port),
132-
},
109+
Healthcheck: &container.HealthConfig{
110+
Interval: healthCheckInterval,
111+
Timeout: healthCheckTimeout,
112+
Retries: healthCheckRetries,
113+
},
114+
},
115+
&container.HostConfig{
116+
PortBindings: map[nat.Port][]nat.PortBinding{
117+
"80/tcp": {
118+
{
119+
HostIP: ui.cfg.Host,
120+
HostPort: strconv.Itoa(ui.cfg.Port),
133121
},
134122
},
135123
},
136-
&network.NetworkingConfig{},
137-
nil,
138-
getEmbeddedUIName(ui.engProps.InstanceID),
139-
)
140-
141-
if err != nil {
142-
return fmt.Errorf("failed to prepare Docker image for embedded UI: %w", err)
143-
}
124+
})
144125

145-
containerID = embeddedUI.ID
126+
if err != nil {
127+
return fmt.Errorf("failed to prepare Docker image for embedded UI: %w", err)
146128
}
147129

148130
if err := networks.Connect(ctx, ui.docker, ui.engProps.InstanceID, containerID); err != nil {

‎engine/internal/retrieval/engine/postgres/logical/dump.go‎

Lines changed: 14 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@ import (
1515

1616
"github.com/docker/docker/api/types"
1717
"github.com/docker/docker/api/types/container"
18-
"github.com/docker/docker/api/types/network"
1918
"github.com/docker/docker/client"
2019
"github.com/jackc/pgx/v4"
2120
"github.com/pkg/errors"
@@ -30,7 +29,6 @@ import (
3029
"gitlab.com/postgres-ai/database-lab/v3/internal/retrieval/engine/postgres/tools/defaults"
3130
"gitlab.com/postgres-ai/database-lab/v3/internal/retrieval/engine/postgres/tools/health"
3231
"gitlab.com/postgres-ai/database-lab/v3/internal/retrieval/options"
33-
3432
"gitlab.com/postgres-ai/database-lab/v3/pkg/config/global"
3533
"gitlab.com/postgres-ai/database-lab/v3/pkg/log"
3634
)
@@ -270,26 +268,23 @@ func (d *DumpJob) Run(ctx context.Context) (err error) {
270268
return errors.Wrap(err, "failed to generate PostgreSQL password")
271269
}
272270

273-
dumpCont, err := d.dockerClient.ContainerCreate(ctx, d.buildContainerConfig(pwd), hostConfig, &network.NetworkingConfig{},
274-
nil, d.dumpContainerName(),
275-
)
276-
if err != nil {
277-
log.Err(err)
271+
containerID, err := tools.CreateContainerIfMissing(ctx, d.dockerClient, d.dumpContainerName(), d.buildContainerConfig(pwd), hostConfig)
278272

279-
return errors.Wrapf(err, "failed to create container %q", d.dumpContainerName())
273+
if err != nil {
274+
return fmt.Errorf("failed to create container %q %w", d.dumpContainerName(), err)
280275
}
281276

282-
defer tools.RemoveContainer(ctx, d.dockerClient, dumpCont.ID, cont.StopTimeout)
277+
defer tools.RemoveContainer(ctx, d.dockerClient, containerID, cont.StopTimeout)
283278

284279
defer func() {
285280
if err != nil {
286281
tools.PrintContainerLogs(ctx, d.dockerClient, d.dumpContainerName())
287282
}
288283
}()
289284

290-
log.Msg(fmt.Sprintf("Running container: %s. ID: %v", d.dumpContainerName(), dumpCont.ID))
285+
log.Msg(fmt.Sprintf("Running container: %s. ID: %v", d.dumpContainerName(), containerID))
291286

292-
if err := d.dockerClient.ContainerStart(ctx, dumpCont.ID, types.ContainerStartOptions{}); err != nil {
287+
if err := d.dockerClient.ContainerStart(ctx, containerID, types.ContainerStartOptions{}); err != nil {
293288
return errors.Wrapf(err, "failed to start container %q", d.dumpContainerName())
294289
}
295290

@@ -299,13 +294,13 @@ func (d *DumpJob) Run(ctx context.Context) (err error) {
299294

300295
log.Msg("Waiting for container readiness")
301296

302-
if err := tools.MakeDir(ctx, d.dockerClient, dumpCont.ID, tmpDBLabPGDataDir); err != nil {
297+
if err := tools.MakeDir(ctx, d.dockerClient, containerID, tmpDBLabPGDataDir); err != nil {
303298
return err
304299
}
305300

306301
dataDir := d.fsPool.DataDir()
307302

308-
if err := tools.CheckContainerReadiness(ctx, d.dockerClient, dumpCont.ID); err != nil {
303+
if err := tools.CheckContainerReadiness(ctx, d.dockerClient, containerID); err != nil {
309304
var errHealthCheck *tools.ErrHealthCheck
310305
if !errors.As(err, &errHealthCheck) {
311306
return errors.Wrap(err, "failed to readiness check")
@@ -316,13 +311,13 @@ func (d *DumpJob) Run(ctx context.Context) (err error) {
316311
pgDataDir = dataDir
317312
}
318313

319-
if err := setupPGData(ctx, d.dockerClient, pgDataDir, dumpCont.ID); err != nil {
314+
if err := setupPGData(ctx, d.dockerClient, pgDataDir, containerID); err != nil {
320315
return errors.Wrap(err, "failed to set up Postgres data")
321316
}
322317
}
323318

324319
if d.DumpOptions.Restore.Enabled && len(d.DumpOptions.Restore.Configs) > 0 {
325-
if err := updateConfigs(ctx, d.dockerClient, dataDir, dumpCont.ID, d.DumpOptions.Restore.Configs); err != nil {
320+
if err := updateConfigs(ctx, d.dockerClient, dataDir, containerID, d.DumpOptions.Restore.Configs); err != nil {
326321
return errors.Wrap(err, "failed to update configs")
327322
}
328323
}
@@ -336,12 +331,12 @@ func (d *DumpJob) Run(ctx context.Context) (err error) {
336331
}
337332
}
338333

339-
if err := d.cleanupDumpLocation(ctx, dumpCont.ID, dbList); err != nil {
334+
if err := d.cleanupDumpLocation(ctx, containerID, dbList); err != nil {
340335
return err
341336
}
342337

343338
for dbName, dbDetails := range dbList {
344-
if err := d.dumpDatabase(ctx, dumpCont.ID, dbName, dbDetails); err != nil {
339+
if err := d.dumpDatabase(ctx, containerID, dbName, dbDetails); err != nil {
345340
return errors.Wrapf(err, "failed to dump the database %s", dbName)
346341
}
347342
}
@@ -358,11 +353,11 @@ func (d *DumpJob) Run(ctx context.Context) (err error) {
358353

359354
log.Msg("Running analyze command: ", analyzeCmd)
360355

361-
if err := tools.ExecCommand(ctx, d.dockerClient, dumpCont.ID, types.ExecConfig{Cmd: analyzeCmd}); err != nil {
356+
if err := tools.ExecCommand(ctx, d.dockerClient, containerID, types.ExecConfig{Cmd: analyzeCmd}); err != nil {
362357
return errors.Wrap(err, "failed to recalculate statistics after restore")
363358
}
364359

365-
if err := tools.StopPostgres(ctx, d.dockerClient, dumpCont.ID, dataDir, tools.DefaultStopTimeout); err != nil {
360+
if err := tools.StopPostgres(ctx, d.dockerClient, containerID, dataDir, tools.DefaultStopTimeout); err != nil {
366361
return errors.Wrap(err, "failed to stop Postgres instance")
367362
}
368363
}

‎engine/internal/retrieval/engine/postgres/logical/dump_integration_test.go‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
//go:build integration
2+
// +build integration
3+
4+
/*
5+
2021 © Postgres.ai
6+
*/
7+
8+
package logical
9+
10+
import (
11+
"context"
12+
"fmt"
13+
"math/rand"
14+
"testing"
15+
"time"
16+
17+
"github.com/docker/docker/api/types"
18+
"github.com/docker/docker/api/types/filters"
19+
"github.com/docker/docker/api/types/network"
20+
"github.com/docker/docker/client"
21+
"github.com/stretchr/testify/assert"
22+
"github.com/stretchr/testify/require"
23+
24+
dockerutils "gitlab.com/postgres-ai/database-lab/v3/internal/provision/docker"
25+
"gitlab.com/postgres-ai/database-lab/v3/internal/provision/resources"
26+
"gitlab.com/postgres-ai/database-lab/v3/internal/retrieval/config"
27+
"gitlab.com/postgres-ai/database-lab/v3/internal/retrieval/engine/postgres/tools"
28+
"gitlab.com/postgres-ai/database-lab/v3/pkg/config/global"
29+
)
30+
31+
func TestStartExisingDumpContainer(t *testing.T) {
32+
t.Parallel()
33+
ctx := context.Background()
34+
35+
docker, err := client.NewClientWithOpts(client.FromEnv)
36+
require.NoError(t, err)
37+
38+
// create dump job
39+
40+
source := rand.NewSource(time.Now().UnixNano())
41+
random := rand.New(source)
42+
43+
engProps := global.EngineProps{
44+
InstanceID: fmt.Sprintf("dumpjob-%d", random.Intn(10000)),
45+
}
46+
47+
job, err := NewDumpJob(
48+
config.JobConfig{
49+
Spec: config.JobSpec{Name: "test"},
50+
FSPool: &resources.Pool{
51+
DataSubDir: t.TempDir(),
52+
},
53+
Docker: docker,
54+
},
55+
&global.Config{},
56+
engProps,
57+
)
58+
assert.NoError(t, err)
59+
job.DockerImage = "postgresai/extended-postgres:14"
60+
job.DumpOptions.DumpLocation = t.TempDir()
61+
62+
err = dockerutils.PrepareImage(ctx, docker, job.DockerImage)
63+
assert.NoError(t, err)
64+
65+
// create dump container and stop it
66+
container, err := docker.ContainerCreate(ctx, job.buildContainerConfig(""), nil, &network.NetworkingConfig{},
67+
nil, job.dumpContainerName(),
68+
)
69+
assert.NoError(t, err)
70+
71+
// clean container in case of any error
72+
defer tools.RemoveContainer(ctx, docker, container.ID, 10*time.Second)
73+
74+
job.Run(ctx)
75+
76+
// list containers and check that container job container got processed
77+
filterArgs := filters.NewArgs()
78+
filterArgs.Add("name", job.dumpContainerName())
79+
80+
list, err := docker.ContainerList(
81+
ctx,
82+
types.ContainerListOptions{
83+
All: false,
84+
Filters: filterArgs,
85+
},
86+
)
87+
88+
require.NoError(t, err)
89+
assert.Empty(t, list)
90+
91+
}

‎engine/internal/retrieval/engine/postgres/logical/restore.go‎

Lines changed: 13 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@ import (
2020

2121
"github.com/docker/docker/api/types"
2222
"github.com/docker/docker/api/types/container"
23-
"github.com/docker/docker/api/types/network"
2423
"github.com/docker/docker/client"
2524
"github.com/docker/docker/pkg/archive"
2625
"github.com/pkg/errors"
@@ -195,61 +194,56 @@ func (r *RestoreJob) Run(ctx context.Context) (err error) {
195194
return errors.Wrap(err, "failed to generate PostgreSQL password")
196195
}
197196

198-
restoreCont, err := r.dockerClient.ContainerCreate(ctx,
199-
r.buildContainerConfig(pwd),
200-
hostConfig,
201-
&network.NetworkingConfig{},
202-
nil,
203-
r.restoreContainerName(),
204-
)
197+
containerID, err := tools.CreateContainerIfMissing(ctx, r.dockerClient, r.restoreContainerName(), r.buildContainerConfig(pwd), hostConfig)
198+
205199
if err != nil {
206-
return errors.Wrapf(err, "failed to create container %q", r.restoreContainerName())
200+
return fmt.Errorf("failed to create container %q %w", r.restoreContainerName(), err)
207201
}
208202

209-
defer tools.RemoveContainer(ctx, r.dockerClient, restoreCont.ID, cont.StopTimeout)
203+
defer tools.RemoveContainer(ctx, r.dockerClient, containerID, cont.StopTimeout)
210204

211205
defer func() {
212206
if err != nil {
213207
tools.PrintContainerLogs(ctx, r.dockerClient, r.restoreContainerName())
214208
}
215209
}()
216210

217-
log.Msg(fmt.Sprintf("Running container: %s. ID: %v", r.restoreContainerName(), restoreCont.ID))
211+
log.Msg(fmt.Sprintf("Running container: %s. ID: %v", r.restoreContainerName(), containerID))
218212

219-
if err := r.dockerClient.ContainerStart(ctx, restoreCont.ID, types.ContainerStartOptions{}); err != nil {
213+
if err := r.dockerClient.ContainerStart(ctx, containerID, types.ContainerStartOptions{}); err != nil {
220214
return errors.Wrapf(err, "failed to start container %q", r.restoreContainerName())
221215
}
222216

223217
dataDir := r.fsPool.DataDir()
224218

225219
log.Msg("Waiting for container readiness")
226220

227-
if err := tools.CheckContainerReadiness(ctx, r.dockerClient, restoreCont.ID); err != nil {
221+
if err := tools.CheckContainerReadiness(ctx, r.dockerClient, containerID); err != nil {
228222
var errHealthCheck *tools.ErrHealthCheck
229223
if !errors.As(err, &errHealthCheck) {
230224
return errors.Wrap(err, "failed to readiness check")
231225
}
232226

233-
if err := setupPGData(ctx, r.dockerClient, dataDir, restoreCont.ID); err != nil {
227+
if err := setupPGData(ctx, r.dockerClient, dataDir, containerID); err != nil {
234228
return errors.Wrap(err, "failed to set up Postgres data")
235229
}
236230
}
237231

238232
if len(r.RestoreOptions.Configs) > 0 {
239-
if err := updateConfigs(ctx, r.dockerClient, dataDir, restoreCont.ID, r.RestoreOptions.Configs); err != nil {
233+
if err := updateConfigs(ctx, r.dockerClient, dataDir, containerID, r.RestoreOptions.Configs); err != nil {
240234
return errors.Wrap(err, "failed to update configs")
241235
}
242236
}
243237

244-
dbList, err := r.getDBList(ctx, restoreCont.ID)
238+
dbList, err := r.getDBList(ctx, containerID)
245239
if err != nil {
246240
return err
247241
}
248242

249243
log.Dbg("Database List to restore: ", dbList)
250244

251245
for dbName, dbDefinition := range dbList {
252-
if err := r.restoreDB(ctx, restoreCont.ID, dbName, dbDefinition); err != nil {
246+
if err := r.restoreDB(ctx, containerID, dbName, dbDefinition); err != nil {
253247
return errors.Wrap(err, "failed to restore a database")
254248
}
255249
}
@@ -261,11 +255,11 @@ func (r *RestoreJob) Run(ctx context.Context) (err error) {
261255

262256
log.Msg("Running analyze command: ", analyzeCmd)
263257

264-
if err := tools.ExecCommand(ctx, r.dockerClient, restoreCont.ID, types.ExecConfig{Cmd: analyzeCmd}); err != nil {
258+
if err := tools.ExecCommand(ctx, r.dockerClient, containerID, types.ExecConfig{Cmd: analyzeCmd}); err != nil {
265259
return errors.Wrap(err, "failed to recalculate statistics after restore")
266260
}
267261

268-
if err := tools.StopPostgres(ctx, r.dockerClient, restoreCont.ID, dataDir, tools.DefaultStopTimeout); err != nil {
262+
if err := tools.StopPostgres(ctx, r.dockerClient, containerID, dataDir, tools.DefaultStopTimeout); err != nil {
269263
return errors.Wrap(err, "failed to stop Postgres instance")
270264
}
271265

0 commit comments

Comments
 (0)