From 45c3b0301745fd7aec0d6da54fb6d9ebd081a71d Mon Sep 17 00:00:00 2001 From: Phil Date: Fri, 23 Oct 2015 14:23:14 +0200 Subject: [PATCH 1/2] Change killContainer to stopContainer and wait for graceful shutdown --- core/docker.go | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/core/docker.go b/core/docker.go index b0c8380..4ed6380 100644 --- a/core/docker.go +++ b/core/docker.go @@ -15,6 +15,7 @@ import ( ) const LatestTag = "latest" +const GracefulShutdownTime = 20 type Docker struct { endPoint string @@ -85,11 +86,13 @@ func (d *Docker) cleanContainers(p *Project) error { for _, c := range l { if c.IsRunning() { Debug("Stoping container and image", "project", p, "container", c.GetShortID(), "end-point", d.endPoint) - if err := d.killContainer(c); err != nil { + if err := d.stopContainer(c); err != nil { return err } } + d.waitForGracefulShutdown(c) + Debug("Removing container", "project", p, "container", c.GetShortID(), "end-point", d.endPoint) if err := d.removeContainer(c); err != nil { return err @@ -99,9 +102,8 @@ func (d *Docker) cleanContainers(p *Project) error { return nil } -func (d *Docker) killContainer(c *Container) error { - kopts := docker.KillContainerOptions{ID: c.ID} - if err := d.client.KillContainer(kopts); err != nil { +func (d *Docker) stopContainer(c *Container) error { + if err := d.client.StopContainer(c.ID, GracefulShutdownTime); err != nil { return err } @@ -455,15 +457,25 @@ func (d *Docker) restartLinkedContainers(p *Project) error { return nil } +func (d *Docker) waitForGracefulShutdown(c * Container) { + timedOut := 0 + for c.IsRunning() && timedOut >= GracefulShutdownTime { + <-time.After(time.Second) + timedOut++ + } +} + func (d *Docker) restartContainer(p *Project, c *Container) error { if !c.IsRunning() { return nil } - if err := d.killContainer(c); err != nil { + if err := d.stopContainer(c); err != nil { return err } + d.waitForGracefulShutdown(c) + if err := d.startContainer(p, c); err != nil { return err } From d0902a6af2925d9039f1049032dc9a4715be78c1 Mon Sep 17 00:00:00 2001 From: Phil Date: Fri, 23 Oct 2015 15:51:15 +0200 Subject: [PATCH 2/2] Add errors on graceful shutdown failure in given time --- core/docker.go | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/core/docker.go b/core/docker.go index 4ed6380..7d374ca 100644 --- a/core/docker.go +++ b/core/docker.go @@ -91,7 +91,10 @@ func (d *Docker) cleanContainers(p *Project) error { } } - d.waitForGracefulShutdown(c) + err := d.waitForGracefulShutdown(c) + if err != nil { + Error(err.Error(), "project", p, "container", c.GetShortID()) + } Debug("Removing container", "project", p, "container", c.GetShortID(), "end-point", d.endPoint) if err := d.removeContainer(c); err != nil { @@ -457,12 +460,17 @@ func (d *Docker) restartLinkedContainers(p *Project) error { return nil } -func (d *Docker) waitForGracefulShutdown(c * Container) { +func (d *Docker) waitForGracefulShutdown(c * Container) error { timedOut := 0 - for c.IsRunning() && timedOut >= GracefulShutdownTime { + isStillRunning := false + for isStillRunning = c.IsRunning(); isStillRunning && timedOut >= GracefulShutdownTime; { <-time.After(time.Second) timedOut++ } + if isStillRunning { + return errors.New("Graceful shutdown timed out on container") + } + return nil } func (d *Docker) restartContainer(p *Project, c *Container) error { @@ -474,7 +482,10 @@ func (d *Docker) restartContainer(p *Project, c *Container) error { return err } - d.waitForGracefulShutdown(c) + err := d.waitForGracefulShutdown(c) + if err != nil { + Error(err.Error(), "project", p, "container", c.GetShortID()) + } if err := d.startContainer(p, c); err != nil { return err