Skip to content

Commit 49ecb74

Browse files
Mark Wilsonmihaibuzgau
Mark Wilson
authored and
mihaibuzgau
committed
(CLOUD-1850)-Restart docker container on unhealthy status
# Conflicts: # manifests/run.pp
1 parent 71c40bc commit 49ecb74

File tree

5 files changed

+206
-57
lines changed

5 files changed

+206
-57
lines changed

‎README.md‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -370,11 +370,21 @@ If using Hiera, you can configure the `docker::run_instance` class:
370370
371371
To remove a running container, add the following code to the manifest file. This will also remove the systemd service file associated with the container.
372372
373-
'''puppet
373+
```puppet
374374
docker::run { 'helloworld':
375375
ensure => absent,
376376
}
377-
'''
377+
```
378+
379+
To enable the restart of an unhealthy container add the following code to the manifest file.
380+
381+
```puppet
382+
docker::run { 'helloworld':
383+
image => 'base',
384+
command => 'command',
385+
health_check_command => '<command_to_execute_to_check_your_containers_health>',
386+
restart_on_unhealthy => true,
387+
```
378388

379389
### Networks
380390

‎lib/puppet/parser/functions/docker_run_flags.rb‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,10 @@ module Puppet::Parser::Functions
4646
flags << '--detach=true'
4747
end
4848

49+
if opts['health_check_cmd']
50+
flags << "--health-cmd '#{opts['health_check_command']} || exit 1'"
51+
end
52+
4953
if opts['tty']
5054
flags << '-t'
5155
end

‎manifests/run.pp‎

Lines changed: 161 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,15 @@
4848
# anything else -> Service[docker_service]
4949
# Default: false
5050
#
51+
# [*health_check_cmd*]
52+
# (optional) Specifies the command to execute to check that the container is healthy using the docker health check functionality.
53+
# Default: undef
54+
#
55+
# [*restart_on_unhealthy*]
56+
# (optional) Checks the health status of Docker container and if it is unhealthy the service will be restarted.
57+
# The health_check_cmd parameter must be set to true to use this functionality.
58+
# Default: undef
59+
#
5160
# [*extra_parameters*]
5261
# An array of additional command line arguments to pass to the `docker run`
5362
# command. Useful for adding additional new or experimental options that the
@@ -108,18 +117,20 @@
108117
Optional[Boolean] $remove_volume_on_start = false,
109118
Optional[Boolean] $remove_volume_on_stop = false,
110119
Optional[Integer] $stop_wait_time = 0,
111-
Optional[String] $syslog_identifier = undef,
120+
Optional[String] $syslog_identifier = undef,
112121
Optional[Boolean] $read_only = false,
122+
Optional[String] $health_check_cmd = undef,
123+
Optional[Boolean] $restart_on_unhealthy = false,
113124
) {
114-
include docker
125+
include docker::params
115126
if ($socket_connect != []) {
116127
$sockopts = join(any2array($socket_connect), ',')
117-
$docker_command = "${docker::docker_command} -H ${sockopts}"
128+
$docker_command = "${docker::params::docker_command} -H ${sockopts}"
118129
}else {
119-
$docker_command = $docker::docker_command
130+
$docker_command = $docker::params::docker_command
120131
}
121-
$service_name = $docker::service_name
122-
$docker_group = $docker::docker_group
132+
$service_name = $docker::params::service_name
133+
$docker_group = $docker::params::docker_group
123134

124135
if $restart {
125136
assert_type(Pattern[/^(no|always|unless-stopped|on-failure)|^on-failure:[\d]+$/], $restart)
@@ -156,30 +167,32 @@
156167
$depend_services_array = any2array($depend_services)
157168

158169
$docker_run_flags = docker_run_flags({
159-
cpuset => any2array($cpuset),
160-
detach => $valid_detach,
161-
disable_network => $disable_network,
162-
dns => any2array($dns),
163-
dns_search => any2array($dns_search),
164-
env => any2array($env),
165-
env_file => any2array($env_file),
166-
expose => any2array($expose),
167-
extra_params => any2array($extra_parameters),
168-
hostentries => any2array($hostentries),
169-
hostname => $hostname,
170-
links => any2array($links),
171-
lxc_conf => any2array($lxc_conf),
172-
memory_limit => $memory_limit,
173-
net => $net,
174-
ports => any2array($ports),
175-
labels => any2array($labels),
176-
privileged => $privileged,
177-
socket_connect => any2array($socket_connect),
178-
tty => $tty,
179-
username => $username,
180-
volumes => any2array($volumes),
181-
volumes_from => any2array($volumes_from),
182-
read_only => $read_only,
170+
cpuset => any2array($cpuset),
171+
detach => $valid_detach,
172+
disable_network => $disable_network,
173+
dns => any2array($dns),
174+
dns_search => any2array($dns_search),
175+
env => any2array($env),
176+
env_file => any2array($env_file),
177+
expose => any2array($expose),
178+
extra_params => any2array($extra_parameters),
179+
hostentries => any2array($hostentries),
180+
hostname => $hostname,
181+
links => any2array($links),
182+
lxc_conf => any2array($lxc_conf),
183+
memory_limit => $memory_limit,
184+
net => $net,
185+
ports => any2array($ports),
186+
labels => any2array($labels),
187+
privileged => $privileged,
188+
socket_connect => any2array($socket_connect),
189+
tty => $tty,
190+
username => $username,
191+
volumes => any2array($volumes),
192+
volumes_from => any2array($volumes_from),
193+
read_only => $read_only,
194+
health_check_cmd => $health_check_cmd,
195+
restart_on_unhealthy => $restart_on_unhealthy,
183196
})
184197

185198
$sanitised_title = regsubst($title, '[^0-9A-Za-z.\-_]', '-', 'G')
@@ -197,21 +210,90 @@
197210
$sanitised_after_array = regsubst($after_array, '[^0-9A-Za-z.\-_]', '-', 'G')
198211
}
199212

213+
if $::osfamily == 'windows' {
214+
$exec_environment = 'PATH=C:/Program Files/Docker/'
215+
$exec_timeout = 3000
216+
$exec_path = ['c:/Windows/Temp/', 'C:/Program Files/Docker/']
217+
$exec_provider = 'powershell'
218+
$cidfile = "c:/Windows/Temp/${service_prefix}${sanitised_title}.cid"
219+
} else {
220+
$exec_environment = 'HOME=/root'
221+
$exec_path = ['/bin', '/usr/bin']
222+
$exec_timeout = 0
223+
$exec_provider = undef
224+
$cidfile = "/var/run/${service_prefix}${sanitised_title}.cid"
225+
$restart_check = "${docker_command} inspect ${sanitised_title} -f '{{ if eq \"unhealthy\" .State.Health.Status }} {{ .Name }}{{ end }}' | grep ${sanitised_title}"
226+
}
227+
228+
if $restart_on_unhealthy {
229+
exec { "Restart unhealthy container ${title} with docker":
230+
command => "${docker_command} restart ${sanitised_title}",
231+
onlyif => $restart_check,
232+
environment => $exec_environment,
233+
path => $exec_path,
234+
provider => $exec_provider,
235+
timeout => $exec_timeout
236+
}
237+
}
238+
200239
if $restart {
240+
if $ensure == 'absent' {
241+
exec { "stop ${title} with docker":
242+
command => "${docker_command} stop --time=${stop_wait_time} ${sanitised_title}",
243+
onlyif => "${docker_command} inspect ${sanitised_title}",
244+
environment => $exec_environment,
245+
path => $exec_path,
246+
provider => $exec_provider,
247+
timeout => $exec_timeout
248+
}
201249

202-
$cidfile = "/var/run/${service_prefix}${sanitised_title}.cid"
250+
exec { "remove ${title} with docker":
251+
command => "${docker_command} rm -v ${sanitised_title}",
252+
onlyif => "${docker_command} inspect ${sanitised_title}",
253+
environment => $exec_environment,
254+
path => $exec_path,
255+
provider => $exec_provider,
256+
timeout => $exec_timeout
257+
}
258+
259+
file { $cidfile:
260+
ensure => absent,
261+
}
262+
}
263+
else {
264+
$run_with_docker_command = [
265+
"${docker_command} run -d ${docker_run_flags}",
266+
"--name ${sanitised_title} --cidfile=${cidfile}",
267+
"--restart=\"${restart}\" ${image} ${command}",
268+
]
269+
exec { "run ${title} with docker":
270+
command => join($run_with_docker_command, ' '),
271+
unless => "${docker_command} inspect ${sanitised_title}",
272+
environment => $exec_environment,
273+
path => $exec_path,
274+
provider => $exec_provider,
275+
timeout => $exec_timeout
276+
}
203277

204-
$run_with_docker_command = [
205-
"${docker_command} run -d ${docker_run_flags}",
206-
"--name ${sanitised_title} --cidfile=${cidfile}",
207-
"--restart=\"${restart}\" ${image} ${command}",
208-
]
209-
exec { "run ${title} with docker":
210-
command => join($run_with_docker_command, ' '),
211-
unless => "${docker_command} ps --no-trunc -a | grep `cat ${cidfile}`",
212-
environment => 'HOME=/root',
213-
path => ['/bin', '/usr/bin'],
214-
timeout => 0
278+
if $running == false {
279+
exec { "stop ${title} with docker":
280+
command => "${docker_command} stop --time=${stop_wait_time} ${sanitised_title}",
281+
unless => "${docker_command} inspect ${sanitised_title} -f \"{{ if (.State.Running) }} {{ nil }}{{ end }}\"",
282+
environment => $exec_environment,
283+
path => $exec_path,
284+
provider => $exec_provider,
285+
timeout => $exec_timeout
286+
}
287+
} else {
288+
exec { "start ${title} with docker":
289+
command => "${docker_command} start ${sanitised_title}",
290+
onlyif => "${docker_command} inspect ${sanitised_title} -f \"{{ if (.State.Running) }} {{ nil }}{{ end }}\"",
291+
environment => $exec_environment,
292+
path => $exec_path,
293+
provider => $exec_provider,
294+
timeout => $exec_timeout
295+
}
296+
}
215297
}
216298
} else {
217299

@@ -227,7 +309,12 @@
227309
$mode = '0750'
228310
}
229311
default: {
230-
fail translate(('Docker needs a Debian or RedHat based system.'))
312+
if $::osfamily != 'windows' {
313+
fail translate(('Docker needs a Debian or RedHat based system.'))
314+
}
315+
elsif $ensure == 'present' {
316+
fail translate(('Restart parameter is required for Windows'))
317+
}
231318
}
232319
}
233320

@@ -237,28 +324,47 @@
237324
$_syslog_identifier = "${service_prefix}${sanitised_title}"
238325
}
239326

240-
241327
if $ensure == 'absent' {
328+
if $::osfamily == 'windows'{
329+
exec {
330+
"stop container ${service_prefix}${sanitised_title}":
331+
command => "${docker_command} stop --time=${stop_wait_time} ${sanitised_title}",
332+
onlyif => "${docker_command} inspect ${sanitised_title}",
333+
environment => $exec_environment,
334+
path => $exec_path,
335+
provider => $exec_provider,
336+
timeout => $exec_timeout,
337+
notify => Exec["remove container ${service_prefix}${sanitised_title}"]
338+
}
339+
}
340+
else {
242341
service { "${service_prefix}${sanitised_title}":
243342
ensure => false,
244343
enable => false,
245344
hasstatus => $docker::params::service_hasstatus,
246345
}
247-
248-
exec {
249-
"remove container ${service_prefix}${sanitised_title}":
250-
command => "${docker_command} rm -v ${sanitised_title}",
251-
refreshonly => true,
252-
onlyif => "${docker_command} ps --no-trunc -a --format='table {{.Names}}' | grep '^${sanitised_title}$'",
253-
path => ['/bin', '/usr/bin'],
254-
environment => 'HOME=/root',
255-
timeout => 0
256-
}
257-
346+
}
347+
exec {
348+
"remove container ${service_prefix}${sanitised_title}":
349+
command => "${docker_command} rm -v ${sanitised_title}",
350+
onlyif => "${docker_command} inspect ${sanitised_title}",
351+
environment => $exec_environment,
352+
path => $exec_path,
353+
refreshonly => true,
354+
provider => $exec_provider,
355+
timeout => $exec_timeout
356+
}
357+
if $::osfamily != 'windows' {
258358
file { "/etc/systemd/system/${service_prefix}${sanitised_title}.service":
259359
ensure => absent,
260360
path => "/etc/systemd/system/${service_prefix}${sanitised_title}.service",
261361
}
362+
}
363+
else {
364+
file { $cidfile:
365+
ensure => absent,
366+
}
367+
}
262368
}
263369
else {
264370
file { $initscript:

‎spec/acceptance/docker_full_spec.rb‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -643,6 +643,21 @@ class { 'docker':}
643643
apply_manifest(pp, :catch_changes => true) unless fact('selinux') == 'true'
644644

645645
end
646+
647+
it 'should restart a unhealthy container' do
648+
pp5=<<-EOS
649+
docker::run { 'container_3_7_3':
650+
image => 'base',
651+
command => '#{default_run_command}',
652+
health_check_cmd => 'pwd',
653+
restart_on_unhealthy => true,
654+
}
655+
EOS
656+
657+
apply_manifest(pp5, :catch_failures => true) do |r|
658+
expect(r.stdout).to match(/docker-container_3_7_3-systemd-reload/)
659+
end
660+
end
646661
end
647662

648663
describe "docker::exec" do

‎spec/defines/run_spec.rb‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -136,6 +136,20 @@
136136
end
137137
end
138138

139+
context 'When restarting an unhealthy container' do
140+
let(:params) {{
141+
'command' => 'command',
142+
'image' => 'base',
143+
'health_check_cmd' => 'pwd',
144+
'restart_on_unhealthy' => true
145+
}}
146+
if (systemd)
147+
it { should contain_file(initscript).with_content(/ExecStop=-\/usr\/bin\/docker stop --time=0 /) }
148+
it { should contain_file(initscript).with_content(/ExecStop=-\/usr\/bin\/docker rm/) }
149+
it { should contain_file(initscript).with_content(/--health-cmd/) }
150+
end
151+
end
152+
139153
context 'when not removing containers on container start and stop' do
140154
let(:params) {{
141155
'command' => 'command',

0 commit comments

Comments
 (0)