Merge pull request #170 from RIAG-Digital/feature/rebootWindow

Feature/reboot window

Author: sdemos

cmd/update-operator/main.go

@@ -19,6 +19,8 @@ var (
 	afterRebootAnnotations  flagutil.StringSliceFlag
 	kubeconfig              = flag.String("kubeconfig", "", "Path to a kubeconfig file. Default to the in-cluster config if not provided.")
 	autoLabelContainerLinux = flag.Bool("auto-label-container-linux", false, "Auto-label Container Linux nodes with agent=true (convenience)")
+	rebootWindowStart       = flag.String("reboot-window-start", "", "Day of week ('Sun', 'Mon', ...; optional) and time of day at which the reboot window starts. E.g. 'Mon 14:00', '11:00'")
+	rebootWindowLength      = flag.String("reboot-window-length", "", "Length of the reboot window. E.g. '1h30m'")
 	printVersion            = flag.Bool("version", false, "Print version and exit")
 	// deprecated
 	analyticsEnabled optValue
@@ -70,6 +72,8 @@ func main() {
 		AgentImageRepo:          *agentImageRepo,
 		BeforeRebootAnnotations: beforeRebootAnnotations,
 		AfterRebootAnnotations:  afterRebootAnnotations,
+		RebootWindowStart:       *rebootWindowStart,
+		RebootWindowLength:      *rebootWindowLength,
 	})
 	if err != nil {
 		glog.Fatalf("Failed to initialize %s: %v", os.Args[0], err)

doc/reboot-windows.md

@@ -0,0 +1,37 @@
+# Reboot windows
+The CLUO `update-operator` can be configured to only reboot nodes during certain timeframes.
+Pre-reboot checks are prevented from running as well.
+
+## Configuring update-operator
+
+The reboot window is configured through the the flags `--reboot-window-start` 
+and `--reboot-window-length`, or through the environment variables 
+`UPDATE_OPERATOR_REBOOT_WINDOW_START` and `UPDATE_OPERATOR_REBOOT_WINDOW_LENGTH`.
+
+Here is an example configuration:
+
+```
+/bin/update-operator \
+ --reboot-window-start=14:00 \
+ --reboot-window-length=1h
+```
+
+This would configure `update-operator` to only reboot between 2pm and 3pm. Optionally,
+a day of week may be specified for the start of the window:
+
+```
+/bin/update-operator \
+ --reboot-window-start="Thu 23:00" \
+ --reboot-window-length=1h30m
+```
+
+This would configure `update-operator` to only reboot the system on Thursday after 11pm,
+or on Friday before 12:30am.
+
+Currently, the only supported values for the day of week are short day names,
+e.g. `Sun`, `Mon`, `Tue`, `Wed`, `Thu`, `Fri`, and `Sat`, but the day of week can
+be upper or lower case. The time of day must be specified in 24-hour time format.
+The window length is expressed as input to go's [time.ParseDuration][time.ParseDuration]
+function.
+
+[time.ParseDuration]: http://godoc.org/time#ParseDuration

glide.lock

@@ -10,6 +10,10 @@ import:
   version: v3
   subpackages:
   - flagutil
+- package: github.com/coreos/locksmith
+  version: ef4279232ecdd23b7a3e3092747367b413da6e6b
+  subpackages:
+  - pkg/timeutil
 - package: github.com/davecgh/go-spew
   version: 782f4967f2dc4564575ca782fe2d04090b5faca8
   subpackages:

glide.yaml

@@ -21,6 +21,7 @@ import (
 
 	"github.com/coreos/container-linux-update-operator/pkg/constants"
 	"github.com/coreos/container-linux-update-operator/pkg/k8sutil"
+	"github.com/coreos/locksmith/pkg/timeutil"
 )
 
 const (
@@ -102,6 +103,9 @@ type Kontroller struct {
 	// auto-label Container Linux nodes for migration compatability
 	autoLabelContainerLinux bool
 
+	// reboot window
+	rebootWindow *timeutil.Periodic
+
 	// Deprecated
 	manageAgent    bool
 	agentImageRepo string
@@ -116,6 +120,9 @@ type Config struct {
 	// annotations to look for before and after reboots
 	BeforeRebootAnnotations []string
 	AfterRebootAnnotations  []string
+	// reboot window
+	RebootWindowStart  string
+	RebootWindowLength string
 	// Deprecated
 	ManageAgent    bool
 	AgentImageRepo string
@@ -159,6 +166,16 @@ func New(config Config) (*Kontroller, error) {
 		return nil, fmt.Errorf("unable to determine operator namespace: please ensure POD_NAMESPACE environment variable is set")
 	}
 
+	var rebootWindow *timeutil.Periodic
+	if config.RebootWindowStart != "" && config.RebootWindowLength != "" {
+		rw, err := timeutil.ParsePeriodic(config.RebootWindowStart, config.RebootWindowLength)
+		if err != nil {
+			return nil, fmt.Errorf("Error parsing reboot window: %s", err)
+		}
+
+		rebootWindow = rw
+	}
+
 	return &Kontroller{
 		kc: kc,
 		nc: nc,
@@ -171,6 +188,7 @@ func New(config Config) (*Kontroller, error) {
 		autoLabelContainerLinux:     config.AutoLabelContainerLinux,
 		manageAgent:                 config.ManageAgent,
 		agentImageRepo:              config.AgentImageRepo,
+		rebootWindow:                rebootWindow,
 	}, nil
 }
 
@@ -424,7 +442,8 @@ func (k *Kontroller) checkAfterReboot() error {
 // before-reboot=true label. This is considered the beginning of the reboot
 // process from the perspective of the update-operator. It will only mark
 // nodes with this label up to the maximum number of concurrently rebootable
-// nodes as configured with the maxRebootingNodes constant.
+// nodes as configured with the maxRebootingNodes constant. It also checks if
+// we are inside the reboot window.
 // It cleans up the before-reboot annotations before it applies the label, in
 // case there are any left over from the last reboot.
 // If there is an error getting the list of nodes or updating any of them, an
@@ -435,6 +454,17 @@ func (k *Kontroller) markBeforeReboot() error {
 		return fmt.Errorf("Failed listing nodes: %v", err)
 	}
 
+	// check if a reboot window is configured
+	if k.rebootWindow != nil {
+		// get previous occurrence relative to now
+		period := k.rebootWindow.Previous(time.Now())
+		// check if we are inside the reboot window
+		if !(period.End.After(time.Now())) {
+			glog.V(4).Info("We are outside the reboot window; not labeling rebootable nodes for now")
+			return nil
+		}
+	}
+
 	// find nodes which are still rebooting
 	rebootingNodes := k8sutil.FilterNodesByAnnotation(nodelist.Items, stillRebootingSelector)
 	// nodes running before and after reboot checks are still considered to be "rebooting" to us