[Feature] Pre OOM Abort (#1197)

Author: ajanikow

CHANGELOG.md

@@ -31,6 +31,7 @@
 - (Bugfix) Do not block reconciliation in case of Resource failure
 - (Improvement) Multi-arch support for ID member
 - (Feature) Allow to change Pod Network and PID settings
+- (Feature) Pre OOM Abort function
 
 ## [1.2.20](https://github.com/arangodb/kube-arangodb/tree/1.2.20) (2022-10-25)
 - (Feature) Add action progress

cmd/cmd.go

@@ -97,6 +97,10 @@ var (
 		Run: executeMain,
 	}
 
+	memoryLimit struct {
+		hardLimit uint64
+	}
+
 	logLevels     []string
 	serverOptions struct {
 		host            string
@@ -209,6 +213,7 @@ func init() {
 	f.IntVar(&operatorKubernetesOptions.burst, "kubernetes.burst", kclient.DefaultBurst, "Burst for the k8s API")
 	f.BoolVar(&crdOptions.install, "crd.install", true, "Install missing CRD if access is possible")
 	f.IntVar(&operatorBackup.concurrentUploads, "backup-concurrent-uploads", globals.DefaultBackupConcurrentUploads, "Number of concurrent uploads per deployment")
+	f.Uint64Var(&memoryLimit.hardLimit, "memory-limit", 0, "Define memory limit for hard shutdown and the dump of goroutines. Used for testing")
 	if err := features.Init(&cmdMain); err != nil {
 		panic(err.Error())
 	}
@@ -236,6 +241,8 @@ func executeMain(cmd *cobra.Command, args []string) {
 	name := os.Getenv(constants.EnvOperatorPodName)
 	ip := os.Getenv(constants.EnvOperatorPodIP)
 
+	go monitorMemoryLimit()
+
 	deploymentApi.DefaultImage = operatorOptions.arangoImage
 
 	globals.GetGlobalTimeouts().Kubernetes().Set(operatorTimeouts.k8s)

cmd/memory.go

@@ -0,0 +1,55 @@
+//
+// DISCLAIMER
+//
+// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package cmd
+
+import (
+	"runtime"
+	"syscall"
+	"time"
+
+	"github.com/arangodb/kube-arangodb/pkg/util/shutdown"
+)
+
+func monitorMemoryLimit() {
+	if memoryLimit.hardLimit == 0 {
+		return
+	}
+
+	var m runtime.MemStats
+
+	t := time.NewTicker(time.Millisecond)
+	defer t.Stop()
+
+	for {
+		select {
+		case <-t.C:
+			runtime.ReadMemStats(&m)
+
+			if m.Sys > 1024*1024*memoryLimit.hardLimit {
+				if err := syscall.Kill(syscall.Getpid(), syscall.SIGABRT); err != nil {
+					panic(err)
+				}
+			}
+		case <-shutdown.Channel():
+			return
+		}
+	}
+}

pkg/util/shutdown/shutdown.go

@@ -0,0 +1,46 @@
+//
+// DISCLAIMER
+//
+// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package shutdown
+
+import (
+	"os"
+	"os/signal"
+	"syscall"
+)
+
+func init() {
+	shutdown = make(chan struct{})
+
+	sigChannel := make(chan os.Signal, 2)
+
+	signal.Notify(sigChannel, os.Interrupt, syscall.SIGTERM)
+
+	go func() {
+		defer close(shutdown)
+		<-sigChannel
+	}()
+}
+
+var shutdown chan struct{}
+
+func Channel() <-chan struct{} {
+	return shutdown
+}

pkg/util/signal.go

@@ -22,30 +22,21 @@ package util
 
 import (
 	"context"
-	"os"
-	"os/signal"
-	"syscall"
+
+	"github.com/arangodb/kube-arangodb/pkg/util/shutdown"
 )
 
 // CreateSignalContext creates and returns the context which is closed when one of the provided signal occurs.
-// If the provided list of signals is empty, then SIGINT and SIGTERM is used by default.
-func CreateSignalContext(ctx context.Context, signals ...os.Signal) context.Context {
+// SIGINT and SIGTERM is used by default.
+func CreateSignalContext(ctx context.Context) context.Context {
 	if ctx == nil {
 		ctx = context.Background()
 	}
 	ctxSignal, cancelSignal := context.WithCancel(ctx)
-	sigChannel := make(chan os.Signal, 2)
-
-	if len(signals) > 0 {
-		signal.Notify(sigChannel, signals...)
-	} else {
-		signal.Notify(sigChannel, os.Interrupt, syscall.SIGTERM)
-	}
 
 	go func() {
-		// Wait until signal occurs.
-		<-sigChannel
-		close(sigChannel)
+		// Wait until shutdown starts
+		<-shutdown.Channel()
 		// Close the context which is used by the caller.
 		cancelSignal()
 	}()