BUG/MEDIUM: cluster: wait until all resources and routines are ready

wait until all resources and routines are ready before starting connection process. previously connection process could be started earlier that other resources expected

Author: oktalz

cmd/dataplaneapi/main.go

@@ -16,10 +16,13 @@
 package main
 
 import (
+	"crypto/tls"
 	"fmt"
+	"net/http"
 	"os"
 	"path"
 	"syscall"
+	"time"
 
 	loads "github.com/go-openapi/loads"
 	"github.com/go-openapi/runtime"
@@ -218,6 +221,28 @@ func startServer(cfg *configuration.Configuration) (reload configuration.AtomicB
 	}()
 
 	server.ConfigureAPI()
+	schema := "http"
+	if len(server.EnabledListeners) > 0 {
+		schema = server.EnabledListeners[0]
+	}
+	path := fmt.Sprintf("%s://%s:%d%s", schema, server.Host, server.Port, cfg.RuntimeData.APIBasePath)
+	go func(path string) {
+		tr := &http.Transport{
+			// we just need to check if we have a response
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, //nolint:gosec
+		}
+		client := &http.Client{Transport: tr}
+		numTrys := 0
+		for ; numTrys < 10; numTrys++ {
+			_, err := client.Get(path)
+			if err == nil {
+				cfg.Notify.ServerStarted.Notify()
+				return
+			}
+			time.Sleep(500 * time.Millisecond)
+		}
+		log.Fatalf("check if dataplane is running failed on %s", path)
+	}(path)
 	if err := server.Serve(); err != nil {
 		log.Fatalln(err)
 	}

configuration/cluster_sync.go

@@ -31,7 +31,6 @@ import (
 	"net/http"
 	"os"
 	"path"
-	"runtime"
 	"strconv"
 	"strings"
 	"time"
@@ -88,11 +87,12 @@ func (c *ClusterSync) Monitor(cfg *Configuration, cli *client_native.HAProxyClie
 	c.certFetch = make(chan struct{}, 2)
 	go c.fetchCert()
 
+	<-c.cfg.Notify.ServerStarted.Subscribe("clusterMonitor")
+
 	key := c.cfg.Cluster.BootstrapKey.Load()
 	certFetched := cfg.Cluster.CertificateFetched.Load()
 
 	if key != "" && !certFetched {
-		runtime.Gosched()
 		c.cfg.Notify.BootstrapKeyChanged.Notify()
 	}
 }

configuration/configuration.go

@@ -134,6 +134,7 @@ type RuntimeData struct {
 
 type NotifyConfiguration struct {
 	BootstrapKeyChanged *ChanNotify `yaml:"-"`
+	ServerStarted       *ChanNotify `yaml:"-"`
 	CertificateRefresh  *ChanNotify `yaml:"-"`
 	Reload              *ChanNotify `yaml:"-"`
 	Shutdown            *ChanNotify `yaml:"-"`
@@ -171,6 +172,7 @@ func Get() *Configuration {
 		cfg = &Configuration{}
 		cfg.initSignalHandler()
 		cfg.Notify.BootstrapKeyChanged = NewChanNotify()
+		cfg.Notify.ServerStarted = NewChanNotify()
 		cfg.Notify.CertificateRefresh = NewChanNotify()
 		cfg.Notify.Reload = NewChanNotify()
 		cfg.Notify.Shutdown = NewChanNotify()
@@ -204,6 +206,7 @@ func (c *Configuration) GetStorageData() *StorageDataplaneAPIConfiguration {
 
 func (c *Configuration) UnSubscribeAll() {
 	c.Notify.BootstrapKeyChanged.UnSubscribeAll()
+	c.Notify.ServerStarted.UnSubscribeAll()
 	c.Notify.CertificateRefresh.UnSubscribeAll()
 	c.Notify.Reload.UnSubscribeAll()
 	c.Notify.Shutdown.UnSubscribeAll()