feat: to have statistics about DLE usage and trends (#299)

* collect aggregated engine data
* store data on the Platform side
* make it optional: add a configuration flag to disable telemetry
* add an endpoint providing configuration flags to be aware of the current telemetry mode
* allow an empty verification token

Author: agneum

api/swagger-spec/dblab_server_swagger.yaml

@@ -404,8 +404,6 @@ paths:
       responses:
         200:
           description: "Successful operation"
-          content:
-            text/json:
         400:
           description: "Bad request"
           schema:
@@ -522,6 +520,8 @@ definitions:
       startedAt:
         type: "string"
         format: "date-time"
+      telemetry:
+        type: boolean
 
   PoolEntry:
     type: "object"
@@ -593,11 +593,9 @@ definitions:
       lastRefresh:
         type: "string"
         format: "date-time"
-        nullable: true
       nextRefresh:
         type: "string"
         format: "date-time"
-        nullable: true
 
   Provisioner:
     type: "object"

cmd/cli/commands/clone/actions.go

@@ -39,7 +39,9 @@ func list(cliCtx *cli.Context) error {
 
 	defer func() { _ = body.Close() }()
 
-	var viewCloneList *models.CloneListView
+	viewCloneList := &models.CloneListView{
+		Clones: make([]*models.CloneView, 0),
+	}
 
 	if err := json.NewDecoder(body).Decode(&viewCloneList); err != nil {
 		return err

cmd/cli/commands/config/command_list.go

@@ -31,7 +31,6 @@ func CommandList() []*cli.Command {
 						&cli.StringFlag{
 							Name:     "token",
 							Usage:    "verification token of Database Lab instance",
-							Required: true,
 						},
 						&cli.BoolFlag{
 							Name:  "insecure",

cmd/cli/commands/global/command_list.go

@@ -32,7 +32,6 @@ func List() []*cli.Command {
 				&cli.StringFlag{
 					Name:     "token",
 					Usage:    "verification token of Database Lab instance",
-					Required: true,
 				},
 				&cli.BoolFlag{
 					Name:  "insecure",

cmd/database-lab/main.go

@@ -32,7 +32,9 @@ import (
 	"gitlab.com/postgres-ai/database-lab/v2/pkg/services/provision/resources"
 	"gitlab.com/postgres-ai/database-lab/v2/pkg/services/provision/runners"
 	"gitlab.com/postgres-ai/database-lab/v2/pkg/srv"
+	"gitlab.com/postgres-ai/database-lab/v2/pkg/telemetry"
 	"gitlab.com/postgres-ai/database-lab/v2/pkg/util/networks"
+	"gitlab.com/postgres-ai/database-lab/v2/version"
 )
 
 const (
@@ -47,6 +49,10 @@ func main() {
 
 	log.Msg("Database Lab Instance ID:", cfg.Global.InstanceID)
 
+	if cfg.Server.VerificationToken == "" {
+		log.Warn("Verification Token is empty. Database Lab Engine is insecure")
+	}
+
 	runner := runners.NewLocalRunner(cfg.Provision.UseSudo)
 
 	pm := pool.NewPoolManager(&cfg.PoolManager, runner)
@@ -96,8 +102,14 @@ func main() {
 		shutdownDatabaseLabEngine(shutdownCtx, dockerCLI, cfg.Global, pm.First().Pool())
 	}
 
+	tm, err := telemetry.New(cfg.Global)
+	if err != nil {
+		log.Errf(errors.WithMessage(err, "failed to initialize a telemetry service").Error())
+		return
+	}
+
 	// Create a new retrieval service to prepare a data directory and start snapshotting.
-	retrievalSvc := retrieval.New(cfg, dockerCLI, pm, runner)
+	retrievalSvc := retrieval.New(cfg, dockerCLI, pm, tm, runner)
 
 	if err := retrievalSvc.Run(ctx); err != nil {
 		log.Err("Failed to run the data retrieval service:", err)
@@ -116,7 +128,7 @@ func main() {
 
 	observingChan := make(chan string, 1)
 
-	cloningSvc := cloning.NewBase(&cfg.Cloning, provisionSvc, observingChan)
+	cloningSvc := cloning.NewBase(&cfg.Cloning, provisionSvc, tm, observingChan)
 	if err = cloningSvc.Run(ctx); err != nil {
 		log.Err(err)
 		emergencyShutdown()
@@ -129,10 +141,17 @@ func main() {
 
 	go removeObservingClones(observingChan, obs)
 
-	server := srv.NewServer(&cfg.Server, &cfg.Global, cloningSvc, retrievalSvc, platformSvc, dockerCLI, obs, est, pm)
+	tm.SendEvent(ctx, telemetry.EngineStartedEvent, telemetry.EngineStarted{
+		EngineVersion: version.GetVersion(),
+		DBVersion:     provisionSvc.DetectDBVersion(),
+		Pools:         pm.CollectPoolStat(),
+		Restore:       retrievalSvc.CollectRestoreTelemetry(),
+	})
+
+	server := srv.NewServer(&cfg.Server, &cfg.Global, cloningSvc, retrievalSvc, platformSvc, dockerCLI, obs, est, pm, tm)
 	shutdownCh := setShutdownListener()
 
-	go setReloadListener(ctx, provisionSvc, retrievalSvc, pm, cloningSvc, platformSvc, est, server)
+	go setReloadListener(ctx, provisionSvc, tm, retrievalSvc, pm, cloningSvc, platformSvc, est, server)
 
 	server.InitHandlers()
 
@@ -155,9 +174,10 @@ func main() {
 
 	shutdownDatabaseLabEngine(shutdownCtx, dockerCLI, cfg.Global, pm.First().Pool())
 	cloningSvc.SaveClonesState()
+	tm.SendEvent(ctx, telemetry.EngineStoppedEvent, telemetry.EngineStopped{Uptime: server.Uptime()})
 }
 
-func reloadConfig(ctx context.Context, provisionSvc *provision.Provisioner, retrievalSvc *retrieval.Retrieval,
+func reloadConfig(ctx context.Context, provisionSvc *provision.Provisioner, tm *telemetry.Agent, retrievalSvc *retrieval.Retrieval,
 	pm *pool.Manager, cloningSvc *cloning.Base, platformSvc *platform.Service, est *estimator.Estimator, server *srv.Server) error {
 	cfg, err := config.LoadConfiguration()
 	if err != nil {
@@ -187,6 +207,7 @@ func reloadConfig(ctx context.Context, provisionSvc *provision.Provisioner, retr
 	}
 
 	provisionSvc.Reload(cfg.Provision, dbCfg)
+	tm.Reload(cfg.Global)
 	retrievalSvc.Reload(ctx, cfg)
 	cloningSvc.Reload(cfg.Cloning)
 	platformSvc.Reload(newPlatformSvc)
@@ -196,15 +217,15 @@ func reloadConfig(ctx context.Context, provisionSvc *provision.Provisioner, retr
 	return nil
 }
 
-func setReloadListener(ctx context.Context, provisionSvc *provision.Provisioner, retrievalSvc *retrieval.Retrieval,
+func setReloadListener(ctx context.Context, provisionSvc *provision.Provisioner, tm *telemetry.Agent, retrievalSvc *retrieval.Retrieval,
 	pm *pool.Manager, cloningSvc *cloning.Base, platformSvc *platform.Service, est *estimator.Estimator, server *srv.Server) {
 	reloadCh := make(chan os.Signal, 1)
 	signal.Notify(reloadCh, syscall.SIGHUP)
 
 	for range reloadCh {
 		log.Msg("Reloading configuration")
 
-		if err := reloadConfig(ctx, provisionSvc, retrievalSvc, pm, cloningSvc, platformSvc, est, server); err != nil {
+		if err := reloadConfig(ctx, provisionSvc, tm, retrievalSvc, pm, cloningSvc, platformSvc, est, server); err != nil {
 			log.Err("Failed to reload configuration", err)
 		}
 

cmd/runci/main.go

@@ -38,6 +38,11 @@ func main() {
 	log.SetDebug(cfg.App.Debug)
 	log.Dbg("Config loaded: ", cfg)
 
+	if cfg.App.VerificationToken == "" {
+		log.Err("DB Migration Checker is insecure since the Verification Token is empty")
+		return
+	}
+
 	networkID := discoverNetwork(ctx, cfg, dockerCLI)
 	if networkID != "" {
 		hostname := os.Getenv("HOSTNAME")

configs/config.example.logical_generic.yml

@@ -42,6 +42,15 @@ global:
     # Default database name.
     dbname: postgres
 
+  # Telemetry: anonymous statistics sent to Postgres.ai.
+  # Used to analyze DLE usage, it helps the DLE maintainers make decisions on product development.
+  # Please leave it enabled if possible – this will contribute to DLE development.
+  # The full list of data points being collected: https://postgres.ai/docs/database-lab/telemetry
+  telemetry:
+    enabled: true
+    # Telemetry API URL. To send anonymous telemetry data, keep it default ("https://postgres.ai/api/general").
+    url: "https://postgres.ai/api/general"
+
 # Manages filesystem pools (in the case of ZFS) or volume groups.
 poolManager:
   # The full path which contains the pool mount directories. mountDir can contain multiple pool directories.

configs/config.example.logical_rds_iam.yml

@@ -42,6 +42,15 @@ global:
     # Default database name.
     dbname: postgres
 
+  # Telemetry: anonymous statistics sent to Postgres.ai.
+  # Used to analyze DLE usage, it helps the DLE maintainers make decisions on product development.
+  # Please leave it enabled if possible – this will contribute to DLE development.
+  # The full list of data points being collected: https://postgres.ai/docs/database-lab/telemetry
+  telemetry:
+    enabled: true
+    # Telemetry API URL. To send anonymous telemetry data, keep it default ("https://postgres.ai/api/general").
+    url: "https://postgres.ai/api/general"
+
 # Manages filesystem pools (in the case of ZFS) or volume groups.
 poolManager:
   # The full path which contains the pool mount directories. mountDir can contain multiple pool directories.

configs/config.example.physical_generic.yml

@@ -42,6 +42,15 @@ global:
     # Default database name.
     dbname: postgres
 
+  # Telemetry: anonymous statistics sent to Postgres.ai.
+  # Used to analyze DLE usage, it helps the DLE maintainers make decisions on product development.
+  # Please leave it enabled if possible – this will contribute to DLE development.
+  # The full list of data points being collected: https://postgres.ai/docs/database-lab/telemetry
+  telemetry:
+    enabled: true
+    # Telemetry API URL. To send anonymous telemetry data, keep it default ("https://postgres.ai/api/general").
+    url: "https://postgres.ai/api/general"
+
 # Manages filesystem pools (in the case of ZFS) or volume groups.
 poolManager:
   # The full path which contains the pool mount directories. mountDir can contain multiple pool directories.

configs/config.example.physical_walg.yml

@@ -42,6 +42,15 @@ global:
     # Default database name.
     dbname: postgres
 
+  # Telemetry: anonymous statistics sent to Postgres.ai.
+  # Used to analyze DLE usage, it helps the DLE maintainers make decisions on product development.
+  # Please leave it enabled if possible – this will contribute to DLE development.
+  # The full list of data points being collected: https://postgres.ai/docs/database-lab/telemetry
+  telemetry:
+    enabled: true
+    # Telemetry API URL. To send anonymous telemetry data, keep it default ("https://postgres.ai/api/general").
+    url: "https://postgres.ai/api/general"
+
 # Manages filesystem pools (in the case of ZFS) or volume groups.
 poolManager:
   # The full path which contains the pool mount directories. mountDir can contain multiple pool directories.