Add VersionCheck and Metadata to Agent labels (#7737)

Author: AlexFenlon

cmd/nginx-ingress/main.go

@@ -26,6 +26,7 @@ import (
 	"github.com/nginx/kubernetes-ingress/internal/k8s"
 	"github.com/nginx/kubernetes-ingress/internal/k8s/secrets"
 	license_reporting "github.com/nginx/kubernetes-ingress/internal/license_reporting"
+	"github.com/nginx/kubernetes-ingress/internal/metadata"
 	"github.com/nginx/kubernetes-ingress/internal/metrics"
 	"github.com/nginx/kubernetes-ingress/internal/metrics/collectors"
 	"github.com/nginx/kubernetes-ingress/internal/nginx"
@@ -128,7 +129,13 @@ func main() {
 		licenseReporter = license_reporting.NewLicenseReporter(kubeClient, eventRecorder, pod)
 	}
 
-	nginxManager, useFakeNginxManager := createNginxManager(ctx, managerCollector, licenseReporter)
+	var deploymentMetadata *metadata.Metadata
+
+	if *agent {
+		deploymentMetadata = metadata.NewMetadataReporter(kubeClient, pod, version)
+	}
+
+	nginxManager, useFakeNginxManager := createNginxManager(ctx, managerCollector, licenseReporter, deploymentMetadata)
 
 	nginxVersion := getNginxVersionInfo(ctx, nginxManager)
 
@@ -562,14 +569,14 @@ func createTemplateExecutors(ctx context.Context) (*version1.TemplateExecutor, *
 	return templateExecutor, templateExecutorV2
 }
 
-func createNginxManager(ctx context.Context, managerCollector collectors.ManagerCollector, licenseReporter *license_reporting.LicenseReporter) (nginx.Manager, bool) {
+func createNginxManager(ctx context.Context, managerCollector collectors.ManagerCollector, licenseReporter *license_reporting.LicenseReporter, deploymentMetadata *metadata.Metadata) (nginx.Manager, bool) {
 	useFakeNginxManager := *proxyURL != ""
 	var nginxManager nginx.Manager
 	if useFakeNginxManager {
 		nginxManager = nginx.NewFakeManager("/etc/nginx")
 	} else {
 		timeout := time.Duration(*nginxReloadTimeout) * time.Millisecond
-		nginxManager = nginx.NewLocalManager(ctx, "/etc/nginx/", *nginxDebug, managerCollector, licenseReporter, timeout, *nginxPlus)
+		nginxManager = nginx.NewLocalManager(ctx, "/etc/nginx/", *nginxDebug, managerCollector, licenseReporter, deploymentMetadata, timeout, *nginxPlus)
 	}
 	return nginxManager, useFakeNginxManager
 }

internal/common_cluster_info/common_cluster_info.go

@@ -9,7 +9,7 @@ import (
 	"k8s.io/client-go/kubernetes"
 )
 
-// This file contains functions for data used in both product telemetry and license reporting
+// This file contains functions for data used in product telemetry, metadata and license reporting
 
 // GetNodeCount returns the number of nodes in the cluster
 func GetNodeCount(ctx context.Context, client kubernetes.Interface) (int, error) {
@@ -43,10 +43,6 @@ func GetInstallationID(ctx context.Context, client kubernetes.Interface, podNSNa
 		return "", err
 	}
 	podOwner := pod.GetOwnerReferences()
-	if len(podOwner) != 1 {
-		return "", fmt.Errorf("expected pod owner reference to be 1, got %d", len(podOwner))
-	}
-
 	switch podOwner[0].Kind {
 	case "ReplicaSet":
 		rs, err := client.AppsV1().ReplicaSets(podNSName.Namespace).Get(ctx, podOwner[0].Name, metav1.GetOptions{})
@@ -61,6 +57,33 @@ func GetInstallationID(ctx context.Context, client kubernetes.Interface, podNSNa
 	case "DaemonSet":
 		return string(podOwner[0].UID), nil
 	default:
-		return "", fmt.Errorf("expected pod owner reference to be ReplicaSet or DeamonSet, got %s", podOwner[0].Kind)
+		return string(podOwner[0].UID), nil
+	}
+}
+
+// GetDeploymentName returns the name of the Deployment
+func GetDeploymentName(ctx context.Context, client kubernetes.Interface, podNSName types.NamespacedName) (string, error) {
+	pod, err := client.CoreV1().Pods(podNSName.Namespace).Get(ctx, podNSName.Name, metav1.GetOptions{})
+	if err != nil {
+		return "", err
+	}
+	owners := pod.GetOwnerReferences()
+	owner := owners[0]
+	switch owner.Kind {
+	case "ReplicaSet":
+		replicaSet, err := client.AppsV1().ReplicaSets(podNSName.Namespace).Get(ctx, owner.Name, metav1.GetOptions{})
+		if err != nil {
+			return "", err
+		}
+		for _, replicaSetOwner := range replicaSet.GetOwnerReferences() {
+			if replicaSetOwner.Kind == "Deployment" {
+				return replicaSetOwner.Name, nil
+			}
+		}
+		return "", fmt.Errorf("replicaset %s has no owner", replicaSet.Name)
+	case "DaemonSet":
+		return owner.Name, nil
+	default:
+		return owner.Name, nil
 	}
 }

internal/metadata/metadata.go

@@ -0,0 +1,70 @@
+package metadata
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	clusterInfo "github.com/nginx/kubernetes-ingress/internal/common_cluster_info"
+	api_v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/kubernetes"
+)
+
+// Labels contains the metadata information needed for reporting to Agent
+type Labels struct {
+	ProductName         string `json:"product_name"`
+	ProductVersion      string `json:"product_version"`
+	ClusterID           string `json:"cluster_id"`
+	DeploymentName      string `json:"deployment_name"`
+	DeploymentID        string `json:"deployment_id"`
+	DeploymentNamespace string `json:"deployment_namespace"`
+}
+
+func newMetadataInfo(deploymentNamespace, clusterID, deploymentID, productVersion, deploymentName string) *Labels {
+	return &Labels{
+		ProductName:         "nic",
+		ProductVersion:      productVersion,
+		ClusterID:           clusterID,
+		DeploymentID:        deploymentID,
+		DeploymentName:      deploymentName,
+		DeploymentNamespace: deploymentNamespace,
+	}
+}
+
+// Metadata contains required information for metadata reporting
+type Metadata struct {
+	K8sClientReader kubernetes.Interface
+	PodNSName       types.NamespacedName
+	Pod             *api_v1.Pod
+	NICVersion      string
+}
+
+// NewMetadataReporter creates a new MetadataConfig
+func NewMetadataReporter(client kubernetes.Interface, pod *api_v1.Pod, version string) *Metadata {
+	return &Metadata{
+		K8sClientReader: client,
+		PodNSName:       types.NamespacedName{Namespace: os.Getenv("POD_NAMESPACE"), Name: os.Getenv("POD_NAME")},
+		Pod:             pod,
+		NICVersion:      version,
+	}
+}
+
+// CollectAndWrite collects the metadata information and returns a Labels struct
+func (md *Metadata) CollectAndWrite(ctx context.Context) (*Labels, error) {
+	deploymentNamespace := md.PodNSName.Namespace
+	clusterID, err := clusterInfo.GetClusterID(ctx, md.K8sClientReader)
+	if err != nil {
+		return nil, fmt.Errorf("error collecting ClusterID: %w", err)
+	}
+	deploymentID, err := clusterInfo.GetInstallationID(ctx, md.K8sClientReader, md.PodNSName)
+	if err != nil {
+		return nil, fmt.Errorf("error collecting InstallationID: %w", err)
+	}
+	deploymentName, err := clusterInfo.GetDeploymentName(ctx, md.K8sClientReader, md.PodNSName)
+	if err != nil {
+		return nil, fmt.Errorf("error collecting DeploymentName: %w", err)
+	}
+	info := newMetadataInfo(deploymentNamespace, clusterID, deploymentID, md.NICVersion, deploymentName)
+	return info, nil
+}

internal/metadata/metadata_test.go

@@ -0,0 +1,92 @@
+package metadata
+
+import (
+	"context"
+	"os"
+	"testing"
+
+	api_v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/kubernetes/fake"
+)
+
+func TestNewMetadataInfo(t *testing.T) {
+	info := newMetadataInfo("nginx-ingress", "e3a5e702-65a7-a55f-753d78cd7ff7", "555-1222-4414test-11223355", "5.0.0", "my-release")
+	if info.ProductName != "nic" {
+		t.Errorf("ProductName = %q, want %q", info.ProductName, "nic")
+	}
+	if info.DeploymentNamespace != "nginx-ingress" {
+		t.Errorf("DeploymentNamespace = %q, want %q", info.DeploymentNamespace, "nginx-ingress")
+	}
+	if info.ClusterID != "e3a5e702-65a7-a55f-753d78cd7ff7" {
+		t.Errorf("ClusterID = %q, want %q", info.ClusterID, "e3a5e702-65a7-a55f-753d78cd7ff7")
+	}
+	if info.DeploymentID != "555-1222-4414test-11223355" {
+		t.Errorf("DeploymentID = %q, want %q", info.DeploymentID, "555-1222-4414test-11223355")
+	}
+	if info.ProductVersion != "5.0.0" {
+		t.Errorf("ProductVersion = %q, want %q", info.ProductVersion, "5.0.0")
+	}
+	if info.DeploymentName != "my-release" {
+		t.Errorf("DeploymentName = %q, want %q", info.DeploymentName, "my-release")
+	}
+}
+
+func TestCollectAndWrite(t *testing.T) {
+	pod := &api_v1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test-pod",
+			Namespace: "test-namespace",
+			OwnerReferences: []metav1.OwnerReference{
+				{
+					APIVersion: "apps/v1",
+					Kind:       "DaemonSet",
+					Name:       "test-pod",
+					UID:        types.UID("install-123"),
+				},
+			},
+		},
+	}
+
+	if err := os.Setenv("POD_NAMESPACE", pod.Namespace); err != nil {
+		t.Errorf("unable to set POD_NAMESPACE: %v", err)
+	}
+	if err := os.Setenv("POD_NAME", pod.Name); err != nil {
+		t.Errorf("unable to set POD_NAME: %v", err)
+	}
+
+	client := fake.NewSimpleClientset(
+		&api_v1.Namespace{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "kube-system",
+				UID:  types.UID("123-abc-456-def"),
+			},
+		},
+		pod,
+	)
+
+	reporter := NewMetadataReporter(client, pod, "5.0.0")
+	if reporter == nil {
+		t.Fatal("expected reporter to be non-nil")
+	}
+
+	info, err := reporter.CollectAndWrite(context.TODO())
+	if err != nil {
+		t.Fatalf("CollectAndWrite() error = %v", err)
+	}
+	if got, want := info.ProductName, "nic"; got != want {
+		t.Errorf("ProductName = %q, want %q", got, want)
+	}
+}
+
+func TestNewMetadataReporter(t *testing.T) {
+	reporter := NewMetadataReporter(
+		fake.NewSimpleClientset(),
+		&api_v1.Pod{},
+		"5.0.0",
+	)
+	if reporter == nil {
+		t.Fatal("Expected NewMetadataReporter to return non-nil")
+	}
+}

internal/nginx/manager.go

@@ -14,6 +14,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/nginx/kubernetes-ingress/internal/metadata"
+
 	license_reporting "github.com/nginx/kubernetes-ingress/internal/license_reporting"
 	nl "github.com/nginx/kubernetes-ingress/internal/logger"
 	"github.com/nginx/kubernetes-ingress/internal/metrics/collectors"
@@ -51,8 +53,9 @@ const (
 )
 
 var (
-	ossre  = regexp.MustCompile(`(?P<name>\S+)/(?P<version>\S+)`)
-	plusre = regexp.MustCompile(`(?P<name>\S+)/(?P<version>\S+).\((?P<plus>\S+plus\S+)\)`)
+	ossre   = regexp.MustCompile(`(?P<name>\S+)/(?P<version>\S+)`)
+	plusre  = regexp.MustCompile(`(?P<name>\S+)/(?P<version>\S+).\((?P<plus>\S+plus\S+)\)`)
+	agentre = regexp.MustCompile(`^v(?P<major>\d+)\.?(?P<minor>\d+)?\.?(?P<patch>\d+)?(-.+)?$`)
 )
 
 // ServerConfig holds the config data for an upstream server in NGINX Plus.
@@ -99,7 +102,7 @@ type Manager interface {
 	DeleteKeyValStateFiles(virtualServerName string)
 }
 
-// LocalManager updates NGINX configuration, starts, reloads and quits NGINX, updates License Reporting file
+// LocalManager updates NGINX configuration, starts, reloads and quits NGINX, updates License Reporting and the Deployment Metadata file
 // updates NGINX Plus upstream servers. It assumes that NGINX is running in the same container.
 type LocalManager struct {
 	confdPath                    string
@@ -119,6 +122,7 @@ type LocalManager struct {
 	metricsCollector             collectors.ManagerCollector
 	licenseReporter              *license_reporting.LicenseReporter
 	licenseReporterCancel        context.CancelFunc
+	deploymentMetadata           *metadata.Metadata
 	appProtectPluginPid          int
 	appProtectDosAgentPid        int
 	agentPid                     int
@@ -127,7 +131,7 @@ type LocalManager struct {
 }
 
 // NewLocalManager creates a LocalManager.
-func NewLocalManager(ctx context.Context, confPath string, debug bool, mc collectors.ManagerCollector, lr *license_reporting.LicenseReporter, timeout time.Duration, nginxPlus bool) *LocalManager {
+func NewLocalManager(ctx context.Context, confPath string, debug bool, mc collectors.ManagerCollector, lr *license_reporting.LicenseReporter, metadata *metadata.Metadata, timeout time.Duration, nginxPlus bool) *LocalManager {
 	l := nl.LoggerFromContext(ctx)
 	verifyConfigGenerator, err := newVerifyConfigGenerator()
 	if err != nil {
@@ -149,6 +153,7 @@ func NewLocalManager(ctx context.Context, confPath string, debug bool, mc collec
 		verifyClient:                newVerifyClient(timeout),
 		metricsCollector:            mc,
 		licenseReporter:             lr,
+		deploymentMetadata:          metadata,
 		nginxPlus:                   nginxPlus,
 		logger:                      l,
 	}
@@ -607,10 +612,34 @@ func (lm *LocalManager) AppProtectDosAgentStart(apdaDone chan error, debug bool,
 
 // AgentStart starts the AppProtect plugin and sets AppProtect log level.
 func (lm *LocalManager) AgentStart(agentDone chan error, instanceGroup string) {
+	ctx := nl.ContextWithLogger(context.Background(), lm.logger)
 	nl.Debugf(lm.logger, "Starting Agent")
 	args := []string{}
-	if len(instanceGroup) > 0 {
-		args = append(args, "--instance-group", instanceGroup)
+	nl.Debug(lm.logger, lm.AgentVersion())
+	major, _, _, err := ExtractAgentVersionValues(lm.AgentVersion())
+	if err != nil {
+		nl.Fatalf(lm.logger, "Failed to extract Agent version: %v", err)
+	}
+	if major <= 2 {
+		if len(instanceGroup) > 0 {
+			args = append(args, "--instance-group", instanceGroup)
+		}
+	}
+	if major >= 3 {
+		metadataInfo, err := lm.deploymentMetadata.CollectAndWrite(ctx)
+		if err != nil {
+			nl.Fatalf(lm.logger, "Failed to start NGINX Agent: %v", err)
+		}
+		labels := []string{
+			fmt.Sprintf("product_name=%s", metadataInfo.ProductName),
+			fmt.Sprintf("product_version=%s", metadataInfo.ProductVersion),
+			fmt.Sprintf("cluster_id=%s", metadataInfo.ClusterID),
+			fmt.Sprintf("deployment_name=%s", metadataInfo.DeploymentName),
+			fmt.Sprintf("deployment_id=%s", metadataInfo.DeploymentID),
+			fmt.Sprintf("deployment_namespace=%s", metadataInfo.DeploymentNamespace),
+		}
+		metadataLabels := "--labels=" + strings.Join(labels, ",")
+		args = append(args, metadataLabels)
 	}
 	cmd := exec.Command(agentPath, args...) // #nosec G204
 

internal/nginx/version.go

@@ -99,3 +99,30 @@ func extractPlusVersionValues(input string) (int, int, error) {
 
 	return rValue, pValue, nil
 }
+
+// ExtractAgentVersionValues splits the agent version string into major, minor, and patch values.
+func ExtractAgentVersionValues(input string) (int, int, int, error) {
+	var major, minor, patch int
+	matches := agentre.FindStringSubmatch(input)
+
+	if len(matches) == 0 {
+		return 0, 0, 0, fmt.Errorf("no matches found in the input string")
+	}
+
+	major, err := strconv.Atoi(matches[1])
+	if err != nil {
+		return 0, 0, 0, fmt.Errorf("failed to convert major version to integer: %w", err)
+	}
+
+	minor, err = strconv.Atoi(matches[2])
+	if err != nil {
+		return major, 0, 0, fmt.Errorf("failed to convert minor version to integer: %w", err)
+	}
+
+	patch, err = strconv.Atoi(matches[3])
+	if err != nil {
+		return major, minor, 0, fmt.Errorf("failed to convert patch version to integer: %w", err)
+	}
+
+	return major, minor, patch, nil
+}