Revert "feat(gfd): enhance device health check with compute capability probe"

This reverts commit 525ea47.

Author: ArangoGutierrez

cmd/gpu-feature-discovery/main.go

@@ -254,29 +254,10 @@ func (d *gfd) run(sigs chan os.Signal) (bool, error) {
 	}()
 
 	timestampLabeler := lm.NewTimestampLabeler(d.config)
-
-	// Stats for tracking labeling health
-	totalRuns := 0
-	failedRuns := 0
-	lastSuccessTime := time.Now()
-
 rerun:
-	totalRuns++
 	loopLabelers, err := lm.NewLabelers(d.manager, d.vgpu, d.config)
 	if err != nil {
-		failedRuns++
-		klog.Errorf("GFD: Error creating labelers (run %d, failures %d): %v",
-			totalRuns, failedRuns, err)
-
-		// In oneshot mode, fail immediately (for testing and CI)
-		if *d.config.Flags.GFD.Oneshot {
-			return false, err
-		}
-
-		// Otherwise, sleep and retry
-		klog.Info("Sleeping for ", *d.config.Flags.GFD.SleepInterval)
-		time.Sleep(time.Duration(*d.config.Flags.GFD.SleepInterval))
-		goto rerun
+		return false, err
 	}
 
 	labelers := lm.Merge(
@@ -286,52 +267,18 @@ rerun:
 
 	labels, err := labelers.Labels()
 	if err != nil {
-		failedRuns++
-		klog.Errorf("GFD: Error generating labels (run %d, failures %d): %v",
-			totalRuns, failedRuns, err)
-
-		// In oneshot mode, fail immediately (for testing and CI)
-		if *d.config.Flags.GFD.Oneshot {
-			return false, err
-		}
-
-		// Otherwise, sleep and retry
-		klog.Info("Sleeping for ", *d.config.Flags.GFD.SleepInterval)
-		time.Sleep(time.Duration(*d.config.Flags.GFD.SleepInterval))
-		goto rerun
+		return false, fmt.Errorf("error generating labels: %v", err)
 	}
 
 	if len(labels) <= 1 {
 		klog.Warning("No labels generated from any source")
 	}
 
-	// Log stats periodically (every 10 successful runs)
-	if totalRuns%10 == 0 {
-		klog.Infof("GFD Stats: runs=%d, failures=%d, success_rate=%.1f%%, last_success=%v",
-			totalRuns, failedRuns, float64(totalRuns-failedRuns)/float64(totalRuns)*100,
-			time.Since(lastSuccessTime))
-	}
-
 	klog.Info("Creating Labels")
 	if err := d.labelOutputer.Output(labels); err != nil {
-		failedRuns++
-		klog.Errorf("GFD: Error outputting labels (run %d, failures %d): %v",
-			totalRuns, failedRuns, err)
-
-		// In oneshot mode, fail immediately
-		if *d.config.Flags.GFD.Oneshot {
-			return false, err
-		}
-
-		// Otherwise, sleep and retry
-		klog.Info("Sleeping for ", *d.config.Flags.GFD.SleepInterval)
-		time.Sleep(time.Duration(*d.config.Flags.GFD.SleepInterval))
-		goto rerun
+		return false, err
 	}
 
-	// Update last success time
-	lastSuccessTime = time.Now()
-
 	if *d.config.Flags.GFD.Oneshot {
 		return false, nil
 	}

internal/lm/imex.go

@@ -17,6 +17,7 @@
 package lm
 
 import (
+	"fmt"
 	"strings"
 
 	"k8s.io/klog/v2"
@@ -47,19 +48,16 @@ func getFabricIDs(devices []resource.Device) (string, string, error) {
 	for i, device := range devices {
 		isFabricAttached, err := device.IsFabricAttached()
 		if err != nil {
-			// Skip unhealthy devices when checking fabric attachment
-			klog.V(2).Infof("Skipping device %d for IMEX capability check: %v", i, err)
-			continue
+			return "", "", fmt.Errorf("error checking imex capability: %v", err)
 		}
 		if !isFabricAttached {
 			continue
 		}
 
 		clusterUUID, cliqueID, err := device.GetFabricIDs()
 		if err != nil {
-			// Skip devices that fail fabric ID retrieval
-			klog.V(2).Infof("Skipping device %d, error getting fabric IDs: %v", i, err)
-			continue
+
+			return "", "", fmt.Errorf("error getting fabric IDs: %w", err)
 		}
 
 		uniqueClusterUUIDs[clusterUUID] = append(uniqueClusterUUIDs[clusterUUID], i)

internal/lm/nvml.go

@@ -50,23 +50,6 @@ func NewDeviceLabeler(manager resource.Manager, config *spec.Config) (Labeler, e
 		return empty{}, nil
 	}
 
-	// Filter to only healthy devices - skip those that fail basic health
-	// checks. This prevents GFD from crashing when a device goes unhealthy.
-	totalDevices := len(devices)
-	healthyDevices := resource.FilterHealthyDevices(devices)
-	if len(healthyDevices) < totalDevices {
-		klog.Warningf("GFD: %d/%d devices failed health check, continuing with healthy devices",
-			totalDevices-len(healthyDevices), totalDevices)
-	}
-
-	if len(healthyDevices) == 0 {
-		klog.Warning("GFD: No healthy devices available for labeling")
-		return empty{}, nil
-	}
-
-	// Use healthyDevices for all subsequent labeling
-	devices = healthyDevices
-
 	machineTypeLabeler, err := newMachineTypeLabeler(*config.Flags.GFD.MachineTypeFile)
 	if err != nil {
 		return nil, fmt.Errorf("failed to construct machine type labeler: %v", err)
@@ -177,9 +160,7 @@ func newMigCapabilityLabeler(manager resource.Manager) (Labeler, error) {
 	for _, d := range devices {
 		isMigCapable, err = d.IsMigCapable()
 		if err != nil {
-			// Skip unhealthy devices, continue checking others
-			klog.V(2).Infof("Skipping device for MIG capability check: %v", err)
-			continue
+			return nil, fmt.Errorf("error getting mig capability: %v", err)
 		}
 		if isMigCapable {
 			break
@@ -220,9 +201,7 @@ func isMPSCapable(manager resource.Manager) (bool, error) {
 	for _, d := range devices {
 		isMigEnabled, err := d.IsMigEnabled()
 		if err != nil {
-			// Skip unhealthy devices when checking MPS capability
-			klog.V(2).Infof("Skipping device for MPS capability check: %v", err)
-			continue
+			return false, fmt.Errorf("failed to check if device is MIG-enabled: %w", err)
 		}
 		if isMigEnabled {
 			return false, fmt.Errorf("%w for mig devices", errMPSSharingNotSupported)
@@ -268,12 +247,10 @@ func getModeForClasses(classes []uint32) string {
 
 func getDeviceClasses(devices []resource.Device) ([]uint32, error) {
 	seenClasses := make(map[uint32]bool)
-	for i, d := range devices {
+	for _, d := range devices {
 		class, err := d.GetPCIClass()
 		if err != nil {
-			// Skip unhealthy devices when determining GPU mode
-			klog.V(2).Infof("Skipping device %d for PCI class check: %v", i, err)
-			continue
+			return nil, err
 		}
 		seenClasses[class] = true
 	}

internal/lm/resource.go

@@ -41,17 +41,9 @@ func NewGPUResourceLabeler(config *spec.Config, device resource.Device, count in
 		return empty{}, nil
 	}
 
-	// Check if device is healthy before querying
-	if err := resource.CheckDeviceHealth(device); err != nil {
-		klog.Warningf("Skipping unhealthy device in GPU resource labeler: %v", err)
-		return empty{}, nil
-	}
-
 	model, err := device.GetName()
 	if err != nil {
-		// If health check passed but GetName fails, log and skip
-		klog.Warningf("Device health check passed but GetName failed: %v", err)
-		return empty{}, nil
+		return nil, fmt.Errorf("failed to get device model: %v", err)
 	}
 
 	totalMemoryMiB, err := device.GetTotalMemoryMiB()
@@ -63,9 +55,7 @@ func NewGPUResourceLabeler(config *spec.Config, device resource.Device, count in
 
 	architectureLabels, err := newArchitectureLabels(resourceLabeler, device)
 	if err != nil {
-		// Don't crash on architecture label failure, log and continue
-		klog.Warningf("Failed to create architecture labels: %v", err)
-		architectureLabels = make(Labels)
+		return nil, fmt.Errorf("failed to create architecture labels: %v", err)
 	}
 
 	memoryLabeler := (Labeler)(&empty{})
@@ -88,35 +78,25 @@ func NewMIGResourceLabeler(resourceName spec.ResourceName, config *spec.Config,
 		return empty{}, nil
 	}
 
-	// Check if device is healthy before querying
-	if err := resource.CheckDeviceHealth(device); err != nil {
-		klog.Warningf("Skipping unhealthy MIG device in resource labeler: %v", err)
-		return empty{}, nil
-	}
-
 	parent, err := device.GetDeviceHandleFromMigDeviceHandle()
 	if err != nil {
-		klog.Warningf("Failed to get parent of MIG device, skipping: %v", err)
-		return empty{}, nil
+		return nil, fmt.Errorf("failed to get parent of MIG device: %v", err)
 	}
 	model, err := parent.GetName()
 	if err != nil {
-		klog.Warningf("Failed to get device model, skipping: %v", err)
-		return empty{}, nil
+		return nil, fmt.Errorf("failed to get device model: %v", err)
 	}
 
 	migProfile, err := device.GetName()
 	if err != nil {
-		klog.Warningf("Failed to get MIG profile name, skipping: %v", err)
-		return empty{}, nil
+		return nil, fmt.Errorf("failed to get MIG profile name: %v", err)
 	}
 
 	resourceLabeler := newResourceLabeler(resourceName, config)
 
 	attributeLabels, err := newMigAttributeLabels(resourceLabeler, device)
 	if err != nil {
-		klog.Warningf("Failed to get MIG attribute labels: %v", err)
-		attributeLabels = make(Labels)
+		return nil, fmt.Errorf("faled to get MIG attribute labels: %v", err)
 	}
 
 	labelers := Merge(
@@ -272,9 +252,7 @@ func (rl resourceLabeler) replicationInfo() *spec.ReplicatedResource {
 func newMigAttributeLabels(rl resourceLabeler, device resource.Device) (Labels, error) {
 	attributes, err := device.GetAttributes()
 	if err != nil {
-		// Return empty labels instead of crashing
-		klog.Warningf("Unable to get attributes of MIG device, skipping: %v", err)
-		return make(Labels), nil
+		return nil, fmt.Errorf("unable to get attributes of MIG device: %v", err)
 	}
 
 	labels := rl.labels(attributes)
@@ -285,9 +263,7 @@ func newMigAttributeLabels(rl resourceLabeler, device resource.Device) (Labels,
 func newArchitectureLabels(rl resourceLabeler, device resource.Device) (Labels, error) {
 	computeMajor, computeMinor, err := device.GetCudaComputeCapability()
 	if err != nil {
-		// Return empty labels instead of error - allows labeling to continue
-		klog.Warningf("Failed to determine CUDA compute capability, skipping architecture labels: %v", err)
-		return make(Labels), nil
+		return nil, fmt.Errorf("failed to determine CUDA compute capability: %v", err)
 	}
 
 	if computeMajor == 0 {