add autoallow and entitlements support to CDI devices

Devices can be marked as "automatically allowed" by TOML config
or by the CDI spec of specific file via annotation.

Device that is is not "automatically allowed" needs to be allowed
by the build request by passing entitlement. For example a Dockerfile
may not use a device without use invoking the build permitting it.

--allow device grants access to any device.
--allow device=kind|name grants access to specific device.
--allow device=kind|name,alias=kind|name allows mapping kind to
a specific device or one device to another. Alias is the name requested
by the build and device is the actual device that is being enabled.

Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>

Author: tonistiigi

client/build_test.go

@@ -1838,7 +1838,7 @@ func testClientGatewayContainerSecurityMode(t *testing.T, sb integration.Sandbox
 
 	command := []string{"sh", "-c", `cat /proc/self/status | grep CapEff | cut -f 2`}
 	mode := llb.SecurityModeSandbox
-	var allowedEntitlements []entitlements.Entitlement
+	var allowedEntitlements []string
 	var assertCaps func(caps uint64)
 	secMode := sb.Value("secmode")
 	if secMode == securitySandbox {
@@ -1850,7 +1850,7 @@ func testClientGatewayContainerSecurityMode(t *testing.T, sb integration.Sandbox
 			*/
 			require.Equal(t, uint64(0xa80425fb), caps)
 		}
-		allowedEntitlements = []entitlements.Entitlement{}
+		allowedEntitlements = []string{}
 		if expectFail {
 			return
 		}
@@ -1869,9 +1869,9 @@ func testClientGatewayContainerSecurityMode(t *testing.T, sb integration.Sandbox
 			require.Equal(t, uint64(0x3fffffffff), caps&0x3fffffffff)
 		}
 		mode = llb.SecurityModeInsecure
-		allowedEntitlements = []entitlements.Entitlement{entitlements.EntitlementSecurityInsecure}
+		allowedEntitlements = []string{entitlements.EntitlementSecurityInsecure.String()}
 		if expectFail {
-			allowedEntitlements = []entitlements.Entitlement{}
+			allowedEntitlements = []string{}
 		}
 	}
 
@@ -2046,13 +2046,13 @@ func testClientGatewayContainerHostNetworking(t *testing.T, sb integration.Sandb
 	ctx := sb.Context()
 	product := "buildkit_test"
 
-	var allowedEntitlements []entitlements.Entitlement
+	var allowedEntitlements []string
 	netMode := pb.NetMode_UNSET
 	if sb.Value("netmode") == hostNetwork {
 		netMode = pb.NetMode_HOST
-		allowedEntitlements = []entitlements.Entitlement{entitlements.EntitlementNetworkHost}
+		allowedEntitlements = []string{entitlements.EntitlementNetworkHost.String()}
 		if expectFail {
-			allowedEntitlements = []entitlements.Entitlement{}
+			allowedEntitlements = []string{}
 		}
 	}
 	c, err := New(sb.Context(), sb.Address())

client/client_test.go

@@ -400,9 +400,9 @@ func testHostNetworking(t *testing.T, sb integration.Sandbox) {
 		t.SkipNow()
 	}
 	netMode := sb.Value("netmode")
-	var allowedEntitlements []entitlements.Entitlement
+	var allowedEntitlements []string
 	if netMode == hostNetwork {
-		allowedEntitlements = []entitlements.Entitlement{entitlements.EntitlementNetworkHost}
+		allowedEntitlements = []string{entitlements.EntitlementNetworkHost.String()}
 	}
 	c, err := New(sb.Context(), sb.Address())
 	require.NoError(t, err)
@@ -1063,7 +1063,7 @@ func testSecurityMode(t *testing.T, sb integration.Sandbox) {
 	workers.CheckFeatureCompat(t, sb, workers.FeatureSecurityMode)
 	command := `sh -c 'cat /proc/self/status | grep CapEff | cut -f 2 > /out'`
 	mode := llb.SecurityModeSandbox
-	var allowedEntitlements []entitlements.Entitlement
+	var allowedEntitlements []string
 	var assertCaps func(caps uint64)
 	secMode := sb.Value("secmode")
 	if secMode == securitySandbox {
@@ -1075,7 +1075,7 @@ func testSecurityMode(t *testing.T, sb integration.Sandbox) {
 			*/
 			require.Equal(t, uint64(0xa80425fb), caps)
 		}
-		allowedEntitlements = []entitlements.Entitlement{}
+		allowedEntitlements = []string{}
 	} else {
 		assertCaps = func(caps uint64) {
 			/*
@@ -1091,7 +1091,7 @@ func testSecurityMode(t *testing.T, sb integration.Sandbox) {
 			require.Equal(t, uint64(0x3fffffffff), caps&0x3fffffffff)
 		}
 		mode = llb.SecurityModeInsecure
-		allowedEntitlements = []entitlements.Entitlement{entitlements.EntitlementSecurityInsecure}
+		allowedEntitlements = []string{entitlements.EntitlementSecurityInsecure.String()}
 	}
 
 	c, err := New(sb.Context(), sb.Address())
@@ -1138,13 +1138,13 @@ func testSecurityModeSysfs(t *testing.T, sb integration.Sandbox) {
 	}
 
 	mode := llb.SecurityModeSandbox
-	var allowedEntitlements []entitlements.Entitlement
+	var allowedEntitlements []string
 	secMode := sb.Value("secmode")
 	if secMode == securitySandbox {
-		allowedEntitlements = []entitlements.Entitlement{}
+		allowedEntitlements = []string{}
 	} else {
 		mode = llb.SecurityModeInsecure
-		allowedEntitlements = []entitlements.Entitlement{entitlements.EntitlementSecurityInsecure}
+		allowedEntitlements = []string{entitlements.EntitlementSecurityInsecure.String()}
 	}
 
 	c, err := New(sb.Context(), sb.Address())
@@ -1191,7 +1191,7 @@ func testSecurityModeErrors(t *testing.T, sb integration.Sandbox) {
 		require.NoError(t, err)
 
 		_, err = c.Solve(sb.Context(), def, SolveOpt{
-			AllowedEntitlements: []entitlements.Entitlement{entitlements.EntitlementSecurityInsecure},
+			AllowedEntitlements: []string{entitlements.EntitlementSecurityInsecure.String()},
 		}, nil)
 		require.Error(t, err)
 		require.Contains(t, err.Error(), "security.insecure is not allowed")
@@ -11054,22 +11054,26 @@ func testCDI(t *testing.T, sb integration.Sandbox) {
 	defer c.Close()
 
 	require.NoError(t, os.WriteFile(filepath.Join(sb.CDISpecDir(), "vendor1-device.yaml"), []byte(`
-cdiVersion: "0.3.0"
+cdiVersion: "0.6.0"
 kind: "vendor1.com/device"
 devices:
 - name: foo
   containerEdits:
     env:
     - FOO=injected
+annotations:
+  org.mobyproject.buildkit.device.autoallow: true
 `), 0600))
 	require.NoError(t, os.WriteFile(filepath.Join(sb.CDISpecDir(), "vendor2-device.yaml"), []byte(`
-cdiVersion: "0.3.0"
+cdiVersion: "0.6.0"
 kind: "vendor2.com/device"
 devices:
 - name: bar
   containerEdits:
     env:
     - BAR=injected
+annotations:
+  org.mobyproject.buildkit.device.autoallow: true
 `), 0600))
 
 	busybox := llb.Image("busybox:latest")
@@ -11119,7 +11123,7 @@ func testCDIFirst(t *testing.T, sb integration.Sandbox) {
 	defer c.Close()
 
 	require.NoError(t, os.WriteFile(filepath.Join(sb.CDISpecDir(), "vendor1-device.yaml"), []byte(`
-cdiVersion: "0.3.0"
+cdiVersion: "0.6.0"
 kind: "vendor1.com/device"
 devices:
 - name: foo
@@ -11138,6 +11142,8 @@ devices:
   containerEdits:
     env:
     - QUX=injected
+annotations:
+  org.mobyproject.buildkit.device.autoallow: true
 `), 0600))
 
 	busybox := llb.Image("busybox:latest")
@@ -11184,7 +11190,7 @@ func testCDIWildcard(t *testing.T, sb integration.Sandbox) {
 	defer c.Close()
 
 	require.NoError(t, os.WriteFile(filepath.Join(sb.CDISpecDir(), "vendor1-device.yaml"), []byte(`
-cdiVersion: "0.3.0"
+cdiVersion: "0.6.0"
 kind: "vendor1.com/device"
 devices:
 - name: foo
@@ -11195,6 +11201,8 @@ devices:
   containerEdits:
     env:
     - BAR=injected
+annotations:
+  org.mobyproject.buildkit.device.autoallow: true
 `), 0600))
 
 	busybox := llb.Image("busybox:latest")
@@ -11243,6 +11251,7 @@ cdiVersion: "0.6.0"
 kind: "vendor1.com/device"
 annotations:
   foo.bar.baz: FOO
+  org.mobyproject.buildkit.device.autoallow: true
 devices:
 - name: foo
   annotations:

client/solve.go

@@ -7,6 +7,7 @@ import (
 	"io"
 	"maps"
 	"os"
+	"slices"
 	"strings"
 	"time"
 
@@ -24,7 +25,6 @@ import (
 	"github.com/moby/buildkit/solver/pb"
 	spb "github.com/moby/buildkit/sourcepolicy/pb"
 	"github.com/moby/buildkit/util/bklog"
-	"github.com/moby/buildkit/util/entitlements"
 	ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 	"github.com/tonistiigi/fsutil"
@@ -45,7 +45,7 @@ type SolveOpt struct {
 	CacheExports          []CacheOptionsEntry
 	CacheImports          []CacheOptionsEntry
 	Session               []session.Attachable
-	AllowedEntitlements   []entitlements.Entitlement
+	AllowedEntitlements   []string
 	SharedSession         *session.Session // TODO: refactor to better session syncing
 	SessionPreInitialized bool             // TODO: refactor to better session syncing
 	Internal              bool
@@ -277,7 +277,7 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG
 			FrontendAttrs:           frontendAttrs,
 			FrontendInputs:          frontendInputs,
 			Cache:                   &cacheOpt.options,
-			Entitlements:            entitlementsToPB(opt.AllowedEntitlements),
+			Entitlements:            slices.Clone(opt.AllowedEntitlements),
 			Internal:                opt.Internal,
 			SourcePolicy:            opt.SourcePolicy,
 		})
@@ -553,11 +553,3 @@ func prepareMounts(opt *SolveOpt) (map[string]fsutil.FS, error) {
 	}
 	return mounts, nil
 }
-
-func entitlementsToPB(entitlements []entitlements.Entitlement) []string {
-	clone := make([]string, len(entitlements))
-	for i, e := range entitlements {
-		clone[i] = string(e)
-	}
-	return clone
-}

cmd/buildctl/build.go

@@ -213,8 +213,7 @@ func buildAction(clicontext *cli.Context) error {
 		attachable = append(attachable, secretProvider)
 	}
 
-	allowed, err := build.ParseAllow(clicontext.StringSlice("allow"))
-	if err != nil {
+	if err := build.ValidateAllow(clicontext.StringSlice("allow")); err != nil {
 		return err
 	}
 
@@ -258,7 +257,7 @@ func buildAction(clicontext *cli.Context) error {
 		CacheExports:        cacheExports,
 		CacheImports:        cacheImports,
 		Session:             attachable,
-		AllowedEntitlements: allowed,
+		AllowedEntitlements: clicontext.StringSlice("allow"),
 		SourcePolicy:        srcPol,
 		Ref:                 ref,
 	}

cmd/buildctl/build/allow.go

@@ -4,15 +4,13 @@ import (
 	"github.com/moby/buildkit/util/entitlements"
 )
 
-// ParseAllow parses --allow
-func ParseAllow(inp []string) ([]entitlements.Entitlement, error) {
-	ent := make([]entitlements.Entitlement, 0, len(inp))
+// ValidateAllow parses --allow
+func ValidateAllow(inp []string) error {
 	for _, v := range inp {
-		e, err := entitlements.Parse(v)
+		_, _, err := entitlements.Parse(v)
 		if err != nil {
-			return nil, err
+			return err
 		}
-		ent = append(ent, e)
 	}
-	return ent, nil
+	return nil
 }

cmd/buildctl/debug/workers.go

@@ -94,7 +94,7 @@ func printWorkersVerbose(tw *tabwriter.Writer, winfo []*client.WorkerInfo) {
 
 				for _, k := range sortedKeys(d.Annotations) {
 					v := d.Annotations[k]
-					fmt.Fprintf(tw, "\t\t%s:\t%s\n", k, v)
+					fmt.Fprintf(tw, "\tAnnotations:\t%s:\t%s\n", k, v)
 				}
 			}
 			fmt.Fprint(tw, "\n")

cmd/buildkitd/config/config.go

@@ -77,8 +77,9 @@ type OTELConfig struct {
 }
 
 type CDIConfig struct {
-	Disabled *bool    `toml:"disabled"`
-	SpecDirs []string `toml:"specDirs"`
+	Disabled    *bool    `toml:"disabled"`
+	SpecDirs    []string `toml:"specDirs"`
+	AutoAllowed []string `toml:"autoAllowed"`
 }
 
 type GCConfig struct {

cmd/buildkitd/main.go

@@ -39,6 +39,7 @@ import (
 	"github.com/moby/buildkit/session"
 	"github.com/moby/buildkit/solver"
 	"github.com/moby/buildkit/solver/bboltcachestorage"
+	"github.com/moby/buildkit/solver/llbsolver/cdidevices"
 	"github.com/moby/buildkit/util/apicaps"
 	"github.com/moby/buildkit/util/appcontext"
 	"github.com/moby/buildkit/util/appdefaults"
@@ -846,6 +847,11 @@ func newController(ctx context.Context, c *cli.Context, cfg *config.Config) (*co
 		"s3":       s3remotecache.ResolveCacheImporterFunc(),
 		"azblob":   azblob.ResolveCacheImporterFunc(),
 	}
+
+	if cfg.CDI.Disabled == nil || !*cfg.CDI.Disabled {
+		cfg.Entitlements = append(cfg.Entitlements, "device")
+	}
+
 	return control.NewController(control.Opt{
 		SessionManager:            sessionManager,
 		WorkerController:          wc,
@@ -1047,16 +1053,16 @@ func newMeterProvider(ctx context.Context) (*sdkmetric.MeterProvider, error) {
 }
 
 // getCDIManager returns a new CDI registry with disabled auto-refresh.
-func getCDIManager(disabled *bool, specDirs []string) (*cdi.Cache, error) {
-	if disabled != nil && *disabled {
+func getCDIManager(cfg config.CDIConfig) (*cdidevices.Manager, error) {
+	if cfg.Disabled != nil && *cfg.Disabled {
 		return nil, nil
 	}
-	if len(specDirs) == 0 {
-		return nil, errors.New("No CDI specification directories specified")
+	if len(cfg.SpecDirs) == 0 {
+		return nil, errors.New("no CDI specification directories specified")
 	}
 	cdiCache, err := func() (*cdi.Cache, error) {
 		cdiCache, err := cdi.NewCache(
-			cdi.WithSpecDirs(specDirs...),
+			cdi.WithSpecDirs(cfg.SpecDirs...),
 			cdi.WithAutoRefresh(false),
 		)
 		if err != nil {
@@ -1070,5 +1076,5 @@ func getCDIManager(disabled *bool, specDirs []string) (*cdi.Cache, error) {
 	if err != nil {
 		return nil, errors.Wrapf(err, "CDI registry initialization failure")
 	}
-	return cdiCache, nil
+	return cdidevices.NewManager(cdiCache, cfg.AutoAllowed), nil
 }

cmd/buildkitd/main_containerd_worker.go

@@ -12,7 +12,6 @@ import (
 	ctd "github.com/containerd/containerd/v2/client"
 	"github.com/containerd/containerd/v2/defaults"
 	"github.com/moby/buildkit/cmd/buildkitd/config"
-	"github.com/moby/buildkit/solver/llbsolver/cdidevices"
 	"github.com/moby/buildkit/util/bklog"
 	"github.com/moby/buildkit/util/disk"
 	"github.com/moby/buildkit/util/network/cniprovider"
@@ -283,7 +282,7 @@ func containerdWorkerInitializer(c *cli.Context, common workerInitializerOpt) ([
 
 	dns := getDNSConfig(common.config.DNS)
 
-	cdiManager, err := getCDIManager(common.config.CDI.Disabled, common.config.CDI.SpecDirs)
+	cdiManager, err := getCDIManager(common.config.CDI)
 	if err != nil {
 		return nil, err
 	}
@@ -345,7 +344,7 @@ func containerdWorkerInitializer(c *cli.Context, common workerInitializerOpt) ([
 		ParallelismSem:  parallelismSem,
 		TraceSocket:     common.traceSocket,
 		Runtime:         runtime,
-		CDIManager:      cdidevices.NewManager(cdiManager),
+		CDIManager:      cdiManager,
 	}
 
 	opt, err := containerd.NewWorkerOpt(workerOpts, ctd.WithTimeout(60*time.Second))

cmd/buildkitd/main_oci_worker.go

@@ -298,7 +298,7 @@ func ociWorkerInitializer(c *cli.Context, common workerInitializerOpt) ([]worker
 
 	dns := getDNSConfig(common.config.DNS)
 
-	cdiManager, err := getCDIManager(common.config.CDI.Disabled, common.config.CDI.SpecDirs)
+	cdiManager, err := getCDIManager(common.config.CDI)
 	if err != nil {
 		return nil, err
 	}