Merge pull request #1303 from elezar/drop-in-in-container

Add support for drop-in config files in a container

Author: elezar

cmd/nvidia-ctk-installer/container/container.go

@@ -35,7 +35,8 @@ const (
 
 // Options defines the shared options for the CLIs to configure containers runtimes.
 type Options struct {
-	DropInConfig string
+	DropInConfig         string
+	DropInConfigHostPath string
 	// TopLevelConfigPath stores the path to the top-level config for the runtime.
 	TopLevelConfigPath string
 	Socket             string

cmd/nvidia-ctk-installer/container/runtime/containerd/config_test.go

@@ -590,6 +590,84 @@ version = 2
 
 				expectedDropIn := `version = 2
 
+[plugins]
+
+  [plugins."io.containerd.grpc.v1.cri"]
+
+    [plugins."io.containerd.grpc.v1.cri".containerd]
+
+      [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
+
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia]
+          privileged_without_host_devices = false
+          runtime_engine = ""
+          runtime_root = ""
+          runtime_type = "io.containerd.runc.v2"
+
+          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia.options]
+            BinaryName = "/usr/bin/nvidia-container-runtime"
+
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-cdi]
+          privileged_without_host_devices = false
+          runtime_engine = ""
+          runtime_root = ""
+          runtime_type = "io.containerd.runc.v2"
+
+          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-cdi.options]
+            BinaryName = "/usr/bin/nvidia-container-runtime.cdi"
+
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-legacy]
+          privileged_without_host_devices = false
+          runtime_engine = ""
+          runtime_root = ""
+          runtime_type = "io.containerd.runc.v2"
+
+          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-legacy.options]
+            BinaryName = "/usr/bin/nvidia-container-runtime.legacy"
+`
+				require.Equal(t, expectedDropIn, string(actualDropIn))
+				return nil
+			},
+			assertCleanupPostConditions: func(t *testing.T, co *container.Options, o *Options) error {
+				require.NoFileExists(t, co.TopLevelConfigPath)
+				require.NoFileExists(t, co.DropInConfig)
+				return nil
+			},
+		},
+		{
+			description: "v2: top-level config does not exist with drop-in-config-host-path",
+			containerOptions: container.Options{
+				TopLevelConfigPath:   "{{ .testRoot }}/etc/containerd/config.toml",
+				DropInConfig:         "{{ .testRoot }}/conf.d/99-nvidia.toml",
+				DropInConfigHostPath: "/some/host/path/conf.d/99-nvidia.toml",
+				RuntimeName:          "nvidia",
+				RuntimeDir:           "/usr/bin",
+				SetAsDefault:         false,
+				RestartMode:          "none",
+				ExecutablePath:       "not-containerd",
+			},
+			options: Options{
+				runtimeType: "io.containerd.runc.v2",
+			},
+			assertSetupPostConditions: func(t *testing.T, co *container.Options, o *Options) error {
+				require.FileExists(t, co.TopLevelConfigPath)
+
+				actual, err := os.ReadFile(co.TopLevelConfigPath)
+				require.NoError(t, err)
+
+				expected := `imports = ["/some/host/path/conf.d/*.toml"]
+version = 2
+`
+				require.Equal(t, expected, string(actual))
+
+				require.NoFileExists(t, co.DropInConfigHostPath)
+				require.FileExists(t, co.DropInConfig)
+
+				actualDropIn, err := os.ReadFile(co.DropInConfig)
+				require.NoError(t, err)
+
+				expectedDropIn := `version = 2
+
 [plugins]
 
   [plugins."io.containerd.grpc.v1.cri"]

cmd/nvidia-ctk-installer/container/runtime/containerd/containerd.go

@@ -19,6 +19,7 @@ package containerd
 import (
 	"encoding/json"
 	"fmt"
+	"path/filepath"
 
 	log "github.com/sirupsen/logrus"
 	cli "github.com/urfave/cli/v3"
@@ -171,7 +172,7 @@ func GetLowlevelRuntimePaths(o *container.Options, co *Options) ([]string, error
 }
 
 func getRuntimeConfig(o *container.Options, co *Options) (engine.Interface, error) {
-	return containerd.New(
+	options := []containerd.Option{
 		containerd.WithTopLevelConfigPath(o.TopLevelConfigPath),
 		containerd.WithConfigSource(
 			toml.LoadFirst(
@@ -182,5 +183,12 @@ func getRuntimeConfig(o *container.Options, co *Options) (engine.Interface, erro
 		containerd.WithRuntimeType(co.runtimeType),
 		containerd.WithUseLegacyConfig(co.useLegacyConfig),
 		containerd.WithContainerAnnotations(co.containerAnnotationsFromCDIPrefixes()...),
-	)
+	}
+	if o.DropInConfigHostPath != "" && o.DropInConfig != "" {
+		options = append(options,
+			containerd.WithContainerPathAsHostPath(filepath.Dir(o.DropInConfig), filepath.Dir(o.DropInConfigHostPath)),
+		)
+	}
+
+	return containerd.New(options...)
 }

cmd/nvidia-ctk-installer/container/runtime/runtime.go

@@ -61,6 +61,13 @@ func Flags(opts *Options) []cli.Flag {
 			Destination: &opts.DropInConfig,
 			Sources:     cli.EnvVars("RUNTIME_DROP_IN_CONFIG"),
 		},
+		&cli.StringFlag{
+			Name: "drop-in-config-host-path",
+			Usage: "When running in a container, this is the path to the drop-in config (--drop-in-config) on the host. " +
+				"This is required to correctly update the top-level config if required since paths there must be host-relative.",
+			Destination: &opts.DropInConfigHostPath,
+			Sources:     cli.EnvVars("RUNTIME_DROP_IN_CONFIG_HOST_PATH"),
+		},
 		&cli.StringFlag{
 			Name:        "executable-path",
 			Usage:       "The path to the runtime executable. This is used to extract the current config",

pkg/config/engine/containerd/config_drop_in.go

@@ -37,15 +37,17 @@ var _ engine.Interface = (*ConfigWithDropIn)(nil)
 // A topLevelConfig stores the original on-disk top-level config.
 // The path to the config is also stored to allow it to be modified if required.
 type topLevelConfig struct {
-	path   string
-	config *Config
+	path                   string
+	containerToHostPathMap map[string]string
+	config                 *Config
 }
 
-func NewConfigWithDropIn(topLevelConfigPath string, tlConfig *Config, dropInConfig engine.Interface) *ConfigWithDropIn {
+func NewConfigWithDropIn(topLevelConfigPath string, containerToHostPathMap map[string]string, tlConfig *Config, dropInConfig engine.Interface) *ConfigWithDropIn {
 	return &ConfigWithDropIn{
 		topLevelConfig: &topLevelConfig{
-			path:   topLevelConfigPath,
-			config: tlConfig,
+			path:                   topLevelConfigPath,
+			containerToHostPathMap: containerToHostPathMap,
+			config:                 tlConfig,
 		},
 		Interface: dropInConfig,
 	}
@@ -117,13 +119,27 @@ func (c *topLevelConfig) removeImports(dropInFilename string) {
 		return
 	}
 
-	requiredImport := filepath.Dir(dropInFilename) + "/*.toml"
+	requiredImport := c.importPattern(dropInFilename)
 	if currentImports[0] != requiredImport {
 		return
 	}
 	c.config.Delete("imports")
 }
 
+func (c *topLevelConfig) importPattern(dropInFilename string) string {
+	return c.asHostPath(filepath.Dir(dropInFilename)) + "/*.toml"
+}
+
+func (c *topLevelConfig) asHostPath(path string) string {
+	if c.containerToHostPathMap == nil {
+		return path
+	}
+	if hostPath, ok := c.containerToHostPathMap[path]; ok {
+		return hostPath
+	}
+	return path
+}
+
 // removeVersion removes the version if it is the ONLY field in the file.
 func (c *topLevelConfig) removeVersion() {
 	if len(c.config.Keys()) > 1 {
@@ -142,7 +158,7 @@ func (c *topLevelConfig) ensureImports(dropInFilename string) {
 		currentImports = ci
 	}
 
-	requiredImport := filepath.Dir(dropInFilename) + "/*.toml"
+	requiredImport := c.importPattern(dropInFilename)
 	for _, currentImport := range currentImports {
 		// If the requiredImport is already present, then we need not update the config.
 		if currentImport == requiredImport {

pkg/config/engine/containerd/containerd.go

@@ -141,7 +141,7 @@ func New(opts ...Option) (engine.Interface, error) {
 			},
 		}
 
-		cfg := NewConfigWithDropIn(b.topLevelConfigPath, topLevelConfig, dropInConfig)
+		cfg := NewConfigWithDropIn(b.topLevelConfigPath, b.containerToHostPathMap, topLevelConfig, dropInConfig)
 		return cfg, nil
 	}
 }

pkg/config/engine/containerd/option.go

@@ -29,11 +29,26 @@ type builder struct {
 	topLevelConfigPath   string
 	runtimeType          string
 	containerAnnotations []string
+
+	containerToHostPathMap map[string]string
 }
 
 // Option defines a function that can be used to configure the config builder
 type Option func(*builder)
 
+// WithContainerPathAsHostPath maps a given container path to a host path.
+func WithContainerPathAsHostPath(containerPath string, hostPath string) Option {
+	return func(b *builder) {
+		if containerPath == "" || hostPath == "" || containerPath == hostPath {
+			return
+		}
+		if b.containerToHostPathMap == nil {
+			b.containerToHostPathMap = make(map[string]string)
+		}
+		b.containerToHostPathMap[containerPath] = hostPath
+	}
+}
+
 // WithLogger sets the logger for the config builder
 func WithLogger(logger logger.Interface) Option {
 	return func(b *builder) {