Merge pull request moby#4101 from crazy-max/fix-rootless-tests

integration: fix rootless tests

Author: tonistiigi

cmd/buildkitd/config/config.go

@@ -17,6 +17,8 @@ type Config struct {
 	// GRPC configuration settings
 	GRPC GRPCConfig `toml:"grpc"`
 
+	OTEL OTELConfig `toml:"otel"`
+
 	Workers struct {
 		OCI        OCIConfig        `toml:"oci"`
 		Containerd ContainerdConfig `toml:"containerd"`
@@ -46,6 +48,10 @@ type TLSConfig struct {
 	CA   string `toml:"ca"`
 }
 
+type OTELConfig struct {
+	SocketPath string `toml:"socketPath"`
+}
+
 type GCConfig struct {
 	GC            *bool      `toml:"gc"`
 	GCKeepStorage DiskSpace  `toml:"gckeepstorage"`

cmd/buildkitd/config/load_test.go

@@ -18,14 +18,16 @@ insecure-entitlements = ["security.insecure"]
 [gc]
 enabled=true
 
-
 [grpc]
 address=["buildkit.sock"]
 debugAddress="debug.sock"
 gid=1234
 [grpc.tls]
 cert="mycert.pem"
 
+[otel]
+socketPath="/tmp/otel-grpc.sock"
+
 [worker.oci]
 enabled=true
 snapshotter="overlay"
@@ -83,6 +85,8 @@ searchDomains=["example.com"]
 	require.Equal(t, 1234, *cfg.GRPC.GID)
 	require.Equal(t, "mycert.pem", cfg.GRPC.TLS.Cert)
 
+	require.Equal(t, "/tmp/otel-grpc.sock", cfg.OTEL.SocketPath)
+
 	require.NotNil(t, cfg.Workers.OCI.Enabled)
 	require.Equal(t, int64(123456789), cfg.Workers.OCI.GCKeepStorage.Bytes)
 	require.Equal(t, true, *cfg.Workers.OCI.Enabled)

cmd/buildkitd/main.go

@@ -200,6 +200,10 @@ func main() {
 			Name:  "allow-insecure-entitlement",
 			Usage: "allows insecure entitlements e.g. network.host, security.insecure",
 		},
+		cli.StringFlag{
+			Name:  "otel-socket-path",
+			Usage: "OTEL collector trace socket path",
+		},
 	)
 	app.Flags = append(app.Flags, appFlags...)
 	app.Flags = append(app.Flags, serviceFlags()...)
@@ -458,6 +462,10 @@ func setDefaultConfig(cfg *config.Config) {
 			appdefaults.EnsureUserAddressDir()
 		}
 	}
+
+	if cfg.OTEL.SocketPath == "" {
+		cfg.OTEL.SocketPath = appdefaults.TraceSocketPath(userns.RunningInUserNS())
+	}
 }
 
 func applyMainFlags(c *cli.Context, cfg *config.Config) error {
@@ -511,6 +519,11 @@ func applyMainFlags(c *cli.Context, cfg *config.Config) error {
 	if tlsca := c.String("tlscacert"); tlsca != "" {
 		cfg.GRPC.TLS.CA = tlsca
 	}
+
+	if c.IsSet("otel-socket-path") {
+		cfg.OTEL.SocketPath = c.String("otel-socket-path")
+	}
+
 	applyPlatformFlags(c)
 
 	return nil
@@ -661,11 +674,7 @@ func newController(c *cli.Context, cfg *config.Config) (*control.Controller, err
 
 	var traceSocket string
 	if tc != nil {
-		if v, ok := os.LookupEnv("BUILDKIT_TRACE_SOCKET"); ok {
-			traceSocket = v
-		} else {
-			traceSocket = appdefaults.TraceSocketPath(userns.RunningInUserNS())
-		}
+		traceSocket = cfg.OTEL.SocketPath
 		if err := runTraceController(traceSocket, tc); err != nil {
 			return nil, err
 		}

docs/buildkitd.toml.md

@@ -29,6 +29,10 @@ insecure-entitlements = [ "network.host", "security.insecure" ]
     key = "/etc/buildkit/tls.key"
     ca = "/etc/buildkit/tlsca.crt"
 
+[otel]
+  # OTEL collector trace socket path
+  socketPath = "/run/buildkit/otel-grpc.sock"
+
 # config for build history API that stores information about completed build commands
 [history]
   # maxAge is the maximum age of history entries to keep, in seconds.

util/testutil/integration/sandbox.go

@@ -126,7 +126,7 @@ func newSandbox(ctx context.Context, w Worker, mirror string, mv matrixValue) (s
 }
 
 func RootlessSupported(uid int) bool {
-	cmd := exec.Command("sudo", "-E", "-u", fmt.Sprintf("#%d", uid), "-i", "--", "exec", "unshare", "-U", "true") //nolint:gosec // test utility
+	cmd := exec.Command("sudo", "-u", fmt.Sprintf("#%d", uid), "-i", "--", "exec", "unshare", "-U", "true") //nolint:gosec // test utility
 	b, err := cmd.CombinedOutput()
 	if err != nil {
 		bklog.L.Warnf("rootless mode is not supported on this host: %v (%s)", err, string(b))

util/testutil/workers/containerd.go

@@ -168,7 +168,7 @@ disabled_plugins = ["cri"]
 	containerdArgs := []string{c.Containerd, "--config", configFile}
 	rootlessKitState := filepath.Join(tmpdir, "rootlesskit-containerd")
 	if rootless {
-		containerdArgs = append(append([]string{"sudo", "-E", "-u", fmt.Sprintf("#%d", c.UID), "-i",
+		containerdArgs = append(append([]string{"sudo", "-u", fmt.Sprintf("#%d", c.UID), "-i",
 			fmt.Sprintf("CONTAINERD_ROOTLESS_ROOTLESSKIT_STATE_DIR=%s", rootlessKitState),
 			// Integration test requires the access to localhost of the host network namespace.
 			// TODO: remove these configurations
@@ -211,7 +211,7 @@ disabled_plugins = ["cri"]
 		if err != nil {
 			return nil, nil, err
 		}
-		buildkitdArgs = append([]string{"sudo", "-E", "-u", fmt.Sprintf("#%d", c.UID), "-i", "--", "exec",
+		buildkitdArgs = append([]string{"sudo", "-u", fmt.Sprintf("#%d", c.UID), "-i", "--", "exec",
 			"nsenter", "-U", "--preserve-credentials", "-m", "-t", fmt.Sprintf("%d", pid)},
 			append(buildkitdArgs, "--containerd-worker-snapshotter=native")...)
 	}

util/testutil/workers/oci.go

@@ -66,7 +66,7 @@ func (s *OCI) New(ctx context.Context, cfg *integration.BackendConfig) (integrat
 			return nil, nil, errors.Errorf("unsupported id pair: uid=%d, gid=%d", s.UID, s.GID)
 		}
 		// TODO: make sure the user exists and subuid/subgid are configured.
-		buildkitdArgs = append([]string{"sudo", "-E", "-u", fmt.Sprintf("#%d", s.UID), "-i", "--", "exec", "rootlesskit"}, buildkitdArgs...)
+		buildkitdArgs = append([]string{"sudo", "-u", fmt.Sprintf("#%d", s.UID), "-i", "--", "exec", "rootlesskit"}, buildkitdArgs...)
 	}
 
 	var extraEnv []string

util/testutil/workers/util.go

@@ -54,9 +54,9 @@ func runBuildkitd(ctx context.Context, conf *integration.BackendConfig, args []s
 
 	address = getBuildkitdAddr(tmpdir)
 
-	args = append(args, "--root", tmpdir, "--addr", address, "--debug")
+	args = append(args, "--root", tmpdir, "--addr", address, "--otel-socket-path", getTraceSocketPath(tmpdir), "--debug")
 	cmd := exec.Command(args[0], args[1:]...) //nolint:gosec // test utility
-	cmd.Env = append(os.Environ(), "BUILDKIT_DEBUG_EXEC_OUTPUT=1", "BUILDKIT_DEBUG_PANIC_ON_ERROR=1", "BUILDKIT_TRACE_SOCKET="+getTraceSocketPath(tmpdir), "TMPDIR="+filepath.Join(tmpdir, "tmp"))
+	cmd.Env = append(os.Environ(), "BUILDKIT_DEBUG_EXEC_OUTPUT=1", "BUILDKIT_DEBUG_PANIC_ON_ERROR=1", "TMPDIR="+filepath.Join(tmpdir, "tmp"))
 	cmd.Env = append(cmd.Env, extraEnv...)
 	cmd.SysProcAttr = getSysProcAttr()