Merge pull request moby#4078 from crazy-max/fix-otel-sock

set tracing socket path to runtime dir

Author: tonistiigi

cmd/buildkitd/main.go

@@ -661,7 +661,11 @@ func newController(c *cli.Context, cfg *config.Config) (*control.Controller, err
 
 	var traceSocket string
 	if tc != nil {
-		traceSocket = traceSocketPath(cfg.Root)
+		if v, ok := os.LookupEnv("BUILDKIT_TRACE_SOCKET"); ok {
+			traceSocket = v
+		} else {
+			traceSocket = appdefaults.TraceSocketPath(userns.RunningInUserNS())
+		}
 		if err := runTraceController(traceSocket, tc); err != nil {
 			return nil, err
 		}

cmd/buildkitd/main_unix.go

@@ -7,7 +7,6 @@ import (
 	"crypto/tls"
 	"net"
 	"os"
-	"path/filepath"
 	"syscall"
 
 	"github.com/containerd/containerd/sys"
@@ -47,10 +46,6 @@ func listenFD(addr string, tlsConfig *tls.Config) (net.Listener, error) {
 	return nil, errors.New("not supported yet")
 }
 
-func traceSocketPath(root string) string {
-	return filepath.Join(root, "otel-grpc.sock")
-}
-
 func getLocalListener(listenerPath string) (net.Listener, error) {
 	uid := os.Getuid()
 	l, err := sys.GetLocalListener(listenerPath, uid, uid)

cmd/buildkitd/main_windows.go

@@ -13,18 +13,10 @@ import (
 	"github.com/pkg/errors"
 )
 
-const (
-	defaultTraceSocketPath = `\\.\pipe\buildkit-otel-grpc`
-)
-
 func listenFD(addr string, tlsConfig *tls.Config) (net.Listener, error) {
 	return nil, errors.New("listening server on fd not supported on windows")
 }
 
-func traceSocketPath(root string) string {
-	return defaultTraceSocketPath
-}
-
 func getLocalListener(listenerPath string) (net.Listener, error) {
 	pc := &winio.PipeConfig{
 		// Allow generic read and generic write access to authenticated users

util/appdefaults/appdefaults_unix.go

@@ -70,3 +70,13 @@ func UserConfigDir() string {
 	}
 	return ConfigDir
 }
+
+func TraceSocketPath(inUserNS bool) string {
+	if inUserNS {
+		if xrd := os.Getenv("XDG_RUNTIME_DIR"); xrd != "" {
+			dirs := strings.Split(xrd, ":")
+			return filepath.Join(dirs[0], "buildkit", "otel-grpc.sock")
+		}
+	}
+	return "/run/buildkit/otel-grpc.sock"
+}

util/appdefaults/appdefaults_windows.go

@@ -31,3 +31,7 @@ func UserRoot() string {
 func UserConfigDir() string {
 	return ConfigDir
 }
+
+func TraceSocketPath(inUserNS bool) string {
+	return `\\.\pipe\buildkit-otel-grpc`
+}

util/testutil/integration/containerd.go

@@ -168,7 +168,7 @@ disabled_plugins = ["cri"]
 	containerdArgs := []string{c.Containerd, "--config", configFile}
 	rootlessKitState := filepath.Join(tmpdir, "rootlesskit-containerd")
 	if rootless {
-		containerdArgs = append(append([]string{"sudo", "-u", fmt.Sprintf("#%d", c.UID), "-i",
+		containerdArgs = append(append([]string{"sudo", "-E", "-u", fmt.Sprintf("#%d", c.UID), "-i",
 			fmt.Sprintf("CONTAINERD_ROOTLESS_ROOTLESSKIT_STATE_DIR=%s", rootlessKitState),
 			// Integration test requires the access to localhost of the host network namespace.
 			// TODO: remove these configurations
@@ -211,7 +211,7 @@ disabled_plugins = ["cri"]
 		if err != nil {
 			return nil, nil, err
 		}
-		buildkitdArgs = append([]string{"sudo", "-u", fmt.Sprintf("#%d", c.UID), "-i", "--", "exec",
+		buildkitdArgs = append([]string{"sudo", "-E", "-u", fmt.Sprintf("#%d", c.UID), "-i", "--", "exec",
 			"nsenter", "-U", "--preserve-credentials", "-m", "-t", fmt.Sprintf("%d", pid)},
 			append(buildkitdArgs, "--containerd-worker-snapshotter=native")...)
 	}

util/testutil/integration/oci.go

@@ -65,7 +65,7 @@ func (s *OCI) New(ctx context.Context, cfg *BackendConfig) (Backend, func() erro
 			return nil, nil, errors.Errorf("unsupported id pair: uid=%d, gid=%d", s.UID, s.GID)
 		}
 		// TODO: make sure the user exists and subuid/subgid are configured.
-		buildkitdArgs = append([]string{"sudo", "-u", fmt.Sprintf("#%d", s.UID), "-i", "--", "exec", "rootlesskit"}, buildkitdArgs...)
+		buildkitdArgs = append([]string{"sudo", "-E", "-u", fmt.Sprintf("#%d", s.UID), "-i", "--", "exec", "rootlesskit"}, buildkitdArgs...)
 	}
 
 	var extraEnv []string

util/testutil/integration/sandbox.go

@@ -8,7 +8,6 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
-	"runtime"
 	"strings"
 	"testing"
 	"time"
@@ -181,14 +180,6 @@ func newSandbox(ctx context.Context, w Worker, mirror string, mv matrixValue) (s
 	}, cl, nil
 }
 
-func getBuildkitdAddr(tmpdir string) string {
-	address := "unix://" + filepath.Join(tmpdir, "buildkitd.sock")
-	if runtime.GOOS == "windows" {
-		address = "//./pipe/buildkitd-" + filepath.Base(tmpdir)
-	}
-	return address
-}
-
 func runBuildkitd(ctx context.Context, conf *BackendConfig, args []string, logs map[string]*bytes.Buffer, uid, gid int, extraEnv []string) (address string, cl func() error, err error) {
 	deferF := &multiCloser{}
 	cl = deferF.F()
@@ -224,7 +215,7 @@ func runBuildkitd(ctx context.Context, conf *BackendConfig, args []string, logs
 
 	args = append(args, "--root", tmpdir, "--addr", address, "--debug")
 	cmd := exec.Command(args[0], args[1:]...) //nolint:gosec // test utility
-	cmd.Env = append(os.Environ(), "BUILDKIT_DEBUG_EXEC_OUTPUT=1", "BUILDKIT_DEBUG_PANIC_ON_ERROR=1", "TMPDIR="+filepath.Join(tmpdir, "tmp"))
+	cmd.Env = append(os.Environ(), "BUILDKIT_DEBUG_EXEC_OUTPUT=1", "BUILDKIT_DEBUG_PANIC_ON_ERROR=1", "BUILDKIT_TRACE_SOCKET="+getTraceSocketPath(tmpdir), "TMPDIR="+filepath.Join(tmpdir, "tmp"))
 	cmd.Env = append(cmd.Env, extraEnv...)
 	cmd.SysProcAttr = getSysProcAttr()
 
@@ -269,7 +260,7 @@ func getBackend(sb Sandbox) (*backend, error) {
 }
 
 func rootlessSupported(uid int) bool {
-	cmd := exec.Command("sudo", "-u", fmt.Sprintf("#%d", uid), "-i", "--", "exec", "unshare", "-U", "true") //nolint:gosec // test utility
+	cmd := exec.Command("sudo", "-E", "-u", fmt.Sprintf("#%d", uid), "-i", "--", "exec", "unshare", "-U", "true") //nolint:gosec // test utility
 	b, err := cmd.CombinedOutput()
 	if err != nil {
 		bklog.L.Warnf("rootless mode is not supported on this host: %v (%s)", err, string(b))

util/testutil/integration/sandbox_unix.go

@@ -3,10 +3,21 @@
 
 package integration
 
-import "syscall"
+import (
+	"path/filepath"
+	"syscall"
+)
 
 func getSysProcAttr() *syscall.SysProcAttr {
 	return &syscall.SysProcAttr{
 		Setsid: true, // stretch sudo needs this for sigterm
 	}
 }
+
+func getBuildkitdAddr(tmpdir string) string {
+	return "unix://" + filepath.Join(tmpdir, "buildkitd.sock")
+}
+
+func getTraceSocketPath(tmpdir string) string {
+	return filepath.Join(tmpdir, "otel-grpc.sock")
+}

util/testutil/integration/sandbox_windows.go

@@ -3,8 +3,19 @@
 
 package integration
 
-import "syscall"
+import (
+	"path/filepath"
+	"syscall"
+)
 
 func getSysProcAttr() *syscall.SysProcAttr {
 	return &syscall.SysProcAttr{}
 }
+
+func getBuildkitdAddr(tmpdir string) string {
+	return "//./pipe/buildkitd-" + filepath.Base(tmpdir)
+}
+
+func getTraceSocketPath(tmpdir string) string {
+	return `\\.\pipe\buildkit-otel-grpc-` + filepath.Base(tmpdir)
+}