ioctl_sniffer: use seccomp_unotify to trap ioctl-s

PiperOrigin-RevId: 806465209

Author: avagin

pkg/abi/linux/ptrace_amd64.go

@@ -67,3 +67,7 @@ func (p *PtraceRegs) StackPointer() uint64 {
 func (p *PtraceRegs) SetStackPointer(sp uint64) {
 	p.Rsp = sp
 }
+
+func (p *PtraceRegs) SyscallRet() uint64 {
+	return p.Rax
+}

pkg/abi/linux/ptrace_arm64.go

@@ -75,3 +75,7 @@ func (p *PtraceRegs) StackPointer() uint64 {
 func (p *PtraceRegs) SetStackPointer(sp uint64) {
 	p.Sp = sp
 }
+
+func (p *PtraceRegs) SyscallRet() uint64 {
+	return p.Regs[0]
+}

pkg/abi/linux/seccomp.go

@@ -31,6 +31,7 @@ const (
 
 	SECCOMP_FILTER_FLAG_TSYNC        = 1
 	SECCOMP_FILTER_FLAG_NEW_LISTENER = 1 << 3
+	SECCOMP_FILTER_FLAG_TSYNC_ESRCH  = 1 << 4
 
 	SECCOMP_USER_NOTIF_FLAG_CONTINUE = 1
 

pkg/seccomp/BUILD

@@ -20,6 +20,7 @@ go_library(
     deps = [
         "//pkg/abi/linux",
         "//pkg/bpf",
+        "//pkg/hostsyscall",
         "//pkg/log",
         "@org_golang_x_sys//unix:go_default_library",
     ],

pkg/seccomp/seccomp.go

@@ -82,8 +82,14 @@ func Install(rules SyscallRules, denyRules SyscallRules, options ProgramOptions)
 	}
 
 	// Perform the actual installation.
-	if err := SetFilter(instrs); err != nil {
-		return fmt.Errorf("failed to set filter: %v", err)
+	if options.LogNotifications {
+		if err := SetFilterAndLogNotifications(instrs, options); err != nil {
+			return fmt.Errorf("failed to set filter: %v", err)
+		}
+	} else {
+		if err := SetFilter(instrs); err != nil {
+			return fmt.Errorf("failed to set filter: %v", err)
+		}
 	}
 
 	log.Infof("Seccomp filters installed.")
@@ -321,6 +327,17 @@ type ProgramOptions struct {
 	// called >10% of the time out of all syscalls made).
 	// It is ordered from most frequent to least frequent.
 	HotSyscalls []uintptr
+
+	// LogNotifications enables logging of user notifications at the
+	// warning level. Syscalls triggered notifications are not blocked.
+	LogNotifications bool
+
+	// NotificationCallback is called when a blocked syscall is triggered.
+	NotificationCallback NotificationCallback
+
+	// NotifyFDNum is a target number for the seccomp notify descriptor.
+	// It can be used in filters to allow ioctl-s on this file descriptor.
+	NotifyFDNum int
 }
 
 // DefaultProgramOptions returns the default program options.
@@ -333,6 +350,7 @@ func DefaultProgramOptions() ProgramOptions {
 		DefaultAction: action,
 		BadArchAction: action,
 		Optimize:      true,
+		NotifyFDNum:   -1,
 	}
 }
 

pkg/seccomp/seccomp_unsafe.go

@@ -16,14 +16,158 @@ package seccomp
 
 import (
 	"fmt"
+	"os"
 	"runtime"
+	"syscall"
+	"time"
 	"unsafe"
 
 	"golang.org/x/sys/unix"
 	"gvisor.dev/gvisor/pkg/abi/linux"
 	"gvisor.dev/gvisor/pkg/bpf"
+	"gvisor.dev/gvisor/pkg/hostsyscall"
+	"gvisor.dev/gvisor/pkg/log"
 )
 
+// NotificationCallback is a callback which is called when a blocked syscall is triggered.
+type NotificationCallback func(f *os.File, req linux.SeccompNotif, ret int)
+
+// SetFilterAndLogNotifications installs the given BPF program and logs user
+// notifications triggered by the seccomp filter. It allows the triggering
+// syscalls to proceed without being blocked.
+//
+// This function is intended for debugging seccomp filter violations and should
+// not be used in production environments.
+//
+// Note: It spawns a background goroutine to monitor a seccomp file descriptor
+// and log any received notifications.
+func SetFilterAndLogNotifications(
+	instrs []bpf.Instruction,
+	options ProgramOptions,
+) error {
+	// PR_SET_NO_NEW_PRIVS is required in order to enable seccomp. See
+	// seccomp(2) for details.
+	//
+	// PR_SET_NO_NEW_PRIVS is specific to the calling thread, not the whole
+	// thread group, so between PR_SET_NO_NEW_PRIVS and seccomp() below we must
+	// remain on the same thread. no_new_privs will be propagated to other
+	// threads in the thread group by seccomp(SECCOMP_FILTER_FLAG_TSYNC), in
+	// kernel/seccomp.c:seccomp_sync_threads().
+	runtime.LockOSThread()
+	defer runtime.UnlockOSThread()
+	if _, _, errno := unix.RawSyscall6(unix.SYS_PRCTL, linux.PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0, 0); errno != 0 {
+		return errno
+	}
+
+	sockProg := linux.SockFprog{
+		Len:    uint16(len(instrs)),
+		Filter: (*linux.BPFInstruction)(unsafe.Pointer(&instrs[0])),
+	}
+	flags := linux.SECCOMP_FILTER_FLAG_TSYNC |
+		linux.SECCOMP_FILTER_FLAG_NEW_LISTENER |
+		linux.SECCOMP_FILTER_FLAG_TSYNC_ESRCH | (1 << 5)
+	fd, errno := seccomp(linux.SECCOMP_SET_MODE_FILTER, uint32(flags), unsafe.Pointer(&sockProg))
+	if errno != 0 {
+		return errno
+	}
+	if options.NotifyFDNum > 0 {
+		if err := unix.Dup2(int(fd), options.NotifyFDNum); err != nil {
+			panic(fmt.Sprintf("dup2 %d -> %d: %v", fd, options.NotifyFDNum, err))
+		}
+		unix.Close(int(fd))
+		fd = uintptr(options.NotifyFDNum)
+	}
+	f := os.NewFile(fd, "seccomp_notify")
+	go func() {
+		// LockOSThread should help minimizing interactions with the scheduler.
+		runtime.LockOSThread()
+		defer runtime.UnlockOSThread()
+		var (
+			req  linux.SeccompNotif
+			resp linux.SeccompNotifResp
+		)
+		for {
+			req = linux.SeccompNotif{}
+			_, _, errno := unix.Syscall(unix.SYS_IOCTL, uintptr(f.Fd()),
+				uintptr(linux.SECCOMP_IOCTL_NOTIF_RECV),
+				uintptr(unsafe.Pointer(&req)))
+			if errno != 0 {
+				if errno == unix.EINTR {
+					continue
+				}
+				panic(fmt.Sprintf("SECCOMP_IOCTL_NOTIF_RECV failed with %d", errno))
+			}
+
+			log.Warningf("req %#v", req)
+			attached := true
+			if errno := hostsyscall.RawSyscallErrno(unix.SYS_PTRACE, unix.PTRACE_ATTACH, uintptr(req.Pid), 0); errno != 0 {
+				log.Warningf("unable to attach: %v", errno)
+				attached = false
+			}
+			resp = linux.SeccompNotifResp{
+				ID:    req.ID,
+				Flags: linux.SECCOMP_USER_NOTIF_FLAG_CONTINUE,
+			}
+			errno = hostsyscall.RawSyscallErrno(unix.SYS_IOCTL, uintptr(f.Fd()),
+				uintptr(linux.SECCOMP_IOCTL_NOTIF_SEND),
+				uintptr(unsafe.Pointer(&resp)))
+			if errno != 0 {
+				panic(fmt.Sprintf("SECCOMP_IOCTL_NOTIF_SEND failed with %d", errno))
+			}
+			if !attached {
+				if options.NotificationCallback != nil {
+					options.NotificationCallback(f, req, 0)
+				} else {
+					log.Warningf("Seccomp violation: %#v", req)
+				}
+				continue
+			}
+			time.Sleep(1 * time.Second)
+			for {
+				var info unix.Siginfo
+				errno := unix.Waitid(unix.P_PID, int(req.Pid), &info, syscall.WALL|syscall.WEXITED, nil)
+				if errno == syscall.EINTR {
+					continue
+				} else if errno != nil {
+					log.Warningf("failed to wait for the child process: %v", errno)
+				}
+				log.Warningf("%d: stopped -> %x", req.Pid, info.Code)
+				break
+			}
+			ret := 0
+			{
+				var regs linux.PtraceRegs
+				iovec := unix.Iovec{
+					Base: (*byte)(unsafe.Pointer(&regs)),
+					Len:  uint64(unsafe.Sizeof(regs)),
+				}
+				_, _, errno := unix.RawSyscall6(
+					unix.SYS_PTRACE,
+					unix.PTRACE_GETREGSET,
+					uintptr(req.Pid),
+					linux.NT_PRSTATUS,
+					uintptr(unsafe.Pointer(&iovec)),
+					0, 0)
+				if errno != 0 {
+					log.Warningf("unable to get registers: %s", errno)
+				}
+				ret = int(regs.SyscallRet())
+			}
+
+			if options.NotificationCallback != nil {
+				options.NotificationCallback(f, req, ret)
+			} else {
+				log.Warningf("Seccomp violation: %#v", req)
+			}
+			log.Warningf("detach from %d", req.Pid)
+			if errno := hostsyscall.RawSyscallErrno(unix.SYS_PTRACE, unix.PTRACE_DETACH, uintptr(req.Pid), 0); errno != 0 {
+				panic(fmt.Sprintf("unable to detach: %v", errno))
+			}
+		}
+	}()
+	return nil
+}
+
 // SetFilter installs the given BPF program.
 func SetFilter(instrs []bpf.Instruction) error {
 	// PR_SET_NO_NEW_PRIVS is required in order to enable seccomp. See

pkg/test/dockerutil/gpu.go

@@ -193,6 +193,10 @@ func (sgo *SniffGPUOpts) GPUCapabilities() string {
 
 // prepend prepends the sniffer arguments to the given command.
 func (sgo *SniffGPUOpts) prepend(argv []string) []string {
+	if *runtime != "" && *runtime != "runc" {
+		// ioctl_sniffer isn't supported in gVisor.
+		return argv
+	}
 	if sgo.DisableSnifferReason != "" {
 		return argv
 	}

runsc/boot/filter/BUILD

@@ -32,6 +32,7 @@ go_library(
         "//pkg/seccomp/precompiledseccomp",
         "//pkg/sync",
         "//runsc/boot/filter/config",
+        "@org_golang_x_sys//unix:go_default_library",
     ],
 )
 

runsc/boot/filter/filter.go

@@ -29,7 +29,10 @@ import (
 // If you suspect the Sentry is getting killed due to a seccomp violation,
 // change this to `true` to get a panic stack trace when there is a
 // violation.
-const debugFilter = false
+const (
+	debugFilterPanic = false // Panic on seccomp violation with stack trace.
+	debugFilterWarn  = false // Log seccomp violation, but continue program execution.
+)
 
 // Options is a re-export of the config Options type under this package.
 type Options = config.Options
@@ -41,7 +44,7 @@ func Install(opt Options) error {
 	}
 	key := opt.ConfigKey()
 	precompiled, usePrecompiled := GetPrecompiled(key)
-	if usePrecompiled && !debugFilter {
+	if usePrecompiled && !debugFilterPanic && !debugFilterWarn {
 		vars := opt.Vars()
 		log.Debugf("Loaded precompiled seccomp instructions for options %v, using variables: %v", key, vars)
 		insns, err := precompiled.RenderInstructions(vars)
@@ -51,9 +54,13 @@ func Install(opt Options) error {
 		return seccomp.SetFilter(insns)
 	}
 	seccompOpts := config.SeccompOptions(opt)
-	if debugFilter {
+	if debugFilterPanic {
 		log.Infof("Seccomp filter debugging is enabled; seccomp failures will result in a panic stack trace.")
 		seccompOpts.DefaultAction = linux.SECCOMP_RET_TRAP
+	} else if debugFilterWarn {
+		log.Infof("Seccomp filter debugging is enabled; seccomp failures will be logged")
+		seccompOpts.DefaultAction = linux.SECCOMP_RET_USER_NOTIF
+		seccompOpts.LogNotifications = true
 	} else {
 		log.Infof("No precompiled program found for config options %v, building seccomp program from scratch. This may slow down container startup.", key)
 		if log.IsLogging(log.Debug) {

tools/ioctl_sniffer/BUILD

@@ -54,7 +54,10 @@ go_binary(
         "//:sandbox",
     ],
     deps = [
+        "//pkg/abi/linux",
         "//pkg/log",
+        "//pkg/seccomp",
         "//tools/ioctl_sniffer/sniffer",
+        "@org_golang_x_sys//unix:go_default_library",
     ],
 )