runsc: add support to control CPU features exposed to user apps

Currently, runsc compares the host feature set between the machine used
for checkpointing and the machine used for restoring. If the former
feature set is not a subset of the latter, the restore will fail, as
user apps might mistakenly use unsupported instructions.

Thanks to cpuid faulting support, it is possible to intercept the cpuid
instruction from user apps and generate results referring to the host
feature set recorded during sandbox boot.

This patch adds support to expose only specified CPU features by
removing features from the list when setting the feature set of kernel.
This makes it possible to checkpoint and restore on machines with
different CPU features. The list of enabled features is passed via
annotation.

It should be noted that currently this feature is not supported on
ARM64, as there is no ARM64 equivalent to cpuid.Static.Remove().

Updates #11486

Signed-off-by: Tianyu Zhou <albert.zty@antgroup.com>
Reviewed-by: Jamie Liu <jamieliu@google.com>

Author: tianyuzhou95

pkg/cpuid/native_amd64.go

@@ -163,6 +163,23 @@ func (fs FeatureSet) query(fn cpuidFunction) (uint32, uint32, uint32, uint32) {
 	return out.Eax, out.Ebx, out.Ecx, out.Edx
 }
 
+// Intersect returns the intersection of features between self and allowedFeatures.
+func (fs FeatureSet) Intersect(allowedFeatures map[Feature]struct{}) (FeatureSet, error) {
+	hs := fs.ToStatic()
+
+	// only keep features inside allowedFeatures.
+	for f, _ := range allFeatures {
+		if fs.HasFeature(f) {
+			if _, ok := allowedFeatures[f]; !ok {
+				log.Infof("Removing CPU feature %v as it is not allowed.", f)
+				hs.Remove(f)
+			}
+		}
+	}
+
+	return hs.ToFeatureSet(), nil
+}
+
 var hostFeatureSet FeatureSet
 
 // HostFeatureSet returns a host CPUID.

pkg/cpuid/native_arm64.go

@@ -18,6 +18,7 @@
 package cpuid
 
 import (
+	"fmt"
 	"os"
 	"runtime"
 	"strconv"
@@ -41,6 +42,13 @@ func (fs FeatureSet) Fixed() FeatureSet {
 	return fs
 }
 
+// Intersect returns the intersection of features between self and allowedFeatures.
+//
+// Just return error as there is no ARM64 equivalent to cpuid.Static.Remove().
+func (fs FeatureSet) Intersect(allowedFeatures map[Feature]struct{}) (FeatureSet, error) {
+	return FeatureSet{}, fmt.Errorf("FeatureSet intersection is not supported on ARM64")
+}
+
 // Reads CPU information from host /proc/cpuinfo.
 //
 // Must run before syscall filter installation. This value is used to create

runsc/boot/loader.go

@@ -21,6 +21,7 @@ import (
 	"os"
 	"runtime"
 	"strconv"
+	"strings"
 	gtime "time"
 
 	"github.com/moby/sys/capability"
@@ -597,6 +598,26 @@ func New(args Args) (*Loader, error) {
 		log.Infof("Setting total memory to %.2f GB", float64(args.TotalMem)/(1<<30))
 	}
 
+	cpufs := cpuid.HostFeatureSet()
+	if value, ok := args.Spec.Annotations[specutils.AnnotationCpuFeatures]; ok {
+		allowedFeatures := make(map[cpuid.Feature]struct{})
+		for str := range strings.SplitSeq(value, ",") {
+			if str == "" {
+				continue // ignore empty feature name rather than return error
+			}
+			f, ok := cpuid.FeatureFromString(str)
+			if !ok {
+				return nil, fmt.Errorf("annotation %s contains unknown feature %s", specutils.AnnotationCpuFeatures, str)
+			}
+			allowedFeatures[f] = struct{}{}
+		}
+		afs, err := cpufs.Intersect(allowedFeatures) // err only needed for now because this isn't implemented on ARM64 yet
+		if err != nil {
+			return nil, err
+		}
+		cpufs = afs
+	}
+
 	maxFDLimit := kernel.MaxFdLimit
 	if args.Spec.Linux != nil && args.Spec.Linux.Sysctl != nil {
 		if val, ok := args.Spec.Linux.Sysctl["fs.nr_open"]; ok {
@@ -616,7 +637,7 @@ func New(args Args) (*Loader, error) {
 		DisconnectOnSave: args.Conf.NetDisconnectOk,
 	}
 	if err = l.k.Init(kernel.InitKernelArgs{
-		FeatureSet:           cpuid.HostFeatureSet().Fixed(),
+		FeatureSet:           cpufs.Fixed(),
 		Timekeeper:           tk,
 		RootUserNamespace:    creds.UserNamespace,
 		RootNetworkNamespace: netns,

runsc/specutils/specutils.go

@@ -70,6 +70,10 @@ const (
 
 	// AnnotationTPU is the annotation used to enable TPU proxy on a pod.
 	AnnotationTPU = "dev.gvisor.internal.tpuproxy"
+
+	// AnnotationCpuFeatures is the annotation used to control cpu features
+	// that exposed to user apps.
+	AnnotationCpuFeatures = "dev.gvisor.internal.cpufeatures"
 )
 
 // ExePath must point to runsc binary, which is normally the same binary. It's