Remove old NICs and routes during save/restore.

During restore there will be new NICs and routes created based on the network
config. Close these old NICs and routes in beforeSave() only for save/restore.
For save/resume, do not close them.

As new link endpoints are created during restore, there is no need to start
the associated old processor goroutines in afterLoad. Start them only for
save/resume.

PiperOrigin-RevId: 804994281

Author: nybidari

pkg/sentry/inet/inet.go

@@ -140,6 +140,10 @@ type Stack interface {
 
 	// Stats returns the network stats.
 	Stats() tcpip.Stats
+
+	// SetRemoveNICs sets removeNICs in stack to true. This should only be
+	// called during save/restore.
+	SetRemoveNICs()
 }
 
 // Interface contains information about a network interface.

pkg/sentry/inet/test_stack.go

@@ -241,3 +241,8 @@ func (*TestStack) Stats() tcpip.Stats {
 	// No-op.
 	return tcpip.Stats{}
 }
+
+// SetRemoveNICs implements Stack.
+func (*TestStack) SetRemoveNICs() {
+	// No-op.
+}

pkg/sentry/kernel/kernel.go

@@ -80,6 +80,7 @@ import (
 	"gvisor.dev/gvisor/pkg/sentry/vfs"
 	"gvisor.dev/gvisor/pkg/state"
 	"gvisor.dev/gvisor/pkg/sync"
+	"gvisor.dev/gvisor/pkg/tcpip/stack"
 )
 
 // IOUringEnabled is set to true when IO_URING is enabled. Added as a global to
@@ -613,7 +614,7 @@ func savePrivateMFs(ctx context.Context, w io.Writer, pw io.Writer, mfsToSave ma
 // SaveTo saves the state of k to w.
 //
 // Preconditions: The kernel must be paused throughout the call to SaveTo.
-func (k *Kernel) SaveTo(ctx context.Context, w, pagesMetadata io.Writer, pagesFile *fd.FD, mfOpts pgalloc.SaveOpts) error {
+func (k *Kernel) SaveTo(ctx context.Context, w, pagesMetadata io.Writer, pagesFile *fd.FD, mfOpts pgalloc.SaveOpts, resume bool) error {
 	saveStart := time.Now()
 
 	// Do not allow other Kernel methods to affect it while it's being saved.
@@ -681,6 +682,11 @@ func (k *Kernel) SaveTo(ctx context.Context, w, pagesMetadata io.Writer, pagesFi
 	if rootNS := k.rootNetworkNamespace; rootNS != nil && rootNS.Stack() != nil {
 		// Pause the network stack.
 		netstackPauseStart := time.Now()
+		if resume {
+			ctx = context.WithValue(ctx, stack.CtxResumeStack, resume)
+		} else {
+			k.rootNetworkNamespace.Stack().SetRemoveNICs()
+		}
 		log.Infof("Pausing root network namespace")
 		k.rootNetworkNamespace.Stack().Pause()
 		defer k.rootNetworkNamespace.Stack().Resume()

pkg/sentry/socket/hostinet/stack.go

@@ -443,3 +443,8 @@ func (s *Stack) IsSaveRestoreEnabled() bool {
 func (s *Stack) Stats() tcpip.Stats {
 	return tcpip.Stats{}
 }
+
+// SetRemoveNICs implements inet.Stack.SetRemoveNICs.
+func (*Stack) SetRemoveNICs() {
+	// No-op.
+}

pkg/sentry/socket/netstack/stack.go

@@ -970,3 +970,8 @@ func (s *Stack) PortRange() (uint16, uint16) {
 func (s *Stack) SetPortRange(start uint16, end uint16) error {
 	return syserr.TranslateNetstackError(s.Stack.SetPortRange(start, end)).ToError()
 }
+
+// SetRemoveNICs implements inet.Stack.SetRemoveNICs.
+func (s *Stack) SetRemoveNICs() {
+	s.Stack.SetRemoveNICs()
+}

pkg/sentry/state/state.go

@@ -110,7 +110,7 @@ func (opts SaveOpts) Save(ctx context.Context, k *kernel.Kernel, w *watchdog.Wat
 		}
 
 		// Save the kernel.
-		err = k.SaveTo(ctx, wc, pagesMetadata, opts.PagesFile, opts.MemoryFileSaveOpts)
+		err = k.SaveTo(ctx, wc, pagesMetadata, opts.PagesFile, opts.MemoryFileSaveOpts, opts.Resume)
 
 		// ENOSPC is a state file error. This error can only come from
 		// writing the state file, and not from fs.FileOperations.Fsync

pkg/tcpip/link/fdbased/processors.go

@@ -127,7 +127,13 @@ func (m *processorManager) start() {
 }
 
 // afterLoad is invoked by stateify.
-func (m *processorManager) afterLoad(context.Context) {
+func (m *processorManager) afterLoad(ctx context.Context) {
+	// Do not start the processors for save/restore. New NICs and
+	// processors will be created during restore.
+	resume := stack.ResumeStackFromContext(ctx)
+	if !resume {
+		return
+	}
 	m.wg.Add(len(m.processors))
 	m.start()
 }
@@ -276,6 +282,7 @@ func (m *processorManager) wakeReady() {
 	}
 }
 
+// beforeSave is invoked by stateify.
 func (m *processorManager) beforeSave() {
 	m.close()
 	m.wg.Wait()

pkg/tcpip/stack/save_restore.go

@@ -22,6 +22,31 @@ import (
 	cryptorand "gvisor.dev/gvisor/pkg/rand"
 )
 
+// beforeSave is invoked by stateify.
+func (s *Stack) beforeSave() {
+	// removeNICs will be set only in case of save/restore.
+	s.mu.Lock()
+	if !s.removeNICs {
+		s.mu.Unlock()
+		return
+	}
+
+	// Remove all the NICs and routes from the stack as they will be
+	// created again during restore based on the new network config.
+	deferActs := make([]func(), 0)
+	for id := range s.nics {
+		act, _ := s.removeNICLocked(id)
+		if act != nil {
+			deferActs = append(deferActs, act)
+		}
+	}
+	s.mu.Unlock()
+
+	for _, act := range deferActs {
+		act()
+	}
+}
+
 // afterLoad is invoked by stateify.
 func (s *Stack) afterLoad(context.Context) {
 	s.insecureRNG = rand.New(rand.NewSource(time.Now().UnixNano()))

pkg/tcpip/stack/stack.go

@@ -185,6 +185,9 @@ type Stack struct {
 
 	// saveRestoreEnabled indicates whether the stack is saved and restored.
 	saveRestoreEnabled bool
+
+	// removeNICs indicates if the NICs and routes should be removed before saving.
+	removeNICs bool `state:"nosave"`
 }
 
 // NetworkProtocolFactory instantiates a network protocol.
@@ -2518,9 +2521,24 @@ type contextID int
 const (
 	// CtxRestoreStack is a Context.Value key for the stack to be used in restore.
 	CtxRestoreStack contextID = iota
+
+	// CtxResumeStack is a Context.Value key for the stack to be used in resume.
+	CtxResumeStack contextID = iota
 )
 
 // RestoreStackFromContext returns the stack to be used during restore.
 func RestoreStackFromContext(ctx context.Context) *Stack {
 	return ctx.Value(CtxRestoreStack).(*Stack)
 }
+
+// ResumeStackFromContext returns the stack to be used during restore.
+func ResumeStackFromContext(ctx context.Context) bool {
+	return ctx.Value(CtxResumeStack).(bool)
+}
+
+// SetRemoveNICs sets the removeNICs in stack to true during save/restore.
+func (s *Stack) SetRemoveNICs() {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.removeNICs = true
+}