Merge pull request #690 from austinvazquez/resolve-integration-test-instability

fix: tune vsock dial for buildkite hosts

Author: Kazuyoshi Kato

.buildkite/pipeline.yml

@@ -88,10 +88,6 @@ steps:
     command:
       - make -C runtime integ-test FICD_DM_POOL=build_${BUILDKITE_BUILD_NUMBER}_runtime
 
-  - wait
-
-  # Let's isolate the remote snapshotter integration tests.
-  # See https://github.com/firecracker-microvm/firecracker-containerd/issues/673
   - label: ":running: snapshotter isolated tests"
     agents:
       queue: "${BUILDKITE_AGENT_META_DATA_QUEUE:-default}"
@@ -108,8 +104,6 @@ steps:
       - make -C snapshotter integ-test FICD_DM_POOL=build_${BUILDKITE_BUILD_NUMBER}_snapshotter
     timeout_in_minutes: 10
 
-  - wait
-
   - label: ":weight_lifter: stress tests"
     concurrency_group: stress
     concurrency: 1

snapshotter/app/service.go

@@ -177,9 +177,12 @@ func initSnapshotter(ctx context.Context, config config.Config, cache cache.Cach
 			return nil, err
 		}
 
-		ackMsgTimeout := time.Duration(config.Snapshotter.Dialer.AckMsgTimeoutInSeconds) * time.Second
+		// TODO: https://github.com/firecracker-microvm/firecracker-containerd/issues/689
 		snapshotterDialer := func(ctx context.Context, namespace string) (net.Conn, error) {
-			return vsock.DialContext(ctx, host, uint32(port), vsock.WithLogger(log.G(ctx)), vsock.WithAckMsgTimeout(ackMsgTimeout))
+			return vsock.DialContext(ctx, host, uint32(port), vsock.WithLogger(log.G(ctx)),
+				vsock.WithAckMsgTimeout(2*time.Second),
+				vsock.WithRetryInterval(200*time.Millisecond),
+			)
 		}
 
 		var metricsProxy *metrics.Proxy

snapshotter/config/config.go

@@ -32,7 +32,6 @@ type Config struct {
 
 type snapshotter struct {
 	Listener listener `toml:"listener"`
-	Dialer   dialer   `toml:"dialer"`
 	Proxy    proxy    `toml:"proxy"`
 	Metrics  metrics  `toml:"metrics"`
 }
@@ -42,10 +41,6 @@ type listener struct {
 	Address string `toml:"address" default:"/var/lib/demux-snapshotter/snapshotter.sock"`
 }
 
-type dialer struct {
-	AckMsgTimeoutInSeconds int `toml:"ack_msg_timeout_in_seconds" default:"1"`
-}
-
 type proxy struct {
 	Address address `toml:"address"`
 }

snapshotter/config/config_test.go

@@ -62,9 +62,6 @@ func defaultConfig() error {
 				Network: "unix",
 				Address: "/var/lib/demux-snapshotter/snapshotter.sock",
 			},
-			Dialer: dialer{
-				AckMsgTimeoutInSeconds: 1,
-			},
 			Metrics: metrics{
 				Enable: false,
 			},
@@ -82,8 +79,6 @@ func parseExampleConfig() error {
 	  [snapshotter.listener]
 	    network = "unix"
 	    address = "/var/lib/demux-snapshotter/non-default-snapshotter.vsock"
-	  [snapshotter.dialer]
-	    ack_msg_timeout_in_seconds = 4
 	  [snapshotter.proxy.address.resolver]
 	    type = "http"
 	    address = "localhost:10001"
@@ -101,9 +96,6 @@ func parseExampleConfig() error {
 				Network: "unix",
 				Address: "/var/lib/demux-snapshotter/non-default-snapshotter.vsock",
 			},
-			Dialer: dialer{
-				AckMsgTimeoutInSeconds: 4,
-			},
 			Proxy: proxy{
 				Address: address{
 					Resolver: resolver{

tools/docker/entrypoint.sh

@@ -42,9 +42,6 @@ EOF
 
 mkdir -p /etc/demux-snapshotter /var/lib/demux-snapshotter
 cat > /etc/demux-snapshotter/config.toml <<EOF
-[snapshotter.dialer]
-  ack_msg_timeout_in_seconds = 2
-
 [snapshotter.proxy.address.resolver]
   type = "http"
   address = "http://127.0.0.1:10001"