fix: isolate remote snapshotter integration tests to increase stability

Signed-off-by: Austin Vazquez <macedonv@amazon.com>

Author: austinvazquez

.buildkite/pipeline.yml

@@ -73,7 +73,7 @@ steps:
       - make test-in-docker
     timeout_in_minutes: 10
 
-  - label: ":running: snapshotter isolated tests"
+  - label: ":running: runtime isolated tests"
     agents:
       queue: "${BUILDKITE_AGENT_META_DATA_QUEUE:-default}"
       distro: "${BUILDKITE_AGENT_META_DATA_DISTRO}"
@@ -82,12 +82,17 @@ steps:
       DOCKER_IMAGE_TAG: "$BUILDKITE_BUILD_NUMBER"
       NUMBER_OF_VMS: 10
       EXTRAGOARGS: "-v -count=1 -race"
+      FICD_DM_VOLUME_GROUP: fcci-vg
     artifact_paths:
-      - "snapshotter/logs/*"
+      - "runtime/logs/*"
     command:
-      - make -C snapshotter integ-test
+      - make -C runtime integ-test FICD_DM_POOL=build_${BUILDKITE_BUILD_NUMBER}_runtime
 
-  - label: ":running: runtime isolated tests"
+  - wait
+
+  # Let's isolate the remote snapshotter integration tests.
+  # See https://github.com/firecracker-microvm/firecracker-containerd/issues/673
+  - label: ":running: snapshotter isolated tests"
     agents:
       queue: "${BUILDKITE_AGENT_META_DATA_QUEUE:-default}"
       distro: "${BUILDKITE_AGENT_META_DATA_DISTRO}"
@@ -96,11 +101,12 @@ steps:
       DOCKER_IMAGE_TAG: "$BUILDKITE_BUILD_NUMBER"
       NUMBER_OF_VMS: 10
       EXTRAGOARGS: "-v -count=1 -race"
-      FICD_DM_VOLUME_GROUP: fcci-vg
     artifact_paths:
-      - "runtime/logs/*"
+      - "snapshotter/logs/*"
     command:
-      - make -C runtime integ-test FICD_DM_POOL=build_${BUILDKITE_BUILD_NUMBER}_runtime
+      - make -C snapshotter integ-test
+
+  - wait
 
   - label: ":weight_lifter: stress tests"
     concurrency_group: stress

snapshotter/service_integ_test.go

@@ -99,8 +99,13 @@ func launchContainerWithRemoteSnapshotterInVM(ctx context.Context, vmID string)
 		return fmt.Errorf("Failed to create fccontrol client. [%v]", err)
 	}
 
+	// Disable 8250 serial device and lessen the number of log messages written to the serial console.
+	// https://github.com/firecracker-microvm/firecracker/blob/v1.1.0/docs/prod-host-setup.md
+	kernelArgs := integtest.DefaultRuntimeConfig.KernelArgs + " 8250.nr_uarts=0 quiet loglevel=1"
+
 	_, err = fcClient.CreateVM(ctx, &proto.CreateVMRequest{
-		VMID: vmID,
+		VMID:       vmID,
+		KernelArgs: kernelArgs,
 		RootDrive: &proto.FirecrackerRootDrive{
 			HostPath: "/var/lib/firecracker-containerd/runtime/rootfs-stargz.img",
 		},
@@ -117,6 +122,7 @@ func launchContainerWithRemoteSnapshotterInVM(ctx context.Context, vmID string)
 			VcpuCount:  1,
 			MemSizeMib: 1024,
 		},
+		TimeoutSeconds: 60,
 		ContainerCount: 1,
 	})
 	if err != nil {