Refactor image management to use transfer service

Dockerfile.d/etc_containerd_config.toml

@@ -5,3 +5,12 @@ version = 2
   [proxy_plugins.stargz]
     type = "snapshot"
     address = "/run/containerd-stargz-grpc/containerd-stargz-grpc.sock"
+  [proxy_plugins.stargz.exports]
+    root = "/var/lib/containerd-stargz-grpc/"
+    enable_remote_snapshot_annotations = "true"
+[[plugins."io.containerd.transfer.v1.local".unpack_config]]
+  platform = "linux"
+  snapshotter = "overlayfs"
+[[plugins."io.containerd.transfer.v1.local".unpack_config]]
+  platform = "linux"
+  snapshotter = "stargz"

Dockerfile.d/test-integration-etc_containerd_config.toml

@@ -5,8 +5,26 @@ version = 2
   [proxy_plugins.stargz]
     type = "snapshot"
     address = "/run/containerd-stargz-grpc/containerd-stargz-grpc.sock"
+  [proxy_plugins.stargz.exports]
+    root = "/var/lib/containerd-stargz-grpc/"
+    enable_remote_snapshot_annotations = "true"
 
 # Enable soci snapshotter
   [proxy_plugins.soci]
     type = "snapshot"
     address = "/run/soci-snapshotter-grpc/soci-snapshotter-grpc.sock"
+  [proxy_plugins.soci.exports]
+    root = "/var/lib/soci-snapshotter-grpc"
+    enable_remote_snapshot_annotations = "true"
+
+[[plugins."io.containerd.transfer.v1.local".unpack_config]]
+  platform = "linux"
+  snapshotter = "overlayfs"
+
+[[plugins."io.containerd.transfer.v1.local".unpack_config]]
+  platform = "linux"
+  snapshotter = "soci"
+
+[[plugins."io.containerd.transfer.v1.local".unpack_config]]
+  platform = "linux"
+  snapshotter = "stargz"

Dockerfile.d/test-integration-rootless.sh

@@ -53,6 +53,15 @@ else
   [proxy_plugins."stargz"]
     type = "snapshot"
     address = "/run/user/$(id -u)/containerd-stargz-grpc/containerd-stargz-grpc.sock"
+  [proxy_plugins.stargz.exports]
+    root = "/home/rootless/.local/share/containerd-stargz-grpc/"
+    enable_remote_snapshot_annotations = "true"
+[[plugins."io.containerd.transfer.v1.local".unpack_config]]
+  platform = "linux"
+  snapshotter = "overlayfs"
+[[plugins."io.containerd.transfer.v1.local".unpack_config]]
+  platform = "linux"
+  snapshotter = "stargz"
 EOF
 	systemctl --user restart containerd.service
 	containerd-rootless-setuptool.sh -- install-ipfs --init --offline # offline ipfs daemon for testing
@@ -64,4 +73,4 @@ EOF
 	cd /go/src/github.com/containerd/nerdctl
 	# We also lose the PATH (and SendEnv=PATH would require sshd config changes)
 	exec env PATH="/usr/local/go/bin:$PATH" "$@"
-fi
+fi

cmd/nerdctl/container/container_run_test.go

@@ -786,7 +786,7 @@ func TestRunFromOCIArchive(t *testing.T) {
 	tarPath := fmt.Sprintf("%s/%s.tar", buildCtx, imageName)
 
 	base.Cmd("build", "--tag", tag, fmt.Sprintf("--output=type=oci,dest=%s", tarPath), buildCtx).AssertOK()
-	base.Cmd("run", "--rm", fmt.Sprintf("oci-archive://%s", tarPath)).AssertOutContainsAll(fmt.Sprintf("Loaded image: %s", tag), sentinel)
+	base.Cmd("run", "--rm", fmt.Sprintf("oci-archive://%s", tarPath)).AssertOutContainsAll(tag, sentinel)
 }
 
 func TestRunDomainname(t *testing.T) {

cmd/nerdctl/image/image_load_test.go

@@ -17,7 +17,6 @@
 package image
 
 import (
-	"fmt"
 	"os"
 	"path/filepath"
 	"strings"
@@ -61,7 +60,7 @@ func TestLoadStdinFromPipe(t *testing.T) {
 			identifier := data.Identifier()
 			return &test.Expected{
 				Output: expect.All(
-					expect.Contains(fmt.Sprintf("Loaded image: %s:latest", identifier)),
+					expect.Contains(identifier),
 					func(stdout string, t tig.T) {
 						assert.Assert(t, strings.Contains(helpers.Capture("images"), identifier))
 					},
@@ -107,7 +106,7 @@ func TestLoadQuiet(t *testing.T) {
 		Expected: func(data test.Data, helpers test.Helpers) *test.Expected {
 			return &test.Expected{
 				Output: expect.All(
-					expect.Contains(fmt.Sprintf("Loaded image: %s:latest", data.Identifier())),
+					expect.Contains(data.Identifier()),
 					expect.DoesNotContain("Loading layer"),
 				),
 			}

extras/rootless/containerd-rootless-setuptool.sh

@@ -404,6 +404,15 @@ cmd_entrypoint_install_fuse_overlayfs() {
 		  [proxy_plugins."fuse-overlayfs"]
 		    type = "snapshot"
 		    address = "${XDG_RUNTIME_DIR}/containerd-fuse-overlayfs.sock"
+		  [proxy_plugins."fuse-overlayfs".exports]
+		    root = "${XDG_DATA_HOME}/containerd-fuse-overlayfs/"
+			enable_remote_snapshot_annotations = "true"
+		[[plugins."io.containerd.transfer.v1.local".unpack_config]]
+			platform = "linux"
+			snapshotter = "fuse-overlayfs"
+		[[plugins."io.containerd.transfer.v1.local".unpack_config]]
+			platform = "linux"
+			snapshotter = "overlayfs"
 		###  END  ###
 	EOT
 	INFO "Set \`export CONTAINERD_SNAPSHOTTER=\"fuse-overlayfs\"\` to use the fuse-overlayfs snapshotter."
@@ -449,6 +458,15 @@ cmd_entrypoint_install_stargz() {
 		  [proxy_plugins."stargz"]
 		    type = "snapshot"
 		    address = "${XDG_RUNTIME_DIR}/containerd-stargz-grpc/containerd-stargz-grpc.sock"
+		  [proxy_plugins.stargz.exports]
+		    root = "${XDG_DATA_HOME}/containerd-stargz-grpc/"
+		    enable_remote_snapshot_annotations = "true"
+		[[plugins."io.containerd.transfer.v1.local".unpack_config]]
+			platform = "linux"
+			snapshotter = "stargz"
+		[[plugins."io.containerd.transfer.v1.local".unpack_config]]
+			platform = "linux"
+			snapshotter = "overlayfs"
 		###  END  ###
 	EOT
 	INFO "Set \`export CONTAINERD_SNAPSHOTTER=\"stargz\"\` to use the stargz snapshotter."
@@ -652,4 +670,4 @@ fi
 
 # main
 shift
-"cmd_entrypoint_${command}" "$@"
+"cmd_entrypoint_${command}" "$@"

mod/tigron/test/data.go

@@ -31,7 +31,6 @@ import (
 
 const (
 	identifierMaxLength       = 76
-	identifierSeparator       = "-"
 	identifierSignatureLength = 8
 )
 
@@ -250,27 +249,18 @@ func newData(t tig.T, seed, parent Data) Data {
 func defaultIdentifierHashing(names ...string) string {
 	// Notes: identifier MAY be used for namespaces, image names, etc.
 	// So, the rules are stringent on what it can contain.
-	replaceWith := []byte(identifierSeparator)
-	name := strings.ToLower(strings.Join(names, string(replaceWith)))
+	// Join names without separator to avoid '-' which causes display issues in progress output
+	name := strings.ToLower(strings.Join(names, ""))
 	// Ensure we have a unique identifier despite characters replacements
 	// (well, as unique as the names collection being passed)
 	signature := fmt.Sprintf("%x", sha256.Sum256([]byte(name)))[0:identifierSignatureLength]
-	// Make sure we do not use any unsafe characters
-	safeName := regexp.MustCompile(`[^a-z0-9-]+`)
-	// And we avoid repeats of the separator
-	noRepeat := regexp.MustCompile(fmt.Sprintf(`[%s]{2,}`, replaceWith))
-	escapedName := safeName.ReplaceAll([]byte(name), replaceWith)
-	escapedName = noRepeat.ReplaceAll(escapedName, replaceWith)
-	// Do not allow trailing or leading dash (as that may stutter)
-	name = strings.Trim(string(escapedName), string(replaceWith))
+	safeName := regexp.MustCompile(`[^a-z0-9]+`)
+	escapedName := safeName.ReplaceAll([]byte(name), []byte(""))
+	name = string(escapedName)
 
 	// Ensure we will never go above 76 characters in length (with signature)
-	if len(name) > (identifierMaxLength - len(signature)) {
-		name = name[0 : identifierMaxLength-identifierSignatureLength-len(identifierSeparator)]
-	}
-
-	if name[len(name)-1:] != identifierSeparator {
-		signature = identifierSeparator + signature
+	if len(name) > (identifierMaxLength - identifierSignatureLength) {
+		name = name[0 : identifierMaxLength-identifierSignatureLength]
 	}
 
 	return name + signature

mod/tigron/test/data_test.go

@@ -71,7 +71,7 @@ func TestDataIdentifier(t *testing.T) {
 	assertive.HasPrefix(t, one, "testdataidentifier")
 
 	three := dataObj.Identifier("Some Add ∞ Funky∞Prefix")
-	assertive.HasPrefix(t, three, "testdataidentifier-some-add-funky-prefix")
+	assertive.HasPrefix(t, three, "testdataidentifiersomeaddfunkyprefix")
 }
 
 func TestDataIdentifierThatIsReallyReallyReallyReallyReallyReallyReallyReallyReallyReallyReallyLong(

pkg/cmd/image/ensure.go

@@ -18,8 +18,6 @@ package image
 
 import (
 	"context"
-	"errors"
-	"net/http"
 	"os"
 
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
@@ -80,7 +78,6 @@ func ensureOne(ctx context.Context, client *containerd.Client, rawRef string, ta
 		// Get a resolver
 		var dOpts []dockerconfigresolver.Opt
 		if options.InsecureRegistry {
-			log.G(ctx).Warnf("skipping verifying HTTPS certs for %q", parsedReference.Domain)
 			dOpts = append(dOpts, dockerconfigresolver.WithSkipVerifyCerts(true))
 		}
 		dOpts = append(dOpts, dockerconfigresolver.WithHostsDirs(options.HostsDir))
@@ -99,7 +96,7 @@ func ensureOne(ctx context.Context, client *containerd.Client, rawRef string, ta
 
 		if err != nil {
 			// In some circumstance (e.g. people just use 80 port to support pure http), the error will contain message like "dial tcp <port>: connection refused".
-			if !errors.Is(err, http.ErrSchemeMismatch) && !errutil.IsErrConnectionRefused(err) {
+			if !errutil.IsErrHTTPSFallbackNeeded(err) {
 				return err
 			}
 			if options.InsecureRegistry {