Support nested OCI indexes

Update `ExportedImageTar.IndexLayerArchiveFactory` to support nested
indexes. Nested indexes support a layer of interaction where the
`index.json` file points to a blob that contains the read index to use.

Prior to this commit, we only supported indexes provided directly by
the `index.json` file. This missing support results in "buildpack.toml:
no such file or directory" errors when referencing specific buildpacks
and using Docker Engine 27.3.1 or above.

See gh-43126

Author: philwebb

spring-boot-project/spring-boot-tools/spring-boot-buildpack-platform/src/main/java/org/springframework/boot/buildpack/platform/docker/ExportedImageTar.java

@@ -30,6 +30,7 @@
 import java.util.Set;
 import java.util.function.Predicate;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
 import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
@@ -142,23 +143,40 @@ private static class IndexLayerArchiveFactory extends LayerArchiveFactory {
 		private final Map<String, String> layerMediaTypes;
 
 		IndexLayerArchiveFactory(Path tarFile, ImageArchiveIndex index) throws IOException {
-			Set<String> manifestDigests = getDigests(index, this::isManifest);
-			List<ManifestList> manifestLists = getManifestLists(tarFile, getDigests(index, this::isManifestList));
+			this(tarFile, withNestedIndexes(tarFile, index));
+		}
+
+		IndexLayerArchiveFactory(Path tarFile, List<ImageArchiveIndex> indexes) throws IOException {
+			Set<String> manifestDigests = getDigests(indexes, this::isManifest);
+			Set<String> manifestListDigests = getDigests(indexes, IndexLayerArchiveFactory::isManifestList);
+			List<ManifestList> manifestLists = getManifestLists(tarFile, manifestListDigests);
 			List<Manifest> manifests = getManifests(tarFile, manifestDigests, manifestLists);
 			this.layerMediaTypes = manifests.stream()
 				.flatMap((manifest) -> manifest.getLayers().stream())
-				.collect(Collectors.toMap(this::getEntryName, BlobReference::getMediaType));
+				.collect(Collectors.toMap(IndexLayerArchiveFactory::getEntryName, BlobReference::getMediaType));
 		}
 
-		private Set<String> getDigests(ImageArchiveIndex index, Predicate<BlobReference> predicate) {
-			return index.getManifests()
-				.stream()
+		private static List<ImageArchiveIndex> withNestedIndexes(Path tarFile, ImageArchiveIndex index)
+				throws IOException {
+			Set<String> indexDigests = getDigests(Stream.of(index), IndexLayerArchiveFactory::isIndex);
+			List<ImageArchiveIndex> indexes = new ArrayList<>();
+			indexes.add(index);
+			indexes.addAll(getDigestMatches(tarFile, indexDigests, ImageArchiveIndex::of));
+			return indexes;
+		}
+
+		private static Set<String> getDigests(List<ImageArchiveIndex> indexes, Predicate<BlobReference> predicate) {
+			return getDigests(indexes.stream(), predicate);
+		}
+
+		private static Set<String> getDigests(Stream<ImageArchiveIndex> indexes, Predicate<BlobReference> predicate) {
+			return indexes.flatMap((index) -> index.getManifests().stream())
 				.filter(predicate)
 				.map(BlobReference::getDigest)
 				.collect(Collectors.toUnmodifiableSet());
 		}
 
-		private List<ManifestList> getManifestLists(Path tarFile, Set<String> digests) throws IOException {
+		private static List<ManifestList> getManifestLists(Path tarFile, Set<String> digests) throws IOException {
 			return getDigestMatches(tarFile, digests, ManifestList::of);
 		}
 
@@ -173,12 +191,14 @@ private List<Manifest> getManifests(Path tarFile, Set<String> manifestDigests, L
 			return getDigestMatches(tarFile, digests, Manifest::of);
 		}
 
-		private <T> List<T> getDigestMatches(Path tarFile, Set<String> digests,
+		private static <T> List<T> getDigestMatches(Path tarFile, Set<String> digests,
 				ThrowingFunction<InputStream, T> factory) throws IOException {
 			if (digests.isEmpty()) {
 				return Collections.emptyList();
 			}
-			Set<String> names = digests.stream().map(this::getEntryName).collect(Collectors.toUnmodifiableSet());
+			Set<String> names = digests.stream()
+				.map(IndexLayerArchiveFactory::getEntryName)
+				.collect(Collectors.toUnmodifiableSet());
 			List<T> result = new ArrayList<>();
 			try (TarArchiveInputStream tar = openTar(tarFile)) {
 				TarArchiveEntry entry = tar.getNextTarEntry();
@@ -197,19 +217,23 @@ private boolean isManifest(BlobReference reference) {
 					|| isJsonWithPrefix(reference.getMediaType(), "application/vnd.docker.distribution.manifest.v");
 		}
 
-		private boolean isManifestList(BlobReference reference) {
+		private static boolean isIndex(BlobReference reference) {
+			return isJsonWithPrefix(reference.getMediaType(), "application/vnd.oci.image.index.v");
+		}
+
+		private static boolean isManifestList(BlobReference reference) {
 			return isJsonWithPrefix(reference.getMediaType(), "application/vnd.docker.distribution.manifest.list.v");
 		}
 
-		private boolean isJsonWithPrefix(String mediaType, String prefix) {
+		private static boolean isJsonWithPrefix(String mediaType, String prefix) {
 			return mediaType.startsWith(prefix) && mediaType.endsWith("+json");
 		}
 
-		private String getEntryName(BlobReference reference) {
+		private static String getEntryName(BlobReference reference) {
 			return getEntryName(reference.getDigest());
 		}
 
-		private String getEntryName(String digest) {
+		private static String getEntryName(String digest) {
 			return "blobs/" + digest.replace(':', '/');
 		}
 

spring-boot-project/spring-boot-tools/spring-boot-buildpack-platform/src/test/java/org/springframework/boot/buildpack/platform/docker/ExportedImageTarTests.java

@@ -16,6 +16,9 @@
 
 package org.springframework.boot.buildpack.platform.docker;
 
+import java.util.ArrayList;
+import java.util.List;
+
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.ValueSource;
 
@@ -34,7 +37,8 @@ class ExportedImageTarTests {
 
 	@ParameterizedTest
 	@ValueSource(strings = { "export-docker-desktop.tar", "export-docker-desktop-containerd.tar",
-			"export-docker-desktop-containerd-manifest-list.tar", "export-docker-engine.tar", "export-podman.tar" })
+			"export-docker-desktop-containerd-manifest-list.tar", "export-docker-engine.tar", "export-podman.tar",
+			"export-docker-desktop-nested-index.tar" })
 	void test(String tarFile) throws Exception {
 		ImageReference reference = ImageReference.of("test:latest");
 		try (ExportedImageTar exportedImageTar = new ExportedImageTar(reference,
@@ -43,10 +47,12 @@ void test(String tarFile) throws Exception {
 			String expectedName = (expectedCompression != Compression.GZIP)
 					? "5caae51697b248b905dca1a4160864b0e1a15c300981736555cdce6567e8d477"
 					: "f0f1fd1bdc71ac6a4dc99cea5f5e45c86c5ec26fe4d1daceeb78207303606429";
+			List<String> names = new ArrayList<>();
 			exportedImageTar.exportLayers((name, tarArchive) -> {
-				assertThat(name).contains(expectedName);
+				names.add(name);
 				assertThat(tarArchive.getCompression()).isEqualTo(expectedCompression);
 			});
+			assertThat(names).filteredOn((name) -> name.contains(expectedName)).isNotEmpty();
 		}
 	}