Use image manifest when exporting layers

A tar archive of a Docker image contains a `mainfest.json` file that
lists the path to each embedded tar file containing the contents of a
layer in the image. This manifest file should be used to identify the
layer files instead of relying on file naming conventions and
assumptions on the directory structure that are not consistent
between container engine implementations.

Fixes gh-34324
This commit is contained in:
Scott Frederick 2023-02-27 14:38:29 -06:00
parent 27ba20f310
commit 7730eee439
11 changed files with 329 additions and 49 deletions

View File

@ -1,5 +1,5 @@
/*
* Copyright 2012-2022 the original author or authors.
* Copyright 2012-2023 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -17,6 +17,7 @@
package org.springframework.boot.buildpack.platform.build;
import java.io.IOException;
import java.nio.file.Path;
import java.util.List;
import java.util.function.Consumer;
@ -32,7 +33,6 @@ import org.springframework.boot.buildpack.platform.docker.transport.DockerEngine
import org.springframework.boot.buildpack.platform.docker.type.Image;
import org.springframework.boot.buildpack.platform.docker.type.ImageReference;
import org.springframework.boot.buildpack.platform.io.IOBiConsumer;
import org.springframework.boot.buildpack.platform.io.TarArchive;
import org.springframework.util.Assert;
import org.springframework.util.StringUtils;
@ -273,9 +273,8 @@ public class Builder {
}
@Override
public void exportImageLayers(ImageReference reference, IOBiConsumer<String, TarArchive> exports)
throws IOException {
Builder.this.docker.image().exportLayers(reference, exports);
public void exportImageLayers(ImageReference reference, IOBiConsumer<String, Path> exports) throws IOException {
Builder.this.docker.image().exportLayerFiles(reference, exports);
}
}

View File

@ -1,5 +1,5 @@
/*
* Copyright 2012-2022 the original author or authors.
* Copyright 2012-2023 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -17,12 +17,12 @@
package org.springframework.boot.buildpack.platform.build;
import java.io.IOException;
import java.nio.file.Path;
import java.util.List;
import org.springframework.boot.buildpack.platform.docker.type.Image;
import org.springframework.boot.buildpack.platform.docker.type.ImageReference;
import org.springframework.boot.buildpack.platform.io.IOBiConsumer;
import org.springframework.boot.buildpack.platform.io.TarArchive;
/**
* Context passed to a {@link BuildpackResolver}.
@ -52,6 +52,6 @@ interface BuildpackResolverContext {
* during the callback)
* @throws IOException on IO error
*/
void exportImageLayers(ImageReference reference, IOBiConsumer<String, TarArchive> exports) throws IOException;
void exportImageLayers(ImageReference reference, IOBiConsumer<String, Path> exports) throws IOException;
}

View File

@ -17,6 +17,7 @@
package org.springframework.boot.buildpack.platform.build;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.file.Files;
import java.nio.file.Path;
@ -35,7 +36,6 @@ import org.springframework.boot.buildpack.platform.docker.type.ImageReference;
import org.springframework.boot.buildpack.platform.docker.type.Layer;
import org.springframework.boot.buildpack.platform.docker.type.LayerId;
import org.springframework.boot.buildpack.platform.io.IOConsumer;
import org.springframework.boot.buildpack.platform.io.TarArchive;
import org.springframework.util.StreamUtils;
/**
@ -115,23 +115,16 @@ final class ImageBuildpack implements Buildpack {
ExportedLayers(BuildpackResolverContext context, ImageReference imageReference) throws IOException {
List<Path> layerFiles = new ArrayList<>();
context.exportImageLayers(imageReference, (name, archive) -> layerFiles.add(copyToTemp(name, archive)));
context.exportImageLayers(imageReference, (name, path) -> layerFiles.add(copyToTemp(path)));
this.layerFiles = Collections.unmodifiableList(layerFiles);
}
private Path copyToTemp(String name, TarArchive archive) throws IOException {
String[] parts = name.split("/");
Path path = Files.createTempFile("create-builder-scratch-", parts[0]);
try (OutputStream out = Files.newOutputStream(path)) {
archive.writeTo(out);
}
return path;
}
void apply(IOConsumer<Layer> layers) throws IOException {
for (Path path : this.layerFiles) {
layers.accept(Layer.fromTarArchive((out) -> copyLayerTar(path, out)));
private Path copyToTemp(Path path) throws IOException {
Path outputPath = Files.createTempFile("create-builder-scratch-", null);
try (OutputStream out = Files.newOutputStream(outputPath)) {
copyLayerTar(path, out);
}
return outputPath;
}
private void copyLayerTar(Path path, OutputStream out) throws IOException {
@ -147,7 +140,16 @@ final class ImageBuildpack implements Buildpack {
}
tarOut.finish();
}
Files.delete(path);
}
void apply(IOConsumer<Layer> layers) throws IOException {
for (Path path : this.layerFiles) {
layers.accept(Layer.fromTarArchive((out) -> {
InputStream in = Files.newInputStream(path);
StreamUtils.copy(in, out);
}));
Files.delete(path);
}
}
}

View File

@ -16,13 +16,24 @@
package org.springframework.boot.buildpack.platform.docker;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
@ -37,6 +48,7 @@ import org.springframework.boot.buildpack.platform.docker.type.ContainerReferenc
import org.springframework.boot.buildpack.platform.docker.type.ContainerStatus;
import org.springframework.boot.buildpack.platform.docker.type.Image;
import org.springframework.boot.buildpack.platform.docker.type.ImageArchive;
import org.springframework.boot.buildpack.platform.docker.type.ImageArchiveManifest;
import org.springframework.boot.buildpack.platform.docker.type.ImageReference;
import org.springframework.boot.buildpack.platform.docker.type.VolumeName;
import org.springframework.boot.buildpack.platform.io.IOBiConsumer;
@ -250,7 +262,7 @@ public class DockerApi {
}
/**
* Export the layers of an image.
* Export the layers of an image as {@link TarArchive}s.
* @param reference the reference to export
* @param exports a consumer to receive the layers (contents can only be accessed
* during the callback)
@ -258,20 +270,49 @@ public class DockerApi {
*/
public void exportLayers(ImageReference reference, IOBiConsumer<String, TarArchive> exports)
throws IOException {
exportLayerFiles(reference, (name, path) -> {
try (InputStream in = Files.newInputStream(path)) {
TarArchive archive = (out) -> StreamUtils.copy(in, out);
exports.accept(name, archive);
}
});
}
/**
* Export the layers of an image as paths to layer tar files.
* @param reference the reference to export
* @param exports a consumer to receive the layer tar file paths (file can only be
* accessed during the callback)
* @throws IOException on IO error
*/
public void exportLayerFiles(ImageReference reference, IOBiConsumer<String, Path> exports) throws IOException {
Assert.notNull(reference, "Reference must not be null");
Assert.notNull(exports, "Exports must not be null");
URI saveUri = buildUrl("/images/" + reference + "/get");
Response response = http().get(saveUri);
ImageArchiveManifest manifest = null;
Map<String, Path> layerFiles = new HashMap<>();
try (TarArchiveInputStream tar = new TarArchiveInputStream(response.getContent())) {
TarArchiveEntry entry = tar.getNextTarEntry();
while (entry != null) {
if (entry.getName().endsWith("/layer.tar")) {
TarArchive archive = (out) -> StreamUtils.copy(tar, out);
exports.accept(entry.getName(), archive);
if (entry.getName().equals("manifest.json")) {
manifest = readManifest(tar);
}
if (entry.getName().endsWith(".tar")) {
layerFiles.put(entry.getName(), copyToTemp(tar));
}
entry = tar.getNextTarEntry();
}
}
Assert.notNull(manifest, "Manifest not found in image " + reference);
for (Map.Entry<String, Path> entry : layerFiles.entrySet()) {
String name = entry.getKey();
Path path = entry.getValue();
if (manifestContainsLayerEntry(manifest, name)) {
exports.accept(name, path);
}
Files.delete(path);
}
}
/**
@ -308,6 +349,24 @@ public class DockerApi {
http().post(uri).close();
}
private ImageArchiveManifest readManifest(TarArchiveInputStream tar) throws IOException {
String manifestContent = new BufferedReader(new InputStreamReader(tar, StandardCharsets.UTF_8)).lines()
.collect(Collectors.joining());
return ImageArchiveManifest.of(new ByteArrayInputStream(manifestContent.getBytes(StandardCharsets.UTF_8)));
}
private Path copyToTemp(TarArchiveInputStream in) throws IOException {
Path path = Files.createTempFile("create-builder-scratch-", null);
try (OutputStream out = Files.newOutputStream(path)) {
StreamUtils.copy(in, out);
}
return path;
}
private boolean manifestContainsLayerEntry(ImageArchiveManifest manifest, String layerId) {
return manifest.getEntries().stream().anyMatch((content) -> content.getLayers().contains(layerId));
}
}
/**

View File

@ -0,0 +1,95 @@
/*
* Copyright 2012-2023 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.boot.buildpack.platform.docker.type;
import java.io.IOException;
import java.io.InputStream;
import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import com.fasterxml.jackson.databind.JsonNode;
import org.springframework.boot.buildpack.platform.json.MappedObject;
/**
* Image archive manifest information.
*
* @author Scott Frederick
* @since 2.7.9
*/
public class ImageArchiveManifest extends MappedObject {
private final List<ManifestEntry> entries = new ArrayList<>();
protected ImageArchiveManifest(JsonNode node) {
super(node, MethodHandles.lookup());
getNode().elements().forEachRemaining((element) -> this.entries.add(ManifestEntry.of(element)));
}
/**
* Return the entries contained in the manifest.
* @return the manifest entries
*/
public List<ManifestEntry> getEntries() {
return this.entries;
}
/**
* Create an {@link ImageArchiveManifest} from the provided JSON input stream.
* @param content the JSON input stream
* @return a new {@link ImageArchiveManifest} instance
* @throws IOException on IO error
*/
public static ImageArchiveManifest of(InputStream content) throws IOException {
return of(content, ImageArchiveManifest::new);
}
public static class ManifestEntry extends MappedObject {
private final List<String> layers;
protected ManifestEntry(JsonNode node) {
super(node, MethodHandles.lookup());
this.layers = extractLayers();
}
/**
* Return the collection of layer IDs from a section of the manifest.
* @return a collection of layer IDs
*/
public List<String> getLayers() {
return this.layers;
}
static ManifestEntry of(JsonNode node) {
return new ManifestEntry(node);
}
@SuppressWarnings("unchecked")
private List<String> extractLayers() {
List<String> layers = valueAt("/Layers", List.class);
if (layers == null) {
return Collections.emptyList();
}
return layers;
}
}
}

View File

@ -18,7 +18,10 @@ package org.springframework.boot.buildpack.platform.build;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;
@ -33,7 +36,6 @@ import org.mockito.invocation.InvocationOnMock;
import org.springframework.boot.buildpack.platform.docker.type.Image;
import org.springframework.boot.buildpack.platform.docker.type.ImageReference;
import org.springframework.boot.buildpack.platform.io.IOBiConsumer;
import org.springframework.boot.buildpack.platform.io.TarArchive;
import org.springframework.boot.buildpack.platform.json.AbstractJsonTests;
import static org.assertj.core.api.Assertions.assertThat;
@ -173,20 +175,20 @@ class ImageBuildpackTests extends AbstractJsonTests {
private Object withMockLayers(InvocationOnMock invocation) {
try {
IOBiConsumer<String, TarArchive> consumer = invocation.getArgument(1);
TarArchive archive = (out) -> {
try (TarArchiveOutputStream tarOut = new TarArchiveOutputStream(out)) {
tarOut.setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX);
writeTarEntry(tarOut, "/cnb/");
writeTarEntry(tarOut, "/cnb/buildpacks/");
writeTarEntry(tarOut, "/cnb/buildpacks/example_buildpack/");
writeTarEntry(tarOut, "/cnb/buildpacks/example_buildpack/0.0.1/");
writeTarEntry(tarOut, "/cnb/buildpacks/example_buildpack/0.0.1/buildpack.toml");
writeTarEntry(tarOut, "/cnb/buildpacks/example_buildpack/0.0.1/" + this.longFilePath);
tarOut.finish();
}
};
consumer.accept("test", archive);
IOBiConsumer<String, Path> consumer = invocation.getArgument(1);
File tarFile = File.createTempFile("create-builder-test-", null);
FileOutputStream out = new FileOutputStream(tarFile);
try (TarArchiveOutputStream tarOut = new TarArchiveOutputStream(out)) {
tarOut.setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX);
writeTarEntry(tarOut, "/cnb/");
writeTarEntry(tarOut, "/cnb/buildpacks/");
writeTarEntry(tarOut, "/cnb/buildpacks/example_buildpack/");
writeTarEntry(tarOut, "/cnb/buildpacks/example_buildpack/0.0.1/");
writeTarEntry(tarOut, "/cnb/buildpacks/example_buildpack/0.0.1/buildpack.toml");
writeTarEntry(tarOut, "/cnb/buildpacks/example_buildpack/0.0.1/" + this.longFilePath);
tarOut.finish();
}
consumer.accept("test", tarFile.toPath());
}
catch (IOException ex) {
fail("Error writing mock layers", ex);

View File

@ -21,6 +21,9 @@ import java.io.ByteArrayOutputStream;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URI;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
@ -310,14 +313,14 @@ class DockerApiTests {
@Test
void exportLayersWhenReferenceIsNullThrowsException() {
assertThatIllegalArgumentException().isThrownBy(() -> this.api.exportLayers(null, (name, archive) -> {
assertThatIllegalArgumentException().isThrownBy(() -> this.api.exportLayerFiles(null, (name, archive) -> {
})).withMessage("Reference must not be null");
}
@Test
void exportLayersWhenExportsIsNullThrowsException() {
ImageReference reference = ImageReference.of("gcr.io/paketo-buildpacks/builder:base");
assertThatIllegalArgumentException().isThrownBy(() -> this.api.exportLayers(reference, null))
assertThatIllegalArgumentException().isThrownBy(() -> this.api.exportLayerFiles(reference, null))
.withMessage("Exports must not be null");
}
@ -340,11 +343,62 @@ class DockerApiTests {
}
});
assertThat(contents).hasSize(3)
.containsKeys("1bf6c63a1e9ed1dd7cb961273bf60b8e0f440361faf273baf866f408e4910601/layer.tar",
"8fdfb915302159a842cbfae6faec5311b00c071ebf14e12da7116ae7532e9319/layer.tar",
"93cd584bb189bfca4f51744bd19d836fd36da70710395af5a1523ee88f208c6a/layer.tar");
assertThat(contents.get("1bf6c63a1e9ed1dd7cb961273bf60b8e0f440361faf273baf866f408e4910601/layer.tar"))
.containsExactly("etc/", "etc/apt/", "etc/apt/sources.list");
.containsKeys("70bb7a3115f3d5c01099852112c7e05bf593789e510468edb06b6a9a11fa3b73/layer.tar",
"74a9a50ece13c025cf10e9110d9ddc86c995079c34e2a22a28d1a3d523222c6e/layer.tar",
"a69532b5b92bb891fbd9fa1a6b3af9087ea7050255f59ba61a796f8555ecd783/layer.tar");
assertThat(contents.get("70bb7a3115f3d5c01099852112c7e05bf593789e510468edb06b6a9a11fa3b73/layer.tar"))
.containsExactly("/cnb/order.toml");
assertThat(contents.get("74a9a50ece13c025cf10e9110d9ddc86c995079c34e2a22a28d1a3d523222c6e/layer.tar"))
.containsExactly("/cnb/stack.toml");
}
@Test
void exportLayersWithSymlinksExportsLayerTars() throws Exception {
ImageReference reference = ImageReference.of("gcr.io/paketo-buildpacks/builder:base");
URI exportUri = new URI(IMAGES_URL + "/gcr.io/paketo-buildpacks/builder:base/get");
given(DockerApiTests.this.http.get(exportUri)).willReturn(responseOf("export-symlinks.tar"));
MultiValueMap<String, String> contents = new LinkedMultiValueMap<>();
this.api.exportLayers(reference, (name, archive) -> {
ByteArrayOutputStream out = new ByteArrayOutputStream();
archive.writeTo(out);
try (TarArchiveInputStream in = new TarArchiveInputStream(
new ByteArrayInputStream(out.toByteArray()))) {
TarArchiveEntry entry = in.getNextTarEntry();
while (entry != null) {
contents.add(name, entry.getName());
entry = in.getNextTarEntry();
}
}
});
assertThat(contents).hasSize(3)
.containsKeys("6aa3691a73805f608e5fce69fb6bc89aec8362f58a6b4be2682515e9cfa3cc1a.tar",
"762e198f655bc2580ef3e56b538810fd2b9981bd707f8a44c70344b58f9aee68.tar",
"d3cc975ad97fdfbb73d9daf157e7f658d6117249fd9c237e3856ad173c87e1d2.tar");
assertThat(contents.get("d3cc975ad97fdfbb73d9daf157e7f658d6117249fd9c237e3856ad173c87e1d2.tar"))
.containsExactly("/cnb/order.toml");
assertThat(contents.get("762e198f655bc2580ef3e56b538810fd2b9981bd707f8a44c70344b58f9aee68.tar"))
.containsExactly("/cnb/stack.toml");
}
@Test
void exportLayerFilesDeletesTempFiles() throws Exception {
ImageReference reference = ImageReference.of("gcr.io/paketo-buildpacks/builder:base");
URI exportUri = new URI(IMAGES_URL + "/gcr.io/paketo-buildpacks/builder:base/get");
given(DockerApiTests.this.http.get(exportUri)).willReturn(responseOf("export.tar"));
List<Path> layerFilePaths = new ArrayList<>();
this.api.exportLayerFiles(reference, (name, path) -> layerFilePaths.add(path));
layerFilePaths.forEach((path) -> assertThat(path.toFile()).doesNotExist());
}
@Test
void exportLayersWithNoManifestThrowsException() throws Exception {
ImageReference reference = ImageReference.of("gcr.io/paketo-buildpacks/builder:base");
URI exportUri = new URI(IMAGES_URL + "/gcr.io/paketo-buildpacks/builder:base/get");
given(DockerApiTests.this.http.get(exportUri)).willReturn(responseOf("export-no-manifest.tar"));
assertThatIllegalArgumentException()
.isThrownBy(() -> this.api.exportLayerFiles(reference, (name, archive) -> {
}))
.withMessageContaining("Manifest not found in image " + reference);
}
@Test

View File

@ -0,0 +1,69 @@
/*
* Copyright 2012-2023 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.boot.buildpack.platform.docker.type;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.junit.jupiter.api.Test;
import org.springframework.boot.buildpack.platform.json.AbstractJsonTests;
import static org.assertj.core.api.Assertions.assertThat;
/**
* Tests for {@link ImageArchiveManifest}.
*
* @author Scott Frederick
* @author Andy Wilkinson
*/
class ImageArchiveManifestTests extends AbstractJsonTests {
@Test
void getLayersReturnsLayers() throws Exception {
ImageArchiveManifest manifest = getManifest();
List<String> expectedLayers = new ArrayList<>();
for (int blankLayersCount = 0; blankLayersCount < 46; blankLayersCount++) {
expectedLayers.add("blank_" + blankLayersCount);
}
expectedLayers.add("bb09e17fd1bd2ee47155f1349645fcd9fff31e1247c7ed99cad469f1c16a4216.tar");
assertThat(manifest.getEntries()).hasSize(1);
assertThat(manifest.getEntries().get(0).getLayers()).hasSize(47);
assertThat(manifest.getEntries().get(0).getLayers()).isEqualTo(expectedLayers);
}
@Test
void getLayersWithNoLayersReturnsEmptyList() throws Exception {
String content = "[{\"Layers\": []}]";
ImageArchiveManifest manifest = new ImageArchiveManifest(getObjectMapper().readTree(content));
assertThat(manifest.getEntries()).hasSize(1);
assertThat(manifest.getEntries().get(0).getLayers()).hasSize(0);
}
@Test
void getLayersWithEmptyManifestReturnsEmptyList() throws Exception {
String content = "[]";
ImageArchiveManifest manifest = new ImageArchiveManifest(getObjectMapper().readTree(content));
assertThat(manifest.getEntries()).isEmpty();
}
private ImageArchiveManifest getManifest() throws IOException {
return new ImageArchiveManifest(getObjectMapper().readTree(getContent("image-archive-manifest.json")));
}
}