Merge branch '3.0.x'

Closes gh-34496
This commit is contained in:
Scott Frederick 2023-03-06 15:31:09 -06:00
commit dab8b07302
11 changed files with 329 additions and 49 deletions

View File

@ -1,5 +1,5 @@
/*
* Copyright 2012-2022 the original author or authors.
* Copyright 2012-2023 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -17,6 +17,7 @@
package org.springframework.boot.buildpack.platform.build;
import java.io.IOException;
import java.nio.file.Path;
import java.util.List;
import java.util.function.Consumer;
@ -32,7 +33,6 @@ import org.springframework.boot.buildpack.platform.docker.transport.DockerEngine
import org.springframework.boot.buildpack.platform.docker.type.Image;
import org.springframework.boot.buildpack.platform.docker.type.ImageReference;
import org.springframework.boot.buildpack.platform.io.IOBiConsumer;
import org.springframework.boot.buildpack.platform.io.TarArchive;
import org.springframework.util.Assert;
import org.springframework.util.StringUtils;
@ -273,9 +273,8 @@ public class Builder {
}
@Override
public void exportImageLayers(ImageReference reference, IOBiConsumer<String, TarArchive> exports)
throws IOException {
Builder.this.docker.image().exportLayers(reference, exports);
public void exportImageLayers(ImageReference reference, IOBiConsumer<String, Path> exports) throws IOException {
Builder.this.docker.image().exportLayerFiles(reference, exports);
}
}

View File

@ -1,5 +1,5 @@
/*
* Copyright 2012-2022 the original author or authors.
* Copyright 2012-2023 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -17,12 +17,12 @@
package org.springframework.boot.buildpack.platform.build;
import java.io.IOException;
import java.nio.file.Path;
import java.util.List;
import org.springframework.boot.buildpack.platform.docker.type.Image;
import org.springframework.boot.buildpack.platform.docker.type.ImageReference;
import org.springframework.boot.buildpack.platform.io.IOBiConsumer;
import org.springframework.boot.buildpack.platform.io.TarArchive;
/**
* Context passed to a {@link BuildpackResolver}.
@ -52,6 +52,6 @@ interface BuildpackResolverContext {
* during the callback)
* @throws IOException on IO error
*/
void exportImageLayers(ImageReference reference, IOBiConsumer<String, TarArchive> exports) throws IOException;
void exportImageLayers(ImageReference reference, IOBiConsumer<String, Path> exports) throws IOException;
}

View File

@ -17,6 +17,7 @@
package org.springframework.boot.buildpack.platform.build;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.file.Files;
import java.nio.file.Path;
@ -35,7 +36,6 @@ import org.springframework.boot.buildpack.platform.docker.type.ImageReference;
import org.springframework.boot.buildpack.platform.docker.type.Layer;
import org.springframework.boot.buildpack.platform.docker.type.LayerId;
import org.springframework.boot.buildpack.platform.io.IOConsumer;
import org.springframework.boot.buildpack.platform.io.TarArchive;
import org.springframework.util.StreamUtils;
/**
@ -115,23 +115,16 @@ final class ImageBuildpack implements Buildpack {
ExportedLayers(BuildpackResolverContext context, ImageReference imageReference) throws IOException {
List<Path> layerFiles = new ArrayList<>();
context.exportImageLayers(imageReference, (name, archive) -> layerFiles.add(copyToTemp(name, archive)));
context.exportImageLayers(imageReference, (name, path) -> layerFiles.add(copyToTemp(path)));
this.layerFiles = Collections.unmodifiableList(layerFiles);
}
private Path copyToTemp(String name, TarArchive archive) throws IOException {
String[] parts = name.split("/");
Path path = Files.createTempFile("create-builder-scratch-", parts[0]);
try (OutputStream out = Files.newOutputStream(path)) {
archive.writeTo(out);
}
return path;
}
void apply(IOConsumer<Layer> layers) throws IOException {
for (Path path : this.layerFiles) {
layers.accept(Layer.fromTarArchive((out) -> copyLayerTar(path, out)));
private Path copyToTemp(Path path) throws IOException {
Path outputPath = Files.createTempFile("create-builder-scratch-", null);
try (OutputStream out = Files.newOutputStream(outputPath)) {
copyLayerTar(path, out);
}
return outputPath;
}
private void copyLayerTar(Path path, OutputStream out) throws IOException {
@ -147,7 +140,16 @@ final class ImageBuildpack implements Buildpack {
}
tarOut.finish();
}
Files.delete(path);
}
void apply(IOConsumer<Layer> layers) throws IOException {
for (Path path : this.layerFiles) {
layers.accept(Layer.fromTarArchive((out) -> {
InputStream in = Files.newInputStream(path);
StreamUtils.copy(in, out);
}));
Files.delete(path);
}
}
}

View File

@ -16,13 +16,24 @@
package org.springframework.boot.buildpack.platform.docker;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
@ -37,6 +48,7 @@ import org.springframework.boot.buildpack.platform.docker.type.ContainerReferenc
import org.springframework.boot.buildpack.platform.docker.type.ContainerStatus;
import org.springframework.boot.buildpack.platform.docker.type.Image;
import org.springframework.boot.buildpack.platform.docker.type.ImageArchive;
import org.springframework.boot.buildpack.platform.docker.type.ImageArchiveManifest;
import org.springframework.boot.buildpack.platform.docker.type.ImageReference;
import org.springframework.boot.buildpack.platform.docker.type.VolumeName;
import org.springframework.boot.buildpack.platform.io.IOBiConsumer;
@ -250,7 +262,7 @@ public class DockerApi {
}
/**
* Export the layers of an image.
* Export the layers of an image as {@link TarArchive}s.
* @param reference the reference to export
* @param exports a consumer to receive the layers (contents can only be accessed
* during the callback)
@ -258,20 +270,49 @@ public class DockerApi {
*/
public void exportLayers(ImageReference reference, IOBiConsumer<String, TarArchive> exports)
throws IOException {
exportLayerFiles(reference, (name, path) -> {
try (InputStream in = Files.newInputStream(path)) {
TarArchive archive = (out) -> StreamUtils.copy(in, out);
exports.accept(name, archive);
}
});
}
/**
* Export the layers of an image as paths to layer tar files.
* @param reference the reference to export
* @param exports a consumer to receive the layer tar file paths (file can only be
* accessed during the callback)
* @throws IOException on IO error
*/
public void exportLayerFiles(ImageReference reference, IOBiConsumer<String, Path> exports) throws IOException {
Assert.notNull(reference, "Reference must not be null");
Assert.notNull(exports, "Exports must not be null");
URI saveUri = buildUrl("/images/" + reference + "/get");
Response response = http().get(saveUri);
ImageArchiveManifest manifest = null;
Map<String, Path> layerFiles = new HashMap<>();
try (TarArchiveInputStream tar = new TarArchiveInputStream(response.getContent())) {
TarArchiveEntry entry = tar.getNextTarEntry();
while (entry != null) {
if (entry.getName().endsWith("/layer.tar")) {
TarArchive archive = (out) -> StreamUtils.copy(tar, out);
exports.accept(entry.getName(), archive);
if (entry.getName().equals("manifest.json")) {
manifest = readManifest(tar);
}
if (entry.getName().endsWith(".tar")) {
layerFiles.put(entry.getName(), copyToTemp(tar));
}
entry = tar.getNextTarEntry();
}
}
Assert.notNull(manifest, "Manifest not found in image " + reference);
for (Map.Entry<String, Path> entry : layerFiles.entrySet()) {
String name = entry.getKey();
Path path = entry.getValue();
if (manifestContainsLayerEntry(manifest, name)) {
exports.accept(name, path);
}
Files.delete(path);
}
}
/**
@ -308,6 +349,24 @@ public class DockerApi {
http().post(uri).close();
}
private ImageArchiveManifest readManifest(TarArchiveInputStream tar) throws IOException {
String manifestContent = new BufferedReader(new InputStreamReader(tar, StandardCharsets.UTF_8)).lines()
.collect(Collectors.joining());
return ImageArchiveManifest.of(new ByteArrayInputStream(manifestContent.getBytes(StandardCharsets.UTF_8)));
}
private Path copyToTemp(TarArchiveInputStream in) throws IOException {
Path path = Files.createTempFile("create-builder-scratch-", null);
try (OutputStream out = Files.newOutputStream(path)) {
StreamUtils.copy(in, out);
}
return path;
}
private boolean manifestContainsLayerEntry(ImageArchiveManifest manifest, String layerId) {
return manifest.getEntries().stream().anyMatch((content) -> content.getLayers().contains(layerId));
}
}
/**

View File

@ -0,0 +1,95 @@
/*
* Copyright 2012-2023 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.boot.buildpack.platform.docker.type;
import java.io.IOException;
import java.io.InputStream;
import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import com.fasterxml.jackson.databind.JsonNode;
import org.springframework.boot.buildpack.platform.json.MappedObject;
/**
* Image archive manifest information.
*
* @author Scott Frederick
* @since 2.7.9
*/
public class ImageArchiveManifest extends MappedObject {
private final List<ManifestEntry> entries = new ArrayList<>();
protected ImageArchiveManifest(JsonNode node) {
super(node, MethodHandles.lookup());
getNode().elements().forEachRemaining((element) -> this.entries.add(ManifestEntry.of(element)));
}
/**
* Return the entries contained in the manifest.
* @return the manifest entries
*/
public List<ManifestEntry> getEntries() {
return this.entries;
}
/**
* Create an {@link ImageArchiveManifest} from the provided JSON input stream.
* @param content the JSON input stream
* @return a new {@link ImageArchiveManifest} instance
* @throws IOException on IO error
*/
public static ImageArchiveManifest of(InputStream content) throws IOException {
return of(content, ImageArchiveManifest::new);
}
public static class ManifestEntry extends MappedObject {
private final List<String> layers;
protected ManifestEntry(JsonNode node) {
super(node, MethodHandles.lookup());
this.layers = extractLayers();
}
/**
* Return the collection of layer IDs from a section of the manifest.
* @return a collection of layer IDs
*/
public List<String> getLayers() {
return this.layers;
}
static ManifestEntry of(JsonNode node) {
return new ManifestEntry(node);
}
@SuppressWarnings("unchecked")
private List<String> extractLayers() {
List<String> layers = valueAt("/Layers", List.class);
if (layers == null) {
return Collections.emptyList();
}
return layers;
}
}
}

View File

@ -18,7 +18,10 @@ package org.springframework.boot.buildpack.platform.build;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;
@ -33,7 +36,6 @@ import org.mockito.invocation.InvocationOnMock;
import org.springframework.boot.buildpack.platform.docker.type.Image;
import org.springframework.boot.buildpack.platform.docker.type.ImageReference;
import org.springframework.boot.buildpack.platform.io.IOBiConsumer;
import org.springframework.boot.buildpack.platform.io.TarArchive;
import org.springframework.boot.buildpack.platform.json.AbstractJsonTests;
import static org.assertj.core.api.Assertions.assertThat;
@ -173,20 +175,20 @@ class ImageBuildpackTests extends AbstractJsonTests {
private Object withMockLayers(InvocationOnMock invocation) {
try {
IOBiConsumer<String, TarArchive> consumer = invocation.getArgument(1);
TarArchive archive = (out) -> {
try (TarArchiveOutputStream tarOut = new TarArchiveOutputStream(out)) {
tarOut.setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX);
writeTarEntry(tarOut, "/cnb/");
writeTarEntry(tarOut, "/cnb/buildpacks/");
writeTarEntry(tarOut, "/cnb/buildpacks/example_buildpack/");
writeTarEntry(tarOut, "/cnb/buildpacks/example_buildpack/0.0.1/");
writeTarEntry(tarOut, "/cnb/buildpacks/example_buildpack/0.0.1/buildpack.toml");
writeTarEntry(tarOut, "/cnb/buildpacks/example_buildpack/0.0.1/" + this.longFilePath);
tarOut.finish();
}
};
consumer.accept("test", archive);
IOBiConsumer<String, Path> consumer = invocation.getArgument(1);
File tarFile = File.createTempFile("create-builder-test-", null);
FileOutputStream out = new FileOutputStream(tarFile);
try (TarArchiveOutputStream tarOut = new TarArchiveOutputStream(out)) {
tarOut.setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX);
writeTarEntry(tarOut, "/cnb/");
writeTarEntry(tarOut, "/cnb/buildpacks/");
writeTarEntry(tarOut, "/cnb/buildpacks/example_buildpack/");
writeTarEntry(tarOut, "/cnb/buildpacks/example_buildpack/0.0.1/");
writeTarEntry(tarOut, "/cnb/buildpacks/example_buildpack/0.0.1/buildpack.toml");
writeTarEntry(tarOut, "/cnb/buildpacks/example_buildpack/0.0.1/" + this.longFilePath);
tarOut.finish();
}
consumer.accept("test", tarFile.toPath());
}
catch (IOException ex) {
fail("Error writing mock layers", ex);

View File

@ -21,6 +21,9 @@ import java.io.ByteArrayOutputStream;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URI;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
@ -310,14 +313,14 @@ class DockerApiTests {
@Test
void exportLayersWhenReferenceIsNullThrowsException() {
assertThatIllegalArgumentException().isThrownBy(() -> this.api.exportLayers(null, (name, archive) -> {
assertThatIllegalArgumentException().isThrownBy(() -> this.api.exportLayerFiles(null, (name, archive) -> {
})).withMessage("Reference must not be null");
}
@Test
void exportLayersWhenExportsIsNullThrowsException() {
ImageReference reference = ImageReference.of("gcr.io/paketo-buildpacks/builder:base");
assertThatIllegalArgumentException().isThrownBy(() -> this.api.exportLayers(reference, null))
assertThatIllegalArgumentException().isThrownBy(() -> this.api.exportLayerFiles(reference, null))
.withMessage("Exports must not be null");
}
@ -340,11 +343,62 @@ class DockerApiTests {
}
});
assertThat(contents).hasSize(3)
.containsKeys("1bf6c63a1e9ed1dd7cb961273bf60b8e0f440361faf273baf866f408e4910601/layer.tar",
"8fdfb915302159a842cbfae6faec5311b00c071ebf14e12da7116ae7532e9319/layer.tar",
"93cd584bb189bfca4f51744bd19d836fd36da70710395af5a1523ee88f208c6a/layer.tar");
assertThat(contents.get("1bf6c63a1e9ed1dd7cb961273bf60b8e0f440361faf273baf866f408e4910601/layer.tar"))
.containsExactly("etc/", "etc/apt/", "etc/apt/sources.list");
.containsKeys("70bb7a3115f3d5c01099852112c7e05bf593789e510468edb06b6a9a11fa3b73/layer.tar",
"74a9a50ece13c025cf10e9110d9ddc86c995079c34e2a22a28d1a3d523222c6e/layer.tar",
"a69532b5b92bb891fbd9fa1a6b3af9087ea7050255f59ba61a796f8555ecd783/layer.tar");
assertThat(contents.get("70bb7a3115f3d5c01099852112c7e05bf593789e510468edb06b6a9a11fa3b73/layer.tar"))
.containsExactly("/cnb/order.toml");
assertThat(contents.get("74a9a50ece13c025cf10e9110d9ddc86c995079c34e2a22a28d1a3d523222c6e/layer.tar"))
.containsExactly("/cnb/stack.toml");
}
@Test
void exportLayersWithSymlinksExportsLayerTars() throws Exception {
ImageReference reference = ImageReference.of("gcr.io/paketo-buildpacks/builder:base");
URI exportUri = new URI(IMAGES_URL + "/gcr.io/paketo-buildpacks/builder:base/get");
given(DockerApiTests.this.http.get(exportUri)).willReturn(responseOf("export-symlinks.tar"));
MultiValueMap<String, String> contents = new LinkedMultiValueMap<>();
this.api.exportLayers(reference, (name, archive) -> {
ByteArrayOutputStream out = new ByteArrayOutputStream();
archive.writeTo(out);
try (TarArchiveInputStream in = new TarArchiveInputStream(
new ByteArrayInputStream(out.toByteArray()))) {
TarArchiveEntry entry = in.getNextTarEntry();
while (entry != null) {
contents.add(name, entry.getName());
entry = in.getNextTarEntry();
}
}
});
assertThat(contents).hasSize(3)
.containsKeys("6aa3691a73805f608e5fce69fb6bc89aec8362f58a6b4be2682515e9cfa3cc1a.tar",
"762e198f655bc2580ef3e56b538810fd2b9981bd707f8a44c70344b58f9aee68.tar",
"d3cc975ad97fdfbb73d9daf157e7f658d6117249fd9c237e3856ad173c87e1d2.tar");
assertThat(contents.get("d3cc975ad97fdfbb73d9daf157e7f658d6117249fd9c237e3856ad173c87e1d2.tar"))
.containsExactly("/cnb/order.toml");
assertThat(contents.get("762e198f655bc2580ef3e56b538810fd2b9981bd707f8a44c70344b58f9aee68.tar"))
.containsExactly("/cnb/stack.toml");
}
@Test
void exportLayerFilesDeletesTempFiles() throws Exception {
ImageReference reference = ImageReference.of("gcr.io/paketo-buildpacks/builder:base");
URI exportUri = new URI(IMAGES_URL + "/gcr.io/paketo-buildpacks/builder:base/get");
given(DockerApiTests.this.http.get(exportUri)).willReturn(responseOf("export.tar"));
List<Path> layerFilePaths = new ArrayList<>();
this.api.exportLayerFiles(reference, (name, path) -> layerFilePaths.add(path));
layerFilePaths.forEach((path) -> assertThat(path.toFile()).doesNotExist());
}
@Test
void exportLayersWithNoManifestThrowsException() throws Exception {
ImageReference reference = ImageReference.of("gcr.io/paketo-buildpacks/builder:base");
URI exportUri = new URI(IMAGES_URL + "/gcr.io/paketo-buildpacks/builder:base/get");
given(DockerApiTests.this.http.get(exportUri)).willReturn(responseOf("export-no-manifest.tar"));
assertThatIllegalArgumentException()
.isThrownBy(() -> this.api.exportLayerFiles(reference, (name, archive) -> {
}))
.withMessageContaining("Manifest not found in image " + reference);
}
@Test

View File

@ -0,0 +1,69 @@
/*
* Copyright 2012-2023 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.boot.buildpack.platform.docker.type;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.junit.jupiter.api.Test;
import org.springframework.boot.buildpack.platform.json.AbstractJsonTests;
import static org.assertj.core.api.Assertions.assertThat;
/**
* Tests for {@link ImageArchiveManifest}.
*
* @author Scott Frederick
* @author Andy Wilkinson
*/
class ImageArchiveManifestTests extends AbstractJsonTests {
@Test
void getLayersReturnsLayers() throws Exception {
ImageArchiveManifest manifest = getManifest();
List<String> expectedLayers = new ArrayList<>();
for (int blankLayersCount = 0; blankLayersCount < 46; blankLayersCount++) {
expectedLayers.add("blank_" + blankLayersCount);
}
expectedLayers.add("bb09e17fd1bd2ee47155f1349645fcd9fff31e1247c7ed99cad469f1c16a4216.tar");
assertThat(manifest.getEntries()).hasSize(1);
assertThat(manifest.getEntries().get(0).getLayers()).hasSize(47);
assertThat(manifest.getEntries().get(0).getLayers()).isEqualTo(expectedLayers);
}
@Test
void getLayersWithNoLayersReturnsEmptyList() throws Exception {
String content = "[{\"Layers\": []}]";
ImageArchiveManifest manifest = new ImageArchiveManifest(getObjectMapper().readTree(content));
assertThat(manifest.getEntries()).hasSize(1);
assertThat(manifest.getEntries().get(0).getLayers()).hasSize(0);
}
@Test
void getLayersWithEmptyManifestReturnsEmptyList() throws Exception {
String content = "[]";
ImageArchiveManifest manifest = new ImageArchiveManifest(getObjectMapper().readTree(content));
assertThat(manifest.getEntries()).isEmpty();
}
private ImageArchiveManifest getManifest() throws IOException {
return new ImageArchiveManifest(getObjectMapper().readTree(getContent("image-archive-manifest.json")));
}
}