Check if promotion has already occurred

Update the release tooling to check for bintray published artifacts
using SHA256 digests and to also check before attempting a promote.

See gh-21474
This commit is contained in:
Phillip Webb 2020-06-11 21:03:01 -07:00
parent d650c5fdf2
commit ce011ca384
11 changed files with 189 additions and 55 deletions

View File

@ -17,6 +17,8 @@
package io.spring.concourse.releasescripts.artifactory;
import java.net.URI;
import java.time.Duration;
import java.util.Set;
import io.spring.concourse.releasescripts.ReleaseInfo;
import io.spring.concourse.releasescripts.artifactory.payload.BuildInfoResponse;
@ -113,8 +115,12 @@ public class ArtifactoryService {
* Deploy builds from Artifactory to Bintray.
* @param sourceRepo the source repo in Artifactory.
* @param releaseInfo the resease info
* @param artifactDigests the artifact digests
*/
public void distribute(String sourceRepo, ReleaseInfo releaseInfo) {
public void distribute(String sourceRepo, ReleaseInfo releaseInfo, Set<String> artifactDigests) {
if (this.bintrayService.isDistributionComplete(releaseInfo, artifactDigests, Duration.ofMinutes(2))) {
console.log("Distribution already complete");
}
DistributionRequest request = new DistributionRequest(new String[] { sourceRepo });
RequestEntity<DistributionRequest> requestEntity = RequestEntity
.post(URI.create(DISTRIBUTION_URL + releaseInfo.getBuildName() + "/" + releaseInfo.getBuildNumber()))
@ -126,7 +132,7 @@ public class ArtifactoryService {
console.log("Failed to distribute.");
throw ex;
}
if (!this.bintrayService.isDistributionComplete(releaseInfo)) {
if (!this.bintrayService.isDistributionComplete(releaseInfo, artifactDigests, Duration.ofMinutes(60))) {
throw new DistributionTimeoutException("Distribution timed out.");
}

View File

@ -16,6 +16,11 @@
package io.spring.concourse.releasescripts.artifactory.payload;
import java.util.Arrays;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
/**
* Represents the response from Artifactory's buildInfo endpoint.
*
@ -54,7 +59,7 @@ public class BuildInfoResponse {
}
public String getName() {
return name;
return this.name;
}
public void setName(String name) {
@ -83,6 +88,14 @@ public class BuildInfoResponse {
public void setVersion(String version) {
this.version = version;
}
public Set<String> getArtifactDigests() {
return Arrays.stream(this.modules).flatMap((module) -> {
Artifact[] artifacts = module.getArtifacts();
return (artifacts != null) ? Arrays.stream(artifacts) : Stream.empty();
}).map(Artifact::getSha256).collect(Collectors.toSet());
}
}
@ -105,6 +118,8 @@ public class BuildInfoResponse {
private String id;
private Artifact[] artifacts;
public String getId() {
return this.id;
}
@ -113,6 +128,38 @@ public class BuildInfoResponse {
this.id = id;
}
public Artifact[] getArtifacts() {
return this.artifacts;
}
public void setArtifacts(Artifact[] artifacts) {
this.artifacts = artifacts;
}
}
public static class Artifact {
private String name;
private String sha256;
public String getName() {
return this.name;
}
public void setName(String name) {
this.name = name;
}
public String getSha256() {
return this.sha256;
}
public void setSha256(String sha256) {
this.sha256 = sha256;
}
}
}

View File

@ -17,8 +17,9 @@
package io.spring.concourse.releasescripts.bintray;
import java.net.URI;
import java.util.Objects;
import java.util.concurrent.TimeUnit;
import java.time.Duration;
import java.util.HashSet;
import java.util.Set;
import io.spring.concourse.releasescripts.ReleaseInfo;
import io.spring.concourse.releasescripts.sonatype.SonatypeProperties;
@ -27,7 +28,6 @@ import io.spring.concourse.releasescripts.system.ConsoleLogger;
import org.awaitility.core.ConditionTimeoutException;
import org.springframework.boot.web.client.RestTemplateBuilder;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.RequestEntity;
import org.springframework.stereotype.Component;
@ -72,24 +72,17 @@ public class BintrayService {
this.restTemplate = builder.build();
}
public boolean isDistributionComplete(ReleaseInfo releaseInfo) {
RequestEntity<Void> allFilesRequest = getRequest(releaseInfo, 1);
Object[] allFiles = waitAtMost(5, TimeUnit.MINUTES).with().pollDelay(20, TimeUnit.SECONDS).until(() -> {
try {
return this.restTemplate.exchange(allFilesRequest, Object[].class).getBody();
}
catch (HttpClientErrorException ex) {
if (ex.getStatusCode() != HttpStatus.NOT_FOUND) {
throw ex;
}
return null;
}
}, Objects::nonNull);
RequestEntity<Void> publishedFilesRequest = getRequest(releaseInfo, 0);
public boolean isDistributionComplete(ReleaseInfo releaseInfo, Set<String> requiredDigets, Duration timeout) {
return isDistributionComplete(releaseInfo, requiredDigets, timeout, Duration.ofSeconds(20));
}
public boolean isDistributionComplete(ReleaseInfo releaseInfo, Set<String> requiredDigets, Duration timeout,
Duration pollDelay) {
RequestEntity<Void> request = getRequest(releaseInfo, 0);
try {
waitAtMost(120, TimeUnit.MINUTES).with().pollDelay(20, TimeUnit.SECONDS).until(() -> {
Object[] publishedFiles = this.restTemplate.exchange(publishedFilesRequest, Object[].class).getBody();
return allFiles.length == publishedFiles.length;
waitAtMost(timeout).with().pollDelay(pollDelay).until(() -> {
PackageFile[] published = this.restTemplate.exchange(request, PackageFile[].class).getBody();
return hasPublishedAll(published, requiredDigets);
});
}
catch (ConditionTimeoutException ex) {
@ -98,6 +91,17 @@ public class BintrayService {
return true;
}
private boolean hasPublishedAll(PackageFile[] published, Set<String> requiredDigets) {
if (published == null || published.length == 0) {
return false;
}
Set<String> remaining = new HashSet<>(requiredDigets);
for (PackageFile publishedFile : published) {
remaining.remove(publishedFile.getSha256());
}
return remaining.isEmpty();
}
private RequestEntity<Void> getRequest(ReleaseInfo releaseInfo, int includeUnpublished) {
return RequestEntity.get(URI.create(BINTRAY_URL + "packages/" + this.bintrayProperties.getSubject() + "/"
+ this.bintrayProperties.getRepo() + "/" + releaseInfo.getGroupId() + "/versions/"

View File

@ -0,0 +1,38 @@
/*
* Copyright 2020 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.spring.concourse.releasescripts.bintray;
/**
* Details for a single packaged file.
*
* @author Phillip Webb
*/
public class PackageFile {
private String name;
private String sha256;
public String getName() {
return this.name;
}
public String getSha256() {
return this.sha256;
}
}

View File

@ -19,12 +19,14 @@ package io.spring.concourse.releasescripts.command;
import java.io.File;
import java.nio.file.Files;
import java.util.List;
import java.util.Set;
import com.fasterxml.jackson.databind.ObjectMapper;
import io.spring.concourse.releasescripts.ReleaseInfo;
import io.spring.concourse.releasescripts.ReleaseType;
import io.spring.concourse.releasescripts.artifactory.ArtifactoryService;
import io.spring.concourse.releasescripts.artifactory.payload.BuildInfoResponse;
import io.spring.concourse.releasescripts.artifactory.payload.BuildInfoResponse.BuildInfo;
import org.springframework.boot.ApplicationArguments;
import org.springframework.stereotype.Component;
@ -38,12 +40,12 @@ import org.springframework.util.Assert;
@Component
public class DistributeCommand implements Command {
private final ArtifactoryService service;
private final ArtifactoryService artifactoryService;
private final ObjectMapper objectMapper;
public DistributeCommand(ArtifactoryService service, ObjectMapper objectMapper) {
this.service = service;
public DistributeCommand(ArtifactoryService artifactoryService, ObjectMapper objectMapper) {
this.artifactoryService = artifactoryService;
this.objectMapper = objectMapper;
}
@ -60,8 +62,10 @@ public class DistributeCommand implements Command {
String buildInfoLocation = nonOptionArgs.get(2);
byte[] content = Files.readAllBytes(new File(buildInfoLocation).toPath());
BuildInfoResponse buildInfoResponse = this.objectMapper.readValue(content, BuildInfoResponse.class);
ReleaseInfo releaseInfo = ReleaseInfo.from(buildInfoResponse.getBuildInfo());
this.service.distribute(type.getRepo(), releaseInfo);
BuildInfo buildInfo = buildInfoResponse.getBuildInfo();
Set<String> artifactDigests = buildInfo.getArtifactDigests();
ReleaseInfo releaseInfo = ReleaseInfo.from(buildInfo);
this.artifactoryService.distribute(type.getRepo(), releaseInfo, artifactDigests);
}
}

View File

@ -16,10 +16,15 @@
package io.spring.concourse.releasescripts.artifactory;
import java.time.Duration;
import java.util.Collections;
import java.util.Set;
import io.spring.concourse.releasescripts.ReleaseInfo;
import io.spring.concourse.releasescripts.bintray.BintrayService;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
import org.mockito.InOrder;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
@ -35,10 +40,13 @@ import org.springframework.util.Base64Utils;
import org.springframework.web.client.HttpClientErrorException;
import static org.assertj.core.api.Assertions.assertThatExceptionOfType;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.BDDMockito.given;
import static org.mockito.Mockito.inOrder;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoInteractions;
import static org.mockito.Mockito.verifyNoMoreInteractions;
import static org.springframework.test.web.client.match.MockRestRequestMatchers.content;
import static org.springframework.test.web.client.match.MockRestRequestMatchers.header;
import static org.springframework.test.web.client.match.MockRestRequestMatchers.method;
@ -55,6 +63,10 @@ import static org.springframework.test.web.client.response.MockRestResponseCreat
@EnableConfigurationProperties(ArtifactoryProperties.class)
class ArtifactoryServiceTests {
private static final Duration SHORT_TIMEOUT = Duration.ofMinutes(2);
private static final Duration LONG_TIMEOUT = Duration.ofMinutes(60);
@Autowired
private ArtifactoryService service;
@ -118,9 +130,10 @@ class ArtifactoryServiceTests {
}
@Test
@SuppressWarnings("unchecked")
void distributeWhenSuccessful() throws Exception {
ReleaseInfo releaseInfo = getReleaseInfo();
given(this.bintrayService.isDistributionComplete(releaseInfo)).willReturn(true);
given(this.bintrayService.isDistributionComplete(eq(releaseInfo), (Set<String>) any(), any())).willReturn(true);
this.server.expect(requestTo("https://repo.spring.io/api/build/distribute/example-build/example-build-1"))
.andExpect(method(HttpMethod.POST))
.andExpect(content().json(
@ -128,9 +141,12 @@ class ArtifactoryServiceTests {
.andExpect(header("Authorization", "Basic " + Base64Utils.encodeToString(String
.format("%s:%s", this.properties.getUsername(), this.properties.getPassword()).getBytes())))
.andExpect(header("Content-Type", MediaType.APPLICATION_JSON.toString())).andRespond(withSuccess());
this.service.distribute("libs-release-local", releaseInfo);
Set<String> artifactDigests = Collections.singleton("602e20176706d3cc7535f01ffdbe91b270ae5014");
this.service.distribute("libs-release-local", releaseInfo, artifactDigests);
this.server.verify();
verify(this.bintrayService, times(1)).isDistributionComplete(releaseInfo);
InOrder ordered = inOrder(this.bintrayService);
ordered.verify(this.bintrayService).isDistributionComplete(releaseInfo, artifactDigests, SHORT_TIMEOUT);
ordered.verify(this.bintrayService).isDistributionComplete(releaseInfo, artifactDigests, LONG_TIMEOUT);
}
@Test
@ -144,16 +160,22 @@ class ArtifactoryServiceTests {
.format("%s:%s", this.properties.getUsername(), this.properties.getPassword()).getBytes())))
.andExpect(header("Content-Type", MediaType.APPLICATION_JSON.toString()))
.andRespond(withStatus(HttpStatus.FORBIDDEN));
Set<String> artifactDigests = Collections.singleton("602e20176706d3cc7535f01ffdbe91b270ae5014");
assertThatExceptionOfType(HttpClientErrorException.class)
.isThrownBy(() -> this.service.distribute("libs-release-local", releaseInfo));
.isThrownBy(() -> this.service.distribute("libs-release-local", releaseInfo, artifactDigests));
this.server.verify();
verifyNoInteractions(this.bintrayService);
verify(this.bintrayService, times(1)).isDistributionComplete(releaseInfo, artifactDigests, SHORT_TIMEOUT);
verifyNoMoreInteractions(this.bintrayService);
}
@Test
@SuppressWarnings("unchecked")
void distributeWhenGettingPackagesTimesOut() throws Exception {
ReleaseInfo releaseInfo = getReleaseInfo();
given(this.bintrayService.isDistributionComplete(releaseInfo)).willReturn(false);
given(this.bintrayService.isDistributionComplete(eq(releaseInfo), (Set<String>) any(), any()))
.willReturn(false);
given(this.bintrayService.isDistributionComplete(eq(releaseInfo), (Set<String>) any(), any()))
.willReturn(false);
this.server.expect(requestTo("https://repo.spring.io/api/build/distribute/example-build/example-build-1"))
.andExpect(method(HttpMethod.POST))
.andExpect(content().json(
@ -161,10 +183,13 @@ class ArtifactoryServiceTests {
.andExpect(header("Authorization", "Basic " + Base64Utils.encodeToString(String
.format("%s:%s", this.properties.getUsername(), this.properties.getPassword()).getBytes())))
.andExpect(header("Content-Type", MediaType.APPLICATION_JSON.toString())).andRespond(withSuccess());
Set<String> artifactDigests = Collections.singleton("602e20176706d3cc7535f01ffdbe91b270ae5014");
assertThatExceptionOfType(DistributionTimeoutException.class)
.isThrownBy(() -> this.service.distribute("libs-release-local", releaseInfo));
.isThrownBy(() -> this.service.distribute("libs-release-local", releaseInfo, artifactDigests));
this.server.verify();
verify(this.bintrayService, times(1)).isDistributionComplete(releaseInfo);
InOrder ordered = inOrder(this.bintrayService);
ordered.verify(this.bintrayService).isDistributionComplete(releaseInfo, artifactDigests, SHORT_TIMEOUT);
ordered.verify(this.bintrayService).isDistributionComplete(releaseInfo, artifactDigests, LONG_TIMEOUT);
}
private ReleaseInfo getReleaseInfo() {

View File

@ -16,6 +16,10 @@
package io.spring.concourse.releasescripts.bintray;
import java.time.Duration;
import java.util.LinkedHashSet;
import java.util.Set;
import io.spring.concourse.releasescripts.ReleaseInfo;
import io.spring.concourse.releasescripts.sonatype.SonatypeProperties;
import io.spring.concourse.releasescripts.sonatype.SonatypeService;
@ -28,7 +32,6 @@ import org.springframework.boot.test.autoconfigure.web.client.RestClientTest;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.core.io.ClassPathResource;
import org.springframework.http.HttpMethod;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.test.web.client.ExpectedCount;
import org.springframework.test.web.client.MockRestServiceServer;
@ -41,7 +44,6 @@ import static org.springframework.test.web.client.match.MockRestRequestMatchers.
import static org.springframework.test.web.client.match.MockRestRequestMatchers.header;
import static org.springframework.test.web.client.match.MockRestRequestMatchers.method;
import static org.springframework.test.web.client.match.MockRestRequestMatchers.requestTo;
import static org.springframework.test.web.client.response.MockRestResponseCreators.withStatus;
import static org.springframework.test.web.client.response.MockRestResponseCreators.withSuccess;
/**
@ -75,15 +77,15 @@ class BintrayServiceTests {
@Test
void isDistributionComplete() throws Exception {
this.server
.expect(requestTo(String.format(
"https://api.bintray.com/packages/%s/%s/%s/versions/%s/files?include_unpublished=%s",
this.properties.getSubject(), this.properties.getRepo(), "example", "1.1.0.RELEASE", 1)))
.andRespond(withStatus(HttpStatus.NOT_FOUND));
setupGetPackageFiles(1, "all-package-files.json");
setupGetPackageFiles(0, "published-files.json");
setupGetPackageFiles(0, "no-package-files.json");
setupGetPackageFiles(0, "some-package-files.json");
setupGetPackageFiles(0, "all-package-files.json");
assertThat(this.service.isDistributionComplete(getReleaseInfo())).isTrue();
Set<String> digests = new LinkedHashSet<>();
digests.add("602e20176706d3cc7535f01ffdbe91b270ae5012");
digests.add("602e20176706d3cc7535f01ffdbe91b270ae5013");
digests.add("602e20176706d3cc7535f01ffdbe91b270ae5014");
assertThat(this.service.isDistributionComplete(getReleaseInfo(), digests, Duration.ofMinutes(1), Duration.ZERO))
.isTrue();
this.server.verify();
}

View File

@ -16,6 +16,8 @@
package io.spring.concourse.releasescripts.command;
import java.util.Set;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import io.spring.concourse.releasescripts.ReleaseInfo;
@ -54,7 +56,7 @@ class DistributeCommandTests {
void setup() {
MockitoAnnotations.initMocks(this);
this.objectMapper = new ObjectMapper().disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES);
this.command = new DistributeCommand(this.service, objectMapper);
this.command = new DistributeCommand(this.service, this.objectMapper);
}
@Test
@ -76,15 +78,20 @@ class DistributeCommandTests {
}
@Test
@SuppressWarnings("unchecked")
void distributeWhenReleaseTypeReleaseShouldCallService() throws Exception {
ArgumentCaptor<ReleaseInfo> captor = ArgumentCaptor.forClass(ReleaseInfo.class);
ArgumentCaptor<ReleaseInfo> releaseInfoCaptor = ArgumentCaptor.forClass(ReleaseInfo.class);
ArgumentCaptor<Set<String>> artifactDigestCaptor = ArgumentCaptor.forClass(Set.class);
this.command.run(new DefaultApplicationArguments("distribute", "RELEASE", getBuildInfoLocation()));
verify(this.service).distribute(eq(ReleaseType.RELEASE.getRepo()), captor.capture());
ReleaseInfo releaseInfo = captor.getValue();
verify(this.service).distribute(eq(ReleaseType.RELEASE.getRepo()), releaseInfoCaptor.capture(),
artifactDigestCaptor.capture());
ReleaseInfo releaseInfo = releaseInfoCaptor.getValue();
assertThat(releaseInfo.getBuildName()).isEqualTo("example");
assertThat(releaseInfo.getBuildNumber()).isEqualTo("example-build-1");
assertThat(releaseInfo.getGroupId()).isEqualTo("org.example.demo");
assertThat(releaseInfo.getVersion()).isEqualTo("2.2.0");
Set<String> artifactDigests = artifactDigestCaptor.getValue();
assertThat(artifactDigests).containsExactly("aaaaaaaaa85f5c5093721f3ed0edda8ff8290yyyyyyyyyy");
}
private String getBuildInfoLocation() throws Exception {

View File

@ -8,7 +8,7 @@
"owner": "jfrog",
"created": "ISO8601 (yyyy-MM-dd'T'HH:mm:ss.SSSZ)",
"size": 1234,
"sha1": "602e20176706d3cc7535f01ffdbe91b270ae5012"
"sha256": "602e20176706d3cc7535f01ffdbe91b270ae5012"
},
{
"name": "nutcracker-1.1.pom",
@ -19,7 +19,7 @@
"owner": "jfrog",
"created": "ISO8601 (yyyy-MM-dd'T'HH:mm:ss.SSSZ)",
"size": 1234,
"sha1": "602e20176706d3cc7535f01ffdbe91b270ae5012"
"sha256": "602e20176706d3cc7535f01ffdbe91b270ae5013"
},
{
"name": "nutcracker-1.1.jar",
@ -30,6 +30,6 @@
"owner": "jfrog",
"created": "ISO8601 (yyyy-MM-dd'T'HH:mm:ss.SSSZ)",
"size": 1234,
"sha1": "602e20176706d3cc7535f01ffdbe91b270ae5012"
"sha256": "602e20176706d3cc7535f01ffdbe91b270ae5014"
}
]

View File

@ -8,6 +8,6 @@
"owner": "jfrog",
"created": "ISO8601 (yyyy-MM-dd'T'HH:mm:ss.SSSZ)",
"size": 1234,
"sha1": "602e20176706d3cc7535f01ffdbe91b270ae5012"
"sha256": "602e20176706d3cc7535f01ffdbe91b270ae5012"
}
]