Fix handling of Zip64 jar files larger than 4,294,967,295 bytes

Previously, a Zip64 jar file was identified by the number of entries
in the central directory being 0xFFFF. This value indicates that
there the number of entries is too big for the 2-byte field. However,
a jar may be in Zip64 format due to it exceeding the Zip format's
maximum size rather than its maximum number of entries so this field
cannot be used as a reliable indicator. The Zip specification doesn't
require any of the fields of the end of central directory record to
have a value of 0xFFFF (2-byte fields) or 0xFFFFFFFF (4-byte fields)
when using Zip64 format so we need to take a different approach.

Additionally, a number of places in the code assumed that an entry's
offset would always be available from the central directory file
header directly. This assumption did not hold true when the jar was
a Zip64 archive due to its size as the offset's value would be
0xFFFFFFF indicating that it should be read from the Zip64 extended
information field within the header's extra field instead.

This commit updates the Zip64 detection to look for the Zip64 end of
central directory locator instead. If present, it begins 20 bytes
before the beginning of the end of central directory record. Its
first four bytes are always 0x07064b50. The code that reads the
local header offset has also been updated to refer to the Zip64
extended information field when the offset is too large to fit in
the 4-byte field in the central directory file header. To allow
greater-than-4-byte offsets to be handled, a number of fields,
method parameters, and local variables have had their type changed
from an int to a long.

Fixes gh-27822

spring-boot-project/spring-boot-tools/spring-boot-loader/src/main/java/org/springframework/boot/loader/jar/CentralDirectoryEndRecord.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2012-2020 the original author or authors.
+ * Copyright 2012-2021 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -34,8 +34,6 @@ class CentralDirectoryEndRecord {
 
 	private static final int MAXIMUM_COMMENT_LENGTH = 0xFFFF;
 
-	private static final int ZIP64_MAGICCOUNT = 0xFFFF;
-
 	private static final int MAXIMUM_SIZE = MINIMUM_SIZE + MAXIMUM_COMMENT_LENGTH;
 
 	private static final int SIGNATURE = 0x06054b50;
@@ -74,8 +72,9 @@ class CentralDirectoryEndRecord {
 			}
 			this.offset = this.block.length - this.size;
 		}
-		int startOfCentralDirectoryEndRecord = (int) (data.getSize() - this.size);
-		this.zip64End = isZip64() ? new Zip64End(data, startOfCentralDirectoryEndRecord) : null;
+		long startOfCentralDirectoryEndRecord = data.getSize() - this.size;
+		Zip64Locator zip64Locator = Zip64Locator.find(data, startOfCentralDirectoryEndRecord);
+		this.zip64End = (zip64Locator != null) ? new Zip64End(data, zip64Locator) : null;
 	}
 
 	private byte[] createBlockFromEndOfData(RandomAccessData data, int size) throws IOException {
@@ -92,10 +91,6 @@ class CentralDirectoryEndRecord {
 		return this.size == MINIMUM_SIZE + commentLength;
 	}
 
-	private boolean isZip64() {
-		return (int) Bytes.littleEndianValue(this.block, this.offset + 10, 2) == ZIP64_MAGICCOUNT;
-	}
-
 	/**
 	 * Returns the location in the data that the archive actually starts. For most files
 	 * the archive data will start at 0, however, it is possible to have prefixed bytes
@@ -105,7 +100,8 @@ class CentralDirectoryEndRecord {
 	 */
 	long getStartOfArchive(RandomAccessData data) {
 		long length = Bytes.littleEndianValue(this.block, this.offset + 12, 4);
-		long specifiedOffset = Bytes.littleEndianValue(this.block, this.offset + 16, 4);
+		long specifiedOffset = (this.zip64End != null) ? this.zip64End.centralDirectoryOffset
+				: Bytes.littleEndianValue(this.block, this.offset + 16, 4);
 		long zip64EndSize = (this.zip64End != null) ? this.zip64End.getSize() : 0L;
 		int zip64LocSize = (this.zip64End != null) ? Zip64Locator.ZIP64_LOCSIZE : 0;
 		long actualOffset = data.getSize() - this.size - length - zip64EndSize - zip64LocSize;
@@ -145,6 +141,10 @@ class CentralDirectoryEndRecord {
 		return comment.toString();
 	}
 
+	boolean isZip64() {
+		return this.zip64End != null;
+	}
+
 	/**
 	 * A Zip64 end of central directory record.
 	 *
@@ -167,10 +167,6 @@ class CentralDirectoryEndRecord {
 
 		private final int numberOfRecords;
 
-		private Zip64End(RandomAccessData data, int centralDirectoryEndOffset) throws IOException {
-			this(data, new Zip64Locator(data, centralDirectoryEndOffset));
-		}
-
 		private Zip64End(RandomAccessData data, Zip64Locator locator) throws IOException {
 			this.locator = locator;
 			byte[] block = data.read(locator.getZip64EndOffset(), 56);
@@ -215,16 +211,18 @@ class CentralDirectoryEndRecord {
 	 */
 	private static final class Zip64Locator {
 
+		static final int SIGNATURE = 0x07064b50;
+
 		static final int ZIP64_LOCSIZE = 20; // locator size
+
 		static final int ZIP64_LOCOFF = 8; // offset of zip64 end
 
 		private final long zip64EndOffset;
 
-		private final int offset;
+		private final long offset;
 
-		private Zip64Locator(RandomAccessData data, int centralDirectoryEndOffset) throws IOException {
-			this.offset = centralDirectoryEndOffset - ZIP64_LOCSIZE;
-			byte[] block = data.read(this.offset, ZIP64_LOCSIZE);
+		private Zip64Locator(long offset, byte[] block) throws IOException {
+			this.offset = offset;
 			this.zip64EndOffset = Bytes.littleEndianValue(block, ZIP64_LOCOFF, 8);
 		}
 
@@ -244,6 +242,17 @@ class CentralDirectoryEndRecord {
 			return this.zip64EndOffset;
 		}
 
+		private static Zip64Locator find(RandomAccessData data, long centralDirectoryEndOffset) throws IOException {
+			long offset = centralDirectoryEndOffset - ZIP64_LOCSIZE;
+			if (offset >= 0) {
+				byte[] block = data.read(offset, ZIP64_LOCSIZE);
+				if (Bytes.littleEndianValue(block, 0, 4) == SIGNATURE) {
+					return new Zip64Locator(offset, block);
+				}
+			}
+			return null;
+		}
+
 	}
 
 }

spring-boot-project/spring-boot-tools/spring-boot-loader/src/main/java/org/springframework/boot/loader/jar/CentralDirectoryFileHeader.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2012-2020 the original author or authors.
+ * Copyright 2012-2021 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -67,15 +67,17 @@ final class CentralDirectoryFileHeader implements FileHeader {
 		this.localHeaderOffset = localHeaderOffset;
 	}
 
-	void load(byte[] data, int dataOffset, RandomAccessData variableData, int variableOffset, JarEntryFilter filter)
+	void load(byte[] data, int dataOffset, RandomAccessData variableData, long variableOffset, JarEntryFilter filter)
 			throws IOException {
 		// Load fixed part
 		this.header = data;
 		this.headerOffset = dataOffset;
+		long compressedSize = Bytes.littleEndianValue(data, dataOffset + 20, 4);
+		long uncompressedSize = Bytes.littleEndianValue(data, dataOffset + 24, 4);
 		long nameLength = Bytes.littleEndianValue(data, dataOffset + 28, 2);
 		long extraLength = Bytes.littleEndianValue(data, dataOffset + 30, 2);
 		long commentLength = Bytes.littleEndianValue(data, dataOffset + 32, 2);
-		this.localHeaderOffset = Bytes.littleEndianValue(data, dataOffset + 42, 4);
+		long localHeaderOffset = Bytes.littleEndianValue(data, dataOffset + 42, 4);
 		// Load variable part
 		dataOffset += 46;
 		if (variableData != null) {
@@ -92,11 +94,37 @@ final class CentralDirectoryFileHeader implements FileHeader {
 			this.extra = new byte[(int) extraLength];
 			System.arraycopy(data, (int) (dataOffset + nameLength), this.extra, 0, this.extra.length);
 		}
+		this.localHeaderOffset = getLocalHeaderOffset(compressedSize, uncompressedSize, localHeaderOffset, this.extra);
 		if (commentLength > 0) {
 			this.comment = new AsciiBytes(data, (int) (dataOffset + nameLength + extraLength), (int) commentLength);
 		}
 	}
 
+	private long getLocalHeaderOffset(long compressedSize, long uncompressedSize, long localHeaderOffset, byte[] extra)
+			throws IOException {
+		if (localHeaderOffset != 0xFFFFFFFFL) {
+			return localHeaderOffset;
+		}
+		int extraOffset = 0;
+		while (extraOffset < extra.length - 2) {
+			int id = (int) Bytes.littleEndianValue(extra, extraOffset, 2);
+			int length = (int) Bytes.littleEndianValue(extra, extraOffset, 2);
+			extraOffset += 4;
+			if (id == 1) {
+				int localHeaderExtraOffset = 0;
+				if (compressedSize == 0xFFFFFFFFL) {
+					localHeaderExtraOffset += 4;
+				}
+				if (uncompressedSize == 0xFFFFFFFFL) {
+					localHeaderExtraOffset += 4;
+				}
+				return Bytes.littleEndianValue(extra, extraOffset + localHeaderExtraOffset, 8);
+			}
+			extraOffset += length;
+		}
+		throw new IOException("Zip64 Extended Information Extra Field not found");
+	}
+
 	AsciiBytes getName() {
 		return this.name;
 	}
@@ -176,7 +204,7 @@ final class CentralDirectoryFileHeader implements FileHeader {
 		return new CentralDirectoryFileHeader(header, 0, this.name, header, this.comment, this.localHeaderOffset);
 	}
 
-	static CentralDirectoryFileHeader fromRandomAccessData(RandomAccessData data, int offset, JarEntryFilter filter)
+	static CentralDirectoryFileHeader fromRandomAccessData(RandomAccessData data, long offset, JarEntryFilter filter)
 			throws IOException {
 		CentralDirectoryFileHeader fileHeader = new CentralDirectoryFileHeader();
 		byte[] bytes = data.read(offset, 46);

spring-boot-project/spring-boot-tools/spring-boot-loader/src/main/java/org/springframework/boot/loader/jar/CentralDirectoryParser.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2012-2019 the original author or authors.
+ * Copyright 2012-2021 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -86,7 +86,7 @@ class CentralDirectoryParser {
 		}
 	}
 
-	private void visitFileHeader(int dataOffset, CentralDirectoryFileHeader fileHeader) {
+	private void visitFileHeader(long dataOffset, CentralDirectoryFileHeader fileHeader) {
 		for (CentralDirectoryVisitor visitor : this.visitors) {
 			visitor.visitFileHeader(fileHeader, dataOffset);
 		}

spring-boot-project/spring-boot-tools/spring-boot-loader/src/main/java/org/springframework/boot/loader/jar/CentralDirectoryVisitor.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2012-2019 the original author or authors.
+ * Copyright 2012-2021 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -27,7 +27,7 @@ interface CentralDirectoryVisitor {
 
 	void visitStart(CentralDirectoryEndRecord endRecord, RandomAccessData centralDirectoryData);
 
-	void visitFileHeader(CentralDirectoryFileHeader fileHeader, int dataOffset);
+	void visitFileHeader(CentralDirectoryFileHeader fileHeader, long dataOffset);
 
 	void visitEnd();
 

spring-boot-project/spring-boot-tools/spring-boot-loader/src/main/java/org/springframework/boot/loader/jar/JarFile.java

@@ -169,7 +169,7 @@ public class JarFile extends AbstractJarFile implements Iterable<java.util.jar.J
 			}
 
 			@Override
-			public void visitFileHeader(CentralDirectoryFileHeader fileHeader, int dataOffset) {
+			public void visitFileHeader(CentralDirectoryFileHeader fileHeader, long dataOffset) {
 				AsciiBytes name = fileHeader.getName();
 				if (name.startsWith(META_INF) && name.endsWith(SIGNATURE_FILE_EXTENSION)) {
 					JarFile.this.signed = true;

spring-boot-project/spring-boot-tools/spring-boot-loader/src/main/java/org/springframework/boot/loader/jar/JarFileEntries.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2012-2020 the original author or authors.
+ * Copyright 2012-2021 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -89,7 +89,7 @@ class JarFileEntries implements CentralDirectoryVisitor, Iterable<JarEntry> {
 
 	private int[] hashCodes;
 
-	private int[] centralDirectoryOffsets;
+	private Offsets centralDirectoryOffsets;
 
 	private int[] positions;
 
@@ -120,21 +120,21 @@ class JarFileEntries implements CentralDirectoryVisitor, Iterable<JarEntry> {
 		int maxSize = endRecord.getNumberOfRecords();
 		this.centralDirectoryData = centralDirectoryData;
 		this.hashCodes = new int[maxSize];
-		this.centralDirectoryOffsets = new int[maxSize];
+		this.centralDirectoryOffsets = Offsets.of(endRecord);
 		this.positions = new int[maxSize];
 	}
 
 	@Override
-	public void visitFileHeader(CentralDirectoryFileHeader fileHeader, int dataOffset) {
+	public void visitFileHeader(CentralDirectoryFileHeader fileHeader, long dataOffset) {
 		AsciiBytes name = applyFilter(fileHeader.getName());
 		if (name != null) {
 			add(name, dataOffset);
 		}
 	}
 
-	private void add(AsciiBytes name, int dataOffset) {
+	private void add(AsciiBytes name, long dataOffset) {
 		this.hashCodes[this.size] = name.hashCode();
-		this.centralDirectoryOffsets[this.size] = dataOffset;
+		this.centralDirectoryOffsets.set(this.size, dataOffset);
 		this.positions[this.size] = this.size;
 		this.size++;
 	}
@@ -183,11 +183,11 @@ class JarFileEntries implements CentralDirectoryVisitor, Iterable<JarEntry> {
 
 	private void swap(int i, int j) {
 		swap(this.hashCodes, i, j);
-		swap(this.centralDirectoryOffsets, i, j);
+		this.centralDirectoryOffsets.swap(i, j);
 		swap(this.positions, i, j);
 	}
 
-	private void swap(int[] array, int i, int j) {
+	private static void swap(int[] array, int i, int j) {
 		int temp = array[i];
 		array[i] = array[j];
 		array[j] = temp;
@@ -316,9 +316,10 @@ class JarFileEntries implements CentralDirectoryVisitor, Iterable<JarEntry> {
 	@SuppressWarnings("unchecked")
 	private <T extends FileHeader> T getEntry(int index, Class<T> type, boolean cacheEntry, AsciiBytes nameAlias) {
 		try {
+			long offset = this.centralDirectoryOffsets.get(index);
 			FileHeader cached = this.entriesCache.get(index);
-			FileHeader entry = (cached != null) ? cached : CentralDirectoryFileHeader
-					.fromRandomAccessData(this.centralDirectoryData, this.centralDirectoryOffsets[index], this.filter);
+			FileHeader entry = (cached != null) ? cached
+					: CentralDirectoryFileHeader.fromRandomAccessData(this.centralDirectoryData, offset, this.filter);
 			if (CentralDirectoryFileHeader.class.equals(entry.getClass()) && type.equals(JarEntry.class)) {
 				entry = new JarEntry(this.jarFile, index, (CentralDirectoryFileHeader) entry, nameAlias);
 			}
@@ -420,4 +421,71 @@ class JarFileEntries implements CentralDirectoryVisitor, Iterable<JarEntry> {
 
 	}
 
+	private interface Offsets {
+
+		void set(int index, long value);
+
+		long get(int index);
+
+		void swap(int i, int j);
+
+		static Offsets of(CentralDirectoryEndRecord endRecord) {
+			return endRecord.isZip64() ? new Zip64Offsets(endRecord.getNumberOfRecords())
+					: new ZipOffsets(endRecord.getNumberOfRecords());
+		}
+
+	}
+
+	private static final class ZipOffsets implements Offsets {
+
+		private final int[] offsets;
+
+		private ZipOffsets(int size) {
+			this.offsets = new int[size];
+		}
+
+		@Override
+		public void swap(int i, int j) {
+			JarFileEntries.swap(this.offsets, i, j);
+		}
+
+		@Override
+		public void set(int index, long value) {
+			this.offsets[index] = (int) value;
+		}
+
+		@Override
+		public long get(int index) {
+			return this.offsets[index];
+		}
+
+	}
+
+	private static final class Zip64Offsets implements Offsets {
+
+		private final long[] offsets;
+
+		private Zip64Offsets(int size) {
+			this.offsets = new long[size];
+		}
+
+		@Override
+		public void swap(int i, int j) {
+			long temp = this.offsets[i];
+			this.offsets[i] = this.offsets[j];
+			this.offsets[j] = temp;
+		}
+
+		@Override
+		public void set(int index, long value) {
+			this.offsets[index] = value;
+		}
+
+		@Override
+		public long get(int index) {
+			return this.offsets[index];
+		}
+
+	}
+
 }

spring-boot-project/spring-boot-tools/spring-boot-loader/src/test/java/org/springframework/boot/loader/jar/CentralDirectoryParserTests.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2012-2019 the original author or authors.
+ * Copyright 2012-2021 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -97,7 +97,7 @@ class CentralDirectoryParserTests {
 		}
 
 		@Override
-		public void visitFileHeader(CentralDirectoryFileHeader fileHeader, int dataOffset) {
+		public void visitFileHeader(CentralDirectoryFileHeader fileHeader, long dataOffset) {
 			this.headers.add(fileHeader.clone());
 		}
 
@@ -121,7 +121,7 @@ class CentralDirectoryParserTests {
 		}
 
 		@Override
-		public void visitFileHeader(CentralDirectoryFileHeader fileHeader, int dataOffset) {
+		public void visitFileHeader(CentralDirectoryFileHeader fileHeader, long dataOffset) {
 			this.invocations.add("visitFileHeader");
 		}
 

spring-boot-project/spring-boot-tools/spring-boot-loader/src/test/java/org/springframework/boot/loader/jar/JarFileTests.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2012-2020 the original author or authors.
+ * Copyright 2012-2021 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -35,6 +35,7 @@ import java.util.Collections;
 import java.util.Enumeration;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Random;
 import java.util.jar.JarEntry;
 import java.util.jar.JarInputStream;
 import java.util.jar.JarOutputStream;
@@ -46,6 +47,7 @@ import java.util.zip.ZipFile;
 
 import org.assertj.core.api.ThrowableAssert.ThrowingCallable;
 import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Assumptions;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
@@ -563,7 +565,7 @@ class JarFileTests {
 	}
 
 	@Test
-	void zip64JarCanBeRead() throws Exception {
+	void zip64JarThatExceedsZipEntryLimitCanBeRead() throws Exception {
 		File zip64Jar = new File(this.tempDir, "zip64.jar");
 		FileCopyUtils.copy(zip64Jar(), zip64Jar);
 		try (JarFile zip64JarFile = new JarFile(zip64Jar)) {
@@ -577,6 +579,39 @@ class JarFileTests {
 		}
 	}
 
+	@Test
+	void zip64JarThatExceedsZipSizeLimitCanBeRead() throws Exception {
+		Assumptions.assumeTrue(this.tempDir.getFreeSpace() > 6 * 1024 * 1024 * 1024, "Insufficient disk space");
+		File zip64Jar = new File(this.tempDir, "zip64.jar");
+		File entry = new File(this.tempDir, "entry.dat");
+		CRC32 crc32 = new CRC32();
+		try (FileOutputStream entryOut = new FileOutputStream(entry)) {
+			byte[] data = new byte[1024 * 1024];
+			new Random().nextBytes(data);
+			for (int i = 0; i < 1024; i++) {
+				entryOut.write(data);
+				crc32.update(data);
+			}
+		}
+		try (JarOutputStream jarOutput = new JarOutputStream(new FileOutputStream(zip64Jar))) {
+			for (int i = 0; i < 6; i++) {
+				JarEntry storedEntry = new JarEntry("huge-" + i);
+				storedEntry.setSize(entry.length());
+				storedEntry.setCompressedSize(entry.length());
+				storedEntry.setCrc(crc32.getValue());
+				storedEntry.setMethod(ZipEntry.STORED);
+				jarOutput.putNextEntry(storedEntry);
+				try (FileInputStream entryIn = new FileInputStream(entry)) {
+					StreamUtils.copy(entryIn, jarOutput);
+				}
+				jarOutput.closeEntry();
+			}
+		}
+		try (JarFile zip64JarFile = new JarFile(zip64Jar)) {
+			assertThat(Collections.list(zip64JarFile.entries())).hasSize(6);
+		}
+	}
+
 	@Test
 	void nestedZip64JarCanBeRead() throws Exception {
 		File outer = new File(this.tempDir, "outer.jar");