From 47163af9b6b426f17d3d0d0fd67070ab50d68e93 Mon Sep 17 00:00:00 2001 From: Andy Wilkinson Date: Tue, 7 Sep 2021 17:50:45 +0100 Subject: [PATCH] Fix handling of Zip64 jar files larger than 4,294,967,295 bytes Previously, a Zip64 jar file was identified by the number of entries in the central directory being 0xFFFF. This value indicates that there the number of entries is too big for the 2-byte field. However, a jar may be in Zip64 format due to it exceeding the Zip format's maximum size rather than its maximum number of entries so this field cannot be used as a reliable indicator. The Zip specification doesn't require any of the fields of the end of central directory record to have a value of 0xFFFF (2-byte fields) or 0xFFFFFFFF (4-byte fields) when using Zip64 format so we need to take a different approach. Additionally, a number of places in the code assumed that an entry's offset would always be available from the central directory file header directly. This assumption did not hold true when the jar was a Zip64 archive due to its size as the offset's value would be 0xFFFFFFF indicating that it should be read from the Zip64 extended information field within the header's extra field instead. This commit updates the Zip64 detection to look for the Zip64 end of central directory locator instead. If present, it begins 20 bytes before the beginning of the end of central directory record. Its first four bytes are always 0x07064b50. The code that reads the local header offset has also been updated to refer to the Zip64 extended information field when the offset is too large to fit in the 4-byte field in the central directory file header. To allow greater-than-4-byte offsets to be handled, a number of fields, method parameters, and local variables have had their type changed from an int to a long. Fixes gh-27822 --- .../loader/jar/CentralDirectoryEndRecord.java | 45 ++++++---- .../jar/CentralDirectoryFileHeader.java | 36 +++++++- .../loader/jar/CentralDirectoryParser.java | 4 +- .../loader/jar/CentralDirectoryVisitor.java | 4 +- .../boot/loader/jar/JarFile.java | 2 +- .../boot/loader/jar/JarFileEntries.java | 88 ++++++++++++++++--- .../jar/CentralDirectoryParserTests.java | 6 +- .../boot/loader/jar/JarFileTests.java | 39 +++++++- 8 files changed, 182 insertions(+), 42 deletions(-) diff --git a/spring-boot-project/spring-boot-tools/spring-boot-loader/src/main/java/org/springframework/boot/loader/jar/CentralDirectoryEndRecord.java b/spring-boot-project/spring-boot-tools/spring-boot-loader/src/main/java/org/springframework/boot/loader/jar/CentralDirectoryEndRecord.java index cfb068f9bac..32c274ba17a 100644 --- a/spring-boot-project/spring-boot-tools/spring-boot-loader/src/main/java/org/springframework/boot/loader/jar/CentralDirectoryEndRecord.java +++ b/spring-boot-project/spring-boot-tools/spring-boot-loader/src/main/java/org/springframework/boot/loader/jar/CentralDirectoryEndRecord.java @@ -1,5 +1,5 @@ /* - * Copyright 2012-2020 the original author or authors. + * Copyright 2012-2021 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -34,8 +34,6 @@ class CentralDirectoryEndRecord { private static final int MAXIMUM_COMMENT_LENGTH = 0xFFFF; - private static final int ZIP64_MAGICCOUNT = 0xFFFF; - private static final int MAXIMUM_SIZE = MINIMUM_SIZE + MAXIMUM_COMMENT_LENGTH; private static final int SIGNATURE = 0x06054b50; @@ -74,8 +72,9 @@ class CentralDirectoryEndRecord { } this.offset = this.block.length - this.size; } - int startOfCentralDirectoryEndRecord = (int) (data.getSize() - this.size); - this.zip64End = isZip64() ? new Zip64End(data, startOfCentralDirectoryEndRecord) : null; + long startOfCentralDirectoryEndRecord = data.getSize() - this.size; + Zip64Locator zip64Locator = Zip64Locator.find(data, startOfCentralDirectoryEndRecord); + this.zip64End = (zip64Locator != null) ? new Zip64End(data, zip64Locator) : null; } private byte[] createBlockFromEndOfData(RandomAccessData data, int size) throws IOException { @@ -92,10 +91,6 @@ class CentralDirectoryEndRecord { return this.size == MINIMUM_SIZE + commentLength; } - private boolean isZip64() { - return (int) Bytes.littleEndianValue(this.block, this.offset + 10, 2) == ZIP64_MAGICCOUNT; - } - /** * Returns the location in the data that the archive actually starts. For most files * the archive data will start at 0, however, it is possible to have prefixed bytes @@ -105,7 +100,8 @@ class CentralDirectoryEndRecord { */ long getStartOfArchive(RandomAccessData data) { long length = Bytes.littleEndianValue(this.block, this.offset + 12, 4); - long specifiedOffset = Bytes.littleEndianValue(this.block, this.offset + 16, 4); + long specifiedOffset = (this.zip64End != null) ? this.zip64End.centralDirectoryOffset + : Bytes.littleEndianValue(this.block, this.offset + 16, 4); long zip64EndSize = (this.zip64End != null) ? this.zip64End.getSize() : 0L; int zip64LocSize = (this.zip64End != null) ? Zip64Locator.ZIP64_LOCSIZE : 0; long actualOffset = data.getSize() - this.size - length - zip64EndSize - zip64LocSize; @@ -145,6 +141,10 @@ class CentralDirectoryEndRecord { return comment.toString(); } + boolean isZip64() { + return this.zip64End != null; + } + /** * A Zip64 end of central directory record. * @@ -167,10 +167,6 @@ class CentralDirectoryEndRecord { private final int numberOfRecords; - private Zip64End(RandomAccessData data, int centralDirectoryEndOffset) throws IOException { - this(data, new Zip64Locator(data, centralDirectoryEndOffset)); - } - private Zip64End(RandomAccessData data, Zip64Locator locator) throws IOException { this.locator = locator; byte[] block = data.read(locator.getZip64EndOffset(), 56); @@ -215,16 +211,18 @@ class CentralDirectoryEndRecord { */ private static final class Zip64Locator { + static final int SIGNATURE = 0x07064b50; + static final int ZIP64_LOCSIZE = 20; // locator size + static final int ZIP64_LOCOFF = 8; // offset of zip64 end private final long zip64EndOffset; - private final int offset; + private final long offset; - private Zip64Locator(RandomAccessData data, int centralDirectoryEndOffset) throws IOException { - this.offset = centralDirectoryEndOffset - ZIP64_LOCSIZE; - byte[] block = data.read(this.offset, ZIP64_LOCSIZE); + private Zip64Locator(long offset, byte[] block) throws IOException { + this.offset = offset; this.zip64EndOffset = Bytes.littleEndianValue(block, ZIP64_LOCOFF, 8); } @@ -244,6 +242,17 @@ class CentralDirectoryEndRecord { return this.zip64EndOffset; } + private static Zip64Locator find(RandomAccessData data, long centralDirectoryEndOffset) throws IOException { + long offset = centralDirectoryEndOffset - ZIP64_LOCSIZE; + if (offset >= 0) { + byte[] block = data.read(offset, ZIP64_LOCSIZE); + if (Bytes.littleEndianValue(block, 0, 4) == SIGNATURE) { + return new Zip64Locator(offset, block); + } + } + return null; + } + } } diff --git a/spring-boot-project/spring-boot-tools/spring-boot-loader/src/main/java/org/springframework/boot/loader/jar/CentralDirectoryFileHeader.java b/spring-boot-project/spring-boot-tools/spring-boot-loader/src/main/java/org/springframework/boot/loader/jar/CentralDirectoryFileHeader.java index 8d4c0e9dbc5..acc05a439f6 100644 --- a/spring-boot-project/spring-boot-tools/spring-boot-loader/src/main/java/org/springframework/boot/loader/jar/CentralDirectoryFileHeader.java +++ b/spring-boot-project/spring-boot-tools/spring-boot-loader/src/main/java/org/springframework/boot/loader/jar/CentralDirectoryFileHeader.java @@ -1,5 +1,5 @@ /* - * Copyright 2012-2020 the original author or authors. + * Copyright 2012-2021 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -67,15 +67,17 @@ final class CentralDirectoryFileHeader implements FileHeader { this.localHeaderOffset = localHeaderOffset; } - void load(byte[] data, int dataOffset, RandomAccessData variableData, int variableOffset, JarEntryFilter filter) + void load(byte[] data, int dataOffset, RandomAccessData variableData, long variableOffset, JarEntryFilter filter) throws IOException { // Load fixed part this.header = data; this.headerOffset = dataOffset; + long compressedSize = Bytes.littleEndianValue(data, dataOffset + 20, 4); + long uncompressedSize = Bytes.littleEndianValue(data, dataOffset + 24, 4); long nameLength = Bytes.littleEndianValue(data, dataOffset + 28, 2); long extraLength = Bytes.littleEndianValue(data, dataOffset + 30, 2); long commentLength = Bytes.littleEndianValue(data, dataOffset + 32, 2); - this.localHeaderOffset = Bytes.littleEndianValue(data, dataOffset + 42, 4); + long localHeaderOffset = Bytes.littleEndianValue(data, dataOffset + 42, 4); // Load variable part dataOffset += 46; if (variableData != null) { @@ -92,11 +94,37 @@ final class CentralDirectoryFileHeader implements FileHeader { this.extra = new byte[(int) extraLength]; System.arraycopy(data, (int) (dataOffset + nameLength), this.extra, 0, this.extra.length); } + this.localHeaderOffset = getLocalHeaderOffset(compressedSize, uncompressedSize, localHeaderOffset, this.extra); if (commentLength > 0) { this.comment = new AsciiBytes(data, (int) (dataOffset + nameLength + extraLength), (int) commentLength); } } + private long getLocalHeaderOffset(long compressedSize, long uncompressedSize, long localHeaderOffset, byte[] extra) + throws IOException { + if (localHeaderOffset != 0xFFFFFFFFL) { + return localHeaderOffset; + } + int extraOffset = 0; + while (extraOffset < extra.length - 2) { + int id = (int) Bytes.littleEndianValue(extra, extraOffset, 2); + int length = (int) Bytes.littleEndianValue(extra, extraOffset, 2); + extraOffset += 4; + if (id == 1) { + int localHeaderExtraOffset = 0; + if (compressedSize == 0xFFFFFFFFL) { + localHeaderExtraOffset += 4; + } + if (uncompressedSize == 0xFFFFFFFFL) { + localHeaderExtraOffset += 4; + } + return Bytes.littleEndianValue(extra, extraOffset + localHeaderExtraOffset, 8); + } + extraOffset += length; + } + throw new IOException("Zip64 Extended Information Extra Field not found"); + } + AsciiBytes getName() { return this.name; } @@ -176,7 +204,7 @@ final class CentralDirectoryFileHeader implements FileHeader { return new CentralDirectoryFileHeader(header, 0, this.name, header, this.comment, this.localHeaderOffset); } - static CentralDirectoryFileHeader fromRandomAccessData(RandomAccessData data, int offset, JarEntryFilter filter) + static CentralDirectoryFileHeader fromRandomAccessData(RandomAccessData data, long offset, JarEntryFilter filter) throws IOException { CentralDirectoryFileHeader fileHeader = new CentralDirectoryFileHeader(); byte[] bytes = data.read(offset, 46); diff --git a/spring-boot-project/spring-boot-tools/spring-boot-loader/src/main/java/org/springframework/boot/loader/jar/CentralDirectoryParser.java b/spring-boot-project/spring-boot-tools/spring-boot-loader/src/main/java/org/springframework/boot/loader/jar/CentralDirectoryParser.java index 941302d99e5..71a76785356 100644 --- a/spring-boot-project/spring-boot-tools/spring-boot-loader/src/main/java/org/springframework/boot/loader/jar/CentralDirectoryParser.java +++ b/spring-boot-project/spring-boot-tools/spring-boot-loader/src/main/java/org/springframework/boot/loader/jar/CentralDirectoryParser.java @@ -1,5 +1,5 @@ /* - * Copyright 2012-2019 the original author or authors. + * Copyright 2012-2021 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -86,7 +86,7 @@ class CentralDirectoryParser { } } - private void visitFileHeader(int dataOffset, CentralDirectoryFileHeader fileHeader) { + private void visitFileHeader(long dataOffset, CentralDirectoryFileHeader fileHeader) { for (CentralDirectoryVisitor visitor : this.visitors) { visitor.visitFileHeader(fileHeader, dataOffset); } diff --git a/spring-boot-project/spring-boot-tools/spring-boot-loader/src/main/java/org/springframework/boot/loader/jar/CentralDirectoryVisitor.java b/spring-boot-project/spring-boot-tools/spring-boot-loader/src/main/java/org/springframework/boot/loader/jar/CentralDirectoryVisitor.java index 993986f742f..d160cbf8477 100644 --- a/spring-boot-project/spring-boot-tools/spring-boot-loader/src/main/java/org/springframework/boot/loader/jar/CentralDirectoryVisitor.java +++ b/spring-boot-project/spring-boot-tools/spring-boot-loader/src/main/java/org/springframework/boot/loader/jar/CentralDirectoryVisitor.java @@ -1,5 +1,5 @@ /* - * Copyright 2012-2019 the original author or authors. + * Copyright 2012-2021 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -27,7 +27,7 @@ interface CentralDirectoryVisitor { void visitStart(CentralDirectoryEndRecord endRecord, RandomAccessData centralDirectoryData); - void visitFileHeader(CentralDirectoryFileHeader fileHeader, int dataOffset); + void visitFileHeader(CentralDirectoryFileHeader fileHeader, long dataOffset); void visitEnd(); diff --git a/spring-boot-project/spring-boot-tools/spring-boot-loader/src/main/java/org/springframework/boot/loader/jar/JarFile.java b/spring-boot-project/spring-boot-tools/spring-boot-loader/src/main/java/org/springframework/boot/loader/jar/JarFile.java index cfef56e5b89..3ee947e4069 100644 --- a/spring-boot-project/spring-boot-tools/spring-boot-loader/src/main/java/org/springframework/boot/loader/jar/JarFile.java +++ b/spring-boot-project/spring-boot-tools/spring-boot-loader/src/main/java/org/springframework/boot/loader/jar/JarFile.java @@ -169,7 +169,7 @@ public class JarFile extends AbstractJarFile implements Iterable { private int[] hashCodes; - private int[] centralDirectoryOffsets; + private Offsets centralDirectoryOffsets; private int[] positions; @@ -120,21 +120,21 @@ class JarFileEntries implements CentralDirectoryVisitor, Iterable { int maxSize = endRecord.getNumberOfRecords(); this.centralDirectoryData = centralDirectoryData; this.hashCodes = new int[maxSize]; - this.centralDirectoryOffsets = new int[maxSize]; + this.centralDirectoryOffsets = Offsets.of(endRecord); this.positions = new int[maxSize]; } @Override - public void visitFileHeader(CentralDirectoryFileHeader fileHeader, int dataOffset) { + public void visitFileHeader(CentralDirectoryFileHeader fileHeader, long dataOffset) { AsciiBytes name = applyFilter(fileHeader.getName()); if (name != null) { add(name, dataOffset); } } - private void add(AsciiBytes name, int dataOffset) { + private void add(AsciiBytes name, long dataOffset) { this.hashCodes[this.size] = name.hashCode(); - this.centralDirectoryOffsets[this.size] = dataOffset; + this.centralDirectoryOffsets.set(this.size, dataOffset); this.positions[this.size] = this.size; this.size++; } @@ -183,11 +183,11 @@ class JarFileEntries implements CentralDirectoryVisitor, Iterable { private void swap(int i, int j) { swap(this.hashCodes, i, j); - swap(this.centralDirectoryOffsets, i, j); + this.centralDirectoryOffsets.swap(i, j); swap(this.positions, i, j); } - private void swap(int[] array, int i, int j) { + private static void swap(int[] array, int i, int j) { int temp = array[i]; array[i] = array[j]; array[j] = temp; @@ -316,9 +316,10 @@ class JarFileEntries implements CentralDirectoryVisitor, Iterable { @SuppressWarnings("unchecked") private T getEntry(int index, Class type, boolean cacheEntry, AsciiBytes nameAlias) { try { + long offset = this.centralDirectoryOffsets.get(index); FileHeader cached = this.entriesCache.get(index); - FileHeader entry = (cached != null) ? cached : CentralDirectoryFileHeader - .fromRandomAccessData(this.centralDirectoryData, this.centralDirectoryOffsets[index], this.filter); + FileHeader entry = (cached != null) ? cached + : CentralDirectoryFileHeader.fromRandomAccessData(this.centralDirectoryData, offset, this.filter); if (CentralDirectoryFileHeader.class.equals(entry.getClass()) && type.equals(JarEntry.class)) { entry = new JarEntry(this.jarFile, index, (CentralDirectoryFileHeader) entry, nameAlias); } @@ -420,4 +421,71 @@ class JarFileEntries implements CentralDirectoryVisitor, Iterable { } + private interface Offsets { + + void set(int index, long value); + + long get(int index); + + void swap(int i, int j); + + static Offsets of(CentralDirectoryEndRecord endRecord) { + return endRecord.isZip64() ? new Zip64Offsets(endRecord.getNumberOfRecords()) + : new ZipOffsets(endRecord.getNumberOfRecords()); + } + + } + + private static final class ZipOffsets implements Offsets { + + private final int[] offsets; + + private ZipOffsets(int size) { + this.offsets = new int[size]; + } + + @Override + public void swap(int i, int j) { + JarFileEntries.swap(this.offsets, i, j); + } + + @Override + public void set(int index, long value) { + this.offsets[index] = (int) value; + } + + @Override + public long get(int index) { + return this.offsets[index]; + } + + } + + private static final class Zip64Offsets implements Offsets { + + private final long[] offsets; + + private Zip64Offsets(int size) { + this.offsets = new long[size]; + } + + @Override + public void swap(int i, int j) { + long temp = this.offsets[i]; + this.offsets[i] = this.offsets[j]; + this.offsets[j] = temp; + } + + @Override + public void set(int index, long value) { + this.offsets[index] = value; + } + + @Override + public long get(int index) { + return this.offsets[index]; + } + + } + } diff --git a/spring-boot-project/spring-boot-tools/spring-boot-loader/src/test/java/org/springframework/boot/loader/jar/CentralDirectoryParserTests.java b/spring-boot-project/spring-boot-tools/spring-boot-loader/src/test/java/org/springframework/boot/loader/jar/CentralDirectoryParserTests.java index 13bfe6f8373..62f35f00102 100644 --- a/spring-boot-project/spring-boot-tools/spring-boot-loader/src/test/java/org/springframework/boot/loader/jar/CentralDirectoryParserTests.java +++ b/spring-boot-project/spring-boot-tools/spring-boot-loader/src/test/java/org/springframework/boot/loader/jar/CentralDirectoryParserTests.java @@ -1,5 +1,5 @@ /* - * Copyright 2012-2019 the original author or authors. + * Copyright 2012-2021 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -97,7 +97,7 @@ class CentralDirectoryParserTests { } @Override - public void visitFileHeader(CentralDirectoryFileHeader fileHeader, int dataOffset) { + public void visitFileHeader(CentralDirectoryFileHeader fileHeader, long dataOffset) { this.headers.add(fileHeader.clone()); } @@ -121,7 +121,7 @@ class CentralDirectoryParserTests { } @Override - public void visitFileHeader(CentralDirectoryFileHeader fileHeader, int dataOffset) { + public void visitFileHeader(CentralDirectoryFileHeader fileHeader, long dataOffset) { this.invocations.add("visitFileHeader"); } diff --git a/spring-boot-project/spring-boot-tools/spring-boot-loader/src/test/java/org/springframework/boot/loader/jar/JarFileTests.java b/spring-boot-project/spring-boot-tools/spring-boot-loader/src/test/java/org/springframework/boot/loader/jar/JarFileTests.java index ba7fe79e8b7..20c955e096d 100644 --- a/spring-boot-project/spring-boot-tools/spring-boot-loader/src/test/java/org/springframework/boot/loader/jar/JarFileTests.java +++ b/spring-boot-project/spring-boot-tools/spring-boot-loader/src/test/java/org/springframework/boot/loader/jar/JarFileTests.java @@ -1,5 +1,5 @@ /* - * Copyright 2012-2020 the original author or authors. + * Copyright 2012-2021 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -35,6 +35,7 @@ import java.util.Collections; import java.util.Enumeration; import java.util.Iterator; import java.util.List; +import java.util.Random; import java.util.jar.JarEntry; import java.util.jar.JarInputStream; import java.util.jar.JarOutputStream; @@ -46,6 +47,7 @@ import java.util.zip.ZipFile; import org.assertj.core.api.ThrowableAssert.ThrowingCallable; import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assumptions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -563,7 +565,7 @@ class JarFileTests { } @Test - void zip64JarCanBeRead() throws Exception { + void zip64JarThatExceedsZipEntryLimitCanBeRead() throws Exception { File zip64Jar = new File(this.tempDir, "zip64.jar"); FileCopyUtils.copy(zip64Jar(), zip64Jar); try (JarFile zip64JarFile = new JarFile(zip64Jar)) { @@ -577,6 +579,39 @@ class JarFileTests { } } + @Test + void zip64JarThatExceedsZipSizeLimitCanBeRead() throws Exception { + Assumptions.assumeTrue(this.tempDir.getFreeSpace() > 6 * 1024 * 1024 * 1024, "Insufficient disk space"); + File zip64Jar = new File(this.tempDir, "zip64.jar"); + File entry = new File(this.tempDir, "entry.dat"); + CRC32 crc32 = new CRC32(); + try (FileOutputStream entryOut = new FileOutputStream(entry)) { + byte[] data = new byte[1024 * 1024]; + new Random().nextBytes(data); + for (int i = 0; i < 1024; i++) { + entryOut.write(data); + crc32.update(data); + } + } + try (JarOutputStream jarOutput = new JarOutputStream(new FileOutputStream(zip64Jar))) { + for (int i = 0; i < 6; i++) { + JarEntry storedEntry = new JarEntry("huge-" + i); + storedEntry.setSize(entry.length()); + storedEntry.setCompressedSize(entry.length()); + storedEntry.setCrc(crc32.getValue()); + storedEntry.setMethod(ZipEntry.STORED); + jarOutput.putNextEntry(storedEntry); + try (FileInputStream entryIn = new FileInputStream(entry)) { + StreamUtils.copy(entryIn, jarOutput); + } + jarOutput.closeEntry(); + } + } + try (JarFile zip64JarFile = new JarFile(zip64Jar)) { + assertThat(Collections.list(zip64JarFile.entries())).hasSize(6); + } + } + @Test void nestedZip64JarCanBeRead() throws Exception { File outer = new File(this.tempDir, "outer.jar");