8029646: [pack200] should support the new zip64 format
Reviewed-by: ksrini, sherman
diff --git a/jdk/src/share/native/com/sun/java/util/jar/pack/zip.cpp b/jdk/src/share/native/com/sun/java/util/jar/pack/zip.cpp
index 2544091..786a832 100644
--- a/jdk/src/share/native/com/sun/java/util/jar/pack/zip.cpp
+++ b/jdk/src/share/native/com/sun/java/util/jar/pack/zip.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2001, 2013, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2014, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -73,8 +73,9 @@
SWAP_BYTES(a & 0xFFFF)
#define GET_INT_HI(a) \
- SWAP_BYTES((a >> 16) & 0xFFFF);
+ SWAP_BYTES((a >> 16) & 0xFFFF)
+static const ushort jarmagic[2] = { SWAP_BYTES(0xCAFE), 0 };
void jar::init(unpacker* u_) {
BYTES_OF(*this).clear();
@@ -105,13 +106,14 @@
header[0] = (ushort)SWAP_BYTES(0x4B50);
header[1] = (ushort)SWAP_BYTES(0x0201);
- header[2] = (ushort)SWAP_BYTES(0xA);
+ header[2] = (ushort)SWAP_BYTES(( store ) ? 0x0A : 0x14);
// required version
- header[3] = (ushort)SWAP_BYTES(0xA);
+ header[3] = (ushort)SWAP_BYTES(( store ) ? 0x0A : 0x14);
- // flags 02 = maximum sub-compression flag
- header[4] = ( store ) ? 0x0 : SWAP_BYTES(0x2);
+ // Flags - UTF-8 compression and separating crc and sizes
+ // into separate headers for deflated file
+ header[4] = ( store ) ? SWAP_BYTES(0x0800) : 0x0808;
// Compression method 8=deflate.
header[5] = ( store ) ? 0x0 : SWAP_BYTES(0x08);
@@ -135,7 +137,8 @@
// Filename length
header[14] = (ushort)SWAP_BYTES(fname_length);
// So called "extra field" length.
- header[15] = 0;
+ // If it's the first record we must add JAR magic sequence
+ header[15] = ( central_directory_count ) ? 0 : (ushort)SWAP_BYTES(4);
// So called "comment" length.
header[16] = 0;
// Disk number start
@@ -155,6 +158,11 @@
// Copy the fname to the header.
central_directory.append(fname, fname_length);
+ // Add jar magic for the first record
+ if (central_directory_count == 0) {
+ central_directory.append((void *)jarmagic, sizeof(jarmagic));
+ }
+
central_directory_count++;
}
@@ -170,10 +178,10 @@
header[1] = (ushort)SWAP_BYTES(0x0403);
// Version
- header[2] = (ushort)SWAP_BYTES(0xA);
+ header[2] = (ushort)SWAP_BYTES(( store ) ? 0x0A : 0x14);
- // flags 02 = maximum sub-compression flag
- header[3] = ( store ) ? 0x0 : SWAP_BYTES(0x2);
+ // General purpose flags - same as in the Central Directory
+ header[3] = ( store ) ? SWAP_BYTES(0x0800) : 0x0808;
// Compression method = deflate
header[4] = ( store ) ? 0x0 : SWAP_BYTES(0x08);
@@ -182,28 +190,51 @@
header[5] = (ushort)GET_INT_LO(dostime);
header[6] = (ushort)GET_INT_HI(dostime);
- // CRC
- header[7] = (ushort)GET_INT_LO(crc);
- header[8] = (ushort)GET_INT_HI(crc);
+ // CRC, 0 if deflated, will come separately in extra header
+ header[7] = ( store ) ? (ushort)GET_INT_LO(crc) : 0;
+ header[8] = ( store ) ? (ushort)GET_INT_HI(crc) : 0;
- // Compressed length:
- header[9] = (ushort)GET_INT_LO(clen);
- header[10] = (ushort)GET_INT_HI(clen);
+ // Compressed length, 0 if deflated
+ header[9] = ( store ) ? (ushort)GET_INT_LO(clen) : 0;
+ header[10] = ( store ) ? (ushort)GET_INT_HI(clen) : 0;
- // Uncompressed length.
- header[11] = (ushort)GET_INT_LO(len);
- header[12] = (ushort)GET_INT_HI(len);
+ // Uncompressed length, 0 if deflated
+ header[11] = ( store ) ? (ushort)GET_INT_LO(len) : 0;
+ header[12] = ( store ) ? (ushort)GET_INT_HI(len) : 0;
// Filename length
header[13] = (ushort)SWAP_BYTES(fname_length);
// So called "extra field" length.
- header[14] = 0;
+ header[14] = ( central_directory_count - 1 ) ? 0 : (ushort)SWAP_BYTES(4);
// Write the LOC header to the output file.
write_data(header, (int)sizeof(header));
// Copy the fname to the header.
write_data((char*)fname, (int)fname_length);
+
+ if (central_directory_count == 1) {
+ // Write JAR magic sequence
+ write_data((void *)jarmagic, (int)sizeof(jarmagic));
+ }
+}
+
+void jar::write_jar_extra(int len, int clen, uint crc) {
+ ushort header[8];
+ // Extra field signature
+ header[0] = (ushort)SWAP_BYTES(0x4B50);
+ header[1] = (ushort)SWAP_BYTES(0x0807);
+ // CRC
+ header[2] = (ushort)GET_INT_LO(crc);
+ header[3] = (ushort)GET_INT_HI(crc);
+ // Compressed length
+ header[4] = (ushort)GET_INT_LO(clen);
+ header[5] = (ushort)GET_INT_HI(clen);
+ // Uncompressed length
+ header[6] = (ushort)GET_INT_LO(len);
+ header[7] = (ushort)GET_INT_HI(len);
+
+ write_data(header, sizeof(header));
}
static const char marker_comment[] = ZIP_ARCHIVE_MARKER_COMMENT;
@@ -212,6 +243,7 @@
bytes mc; mc.set(marker_comment);
ushort header[11];
+ ushort header64[38];
// Create the End of Central Directory structure.
header[0] = (ushort)SWAP_BYTES(0x4B50);
@@ -220,8 +252,8 @@
header[2] = 0;
header[3] = 0;
// Number of entries in central directory.
- header[4] = (ushort)SWAP_BYTES(central_directory_count);
- header[5] = (ushort)SWAP_BYTES(central_directory_count);
+ header[4] = ( central_directory_count >= 0xffff ) ? 0xffff : (ushort)SWAP_BYTES(central_directory_count);
+ header[5] = ( central_directory_count >= 0xffff ) ? 0xffff : (ushort)SWAP_BYTES(central_directory_count);
// Size of the central directory}
header[6] = (ushort)GET_INT_LO((int)central_directory.size());
header[7] = (ushort)GET_INT_HI((int)central_directory.size());
@@ -229,12 +261,71 @@
header[8] = (ushort)GET_INT_LO(output_file_offset);
header[9] = (ushort)GET_INT_HI(output_file_offset);
// zipfile comment length;
- header [10] = (ushort)SWAP_BYTES((int)mc.len);
+ header[10] = (ushort)SWAP_BYTES((int)mc.len);
// Write the central directory.
PRINTCR((2, "Central directory at %d\n", output_file_offset));
write_data(central_directory.b);
+ // If number of records exceeds the 0xFFFF we need to prepend extended
+ // Zip64 End of Central Directory record and its locator to the old
+ // style ECD record
+ if (central_directory_count > 0xFFFF) {
+ // Zip64 END signature
+ header64[0] = (ushort)SWAP_BYTES(0x4B50);
+ header64[1] = (ushort)0x0606;
+ // Size of header (long)
+ header64[2] = (ushort)SWAP_BYTES(44);;
+ header64[3] = 0;
+ header64[4] = 0;
+ header64[5] = 0;
+ // Version produced and required (short)
+ header64[6] = (ushort)SWAP_BYTES(45);
+ header64[7] = (ushort)SWAP_BYTES(45);
+ // Current disk number (int)
+ header64[8] = 0;
+ header64[9] = 0;
+ // Central directory start disk (int)
+ header64[10] = 0;
+ header64[11] = 0;
+ // Count of records on disk (long)
+ header64[12] = (ushort)GET_INT_LO(central_directory_count);
+ header64[13] = (ushort)GET_INT_HI(central_directory_count);
+ header64[14] = 0;
+ header64[15] = 0;
+ // Count of records totally (long)
+ header64[16] = (ushort)GET_INT_LO(central_directory_count);
+ header64[17] = (ushort)GET_INT_HI(central_directory_count);
+ header64[18] = 0;
+ header64[19] = 0;
+ // Length of the central directory (long)
+ header64[20] = header[6];
+ header64[21] = header[7];
+ header64[22] = 0;
+ header64[23] = 0;
+ // Offset of central directory (long)
+ header64[24] = header[8];
+ header64[25] = header[9];
+ header64[26] = 0;
+ header64[27] = 0;
+ // Zip64 end of central directory locator
+ // Locator signature
+ header64[28] = (ushort)SWAP_BYTES(0x4B50);
+ header64[29] = (ushort)SWAP_BYTES(0x0706);
+ // Start disk number (int)
+ header64[30] = 0;
+ header64[31] = 0;
+ // Offset of zip64 END record (long)
+ header64[32] = (ushort)GET_INT_LO(output_file_offset);
+ header64[33] = (ushort)GET_INT_HI(output_file_offset);
+ header64[34] = 0;
+ header64[35] = 0;
+ // Total number of disks (int)
+ header64[36] = (ushort)SWAP_BYTES(1);
+ header64[37] = 0;
+ write_data(header64, (int)sizeof(header64));
+ }
+
// Write the End of Central Directory structure.
PRINTCR((2, "end-of-directory at %d\n", output_file_offset));
write_data(header, (int)sizeof(header));
@@ -286,6 +377,8 @@
if (deflate) {
write_data(deflated.b);
+ // Write deflated information in extra header
+ write_jar_extra(len, clen, crc);
} else {
write_data(head);
write_data(tail);
@@ -368,7 +461,7 @@
// NOTE: the window size should always be -MAX_WBITS normally -15.
// unzip/zipup.c and java/Deflater.c
- int error = deflateInit2(&zs, Z_BEST_COMPRESSION, Z_DEFLATED,
+ int error = deflateInit2(&zs, Z_DEFAULT_COMPRESSION, Z_DEFLATED,
-MAX_WBITS, 8, Z_DEFAULT_STRATEGY);
if (error != Z_OK) {
switch (error) {
@@ -414,7 +507,8 @@
error = deflate(&zs, Z_FINISH);
}
if (error == Z_STREAM_END) {
- if (len > (int)zs.total_out ) {
+ if ((int)zs.total_out > 0) {
+ // Even if compressed size is bigger than uncompressed, write it
PRINTCR((2, "deflate compressed data %d -> %d\n", len, zs.total_out));
deflated.b.len = zs.total_out;
deflateEnd(&zs);
diff --git a/jdk/src/share/native/com/sun/java/util/jar/pack/zip.h b/jdk/src/share/native/com/sun/java/util/jar/pack/zip.h
index d7c6b34..14ffc9d 100644
--- a/jdk/src/share/native/com/sun/java/util/jar/pack/zip.h
+++ b/jdk/src/share/native/com/sun/java/util/jar/pack/zip.h
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2001, 2008, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2014, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -40,7 +40,7 @@
// Private members
fillbytes central_directory;
- ushort central_directory_count;
+ uint central_directory_count;
uint output_file_offset;
fillbytes deflated; // temporary buffer
@@ -74,6 +74,7 @@
int len, int clen, uLong crc);
void write_jar_header(const char* fname, bool store, int modtime,
int len, int clen, unsigned int crc);
+ void write_jar_extra(int len, int clen, unsigned int crc);
void write_central_directory();
uLong dostime(int y, int n, int d, int h, int m, int s);
uLong get_dostime(int modtime);
diff --git a/jdk/test/tools/pack200/PackTestZip64.java b/jdk/test/tools/pack200/PackTestZip64.java
new file mode 100644
index 0000000..edfeb5a
--- /dev/null
+++ b/jdk/test/tools/pack200/PackTestZip64.java
@@ -0,0 +1,152 @@
+/*
+ * Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+import java.io.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Enumeration;
+import java.util.List;
+import java.util.jar.JarEntry;
+import java.util.jar.JarFile;
+import java.util.jar.JarInputStream;
+import java.util.jar.JarOutputStream;
+import java.util.zip.ZipEntry;
+/*
+ * @test
+ * @bug 8029646
+ * @summary tests that native unpacker produces the same result as Java one
+ * @compile -XDignore.symbol.file Utils.java PackTestZip64.java
+ * @run main PackTestZip64
+ * @author kizune
+ */
+
+public class PackTestZip64 {
+ public static void main(String... args) throws Exception {
+ testPacking();
+ Utils.cleanup();
+ }
+
+ // 1KB buffer is enough to copy jar content
+ private static final byte[] BUFFER = new byte[1024];
+
+ static void testPacking() throws IOException {
+ // make a copy of the test specimen to local directory
+ File testFile = new File("tools_java.jar");
+ // Add a large number of small files to the golden jar
+ generateLargeJar(testFile, Utils.locateJar("golden.jar"));
+
+ List<String> cmdsList = new ArrayList<>();
+
+ // Repack file to get the Java-based result
+ cmdsList.add(Utils.getPack200Cmd());
+ cmdsList.add("--repack");
+ cmdsList.add(testFile.getName());
+ Utils.runExec(cmdsList);
+ cmdsList.clear();
+
+ // Pack file with pack200 and unpack in with unpack200
+ File packedFile = new File("tools.pack.gz");
+ cmdsList.add(Utils.getPack200Cmd());
+ cmdsList.add(packedFile.getName());
+ cmdsList.add(testFile.getName());
+ Utils.runExec(cmdsList);
+ cmdsList.clear();
+
+ File unpackedFile = new File("tools_native.jar");
+ cmdsList.add(Utils.getUnpack200Cmd());
+ cmdsList.add(packedFile.getName());
+ cmdsList.add(unpackedFile.getName());
+ Utils.runExec(cmdsList);
+
+ // Compare files binary
+ compareTwoFiles(testFile, unpackedFile);
+
+ // Cleaning up generated files
+ testFile.delete();
+ packedFile.delete();
+ unpackedFile.delete();
+ }
+
+ static void compareTwoFiles(File src, File dst) throws IOException {
+ if (!src.exists()) {
+ throw new IOException("File " + src.getName() + " does not exist!");
+ }
+
+ if(!dst.exists()) {
+ throw new IOException("File " + dst.getName() + " does not exist!");
+ }
+
+ BufferedInputStream srcis, dstis;
+ srcis = new BufferedInputStream(new FileInputStream(src));
+ dstis = new BufferedInputStream(new FileInputStream(dst));
+
+ int s = 0, d, pos = 0;
+ while (s != -1) { // Checking of just one result for EOF is enough
+ s = srcis.read();
+ d = dstis.read();
+
+ if (s != d) {
+ throw new IOException("Files are differ starting at position: "
+ + Integer.toHexString(pos));
+ }
+
+ pos++;
+ }
+
+ srcis.close();
+ dstis.close();
+ }
+
+ static void generateLargeJar(File result, File source) throws IOException {
+ if (result.exists()) {
+ result.delete();
+ }
+
+ try (JarOutputStream copyTo = new JarOutputStream(new FileOutputStream(result));
+ JarFile srcJar = new JarFile(source)) {
+
+ for (JarEntry je : Collections.list(srcJar.entries())) {
+ copyTo.putNextEntry(je);
+ if (!je.isDirectory()) {
+ copyStream(srcJar.getInputStream(je), copyTo);
+ }
+ copyTo.closeEntry();
+ }
+
+ int many = Short.MAX_VALUE * 2 + 2;
+
+ for (int i = 0 ; i < many ; i++) {
+ JarEntry e = new JarEntry("F-" + i + ".txt");
+ copyTo.putNextEntry(e);
+ }
+ copyTo.flush();
+ copyTo.close();
+ }
+ }
+
+ static void copyStream(InputStream in, OutputStream out) throws IOException {
+ int bytesRead;
+ while ((bytesRead = in.read(BUFFER))!= -1) {
+ out.write(BUFFER, 0, bytesRead);
+ }
+ }
+}
