Add utilities for parsing/extracting individual files from a zip file.
Main features added are:
1. Parse zip file structure from partial zip file.
2. Extract a specific file from partial zip file.
3. Confirm file CRC check and permissions are correct.
Bug: 73786521
Test: unittest
Change-Id: I7d2a497eb0f6dbf53602371ad8eca5fc911988be
diff --git a/common_util/com/android/tradefed/util/ByteArrayUtil.java b/common_util/com/android/tradefed/util/ByteArrayUtil.java
new file mode 100644
index 0000000..510321b
--- /dev/null
+++ b/common_util/com/android/tradefed/util/ByteArrayUtil.java
@@ -0,0 +1,92 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.tradefed.util;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+
+/**
+ * Utilities to operate on byte array, e.g., convert bytes to integer.
+ *
+ * <p>Java doesn't have an unsigned value type, so expansion is needed to convert an unsigned
+ * integer stored in 4 bytes to a long value, or unsigned short stored in 2 bytes to an integer
+ * value.
+ */
+public class ByteArrayUtil {
+
+ /**
+ * Get a {@link ByteBuffer} for the given bytes wrapped in a byte array of given size.
+ *
+ * <p>java doesn't have an unsigned value type, so expansion is needed to convert an unsigned
+ * short stored in 2 bytes to an integer value.
+ *
+ * @param bytes an array of bytes.
+ * @param offset the start offset of the integer data.
+ * @param length the length of the integer data.
+ * @param containerSize the size of the array to store the given bytes, append zero to unfilled
+ * items.
+ * @return a {@link ByteBuffer} for the given bytes wrapped in a byte array of given size.
+ */
+ private static ByteBuffer getByteBuffer(
+ byte[] bytes, int offset, int length, int containerSize) {
+ byte[] data = new byte[containerSize];
+ for (int i = 0; i < length; i++) {
+ data[i] = bytes[offset + i];
+ }
+ return ByteBuffer.wrap(data).order(java.nio.ByteOrder.LITTLE_ENDIAN);
+ }
+
+ /**
+ * Get an integer from the given bytes.
+ *
+ * <p>java doesn't have an unsigned value type, so expansion is needed to convert an unsigned
+ * short stored in 2 bytes to an integer value.
+ *
+ * @param bytes an array of bytes.
+ * @param offset the start offset of the integer data.
+ * @param length the length of the integer data.
+ * @return an int value from the given bytes.
+ */
+ public static int getInt(byte[] bytes, int offset, int length) {
+ return getByteBuffer(bytes, offset, length, 4).getInt();
+ }
+
+ /**
+ * Get a long value from the given bytes.
+ *
+ * <p>java doesn't have an unsigned value type, so expansion is needed to convert an unsigned
+ * integer stored in 4 bytes to a long value.
+ *
+ * @param bytes an array of bytes.
+ * @param offset the start offset of the long value.
+ * @param length the length of the long value.
+ * @return a long value from the given bytes.
+ */
+ public static long getLong(byte[] bytes, int offset, int length) {
+ return getByteBuffer(bytes, offset, length, 8).getLong();
+ }
+
+ /**
+ * Get the string from the given bytes.
+ *
+ * @param bytes an array of bytes.
+ * @param offset the start offset of the string data.
+ * @param length the length of the string data.
+ */
+ public static String getString(byte[] bytes, int offset, int length) {
+ return new String(Arrays.copyOfRange(bytes, offset, offset + length));
+ }
+}
diff --git a/common_util/com/android/tradefed/util/FileUtil.java b/common_util/com/android/tradefed/util/FileUtil.java
index ee158e2..2034972 100644
--- a/common_util/com/android/tradefed/util/FileUtil.java
+++ b/common_util/com/android/tradefed/util/FileUtil.java
@@ -616,12 +616,29 @@
*/
public static void writeToFile(
InputStream input, File destFile, boolean append) throws IOException {
+ // Set size to a negative value to write all content starting at the given offset.
+ writeToFile(input, destFile, append, 0, -1);
+ }
+
+ /**
+ * A helper method for writing stream data to file
+ *
+ * @param input the unbuffered input stream
+ * @param destFile the destination file to write or append to
+ * @param append append to end of file if true, overwrite otherwise
+ * @param startOffset the start offset of the input stream to retrieve data
+ * @param size number of bytes to retrieve from the input stream, set it to a negative value to
+ * retrieve all content starting at the given offset.
+ */
+ public static void writeToFile(
+ InputStream input, File destFile, boolean append, long startOffset, long size)
+ throws IOException {
InputStream origStream = null;
OutputStream destStream = null;
try {
origStream = new BufferedInputStream(input);
destStream = new BufferedOutputStream(new FileOutputStream(destFile, append));
- StreamUtil.copyStreams(origStream, destStream);
+ StreamUtil.copyStreams(origStream, destStream, startOffset, size);
} finally {
StreamUtil.close(origStream);
StreamUtil.flushAndCloseStream(destStream);
diff --git a/common_util/com/android/tradefed/util/StreamUtil.java b/common_util/com/android/tradefed/util/StreamUtil.java
index 15d32a1..f6fce21 100644
--- a/common_util/com/android/tradefed/util/StreamUtil.java
+++ b/common_util/com/android/tradefed/util/StreamUtil.java
@@ -173,16 +173,51 @@
*
* @param inStream the {@link InputStream}
* @param outStream the {@link OutputStream}
- * @param offset The offset of when to start copying the data.
+ * @param offset the offset of when to start copying the data.
* @throws IOException
*/
public static void copyStreams(InputStream inStream, OutputStream outStream, int offset)
throws IOException {
+ // Set size to a negative value to copy all content starting at the given offset.
+ copyStreams(inStream, outStream, offset, -1);
+ }
+
+ /**
+ * Copies contents of origStream to destStream starting at a given offset with a specific size.
+ *
+ * <p>Recommended to provide a buffered stream for input and output
+ *
+ * @param inStream the {@link InputStream}
+ * @param outStream the {@link OutputStream}
+ * @param offset the offset of when to start copying the data.
+ * @param size the number of bytes to copy. A negative value means to copy all content.
+ * @throws IOException
+ */
+ public static void copyStreams(
+ InputStream inStream, OutputStream outStream, long offset, long size)
+ throws IOException {
+ assert offset >= 0 : "offset must be greater or equal to zero.";
+ assert size != 0 : "size cannot be zero.";
inStream.skip(offset);
byte[] buf = new byte[BUF_SIZE];
- int size = -1;
- while ((size = inStream.read(buf)) != -1) {
- outStream.write(buf, 0, size);
+ long totalRetrievedSize = 0;
+ int retrievedSize = -1;
+ while ((retrievedSize = inStream.read(buf)) != -1) {
+ if (size > 0 && size < totalRetrievedSize + retrievedSize) {
+ retrievedSize = (int) (size - totalRetrievedSize);
+ }
+ outStream.write(buf, 0, retrievedSize);
+ totalRetrievedSize += retrievedSize;
+ if (size == totalRetrievedSize) {
+ break;
+ }
+ }
+ if (size > 0 && size > totalRetrievedSize) {
+ throw new IOException(
+ String.format(
+ "Failed to read %d bytes starting at offset %d, only %d bytes "
+ + "retrieved.",
+ size, offset, totalRetrievedSize));
}
}
diff --git a/common_util/com/android/tradefed/util/ZipUtil.java b/common_util/com/android/tradefed/util/ZipUtil.java
index 61f46a2..6204b8a 100644
--- a/common_util/com/android/tradefed/util/ZipUtil.java
+++ b/common_util/com/android/tradefed/util/ZipUtil.java
@@ -16,6 +16,9 @@
package com.android.tradefed.util;
import com.android.tradefed.log.LogUtil.CLog;
+import com.android.tradefed.util.zip.CentralDirectoryInfo;
+import com.android.tradefed.util.zip.EndCentralDirectoryInfo;
+import com.android.tradefed.util.zip.LocalFileHeader;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
@@ -25,10 +28,14 @@
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
+import java.nio.file.Files;
+import java.util.ArrayList;
import java.util.Enumeration;
import java.util.LinkedList;
import java.util.List;
+import java.util.zip.DataFormatException;
import java.util.zip.GZIPOutputStream;
+import java.util.zip.Inflater;
import java.util.zip.ZipEntry;
import java.util.zip.ZipException;
import java.util.zip.ZipFile;
@@ -39,9 +46,19 @@
*/
public class ZipUtil {
+ private static final int COMPRESSION_METHOD_STORED = 0;
+ private static final int COMPRESSION_METHOD_DEFLATE = 8;
private static final String DEFAULT_DIRNAME = "dir";
private static final String DEFAULT_FILENAME = "files";
private static final String ZIP_EXTENSION = ".zip";
+ private static final String PARTIAL_ZIP_DATA = "compressed_data";
+
+ private static final boolean IS_UNIX;
+
+ static {
+ String OS = System.getProperty("os.name").toLowerCase();
+ IS_UNIX = (OS.contains("nix") || OS.contains("nux") || OS.contains("aix"));
+ }
/**
* Utility method to verify that a zip file is not corrupt.
@@ -347,4 +364,212 @@
throw e;
}
}
+
+ /**
+ * Get a list of {link CentralDirectoryInfo} for files in a zip file.
+ *
+ * @param partialZipFile a {@link File} object of the partial zip file that contains central
+ * directory entries.
+ * @param endCentralDirInfo a {@link EndCentralDirectoryInfo} object of the zip file.
+ * @return A list of {@link CentralDirectoryInfo} of the zip file
+ * @throws IOException
+ */
+ public static List<CentralDirectoryInfo> getZipCentralDirectoryInfos(
+ File partialZipFile, EndCentralDirectoryInfo endCentralDirInfo) throws IOException {
+ return getZipCentralDirectoryInfos(partialZipFile, endCentralDirInfo, 0);
+ }
+
+ /**
+ * Get a list of {link CentralDirectoryInfo} for files in a zip file.
+ *
+ * @param partialZipFile a {@link File} object of the partial zip file that contains central
+ * directory entries.
+ * @param endCentralDirInfo a {@link EndCentralDirectoryInfo} object of the zip file.
+ * @param offset the offset in the partial zip file where the content of central directory
+ * entries starts.
+ * @return A list of {@link CentralDirectoryInfo} of the zip file
+ * @throws IOException
+ */
+ public static List<CentralDirectoryInfo> getZipCentralDirectoryInfos(
+ File partialZipFile, EndCentralDirectoryInfo endCentralDirInfo, long offset)
+ throws IOException {
+ List<CentralDirectoryInfo> infos = new ArrayList<>();
+ byte[] data;
+ try (FileInputStream stream = new FileInputStream(partialZipFile)) {
+ // Read in the entire central directory block for a zip file till the end. The block
+ // should be small even for a large zip file.
+ long totalSize = stream.getChannel().size();
+ stream.skip(offset);
+ data = new byte[(int) (totalSize - offset)];
+ stream.read(data);
+ }
+ int startOffset = 0;
+ for (int i = 0; i < endCentralDirInfo.getEntryNumber(); i++) {
+ CentralDirectoryInfo info = new CentralDirectoryInfo(data, startOffset);
+ infos.add(info);
+ startOffset += info.getInfoSize();
+ }
+
+ return infos;
+ }
+
+ /**
+ * Apply the file permission configured in the central directory entry.
+ *
+ * @param targetFile the {@link File} to set permission to.
+ * @param zipEntry a {@link CentralDirectoryInfo} object that contains the file permissions.
+ * @throws IOException if fail to access the file.
+ */
+ public static void applyPermission(File targetFile, CentralDirectoryInfo zipEntry)
+ throws IOException {
+ if (!IS_UNIX) {
+ CLog.w("Permission setting is only supported in Unix/Linux system.");
+ return;
+ }
+
+ if (zipEntry.getFilePermission() != 0) {
+ Files.setPosixFilePermissions(
+ targetFile.toPath(), FileUtil.unixModeToPosix(zipEntry.getFilePermission()));
+ }
+ }
+
+ /**
+ * Extract the requested folder from a partial zip file and apply proper permission.
+ *
+ * @param targetFile the {@link File} to save the extracted file to.
+ * @param zipEntry a {@link CentralDirectoryInfo} object of the file to extract from the partial
+ * zip file.
+ * @throws IOException
+ */
+ public static void unzipPartialZipFolder(File targetFile, CentralDirectoryInfo zipEntry)
+ throws IOException {
+ unzipPartialZipFile(null, targetFile, zipEntry, null, -1);
+ }
+
+ /**
+ * Extract the requested file from a partial zip file.
+ *
+ * <p>This method assumes all files are on the same disk when compressed. It doesn't support
+ * following features yet:
+ *
+ * <p>Zip file larger than 4GB
+ *
+ * <p>ZIP64(require ZipLocalFileHeader update on compressed size)
+ *
+ * <p>Encrypted zip file
+ *
+ * <p>Symlink
+ *
+ * @param partialZip a {@link File} that's a partial of the zip file.
+ * @param targetFile the {@link File} to save the extracted file to.
+ * @param zipEntry a {@link CentralDirectoryInfo} object of the file to extract from the partial
+ * zip file.
+ * @param localFileHeader a {@link LocalFileHeader} object of the file to extract from the
+ * partial zip file.
+ * @param startOffset start offset of the file to extract.
+ * @throws IOException
+ */
+ public static void unzipPartialZipFile(
+ File partialZip,
+ File targetFile,
+ CentralDirectoryInfo zipEntry,
+ LocalFileHeader localFileHeader,
+ long startOffset)
+ throws IOException {
+ try {
+ if (zipEntry.getFileName().endsWith("/")) {
+ // Create a folder.
+ targetFile.mkdir();
+ return;
+ } else if (zipEntry.getCompressedSize() == 0) {
+ // The file is empty, just create an empty file.
+ targetFile.createNewFile();
+ return;
+ }
+
+ File zipFile = targetFile;
+ if (zipEntry.getCompressionMethod() != COMPRESSION_METHOD_STORED)
+ // Create a temp file to store the compressed data, then unzip it.
+ zipFile = FileUtil.createTempFile(PARTIAL_ZIP_DATA, ZIP_EXTENSION);
+ else {
+ // The file is not compressed, stream it directly to the target.
+ zipFile.getParentFile().mkdirs();
+ zipFile.createNewFile();
+ }
+
+ // Save compressed data to zipFile
+ try (FileInputStream stream = new FileInputStream(partialZip)) {
+ FileUtil.writeToFile(
+ stream,
+ zipFile,
+ false,
+ startOffset + localFileHeader.getHeaderSize(),
+ zipEntry.getCompressedSize());
+ }
+
+ if (zipEntry.getCompressionMethod() == COMPRESSION_METHOD_STORED) {
+ return;
+ } else if (zipEntry.getCompressionMethod() == COMPRESSION_METHOD_DEFLATE) {
+ boolean success = false;
+ try {
+ unzipRawZip(zipFile, targetFile, zipEntry);
+ success = true;
+ } catch (DataFormatException e) {
+ throw new IOException(e);
+ } finally {
+ zipFile.delete();
+ if (!success) {
+ CLog.e("Failed to unzip %s", zipEntry.getFileName());
+ targetFile.delete();
+ }
+ }
+ } else {
+ throw new RuntimeException(
+ String.format(
+ "Compression method %d is not supported.",
+ localFileHeader.getCompressionMethod()));
+ }
+ } finally {
+ if (targetFile.exists()) {
+ applyPermission(targetFile, zipEntry);
+ }
+ }
+ }
+
+ /**
+ * Unzip the raw compressed content without wrapper (local file header).
+ *
+ * @param zipFile the {@link File} that contains the compressed data of the target file.
+ * @param targetFile {@link File} to same the decompressed data to.
+ * @throws DataFormatException if decompression failed due to zip format issue.
+ * @throws IOException if failed to access the compressed data or the decompressed file has
+ * mismatched CRC.
+ */
+ private static void unzipRawZip(File zipFile, File targetFile, CentralDirectoryInfo zipEntry)
+ throws IOException, DataFormatException {
+ Inflater decompresser = new Inflater(true);
+
+ targetFile.getParentFile().mkdirs();
+ targetFile.createNewFile();
+
+ try (FileInputStream inputStream = new FileInputStream(zipFile);
+ FileOutputStream outputStream = new FileOutputStream(targetFile)) {
+ byte[] data = new byte[32768];
+ byte[] buffer = new byte[65536];
+ while (inputStream.read(data) > 0) {
+ decompresser.setInput(data);
+ while (!decompresser.finished() && !decompresser.needsInput()) {
+ int size = decompresser.inflate(buffer);
+ outputStream.write(buffer, 0, size);
+ }
+ }
+ } finally {
+ decompresser.end();
+ }
+
+ // Validate CRC
+ if (FileUtil.calculateCrc32(targetFile) != zipEntry.getCrc()) {
+ throw new IOException(String.format("Failed to match CRC for file %s", targetFile));
+ }
+ }
}
diff --git a/common_util/com/android/tradefed/util/zip/CentralDirectoryInfo.java b/common_util/com/android/tradefed/util/zip/CentralDirectoryInfo.java
new file mode 100644
index 0000000..5b16625
--- /dev/null
+++ b/common_util/com/android/tradefed/util/zip/CentralDirectoryInfo.java
@@ -0,0 +1,217 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.tradefed.util.zip;
+
+import com.android.tradefed.util.ByteArrayUtil;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+/**
+ * CentralDirectoryInfo is a class containing the information of a file/folder inside a zip file.
+ *
+ * <p>Overall zipfile format: [Local file header + Compressed data [+ Extended local header]?]*
+ * [Central directory]* [End of central directory record]
+ *
+ * <p>Refer to following link for more details: https://en.wikipedia.org/wiki/Zip_(file_format)
+ */
+public final class CentralDirectoryInfo {
+
+ private static final byte[] CENTRAL_DIRECTORY_SIGNATURE = {0x50, 0x4b, 0x01, 0x02};
+
+ private int mCompressionMethod;
+ private long mCrc;
+ private long mCompressedSize;
+ private long mUncompressedSize;
+ private long mLocalHeaderOffset;
+ private int mInternalFileAttributes;
+ private long mExternalFileAttributes;
+ private String mFileName;
+ private int mFileNameLength;
+ private int mExtraFieldLength;
+ private int mFileCommentLength;
+
+ /** Get the compression method. */
+ public int getCompressionMethod() {
+ return mCompressionMethod;
+ }
+
+ /** Set the compression method. */
+ public void setCompressionMethod(int compressionMethod) {
+ mCompressionMethod = compressionMethod;
+ }
+
+ /** Get the CRC of the file. */
+ public long getCrc() {
+ return mCrc;
+ }
+
+ /** Set the CRC of the file. */
+ public void setCrc(long crc) {
+ mCrc = crc;
+ }
+
+ /** Get the compressed size. */
+ public int getCompressedSize() {
+ return (int) mCompressedSize;
+ }
+
+ /** Set the compressed size. */
+ public void setCompressedSize(long compressionSize) {
+ mCompressedSize = compressionSize;
+ }
+
+ /** Get the uncompressed size. */
+ public long getUncompressedSize() {
+ return mUncompressedSize;
+ }
+
+ /** Set the uncompressed size. */
+ public void setUncompressedSize(long uncompressedSize) {
+ mUncompressedSize = uncompressedSize;
+ }
+
+ /** Get the offset of local file header entry. */
+ public long getLocalHeaderOffset() {
+ return mLocalHeaderOffset;
+ }
+
+ /** Set the offset of local file header entry. */
+ public void setLocalHeaderOffset(long localHeaderOffset) {
+ mLocalHeaderOffset = localHeaderOffset;
+ }
+
+ /** Get the internal file attributes. */
+ public int getInternalFileAttributes() {
+ return mInternalFileAttributes;
+ }
+
+ /** Set the internal file attributes. */
+ public void setInternalFileAttributes(int internalFileAttributes) {
+ mInternalFileAttributes = internalFileAttributes;
+ }
+
+ /** Get the external file attributes. */
+ public long getExternalFileAttributes() {
+ return mExternalFileAttributes;
+ }
+
+ /** Set the external file attributes. */
+ public void setExternalFileAttributes(long externalFileAttributes) {
+ mExternalFileAttributes = externalFileAttributes;
+ }
+
+ /** Get the Linux file permission, stored in the last 9 bits of external file attributes. */
+ public int getFilePermission() {
+ return ((int) mExternalFileAttributes & (0777 << 16L)) >> 16L;
+ }
+
+ /** Get the file name including the relative path. */
+ public String getFileName() {
+ return mFileName;
+ }
+
+ /** Set the file name including the relative path. */
+ public void setFileName(String fileName) {
+ mFileName = fileName;
+ }
+
+ /** Get the file name length. */
+ public int getFileNameLength() {
+ return mFileNameLength;
+ }
+
+ /** Set the file name length. */
+ public void setFileNameLength(int fileNameLength) {
+ mFileNameLength = fileNameLength;
+ }
+
+ /** Get the extra field length. */
+ public int getExtraFieldLength() {
+ return mExtraFieldLength;
+ }
+
+ /** Set the extra field length. */
+ public void setExtraFieldLength(int extraFieldLength) {
+ mExtraFieldLength = extraFieldLength;
+ }
+
+ /** Get the file comment length. */
+ public int getFileCommentLength() {
+ return mFileCommentLength;
+ }
+
+ /** Set the file comment length. */
+ public void setFileCommentLength(int fileCommentLength) {
+ mFileCommentLength = fileCommentLength;
+ }
+
+ /** Get the size of the central directory entry. */
+ public int getInfoSize() {
+ return 46 + mFileNameLength + mExtraFieldLength + mFileCommentLength;
+ }
+
+ /**
+ * Constructor to collect the information of a file entry inside zip file.
+ *
+ * @param data {@code byte[]} of data that contains the information of a file entry.
+ * @param startOffset start offset of the information block.
+ * @throws IOException
+ */
+ public CentralDirectoryInfo(byte[] data, int startOffset) throws IOException {
+ // Central directory:
+ // Offset Length Contents
+ // 0 4 bytes Central file header signature (0x02014b50)
+ // 4 2 bytes Version made by
+ // 6 2 bytes Version needed to extract
+ // 8 2 bytes General purpose bit flag
+ // 10 2 bytes Compression method
+ // 12 2 bytes Last mod file time
+ // 14 2 bytes Last mod file date
+ // 16 4 bytes CRC-32
+ // 20 4 bytes Compressed size
+ // 24 4 bytes Uncompressed size
+ // 28 2 bytes Filename length (f)
+ // 30 2 bytes Extra field length (e)
+ // 32 2 bytes File comment length (c)
+ // 34 2 bytes Disk number start
+ // 36 2 bytes Internal file attributes
+ // 38 4 bytes External file attributes (file permission stored in the last 9 bits)
+ // 42 4 bytes Relative offset of local header
+ // 46 (f)bytes Filename
+ // (e)bytes Extra field
+ // (c)bytes File comment
+
+ // Check signature
+ if (!Arrays.equals(
+ CENTRAL_DIRECTORY_SIGNATURE,
+ Arrays.copyOfRange(data, startOffset, startOffset + 4))) {
+ throw new IOException("Invalid central directory info for zip file is found.");
+ }
+ mCompressionMethod = ByteArrayUtil.getInt(data, startOffset + 10, 2);
+ mCrc = ByteArrayUtil.getLong(data, startOffset + 16, 4);
+ mCompressedSize = ByteArrayUtil.getLong(data, startOffset + 20, 4);
+ mUncompressedSize = ByteArrayUtil.getLong(data, startOffset + 24, 4);
+ mInternalFileAttributes = ByteArrayUtil.getInt(data, startOffset + 36, 2);
+ mExternalFileAttributes = ByteArrayUtil.getLong(data, startOffset + 38, 4);
+ mLocalHeaderOffset = ByteArrayUtil.getLong(data, startOffset + 42, 4);
+ mFileNameLength = ByteArrayUtil.getInt(data, startOffset + 28, 2);
+ mFileName = ByteArrayUtil.getString(data, startOffset + 46, mFileNameLength);
+ mExtraFieldLength = ByteArrayUtil.getInt(data, startOffset + 30, 2);
+ mFileCommentLength = ByteArrayUtil.getInt(data, startOffset + 32, 2);
+ }
+}
diff --git a/common_util/com/android/tradefed/util/zip/EndCentralDirectoryInfo.java b/common_util/com/android/tradefed/util/zip/EndCentralDirectoryInfo.java
new file mode 100644
index 0000000..b675ebb
--- /dev/null
+++ b/common_util/com/android/tradefed/util/zip/EndCentralDirectoryInfo.java
@@ -0,0 +1,119 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.tradefed.util.zip;
+
+import com.android.tradefed.util.ByteArrayUtil;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.util.Arrays;
+
+/**
+ * EndCentralDirectoryInfo is a class containing the overall information of a zip file. It's at the
+ * end of the zip file.
+ *
+ * <p>Overall zipfile format: [Local file header + Compressed data [+ Extended local header]?]*
+ * [Central directory]* [End of central directory record]
+ *
+ * <p>Refer to following link for more details: https://en.wikipedia.org/wiki/Zip_(file_format)
+ */
+public final class EndCentralDirectoryInfo {
+
+ // End central directory of a zip file is at the end of the file, and its size shouldn't be
+ // larger than 64k.
+ public static final int MAX_LOOKBACK = 64 * 1024;
+
+ private static final byte[] END_CENTRAL_DIRECTORY_SIGNATURE = {0x50, 0x4b, 0x05, 0x06};
+ // Central directory signature is always 4 bytes.
+ private static final int CENTRAL_DIRECTORY_MAGIC_LENGTH = 4;
+
+ private int mEntryNumber;
+ private long mCentralDirSize;
+ private long mCentralDirOffset;
+
+ public int getEntryNumber() {
+ return mEntryNumber;
+ }
+
+ public long getCentralDirSize() {
+ return mCentralDirSize;
+ }
+
+ public long getCentralDirOffset() {
+ return mCentralDirOffset;
+ }
+
+ /**
+ * Constructor to collect end central directory information of a zip file.
+ *
+ * @param zipFile a {@link File} contains the end central directory information. It's likely the
+ * ending part of the zip file.
+ * @throws IOException
+ */
+ public EndCentralDirectoryInfo(File zipFile) throws IOException {
+ // End of central directory record:
+ // Offset Length Contents
+ // 0 4 bytes End of central dir signature (0x06054b50)
+ // 4 2 bytes Number of this disk
+ // 6 2 bytes Number of the disk with the start of the central directory
+ // 8 2 bytes Total number of entries in the central dir on this disk
+ // 10 2 bytes Total number of entries in the central dir
+ // 12 4 bytes Size of the central directory
+ // 16 4 bytes Offset of start of central directory with respect to the starting
+ // disk number
+ // 20 2 bytes zipfile comment length (c)
+ // 22 (c)bytes zipfile comment
+
+ try (FileInputStream stream = new FileInputStream(zipFile)) {
+ long size = stream.getChannel().size();
+ if (size > MAX_LOOKBACK) {
+ stream.skip(size - MAX_LOOKBACK);
+ size = MAX_LOOKBACK;
+ }
+ byte[] endCentralDir = new byte[(int) size];
+ stream.read(endCentralDir);
+ int offset = (int) size - CENTRAL_DIRECTORY_MAGIC_LENGTH - 1;
+ // Seek from the end of the file, searching for the end central directory signature.
+ while (offset >= 0) {
+ if (!java.util.Arrays.equals(
+ END_CENTRAL_DIRECTORY_SIGNATURE,
+ Arrays.copyOfRange(endCentralDir, offset, offset + 4))) {
+ offset--;
+ continue;
+ }
+ // Get the total number of entries in the central directory
+ mEntryNumber = ByteArrayUtil.getInt(endCentralDir, offset + 10, 2);
+ // Get the size of the central directory block
+ mCentralDirSize = ByteArrayUtil.getLong(endCentralDir, offset + 12, 4);
+ // Get the offset of start of central directory
+ mCentralDirOffset = ByteArrayUtil.getLong(endCentralDir, offset + 16, 4);
+
+ if (mCentralDirOffset < 0) {
+ throw new IOException(
+ "Failed to get offset of EndCentralDirectoryInfo. Partial unzip doesn't support zip files larger than 4GB.");
+ }
+ break;
+ }
+ if (offset < 0) {
+ throw new RuntimeException(
+ "Failed to find end central directory info for zip file: "
+ + zipFile.getPath());
+ }
+ }
+ }
+}
diff --git a/common_util/com/android/tradefed/util/zip/LocalFileHeader.java b/common_util/com/android/tradefed/util/zip/LocalFileHeader.java
new file mode 100644
index 0000000..2012189
--- /dev/null
+++ b/common_util/com/android/tradefed/util/zip/LocalFileHeader.java
@@ -0,0 +1,121 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.tradefed.util.zip;
+
+import com.android.tradefed.util.ByteArrayUtil;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.util.Arrays;
+
+/**
+ * LocalFileHeader is a class containing the information of a file/folder inside a zip file. The
+ * block of data is at the beginning part of each file entry.
+ *
+ * <p>Overall zipfile format: [Local file header + Compressed data [+ Extended local header]?]*
+ * [Central directory]* [End of central directory record]
+ *
+ * <p>Refer to following link for more details: https://en.wikipedia.org/wiki/Zip_(file_format)
+ */
+public final class LocalFileHeader {
+
+ public static final int LOCAL_FILE_HEADER_SIZE = 30;
+ private static final byte[] LOCAL_FILE_HEADER_SIGNATURE = {0x50, 0x4b, 0x03, 0x04};
+
+ private int mCompressionMethod;
+ private long mCrc;
+ private long mCompressedSize;
+ private long mUncompressedSize;
+ private int mFileNameLength;
+ private int mExtraFieldLength;
+
+ public int getCompressionMethod() {
+ return mCompressionMethod;
+ }
+
+ public long getCrc() {
+ return mCrc;
+ }
+
+ public long getCompressedSize() {
+ return mCompressedSize;
+ }
+
+ public long getUncompressedSize() {
+ return mUncompressedSize;
+ }
+
+ public int getFileNameLength() {
+ return mFileNameLength;
+ }
+
+ public int getExtraFieldLength() {
+ return mExtraFieldLength;
+ }
+
+ public int getHeaderSize() {
+ return LOCAL_FILE_HEADER_SIZE + mFileNameLength + mExtraFieldLength;
+ }
+
+ public LocalFileHeader(File partialZipFile) throws IOException {
+ this(partialZipFile, 0);
+ }
+
+ /**
+ * Constructor to collect local file header information of a file entry in a zip file.
+ *
+ * @param partialZipFile a {@link File} contains the local file header information.
+ * @param startOffset the start offset of the block of data for a local file header.
+ * @throws IOException
+ */
+ public LocalFileHeader(File partialZipFile, int startOffset) throws IOException {
+ // Local file header:
+ // Offset Length Contents
+ // 0 4 bytes Local file header signature (0x04034b50)
+ // 4 2 bytes Version needed to extract
+ // 6 2 bytes General purpose bit flag
+ // 8 2 bytes Compression method
+ // 10 2 bytes Last mod file time
+ // 12 2 bytes Last mod file date
+ // 14 4 bytes CRC-32
+ // 18 4 bytes Compressed size (n)
+ // 22 4 bytes Uncompressed size
+ // 26 2 bytes Filename length (f)
+ // 28 2 bytes Extra field length (e)
+ // (f)bytes Filename
+ // (e)bytes Extra field
+ // (n)bytes Compressed data
+ byte[] data;
+ try (FileInputStream stream = new FileInputStream(partialZipFile)) {
+ stream.skip(startOffset);
+ data = new byte[LOCAL_FILE_HEADER_SIZE];
+ stream.read(data);
+ }
+
+ // Check signature
+ if (!Arrays.equals(LOCAL_FILE_HEADER_SIGNATURE, Arrays.copyOfRange(data, 0, 4))) {
+ throw new IOException("Invalid local file header for zip file is found.");
+ }
+ mCompressionMethod = ByteArrayUtil.getInt(data, 8, 2);
+ mCrc = ByteArrayUtil.getLong(data, 14, 4);
+ mCompressedSize = ByteArrayUtil.getLong(data, 18, 2);
+ mUncompressedSize = ByteArrayUtil.getLong(data, 22, 2);
+ mFileNameLength = ByteArrayUtil.getInt(data, 26, 2);
+ mExtraFieldLength = ByteArrayUtil.getInt(data, 28, 2);
+ }
+}
diff --git a/tests/res/util/partial_zip.zip b/tests/res/util/partial_zip.zip
new file mode 100644
index 0000000..1f50fef
--- /dev/null
+++ b/tests/res/util/partial_zip.zip
Binary files differ
diff --git a/tests/src/com/android/tradefed/util/StreamUtilTest.java b/tests/src/com/android/tradefed/util/StreamUtilTest.java
index 6cd4995..f2e774c 100644
--- a/tests/src/com/android/tradefed/util/StreamUtilTest.java
+++ b/tests/src/com/android/tradefed/util/StreamUtilTest.java
@@ -172,6 +172,46 @@
assertEquals(text, baos.toString());
}
+ /**
+ * Verify that {@link com.android.tradefed.util.StreamUtil#copyStreams(InputStream,
+ * OutputStream, int, int)} can copy partial content.
+ */
+ public void testCopyStreams_partialSuccess() throws Exception {
+ String text = getLargeText();
+ StringBuilder builder = new StringBuilder(33 * 1024);
+ // Create a string longer than StreamUtil.BUF_SIZE
+ while (builder.length() < 32 * 1024) {
+ builder.append(text);
+ }
+ ByteArrayInputStream bais = new ByteArrayInputStream(builder.toString().getBytes());
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ // Skip the first 1kB, and read longer than StreamUtil.BUF_SIZE
+ StreamUtil.copyStreams(bais, baos, 1024, 20 * 1024);
+ bais.close();
+ baos.close();
+ assertEquals(builder.toString().substring(1024, 21 * 1024), baos.toString());
+ }
+
+ /**
+ * Verify that {@link com.android.tradefed.util.StreamUtil#copyStreams(InputStream,
+ * OutputStream, int, int)} cannot copy partial content if requested size is larger than what's
+ * available.
+ */
+ public void testCopyStreams_partialFail() throws Exception {
+ ByteArrayInputStream bais = null;
+ try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
+ String text = getLargeText();
+ bais = new ByteArrayInputStream(text.getBytes());
+ // Skip the first 1kB, and read longer than the size of text
+ StreamUtil.copyStreams(bais, baos, 10, text.length() + 1024);
+ fail("IOException should be thrown when reading too much data.");
+ } catch (IOException e) {
+ // Ignore expected error.
+ } finally {
+ StreamUtil.close(bais);
+ }
+ }
+
public void testCopyStreamToWriter() throws Exception {
String text = getLargeText();
ByteArrayInputStream bais = new ByteArrayInputStream(text.getBytes());
diff --git a/tests/src/com/android/tradefed/util/ZipUtilTest.java b/tests/src/com/android/tradefed/util/ZipUtilTest.java
index 4e4fb2f..51dc090 100644
--- a/tests/src/com/android/tradefed/util/ZipUtilTest.java
+++ b/tests/src/com/android/tradefed/util/ZipUtilTest.java
@@ -15,13 +15,24 @@
*/
package com.android.tradefed.util;
+import com.android.tradefed.util.zip.CentralDirectoryInfo;
+import com.android.tradefed.util.zip.EndCentralDirectoryInfo;
+import com.android.tradefed.util.zip.LocalFileHeader;
+
import junit.framework.TestCase;
+import java.io.BufferedReader;
import java.io.File;
+import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.nio.file.attribute.PosixFilePermission;
+import java.nio.file.attribute.PosixFilePermissions;
import java.util.Arrays;
import java.util.HashSet;
+import java.util.List;
import java.util.Set;
import java.util.zip.ZipFile;
@@ -191,6 +202,159 @@
}
}
+ public void testPartipUnzip() throws Exception {
+ File partialZipFile = null;
+ File tmpDir = null;
+ Set<PosixFilePermission> permissions;
+ try {
+ // The zip file is small, read the whole file and assume it's partial.
+ // This does not affect testing the behavior of partial unzipping.
+ partialZipFile = getTestDataFile("partial_zip");
+ EndCentralDirectoryInfo endCentralDirInfo = new EndCentralDirectoryInfo(partialZipFile);
+ List<CentralDirectoryInfo> zipEntries =
+ ZipUtil.getZipCentralDirectoryInfos(
+ partialZipFile,
+ endCentralDirInfo,
+ endCentralDirInfo.getCentralDirOffset());
+ // The zip file has 3 folders, 4 files.
+ assertEquals(7, zipEntries.size());
+
+ CentralDirectoryInfo zipEntry;
+ LocalFileHeader localFileHeader;
+ File targetFile;
+ tmpDir = FileUtil.createTempDir("partial_unzip");
+
+ // Unzip empty file
+ zipEntry =
+ zipEntries
+ .stream()
+ .filter(e -> e.getFileName().equals("empty_file"))
+ .findFirst()
+ .get();
+ targetFile = new File(Paths.get(tmpDir.toString(), zipEntry.getFileName()).toString());
+ localFileHeader =
+ new LocalFileHeader(partialZipFile, (int) zipEntry.getLocalHeaderOffset());
+ ZipUtil.unzipPartialZipFile(
+ partialZipFile,
+ targetFile,
+ zipEntry,
+ localFileHeader,
+ zipEntry.getLocalHeaderOffset());
+ // Verify file permissions - readonly - 644 rw-r--r--
+ permissions = Files.getPosixFilePermissions(targetFile.toPath());
+ assertEquals(PosixFilePermissions.fromString("rw-r--r--"), permissions);
+
+ // Unzip text file
+ zipEntry =
+ zipEntries
+ .stream()
+ .filter(e -> e.getFileName().equals("large_text/file.txt"))
+ .findFirst()
+ .get();
+ targetFile = new File(Paths.get(tmpDir.toString(), zipEntry.getFileName()).toString());
+ localFileHeader =
+ new LocalFileHeader(partialZipFile, (int) zipEntry.getLocalHeaderOffset());
+ ZipUtil.unzipPartialZipFile(
+ partialZipFile,
+ targetFile,
+ zipEntry,
+ localFileHeader,
+ zipEntry.getLocalHeaderOffset());
+ // Verify CRC
+ long crc = FileUtil.calculateCrc32(targetFile);
+ assertEquals(4146093769L, crc);
+ try (BufferedReader br = new BufferedReader(new FileReader(targetFile))) {
+ String line = br.readLine();
+ assertTrue(line.endsWith("this is a text file."));
+ } catch (IOException e) {
+ // fail if the file is corrupt in any way
+ fail("failed reading text file");
+ }
+ // Verify file permissions - read/write - 666 rw-rw-rw-
+ permissions = Files.getPosixFilePermissions(targetFile.toPath());
+ assertEquals(PosixFilePermissions.fromString("rw-rw-rw-"), permissions);
+
+ // Verify file permissions - executable - 755 rwxr-xr-x
+ zipEntry =
+ zipEntries
+ .stream()
+ .filter(e -> e.getFileName().equals("executable/executable_file"))
+ .findFirst()
+ .get();
+ targetFile = new File(Paths.get(tmpDir.toString(), zipEntry.getFileName()).toString());
+ localFileHeader =
+ new LocalFileHeader(partialZipFile, (int) zipEntry.getLocalHeaderOffset());
+ ZipUtil.unzipPartialZipFile(
+ partialZipFile,
+ targetFile,
+ zipEntry,
+ localFileHeader,
+ zipEntry.getLocalHeaderOffset());
+ permissions = Files.getPosixFilePermissions(targetFile.toPath());
+ assertEquals(PosixFilePermissions.fromString("rwxr-xr-x"), permissions);
+
+ // Verify file permissions - readonly - 444 r--r--r--
+ zipEntry =
+ zipEntries
+ .stream()
+ .filter(e -> e.getFileName().equals("read_only/readonly_file"))
+ .findFirst()
+ .get();
+ targetFile = new File(Paths.get(tmpDir.toString(), zipEntry.getFileName()).toString());
+ localFileHeader =
+ new LocalFileHeader(partialZipFile, (int) zipEntry.getLocalHeaderOffset());
+ ZipUtil.unzipPartialZipFile(
+ partialZipFile,
+ targetFile,
+ zipEntry,
+ localFileHeader,
+ zipEntry.getLocalHeaderOffset());
+ permissions = Files.getPosixFilePermissions(targetFile.toPath());
+ assertEquals(PosixFilePermissions.fromString("r--r--r--"), permissions);
+
+ // Verify folder permissions - readonly - 744 rwxr--r--
+ zipEntry =
+ zipEntries
+ .stream()
+ .filter(e -> e.getFileName().equals("read_only/"))
+ .findFirst()
+ .get();
+ targetFile = new File(Paths.get(tmpDir.toString(), zipEntry.getFileName()).toString());
+ localFileHeader =
+ new LocalFileHeader(partialZipFile, (int) zipEntry.getLocalHeaderOffset());
+ ZipUtil.unzipPartialZipFile(
+ partialZipFile,
+ targetFile,
+ zipEntry,
+ localFileHeader,
+ zipEntry.getLocalHeaderOffset());
+ permissions = Files.getPosixFilePermissions(targetFile.toPath());
+ assertEquals(PosixFilePermissions.fromString("rwxr--r--"), permissions);
+
+ // Verify folder permissions - read/write - 755 rwxr-xr-x
+ zipEntry =
+ zipEntries
+ .stream()
+ .filter(e -> e.getFileName().equals("large_text/"))
+ .findFirst()
+ .get();
+ targetFile = new File(Paths.get(tmpDir.toString(), zipEntry.getFileName()).toString());
+ localFileHeader =
+ new LocalFileHeader(partialZipFile, (int) zipEntry.getLocalHeaderOffset());
+ ZipUtil.unzipPartialZipFile(
+ partialZipFile,
+ targetFile,
+ zipEntry,
+ localFileHeader,
+ zipEntry.getLocalHeaderOffset());
+ permissions = Files.getPosixFilePermissions(targetFile.toPath());
+ assertEquals(PosixFilePermissions.fromString("rwxr-xr-x"), permissions);
+ } finally {
+ FileUtil.deleteFile(partialZipFile);
+ FileUtil.recursiveDelete(tmpDir);
+ }
+ }
+
// Helpers
private File createTempDir(String prefix) throws IOException {
return createTempDir(prefix, null);
