| /* |
| * Copyright (C) 2016 The Android Open Source Project |
| * |
| * Licensed under the Apache License, Version 2.0 (the "License"); |
| * you may not use this file except in compliance with the License. |
| * You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| package com.android.compatibility.common.util; |
| |
| import com.google.common.annotations.VisibleForTesting; |
| import com.google.common.base.Joiner; |
| import com.google.common.base.Strings; |
| import com.google.common.hash.BloomFilter; |
| import com.google.common.hash.Funnels; |
| |
| import java.io.BufferedInputStream; |
| import java.io.BufferedOutputStream; |
| import java.io.File; |
| import java.io.FileInputStream; |
| import java.io.FileOutputStream; |
| import java.io.IOException; |
| import java.io.InputStream; |
| import java.io.ObjectInput; |
| import java.io.ObjectInputStream; |
| import java.io.ObjectOutput; |
| import java.io.ObjectOutputStream; |
| import java.io.OutputStream; |
| import java.io.Serializable; |
| import java.security.DigestException; |
| import java.security.MessageDigest; |
| import java.security.NoSuchAlgorithmException; |
| import java.util.Arrays; |
| import java.util.HashMap; |
| |
| /*** |
| * Calculate and store checksum values for files and test results |
| */ |
| public final class ChecksumReporter implements Serializable { |
| |
| public static final String NAME = "checksum.data"; |
| public static final String PREV_NAME = "checksum.previous.data"; |
| |
| private static final double DEFAULT_FPP = 0.05; |
| private static final String SEPARATOR = "/"; |
| private static final String ID_SEPARATOR = "@"; |
| private static final String NAME_SEPARATOR = "."; |
| |
| private static final short CURRENT_VERSION = 1; |
| // Serialized format Id (ie magic number) used to identify serialized data. |
| static final short SERIALIZED_FORMAT_CODE = 650; |
| |
| private final BloomFilter<CharSequence> mResultChecksum; |
| private final HashMap<String, byte[]> mFileChecksum; |
| private final short mVersion; |
| |
| /*** |
| * Calculate checksum of test results and files in result directory and write to disk |
| * @param dir test results directory |
| * @param result test results |
| * @return true if successful, false if unable to calculate or store the checksum |
| */ |
| public static boolean tryCreateChecksum(File dir, IInvocationResult result) { |
| try { |
| int totalCount = countTestResults(result); |
| ChecksumReporter checksumReporter = |
| new ChecksumReporter(totalCount, DEFAULT_FPP, CURRENT_VERSION); |
| checksumReporter.addInvocation(result); |
| checksumReporter.addDirectory(dir); |
| checksumReporter.saveToFile(dir); |
| } catch (Exception e) { |
| return false; |
| } |
| return true; |
| } |
| |
| /*** |
| * Create Checksum Reporter from data saved on disk |
| * @param directory |
| * @return |
| * @throws ChecksumValidationException |
| */ |
| public static ChecksumReporter load(File directory) throws ChecksumValidationException { |
| ChecksumReporter reporter = new ChecksumReporter(directory); |
| if (reporter.getCapacity() > 1.1) { |
| throw new ChecksumValidationException("Capacity exceeded."); |
| } |
| return reporter; |
| } |
| |
| /*** |
| * Deserialize checksum from file |
| * @param directory the parent directory containing the checksum file |
| * @throws ChecksumValidationException |
| */ |
| public ChecksumReporter(File directory) throws ChecksumValidationException { |
| File file = new File(directory, ChecksumReporter.NAME); |
| try (FileInputStream fileStream = new FileInputStream(file); |
| InputStream outputStream = new BufferedInputStream(fileStream); |
| ObjectInput objectInput = new ObjectInputStream(outputStream)) { |
| short magicNumber = objectInput.readShort(); |
| switch (magicNumber) { |
| case SERIALIZED_FORMAT_CODE: |
| mVersion = objectInput.readShort(); |
| mResultChecksum = (BloomFilter<CharSequence>) objectInput.readObject(); |
| mFileChecksum = (HashMap<String, byte[]>) objectInput.readObject(); |
| break; |
| default: |
| throw new ChecksumValidationException("Unknown format of serialized data."); |
| } |
| } catch (Exception e) { |
| throw new ChecksumValidationException("Unable to load checksum from file", e); |
| } |
| if (mVersion > CURRENT_VERSION) { |
| throw new ChecksumValidationException( |
| "File contains a newer version of ChecksumReporter"); |
| } |
| } |
| |
| /*** |
| * Create new instance of ChecksumReporter |
| * @param testCount the number of test results that will be stored |
| * @param fpp the false positive percentage for result lookup misses |
| */ |
| public ChecksumReporter(int testCount, double fpp, short version) { |
| mResultChecksum = BloomFilter.create(Funnels.unencodedCharsFunnel(), |
| testCount, fpp); |
| mFileChecksum = new HashMap<>(); |
| mVersion = version; |
| } |
| |
| /*** |
| * Add each test result from each module and test case |
| */ |
| public void addInvocation(IInvocationResult invocationResult) { |
| for (IModuleResult module : invocationResult.getModules()) { |
| String buildFingerprint = invocationResult.getBuildFingerprint(); |
| addModuleResult(module, buildFingerprint); |
| for (ICaseResult caseResult : module.getResults()) { |
| for (ITestResult testResult : caseResult.getResults()) { |
| addTestResult(testResult, module, buildFingerprint); |
| } |
| } |
| } |
| } |
| |
| /*** |
| * Calculate CRC of file and store the result |
| * @param file crc calculated on this file |
| * @param path part of the key to identify the files crc |
| */ |
| public void addFile(File file, String path) { |
| byte[] crc; |
| try { |
| crc = calculateFileChecksum(file); |
| } catch (ChecksumValidationException e) { |
| crc = new byte[0]; |
| } |
| String key = path + SEPARATOR + file.getName(); |
| mFileChecksum.put(key, crc); |
| } |
| |
| @VisibleForTesting |
| public boolean containsFile(File file, String path) { |
| String key = path + SEPARATOR + file.getName(); |
| if (mFileChecksum.containsKey(key)) |
| { |
| try { |
| byte[] crc = calculateFileChecksum(file); |
| return Arrays.equals(mFileChecksum.get(key), crc); |
| } catch (ChecksumValidationException e) { |
| return false; |
| } |
| } |
| return false; |
| } |
| |
| /*** |
| * Adds all child files recursively through all sub directories |
| * @param directory target that is deeply searched for files |
| */ |
| public void addDirectory(File directory) { |
| addDirectory(directory, directory.getName()); |
| } |
| |
| /*** |
| * @param path the relative path to the current directory from the base directory |
| */ |
| private void addDirectory(File directory, String path) { |
| for(String childName : directory.list()) { |
| File child = new File(directory, childName); |
| if (child.isDirectory()) { |
| addDirectory(child, path + SEPARATOR + child.getName()); |
| } else { |
| addFile(child, path); |
| } |
| } |
| } |
| |
| /*** |
| * Calculate checksum of test result and store the value |
| * @param testResult the target of the checksum |
| * @param moduleResult the module that contains the test result |
| * @param buildFingerprint the fingerprint the test execution is running against |
| */ |
| public void addTestResult( |
| ITestResult testResult, IModuleResult moduleResult, String buildFingerprint) { |
| |
| String signature = generateTestResultSignature(testResult, moduleResult, buildFingerprint); |
| mResultChecksum.put(signature); |
| } |
| |
| @VisibleForTesting |
| public boolean containsTestResult( |
| ITestResult testResult, IModuleResult moduleResult, String buildFingerprint) { |
| |
| String signature = generateTestResultSignature(testResult, moduleResult, buildFingerprint); |
| return mResultChecksum.mightContain(signature); |
| } |
| |
| /*** |
| * Calculate checksm of module result and store value |
| * @param moduleResult the target of the checksum |
| * @param buildFingerprint the fingerprint the test execution is running against |
| */ |
| public void addModuleResult(IModuleResult moduleResult, String buildFingerprint) { |
| mResultChecksum.put( |
| generateModuleResultSignature(moduleResult, buildFingerprint)); |
| mResultChecksum.put( |
| generateModuleSummarySignature(moduleResult, buildFingerprint)); |
| } |
| |
| @VisibleForTesting |
| public Boolean containsModuleResult(IModuleResult moduleResult, String buildFingerprint) { |
| return mResultChecksum.mightContain( |
| generateModuleResultSignature(moduleResult, buildFingerprint)); |
| } |
| |
| /*** |
| * Write the checksum data to disk. |
| * Overwrites existing file |
| * @param directory |
| * @throws IOException |
| */ |
| public void saveToFile(File directory) throws IOException { |
| File file = new File(directory, NAME); |
| |
| try (FileOutputStream fileStream = new FileOutputStream(file, false); |
| OutputStream outputStream = new BufferedOutputStream(fileStream); |
| ObjectOutput objectOutput = new ObjectOutputStream(outputStream)) { |
| objectOutput.writeShort(SERIALIZED_FORMAT_CODE); |
| objectOutput.writeShort(mVersion); |
| objectOutput.writeObject(mResultChecksum); |
| objectOutput.writeObject(mFileChecksum); |
| } |
| } |
| |
| @VisibleForTesting |
| double getCapacity() { |
| // If default FPP changes: |
| // increment the CURRENT_VERSION and set the denominator based on this.mVersion |
| return mResultChecksum.expectedFpp() / DEFAULT_FPP; |
| } |
| |
| static String generateTestResultSignature(ITestResult testResult, IModuleResult module, |
| String buildFingerprint) { |
| StringBuilder sb = new StringBuilder(); |
| String stacktrace = testResult.getStackTrace(); |
| |
| stacktrace = stacktrace == null ? "" : stacktrace.trim(); |
| // Line endings for stacktraces are somewhat unpredictable and there is no need to |
| // actually read the result they are all removed for consistency. |
| stacktrace = stacktrace.replaceAll("\\r?\\n|\\r", ""); |
| sb.append(buildFingerprint).append(SEPARATOR) |
| .append(module.getId()).append(SEPARATOR) |
| .append(testResult.getFullName()).append(SEPARATOR) |
| .append(testResult.getResultStatus().getValue()).append(SEPARATOR) |
| .append(stacktrace).append(SEPARATOR); |
| return sb.toString(); |
| } |
| |
| static String generateTestResultSignature( |
| String packageName, String suiteName, String caseName, String testName, String abi, |
| String status, |
| String stacktrace, |
| String buildFingerprint) { |
| |
| String testId = buildTestId(suiteName, caseName, testName, abi); |
| StringBuilder sb = new StringBuilder(); |
| |
| stacktrace = stacktrace == null ? "" : stacktrace.trim(); |
| // Line endings for stacktraces are somewhat unpredictable and there is no need to |
| // actually read the result they are all removed for consistency. |
| stacktrace = stacktrace.replaceAll("\\r?\\n|\\r", ""); |
| sb.append(buildFingerprint) |
| .append(SEPARATOR) |
| .append(packageName) |
| .append(SEPARATOR) |
| .append(testId) |
| .append(SEPARATOR) |
| .append(status) |
| .append(SEPARATOR) |
| .append(stacktrace) |
| .append(SEPARATOR); |
| return sb.toString(); |
| } |
| |
| private static String buildTestId( |
| String suiteName, String caseName, String testName, String abi) { |
| String name = Joiner.on(NAME_SEPARATOR).skipNulls().join( |
| Strings.emptyToNull(suiteName), |
| Strings.emptyToNull(caseName), |
| Strings.emptyToNull(testName)); |
| return Joiner.on(ID_SEPARATOR).skipNulls().join( |
| Strings.emptyToNull(name), |
| Strings.emptyToNull(abi)); |
| } |
| |
| |
| private static String generateModuleResultSignature(IModuleResult module, |
| String buildFingerprint) { |
| StringBuilder sb = new StringBuilder(); |
| sb.append(buildFingerprint).append(SEPARATOR) |
| .append(module.getId()).append(SEPARATOR) |
| .append(module.isDone()).append(SEPARATOR) |
| .append(module.countResults(TestStatus.FAIL)); |
| return sb.toString(); |
| } |
| |
| private static String generateModuleSummarySignature(IModuleResult module, |
| String buildFingerprint) { |
| StringBuilder sb = new StringBuilder(); |
| sb.append(buildFingerprint).append(SEPARATOR) |
| .append(module.getId()).append(SEPARATOR) |
| .append(module.countResults(TestStatus.FAIL)); |
| return sb.toString(); |
| } |
| |
| static byte[] calculateFileChecksum(File file) throws ChecksumValidationException { |
| |
| try (FileInputStream fis = new FileInputStream(file); |
| InputStream inputStream = new BufferedInputStream(fis)) { |
| MessageDigest hashSum = MessageDigest.getInstance("SHA-256"); |
| int cnt; |
| int bufferSize = 8192; |
| byte [] buffer = new byte[bufferSize]; |
| while ((cnt = inputStream.read(buffer)) != -1) { |
| hashSum.update(buffer, 0, cnt); |
| } |
| |
| byte[] partialHash = new byte[32]; |
| hashSum.digest(partialHash, 0, 32); |
| return partialHash; |
| } catch (NoSuchAlgorithmException e) { |
| throw new ChecksumValidationException("Unable to hash file.", e); |
| } catch (IOException e) { |
| throw new ChecksumValidationException("Unable to hash file.", e); |
| } catch (DigestException e) { |
| throw new ChecksumValidationException("Unable to hash file.", e); |
| } |
| } |
| |
| |
| private static int countTestResults(IInvocationResult invocation) { |
| int count = 0; |
| for (IModuleResult module : invocation.getModules()) { |
| // Two entries per module (result & summary) |
| count += 2; |
| for (ICaseResult caseResult : module.getResults()) { |
| count += caseResult.getResults().size(); |
| } |
| } |
| return count; |
| } |
| |
| public static class ChecksumValidationException extends Exception { |
| public ChecksumValidationException(String detailMessage) { |
| super(detailMessage); |
| } |
| |
| public ChecksumValidationException(String detailMessage, Throwable throwable) { |
| super(detailMessage, throwable); |
| } |
| } |
| } |