Generate and include checksum data with CTS report submissions
Test: Used exploratory tests and unit tests in ChecksumReporterTest.java
to verify the behavior.
* All TestResults are added from an InvocationResult
* Able to verify TestResult against previously calculated checksum
* CRCs are created for all files in a directory including subdirectories
* Checksum integrity is maintained while serializing to/from disk
* When possible checksum data is compressed before report submission
Change-Id: I8c563499991939c777892c00f24a339980916f04
diff --git a/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/result/ResultReporter.java b/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/result/ResultReporter.java
index 6a9a9ab..fa8d8ad 100644
--- a/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/result/ResultReporter.java
+++ b/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/result/ResultReporter.java
@@ -408,7 +408,6 @@
// NOTE: Everything after this line only applies to the master ResultReporter.
-
synchronized(this) {
// The master ResultReporter tracks the progress of all invocations across
// shard ResultReporters. Writing results should not proceed until all
@@ -460,16 +459,16 @@
long startTime = mResult.getStartTime();
try {
+ // Zip the full test results directory.
+ copyDynamicConfigFiles(mBuildHelper.getDynamicConfigFiles(), mResultDir);
+ copyFormattingFiles(mResultDir);
+
File resultFile = ResultHandler.writeResults(mBuildHelper.getSuiteName(),
mBuildHelper.getSuiteVersion(), mBuildHelper.getSuitePlan(),
mBuildHelper.getSuiteBuild(), mResult, mResultDir, startTime,
elapsedTime + startTime, mReferenceUrl, getLogUrl(),
mBuildHelper.getCommandLineArgs());
info("Test Result: %s", resultFile.getCanonicalPath());
-
- // Zip the full test results directory.
- copyDynamicConfigFiles(mBuildHelper.getDynamicConfigFiles(), mResultDir);
- copyFormattingFiles(mResultDir);
File zippedResults = zipResults(mResultDir);
info("Full Result: %s", zippedResults.getCanonicalPath());
diff --git a/common/host-side/tradefed/tests/src/com/android/compatibility/common/tradefed/UnitTests.java b/common/host-side/tradefed/tests/src/com/android/compatibility/common/tradefed/UnitTests.java
index a7a31f6..b8b7858 100644
--- a/common/host-side/tradefed/tests/src/com/android/compatibility/common/tradefed/UnitTests.java
+++ b/common/host-side/tradefed/tests/src/com/android/compatibility/common/tradefed/UnitTests.java
@@ -17,6 +17,7 @@
import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelperTest;
import com.android.compatibility.common.tradefed.command.CompatibilityConsoleTest;
+import com.android.compatibility.common.tradefed.result.ChecksumReporterTest;
import com.android.compatibility.common.tradefed.result.ConsoleReporterTest;
import com.android.compatibility.common.tradefed.result.ResultReporterTest;
import com.android.compatibility.common.tradefed.result.SubPlanCreatorTest;
@@ -45,6 +46,7 @@
addTestSuite(CompatibilityConsoleTest.class);
addTestSuite(CompatibilityTestTest.class);
addTestSuite(ConsoleReporterTest.class);
+ addTestSuite(ChecksumReporterTest.class);
addTestSuite(ResultReporterTest.class);
addTestSuite(CompatibilityTestTest.class);
addTestSuite(OptionHelperTest.class);
diff --git a/common/host-side/tradefed/tests/src/com/android/compatibility/common/tradefed/result/ChecksumReporterTest.java b/common/host-side/tradefed/tests/src/com/android/compatibility/common/tradefed/result/ChecksumReporterTest.java
new file mode 100644
index 0000000..068ce45
--- /dev/null
+++ b/common/host-side/tradefed/tests/src/com/android/compatibility/common/tradefed/result/ChecksumReporterTest.java
@@ -0,0 +1,194 @@
+/*
+ * Copyright (C) 2015 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.compatibility.common.tradefed.result;
+
+import com.android.compatibility.common.tradefed.build.CompatibilityBuildProvider;
+import com.android.compatibility.common.tradefed.result.ResultReporter;
+import com.android.compatibility.common.util.ChecksumReporter;
+import com.android.compatibility.common.util.ChecksumReporter.ChecksumValidationException;
+import com.android.compatibility.common.util.ICaseResult;
+import com.android.compatibility.common.util.IInvocationResult;
+import com.android.compatibility.common.util.IModuleResult;
+import com.android.compatibility.common.util.ITestResult;
+import com.android.compatibility.common.util.ReportLog;
+import com.android.compatibility.common.util.TestStatus;
+import com.android.tradefed.build.IBuildInfo;
+import com.android.tradefed.config.OptionSetter;
+import com.android.tradefed.util.FileUtil;
+
+import junit.framework.TestCase;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+
+public class ChecksumReporterTest extends TestCase {
+
+ private static final String ROOT_PROPERTY = "TESTS_ROOT";
+ private static final String ROOT_DIR_NAME = "root";
+ private static final String SUITE_NAME = "TESTS";
+ private static final String BUILD_NUMBER = "2";
+ private static final String SUITE_PLAN = "cts";
+ private static final String BASE_DIR_NAME = "android-tests";
+ private static final String TESTCASES = "testcases";
+
+ private ChecksumReporter mReporter;
+ private File mRoot = null;
+ private IBuildInfo mBuildInfo;
+ private ReportLog mReportLog = null;
+ private IInvocationResult mInvocationResult;
+ private IModuleResult mModuleResult;
+ private ITestResult mFailedTest;
+
+ @Override
+ public void setUp() throws Exception {
+ mReporter = new ChecksumReporter(100, .001, (short)1);
+ mRoot = FileUtil.createTempDir(ROOT_DIR_NAME);
+ File baseDir = new File(mRoot, BASE_DIR_NAME);
+ baseDir.mkdirs();
+ File testDir = new File(baseDir, TESTCASES);
+ testDir.mkdirs();
+ System.setProperty(ROOT_PROPERTY, mRoot.getAbsolutePath());
+
+ ResultReporter resultReporter = new ResultReporter();
+ CompatibilityBuildProvider provider = new CompatibilityBuildProvider() {
+ @Override
+ protected String getSuiteInfoName() {
+ return SUITE_NAME;
+ }
+ @Override
+ protected String getSuiteInfoBuildNumber() {
+ return BUILD_NUMBER;
+ }
+ @Override
+ protected String getSuiteInfoVersion() {
+ return BUILD_NUMBER;
+ }
+ };
+ OptionSetter setter = new OptionSetter(provider);
+ setter.setOptionValue("plan", SUITE_PLAN);
+ setter.setOptionValue("dynamic-config-url", "");
+ mBuildInfo = provider.getBuild();
+
+ resultReporter.invocationStarted(mBuildInfo);
+ mInvocationResult = resultReporter.getResult();
+ mModuleResult = mInvocationResult.getOrCreateModule("Module-1");
+ mModuleResult.setDone(true);
+ mModuleResult.setNotExecuted(0);
+ ICaseResult caseResult = mModuleResult.getOrCreateResult("Case-1");
+ ITestResult test1 = caseResult.getOrCreateResult("Test1");
+ test1.passed(mReportLog);
+ mFailedTest = caseResult.getOrCreateResult("Test2");
+ mFailedTest.failed("stack-trace - error happened");
+
+ IModuleResult moduleResult2 = mInvocationResult.getOrCreateModule("Module-2");
+ ICaseResult caseResult2 = moduleResult2.getOrCreateResult("Case-2");
+ mModuleResult.setDone(false);
+ mModuleResult.setNotExecuted(1);
+ ITestResult test3 = caseResult2.getOrCreateResult("Test3");
+ test3.passed(mReportLog);
+
+ }
+
+ @Override
+ public void tearDown() throws Exception {
+ mReporter = null;
+ }
+
+ public void testStoreAndRetrieveTestResults() {
+ mReporter.addInvocation(mInvocationResult);
+ VerifyInvocationResults(mInvocationResult, mReporter);
+ }
+
+ /***
+ * By definition this test is flaky since the checksum has a false positive probability of .1%
+ */
+ public void testInvalidChecksums() {
+ mReporter.addInvocation(mInvocationResult);
+ IModuleResult module = mInvocationResult.getModules().get(1);
+ module.setDone(!module.isDone());
+ String fingerprint = mInvocationResult.getBuildFingerprint();
+ assertFalse("Checksum should contain module: " + module.getName(),
+ mReporter.containsModuleResult(module, fingerprint));
+
+ mFailedTest.setResultStatus(TestStatus.PASS);
+ assertFalse("Checksum should not contain test: " + mFailedTest.getName(),
+ mReporter.containsTestResult(mFailedTest, mModuleResult, fingerprint));
+ assertFalse("Module checksum should verify number of tests",
+ mReporter.containsModuleResult(mModuleResult, fingerprint));
+ }
+
+ public void testFileSerialization()
+ throws IOException, ClassNotFoundException, ChecksumValidationException {
+ mReporter.addInvocation(mInvocationResult);
+
+ File file1 = new File(mRoot, "file1.txt");
+ try (FileWriter fileWriter = new FileWriter(file1, false)) {
+ fileWriter.append("This is a test file");
+ }
+
+ mReporter.addDirectory(mRoot);
+ mReporter.saveToFile(mRoot);
+
+ ChecksumReporter storedChecksum = ChecksumReporter.load(mRoot);
+ VerifyInvocationResults(mInvocationResult, storedChecksum);
+ assertTrue("Serializing checksum maintains file hash",
+ storedChecksum.containsFile(file1, ""));
+ }
+
+ public void testFileCRCOperations() throws IOException {
+ File subDirectory = new File(mRoot, "child");
+ subDirectory.mkdir();
+ File file1 = new File(mRoot, "file1.txt");
+ try (FileWriter fileWriter = new FileWriter(file1, false)) {
+ fileWriter.append("This is a test file");
+ }
+
+ File file2 = new File(subDirectory, "file2.txt");
+ try (FileWriter fileWriter = new FileWriter(file2, false)) {
+ fileWriter.append("This is another test file with a different crc");
+ }
+
+ mReporter.addDirectory(mRoot);
+
+ assertTrue(mReporter.containsFile(file1, ""));
+ assertTrue(mReporter.containsFile(file2, "/child"));
+ assertFalse("Should not contain non-existent file",
+ mReporter.containsFile(new File(mRoot, "fake.txt"), ""));
+
+ File file3 = new File(mRoot, "file3.txt");
+ try (FileWriter fileWriter = new FileWriter(file3, false)) {
+ fileWriter.append("This is a test file added after crc calculated");
+ }
+ assertFalse("Should not contain file created after crc calculated",
+ mReporter.containsFile(file3, ""));
+
+ }
+
+ private void VerifyInvocationResults(IInvocationResult invocation, ChecksumReporter reporter) {
+ for (IModuleResult module : invocation.getModules()) {
+ String buildFingerprint = invocation.getBuildFingerprint();
+ assertTrue("Checksum should contain module: " + module.getName(),
+ reporter.containsModuleResult(module, buildFingerprint));
+ for (ICaseResult caseResult : module.getResults()) {
+ for (ITestResult result : caseResult.getResults()) {
+ assertTrue("Checksum should contain test: " + result.getName(),
+ reporter.containsTestResult(result, module, buildFingerprint));
+ }
+ }
+ }
+ }
+}
diff --git a/common/host-side/util/src/com/android/compatibility/common/util/ChecksumReporter.java b/common/host-side/util/src/com/android/compatibility/common/util/ChecksumReporter.java
new file mode 100644
index 0000000..faac61f
--- /dev/null
+++ b/common/host-side/util/src/com/android/compatibility/common/util/ChecksumReporter.java
@@ -0,0 +1,396 @@
+/*
+ * Copyright (C) 2016 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.compatibility.common.util;
+
+import com.android.annotations.Nullable;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Joiner;
+import com.google.common.base.Strings;
+import com.google.common.hash.BloomFilter;
+import com.google.common.hash.Funnels;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.ObjectInput;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutput;
+import java.io.ObjectOutputStream;
+import java.io.OutputStream;
+import java.io.Serializable;
+import java.security.DigestException;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.Arrays;
+import java.util.HashMap;
+
+/***
+ * Calculate and store checksum values for files and test results
+ */
+public final class ChecksumReporter implements Serializable {
+
+ public static final String NAME = "checksum.data";
+ public static final String PREV_NAME = "checksum.previous.data";
+
+ private static final double DEFAULT_FPP = 0.05;
+ private static final String SEPARATOR = "/";
+ private static final String ID_SEPARATOR = "@";
+ private static final String NAME_SEPARATOR = ".";
+
+ private static final short CURRENT_VERSION = 1;
+ // Serialized format Id (ie magic number) used to identify serialized data.
+ static final short SERIALIZED_FORMAT_CODE = 650;
+
+ private final BloomFilter<CharSequence> mResultChecksum;
+ private final HashMap<String, byte[]> mFileChecksum;
+ private final short mVersion;
+
+ /***
+ * Calculate checksum of test results and files in result directory and write to disk
+ * @param dir test results directory
+ * @param result test results
+ * @return true if successful, false if unable to calculate or store the checksum
+ */
+ public static boolean tryCreateChecksum(File dir, IInvocationResult result) {
+ try {
+ int totalCount = countTestResults(result);
+ ChecksumReporter checksumReporter =
+ new ChecksumReporter(totalCount, DEFAULT_FPP, CURRENT_VERSION);
+ checksumReporter.addInvocation(result);
+ checksumReporter.addDirectory(dir);
+ checksumReporter.saveToFile(dir);
+ } catch (Exception e) {
+ return false;
+ }
+ return true;
+ }
+
+ /***
+ * Create Checksum Reporter from data saved on disk
+ * @param directory
+ * @return
+ * @throws ChecksumValidationException
+ */
+ public static ChecksumReporter load(File directory) throws ChecksumValidationException {
+ ChecksumReporter reporter = new ChecksumReporter(directory);
+ if (reporter.getCapacity() > 1.1) {
+ throw new ChecksumValidationException("Capacity exceeded.");
+ }
+ return reporter;
+ }
+
+ /***
+ * Deserialize checksum from file
+ * @param directory the parent directory containing the checksum file
+ * @throws ChecksumValidationException
+ */
+ public ChecksumReporter(File directory) throws ChecksumValidationException {
+ File file = new File(directory, ChecksumReporter.NAME);
+ try (FileInputStream fileStream = new FileInputStream(file);
+ InputStream outputStream = new BufferedInputStream(fileStream);
+ ObjectInput objectInput = new ObjectInputStream(outputStream)) {
+ short magicNumber = objectInput.readShort();
+ switch (magicNumber) {
+ case SERIALIZED_FORMAT_CODE:
+ mVersion = objectInput.readShort();
+ mResultChecksum = (BloomFilter<CharSequence>) objectInput.readObject();
+ mFileChecksum = (HashMap<String, byte[]>) objectInput.readObject();
+ break;
+ default:
+ throw new ChecksumValidationException("Unknown format of serialized data.");
+ }
+ } catch (Exception e) {
+ throw new ChecksumValidationException("Unable to load checksum from file", e);
+ }
+ if (mVersion > CURRENT_VERSION) {
+ throw new ChecksumValidationException(
+ "File contains a newer version of ChecksumReporter");
+ }
+ }
+
+ /***
+ * Create new instance of ChecksumReporter
+ * @param testCount the number of test results that will be stored
+ * @param fpp the false positive percentage for result lookup misses
+ */
+ public ChecksumReporter(int testCount, double fpp, short version) {
+ mResultChecksum = BloomFilter.create(Funnels.unencodedCharsFunnel(),
+ testCount, fpp);
+ mFileChecksum = new HashMap<>();
+ mVersion = version;
+ }
+
+ /***
+ * Add each test result from each module and test case
+ */
+ public void addInvocation(IInvocationResult invocationResult) {
+ for (IModuleResult module : invocationResult.getModules()) {
+ String buildFingerprint = invocationResult.getBuildFingerprint();
+ addModuleResult(module, buildFingerprint);
+ for (ICaseResult caseResult : module.getResults()) {
+ for (ITestResult testResult : caseResult.getResults()) {
+ addTestResult(testResult, module, buildFingerprint);
+ }
+ }
+ }
+ }
+
+ /***
+ * Calculate CRC of file and store the result
+ * @param file crc calculated on this file
+ * @param path part of the key to identify the files crc
+ */
+ public void addFile(File file, String path) {
+ byte[] crc;
+ try {
+ crc = calculateFileChecksum(file);
+ } catch (ChecksumValidationException e) {
+ crc = new byte[0];
+ }
+ String key = path + SEPARATOR + file.getName();
+ mFileChecksum.put(key, crc);
+ }
+
+ @VisibleForTesting
+ public boolean containsFile(File file, String path) {
+ String key = path + SEPARATOR + file.getName();
+ if (mFileChecksum.containsKey(key))
+ {
+ try {
+ byte[] crc = calculateFileChecksum(file);
+ return Arrays.equals(mFileChecksum.get(key), crc);
+ } catch (ChecksumValidationException e) {
+ return false;
+ }
+ }
+ return false;
+ }
+
+ /***
+ * Adds all child files recursively through all sub directories
+ * @param directory target that is deeply searched for files
+ */
+ public void addDirectory(File directory) {
+ addDirectory(directory, directory.getName());
+ }
+
+ /***
+ * @param path the relative path to the current directory from the base directory
+ */
+ private void addDirectory(File directory, String path) {
+ for(String childName : directory.list()) {
+ File child = new File(directory, childName);
+ if (child.isDirectory()) {
+ addDirectory(child, path + SEPARATOR + child.getName());
+ } else {
+ addFile(child, path);
+ }
+ }
+ }
+
+ /***
+ * Calculate checksum of test result and store the value
+ * @param testResult the target of the checksum
+ * @param moduleResult the module that contains the test result
+ * @param buildFingerprint the fingerprint the test execution is running against
+ */
+ public void addTestResult(
+ ITestResult testResult, IModuleResult moduleResult, String buildFingerprint) {
+
+ String signature = generateTestResultSignature(testResult, moduleResult, buildFingerprint);
+ mResultChecksum.put(signature);
+ }
+
+ @VisibleForTesting
+ public boolean containsTestResult(
+ ITestResult testResult, IModuleResult moduleResult, String buildFingerprint) {
+
+ String signature = generateTestResultSignature(testResult, moduleResult, buildFingerprint);
+ return mResultChecksum.mightContain(signature);
+ }
+
+ /***
+ * Calculate checksm of module result and store value
+ * @param moduleResult the target of the checksum
+ * @param buildFingerprint the fingerprint the test execution is running against
+ */
+ public void addModuleResult(IModuleResult moduleResult, String buildFingerprint) {
+ mResultChecksum.put(
+ generateModuleResultSignature(moduleResult, buildFingerprint));
+ mResultChecksum.put(
+ generateModuleSummarySignature(moduleResult, buildFingerprint));
+ }
+
+ @VisibleForTesting
+ public Boolean containsModuleResult(IModuleResult moduleResult, String buildFingerprint) {
+ return mResultChecksum.mightContain(
+ generateModuleResultSignature(moduleResult, buildFingerprint));
+ }
+
+ /***
+ * Write the checksum data to disk.
+ * Overwrites existing file
+ * @param directory
+ * @throws IOException
+ */
+ public void saveToFile(File directory) throws IOException {
+ File file = new File(directory, NAME);
+
+ try (FileOutputStream fileStream = new FileOutputStream(file, false);
+ OutputStream outputStream = new BufferedOutputStream(fileStream);
+ ObjectOutput objectOutput = new ObjectOutputStream(outputStream)) {
+ objectOutput.writeShort(SERIALIZED_FORMAT_CODE);
+ objectOutput.writeShort(mVersion);
+ objectOutput.writeObject(mResultChecksum);
+ objectOutput.writeObject(mFileChecksum);
+ }
+ }
+
+ @VisibleForTesting
+ double getCapacity() {
+ // If default FPP changes:
+ // increment the CURRENT_VERSION and set the denominator based on this.mVersion
+ return mResultChecksum.expectedFpp() / DEFAULT_FPP;
+ }
+
+ static String generateTestResultSignature(ITestResult testResult, IModuleResult module,
+ String buildFingerprint) {
+ StringBuilder sb = new StringBuilder();
+ String stacktrace = testResult.getStackTrace();
+
+ stacktrace = stacktrace == null ? "" : stacktrace.trim();
+ // Line endings for stacktraces are somewhat unpredictable and there is no need to
+ // actually read the result they are all removed for consistency.
+ stacktrace = stacktrace.replaceAll("\\r?\\n|\\r", "");
+ sb.append(buildFingerprint).append(SEPARATOR)
+ .append(module.getId()).append(SEPARATOR)
+ .append(testResult.getFullName()).append(SEPARATOR)
+ .append(testResult.getResultStatus().getValue()).append(SEPARATOR)
+ .append(stacktrace).append(SEPARATOR);
+ return sb.toString();
+ }
+
+ static String generateTestResultSignature(
+ String packageName, String suiteName, String caseName, String testName, String abi,
+ String status,
+ String stacktrace,
+ String buildFingerprint) {
+
+ String testId = buildTestId(suiteName, caseName, testName, abi);
+ StringBuilder sb = new StringBuilder();
+
+ stacktrace = stacktrace == null ? "" : stacktrace.trim();
+ // Line endings for stacktraces are somewhat unpredictable and there is no need to
+ // actually read the result they are all removed for consistency.
+ stacktrace = stacktrace.replaceAll("\\r?\\n|\\r", "");
+ sb.append(buildFingerprint)
+ .append(SEPARATOR)
+ .append(packageName)
+ .append(SEPARATOR)
+ .append(testId)
+ .append(SEPARATOR)
+ .append(status)
+ .append(SEPARATOR)
+ .append(stacktrace)
+ .append(SEPARATOR);
+ return sb.toString();
+ }
+
+ private static String buildTestId(
+ String suiteName, String caseName, String testName, @Nullable String abi) {
+ String name = Joiner.on(NAME_SEPARATOR).skipNulls().join(
+ Strings.emptyToNull(suiteName),
+ Strings.emptyToNull(caseName),
+ Strings.emptyToNull(testName));
+ return Joiner.on(ID_SEPARATOR).skipNulls().join(
+ Strings.emptyToNull(name),
+ Strings.emptyToNull(abi));
+ }
+
+
+ private static String generateModuleResultSignature(IModuleResult module,
+ String buildFingerprint) {
+ StringBuilder sb = new StringBuilder();
+ sb.append(buildFingerprint).append(SEPARATOR)
+ .append(module.getId()).append(SEPARATOR)
+ .append(module.isDone()).append(SEPARATOR)
+ .append(module.getNotExecuted()).append(SEPARATOR)
+ .append(module.countResults(TestStatus.FAIL));
+ return sb.toString();
+ }
+
+ private static String generateModuleSummarySignature(IModuleResult module,
+ String buildFingerprint) {
+ StringBuilder sb = new StringBuilder();
+ sb.append(buildFingerprint).append(SEPARATOR)
+ .append(module.getId()).append(SEPARATOR)
+ .append(module.countResults(TestStatus.FAIL));
+ return sb.toString();
+ }
+
+ static byte[] calculateFileChecksum(File file) throws ChecksumValidationException {
+
+ try (FileInputStream fis = new FileInputStream(file);
+ InputStream inputStream = new BufferedInputStream(fis)) {
+ MessageDigest hashSum = MessageDigest.getInstance("SHA-256");
+ int cnt;
+ int bufferSize = 8192;
+ byte [] buffer = new byte[bufferSize];
+ while ((cnt = inputStream.read(buffer)) != -1) {
+ hashSum.update(buffer, 0, cnt);
+ }
+
+ byte[] partialHash = new byte[32];
+ hashSum.digest(partialHash, 0, 32);
+ return partialHash;
+ } catch (NoSuchAlgorithmException e) {
+ throw new ChecksumValidationException("Unable to hash file.", e);
+ } catch (IOException e) {
+ throw new ChecksumValidationException("Unable to hash file.", e);
+ } catch (DigestException e) {
+ throw new ChecksumValidationException("Unable to hash file.", e);
+ }
+ }
+
+
+ private static int countTestResults(IInvocationResult invocation) {
+ int count = 0;
+ for (IModuleResult module : invocation.getModules()) {
+ // Two entries per module (result & summary)
+ count += 2;
+ for (ICaseResult caseResult : module.getResults()) {
+ count += caseResult.getResults().size();
+ }
+ }
+ return count;
+ }
+
+ public static class ChecksumValidationException extends Exception {
+ public ChecksumValidationException(String detailMessage) {
+ super(detailMessage);
+ }
+
+ public ChecksumValidationException(String detailMessage, Throwable throwable) {
+ super(detailMessage, throwable);
+ }
+ }
+}
diff --git a/common/util/src/com/android/compatibility/common/util/ResultHandler.java b/common/host-side/util/src/com/android/compatibility/common/util/ResultHandler.java
similarity index 79%
rename from common/util/src/com/android/compatibility/common/util/ResultHandler.java
rename to common/host-side/util/src/com/android/compatibility/common/util/ResultHandler.java
index 435f515..a6f990e 100644
--- a/common/util/src/com/android/compatibility/common/util/ResultHandler.java
+++ b/common/host-side/util/src/com/android/compatibility/common/util/ResultHandler.java
@@ -15,6 +15,10 @@
*/
package com.android.compatibility.common.util;
+import com.android.compatibility.common.util.ChecksumReporter.ChecksumValidationException;
+
+import com.google.common.base.Strings;
+
import org.xmlpull.v1.XmlPullParser;
import org.xmlpull.v1.XmlPullParserException;
import org.xmlpull.v1.XmlPullParserFactory;
@@ -28,6 +32,9 @@
import java.io.OutputStream;
import java.net.InetAddress;
import java.net.UnknownHostException;
+import java.nio.file.FileSystems;
+import java.nio.file.Files;
+import java.nio.file.Path;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Comparator;
@@ -98,12 +105,17 @@
* @param resultsDir
*/
public static List<IInvocationResult> getResults(File resultsDir) {
+ return getResults(resultsDir, false);
+ }
+
+ /**
+ * @param resultsDir
+ * @param useChecksum
+ */
+ public static List<IInvocationResult> getResults(
+ File resultsDir, Boolean useChecksum) {
List<IInvocationResult> results = new ArrayList<>();
- File[] files = resultsDir.listFiles();
- if (files == null || files.length == 0) {
- // No results, just return the empty list
- return results;
- }
+ List<File> files = getResultDirectories(resultsDir);;
for (File resultDir : files) {
if (!resultDir.isDirectory()) {
continue;
@@ -113,7 +125,20 @@
if (!resultFile.exists()) {
continue;
}
+ Boolean invocationUseChecksum = useChecksum;
IInvocationResult invocation = new InvocationResult();
+ invocation.setRetryDirectory(resultDir);
+ ChecksumReporter checksumReporter = null;
+ if (invocationUseChecksum) {
+ try {
+ checksumReporter = ChecksumReporter.load(resultDir);
+ invocation.setRetryChecksumStatus(RetryChecksumStatus.RetryWithChecksum);
+ } catch (ChecksumValidationException e) {
+ // Unable to read checksum form previous execution
+ invocation.setRetryChecksumStatus(RetryChecksumStatus.RetryWithoutChecksum);
+ invocationUseChecksum = false;
+ }
+ }
XmlPullParserFactory factory = XmlPullParserFactory.newInstance();
XmlPullParser parser = factory.newPullParser();
parser.setInput(new FileReader(resultFile));
@@ -194,10 +219,22 @@
}
}
parser.require(XmlPullParser.END_TAG, NS, TEST_TAG);
+ Boolean checksumMismatch = invocationUseChecksum
+ && !checksumReporter.containsTestResult(
+ test, module, invocation.getBuildFingerprint());
+ if (checksumMismatch) {
+ test.removeResult();
+ }
}
parser.require(XmlPullParser.END_TAG, NS, CASE_TAG);
}
parser.require(XmlPullParser.END_TAG, NS, MODULE_TAG);
+ Boolean checksumMismatch = invocationUseChecksum
+ && !checksumReporter.containsModuleResult(
+ module, invocation.getBuildFingerprint());
+ if (checksumMismatch) {
+ module.setDone(false);
+ }
}
parser.require(XmlPullParser.END_TAG, NS, RESULT_TAG);
results.add(invocation);
@@ -296,6 +333,10 @@
serializer.startTag(NS, BUILD_TAG);
for (Entry<String, String> entry : result.getInvocationInfo().entrySet()) {
serializer.attribute(NS, entry.getKey(), entry.getValue());
+ if (Strings.isNullOrEmpty(result.getBuildFingerprint()) &&
+ entry.getKey().equals(BUILD_FINGERPRINT)) {
+ result.setBuildFingerprint(entry.getValue());
+ }
}
serializer.endTag(NS, BUILD_TAG);
@@ -370,20 +411,59 @@
serializer.endTag(NS, MODULE_TAG);
}
serializer.endDocument();
+ createChecksum(resultDir, result);
return resultFile;
}
+ private static void createChecksum(File resultDir, IInvocationResult invocationResult) {
+ RetryChecksumStatus retryStatus = invocationResult.getRetryChecksumStatus();
+ switch (retryStatus) {
+ case NotRetry: case RetryWithChecksum:
+ // Do not disrupt the process if there is a problem generating checksum.
+ ChecksumReporter.tryCreateChecksum(resultDir, invocationResult);
+ break;
+ case RetryWithoutChecksum:
+ // If the previous run has an invalid checksum file,
+ // copy it into current results folder for future troubleshooting
+ File retryDirectory = invocationResult.getRetryDirectory();
+ Path retryChecksum = FileSystems.getDefault().getPath(
+ retryDirectory.getAbsolutePath(), ChecksumReporter.NAME);
+ if (!retryChecksum.toFile().exists()) {
+ // if no checksum file, check for a copy from a previous retry
+ retryChecksum = FileSystems.getDefault().getPath(
+ retryDirectory.getAbsolutePath(), ChecksumReporter.PREV_NAME);
+ }
+
+ if (retryChecksum.toFile().exists()) {
+ File checksumCopy = new File(resultDir, ChecksumReporter.PREV_NAME);
+ try (FileOutputStream stream = new FileOutputStream(checksumCopy)) {
+ Files.copy(retryChecksum, stream);
+ } catch (IOException e) {
+ // Do not disrupt the process if there is a problem copying checksum
+ }
+ }
+ }
+ }
+
/**
* Find the IInvocationResult for the given sessionId.
*/
public static IInvocationResult findResult(File resultsDir, Integer sessionId)
throws FileNotFoundException {
+ return findResult(resultsDir, sessionId, true);
+ }
+
+ /**
+ * Find the IInvocationResult for the given sessionId.
+ */
+ private static IInvocationResult findResult(
+ File resultsDir, Integer sessionId, Boolean useChecksum) throws FileNotFoundException {
if (sessionId < 0) {
throw new IllegalArgumentException(
String.format("Invalid session id [%d] ", sessionId));
}
- List<IInvocationResult> results = getResults(resultsDir);
+ List<IInvocationResult> results = getResults(resultsDir, useChecksum);
if (results == null || sessionId >= results.size()) {
throw new RuntimeException(String.format("Could not find session [%d]", sessionId));
}
@@ -391,6 +471,33 @@
}
/**
+ * Get a list of child directories that contain test invocation results
+ * @param resultsDir the root test result directory
+ * @return
+ */
+ public static List<File> getResultDirectories(File resultsDir) {
+ List<File> directoryList = new ArrayList<>();
+ File[] files = resultsDir.listFiles();
+ if (files == null || files.length == 0) {
+ // No results, just return the empty list
+ return directoryList;
+ }
+ for (File resultDir : files) {
+ if (!resultDir.isDirectory()) {
+ continue;
+ }
+ // Only include if it contain results file
+ File resultFile = new File(resultDir, TEST_RESULT_FILE_NAME);
+ if (!resultFile.exists()) {
+ continue;
+ }
+ directoryList.add(resultDir);
+ }
+ Collections.sort(directoryList, (d1, d2) -> d1.getName().compareTo(d2.getName()));
+ return directoryList;
+ }
+
+ /**
* Return the given time as a {@link String} suitable for displaying.
* <p/>
* Example: Fri Aug 20 15:13:03 PDT 2010
diff --git a/common/util/src/com/android/compatibility/common/util/IInvocationResult.java b/common/util/src/com/android/compatibility/common/util/IInvocationResult.java
index 739dd48..2b75371 100644
--- a/common/util/src/com/android/compatibility/common/util/IInvocationResult.java
+++ b/common/util/src/com/android/compatibility/common/util/IInvocationResult.java
@@ -114,4 +114,24 @@
* Return the number of completed test modules for this invocation.
*/
int getModuleCompleteCount();
+
+ /**
+ * Return status of checksum from previous session
+ */
+ RetryChecksumStatus getRetryChecksumStatus();
+
+ /**
+ * Set status of checksum from previous session
+ */
+ void setRetryChecksumStatus(RetryChecksumStatus retryStatus);
+
+ /**
+ * Return the directory of the previous sessions results
+ */
+ File getRetryDirectory();
+
+ /**
+ * Set the directory of the previous sessions results
+ */
+ void setRetryDirectory(File resultDir);
}
diff --git a/common/util/src/com/android/compatibility/common/util/ITestResult.java b/common/util/src/com/android/compatibility/common/util/ITestResult.java
index c35b997..24ee1aa 100644
--- a/common/util/src/com/android/compatibility/common/util/ITestResult.java
+++ b/common/util/src/com/android/compatibility/common/util/ITestResult.java
@@ -142,4 +142,8 @@
*/
boolean isRetry();
+ /**
+ * Clear the existing result and default to 'failed'
+ */
+ void removeResult();
}
diff --git a/common/util/src/com/android/compatibility/common/util/InvocationResult.java b/common/util/src/com/android/compatibility/common/util/InvocationResult.java
index f74c61d..83f1dac 100644
--- a/common/util/src/com/android/compatibility/common/util/InvocationResult.java
+++ b/common/util/src/com/android/compatibility/common/util/InvocationResult.java
@@ -15,6 +15,7 @@
*/
package com.android.compatibility.common.util;
+import java.io.File;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@@ -37,7 +38,8 @@
private String mTestPlan;
private String mCommandLineArgs;
private int mNotExecuted = 0;
-
+ private RetryChecksumStatus mRetryChecksumStatus = RetryChecksumStatus.NotRetry;
+ private File mRetryDirectory = null;
/**
* {@inheritDoc}
*/
@@ -201,4 +203,36 @@
}
return completeModules;
}
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public RetryChecksumStatus getRetryChecksumStatus() {
+ return mRetryChecksumStatus;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void setRetryChecksumStatus(RetryChecksumStatus retryStatus) {
+ mRetryChecksumStatus = retryStatus;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public File getRetryDirectory() {
+ return mRetryDirectory;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void setRetryDirectory(File resultDir) {
+ mRetryDirectory = resultDir;
+ }
}
diff --git a/common/util/src/com/android/compatibility/common/util/RetryChecksumStatus.java b/common/util/src/com/android/compatibility/common/util/RetryChecksumStatus.java
new file mode 100644
index 0000000..a86ab37
--- /dev/null
+++ b/common/util/src/com/android/compatibility/common/util/RetryChecksumStatus.java
@@ -0,0 +1,8 @@
+package com.android.compatibility.common.util;
+
+
+public enum RetryChecksumStatus {
+ NotRetry,
+ RetryWithChecksum,
+ RetryWithoutChecksum
+}
diff --git a/common/util/src/com/android/compatibility/common/util/TestResult.java b/common/util/src/com/android/compatibility/common/util/TestResult.java
index ba378d0..36b6e5a 100644
--- a/common/util/src/com/android/compatibility/common/util/TestResult.java
+++ b/common/util/src/com/android/compatibility/common/util/TestResult.java
@@ -242,6 +242,15 @@
* {@inheritDoc}
*/
@Override
+ public void removeResult() {
+ setResultStatus(TestStatus.FAIL);
+ setStackTrace("");
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
public int compareTo(ITestResult another) {
return getName().compareTo(another.getName());
}
