src/com/android/tradefed/result/CollectingTestListener.java - platform/tools/tradefederation - Git at Google

 /*
  * Copyright (C) 2010 The Android Open Source Project
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
  *      http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
 package com.android.tradefed.result;

 import com.android.ddmlib.testrunner.TestResult.TestStatus;
 import com.android.tradefed.build.IBuildInfo;
 import com.android.tradefed.config.Option;
 import com.android.tradefed.invoker.IInvocationContext;
 import com.android.tradefed.log.LogUtil.CLog;
 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric;
 import com.android.tradefed.retry.MergeStrategy;

 import com.google.common.annotations.VisibleForTesting;

 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.concurrent.atomic.AtomicBoolean;

 /**
  * A {@link ITestInvocationListener} that will collect all test results.
  *
  * <p>Although the data structures used in this object are thread-safe, the {@link
  * ITestInvocationListener} callbacks must be called in the correct order.
  */
 public class CollectingTestListener implements ITestInvocationListener, ILogSaverListener {

     @Option(
             name = "aggregate-metrics",
             description = "attempt to add test metrics values for test runs with the same name.")
     private boolean mIsAggregateMetrics = false;

     /** Toggle the 'aggregate metrics' option */
     protected void setIsAggregrateMetrics(boolean aggregate) {
         mIsAggregateMetrics = aggregate;
     }

     private IInvocationContext mContext;
     private IBuildInfo mBuildInfo;
     private Map<String, IInvocationContext> mModuleContextMap = new HashMap<>();
     // Use LinkedHashMap to provide consistent iterations over the keys.
     private Map<String, List<TestRunResult>> mTestRunResultMap =
             Collections.synchronizedMap(new LinkedHashMap<>());

     private IInvocationContext mCurrentModuleContext = null;
     private TestRunResult mCurrentTestRunResult = new TestRunResult();
     /** True if the default initialized mCurrentTestRunResult has its original value. */
     private boolean mDefaultRun = true;
     /** Track whether or not a test run is currently in progress */
     private boolean mRunInProgress = false;

     private Map<String, LogFile> mNonAssociatedLogFiles = new LinkedHashMap<>();

     // Tracks if mStatusCounts are accurate, or if they need to be recalculated
     private AtomicBoolean mIsCountDirty = new AtomicBoolean(true);
     // Tracks if the expected count is accurate, or if it needs to be recalculated.
     private AtomicBoolean mIsExpectedCountDirty = new AtomicBoolean(true);
     private int mExpectedCount = 0;

     // Represents the merged test results. This should not be accessed directly since it's only
     // calculated when needed.
     private final List<TestRunResult> mMergedTestRunResults = new ArrayList<>();
     // Represents the number of tests in each TestStatus state of the merged test results. Indexed
     // by TestStatus.ordinal()
     private int[] mStatusCounts = new int[TestStatus.values().length];

     private MergeStrategy mStrategy = MergeStrategy.ONE_TESTCASE_PASS_IS_PASS;

     /** Sets the {@link MergeStrategy} to use when merging results. */
     public void setMergeStrategy(MergeStrategy strategy) {
         mStrategy = strategy;
     }

     /**
      * Return the primary build info that was reported via {@link
      * #invocationStarted(IInvocationContext)}. Primary build is the build returned by the first
      * build provider of the running configuration. Returns null if there is no context (no build to
      * test case).
      */
     public IBuildInfo getPrimaryBuildInfo() {
         if (mContext == null) {
             return null;
         } else {
             return mContext.getBuildInfos().get(0);
         }
     }

     /**
      * Return the invocation context that was reported via {@link
      * #invocationStarted(IInvocationContext)}
      */
     public IInvocationContext getInvocationContext() {
         return mContext;
     }

     /**
      * Returns the build info.
      *
      * @deprecated rely on the {@link IBuildInfo} from {@link #getInvocationContext()}.
      */
     @Deprecated
     public IBuildInfo getBuildInfo() {
         return mBuildInfo;
     }

     /**
      * Set the build info. Should only be used for testing.
      *
      * @deprecated Not necessary for testing anymore.
      */
     @VisibleForTesting
     @Deprecated
     public void setBuildInfo(IBuildInfo buildInfo) {
         mBuildInfo = buildInfo;
     }

     /** {@inheritDoc} */
     @Override
     public void invocationStarted(IInvocationContext context) {
         mContext = context;
         mBuildInfo = getPrimaryBuildInfo();
     }

     /** {@inheritDoc} */
     @Override
     public void invocationEnded(long elapsedTime) {
         // ignore
     }

     /** {@inheritDoc} */
     @Override
     public void invocationFailed(Throwable cause) {
         // ignore
     }

     @Override
     public void testModuleStarted(IInvocationContext moduleContext) {
         mCurrentModuleContext = moduleContext;
     }

     @Override
     public void testModuleEnded() {
         mCurrentModuleContext = null;
     }

     /** {@inheritDoc} */
     @Override
     public void testRunStarted(String name, int numTests) {
         testRunStarted(name, numTests, 0);
     }

     private TestRunResult getNewRunResult() {
         TestRunResult result = new TestRunResult();
         if (mDefaultRun) {
             result = mCurrentTestRunResult;
             mDefaultRun = false;
         }
         result.setAggregateMetrics(mIsAggregateMetrics);
         return result;
     }

     /** {@inheritDoc} */
     @Override
     public void testRunStarted(String name, int numTests, int attemptNumber) {
         testRunStarted(name, numTests, attemptNumber, System.currentTimeMillis());
     }

     /** {@inheritDoc} */
     @Override
     public void testRunStarted(String name, int numTests, int attemptNumber, long startTime) {
         setCountDirty();
         // Only testRunStarted can affect the expected count.
         mIsExpectedCountDirty.set(true);

         // Associate the run name with the current module context
         if (mCurrentModuleContext != null) {
             mModuleContextMap.put(name, mCurrentModuleContext);
         }

         // Add the list of maps if the run doesn't exist
         if (!mTestRunResultMap.containsKey(name)) {
             mTestRunResultMap.put(name, new LinkedList<>());
         }
         List<TestRunResult> results = mTestRunResultMap.get(name);

         // Set the current test run result based on the attempt
         if (attemptNumber < results.size()) {
             if (results.get(attemptNumber) == null) {
                 throw new RuntimeException(
                         "Test run results should never be null in internal structure.");
             }
         } else if (attemptNumber == results.size()) {
             // new run
             TestRunResult result = getNewRunResult();
             results.add(result);
         } else {
             int size = results.size();
             for (int i = size; i < attemptNumber; i++) {
                 TestRunResult result = getNewRunResult();
                 result.testRunStarted(name, numTests, startTime);
                 String errorMessage =
                         String.format(
                                 "Run attempt %s of %s did not exists, but got attempt %s. This is a placeholder for the missing attempt.",
                                 i, name, attemptNumber);
                 result.testRunFailed(errorMessage);
                 result.testRunEnded(0L, new HashMap<String, Metric>());
                 results.add(result);
             }
             // New current run
             TestRunResult newResult = getNewRunResult();
             results.add(newResult);
         }
         mCurrentTestRunResult = results.get(attemptNumber);

         mCurrentTestRunResult.testRunStarted(name, numTests, startTime);
         mRunInProgress = true;
     }

     /** {@inheritDoc} */
     @Override
     public void testRunEnded(long elapsedTime, HashMap<String, Metric> runMetrics) {
         setCountDirty();
         mCurrentTestRunResult.testRunEnded(elapsedTime, runMetrics);
         mRunInProgress = false;
     }

     /** {@inheritDoc} */
     @Override
     public void testRunFailed(String errorMessage) {
         setCountDirty();
         mCurrentTestRunResult.testRunFailed(errorMessage);
     }

     /** {@inheritDoc} */
     @Override
     public void testRunFailed(FailureDescription failure) {
         setCountDirty();
         mCurrentTestRunResult.testRunFailed(failure);
     }

     /** {@inheritDoc} */
     @Override
     public void testRunStopped(long elapsedTime) {
         setCountDirty();
         mCurrentTestRunResult.testRunStopped(elapsedTime);
     }

     /** {@inheritDoc} */
     @Override
     public void testStarted(TestDescription test) {
         testStarted(test, System.currentTimeMillis());
     }

     /** {@inheritDoc} */
     @Override
     public void testStarted(TestDescription test, long startTime) {
         setCountDirty();
         mCurrentTestRunResult.testStarted(test, startTime);
     }

     /** {@inheritDoc} */
     @Override
     public void testEnded(TestDescription test, HashMap<String, Metric> testMetrics) {
         testEnded(test, System.currentTimeMillis(), testMetrics);
     }

     /** {@inheritDoc} */
     @Override
     public void testEnded(TestDescription test, long endTime, HashMap<String, Metric> testMetrics) {
         setCountDirty();
         mCurrentTestRunResult.testEnded(test, endTime, testMetrics);
     }

     /** {@inheritDoc} */
     @Override
     public void testFailed(TestDescription test, String trace) {
         setCountDirty();
         mCurrentTestRunResult.testFailed(test, trace);
     }

     @Override
     public void testAssumptionFailure(TestDescription test, String trace) {
         setCountDirty();
         mCurrentTestRunResult.testAssumptionFailure(test, trace);
     }

     @Override
     public void testIgnored(TestDescription test) {
         setCountDirty();
         mCurrentTestRunResult.testIgnored(test);
     }

     /** {@inheritDoc} */
     @Override
     public void logAssociation(String dataName, LogFile logFile) {
         if (mRunInProgress) {
             mCurrentTestRunResult.testLogSaved(dataName, logFile);
         } else {
             mNonAssociatedLogFiles.put(dataName, logFile);
         }
     }

     /**
      * Gets the results for the current test run.
      *
      * <p>Note the results may not be complete. It is recommended to test the value of {@link
      * TestRunResult#isRunComplete()} and/or (@link TestRunResult#isRunFailure()} as appropriate
      * before processing the results.
      *
      * @return the {@link TestRunResult} representing data collected during last test run
      */
     public TestRunResult getCurrentRunResults() {
         return mCurrentTestRunResult;
     }

     /** Returns the total number of complete tests for all runs. */
     public int getNumTotalTests() {
         computeMergedResults();
         int total = 0;
         for (TestStatus s : TestStatus.values()) {
             total += mStatusCounts[s.ordinal()];
         }
         return total;
     }

     /**
      * Returns the number of expected tests count. Could differ from {@link #getNumTotalTests()} if
      * some tests did not run.
      */
     public synchronized int getExpectedTests() {
         // If expected count is not dirty, no need to do anything
         if (!mIsExpectedCountDirty.compareAndSet(true, false)) {
             return mExpectedCount;
         }

         computeMergedResults();
         mExpectedCount = 0;
         for (TestRunResult result : getMergedTestRunResults()) {
             mExpectedCount += result.getExpectedTestCount();
         }
         return mExpectedCount;
     }

     /** Returns the number of tests in given state for this run. */
     public int getNumTestsInState(TestStatus status) {
         computeMergedResults();
         return mStatusCounts[status.ordinal()];
     }

     /** Returns if the invocation had any failed or assumption failed tests. */
     public boolean hasFailedTests() {
         return getNumAllFailedTests() > 0;
     }

     /** Returns the total number of test runs in a failure state */
     public int getNumAllFailedTestRuns() {
         int count = 0;
         for (TestRunResult result : getMergedTestRunResults()) {
             if (result.isRunFailure()) {
                 count++;
             }
         }
         return count;
     }

     /**
      * Returns the total number of tests in a failure state (only failed, assumption failures do not
      * count toward it).
      */
     public int getNumAllFailedTests() {
         return getNumTestsInState(TestStatus.FAILURE);
     }

     /**
      * Return the merged collection of results for all runs across different attempts.
      *
      * <p>If there are multiple results, each test run is merged, with the latest test result
      * overwriting test results of previous runs. Test runs are ordered by attempt number.
      *
      * <p>Metrics for the same attempt will be merged based on the preference set by {@code
      * aggregate-metrics}. The final metrics will be the metrics of the last attempt.
      */
     public List<TestRunResult> getMergedTestRunResults() {
         computeMergedResults();
         return new ArrayList<>(mMergedTestRunResults);
     }

     /**
      * Returns the results for all test runs.
      *
      * @deprecated Use {@link #getMergedTestRunResults()}
      */
     @Deprecated
     public Collection<TestRunResult> getRunResults() {
         return getMergedTestRunResults();
     }

     /**
      * Computes and stores the merged results and the total status counts since both operations are
      * expensive.
      */
     private synchronized void computeMergedResults() {
         // If not dirty, nothing to be done
         if (!mIsCountDirty.compareAndSet(true, false)) {
             return;
         }

         mMergedTestRunResults.clear();
         // Merge results
         if (mTestRunResultMap.isEmpty() && mCurrentTestRunResult.isRunFailure()) {
             // In case of early failure that is a bit untracked, still add it to the list to
             // not loose it.
             CLog.e("Early failure resulting in no testRunStart. Results might be inconsistent.");
             mMergedTestRunResults.add(mCurrentTestRunResult);
         } else {
             for (Entry<String, List<TestRunResult>> results : mTestRunResultMap.entrySet()) {
                 TestRunResult res = TestRunResult.merge(results.getValue(), mStrategy);
                 if (res == null) {
                     // Merge can return null in case of results being empty.
                     CLog.w("No results for %s", results.getKey());
                 } else {
                     mMergedTestRunResults.add(res);
                 }
             }
         }
         // Reset counts
         for (TestStatus s : TestStatus.values()) {
             mStatusCounts[s.ordinal()] = 0;
         }

         // Calculate results
         for (TestRunResult result : mMergedTestRunResults) {
             for (TestStatus s : TestStatus.values()) {
                 mStatusCounts[s.ordinal()] += result.getNumTestsInState(s);
             }
         }
     }

     /**
      * Keep dirty count as AtomicBoolean to ensure when accessed from another thread the state is
      * consistent.
      */
     private void setCountDirty() {
         mIsCountDirty.set(true);
     }

     /**
      * Return all the names for all the test runs.
      *
      * <p>These test runs may have run multiple times with different attempts.
      */
     public Collection<String> getTestRunNames() {
         return new ArrayList<String>(mTestRunResultMap.keySet());
     }

     /**
      * Gets all the attempts for a {@link TestRunResult} of a given test run.
      *
      * @param testRunName The name given by {{@link #testRunStarted(String, int)}.
      * @return All {@link TestRunResult} for a given test run, ordered by attempts.
      */
     public List<TestRunResult> getTestRunAttempts(String testRunName) {
         return mTestRunResultMap.get(testRunName);
     }

     /**
      * Gets all the results for a given attempt.
      *
      * @param attempt The attempt we want results for.
      * @return All {@link TestRunResult} for a given attempt.
      */
     public List<TestRunResult> getTestRunForAttempts(int attempt) {
         List<TestRunResult> allResultForAttempts = new ArrayList<>();
         for (Entry<String, List<TestRunResult>> runInfo : mTestRunResultMap.entrySet()) {
             if (attempt < runInfo.getValue().size()) {
                 TestRunResult attemptRes = runInfo.getValue().get(attempt);
                 allResultForAttempts.add(attemptRes);
             }
         }
         return allResultForAttempts;
     }

     /**
      * Returns whether a given test run name has any results.
      *
      * @param testRunName The name given by {{@link #testRunStarted(String, int)}.
      */
     public boolean hasTestRunResultsForName(String testRunName) {
         return mTestRunResultMap.containsKey(testRunName);
     }

     /**
      * Returns the number of attempts for a given test run name.
      *
      * @param testRunName The name given by {{@link #testRunStarted(String, int)}.
      */
     public int getTestRunAttemptCount(String testRunName) {
         List<TestRunResult> results = mTestRunResultMap.get(testRunName);
         if (results == null) {
             return 0;
         }
         return results.size();
     }

     /**
      * Return the {@link TestRunResult} for a single attempt.
      *
      * @param testRunName The name given by {{@link #testRunStarted(String, int)}.
      * @param attempt The attempt id.
      * @return The {@link TestRunResult} for the given name and attempt id or {@code null} if it
      *     does not exist.
      */
     public TestRunResult getTestRunAtAttempt(String testRunName, int attempt) {
         List<TestRunResult> results = mTestRunResultMap.get(testRunName);
         if (results == null || attempt < 0 || attempt >= results.size()) {
             return null;
         }

         return results.get(attempt);
     }

     /**
      * Returns the {@link IInvocationContext} of the module associated with the results.
      *
      * @param testRunName The name given by {{@link #testRunStarted(String, int)}.
      * @return The {@link IInvocationContext} of the module for a given test run name {@code null}
      *     if there are no results for that name.
      */
     public IInvocationContext getModuleContextForRunResult(String testRunName) {
         return mModuleContextMap.get(testRunName);
     }

     /** Returns a copy of the map containing all the logged file not associated with a test run. */
     public Map<String, LogFile> getNonAssociatedLogFiles() {
         return new LinkedHashMap<>(mNonAssociatedLogFiles);
     }

     /**
      * Allows to clear the results for a given run name. Should only be used in some cases like the
      * aggregator of results.
      */
     protected final synchronized void clearResultsForName(String testRunName) {
         setCountDirty();
         mTestRunResultMap.remove(testRunName);
     }
 }
	/*
	* Copyright (C) 2010 The Android Open Source Project
	*
	* Licensed under the Apache License, Version 2.0 (the "License");
	* you may not use this file except in compliance with the License.
	* You may obtain a copy of the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS,
	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	* See the License for the specific language governing permissions and
	* limitations under the License.
	*/
	package com.android.tradefed.result;

	import com.android.ddmlib.testrunner.TestResult.TestStatus;
	import com.android.tradefed.build.IBuildInfo;
	import com.android.tradefed.config.Option;
	import com.android.tradefed.invoker.IInvocationContext;
	import com.android.tradefed.log.LogUtil.CLog;
	import com.android.tradefed.metrics.proto.MetricMeasurement.Metric;
	import com.android.tradefed.retry.MergeStrategy;

	import com.google.common.annotations.VisibleForTesting;

	import java.util.ArrayList;
	import java.util.Collection;
	import java.util.Collections;
	import java.util.HashMap;
	import java.util.LinkedHashMap;
	import java.util.LinkedList;
	import java.util.List;
	import java.util.Map;
	import java.util.Map.Entry;
	import java.util.concurrent.atomic.AtomicBoolean;

	/**
	* A {@link ITestInvocationListener} that will collect all test results.
	*
	* <p>Although the data structures used in this object are thread-safe, the {@link
	* ITestInvocationListener} callbacks must be called in the correct order.
	*/
	public class CollectingTestListener implements ITestInvocationListener, ILogSaverListener {

	@Option(
	name = "aggregate-metrics",
	description = "attempt to add test metrics values for test runs with the same name.")
	private boolean mIsAggregateMetrics = false;

	/** Toggle the 'aggregate metrics' option */
	protected void setIsAggregrateMetrics(boolean aggregate) {
	mIsAggregateMetrics = aggregate;
	}

	private IInvocationContext mContext;
	private IBuildInfo mBuildInfo;
	private Map<String, IInvocationContext> mModuleContextMap = new HashMap<>();
	// Use LinkedHashMap to provide consistent iterations over the keys.
	private Map<String, List<TestRunResult>> mTestRunResultMap =
	Collections.synchronizedMap(new LinkedHashMap<>());

	private IInvocationContext mCurrentModuleContext = null;
	private TestRunResult mCurrentTestRunResult = new TestRunResult();
	/** True if the default initialized mCurrentTestRunResult has its original value. */
	private boolean mDefaultRun = true;
	/** Track whether or not a test run is currently in progress */
	private boolean mRunInProgress = false;

	private Map<String, LogFile> mNonAssociatedLogFiles = new LinkedHashMap<>();

	// Tracks if mStatusCounts are accurate, or if they need to be recalculated
	private AtomicBoolean mIsCountDirty = new AtomicBoolean(true);
	// Tracks if the expected count is accurate, or if it needs to be recalculated.
	private AtomicBoolean mIsExpectedCountDirty = new AtomicBoolean(true);
	private int mExpectedCount = 0;

	// Represents the merged test results. This should not be accessed directly since it's only
	// calculated when needed.
	private final List<TestRunResult> mMergedTestRunResults = new ArrayList<>();
	// Represents the number of tests in each TestStatus state of the merged test results. Indexed
	// by TestStatus.ordinal()
	private int[] mStatusCounts = new int[TestStatus.values().length];

	private MergeStrategy mStrategy = MergeStrategy.ONE_TESTCASE_PASS_IS_PASS;

	/** Sets the {@link MergeStrategy} to use when merging results. */
	public void setMergeStrategy(MergeStrategy strategy) {
	mStrategy = strategy;
	}

	/**
	* Return the primary build info that was reported via {@link
	* #invocationStarted(IInvocationContext)}. Primary build is the build returned by the first
	* build provider of the running configuration. Returns null if there is no context (no build to
	* test case).
	*/
	public IBuildInfo getPrimaryBuildInfo() {
	if (mContext == null) {
	return null;
	} else {
	return mContext.getBuildInfos().get(0);
	}
	}

	/**
	* Return the invocation context that was reported via {@link
	* #invocationStarted(IInvocationContext)}
	*/
	public IInvocationContext getInvocationContext() {
	return mContext;
	}

	/**
	* Returns the build info.
	*
	* @deprecated rely on the {@link IBuildInfo} from {@link #getInvocationContext()}.
	*/
	@Deprecated
	public IBuildInfo getBuildInfo() {
	return mBuildInfo;
	}

	/**
	* Set the build info. Should only be used for testing.
	*
	* @deprecated Not necessary for testing anymore.
	*/
	@VisibleForTesting
	@Deprecated
	public void setBuildInfo(IBuildInfo buildInfo) {
	mBuildInfo = buildInfo;
	}

	/** {@inheritDoc} */
	@Override
	public void invocationStarted(IInvocationContext context) {
	mContext = context;
	mBuildInfo = getPrimaryBuildInfo();
	}

	/** {@inheritDoc} */
	@Override
	public void invocationEnded(long elapsedTime) {
	// ignore
	}

	/** {@inheritDoc} */
	@Override
	public void invocationFailed(Throwable cause) {
	// ignore
	}

	@Override
	public void testModuleStarted(IInvocationContext moduleContext) {
	mCurrentModuleContext = moduleContext;
	}

	@Override
	public void testModuleEnded() {
	mCurrentModuleContext = null;
	}

	/** {@inheritDoc} */
	@Override
	public void testRunStarted(String name, int numTests) {
	testRunStarted(name, numTests, 0);
	}

	private TestRunResult getNewRunResult() {
	TestRunResult result = new TestRunResult();
	if (mDefaultRun) {
	result = mCurrentTestRunResult;
	mDefaultRun = false;
	}
	result.setAggregateMetrics(mIsAggregateMetrics);
	return result;
	}

	/** {@inheritDoc} */
	@Override
	public void testRunStarted(String name, int numTests, int attemptNumber) {
	testRunStarted(name, numTests, attemptNumber, System.currentTimeMillis());
	}

	/** {@inheritDoc} */
	@Override
	public void testRunStarted(String name, int numTests, int attemptNumber, long startTime) {
	setCountDirty();
	// Only testRunStarted can affect the expected count.
	mIsExpectedCountDirty.set(true);

	// Associate the run name with the current module context
	if (mCurrentModuleContext != null) {
	mModuleContextMap.put(name, mCurrentModuleContext);
	}

	// Add the list of maps if the run doesn't exist
	if (!mTestRunResultMap.containsKey(name)) {
	mTestRunResultMap.put(name, new LinkedList<>());
	}
	List<TestRunResult> results = mTestRunResultMap.get(name);

	// Set the current test run result based on the attempt
	if (attemptNumber < results.size()) {
	if (results.get(attemptNumber) == null) {
	throw new RuntimeException(
	"Test run results should never be null in internal structure.");
	}
	} else if (attemptNumber == results.size()) {
	// new run
	TestRunResult result = getNewRunResult();
	results.add(result);
	} else {
	int size = results.size();
	for (int i = size; i < attemptNumber; i++) {
	TestRunResult result = getNewRunResult();
	result.testRunStarted(name, numTests, startTime);
	String errorMessage =
	String.format(
	"Run attempt %s of %s did not exists, but got attempt %s. This is a placeholder for the missing attempt.",
	i, name, attemptNumber);
	result.testRunFailed(errorMessage);
	result.testRunEnded(0L, new HashMap<String, Metric>());
	results.add(result);
	}
	// New current run
	TestRunResult newResult = getNewRunResult();
	results.add(newResult);
	}
	mCurrentTestRunResult = results.get(attemptNumber);

	mCurrentTestRunResult.testRunStarted(name, numTests, startTime);
	mRunInProgress = true;
	}

	/** {@inheritDoc} */
	@Override
	public void testRunEnded(long elapsedTime, HashMap<String, Metric> runMetrics) {
	setCountDirty();
	mCurrentTestRunResult.testRunEnded(elapsedTime, runMetrics);
	mRunInProgress = false;
	}

	/** {@inheritDoc} */
	@Override
	public void testRunFailed(String errorMessage) {
	setCountDirty();
	mCurrentTestRunResult.testRunFailed(errorMessage);
	}

	/** {@inheritDoc} */
	@Override
	public void testRunFailed(FailureDescription failure) {
	setCountDirty();
	mCurrentTestRunResult.testRunFailed(failure);
	}

	/** {@inheritDoc} */
	@Override
	public void testRunStopped(long elapsedTime) {
	setCountDirty();
	mCurrentTestRunResult.testRunStopped(elapsedTime);
	}

	/** {@inheritDoc} */
	@Override
	public void testStarted(TestDescription test) {
	testStarted(test, System.currentTimeMillis());
	}

	/** {@inheritDoc} */
	@Override
	public void testStarted(TestDescription test, long startTime) {
	setCountDirty();
	mCurrentTestRunResult.testStarted(test, startTime);
	}

	/** {@inheritDoc} */
	@Override
	public void testEnded(TestDescription test, HashMap<String, Metric> testMetrics) {
	testEnded(test, System.currentTimeMillis(), testMetrics);
	}

	/** {@inheritDoc} */
	@Override
	public void testEnded(TestDescription test, long endTime, HashMap<String, Metric> testMetrics) {
	setCountDirty();
	mCurrentTestRunResult.testEnded(test, endTime, testMetrics);
	}

	/** {@inheritDoc} */
	@Override
	public void testFailed(TestDescription test, String trace) {
	setCountDirty();
	mCurrentTestRunResult.testFailed(test, trace);
	}

	@Override
	public void testAssumptionFailure(TestDescription test, String trace) {
	setCountDirty();
	mCurrentTestRunResult.testAssumptionFailure(test, trace);
	}

	@Override
	public void testIgnored(TestDescription test) {
	setCountDirty();
	mCurrentTestRunResult.testIgnored(test);
	}

	/** {@inheritDoc} */
	@Override
	public void logAssociation(String dataName, LogFile logFile) {
	if (mRunInProgress) {
	mCurrentTestRunResult.testLogSaved(dataName, logFile);
	} else {
	mNonAssociatedLogFiles.put(dataName, logFile);
	}
	}

	/**
	* Gets the results for the current test run.
	*
	* <p>Note the results may not be complete. It is recommended to test the value of {@link
	* TestRunResult#isRunComplete()} and/or (@link TestRunResult#isRunFailure()} as appropriate
	* before processing the results.
	*
	* @return the {@link TestRunResult} representing data collected during last test run
	*/
	public TestRunResult getCurrentRunResults() {
	return mCurrentTestRunResult;
	}

	/** Returns the total number of complete tests for all runs. */
	public int getNumTotalTests() {
	computeMergedResults();
	int total = 0;
	for (TestStatus s : TestStatus.values()) {
	total += mStatusCounts[s.ordinal()];
	}
	return total;
	}

	/**
	* Returns the number of expected tests count. Could differ from {@link #getNumTotalTests()} if
	* some tests did not run.
	*/
	public synchronized int getExpectedTests() {
	// If expected count is not dirty, no need to do anything
	if (!mIsExpectedCountDirty.compareAndSet(true, false)) {
	return mExpectedCount;
	}

	computeMergedResults();
	mExpectedCount = 0;
	for (TestRunResult result : getMergedTestRunResults()) {
	mExpectedCount += result.getExpectedTestCount();
	}
	return mExpectedCount;
	}

	/** Returns the number of tests in given state for this run. */
	public int getNumTestsInState(TestStatus status) {
	computeMergedResults();
	return mStatusCounts[status.ordinal()];
	}

	/** Returns if the invocation had any failed or assumption failed tests. */
	public boolean hasFailedTests() {
	return getNumAllFailedTests() > 0;
	}

	/** Returns the total number of test runs in a failure state */
	public int getNumAllFailedTestRuns() {
	int count = 0;
	for (TestRunResult result : getMergedTestRunResults()) {
	if (result.isRunFailure()) {
	count++;
	}
	}
	return count;
	}

	/**
	* Returns the total number of tests in a failure state (only failed, assumption failures do not
	* count toward it).
	*/
	public int getNumAllFailedTests() {
	return getNumTestsInState(TestStatus.FAILURE);
	}

	/**
	* Return the merged collection of results for all runs across different attempts.
	*
	* <p>If there are multiple results, each test run is merged, with the latest test result
	* overwriting test results of previous runs. Test runs are ordered by attempt number.
	*
	* <p>Metrics for the same attempt will be merged based on the preference set by {@code
	* aggregate-metrics}. The final metrics will be the metrics of the last attempt.
	*/
	public List<TestRunResult> getMergedTestRunResults() {
	computeMergedResults();
	return new ArrayList<>(mMergedTestRunResults);
	}

	/**
	* Returns the results for all test runs.
	*
	* @deprecated Use {@link #getMergedTestRunResults()}
	*/
	@Deprecated
	public Collection<TestRunResult> getRunResults() {
	return getMergedTestRunResults();
	}

	/**
	* Computes and stores the merged results and the total status counts since both operations are
	* expensive.
	*/
	private synchronized void computeMergedResults() {
	// If not dirty, nothing to be done
	if (!mIsCountDirty.compareAndSet(true, false)) {
	return;
	}

	mMergedTestRunResults.clear();
	// Merge results
	if (mTestRunResultMap.isEmpty() && mCurrentTestRunResult.isRunFailure()) {
	// In case of early failure that is a bit untracked, still add it to the list to
	// not loose it.
	CLog.e("Early failure resulting in no testRunStart. Results might be inconsistent.");
	mMergedTestRunResults.add(mCurrentTestRunResult);
	} else {
	for (Entry<String, List<TestRunResult>> results : mTestRunResultMap.entrySet()) {
	TestRunResult res = TestRunResult.merge(results.getValue(), mStrategy);
	if (res == null) {
	// Merge can return null in case of results being empty.
	CLog.w("No results for %s", results.getKey());
	} else {
	mMergedTestRunResults.add(res);
	}
	}
	}
	// Reset counts
	for (TestStatus s : TestStatus.values()) {
	mStatusCounts[s.ordinal()] = 0;
	}

	// Calculate results
	for (TestRunResult result : mMergedTestRunResults) {
	for (TestStatus s : TestStatus.values()) {
	mStatusCounts[s.ordinal()] += result.getNumTestsInState(s);
	}
	}
	}

	/**
	* Keep dirty count as AtomicBoolean to ensure when accessed from another thread the state is
	* consistent.
	*/
	private void setCountDirty() {
	mIsCountDirty.set(true);
	}

	/**
	* Return all the names for all the test runs.
	*
	* <p>These test runs may have run multiple times with different attempts.
	*/
	public Collection<String> getTestRunNames() {
	return new ArrayList<String>(mTestRunResultMap.keySet());
	}

	/**
	* Gets all the attempts for a {@link TestRunResult} of a given test run.
	*
	* @param testRunName The name given by {{@link #testRunStarted(String, int)}.
	* @return All {@link TestRunResult} for a given test run, ordered by attempts.
	*/
	public List<TestRunResult> getTestRunAttempts(String testRunName) {
	return mTestRunResultMap.get(testRunName);
	}

	/**
	* Gets all the results for a given attempt.
	*
	* @param attempt The attempt we want results for.
	* @return All {@link TestRunResult} for a given attempt.
	*/
	public List<TestRunResult> getTestRunForAttempts(int attempt) {
	List<TestRunResult> allResultForAttempts = new ArrayList<>();
	for (Entry<String, List<TestRunResult>> runInfo : mTestRunResultMap.entrySet()) {
	if (attempt < runInfo.getValue().size()) {
	TestRunResult attemptRes = runInfo.getValue().get(attempt);
	allResultForAttempts.add(attemptRes);
	}
	}
	return allResultForAttempts;
	}

	/**
	* Returns whether a given test run name has any results.
	*
	* @param testRunName The name given by {{@link #testRunStarted(String, int)}.
	*/
	public boolean hasTestRunResultsForName(String testRunName) {
	return mTestRunResultMap.containsKey(testRunName);
	}

	/**
	* Returns the number of attempts for a given test run name.
	*
	* @param testRunName The name given by {{@link #testRunStarted(String, int)}.
	*/
	public int getTestRunAttemptCount(String testRunName) {
	List<TestRunResult> results = mTestRunResultMap.get(testRunName);
	if (results == null) {
	return 0;
	}
	return results.size();
	}

	/**
	* Return the {@link TestRunResult} for a single attempt.
	*
	* @param testRunName The name given by {{@link #testRunStarted(String, int)}.
	* @param attempt The attempt id.
	* @return The {@link TestRunResult} for the given name and attempt id or {@code null} if it
	* does not exist.
	*/
	public TestRunResult getTestRunAtAttempt(String testRunName, int attempt) {
	List<TestRunResult> results = mTestRunResultMap.get(testRunName);
	if (results == null \|\| attempt < 0 \|\| attempt >= results.size()) {
	return null;
	}

	return results.get(attempt);
	}

	/**
	* Returns the {@link IInvocationContext} of the module associated with the results.
	*
	* @param testRunName The name given by {{@link #testRunStarted(String, int)}.
	* @return The {@link IInvocationContext} of the module for a given test run name {@code null}
	* if there are no results for that name.
	*/
	public IInvocationContext getModuleContextForRunResult(String testRunName) {
	return mModuleContextMap.get(testRunName);
	}

	/** Returns a copy of the map containing all the logged file not associated with a test run. */
	public Map<String, LogFile> getNonAssociatedLogFiles() {
	return new LinkedHashMap<>(mNonAssociatedLogFiles);
	}

	/**
	* Allows to clear the results for a given run name. Should only be used in some cases like the
	* aggregator of results.
	*/
	protected final synchronized void clearResultsForName(String testRunName) {
	setCountDirty();
	mTestRunResultMap.remove(testRunName);
	}
	}