Google Git
Sign in
android / platform / tools / tradefederation / af7303e50 / . / javatests / com / android / tradefed / retry / ResultAggregatorTest.java
blob: 46a8689e0b20fc23d1991780eb0529b1c32e0a3b [file] [log] [blame]
/*
* Copyright (C) 2019 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.tradefed.retry;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import com.android.tradefed.build.BuildInfo;
import com.android.tradefed.config.ConfigurationDef;
import com.android.tradefed.invoker.IInvocationContext;
import com.android.tradefed.invoker.InvocationContext;
import com.android.tradefed.metrics.proto.MetricMeasurement.Metric;
import com.android.tradefed.result.FailureDescription;
import com.android.tradefed.result.ILogSaver;
import com.android.tradefed.result.ILogSaverListener;
import com.android.tradefed.result.ITestInvocationListener;
import com.android.tradefed.result.LogDataType;
import com.android.tradefed.result.LogFile;
import com.android.tradefed.result.MultiFailureDescription;
import com.android.tradefed.result.TestDescription;
import com.android.tradefed.result.retry.ISupportGranularResults;
import com.android.tradefed.util.FileUtil;
import org.easymock.EasyMock;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
import java.io.File;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
/** Unit tests for {@link ResultAggregator}. */
@RunWith(JUnit4.class)
public class ResultAggregatorTest {
private TestableResultAggregator mAggregator;
private ILogSaverListener mAggListener;
private ITestDetailedReceiver mDetailedListener;
private ILogSaver mLogger;
private IInvocationContext mInvocationContext;
private IInvocationContext mModuleContext;
private interface ITestDetailedReceiver
extends ISupportGranularResults, ITestInvocationListener, ILogSaverListener {}
private class TestableResultAggregator extends ResultAggregator {
private String mCurrentRunError = null;
public TestableResultAggregator(
List<ITestInvocationListener> listeners, RetryStrategy strategy) {
super(listeners, strategy);
}
@Override
String getInvocationMetricRunError() {
return mCurrentRunError;
}
@Override
void addInvocationMetricRunError(String errors) {
mCurrentRunError = errors;
}
}
@Before
public void setUp() throws Exception {
mAggListener = EasyMock.createStrictMock(ILogSaverListener.class);
mDetailedListener = EasyMock.createStrictMock(ITestDetailedReceiver.class);
mLogger = EasyMock.createMock(ILogSaver.class);
mInvocationContext = new InvocationContext();
mInvocationContext.addDeviceBuildInfo(
ConfigurationDef.DEFAULT_DEVICE_NAME, new BuildInfo());
mModuleContext = new InvocationContext();
}
@After
public void tearDown() throws Exception {
if (mAggregator != null) {
for (File f : mAggregator.getEventsLogs()) {
FileUtil.deleteFile(f);
}
}
}
@Test
public void testForwarding() throws Exception {
LogFile beforeModule = new LogFile("before-module", "url", LogDataType.TEXT);
LogFile test1Log = new LogFile("test1", "url", LogDataType.TEXT);
LogFile test2LogBefore = new LogFile("test2-before", "url", LogDataType.TEXT);
LogFile test2LogAfter = new LogFile("test2-after", "url", LogDataType.TEXT);
LogFile testRun1LogBefore = new LogFile("test-run1-before", "url", LogDataType.TEXT);
LogFile testRun1LogAfter = new LogFile("test-run1-after", "url", LogDataType.TEXT);
LogFile beforeEnd = new LogFile("path", "url", LogDataType.TEXT);
LogFile betweenAttemptsLog = new LogFile("between-attempts", "url", LogDataType.TEXT);
LogFile moduleLog = new LogFile("module-log", "url", LogDataType.TEXT);
TestDescription test1 = new TestDescription("classname", "test1");
TestDescription test2 = new TestDescription("classname", "test2");
EasyMock.expect(mDetailedListener.supportGranularResults()).andStubReturn(true);
// Invocation level
mAggListener.setLogSaver(mLogger);
mAggListener.invocationStarted(mInvocationContext);
EasyMock.expect(mAggListener.getSummary()).andStubReturn(null);
mDetailedListener.setLogSaver(mLogger);
mDetailedListener.invocationStarted(mInvocationContext);
EasyMock.expect(mDetailedListener.getSummary()).andStubReturn(null);
mDetailedListener.logAssociation("before-module-log", beforeModule);
mAggListener.testModuleStarted(mModuleContext);
mDetailedListener.testModuleStarted(mModuleContext);
// Detailed receives the breakdown
mDetailedListener.testRunStarted(
EasyMock.eq("run1"), EasyMock.eq(2), EasyMock.eq(0), EasyMock.anyLong());
mDetailedListener.testStarted(EasyMock.eq(test1), EasyMock.anyLong());
mDetailedListener.logAssociation("test1-log", test1Log);
mDetailedListener.testEnded(
EasyMock.eq(test1),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mDetailedListener.testStarted(EasyMock.eq(test2), EasyMock.anyLong());
mDetailedListener.logAssociation("test2-before-log", test2LogBefore);
mDetailedListener.testFailed(test2, FailureDescription.create("I failed. retry me."));
mDetailedListener.logAssociation("test2-after-log", test2LogAfter);
mDetailedListener.testEnded(
EasyMock.eq(test2),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mDetailedListener.logAssociation("test-run1-before-log", testRun1LogBefore);
mDetailedListener.logAssociation("test-run1-after-log", testRun1LogAfter);
mDetailedListener.testRunEnded(450L, new HashMap<String, Metric>());
mDetailedListener.testRunStarted(
EasyMock.eq("run1"), EasyMock.eq(2), EasyMock.eq(1), EasyMock.anyLong());
mDetailedListener.testStarted(EasyMock.eq(test2), EasyMock.anyLong());
mDetailedListener.testEnded(
EasyMock.eq(test2),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mDetailedListener.testRunEnded(450L, new HashMap<String, Metric>());
mDetailedListener.logAssociation("between-attempts", betweenAttemptsLog);
mDetailedListener.logAssociation("module-log", moduleLog);
// Aggregated listeners receives the aggregated results
mAggListener.testRunStarted(
EasyMock.eq("run1"), EasyMock.eq(2), EasyMock.eq(0), EasyMock.anyLong());
mAggListener.testStarted(EasyMock.eq(test1), EasyMock.anyLong());
mAggListener.logAssociation("test1-log", test1Log);
mAggListener.testEnded(
EasyMock.eq(test1),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mAggListener.testStarted(EasyMock.eq(test2), EasyMock.anyLong());
mAggListener.logAssociation("test2-before-log", test2LogBefore);
mAggListener.logAssociation("test2-after-log", test2LogAfter);
mAggListener.testEnded(
EasyMock.eq(test2),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mAggListener.logAssociation("test-run1-before-log", testRun1LogBefore);
mAggListener.logAssociation("test-run1-after-log", testRun1LogAfter);
mAggListener.testRunEnded(450L, new HashMap<String, Metric>());
mAggListener.logAssociation("between-attempts", betweenAttemptsLog);
mAggListener.logAssociation("module-log", moduleLog);
mAggListener.testModuleEnded();
mDetailedListener.testModuleEnded();
mAggListener.logAssociation("before-module-log", beforeModule);
mAggListener.logAssociation("before-end", beforeEnd);
mAggListener.invocationEnded(500L);
mDetailedListener.logAssociation("before-end", beforeEnd);
mDetailedListener.invocationEnded(500L);
EasyMock.expect(
mLogger.saveLogData(
EasyMock.contains("aggregated-events"),
EasyMock.eq(LogDataType.TF_EVENTS),
EasyMock.anyObject()))
.andReturn(null);
EasyMock.expect(
mLogger.saveLogData(
EasyMock.contains("detailed-events"),
EasyMock.eq(LogDataType.TF_EVENTS),
EasyMock.anyObject()))
.andReturn(null);
EasyMock.replay(mAggListener, mDetailedListener, mLogger);
mAggregator =
new TestableResultAggregator(
Arrays.asList(mAggListener, mDetailedListener),
RetryStrategy.RETRY_ANY_FAILURE);
mAggregator.setLogSaver(mLogger);
mAggregator.invocationStarted(mInvocationContext);
mAggregator.logAssociation("before-module-log", beforeModule);
mAggregator.testModuleStarted(mModuleContext);
// Attempt 1
mAggregator.testRunStarted("run1", 2, 0);
mAggregator.testStarted(test1);
mAggregator.logAssociation("test1-log", test1Log);
mAggregator.testEnded(test1, new HashMap<String, Metric>());
mAggregator.testStarted(test2);
mAggregator.logAssociation("test2-before-log", test2LogBefore);
mAggregator.testFailed(test2, FailureDescription.create("I failed. retry me."));
mAggregator.logAssociation("test2-after-log", test2LogAfter);
mAggregator.testEnded(test2, new HashMap<String, Metric>());
mAggregator.logAssociation("test-run1-before-log", testRun1LogBefore);
mAggregator.testRunFailed("run fail");
mAggregator.logAssociation("test-run1-after-log", testRun1LogAfter);
mAggregator.testRunEnded(450L, new HashMap<String, Metric>());
mAggregator.logAssociation("between-attempts", betweenAttemptsLog);
// Attempt 2
mAggregator.testRunStarted("run1", 2, 1);
mAggregator.testStarted(test2);
mAggregator.testEnded(test2, new HashMap<String, Metric>());
mAggregator.testRunEnded(450L, new HashMap<String, Metric>());
mAggregator.logAssociation("module-log", moduleLog);
mAggregator.testModuleEnded();
mAggregator.logAssociation("before-end", beforeEnd);
mAggregator.invocationEnded(500L);
EasyMock.verify(mAggListener, mDetailedListener, mLogger);
assertEquals("run fail", mAggregator.getInvocationMetricRunError());
}
@Test
public void testForwarding_newResult() throws Exception {
LogFile beforeModule = new LogFile("before-module", "url", LogDataType.TEXT);
LogFile test1Log = new LogFile("test1", "url", LogDataType.TEXT);
LogFile test2LogBefore = new LogFile("test2-before", "url", LogDataType.TEXT);
LogFile test2LogAfter = new LogFile("test2-after", "url", LogDataType.TEXT);
LogFile testRun1LogBefore = new LogFile("test-run1-before", "url", LogDataType.TEXT);
LogFile testRun1LogAfter = new LogFile("test-run1-after", "url", LogDataType.TEXT);
LogFile beforeEnd = new LogFile("path", "url", LogDataType.TEXT);
LogFile betweenAttemptsLog = new LogFile("between-attempts", "url", LogDataType.TEXT);
LogFile moduleLog = new LogFile("module-log", "url", LogDataType.TEXT);
TestDescription test1 = new TestDescription("classname", "test1");
TestDescription test2 = new TestDescription("classname", "test2");
EasyMock.expect(mDetailedListener.supportGranularResults()).andStubReturn(true);
// Invocation level
mAggListener.setLogSaver(mLogger);
mAggListener.invocationStarted(mInvocationContext);
EasyMock.expect(mAggListener.getSummary()).andStubReturn(null);
mDetailedListener.setLogSaver(mLogger);
mDetailedListener.invocationStarted(mInvocationContext);
EasyMock.expect(mDetailedListener.getSummary()).andStubReturn(null);
mDetailedListener.logAssociation("before-module-log", beforeModule);
mAggListener.testModuleStarted(mModuleContext);
mDetailedListener.testModuleStarted(mModuleContext);
// Detailed receives the breakdown
mDetailedListener.testRunStarted(
EasyMock.eq("run1"), EasyMock.eq(2), EasyMock.eq(0), EasyMock.anyLong());
mDetailedListener.testStarted(EasyMock.eq(test1), EasyMock.anyLong());
mDetailedListener.logAssociation("test1-log", test1Log);
mDetailedListener.testEnded(
EasyMock.eq(test1),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mDetailedListener.testStarted(EasyMock.eq(test2), EasyMock.anyLong());
mDetailedListener.logAssociation("test2-before-log", test2LogBefore);
mDetailedListener.testFailed(test2, FailureDescription.create("I failed. retry me."));
mDetailedListener.logAssociation("test2-after-log", test2LogAfter);
mDetailedListener.testEnded(
EasyMock.eq(test2),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mDetailedListener.logAssociation("test-run1-before-log", testRun1LogBefore);
mDetailedListener.testRunFailed("run fail");
mDetailedListener.logAssociation("test-run1-after-log", testRun1LogAfter);
mDetailedListener.testRunEnded(450L, new HashMap<String, Metric>());
mDetailedListener.logAssociation("between-attempts", betweenAttemptsLog);
mDetailedListener.testRunStarted(
EasyMock.eq("run1"), EasyMock.eq(2), EasyMock.eq(1), EasyMock.anyLong());
mDetailedListener.testStarted(EasyMock.eq(test2), EasyMock.anyLong());
mDetailedListener.testEnded(
EasyMock.eq(test2),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mDetailedListener.testRunEnded(450L, new HashMap<String, Metric>());
mDetailedListener.logAssociation("module-log", moduleLog);
// Aggregated listeners receives the aggregated results
mAggListener.testRunStarted(
EasyMock.eq("run1"), EasyMock.eq(2), EasyMock.eq(0), EasyMock.anyLong());
mAggListener.testStarted(EasyMock.eq(test1), EasyMock.anyLong());
mAggListener.logAssociation("test1-log", test1Log);
mAggListener.testEnded(
EasyMock.eq(test1),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mAggListener.testStarted(EasyMock.eq(test2), EasyMock.anyLong());
mAggListener.logAssociation("test2-before-log", test2LogBefore);
mAggListener.logAssociation("test2-after-log", test2LogAfter);
mAggListener.testEnded(
EasyMock.eq(test2),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mAggListener.logAssociation("test-run1-before-log", testRun1LogBefore);
mAggListener.logAssociation("test-run1-after-log", testRun1LogAfter);
mAggListener.testRunEnded(450L, new HashMap<String, Metric>());
mAggListener.logAssociation("between-attempts", betweenAttemptsLog);
mAggListener.logAssociation("module-log", moduleLog);
mAggListener.testModuleEnded();
mDetailedListener.testModuleEnded();
mAggListener.logAssociation("before-module-log", beforeModule);
mAggListener.logAssociation("before-end", beforeEnd);
mAggListener.invocationEnded(500L);
mDetailedListener.logAssociation("before-end", beforeEnd);
mDetailedListener.invocationEnded(500L);
EasyMock.expect(
mLogger.saveLogData(
EasyMock.contains("aggregated-events"),
EasyMock.eq(LogDataType.TF_EVENTS),
EasyMock.anyObject()))
.andReturn(null);
EasyMock.expect(
mLogger.saveLogData(
EasyMock.contains("detailed-events"),
EasyMock.eq(LogDataType.TF_EVENTS),
EasyMock.anyObject()))
.andReturn(null);
EasyMock.replay(mAggListener, mDetailedListener, mLogger);
mAggregator =
new TestableResultAggregator(
Arrays.asList(mAggListener, mDetailedListener),
RetryStrategy.RETRY_ANY_FAILURE);
mAggregator.setUpdatedReporting(true);
mAggregator.setLogSaver(mLogger);
mAggregator.invocationStarted(mInvocationContext);
mAggregator.logAssociation("before-module-log", beforeModule);
mAggregator.testModuleStarted(mModuleContext);
// Attempt 1
mAggregator.testRunStarted("run1", 2, 0);
mAggregator.testStarted(test1);
mAggregator.logAssociation("test1-log", test1Log);
mAggregator.testEnded(test1, new HashMap<String, Metric>());
mAggregator.testStarted(test2);
mAggregator.logAssociation("test2-before-log", test2LogBefore);
mAggregator.testFailed(test2, FailureDescription.create("I failed. retry me."));
mAggregator.logAssociation("test2-after-log", test2LogAfter);
mAggregator.testEnded(test2, new HashMap<String, Metric>());
mAggregator.logAssociation("test-run1-before-log", testRun1LogBefore);
mAggregator.testRunFailed("run fail");
mAggregator.logAssociation("test-run1-after-log", testRun1LogAfter);
mAggregator.testRunEnded(450L, new HashMap<String, Metric>());
mAggregator.logAssociation("between-attempts", betweenAttemptsLog);
// Attempt 2
mAggregator.testRunStarted("run1", 2, 1);
mAggregator.testStarted(test2);
mAggregator.testEnded(test2, new HashMap<String, Metric>());
mAggregator.testRunEnded(450L, new HashMap<String, Metric>());
mAggregator.logAssociation("module-log", moduleLog);
mAggregator.testModuleEnded();
mAggregator.logAssociation("before-end", beforeEnd);
mAggregator.invocationEnded(500L);
EasyMock.verify(mAggListener, mDetailedListener, mLogger);
assertNull(mAggregator.getInvocationMetricRunError());
}
@Test
public void testForwarding_assumptionFailure() throws Exception {
mDetailedListener = EasyMock.createStrictMock(ITestDetailedReceiver.class);
LogFile test1Log = new LogFile("test1", "url", LogDataType.TEXT);
LogFile test2LogBefore = new LogFile("test2-before", "url", LogDataType.TEXT);
LogFile test2LogAfter = new LogFile("test2-after", "url", LogDataType.TEXT);
LogFile testRun1LogBefore = new LogFile("test-run1-before", "url", LogDataType.TEXT);
LogFile testRun1LogAfter = new LogFile("test-run1-after", "url", LogDataType.TEXT);
LogFile beforeEnd = new LogFile("path", "url", LogDataType.TEXT);
LogFile betweenAttemptsLog = new LogFile("between-attempts", "url", LogDataType.TEXT);
LogFile moduleLog = new LogFile("module-log", "url", LogDataType.TEXT);
TestDescription test1 = new TestDescription("classname", "test1");
TestDescription test2 = new TestDescription("classname", "test2");
EasyMock.expect(mDetailedListener.supportGranularResults()).andStubReturn(true);
// Invocation level
mAggListener.setLogSaver(mLogger);
mAggListener.invocationStarted(mInvocationContext);
EasyMock.expect(mAggListener.getSummary()).andStubReturn(null);
mDetailedListener.setLogSaver(mLogger);
mDetailedListener.invocationStarted(mInvocationContext);
EasyMock.expect(mDetailedListener.getSummary()).andStubReturn(null);
mAggListener.testModuleStarted(mModuleContext);
mDetailedListener.testModuleStarted(mModuleContext);
// Detailed receives the breakdown
mDetailedListener.testRunStarted(
EasyMock.eq("run1"), EasyMock.eq(2), EasyMock.eq(0), EasyMock.anyLong());
mDetailedListener.testStarted(EasyMock.eq(test1), EasyMock.anyLong());
mDetailedListener.logAssociation("test1-log", test1Log);
mDetailedListener.testEnded(
EasyMock.eq(test1),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mDetailedListener.testStarted(EasyMock.eq(test2), EasyMock.anyLong());
mDetailedListener.logAssociation("test2-before-log", test2LogBefore);
mDetailedListener.testFailed(test2, FailureDescription.create("I failed. retry me."));
mDetailedListener.logAssociation("test2-after-log", test2LogAfter);
mDetailedListener.testEnded(
EasyMock.eq(test2),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mDetailedListener.logAssociation("test-run1-before-log", testRun1LogBefore);
mDetailedListener.logAssociation("test-run1-after-log", testRun1LogAfter);
mDetailedListener.testRunEnded(450L, new HashMap<String, Metric>());
mDetailedListener.testRunStarted(
EasyMock.eq("run1"), EasyMock.eq(2), EasyMock.eq(1), EasyMock.anyLong());
mDetailedListener.testStarted(EasyMock.eq(test2), EasyMock.anyLong());
mDetailedListener.testAssumptionFailure(
EasyMock.eq(test2), EasyMock.eq(FailureDescription.create("Assump failure")));
mDetailedListener.testEnded(
EasyMock.eq(test2),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mDetailedListener.testRunEnded(450L, new HashMap<String, Metric>());
mDetailedListener.logAssociation("between-attempts", betweenAttemptsLog);
mDetailedListener.logAssociation("module-log", moduleLog);
// Aggregated listeners receives the aggregated results
mAggListener.testRunStarted(
EasyMock.eq("run1"), EasyMock.eq(2), EasyMock.eq(0), EasyMock.anyLong());
mAggListener.testStarted(EasyMock.eq(test1), EasyMock.anyLong());
mAggListener.logAssociation("test1-log", test1Log);
mAggListener.testEnded(
EasyMock.eq(test1),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mAggListener.testStarted(EasyMock.eq(test2), EasyMock.anyLong());
mAggListener.testAssumptionFailure(
EasyMock.eq(test2), (FailureDescription) EasyMock.anyObject());
mAggListener.logAssociation("test2-before-log", test2LogBefore);
mAggListener.logAssociation("test2-after-log", test2LogAfter);
mAggListener.testEnded(
EasyMock.eq(test2),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mAggListener.logAssociation("test-run1-before-log", testRun1LogBefore);
mAggListener.logAssociation("test-run1-after-log", testRun1LogAfter);
mAggListener.testRunEnded(450L, new HashMap<String, Metric>());
mAggListener.logAssociation("between-attempts", betweenAttemptsLog);
mAggListener.logAssociation("module-log", moduleLog);
mAggListener.testModuleEnded();
mDetailedListener.testModuleEnded();
mAggListener.logAssociation("before-end", beforeEnd);
mAggListener.invocationEnded(500L);
mDetailedListener.logAssociation("before-end", beforeEnd);
mDetailedListener.invocationEnded(500L);
EasyMock.expect(
mLogger.saveLogData(
EasyMock.contains("aggregated-events"),
EasyMock.eq(LogDataType.TF_EVENTS),
EasyMock.anyObject()))
.andReturn(null);
EasyMock.expect(
mLogger.saveLogData(
EasyMock.contains("detailed-events"),
EasyMock.eq(LogDataType.TF_EVENTS),
EasyMock.anyObject()))
.andReturn(null);
EasyMock.replay(mAggListener, mDetailedListener, mLogger);
mAggregator =
new TestableResultAggregator(
Arrays.asList(mAggListener, mDetailedListener),
RetryStrategy.RETRY_ANY_FAILURE);
mAggregator.setLogSaver(mLogger);
mAggregator.invocationStarted(mInvocationContext);
mAggregator.testModuleStarted(mModuleContext);
// Attempt 1
mAggregator.testRunStarted("run1", 2, 0);
mAggregator.testStarted(test1);
mAggregator.logAssociation("test1-log", test1Log);
mAggregator.testEnded(test1, new HashMap<String, Metric>());
mAggregator.testStarted(test2);
mAggregator.logAssociation("test2-before-log", test2LogBefore);
mAggregator.testFailed(test2, FailureDescription.create("I failed. retry me."));
mAggregator.logAssociation("test2-after-log", test2LogAfter);
mAggregator.testEnded(test2, new HashMap<String, Metric>());
mAggregator.logAssociation("test-run1-before-log", testRun1LogBefore);
mAggregator.testRunFailed("run fail");
mAggregator.logAssociation("test-run1-after-log", testRun1LogAfter);
mAggregator.testRunEnded(450L, new HashMap<String, Metric>());
mAggregator.logAssociation("between-attempts", betweenAttemptsLog);
// Attempt 2
mAggregator.testRunStarted("run1", 2, 1);
mAggregator.testStarted(test2);
mAggregator.testAssumptionFailure(test2, FailureDescription.create("Assump failure"));
mAggregator.testEnded(test2, new HashMap<String, Metric>());
mAggregator.testRunEnded(450L, new HashMap<String, Metric>());
mAggregator.logAssociation("module-log", moduleLog);
mAggregator.testModuleEnded();
mAggregator.logAssociation("before-end", beforeEnd);
mAggregator.invocationEnded(500L);
EasyMock.verify(mAggListener, mDetailedListener, mLogger);
assertEquals("run fail", mAggregator.getInvocationMetricRunError());
}
@Test
public void testForwarding_runFailure() throws Exception {
mDetailedListener = EasyMock.createStrictMock(ITestDetailedReceiver.class);
TestDescription test1 = new TestDescription("classname", "test1");
TestDescription test2 = new TestDescription("classname", "test2");
MultiFailureDescription aggFailure =
new MultiFailureDescription(
FailureDescription.create("run fail"),
FailureDescription.create("run fail 2"));
EasyMock.expect(mDetailedListener.supportGranularResults()).andStubReturn(true);
// Invocation level
mAggListener.setLogSaver(mLogger);
mAggListener.invocationStarted(mInvocationContext);
EasyMock.expect(mAggListener.getSummary()).andStubReturn(null);
mDetailedListener.setLogSaver(mLogger);
mDetailedListener.invocationStarted(mInvocationContext);
EasyMock.expect(mDetailedListener.getSummary()).andStubReturn(null);
mAggListener.testModuleStarted(mModuleContext);
mDetailedListener.testModuleStarted(mModuleContext);
// Detailed receives the breakdown
mDetailedListener.testRunStarted(
EasyMock.eq("run1"), EasyMock.eq(2), EasyMock.eq(0), EasyMock.anyLong());
mDetailedListener.testStarted(EasyMock.eq(test1), EasyMock.anyLong());
mDetailedListener.testEnded(
EasyMock.eq(test1),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mDetailedListener.testStarted(EasyMock.eq(test2), EasyMock.anyLong());
mDetailedListener.testFailed(test2, "I failed. retry me.");
mDetailedListener.testEnded(
EasyMock.eq(test2),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mDetailedListener.testRunEnded(450L, new HashMap<String, Metric>());
mDetailedListener.testRunStarted(
EasyMock.eq("run1"), EasyMock.eq(2), EasyMock.eq(1), EasyMock.anyLong());
mDetailedListener.testStarted(EasyMock.eq(test2), EasyMock.anyLong());
mDetailedListener.testEnded(
EasyMock.eq(test2),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mDetailedListener.testRunFailed(aggFailure);
mDetailedListener.testRunEnded(450L, new HashMap<String, Metric>());
// Aggregated listeners receives the aggregated results
mAggListener.testRunStarted(
EasyMock.eq("run1"), EasyMock.eq(2), EasyMock.eq(0), EasyMock.anyLong());
mAggListener.testStarted(EasyMock.eq(test1), EasyMock.anyLong());
mAggListener.testEnded(
EasyMock.eq(test1),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mAggListener.testStarted(EasyMock.eq(test2), EasyMock.anyLong());
mAggListener.testEnded(
EasyMock.eq(test2),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mAggListener.testRunFailed(aggFailure);
mAggListener.testRunEnded(450L, new HashMap<String, Metric>());
mAggListener.testModuleEnded();
mDetailedListener.testModuleEnded();
mAggListener.invocationEnded(500L);
mDetailedListener.invocationEnded(500L);
EasyMock.expect(
mLogger.saveLogData(
EasyMock.contains("aggregated-events"),
EasyMock.eq(LogDataType.TF_EVENTS),
EasyMock.anyObject()))
.andReturn(null);
EasyMock.expect(
mLogger.saveLogData(
EasyMock.contains("detailed-events"),
EasyMock.eq(LogDataType.TF_EVENTS),
EasyMock.anyObject()))
.andReturn(null);
EasyMock.replay(mAggListener, mDetailedListener, mLogger);
mAggregator =
new TestableResultAggregator(
Arrays.asList(mAggListener, mDetailedListener),
RetryStrategy.RETRY_ANY_FAILURE);
mAggregator.setLogSaver(mLogger);
mAggregator.invocationStarted(mInvocationContext);
mAggregator.testModuleStarted(mModuleContext);
// Attempt 1
mAggregator.testRunStarted("run1", 2, 0);
mAggregator.testStarted(test1);
mAggregator.testEnded(test1, new HashMap<String, Metric>());
mAggregator.testStarted(test2);
mAggregator.testFailed(test2, "I failed. retry me.");
mAggregator.testEnded(test2, new HashMap<String, Metric>());
mAggregator.testRunFailed("run fail");
mAggregator.testRunEnded(450L, new HashMap<String, Metric>());
// Attempt 2
mAggregator.testRunStarted("run1", 2, 1);
mAggregator.testStarted(test2);
mAggregator.testEnded(test2, new HashMap<String, Metric>());
mAggregator.testRunFailed("run fail 2");
mAggregator.testRunEnded(450L, new HashMap<String, Metric>());
mAggregator.testModuleEnded();
mAggregator.invocationEnded(500L);
EasyMock.verify(mAggListener, mDetailedListener, mLogger);
assertNull(mAggregator.getInvocationMetricRunError());
}
@Test
public void testForwarding_runFailure_noRerun() throws Exception {
TestDescription test1 = new TestDescription("classname", "test1");
TestDescription test2 = new TestDescription("classname", "test2");
EasyMock.expect(mDetailedListener.supportGranularResults()).andStubReturn(true);
// Invocation level
mAggListener.setLogSaver(mLogger);
mAggListener.invocationStarted(mInvocationContext);
EasyMock.expect(mAggListener.getSummary()).andStubReturn(null);
mDetailedListener.setLogSaver(mLogger);
mDetailedListener.invocationStarted(mInvocationContext);
EasyMock.expect(mDetailedListener.getSummary()).andStubReturn(null);
mAggListener.testModuleStarted(mModuleContext);
mDetailedListener.testModuleStarted(mModuleContext);
// Detailed receives the breakdown
mDetailedListener.testRunStarted(
EasyMock.eq("run1"), EasyMock.eq(2), EasyMock.eq(0), EasyMock.anyLong());
mDetailedListener.testStarted(EasyMock.eq(test1), EasyMock.anyLong());
mDetailedListener.testEnded(
EasyMock.eq(test1),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mDetailedListener.testStarted(EasyMock.eq(test2), EasyMock.anyLong());
mDetailedListener.testFailed(test2, "I failed. retry me.");
mDetailedListener.testEnded(
EasyMock.eq(test2),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mDetailedListener.testRunFailed(EasyMock.eq(FailureDescription.create("run fail")));
mDetailedListener.testRunEnded(450L, new HashMap<String, Metric>());
// Aggregated listeners receives the aggregated results
mAggListener.testRunStarted(
EasyMock.eq("run1"), EasyMock.eq(2), EasyMock.eq(0), EasyMock.anyLong());
mAggListener.testStarted(EasyMock.eq(test1), EasyMock.anyLong());
mAggListener.testEnded(
EasyMock.eq(test1),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mAggListener.testStarted(EasyMock.eq(test2), EasyMock.anyLong());
mAggListener.testFailed(test2, FailureDescription.create("I failed. retry me."));
mAggListener.testEnded(
EasyMock.eq(test2),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mAggListener.testRunFailed(EasyMock.eq(FailureDescription.create("run fail")));
mAggListener.testRunEnded(450L, new HashMap<String, Metric>());
mAggListener.testModuleEnded();
mDetailedListener.testModuleEnded();
mAggListener.invocationEnded(500L);
mDetailedListener.invocationEnded(500L);
EasyMock.expect(
mLogger.saveLogData(
EasyMock.contains("aggregated-events"),
EasyMock.eq(LogDataType.TF_EVENTS),
EasyMock.anyObject()))
.andReturn(null);
EasyMock.expect(
mLogger.saveLogData(
EasyMock.contains("detailed-events"),
EasyMock.eq(LogDataType.TF_EVENTS),
EasyMock.anyObject()))
.andReturn(null);
EasyMock.replay(mAggListener, mDetailedListener, mLogger);
mAggregator =
new TestableResultAggregator(
Arrays.asList(mAggListener, mDetailedListener),
RetryStrategy.RETRY_ANY_FAILURE);
mAggregator.setLogSaver(mLogger);
mAggregator.invocationStarted(mInvocationContext);
mAggregator.testModuleStarted(mModuleContext);
// Attempt 1
mAggregator.testRunStarted("run1", 2, 0);
mAggregator.testStarted(test1);
mAggregator.testEnded(test1, new HashMap<String, Metric>());
mAggregator.testStarted(test2);
mAggregator.testFailed(test2, "I failed. retry me.");
mAggregator.testEnded(test2, new HashMap<String, Metric>());
mAggregator.testRunFailed("run fail");
mAggregator.testRunEnded(450L, new HashMap<String, Metric>());
mAggregator.testModuleEnded();
mAggregator.invocationEnded(500L);
EasyMock.verify(mAggListener, mDetailedListener, mLogger);
assertNull(mAggregator.getInvocationMetricRunError());
}
@Test
public void testForwarding_runFailure_aggregation() throws Exception {
TestDescription test1 = new TestDescription("classname", "test1");
TestDescription test2 = new TestDescription("classname", "test2");
EasyMock.expect(mDetailedListener.supportGranularResults()).andStubReturn(true);
// Invocation level
mAggListener.setLogSaver(mLogger);
mAggListener.invocationStarted(mInvocationContext);
EasyMock.expect(mAggListener.getSummary()).andStubReturn(null);
mDetailedListener.setLogSaver(mLogger);
mDetailedListener.invocationStarted(mInvocationContext);
EasyMock.expect(mDetailedListener.getSummary()).andStubReturn(null);
mAggListener.testModuleStarted(mModuleContext);
mDetailedListener.testModuleStarted(mModuleContext);
// Detailed receives the breakdown
mDetailedListener.testRunStarted(
EasyMock.eq("run1"), EasyMock.eq(2), EasyMock.eq(0), EasyMock.anyLong());
mDetailedListener.testStarted(EasyMock.eq(test1), EasyMock.anyLong());
mDetailedListener.testEnded(
EasyMock.eq(test1),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mDetailedListener.testStarted(EasyMock.eq(test2), EasyMock.anyLong());
mDetailedListener.testFailed(test2, "I failed. retry me.");
mDetailedListener.testEnded(
EasyMock.eq(test2),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mDetailedListener.testRunEnded(450L, new HashMap<String, Metric>());
mDetailedListener.testRunStarted(
EasyMock.eq("run1"), EasyMock.eq(2), EasyMock.eq(1), EasyMock.anyLong());
mDetailedListener.testStarted(EasyMock.eq(test2), EasyMock.anyLong());
mDetailedListener.testFailed(test2, "I failed. retry me.");
mDetailedListener.testEnded(
EasyMock.eq(test2),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mDetailedListener.testRunEnded(450L, new HashMap<String, Metric>());
mDetailedListener.testModuleEnded();
mDetailedListener.testModuleStarted(mModuleContext);
mDetailedListener.testRunStarted(
EasyMock.eq("run2"), EasyMock.eq(1), EasyMock.eq(0), EasyMock.anyLong());
mDetailedListener.testRunFailed(EasyMock.eq(FailureDescription.create("run fail")));
mDetailedListener.testRunEnded(450L, new HashMap<String, Metric>());
// Aggregated listeners receives the aggregated results
mAggListener.testRunStarted(
EasyMock.eq("run1"), EasyMock.eq(2), EasyMock.eq(0), EasyMock.anyLong());
mAggListener.testStarted(EasyMock.eq(test1), EasyMock.anyLong());
mAggListener.testEnded(
EasyMock.eq(test1),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mAggListener.testStarted(EasyMock.eq(test2), EasyMock.anyLong());
mAggListener.testFailed(
test2,
new MultiFailureDescription(
FailureDescription.create("I failed. retry me."),
FailureDescription.create("I failed. retry me.")));
mAggListener.testEnded(
EasyMock.eq(test2),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mAggListener.testRunEnded(450L, new HashMap<String, Metric>());
mAggListener.testModuleEnded();
mAggListener.testModuleStarted(mModuleContext);
mAggListener.testRunStarted(
EasyMock.eq("run2"), EasyMock.eq(1), EasyMock.eq(0), EasyMock.anyLong());
mAggListener.testRunFailed(EasyMock.eq(FailureDescription.create("run fail")));
mAggListener.testRunEnded(450L, new HashMap<String, Metric>());
mAggListener.testModuleEnded();
mDetailedListener.testModuleEnded();
mAggListener.invocationEnded(500L);
mDetailedListener.invocationEnded(500L);
EasyMock.expect(
mLogger.saveLogData(
EasyMock.contains("aggregated-events"),
EasyMock.eq(LogDataType.TF_EVENTS),
EasyMock.anyObject()))
.andReturn(null);
EasyMock.expect(
mLogger.saveLogData(
EasyMock.contains("detailed-events"),
EasyMock.eq(LogDataType.TF_EVENTS),
EasyMock.anyObject()))
.andReturn(null);
EasyMock.replay(mAggListener, mDetailedListener, mLogger);
mAggregator =
new TestableResultAggregator(
Arrays.asList(mAggListener, mDetailedListener),
RetryStrategy.RETRY_ANY_FAILURE);
mAggregator.setLogSaver(mLogger);
mAggregator.invocationStarted(mInvocationContext);
mAggregator.testModuleStarted(mModuleContext);
// Attempt 1
mAggregator.testRunStarted("run1", 2, 0);
mAggregator.testStarted(test1);
mAggregator.testEnded(test1, new HashMap<String, Metric>());
mAggregator.testStarted(test2);
mAggregator.testFailed(test2, "I failed. retry me.");
mAggregator.testEnded(test2, new HashMap<String, Metric>());
mAggregator.testRunEnded(450L, new HashMap<String, Metric>());
mAggregator.testRunStarted("run1", 2, 1);
mAggregator.testStarted(test2);
mAggregator.testFailed(test2, "I failed. retry me.");
mAggregator.testEnded(test2, new HashMap<String, Metric>());
mAggregator.testRunEnded(450L, new HashMap<String, Metric>());
mAggregator.testModuleEnded();
mAggregator.testModuleStarted(mModuleContext);
mAggregator.testRunStarted("run2", 1, 0);
mAggregator.testRunFailed(FailureDescription.create("run fail"));
mAggregator.testRunEnded(450L, new HashMap<String, Metric>());
mAggregator.testModuleEnded();
mAggregator.invocationEnded(500L);
EasyMock.verify(mAggListener, mDetailedListener, mLogger);
assertNull(mAggregator.getInvocationMetricRunError());
}
@Test
public void testForwarding_noModules() throws Exception {
TestDescription test1 = new TestDescription("classname", "test1");
TestDescription test2 = new TestDescription("classname", "test2");
LogFile afterRunLog = new LogFile("after-run", "url", LogDataType.TEXT);
mDetailedListener = EasyMock.createStrictMock(ITestDetailedReceiver.class);
EasyMock.expect(mDetailedListener.supportGranularResults()).andStubReturn(true);
// Invocation level
mAggListener.setLogSaver(mLogger);
mAggListener.invocationStarted(mInvocationContext);
EasyMock.expect(mAggListener.getSummary()).andStubReturn(null);
mDetailedListener.setLogSaver(mLogger);
mDetailedListener.invocationStarted(mInvocationContext);
EasyMock.expect(mDetailedListener.getSummary()).andStubReturn(null);
// Detailed receives the breakdown
mDetailedListener.testRunStarted(
EasyMock.eq("run1"), EasyMock.eq(2), EasyMock.eq(0), EasyMock.anyLong());
mDetailedListener.testStarted(EasyMock.eq(test1), EasyMock.anyLong());
mDetailedListener.testEnded(
EasyMock.eq(test1),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mDetailedListener.testStarted(EasyMock.eq(test2), EasyMock.anyLong());
mDetailedListener.testFailed(test2, "I failed. retry me.");
mDetailedListener.testEnded(
EasyMock.eq(test2),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mDetailedListener.testRunEnded(450L, new HashMap<String, Metric>());
mDetailedListener.testRunStarted(
EasyMock.eq("run1"), EasyMock.eq(2), EasyMock.eq(1), EasyMock.anyLong());
mDetailedListener.testStarted(EasyMock.eq(test2), EasyMock.anyLong());
mDetailedListener.testEnded(
EasyMock.eq(test2),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mDetailedListener.testRunEnded(450L, new HashMap<String, Metric>());
mDetailedListener.logAssociation("after-run", afterRunLog);
// Aggregated listeners receives the aggregated results
mAggListener.testRunStarted(
EasyMock.eq("run1"), EasyMock.eq(2), EasyMock.eq(0), EasyMock.anyLong());
mAggListener.testStarted(EasyMock.eq(test1), EasyMock.anyLong());
mAggListener.testEnded(
EasyMock.eq(test1),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mAggListener.testStarted(EasyMock.eq(test2), EasyMock.anyLong());
mAggListener.testEnded(
EasyMock.eq(test2),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mAggListener.testRunEnded(900L, new HashMap<String, Metric>());
mAggListener.logAssociation("after-run", afterRunLog);
mAggListener.invocationEnded(500L);
mDetailedListener.invocationEnded(500L);
EasyMock.expect(
mLogger.saveLogData(
EasyMock.contains("aggregated-events"),
EasyMock.eq(LogDataType.TF_EVENTS),
EasyMock.anyObject()))
.andReturn(null);
EasyMock.expect(
mLogger.saveLogData(
EasyMock.contains("detailed-events"),
EasyMock.eq(LogDataType.TF_EVENTS),
EasyMock.anyObject()))
.andReturn(null);
EasyMock.replay(mAggListener, mDetailedListener, mLogger);
mAggregator =
new TestableResultAggregator(
Arrays.asList(mAggListener, mDetailedListener),
RetryStrategy.RETRY_ANY_FAILURE);
mAggregator.setLogSaver(mLogger);
mAggregator.invocationStarted(mInvocationContext);
// Attempt 1
mAggregator.testRunStarted("run1", 2, 0);
mAggregator.testStarted(test1);
mAggregator.testEnded(test1, new HashMap<String, Metric>());
mAggregator.testStarted(test2);
mAggregator.testFailed(test2, "I failed. retry me.");
mAggregator.testEnded(test2, new HashMap<String, Metric>());
mAggregator.testRunFailed("I failed");
mAggregator.testRunEnded(450L, new HashMap<String, Metric>());
// Attempt 2
mAggregator.testRunStarted("run1", 2, 1);
mAggregator.testStarted(test2);
mAggregator.testEnded(test2, new HashMap<String, Metric>());
mAggregator.testRunEnded(450L, new HashMap<String, Metric>());
mAggregator.logAssociation("after-run", afterRunLog);
mAggregator.invocationEnded(500L);
EasyMock.verify(mAggListener, mDetailedListener, mLogger);
assertEquals("I failed", mAggregator.getInvocationMetricRunError());
}
@Test
public void testForwarding_singleRun_noModules_runFailures() throws Exception {
TestDescription test1 = new TestDescription("classname", "test1");
TestDescription test2 = new TestDescription("classname", "test2");
MultiFailureDescription aggFailure =
new MultiFailureDescription(
FailureDescription.create("I failed"),
FailureDescription.create("I failed 2"));
EasyMock.expect(mDetailedListener.supportGranularResults()).andStubReturn(true);
// Invocation level
mAggListener.setLogSaver(mLogger);
mAggListener.invocationStarted(mInvocationContext);
EasyMock.expect(mAggListener.getSummary()).andStubReturn(null);
mDetailedListener.setLogSaver(mLogger);
mDetailedListener.invocationStarted(mInvocationContext);
EasyMock.expect(mDetailedListener.getSummary()).andStubReturn(null);
// Detailed receives the breakdown
mDetailedListener.testRunStarted(
EasyMock.eq("run1"), EasyMock.eq(2), EasyMock.eq(0), EasyMock.anyLong());
mDetailedListener.testStarted(EasyMock.eq(test1), EasyMock.anyLong());
mDetailedListener.testEnded(
EasyMock.eq(test1),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mDetailedListener.testStarted(EasyMock.eq(test2), EasyMock.anyLong());
mDetailedListener.testFailed(test2, "I failed. retry me.");
mDetailedListener.testEnded(
EasyMock.eq(test2),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mDetailedListener.testRunEnded(450L, new HashMap<String, Metric>());
mDetailedListener.testRunStarted(
EasyMock.eq("run1"), EasyMock.eq(2), EasyMock.eq(1), EasyMock.anyLong());
mDetailedListener.testStarted(EasyMock.eq(test2), EasyMock.anyLong());
mDetailedListener.testEnded(
EasyMock.eq(test2),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mDetailedListener.testRunFailed(aggFailure);
mDetailedListener.testRunEnded(450L, new HashMap<String, Metric>());
// Aggregated listeners receives the aggregated results
mAggListener.testRunStarted(
EasyMock.eq("run1"), EasyMock.eq(2), EasyMock.eq(0), EasyMock.anyLong());
mAggListener.testStarted(EasyMock.eq(test1), EasyMock.anyLong());
mAggListener.testEnded(
EasyMock.eq(test1),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mAggListener.testStarted(EasyMock.eq(test2), EasyMock.anyLong());
mAggListener.testEnded(
EasyMock.eq(test2),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mAggListener.testRunFailed(aggFailure);
mAggListener.testRunEnded(900L, new HashMap<String, Metric>());
mAggListener.invocationEnded(500L);
mDetailedListener.invocationEnded(500L);
EasyMock.expect(
mLogger.saveLogData(
EasyMock.contains("aggregated-events"),
EasyMock.eq(LogDataType.TF_EVENTS),
EasyMock.anyObject()))
.andReturn(null);
EasyMock.expect(
mLogger.saveLogData(
EasyMock.contains("detailed-events"),
EasyMock.eq(LogDataType.TF_EVENTS),
EasyMock.anyObject()))
.andReturn(null);
EasyMock.replay(mAggListener, mDetailedListener, mLogger);
mAggregator =
new TestableResultAggregator(
Arrays.asList(mAggListener, mDetailedListener),
RetryStrategy.RETRY_ANY_FAILURE);
mAggregator.setLogSaver(mLogger);
mAggregator.invocationStarted(mInvocationContext);
// Attempt 1
mAggregator.testRunStarted("run1", 2, 0);
mAggregator.testStarted(test1);
mAggregator.testEnded(test1, new HashMap<String, Metric>());
mAggregator.testStarted(test2);
mAggregator.testFailed(test2, "I failed. retry me.");
mAggregator.testEnded(test2, new HashMap<String, Metric>());
mAggregator.testRunFailed("I failed");
mAggregator.testRunEnded(450L, new HashMap<String, Metric>());
// Attempt 2
mAggregator.testRunStarted("run1", 2, 1);
mAggregator.testStarted(test2);
mAggregator.testEnded(test2, new HashMap<String, Metric>());
mAggregator.testRunFailed("I failed 2");
mAggregator.testRunEnded(450L, new HashMap<String, Metric>());
mAggregator.invocationEnded(500L);
EasyMock.verify(mAggListener, mDetailedListener, mLogger);
assertNull(mAggregator.getInvocationMetricRunError());
}
@Test
public void testForwarding_noModules_runFailures() throws Exception {
TestDescription test1 = new TestDescription("classname", "test1");
TestDescription test2 = new TestDescription("classname", "test2");
MultiFailureDescription aggFailure =
new MultiFailureDescription(
FailureDescription.create("I failed"),
FailureDescription.create("I failed 2"));
EasyMock.expect(mDetailedListener.supportGranularResults()).andStubReturn(true);
// Invocation level
mAggListener.setLogSaver(mLogger);
mAggListener.invocationStarted(mInvocationContext);
EasyMock.expect(mAggListener.getSummary()).andStubReturn(null);
mDetailedListener.setLogSaver(mLogger);
mDetailedListener.invocationStarted(mInvocationContext);
EasyMock.expect(mDetailedListener.getSummary()).andStubReturn(null);
// Detailed receives the breakdown
mDetailedListener.testRunStarted(
EasyMock.eq("run1"), EasyMock.eq(2), EasyMock.eq(0), EasyMock.anyLong());
mDetailedListener.testStarted(EasyMock.eq(test1), EasyMock.anyLong());
mDetailedListener.testEnded(
EasyMock.eq(test1),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mDetailedListener.testStarted(EasyMock.eq(test2), EasyMock.anyLong());
mDetailedListener.testFailed(test2, "I failed. retry me.");
mDetailedListener.testEnded(
EasyMock.eq(test2),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mDetailedListener.testRunEnded(450L, new HashMap<String, Metric>());
mDetailedListener.testRunStarted(
EasyMock.eq("run1"), EasyMock.eq(2), EasyMock.eq(1), EasyMock.anyLong());
mDetailedListener.testStarted(EasyMock.eq(test2), EasyMock.anyLong());
mDetailedListener.testEnded(
EasyMock.eq(test2),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mDetailedListener.testRunFailed(EasyMock.eq(aggFailure));
mDetailedListener.testRunEnded(450L, new HashMap<String, Metric>());
mDetailedListener.testRunStarted(
EasyMock.eq("run2"), EasyMock.eq(1), EasyMock.eq(0), EasyMock.anyLong());
mDetailedListener.testStarted(EasyMock.eq(test1), EasyMock.anyLong());
mDetailedListener.testEnded(
EasyMock.eq(test1),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mDetailedListener.testRunEnded(450L, new HashMap<String, Metric>());
// Aggregated listeners receives the aggregated results
mAggListener.testRunStarted(
EasyMock.eq("run1"), EasyMock.eq(2), EasyMock.eq(0), EasyMock.anyLong());
mAggListener.testStarted(EasyMock.eq(test1), EasyMock.anyLong());
mAggListener.testEnded(
EasyMock.eq(test1),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mAggListener.testStarted(EasyMock.eq(test2), EasyMock.anyLong());
mAggListener.testEnded(
EasyMock.eq(test2),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mAggListener.testRunFailed(EasyMock.eq(aggFailure));
mAggListener.testRunEnded(900L, new HashMap<String, Metric>());
mAggListener.testRunStarted(
EasyMock.eq("run2"), EasyMock.eq(1), EasyMock.eq(0), EasyMock.anyLong());
mAggListener.testStarted(EasyMock.eq(test1), EasyMock.anyLong());
mAggListener.testEnded(
EasyMock.eq(test1),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mAggListener.testRunEnded(450L, new HashMap<String, Metric>());
mAggListener.invocationEnded(500L);
mDetailedListener.invocationEnded(500L);
EasyMock.expect(
mLogger.saveLogData(
EasyMock.contains("aggregated-events"),
EasyMock.eq(LogDataType.TF_EVENTS),
EasyMock.anyObject()))
.andReturn(null);
EasyMock.expect(
mLogger.saveLogData(
EasyMock.contains("detailed-events"),
EasyMock.eq(LogDataType.TF_EVENTS),
EasyMock.anyObject()))
.andReturn(null);
EasyMock.replay(mAggListener, mDetailedListener, mLogger);
mAggregator =
new TestableResultAggregator(
Arrays.asList(mAggListener, mDetailedListener),
RetryStrategy.RETRY_ANY_FAILURE);
mAggregator.setLogSaver(mLogger);
mAggregator.invocationStarted(mInvocationContext);
// Attempt 1
mAggregator.testRunStarted("run1", 2, 0);
mAggregator.testStarted(test1);
mAggregator.testEnded(test1, new HashMap<String, Metric>());
mAggregator.testStarted(test2);
mAggregator.testFailed(test2, "I failed. retry me.");
mAggregator.testEnded(test2, new HashMap<String, Metric>());
mAggregator.testRunFailed("I failed");
mAggregator.testRunEnded(450L, new HashMap<String, Metric>());
// Attempt 2
mAggregator.testRunStarted("run1", 2, 1);
mAggregator.testStarted(test2);
mAggregator.testEnded(test2, new HashMap<String, Metric>());
mAggregator.testRunFailed("I failed 2");
mAggregator.testRunEnded(450L, new HashMap<String, Metric>());
// run 2
mAggregator.testRunStarted("run2", 1, 0);
mAggregator.testStarted(test1);
mAggregator.testEnded(test1, new HashMap<String, Metric>());
mAggregator.testRunEnded(450L, new HashMap<String, Metric>());
mAggregator.invocationEnded(500L);
EasyMock.verify(mAggListener, mDetailedListener, mLogger);
assertNull(mAggregator.getInvocationMetricRunError());
}
/** Test aggregation of results coming from a module first then from a simple test run. */
@Test
public void testForwarding_module_noModule() throws Exception {
TestDescription test1 = new TestDescription("classname", "test1");
TestDescription test2 = new TestDescription("classname", "test2");
EasyMock.expect(mDetailedListener.supportGranularResults()).andStubReturn(true);
// Invocation level
mAggListener.setLogSaver(mLogger);
mAggListener.invocationStarted(mInvocationContext);
EasyMock.expect(mAggListener.getSummary()).andStubReturn(null);
mDetailedListener.setLogSaver(mLogger);
mDetailedListener.invocationStarted(mInvocationContext);
EasyMock.expect(mDetailedListener.getSummary()).andStubReturn(null);
mAggListener.testModuleStarted(mModuleContext);
mDetailedListener.testModuleStarted(mModuleContext);
// Detailed receives the breakdown
mDetailedListener.testRunStarted(
EasyMock.eq("run1"), EasyMock.eq(2), EasyMock.eq(0), EasyMock.anyLong());
mDetailedListener.testStarted(EasyMock.eq(test1), EasyMock.anyLong());
mDetailedListener.testEnded(
EasyMock.eq(test1),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mDetailedListener.testStarted(EasyMock.eq(test2), EasyMock.anyLong());
mDetailedListener.testFailed(test2, "I failed. retry me.");
mDetailedListener.testEnded(
EasyMock.eq(test2),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mDetailedListener.testRunEnded(450L, new HashMap<String, Metric>());
mDetailedListener.testRunStarted(
EasyMock.eq("run1"), EasyMock.eq(2), EasyMock.eq(1), EasyMock.anyLong());
mDetailedListener.testStarted(EasyMock.eq(test2), EasyMock.anyLong());
mDetailedListener.testEnded(
EasyMock.eq(test2),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mDetailedListener.testRunEnded(450L, new HashMap<String, Metric>());
// Aggregated listeners receives the aggregated results
mAggListener.testRunStarted(
EasyMock.eq("run1"), EasyMock.eq(2), EasyMock.eq(0), EasyMock.anyLong());
mAggListener.testStarted(EasyMock.eq(test1), EasyMock.anyLong());
mAggListener.testEnded(
EasyMock.eq(test1),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mAggListener.testStarted(EasyMock.eq(test2), EasyMock.anyLong());
mAggListener.testEnded(
EasyMock.eq(test2),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mAggListener.testRunEnded(450L, new HashMap<String, Metric>());
mAggListener.testModuleEnded();
mDetailedListener.testModuleEnded();
// Detailed receives the breakdown for non-module
mDetailedListener.testRunStarted(
EasyMock.eq("run2"), EasyMock.eq(1), EasyMock.eq(0), EasyMock.anyLong());
mDetailedListener.testStarted(EasyMock.eq(test1), EasyMock.anyLong());
mDetailedListener.testFailed(test1, "I failed. retry me.");
mDetailedListener.testEnded(
EasyMock.eq(test1),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mDetailedListener.testRunEnded(450L, new HashMap<String, Metric>());
mDetailedListener.testRunStarted(
EasyMock.eq("run2"), EasyMock.eq(1), EasyMock.eq(1), EasyMock.anyLong());
mDetailedListener.testStarted(EasyMock.eq(test1), EasyMock.anyLong());
mDetailedListener.testEnded(
EasyMock.eq(test1),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mDetailedListener.testRunEnded(450L, new HashMap<String, Metric>());
// Aggregated listeners receives the aggregated results for non-module
mAggListener.testRunStarted(
EasyMock.eq("run2"), EasyMock.eq(1), EasyMock.eq(0), EasyMock.anyLong());
mAggListener.testStarted(EasyMock.eq(test1), EasyMock.anyLong());
mAggListener.testEnded(
EasyMock.eq(test1),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mAggListener.testRunEnded(900L, new HashMap<String, Metric>());
EasyMock.expect(
mLogger.saveLogData(
EasyMock.contains("aggregated-events"),
EasyMock.eq(LogDataType.TF_EVENTS),
EasyMock.anyObject()))
.andReturn(null);
EasyMock.expect(
mLogger.saveLogData(
EasyMock.contains("detailed-events"),
EasyMock.eq(LogDataType.TF_EVENTS),
EasyMock.anyObject()))
.andReturn(null);
mAggListener.invocationEnded(500L);
mDetailedListener.invocationEnded(500L);
EasyMock.replay(mAggListener, mDetailedListener, mLogger);
mAggregator =
new TestableResultAggregator(
Arrays.asList(mAggListener, mDetailedListener),
RetryStrategy.RETRY_ANY_FAILURE);
mAggregator.setLogSaver(mLogger);
mAggregator.invocationStarted(mInvocationContext);
mAggregator.testModuleStarted(mModuleContext);
// Attempt 1
mAggregator.testRunStarted("run1", 2, 0);
mAggregator.testStarted(test1);
mAggregator.testEnded(test1, new HashMap<String, Metric>());
mAggregator.testStarted(test2);
mAggregator.testFailed(test2, "I failed. retry me.");
mAggregator.testEnded(test2, new HashMap<String, Metric>());
mAggregator.testRunFailed("I failed");
mAggregator.testRunEnded(450L, new HashMap<String, Metric>());
// Attempt 2
mAggregator.testRunStarted("run1", 2, 1);
mAggregator.testStarted(test2);
mAggregator.testEnded(test2, new HashMap<String, Metric>());
mAggregator.testRunEnded(450L, new HashMap<String, Metric>());
mAggregator.testModuleEnded();
// New run that is not a module
mAggregator.testRunStarted("run2", 1, 0);
mAggregator.testStarted(test1);
mAggregator.testFailed(test1, "I failed. retry me.");
mAggregator.testEnded(test1, new HashMap<String, Metric>());
mAggregator.testRunEnded(450L, new HashMap<String, Metric>());
mAggregator.testRunStarted("run2", 1, 1);
mAggregator.testStarted(test1);
mAggregator.testEnded(test1, new HashMap<String, Metric>());
mAggregator.testRunEnded(450L, new HashMap<String, Metric>());
mAggregator.invocationEnded(500L);
EasyMock.verify(mAggListener, mDetailedListener, mLogger);
assertEquals("I failed", mAggregator.getInvocationMetricRunError());
}
/** Test aggregation of results coming from a simple test run first then from a module. */
@Test
public void testForwarding_noModule_module() throws Exception {
TestDescription test1 = new TestDescription("classname", "test1");
TestDescription test2 = new TestDescription("classname", "test2");
EasyMock.expect(mDetailedListener.supportGranularResults()).andStubReturn(true);
// Invocation level
mAggListener.setLogSaver(mLogger);
mAggListener.invocationStarted(mInvocationContext);
EasyMock.expect(mAggListener.getSummary()).andStubReturn(null);
mDetailedListener.setLogSaver(mLogger);
mDetailedListener.invocationStarted(mInvocationContext);
EasyMock.expect(mDetailedListener.getSummary()).andStubReturn(null);
// Detailed receives the breakdown for non-module
mDetailedListener.testRunStarted(
EasyMock.eq("run2"), EasyMock.eq(1), EasyMock.eq(0), EasyMock.anyLong());
mDetailedListener.testStarted(EasyMock.eq(test1), EasyMock.anyLong());
mDetailedListener.testFailed(test1, "I failed. retry me.");
mDetailedListener.testEnded(
EasyMock.eq(test1),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mDetailedListener.testRunEnded(450L, new HashMap<String, Metric>());
mDetailedListener.testRunStarted(
EasyMock.eq("run2"), EasyMock.eq(1), EasyMock.eq(1), EasyMock.anyLong());
mDetailedListener.testStarted(EasyMock.eq(test1), EasyMock.anyLong());
mDetailedListener.testEnded(
EasyMock.eq(test1),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mDetailedListener.testRunEnded(450L, new HashMap<String, Metric>());
// Aggregated listeners receives the aggregated results for non-module
mAggListener.testRunStarted(
EasyMock.eq("run2"), EasyMock.eq(1), EasyMock.eq(0), EasyMock.anyLong());
mAggListener.testStarted(EasyMock.eq(test1), EasyMock.anyLong());
mAggListener.testEnded(
EasyMock.eq(test1),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mAggListener.testRunEnded(900L, new HashMap<String, Metric>());
// Detailed receives the breakdown
mDetailedListener.testModuleStarted(mModuleContext);
mDetailedListener.testRunStarted(
EasyMock.eq("run1"), EasyMock.eq(2), EasyMock.eq(0), EasyMock.anyLong());
mDetailedListener.testStarted(EasyMock.eq(test1), EasyMock.anyLong());
mDetailedListener.testEnded(
EasyMock.eq(test1),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mDetailedListener.testStarted(EasyMock.eq(test2), EasyMock.anyLong());
mDetailedListener.testFailed(test2, "I failed. retry me.");
mDetailedListener.testEnded(
EasyMock.eq(test2),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mDetailedListener.testRunEnded(450L, new HashMap<String, Metric>());
mDetailedListener.testRunStarted(
EasyMock.eq("run1"), EasyMock.eq(2), EasyMock.eq(1), EasyMock.anyLong());
mDetailedListener.testStarted(EasyMock.eq(test2), EasyMock.anyLong());
mDetailedListener.testEnded(
EasyMock.eq(test2),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mDetailedListener.testRunEnded(450L, new HashMap<String, Metric>());
mDetailedListener.testModuleEnded();
// Aggregated listeners receives the aggregated results
mAggListener.testModuleStarted(mModuleContext);
mAggListener.testRunStarted(
EasyMock.eq("run1"), EasyMock.eq(2), EasyMock.eq(0), EasyMock.anyLong());
mAggListener.testStarted(EasyMock.eq(test1), EasyMock.anyLong());
mAggListener.testEnded(
EasyMock.eq(test1),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mAggListener.testStarted(EasyMock.eq(test2), EasyMock.anyLong());
mAggListener.testEnded(
EasyMock.eq(test2),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mAggListener.testRunEnded(450L, new HashMap<String, Metric>());
mAggListener.testModuleEnded();
mAggListener.invocationEnded(500L);
mDetailedListener.invocationEnded(500L);
EasyMock.expect(
mLogger.saveLogData(
EasyMock.contains("aggregated-events"),
EasyMock.eq(LogDataType.TF_EVENTS),
EasyMock.anyObject()))
.andReturn(null);
EasyMock.expect(
mLogger.saveLogData(
EasyMock.contains("detailed-events"),
EasyMock.eq(LogDataType.TF_EVENTS),
EasyMock.anyObject()))
.andReturn(null);
EasyMock.replay(mAggListener, mDetailedListener, mLogger);
mAggregator =
new TestableResultAggregator(
Arrays.asList(mAggListener, mDetailedListener),
RetryStrategy.RETRY_ANY_FAILURE);
mAggregator.setLogSaver(mLogger);
mAggregator.invocationStarted(mInvocationContext);
// First run that is not a module
mAggregator.testRunStarted("run2", 1, 0);
mAggregator.testStarted(test1);
mAggregator.testFailed(test1, "I failed. retry me.");
mAggregator.testEnded(test1, new HashMap<String, Metric>());
mAggregator.testRunEnded(450L, new HashMap<String, Metric>());
mAggregator.testRunStarted("run2", 1, 1);
mAggregator.testStarted(test1);
mAggregator.testEnded(test1, new HashMap<String, Metric>());
mAggregator.testRunEnded(450L, new HashMap<String, Metric>());
// Module start
mAggregator.testModuleStarted(mModuleContext);
// Attempt 1
mAggregator.testRunStarted("run1", 2, 0);
mAggregator.testStarted(test1);
mAggregator.testEnded(test1, new HashMap<String, Metric>());
mAggregator.testStarted(test2);
mAggregator.testFailed(test2, "I failed. retry me.");
mAggregator.testEnded(test2, new HashMap<String, Metric>());
mAggregator.testRunEnded(450L, new HashMap<String, Metric>());
// Attempt 2
mAggregator.testRunStarted("run1", 2, 1);
mAggregator.testStarted(test2);
mAggregator.testEnded(test2, new HashMap<String, Metric>());
mAggregator.testRunEnded(450L, new HashMap<String, Metric>());
mAggregator.testModuleEnded();
mAggregator.invocationEnded(500L);
EasyMock.verify(mAggListener, mDetailedListener, mLogger);
assertNull(mAggregator.getInvocationMetricRunError());
}
@Test
public void testForwarding_noModule_module_runFailure() throws Exception {
TestDescription test1 = new TestDescription("classname", "test1");
TestDescription test2 = new TestDescription("classname", "test2");
MultiFailureDescription aggFailure =
new MultiFailureDescription(
FailureDescription.create("I failed"),
FailureDescription.create("I failed 2"));
EasyMock.expect(mDetailedListener.supportGranularResults()).andStubReturn(true);
// Invocation level
mAggListener.setLogSaver(mLogger);
mAggListener.invocationStarted(mInvocationContext);
EasyMock.expect(mAggListener.getSummary()).andStubReturn(null);
mDetailedListener.setLogSaver(mLogger);
mDetailedListener.invocationStarted(mInvocationContext);
EasyMock.expect(mDetailedListener.getSummary()).andStubReturn(null);
// Detailed receives the breakdown for non-module
mDetailedListener.testRunStarted(
EasyMock.eq("run2"), EasyMock.eq(1), EasyMock.eq(0), EasyMock.anyLong());
mDetailedListener.testStarted(EasyMock.eq(test1), EasyMock.anyLong());
mDetailedListener.testFailed(test1, "I failed. retry me.");
mDetailedListener.testEnded(
EasyMock.eq(test1),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mDetailedListener.testRunEnded(450L, new HashMap<String, Metric>());
mDetailedListener.testRunStarted(
EasyMock.eq("run2"), EasyMock.eq(1), EasyMock.eq(1), EasyMock.anyLong());
mDetailedListener.testStarted(EasyMock.eq(test1), EasyMock.anyLong());
mDetailedListener.testEnded(
EasyMock.eq(test1),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mDetailedListener.testRunFailed(aggFailure);
mDetailedListener.testRunEnded(450L, new HashMap<String, Metric>());
// Aggregated listeners receives the aggregated results for non-module
mAggListener.testRunStarted(
EasyMock.eq("run2"), EasyMock.eq(1), EasyMock.eq(0), EasyMock.anyLong());
mAggListener.testStarted(EasyMock.eq(test1), EasyMock.anyLong());
mAggListener.testEnded(
EasyMock.eq(test1),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mAggListener.testRunFailed(aggFailure);
mAggListener.testRunEnded(900L, new HashMap<String, Metric>());
// Detailed receives the breakdown
mDetailedListener.testModuleStarted(mModuleContext);
mDetailedListener.testRunStarted(
EasyMock.eq("run1"), EasyMock.eq(2), EasyMock.eq(0), EasyMock.anyLong());
mDetailedListener.testStarted(EasyMock.eq(test1), EasyMock.anyLong());
mDetailedListener.testEnded(
EasyMock.eq(test1),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mDetailedListener.testStarted(EasyMock.eq(test2), EasyMock.anyLong());
mDetailedListener.testFailed(test2, "I failed. retry me.");
mDetailedListener.testEnded(
EasyMock.eq(test2),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mDetailedListener.testRunEnded(450L, new HashMap<String, Metric>());
mDetailedListener.testRunStarted(
EasyMock.eq("run1"), EasyMock.eq(2), EasyMock.eq(1), EasyMock.anyLong());
mDetailedListener.testStarted(EasyMock.eq(test2), EasyMock.anyLong());
mDetailedListener.testEnded(
EasyMock.eq(test2),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mDetailedListener.testRunEnded(450L, new HashMap<String, Metric>());
mDetailedListener.testModuleEnded();
// Aggregated listeners receives the aggregated results
mAggListener.testModuleStarted(mModuleContext);
mAggListener.testRunStarted(
EasyMock.eq("run1"), EasyMock.eq(2), EasyMock.eq(0), EasyMock.anyLong());
mAggListener.testStarted(EasyMock.eq(test1), EasyMock.anyLong());
mAggListener.testEnded(
EasyMock.eq(test1),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mAggListener.testStarted(EasyMock.eq(test2), EasyMock.anyLong());
mAggListener.testEnded(
EasyMock.eq(test2),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mAggListener.testRunEnded(450L, new HashMap<String, Metric>());
mAggListener.testModuleEnded();
mAggListener.invocationEnded(500L);
mDetailedListener.invocationEnded(500L);
EasyMock.expect(
mLogger.saveLogData(
EasyMock.contains("aggregated-events"),
EasyMock.eq(LogDataType.TF_EVENTS),
EasyMock.anyObject()))
.andReturn(null);
EasyMock.expect(
mLogger.saveLogData(
EasyMock.contains("detailed-events"),
EasyMock.eq(LogDataType.TF_EVENTS),
EasyMock.anyObject()))
.andReturn(null);
EasyMock.replay(mAggListener, mDetailedListener, mLogger);
mAggregator =
new TestableResultAggregator(
Arrays.asList(mAggListener, mDetailedListener),
RetryStrategy.RETRY_ANY_FAILURE);
mAggregator.setLogSaver(mLogger);
mAggregator.invocationStarted(mInvocationContext);
// First run that is not a module
mAggregator.testRunStarted("run2", 1, 0);
mAggregator.testStarted(test1);
mAggregator.testFailed(test1, "I failed. retry me.");
mAggregator.testEnded(test1, new HashMap<String, Metric>());
mAggregator.testRunFailed("I failed");
mAggregator.testRunEnded(450L, new HashMap<String, Metric>());
mAggregator.testRunStarted("run2", 1, 1);
mAggregator.testStarted(test1);
mAggregator.testEnded(test1, new HashMap<String, Metric>());
mAggregator.testRunFailed("I failed 2");
mAggregator.testRunEnded(450L, new HashMap<String, Metric>());
// Module start
mAggregator.testModuleStarted(mModuleContext);
// Attempt 1
mAggregator.testRunStarted("run1", 2, 0);
mAggregator.testStarted(test1);
mAggregator.testEnded(test1, new HashMap<String, Metric>());
mAggregator.testStarted(test2);
mAggregator.testFailed(test2, "I failed. retry me.");
mAggregator.testEnded(test2, new HashMap<String, Metric>());
mAggregator.testRunEnded(450L, new HashMap<String, Metric>());
// Attempt 2
mAggregator.testRunStarted("run1", 2, 1);
mAggregator.testStarted(test2);
mAggregator.testEnded(test2, new HashMap<String, Metric>());
mAggregator.testRunEnded(450L, new HashMap<String, Metric>());
mAggregator.testModuleEnded();
mAggregator.invocationEnded(500L);
EasyMock.verify(mAggListener, mDetailedListener, mLogger);
assertNull(mAggregator.getInvocationMetricRunError());
}
/** Test when two modules follow each others. */
@Test
public void testForwarding_module_module() throws Exception {
TestDescription test1 = new TestDescription("classname", "test1");
TestDescription test2 = new TestDescription("classname", "test2");
LogFile moduleLog = new LogFile("module-log", "url", LogDataType.TEXT);
EasyMock.expect(mDetailedListener.supportGranularResults()).andStubReturn(true);
// Invocation level
mAggListener.setLogSaver(mLogger);
mAggListener.invocationStarted(mInvocationContext);
EasyMock.expect(mAggListener.getSummary()).andStubReturn(null);
mDetailedListener.setLogSaver(mLogger);
mDetailedListener.invocationStarted(mInvocationContext);
EasyMock.expect(mDetailedListener.getSummary()).andStubReturn(null);
mAggListener.testModuleStarted(mModuleContext);
mDetailedListener.testModuleStarted(mModuleContext);
// Detailed receives the breakdown
mDetailedListener.testRunStarted(
EasyMock.eq("run1"), EasyMock.eq(2), EasyMock.eq(0), EasyMock.anyLong());
mDetailedListener.testStarted(EasyMock.eq(test1), EasyMock.anyLong());
mDetailedListener.testEnded(
EasyMock.eq(test1),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mDetailedListener.testStarted(EasyMock.eq(test2), EasyMock.anyLong());
mDetailedListener.testFailed(test2, "I failed. retry me.");
mDetailedListener.testEnded(
EasyMock.eq(test2),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mDetailedListener.testRunEnded(450L, new HashMap<String, Metric>());
mDetailedListener.testRunStarted(
EasyMock.eq("run1"), EasyMock.eq(2), EasyMock.eq(1), EasyMock.anyLong());
mDetailedListener.testStarted(EasyMock.eq(test2), EasyMock.anyLong());
mDetailedListener.testEnded(
EasyMock.eq(test2),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mDetailedListener.testRunEnded(450L, new HashMap<String, Metric>());
mDetailedListener.logAssociation("module-log", moduleLog);
mDetailedListener.testModuleEnded();
// Aggregated listeners receives the aggregated results
mAggListener.testRunStarted(
EasyMock.eq("run1"), EasyMock.eq(2), EasyMock.eq(0), EasyMock.anyLong());
mAggListener.testStarted(EasyMock.eq(test1), EasyMock.anyLong());
mAggListener.testEnded(
EasyMock.eq(test1),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mAggListener.testStarted(EasyMock.eq(test2), EasyMock.anyLong());
mAggListener.testEnded(
EasyMock.eq(test2),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mAggListener.testRunEnded(450L, new HashMap<String, Metric>());
mAggListener.logAssociation("module-log", moduleLog);
mAggListener.testModuleEnded();
// second module
mAggListener.testModuleStarted(mModuleContext);
mDetailedListener.testModuleStarted(mModuleContext);
mDetailedListener.testRunStarted(
EasyMock.eq("run2"), EasyMock.eq(1), EasyMock.eq(0), EasyMock.anyLong());
mDetailedListener.testStarted(EasyMock.eq(test1), EasyMock.anyLong());
mDetailedListener.testFailed(test1, "I failed. retry me.");
mDetailedListener.testEnded(
EasyMock.eq(test1),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mDetailedListener.testRunEnded(450L, new HashMap<String, Metric>());
mDetailedListener.testRunStarted(
EasyMock.eq("run2"), EasyMock.eq(1), EasyMock.eq(1), EasyMock.anyLong());
mDetailedListener.testStarted(EasyMock.eq(test1), EasyMock.anyLong());
mDetailedListener.testEnded(
EasyMock.eq(test1),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mDetailedListener.testRunEnded(450L, new HashMap<String, Metric>());
// Aggregated listeners receives the aggregated results for non-module
mAggListener.testRunStarted(
EasyMock.eq("run2"), EasyMock.eq(1), EasyMock.eq(0), EasyMock.anyLong());
mAggListener.testStarted(EasyMock.eq(test1), EasyMock.anyLong());
mAggListener.testEnded(
EasyMock.eq(test1),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mAggListener.testRunEnded(450L, new HashMap<String, Metric>());
mAggListener.testModuleEnded();
mDetailedListener.testModuleEnded();
mAggListener.invocationEnded(500L);
mDetailedListener.invocationEnded(500L);
EasyMock.expect(
mLogger.saveLogData(
EasyMock.contains("aggregated-events"),
EasyMock.eq(LogDataType.TF_EVENTS),
EasyMock.anyObject()))
.andReturn(null);
EasyMock.expect(
mLogger.saveLogData(
EasyMock.contains("detailed-events"),
EasyMock.eq(LogDataType.TF_EVENTS),
EasyMock.anyObject()))
.andReturn(null);
EasyMock.replay(mAggListener, mDetailedListener, mLogger);
mAggregator =
new TestableResultAggregator(
Arrays.asList(mAggListener, mDetailedListener),
RetryStrategy.RETRY_ANY_FAILURE);
mAggregator.setLogSaver(mLogger);
mAggregator.invocationStarted(mInvocationContext);
// Module 1 starts
mAggregator.testModuleStarted(mModuleContext);
// Attempt 1
mAggregator.testRunStarted("run1", 2, 0);
mAggregator.testStarted(test1);
mAggregator.testEnded(test1, new HashMap<String, Metric>());
mAggregator.testStarted(test2);
mAggregator.testFailed(test2, "I failed. retry me.");
mAggregator.testEnded(test2, new HashMap<String, Metric>());
mAggregator.testRunEnded(450L, new HashMap<String, Metric>());
// Attempt 2
mAggregator.testRunStarted("run1", 2, 1);
mAggregator.testStarted(test2);
mAggregator.testEnded(test2, new HashMap<String, Metric>());
mAggregator.testRunEnded(450L, new HashMap<String, Metric>());
mAggregator.logAssociation("module-log", moduleLog);
mAggregator.testModuleEnded();
// Module 2 starts
mAggregator.testModuleStarted(mModuleContext);
mAggregator.testRunStarted("run2", 1, 0);
mAggregator.testStarted(test1);
mAggregator.testFailed(test1, "I failed. retry me.");
mAggregator.testEnded(test1, new HashMap<String, Metric>());
mAggregator.testRunEnded(450L, new HashMap<String, Metric>());
mAggregator.testRunStarted("run2", 1, 1);
mAggregator.testStarted(test1);
mAggregator.testEnded(test1, new HashMap<String, Metric>());
mAggregator.testRunEnded(450L, new HashMap<String, Metric>());
mAggregator.testModuleEnded();
mAggregator.invocationEnded(500L);
EasyMock.verify(mAggListener, mDetailedListener, mLogger);
assertNull(mAggregator.getInvocationMetricRunError());
}
@Test
public void testForwarding_module_pass_fail_fail() throws Exception {
TestDescription test1 = new TestDescription("classname", "test1");
TestDescription test2 = new TestDescription("classname", "test2");
EasyMock.expect(mDetailedListener.supportGranularResults()).andStubReturn(true);
// Invocation level
mAggListener.setLogSaver(mLogger);
mAggListener.invocationStarted(mInvocationContext);
EasyMock.expect(mAggListener.getSummary()).andStubReturn(null);
mDetailedListener.setLogSaver(mLogger);
mDetailedListener.invocationStarted(mInvocationContext);
EasyMock.expect(mDetailedListener.getSummary()).andStubReturn(null);
mAggListener.testModuleStarted(mModuleContext);
mDetailedListener.testModuleStarted(mModuleContext);
// Detailed receives the breakdown
mDetailedListener.testRunStarted(
EasyMock.eq("run1"), EasyMock.eq(2), EasyMock.eq(0), EasyMock.anyLong());
mDetailedListener.testStarted(EasyMock.eq(test1), EasyMock.anyLong());
mDetailedListener.testEnded(
EasyMock.eq(test1),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mDetailedListener.testStarted(EasyMock.eq(test2), EasyMock.anyLong());
mDetailedListener.testFailed(test2, "I failed. retry me.");
mDetailedListener.testEnded(
EasyMock.eq(test2),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mDetailedListener.testRunEnded(450L, new HashMap<String, Metric>());
mDetailedListener.testRunStarted(
EasyMock.eq("run1"), EasyMock.eq(2), EasyMock.eq(1), EasyMock.anyLong());
mDetailedListener.testStarted(EasyMock.eq(test2), EasyMock.anyLong());
mDetailedListener.testEnded(
EasyMock.eq(test2),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mDetailedListener.testRunEnded(450L, new HashMap<String, Metric>());
mDetailedListener.testRunStarted(
EasyMock.eq("run1"), EasyMock.eq(2), EasyMock.eq(2), EasyMock.anyLong());
mDetailedListener.testStarted(EasyMock.eq(test2), EasyMock.anyLong());
mDetailedListener.testEnded(
EasyMock.eq(test2),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mDetailedListener.testRunEnded(450L, new HashMap<String, Metric>());
// Aggregated listeners receives the aggregated results
mAggListener.testRunStarted(
EasyMock.eq("run1"), EasyMock.eq(2), EasyMock.eq(0), EasyMock.anyLong());
mAggListener.testStarted(EasyMock.eq(test1), EasyMock.anyLong());
mAggListener.testEnded(
EasyMock.eq(test1),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mAggListener.testStarted(EasyMock.eq(test2), EasyMock.anyLong());
mAggListener.testEnded(
EasyMock.eq(test2),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mAggListener.testRunEnded(450L, new HashMap<String, Metric>());
mAggListener.testModuleEnded();
mDetailedListener.testModuleEnded();
mAggListener.invocationEnded(500L);
mDetailedListener.invocationEnded(500L);
EasyMock.expect(
mLogger.saveLogData(
EasyMock.contains("aggregated-events"),
EasyMock.eq(LogDataType.TF_EVENTS),
EasyMock.anyObject()))
.andReturn(null);
EasyMock.expect(
mLogger.saveLogData(
EasyMock.contains("detailed-events"),
EasyMock.eq(LogDataType.TF_EVENTS),
EasyMock.anyObject()))
.andReturn(null);
EasyMock.replay(mAggListener, mDetailedListener, mLogger);
mAggregator =
new TestableResultAggregator(
Arrays.asList(mAggListener, mDetailedListener),
RetryStrategy.RETRY_ANY_FAILURE);
mAggregator.setLogSaver(mLogger);
mAggregator.invocationStarted(mInvocationContext);
// Module 1 starts
mAggregator.testModuleStarted(mModuleContext);
// Attempt 1
mAggregator.testRunStarted("run1", 2, 0);
mAggregator.testStarted(test1);
mAggregator.testEnded(test1, new HashMap<String, Metric>());
mAggregator.testStarted(test2);
mAggregator.testFailed(test2, "I failed. retry me.");
mAggregator.testEnded(test2, new HashMap<String, Metric>());
mAggregator.testRunEnded(450L, new HashMap<String, Metric>());
// Attempt 2
mAggregator.testRunStarted("run1", 2, 1);
mAggregator.testStarted(test2);
mAggregator.testEnded(test2, new HashMap<String, Metric>());
mAggregator.testRunFailed("failed2");
mAggregator.testRunEnded(450L, new HashMap<String, Metric>());
// Attempt 3
mAggregator.testRunStarted("run1", 2, 2);
mAggregator.testStarted(test2);
mAggregator.testEnded(test2, new HashMap<String, Metric>());
mAggregator.testRunFailed("failed3");
mAggregator.testRunEnded(450L, new HashMap<String, Metric>());
mAggregator.testModuleEnded();
mAggregator.invocationEnded(500L);
EasyMock.verify(mAggListener, mDetailedListener, mLogger);
assertEquals(
"There were 2 failures:\n failed2\n failed3",
mAggregator.getInvocationMetricRunError());
}
/**
* Ensure that we handle the aggregation properly when a single IRemoteTest generates several
* testRuns and we retry them all (due to no filter support). In this case, the retry attempt
* will not be right after the original attempt.
*/
@Test
public void testForwarding_noModules_runFailures_unordered() throws Exception {
TestDescription test1 = new TestDescription("classname", "test1");
TestDescription test2 = new TestDescription("classname", "test2");
EasyMock.expect(mDetailedListener.supportGranularResults()).andStubReturn(true);
// Invocation level
mAggListener.setLogSaver(mLogger);
mAggListener.invocationStarted(mInvocationContext);
EasyMock.expect(mAggListener.getSummary()).andStubReturn(null);
mDetailedListener.setLogSaver(mLogger);
mDetailedListener.invocationStarted(mInvocationContext);
EasyMock.expect(mDetailedListener.getSummary()).andStubReturn(null);
// Detailed receives the breakdown
mDetailedListener.testRunStarted(
EasyMock.eq("run1"), EasyMock.eq(2), EasyMock.eq(0), EasyMock.anyLong());
mDetailedListener.testStarted(EasyMock.eq(test1), EasyMock.anyLong());
mDetailedListener.testEnded(
EasyMock.eq(test1),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mDetailedListener.testStarted(EasyMock.eq(test2), EasyMock.anyLong());
mDetailedListener.testFailed(test2, "I failed. retry me.");
mDetailedListener.testEnded(
EasyMock.eq(test2),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
// TODO: Fix detailed reporting that should clear this failure.
mDetailedListener.testRunFailed(FailureDescription.create("I failed"));
mDetailedListener.testRunEnded(450L, new HashMap<String, Metric>());
mDetailedListener.testRunStarted(
EasyMock.eq("run2"), EasyMock.eq(1), EasyMock.eq(0), EasyMock.anyLong());
mDetailedListener.testStarted(EasyMock.eq(test1), EasyMock.anyLong());
mDetailedListener.testEnded(
EasyMock.eq(test1),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mDetailedListener.testRunEnded(450L, new HashMap<String, Metric>());
mDetailedListener.testRunStarted(
EasyMock.eq("run1"), EasyMock.eq(2), EasyMock.eq(1), EasyMock.anyLong());
mDetailedListener.testStarted(EasyMock.eq(test2), EasyMock.anyLong());
mDetailedListener.testEnded(
EasyMock.eq(test2),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mDetailedListener.testRunEnded(450L, new HashMap<String, Metric>());
// Aggregated listeners receives the aggregated results
mAggListener.testRunStarted(
EasyMock.eq("run1"), EasyMock.eq(2), EasyMock.eq(0), EasyMock.anyLong());
mAggListener.testStarted(EasyMock.eq(test1), EasyMock.anyLong());
mAggListener.testEnded(
EasyMock.eq(test1),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mAggListener.testStarted(EasyMock.eq(test2), EasyMock.anyLong());
mAggListener.testEnded(
EasyMock.eq(test2),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mAggListener.testRunEnded(900L, new HashMap<String, Metric>());
mAggListener.testRunStarted(
EasyMock.eq("run2"), EasyMock.eq(1), EasyMock.eq(0), EasyMock.anyLong());
mAggListener.testStarted(EasyMock.eq(test1), EasyMock.anyLong());
mAggListener.testEnded(
EasyMock.eq(test1),
EasyMock.anyLong(),
EasyMock.<HashMap<String, Metric>>anyObject());
mAggListener.testRunEnded(450L, new HashMap<String, Metric>());
mAggListener.invocationEnded(500L);
mDetailedListener.invocationEnded(500L);
EasyMock.expect(
mLogger.saveLogData(
EasyMock.contains("aggregated-events"),
EasyMock.eq(LogDataType.TF_EVENTS),
EasyMock.anyObject()))
.andReturn(null);
EasyMock.expect(
mLogger.saveLogData(
EasyMock.contains("detailed-events"),
EasyMock.eq(LogDataType.TF_EVENTS),
EasyMock.anyObject()))
.andReturn(null);
EasyMock.replay(mAggListener, mDetailedListener, mLogger);
mAggregator =
new TestableResultAggregator(
Arrays.asList(mAggListener, mDetailedListener),
RetryStrategy.RETRY_ANY_FAILURE);
mAggregator.setLogSaver(mLogger);
mAggregator.invocationStarted(mInvocationContext);
// Run 1 - Attempt 1
mAggregator.testRunStarted("run1", 2, 0);
mAggregator.testStarted(test1);
mAggregator.testEnded(test1, new HashMap<String, Metric>());
mAggregator.testStarted(test2);
mAggregator.testFailed(test2, "I failed. retry me.");
mAggregator.testEnded(test2, new HashMap<String, Metric>());
mAggregator.testRunFailed("I failed");
mAggregator.testRunEnded(450L, new HashMap<String, Metric>());
// Run 2 - Attempt 1
mAggregator.testRunStarted("run2", 1, 0);
mAggregator.testStarted(test1);
mAggregator.testEnded(test1, new HashMap<String, Metric>());
mAggregator.testRunEnded(450L, new HashMap<String, Metric>());
// Run 1 - Attempt 2
mAggregator.testRunStarted("run1", 2, 1);
mAggregator.testStarted(test2);
mAggregator.testEnded(test2, new HashMap<String, Metric>());
mAggregator.testRunEnded(450L, new HashMap<String, Metric>());
mAggregator.invocationEnded(500L);
EasyMock.verify(mAggListener, mDetailedListener, mLogger);
assertNull(mAggregator.getInvocationMetricRunError());
}
}