Mobly: report test events to listener in real-time
Also:
* Added complete support for include/exclude filters.
Bug: 270963463
Test: None
Change-Id: Ic93fece249b447c5a3c35de3ff37c59ca5d703cc
diff --git a/javatests/com/android/tradefed/testtype/mobly/MoblyBinaryHostTestTest.java b/javatests/com/android/tradefed/testtype/mobly/MoblyBinaryHostTestTest.java
index 311a1a7..96bf479 100644
--- a/javatests/com/android/tradefed/testtype/mobly/MoblyBinaryHostTestTest.java
+++ b/javatests/com/android/tradefed/testtype/mobly/MoblyBinaryHostTestTest.java
@@ -20,12 +20,12 @@
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull;
-import static org.junit.Assert.fail;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.contains;
import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.inOrder;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
@@ -53,9 +53,8 @@
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
+import org.mockito.InOrder;
import org.mockito.Mockito;
-import org.mockito.invocation.InvocationOnMock;
-import org.mockito.stubbing.Answer;
import java.io.ByteArrayInputStream;
import java.io.File;
@@ -135,21 +134,23 @@
// Mimics the behavior of a successful test run.
Mockito.when(mMockRunUtil.runTimedCmd(anyLong(), any()))
.thenAnswer(
- new Answer<CommandResult>() {
- @Override
- public CommandResult answer(InvocationOnMock invocation)
- throws Throwable {
- FileUtils.createFile(testResult, "");
- FileUtils.createFile(
- new File(mSpyTest.getLogDirAbsolutePath(), "log"),
- "log content");
- return new CommandResult(CommandStatus.SUCCESS);
- }
+ invocation -> {
+ CommandResult res = new CommandResult(CommandStatus.SUCCESS);
+ res.setStdout("test_foo");
+ return res;
+ })
+ .thenAnswer(
+ invocation -> {
+ FileUtils.createFile(testResult, "");
+ FileUtils.createFile(
+ new File(mSpyTest.getLogDirAbsolutePath(), "log"),
+ "log content");
+ return new CommandResult(CommandStatus.SUCCESS);
});
mSpyTest.run(mTestInfo, Mockito.mock(ITestInvocationListener.class));
- verify(mSpyTest.getRunUtil()).runTimedCmd(anyLong(), any());
+ verify(mSpyTest.getRunUtil(), times(2)).runTimedCmd(anyLong(), any());
assertNull(mSpyTest.getLogDirFile());
}
@@ -174,21 +175,23 @@
File testResult = new File(mSpyTest.getLogDirAbsolutePath(), TEST_RESULT_FILE_NAME);
Mockito.when(mMockRunUtil.runTimedCmd(anyLong(), any()))
.thenAnswer(
- new Answer<CommandResult>() {
- @Override
- public CommandResult answer(InvocationOnMock invocation)
- throws Throwable {
- FileUtils.createFile(testResult, "");
- FileUtils.createFile(
- new File(mSpyTest.getLogDirAbsolutePath(), "log"),
- "log content");
- return new CommandResult(CommandStatus.SUCCESS);
- }
+ invocation -> {
+ CommandResult res = new CommandResult(CommandStatus.SUCCESS);
+ res.setStdout("test_foo");
+ return res;
+ })
+ .thenAnswer(
+ invocation -> {
+ FileUtils.createFile(testResult, "");
+ FileUtils.createFile(
+ new File(mSpyTest.getLogDirAbsolutePath(), "log"),
+ "log content");
+ return new CommandResult(CommandStatus.SUCCESS);
});
mSpyTest.run(mTestInfo, Mockito.mock(ITestInvocationListener.class));
- verify(mSpyTest.getRunUtil()).runTimedCmd(anyLong(), any());
+ verify(mSpyTest.getRunUtil(), times(2)).runTimedCmd(anyLong(), any());
assertNull(mSpyTest.getLogDirFile());
}
@@ -216,20 +219,23 @@
setter.setOptionValue("mobly-std-log", "true");
File testResult = new File(mSpyTest.getLogDirAbsolutePath(), TEST_RESULT_FILE_NAME);
// Mimics the behavior of a successful test run.
+ Mockito.when(mMockRunUtil.runTimedCmd(anyLong(), any()))
+ .thenAnswer(
+ invocation -> {
+ CommandResult res = new CommandResult(CommandStatus.SUCCESS);
+ res.setStdout("test_foo");
+ return res;
+ });
Mockito.when(
mMockRunUtil.runTimedCmd(
anyLong(), any(OutputStream.class), any(OutputStream.class), any()))
.thenAnswer(
- new Answer<CommandResult>() {
- @Override
- public CommandResult answer(InvocationOnMock invocation)
- throws Throwable {
- FileUtils.createFile(testResult, "");
- FileUtils.createFile(
- new File(mSpyTest.getLogDirAbsolutePath(), "log"),
- "log content");
- return new CommandResult(CommandStatus.SUCCESS);
- }
+ invocation -> {
+ FileUtils.createFile(testResult, "");
+ FileUtils.createFile(
+ new File(mSpyTest.getLogDirAbsolutePath(), "log"),
+ "log content");
+ return new CommandResult(CommandStatus.SUCCESS);
});
mSpyTest.run(mTestInfo, Mockito.mock(ITestInvocationListener.class));
@@ -239,26 +245,22 @@
public void testRun_testResultIsMissing() throws Exception {
OptionSetter setter = new OptionSetter(mSpyTest);
setter.setOptionValue("mobly-binaries", mMoblyBinary.getAbsolutePath());
+ File testResult = new File(mSpyTest.getLogDirAbsolutePath(), TEST_RESULT_FILE_NAME);
// Test result and log files were not created for some reasons during test run.
Mockito.when(mMockRunUtil.runTimedCmd(anyLong(), any()))
.thenAnswer(
- new Answer<CommandResult>() {
- @Override
- public CommandResult answer(InvocationOnMock invocation)
- throws Throwable {
- return new CommandResult(CommandStatus.SUCCESS);
- }
+ invocation -> {
+ CommandResult res = new CommandResult(CommandStatus.SUCCESS);
+ res.setStdout("test_foo");
+ return res;
+ })
+ .thenAnswer(
+ invocation -> {
+ return new CommandResult(CommandStatus.SUCCESS);
});
- try {
- mSpyTest.run(mTestInfo, Mockito.mock(ITestInvocationListener.class));
- fail("Should have thrown an exception");
- } catch (RuntimeException e) {
- assertThat(e)
- .hasMessageThat()
- .contains("Fail to find test summary file test_summary.yaml under directory");
- assertNull(mSpyTest.getLogDirFile());
- }
+ mSpyTest.run(mTestInfo, Mockito.mock(ITestInvocationListener.class));
+ assertFalse(testResult.exists());
}
@Test
@@ -267,25 +269,30 @@
OptionSetter setter = new OptionSetter(mSpyTest);
setter.setOptionValue("mobly-binaries", mMoblyBinary.getAbsolutePath());
File testResult = new File(mSpyTest.getLogDirAbsolutePath(), TEST_RESULT_FILE_NAME);
- Mockito.when(
- mMockRunUtil.runTimedCmd(
- anyLong(),
- anyString(),
- eq("--"),
- contains("--config="),
- contains("--device_serial="),
- contains("--log_path=")))
+ Mockito.when(mMockRunUtil.runTimedCmd(anyLong(), any()))
.thenAnswer(
- new Answer<CommandResult>() {
- @Override
- public CommandResult answer(InvocationOnMock invocation)
- throws Throwable {
- FileUtils.createFile(testResult, "");
- FileUtils.createFile(
- new File(mSpyTest.getLogDirAbsolutePath(), "log"),
- "log content");
- return new CommandResult(CommandStatus.SUCCESS);
- }
+ invocation -> {
+ CommandResult result = new CommandResult(CommandStatus.SUCCESS);
+ result.setStdout(
+ "Name: pip\nLocation: "
+ + new File(
+ mVenvDir.getAbsolutePath(),
+ "lib/python3.8/site-packages"));
+ return result;
+ })
+ .thenAnswer(
+ invocation -> {
+ CommandResult res = new CommandResult(CommandStatus.SUCCESS);
+ res.setStdout("test_foo");
+ return res;
+ })
+ .thenAnswer(
+ invocation -> {
+ FileUtils.createFile(testResult, "");
+ FileUtils.createFile(
+ new File(mSpyTest.getLogDirAbsolutePath(), "log"),
+ "log content");
+ return new CommandResult(CommandStatus.SUCCESS);
});
CommandResult result = new CommandResult(CommandStatus.SUCCESS);
result.setStdout(
@@ -296,6 +303,7 @@
mSpyTest.run(mTestInfo, Mockito.mock(ITestInvocationListener.class));
+ verify(mSpyTest.getRunUtil(), times(3)).runTimedCmd(anyLong(), any());
verify(mSpyTest.getRunUtil(), times(1))
.setEnvVariable(eq("VIRTUAL_ENV"), eq(mVenvDir.getAbsolutePath()));
assertFalse(mVenvDir.exists());
@@ -309,16 +317,18 @@
File testResult = new File(mSpyTest.getLogDirAbsolutePath(), TEST_RESULT_FILE_NAME);
Mockito.when(mMockRunUtil.runTimedCmd(anyLong(), any()))
.thenAnswer(
- new Answer<CommandResult>() {
- @Override
- public CommandResult answer(InvocationOnMock invocation)
- throws Throwable {
- FileUtils.createFile(testResult, "");
- FileUtils.createFile(
- new File(mSpyTest.getLogDirAbsolutePath(), "log"),
- "log content");
- return new CommandResult(CommandStatus.SUCCESS);
- }
+ invocation -> {
+ CommandResult res = new CommandResult(CommandStatus.SUCCESS);
+ res.setStdout("test_foo");
+ return res;
+ })
+ .thenAnswer(
+ invocation -> {
+ FileUtils.createFile(testResult, "");
+ FileUtils.createFile(
+ new File(mSpyTest.getLogDirAbsolutePath(), "log"),
+ "log content");
+ return new CommandResult(CommandStatus.SUCCESS);
});
mSpyTest.run(mTestInfo, Mockito.mock(ITestInvocationListener.class));
@@ -358,31 +368,29 @@
.when(mTestInfo)
.getDependencyFile(eq(mMoblyBinary2.getName()), eq(false));
File testResult = new File(mSpyTest.getLogDirAbsolutePath(), TEST_RESULT_FILE_NAME);
- Mockito.when(
- mMockRunUtil.runTimedCmd(
- anyLong(),
- any(),
- eq("--"),
- contains("--config="),
- contains("--device_serial="),
- contains("--log_path=")))
+ Mockito.when(mMockRunUtil.runTimedCmd(anyLong(), any()))
.thenAnswer(
- new Answer<CommandResult>() {
- @Override
- public CommandResult answer(InvocationOnMock invocation)
- throws Throwable {
- FileUtils.createFile(testResult, "");
- FileUtils.createFile(
- new File(mSpyTest.getLogDirAbsolutePath(), "log"),
- "log content");
- return new CommandResult(CommandStatus.SUCCESS);
- }
+ invocation -> {
+ CommandResult res = new CommandResult(CommandStatus.SUCCESS);
+ res.setStdout("test_foo");
+ return res;
+ })
+ .thenAnswer(
+ invocation -> {
+ FileUtils.createFile(testResult, "");
+ FileUtils.createFile(
+ new File(mSpyTest.getLogDirAbsolutePath(), "log"),
+ "log content");
+ return new CommandResult(CommandStatus.SUCCESS);
});
mSpyTest.run(mTestInfo, Mockito.mock(ITestInvocationListener.class));
// Verify the command line contains "--config"
- verify(mSpyTest.getRunUtil(), times(1))
+ InOrder inOrder = inOrder(mSpyTest.getRunUtil());
+ inOrder.verify(mSpyTest.getRunUtil())
+ .runTimedCmd(anyLong(), any(), eq("--"), eq("--list_tests"));
+ inOrder.verify(mSpyTest.getRunUtil())
.runTimedCmd(
anyLong(),
any(),
@@ -393,6 +401,387 @@
}
@Test
+ public void testRun_withoutTests() throws Exception {
+ Mockito.doNothing().when(mSpyTest).reportLogs(any(), any());
+ OptionSetter setter = new OptionSetter(mSpyTest);
+ setter.setOptionValue("mobly-par-file-name", mMoblyBinary2.getName());
+ Mockito.doReturn(mMoblyTestDir)
+ .when(mTestInfo)
+ .getDependencyFile(eq(mMoblyBinary2.getName()), eq(false));
+ File testResult = new File(mSpyTest.getLogDirAbsolutePath(), TEST_RESULT_FILE_NAME);
+ Mockito.when(mMockRunUtil.runTimedCmd(anyLong(), any()))
+ .thenAnswer(
+ invocation -> {
+ CommandResult res = new CommandResult(CommandStatus.SUCCESS);
+ res.setStdout(""); // No tests.
+ return res;
+ })
+ .thenAnswer(
+ invocation -> {
+ FileUtils.createFile(testResult, "");
+ FileUtils.createFile(
+ new File(mSpyTest.getLogDirAbsolutePath(), "log"),
+ "log content");
+ return new CommandResult(CommandStatus.SUCCESS);
+ });
+
+ mSpyTest.run(mTestInfo, Mockito.mock(ITestInvocationListener.class));
+
+ // Verify no tests where run.
+ verify(mSpyTest.getRunUtil()).runTimedCmd(anyLong(), any(), eq("--"), eq("--list_tests"));
+ }
+
+ @Test
+ public void testRun_withoutFilters() throws Exception {
+ Mockito.doNothing().when(mSpyTest).reportLogs(any(), any());
+ OptionSetter setter = new OptionSetter(mSpyTest);
+ setter.setOptionValue("mobly-par-file-name", mMoblyBinary2.getName());
+ Mockito.doReturn(mMoblyTestDir)
+ .when(mTestInfo)
+ .getDependencyFile(eq(mMoblyBinary2.getName()), eq(false));
+ File testResult = new File(mSpyTest.getLogDirAbsolutePath(), TEST_RESULT_FILE_NAME);
+ Mockito.when(mMockRunUtil.runTimedCmd(anyLong(), any()))
+ .thenAnswer(
+ invocation -> {
+ CommandResult res = new CommandResult(CommandStatus.SUCCESS);
+ res.setStdout("test_foo");
+ return res;
+ })
+ .thenAnswer(
+ invocation -> {
+ FileUtils.createFile(testResult, "");
+ FileUtils.createFile(
+ new File(mSpyTest.getLogDirAbsolutePath(), "log"),
+ "log content");
+ return new CommandResult(CommandStatus.SUCCESS);
+ });
+
+ mSpyTest.run(mTestInfo, Mockito.mock(ITestInvocationListener.class));
+
+ // Verify the command line contains "--tests"
+ InOrder inOrder = inOrder(mSpyTest.getRunUtil());
+ inOrder.verify(mSpyTest.getRunUtil())
+ .runTimedCmd(anyLong(), any(), eq("--"), eq("--list_tests"));
+ inOrder.verify(mSpyTest.getRunUtil())
+ .runTimedCmd(
+ anyLong(),
+ any(),
+ eq("--"),
+ contains("--config="),
+ contains("--device_serial="),
+ contains("--log_path="));
+ }
+
+ @Test
+ public void testRun_withInvalidIncludeFilters() throws Exception {
+ Mockito.doNothing().when(mSpyTest).reportLogs(any(), any());
+ OptionSetter setter = new OptionSetter(mSpyTest);
+ setter.setOptionValue("mobly-binaries", mMoblyBinary.getAbsolutePath());
+ File testResult = new File(mSpyTest.getLogDirAbsolutePath(), TEST_RESULT_FILE_NAME);
+ mSpyTest.addIncludeFilter("test_bar");
+ Mockito.when(mMockRunUtil.runTimedCmd(anyLong(), any()))
+ .thenAnswer(
+ invocation -> {
+ CommandResult res = new CommandResult(CommandStatus.SUCCESS);
+ res.setStdout("test_foo");
+ return res;
+ })
+ .thenAnswer(
+ invocation -> {
+ FileUtils.createFile(testResult, "");
+ FileUtils.createFile(
+ new File(mSpyTest.getLogDirAbsolutePath(), "log"),
+ "log content");
+ return new CommandResult(CommandStatus.SUCCESS);
+ });
+
+ ITestInvocationListener mockListener = Mockito.mock(ITestInvocationListener.class);
+
+ mSpyTest.run(mTestInfo, mockListener);
+
+ verify(mockListener, times(1)).testRunStarted(anyString(), eq(0));
+ verify(mockListener, times(1)).testRunFailed(any(FailureDescription.class));
+ verify(mockListener, times(1)).testRunEnded(eq(0L), eq(new HashMap<String, Metric>()));
+ }
+
+ @Test
+ public void testRun_withInvalidExcludeFilters() throws Exception {
+ Mockito.doNothing().when(mSpyTest).reportLogs(any(), any());
+ OptionSetter setter = new OptionSetter(mSpyTest);
+ setter.setOptionValue("mobly-binaries", mMoblyBinary.getAbsolutePath());
+ File testResult = new File(mSpyTest.getLogDirAbsolutePath(), TEST_RESULT_FILE_NAME);
+ mSpyTest.addExcludeFilter("test_bar");
+ Mockito.when(mMockRunUtil.runTimedCmd(anyLong(), any()))
+ .thenAnswer(
+ invocation -> {
+ CommandResult res = new CommandResult(CommandStatus.SUCCESS);
+ res.setStdout("test_foo");
+ return res;
+ })
+ .thenAnswer(
+ invocation -> {
+ FileUtils.createFile(testResult, "");
+ FileUtils.createFile(
+ new File(mSpyTest.getLogDirAbsolutePath(), "log"),
+ "log content");
+ return new CommandResult(CommandStatus.SUCCESS);
+ });
+
+ ITestInvocationListener mockListener = Mockito.mock(ITestInvocationListener.class);
+
+ mSpyTest.run(mTestInfo, mockListener);
+
+ verify(mockListener, times(1)).testRunStarted(anyString(), eq(0));
+ }
+
+ @Test
+ public void testRun_withInvalidExcludeFiltersPrefix() throws Exception {
+ Mockito.doNothing().when(mSpyTest).reportLogs(any(), any());
+ OptionSetter setter = new OptionSetter(mSpyTest);
+ setter.setOptionValue("mobly-binaries", mMoblyBinary.getAbsolutePath());
+ File testResult = new File(mSpyTest.getLogDirAbsolutePath(), TEST_RESULT_FILE_NAME);
+ mSpyTest.addExcludeFilter("test_f");
+ Mockito.when(mMockRunUtil.runTimedCmd(anyLong(), any()))
+ .thenAnswer(
+ invocation -> {
+ CommandResult res = new CommandResult(CommandStatus.SUCCESS);
+ res.setStdout("test_foo");
+ return res;
+ })
+ .thenAnswer(
+ invocation -> {
+ FileUtils.createFile(testResult, "");
+ FileUtils.createFile(
+ new File(mSpyTest.getLogDirAbsolutePath(), "log"),
+ "log content");
+ return new CommandResult(CommandStatus.SUCCESS);
+ });
+
+ ITestInvocationListener mockListener = Mockito.mock(ITestInvocationListener.class);
+
+ mSpyTest.run(mTestInfo, mockListener);
+
+ verify(mockListener, times(1)).testRunStarted(anyString(), eq(0));
+ }
+
+ @Test
+ public void testRun_withIncludeFiltersExact() throws Exception {
+ Mockito.doNothing().when(mSpyTest).reportLogs(any(), any());
+ OptionSetter setter = new OptionSetter(mSpyTest);
+ setter.setOptionValue("mobly-binaries", mMoblyBinary.getAbsolutePath());
+ File testResult = new File(mSpyTest.getLogDirAbsolutePath(), TEST_RESULT_FILE_NAME);
+ mSpyTest.addIncludeFilter("test_bar");
+ mSpyTest.addIncludeFilter("test_foo");
+ Mockito.when(mMockRunUtil.runTimedCmd(anyLong(), any()))
+ .thenAnswer(
+ invocation -> {
+ CommandResult res = new CommandResult(CommandStatus.SUCCESS);
+ res.setStdout("test_foo\ntest_baz\ntest_bar");
+ return res;
+ })
+ .thenAnswer(
+ invocation -> {
+ FileUtils.createFile(testResult, "");
+ FileUtils.createFile(
+ new File(mSpyTest.getLogDirAbsolutePath(), "log"),
+ "log content");
+ return new CommandResult(CommandStatus.SUCCESS);
+ });
+
+ ITestInvocationListener mockListener = Mockito.mock(ITestInvocationListener.class);
+
+ mSpyTest.run(mTestInfo, mockListener);
+
+ verify(mockListener, times(1)).testRunStarted(anyString(), eq(2));
+
+ // Verify the command line contains "--tests"
+ InOrder inOrder = inOrder(mSpyTest.getRunUtil());
+ inOrder.verify(mSpyTest.getRunUtil())
+ .runTimedCmd(anyLong(), any(), eq("--"), eq("--list_tests"));
+ inOrder.verify(mSpyTest.getRunUtil())
+ .runTimedCmd(
+ anyLong(),
+ any(),
+ eq("--"),
+ contains("--config="),
+ contains("--device_serial="),
+ contains("--log_path="),
+ eq("--tests"),
+ eq("test_foo"),
+ eq("test_bar"));
+ }
+
+ @Test
+ public void testRun_withPrefixIncludeFiltersPrefix() throws Exception {
+ Mockito.doNothing().when(mSpyTest).reportLogs(any(), any());
+ OptionSetter setter = new OptionSetter(mSpyTest);
+ setter.setOptionValue("mobly-binaries", mMoblyBinary.getAbsolutePath());
+ File testResult = new File(mSpyTest.getLogDirAbsolutePath(), TEST_RESULT_FILE_NAME);
+ mSpyTest.addIncludeFilter("test_b");
+ Mockito.when(mMockRunUtil.runTimedCmd(anyLong(), any()))
+ .thenAnswer(
+ invocation -> {
+ CommandResult res = new CommandResult(CommandStatus.SUCCESS);
+ res.setStdout("test_foo\ntest_baz\ntest_bar");
+ return res;
+ })
+ .thenAnswer(
+ invocation -> {
+ FileUtils.createFile(testResult, "");
+ FileUtils.createFile(
+ new File(mSpyTest.getLogDirAbsolutePath(), "log"),
+ "log content");
+ return new CommandResult(CommandStatus.SUCCESS);
+ });
+
+ ITestInvocationListener mockListener = Mockito.mock(ITestInvocationListener.class);
+
+ mSpyTest.run(mTestInfo, mockListener);
+
+ verify(mockListener, times(1)).testRunStarted(anyString(), eq(2));
+
+ // Verify the command line contains "--tests"
+ InOrder inOrder = inOrder(mSpyTest.getRunUtil());
+ inOrder.verify(mSpyTest.getRunUtil())
+ .runTimedCmd(anyLong(), any(), eq("--"), eq("--list_tests"));
+ inOrder.verify(mSpyTest.getRunUtil())
+ .runTimedCmd(
+ anyLong(),
+ any(),
+ eq("--"),
+ contains("--config="),
+ contains("--device_serial="),
+ contains("--log_path="),
+ eq("--tests"),
+ eq("test_baz"),
+ eq("test_bar"));
+ }
+
+ @Test
+ public void testRun_withExcludeFiltersExact() throws Exception {
+ Mockito.doNothing().when(mSpyTest).reportLogs(any(), any());
+ OptionSetter setter = new OptionSetter(mSpyTest);
+ setter.setOptionValue("mobly-binaries", mMoblyBinary.getAbsolutePath());
+ File testResult = new File(mSpyTest.getLogDirAbsolutePath(), TEST_RESULT_FILE_NAME);
+ mSpyTest.addExcludeFilter("test_bar");
+ mSpyTest.addExcludeFilter("test_foo");
+ Mockito.when(mMockRunUtil.runTimedCmd(anyLong(), any()))
+ .thenAnswer(
+ invocation -> {
+ CommandResult res = new CommandResult(CommandStatus.SUCCESS);
+ res.setStdout("test_foo\ntest_baz\ntest_bar");
+ return res;
+ })
+ .thenAnswer(
+ invocation -> {
+ FileUtils.createFile(testResult, "");
+ FileUtils.createFile(
+ new File(mSpyTest.getLogDirAbsolutePath(), "log"),
+ "log content");
+ return new CommandResult(CommandStatus.SUCCESS);
+ });
+
+ ITestInvocationListener mockListener = Mockito.mock(ITestInvocationListener.class);
+
+ mSpyTest.run(mTestInfo, mockListener);
+
+ verify(mockListener, times(1)).testRunStarted(anyString(), eq(1));
+
+ // Verify the command line contains "--tests"
+ InOrder inOrder = inOrder(mSpyTest.getRunUtil());
+ inOrder.verify(mSpyTest.getRunUtil())
+ .runTimedCmd(anyLong(), any(), eq("--"), eq("--list_tests"));
+ inOrder.verify(mSpyTest.getRunUtil())
+ .runTimedCmd(
+ anyLong(),
+ any(),
+ eq("--"),
+ contains("--config="),
+ contains("--device_serial="),
+ contains("--log_path="),
+ eq("--tests"),
+ eq("test_baz"));
+ }
+
+ @Test
+ public void testRun_withExcludeFiltersNoTests() throws Exception {
+ Mockito.doNothing().when(mSpyTest).reportLogs(any(), any());
+ OptionSetter setter = new OptionSetter(mSpyTest);
+ setter.setOptionValue("mobly-binaries", mMoblyBinary.getAbsolutePath());
+ File testResult = new File(mSpyTest.getLogDirAbsolutePath(), TEST_RESULT_FILE_NAME);
+ mSpyTest.addExcludeFilter("test_bar");
+ mSpyTest.addExcludeFilter("test_baz");
+ mSpyTest.addExcludeFilter("test_foo");
+ Mockito.when(mMockRunUtil.runTimedCmd(anyLong(), any()))
+ .thenAnswer(
+ invocation -> {
+ CommandResult res = new CommandResult(CommandStatus.SUCCESS);
+ res.setStdout("test_foo\ntest_baz\ntest_bar");
+ return res;
+ })
+ .thenAnswer(
+ invocation -> {
+ FileUtils.createFile(testResult, "");
+ FileUtils.createFile(
+ new File(mSpyTest.getLogDirAbsolutePath(), "log"),
+ "log content");
+ return new CommandResult(CommandStatus.SUCCESS);
+ });
+
+ ITestInvocationListener mockListener = Mockito.mock(ITestInvocationListener.class);
+
+ mSpyTest.run(mTestInfo, mockListener);
+
+ verify(mockListener, times(1)).testRunStarted(anyString(), eq(0));
+ }
+
+ @Test
+ public void testRun_withBothIncludeAndExcludeFilters() throws Exception {
+ Mockito.doNothing().when(mSpyTest).reportLogs(any(), any());
+ OptionSetter setter = new OptionSetter(mSpyTest);
+ setter.setOptionValue("mobly-binaries", mMoblyBinary.getAbsolutePath());
+ File testResult = new File(mSpyTest.getLogDirAbsolutePath(), TEST_RESULT_FILE_NAME);
+ mSpyTest.addIncludeFilter("test_b");
+ mSpyTest.addExcludeFilter("test_bar");
+ Mockito.when(mMockRunUtil.runTimedCmd(anyLong(), any()))
+ .thenAnswer(
+ invocation -> {
+ CommandResult res = new CommandResult(CommandStatus.SUCCESS);
+ res.setStdout("test_foo\ntest_baz\ntest_bar");
+ return res;
+ })
+ .thenAnswer(
+ invocation -> {
+ FileUtils.createFile(testResult, "");
+ FileUtils.createFile(
+ new File(mSpyTest.getLogDirAbsolutePath(), "log"),
+ "log content");
+ return new CommandResult(CommandStatus.SUCCESS);
+ });
+
+ ITestInvocationListener mockListener = Mockito.mock(ITestInvocationListener.class);
+
+ mSpyTest.run(mTestInfo, mockListener);
+
+ verify(mockListener, times(1)).testRunStarted(anyString(), eq(1));
+
+ // Verify the command line contains "--tests"
+ InOrder inOrder = inOrder(mSpyTest.getRunUtil());
+ inOrder.verify(mSpyTest.getRunUtil())
+ .runTimedCmd(anyLong(), any(), eq("--"), eq("--list_tests"));
+ inOrder.verify(mSpyTest.getRunUtil())
+ .runTimedCmd(
+ anyLong(),
+ any(),
+ eq("--"),
+ contains("--config="),
+ contains("--device_serial="),
+ contains("--log_path="),
+ eq("--tests"),
+ eq("test_baz"));
+ }
+
+ @Test
public void testBuildCommandLineArrayWithConfig() throws Exception {
Mockito.doNothing().when(mSpyTest).reportLogs(any(), any());
Mockito.doReturn(DEVICE_SERIAL).when(mMockDevice).getSerialNumber();
diff --git a/javatests/com/android/tradefed/testtype/mobly/MoblyYamlResultParserTest.java b/javatests/com/android/tradefed/testtype/mobly/MoblyYamlResultParserTest.java
index 9d87fff..daa2108 100644
--- a/javatests/com/android/tradefed/testtype/mobly/MoblyYamlResultParserTest.java
+++ b/javatests/com/android/tradefed/testtype/mobly/MoblyYamlResultParserTest.java
@@ -24,7 +24,6 @@
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.verifyNoMoreInteractions;
import com.android.tradefed.result.FailureDescription;
import com.android.tradefed.result.ITestInvocationListener;
@@ -34,8 +33,6 @@
import com.android.tradefed.testtype.mobly.MoblyYamlResultRecordHandler.Record;
import com.android.tradefed.testtype.mobly.MoblyYamlResultSummaryHandler.Summary;
import com.android.tradefed.testtype.mobly.MoblyYamlResultUserDataHandler.UserData;
-import com.android.tradefed.util.FileUtil;
-import com.android.tradefed.util.ResourceUtil;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
@@ -45,12 +42,9 @@
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
import org.mockito.ArgumentCaptor;
-import org.mockito.InOrder;
import org.mockito.Mockito;
import java.io.ByteArrayInputStream;
-import java.io.File;
-import java.io.FileInputStream;
import java.io.InputStream;
import java.util.HashMap;
import java.util.Iterator;
@@ -63,7 +57,7 @@
private static final String DEFAULT_BEGIN_TIME = "1571681517464";
private static final String DEFAULT_END_TIME = "1571681520407";
private static final String DEFAULT_TEST_CLASS = "DefaultTestClass";
- private static final String DEFAULT_TEST_NAME = "default_test_name";
+ private static final String DEFAULT_TEST_NAME = "test_default_name";
private static final String TEST_ERROR_YAML = "/testtype/test_summary_error.yaml";
private static final String SAMPLE_STACK_TRACE =
"\"Traceback (most recent call last):\\n"
@@ -137,16 +131,12 @@
private MoblyYamlResultParser mParser;
private ITestInvocationListener mMockListener;
private ImmutableList<ITestInvocationListener> mListeners;
- private String mRunName;
- private ArgumentCaptor<String> mRunNameCaptor;
- private ArgumentCaptor<Integer> mCountCaptor;
private ArgumentCaptor<TestDescription> mStartedDescCaptor;
private ArgumentCaptor<Long> mBeginTimeCaptor;
private ArgumentCaptor<TestDescription> mFailedDescCaptor;
private ArgumentCaptor<FailureDescription> mFailureDescriptionCaptor;
private ArgumentCaptor<TestDescription> mEndDescCaptor;
private ArgumentCaptor<Long> mEndTimeCaptor;
- private ArgumentCaptor<Long> mElapseTimeCaptor;
@Before
public void setUp() throws Exception {
@@ -156,12 +146,6 @@
}
private void setUpArgumentCaptors() {
- // Setup testRunStarted
- mRunNameCaptor = ArgumentCaptor.forClass(String.class);
- mCountCaptor = ArgumentCaptor.forClass(Integer.class);
- Mockito.doNothing()
- .when(mMockListener)
- .testRunStarted(mRunNameCaptor.capture(), mCountCaptor.capture());
// Setup testStarted
mStartedDescCaptor = ArgumentCaptor.forClass(TestDescription.class);
mBeginTimeCaptor = ArgumentCaptor.forClass(Long.class);
@@ -180,17 +164,11 @@
Mockito.doNothing()
.when(mMockListener)
.testEnded(mEndDescCaptor.capture(), mEndTimeCaptor.capture(), any(Map.class));
- // Setup testRunEnded
- mElapseTimeCaptor = ArgumentCaptor.forClass(Long.class);
- Mockito.doNothing()
- .when(mMockListener)
- .testRunEnded(mElapseTimeCaptor.capture(), any(Map.class));
}
@Test
public void testReportToListenersPassRecord() {
- mRunName = new Object() {}.getClass().getEnclosingMethod().getName();
- mParser = new MoblyYamlResultParser(mMockListener, mRunName);
+ mParser = new MoblyYamlResultParser(mMockListener);
IMoblyYamlResultHandler.ITestResult passRecord =
new Record.Builder()
.setTestName(DEFAULT_TEST_NAME)
@@ -202,8 +180,6 @@
List<IMoblyYamlResultHandler.ITestResult> resultCache = ImmutableList.of(passRecord);
mParser.reportToListeners(mListeners, resultCache);
- assertEquals(mRunName, mRunNameCaptor.getValue());
- assertEquals(0, (int) mCountCaptor.getValue());
assertEquals(DEFAULT_TEST_CLASS, mStartedDescCaptor.getValue().getClassName());
assertEquals(DEFAULT_TEST_NAME, mStartedDescCaptor.getValue().getTestName());
assertEquals(Long.parseLong(DEFAULT_BEGIN_TIME), (long) mBeginTimeCaptor.getValue());
@@ -211,15 +187,11 @@
assertEquals(DEFAULT_TEST_CLASS, mEndDescCaptor.getValue().getClassName());
assertEquals(DEFAULT_TEST_NAME, mEndDescCaptor.getValue().getTestName());
assertEquals(Long.parseLong(DEFAULT_END_TIME), (long) mEndTimeCaptor.getValue());
- assertEquals(
- Long.parseLong(DEFAULT_END_TIME) - Long.parseLong(DEFAULT_BEGIN_TIME),
- (long) mElapseTimeCaptor.getValue());
}
@Test
public void testReportToListenersFailRecord() {
- mRunName = new Object() {}.getClass().getEnclosingMethod().getName();
- mParser = new MoblyYamlResultParser(mMockListener, mRunName);
+ mParser = new MoblyYamlResultParser(mMockListener);
IMoblyYamlResultHandler.ITestResult failRecord =
new Record.Builder()
.setTestName(DEFAULT_TEST_NAME)
@@ -232,8 +204,6 @@
List<IMoblyYamlResultHandler.ITestResult> resultCache = ImmutableList.of(failRecord);
mParser.reportToListeners(mListeners, resultCache);
- assertEquals(mRunName, mRunNameCaptor.getValue());
- assertEquals(0, (int) mCountCaptor.getValue());
assertEquals(DEFAULT_TEST_CLASS, mStartedDescCaptor.getValue().getClassName());
assertEquals(DEFAULT_TEST_NAME, mStartedDescCaptor.getValue().getTestName());
assertEquals(Long.parseLong(DEFAULT_BEGIN_TIME), (long) mBeginTimeCaptor.getValue());
@@ -252,46 +222,34 @@
assertEquals(DEFAULT_TEST_CLASS, mEndDescCaptor.getValue().getClassName());
assertEquals(DEFAULT_TEST_NAME, mEndDescCaptor.getValue().getTestName());
assertEquals(Long.parseLong(DEFAULT_END_TIME), (long) mEndTimeCaptor.getValue());
- assertEquals(
- Long.parseLong(DEFAULT_END_TIME) - Long.parseLong(DEFAULT_BEGIN_TIME),
- (long) mElapseTimeCaptor.getValue());
}
@Test
public void testReportToListenersUserData() {
- mRunName = new Object() {}.getClass().getEnclosingMethod().getName();
- mParser = new MoblyYamlResultParser(mMockListener, mRunName);
+ mParser = new MoblyYamlResultParser(mMockListener);
List<IMoblyYamlResultHandler.ITestResult> resultCache =
ImmutableList.of(new UserData.Builder().setTimestamp(DEFAULT_BEGIN_TIME).build());
mParser.reportToListeners(mListeners, resultCache);
- assertEquals(mRunName, mRunNameCaptor.getValue());
- assertEquals(0, (int) mCountCaptor.getValue());
verify(mMockListener, never()).testStarted(any(), anyLong());
verify(mMockListener, never()).testFailed(any(), anyString());
- assertEquals(0L - Long.parseLong(DEFAULT_BEGIN_TIME), (long) mElapseTimeCaptor.getValue());
}
@Test
public void testReportToListenersControllerInfo() {
- mRunName = new Object() {}.getClass().getEnclosingMethod().getName();
- mParser = new MoblyYamlResultParser(mMockListener, mRunName);
+ mParser = new MoblyYamlResultParser(mMockListener);
List<IMoblyYamlResultHandler.ITestResult> resultCache =
ImmutableList.of(
new ControllerInfo.Builder().setTimestamp("1571681322.791003").build());
mParser.reportToListeners(mListeners, resultCache);
- assertEquals(mRunName, mRunNameCaptor.getValue());
- assertEquals(0, (int) mCountCaptor.getValue());
verify(mMockListener, never()).testStarted(any(), anyLong());
verify(mMockListener, never()).testFailed(any(), anyString());
- assertEquals(1571681322791L, (long) mElapseTimeCaptor.getValue());
}
@Test
public void testParseDocumentMapRecordPass() throws Exception {
- mRunName = new Object() {}.getClass().getEnclosingMethod().getName();
- mParser = new MoblyYamlResultParser(mMockListener, mRunName);
+ mParser = new MoblyYamlResultParser(mMockListener);
Map<String, Object> detailMap = new HashMap<>();
detailMap.put("Result", "PASS");
detailMap.put("Stacktrace", "null");
@@ -307,8 +265,7 @@
@Test
public void testParseDocumentMapRecordFail() throws Exception {
- mRunName = new Object() {}.getClass().getEnclosingMethod().getName();
- mParser = new MoblyYamlResultParser(mMockListener, mRunName);
+ mParser = new MoblyYamlResultParser(mMockListener);
Map<String, Object> detailMap = new HashMap<>();
detailMap.put("Stacktrace", SAMPLE_STACK_TRACE);
detailMap.put("Result", "FAIL");
@@ -325,8 +282,7 @@
@Test
public void testParseDocumentMapSummary() throws Exception {
- mRunName = new Object() {}.getClass().getEnclosingMethod().getName();
- mParser = new MoblyYamlResultParser(mMockListener, mRunName);
+ mParser = new MoblyYamlResultParser(mMockListener);
Map<String, Object> docMap = new HashMap<>();
docMap.put("Type", "Summary");
docMap.put("Executed", "10");
@@ -339,8 +295,7 @@
@Test
public void testParseDocumentMapControllerInfo() throws Exception {
- mRunName = new Object() {}.getClass().getEnclosingMethod().getName();
- mParser = new MoblyYamlResultParser(mMockListener, mRunName);
+ mParser = new MoblyYamlResultParser(mMockListener);
Map<String, Object> docMap = new HashMap<>();
docMap.put("Type", "ControllerInfo");
docMap.put("Timestamp", "1571681322.791003");
@@ -352,8 +307,7 @@
@Test
public void testParseDocumentMapUserData() throws Exception {
- mRunName = new Object() {}.getClass().getEnclosingMethod().getName();
- mParser = new MoblyYamlResultParser(mMockListener, mRunName);
+ mParser = new MoblyYamlResultParser(mMockListener);
Map<String, Object> docMap = new HashMap<>();
docMap.put("Type", "UserData");
docMap.put("timestamp", DEFAULT_BEGIN_TIME);
@@ -364,8 +318,7 @@
@Test
public void testParseDocumentMapTestNameList() throws Exception {
- mRunName = new Object() {}.getClass().getEnclosingMethod().getName();
- mParser = new MoblyYamlResultParser(mMockListener, mRunName);
+ mParser = new MoblyYamlResultParser(mMockListener);
Map<String, Object> docMap = new HashMap<>();
docMap.put("Type", "TestNameList");
IMoblyYamlResultHandler.ITestResult result = mParser.parseDocumentMap(docMap);
@@ -374,8 +327,7 @@
@Test
public void testParse() throws Exception {
- mRunName = new Object() {}.getClass().getEnclosingMethod().getName();
- mParser = new MoblyYamlResultParser(mMockListener, mRunName);
+ mParser = new MoblyYamlResultParser(mMockListener);
MoblyYamlResultParser spyParser = Mockito.spy(mParser);
String passRecord = buildTestRecordString(new HashMap<>());
@@ -387,41 +339,22 @@
StringBuilder strBuilder = new StringBuilder();
strBuilder.append("---\n");
strBuilder.append(TESTNAME_LIST);
- strBuilder.append("\n---\n");
+ strBuilder.append("\n...\n---\n");
strBuilder.append(USER_DATA);
- strBuilder.append("\n---\n");
+ strBuilder.append("\n...\n---\n");
strBuilder.append(passRecord);
- strBuilder.append("\n---\n");
+ strBuilder.append("\n...\n---\n");
strBuilder.append(failRecord);
- strBuilder.append("\n---\n");
+ strBuilder.append("\n...\n---\n");
strBuilder.append(CONTROLLER_INFO);
- strBuilder.append("\n--- ");
+ strBuilder.append("\n...\n---\n");
strBuilder.append(SUMMARY);
+ strBuilder.append("\n...\n");
InputStream inputStream = new ByteArrayInputStream(strBuilder.toString().getBytes());
spyParser.parse(inputStream);
verify(spyParser, times(6)).parseDocumentMap(any(Map.class));
- verify(spyParser, times(1)).reportToListeners(any(), any());
- }
-
- @Test
- public void testParseError() throws Exception {
- mRunName = new Object() {}.getClass().getEnclosingMethod().getName();
- mParser = new MoblyYamlResultParser(mMockListener, mRunName);
- File testErrorYaml = FileUtil.createTempFile("test_summary", "yaml");
- ResourceUtil.extractResourceAsFile(TEST_ERROR_YAML, testErrorYaml);
- try (FileInputStream inputStream = new FileInputStream(testErrorYaml)) {
- mParser.parse(inputStream);
- } finally {
- FileUtil.deleteFile(testErrorYaml);
- }
- InOrder inOrder = Mockito.inOrder(mMockListener);
- inOrder.verify(mMockListener).testRunStarted(mRunName, 1);
- inOrder.verify(mMockListener).testRunFailed((FailureDescription) any());
- inOrder.verify(mMockListener).testRunEnded(anyLong(), any(Map.class));
-
- inOrder.verifyNoMoreInteractions();
- verifyNoMoreInteractions(mMockListener);
+ verify(spyParser, times(6)).reportToListeners(any(), any());
}
private ImmutableMap<String, Object> buildTestRecordDocMap(Map<String, Object> propertyMap) {
diff --git a/test_framework/com/android/tradefed/testtype/mobly/MoblyBinaryHostTest.java b/test_framework/com/android/tradefed/testtype/mobly/MoblyBinaryHostTest.java
index abeb2b4..eb1dbd7 100644
--- a/test_framework/com/android/tradefed/testtype/mobly/MoblyBinaryHostTest.java
+++ b/test_framework/com/android/tradefed/testtype/mobly/MoblyBinaryHostTest.java
@@ -56,12 +56,19 @@
import java.io.InputStream;
import java.io.Writer;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
/** Host test meant to run a mobly python binary file from the Android Build system (Soong) */
@OptionClass(alias = "mobly-host")
@@ -217,8 +224,7 @@
}
parFile.setExecutable(true);
try {
- runSingleParFile(parFile.getAbsolutePath(), listener);
- processTestResults(listener, parFile.getName());
+ runSingleParFile(parFile.getAbsolutePath(), parFile.getName(), listener);
} finally {
reportLogs(getLogDir(), listener);
}
@@ -247,7 +253,8 @@
return files;
}
- private void runSingleParFile(String parFilePath, ITestInvocationListener listener) {
+ private void runSingleParFile(
+ String parFilePath, String runName, ITestInvocationListener listener) {
if (mInjectAndroidSerialVar) {
getRunUtil().setEnvVariable(ANDROID_SERIAL_VAR, getDevice().getSerialNumber());
}
@@ -263,60 +270,145 @@
configPath = updateTemplateConfigFile(configFile, mWildcardConfig);
} catch (FileNotFoundException e) {
reportFailure(
- listener,
- mConfigFileName,
- "Couldn't find Mobly config file " + mConfigFileName);
+ listener, runName, "Couldn't find Mobly config file " + mConfigFileName);
+ return;
}
}
- CommandResult result;
- if (isStdLogging()) {
- result =
- getRunUtil()
- .runTimedCmd(
- getTestTimeout(),
- System.out,
- System.err,
- buildCommandLineArray(parFilePath, configPath));
- } else {
- result =
- getRunUtil()
- .runTimedCmd(
- getTestTimeout(),
- buildCommandLineArray(parFilePath, configPath));
+ CommandResult list_result =
+ getRunUtil().runTimedCmd(6000, parFilePath, "--", "--list_tests");
+ if (!CommandStatus.SUCCESS.equals(list_result.getStatus())) {
+ String message;
+ if (CommandStatus.TIMED_OUT.equals(list_result.getStatus())) {
+ message = "Unable to list tests from the python binary: Timed out";
+ } else {
+ message =
+ "Unable to list tests from the python binary\nstdout: "
+ + list_result.getStdout()
+ + "\nstderr: "
+ + list_result.getStderr();
+ }
+ reportFailure(listener, runName, message);
+ return;
}
- if (!CommandStatus.SUCCESS.equals(result.getStatus())) {
- CLog.e(
- "Something went wrong when running the python binary:\nstdout: "
- + "%s\nstderr:%s\nStatus:%s",
- result.getStdout(), result.getStderr(), result.getStatus());
+ // Compute all tests.
+ final String[] all_tests =
+ Arrays.stream(list_result.getStdout().split(System.lineSeparator()))
+ .filter(line -> !line.startsWith("==========>"))
+ .toArray(String[]::new);
+ Stream<String> includedTests = Arrays.stream(all_tests);
+ // Process include filters.
+ String[] includeFilters =
+ getIncludeFilters().stream()
+ .map(filter -> filter.replace("#", "."))
+ .toArray(String[]::new);
+ if (includeFilters.length > 0) {
+ String invalidIncludeFilters =
+ Arrays.stream(includeFilters)
+ .filter(
+ filter ->
+ !Arrays.stream(all_tests)
+ .anyMatch(test -> test.startsWith(filter)))
+ .collect(Collectors.joining(", "));
+ if (!invalidIncludeFilters.isEmpty()) {
+ reportFailure(
+ listener,
+ runName,
+ "Invalid include filters: [" + invalidIncludeFilters + "]");
+ return;
+ }
+ includedTests =
+ includedTests.filter(
+ test ->
+ Arrays.stream(includeFilters)
+ .anyMatch(filter -> test.startsWith(filter)));
}
- }
-
- private void processTestResults(ITestInvocationListener listener, String runName)
- throws HarnessRuntimeException {
- // Convert yaml test summary to xml.
- File yamlSummaryFile = FileUtil.findFile(getLogDir(), MOBLY_TEST_SUMMARY);
- if (yamlSummaryFile == null) {
- throw new HarnessRuntimeException(
- String.format(
- "Fail to find test summary file %s under directory %s",
- MOBLY_TEST_SUMMARY, getLogDir()),
- TestErrorIdentifier.UNEXPECTED_MOBLY_BEHAVIOR);
+ // Process exclude filters.
+ String[] excludeFilters =
+ getExcludeFilters().stream()
+ .map(filter -> filter.replace("#", "."))
+ .toArray(String[]::new);
+ if (excludeFilters.length > 0) {
+ String invalidExcludeFilters =
+ Arrays.stream(excludeFilters)
+ .filter(
+ filter ->
+ !Arrays.stream(all_tests)
+ .anyMatch(test -> test.equals(filter)))
+ .collect(Collectors.joining(", "));
+ if (!invalidExcludeFilters.isEmpty()) {
+ reportFailure(
+ listener,
+ runName,
+ "Invalid exclude filters: [" + invalidExcludeFilters + "]");
+ return;
+ }
+ includedTests =
+ includedTests.filter(
+ test ->
+ !Arrays.stream(excludeFilters)
+ .anyMatch(filter -> test.equals(filter)));
}
-
- MoblyYamlResultParser parser = new MoblyYamlResultParser(listener, runName);
+ // Collect final filtered tests list.
+ Set<String> tests = includedTests.collect(Collectors.toSet());
+ // Start run.
+ long startTime = System.currentTimeMillis();
+ listener.testRunStarted(runName, tests.size());
+ // No test to run, abort early.
+ if (tests.isEmpty()) {
+ listener.testRunEnded(0, new HashMap<String, String>());
+ return;
+ }
+ // Do not pass tests to command line if all included.
+ if (tests.size() == all_tests.length) {
+ tests.clear();
+ }
+ String[] command = buildCommandLineArray(parFilePath, configPath, tests);
+ ExecutorService executor = Executors.newSingleThreadExecutor();
+ CompletableFuture<CommandResult> future =
+ CompletableFuture.supplyAsync(
+ () -> {
+ if (isStdLogging()) {
+ return getRunUtil()
+ .runTimedCmd(
+ getTestTimeout(), System.out, System.err, command);
+ }
+ return getRunUtil().runTimedCmd(getTestTimeout(), command);
+ },
+ executor);
+ MoblyYamlResultParser parser = new MoblyYamlResultParser(listener);
+ File yamlSummaryFile = null;
InputStream inputStream = null;
- try {
- inputStream = new FileInputStream(yamlSummaryFile);
- processYamlTestResults(inputStream, parser, listener, runName);
- } catch (FileNotFoundException ex) {
- reportFailure(
- listener,
- runName,
- "Fail processing test results, result file not found.\n" + ex);
- } finally {
+ boolean runFailed = false;
+ while (!future.isDone() && yamlSummaryFile == null) {
+ yamlSummaryFile = FileUtil.findFile(getLogDir(), MOBLY_TEST_SUMMARY);
+ if (yamlSummaryFile != null) {
+ try {
+ inputStream = new FileInputStream(yamlSummaryFile);
+ } catch (FileNotFoundException ex) {
+ listener.testRunFailed(ex.toString());
+ runFailed = true;
+ }
+ }
+ }
+ if (inputStream != null) {
+ while (!future.isDone()) processYamlTestResults(inputStream, parser, listener, runName);
+ if (!processYamlTestResults(inputStream, parser, listener, runName))
+ CLog.e("Did not get a complete summary file from python binary.");
+ runFailed = parser.getRunFailed();
StreamUtil.close(inputStream);
}
+ try {
+ CommandResult result = future.get();
+ if (!CommandStatus.SUCCESS.equals(result.getStatus()) && !runFailed)
+ listener.testRunFailed(result.getStderr());
+ } catch (InterruptedException ex) {
+ listener.testRunFailed(ex.toString());
+ } catch (ExecutionException ex) {
+ listener.testRunFailed(ex.toString());
+ }
+ executor.shutdownNow();
+ listener.testRunEnded(
+ System.currentTimeMillis() - startTime, new HashMap<String, String>());
}
/**
@@ -328,17 +420,19 @@
* @param runName str, the name of the Mobly test binary run.
*/
@VisibleForTesting
- protected void processYamlTestResults(
+ protected boolean processYamlTestResults(
InputStream inputStream,
MoblyYamlResultParser parser,
ITestInvocationListener listener,
String runName) {
try {
- parser.parse(inputStream);
+ return parser.parse(inputStream);
} catch (MoblyYamlResultHandlerFactory.InvalidResultTypeException
+ | IOException
| IllegalAccessException
| InstantiationException ex) {
- reportFailure(listener, runName, "Failed to parse the result file.\n" + ex);
+ CLog.e("Failed to parse the result file.\n" + ex);
+ return false;
}
}
@@ -423,6 +517,8 @@
androidDeviceList.add(deviceMap);
deviceMap.put("serial", devices.get(index).getSerialNumber());
}
+ } else if (androidDeviceValue == null) {
+ CLog.d("No Android device provided.");
} else {
throw new HarnessRuntimeException(
String.format("Unsupported value for AndroidDevice: %s", androidDeviceValue),
@@ -496,6 +592,11 @@
@VisibleForTesting
protected String[] buildCommandLineArray(String filePath, String configPath) {
+ return buildCommandLineArray(filePath, configPath, getIncludeFilters());
+ }
+
+ protected String[] buildCommandLineArray(
+ String filePath, String configPath, Set<String> tests) {
List<String> commandLine = new ArrayList<>();
commandLine.add(filePath);
// TODO(b/166468397): some test binaries are actually a wrapper of Mobly runner and need --
@@ -511,9 +612,9 @@
commandLine.add("--device_serial=" + device.getSerialNumber());
}
commandLine.add("--log_path=" + getLogDirAbsolutePath());
- if (!mIncludeFilters.isEmpty()) {
+ if (!tests.isEmpty()) {
commandLine.add("--tests");
- commandLine.addAll(cleanFilters(mIncludeFilters));
+ commandLine.addAll(cleanFilters(tests));
}
// Add all the other options
commandLine.addAll(getTestOptions());
diff --git a/test_framework/com/android/tradefed/testtype/mobly/MoblyYamlResultParser.java b/test_framework/com/android/tradefed/testtype/mobly/MoblyYamlResultParser.java
index 0d3d4af..c83580b 100644
--- a/test_framework/com/android/tradefed/testtype/mobly/MoblyYamlResultParser.java
+++ b/test_framework/com/android/tradefed/testtype/mobly/MoblyYamlResultParser.java
@@ -23,7 +23,6 @@
import com.android.tradefed.result.proto.TestRecordProto;
import com.android.tradefed.testtype.mobly.IMoblyYamlResultHandler.ITestResult;
import com.android.tradefed.testtype.mobly.MoblyYamlResultHandlerFactory.InvalidResultTypeException;
-import com.android.tradefed.testtype.mobly.MoblyYamlResultSummaryHandler.Summary;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableList;
@@ -31,7 +30,11 @@
import org.yaml.snakeyaml.Yaml;
import org.yaml.snakeyaml.constructor.SafeConstructor;
+import java.io.BufferedReader;
+import java.io.IOException;
import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -41,122 +44,122 @@
private static final String TYPE = "Type";
private ImmutableList.Builder<ITestInvocationListener> mListenersBuilder =
new ImmutableList.Builder<>();
- private final String mRunName;
private ImmutableList.Builder<ITestResult> mResultCacheBuilder = new ImmutableList.Builder<>();
- private int mTestCount;
private long mRunStartTime;
private long mRunEndTime;
+ private boolean mEnded;
+ private boolean mRunFailed;
- public MoblyYamlResultParser(ITestInvocationListener listener, String runName) {
+ public MoblyYamlResultParser(ITestInvocationListener listener) {
mListenersBuilder.add(listener);
- mRunName = runName;
}
- public void parse(InputStream inputStream)
- throws InvalidResultTypeException, IllegalAccessException, InstantiationException {
- Yaml yaml = new Yaml(new SafeConstructor());
- for (Object doc : yaml.loadAll(inputStream)) {
- Map<String, Object> docMap = (Map<String, Object>) doc;
- mResultCacheBuilder.add(parseDocumentMap(docMap));
+ public boolean parse(InputStream inputStream)
+ throws InvalidResultTypeException,
+ IllegalAccessException,
+ InstantiationException,
+ IOException {
+ InputStreamReader isr = new InputStreamReader(inputStream);
+ BufferedReader in = new BufferedReader(isr);
+ while (in.ready() == true) {
+ String line = null;
+ String yaml_string = "";
+ while (true) {
+ line = in.readLine();
+ if (line == null) continue;
+ if (line.equals("...")) break;
+ yaml_string = yaml_string + line + "\n";
+ }
+ Yaml yaml = new Yaml(new SafeConstructor());
+ ArrayList<ITestResult> resultCache = new ArrayList<ITestResult>();
+ for (Object doc : yaml.loadAll(yaml_string)) {
+ Map<String, Object> docMap = (Map<String, Object>) doc;
+ resultCache.add(parseDocumentMap(docMap));
+ }
+ reportToListeners(mListenersBuilder.build(), resultCache);
}
- reportToListeners(mListenersBuilder.build(), mResultCacheBuilder.build());
+ return mEnded;
+ }
+
+ public boolean getRunFailed() {
+ return mRunFailed;
}
@VisibleForTesting
protected ITestResult parseDocumentMap(Map<String, Object> docMap)
throws InvalidResultTypeException, IllegalAccessException, InstantiationException {
- LogUtil.CLog.i("Parsed object: %s", docMap.toString());
+ LogUtil.CLog.d("Parsed object: %s", docMap.toString());
String docType = String.valueOf(docMap.get(TYPE));
- LogUtil.CLog.d("Parsing result type: %s", docType);
+ LogUtil.CLog.v("Parsing result type: %s", docType);
IMoblyYamlResultHandler resultHandler =
new MoblyYamlResultHandlerFactory().getHandler(docType);
- ITestResult testResult = resultHandler.handle(docMap);
- if ("Summary".equals(docType)) {
- mTestCount = ((Summary) testResult).getExecuted() + ((Summary) testResult).getSkipped();
- }
- return testResult;
+ return resultHandler.handle(docMap);
}
@VisibleForTesting
protected void reportToListeners(
List<ITestInvocationListener> listeners,
List<IMoblyYamlResultHandler.ITestResult> resultCache) {
- for (ITestInvocationListener listener : listeners) {
- listener.testRunStarted(mRunName, mTestCount);
- }
- try {
- boolean abort = false;
- for (IMoblyYamlResultHandler.ITestResult result : resultCache) {
- if (abort) {
- break;
- }
- switch (result.getType()) {
- case RECORD:
- MoblyYamlResultRecordHandler.Record record =
- (MoblyYamlResultRecordHandler.Record) result;
- TestDescription testDescription =
- new TestDescription(record.getTestClass(), record.getTestName());
- FailureDescription failureDescription =
- FailureDescription.create(
- record.getStackTrace(),
- TestRecordProto.FailureStatus.TEST_FAILURE);
- if (MoblyYamlResultRecordHandler.RecordResult.ERROR.equals(
+ for (IMoblyYamlResultHandler.ITestResult result : resultCache) {
+ switch (result.getType()) {
+ case RECORD:
+ MoblyYamlResultRecordHandler.Record record =
+ (MoblyYamlResultRecordHandler.Record) result;
+ TestDescription testDescription =
+ new TestDescription(record.getTestClass(), record.getTestName());
+ FailureDescription failureDescription =
+ FailureDescription.create(
+ record.getStackTrace(),
+ TestRecordProto.FailureStatus.TEST_FAILURE);
+ if (MoblyYamlResultRecordHandler.RecordResult.ERROR.equals(
+ record.getResult())) {
+ // Non-test failure reports indicates some early failure so we fail the run
+ if (!testDescription.getTestName().startsWith("test_")) {
+ for (ITestInvocationListener listener : listeners) {
+ listener.testRunFailed(failureDescription);
+ }
+ mRunFailed = true;
+ continue;
+ }
+ }
+ mRunStartTime =
+ mRunStartTime == 0L
+ ? record.getBeginTime()
+ : Math.min(mRunStartTime, record.getBeginTime());
+ mRunEndTime = Math.max(mRunEndTime, record.getEndTime());
+ for (ITestInvocationListener listener : listeners) {
+ listener.testStarted(testDescription, record.getBeginTime());
+ if (MoblyYamlResultRecordHandler.RecordResult.SKIP.equals(
record.getResult())) {
- // Setup_class indicates some early failure so we stop parsing
- if (testDescription.getTestName().equals("setup_class")) {
- for (ITestInvocationListener listener : listeners) {
- listener.testRunFailed(failureDescription);
- }
- abort = true;
- break;
- }
+ listener.testIgnored(testDescription);
+ } else if (!MoblyYamlResultRecordHandler.RecordResult.PASS.equals(
+ record.getResult())) {
+ listener.testFailed(testDescription, failureDescription);
}
- mRunStartTime =
- mRunStartTime == 0L
- ? record.getBeginTime()
- : Math.min(mRunStartTime, record.getBeginTime());
- mRunEndTime = Math.max(mRunEndTime, record.getEndTime());
- for (ITestInvocationListener listener : listeners) {
- listener.testStarted(testDescription, record.getBeginTime());
- if (MoblyYamlResultRecordHandler.RecordResult.SKIP.equals(
- record.getResult())) {
- listener.testIgnored(testDescription);
- } else if (!MoblyYamlResultRecordHandler.RecordResult.PASS.equals(
- record.getResult())) {
- listener.testFailed(testDescription, failureDescription);
- }
- listener.testEnded(
- testDescription,
- record.getEndTime(),
- new HashMap<String, String>());
- }
- break;
- case USER_DATA:
- long timestamp =
- ((MoblyYamlResultUserDataHandler.UserData) result).getTimeStamp();
- mRunStartTime =
- mRunStartTime == 0L
- ? timestamp
- : Math.min(mRunStartTime, timestamp);
- break;
- case CONTROLLER_INFO:
- mRunEndTime =
- Math.max(
- mRunEndTime,
- ((MoblyYamlResultControllerInfoHandler.ControllerInfo)
- result)
- .getTimeStamp());
- break;
- case TEST_NAME_LIST:
- // Do nothing
- break;
- default:
- // Do nothing
- }
- }
- } finally {
- for (ITestInvocationListener listener : listeners) {
- listener.testRunEnded(mRunEndTime - mRunStartTime, new HashMap<String, String>());
+ listener.testEnded(
+ testDescription,
+ record.getEndTime(),
+ new HashMap<String, String>());
+ }
+ break;
+ case USER_DATA:
+ long timestamp =
+ ((MoblyYamlResultUserDataHandler.UserData) result).getTimeStamp();
+ mRunStartTime =
+ mRunStartTime == 0L ? timestamp : Math.min(mRunStartTime, timestamp);
+ break;
+ case CONTROLLER_INFO:
+ mRunEndTime =
+ Math.max(
+ mRunEndTime,
+ ((MoblyYamlResultControllerInfoHandler.ControllerInfo) result)
+ .getTimeStamp());
+ break;
+ case SUMMARY:
+ mEnded = true;
+ break;
+ default:
+ // Do nothing
}
}
}
