Add support for python unit tests and adb use cases
- Handle pure python unit tests in par files.
- Handle the adb required setup in a preparer for adb
testing.
Test: tf unit tests
atest adb_integration_test_adb with a temporary AndroidTest.xml
http://sponge.corp.google.com/invocation?tab=Test+Cases&show=FAILED&show=INTERNAL_ERROR&id=3d282e52-4cbe-434c-9d16-36323902419c
Bug: 112104122
Change-Id: I95f3a3ac9b41543d9f332b20c97716d5c6d949b0
diff --git a/src/com/android/tradefed/targetprep/adb/AdbStopServerPreparer.java b/src/com/android/tradefed/targetprep/adb/AdbStopServerPreparer.java
index b070bd6..66462ed 100644
--- a/src/com/android/tradefed/targetprep/adb/AdbStopServerPreparer.java
+++ b/src/com/android/tradefed/targetprep/adb/AdbStopServerPreparer.java
@@ -15,24 +15,42 @@
*/
package com.android.tradefed.targetprep.adb;
+import com.android.annotations.VisibleForTesting;
import com.android.tradefed.build.IBuildInfo;
import com.android.tradefed.config.GlobalConfiguration;
import com.android.tradefed.device.DeviceNotAvailableException;
import com.android.tradefed.device.IDeviceManager;
import com.android.tradefed.device.ITestDevice;
+import com.android.tradefed.log.LogUtil.CLog;
import com.android.tradefed.targetprep.BaseTargetPreparer;
import com.android.tradefed.targetprep.BuildError;
import com.android.tradefed.targetprep.ITargetCleaner;
+import com.android.tradefed.targetprep.SemaphoreTokenTargetPreparer;
import com.android.tradefed.targetprep.TargetSetupError;
+import com.android.tradefed.util.CommandResult;
+import com.android.tradefed.util.CommandStatus;
+import com.android.tradefed.util.FileUtil;
import com.android.tradefed.util.IRunUtil;
import com.android.tradefed.util.RunUtil;
-/** Target preparer to stop adb server on the host before and after running adb tests. */
+import java.io.File;
+import java.io.IOException;
+
+/**
+ * Target preparer to stop adb server on the host before and after running adb tests.
+ *
+ * <p>This preparer should be used with care as it stops and restart adb on the hosts. It should
+ * usually be tight with {@link SemaphoreTokenTargetPreparer} to avoid other tests from running at
+ * the same time.
+ */
public class AdbStopServerPreparer extends BaseTargetPreparer implements ITargetCleaner {
- private IRunUtil mRunUtil = RunUtil.getDefault();
-
+ private static final String PATH = "PATH";
private static final long CMD_TIMEOUT = 60000L;
+ private static final String ANDROID_HOST_OUT = "ANDROID_HOST_OUT";
+
+ private IRunUtil mRunUtil;
+ private File mTmpDir;
/** {@inheritDoc} */
@Override
@@ -42,29 +60,89 @@
getDeviceManager().stopAdbBridge();
// Kill the default adb server
- mRunUtil.runTimedCmd(CMD_TIMEOUT, "adb", "kill-server");
- // Wait 1000ms for "adb kill-server" to finish. See b/37104408
- mRunUtil.sleep(1 * 1000);
+ getRunUtil().runTimedCmd(CMD_TIMEOUT, "adb", "kill-server");
+
+ File adb = null;
+ if (System.getenv(ANDROID_HOST_OUT) != null) {
+ String hostOut = System.getenv(ANDROID_HOST_OUT);
+ adb = new File(hostOut, "bin/adb");
+ if (adb.exists()) {
+ adb.setExecutable(true);
+ } else {
+ adb = null;
+ }
+ }
+
+ if (adb == null && buildInfo.getFile("adb") != null) {
+ adb = buildInfo.getFile("adb");
+ adb = renameAdbBinary(adb);
+ }
+
+ if (adb != null) {
+ CLog.d("Restarting adb from %s", adb.getAbsolutePath());
+ IRunUtil restartAdb = createRunUtil();
+ restartAdb.setEnvVariable(PATH, adb.getAbsolutePath());
+ CommandResult result =
+ restartAdb.runTimedCmd(CMD_TIMEOUT, adb.getAbsolutePath(), "start-server");
+ if (!CommandStatus.SUCCESS.equals(result.getStatus())) {
+ throw new TargetSetupError(
+ "Failed to restart adb with the build info one.",
+ device.getDeviceDescriptor());
+ }
+ } else {
+ getRunUtil().runTimedCmd(CMD_TIMEOUT, "adb", "start-server");
+ throw new TargetSetupError(
+ "Could not find a new version of adb to tests.", device.getDeviceDescriptor());
+ }
}
/** {@inheritDoc} */
@Override
public void tearDown(ITestDevice device, IBuildInfo buildInfo, Throwable e)
throws DeviceNotAvailableException {
+ FileUtil.recursiveDelete(mTmpDir);
// Kill the test adb server
- mRunUtil.runTimedCmd(CMD_TIMEOUT, "adb", "kill-server");
- // Wait 1000ms for "adb kill-server" to finish. See b/37104408
- mRunUtil.sleep(1 * 1000);
- // Restart the default adb server found in system PATH. Ensure adb server is already
- // running the next time TF executes a command over adb bridge. Otherwise adb bridge may
- // see a "Connection refused" exception.
- mRunUtil.runTimedCmd(CMD_TIMEOUT, "adb", "start-server");
- mRunUtil.sleep(1 * 1000);
-
+ getRunUtil().runTimedCmd(CMD_TIMEOUT, "adb", "kill-server");
+ // Restart the one from the parent PATH (original one)
+ getRunUtil().runTimedCmd(CMD_TIMEOUT, "adb", "start-server");
+ // Restart device manager monitor
getDeviceManager().restartAdbBridge();
}
+ @VisibleForTesting
IDeviceManager getDeviceManager() {
return GlobalConfiguration.getDeviceManagerInstance();
}
+
+ @VisibleForTesting
+ IRunUtil createRunUtil() {
+ return new RunUtil();
+ }
+
+ private IRunUtil getRunUtil() {
+ if (mRunUtil == null) {
+ mRunUtil = createRunUtil();
+ }
+ return mRunUtil;
+ }
+
+ private File renameAdbBinary(File originalAdb) {
+ try {
+ mTmpDir = FileUtil.createTempDir("adb");
+ } catch (IOException e) {
+ CLog.e("Cannot create temp directory");
+ FileUtil.recursiveDelete(mTmpDir);
+ return null;
+ }
+ File renamedAdbBinary = new File(mTmpDir, "adb");
+ if (!originalAdb.renameTo(renamedAdbBinary)) {
+ CLog.e("Cannot rename adb binary");
+ return null;
+ }
+ if (!renamedAdbBinary.setExecutable(true)) {
+ CLog.e("Cannot set adb binary executable");
+ return null;
+ }
+ return renamedAdbBinary;
+ }
}
diff --git a/src/com/android/tradefed/testtype/PythonUnitTestResultParser.java b/src/com/android/tradefed/testtype/PythonUnitTestResultParser.java
index 70e68b4..b36c5e4 100644
--- a/src/com/android/tradefed/testtype/PythonUnitTestResultParser.java
+++ b/src/com/android/tradefed/testtype/PythonUnitTestResultParser.java
@@ -20,6 +20,8 @@
import com.android.tradefed.result.ITestInvocationListener;
import com.android.tradefed.result.TestDescription;
+import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
@@ -95,7 +97,7 @@
private int mFailedTestCount;
// General state
- private final Collection<ITestInvocationListener> mListeners;
+ private Collection<ITestInvocationListener> mListeners = new ArrayList<>();
private final String mRunName;
private Map<TestDescription, String> mTestResultCache;
// Use a special entry to mark skipped test in mTestResultCache
@@ -145,11 +147,19 @@
/**
* Create a new {@link PythonUnitTestResultParser} that reports to the given {@link
+ * ITestInvocationListener}.
+ */
+ public PythonUnitTestResultParser(ITestInvocationListener listener, String runName) {
+ this(Arrays.asList(listener), runName);
+ }
+
+ /**
+ * Create a new {@link PythonUnitTestResultParser} that reports to the given {@link
* ITestInvocationListener}s.
*/
public PythonUnitTestResultParser(
Collection<ITestInvocationListener> listeners, String runName) {
- mListeners = listeners;
+ mListeners.addAll(listeners);
mRunName = runName;
mTestResultCache = new HashMap<>();
}
diff --git a/src/com/android/tradefed/testtype/python/PythonBinaryHostTest.java b/src/com/android/tradefed/testtype/python/PythonBinaryHostTest.java
index dd07c17..ce73bd4 100644
--- a/src/com/android/tradefed/testtype/python/PythonBinaryHostTest.java
+++ b/src/com/android/tradefed/testtype/python/PythonBinaryHostTest.java
@@ -33,12 +33,12 @@
import com.android.tradefed.testtype.IDeviceTest;
import com.android.tradefed.testtype.IInvocationContextReceiver;
import com.android.tradefed.testtype.IRemoteTest;
+import com.android.tradefed.testtype.PythonUnitTestResultParser;
import com.android.tradefed.util.CommandResult;
import com.android.tradefed.util.CommandStatus;
import com.android.tradefed.util.FileUtil;
import com.android.tradefed.util.IRunUtil;
import com.android.tradefed.util.RunUtil;
-import com.android.tradefed.util.StreamUtil;
import com.android.tradefed.util.SubprocessTestResultsParser;
import java.io.File;
@@ -72,7 +72,14 @@
)
private long mTestTimeout = 20 * 1000L;
- @Option(name = "inject-android-serial")
+ @Option(
+ name = "inject-serial-option",
+ description = "Whether or not to pass a -s <serialnumber> option to the binary")
+ private boolean mInjectSerial = false;
+
+ @Option(
+ name = "inject-android-serial",
+ description = "Whether or not to pass a ANDROID_SERIAL variable to the process.")
private boolean mInjectAndroidSerialVar = true;
private ITestDevice mDevice;
@@ -143,10 +150,8 @@
private void runSinglePythonFile(ITestInvocationListener listener, File pyFile) {
List<String> commandLine = new ArrayList<>();
commandLine.add(pyFile.getAbsolutePath());
- // Run with -q (quiet) to avoid extraneous outputs
- commandLine.add("-q");
// If we have a physical device, pass it to the python test by serial
- if (!(getDevice().getIDevice() instanceof StubDevice)) {
+ if (!(getDevice().getIDevice() instanceof StubDevice) && mInjectSerial) {
// TODO: support multi-device python tests?
commandLine.add("-s");
commandLine.add(getDevice().getSerialNumber());
@@ -160,17 +165,12 @@
getRunUtil().runTimedCmd(mTestTimeout, commandLine.toArray(new String[0]));
PythonForwarder forwarder = new PythonForwarder(listener, pyFile.getName());
if (!CommandStatus.SUCCESS.equals(result.getStatus())) {
- // If the binary finishes we an error code, it could simply be a test failure, but if
- // it does not even have a TEST_RUN_STARTED tag, then we probably have binary setup
- // issue.
- if (!result.getStderr().contains("TEST_RUN_STARTED")) {
- throw new RuntimeException(
- String.format(
- "Something went wrong when running the python binary: %s",
- result.getStderr()));
- }
+ CLog.e(
+ "Something went wrong when running the python binary:\nstdout: "
+ + "%s\nstderr:%s",
+ result.getStdout(), result.getStderr());
}
- SubprocessTestResultsParser parser = new SubprocessTestResultsParser(forwarder, mContext);
+
File resultFile = null;
try {
resultFile = FileUtil.createTempFile("python-res", ".txt");
@@ -178,12 +178,22 @@
try (FileInputStreamSource data = new FileInputStreamSource(resultFile)) {
listener.testLog(PYTHON_OUTPUT, LogDataType.TEXT, data);
}
- parser.parseFile(resultFile);
+ // If it doesn't have the std output TEST_RUN_STARTED, use regular parser.
+ if (!result.getStderr().contains("TEST_RUN_STARTED")) {
+ // Attempt to parse the pure python output
+ PythonUnitTestResultParser pythonParser =
+ new PythonUnitTestResultParser(forwarder, "python-run");
+ pythonParser.processNewLines(result.getStderr().split("\n"));
+ } else {
+ try (SubprocessTestResultsParser parser =
+ new SubprocessTestResultsParser(forwarder, mContext)) {
+ parser.parseFile(resultFile);
+ }
+ }
} catch (IOException e) {
throw new RuntimeException(e);
} finally {
FileUtil.deleteFile(resultFile);
- StreamUtil.close(parser);
}
}
diff --git a/tests/src/com/android/tradefed/UnitTests.java b/tests/src/com/android/tradefed/UnitTests.java
index 1a51701..b571ad3 100644
--- a/tests/src/com/android/tradefed/UnitTests.java
+++ b/tests/src/com/android/tradefed/UnitTests.java
@@ -161,6 +161,7 @@
import com.android.tradefed.targetprep.TestAppInstallSetupTest;
import com.android.tradefed.targetprep.TestFilePushSetupTest;
import com.android.tradefed.targetprep.TimeSetterTargetPreparerTest;
+import com.android.tradefed.targetprep.adb.AdbStopServerPreparerTest;
import com.android.tradefed.targetprep.multi.MergeMultiBuildTargetPreparerTest;
import com.android.tradefed.targetprep.suite.SuiteApkInstallerTest;
import com.android.tradefed.testtype.AndroidJUnitTestTest;
@@ -471,6 +472,9 @@
TestFilePushSetupTest.class,
TimeSetterTargetPreparerTest.class,
+ // targetprep.adb
+ AdbStopServerPreparerTest.class,
+
// targetprep.multi
MergeMultiBuildTargetPreparerTest.class,
diff --git a/tests/src/com/android/tradefed/targetprep/adb/AdbStopServerPreparerTest.java b/tests/src/com/android/tradefed/targetprep/adb/AdbStopServerPreparerTest.java
new file mode 100644
index 0000000..0eba115
--- /dev/null
+++ b/tests/src/com/android/tradefed/targetprep/adb/AdbStopServerPreparerTest.java
@@ -0,0 +1,182 @@
+/*
+ * Copyright (C) 2018 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.tradefed.targetprep.adb;
+
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import com.android.tradefed.build.BuildInfo;
+import com.android.tradefed.build.IBuildInfo;
+import com.android.tradefed.device.IDeviceManager;
+import com.android.tradefed.device.ITestDevice;
+import com.android.tradefed.targetprep.TargetSetupError;
+import com.android.tradefed.util.CommandResult;
+import com.android.tradefed.util.CommandStatus;
+import com.android.tradefed.util.FileUtil;
+import com.android.tradefed.util.IRunUtil;
+
+import org.easymock.EasyMock;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.io.File;
+
+/** Unit tests for {@link AdbStopServerPreparer}. */
+@RunWith(JUnit4.class)
+public class AdbStopServerPreparerTest {
+
+ private AdbStopServerPreparer mPreparer;
+ private IRunUtil mMockRunUtil;
+ private IDeviceManager mMockManager;
+
+ private ITestDevice mMockDevice;
+ private IBuildInfo mMockBuild;
+ private File mFakeAdbFile;
+
+ @Before
+ public void setUp() throws Exception {
+ mMockRunUtil = EasyMock.createMock(IRunUtil.class);
+ mMockManager = EasyMock.createMock(IDeviceManager.class);
+ mMockDevice = EasyMock.createMock(ITestDevice.class);
+ mPreparer =
+ new AdbStopServerPreparer() {
+ @Override
+ IRunUtil createRunUtil() {
+ return mMockRunUtil;
+ }
+
+ @Override
+ IDeviceManager getDeviceManager() {
+ return mMockManager;
+ }
+ };
+ mMockBuild = new BuildInfo();
+ mFakeAdbFile = FileUtil.createTempFile("adb", "");
+ mMockBuild.setFile("adb", mFakeAdbFile, "v1");
+
+ EasyMock.expect(mMockDevice.getDeviceDescriptor()).andStubReturn(null);
+ }
+
+ @After
+ public void tearDown() {
+ FileUtil.deleteFile(mFakeAdbFile);
+ }
+
+ /** Test that during setup the adb server is stopped then restart with a new found version. */
+ @Test
+ public void testSetup_tearDown() throws Exception {
+ mMockManager.stopAdbBridge();
+ CommandResult result = new CommandResult(CommandStatus.SUCCESS);
+ // setUp
+ EasyMock.expect(
+ mMockRunUtil.runTimedCmd(
+ EasyMock.anyLong(), EasyMock.eq("adb"), EasyMock.eq("kill-server")))
+ .andReturn(result);
+ mMockRunUtil.setEnvVariable(EasyMock.eq("PATH"), EasyMock.anyObject());
+ EasyMock.expect(
+ mMockRunUtil.runTimedCmd(
+ EasyMock.anyLong(),
+ EasyMock.contains("adb"),
+ EasyMock.eq("start-server")))
+ .andReturn(result);
+ // tear down
+ mockTearDown();
+ EasyMock.replay(mMockRunUtil, mMockManager, mMockDevice);
+ mPreparer.setUp(mMockDevice, mMockBuild);
+ mPreparer.tearDown(mMockDevice, mMockBuild, null);
+ EasyMock.verify(mMockRunUtil, mMockManager, mMockDevice);
+ }
+
+ /** Test when restarting the adb server fails. */
+ @Test
+ public void testSetup_fail_tearDown() throws Exception {
+ mMockManager.stopAdbBridge();
+ CommandResult result = new CommandResult(CommandStatus.SUCCESS);
+ // setUp
+ EasyMock.expect(
+ mMockRunUtil.runTimedCmd(
+ EasyMock.anyLong(), EasyMock.eq("adb"), EasyMock.eq("kill-server")))
+ .andReturn(result);
+ mMockRunUtil.setEnvVariable(EasyMock.eq("PATH"), EasyMock.anyObject());
+
+ CommandResult failedResult = new CommandResult(CommandStatus.FAILED);
+ EasyMock.expect(
+ mMockRunUtil.runTimedCmd(
+ EasyMock.anyLong(),
+ EasyMock.contains("adb"),
+ EasyMock.eq("start-server")))
+ .andReturn(failedResult);
+
+ // tear down
+ mockTearDown();
+ EasyMock.replay(mMockRunUtil, mMockManager, mMockDevice);
+ try {
+ mPreparer.setUp(mMockDevice, mMockBuild);
+ fail("Should have thrown an exception.");
+ } catch (TargetSetupError expected) {
+ // Expected
+ }
+ mPreparer.tearDown(mMockDevice, mMockBuild, null);
+ EasyMock.verify(mMockRunUtil, mMockManager, mMockDevice);
+ }
+
+ /** Test that we fail the preparation when there is no adb to test. */
+ @Test
+ public void testNoAdb() throws Exception {
+ mMockManager.stopAdbBridge();
+ CommandResult result = new CommandResult(CommandStatus.SUCCESS);
+ // setUp
+ EasyMock.expect(
+ mMockRunUtil.runTimedCmd(
+ EasyMock.anyLong(), EasyMock.eq("adb"), EasyMock.eq("kill-server")))
+ .andReturn(result);
+ EasyMock.expect(
+ mMockRunUtil.runTimedCmd(
+ EasyMock.anyLong(),
+ EasyMock.eq("adb"),
+ EasyMock.eq("start-server")))
+ .andReturn(result);
+ EasyMock.replay(mMockRunUtil, mMockManager, mMockDevice);
+ try {
+ mPreparer.setUp(mMockDevice, new BuildInfo());
+ fail("Should have thrown an exception.");
+ } catch (TargetSetupError expected) {
+ // Expected
+ assertTrue(
+ expected.getMessage()
+ .contains("Could not find a new version of adb to tests."));
+ }
+ EasyMock.verify(mMockRunUtil, mMockManager, mMockDevice);
+ }
+
+ private void mockTearDown() {
+ CommandResult result = new CommandResult(CommandStatus.SUCCESS);
+ EasyMock.expect(
+ mMockRunUtil.runTimedCmd(
+ EasyMock.anyLong(), EasyMock.eq("adb"), EasyMock.eq("kill-server")))
+ .andReturn(result);
+ EasyMock.expect(
+ mMockRunUtil.runTimedCmd(
+ EasyMock.anyLong(),
+ EasyMock.eq("adb"),
+ EasyMock.eq("start-server")))
+ .andReturn(result);
+ mMockManager.restartAdbBridge();
+ }
+}
diff --git a/tests/src/com/android/tradefed/testtype/python/PythonBinaryHostTestTest.java b/tests/src/com/android/tradefed/testtype/python/PythonBinaryHostTestTest.java
index 26c3b83..fd01f4f 100644
--- a/tests/src/com/android/tradefed/testtype/python/PythonBinaryHostTestTest.java
+++ b/tests/src/com/android/tradefed/testtype/python/PythonBinaryHostTestTest.java
@@ -15,6 +15,7 @@
*/
package com.android.tradefed.testtype.python;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
import com.android.tradefed.build.IBuildInfo;
@@ -76,9 +77,7 @@
res.setStderr("TEST_RUN_STARTED {\"testCount\": 5, \"runName\": \"TestSuite\"}");
EasyMock.expect(
mMockRunUtil.runTimedCmd(
- EasyMock.anyLong(),
- EasyMock.eq(binary.getAbsolutePath()),
- EasyMock.eq("-q")))
+ EasyMock.anyLong(), EasyMock.eq(binary.getAbsolutePath())))
.andReturn(res);
mMockListener.testRunStarted(binary.getName(), 5);
mMockListener.testLog(
@@ -110,17 +109,22 @@
res.setStderr("Could not execute.");
EasyMock.expect(
mMockRunUtil.runTimedCmd(
- EasyMock.anyLong(),
- EasyMock.eq(binary.getAbsolutePath()),
- EasyMock.eq("-q")))
+ EasyMock.anyLong(), EasyMock.eq(binary.getAbsolutePath())))
.andReturn(res);
EasyMock.expect(mMockDevice.getIDevice()).andReturn(new StubDevice("serial"));
+
+ mMockListener.testLog(
+ EasyMock.eq("python-output"),
+ EasyMock.eq(LogDataType.TEXT),
+ EasyMock.anyObject());
+
EasyMock.replay(mMockRunUtil, mMockBuildInfo, mMockListener, mMockDevice);
try {
mTest.run(mMockListener);
fail("Should have thrown an exception.");
} catch (RuntimeException expected) {
// expected
+ assertEquals("Failed to parse Python unittest result", expected.getMessage());
}
EasyMock.verify(mMockRunUtil, mMockBuildInfo, mMockListener, mMockDevice);
} finally {
@@ -143,9 +147,7 @@
res.setStderr("TEST_RUN_STARTED {\"testCount\": 5, \"runName\": \"TestSuite\"}");
EasyMock.expect(
mMockRunUtil.runTimedCmd(
- EasyMock.anyLong(),
- EasyMock.eq(binary.getAbsolutePath()),
- EasyMock.eq("-q")))
+ EasyMock.anyLong(), EasyMock.eq(binary.getAbsolutePath())))
.andReturn(res);
mMockListener.testRunStarted(binary.getName(), 5);
mMockListener.testLog(
