Adding additional reporting to tests and features
Implement MetricsReportLog to generate reportlog.json.
results/latest/report-log-files/StsTestCases.reportlog.json
see json prettified with:
StsTestCases.reportlog.json | python3 -m json.tool | less
Bug: 157920496
Test: run sts
Change-Id: I23d4a50673067b15e26b20dcdbee3dfe585b0331
diff --git a/hostsidetests/securitybulletin/AndroidTest.xml b/hostsidetests/securitybulletin/AndroidTest.xml
index 7a25642..4f35211 100644
--- a/hostsidetests/securitybulletin/AndroidTest.xml
+++ b/hostsidetests/securitybulletin/AndroidTest.xml
@@ -255,4 +255,11 @@
<option name="jar" value="CtsSecurityBulletinHostTestCases.jar" />
<option name="runtime-hint" value="18m26s" />
</test>
+
+ <target_preparer class="com.android.compatibility.common.tradefed.targetprep.ReportLogCollector">
+ <option name="src-dir" value="/sdcard/report-log-files/"/>
+ <option name="dest-dir" value="report-log-files/"/>
+ <option name="temp-dir" value="temp-report-logs/"/>
+ <option name="device-dir" value="true"/>
+ </target_preparer>
</configuration>
diff --git a/hostsidetests/securitybulletin/src/android/security/cts/AdbUtils.java b/hostsidetests/securitybulletin/src/android/security/cts/AdbUtils.java
index e5e5c69..c780ab0 100644
--- a/hostsidetests/securitybulletin/src/android/security/cts/AdbUtils.java
+++ b/hostsidetests/securitybulletin/src/android/security/cts/AdbUtils.java
@@ -17,6 +17,9 @@
package android.security.cts;
import com.android.compatibility.common.util.CrashUtils;
+import com.android.compatibility.common.util.MetricsReportLog;
+import com.android.compatibility.common.util.ResultType;
+import com.android.compatibility.common.util.ResultUnit;
import com.android.ddmlib.IShellOutputReceiver;
import com.android.ddmlib.NullOutputReceiver;
import com.android.ddmlib.CollectingOutputReceiver;
@@ -148,8 +151,30 @@
if (arguments == null) {
arguments = "";
}
- device.executeShellCommand(TMP_PATH + pocName + " " + arguments,
+
+ // since we have to return the exit status AND the poc stdout+stderr we redirect the exit
+ // status to a file temporarily
+ String exitStatusFilepath = TMP_PATH + "exit_status";
+ runCommandLine("rm " + exitStatusFilepath, device); // remove any old exit status
+ device.executeShellCommand(TMP_PATH + pocName + " " + arguments +
+ "; echo $? > " + exitStatusFilepath, // echo exit status to file
receiver, timeout, TimeUnit.SECONDS, 0);
+
+ // cat the exit status
+ String exitStatusString = runCommandLine("cat " + exitStatusFilepath, device).trim();
+
+ MetricsReportLog reportLog = SecurityTestCase.buildMetricsReportLog(device);
+ reportLog.addValue("poc_name", pocName, ResultType.NEUTRAL, ResultUnit.NONE);
+ try {
+ int exitStatus = Integer.parseInt(exitStatusString);
+ reportLog.addValue("exit_status", exitStatus, ResultType.NEUTRAL, ResultUnit.NONE);
+ } catch (NumberFormatException e) {
+ // Getting the exit status is a bonus. We can continue without it.
+ CLog.w("Could not parse exit status to int: %s", exitStatusString);
+ }
+ reportLog.submit();
+
+ runCommandLine("rm " + exitStatusFilepath, device);
}
/**
@@ -304,15 +329,21 @@
*/
public static int runCommandGetExitCode(String cmd, ITestDevice device) throws Exception {
long time = System.currentTimeMillis();
- String exitStatus = runCommandLine(
+ String exitStatusString = runCommandLine(
"(" + cmd + ") > /dev/null 2>&1; echo $?", device).trim();
time = System.currentTimeMillis() - time;
+
try {
- return Integer.parseInt(exitStatus);
+ int exitStatus = Integer.parseInt(exitStatusString);
+ MetricsReportLog reportLog = SecurityTestCase.buildMetricsReportLog(device);
+ reportLog.addValue("command", cmd, ResultType.NEUTRAL, ResultUnit.NONE);
+ reportLog.addValue("exit_status", exitStatus, ResultType.NEUTRAL, ResultUnit.NONE);
+ reportLog.submit();
+ return exitStatus;
} catch (NumberFormatException e) {
throw new IllegalArgumentException(String.format(
"Could not get the exit status (%s) for '%s' (%d ms).",
- exitStatus, cmd, time));
+ exitStatusString, cmd, time));
}
}
@@ -359,13 +390,19 @@
long time = System.currentTimeMillis();
device.executeShellCommand(cmd, receiver, timeout, TimeUnit.SECONDS, 0);
time = System.currentTimeMillis() - time;
- String exitStatus = receiver.getOutput().trim();
+ String exitStatusString = receiver.getOutput().trim();
+
try {
- return Integer.parseInt(exitStatus);
+ int exitStatus = Integer.parseInt(exitStatusString);
+ MetricsReportLog reportLog = SecurityTestCase.buildMetricsReportLog(device);
+ reportLog.addValue("poc_name", pocName, ResultType.NEUTRAL, ResultUnit.NONE);
+ reportLog.addValue("exit_status", exitStatus, ResultType.NEUTRAL, ResultUnit.NONE);
+ reportLog.submit();
+ return exitStatus;
} catch (NumberFormatException e) {
throw new IllegalArgumentException(String.format(
"Could not get the exit status (%s) for '%s' (%d ms).",
- exitStatus, cmd, time));
+ exitStatusString, cmd, time));
}
}
@@ -569,6 +606,12 @@
JSONArray crashes = CrashUtils.addAllCrashes(logcat, new JSONArray());
JSONArray securityCrashes = CrashUtils.matchSecurityCrashes(crashes, config);
+ MetricsReportLog reportLog = SecurityTestCase.buildMetricsReportLog(device);
+ reportLog.addValue("all_crashes", crashes.toString(), ResultType.NEUTRAL, ResultUnit.NONE);
+ reportLog.addValue("security_crashes", securityCrashes.toString(),
+ ResultType.NEUTRAL, ResultUnit.NONE);
+ reportLog.submit();
+
if (securityCrashes.length() == 0) {
return; // no security crashes detected
}
diff --git a/hostsidetests/securitybulletin/src/android/security/cts/SecurityTestCase.java b/hostsidetests/securitybulletin/src/android/security/cts/SecurityTestCase.java
index 038eefe..912d255 100644
--- a/hostsidetests/securitybulletin/src/android/security/cts/SecurityTestCase.java
+++ b/hostsidetests/securitybulletin/src/android/security/cts/SecurityTestCase.java
@@ -16,6 +16,13 @@
package android.security.cts;
+import com.android.compatibility.common.util.MetricsReportLog;
+import com.android.compatibility.common.util.ResultType;
+import com.android.compatibility.common.util.ResultUnit;
+import com.android.tradefed.build.IBuildInfo;
+import com.android.tradefed.testtype.IBuildReceiver;
+import com.android.tradefed.testtype.IAbi;
+import com.android.tradefed.testtype.IAbiReceiver;
import com.android.tradefed.testtype.junit4.BaseHostJUnit4Test;
import com.android.tradefed.device.DeviceNotAvailableException;
import com.android.tradefed.device.ITestDevice;
@@ -24,10 +31,14 @@
import com.android.tradefed.testtype.DeviceJUnit4ClassRunner;
import com.android.ddmlib.Log;
+import org.junit.rules.TestName;
+import org.junit.Rule;
import org.junit.After;
import org.junit.Before;
import org.junit.runner.RunWith;
+import java.util.Map;
+import java.util.HashMap;
import java.util.regex.Pattern;
import java.util.regex.Matcher;
import java.util.concurrent.Callable;
@@ -49,6 +60,12 @@
private HostsideOomCatcher oomCatcher = new HostsideOomCatcher(this);
+ @Rule public TestName testName = new TestName();
+
+ private static Map<ITestDevice, IBuildInfo> sBuildInfo = new HashMap<>();
+ private static Map<ITestDevice, IAbi> sAbi = new HashMap<>();
+ private static Map<ITestDevice, String> sTestName = new HashMap<>();
+
/**
* Waits for device to be online, marks the most recent boottime of the device
*/
@@ -61,6 +78,9 @@
// Specifically time when app framework starts
oomCatcher.start();
+ sBuildInfo.put(getDevice(), getBuild());
+ sAbi.put(getDevice(), getAbi());
+ sTestName.put(getDevice(), testName.getMethodName());
}
/**
@@ -102,6 +122,18 @@
}
}
+ public static IBuildInfo getBuildInfo(ITestDevice device) {
+ return sBuildInfo.get(device);
+ }
+
+ public static IAbi getAbi(ITestDevice device) {
+ return sAbi.get(device);
+ }
+
+ public static String getTestName(ITestDevice device) {
+ return sTestName.get(device);
+ }
+
// TODO convert existing assertMatches*() to RegexUtils.assertMatches*()
// b/123237827
@Deprecated
@@ -189,7 +221,39 @@
* Check if a driver is present on a machine.
*/
protected boolean containsDriver(ITestDevice device, String driver) throws Exception {
- return AdbUtils.runCommandGetExitCode("test -r " + driver, device) == 0;
+ boolean containsDriver = AdbUtils.runCommandGetExitCode("test -r " + driver, device) == 0;
+
+ MetricsReportLog reportLog = buildMetricsReportLog(getDevice());
+ reportLog.addValue("path", driver, ResultType.NEUTRAL, ResultUnit.NONE);
+ reportLog.addValue("exists", containsDriver, ResultType.NEUTRAL, ResultUnit.NONE);
+ reportLog.submit();
+
+ return containsDriver;
+ }
+
+ protected static MetricsReportLog buildMetricsReportLog(ITestDevice device) {
+ IBuildInfo buildInfo = getBuildInfo(device);
+ IAbi abi = getAbi(device);
+ String testName = getTestName(device);
+
+ StackTraceElement[] stacktraces = Thread.currentThread().getStackTrace();
+ int stackDepth = 2; // 0: getStackTrace(), 1: buildMetricsReportLog, 2: caller
+ String className = stacktraces[stackDepth].getClassName();
+ String methodName = stacktraces[stackDepth].getMethodName();
+ String classMethodName = String.format("%s#%s", className, methodName);
+
+ // The stream name must be snake_case or else json formatting breaks
+ String streamName = methodName.replaceAll("(\\p{Upper})", "_$1").toLowerCase();
+
+ MetricsReportLog reportLog = new MetricsReportLog(
+ buildInfo,
+ abi.getName(),
+ classMethodName,
+ "CtsSecurityBulletinHostTestCases",
+ streamName,
+ true);
+ reportLog.addValue("test_name", testName, ResultType.NEUTRAL, ResultUnit.NONE);
+ return reportLog;
}
private long getDeviceUptime() throws DeviceNotAvailableException {
