| /* |
| * Copyright (C) 2015 The Android Open Source Project |
| * |
| * Licensed under the Apache License, Version 2.0 (the "License"); |
| * you may not use this file except in compliance with the License. |
| * You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| package com.android.compatibility.common.tradefed.result; |
| |
| import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper; |
| import com.android.compatibility.common.tradefed.testtype.retry.RetryFactoryTest; |
| import com.android.compatibility.common.tradefed.testtype.suite.CompatibilityTestSuite; |
| import com.android.compatibility.common.tradefed.util.FingerprintComparisonException; |
| import com.android.compatibility.common.tradefed.util.RetryType; |
| import com.android.compatibility.common.util.ChecksumReporter; |
| import com.android.compatibility.common.util.DeviceInfo; |
| import com.android.compatibility.common.util.ICaseResult; |
| import com.android.compatibility.common.util.IInvocationResult; |
| import com.android.compatibility.common.util.IModuleResult; |
| import com.android.compatibility.common.util.ITestResult; |
| import com.android.compatibility.common.util.InvocationResult; |
| import com.android.compatibility.common.util.InvocationResult.RunHistory; |
| import com.android.compatibility.common.util.MetricsStore; |
| import com.android.compatibility.common.util.ReportLog; |
| import com.android.compatibility.common.util.ResultHandler; |
| import com.android.compatibility.common.util.ResultUploader; |
| import com.android.compatibility.common.util.TestStatus; |
| import com.android.ddmlib.Log.LogLevel; |
| import com.android.tradefed.build.IBuildInfo; |
| import com.android.tradefed.config.IConfiguration; |
| import com.android.tradefed.config.IConfigurationReceiver; |
| import com.android.tradefed.config.Option; |
| import com.android.tradefed.config.Option.Importance; |
| import com.android.tradefed.config.OptionClass; |
| import com.android.tradefed.config.OptionCopier; |
| import com.android.tradefed.invoker.IInvocationContext; |
| import com.android.tradefed.log.LogUtil.CLog; |
| import com.android.tradefed.metrics.proto.MetricMeasurement.Metric; |
| import com.android.tradefed.result.FileInputStreamSource; |
| import com.android.tradefed.result.ILogSaver; |
| import com.android.tradefed.result.ILogSaverListener; |
| import com.android.tradefed.result.IShardableListener; |
| import com.android.tradefed.result.ITestInvocationListener; |
| import com.android.tradefed.result.ITestSummaryListener; |
| import com.android.tradefed.result.InputStreamSource; |
| import com.android.tradefed.result.LogDataType; |
| import com.android.tradefed.result.LogFile; |
| import com.android.tradefed.result.LogFileSaver; |
| import com.android.tradefed.result.TestDescription; |
| import com.android.tradefed.result.TestSummary; |
| import com.android.tradefed.result.suite.SuiteResultReporter; |
| import com.android.tradefed.util.FileUtil; |
| import com.android.tradefed.util.StreamUtil; |
| import com.android.tradefed.util.TimeUtil; |
| import com.android.tradefed.util.ZipUtil; |
| import com.android.tradefed.util.proto.TfMetricProtoUtil; |
| |
| import com.google.common.annotations.VisibleForTesting; |
| import com.google.common.xml.XmlEscapers; |
| import com.google.gson.Gson; |
| |
| import org.xmlpull.v1.XmlPullParserException; |
| |
| import java.io.File; |
| import java.io.FileInputStream; |
| import java.io.FileNotFoundException; |
| import java.io.IOException; |
| import java.io.InputStream; |
| import java.nio.file.Files; |
| import java.nio.file.Path; |
| import java.util.Arrays; |
| import java.util.Collection; |
| import java.util.Collections; |
| import java.util.HashMap; |
| import java.util.HashSet; |
| import java.util.List; |
| import java.util.Map; |
| import java.util.Set; |
| import java.util.concurrent.CountDownLatch; |
| import java.util.concurrent.TimeUnit; |
| |
| /** |
| * Collect test results for an entire invocation and output test results to disk. |
| */ |
| @OptionClass(alias="result-reporter") |
| public class ResultReporter implements ILogSaverListener, ITestInvocationListener, |
| ITestSummaryListener, IShardableListener, IConfigurationReceiver { |
| |
| public static final String INCLUDE_HTML_IN_ZIP = "html-in-zip"; |
| private static final String UNKNOWN_DEVICE = "unknown_device"; |
| private static final String RESULT_KEY = "COMPATIBILITY_TEST_RESULT"; |
| private static final String CTS_PREFIX = "cts:"; |
| private static final String BUILD_INFO = CTS_PREFIX + "build_"; |
| private static final String LATEST_LINK_NAME = "latest"; |
| /** Used to get run history from the test result of last run. */ |
| private static final String RUN_HISTORY_KEY = "run_history"; |
| |
| |
| public static final String BUILD_BRAND = "build_brand"; |
| public static final String BUILD_DEVICE = "build_device"; |
| public static final String BUILD_FINGERPRINT = "build_fingerprint"; |
| public static final String BUILD_ID = "build_id"; |
| public static final String BUILD_MANUFACTURER = "build_manufacturer"; |
| public static final String BUILD_MODEL = "build_model"; |
| public static final String BUILD_PRODUCT = "build_product"; |
| public static final String BUILD_VERSION_RELEASE = "build_version_release"; |
| |
| private static final List<String> NOT_RETRY_FILES = Arrays.asList( |
| ChecksumReporter.NAME, |
| ChecksumReporter.PREV_NAME, |
| ResultHandler.FAILURE_REPORT_NAME, |
| "diffs"); |
| |
| @Option(name = RetryFactoryTest.RETRY_OPTION, |
| shortName = 'r', |
| description = "retry a previous session.", |
| importance = Importance.IF_UNSET) |
| private Integer mRetrySessionId = null; |
| |
| @Option(name = RetryFactoryTest.RETRY_TYPE_OPTION, |
| description = "used with " + RetryFactoryTest.RETRY_OPTION |
| + ", retry tests of a certain status. Possible values include \"failed\", " |
| + "\"not_executed\", and \"custom\".", |
| importance = Importance.IF_UNSET) |
| private RetryType mRetryType = null; |
| |
| @Option(name = "result-server", description = "Server to publish test results.") |
| private String mResultServer; |
| |
| @Option(name = "disable-result-posting", description = "Disable result posting into report server.") |
| private boolean mDisableResultPosting = false; |
| |
| @Option(name = "include-test-log-tags", description = "Include test log tags in report.") |
| private boolean mIncludeTestLogTags = false; |
| |
| @Option(name = "use-log-saver", description = "Also saves generated result with log saver") |
| private boolean mUseLogSaver = false; |
| |
| @Option(name = "compress-logs", description = "Whether logs will be saved with compression") |
| private boolean mCompressLogs = true; |
| |
| @Option(name = INCLUDE_HTML_IN_ZIP, |
| description = "Whether failure summary report is included in the zip fie.") |
| private boolean mIncludeHtml = false; |
| |
| @Option( |
| name = "result-attribute", |
| description = |
| "Extra key-value pairs to be added as attributes and corresponding" |
| + "values of the \"Result\" tag in the result XML.") |
| private Map<String, String> mResultAttributes = new HashMap<String, String>(); |
| |
| private CompatibilityBuildHelper mBuildHelper; |
| private File mResultDir = null; |
| private File mLogDir = null; |
| private ResultUploader mUploader; |
| private String mReferenceUrl; |
| private ILogSaver mLogSaver; |
| private int invocationEndedCount = 0; |
| private CountDownLatch mFinalized = null; |
| |
| protected IInvocationResult mResult = new InvocationResult(); |
| private IModuleResult mCurrentModuleResult; |
| private ICaseResult mCurrentCaseResult; |
| private ITestResult mCurrentResult; |
| private String mDeviceSerial = UNKNOWN_DEVICE; |
| private Set<String> mMasterDeviceSerials = new HashSet<>(); |
| private Set<IBuildInfo> mMasterBuildInfos = new HashSet<>(); |
| // Whether or not we failed the fingerprint check |
| private boolean mFingerprintFailure = false; |
| |
| // mCurrentTestNum and mTotalTestsInModule track the progress within the module |
| // Note that this count is not necessarily equal to the count of tests contained |
| // in mCurrentModuleResult because of how special cases like ignored tests are reported. |
| private int mCurrentTestNum; |
| private int mTotalTestsInModule; |
| |
| // Whether modules can be marked done for this invocation. Initialized in invocationStarted() |
| // Visible for unit testing |
| protected boolean mCanMarkDone; |
| // Whether the current test run has failed. If true, we will not mark the current module done |
| protected boolean mTestRunFailed; |
| // Whether the current module has previously been marked done |
| private boolean mModuleWasDone; |
| |
| // Nullable. If null, "this" is considered the master and must handle |
| // result aggregation and reporting. When not null, it should forward events |
| // to the master. |
| private final ResultReporter mMasterResultReporter; |
| |
| private LogFileSaver mTestLogSaver; |
| |
| // Elapsed time from invocation started to ended. |
| private long mElapsedTime; |
| |
| /** Invocation level configuration */ |
| private IConfiguration mConfiguration = null; |
| |
| /** |
| * Default constructor. |
| */ |
| public ResultReporter() { |
| this(null); |
| mFinalized = new CountDownLatch(1); |
| } |
| |
| /** |
| * Construct a shard ResultReporter that forwards module results to the |
| * masterResultReporter. |
| */ |
| public ResultReporter(ResultReporter masterResultReporter) { |
| mMasterResultReporter = masterResultReporter; |
| } |
| |
| /** {@inheritDoc} */ |
| @Override |
| public void setConfiguration(IConfiguration configuration) { |
| mConfiguration = configuration; |
| } |
| |
| /** |
| * {@inheritDoc} |
| */ |
| @Override |
| public void invocationStarted(IInvocationContext context) { |
| IBuildInfo primaryBuild = context.getBuildInfos().get(0); |
| synchronized(this) { |
| if (mBuildHelper == null) { |
| mBuildHelper = new CompatibilityBuildHelper(primaryBuild); |
| } |
| if (mDeviceSerial == null && primaryBuild.getDeviceSerial() != null) { |
| mDeviceSerial = primaryBuild.getDeviceSerial(); |
| } |
| mCanMarkDone = canMarkDone(mBuildHelper.getRecentCommandLineArgs()); |
| } |
| |
| if (isShardResultReporter()) { |
| // Shard ResultReporters forward invocationStarted to the mMasterResultReporter |
| mMasterResultReporter.invocationStarted(context); |
| return; |
| } |
| |
| // NOTE: Everything after this line only applies to the master ResultReporter. |
| |
| synchronized(this) { |
| if (primaryBuild.getDeviceSerial() != null) { |
| // The master ResultReporter collects all device serials being used |
| // for the current implementation. |
| mMasterDeviceSerials.add(primaryBuild.getDeviceSerial()); |
| } |
| |
| // The master ResultReporter collects all buildInfos. |
| mMasterBuildInfos.add(primaryBuild); |
| |
| if (mResultDir == null) { |
| // For the non-sharding case, invocationStarted is only called once, |
| // but for the sharding case, this might be called multiple times. |
| // Logic used to initialize the result directory should not be |
| // invoked twice during the same invocation. |
| initializeResultDirectories(); |
| } |
| } |
| } |
| |
| /** |
| * Create directory structure where results and logs will be written. |
| */ |
| private void initializeResultDirectories() { |
| debug("Initializing result directory"); |
| |
| try { |
| // Initialize the result directory. Either a new directory or reusing |
| // an existing session. |
| if (mRetrySessionId != null) { |
| // Overwrite the mResult with the test results of the previous session |
| mResult = ResultHandler.findResult(mBuildHelper.getResultsDir(), mRetrySessionId); |
| } |
| mResult.setStartTime(mBuildHelper.getStartTime()); |
| mResultDir = mBuildHelper.getResultDir(); |
| if (mResultDir != null) { |
| mResultDir.mkdirs(); |
| } |
| } catch (FileNotFoundException e) { |
| throw new RuntimeException(e); |
| } |
| |
| if (mResultDir == null) { |
| throw new RuntimeException("Result Directory was not created"); |
| } |
| if (!mResultDir.exists()) { |
| throw new RuntimeException("Result Directory was not created: " + |
| mResultDir.getAbsolutePath()); |
| } |
| |
| debug("Results Directory: %s", mResultDir.getAbsolutePath()); |
| |
| mUploader = new ResultUploader(mResultServer, mBuildHelper.getSuiteName()); |
| try { |
| mLogDir = new File(mBuildHelper.getLogsDir(), |
| CompatibilityBuildHelper.getDirSuffix(mBuildHelper.getStartTime())); |
| } catch (FileNotFoundException e) { |
| CLog.e(e); |
| } |
| if (mLogDir != null && mLogDir.mkdirs()) { |
| debug("Created log dir %s", mLogDir.getAbsolutePath()); |
| } |
| if (mLogDir == null || !mLogDir.exists()) { |
| throw new IllegalArgumentException(String.format("Could not create log dir %s", |
| mLogDir.getAbsolutePath())); |
| } |
| if (mTestLogSaver == null) { |
| mTestLogSaver = new LogFileSaver(mLogDir); |
| } |
| } |
| |
| /** |
| * {@inheritDoc} |
| */ |
| @Override |
| public void testRunStarted(String id, int numTests) { |
| if (mCurrentModuleResult != null && mCurrentModuleResult.getId().equals(id) |
| && mCurrentModuleResult.isDone()) { |
| // Modules run with JarHostTest treat each test class as a separate module, |
| // resulting in additional unexpected test runs. |
| // This case exists only for N |
| mTotalTestsInModule += numTests; |
| } else { |
| // Handle non-JarHostTest case |
| mCurrentModuleResult = mResult.getOrCreateModule(id); |
| mModuleWasDone = mCurrentModuleResult.isDone(); |
| mTestRunFailed = false; |
| if (!mModuleWasDone) { |
| // we only want to update testRun variables if the IModuleResult is not yet done |
| // otherwise leave testRun variables alone so isDone evaluates to true. |
| if (mCurrentModuleResult.getExpectedTestRuns() == 0) { |
| mCurrentModuleResult.setExpectedTestRuns(TestRunHandler.getTestRuns( |
| mBuildHelper, mCurrentModuleResult.getId())); |
| } |
| mCurrentModuleResult.addTestRun(); |
| } |
| // Reset counters |
| mTotalTestsInModule = numTests; |
| mCurrentTestNum = 0; |
| } |
| mCurrentModuleResult.inProgress(true); |
| } |
| |
| /** |
| * {@inheritDoc} |
| */ |
| @Override |
| public void testStarted(TestDescription test) { |
| mCurrentCaseResult = mCurrentModuleResult.getOrCreateResult(test.getClassName()); |
| mCurrentResult = mCurrentCaseResult.getOrCreateResult(test.getTestName().trim()); |
| if (mCurrentResult.isRetry()) { |
| mCurrentResult.reset(); // clear result status for this invocation |
| } |
| mCurrentTestNum++; |
| } |
| |
| /** |
| * {@inheritDoc} |
| */ |
| @Override |
| public void testEnded(TestDescription test, HashMap<String, Metric> metrics) { |
| if (mCurrentResult.getResultStatus() == TestStatus.FAIL) { |
| // Test has previously failed. |
| return; |
| } |
| // device test can have performance results in test metrics |
| Metric perfResult = metrics.get(RESULT_KEY); |
| ReportLog report = null; |
| if (perfResult != null) { |
| try { |
| report = ReportLog.parse(perfResult.getMeasurements().getSingleString()); |
| } catch (XmlPullParserException | IOException e) { |
| e.printStackTrace(); |
| } |
| } else { |
| // host test should be checked into MetricsStore. |
| report = MetricsStore.removeResult(mBuildHelper.getBuildInfo(), |
| mCurrentModuleResult.getAbi(), test.toString()); |
| } |
| if (mCurrentResult.getResultStatus() == null) { |
| // Only claim that we passed when we're certain our result was |
| // not any other state. |
| mCurrentResult.passed(report); |
| } |
| } |
| |
| /** |
| * {@inheritDoc} |
| */ |
| @Override |
| public void testIgnored(TestDescription test) { |
| mCurrentResult.skipped(); |
| } |
| |
| /** |
| * {@inheritDoc} |
| */ |
| @Override |
| public void testFailed(TestDescription test, String trace) { |
| mCurrentResult.failed(sanitizeXmlContent(trace)); |
| } |
| |
| /** |
| * {@inheritDoc} |
| */ |
| @Override |
| public void testAssumptionFailure(TestDescription test, String trace) { |
| mCurrentResult.skipped(); |
| } |
| |
| /** |
| * {@inheritDoc} |
| */ |
| @Override |
| public void testRunStopped(long elapsedTime) { |
| // ignore |
| } |
| |
| /** |
| * {@inheritDoc} |
| */ |
| @Override |
| public void testRunEnded(long elapsedTime, Map<String, String> metrics) { |
| testRunEnded(elapsedTime, TfMetricProtoUtil.upgradeConvert(metrics)); |
| } |
| |
| /** |
| * {@inheritDoc} |
| */ |
| @Override |
| public void testRunEnded(long elapsedTime, HashMap<String, Metric> metrics) { |
| mCurrentModuleResult.inProgress(false); |
| mCurrentModuleResult.addRuntime(elapsedTime); |
| if (!mModuleWasDone && mCanMarkDone) { |
| if (mTestRunFailed) { |
| // set done to false for test run failures |
| mCurrentModuleResult.setDone(false); |
| } else { |
| // Only mark module done if: |
| // - status of the invocation allows it (mCanMarkDone), and |
| // - module has not already been marked done, and |
| // - no test run failure has been detected |
| mCurrentModuleResult.setDone(mCurrentTestNum >= mTotalTestsInModule); |
| } |
| } |
| if (isShardResultReporter()) { |
| // Forward module results to the master. |
| mMasterResultReporter.mergeModuleResult(mCurrentModuleResult); |
| mCurrentModuleResult.resetTestRuns(); |
| mCurrentModuleResult.resetRuntime(); |
| } |
| } |
| |
| /** |
| * Directly add a module result. Note: this method is meant to be used by |
| * a shard ResultReporter. |
| */ |
| private void mergeModuleResult(IModuleResult moduleResult) { |
| // This merges the results in moduleResult to any existing results already |
| // contained in mResult. This is useful for retries and allows the final |
| // report from a retry to contain all test results. |
| synchronized(this) { |
| mResult.mergeModuleResult(moduleResult); |
| } |
| } |
| |
| /** |
| * {@inheritDoc} |
| */ |
| @Override |
| public void testRunFailed(String errorMessage) { |
| mTestRunFailed = true; |
| mCurrentModuleResult.setFailed(); |
| } |
| |
| /** |
| * {@inheritDoc} |
| */ |
| @Override |
| public TestSummary getSummary() { |
| // ignore |
| return null; |
| } |
| |
| /** |
| * {@inheritDoc} |
| */ |
| @Override |
| public void putSummary(List<TestSummary> summaries) { |
| for (TestSummary summary : summaries) { |
| // If one summary is from SuiteResultReporter, log it as an extra file. |
| if (SuiteResultReporter.SUITE_REPORTER_SOURCE.equals(summary.getSource())) { |
| File summaryFile = null; |
| try { |
| summaryFile = FileUtil.createTempFile("summary", ".txt"); |
| FileUtil.writeToFile(summary.getSummary().getString(), summaryFile); |
| try (InputStreamSource stream = new FileInputStreamSource(summaryFile)) { |
| testLog("summary", LogDataType.TEXT, stream); |
| } |
| } catch (IOException e) { |
| CLog.e(e); |
| } finally { |
| FileUtil.deleteFile(summaryFile); |
| } |
| } else if (mReferenceUrl == null && summary.getSummary().getString() != null) { |
| mReferenceUrl = summary.getSummary().getString(); |
| } |
| } |
| } |
| |
| /** |
| * {@inheritDoc} |
| */ |
| @Override |
| public void invocationEnded(long elapsedTime) { |
| if (isShardResultReporter()) { |
| // Shard ResultReporters report |
| mMasterResultReporter.invocationEnded(elapsedTime); |
| return; |
| } |
| |
| // NOTE: Everything after this line only applies to the master ResultReporter. |
| |
| synchronized(this) { |
| // The master ResultReporter tracks the progress of all invocations across |
| // shard ResultReporters. Writing results should not proceed until all |
| // ResultReporters have completed. |
| if (++invocationEndedCount < mMasterBuildInfos.size()) { |
| return; |
| } |
| mElapsedTime = elapsedTime; |
| finalizeResults(); |
| mFinalized.countDown(); |
| } |
| } |
| |
| /** |
| * Returns whether a report creation should be skipped. |
| */ |
| protected boolean shouldSkipReportCreation() { |
| // This value is always false here for backwards compatibility. |
| // Extended classes have the option to override this. |
| return false; |
| } |
| |
| private void finalizeResults() { |
| if (mFingerprintFailure) { |
| CLog.w("Failed the fingerprint check. Skip result reporting."); |
| return; |
| } |
| // Add all device serials into the result to be serialized |
| for (String deviceSerial : mMasterDeviceSerials) { |
| mResult.addDeviceSerial(deviceSerial); |
| } |
| |
| addDeviceBuildInfoToResult(); |
| |
| Set<String> allExpectedModules = new HashSet<>(); |
| for (IBuildInfo buildInfo : mMasterBuildInfos) { |
| for (Map.Entry<String, String> entry : buildInfo.getBuildAttributes().entrySet()) { |
| String key = entry.getKey(); |
| String value = entry.getValue(); |
| if (key.equals(CompatibilityBuildHelper.MODULE_IDS) && value.length() > 0) { |
| Collections.addAll(allExpectedModules, value.split(",")); |
| } |
| } |
| } |
| |
| // Include a record in the report of all expected modules ids, even if they weren't |
| // executed. |
| for (String moduleId : allExpectedModules) { |
| mResult.getOrCreateModule(moduleId); |
| } |
| |
| String moduleProgress = String.format("%d of %d", |
| mResult.getModuleCompleteCount(), mResult.getModules().size()); |
| |
| |
| if (shouldSkipReportCreation()) { |
| return; |
| } |
| |
| // Get run history from the test result of last run and add the run history of the current |
| // run to it. |
| // TODO(b/137973382): avoid casting by move the method to interface level. |
| Collection<RunHistory> runHistories = ((InvocationResult) mResult).getRunHistories(); |
| String runHistoryJSON = mResult.getInvocationInfo().get(RUN_HISTORY_KEY); |
| Gson gson = new Gson(); |
| if (runHistoryJSON != null) { |
| RunHistory[] runHistoryArray = gson.fromJson(runHistoryJSON, RunHistory[].class); |
| Collections.addAll(runHistories, runHistoryArray); |
| } |
| RunHistory newRun = new RunHistory(); |
| newRun.startTime = mResult.getStartTime(); |
| newRun.endTime = newRun.startTime + mElapsedTime; |
| runHistories.add(newRun); |
| mResult.addInvocationInfo(RUN_HISTORY_KEY, gson.toJson(runHistories)); |
| |
| try { |
| // Zip the full test results directory. |
| copyDynamicConfigFiles(); |
| copyFormattingFiles(mResultDir, mBuildHelper.getSuiteName()); |
| |
| File resultFile = generateResultXmlFile(); |
| if (mRetrySessionId != null) { |
| copyRetryFiles(ResultHandler.getResultDirectory( |
| mBuildHelper.getResultsDir(), mRetrySessionId), mResultDir); |
| } |
| File failureReport = null; |
| if (mIncludeHtml) { |
| // Create the html report before the zip file. |
| failureReport = ResultHandler.createFailureReport(resultFile); |
| } |
| File zippedResults = zipResults(mResultDir); |
| if (!mIncludeHtml) { |
| // Create failure report after zip file so extra data is not uploaded |
| failureReport = ResultHandler.createFailureReport(resultFile); |
| } |
| if (failureReport != null && failureReport.exists()) { |
| info("Test Result: %s", failureReport.getCanonicalPath()); |
| } else { |
| info("Test Result: %s", resultFile.getCanonicalPath()); |
| } |
| info("Test Logs: %s", mLogDir.getCanonicalPath()); |
| debug("Full Result: %s", zippedResults.getCanonicalPath()); |
| |
| Path latestLink = createLatestLinkDirectory(mResultDir.toPath()); |
| if (latestLink != null) { |
| info("Latest results link: " + latestLink.toAbsolutePath()); |
| } |
| |
| latestLink = createLatestLinkDirectory(mLogDir.toPath()); |
| if (latestLink != null) { |
| info("Latest logs link: " + latestLink.toAbsolutePath()); |
| } |
| |
| saveLog(resultFile, zippedResults); |
| |
| uploadResult(resultFile); |
| |
| } catch (IOException | XmlPullParserException e) { |
| CLog.e("[%s] Exception while saving result XML.", mDeviceSerial); |
| CLog.e(e); |
| } |
| // print the run results last. |
| info("Invocation finished in %s. PASSED: %d, FAILED: %d, MODULES: %s", |
| TimeUtil.formatElapsedTime(mElapsedTime), |
| mResult.countResults(TestStatus.PASS), |
| mResult.countResults(TestStatus.FAIL), |
| moduleProgress); |
| } |
| |
| private Path createLatestLinkDirectory(Path directory) { |
| Path link = null; |
| |
| Path parent = directory.getParent(); |
| |
| if (parent != null) { |
| link = parent.resolve(LATEST_LINK_NAME); |
| try { |
| // if latest already exists, we have to remove it before creating |
| Files.deleteIfExists(link); |
| Files.createSymbolicLink(link, directory); |
| } catch (IOException ioe) { |
| CLog.e("Exception while attempting to create 'latest' link to: [%s]", |
| directory); |
| CLog.e(ioe); |
| return null; |
| } catch (UnsupportedOperationException uoe) { |
| CLog.e("Failed to create 'latest' symbolic link - unsupported operation"); |
| return null; |
| } |
| } |
| return link; |
| } |
| |
| /** |
| * {@inheritDoc} |
| */ |
| @Override |
| public void invocationFailed(Throwable cause) { |
| warn("Invocation failed: %s", cause); |
| InvocationFailureHandler.setFailed(mBuildHelper, cause); |
| if (cause instanceof FingerprintComparisonException) { |
| mFingerprintFailure = true; |
| } |
| } |
| |
| /** |
| * {@inheritDoc} |
| */ |
| @Override |
| public void testLog(String name, LogDataType type, InputStreamSource stream) { |
| // This is safe to be invoked on either the master or a shard ResultReporter |
| if (isShardResultReporter()) { |
| // Shard ResultReporters forward testLog to the mMasterResultReporter |
| mMasterResultReporter.testLog(name, type, stream); |
| return; |
| } |
| if (name.endsWith(DeviceInfo.FILE_SUFFIX)) { |
| // Handle device info file case |
| testLogDeviceInfo(name, stream); |
| } else { |
| // Handle default case |
| try { |
| File logFile = null; |
| if (mCompressLogs) { |
| try (InputStream inputStream = stream.createInputStream()) { |
| logFile = mTestLogSaver.saveAndGZipLogData(name, type, inputStream); |
| } |
| } else { |
| try (InputStream inputStream = stream.createInputStream()) { |
| logFile = mTestLogSaver.saveLogData(name, type, inputStream); |
| } |
| } |
| debug("Saved logs for %s in %s", name, logFile.getAbsolutePath()); |
| } catch (IOException e) { |
| warn("Failed to write log for %s", name); |
| CLog.e(e); |
| } |
| } |
| } |
| |
| /* Write device-info files to the result, invoked only by the master result reporter */ |
| private void testLogDeviceInfo(String name, InputStreamSource stream) { |
| try { |
| File ediDir = new File(mResultDir, DeviceInfo.RESULT_DIR_NAME); |
| ediDir.mkdirs(); |
| File ediFile = new File(ediDir, name); |
| if (!ediFile.exists()) { |
| // only write this file to the results if not already present |
| FileUtil.writeToFile(stream.createInputStream(), ediFile); |
| } |
| } catch (IOException e) { |
| warn("Failed to write device info %s to result", name); |
| CLog.e(e); |
| } |
| } |
| |
| /** |
| * {@inheritDoc} |
| */ |
| @Override |
| public void testLogSaved(String dataName, LogDataType dataType, InputStreamSource dataStream, |
| LogFile logFile) { |
| // This is safe to be invoked on either the master or a shard ResultReporter |
| if (mIncludeTestLogTags && mCurrentResult != null |
| && dataName.startsWith(mCurrentResult.getFullName())) { |
| |
| if (dataType == LogDataType.BUGREPORT) { |
| mCurrentResult.setBugReport(logFile.getUrl()); |
| } else if (dataType == LogDataType.LOGCAT) { |
| mCurrentResult.setLog(logFile.getUrl()); |
| } else if (dataType == LogDataType.PNG) { |
| mCurrentResult.setScreenshot(logFile.getUrl()); |
| } |
| } |
| } |
| |
| /** |
| * {@inheritDoc} |
| */ |
| @Override |
| public void setLogSaver(ILogSaver saver) { |
| // This is safe to be invoked on either the master or a shard ResultReporter |
| mLogSaver = saver; |
| } |
| |
| /** |
| * When enabled, save log data using log saver |
| */ |
| private void saveLog(File resultFile, File zippedResults) throws IOException { |
| if (!mUseLogSaver) { |
| return; |
| } |
| |
| FileInputStream fis = null; |
| LogFile logFile = null; |
| try { |
| fis = new FileInputStream(resultFile); |
| logFile = mLogSaver.saveLogData("log-result", LogDataType.XML, fis); |
| debug("Result XML URL: %s", logFile.getUrl()); |
| logReportFiles(mConfiguration, resultFile, resultFile.getName(), LogDataType.XML); |
| } catch (IOException ioe) { |
| CLog.e("[%s] error saving XML with log saver", mDeviceSerial); |
| CLog.e(ioe); |
| } finally { |
| StreamUtil.close(fis); |
| } |
| // Save the full results folder. |
| if (zippedResults != null) { |
| FileInputStream zipResultStream = null; |
| try { |
| zipResultStream = new FileInputStream(zippedResults); |
| logFile = mLogSaver.saveLogData("results", LogDataType.ZIP, zipResultStream); |
| debug("Result zip URL: %s", logFile.getUrl()); |
| logReportFiles( |
| mConfiguration, zippedResults, "results", LogDataType.ZIP); |
| } finally { |
| StreamUtil.close(zipResultStream); |
| } |
| } |
| } |
| |
| /** |
| * Return the path in which log saver persists log files or null if |
| * logSaver is not enabled. |
| */ |
| private String getLogUrl() { |
| if (!mUseLogSaver || mLogSaver == null) { |
| return null; |
| } |
| |
| return mLogSaver.getLogReportDir().getUrl(); |
| } |
| |
| @Override |
| public IShardableListener clone() { |
| ResultReporter clone = new ResultReporter(this); |
| OptionCopier.copyOptionsNoThrow(this, clone); |
| return clone; |
| } |
| |
| /** |
| * Create results file compatible with CTSv2 (xml) report format. |
| */ |
| protected File generateResultXmlFile() |
| throws IOException, XmlPullParserException { |
| return ResultHandler.writeResults( |
| mBuildHelper.getSuiteName(), |
| mBuildHelper.getSuiteVersion(), |
| getSuitePlan(mBuildHelper), |
| mBuildHelper.getSuiteBuild(), |
| mResult, |
| mResultDir, |
| mResult.getStartTime(), |
| mElapsedTime + mResult.getStartTime(), |
| mReferenceUrl, |
| getLogUrl(), |
| mBuildHelper.getCommandLineArgs(), |
| mResultAttributes); |
| } |
| |
| /** |
| * Add build info collected from the device attributes to the results. |
| */ |
| protected void addDeviceBuildInfoToResult() { |
| // Add all build info to the result to be serialized |
| Map<String, String> buildProperties = mapBuildInfo(); |
| addBuildInfoToResult(buildProperties, mResult); |
| } |
| |
| /** |
| * Override specific build properties so the report will be associated with the |
| * build fingerprint being certified. |
| */ |
| protected void addDeviceBuildInfoToResult(String buildFingerprintOverride, |
| String manufactureOverride, String modelOverride) { |
| |
| Map<String, String> buildProperties = mapBuildInfo(); |
| |
| // Extract and override values from build fingerprint. |
| // Build fingerprint format: brand/product/device:version/build_id/tags |
| String fingerprintPrefix = buildFingerprintOverride.split(":")[0]; |
| String fingerprintTail = buildFingerprintOverride.split(":")[1]; |
| String buildIdOverride = fingerprintTail.split("/")[1]; |
| buildProperties.put(BUILD_ID, buildIdOverride); |
| String brandOverride = fingerprintPrefix.split("/")[0]; |
| buildProperties.put(BUILD_BRAND, brandOverride); |
| String deviceOverride = fingerprintPrefix.split("/")[2]; |
| buildProperties.put(BUILD_DEVICE, deviceOverride); |
| String productOverride = fingerprintPrefix.split("/")[1]; |
| buildProperties.put(BUILD_PRODUCT, productOverride); |
| String versionOverride = fingerprintTail.split("/")[0]; |
| buildProperties.put(BUILD_VERSION_RELEASE, versionOverride); |
| buildProperties.put(BUILD_FINGERPRINT, buildFingerprintOverride); |
| buildProperties.put(BUILD_MANUFACTURER, manufactureOverride); |
| buildProperties.put(BUILD_MODEL, modelOverride); |
| |
| // Add modified values to results. |
| addBuildInfoToResult(buildProperties, mResult); |
| mResult.setBuildFingerprint(buildFingerprintOverride); |
| } |
| /** Aggregate build info from member device info. */ |
| protected Map<String, String> mapBuildInfo() { |
| Map<String, String> buildProperties = new HashMap<>(); |
| for (IBuildInfo buildInfo : mMasterBuildInfos) { |
| for (Map.Entry<String, String> entry : buildInfo.getBuildAttributes().entrySet()) { |
| String key = entry.getKey(); |
| String value = entry.getValue(); |
| if (key.startsWith(BUILD_INFO)) { |
| buildProperties.put(key.substring(CTS_PREFIX.length()), value); |
| } |
| } |
| } |
| return buildProperties; |
| } |
| |
| /** |
| * Add build info to results. |
| * @param buildProperties Build info to add. |
| */ |
| protected static void addBuildInfoToResult(Map<String, String> buildProperties, |
| IInvocationResult invocationResult) { |
| buildProperties.entrySet().stream().forEach(entry -> |
| invocationResult.addInvocationInfo(entry.getKey(), entry.getValue())); |
| } |
| |
| /** |
| * Get the suite plan. This protected method was created for overrides. |
| * Extending classes can decide on the content of the output's suite_plan field. |
| * |
| * @param mBuildHelper Helper that contains build information. |
| * @return string Suite plan to use. |
| */ |
| protected String getSuitePlan(CompatibilityBuildHelper mBuildHelper) { |
| return mBuildHelper.getSuitePlan(); |
| } |
| |
| /** |
| * Return true if this instance is a shard ResultReporter and should propagate |
| * certain events to the master. |
| */ |
| private boolean isShardResultReporter() { |
| return mMasterResultReporter != null; |
| } |
| |
| /** |
| * When enabled, upload the result to a server. |
| */ |
| private void uploadResult(File resultFile) { |
| if (mResultServer != null && !mResultServer.trim().isEmpty() && !mDisableResultPosting) { |
| try { |
| debug("Result Server: %d", mUploader.uploadResult(resultFile, mReferenceUrl)); |
| } catch (IOException ioe) { |
| CLog.e("[%s] IOException while uploading result.", mDeviceSerial); |
| CLog.e(ioe); |
| } |
| } |
| } |
| |
| /** |
| * Returns whether it is safe to mark modules as "done", given the invocation command-line |
| * arguments. Returns true unless this is a retry and specific filtering techniques are applied |
| * on the command-line, such as: |
| * --retry-type failed |
| * --include-filter |
| * --exclude-filter |
| * -t/--test |
| * --subplan |
| */ |
| private boolean canMarkDone(String args) { |
| if (mRetrySessionId == null) { |
| return true; // always allow modules to be marked done if not retry |
| } |
| return !(RetryType.FAILED.equals(mRetryType) |
| || RetryType.CUSTOM.equals(mRetryType) |
| || args.contains(CompatibilityTestSuite.INCLUDE_FILTER_OPTION) |
| || args.contains(CompatibilityTestSuite.EXCLUDE_FILTER_OPTION) |
| || args.contains(CompatibilityTestSuite.SUBPLAN_OPTION) |
| || args.matches(String.format(".* (-%s|--%s) .*", |
| CompatibilityTestSuite.TEST_OPTION_SHORT_NAME, CompatibilityTestSuite.TEST_OPTION))); |
| } |
| |
| /** |
| * Copy the xml formatting files stored in this jar to the results directory |
| * |
| * @param resultsDir |
| */ |
| static void copyFormattingFiles(File resultsDir, String suiteName) { |
| for (String resultFileName : ResultHandler.RESULT_RESOURCES) { |
| InputStream configStream = ResultHandler.class.getResourceAsStream( |
| String.format("/report/%s-%s", suiteName, resultFileName)); |
| if (configStream == null) { |
| // If suite specific files are not available, fallback to common. |
| configStream = ResultHandler.class.getResourceAsStream( |
| String.format("/report/%s", resultFileName)); |
| } |
| if (configStream != null) { |
| File resultFile = new File(resultsDir, resultFileName); |
| try { |
| FileUtil.writeToFile(configStream, resultFile); |
| } catch (IOException e) { |
| warn("Failed to write %s to file", resultFileName); |
| } |
| } else { |
| warn("Failed to load %s from jar", resultFileName); |
| } |
| } |
| } |
| |
| /** |
| * move the dynamic config files to the results directory |
| */ |
| private void copyDynamicConfigFiles() { |
| File configDir = new File(mResultDir, "config"); |
| if (!configDir.mkdir()) { |
| warn("Failed to make dynamic config directory \"%s\" in the result", |
| configDir.getAbsolutePath()); |
| } |
| |
| Set<String> uniqueModules = new HashSet<>(); |
| for (IBuildInfo buildInfo : mMasterBuildInfos) { |
| CompatibilityBuildHelper helper = new CompatibilityBuildHelper(buildInfo); |
| Map<String, File> dcFiles = helper.getDynamicConfigFiles(); |
| for (String moduleName : dcFiles.keySet()) { |
| File srcFile = dcFiles.get(moduleName); |
| if (!uniqueModules.contains(moduleName)) { |
| // have not seen config for this module yet, copy into result |
| File destFile = new File(configDir, moduleName + ".dynamic"); |
| try { |
| FileUtil.copyFile(srcFile, destFile); |
| uniqueModules.add(moduleName); // Add to uniqueModules if copy succeeds |
| } catch (IOException e) { |
| warn("Failure when copying config file \"%s\" to \"%s\" for module %s", |
| srcFile.getAbsolutePath(), destFile.getAbsolutePath(), moduleName); |
| CLog.e(e); |
| } |
| } |
| FileUtil.deleteFile(srcFile); |
| } |
| } |
| } |
| |
| /** |
| * Recursively copy any other files found in the previous session's result directory to the |
| * new result directory, so long as they don't already exist. For example, a "screenshots" |
| * directory generated in a previous session by a passing test will not be generated on retry |
| * unless copied from the old result directory. |
| * |
| * @param oldDir |
| * @param newDir |
| */ |
| static void copyRetryFiles(File oldDir, File newDir) { |
| File[] oldChildren = oldDir.listFiles(); |
| for (File oldChild : oldChildren) { |
| if (NOT_RETRY_FILES.contains(oldChild.getName())) { |
| continue; // do not copy this file/directory or its children |
| } |
| File newChild = new File(newDir, oldChild.getName()); |
| if (!newChild.exists()) { |
| // If this old file or directory doesn't exist in new dir, simply copy it |
| try { |
| if (oldChild.isDirectory()) { |
| FileUtil.recursiveCopy(oldChild, newChild); |
| } else { |
| FileUtil.copyFile(oldChild, newChild); |
| } |
| } catch (IOException e) { |
| warn("Failed to copy file \"%s\" from previous session", oldChild.getName()); |
| } |
| } else if (oldChild.isDirectory() && newChild.isDirectory()) { |
| // If both children exist as directories, make sure the children of the old child |
| // directory exist in the new child directory. |
| copyRetryFiles(oldChild, newChild); |
| } |
| } |
| } |
| |
| /** |
| * Zip the contents of the given results directory. |
| * |
| * @param resultsDir |
| */ |
| private static File zipResults(File resultsDir) { |
| File zipResultFile = null; |
| try { |
| // create a file in parent directory, with same name as resultsDir |
| zipResultFile = new File(resultsDir.getParent(), String.format("%s.zip", |
| resultsDir.getName())); |
| ZipUtil.createZip(resultsDir, zipResultFile); |
| } catch (IOException e) { |
| warn("Failed to create zip for %s", resultsDir.getName()); |
| } |
| return zipResultFile; |
| } |
| |
| /** |
| * Log info to the console. |
| */ |
| private static void info(String format, Object... args) { |
| log(LogLevel.INFO, format, args); |
| } |
| |
| /** |
| * Log debug to the console. |
| */ |
| private static void debug(String format, Object... args) { |
| log(LogLevel.DEBUG, format, args); |
| } |
| |
| /** |
| * Log a warning to the console. |
| */ |
| private static void warn(String format, Object... args) { |
| log(LogLevel.WARN, format, args); |
| } |
| |
| /** |
| * Log a message to the console |
| */ |
| private static void log(LogLevel level, String format, Object... args) { |
| CLog.logAndDisplay(level, format, args); |
| } |
| |
| /** |
| * For testing purpose. |
| */ |
| @VisibleForTesting |
| public IInvocationResult getResult() { |
| return mResult; |
| } |
| |
| /** |
| * Returns true if the reporter is finalized before the end of the timeout. False otherwise. |
| */ |
| @VisibleForTesting |
| public boolean waitForFinalized(long timeout, TimeUnit unit) throws InterruptedException { |
| return mFinalized.await(timeout, unit); |
| } |
| |
| private static String sanitizeXmlContent(String s) { |
| return XmlEscapers.xmlContentEscaper().escape(s); |
| } |
| |
| /** Re-log a result file to all reporters so they are aware of it. */ |
| private void logReportFiles( |
| IConfiguration configuration, File resultFile, String dataName, LogDataType type) { |
| if (configuration == null) { |
| return; |
| } |
| List<ITestInvocationListener> listeners = configuration.getTestInvocationListeners(); |
| try (FileInputStreamSource source = new FileInputStreamSource(resultFile)) { |
| for (ITestInvocationListener listener : listeners) { |
| if (listener.equals(this)) { |
| // Avoid logging agaisnt itself |
| continue; |
| } |
| listener.testLog(dataName, type, source); |
| } |
| } |
| } |
| } |
