| /* |
| * Copyright (c) 2017 Google Inc. All Rights Reserved. |
| * |
| * Licensed under the Apache License, Version 2.0 (the "License"); you |
| * may not use this file except in compliance with the License. You may |
| * obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or |
| * implied. See the License for the specific language governing |
| * permissions and limitations under the License. |
| */ |
| |
| package com.android.vts.util; |
| |
| import com.android.vts.entity.CodeCoverageEntity; |
| import com.android.vts.entity.DeviceInfoEntity; |
| import com.android.vts.entity.ProfilingPointRunEntity; |
| import com.android.vts.entity.TestCaseRunEntity; |
| import com.android.vts.entity.TestCaseRunEntity.TestCase; |
| import com.android.vts.entity.TestEntity; |
| import com.android.vts.entity.TestRunEntity; |
| import com.android.vts.proto.VtsReportMessage.TestCaseResult; |
| import com.android.vts.util.UrlUtil.LinkDisplay; |
| import com.google.appengine.api.datastore.DatastoreService; |
| import com.google.appengine.api.datastore.DatastoreServiceFactory; |
| import com.google.appengine.api.datastore.Entity; |
| import com.google.appengine.api.datastore.Key; |
| import com.google.appengine.api.datastore.KeyFactory; |
| import com.google.appengine.api.datastore.Query; |
| import java.util.ArrayList; |
| import java.util.Arrays; |
| import java.util.HashMap; |
| import java.util.HashSet; |
| import java.util.List; |
| import java.util.Map; |
| import java.util.Set; |
| import java.util.logging.Level; |
| import java.util.logging.Logger; |
| import org.apache.commons.lang.StringUtils; |
| |
| /** Helper object for describing test results data. */ |
| public class TestResults { |
| private final Logger logger = Logger.getLogger(getClass().getName()); |
| |
| private List<TestRunEntity> testRuns; // list of all test runs |
| private Map<Key, List<TestCaseRunEntity>> |
| testCaseRunMap; // map from test run key to the test run information |
| private Map<Key, List<DeviceInfoEntity>> deviceInfoMap; // map from test run key to device info |
| private Map<String, Integer> testCaseNameMap; // map from test case name to its order |
| private Set<String> profilingPointNameSet; // set of profiling point names |
| |
| public String testName; |
| public String[] headerRow; // row to display above the test results table |
| public String[][] timeGrid; // grid of data storing timestamps to render as dates |
| public String[][] durationGrid; // grid of data storing timestamps to render as time intervals |
| public String[][] summaryGrid; // grid of data displaying a summary of the test run |
| public String[][] resultsGrid; // grid of data displaying test case results |
| public String[] profilingPointNames; // list of profiling point names in the test run |
| public Map<String, List<String[]>> logInfoMap; // map from test run index to url/display pairs |
| public int[] totResultCounts; // array of test result counts for the tip-of-tree runs |
| public String totBuildId = ""; // build ID of tip-of-tree run |
| public long startTime = Long.MAX_VALUE; // oldest timestamp displayed in the results table |
| public long endTime = Long.MIN_VALUE; // newest timestamp displayed in the results table |
| |
| // Row labels for the test time-formatted information. |
| private static final String[] TIME_INFO_NAMES = {"Test Start", "Test End"}; |
| |
| // Row labels for the test duration information. |
| private static final String[] DURATION_INFO_NAMES = {"<b>Test Duration</b>"}; |
| |
| // Row labels for the test summary grid. |
| private static final String[] SUMMARY_NAMES = { |
| "Total", "Passing #", "Non-Passing #", "Passing %", "Covered Lines", "Coverage %", "Links" |
| }; |
| |
| // Row labels for the device summary information in the table header. |
| private static final String[] HEADER_NAMES = { |
| "<b>Stats Type \\ Device Build ID</b>", |
| "Branch", |
| "Build Target", |
| "Device", |
| "ABI Target", |
| "VTS Build ID", |
| "Hostname" |
| }; |
| |
| /** |
| * Create a test results object. |
| * |
| * @param testName The name of the test. |
| */ |
| public TestResults(String testName) { |
| this.testName = testName; |
| this.testRuns = new ArrayList<>(); |
| this.deviceInfoMap = new HashMap<>(); |
| this.testCaseRunMap = new HashMap<>(); |
| this.testCaseNameMap = new HashMap<>(); |
| this.logInfoMap = new HashMap<>(); |
| this.profilingPointNameSet = new HashSet<>(); |
| } |
| |
| /** |
| * Add a test run to the test results. |
| * |
| * @param testRun The Entity containing the test run information. |
| * @param testCaseRuns The collection of test case executions within the test run. |
| */ |
| public void addTestRun(Entity testRun, Iterable<Entity> testCaseRuns) { |
| TestRunEntity testRunEntity = TestRunEntity.fromEntity(testRun); |
| if (testRunEntity == null) return; |
| if (testRunEntity.getStartTimestamp() < startTime) { |
| startTime = testRunEntity.getStartTimestamp(); |
| } |
| if (testRunEntity.getStartTimestamp() > endTime) { |
| endTime = testRunEntity.getStartTimestamp(); |
| } |
| testRuns.add(testRunEntity); |
| testCaseRunMap.put(testRun.getKey(), new ArrayList<TestCaseRunEntity>()); |
| |
| // Process the test cases in the test run |
| for (Entity e : testCaseRuns) { |
| TestCaseRunEntity testCaseRunEntity = TestCaseRunEntity.fromEntity(e); |
| if (testCaseRunEntity == null) continue; |
| testCaseRunMap.get(testRun.getKey()).add(testCaseRunEntity); |
| for (TestCase testCase : testCaseRunEntity.testCases) { |
| if (!testCaseNameMap.containsKey(testCase.name)) { |
| testCaseNameMap.put(testCase.name, testCaseNameMap.size()); |
| } |
| } |
| } |
| } |
| |
| /** Creates a test case breakdown of the most recent test run. */ |
| private void generateToTBreakdown() { |
| totResultCounts = new int[TestCaseResult.values().length]; |
| if (testRuns.size() == 0) return; |
| |
| TestRunEntity mostRecentRun = testRuns.get(0); |
| List<TestCaseRunEntity> testCaseResults = testCaseRunMap.get(mostRecentRun.getKey()); |
| List<DeviceInfoEntity> deviceInfos = deviceInfoMap.get(mostRecentRun.getKey()); |
| if (deviceInfos.size() > 0) { |
| DeviceInfoEntity totDevice = deviceInfos.get(0); |
| totBuildId = totDevice.getBuildId(); |
| } |
| // Count array for each test result |
| for (TestCaseRunEntity testCaseRunEntity : testCaseResults) { |
| for (TestCase testCase : testCaseRunEntity.testCases) { |
| totResultCounts[testCase.result]++; |
| } |
| } |
| } |
| |
| /** |
| * Get the number of test runs observed. |
| * |
| * @return The number of test runs observed. |
| */ |
| public int getSize() { |
| return testRuns.size(); |
| } |
| |
| /** Fetch and process profiling point names for the set of test runs. */ |
| private void processProfilingPoints() { |
| DatastoreService datastore = DatastoreServiceFactory.getDatastoreService(); |
| Key testKey = KeyFactory.createKey(TestEntity.KIND, this.testName); |
| Query.Filter profilingFilter = |
| FilterUtil.getProfilingTimeFilter( |
| testKey, TestRunEntity.KIND, this.startTime, this.endTime); |
| Query profilingPointQuery = |
| new Query(ProfilingPointRunEntity.KIND) |
| .setAncestor(testKey) |
| .setFilter(profilingFilter) |
| .setKeysOnly(); |
| Iterable<Entity> profilingPoints = datastore.prepare(profilingPointQuery).asIterable(); |
| // Process the profiling point observations in the test run |
| for (Entity e : profilingPoints) { |
| if (e.getKey().getName() != null) { |
| profilingPointNameSet.add(e.getKey().getName()); |
| } |
| } |
| } |
| |
| /** Fetch and process device information for the set of test runs. */ |
| private void processDeviceInfos() { |
| DatastoreService datastore = DatastoreServiceFactory.getDatastoreService(); |
| Key testKey = KeyFactory.createKey(TestEntity.KIND, this.testName); |
| Query.Filter deviceFilter = |
| FilterUtil.getDeviceTimeFilter( |
| testKey, TestRunEntity.KIND, this.startTime, this.endTime); |
| Query deviceQuery = |
| new Query(DeviceInfoEntity.KIND) |
| .setAncestor(testKey) |
| .setFilter(deviceFilter) |
| .setKeysOnly(); |
| List<Key> deviceGets = new ArrayList<>(); |
| for (Entity device : |
| datastore.prepare(deviceQuery).asIterable(DatastoreHelper.getLargeBatchOptions())) { |
| if (testCaseRunMap.containsKey(device.getParent())) { |
| deviceGets.add(device.getKey()); |
| } |
| } |
| Map<Key, Entity> devices = datastore.get(deviceGets); |
| for (Key key : devices.keySet()) { |
| Entity device = devices.get(key); |
| if (!testCaseRunMap.containsKey(device.getParent())) return; |
| DeviceInfoEntity deviceEntity = DeviceInfoEntity.fromEntity(device); |
| if (deviceEntity == null) return; |
| if (!deviceInfoMap.containsKey(device.getParent())) { |
| deviceInfoMap.put(device.getParent(), new ArrayList<DeviceInfoEntity>()); |
| } |
| deviceInfoMap.get(device.getParent()).add(deviceEntity); |
| } |
| } |
| |
| /** Post-process the test runs to generate reports of the results. */ |
| public void processReport() { |
| if (getSize() > 0) { |
| processDeviceInfos(); |
| processProfilingPoints(); |
| } |
| testRuns.sort((t1, t2) -> new Long(t2.getStartTimestamp()).compareTo(t1.getStartTimestamp())); |
| generateToTBreakdown(); |
| |
| headerRow = new String[testRuns.size() + 1]; |
| headerRow[0] = StringUtils.join(HEADER_NAMES, "<br>"); |
| |
| summaryGrid = new String[SUMMARY_NAMES.length][testRuns.size() + 1]; |
| for (int i = 0; i < SUMMARY_NAMES.length; i++) { |
| summaryGrid[i][0] = "<b>" + SUMMARY_NAMES[i] + "</b>"; |
| } |
| |
| timeGrid = new String[TIME_INFO_NAMES.length][testRuns.size() + 1]; |
| for (int i = 0; i < TIME_INFO_NAMES.length; i++) { |
| timeGrid[i][0] = "<b>" + TIME_INFO_NAMES[i] + "</b>"; |
| } |
| |
| durationGrid = new String[DURATION_INFO_NAMES.length][testRuns.size() + 1]; |
| for (int i = 0; i < DURATION_INFO_NAMES.length; i++) { |
| durationGrid[i][0] = "<b>" + DURATION_INFO_NAMES[i] + "</b>"; |
| } |
| |
| resultsGrid = new String[testCaseNameMap.size()][testRuns.size() + 1]; |
| // first column for results grid |
| for (String testCaseName : testCaseNameMap.keySet()) { |
| resultsGrid[testCaseNameMap.get(testCaseName)][0] = testCaseName; |
| } |
| |
| // Iterate through the test runs |
| for (int col = 0; col < testRuns.size(); col++) { |
| TestRunEntity testRun = testRuns.get(col); |
| CodeCoverageEntity codeCoverageEntity = testRun.getCodeCoverageEntity(); |
| |
| // Process the device information |
| List<DeviceInfoEntity> devices = deviceInfoMap.get(testRun.getKey()); |
| List<String> buildIdList = new ArrayList<>(); |
| List<String> buildAliasList = new ArrayList<>(); |
| List<String> buildFlavorList = new ArrayList<>(); |
| List<String> productVariantList = new ArrayList<>(); |
| List<String> abiInfoList = new ArrayList<>(); |
| for (DeviceInfoEntity deviceInfoEntity : devices) { |
| buildAliasList.add(deviceInfoEntity.getBranch()); |
| buildFlavorList.add(deviceInfoEntity.getBuildFlavor()); |
| productVariantList.add(deviceInfoEntity.getProduct()); |
| buildIdList.add(deviceInfoEntity.getBuildId()); |
| String abi = ""; |
| String abiName = deviceInfoEntity.getAbiName(); |
| String abiBitness = deviceInfoEntity.getAbiBitness(); |
| if (abiName.length() > 0) { |
| abi += abiName; |
| if (abiBitness.length() > 0) { |
| abi += " (" + abiBitness + " bit)"; |
| } |
| } |
| abiInfoList.add(abi); |
| } |
| |
| String buildAlias = StringUtils.join(buildAliasList, ","); |
| String buildFlavor = StringUtils.join(buildFlavorList, ","); |
| String productVariant = StringUtils.join(productVariantList, ","); |
| String buildIds = StringUtils.join(buildIdList, ","); |
| String abiInfo = StringUtils.join(abiInfoList, ","); |
| String vtsBuildId = testRun.getTestBuildId(); |
| |
| int totalCount = 0; |
| int passCount = (int) testRun.getPassCount(); |
| int nonpassCount = (int) testRun.getFailCount(); |
| TestCaseResult aggregateStatus = TestCaseResult.UNKNOWN_RESULT; |
| |
| long totalLineCount = 0; |
| long coveredLineCount = 0; |
| if (testRun.getHasCodeCoverage()) { |
| totalLineCount = codeCoverageEntity.getTotalLineCount(); |
| coveredLineCount = codeCoverageEntity.getCoveredLineCount(); |
| } |
| |
| // Process test case results |
| for (TestCaseRunEntity testCaseEntity : testCaseRunMap.get(testRun.getKey())) { |
| // Update the aggregated test run status |
| totalCount += testCaseEntity.testCases.size(); |
| for (TestCase testCase : testCaseEntity.testCases) { |
| int result = testCase.result; |
| String name = testCase.name; |
| if (result == TestCaseResult.TEST_CASE_RESULT_PASS.getNumber()) { |
| if (aggregateStatus == TestCaseResult.UNKNOWN_RESULT) { |
| aggregateStatus = TestCaseResult.TEST_CASE_RESULT_PASS; |
| } |
| } else if (result != TestCaseResult.TEST_CASE_RESULT_SKIP.getNumber()) { |
| aggregateStatus = TestCaseResult.TEST_CASE_RESULT_FAIL; |
| } |
| |
| String systraceUrl = null; |
| |
| if (testCaseEntity.getSystraceUrl() != null) { |
| String url = testCaseEntity.getSystraceUrl(); |
| LinkDisplay validatedLink = UrlUtil.processUrl(url); |
| if (validatedLink != null) { |
| systraceUrl = validatedLink.url; |
| } else { |
| logger.log(Level.WARNING, "Invalid systrace URL : " + url); |
| } |
| } |
| |
| int index = testCaseNameMap.get(name); |
| String classNames = "test-case-status "; |
| String glyph = ""; |
| TestCaseResult testCaseResult = TestCaseResult.valueOf(result); |
| if (testCaseResult != null) classNames += testCaseResult.toString(); |
| else classNames += TestCaseResult.UNKNOWN_RESULT.toString(); |
| |
| if (systraceUrl != null) { |
| classNames += " width-1"; |
| glyph += |
| "<a href=\"" |
| + systraceUrl |
| + "\" " |
| + "class=\"waves-effect waves-light btn red right inline-btn\">" |
| + "<i class=\"material-icons inline-icon\">info_outline</i></a>"; |
| } |
| resultsGrid[index][col + 1] = |
| "<div class=\"" + classNames + "\"> </div>" + glyph; |
| } |
| } |
| String passInfo; |
| try { |
| double passPct = |
| Math.round((100 * passCount / (passCount + nonpassCount)) * 100f) / 100f; |
| passInfo = Double.toString(passPct) + "%"; |
| } catch (ArithmeticException e) { |
| passInfo = " - "; |
| } |
| |
| // Process coverage metadata |
| String coverageInfo; |
| String coveragePctInfo; |
| try { |
| double coveragePct = |
| Math.round((100 * coveredLineCount / totalLineCount) * 100f) / 100f; |
| coveragePctInfo = |
| Double.toString(coveragePct) |
| + "%" |
| + "<a href=\"/show_coverage?testName=" |
| + testName |
| + "&startTime=" |
| + testRun.getStartTimestamp() |
| + "\" class=\"waves-effect waves-light btn red right inline-btn\">" |
| + "<i class=\"material-icons inline-icon\">menu</i></a>"; |
| coverageInfo = coveredLineCount + "/" + totalLineCount; |
| } catch (ArithmeticException e) { |
| coveragePctInfo = " - "; |
| coverageInfo = " - "; |
| } |
| |
| // Process log information |
| String linkSummary = " - "; |
| List<String[]> linkEntries = new ArrayList<>(); |
| logInfoMap.put(Integer.toString(col), linkEntries); |
| |
| if (testRun.getLogLinks() != null) { |
| for (String rawUrl : testRun.getLogLinks()) { |
| LinkDisplay validatedLink = UrlUtil.processUrl(rawUrl); |
| if (validatedLink == null) { |
| logger.log(Level.WARNING, "Invalid logging URL : " + rawUrl); |
| continue; |
| } |
| String[] logInfo = |
| new String[] { |
| validatedLink.name, |
| validatedLink.url // TODO: process the name from the URL |
| }; |
| linkEntries.add(logInfo); |
| } |
| } |
| if (linkEntries.size() > 0) { |
| linkSummary = Integer.toString(linkEntries.size()); |
| linkSummary += |
| "<i class=\"waves-effect waves-light btn red right inline-btn" |
| + " info-btn material-icons inline-icon\"" |
| + " data-col=\"" |
| + Integer.toString(col) |
| + "\"" |
| + ">launch</i>"; |
| } |
| |
| String icon = "<div class='status-icon " + aggregateStatus.toString() + "'> </div>"; |
| String hostname = testRun.getHostName(); |
| |
| // Populate the header row |
| headerRow[col + 1] = |
| "<span class='valign-wrapper'><b>" |
| + buildIds |
| + "</b>" |
| + icon |
| + "</span>" |
| + buildAlias |
| + "<br>" |
| + buildFlavor |
| + "<br>" |
| + productVariant |
| + "<br>" |
| + abiInfo |
| + "<br>" |
| + vtsBuildId |
| + "<br>" |
| + hostname; |
| |
| // Populate the test summary grid |
| summaryGrid[0][col + 1] = Integer.toString(totalCount); |
| summaryGrid[1][col + 1] = Integer.toString(passCount); |
| summaryGrid[2][col + 1] = Integer.toString(nonpassCount); |
| summaryGrid[3][col + 1] = passInfo; |
| summaryGrid[4][col + 1] = coverageInfo; |
| summaryGrid[5][col + 1] = coveragePctInfo; |
| summaryGrid[6][col + 1] = linkSummary; |
| |
| // Populate the test time info grid |
| timeGrid[0][col + 1] = Long.toString(testRun.getStartTimestamp()); |
| timeGrid[1][col + 1] = Long.toString(testRun.getEndTimestamp()); |
| |
| // Populate the test duration info grid |
| durationGrid[0][col + 1] = Long.toString(testRun.getEndTimestamp() - testRun.getStartTimestamp()); |
| } |
| |
| profilingPointNames = |
| profilingPointNameSet.toArray(new String[profilingPointNameSet.size()]); |
| Arrays.sort(profilingPointNames); |
| } |
| } |
