tools/tradefed-host/src/com/android/cts/tradefed/result/TestPackageResult.java - platform/cts - Git at Google

 /*
  * Copyright (C) 2011 The Android Open Source Project
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
  *      http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
 package com.android.cts.tradefed.result;

 import android.tests.getinfo.DeviceInfoConstants;

 import com.android.cts.tradefed.device.DeviceInfoCollector;
 import com.android.cts.tradefed.testtype.CtsTest;
 import com.android.ddmlib.testrunner.TestIdentifier;
 import com.android.tradefed.log.LogUtil.CLog;

 import org.kxml2.io.KXmlSerializer;
 import org.xmlpull.v1.XmlPullParser;
 import org.xmlpull.v1.XmlPullParserException;

 import java.io.IOException;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Deque;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;

 /**
  * Data structure for a CTS test package result.
  * <p/>
  * Provides methods to serialize to XML.
  */
 class TestPackageResult  extends AbstractXmlPullParser {

     static final String TAG = "TestPackage";
     private static final String DIGEST_ATTR = "digest";
     private static final String APP_PACKAGE_NAME_ATTR = "appPackageName";
     private static final String NAME_ATTR = "name";
     private static final String ns = CtsXmlResultReporter.ns;
     private static final String SIGNATURE_TEST_PKG = "android.tests.sigtest";

     private String mAppPackageName;
     private String mName;
     private String mDigest;

     private Map<String, String> mMetrics = new HashMap<String, String>();

     private TestSuite mSuiteRoot = new TestSuite(null);

     public void setAppPackageName(String appPackageName) {
         mAppPackageName = appPackageName;
     }

     public String getAppPackageName() {
         return mAppPackageName;
     }

     public void setName(String name) {
         mName = name;
     }

     public String getName() {
         return mName;
     }

     public void setDigest(String digest) {
         mDigest = digest;
     }

     public String getDigest() {
         return mDigest;
     }

     /**
      * Return the {@link TestSuite}s
      */
     public Collection<TestSuite> getTestSuites() {
         return mSuiteRoot.getTestSuites();
     }

     /**
      * Adds a test result to this test package
      *
      * @param testId
      * @param testResult
      */
     public Test insertTest(TestIdentifier testId) {
         return findTest(testId, true);
     }

     private Test findTest(TestIdentifier testId, boolean insertIfMissing) {
         List<String> classNameSegments = new LinkedList<String>();
         Collections.addAll(classNameSegments, testId.getClassName().split("\\."));
         if (classNameSegments.size() <= 0) {
             CLog.e("Unrecognized package name format for test class '%s'",
                     testId.getClassName());
             // should never happen
             classNameSegments.add("UnknownTestClass");
         }
             String testCaseName = classNameSegments.remove(classNameSegments.size()-1);
             return mSuiteRoot.findTest(classNameSegments, testCaseName, testId.getTestName(), insertIfMissing);
     }


     /**
      * Find the test result for given {@link TestIdentifier}.
      * @param testId
      * @return the {@link Test} or <code>null</code>
      */
     public Test findTest(TestIdentifier testId) {
         return findTest(testId, false);
     }

     /**
      * Serialize this object and all its contents to XML.
      *
      * @param serializer
      * @throws IOException
      */
     public void serialize(KXmlSerializer serializer) throws IOException {
         serializer.startTag(ns, TAG);
         serializeAttribute(serializer, NAME_ATTR, mName);
         serializeAttribute(serializer, APP_PACKAGE_NAME_ATTR, mAppPackageName);
         serializeAttribute(serializer, DIGEST_ATTR, getDigest());
         if (SIGNATURE_TEST_PKG.equals(mName)) {
             serializer.attribute(ns, "signatureCheck", "true");
         }
         mSuiteRoot.serialize(serializer);
         serializer.endTag(ns, TAG);
     }

     /**
      * Helper method to serialize attributes.
      * Can handle null values. Useful for cases where test package has not been fully populated
      * such as when unit testing.
      *
      * @param attrName
      * @param attrValue
      * @throws IOException
      */
     private void serializeAttribute(KXmlSerializer serializer, String attrName, String attrValue)
             throws IOException {
         attrValue = attrValue == null ? "" : attrValue;
         serializer.attribute(ns, attrName, attrValue);
     }

     /**
      * Populates this class with package result data parsed from XML.
      *
      * @param parser the {@link XmlPullParser}. Expected to be pointing at start
      *            of TestPackage tag
      */
     @Override
     void parse(XmlPullParser parser) throws XmlPullParserException, IOException {
         if (!parser.getName().equals(TAG)) {
             throw new XmlPullParserException(String.format(
                     "invalid XML: Expected %s tag but received %s", TAG, parser.getName()));
         }
         setAppPackageName(getAttribute(parser, APP_PACKAGE_NAME_ATTR));
         setName(getAttribute(parser, NAME_ATTR));
         setDigest(getAttribute(parser, DIGEST_ATTR));
         int eventType = parser.getEventType();
         while (eventType != XmlPullParser.END_DOCUMENT) {
             if (eventType == XmlPullParser.START_TAG && parser.getName().equals(TestSuite.TAG)) {
                 TestSuite suite = new TestSuite();
                 suite.parse(parser);
                 mSuiteRoot.insertSuite(suite);
             }
             if (eventType == XmlPullParser.END_TAG && parser.getName().equals(TAG)) {
                 return;
             }
             eventType = parser.next();
         }
     }

     /**
      * Return a list of {@link TestIdentifer}s contained in this result with the given status
      *
      * @param resultFilter the {@link CtsTestStatus} to filter by
      * @return a collection of {@link TestIdentifer}s
      */
     public Collection<TestIdentifier> getTestsWithStatus(CtsTestStatus resultFilter) {
         Collection<TestIdentifier> tests = new LinkedList<TestIdentifier>();
         Deque<String> suiteNames = new LinkedList<String>();
         mSuiteRoot.addTestsWithStatus(tests, suiteNames, resultFilter);
         return tests;
     }

     /**
      * Populate values in this package result from run metrics
      * @param runResult
      */
     public void populateMetrics(Map<String, String> metrics) {
         String name = metrics.get(CtsTest.PACKAGE_NAME_METRIC);
         if (name != null) {
             setName(name);
         }
         String digest = metrics.get(CtsTest.PACKAGE_DIGEST_METRIC);
         if (digest != null) {
             setDigest(digest);
         }
         if (DeviceInfoCollector.APP_PACKAGE_NAME.equals(getAppPackageName())) {
             storeDeviceMetrics(metrics);
         } else {
             mMetrics.putAll(metrics);
         }
     }

     /**
      * Check that the provided device info metrics are consistent with the currently stored metrics.
      * <p/>
      * If any inconsistencies occur, logs errors and stores error messages in the metrics map
      *
      * @param metrics
      */
     private void storeDeviceMetrics(Map<String, String> metrics) {
         // TODO centralize all the device metrics handling into a single class
         if (mMetrics.isEmpty()) {
             // nothing to check!
             mMetrics.putAll(metrics);
             return;
         }
         // ensure all the metrics we expect to be identical actually are
         checkMetrics(metrics, DeviceInfoConstants.BUILD_FINGERPRINT,
                 DeviceInfoConstants.BUILD_MODEL, DeviceInfoConstants.BUILD_BRAND,
                 DeviceInfoConstants.BUILD_MANUFACTURER, DeviceInfoConstants.BUILD_BOARD,
                 DeviceInfoConstants.BUILD_DEVICE, DeviceInfoConstants.PRODUCT_NAME,
                 DeviceInfoConstants.BUILD_ABI, DeviceInfoConstants.BUILD_ABI2,
                 DeviceInfoConstants.SCREEN_SIZE);
     }

     private void checkMetrics(Map<String, String> metrics, String... keysToCheck) {
         Set<String> keyCheckSet = new HashSet<String>();
         Collections.addAll(keyCheckSet, keysToCheck);
         for (Map.Entry<String, String> metricEntry : metrics.entrySet()) {
             String currentValue = mMetrics.get(metricEntry.getKey());
             if (keyCheckSet.contains(metricEntry.getKey()) && currentValue != null
                     && !metricEntry.getValue().equals(currentValue)) {
                 CLog.e("Inconsistent info collected from devices. "
                         + "Current result has %s='%s', Received '%s'. Are you sharding or " +
                         "resuming a test run across different devices and/or builds?",
                         metricEntry.getKey(), currentValue, metricEntry.getValue());
                 mMetrics.put(metricEntry.getKey(),
                         String.format("ERROR: Inconsistent results: %s, %s",
                                 metricEntry.getValue(), currentValue));
             } else {
                 mMetrics.put(metricEntry.getKey(), metricEntry.getValue());
             }
         }
     }

     /**
      * Report the given test as a failure.
      *
      * @param test
      * @param status
      * @param trace
      */
     public void reportTestFailure(TestIdentifier test, CtsTestStatus status, String trace) {
         Test result = findTest(test);
         result.setResultStatus(status);
         result.setStackTrace(trace);
     }

     /**
      * Report that the given test has completed.
      *
      * @param test
      */
     public void reportTestEnded(TestIdentifier test) {
         Test result = findTest(test);
         if (!result.getResult().equals(CtsTestStatus.FAIL)) {
             result.setResultStatus(CtsTestStatus.PASS);
         }
         result.updateEndTime();
     }

     /**
      * Return the number of tests with given status
      *
      * @param status
      * @return the total number of tests with given status
      */
     public int countTests(CtsTestStatus status) {
         return mSuiteRoot.countTests(status);
     }

     /**
      * @return
      */
     public Map<String, String> getMetrics() {
         return mMetrics;
     }
 }
	/*
	* Copyright (C) 2011 The Android Open Source Project
	*
	* Licensed under the Apache License, Version 2.0 (the "License");
	* you may not use this file except in compliance with the License.
	* You may obtain a copy of the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS,
	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	* See the License for the specific language governing permissions and
	* limitations under the License.
	*/
	package com.android.cts.tradefed.result;

	import android.tests.getinfo.DeviceInfoConstants;

	import com.android.cts.tradefed.device.DeviceInfoCollector;
	import com.android.cts.tradefed.testtype.CtsTest;
	import com.android.ddmlib.testrunner.TestIdentifier;
	import com.android.tradefed.log.LogUtil.CLog;

	import org.kxml2.io.KXmlSerializer;
	import org.xmlpull.v1.XmlPullParser;
	import org.xmlpull.v1.XmlPullParserException;

	import java.io.IOException;
	import java.util.Collection;
	import java.util.Collections;
	import java.util.Deque;
	import java.util.HashMap;
	import java.util.HashSet;
	import java.util.LinkedList;
	import java.util.List;
	import java.util.Map;
	import java.util.Set;

	/**
	* Data structure for a CTS test package result.
	* <p/>
	* Provides methods to serialize to XML.
	*/
	class TestPackageResult extends AbstractXmlPullParser {

	static final String TAG = "TestPackage";
	private static final String DIGEST_ATTR = "digest";
	private static final String APP_PACKAGE_NAME_ATTR = "appPackageName";
	private static final String NAME_ATTR = "name";
	private static final String ns = CtsXmlResultReporter.ns;
	private static final String SIGNATURE_TEST_PKG = "android.tests.sigtest";

	private String mAppPackageName;
	private String mName;
	private String mDigest;

	private Map<String, String> mMetrics = new HashMap<String, String>();

	private TestSuite mSuiteRoot = new TestSuite(null);

	public void setAppPackageName(String appPackageName) {
	mAppPackageName = appPackageName;
	}

	public String getAppPackageName() {
	return mAppPackageName;
	}

	public void setName(String name) {
	mName = name;
	}

	public String getName() {
	return mName;
	}

	public void setDigest(String digest) {
	mDigest = digest;
	}

	public String getDigest() {
	return mDigest;
	}

	/**
	* Return the {@link TestSuite}s
	*/
	public Collection<TestSuite> getTestSuites() {
	return mSuiteRoot.getTestSuites();
	}

	/**
	* Adds a test result to this test package
	*
	* @param testId
	* @param testResult
	*/
	public Test insertTest(TestIdentifier testId) {
	return findTest(testId, true);
	}

	private Test findTest(TestIdentifier testId, boolean insertIfMissing) {
	List<String> classNameSegments = new LinkedList<String>();
	Collections.addAll(classNameSegments, testId.getClassName().split("\\."));
	if (classNameSegments.size() <= 0) {
	CLog.e("Unrecognized package name format for test class '%s'",
	testId.getClassName());
	// should never happen
	classNameSegments.add("UnknownTestClass");
	}
	String testCaseName = classNameSegments.remove(classNameSegments.size()-1);
	return mSuiteRoot.findTest(classNameSegments, testCaseName, testId.getTestName(), insertIfMissing);
	}


	/**
	* Find the test result for given {@link TestIdentifier}.
	* @param testId
	* @return the {@link Test} or <code>null</code>
	*/
	public Test findTest(TestIdentifier testId) {
	return findTest(testId, false);
	}

	/**
	* Serialize this object and all its contents to XML.
	*
	* @param serializer
	* @throws IOException
	*/
	public void serialize(KXmlSerializer serializer) throws IOException {
	serializer.startTag(ns, TAG);
	serializeAttribute(serializer, NAME_ATTR, mName);
	serializeAttribute(serializer, APP_PACKAGE_NAME_ATTR, mAppPackageName);
	serializeAttribute(serializer, DIGEST_ATTR, getDigest());
	if (SIGNATURE_TEST_PKG.equals(mName)) {
	serializer.attribute(ns, "signatureCheck", "true");
	}
	mSuiteRoot.serialize(serializer);
	serializer.endTag(ns, TAG);
	}

	/**
	* Helper method to serialize attributes.
	* Can handle null values. Useful for cases where test package has not been fully populated
	* such as when unit testing.
	*
	* @param attrName
	* @param attrValue
	* @throws IOException
	*/
	private void serializeAttribute(KXmlSerializer serializer, String attrName, String attrValue)
	throws IOException {
	attrValue = attrValue == null ? "" : attrValue;
	serializer.attribute(ns, attrName, attrValue);
	}

	/**
	* Populates this class with package result data parsed from XML.
	*
	* @param parser the {@link XmlPullParser}. Expected to be pointing at start
	* of TestPackage tag
	*/
	@Override
	void parse(XmlPullParser parser) throws XmlPullParserException, IOException {
	if (!parser.getName().equals(TAG)) {
	throw new XmlPullParserException(String.format(
	"invalid XML: Expected %s tag but received %s", TAG, parser.getName()));
	}
	setAppPackageName(getAttribute(parser, APP_PACKAGE_NAME_ATTR));
	setName(getAttribute(parser, NAME_ATTR));
	setDigest(getAttribute(parser, DIGEST_ATTR));
	int eventType = parser.getEventType();
	while (eventType != XmlPullParser.END_DOCUMENT) {
	if (eventType == XmlPullParser.START_TAG && parser.getName().equals(TestSuite.TAG)) {
	TestSuite suite = new TestSuite();
	suite.parse(parser);
	mSuiteRoot.insertSuite(suite);
	}
	if (eventType == XmlPullParser.END_TAG && parser.getName().equals(TAG)) {
	return;
	}
	eventType = parser.next();
	}
	}

	/**
	* Return a list of {@link TestIdentifer}s contained in this result with the given status
	*
	* @param resultFilter the {@link CtsTestStatus} to filter by
	* @return a collection of {@link TestIdentifer}s
	*/
	public Collection<TestIdentifier> getTestsWithStatus(CtsTestStatus resultFilter) {
	Collection<TestIdentifier> tests = new LinkedList<TestIdentifier>();
	Deque<String> suiteNames = new LinkedList<String>();
	mSuiteRoot.addTestsWithStatus(tests, suiteNames, resultFilter);
	return tests;
	}

	/**
	* Populate values in this package result from run metrics
	* @param runResult
	*/
	public void populateMetrics(Map<String, String> metrics) {
	String name = metrics.get(CtsTest.PACKAGE_NAME_METRIC);
	if (name != null) {
	setName(name);
	}
	String digest = metrics.get(CtsTest.PACKAGE_DIGEST_METRIC);
	if (digest != null) {
	setDigest(digest);
	}
	if (DeviceInfoCollector.APP_PACKAGE_NAME.equals(getAppPackageName())) {
	storeDeviceMetrics(metrics);
	} else {
	mMetrics.putAll(metrics);
	}
	}

	/**
	* Check that the provided device info metrics are consistent with the currently stored metrics.
	* <p/>
	* If any inconsistencies occur, logs errors and stores error messages in the metrics map
	*
	* @param metrics
	*/
	private void storeDeviceMetrics(Map<String, String> metrics) {
	// TODO centralize all the device metrics handling into a single class
	if (mMetrics.isEmpty()) {
	// nothing to check!
	mMetrics.putAll(metrics);
	return;
	}
	// ensure all the metrics we expect to be identical actually are
	checkMetrics(metrics, DeviceInfoConstants.BUILD_FINGERPRINT,
	DeviceInfoConstants.BUILD_MODEL, DeviceInfoConstants.BUILD_BRAND,
	DeviceInfoConstants.BUILD_MANUFACTURER, DeviceInfoConstants.BUILD_BOARD,
	DeviceInfoConstants.BUILD_DEVICE, DeviceInfoConstants.PRODUCT_NAME,
	DeviceInfoConstants.BUILD_ABI, DeviceInfoConstants.BUILD_ABI2,
	DeviceInfoConstants.SCREEN_SIZE);
	}

	private void checkMetrics(Map<String, String> metrics, String... keysToCheck) {
	Set<String> keyCheckSet = new HashSet<String>();
	Collections.addAll(keyCheckSet, keysToCheck);
	for (Map.Entry<String, String> metricEntry : metrics.entrySet()) {
	String currentValue = mMetrics.get(metricEntry.getKey());
	if (keyCheckSet.contains(metricEntry.getKey()) && currentValue != null
	&& !metricEntry.getValue().equals(currentValue)) {
	CLog.e("Inconsistent info collected from devices. "
	+ "Current result has %s='%s', Received '%s'. Are you sharding or " +
	"resuming a test run across different devices and/or builds?",
	metricEntry.getKey(), currentValue, metricEntry.getValue());
	mMetrics.put(metricEntry.getKey(),
	String.format("ERROR: Inconsistent results: %s, %s",
	metricEntry.getValue(), currentValue));
	} else {
	mMetrics.put(metricEntry.getKey(), metricEntry.getValue());
	}
	}
	}

	/**
	* Report the given test as a failure.
	*
	* @param test
	* @param status
	* @param trace
	*/
	public void reportTestFailure(TestIdentifier test, CtsTestStatus status, String trace) {
	Test result = findTest(test);
	result.setResultStatus(status);
	result.setStackTrace(trace);
	}

	/**
	* Report that the given test has completed.
	*
	* @param test
	*/
	public void reportTestEnded(TestIdentifier test) {
	Test result = findTest(test);
	if (!result.getResult().equals(CtsTestStatus.FAIL)) {
	result.setResultStatus(CtsTestStatus.PASS);
	}
	result.updateEndTime();
	}

	/**
	* Return the number of tests with given status
	*
	* @param status
	* @return the total number of tests with given status
	*/
	public int countTests(CtsTestStatus status) {
	return mSuiteRoot.countTests(status);
	}

	/**
	* @return
	*/
	public Map<String, String> getMetrics() {
	return mMetrics;
	}
	}