blob: 5ecede83cef2ca0d4c658cb3ff9795be62bbabda [file] [log] [blame]
#!/usr/bin/python
# Copyright (C) 2009 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Module for generating CTS test descriptions and test plans."""
import glob
import os
import re
import shutil
import subprocess
import sys
import xml.dom.minidom as dom
from cts import tools
from multiprocessing import Pool
def GetSubDirectories(root):
"""Return all directories under the given root directory."""
return [x for x in os.listdir(root) if os.path.isdir(os.path.join(root, x))]
def ReadFileLines(filePath):
"""Reads a file and returns its contents as a line list."""
f = open(filePath, 'r');
lines = [line.strip() for line in f.readlines()]
f.close()
return lines
def GetMakeFileVars(makefile_path):
"""Extracts variable definitions from the given make file.
Args:
makefile_path: Path to the make file.
Returns:
A dictionary mapping variable names to their assigned value.
"""
result = {}
pattern = re.compile(r'^\s*([^:#=\s]+)\s*:=\s*(.*?[^\\])$', re.MULTILINE + re.DOTALL)
stream = open(makefile_path, 'r')
content = stream.read()
for match in pattern.finditer(content):
result[match.group(1)] = match.group(2)
stream.close()
return result
class CtsBuilder(object):
"""Main class for generating test descriptions and test plans."""
def __init__(self, argv):
"""Initialize the CtsBuilder from command line arguments."""
if len(argv) != 6:
print 'Usage: %s <testRoot> <ctsOutputDir> <tempDir> <androidRootDir> <docletPath>' % argv[0]
print ''
print 'testRoot: Directory under which to search for CTS tests.'
print 'ctsOutputDir: Directory in which the CTS repository should be created.'
print 'tempDir: Directory to use for storing temporary files.'
print 'androidRootDir: Root directory of the Android source tree.'
print 'docletPath: Class path where the DescriptionGenerator doclet can be found.'
sys.exit(1)
self.test_root = sys.argv[1]
self.out_dir = sys.argv[2]
self.temp_dir = sys.argv[3]
self.android_root = sys.argv[4]
self.doclet_path = sys.argv[5]
self.test_repository = os.path.join(self.out_dir, 'repository/testcases')
self.plan_repository = os.path.join(self.out_dir, 'repository/plans')
self.definedplans_repository = os.path.join(self.android_root, 'cts/tests/plans')
def GenerateTestDescriptions(self):
"""Generate test descriptions for all packages."""
pool = Pool(processes=2)
# generate test descriptions for android tests
results = []
pool.close()
pool.join()
return sum(map(lambda result: result.get(), results))
def __WritePlan(self, plan, plan_name):
print 'Generating test plan %s' % plan_name
plan.Write(os.path.join(self.plan_repository, plan_name + '.xml'))
def GenerateTestPlans(self):
"""Generate default test plans."""
# TODO: Instead of hard-coding the plans here, use a configuration file,
# such as test_defs.xml
packages = []
descriptions = sorted(glob.glob(os.path.join(self.test_repository, '*.xml')))
for description in descriptions:
doc = tools.XmlFile(description)
packages.append(doc.GetAttr('TestPackage', 'appPackageName'))
# sort the list to give the same sequence based on name
packages.sort()
plan = tools.TestPlan(packages)
plan.Exclude('android\.performance.*')
self.__WritePlan(plan, 'CTS')
self.__WritePlan(plan, 'CTS-TF')
plan = tools.TestPlan(packages)
plan.Exclude('android\.performance.*')
plan.Exclude('android\.media\.cts\.StreamingMediaPlayerTest.*')
# Test plan to not include media streaming tests
self.__WritePlan(plan, 'CTS-No-Media-Stream')
plan = tools.TestPlan(packages)
plan.Exclude('android\.performance.*')
self.__WritePlan(plan, 'SDK')
plan.Exclude(r'android\.signature')
plan.Exclude(r'android\.core.*')
self.__WritePlan(plan, 'Android')
plan = tools.TestPlan(packages)
plan.Include(r'android\.core\.tests.*')
plan.Exclude(r'android\.core\.tests\.libcore.\package.\harmony*')
self.__WritePlan(plan, 'Java')
# TODO: remove this once the tests are fixed and merged into Java plan above.
plan = tools.TestPlan(packages)
plan.Include(r'android\.core\.tests\.libcore.\package.\harmony*')
self.__WritePlan(plan, 'Harmony')
plan = tools.TestPlan(packages)
plan.Include(r'android\.core\.vm-tests-tf')
self.__WritePlan(plan, 'VM-TF')
plan = tools.TestPlan(packages)
plan.Include(r'android\.tests\.appsecurity')
self.__WritePlan(plan, 'AppSecurity')
# hard-coded white list for PDK plan
plan.Exclude('.*')
plan.Include('android\.aadb')
plan.Include('android\.bluetooth')
plan.Include('android\.graphics.*')
plan.Include('android\.hardware')
plan.Include('android\.media')
plan.Exclude('android\.mediastress')
plan.Include('android\.net')
plan.Include('android\.opengl.*')
plan.Include('android\.renderscript')
plan.Include('android\.telephony')
plan.Include('android\.nativemedia.*')
plan.Include('com\.android\.cts\..*')#TODO(stuartscott): Should PDK have all these?
self.__WritePlan(plan, 'PDK')
temporarily_known_failure_tests = BuildCtsTemporarilyKnownFailureList();
flaky_tests = BuildCtsFlakyTestList()
releasekey_tests = BuildListForReleaseBuildTest()
# CTS Stable plan
plan = tools.TestPlan(packages)
plan.Exclude(r'com\.android\.cts\.browserbench')
for package, test_list in flaky_tests.iteritems():
plan.ExcludeTests(package, test_list)
for package, test_list in releasekey_tests.iteritems():
plan.ExcludeTests(package, test_list)
self.__WritePlan(plan, 'CTS-stable')
# CTS Flaky plan - list of tests known to be flaky in lab environment
plan = tools.TestPlan(packages)
plan.Exclude('.*')
plan.Include(r'com\.android\.cts\.browserbench')
for package, test_list in flaky_tests.iteritems():
plan.Include(package+'$')
plan.IncludeTests(package, test_list)
self.__WritePlan(plan, 'CTS-flaky')
small_tests = BuildAospSmallSizeTestList()
medium_tests = BuildAospMediumSizeTestList()
new_test_packages = BuildCtsVettedNewPackagesList()
# CTS - sub plan for public, small size tests
plan = tools.TestPlan(packages)
plan.Exclude('.*')
for package, test_list in small_tests.iteritems():
plan.Include(package+'$')
plan.Exclude(r'com\.android\.cts\.browserbench')
for package, test_list in temporarily_known_failure_tests.iteritems():
plan.ExcludeTests(package, test_list)
for package, test_list in flaky_tests.iteritems():
plan.ExcludeTests(package, test_list)
for package, test_list in releasekey_tests.iteritems():
plan.ExcludeTests(package, test_list)
self.__WritePlan(plan, 'CTS-kitkat-small')
self.__WritePlan(plan, 'CTS-public-small')
# CTS - sub plan for public, medium size tests
plan = tools.TestPlan(packages)
plan.Exclude('.*')
for package, test_list in medium_tests.iteritems():
plan.Include(package+'$')
plan.Exclude(r'com\.android\.cts\.browserbench')
for package, test_list in flaky_tests.iteritems():
plan.ExcludeTests(package, test_list)
for package, test_list in releasekey_tests.iteritems():
plan.ExcludeTests(package, test_list)
self.__WritePlan(plan, 'CTS-kitkat-medium')
self.__WritePlan(plan, 'CTS-public-medium')
# CTS - sub plan for hardware tests which is public, large
plan = tools.TestPlan(packages)
plan.Exclude('.*')
plan.Include(r'android\.hardware$')
plan.Exclude(r'com\.android\.cts\.browserbench')
for package, test_list in flaky_tests.iteritems():
plan.ExcludeTests(package, test_list)
for package, test_list in releasekey_tests.iteritems():
plan.ExcludeTests(package, test_list)
self.__WritePlan(plan, 'CTS-hardware')
# CTS - sub plan for media tests which is public, large
plan = tools.TestPlan(packages)
plan.Exclude('.*')
plan.Include(r'android\.media$')
plan.Include(r'android\.view$')
plan.Exclude(r'com\.android\.cts\.browserbench')
for package, test_list in flaky_tests.iteritems():
plan.ExcludeTests(package, test_list)
for package, test_list in releasekey_tests.iteritems():
plan.ExcludeTests(package, test_list)
self.__WritePlan(plan, 'CTS-media')
# CTS - sub plan for mediastress tests which is public, large
plan = tools.TestPlan(packages)
plan.Exclude('.*')
plan.Include(r'android\.mediastress$')
plan.Exclude(r'com\.android\.cts\.browserbench')
for package, test_list in flaky_tests.iteritems():
plan.ExcludeTests(package, test_list)
for package, test_list in releasekey_tests.iteritems():
plan.ExcludeTests(package, test_list)
self.__WritePlan(plan, 'CTS-mediastress')
# CTS - sub plan for new tests that is vetted for L launch
plan = tools.TestPlan(packages)
plan.Exclude('.*')
for package, test_list in new_test_packages.iteritems():
plan.Include(package+'$')
plan.Exclude(r'com\.android\.cts\.browserbench')
for package, test_list in flaky_tests.iteritems():
plan.ExcludeTests(package, test_list)
for package, test_list in releasekey_tests.iteritems():
plan.ExcludeTests(package, test_list)
self.__WritePlan(plan, 'CTS-l-tests')
# CTS - sub plan for tests in drawelement packages
plan = tools.TestPlan(packages)
plan.Exclude('.*')
plan.Include(r'com\.drawelements\.')
# STOPSHIP(jpoyry): Do not ship with reduced test plan to avoid confusion of having multiple
# different deqp sets in different plans.
plan.ExcludeTests('com.drawelements.deqp.gles3', ReadFileLines(os.path.join(self.test_root, 'deqp/gles3-temporary-failures.txt')))
plan.ExcludeTests('com.drawelements.deqp.gles31', ReadFileLines(os.path.join(self.test_root, 'deqp/gles31-temporary-failures.txt')))
self.__WritePlan(plan, 'CTS-DEQP')
# CTS - sub plan for new test packages added for staging
plan = tools.TestPlan(packages)
for package, test_list in small_tests.iteritems():
plan.Exclude(package+'$')
for package, test_list in medium_tests.iteritems():
plan.Exclude(package+'$')
for package, tests_list in new_test_packages.iteritems():
plan.Exclude(package+'$')
for package, test_list in temporarily_known_failure_tests.iteritems():
plan.Include(package+'$')
plan.IncludeTests(package, test_list)
plan.Exclude(r'com\.drawelements\.')
plan.Exclude(r'android\.hardware$')
plan.Exclude(r'android\.media$')
plan.Exclude(r'android\.view$')
plan.Exclude(r'android\.mediastress$')
plan.Exclude(r'com\.android\.cts\.browserbench')
for package, test_list in flaky_tests.iteritems():
plan.ExcludeTests(package, test_list)
for package, test_list in releasekey_tests.iteritems():
plan.ExcludeTests(package, test_list)
self.__WritePlan(plan, 'CTS-staging')
plan = tools.TestPlan(packages)
plan.Exclude('.*')
plan.Include(r'android\.webgl')
self.__WritePlan(plan, 'CTS-webview')
def BuildAospMediumSizeTestList():
""" Construct a defaultdic that lists package names of medium tests
already published to aosp. """
return {
'android.app' : [],
'android.core.tests.libcore.package.libcore' : [],
'android.core.tests.libcore.package.org' : [],
'android.core.vm-tests-tf' : [],
'android.dpi' : [],
'android.host.security' : [],
'android.net' : [],
'android.os' : [],
'android.permission2' : [],
'android.security' : [],
'android.telephony' : [],
'android.webkit' : [],
'android.widget' : [],
'com.android.cts.browserbench' : []}
def BuildAospSmallSizeTestList():
""" Construct a default dict that lists packages names of small tests
already published to aosp. """
return {
'android.aadb' : [],
'android.acceleration' : [],
'android.accessibility' : [],
'android.accessibilityservice' : [],
'android.accounts' : [],
'android.admin' : [],
'android.animation' : [],
'android.bionic' : [],
'android.bluetooth' : [],
'android.calendarcommon' : [],
'android.content' : [],
'android.core.tests.libcore.package.com' : [],
'android.core.tests.libcore.package.conscrypt' : [],
'android.core.tests.libcore.package.dalvik' : [],
'android.core.tests.libcore.package.sun' : [],
'android.core.tests.libcore.package.tests' : [],
'android.database' : [],
'android.dreams' : [],
'android.drm' : [],
'android.effect' : [],
'android.gesture' : [],
'android.graphics' : [],
'android.graphics2' : [],
'android.jni' : [],
'android.keystore' : [],
'android.location' : [],
'android.nativemedia.sl' : [],
'android.nativemedia.xa' : [],
'android.nativeopengl' : [],
'android.ndef' : [],
'android.opengl' : [],
'android.openglperf' : [],
'android.permission' : [],
'android.preference' : [],
'android.preference2' : [],
'android.provider' : [],
'android.renderscript' : [],
'android.rscpp' : [],
'android.rsg' : [],
'android.sax' : [],
'android.signature' : [],
'android.speech' : [],
'android.tests.appsecurity' : [],
'android.text' : [],
'android.textureview' : [],
'android.theme' : [],
'android.usb' : [],
'android.util' : [],
'com.android.cts.dram' : [],
'com.android.cts.filesystemperf' : [],
'com.android.cts.jank' : [],
'com.android.cts.jank2' : [],
'com.android.cts.opengl' : [],
'com.android.cts.simplecpu' : [],
'com.android.cts.ui' : [],
'com.android.cts.uihost' : [],
'com.android.cts.videoperf' : [],
'zzz.android.monkey' : []}
def BuildCtsVettedNewPackagesList():
""" Construct a defaultdict that maps package names that is vetted for L. """
return {
'android.JobScheduler' : [],
'android.core.tests.libcore.package.harmony_annotation' : [],
'android.core.tests.libcore.package.harmony_beans' : [],
'android.core.tests.libcore.package.harmony_java_io' : [],
'android.core.tests.libcore.package.harmony_java_lang' : [],
'android.core.tests.libcore.package.harmony_java_math' : [],
'android.core.tests.libcore.package.harmony_java_net' : [],
'android.core.tests.libcore.package.harmony_java_nio' : [],
'android.core.tests.libcore.package.harmony_java_util' : [],
'android.core.tests.libcore.package.harmony_java_text' : [],
'android.core.tests.libcore.package.harmony_javax_security' : [],
'android.core.tests.libcore.package.harmony_logging' : [],
'android.core.tests.libcore.package.harmony_prefs' : [],
'android.core.tests.libcore.package.harmony_sql' : [],
'android.core.tests.libcore.package.jsr166' : [],
'android.core.tests.libcore.package.okhttp' : [],
'android.display' : [],
'android.host.theme' : [],
'android.jdwp' : [],
'android.location2' : [],
'android.print' : [],
'android.renderscriptlegacy' : [],
'android.signature' : [],
'android.tv' : [],
'android.uiautomation' : [],
'android.uirendering' : [],
'android.webgl' : []}
def BuildListForReleaseBuildTest():
""" Construct a defaultdict that maps package name to a list of tests
that are expected to pass only when running against a user/release-key build. """
return {
'android.app' : [
'android.app.cts.ActivityManagerTest#testIsRunningInTestHarness',],
'android.dpi' : [
'android.dpi.cts.DefaultManifestAttributesSdkTest#testPackageHasExpectedSdkVersion',],
'android.host.security' : [
'android.cts.security.SELinuxHostTest#testAllEnforcing',
'android.cts.security.SELinuxHostTest#testSuDomain',],
'android.os' : [
'android.os.cts.BuildVersionTest#testReleaseVersion',
'android.os.cts.BuildTest#testIsSecureUserBuild',],
'android.security' : [
'android.security.cts.BannedFilesTest#testNoSu',
'android.security.cts.BannedFilesTest#testNoSuInPath',
'android.security.cts.PackageSignatureTest#testPackageSignatures',
'android.security.cts.SELinuxDomainTest#testSuDomain',],
'' : []}
def BuildCtsFlakyTestList():
""" Construct a defaultdict that maps package name to a list of tests
that flaky during dev cycle and cause other subsequent tests to fail. """
return {
'android.hardware' : [
'android.hardware.cts.CameraTest#testVideoSnapshot',
'android.hardware.cts.CameraGLTest#testCameraToSurfaceTextureMetadata',
'android.hardware.cts.CameraGLTest#testSetPreviewTextureBothCallbacks',
'android.hardware.cts.CameraGLTest#testSetPreviewTexturePreviewCallback',],
'android.media' : [
'android.media.cts.DecoderTest#testCodecResetsH264WithSurface',
'android.media.cts.StreamingMediaPlayerTest#testHLS',],
'android.net' : [
'android.net.cts.ConnectivityManagerTest#testStartUsingNetworkFeature_enableHipri',
'android.net.cts.DnsTest#testDnsWorks',
'android.net.cts.SSLCertificateSocketFactoryTest#testCreateSocket',
'android.net.cts.SSLCertificateSocketFactoryTest#test_createSocket_bind',
'android.net.cts.SSLCertificateSocketFactoryTest#test_createSocket_simple',
'android.net.cts.SSLCertificateSocketFactoryTest#test_createSocket_wrapping',
'android.net.cts.TrafficStatsTest#testTrafficStatsForLocalhost',
'android.net.wifi.cts.NsdManagerTest#testAndroidTestCaseSetupProperly',],
'android.security' : [
'android.security.cts.ListeningPortsTest#testNoRemotelyAccessibleListeningUdp6Ports',
'android.security.cts.ListeningPortsTest#testNoRemotelyAccessibleListeningUdpPorts',],
'android.webkit' : [
'android.webkit.cts.WebViewClientTest#testOnUnhandledKeyEvent',],
'com.android.cts.filesystemperf' : [
'com.android.cts.filesystemperf.RandomRWTest#testRandomRead',
'com.android.cts.filesystemperf.RandomRWTest#testRandomUpdate',],
'' : []}
def BuildCtsTemporarilyKnownFailureList():
""" Construct a defaultdict that maps package name to a list of tests
that are known failures during dev cycle but expected to be fixed before launch """
return {
'android.content' : [
'android.content.cts.ContentResolverTest#testAndroidTestCaseSetupProperly',
'android.content.cts.ContentResolverTest#testBulkInsert',
'android.content.cts.ContentResolverTest#testCancelableQuery_WhenCanceledBeforeQuery_ThrowsImmediately',
'android.content.cts.ContentResolverTest#testCancelableQuery_WhenCanceledDuringLongRunningQuery_CancelsQueryAndThrows',
'android.content.cts.ContentResolverTest#testCancelableQuery_WhenNotCanceled_ReturnsResultSet',
'android.content.cts.ContentResolverTest#testConstructor',
'android.content.cts.ContentResolverTest#testCrashOnLaunch',
'android.content.cts.ContentResolverTest#testCrashingOpenAssetFileDescriptor',
'android.content.cts.ContentResolverTest#testCrashingOpenTypedAssetFileDescriptor',
'android.content.cts.ContentResolverTest#testCrashingQuery',
'android.content.cts.ContentResolverTest#testDelete',
'android.content.cts.ContentResolverTest#testGetType',
'android.content.cts.ContentResolverTest#testInsert',
'android.content.cts.ContentResolverTest#testNotifyChange1',
'android.content.cts.ContentResolverTest#testNotifyChange2',
'android.content.cts.ContentResolverTest#testOpenAssetFileDescriptor',
'android.content.cts.ContentResolverTest#testOpenFileDescriptor',
'android.content.cts.ContentResolverTest#testOpenInputStream',
'android.content.cts.ContentResolverTest#testOpenOutputStream',
'android.content.cts.ContentResolverTest#testQuery',
'android.content.cts.ContentResolverTest#testRegisterContentObserver',
'android.content.cts.ContentResolverTest#testStableToUnstableRefs',
'android.content.cts.ContentResolverTest#testStartCancelSync',
'android.content.cts.ContentResolverTest#testStartSyncFailure',
'android.content.cts.ContentResolverTest#testUnstableGetType',
'android.content.cts.ContentResolverTest#testUnstableToStableRefs',
'android.content.cts.ContentResolverTest#testUpdate',
'android.content.cts.ContentResolverTest#testValidateSyncExtrasBundle',],
'' : []}
def LogGenerateDescription(name):
print 'Generating test description for package %s' % name
if __name__ == '__main__':
builder = CtsBuilder(sys.argv)
result = builder.GenerateTestDescriptions()
if result != 0:
sys.exit(result)
builder.GenerateTestPlans()