# -*- coding: utf-8 -*- | |
#------------------------------------------------------------------------- | |
# drawElements Quality Program utilities | |
# -------------------------------------- | |
# | |
# Copyright 2015 The Android Open Source Project | |
# | |
# Licensed under the Apache License, Version 2.0 (the "License"); | |
# you may not use this file except in compliance with the License. | |
# You may obtain a copy of the License at | |
# | |
# http://www.apache.org/licenses/LICENSE-2.0 | |
# | |
# Unless required by applicable law or agreed to in writing, software | |
# distributed under the License is distributed on an "AS IS" BASIS, | |
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
# See the License for the specific language governing permissions and | |
# limitations under the License. | |
# | |
#------------------------------------------------------------------------- | |
from build.common import * | |
from build.config import ANY_GENERATOR | |
from build.build import build | |
from build_caselists import Module, getBuildConfig, genCaseList, getCaseListPath, DEFAULT_BUILD_DIR, DEFAULT_TARGET | |
from fnmatch import fnmatch | |
from copy import copy | |
import xml.etree.cElementTree as ElementTree | |
import xml.dom.minidom as minidom | |
CTS_DATA_DIR = os.path.join(DEQP_DIR, "android", "cts") | |
class Configuration: | |
def __init__ (self, name, glconfig, rotation, surfacetype, filters): | |
self.name = name | |
self.glconfig = glconfig | |
self.rotation = rotation | |
self.surfacetype = surfacetype | |
self.filters = filters | |
class Package: | |
def __init__ (self, module, configurations): | |
self.module = module | |
self.configurations = configurations | |
class Mustpass: | |
def __init__ (self, version, packages): | |
self.version = version | |
self.packages = packages | |
class Filter: | |
TYPE_INCLUDE = 0 | |
TYPE_EXCLUDE = 1 | |
def __init__ (self, type, filename): | |
self.type = type | |
self.filename = filename | |
class TestRoot: | |
def __init__ (self): | |
self.children = [] | |
class TestGroup: | |
def __init__ (self, name): | |
self.name = name | |
self.children = [] | |
class TestCase: | |
def __init__ (self, name): | |
self.name = name | |
self.configurations = [] | |
class GLESVersion: | |
def __init__(self, major, minor): | |
self.major = major | |
self.minor = minor | |
def encode (self): | |
return (self.major << 16) | (self.minor) | |
def getModuleGLESVersion (module): | |
versions = { | |
'dEQP-GLES2': GLESVersion(2,0), | |
'dEQP-GLES3': GLESVersion(3,0), | |
'dEQP-GLES31': GLESVersion(3,1) | |
} | |
return versions[module.name] | |
def getSrcDir (mustpass): | |
return os.path.join(CTS_DATA_DIR, mustpass.version, "src") | |
def getTmpDir (mustpass): | |
return os.path.join(CTS_DATA_DIR, mustpass.version, "tmp") | |
def getModuleShorthand (module): | |
assert module.name[:5] == "dEQP-" | |
return module.name[5:].lower() | |
def getCaseListFileName (package, configuration): | |
return "%s-%s.txt" % (getModuleShorthand(package.module), configuration.name) | |
def getDstCaseListPath (mustpass, package, configuration): | |
return os.path.join(CTS_DATA_DIR, mustpass.version, getCaseListFileName(package, configuration)) | |
def getCTSPackageName (package): | |
return "com.drawelements.deqp." + getModuleShorthand(package.module) | |
def getCommandLine (config): | |
return "--deqp-gl-config-name=%s --deqp-screen-rotation=%s --deqp-surface-type=%s" % (config.glconfig, config.rotation, config.surfacetype) | |
def readCaseList (filename): | |
cases = [] | |
with open(filename, 'rb') as f: | |
for line in f: | |
if line[:6] == "TEST: ": | |
cases.append(line[6:].strip()) | |
return cases | |
def getCaseList (mustpass, module): | |
generator = ANY_GENERATOR | |
buildCfg = getBuildConfig(DEFAULT_BUILD_DIR, DEFAULT_TARGET, "Debug") | |
build(buildCfg, generator, [module.binName]) | |
genCaseList(buildCfg, generator, module, "txt") | |
return readCaseList(getCaseListPath(buildCfg, module, "txt")) | |
def readPatternList (filename): | |
ptrns = [] | |
with open(filename, 'rb') as f: | |
for line in f: | |
line = line.strip() | |
if len(line) > 0 and line[0] != '#': | |
ptrns.append(line) | |
return ptrns | |
def applyPatterns (caseList, patterns, op): | |
matched = set() | |
errors = [] | |
curList = copy(caseList) | |
trivialPtrns = [p for p in patterns if p.find('*') < 0] | |
regularPtrns = [p for p in patterns if p.find('*') >= 0] | |
# Apply trivial (just case paths) | |
allCasesSet = set(caseList) | |
for path in trivialPtrns: | |
if path in allCasesSet: | |
if path in matched: | |
errors.append((path, "Same case specified more than once")) | |
matched.add(path) | |
else: | |
errors.append((path, "Test case not found")) | |
curList = [c for c in curList if c not in matched] | |
for pattern in regularPtrns: | |
matchedThisPtrn = set() | |
for case in curList: | |
if fnmatch(case, pattern): | |
matchedThisPtrn.add(case) | |
if len(matchedThisPtrn) == 0: | |
errors.append((pattern, "Pattern didn't match any cases")) | |
matched = matched | matchedThisPtrn | |
curList = [c for c in curList if c not in matched] | |
for pattern, reason in errors: | |
print "ERROR: %s: %s" % (reason, pattern) | |
if len(errors) > 0: | |
die("Found %s invalid patterns" % len(errors)) | |
return [c for c in caseList if op(c in matched)] | |
def applyInclude (caseList, patterns): | |
return applyPatterns(caseList, patterns, lambda b: b) | |
def applyExclude (caseList, patterns): | |
return applyPatterns(caseList, patterns, lambda b: not b) | |
def readPatternLists (mustpass): | |
lists = {} | |
for package in mustpass.packages: | |
for cfg in package.configurations: | |
for filter in cfg.filters: | |
if not filter.filename in lists: | |
lists[filter.filename] = readPatternList(os.path.join(getSrcDir(mustpass), filter.filename)) | |
return lists | |
def applyFilters (caseList, patternLists, filters): | |
res = copy(caseList) | |
for filter in filters: | |
ptrnList = patternLists[filter.filename] | |
if filter.type == Filter.TYPE_INCLUDE: | |
res = applyInclude(res, ptrnList) | |
else: | |
assert filter.type == Filter.TYPE_EXCLUDE | |
res = applyExclude(res, ptrnList) | |
return res | |
def appendToHierarchy (root, casePath): | |
def findChild (node, name): | |
for child in node.children: | |
if child.name == name: | |
return child | |
return None | |
curNode = root | |
components = casePath.split('.') | |
for component in components[:-1]: | |
nextNode = findChild(curNode, component) | |
if not nextNode: | |
nextNode = TestGroup(component) | |
curNode.children.append(nextNode) | |
curNode = nextNode | |
if not findChild(curNode, components[-1]): | |
curNode.children.append(TestCase(components[-1])) | |
def buildTestHierachy (caseList): | |
root = TestRoot() | |
for case in caseList: | |
appendToHierarchy(root, case) | |
return root | |
def buildTestCaseMap (root): | |
caseMap = {} | |
def recursiveBuild (curNode, prefix): | |
curPath = prefix + curNode.name | |
if isinstance(curNode, TestCase): | |
caseMap[curPath] = curNode | |
else: | |
for child in curNode.children: | |
recursiveBuild(child, curPath + '.') | |
for child in root.children: | |
recursiveBuild(child, '') | |
return caseMap | |
def include (filename): | |
return Filter(Filter.TYPE_INCLUDE, filename) | |
def exclude (filename): | |
return Filter(Filter.TYPE_EXCLUDE, filename) | |
def prettifyXML (doc): | |
uglyString = ElementTree.tostring(doc, 'utf-8') | |
reparsed = minidom.parseString(uglyString) | |
return reparsed.toprettyxml(indent='\t', encoding='utf-8') | |
def genCTSPackageXML (package, root): | |
def isLeafGroup (testGroup): | |
numGroups = 0 | |
numTests = 0 | |
for child in testGroup.children: | |
if isinstance(child, TestCase): | |
numTests += 1 | |
else: | |
numGroups += 1 | |
assert numGroups + numTests > 0 | |
if numGroups > 0 and numTests > 0: | |
die("Mixed groups and cases in %s" % testGroup.name) | |
return numGroups == 0 | |
def makeConfiguration (parentElem, configuration): | |
return ElementTree.SubElement(parentElem, "TestInstance", glconfig=configuration.glconfig, rotation=configuration.rotation, surfacetype=configuration.surfacetype) | |
def makeTestCase (parentElem, testCase): | |
caseElem = ElementTree.SubElement(parentElem, "Test", name=testCase.name) | |
for config in testCase.configurations: | |
makeConfiguration(caseElem, config) | |
return caseElem | |
def makeTestGroup (parentElem, testGroup): | |
groupElem = ElementTree.SubElement(parentElem, "TestCase" if isLeafGroup(testGroup) else "TestSuite", name=testGroup.name) | |
for child in testGroup.children: | |
if isinstance(child, TestCase): | |
makeTestCase(groupElem, child) | |
else: | |
makeTestGroup(groupElem, child) | |
return groupElem | |
pkgElem = ElementTree.Element("TestPackage", | |
name = package.module.name, | |
appPackageName = getCTSPackageName(package), | |
testType = "deqpTest") | |
pkgElem.set("xmlns:deqp", "http://drawelements.com/deqp") | |
pkgElem.set("deqp:glesVersion", str(getModuleGLESVersion(package.module).encode())) | |
for child in root.children: | |
makeTestGroup(pkgElem, child) | |
return pkgElem | |
def genSpecXML (mustpass): | |
mustpassElem = ElementTree.Element("Mustpass", version = mustpass.version) | |
for package in mustpass.packages: | |
packageElem = ElementTree.SubElement(mustpassElem, "TestPackage", name = package.module.name) | |
for config in package.configurations: | |
configElem = ElementTree.SubElement(packageElem, "Configuration", | |
name = config.name, | |
caseListFile = getCaseListFileName(package, config), | |
commandLine = getCommandLine(config)) | |
return mustpassElem | |
def genMustpass (mustpass, moduleCaseLists): | |
print "Generating mustpass '%s'" % mustpass.version | |
patternLists = readPatternLists(mustpass) | |
for package in mustpass.packages: | |
allCasesInPkg = moduleCaseLists[package.module] | |
matchingByConfig = {} | |
allMatchingSet = set() | |
for config in package.configurations: | |
filtered = applyFilters(allCasesInPkg, patternLists, config.filters) | |
dstFile = getDstCaseListPath(mustpass, package, config) | |
print " Writing deqp caselist: " + dstFile | |
writeFile(dstFile, "\n".join(filtered) + "\n") | |
matchingByConfig[config] = filtered | |
allMatchingSet = allMatchingSet | set(filtered) | |
allMatchingCases = [c for c in allCasesInPkg if c in allMatchingSet] # To preserve ordering | |
root = buildTestHierachy(allMatchingCases) | |
testCaseMap = buildTestCaseMap(root) | |
for config in package.configurations: | |
for case in matchingByConfig[config]: | |
testCaseMap[case].configurations.append(config) | |
packageXml = genCTSPackageXML(package, root) | |
xmlFilename = os.path.join(CTS_DATA_DIR, mustpass.version, getCTSPackageName(package) + ".xml") | |
print " Writing CTS caselist: " + xmlFilename | |
writeFile(xmlFilename, prettifyXML(packageXml)) | |
specXML = genSpecXML(mustpass) | |
specFilename = os.path.join(CTS_DATA_DIR, mustpass.version, "mustpass.xml") | |
print " Writing spec: " + specFilename | |
writeFile(specFilename, prettifyXML(specXML)) | |
print "Done!" | |
def genMustpassLists (mustpassLists): | |
moduleCaseLists = {} | |
# Getting case lists involves invoking build, so we want to cache the results | |
for mustpass in mustpassLists: | |
for package in mustpass.packages: | |
if not package.module in moduleCaseLists: | |
moduleCaseLists[package.module] = getCaseList(mustpass, package.module) | |
for mustpass in mustpassLists: | |
genMustpass(mustpass, moduleCaseLists) | |
GLES3_MODULE = Module(name = "dEQP-GLES3", dirName = "gles3", binName = "deqp-gles3") | |
GLES31_MODULE = Module(name = "dEQP-GLES31", dirName = "gles31", binName = "deqp-gles31") | |
LMP_GLES3_PKG = Package(module = GLES3_MODULE, configurations = [ | |
Configuration(name = "master", | |
glconfig = "rgba8888d24s8ms0", | |
rotation = "unspecified", | |
surfacetype = "window", | |
filters = [include("es30-lmp.txt")]), | |
]) | |
LMP_GLES31_PKG = Package(module = GLES31_MODULE, configurations = [ | |
Configuration(name = "master", | |
glconfig = "rgba8888d24s8ms0", | |
rotation = "unspecified", | |
surfacetype = "window", | |
filters = [include("es31-lmp.txt")]), | |
]) | |
LMP_MR1_GLES3_PKG = Package(module = GLES3_MODULE, configurations = [ | |
Configuration(name = "master", | |
glconfig = "rgba8888d24s8ms0", | |
rotation = "unspecified", | |
surfacetype = "window", | |
filters = [include("es30-lmp-mr1.txt")]), | |
]) | |
LMP_MR1_GLES31_PKG = Package(module = GLES31_MODULE, configurations = [ | |
Configuration(name = "master", | |
glconfig = "rgba8888d24s8ms0", | |
rotation = "unspecified", | |
surfacetype = "window", | |
filters = [include("es31-lmp-mr1.txt")]), | |
]) | |
MNC_GLES3_COMMON_FILTERS = [exclude("gles3-hw-issues.txt"), exclude("gles3-test-issues.txt")] | |
MNC_GLES3_PKG = Package(module = GLES3_MODULE, configurations = [ | |
Configuration(name = "master", | |
glconfig = "rgba8888d24s8ms0", | |
rotation = "unspecified", | |
surfacetype = "window", | |
filters = MNC_GLES3_COMMON_FILTERS), | |
Configuration(name = "rotate-0", | |
glconfig = "rgba8888d24s8ms0", | |
rotation = "0", | |
surfacetype = "window", | |
filters = MNC_GLES3_COMMON_FILTERS + [include("gles3-rotation.txt")]), | |
]) | |
MNC_GLES31_COMMON_FILTERS = [exclude("gles31-hw-issues.txt"), exclude("gles31-test-issues.txt")] | |
MNC_GLES31_PKG = Package(module = GLES31_MODULE, configurations = [ | |
Configuration(name = "master", | |
glconfig = "rgba8888d24s8ms0", | |
rotation = "unspecified", | |
surfacetype = "window", | |
filters = MNC_GLES31_COMMON_FILTERS), | |
]) | |
MUSTPASS_LISTS = [ | |
Mustpass(version = "lmp", packages = [LMP_GLES3_PKG, LMP_GLES31_PKG]), | |
Mustpass(version = "lmp-mr1", packages = [LMP_MR1_GLES3_PKG, LMP_MR1_GLES31_PKG]), | |
Mustpass(version = "mnc", packages = [MNC_GLES3_PKG, MNC_GLES31_PKG]) | |
] | |
if __name__ == "__main__": | |
genMustpassLists(MUSTPASS_LISTS) |