| # -*- coding: utf-8 -*- |
| |
| #------------------------------------------------------------------------- |
| # drawElements Quality Program utilities |
| # -------------------------------------- |
| # |
| # Copyright 2015 The Android Open Source Project |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| # |
| #------------------------------------------------------------------------- |
| |
| from build.common import * |
| from build.config import ANY_GENERATOR |
| from build.build import build |
| from build_caselists import Module, getBuildConfig, genCaseList, getCaseListPath, DEFAULT_BUILD_DIR, DEFAULT_TARGET |
| from fnmatch import fnmatch |
| from copy import copy |
| |
| import xml.etree.cElementTree as ElementTree |
| import xml.dom.minidom as minidom |
| |
| CTS_DATA_DIR = os.path.join(DEQP_DIR, "android", "cts") |
| |
| class Configuration: |
| def __init__ (self, name, glconfig, rotation, surfacetype, filters): |
| self.name = name |
| self.glconfig = glconfig |
| self.rotation = rotation |
| self.surfacetype = surfacetype |
| self.filters = filters |
| |
| class Package: |
| def __init__ (self, module, configurations, splitFilters = {}): |
| self.module = module |
| self.configurations = configurations |
| # Map of name:[include filters]. Each will generate <api>.<name> package |
| # Test cases that didn't match any split filter will be in <api> package, |
| # i.e., the default value keeps everything in one package. |
| self.splitFilters = splitFilters |
| |
| class Mustpass: |
| def __init__ (self, version, packages): |
| self.version = version |
| self.packages = packages |
| |
| class Filter: |
| TYPE_INCLUDE = 0 |
| TYPE_EXCLUDE = 1 |
| |
| def __init__ (self, type, filename): |
| self.type = type |
| self.filename = filename |
| |
| class TestRoot: |
| def __init__ (self): |
| self.children = [] |
| |
| class TestGroup: |
| def __init__ (self, name): |
| self.name = name |
| self.children = [] |
| |
| class TestCase: |
| def __init__ (self, name): |
| self.name = name |
| self.configurations = [] |
| |
| class GLESVersion: |
| def __init__(self, major, minor): |
| self.major = major |
| self.minor = minor |
| |
| def encode (self): |
| return (self.major << 16) | (self.minor) |
| |
| def getModuleGLESVersion (module): |
| versions = { |
| 'dEQP-EGL': GLESVersion(2,0), |
| 'dEQP-GLES2': GLESVersion(2,0), |
| 'dEQP-GLES3': GLESVersion(3,0), |
| 'dEQP-GLES31': GLESVersion(3,1) |
| } |
| return versions[module.name] |
| |
| def getSrcDir (mustpass): |
| return os.path.join(CTS_DATA_DIR, mustpass.version, "src") |
| |
| def getTmpDir (mustpass): |
| return os.path.join(CTS_DATA_DIR, mustpass.version, "tmp") |
| |
| def getModuleShorthand (module): |
| assert module.name[:5] == "dEQP-" |
| return module.name[5:].lower() |
| |
| def getCaseListFileName (package, configuration): |
| return "%s-%s.txt" % (getModuleShorthand(package.module), configuration.name) |
| |
| def getDstCaseListPath (mustpass, package, configuration): |
| return os.path.join(CTS_DATA_DIR, mustpass.version, getCaseListFileName(package, configuration)) |
| |
| def getCTSPackageName (package, splitName): |
| if splitName == None: |
| return "com.drawelements.deqp." + getModuleShorthand(package.module) |
| return "com.drawelements.deqp." + getModuleShorthand(package.module) + "." + splitName |
| |
| def getCommandLine (config): |
| return "--deqp-gl-config-name=%s --deqp-screen-rotation=%s --deqp-surface-type=%s --deqp-watchdog=enable" % (config.glconfig, config.rotation, config.surfacetype) |
| |
| def readCaseList (filename): |
| cases = [] |
| with open(filename, 'rb') as f: |
| for line in f: |
| if line[:6] == "TEST: ": |
| cases.append(line[6:].strip()) |
| return cases |
| |
| def getCaseList (mustpass, module): |
| generator = ANY_GENERATOR |
| buildCfg = getBuildConfig(DEFAULT_BUILD_DIR, DEFAULT_TARGET, "Debug") |
| |
| #build(buildCfg, generator, [module.binName]) |
| genCaseList(buildCfg, generator, module, "txt") |
| |
| return readCaseList(getCaseListPath(buildCfg, module, "txt")) |
| |
| def readPatternList (filename): |
| ptrns = [] |
| with open(filename, 'rb') as f: |
| for line in f: |
| line = line.strip() |
| if len(line) > 0 and line[0] != '#': |
| ptrns.append(line) |
| return ptrns |
| |
| def applyPatterns (caseList, patterns, op): |
| matched = set() |
| errors = [] |
| curList = copy(caseList) |
| trivialPtrns = [p for p in patterns if p.find('*') < 0] |
| regularPtrns = [p for p in patterns if p.find('*') >= 0] |
| |
| # Apply trivial (just case paths) |
| allCasesSet = set(caseList) |
| for path in trivialPtrns: |
| if path in allCasesSet: |
| if path in matched: |
| errors.append((path, "Same case specified more than once")) |
| matched.add(path) |
| else: |
| errors.append((path, "Test case not found")) |
| |
| curList = [c for c in curList if c not in matched] |
| |
| for pattern in regularPtrns: |
| matchedThisPtrn = set() |
| |
| for case in curList: |
| if fnmatch(case, pattern): |
| matchedThisPtrn.add(case) |
| |
| if len(matchedThisPtrn) == 0: |
| errors.append((pattern, "Pattern didn't match any cases")) |
| |
| matched = matched | matchedThisPtrn |
| curList = [c for c in curList if c not in matched] |
| |
| for pattern, reason in errors: |
| print "ERROR: %s: %s" % (reason, pattern) |
| |
| if len(errors) > 0: |
| die("Found %s invalid patterns" % len(errors)) |
| |
| return [c for c in caseList if op(c in matched)] |
| |
| def applyInclude (caseList, patterns): |
| return applyPatterns(caseList, patterns, lambda b: b) |
| |
| def applyExclude (caseList, patterns): |
| return applyPatterns(caseList, patterns, lambda b: not b) |
| |
| def readPatternLists (mustpass): |
| lists = {} |
| for package in mustpass.packages: |
| for cfg in package.configurations: |
| for filter in cfg.filters: |
| if not filter.filename in lists: |
| lists[filter.filename] = readPatternList(os.path.join(getSrcDir(mustpass), filter.filename)) |
| return lists |
| |
| def applyFilters (caseList, patternLists, filters): |
| res = copy(caseList) |
| for filter in filters: |
| ptrnList = patternLists[filter.filename] |
| if filter.type == Filter.TYPE_INCLUDE: |
| res = applyInclude(res, ptrnList) |
| else: |
| assert filter.type == Filter.TYPE_EXCLUDE |
| res = applyExclude(res, ptrnList) |
| return res |
| |
| def appendToHierarchy (root, casePath): |
| def findChild (node, name): |
| for child in node.children: |
| if child.name == name: |
| return child |
| return None |
| |
| curNode = root |
| components = casePath.split('.') |
| |
| for component in components[:-1]: |
| nextNode = findChild(curNode, component) |
| if not nextNode: |
| nextNode = TestGroup(component) |
| curNode.children.append(nextNode) |
| curNode = nextNode |
| |
| if not findChild(curNode, components[-1]): |
| curNode.children.append(TestCase(components[-1])) |
| |
| def buildTestHierachy (caseList): |
| root = TestRoot() |
| for case in caseList: |
| appendToHierarchy(root, case) |
| return root |
| |
| def buildTestCaseMap (root): |
| caseMap = {} |
| |
| def recursiveBuild (curNode, prefix): |
| curPath = prefix + curNode.name |
| if isinstance(curNode, TestCase): |
| caseMap[curPath] = curNode |
| else: |
| for child in curNode.children: |
| recursiveBuild(child, curPath + '.') |
| |
| for child in root.children: |
| recursiveBuild(child, '') |
| |
| return caseMap |
| |
| def include (filename): |
| return Filter(Filter.TYPE_INCLUDE, filename) |
| |
| def exclude (filename): |
| return Filter(Filter.TYPE_EXCLUDE, filename) |
| |
| def prettifyXML (doc): |
| uglyString = ElementTree.tostring(doc, 'utf-8') |
| reparsed = minidom.parseString(uglyString) |
| return reparsed.toprettyxml(indent='\t', encoding='utf-8') |
| |
| def genCTSPackageXML (package, root, name): |
| def isLeafGroup (testGroup): |
| numGroups = 0 |
| numTests = 0 |
| |
| for child in testGroup.children: |
| if isinstance(child, TestCase): |
| numTests += 1 |
| else: |
| numGroups += 1 |
| |
| assert numGroups + numTests > 0 |
| |
| if numGroups > 0 and numTests > 0: |
| die("Mixed groups and cases in %s" % testGroup.name) |
| |
| return numGroups == 0 |
| |
| def makeConfiguration (parentElem, configuration): |
| return ElementTree.SubElement(parentElem, "TestInstance", glconfig=configuration.glconfig, rotation=configuration.rotation, surfacetype=configuration.surfacetype) |
| |
| def makeTestCase (parentElem, testCase): |
| caseElem = ElementTree.SubElement(parentElem, "Test", name=testCase.name) |
| for config in testCase.configurations: |
| makeConfiguration(caseElem, config) |
| return caseElem |
| |
| def makeTestGroup (parentElem, testGroup): |
| groupElem = ElementTree.SubElement(parentElem, "TestCase" if isLeafGroup(testGroup) else "TestSuite", name=testGroup.name) |
| for child in testGroup.children: |
| if isinstance(child, TestCase): |
| makeTestCase(groupElem, child) |
| else: |
| makeTestGroup(groupElem, child) |
| return groupElem |
| |
| pkgElem = ElementTree.Element("TestPackage", |
| name = package.module.name, |
| appPackageName = name, |
| testType = "deqpTest") |
| |
| pkgElem.set("xmlns:deqp", "http://drawelements.com/deqp") |
| pkgElem.set("deqp:glesVersion", str(getModuleGLESVersion(package.module).encode())) |
| |
| for child in root.children: |
| makeTestGroup(pkgElem, child) |
| |
| return pkgElem |
| |
| def genSpecXML (mustpass): |
| mustpassElem = ElementTree.Element("Mustpass", version = mustpass.version) |
| |
| for package in mustpass.packages: |
| packageElem = ElementTree.SubElement(mustpassElem, "TestPackage", name = package.module.name) |
| |
| for config in package.configurations: |
| configElem = ElementTree.SubElement(packageElem, "Configuration", |
| name = config.name, |
| caseListFile = getCaseListFileName(package, config), |
| commandLine = getCommandLine(config)) |
| |
| return mustpassElem |
| |
| def genCTSPackage (package, cases, matchingByConfig, packageName, xmlFilename): |
| root = buildTestHierachy(cases) |
| testCaseMap = buildTestCaseMap(root) |
| |
| for config in package.configurations: |
| for case in matchingByConfig[config]: |
| if case in testCaseMap: |
| testCaseMap[case].configurations.append(config) |
| |
| packageXml = genCTSPackageXML(package, root, packageName) |
| |
| print " Writing CTS caselist: " + xmlFilename |
| writeFile(xmlFilename, prettifyXML(packageXml)) |
| |
| def genMustpass (mustpass, moduleCaseLists): |
| print "Generating mustpass '%s'" % mustpass.version |
| |
| patternLists = readPatternLists(mustpass) |
| |
| for package in mustpass.packages: |
| allCasesInPkg = moduleCaseLists[package.module] |
| matchingByConfig = {} |
| allMatchingSet = set() |
| |
| for config in package.configurations: |
| filtered = applyFilters(allCasesInPkg, patternLists, config.filters) |
| dstFile = getDstCaseListPath(mustpass, package, config) |
| |
| print " Writing deqp caselist: " + dstFile |
| writeFile(dstFile, "\n".join(filtered) + "\n") |
| |
| matchingByConfig[config] = filtered |
| allMatchingSet = allMatchingSet | set(filtered) |
| |
| allMatchingCases = [c for c in allCasesInPkg if c in allMatchingSet] # To preserve ordering |
| splitFilters = package.splitFilters |
| for splitName in splitFilters.keys(): |
| splitIncludeFilters = splitFilters[splitName] |
| splitCases = applyInclude(allMatchingCases, splitIncludeFilters) |
| packageName = getCTSPackageName(package, splitName) |
| xmlFilename = os.path.join(CTS_DATA_DIR, mustpass.version, packageName + ".xml") |
| genCTSPackage(package, splitCases, matchingByConfig, packageName, xmlFilename) |
| |
| # The cases not matching any of the includes |
| combinedSplitFilters = reduce(lambda x,y: x+y, splitFilters.values(), []) |
| restOfCases = applyExclude(allMatchingCases, combinedSplitFilters) |
| packageName = getCTSPackageName(package, None) |
| xmlFilename = os.path.join(CTS_DATA_DIR, mustpass.version, packageName + ".xml") |
| genCTSPackage(package, restOfCases, matchingByConfig, packageName, xmlFilename) |
| |
| specXML = genSpecXML(mustpass) |
| specFilename = os.path.join(CTS_DATA_DIR, mustpass.version, "mustpass.xml") |
| |
| print " Writing spec: " + specFilename |
| writeFile(specFilename, prettifyXML(specXML)) |
| |
| print "Done!" |
| |
| def genMustpassLists (mustpassLists): |
| moduleCaseLists = {} |
| |
| # Getting case lists involves invoking build, so we want to cache the results |
| for mustpass in mustpassLists: |
| for package in mustpass.packages: |
| if not package.module in moduleCaseLists: |
| moduleCaseLists[package.module] = getCaseList(mustpass, package.module) |
| |
| for mustpass in mustpassLists: |
| genMustpass(mustpass, moduleCaseLists) |
| |
| EGL_MODULE = Module(name = "dEQP-EGL", dirName = "egl", binName = "deqp-egl") |
| GLES2_MODULE = Module(name = "dEQP-GLES2", dirName = "gles2", binName = "deqp-gles2") |
| GLES3_MODULE = Module(name = "dEQP-GLES3", dirName = "gles3", binName = "deqp-gles3") |
| GLES31_MODULE = Module(name = "dEQP-GLES31", dirName = "gles31", binName = "deqp-gles31") |
| |
| LMP_GLES3_PKG = Package(module = GLES3_MODULE, configurations = [ |
| Configuration(name = "master", |
| glconfig = "rgba8888d24s8ms0", |
| rotation = "unspecified", |
| surfacetype = "window", |
| filters = [include("es30-lmp.txt")]), |
| ]) |
| LMP_GLES31_PKG = Package(module = GLES31_MODULE, configurations = [ |
| Configuration(name = "master", |
| glconfig = "rgba8888d24s8ms0", |
| rotation = "unspecified", |
| surfacetype = "window", |
| filters = [include("es31-lmp.txt")]), |
| ]) |
| |
| LMP_MR1_GLES3_PKG = Package(module = GLES3_MODULE, configurations = [ |
| Configuration(name = "master", |
| glconfig = "rgba8888d24s8ms0", |
| rotation = "unspecified", |
| surfacetype = "window", |
| filters = [include("es30-lmp-mr1.txt")]), |
| ]) |
| LMP_MR1_GLES31_PKG = Package(module = GLES31_MODULE, configurations = [ |
| Configuration(name = "master", |
| glconfig = "rgba8888d24s8ms0", |
| rotation = "unspecified", |
| surfacetype = "window", |
| filters = [include("es31-lmp-mr1.txt")]), |
| ]) |
| |
| MASTER_EGL_COMMON_FILTERS = [include("egl-master.txt"), exclude("egl-failures.txt")] |
| MASTER_EGL_PKG = Package(module = EGL_MODULE, configurations = [ |
| # Master |
| Configuration(name = "master", |
| glconfig = "rgba8888d24s8ms0", |
| rotation = "unspecified", |
| surfacetype = "window", |
| filters = MASTER_EGL_COMMON_FILTERS), |
| ]) |
| |
| MASTER_GLES2_COMMON_FILTERS = [ |
| include("gles2-master.txt"), |
| exclude("gles2-test-issues.txt"), |
| exclude("gles2-failures.txt") |
| ] |
| MASTER_GLES2_PKG = Package(module = GLES2_MODULE, configurations = [ |
| # Master |
| Configuration(name = "master", |
| glconfig = "rgba8888d24s8ms0", |
| rotation = "unspecified", |
| surfacetype = "window", |
| filters = MASTER_GLES2_COMMON_FILTERS), |
| ]) |
| |
| MASTER_GLES3_COMMON_FILTERS = [ |
| include("gles3-master.txt"), |
| exclude("gles3-hw-issues.txt"), |
| exclude("gles3-driver-issues.txt"), |
| exclude("gles3-test-issues.txt"), |
| exclude("gles3-spec-issues.txt") |
| ] |
| MASTER_GLES3_PKG = Package(module = GLES3_MODULE, configurations = [ |
| # Master |
| Configuration(name = "master", |
| glconfig = "rgba8888d24s8ms0", |
| rotation = "unspecified", |
| surfacetype = "window", |
| filters = MASTER_GLES3_COMMON_FILTERS), |
| # Rotations |
| Configuration(name = "rotate-portrait", |
| glconfig = "rgba8888d24s8ms0", |
| rotation = "0", |
| surfacetype = "window", |
| filters = MASTER_GLES3_COMMON_FILTERS + [include("gles3-rotation.txt")]), |
| Configuration(name = "rotate-landscape", |
| glconfig = "rgba8888d24s8ms0", |
| rotation = "90", |
| surfacetype = "window", |
| filters = MASTER_GLES3_COMMON_FILTERS + [include("gles3-rotation.txt")]), |
| Configuration(name = "rotate-reverse-portrait", |
| glconfig = "rgba8888d24s8ms0", |
| rotation = "180", |
| surfacetype = "window", |
| filters = MASTER_GLES3_COMMON_FILTERS + [include("gles3-rotation.txt")]), |
| Configuration(name = "rotate-reverse-landscape", |
| glconfig = "rgba8888d24s8ms0", |
| rotation = "270", |
| surfacetype = "window", |
| filters = MASTER_GLES3_COMMON_FILTERS + [include("gles3-rotation.txt")]), |
| |
| # MSAA |
| Configuration(name = "multisample", |
| glconfig = "rgba8888d24s8ms4", |
| rotation = "unspecified", |
| surfacetype = "window", |
| filters = MASTER_GLES3_COMMON_FILTERS + [include("gles3-multisample.txt"), |
| exclude("gles3-multisample-issues.txt")]), |
| |
| # Pixel format |
| Configuration(name = "565-no-depth-no-stencil", |
| glconfig = "rgb565d0s0ms0", |
| rotation = "unspecified", |
| surfacetype = "window", |
| filters = MASTER_GLES3_COMMON_FILTERS + [include("gles3-pixelformat.txt"), |
| exclude("gles3-pixelformat-issues.txt")]), |
| ]) |
| |
| MASTER_GLES31_COMMON_FILTERS = [ |
| include("gles31-master.txt"), |
| exclude("gles31-hw-issues.txt"), |
| exclude("gles31-driver-issues.txt"), |
| exclude("gles31-test-issues.txt"), |
| exclude("gles31-spec-issues.txt"), |
| ] |
| MASTER_GLES31_PKG = Package(module = GLES31_MODULE, configurations = [ |
| # Master |
| Configuration(name = "master", |
| glconfig = "rgba8888d24s8ms0", |
| rotation = "unspecified", |
| surfacetype = "window", |
| filters = MASTER_GLES31_COMMON_FILTERS), |
| |
| # Rotations |
| Configuration(name = "rotate-portrait", |
| glconfig = "rgba8888d24s8ms0", |
| rotation = "0", |
| surfacetype = "window", |
| filters = MASTER_GLES31_COMMON_FILTERS + [include("gles31-rotation.txt")]), |
| Configuration(name = "rotate-landscape", |
| glconfig = "rgba8888d24s8ms0", |
| rotation = "90", |
| surfacetype = "window", |
| filters = MASTER_GLES31_COMMON_FILTERS + [include("gles31-rotation.txt")]), |
| Configuration(name = "rotate-reverse-portrait", |
| glconfig = "rgba8888d24s8ms0", |
| rotation = "180", |
| surfacetype = "window", |
| filters = MASTER_GLES31_COMMON_FILTERS + [include("gles31-rotation.txt")]), |
| Configuration(name = "rotate-reverse-landscape", |
| glconfig = "rgba8888d24s8ms0", |
| rotation = "270", |
| surfacetype = "window", |
| filters = MASTER_GLES31_COMMON_FILTERS + [include("gles31-rotation.txt")]), |
| |
| # MSAA |
| Configuration(name = "multisample", |
| glconfig = "rgba8888d24s8ms4", |
| rotation = "unspecified", |
| surfacetype = "window", |
| filters = MASTER_GLES31_COMMON_FILTERS + [include("gles31-multisample.txt")]), |
| |
| # Pixel format |
| Configuration(name = "565-no-depth-no-stencil", |
| glconfig = "rgb565d0s0ms0", |
| rotation = "unspecified", |
| surfacetype = "window", |
| filters = MASTER_GLES31_COMMON_FILTERS + [include("gles31-pixelformat.txt")]), |
| ], |
| splitFilters = {"copy_image_compressed": ["dEQP-GLES31.functional.copy_image.compressed.*"], |
| "copy_image_non_compressed": ["dEQP-GLES31.functional.copy_image.non_compressed.*"], |
| "copy_image_mixed": ["dEQP-GLES31.functional.copy_image.mixed.*"], |
| } |
| ) |
| |
| MUSTPASS_LISTS = [ |
| Mustpass(version = "lmp", packages = [LMP_GLES3_PKG, LMP_GLES31_PKG]), |
| Mustpass(version = "lmp-mr1", packages = [LMP_MR1_GLES3_PKG, LMP_MR1_GLES31_PKG]), |
| Mustpass(version = "master", packages = [MASTER_EGL_PKG, MASTER_GLES2_PKG, MASTER_GLES3_PKG, MASTER_GLES31_PKG]) |
| ] |
| |
| if __name__ == "__main__": |
| genMustpassLists(MUSTPASS_LISTS) |
