| # -*- coding: utf-8 -*- |
| #------------------------------------------------------------------------- |
| # drawElements Quality Program utilities |
| # -------------------------------------- |
| # |
| # Copyright 2016 The Android Open Source Project |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| # |
| #------------------------------------------------------------------------- |
| |
| from ctsbuild.common import * |
| from ctsbuild.build import build |
| from build_caselists import Module, getModuleByName, getBuildConfig, genCaseList, getCaseListPath, DEFAULT_BUILD_DIR, DEFAULT_TARGET |
| from fnmatch import fnmatch |
| from copy import copy |
| from collections import defaultdict |
| import logging |
| import argparse |
| import re |
| import xml.etree.cElementTree as ElementTree |
| import xml.dom.minidom as minidom |
| |
| GENERATED_FILE_WARNING = """ |
| This file has been automatically generated. Edit with caution. |
| """ |
| |
| class Project: |
| def __init__ (self, path, copyright = None): |
| self.path = path |
| self.copyright = copyright |
| |
| class Configuration: |
| def __init__ (self, name, filters, glconfig = None, rotation = None, surfacetype = None, required = False, runtime = None, runByDefault = True, listOfGroupsToSplit = []): |
| self.name = name |
| self.glconfig = glconfig |
| self.rotation = rotation |
| self.surfacetype = surfacetype |
| self.required = required |
| self.filters = filters |
| self.expectedRuntime = runtime |
| self.runByDefault = runByDefault |
| self.listOfGroupsToSplit = listOfGroupsToSplit |
| |
| class Package: |
| def __init__ (self, module, configurations): |
| self.module = module |
| self.configurations = configurations |
| |
| class Mustpass: |
| def __init__ (self, project, version, packages): |
| self.project = project |
| self.version = version |
| self.packages = packages |
| |
| class Filter: |
| TYPE_INCLUDE = 0 |
| TYPE_EXCLUDE = 1 |
| |
| def __init__ (self, type, filenames): |
| self.type = type |
| self.filenames = filenames |
| self.key = ",".join(filenames) |
| |
| class TestRoot: |
| def __init__ (self): |
| self.children = [] |
| |
| class TestGroup: |
| def __init__ (self, name): |
| self.name = name |
| self.children = [] |
| |
| class TestCase: |
| def __init__ (self, name): |
| self.name = name |
| self.configurations = [] |
| |
| def getSrcDir (mustpass): |
| return os.path.join(mustpass.project.path, mustpass.version, "src") |
| |
| def getModuleShorthand (module): |
| assert module.name[:5] == "dEQP-" |
| return module.name[5:].lower() |
| |
| def getCaseListFileName (package, configuration): |
| return "%s-%s.txt" % (getModuleShorthand(package.module), configuration.name) |
| |
| def getDstCaseListPath (mustpass): |
| return os.path.join(mustpass.project.path, mustpass.version) |
| |
| def getCommandLine (config): |
| cmdLine = "" |
| |
| if config.glconfig != None: |
| cmdLine += "--deqp-gl-config-name=%s " % config.glconfig |
| |
| if config.rotation != None: |
| cmdLine += "--deqp-screen-rotation=%s " % config.rotation |
| |
| if config.surfacetype != None: |
| cmdLine += "--deqp-surface-type=%s " % config.surfacetype |
| |
| cmdLine += "--deqp-watchdog=enable" |
| |
| return cmdLine |
| |
| class CaseList: |
| def __init__(self, filePath, sortedLines): |
| self.filePath = filePath |
| self.sortedLines = sortedLines |
| |
| def readAndSortCaseList (buildCfg, generator, module): |
| build(buildCfg, generator, [module.binName]) |
| genCaseList(buildCfg, generator, module, "txt") |
| filePath = getCaseListPath(buildCfg, module, "txt") |
| with open(filePath, 'r') as first_file: |
| lines = first_file.readlines() |
| lines.sort() |
| caseList = CaseList(filePath, lines) |
| return caseList |
| |
| def readPatternList (filename, patternList): |
| with open(filename, 'rt') as f: |
| for line in f: |
| line = line.strip() |
| if len(line) > 0 and line[0] != '#': |
| patternList.append(line) |
| |
| def include (*filenames): |
| return Filter(Filter.TYPE_INCLUDE, filenames) |
| |
| def exclude (*filenames): |
| return Filter(Filter.TYPE_EXCLUDE, filenames) |
| |
| def insertXMLHeaders (mustpass, doc): |
| if mustpass.project.copyright != None: |
| doc.insert(0, ElementTree.Comment(mustpass.project.copyright)) |
| doc.insert(1, ElementTree.Comment(GENERATED_FILE_WARNING)) |
| |
| def prettifyXML (doc): |
| uglyString = ElementTree.tostring(doc, 'utf-8') |
| reparsed = minidom.parseString(uglyString) |
| return reparsed.toprettyxml(indent='\t', encoding='utf-8') |
| |
| def genSpecXML (mustpass): |
| mustpassElem = ElementTree.Element("Mustpass", version = mustpass.version) |
| insertXMLHeaders(mustpass, mustpassElem) |
| |
| for package in mustpass.packages: |
| packageElem = ElementTree.SubElement(mustpassElem, "TestPackage", name = package.module.name) |
| |
| for config in package.configurations: |
| configElem = ElementTree.SubElement(packageElem, "Configuration", |
| caseListFile = getCaseListFileName(package, config), |
| commandLine = getCommandLine(config), |
| name = config.name) |
| |
| return mustpassElem |
| |
| def addOptionElement (parent, optionName, optionValue): |
| ElementTree.SubElement(parent, "option", name=optionName, value=optionValue) |
| |
| def genAndroidTestXml (mustpass): |
| RUNNER_CLASS = "com.drawelements.deqp.runner.DeqpTestRunner" |
| configElement = ElementTree.Element("configuration") |
| |
| # have the deqp package installed on the device for us |
| preparerElement = ElementTree.SubElement(configElement, "target_preparer") |
| preparerElement.set("class", "com.android.tradefed.targetprep.suite.SuiteApkInstaller") |
| addOptionElement(preparerElement, "cleanup-apks", "true") |
| addOptionElement(preparerElement, "test-file-name", "com.drawelements.deqp.apk") |
| |
| # Target preparer for incremental dEQP |
| preparerElement = ElementTree.SubElement(configElement, "target_preparer") |
| preparerElement.set("class", "com.android.compatibility.common.tradefed.targetprep.FilePusher") |
| addOptionElement(preparerElement, "cleanup", "true") |
| addOptionElement(preparerElement, "disable", "true") |
| addOptionElement(preparerElement, "push", "deqp-binary32->/data/local/tmp/deqp-binary32") |
| addOptionElement(preparerElement, "push", "deqp-binary64->/data/local/tmp/deqp-binary64") |
| addOptionElement(preparerElement, "push", "gles2->/data/local/tmp/gles2") |
| addOptionElement(preparerElement, "push", "gles3->/data/local/tmp/gles3") |
| addOptionElement(preparerElement, "push", "gles3-incremental-deqp.txt->/data/local/tmp/gles3-incremental-deqp.txt") |
| addOptionElement(preparerElement, "push", "gles31->/data/local/tmp/gles31") |
| addOptionElement(preparerElement, "push", "internal->/data/local/tmp/internal") |
| addOptionElement(preparerElement, "push", "vk-incremental-deqp.txt->/data/local/tmp/vk-incremental-deqp.txt") |
| addOptionElement(preparerElement, "push", "vulkan->/data/local/tmp/vulkan") |
| preparerElement = ElementTree.SubElement(configElement, "target_preparer") |
| preparerElement.set("class", "com.android.compatibility.common.tradefed.targetprep.IncrementalDeqpPreparer") |
| addOptionElement(preparerElement, "disable", "true") |
| |
| # add in metadata option for component name |
| ElementTree.SubElement(configElement, "option", name="test-suite-tag", value="cts") |
| ElementTree.SubElement(configElement, "option", key="component", name="config-descriptor:metadata", value="deqp") |
| ElementTree.SubElement(configElement, "option", key="parameter", name="config-descriptor:metadata", value="not_instant_app") |
| ElementTree.SubElement(configElement, "option", key="parameter", name="config-descriptor:metadata", value="multi_abi") |
| ElementTree.SubElement(configElement, "option", key="parameter", name="config-descriptor:metadata", value="secondary_user") |
| ElementTree.SubElement(configElement, "option", key="parameter", name="config-descriptor:metadata", value="no_foldable_states") |
| controllerElement = ElementTree.SubElement(configElement, "object") |
| controllerElement.set("class", "com.android.tradefed.testtype.suite.module.TestFailureModuleController") |
| controllerElement.set("type", "module_controller") |
| addOptionElement(controllerElement, "screenshot-on-failure", "false") |
| |
| for package in mustpass.packages: |
| for config in package.configurations: |
| if not config.runByDefault: |
| continue |
| |
| testElement = ElementTree.SubElement(configElement, "test") |
| testElement.set("class", RUNNER_CLASS) |
| addOptionElement(testElement, "deqp-package", package.module.name) |
| caseListFile = getCaseListFileName(package,config) |
| addOptionElement(testElement, "deqp-caselist-file", caseListFile) |
| if caseListFile.startswith("gles3"): |
| addOptionElement(testElement, "incremental-deqp-include-file", "gles3-incremental-deqp.txt") |
| elif caseListFile.startswith("vk"): |
| addOptionElement(testElement, "incremental-deqp-include-file", "vk-incremental-deqp.txt") |
| # \todo [2015-10-16 kalle]: Replace with just command line? - requires simplifications in the runner/tests as well. |
| if config.glconfig != None: |
| addOptionElement(testElement, "deqp-gl-config-name", config.glconfig) |
| |
| if config.surfacetype != None: |
| addOptionElement(testElement, "deqp-surface-type", config.surfacetype) |
| |
| if config.rotation != None: |
| addOptionElement(testElement, "deqp-screen-rotation", config.rotation) |
| |
| if config.expectedRuntime != None: |
| addOptionElement(testElement, "runtime-hint", config.expectedRuntime) |
| |
| if config.required: |
| addOptionElement(testElement, "deqp-config-required", "true") |
| |
| insertXMLHeaders(mustpass, configElement) |
| |
| return configElement |
| |
| class PatternSet: |
| def __init__(self): |
| self.namedPatternsTree = {} |
| self.namedPatternsDict = {} |
| self.wildcardPatternsDict = {} |
| |
| def readPatternSets (mustpass): |
| patternSets = {} |
| for package in mustpass.packages: |
| for cfg in package.configurations: |
| for filter in cfg.filters: |
| if not filter.key in patternSets: |
| patternList = [] |
| for filename in filter.filenames: |
| readPatternList(os.path.join(getSrcDir(mustpass), filename), patternList) |
| patternSet = PatternSet() |
| for pattern in patternList: |
| if pattern.find('*') == -1: |
| patternSet.namedPatternsDict[pattern] = 0 |
| t = patternSet.namedPatternsTree |
| parts = pattern.split('.') |
| for part in parts: |
| t = t.setdefault(part, {}) |
| else: |
| # We use regex instead of fnmatch because it's faster |
| patternSet.wildcardPatternsDict[re.compile("^" + pattern.replace(".", r"\.").replace("*", ".*?") + "$")] = 0 |
| patternSets[filter.key] = patternSet |
| return patternSets |
| |
| def genMustpassFromLists (mustpass, moduleCaseLists): |
| print("Generating mustpass '%s'" % mustpass.version) |
| patternSets = readPatternSets(mustpass) |
| |
| for package in mustpass.packages: |
| currentCaseList = moduleCaseLists[package.module] |
| logging.debug("Reading " + currentCaseList.filePath) |
| |
| for config in package.configurations: |
| # construct components of path to main destination file |
| mainDstFileDir = getDstCaseListPath(mustpass) |
| mainDstFileName = getCaseListFileName(package, config) |
| mainDstFilePath = os.path.join(mainDstFileDir, mainDstFileName) |
| mainGroupSubDir = mainDstFileName[:-4] |
| |
| if not os.path.exists(mainDstFileDir): |
| os.makedirs(mainDstFileDir) |
| mainDstFile = open(mainDstFilePath, 'w') |
| print(mainDstFilePath) |
| output_files = {} |
| def openAndStoreFile(filePath, testFilePath, parentFile): |
| if filePath not in output_files: |
| try: |
| print(" " + filePath) |
| parentFile.write(mainGroupSubDir + "/" + testFilePath + "\n") |
| currentDir = os.path.dirname(filePath) |
| if not os.path.exists(currentDir): |
| os.makedirs(currentDir) |
| output_files[filePath] = open(filePath, 'w') |
| |
| except FileNotFoundError: |
| print(f"File not found: {filePath}") |
| return output_files[filePath] |
| |
| lastOutputFile = "" |
| currentOutputFile = None |
| for line in currentCaseList.sortedLines: |
| if not line.startswith("TEST: "): |
| continue |
| caseName = line.replace("TEST: ", "").strip("\n") |
| caseParts = caseName.split(".") |
| keep = True |
| # Do the includes with the complex patterns first |
| for filter in config.filters: |
| if filter.type == Filter.TYPE_INCLUDE: |
| keep = False |
| patterns = patternSets[filter.key].wildcardPatternsDict |
| for pattern in patterns.keys(): |
| keep = pattern.match(caseName) |
| if keep: |
| patterns[pattern] += 1 |
| break |
| |
| if not keep: |
| t = patternSets[filter.key].namedPatternsTree |
| if len(t.keys()) == 0: |
| continue |
| for part in caseParts: |
| if part in t: |
| t = t[part] |
| else: |
| t = None # Not found |
| break |
| keep = t == {} |
| if keep: |
| patternSets[filter.key].namedPatternsDict[caseName] += 1 |
| |
| # Do the excludes |
| if filter.type == Filter.TYPE_EXCLUDE: |
| patterns = patternSets[filter.key].wildcardPatternsDict |
| for pattern in patterns.keys(): |
| discard = pattern.match(caseName) |
| if discard: |
| patterns[pattern] += 1 |
| keep = False |
| break |
| if keep: |
| t = patternSets[filter.key].namedPatternsTree |
| if len(t.keys()) == 0: |
| continue |
| for part in caseParts: |
| if part in t: |
| t = t[part] |
| else: |
| t = None # Not found |
| break |
| if t == {}: |
| patternSets[filter.key].namedPatternsDict[caseName] += 1 |
| keep = False |
| if not keep: |
| break |
| if not keep: |
| continue |
| |
| parts = caseName.split('.') |
| if len(config.listOfGroupsToSplit) > 0: |
| if len(parts) > 2: |
| groupName = parts[1].replace("_", "-") |
| for splitPattern in config.listOfGroupsToSplit: |
| splitParts = splitPattern.split(".") |
| if len(splitParts) > 1 and caseName.startswith(splitPattern + "."): |
| groupName = groupName + "/" + parts[2].replace("_", "-") |
| filePath = os.path.join(mainDstFileDir, mainGroupSubDir, groupName + ".txt") |
| if lastOutputFile != filePath: |
| currentOutputFile = openAndStoreFile(filePath, groupName + ".txt", mainDstFile) |
| lastOutputFile = filePath |
| currentOutputFile.write(caseName + "\n") |
| else: |
| mainDstFile.write(caseName + "\n") |
| |
| # Check that all patterns have been used in the filters |
| # This check will help identifying typos and patterns becoming stale |
| for filter in config.filters: |
| if filter.type == Filter.TYPE_INCLUDE: |
| patternSet = patternSets[filter.key] |
| for pattern, usage in patternSet.namedPatternsDict.items(): |
| if usage == 0: |
| logging.warning("Case %s in file %s for module %s was never used!" % (pattern, filter.key, config.name)) |
| for pattern, usage in patternSet.wildcardPatternsDict.items(): |
| if usage == 0: |
| logging.warning("Pattern %s in file %s for module %s was never used!" % (pattern, filter.key, config.name)) |
| |
| # Generate XML |
| specXML = genSpecXML(mustpass) |
| specFilename = os.path.join(mustpass.project.path, mustpass.version, "mustpass.xml") |
| |
| print(" Writing spec: " + specFilename) |
| writeFile(specFilename, prettifyXML(specXML).decode()) |
| |
| # TODO: Which is the best selector mechanism? |
| if (mustpass.version == "main"): |
| androidTestXML = genAndroidTestXml(mustpass) |
| androidTestFilename = os.path.join(mustpass.project.path, "AndroidTest.xml") |
| |
| print(" Writing AndroidTest.xml: " + androidTestFilename) |
| writeFile(androidTestFilename, prettifyXML(androidTestXML).decode()) |
| |
| print("Done!") |
| |
| |
| def genMustpassLists (mustpassLists, generator, buildCfg): |
| moduleCaseLists = {} |
| |
| # Getting case lists involves invoking build, so we want to cache the results |
| for mustpass in mustpassLists: |
| for package in mustpass.packages: |
| if not package.module in moduleCaseLists: |
| moduleCaseLists[package.module] = readAndSortCaseList(buildCfg, generator, package.module) |
| |
| for mustpass in mustpassLists: |
| genMustpassFromLists(mustpass, moduleCaseLists) |
| |
| def parseCmdLineArgs (): |
| parser = argparse.ArgumentParser(description = "Build Android CTS mustpass", |
| formatter_class=argparse.ArgumentDefaultsHelpFormatter) |
| parser.add_argument("-b", |
| "--build-dir", |
| dest="buildDir", |
| default=DEFAULT_BUILD_DIR, |
| help="Temporary build directory") |
| parser.add_argument("-t", |
| "--build-type", |
| dest="buildType", |
| default="Debug", |
| help="Build type") |
| parser.add_argument("-c", |
| "--deqp-target", |
| dest="targetName", |
| default=DEFAULT_TARGET, |
| help="dEQP build target") |
| parser.add_argument("-v", "--verbose", |
| dest="verbose", |
| action="store_true", |
| help="Enable verbose logging") |
| return parser.parse_args() |
| |
| def parseBuildConfigFromCmdLineArgs (): |
| args = parseCmdLineArgs() |
| initializeLogger(args.verbose) |
| return getBuildConfig(args.buildDir, args.targetName, args.buildType) |
