external/vulkancts/scripts/verify_submission.py

# -*- coding: utf-8 -*-

#-------------------------------------------------------------------------
# Vulkan CTS
# ----------
#
# Copyright (c) 2016 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
#-------------------------------------------------------------------------

import os
import re
import sys

from fnmatch import fnmatch

sys.path.append(os.path.join(os.path.dirname(__file__), "..", "..", "..", "scripts"))
sys.path.append(os.path.join(os.path.dirname(__file__), "..", "..", "..", "scripts", "log"))

from build.common import readFile
from log_parser import StatusCode, BatchResultParser

ALLOWED_STATUS_CODES = set([
		StatusCode.PASS,
		StatusCode.NOT_SUPPORTED,
		StatusCode.QUALITY_WARNING,
		StatusCode.COMPATIBILITY_WARNING
	])

STATEMENT_PATTERN	= "STATEMENT-*"
TEST_LOG_PATTERN	= "*.qpa"
GIT_STATUS_PATTERN	= "git-status.txt"
GIT_LOG_PATTERN		= "git-log.txt"
PATCH_PATTERN		= "*.patch"

class PackageDescription:
	def __init__ (self, basePath, statement, testLogs, gitStatus, gitLog, patches, otherItems):
		self.basePath		= basePath
		self.statement		= statement
		self.testLogs		= testLogs
		self.gitStatus		= gitStatus
		self.gitLog			= gitLog
		self.patches		= patches
		self.otherItems		= otherItems

class ValidationMessage:
	TYPE_ERROR		= 0
	TYPE_WARNING	= 1

	def __init__ (self, type, filename, message):
		self.type		= type
		self.filename	= filename
		self.message	= message

	def __str__ (self):
		prefix = {self.TYPE_ERROR: "ERROR: ", self.TYPE_WARNING: "WARNING: "}
		return prefix[self.type] + os.path.basename(self.filename) + ": " + self.message

def error (filename, message):
	return ValidationMessage(ValidationMessage.TYPE_ERROR, filename, message)

def warning (filename, message):
	return ValidationMessage(ValidationMessage.TYPE_WARNING, filename, message)

def getPackageDescription (packagePath):
	allItems	= os.listdir(packagePath)
	statement	= None
	testLogs	= []
	gitStatus	= None
	gitLog		= None
	patches		= []
	otherItems	= []

	for item in allItems:
		if fnmatch(item, STATEMENT_PATTERN):
			assert statement == None
			statement = item
		elif fnmatch(item, TEST_LOG_PATTERN):
			testLogs.append(item)
		elif fnmatch(item, GIT_STATUS_PATTERN):
			assert gitStatus == None
			gitStatus = item
		elif fnmatch(item, GIT_LOG_PATTERN):
			assert gitLog == None
			gitLog = item
		elif fnmatch(item, PATCH_PATTERN):
			patches.append(item)
		else:
			otherItems.append(item)

	return PackageDescription(packagePath, statement, testLogs, gitStatus, gitLog, patches, otherItems)

def readMustpass (filename):
	f = open(filename, 'rb')
	cases = []
	for line in f:
		s = line.strip()
		if len(s) > 0:
			cases.append(s)
	return cases

def readTestLog (filename):
	parser = BatchResultParser()
	return parser.parseFile(filename)

def verifyTestLog (filename, mustpass):
	results			= readTestLog(filename)
	messages			= []
	resultOrderOk	= True

	# Mustpass case names must be unique
	assert len(mustpass) == len(set(mustpass))

	# Verify number of results
	if len(results) != len(mustpass):
		messages.append(error(filename, "Wrong number of test results, expected %d, found %d" % (len(mustpass), len(results))))

	caseNameToResultNdx = {}
	for ndx in xrange(len(results)):
		result = results[ndx]
		if not result in caseNameToResultNdx:
			caseNameToResultNdx[result.name] = ndx
		else:
			messages.append(error(filename, "Multiple results for " + result.name))

	# Verify that all results are present and valid
	for ndx in xrange(len(mustpass)):
		caseName = mustpass[ndx]

		if caseName in caseNameToResultNdx:
			resultNdx	= caseNameToResultNdx[caseName]
			result		= results[resultNdx]

			if resultNdx != ndx:
				resultOrderOk = False

			if not result.statusCode in ALLOWED_STATUS_CODES:
				messages.append(error(filename, result.name + ": " + result.statusCode))
		else:
			messages.append(error(filename, "Missing result for " + caseName))

	if len(results) == len(mustpass) and not resultOrderOk:
		messages.append(error(filename, "Results are not in the expected order"))

	return messages

def beginsWith (str, prefix):
	return str[:len(prefix)] == prefix

def verifyStatement (package):
	messages	= []

	if package.statement != None:
		statementPath	= os.path.join(package.basePath, package.statement)
		statement		= readFile(statementPath)
		hasVersion		= False
		hasProduct		= False
		hasCpu			= False
		hasOs			= False

		for line in statement.splitlines():
			if beginsWith(line, "CONFORM_VERSION:"):
				if hasVersion:
					messages.append(error(statementPath, "Multiple CONFORM_VERSIONs"))
				else:
					hasVersion = True
			elif beginsWith(line, "PRODUCT:"):
				hasProduct = True # Multiple products allowed
			elif beginsWith(line, "CPU:"):
				if hasCpu:
					messages.append(error(statementPath, "Multiple PRODUCTs"))
				else:
					hasCpu = True
			elif beginsWith(line, "OS:"):
				if hasOs:
					messages.append(error(statementPath, "Multiple OSes"))
				else:
					hasOs = True

		if not hasVersion:
			messages.append(error(statementPath, "No CONFORM_VERSION"))
		if not hasProduct:
			messages.append(error(statementPath, "No PRODUCT"))
		if not hasCpu:
			messages.append(error(statementPath, "No CPU"))
		if not hasOs:
			messages.append(error(statementPath, "No OS"))
	else:
		messages.append(error(package.basePath, "Missing conformance statement file"))

	return messages

def verifyGitStatus (package):
	messages = []

	if package.gitStatus != None:
		statusPath	= os.path.join(package.basePath, package.gitStatus)
		status		= readFile(statusPath)

		if status.find("nothing to commit, working directory clean") < 0 and status.find("nothing to commit, working tree clean") < 0:
			messages.append(error(package.basePath, "Working directory is not clean"))
	else:
		messages.append(error(package.basePath, "Missing git-status.txt"))

	return messages

def isGitLogEmpty (package):
	assert package.gitLog != None

	logPath	= os.path.join(package.basePath, package.gitLog)
	log		= readFile(logPath)

	return len(log.strip()) == 0

def verifyGitLog (package):
	messages = []

	if package.gitLog != None:
		if not isGitLogEmpty(package):
			messages.append(warning(os.path.join(package.basePath, package.gitLog), "Log is not empty"))
	else:
		messages.append(error(package.basePath, "Missing git-log.txt"))

	return messages

def verifyPatches (package):
	messages	= []
	hasPatches	= len(package.patches)
	logEmpty	= package.gitLog and isGitLogEmpty(package)

	if hasPatches and logEmpty:
		messages.append(error(package.basePath, "Package includes patches but log is empty"))
	elif not hasPatches and not logEmpty:
		messages.append(error(package.basePath, "Test log is not empty but package doesn't contain patches"))

	return messages

def verifyTestLogs (package, mustpass):
	messages	= []

	for testLogFile in package.testLogs:
		messages += verifyTestLog(os.path.join(package.basePath, testLogFile), mustpass)

	if len(package.testLogs) == 0:
		messages.append(error(package.basePath, "No test log files found"))

	return messages

def verifyPackage (package, mustpass):
	messages = []

	messages += verifyStatement(package)
	messages += verifyGitStatus(package)
	messages += verifyGitLog(package)
	messages += verifyPatches(package)
	messages += verifyTestLogs(package, mustpass)

	for item in package.otherItems:
		messages.append(warning(os.path.join(package.basePath, item), "Unknown file"))

	return messages

if __name__ == "__main__":
	if len(sys.argv) != 3:
		print "%s: [extracted submission package] [mustpass]" % sys.argv[0]
		sys.exit(-1)

	packagePath		= os.path.normpath(sys.argv[1])
	mustpassPath	= sys.argv[2]
	package			= getPackageDescription(packagePath)
	mustpass		= readMustpass(mustpassPath)
	messages		= verifyPackage(package, mustpass)

	errors			= [m for m in messages if m.type == ValidationMessage.TYPE_ERROR]
	warnings		= [m for m in messages if m.type == ValidationMessage.TYPE_WARNING]

	for message in messages:
		print str(message)

	print ""

	if len(errors) > 0:
		print "Found %d validation errors and %d warnings!" % (len(errors), len(warnings))
		sys.exit(-2)
	elif len(warnings) > 0:
		print "Found %d warnings, manual review required" % len(warnings)
		sys.exit(-1)
	else:
		print "All validation checks passed"