scripts/verify/verify.py

# -*- coding: utf-8 -*-

#-------------------------------------------------------------------------
# Vulkan CTS
# ----------
#
# Copyright (c) 2016 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
#-------------------------------------------------------------------------

import os
import sys

sys.path.append(os.path.join(os.path.dirname(__file__), "..", "log"))
sys.path.append(os.path.join(os.path.dirname(__file__), "..", "build"))

from common import readFile
from message import *
from log_parser import StatusCode, BatchResultParser

ALLOWED_STATUS_CODES = set([
		StatusCode.PASS,
		StatusCode.NOT_SUPPORTED,
		StatusCode.QUALITY_WARNING,
		StatusCode.COMPATIBILITY_WARNING
	])

def readMustpass (filename):
	f = open(filename, 'rb')
	cases = []
	for line in f:
		s = line.strip()
		if len(s) > 0:
			cases.append(s)
	return cases

def readTestLog (filename):
	parser = BatchResultParser()
	return parser.parseFile(filename)

def verifyTestLog (filename, mustpass):
	results			= readTestLog(filename)
	messages			= []
	resultOrderOk	= True

	# Mustpass case names must be unique
	assert len(mustpass) == len(set(mustpass))

	# Verify number of results
	if len(results) != len(mustpass):
		messages.append(error(filename, "Wrong number of test results, expected %d, found %d" % (len(mustpass), len(results))))

	caseNameToResultNdx = {}
	for ndx in xrange(len(results)):
		result = results[ndx]
		if not result in caseNameToResultNdx:
			caseNameToResultNdx[result.name] = ndx
		else:
			messages.append(error(filename, "Multiple results for " + result.name))

	# Verify that all results are present and valid
	for ndx in xrange(len(mustpass)):
		caseName = mustpass[ndx]

		if caseName in caseNameToResultNdx:
			resultNdx	= caseNameToResultNdx[caseName]
			result		= results[resultNdx]

			if resultNdx != ndx:
				resultOrderOk = False

			if not result.statusCode in ALLOWED_STATUS_CODES:
				messages.append(error(filename, result.name + ": " + result.statusCode))
		else:
			messages.append(error(filename, "Missing result for " + caseName))

	if len(results) == len(mustpass) and not resultOrderOk:
		messages.append(error(filename, "Results are not in the expected order"))

	return messages

def beginsWith (str, prefix):
	return str[:len(prefix)] == prefix

def verifyStatement (package):
	messages	= []

	if package.statement != None:
		statementPath	= os.path.join(package.basePath, package.statement)
		statement		= readFile(statementPath)
		hasVersion		= False
		hasProduct		= False
		hasCpu			= False
		hasOs			= False

		for line in statement.splitlines():
			if beginsWith(line, "CONFORM_VERSION:"):
				if hasVersion:
					messages.append(error(statementPath, "Multiple CONFORM_VERSIONs"))
				else:
					assert len(line.split()) >= 2
					package.conformVersion = line.split()[1]
					hasVersion = True
			elif beginsWith(line, "PRODUCT:"):
				hasProduct = True # Multiple products allowed
			elif beginsWith(line, "CPU:"):
				if hasCpu:
					messages.append(error(statementPath, "Multiple PRODUCTs"))
				else:
					hasCpu = True
			elif beginsWith(line, "OS:"):
				if hasOs:
					messages.append(error(statementPath, "Multiple OSes"))
				else:
					assert len(line.split()) >= 2
					package.conformOs = line.split()[1]
					hasOs = True

		if not hasVersion:
			messages.append(error(statementPath, "No CONFORM_VERSION"))
		if not hasProduct:
			messages.append(error(statementPath, "No PRODUCT"))
		if not hasCpu:
			messages.append(error(statementPath, "No CPU"))
		if not hasOs:
			messages.append(error(statementPath, "No OS"))
	else:
		messages.append(error(package.basePath, "Missing conformance statement file"))

	return messages

def verifyGitStatus (package):
	messages = []

	if len(package.gitStatus) > 0:
		for s in package.gitStatus:
			statusPath	= os.path.join(package.basePath, s)
			status		= readFile(statusPath)

			if status.find("nothing to commit, working directory clean") < 0 and status.find("nothing to commit, working tree clean") < 0:
				messages.append(error(package.basePath, "Working directory is not clean"))
	else:
		messages.append(error(package.basePath, "Missing git status files"))

	return messages

def isGitLogEmpty (package, gitLog):
	logPath	= os.path.join(package.basePath, gitLog)
	log		= readFile(logPath)

	return len(log.strip()) == 0

def verifyGitLog (package):
	messages = []

	if len(package.gitLog) > 0:
		for log, path in package.gitLog:
			if not isGitLogEmpty(package, log):
				messages.append(warning(os.path.join(package.basePath, log), "Log is not empty"))
	else:
		messages.append(error(package.basePath, "Missing git log files"))

	return messages

def verifyPatches (package):
	messages	= []
	hasPatches	= len(package.patches)
	logEmpty	= True
	for log, path in package.gitLog:
		logEmpty &= isGitLogEmpty(package, log)

	if hasPatches and logEmpty:
		messages.append(error(package.basePath, "Package includes patches but log is empty"))
	elif not hasPatches and not logEmpty:
		messages.append(error(package.basePath, "Test log is not empty but package doesn't contain patches"))

	return messages