blob: ec63839bb5d43169fb3844cec342254636cbcfce [file] [log] [blame]
/*-------------------------------------------------------------------------
* drawElements Quality Program Tester Core
* ----------------------------------------
*
* Copyright 2014 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*//*!
* \file
* \brief Test executor.
*//*--------------------------------------------------------------------*/
#include "tcuTestSessionExecutor.hpp"
#include "qpTestLog.h"
#include "tcuCommandLine.hpp"
#include "tcuTestLog.hpp"
#include "deClock.h"
namespace tcu
{
using std::vector;
static qpTestCaseType nodeTypeToTestCaseType(TestNodeType nodeType)
{
switch (nodeType)
{
case NODETYPE_SELF_VALIDATE:
return QP_TEST_CASE_TYPE_SELF_VALIDATE;
case NODETYPE_PERFORMANCE:
return QP_TEST_CASE_TYPE_PERFORMANCE;
case NODETYPE_CAPABILITY:
return QP_TEST_CASE_TYPE_CAPABILITY;
case NODETYPE_ACCURACY:
return QP_TEST_CASE_TYPE_ACCURACY;
default:
DE_ASSERT(false);
return QP_TEST_CASE_TYPE_LAST;
}
}
TestSessionExecutor::TestSessionExecutor(TestPackageRoot &root, TestContext &testCtx)
: m_testCtx(testCtx)
, m_inflater(testCtx)
, m_caseListFilter(testCtx.getCommandLine().createCaseListFilter(testCtx.getArchive()))
, m_iterator(root, m_inflater, *m_caseListFilter)
, m_state(STATE_TRAVERSE_HIERARCHY)
, m_abortSession(false)
, m_isInTestCase(false)
, m_testStartTime(0)
, m_packageStartTime(0)
{
}
TestSessionExecutor::~TestSessionExecutor(void)
{
}
bool TestSessionExecutor::iterate(void)
{
while (!m_abortSession)
{
switch (m_state)
{
case STATE_TRAVERSE_HIERARCHY:
{
const TestHierarchyIterator::State hierIterState = m_iterator.getState();
if (hierIterState == TestHierarchyIterator::STATE_ENTER_NODE ||
hierIterState == TestHierarchyIterator::STATE_LEAVE_NODE)
{
TestNode *const curNode = m_iterator.getNode();
const TestNodeType nodeType = curNode->getNodeType();
const bool isEnter = hierIterState == TestHierarchyIterator::STATE_ENTER_NODE;
switch (nodeType)
{
case NODETYPE_PACKAGE:
{
TestPackage *const testPackage = static_cast<TestPackage *>(curNode);
isEnter ? enterTestPackage(testPackage) : leaveTestPackage(testPackage);
break;
}
case NODETYPE_GROUP:
{
isEnter ? enterTestGroup(m_iterator.getNodePath()) : leaveTestGroup(m_iterator.getNodePath());
break; // nada
}
case NODETYPE_SELF_VALIDATE:
case NODETYPE_PERFORMANCE:
case NODETYPE_CAPABILITY:
case NODETYPE_ACCURACY:
{
TestCase *const testCase = static_cast<TestCase *>(curNode);
if (isEnter)
{
if (enterTestCase(testCase, m_iterator.getNodePath()))
m_state = STATE_EXECUTE_TEST_CASE;
// else remain in TRAVERSING_HIERARCHY => node will be exited from in the next iteration
}
else
leaveTestCase(testCase);
break;
}
default:
DE_ASSERT(false);
break;
}
m_iterator.next();
break;
}
else
{
DE_ASSERT(hierIterState == TestHierarchyIterator::STATE_FINISHED);
m_status.isComplete = true;
return false;
}
}
case STATE_EXECUTE_TEST_CASE:
{
DE_ASSERT(m_iterator.getState() == TestHierarchyIterator::STATE_LEAVE_NODE &&
isTestNodeTypeExecutable(m_iterator.getNode()->getNodeType()));
TestCase *const testCase = static_cast<TestCase *>(m_iterator.getNode());
const TestCase::IterateResult iterResult = iterateTestCase(testCase);
if (iterResult == TestCase::STOP)
m_state = STATE_TRAVERSE_HIERARCHY;
return true;
}
default:
DE_ASSERT(false);
break;
}
}
return false;
}
void TestSessionExecutor::enterTestPackage(TestPackage *testPackage)
{
// Create test case wrapper
DE_ASSERT(!m_caseExecutor);
m_caseExecutor = de::MovePtr<TestCaseExecutor>(testPackage->createExecutor());
testPackage->setCaseListFilter(m_caseListFilter.get());
m_packageStartTime = deGetMicroseconds();
}
void TestSessionExecutor::leaveTestPackage(TestPackage *testPackage)
{
DE_UNREF(testPackage);
m_caseExecutor->deinitTestPackage(m_testCtx);
// If m_caseExecutor uses local status then it may perform some tests in deinitTestPackage(). We have to update TestSessionExecutor::m_status
if (m_caseExecutor->usesLocalStatus())
m_caseExecutor->updateGlobalStatus(m_status);
const int64_t duration = deGetMicroseconds() - m_packageStartTime;
m_packageStartTime = 0;
if (!std::string(m_testCtx.getCommandLine().getServerAddress()).empty())
m_caseExecutor->reportDurations(m_testCtx, std::string(testPackage->getName()), duration, m_groupsDurationTime);
m_caseExecutor.clear();
if (!std::string(m_testCtx.getCommandLine().getServerAddress()).empty())
{
m_testCtx.getLog().startTestsCasesTime();
m_testCtx.getLog() << TestLog::Integer(testPackage->getName(), "Total tests case duration in microseconds",
"us", QP_KEY_TAG_TIME, duration);
for (std::map<std::string, uint64_t>::iterator it = m_groupsDurationTime.begin();
it != m_groupsDurationTime.end(); ++it)
m_testCtx.getLog() << TestLog::Integer(it->first, "The test group case duration in microseconds", "us",
QP_KEY_TAG_TIME, it->second);
m_testCtx.getLog().endTestsCasesTime();
}
}
void TestSessionExecutor::enterTestGroup(const std::string &casePath)
{
m_groupsDurationTime[casePath] = deGetMicroseconds();
}
void TestSessionExecutor::leaveTestGroup(const std::string &casePath)
{
m_groupsDurationTime[casePath] = deGetMicroseconds() - m_groupsDurationTime[casePath];
}
bool TestSessionExecutor::enterTestCase(TestCase *testCase, const std::string &casePath)
{
TestLog &log = m_testCtx.getLog();
const qpTestCaseType caseType = nodeTypeToTestCaseType(testCase->getNodeType());
bool initOk = false;
print("\nTest case '%s'..\n", casePath.c_str());
#if (DE_OS == DE_OS_WIN32)
fflush(stdout);
#endif
m_testCtx.setTestResult(QP_TEST_RESULT_LAST, "");
m_testCtx.setTerminateAfter(false);
log.startCase(casePath.c_str(), caseType);
m_isInTestCase = true;
m_testStartTime = deGetMicroseconds();
try
{
m_caseExecutor->init(testCase, casePath);
initOk = true;
}
catch (const std::bad_alloc &)
{
DE_ASSERT(!initOk);
m_testCtx.setTestResult(QP_TEST_RESULT_RESOURCE_ERROR, "Failed to allocate memory in test case init");
m_testCtx.setTerminateAfter(true);
}
catch (const tcu::TestException &e)
{
DE_ASSERT(!initOk);
DE_ASSERT(e.getTestResult() != QP_TEST_RESULT_LAST);
m_testCtx.setTestResult(e.getTestResult(), e.getMessage());
m_testCtx.setTerminateAfter(e.isFatal());
log << e;
}
catch (const tcu::Exception &e)
{
DE_ASSERT(!initOk);
m_testCtx.setTestResult(QP_TEST_RESULT_FAIL, e.getMessage());
log << e;
}
DE_ASSERT(initOk || m_testCtx.getTestResult() != QP_TEST_RESULT_LAST);
return initOk;
}
void TestSessionExecutor::leaveTestCase(TestCase *testCase)
{
TestLog &log = m_testCtx.getLog();
// De-init case.
try
{
m_caseExecutor->deinit(testCase);
}
catch (const tcu::Exception &e)
{
const bool suppressLogging = m_testCtx.getLog().isSupressLogging();
if (suppressLogging)
m_testCtx.getLog().supressLogging(false);
log << e << TestLog::Message << "Error in test case deinit, test program will terminate."
<< TestLog::EndMessage;
m_testCtx.setTerminateAfter(true);
m_testCtx.getLog().supressLogging(suppressLogging);
}
{
const int64_t duration = deGetMicroseconds() - m_testStartTime;
m_testStartTime = 0;
m_testCtx.getLog() << TestLog::Integer("TestDuration", "Test case duration in microseconds", "us",
QP_KEY_TAG_TIME, duration);
}
{
const qpTestResult testResult = m_testCtx.getTestResult();
const char *const testResultDesc = m_testCtx.getTestResultDesc();
const bool terminateAfter = m_testCtx.getTerminateAfter();
DE_ASSERT(testResult != QP_TEST_RESULT_LAST);
m_isInTestCase = false;
m_testCtx.getLog().endCase(testResult, testResultDesc);
// Update statistics.
print(" %s (%s)\n", qpGetTestResultName(testResult), testResultDesc);
#if (DE_OS == DE_OS_WIN32)
fflush(stdout);
#endif
if (!m_caseExecutor->usesLocalStatus())
{
m_status.numExecuted += 1;
switch (testResult)
{
case QP_TEST_RESULT_PASS:
m_status.numPassed += 1;
break;
case QP_TEST_RESULT_NOT_SUPPORTED:
m_status.numNotSupported += 1;
break;
case QP_TEST_RESULT_QUALITY_WARNING:
m_status.numWarnings += 1;
break;
case QP_TEST_RESULT_COMPATIBILITY_WARNING:
m_status.numWarnings += 1;
break;
case QP_TEST_RESULT_WAIVER:
m_status.numWaived += 1;
break;
case QP_TEST_RESULT_DEVICE_LOST:
m_status.numDeviceLost += 1;
m_status.numFailed += 1;
break;
default:
m_status.numFailed += 1;
break;
}
}
else
{
m_caseExecutor->updateGlobalStatus(m_status);
}
// terminateAfter, Resource error or any error in deinit means that execution should end
if (terminateAfter || testResult == QP_TEST_RESULT_RESOURCE_ERROR ||
(m_status.numFailed > 0 && m_testCtx.getCommandLine().isTerminateOnFailEnabled()) ||
(m_status.numDeviceLost > 0 && m_testCtx.getCommandLine().isTerminateOnDeviceLostEnabled()))
m_abortSession = true;
}
if (m_testCtx.getWatchDog())
qpWatchDog_reset(m_testCtx.getWatchDog());
}
TestCase::IterateResult TestSessionExecutor::iterateTestCase(TestCase *testCase)
{
TestLog &log = m_testCtx.getLog();
TestCase::IterateResult iterateResult = TestCase::STOP;
m_testCtx.touchWatchdog();
try
{
iterateResult = m_caseExecutor->iterate(testCase);
}
catch (const std::bad_alloc &)
{
m_testCtx.setTestResult(QP_TEST_RESULT_RESOURCE_ERROR, "Failed to allocate memory during test execution");
m_testCtx.setTerminateAfter(true);
}
catch (const tcu::TestException &e)
{
log << e;
m_testCtx.setTestResult(e.getTestResult(), e.getMessage());
m_testCtx.setTerminateAfter(e.isFatal());
}
catch (const tcu::Exception &e)
{
log << e;
m_testCtx.setTestResult(QP_TEST_RESULT_FAIL, e.getMessage());
}
return iterateResult;
}
} // namespace tcu