Merge topic 'ctest-resource-fixes' into release-3.16
a033bafbe0 Help: Clarify how tests are run if no resource spec file is specified
a64ba0235f CTest: Clarify that resource requirements can be split
f9f294f5fa CTest: Add version field to resource spec file
Acked-by: Kitware Robot <kwrobot@kitware.com>
Merge-request: !4080
diff --git a/Help/manual/ctest.1.rst b/Help/manual/ctest.1.rst
index dbefb91..e29ebca 100644
--- a/Help/manual/ctest.1.rst
+++ b/Help/manual/ctest.1.rst
@@ -1331,6 +1331,23 @@
determine which resources have been allocated to each group. For example,
each group may correspond to a process the test will spawn when executed.
+Note that even if a test specifies a ``RESOURCE_GROUPS`` property, it is still
+possible for that to test to run without any resource allocation (and without
+the corresponding
+:ref:`environment variables <ctest-resource-environment-variables>`)
+if the user does not pass a resource specification file. Passing this file,
+either through the ``--resource-spec-file`` command-line argument or the
+``RESOURCE_SPEC_FILE`` argument to :command:`ctest_test`, is what activates the
+resource allocation feature. Tests should check the
+``CTEST_RESOURCE_GROUP_COUNT`` environment variable to find out whether or not
+resource allocation is activated. This variable will always (and only) be
+defined if resource allocation is activated. If resource allocation is not
+activated, then the ``CTEST_RESOURCE_GROUP_COUNT`` variable will not exist,
+even if it exists for the parent ``ctest`` process. If a test absolutely must
+have resource allocation, then it can return a failing exit code or use the
+:prop_test:`SKIP_RETURN_CODE` or :prop_test:`SKIP_REGULAR_EXPRESSION`
+properties to indicate a skipped test.
+
.. _`ctest-resource-specification-file`:
Resource Specification File
@@ -1345,6 +1362,10 @@
.. code-block:: json
{
+ "version": {
+ "major": 1,
+ "minor": 0
+ },
"local": [
{
"gpus": [
@@ -1376,6 +1397,11 @@
The members are:
+``version``
+ An object containing a ``major`` integer field and a ``minor`` integer field.
+ Currently, the only supported version is major ``1``, minor ``0``. Any other
+ value is an error.
+
``local``
A JSON array of resource sets present on the system. Currently, this array
is restricted to being of size 1.
diff --git a/Help/prop_test/RESOURCE_GROUPS.rst b/Help/prop_test/RESOURCE_GROUPS.rst
index 436451c..63c56ce 100644
--- a/Help/prop_test/RESOURCE_GROUPS.rst
+++ b/Help/prop_test/RESOURCE_GROUPS.rst
@@ -38,6 +38,22 @@
single cryptography chip. In total, 3 resource groups are specified for this
test, each with its own unique requirements.
+Note that the number of slots following the resource type specifies slots from
+a *single* instance of the resource. If the resource group can tolerate
+receiving slots from different instances of the same resource, it can indicate
+this by splitting the specification into multiple requirements of one slot. For
+example:
+
+.. code-block:: cmake
+
+ add_test(NAME MyTest COMMAND MyExe)
+ set_property(TEST MyTest PROPERTY RESOURCE_GROUPS
+ "gpus:1,gpus:1,gpus:1,gpus:1")
+
+In this case, the single resource group indicates that it needs four GPU slots,
+all of which may come from separate GPUs (though they don't have to; CTest may
+still assign slots from the same GPU.)
+
When CTest sets the :ref:`environment variables
<ctest-resource-environment-variables>` for a test, it assigns a group number
based on the group description, starting at 0 on the left and the number of
diff --git a/Source/CTest/cmCTestResourceSpec.cxx b/Source/CTest/cmCTestResourceSpec.cxx
index b4a2b30..237a745 100644
--- a/Source/CTest/cmCTestResourceSpec.cxx
+++ b/Source/CTest/cmCTestResourceSpec.cxx
@@ -33,6 +33,32 @@
return false;
}
+ int majorVersion = 1;
+ int minorVersion = 0;
+ if (root.isMember("version")) {
+ auto const& version = root["version"];
+ if (version.isObject()) {
+ if (!version.isMember("major") || !version.isMember("minor")) {
+ return false;
+ }
+ auto const& major = version["major"];
+ auto const& minor = version["minor"];
+ if (!major.isInt() || !minor.isInt()) {
+ return false;
+ }
+ majorVersion = major.asInt();
+ minorVersion = minor.asInt();
+ } else {
+ return false;
+ }
+ } else {
+ return false;
+ }
+
+ if (majorVersion != 1 || minorVersion != 0) {
+ return false;
+ }
+
auto const& local = root["local"];
if (!local.isArray()) {
return false;
diff --git a/Tests/CMakeLib/testCTestResourceGroups.cxx b/Tests/CMakeLib/testCTestResourceGroups.cxx
index 5fd7d4a..c3532a6 100644
--- a/Tests/CMakeLib/testCTestResourceGroups.cxx
+++ b/Tests/CMakeLib/testCTestResourceGroups.cxx
@@ -74,6 +74,9 @@
{ "1,threads:1,", true, {
{ { "threads", 1, 1 } },
} },
+ { "threads:1,threads:1", true, {
+ { { "threads", 1, 1 }, { "threads", 1, 1 } },
+ } },
{ "threads:1;;threads:2", true, {
{ { "threads", 1, 1 } },
{ { "threads", 2, 1 } },
diff --git a/Tests/CMakeLib/testCTestResourceSpec.cxx b/Tests/CMakeLib/testCTestResourceSpec.cxx
index e4bc770..b981387 100644
--- a/Tests/CMakeLib/testCTestResourceSpec.cxx
+++ b/Tests/CMakeLib/testCTestResourceSpec.cxx
@@ -39,6 +39,23 @@
{"spec16.json", true, {{{}}}},
{"spec17.json", false, {{{}}}},
{"spec18.json", false, {{{}}}},
+ {"spec19.json", false, {{{}}}},
+ {"spec20.json", true, {{{}}}},
+ {"spec21.json", false, {{{}}}},
+ {"spec22.json", false, {{{}}}},
+ {"spec23.json", false, {{{}}}},
+ {"spec24.json", false, {{{}}}},
+ {"spec25.json", false, {{{}}}},
+ {"spec26.json", false, {{{}}}},
+ {"spec27.json", false, {{{}}}},
+ {"spec28.json", false, {{{}}}},
+ {"spec29.json", false, {{{}}}},
+ {"spec30.json", false, {{{}}}},
+ {"spec31.json", false, {{{}}}},
+ {"spec32.json", false, {{{}}}},
+ {"spec33.json", false, {{{}}}},
+ {"spec34.json", false, {{{}}}},
+ {"spec35.json", false, {{{}}}},
{"noexist.json", false, {{{}}}},
/* clang-format on */
};
diff --git a/Tests/CMakeLib/testCTestResourceSpec_data/spec1.json b/Tests/CMakeLib/testCTestResourceSpec_data/spec1.json
index ee3d0ce..b01aa6b 100644
--- a/Tests/CMakeLib/testCTestResourceSpec_data/spec1.json
+++ b/Tests/CMakeLib/testCTestResourceSpec_data/spec1.json
@@ -1,4 +1,8 @@
{
+ "version": {
+ "major": 1,
+ "minor": 0
+ },
"local": [
{
"gpus": [
diff --git a/Tests/CMakeLib/testCTestResourceSpec_data/spec10.json b/Tests/CMakeLib/testCTestResourceSpec_data/spec10.json
index 22105d7..8764907 100644
--- a/Tests/CMakeLib/testCTestResourceSpec_data/spec10.json
+++ b/Tests/CMakeLib/testCTestResourceSpec_data/spec10.json
@@ -1,4 +1,8 @@
{
+ "version": {
+ "major": 1,
+ "minor": 0
+ },
"local": [
{
"gpus": [
diff --git a/Tests/CMakeLib/testCTestResourceSpec_data/spec11.json b/Tests/CMakeLib/testCTestResourceSpec_data/spec11.json
index 1e37ef5..7551ea2 100644
--- a/Tests/CMakeLib/testCTestResourceSpec_data/spec11.json
+++ b/Tests/CMakeLib/testCTestResourceSpec_data/spec11.json
@@ -1,4 +1,8 @@
{
+ "version": {
+ "major": 1,
+ "minor": 0
+ },
"local": [
{
"gpus": [
diff --git a/Tests/CMakeLib/testCTestResourceSpec_data/spec14.json b/Tests/CMakeLib/testCTestResourceSpec_data/spec14.json
index ce708c7..83f480c 100644
--- a/Tests/CMakeLib/testCTestResourceSpec_data/spec14.json
+++ b/Tests/CMakeLib/testCTestResourceSpec_data/spec14.json
@@ -1,4 +1,8 @@
{
+ "version": {
+ "major": 1,
+ "minor": 0
+ },
"local": [
{
"0": [
diff --git a/Tests/CMakeLib/testCTestResourceSpec_data/spec15.json b/Tests/CMakeLib/testCTestResourceSpec_data/spec15.json
index 78b6990..10fe2e3 100644
--- a/Tests/CMakeLib/testCTestResourceSpec_data/spec15.json
+++ b/Tests/CMakeLib/testCTestResourceSpec_data/spec15.json
@@ -1,4 +1,8 @@
{
+ "version": {
+ "major": 1,
+ "minor": 0
+ },
"local": [
{
"-": [
diff --git a/Tests/CMakeLib/testCTestResourceSpec_data/spec16.json b/Tests/CMakeLib/testCTestResourceSpec_data/spec16.json
index 95c7d26..8546759 100644
--- a/Tests/CMakeLib/testCTestResourceSpec_data/spec16.json
+++ b/Tests/CMakeLib/testCTestResourceSpec_data/spec16.json
@@ -1,4 +1,8 @@
{
+ "version": {
+ "major": 1,
+ "minor": 0
+ },
"local": [
{
"A": [
diff --git a/Tests/CMakeLib/testCTestResourceSpec_data/spec17.json b/Tests/CMakeLib/testCTestResourceSpec_data/spec17.json
index 1b6ab4b..e4cdfc9 100644
--- a/Tests/CMakeLib/testCTestResourceSpec_data/spec17.json
+++ b/Tests/CMakeLib/testCTestResourceSpec_data/spec17.json
@@ -1,4 +1,8 @@
{
+ "version": {
+ "major": 1,
+ "minor": 0
+ },
"local": [
{
"gpus": [
diff --git a/Tests/CMakeLib/testCTestResourceSpec_data/spec18.json b/Tests/CMakeLib/testCTestResourceSpec_data/spec18.json
index 1a17df7..26a7c70 100644
--- a/Tests/CMakeLib/testCTestResourceSpec_data/spec18.json
+++ b/Tests/CMakeLib/testCTestResourceSpec_data/spec18.json
@@ -1,4 +1,8 @@
{
+ "version": {
+ "major": 1,
+ "minor": 0
+ },
"local": [
{
"gpus": [
diff --git a/Tests/CMakeLib/testCTestResourceSpec_data/spec19.json b/Tests/CMakeLib/testCTestResourceSpec_data/spec19.json
new file mode 100644
index 0000000..3067f0c
--- /dev/null
+++ b/Tests/CMakeLib/testCTestResourceSpec_data/spec19.json
@@ -0,0 +1,5 @@
+{
+ "version": 1,
+ "local": [
+ ]
+}
diff --git a/Tests/CMakeLib/testCTestResourceSpec_data/spec2.json b/Tests/CMakeLib/testCTestResourceSpec_data/spec2.json
index 6175b1a..df49390 100644
--- a/Tests/CMakeLib/testCTestResourceSpec_data/spec2.json
+++ b/Tests/CMakeLib/testCTestResourceSpec_data/spec2.json
@@ -1,4 +1,8 @@
{
+ "version": {
+ "major": 1,
+ "minor": 0
+ },
"local": [
]
}
diff --git a/Tests/CMakeLib/testCTestResourceSpec_data/spec20.json b/Tests/CMakeLib/testCTestResourceSpec_data/spec20.json
new file mode 100644
index 0000000..df49390
--- /dev/null
+++ b/Tests/CMakeLib/testCTestResourceSpec_data/spec20.json
@@ -0,0 +1,8 @@
+{
+ "version": {
+ "major": 1,
+ "minor": 0
+ },
+ "local": [
+ ]
+}
diff --git a/Tests/CMakeLib/testCTestResourceSpec_data/spec21.json b/Tests/CMakeLib/testCTestResourceSpec_data/spec21.json
new file mode 100644
index 0000000..7459ff2
--- /dev/null
+++ b/Tests/CMakeLib/testCTestResourceSpec_data/spec21.json
@@ -0,0 +1,5 @@
+{
+ "version": [1, 0],
+ "local": [
+ ]
+}
diff --git a/Tests/CMakeLib/testCTestResourceSpec_data/spec22.json b/Tests/CMakeLib/testCTestResourceSpec_data/spec22.json
new file mode 100644
index 0000000..23c57d8
--- /dev/null
+++ b/Tests/CMakeLib/testCTestResourceSpec_data/spec22.json
@@ -0,0 +1,5 @@
+{
+ "version": 2,
+ "local": [
+ ]
+}
diff --git a/Tests/CMakeLib/testCTestResourceSpec_data/spec23.json b/Tests/CMakeLib/testCTestResourceSpec_data/spec23.json
new file mode 100644
index 0000000..a3d0a27
--- /dev/null
+++ b/Tests/CMakeLib/testCTestResourceSpec_data/spec23.json
@@ -0,0 +1,7 @@
+{
+ "version": {
+ "major": 1
+ },
+ "local": [
+ ]
+}
diff --git a/Tests/CMakeLib/testCTestResourceSpec_data/spec24.json b/Tests/CMakeLib/testCTestResourceSpec_data/spec24.json
new file mode 100644
index 0000000..d5f6b08
--- /dev/null
+++ b/Tests/CMakeLib/testCTestResourceSpec_data/spec24.json
@@ -0,0 +1,7 @@
+{
+ "version": {
+ "minor": 0
+ },
+ "local": [
+ ]
+}
diff --git a/Tests/CMakeLib/testCTestResourceSpec_data/spec25.json b/Tests/CMakeLib/testCTestResourceSpec_data/spec25.json
new file mode 100644
index 0000000..914da4b
--- /dev/null
+++ b/Tests/CMakeLib/testCTestResourceSpec_data/spec25.json
@@ -0,0 +1,8 @@
+{
+ "version": {
+ "major": 1,
+ "minor": 1
+ },
+ "local": [
+ ]
+}
diff --git a/Tests/CMakeLib/testCTestResourceSpec_data/spec26.json b/Tests/CMakeLib/testCTestResourceSpec_data/spec26.json
new file mode 100644
index 0000000..c06de22
--- /dev/null
+++ b/Tests/CMakeLib/testCTestResourceSpec_data/spec26.json
@@ -0,0 +1,8 @@
+{
+ "version": {
+ "major": 2,
+ "minor": 0
+ },
+ "local": [
+ ]
+}
diff --git a/Tests/CMakeLib/testCTestResourceSpec_data/spec27.json b/Tests/CMakeLib/testCTestResourceSpec_data/spec27.json
new file mode 100644
index 0000000..9e2b6c3
--- /dev/null
+++ b/Tests/CMakeLib/testCTestResourceSpec_data/spec27.json
@@ -0,0 +1,8 @@
+{
+ "version": {
+ "major": "1",
+ "minor": 0
+ },
+ "local": [
+ ]
+}
diff --git a/Tests/CMakeLib/testCTestResourceSpec_data/spec28.json b/Tests/CMakeLib/testCTestResourceSpec_data/spec28.json
new file mode 100644
index 0000000..ce3b76a
--- /dev/null
+++ b/Tests/CMakeLib/testCTestResourceSpec_data/spec28.json
@@ -0,0 +1,8 @@
+{
+ "version": {
+ "major": 1,
+ "minor": "0"
+ },
+ "local": [
+ ]
+}
diff --git a/Tests/CMakeLib/testCTestResourceSpec_data/spec29.json b/Tests/CMakeLib/testCTestResourceSpec_data/spec29.json
new file mode 100644
index 0000000..58afd2b
--- /dev/null
+++ b/Tests/CMakeLib/testCTestResourceSpec_data/spec29.json
@@ -0,0 +1,5 @@
+{
+ "version": [1, 0, 0],
+ "local": [
+ ]
+}
diff --git a/Tests/CMakeLib/testCTestResourceSpec_data/spec3.json b/Tests/CMakeLib/testCTestResourceSpec_data/spec3.json
index 82453ec..2f1045f 100644
--- a/Tests/CMakeLib/testCTestResourceSpec_data/spec3.json
+++ b/Tests/CMakeLib/testCTestResourceSpec_data/spec3.json
@@ -1,4 +1,8 @@
{
+ "version": {
+ "major": 1,
+ "minor": 0
+ },
"local": [
{
},
diff --git a/Tests/CMakeLib/testCTestResourceSpec_data/spec30.json b/Tests/CMakeLib/testCTestResourceSpec_data/spec30.json
new file mode 100644
index 0000000..9e13ff0
--- /dev/null
+++ b/Tests/CMakeLib/testCTestResourceSpec_data/spec30.json
@@ -0,0 +1,5 @@
+{
+ "version": [1],
+ "local": [
+ ]
+}
diff --git a/Tests/CMakeLib/testCTestResourceSpec_data/spec31.json b/Tests/CMakeLib/testCTestResourceSpec_data/spec31.json
new file mode 100644
index 0000000..c0ef7f4
--- /dev/null
+++ b/Tests/CMakeLib/testCTestResourceSpec_data/spec31.json
@@ -0,0 +1,5 @@
+{
+ "version": [1, 1],
+ "local": [
+ ]
+}
diff --git a/Tests/CMakeLib/testCTestResourceSpec_data/spec32.json b/Tests/CMakeLib/testCTestResourceSpec_data/spec32.json
new file mode 100644
index 0000000..abe977e
--- /dev/null
+++ b/Tests/CMakeLib/testCTestResourceSpec_data/spec32.json
@@ -0,0 +1,5 @@
+{
+ "version": [2, 0],
+ "local": [
+ ]
+}
diff --git a/Tests/CMakeLib/testCTestResourceSpec_data/spec33.json b/Tests/CMakeLib/testCTestResourceSpec_data/spec33.json
new file mode 100644
index 0000000..c6ca749
--- /dev/null
+++ b/Tests/CMakeLib/testCTestResourceSpec_data/spec33.json
@@ -0,0 +1,5 @@
+{
+ "version": ["1", 0],
+ "local": [
+ ]
+}
diff --git a/Tests/CMakeLib/testCTestResourceSpec_data/spec34.json b/Tests/CMakeLib/testCTestResourceSpec_data/spec34.json
new file mode 100644
index 0000000..be258f1
--- /dev/null
+++ b/Tests/CMakeLib/testCTestResourceSpec_data/spec34.json
@@ -0,0 +1,5 @@
+{
+ "version": [1, "0"],
+ "local": [
+ ]
+}
diff --git a/Tests/CMakeLib/testCTestResourceSpec_data/spec35.json b/Tests/CMakeLib/testCTestResourceSpec_data/spec35.json
new file mode 100644
index 0000000..137648c
--- /dev/null
+++ b/Tests/CMakeLib/testCTestResourceSpec_data/spec35.json
@@ -0,0 +1,5 @@
+{
+ "version": "1",
+ "local": [
+ ]
+}
diff --git a/Tests/CMakeLib/testCTestResourceSpec_data/spec4.json b/Tests/CMakeLib/testCTestResourceSpec_data/spec4.json
index 05e73d7..17349bd 100644
--- a/Tests/CMakeLib/testCTestResourceSpec_data/spec4.json
+++ b/Tests/CMakeLib/testCTestResourceSpec_data/spec4.json
@@ -1,4 +1,8 @@
{
+ "version": {
+ "major": 1,
+ "minor": 0
+ },
"local": {
}
}
diff --git a/Tests/CMakeLib/testCTestResourceSpec_data/spec5.json b/Tests/CMakeLib/testCTestResourceSpec_data/spec5.json
index 2c63c08..f6e6cb4 100644
--- a/Tests/CMakeLib/testCTestResourceSpec_data/spec5.json
+++ b/Tests/CMakeLib/testCTestResourceSpec_data/spec5.json
@@ -1,2 +1,6 @@
{
+ "version": {
+ "major": 1,
+ "minor": 0
+ }
}
diff --git a/Tests/CMakeLib/testCTestResourceSpec_data/spec6.json b/Tests/CMakeLib/testCTestResourceSpec_data/spec6.json
index 93c790d..eb8b14c 100644
--- a/Tests/CMakeLib/testCTestResourceSpec_data/spec6.json
+++ b/Tests/CMakeLib/testCTestResourceSpec_data/spec6.json
@@ -1,4 +1,8 @@
{
+ "version": {
+ "major": 1,
+ "minor": 0
+ },
"local": [
[]
]
diff --git a/Tests/CMakeLib/testCTestResourceSpec_data/spec7.json b/Tests/CMakeLib/testCTestResourceSpec_data/spec7.json
index 28b6a4f..0447981 100644
--- a/Tests/CMakeLib/testCTestResourceSpec_data/spec7.json
+++ b/Tests/CMakeLib/testCTestResourceSpec_data/spec7.json
@@ -1,4 +1,8 @@
{
+ "version": {
+ "major": 1,
+ "minor": 0
+ },
"local": [
{
"gpus": {
diff --git a/Tests/CMakeLib/testCTestResourceSpec_data/spec8.json b/Tests/CMakeLib/testCTestResourceSpec_data/spec8.json
index 79bd224..ee7a9c2 100644
--- a/Tests/CMakeLib/testCTestResourceSpec_data/spec8.json
+++ b/Tests/CMakeLib/testCTestResourceSpec_data/spec8.json
@@ -1,4 +1,8 @@
{
+ "version": {
+ "major": 1,
+ "minor": 0
+ },
"local": [
{
"gpus": [
diff --git a/Tests/CMakeLib/testCTestResourceSpec_data/spec9.json b/Tests/CMakeLib/testCTestResourceSpec_data/spec9.json
index 6bb1def..ae1117b 100644
--- a/Tests/CMakeLib/testCTestResourceSpec_data/spec9.json
+++ b/Tests/CMakeLib/testCTestResourceSpec_data/spec9.json
@@ -1,4 +1,8 @@
{
+ "version": {
+ "major": 1,
+ "minor": 0
+ },
"local": [
{
"gpus": [
diff --git a/Tests/RunCMake/CTestResourceAllocation/RunCMakeTest.cmake b/Tests/RunCMake/CTestResourceAllocation/RunCMakeTest.cmake
index d52a63e..8584786 100644
--- a/Tests/RunCMake/CTestResourceAllocation/RunCMakeTest.cmake
+++ b/Tests/RunCMake/CTestResourceAllocation/RunCMakeTest.cmake
@@ -160,6 +160,8 @@
run_ctest_resource(checkfree2 1 0)
run_ctest_resource(notenough1 1 0)
run_ctest_resource(notenough2 1 0)
+run_ctest_resource(notenough3 1 0)
+run_ctest_resource(combine 1 0)
run_ctest_resource(ensure_parallel 2 0)
set(ENV{CTEST_RESOURCE_GROUP_COUNT} 2)
diff --git a/Tests/RunCMake/CTestResourceAllocation/combine.cmake b/Tests/RunCMake/CTestResourceAllocation/combine.cmake
new file mode 100644
index 0000000..ed5b251
--- /dev/null
+++ b/Tests/RunCMake/CTestResourceAllocation/combine.cmake
@@ -0,0 +1,5 @@
+setup_resource_tests()
+
+add_resource_test(Test1 0 "widgets:8,widgets:4")
+
+cleanup_resource_tests()
diff --git a/Tests/RunCMake/CTestResourceAllocation/notenough1.cmake b/Tests/RunCMake/CTestResourceAllocation/notenough1.cmake
index 2908812..f820c37 100644
--- a/Tests/RunCMake/CTestResourceAllocation/notenough1.cmake
+++ b/Tests/RunCMake/CTestResourceAllocation/notenough1.cmake
@@ -1,5 +1,5 @@
setup_resource_tests()
-add_resource_test(Test1 1 "fluxcapacitors:200")
+add_resource_test(Test1 0 "fluxcapacitors:200")
cleanup_resource_tests()
diff --git a/Tests/RunCMake/CTestResourceAllocation/notenough2.cmake b/Tests/RunCMake/CTestResourceAllocation/notenough2.cmake
index d7600c8..5b81776 100644
--- a/Tests/RunCMake/CTestResourceAllocation/notenough2.cmake
+++ b/Tests/RunCMake/CTestResourceAllocation/notenough2.cmake
@@ -1,5 +1,5 @@
setup_resource_tests()
-add_resource_test(Test1 1 "terminators:2")
+add_resource_test(Test1 0 "terminators:2")
cleanup_resource_tests()
diff --git a/Tests/RunCMake/CTestResourceAllocation/notenough3-ctest-s-res-check.cmake b/Tests/RunCMake/CTestResourceAllocation/notenough3-ctest-s-res-check.cmake
new file mode 100644
index 0000000..321e9a2
--- /dev/null
+++ b/Tests/RunCMake/CTestResourceAllocation/notenough3-ctest-s-res-check.cmake
@@ -0,0 +1,3 @@
+if(EXISTS "${RunCMake_TEST_BINARY_DIR}/ctresalloc.log")
+ set(RunCMake_TEST_FAILED "ctresalloc.log should not exist")
+endif()
diff --git a/Tests/RunCMake/CTestResourceAllocation/notenough3-ctest-s-res-result.txt b/Tests/RunCMake/CTestResourceAllocation/notenough3-ctest-s-res-result.txt
new file mode 100644
index 0000000..b57e2de
--- /dev/null
+++ b/Tests/RunCMake/CTestResourceAllocation/notenough3-ctest-s-res-result.txt
@@ -0,0 +1 @@
+(-1|255)
diff --git a/Tests/RunCMake/CTestResourceAllocation/notenough3-ctest-s-res-stderr.txt b/Tests/RunCMake/CTestResourceAllocation/notenough3-ctest-s-res-stderr.txt
new file mode 100644
index 0000000..82dfdef
--- /dev/null
+++ b/Tests/RunCMake/CTestResourceAllocation/notenough3-ctest-s-res-stderr.txt
@@ -0,0 +1,4 @@
+^Insufficient resources
+CMake Error at [^
+]*/Tests/RunCMake/CTestResourceAllocation/notenough3-ctest-s-res/test\.cmake:[0-9]+ \(message\):
+ Tests did not pass$
diff --git a/Tests/RunCMake/CTestResourceAllocation/notenough3.cmake b/Tests/RunCMake/CTestResourceAllocation/notenough3.cmake
new file mode 100644
index 0000000..ddf3a9c
--- /dev/null
+++ b/Tests/RunCMake/CTestResourceAllocation/notenough3.cmake
@@ -0,0 +1,5 @@
+setup_resource_tests()
+
+add_resource_test(Test1 0 "widgets:12")
+
+cleanup_resource_tests()
diff --git a/Tests/RunCMake/CTestResourceAllocation/resspec.json b/Tests/RunCMake/CTestResourceAllocation/resspec.json
index c67fcca..48321ec 100644
--- a/Tests/RunCMake/CTestResourceAllocation/resspec.json
+++ b/Tests/RunCMake/CTestResourceAllocation/resspec.json
@@ -1,4 +1,8 @@
{
+ "version": {
+ "major": 1,
+ "minor": 0
+ },
"local": [
{
"widgets": [
