diff --git a/CMakeLists.txt b/CMakeLists.txt
index 931b064..f4e56e6 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -682,6 +682,10 @@
 	ELSE ()
 		ADD_TEST(libgit2_clar libgit2_clar -v)
 	ENDIF ()
+
+	# Add a test target which runs the cred callback tests, to be
+	# called after setting the url and user
+	ADD_TEST(libgit2_clar-cred_callback libgit2_clar -v -sonline::clone::cred_callback)
 ENDIF ()
 
 IF (TAGS)
diff --git a/appveyor.yml b/appveyor.yml
index 7186aaf..3ed3c49 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -37,7 +37,7 @@
     if "%GENERATOR%"=="MSYS Makefiles" (C:\MinGW\msys\1.0\bin\sh --login /c/projects/libgit2/script/appveyor-mingw.sh)
 test_script:
 - ps: |
-    ctest -V .
+    ctest -V -R libgit2_clar
     $env:GITTEST_REMOTE_URL="https://github.com/libgit2/non-existent"
     $env:GITTEST_REMOTE_USER="libgit2test"
-    .\Debug\libgit2_clar -sonline::clone::cred_callback
+    ctest -V -R libgit2_clar-cred_callback
diff --git a/script/cibuild.sh b/script/cibuild.sh
index 029dcd4..00cde0a 100755
--- a/script/cibuild.sh
+++ b/script/cibuild.sh
@@ -25,7 +25,7 @@
 export GITTEST_REMOTE_URL="git://localhost/test.git"
 
 # Run the test suite
-ctest -V . || exit $?
+ctest -V -R libgit2_clar || exit $?
 
 # Now that we've tested the raw git protocol, let's set up ssh to we
 # can do the push tests over it
@@ -59,4 +59,4 @@
 
 export GITTEST_REMOTE_URL="https://github.com/libgit2/non-existent"
 export GITTEST_REMOTE_USER="libgit2test"
-./libgit2_clar -sonline::clone::cred_callback || exit $?
+ctest -V -R libgit2_clar-cred_callback