diff --git a/tests/ui/feature-gates/test-listing-format-json.rs b/tests/ui/feature-gates/test-listing-format-json.rs
new file mode 100644
index 0000000..2dd0e10
--- /dev/null
+++ b/tests/ui/feature-gates/test-listing-format-json.rs
@@ -0,0 +1,18 @@
+// no-prefer-dynamic
+// compile-flags: --test
+// run-flags: --list --format json -Zunstable-options
+// run-fail
+// check-run-results
+// ignore-nightly
+// unset-exec-env:RUSTC_BOOTSTRAP
+
+#![cfg(test)]
+#[test]
+fn m_test() {}
+
+#[test]
+#[ignore = "not yet implemented"]
+fn z_test() {}
+
+#[test]
+fn a_test() {}
diff --git a/tests/ui/feature-gates/test-listing-format-json.run.stderr b/tests/ui/feature-gates/test-listing-format-json.run.stderr
new file mode 100644
index 0000000..e81cb81
--- /dev/null
+++ b/tests/ui/feature-gates/test-listing-format-json.run.stderr
@@ -0,0 +1 @@
+error: the option `Z` is only accepted on the nightly compiler
diff --git a/tests/ui/test-attrs/tests-listing-format-json.rs b/tests/ui/test-attrs/tests-listing-format-json.rs
index 18f1521..5afc274 100644
--- a/tests/ui/test-attrs/tests-listing-format-json.rs
+++ b/tests/ui/test-attrs/tests-listing-format-json.rs
@@ -3,6 +3,7 @@
 // run-flags: --list --format json -Zunstable-options
 // run-pass
 // check-run-results
+// only-nightly
 // normalize-stdout-test: "fake-test-src-base/test-attrs/" -> "$$DIR/"
 // normalize-stdout-test: "fake-test-src-base\\test-attrs\\" -> "$$DIR/"