[test] Make test discovery robust with retries
Previously, if `fx test --list` failed, the test discovery user
interaction would silently fail, leaving the user with an empty list of
tests without any notification or way to recover.
This change introduces a `TestcaseDiscoveryQueue` to manage test case
discovery, bringing the following benefits:
- Error Handling and Retries: When a test discovery fails, an error
notification is displayed to the user with options to "Retry",
"Show Logs", or "Dismiss".
- JSON parsing: Errors from parsing the output of `fx test --list` are
now explicitly surfaced to users.
- Batching: Multiple discovery requests are batched together, limiting
the amount of notification spam should multiple TestItem discoveries
fail.
The following test improvements are also included:
- Test coverage of `TestcaseDiscoveryQueue`, and implicit coverage of
`discoverTestCases()`.
- `setupMockFfx()` is now generalized to `setupMockedCommand()`,
allowing `fx` to be expressively mocked in the same way.
Fixed: 436612350, 436866233
Change-Id: I8deac64788aad994ef762b7e034ce706446e80a4
Reviewed-on: https://fuchsia-review.googlesource.com/c/vscode-plugins/+/1351884
Reviewed-by: Amy Hu <amyhu@google.com>
Kokoro: Kokoro <noreply+kokoro@google.com>
diff --git a/src/test/suite/ffx.test.ts b/src/test/suite/ffx.test.ts
index f069313..3280b12 100644
--- a/src/test/suite/ffx.test.ts
+++ b/src/test/suite/ffx.test.ts
@@ -12,7 +12,7 @@
import { Ffx, FfxEventType, FuchsiaDevice } from '../../ffx';
import * as logger from '../../logger';
-import { MockedFfxInvocation, StubbedSpawn, noTimeout, setupMockFfx } from './utils';
+import { MockedCommandInvocation, StubbedSpawn, noTimeout, setupMockFfx } from './utils';
describe('FuchsiaDevice', function() {
describe('#constructor()', function() {
@@ -75,7 +75,7 @@
'is_default': false
});
- const mockFfx = (...ffxInvocations: MockedFfxInvocation[]) => setupMockFfx(
+ const mockFfx = (...ffxInvocations: MockedCommandInvocation[]) => setupMockFfx(
stubbedSpawn,
TEST_CWD,
TEST_FFX,
diff --git a/src/test/suite/test_controller.test.ts b/src/test/suite/test_controller.test.ts
new file mode 100644
index 0000000..29a3eb2
--- /dev/null
+++ b/src/test/suite/test_controller.test.ts
@@ -0,0 +1,891 @@
+// Copyright 2025 The Fuchsia Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import * as assert from 'assert';
+import * as vscode from 'vscode';
+import { describe, it } from 'mocha';
+import {createSandbox} from 'sinon';
+
+import {Fx} from '../../fx';
+import {TestcaseDiscoveryQueue} from '../../test_controller';
+
+import {macrotask, MockedCommandInvocation, MOCKED_COMMAND_DURATION, setupMockedCommand, StubbedSpawn} from './utils';
+import {Logger, initLogger} from '../../logger';
+import {Ffx} from '../../ffx';
+
+describe('TestcaseDiscoveryQueue', function() {
+ const sandbox = createSandbox();
+ const TEST_CWD = '/path/to/workspace';
+ let stubbedSpawn: StubbedSpawn;
+ let controller: vscode.TestController;
+ let fx: Fx;
+
+ this.beforeEach(function() {
+ stubbedSpawn = new StubbedSpawn(sandbox);
+ controller = vscode.tests.createTestController('TestcaseDiscoveryQueue', 'TestcaseDiscoveryQueue');
+ const log = vscode.window.createOutputChannel('test_controller.test', { log: true });
+ initLogger(log);
+ const ffx = new Ffx(TEST_CWD);
+ fx = new Fx(TEST_CWD, ffx);
+ });
+
+ this.afterEach(function() {
+ controller.dispose();
+ sandbox.restore();
+ });
+
+ const mockCommands = (...commandInvocations: MockedCommandInvocation[]) => setupMockedCommand(
+ stubbedSpawn,
+ TEST_CWD,
+ commandInvocations,
+ );
+
+ describe('#discover', function() {
+ it('resolves if no tests are discovered', async function() {
+ const TEST_URL = 'fuchsia-pkg://a_repo/test_pkg#meta/foo_test_component.cm';
+ const {actualInvocations, expectedInvocations} = mockCommands(
+ {
+ args: ['fx', 'test', '--logpath=-', '--list', TEST_URL],
+ output: {},
+ },
+ );
+
+ const queue = new TestcaseDiscoveryQueue(controller, fx);
+ const testItem = controller.createTestItem(TEST_URL, 'foo');
+
+ await queue.discover(testItem);
+
+ assert.strictEqual(testItem.children.size, 0);
+ assert.deepStrictEqual(actualInvocations, expectedInvocations);
+ });
+
+ it('populates a TestItem', async function() {
+ const TEST_URL = 'fuchsia-pkg://a_repo/test_pkg#meta/bar_test_component.cm';
+ const {actualInvocations, expectedInvocations} = mockCommands(
+ {
+ args: ['fx', 'test', '--logpath=-', '--list', TEST_URL],
+ output: {
+ payload: {
+ // eslint-disable-next-line @typescript-eslint/naming-convention
+ enumerate_test_cases: {
+ // eslint-disable-next-line @typescript-eslint/naming-convention
+ test_case_names: [
+ 'BarTest.testA',
+ 'BarTest.testB',
+ 'BarTest.testC',
+ ]
+ }
+ }
+ },
+ },
+ );
+
+ const queue = new TestcaseDiscoveryQueue(controller, fx);
+ const testItem = controller.createTestItem(TEST_URL, 'bar');
+
+ await queue.discover(testItem);
+
+ assert.strictEqual(testItem.children.size, 3);
+ assert.ok(testItem.children.get('BarTest.testA'));
+ assert.ok(testItem.children.get('BarTest.testB'));
+ assert.ok(testItem.children.get('BarTest.testC'));
+ assert.deepStrictEqual(actualInvocations, expectedInvocations);
+ });
+
+ it('populates independent TestItem discovery requests', async function() {
+ const FOO_TEST_URL = 'fuchsia-pkg://a_repo/test_pkg#meta/foo_test_component.cm';
+ const BAR_TEST_URL = 'fuchsia-pkg://a_repo/test_pkg#meta/bar_test_component.cm';
+ const {actualInvocations, expectedInvocations} = mockCommands(
+ {
+ args: ['fx', 'test', '--logpath=-', '--list', FOO_TEST_URL],
+ output: {
+ payload: {
+ // eslint-disable-next-line @typescript-eslint/naming-convention
+ enumerate_test_cases: {
+ // eslint-disable-next-line @typescript-eslint/naming-convention
+ test_case_names: [
+ 'FooTest.testCase',
+ ]
+ }
+ }
+ },
+ },
+ {
+ args: ['fx', 'test', '--logpath=-', '--list', BAR_TEST_URL],
+ output: {
+ payload: {
+ // eslint-disable-next-line @typescript-eslint/naming-convention
+ enumerate_test_cases: {
+ // eslint-disable-next-line @typescript-eslint/naming-convention
+ test_case_names: [
+ 'BarTest.testCase',
+ ]
+ }
+ }
+ },
+ },
+ );
+
+ const queue = new TestcaseDiscoveryQueue(controller, fx);
+ const fooTestItem = controller.createTestItem(FOO_TEST_URL, 'foo');
+ const barTestItem = controller.createTestItem(BAR_TEST_URL, 'bar');
+
+ await queue.discover(fooTestItem);
+ await queue.discover(barTestItem);
+
+ assert.strictEqual(fooTestItem.children.size, 1);
+ assert.ok(fooTestItem.children.get('FooTest.testCase'));
+ assert.strictEqual(barTestItem.children.size, 1);
+ assert.ok(barTestItem.children.get('BarTest.testCase'));
+ assert.deepStrictEqual(actualInvocations, expectedInvocations);
+ });
+
+ it('populates varied concurrent TestItem discovery requests', async function() {
+ const INITIAL_TEST_URL = 'fuchsia-pkg://a_repo/test_pkg#meta/initial_discovery.cm';
+ const SYNCHRONOUS_TEST_URL = 'fuchsia-pkg://a_repo/test_pkg#meta/synchronous_discovery.cm';
+ const MICROTASK_TEST_URL = 'fuchsia-pkg://a_repo/test_pkg#meta/microtask_discovery.cm';
+ const MACROTASK_TEST_URL = 'fuchsia-pkg://a_repo/test_pkg#meta/macrotask_discovery.cm';
+ const TIMED_TEST_URL = 'fuchsia-pkg://a_repo/test_pkg#meta/timed_discovery.cm';
+ const {actualInvocations, expectedInvocations, singleCommandDuration} = mockCommands(
+ {
+ args: ['fx', 'test', '--logpath=-', '--list', INITIAL_TEST_URL],
+ output: {
+ payload: {
+ // eslint-disable-next-line @typescript-eslint/naming-convention
+ enumerate_test_cases: {
+ // eslint-disable-next-line @typescript-eslint/naming-convention
+ test_case_names: [
+ 'InitialDiscovery.testCase',
+ ]
+ }
+ }
+ },
+ },
+ {
+ args: ['fx', 'test', '--logpath=-', '--list', SYNCHRONOUS_TEST_URL],
+ output: {
+ payload: {
+ // eslint-disable-next-line @typescript-eslint/naming-convention
+ enumerate_test_cases: {
+ // eslint-disable-next-line @typescript-eslint/naming-convention
+ test_case_names: [
+ 'SynchronousDiscovery.testCase',
+ ]
+ }
+ }
+ },
+ },
+ {
+ args: ['fx', 'test', '--logpath=-', '--list', MICROTASK_TEST_URL],
+ output: {
+ payload: {
+ // eslint-disable-next-line @typescript-eslint/naming-convention
+ enumerate_test_cases: {
+ // eslint-disable-next-line @typescript-eslint/naming-convention
+ test_case_names: [
+ 'MicrotaskDiscovery.testCase',
+ ]
+ }
+ }
+ },
+ },
+ {
+ args: ['fx', 'test', '--logpath=-', '--list', MACROTASK_TEST_URL],
+ output: {
+ payload: {
+ // eslint-disable-next-line @typescript-eslint/naming-convention
+ enumerate_test_cases: {
+ // eslint-disable-next-line @typescript-eslint/naming-convention
+ test_case_names: [
+ 'MacrotaskDiscovery.testCase',
+ ]
+ }
+ }
+ },
+ },
+ {
+ args: ['fx', 'test', '--logpath=-', '--list', TIMED_TEST_URL],
+ output: {
+ payload: {
+ // eslint-disable-next-line @typescript-eslint/naming-convention
+ enumerate_test_cases: {
+ // eslint-disable-next-line @typescript-eslint/naming-convention
+ test_case_names: [
+ 'TimedDiscovery.testCase',
+ ]
+ }
+ }
+ },
+ },
+ );
+
+ const queue = new TestcaseDiscoveryQueue(controller, fx);
+ const initialTestItem = controller.createTestItem(INITIAL_TEST_URL, 'initial');
+ const synchronousTestItem = controller.createTestItem(SYNCHRONOUS_TEST_URL, 'synchronous');
+ const microtaskTestItem = controller.createTestItem(MICROTASK_TEST_URL, 'microtask');
+ const macrotaskTestItem = controller.createTestItem(MACROTASK_TEST_URL, 'macrotask');
+ const timedTestItem = controller.createTestItem(TIMED_TEST_URL, 'timed');
+
+ const discovered = {
+ initial: false,
+ synchronous: false,
+ microtask: false,
+ macrotask: false,
+ timed: false,
+ };
+
+ // Test synchronous concurrent discovery requests.
+ void queue.discover(initialTestItem).then(() => discovered.initial = true);
+ void queue.discover(synchronousTestItem).then(() => discovered.synchronous = true);
+
+ // Wait for the `initial` discovery to complete; keep the `synchronous` one in the queue.
+ await singleCommandDuration();
+ assert.deepStrictEqual(
+ discovered,
+ {initial: true, synchronous: false, microtask: false, macrotask: false, timed: false},
+ );
+ assert.strictEqual(initialTestItem.children.size, 1);
+ assert.ok(initialTestItem.children.get('InitialDiscovery.testCase'));
+
+ // Test concurrent discovery request after 1 microtask.
+ await Promise.resolve();
+ void queue.discover(microtaskTestItem).then(() => discovered.microtask = true);
+
+ // Wait for the `synchronous` discovery to complete; keep the `microtask` one in the queue.
+ await singleCommandDuration();
+ assert.deepStrictEqual(
+ discovered,
+ {initial: true, synchronous: true, microtask: false, macrotask: false, timed: false},
+ );
+ assert.strictEqual(synchronousTestItem.children.size, 1);
+ assert.ok(synchronousTestItem.children.get('SynchronousDiscovery.testCase'));
+
+ // Test concurrent discovery request after 1 macrotask.
+ await macrotask();
+ const macrotaskPromise = queue.discover(macrotaskTestItem)
+ .then(() => discovered.macrotask = true);
+
+ // Wait for the `microtask` discovery to complete; keep the `macrotask` one in the queue.
+ await singleCommandDuration();
+ assert.deepStrictEqual(
+ discovered,
+ {initial: true, synchronous: true, microtask: true, macrotask: false, timed: false},
+ );
+ assert.strictEqual(microtaskTestItem.children.size, 1);
+ assert.ok(microtaskTestItem.children.get('MicrotaskDiscovery.testCase'));
+
+ // Test concurrent discovery request halfway through command execution.
+ await new Promise(resolve => setTimeout(resolve, MOCKED_COMMAND_DURATION / 2));
+ void queue.discover(timedTestItem).then(() => discovered.timed = true);
+
+ // Wait for the `macrotask` discovery to complete; keep the `timed` one in the queue.
+ await macrotaskPromise;
+ assert.deepStrictEqual(
+ discovered,
+ {initial: true, synchronous: true, microtask: true, macrotask: true, timed: false},
+ );
+ assert.strictEqual(macrotaskTestItem.children.size, 1);
+ assert.ok(macrotaskTestItem.children.get('MacrotaskDiscovery.testCase'));
+
+ // Wait for the `timed` discovery to complete; the discovery queue should be empty now.
+ await singleCommandDuration();
+ assert.deepStrictEqual(
+ discovered,
+ {initial: true, synchronous: true, microtask: true, macrotask: true, timed: true},
+ );
+ assert.strictEqual(timedTestItem.children.size, 1);
+ assert.ok(timedTestItem.children.get('TimedDiscovery.testCase'));
+
+ // Check whether `TestcaseDiscoveryQueue` makes any extra unnecessary `fx` invocations.
+ await singleCommandDuration();
+ assert.deepStrictEqual(actualInvocations, expectedInvocations);
+ });
+
+ it('issues a notification without resolving if fx test errors', async function() {
+ const TEST_URL = 'fuchsia-pkg://a_repo/test_pkg#meta/foo_test_component.cm';
+ const {actualInvocations, expectedInvocations, singleCommandDuration} = mockCommands(
+ {
+ args: ['fx', 'test', '--logpath=-', '--list', TEST_URL],
+ output: new Error('{}'),
+ },
+ );
+ const stubShowErrorMessage = sandbox.stub(vscode.window, 'showErrorMessage')
+ .resolves('Dismiss' as any);
+
+ const queue = new TestcaseDiscoveryQueue(controller, fx);
+ const testItem = controller.createTestItem(TEST_URL, 'foo');
+
+ let discoverDidResolve = false;
+ void queue.discover(testItem).then(() => {
+ discoverDidResolve = true;
+ });
+ // Wait for `fx test --list` complete and propagate up to `queue.discover()`.
+ await singleCommandDuration();
+
+ assert.ok(stubShowErrorMessage.calledOnce);
+ assert.strictEqual(discoverDidResolve, false);
+ assert.strictEqual(testItem.children.size, 0);
+ assert.deepStrictEqual(actualInvocations, expectedInvocations);
+ });
+
+ it('ignores irrelevant JSON objects', async function() {
+ const TEST_URL = 'fuchsia-pkg://a_repo/test_pkg#meta/foo_test_component.cm';
+ const MOCK_FX_JSON_OUTPUTS = [
+ undefined,
+ true,
+ {},
+ {
+ payload: -1,
+ },
+ {
+ payload: {},
+ },
+ {
+ payload: {
+ // eslint-disable-next-line @typescript-eslint/naming-convention
+ enumerate_test_cases: 'invalid',
+ },
+ },
+ {
+ payload: {
+ // eslint-disable-next-line @typescript-eslint/naming-convention
+ enumerate_test_cases: [],
+ },
+ },
+ {
+ payload: {
+ // eslint-disable-next-line @typescript-eslint/naming-convention
+ enumerate_test_cases: {
+ // eslint-disable-next-line @typescript-eslint/naming-convention
+ test_case_names: [],
+ },
+ },
+ },
+ {
+ payload: {
+ // eslint-disable-next-line @typescript-eslint/naming-convention
+ enumerate_test_cases: {
+ // eslint-disable-next-line @typescript-eslint/naming-convention
+ test_case_names: ['FooTest.testA'],
+ },
+ },
+ },
+ ];
+ const {actualInvocations, expectedInvocations} = mockCommands(
+ {
+ args: ['fx', 'test', '--logpath=-', '--list', TEST_URL],
+ output: MOCK_FX_JSON_OUTPUTS.map(obj => JSON.stringify(obj)).join('\n'),
+ },
+ );
+
+ const queue = new TestcaseDiscoveryQueue(controller, fx);
+ const testItem = controller.createTestItem(TEST_URL, 'foo');
+
+ await queue.discover(testItem);
+
+ assert.strictEqual(testItem.children.size, 1);
+ assert.ok(testItem.children.get('FooTest.testA'));
+ assert.deepStrictEqual(actualInvocations, expectedInvocations);
+ });
+
+ it('deduplicates test cases', async function() {
+ const TEST_URL = 'fuchsia-pkg://a_repo/test_pkg#meta/foo_test_component.cm';
+ const {actualInvocations, expectedInvocations} = mockCommands(
+ {
+ args: ['fx', 'test', '--logpath=-', '--list', TEST_URL],
+ output: JSON.stringify({
+ payload: {
+ // eslint-disable-next-line @typescript-eslint/naming-convention
+ enumerate_test_cases: {
+ // eslint-disable-next-line @typescript-eslint/naming-convention
+ test_case_names: ['FooTest.testA', 'FooTest.testB'],
+ },
+ },
+ }) + '\n' + JSON.stringify({
+ payload: {
+ // eslint-disable-next-line @typescript-eslint/naming-convention
+ enumerate_test_cases: {
+ // eslint-disable-next-line @typescript-eslint/naming-convention
+ test_case_names: ['FooTest.testB', 'FooTest.testC'],
+ },
+ },
+ }),
+ },
+ );
+
+ const queue = new TestcaseDiscoveryQueue(controller, fx);
+ const testItem = controller.createTestItem(TEST_URL, 'foo');
+
+ await queue.discover(testItem);
+
+ assert.strictEqual(testItem.children.size, 3);
+ assert.ok(testItem.children.get('FooTest.testA'));
+ assert.ok(testItem.children.get('FooTest.testB'));
+ assert.ok(testItem.children.get('FooTest.testC'));
+ assert.deepStrictEqual(actualInvocations, expectedInvocations);
+ });
+
+ it('counts as an error if test_case_names is the wrong type', async function() {
+ const TEST_URL = 'fuchsia-pkg://a_repo/test_pkg#meta/foo_test_component.cm';
+ const {actualInvocations, expectedInvocations, singleCommandDuration} = mockCommands(
+ {
+ args: ['fx', 'test', '--logpath=-', '--list', TEST_URL],
+ output: {
+ payload: {
+ // eslint-disable-next-line @typescript-eslint/naming-convention
+ enumerate_test_cases: {
+ // eslint-disable-next-line @typescript-eslint/naming-convention
+ test_case_names: {
+ fooTest: TEST_URL
+ }
+ }
+ }
+ },
+ },
+ );
+ const stubShowErrorMessage = sandbox.stub(vscode.window, 'showErrorMessage')
+ .resolves('Dismiss' as any);
+
+ const queue = new TestcaseDiscoveryQueue(controller, fx);
+ const testItem = controller.createTestItem(TEST_URL, 'foo');
+
+ let discoverDidResolve = false;
+ void queue.discover(testItem).then(() => {
+ discoverDidResolve = true;
+ });
+ await singleCommandDuration();
+
+ assert.ok(stubShowErrorMessage.calledOnce);
+ assert.strictEqual(discoverDidResolve, false);
+ assert.strictEqual(testItem.children.size, 0);
+ assert.deepStrictEqual(actualInvocations, expectedInvocations);
+ });
+
+ it('counts as an error if test_case_names entries are the wrong type', async function() {
+ const TEST_URL = 'fuchsia-pkg://a_repo/test_pkg#meta/foo_test_component.cm';
+ const {actualInvocations, expectedInvocations, singleCommandDuration} = mockCommands(
+ {
+ args: ['fx', 'test', '--logpath=-', '--list', TEST_URL],
+ output: {
+ payload: {
+ // eslint-disable-next-line @typescript-eslint/naming-convention
+ enumerate_test_cases: {
+ // eslint-disable-next-line @typescript-eslint/naming-convention
+ test_case_names: [
+ 123,
+ ]
+ }
+ }
+ },
+ },
+ );
+ const stubShowErrorMessage = sandbox.stub(vscode.window, 'showErrorMessage')
+ .resolves(undefined);
+
+ const queue = new TestcaseDiscoveryQueue(controller, fx);
+ const testItem = controller.createTestItem(TEST_URL, 'foo');
+
+ let discoverDidResolve = false;
+ void queue.discover(testItem).then(() => {
+ discoverDidResolve = true;
+ });
+ await singleCommandDuration();
+
+ assert.ok(stubShowErrorMessage.calledOnce);
+ assert.strictEqual(discoverDidResolve, false);
+ assert.strictEqual(testItem.children.size, 0);
+ assert.deepStrictEqual(actualInvocations, expectedInvocations);
+ });
+
+ it('shows error logs if the user clicks show logs', async function() {
+ const TEST_URL = 'fuchsia-pkg://a_repo/test_pkg#meta/foo_test_component.cm';
+ const {actualInvocations, expectedInvocations, singleCommandDuration} = mockCommands(
+ {
+ args: ['fx', 'test', '--logpath=-', '--list', TEST_URL],
+ output: new Error('{}'),
+ },
+ );
+ const CHOICES = ['Show Logs'];
+ const stubShowErrorMessage = sandbox.stub(vscode.window, 'showErrorMessage')
+ .callsFake(() => new Promise(resolve => resolve(CHOICES.shift() as any)));
+ const spyLoggerShow = sandbox.spy(Logger.prototype, 'show');
+
+ const queue = new TestcaseDiscoveryQueue(controller, fx);
+ const testItem = controller.createTestItem(TEST_URL, 'foo');
+
+ void queue.discover(testItem);
+ // Wait for `fx test --list` complete and propagate up to `queue.discover()`.
+ await singleCommandDuration();
+
+ assert.strictEqual(stubShowErrorMessage.callCount, 2);
+ assert.ok(spyLoggerShow.calledOnce);
+ assert.strictEqual(testItem.children.size, 0);
+ assert.deepStrictEqual(actualInvocations, expectedInvocations);
+ });
+
+ it('reattempts a discovery and fails again', async function() {
+ const TEST_URL = 'fuchsia-pkg://a_repo/test_pkg#meta/foo_test_component.cm';
+ const {actualInvocations, expectedInvocations, singleCommandDuration} = mockCommands(
+ {
+ args: ['fx', 'test', '--logpath=-', '--list', TEST_URL],
+ output: new Error('{}'),
+ },
+ {
+ args: ['fx', 'test', '--logpath=-', '--list', TEST_URL],
+ output: new Error('{}'),
+ },
+ );
+ const CHOICES = ['Retry'];
+ const stubShowErrorMessage = sandbox.stub(vscode.window, 'showErrorMessage')
+ .callsFake(() => new Promise(resolve => resolve(CHOICES.shift() as any)));
+
+ const queue = new TestcaseDiscoveryQueue(controller, fx);
+ const testItem = controller.createTestItem(TEST_URL, 'foo');
+
+ let discoverDidResolve = false;
+ void queue.discover(testItem).then(() => {
+ discoverDidResolve = true;
+ });
+ // Wait for both discovery failures to complete.
+ await singleCommandDuration();
+ await singleCommandDuration();
+
+ assert.strictEqual(stubShowErrorMessage.callCount, 2);
+ assert.strictEqual(discoverDidResolve, false);
+ assert.strictEqual(testItem.children.size, 0);
+ assert.deepStrictEqual(actualInvocations, expectedInvocations);
+ });
+
+ it('issues one notification when multiple TestItems fail in a batch', async function() {
+ const FOO_TEST_URL = 'fuchsia-pkg://a_repo/test_pkg#meta/foo_test_component.cm';
+ const BAR_TEST_URL = 'fuchsia-pkg://a_repo/test_pkg#meta/bar_test_component.cm';
+ const {actualInvocations, expectedInvocations, singleCommandDuration} = mockCommands(
+ {
+ args: ['fx', 'test', '--logpath=-', '--list', FOO_TEST_URL],
+ output: new Error('{}'),
+ },
+ {
+ args: ['fx', 'test', '--logpath=-', '--list', BAR_TEST_URL],
+ output: new Error('{}'),
+ },
+ );
+ const spyShowErrorMessage = sandbox.spy(vscode.window, 'showErrorMessage');
+
+ const queue = new TestcaseDiscoveryQueue(controller, fx);
+ const fooTestItem = controller.createTestItem(FOO_TEST_URL, 'foo');
+ const barTestItem = controller.createTestItem(BAR_TEST_URL, 'bar');
+
+ let discoverDidResolve = false;
+ void queue.discover(fooTestItem).then(() => {
+ discoverDidResolve = true;
+ });
+ void queue.discover(barTestItem).then(() => {
+ discoverDidResolve = true;
+ });
+ // Wait for both `fx test --list` (not run in parallel) to complete and propagate up to
+ // `queue.discover()`.
+ await singleCommandDuration();
+ await singleCommandDuration();
+
+ assert.ok(spyShowErrorMessage.calledOnce);
+ assert.strictEqual(discoverDidResolve, false);
+ assert.strictEqual(fooTestItem.children.size, 0);
+ assert.strictEqual(barTestItem.children.size, 0);
+ assert.deepStrictEqual(actualInvocations, expectedInvocations);
+ });
+
+ it('batches previous failures onto future discoveries', async function() {
+ const FOO_TEST_URL = 'fuchsia-pkg://a_repo/test_pkg#meta/foo_test_component.cm';
+ const BAR_TEST_URL = 'fuchsia-pkg://a_repo/test_pkg#meta/bar_test_component.cm';
+ const {actualInvocations, expectedInvocations, singleCommandDuration} = mockCommands(
+ {
+ args: ['fx', 'test', '--logpath=-', '--list', FOO_TEST_URL],
+ output: new Error('{}'),
+ },
+ {
+ args: ['fx', 'test', '--logpath=-', '--list', FOO_TEST_URL],
+ output: new Error('{}'),
+ },
+ {
+ args: ['fx', 'test', '--logpath=-', '--list', BAR_TEST_URL],
+ output: new Error('{}'),
+ },
+ {
+ args: ['fx', 'test', '--logpath=-', '--list', FOO_TEST_URL],
+ output: new Error('{}'),
+ },
+ {
+ args: ['fx', 'test', '--logpath=-', '--list', BAR_TEST_URL],
+ output: new Error('{}'),
+ },
+ );
+ const CHOICES = [
+ undefined, // User clicks the (x) button on the notification.
+ 'Retry',
+ 'Dismiss',
+ ];
+ const stubShowErrorMessage = sandbox.stub(vscode.window, 'showErrorMessage')
+ .callsFake(() => new Promise(resolve => resolve(CHOICES.shift() as any)));
+
+ const queue = new TestcaseDiscoveryQueue(controller, fx);
+ const fooTestItem = controller.createTestItem(FOO_TEST_URL, 'foo');
+ const barTestItem = controller.createTestItem(BAR_TEST_URL, 'bar');
+
+ let discoverDidResolve = false;
+ void queue.discover(fooTestItem).then(() => {
+ discoverDidResolve = true;
+ });
+ // Wait the first discovery to fail.
+ await singleCommandDuration();
+
+ void queue.discover(barTestItem).then(() => {
+ discoverDidResolve = true;
+ });
+ // Wait the second and third discoveries to fail.
+ await singleCommandDuration();
+ await singleCommandDuration();
+ await singleCommandDuration();
+ await singleCommandDuration();
+
+ assert.strictEqual(stubShowErrorMessage.callCount, 3);
+ assert.strictEqual(discoverDidResolve, false);
+ assert.strictEqual(fooTestItem.children.size, 0);
+ assert.strictEqual(barTestItem.children.size, 0);
+ assert.deepStrictEqual(actualInvocations, expectedInvocations);
+ });
+
+ it('reattempts a discovery and succeeds', async function() {
+ const TEST_URL = 'fuchsia-pkg://a_repo/test_pkg#meta/foo_test_component.cm';
+ const {actualInvocations, expectedInvocations} = mockCommands(
+ {
+ args: ['fx', 'test', '--logpath=-', '--list', TEST_URL],
+ output: new Error('{}'),
+ },
+ {
+ args: ['fx', 'test', '--logpath=-', '--list', TEST_URL],
+ output: {
+ payload: {
+ // eslint-disable-next-line @typescript-eslint/naming-convention
+ enumerate_test_cases: {
+ // eslint-disable-next-line @typescript-eslint/naming-convention
+ test_case_names: [
+ 'FooTest.testA',
+ 'FooTest.testB',
+ ]
+ }
+ }
+ },
+ },
+ );
+ const CHOICES = ['Retry'];
+ const stubShowErrorMessage = sandbox.stub(vscode.window, 'showErrorMessage')
+ .callsFake(() => new Promise(resolve => resolve(CHOICES.shift() as any)));
+
+ const queue = new TestcaseDiscoveryQueue(controller, fx);
+ const testItem = controller.createTestItem(TEST_URL, 'foo');
+
+ await queue.discover(testItem);
+
+ assert.ok(stubShowErrorMessage.calledOnce);
+ assert.strictEqual(testItem.children.size, 2);
+ assert.ok(testItem.children.get('FooTest.testA'));
+ assert.ok(testItem.children.get('FooTest.testB'));
+ assert.deepStrictEqual(actualInvocations, expectedInvocations);
+ });
+
+ it('only retries failing discoveries when mixed with successful discoveries', async function() {
+ const FOO_TEST_URL = 'fuchsia-pkg://a_repo/test_pkg#meta/foo_test_component.cm';
+ const BAR_TEST_URL = 'fuchsia-pkg://a_repo/test_pkg#meta/bar_test_component.cm';
+ const {actualInvocations, expectedInvocations} = mockCommands(
+ {
+ args: ['fx', 'test', '--logpath=-', '--list', FOO_TEST_URL],
+ output: new Error('{}'),
+ },
+ {
+ args: ['fx', 'test', '--logpath=-', '--list', BAR_TEST_URL],
+ output: {
+ payload: {
+ // eslint-disable-next-line @typescript-eslint/naming-convention
+ enumerate_test_cases: {
+ // eslint-disable-next-line @typescript-eslint/naming-convention
+ test_case_names: [
+ 'BarTest.testA',
+ ]
+ }
+ }
+ },
+ },
+ {
+ args: ['fx', 'test', '--logpath=-', '--list', FOO_TEST_URL],
+ output: {
+ payload: {
+ // eslint-disable-next-line @typescript-eslint/naming-convention
+ enumerate_test_cases: {
+ // eslint-disable-next-line @typescript-eslint/naming-convention
+ test_case_names: [
+ 'FooTest.testA',
+ ]
+ }
+ }
+ },
+ },
+ );
+ const CHOICES = ['Retry'];
+ const stubShowErrorMessage = sandbox.stub(vscode.window, 'showErrorMessage')
+ .callsFake(() => new Promise(resolve => resolve(CHOICES.shift() as any)));
+
+ const queue = new TestcaseDiscoveryQueue(controller, fx);
+ const fooTestItem = controller.createTestItem(FOO_TEST_URL, 'foo');
+ const barTestItem = controller.createTestItem(BAR_TEST_URL, 'bar');
+
+ const fooPromise = queue.discover(fooTestItem);
+ await queue.discover(barTestItem);
+ await fooPromise;
+
+ assert.ok(stubShowErrorMessage.calledOnce);
+ assert.strictEqual(fooTestItem.children.size, 1);
+ assert.ok(fooTestItem.children.get('FooTest.testA'));
+ assert.strictEqual(barTestItem.children.size, 1);
+ assert.ok(barTestItem.children.get('BarTest.testA'));
+ assert.deepStrictEqual(actualInvocations, expectedInvocations);
+ });
+
+ it('batches failing discoveries onto future discoveries when mixed with successful discoveries', async function() {
+ const FOO_TEST_URL = 'fuchsia-pkg://a_repo/test_pkg#meta/foo_test_component.cm';
+ const BAR_TEST_URL = 'fuchsia-pkg://a_repo/test_pkg#meta/bar_test_component.cm';
+ const BAZ_TEST_URL = 'fuchsia-pkg://a_repo/test_pkg#meta/baz_test_component.cm';
+ const {actualInvocations, expectedInvocations} = mockCommands(
+ {
+ args: ['fx', 'test', '--logpath=-', '--list', FOO_TEST_URL],
+ output: new Error('{}'),
+ },
+ {
+ args: ['fx', 'test', '--logpath=-', '--list', BAR_TEST_URL],
+ output: {
+ payload: {
+ // eslint-disable-next-line @typescript-eslint/naming-convention
+ enumerate_test_cases: {
+ // eslint-disable-next-line @typescript-eslint/naming-convention
+ test_case_names: [
+ 'BarTest.testA',
+ ]
+ }
+ }
+ },
+ },
+ {
+ args: ['fx', 'test', '--logpath=-', '--list', FOO_TEST_URL],
+ output: {
+ payload: {
+ // eslint-disable-next-line @typescript-eslint/naming-convention
+ enumerate_test_cases: {
+ // eslint-disable-next-line @typescript-eslint/naming-convention
+ test_case_names: [
+ 'FooTest.testA',
+ ],
+ },
+ },
+ },
+ },
+ {
+ args: ['fx', 'test', '--logpath=-', '--list', BAZ_TEST_URL],
+ output: {
+ payload: {
+ // eslint-disable-next-line @typescript-eslint/naming-convention
+ enumerate_test_cases: {
+ // eslint-disable-next-line @typescript-eslint/naming-convention
+ test_case_names: [],
+ },
+ },
+ },
+ },
+ );
+ const spyShowErrorMessage = sandbox.spy(vscode.window, 'showErrorMessage');
+
+ const queue = new TestcaseDiscoveryQueue(controller, fx);
+ const fooTestItem = controller.createTestItem(FOO_TEST_URL, 'foo');
+ const barTestItem = controller.createTestItem(BAR_TEST_URL, 'bar');
+ const bazTestItem = controller.createTestItem(BAZ_TEST_URL, 'baz');
+
+ let isFooResolved = false;
+
+ // Discover `foo` and `bar` in the same batch; `foo` should fail while `bar` passes.
+ void queue.discover(fooTestItem).then(() => isFooResolved = true);
+ await queue.discover(barTestItem);
+ assert.strictEqual(isFooResolved, false);
+ assert.strictEqual(spyShowErrorMessage.callCount, 1);
+ assert.strictEqual(barTestItem.children.size, 1);
+ assert.ok(barTestItem.children.get('BarTest.testA'));
+
+ // Now, trigger a new discovery for `baz`. This will start a new discovery batch that also
+ // includes the previously failed `foo` item from the queue, effectively retrying it.
+ await queue.discover(bazTestItem);
+ assert.strictEqual(isFooResolved, true);
+ assert.strictEqual(spyShowErrorMessage.callCount, 1);
+ assert.strictEqual(fooTestItem.children.size, 1);
+ assert.ok(fooTestItem.children.get('FooTest.testA'));
+ assert.strictEqual(bazTestItem.children.size, 0);
+ assert.deepStrictEqual(actualInvocations, expectedInvocations);
+ });
+
+ it('deduplicates 2 retry buttons rapidly clicked', async function() {
+ const FOO_TEST_URL = 'fuchsia-pkg://a_repo/test_pkg#meta/foo_test_component.cm';
+ const BAR_TEST_URL = 'fuchsia-pkg://a_repo/test_pkg#meta/bar_test_component.cm';
+ const {actualInvocations, expectedInvocations, singleCommandDuration} = mockCommands(
+ {
+ args: ['fx', 'test', '--logpath=-', '--list', FOO_TEST_URL],
+ output: new Error('{}'),
+ },
+ {
+ args: ['fx', 'test', '--logpath=-', '--list', FOO_TEST_URL],
+ output: new Error('{}'),
+ },
+ {
+ args: ['fx', 'test', '--logpath=-', '--list', BAR_TEST_URL],
+ output: new Error('{}'),
+ },
+ {
+ args: ['fx', 'test', '--logpath=-', '--list', FOO_TEST_URL],
+ output: new Error('{}'),
+ },
+ {
+ args: ['fx', 'test', '--logpath=-', '--list', BAR_TEST_URL],
+ output: new Error('{}'),
+ },
+ );
+ const pendingErrorMessageToasts: ((choice: string | undefined) => void)[] = [];
+ const stubShowErrorMessage = sandbox.stub(vscode.window, 'showErrorMessage')
+ .callsFake(() => new Promise(resolve => pendingErrorMessageToasts.push(resolve as any)));
+
+ const queue = new TestcaseDiscoveryQueue(controller, fx);
+ const fooTestItem = controller.createTestItem(FOO_TEST_URL, 'foo');
+ const barTestItem = controller.createTestItem(BAR_TEST_URL, 'bar');
+
+ const resolved = {
+ foo: false,
+ bar: false,
+ };
+
+ // Discover `bar` after `foo` sequentially, in another batch.
+ void queue.discover(fooTestItem).then(() => resolved.foo = true);
+ await singleCommandDuration();
+ void queue.discover(barTestItem).then(() => resolved.bar = true);
+ await singleCommandDuration();
+ await singleCommandDuration();
+ assert.strictEqual(stubShowErrorMessage.callCount, 2);
+
+ // Now click the 'Retry' button on both notifications simultaneously.
+ pendingErrorMessageToasts.forEach(makeSelection => makeSelection('Retry'));
+ await singleCommandDuration();
+ await singleCommandDuration();
+ await singleCommandDuration(); // Give time to check that we've properly deduped.
+ await singleCommandDuration(); // Give time to check that we've properly deduped.
+ assert.strictEqual(stubShowErrorMessage.callCount, 3);
+ assert.deepStrictEqual(resolved, {foo: false, bar: false});
+ assert.strictEqual(fooTestItem.children.size, 0);
+ assert.strictEqual(barTestItem.children.size, 0);
+ assert.deepStrictEqual(actualInvocations, expectedInvocations);
+ });
+ });
+});
diff --git a/src/test/suite/utils.ts b/src/test/suite/utils.ts
index cb67eb0..cd5c755 100644
--- a/src/test/suite/utils.ts
+++ b/src/test/suite/utils.ts
@@ -83,7 +83,7 @@
* Creates a fake child process.
* @returns the child process object.
*/
-export function createFakeChildProcess() {
+export function createFakeChildProcess(): child_process.ChildProcess {
let spawnEvent: child_process.ChildProcess = new EventEmitter() as child_process.ChildProcess;
spawnEvent.stdout = new EventEmitter() as Readable;
spawnEvent.stderr = new EventEmitter() as Readable;
@@ -124,62 +124,88 @@
}
}
-export type MockedFfxInvocation = {
+export const MOCKED_COMMAND_DURATION = 10;
+
+export type MockedCommandInvocation = {
args: string[];
output: (string | Error | Object);
};
/**
- * Mocks a sequence of expected ffx commands and their outputs.
+ * An ffx-specific version of `setupMockedCommand`.
*
- * All command executions will be captured by this mock.
- * Non-ffx commands will immediately exit with code 0 and be included in the
- * list of `actualInvocations`.
- * Mismatching ffx commands will emit a warning, exit with code 1, and be
- * included in the list of `actualInvocations.
+ * Prepends the specified `ffxPath` and ffx analytics flags onto the `args` in `ffxInvocations`.
+ *
+ * See `setupMockedCommand` for more details.
+ */
+export function setupMockFfx(
+ stubbedSpawn: StubbedSpawn,
+ cwd: string,
+ ffxPath: string,
+ ffxInvocations: MockedCommandInvocation[]
+): {
+ actualInvocations: string[][],
+ expectedInvocations: string[][],
+ singleCommandDuration: () => Promise<void>
+} {
+ const ANALYTICS_FLAG = ['--config', 'fuchsia.analytics.ffx_invoker=vscode-fuchsia'];
+ return setupMockedCommand(
+ stubbedSpawn,
+ cwd,
+ ffxInvocations.map(inv => ({
+ ...inv,
+ args: [ffxPath, ...ANALYTICS_FLAG, ...inv.args],
+ })),
+ );
+}
+
+/**
+ * Mocks a sequence of expected commands and their outputs.
+ *
+ * All command executions are handled by this mock.
+ * Non-matching command binaries will immediately exit with code 0 and are excluded in the list of
+ * `actualInvocations`.
+ * Mismatching command arguments will emit a warning, exit with code 1, and are included in the list
+ * of `actualInvocations`.
*
* Example usage:
* ```
- * const mockFfx = (...ffxInvocations: MockedFfxInvocation[]) => setupMockFfx(
+ * const mockCommands = (...commandInvocations: MockedCommandInvocation[]) => setupMockedCommand(
* stubbedSpawn,
* TEST_CWD,
- * TEST_FFX,
- * ffxInvocations,
+ * commandInvocations,
* );
*
- * const {actualInvocations, expectedInvocations} = mockFfx(
+ * const {actualInvocations, expectedInvocations} = mockCommands(
* {
- * 'args': ['--target', 'sample-device', 'target', 'reboot'],
- * 'output': 'restarted sample-device',
+ * args: ['fx', 'list-devices'],
+ * output: '127.0.0.1 fuchsia-emulator',
* },
* {
- * 'args': ['--target', 'sample-device', 'target', 'wait', '-t', '300'],
- * 'output': new Error(errorMessage),
+ * args: ['/path/to/ffx', '--target', 'sample-device', 'target', 'reboot'],
+ * output: {message: 'restarted sample-device', success: true},
* },
* {
- * 'args': ['--machine', 'json', 'target', 'list'],
- * 'output': [
- * {'nodename': 'sample-device', 'rcs_state': 'N'},
- * ],
+ * args: ['/path/to/ffx', '--target', 'sample-device', 'target', 'wait', '-t', '300'],
+ * output: new Error('Failed to establish RCS connection to device.'),
* },
* );
*
- * const ffx = new Ffx(TEST_CWD, TEST_FFX);
- * ffx.targetDevice = SAMPLE_DEVICE;
- *
+ * // This block is fake sample code.
+ * const devices = await enumerateDevices();
+ * assert.strictEqual(devices.length, 1);
* await assert.rejects(
- * () => ffx.rebootTarget(),
- * new Error(`ffx returned with non-zero exit code 1: ${errorMessage}`),
+ * () => device.rebootAndReconnect(),
+ * new Error('fuchsia-emulator failed to come back online!').
* );
*
* assert.deepStrictEqual(actualInvocations, expectedInvocations);
* ```
*
* @param stubbedSpawn: A StubbedSpawn instance.
- * @param cwd: The expected working directory that ffx commands are run in.
- * @param ffxPath: The expected ffx binary path to call.
- * @param ffxInvocations an array of objects with the following properties:
- * args: An array of ffx arguments to match against.
+ * @param cwd: The expected working directory that commands are run in.
+ * @param commandInvocations an array of objects with the following properties:
+ * args: An array of command and arguments to match against.
* output: A string or object to be serialized (for stdout), or an Error
* which message will be emitted for stderr. The former will mock exit
* code 0 whereas the latter will mock exit code 1.
@@ -187,24 +213,22 @@
* actualInvocations: An array of string arrays representing the commands
* that were run.
* expectedInvocations: An array of string arrays mapped from the input
- * `ffxInvocations` `args`.
+ * `commandInvocations` `args`.
* singleCommandDuration: A promise that can be `awaited` to wait for a
* single ffx command execution macrotask to complete.
* This is useful for getter/setter tests in which test code cannot await
* getter/setter side-effects involving ffx command execution.
*/
-export function setupMockFfx(
+export function setupMockedCommand(
stubbedSpawn: StubbedSpawn,
cwd: string,
- ffxPath: string,
- ffxInvocations: MockedFfxInvocation[]
+ commandInvocations: MockedCommandInvocation[]
): {
actualInvocations: string[][],
expectedInvocations: string[][],
singleCommandDuration: () => Promise<void>
} {
- const ANALYTICS_FLAG = ['--config', 'fuchsia.analytics.ffx_invoker=vscode-fuchsia'];
- const expectedInvocations = ffxInvocations.map(inv => [ffxPath, ...ANALYTICS_FLAG, ...inv.args]);
+ const expectedInvocations = commandInvocations.map(inv => inv.args);
const actualInvocations: string[][] = [];
let idx = 0;
stubbedSpawn.spawnStubInfo.callsFake((
@@ -212,55 +236,63 @@
args: readonly string[],
options: SpawnOptions
) => {
- // Push onto macrotask queue since we need time for ffx to add listeners
- // for `.on('data')` and `.on('close')`, which may be deferred onto the
- // microtask queue when invoked within an `async` function context.
+ const fakeSpawnedProcess = createFakeChildProcess();
+
+ // Push onto macrotask queue since we may need time for the command to add listeners for
+ // `.on('data')` and `.on('close')`, which may be deferred onto the microtask queue when invoked
+ // within an `async` function context.
//
// Add a 10ms delay to better simulate real-world execution.
//
- // For getter/setter tests in which test code cannot await getter/setter
- // side-effects involving ffx command execution you can use
- // `await mockFfxResult.singleCommandDuration()` to wait for a single ffx
- // command execution macrotask to complete.
- // Note: This alone is insufficient if the getter/setter chains a command
- // execution macrotask with other macrotasks in series.
- // In this case, test code will need to await multiple
- // `singleCommandDuration()` and/or `macrotask()` in series.
+ // For getter/setter tests in which test code cannot await getter/setter side-effects involving
+ // command execution you can use `await mockFfxResult.singleCommandDuration()` to wait for a
+ // single command execution macrotask to complete.
+ // Note: This alone is insufficient if the getter/setter chains a command execution macrotask
+ // with other macrotasks in series.
+ // In this case, test code will need to await multiple `singleCommandDuration()` and/or
+ // `macrotask()` in series.
setTimeout(() => {
- if (command !== ffxPath) {
- console.warn(`[mockFfx] Ignoring non-ffx command ${command}, ${args}, ${options}`);
- stubbedSpawn.spawnEvent.kill(0);
+ if (command !== expectedInvocations[idx]?.at(0)) {
+ console.warn(
+ `[setupMockedCommand] Ignoring non-matching command ${command}, ${args}, ${options}`
+ );
+ fakeSpawnedProcess.emit('exit', 0);
+ fakeSpawnedProcess.emit('close', 0);
return;
}
actualInvocations.push([command, ...args]);
const [expectedArgs, nextOutput] = [
expectedInvocations[idx]?.slice(1),
- ffxInvocations[idx]?.output,
+ commandInvocations[idx]?.output,
];
idx++;
if (JSON.stringify(expectedArgs) !== JSON.stringify(args) || options.cwd !== cwd) {
- const errmsg = `[mockFfx] Unexpected call to ffx.
+ const errmsg = `[setupMockedCommand] Unexpected call.
Expected invocations:\n${expectedInvocations.join('\n')}, CWD=${cwd}
Actual invocations:\n${actualInvocations.join('\n')}, CWD=${options.cwd}`;
console.error(errmsg);
- stubbedSpawn.spawnEvent.stderr?.emit('data', JSON.stringify(errmsg));
- stubbedSpawn.spawnEvent.kill(1);
+ fakeSpawnedProcess.stderr?.emit('data', JSON.stringify(errmsg));
+ fakeSpawnedProcess.emit('exit', 1);
+ fakeSpawnedProcess.emit('close', 1);
return;
}
if (nextOutput instanceof Error) {
- stubbedSpawn.spawnEvent.stderr?.emit('data', nextOutput.message);
- stubbedSpawn.spawnEvent.kill(1);
+ fakeSpawnedProcess.stderr?.emit('data', nextOutput.message);
+ fakeSpawnedProcess.emit('exit', 1);
+ fakeSpawnedProcess.emit('close', 1);
} else if (typeof nextOutput === 'string') {
- stubbedSpawn.spawnEvent.stdout?.emit('data', nextOutput);
- stubbedSpawn.spawnEvent.kill(0);
+ fakeSpawnedProcess.stdout?.emit('data', nextOutput);
+ fakeSpawnedProcess.emit('exit', 0);
+ fakeSpawnedProcess.emit('close', 0);
} else {
- stubbedSpawn.spawnEvent.stdout?.emit('data', JSON.stringify(nextOutput));
- stubbedSpawn.spawnEvent.kill(0);
+ fakeSpawnedProcess.stdout?.emit('data', JSON.stringify(nextOutput));
+ fakeSpawnedProcess.emit('exit', 0);
+ fakeSpawnedProcess.emit('close', 0);
}
- }, 10);
- return stubbedSpawn.spawnEvent;
+ }, MOCKED_COMMAND_DURATION);
+ return fakeSpawnedProcess;
});
return {
diff --git a/src/test_controller.ts b/src/test_controller.ts
index 2daf080..c46d232 100644
--- a/src/test_controller.ts
+++ b/src/test_controller.ts
@@ -12,17 +12,129 @@
import { Fx } from './fx';
/**
+ * Manages a queue of pending `TestItem` discovery requests, allowing retries and batched background
+ * processing.
+ *
+ * Why this is needed:
+ * VSCode treats `TestController.resolveHandler` `reject()` the same as `resolve()`.
+ * The `resolveHandler` also won't be reinvoked/retried against the same `TestItem` if the user
+ * collapses/re-expands the `TestItem` that failed to discover.
+ * Practically this means throwing/`reject()`-ing during a `TestItem` discovery will silently fail,
+ * resulting in an empty list of test cases with no way to recover.
+ *
+ * How `TestcaseDiscoveryQueue` helps:
+ * Rather than propagating `reject()` from `discoverTestCases()` up to
+ * `TestController.resolveHandler`, we display an error toast notification allowing users to retry
+ * discovery across all failed `TestItem` discoveries and defer `resolve()`-ing to a future
+ * `TestcaseDiscoveryQueue.discover()` attempt.
+ * Batching `TestItem` discovery requests helps avoid duplicate error toast notifications.
+ */
+export class TestcaseDiscoveryQueue {
+ private queue: [TestItem, () => void][] = [];
+
+ // If this is `true`, any discoveries pushed onto `this.queue` will be picked up by the current
+ // discovery background task.
+ private isDiscoveringFromQueue: boolean = false;
+
+ constructor(private readonly controller: vscode.TestController, private readonly fx: Fx) {
+ }
+
+ /**
+ * Enqueues a new `TestItem` to be discovered, returning a `Promise` that never rejects and only
+ * resolves when the test cases on `TestItem` are discovered.
+ *
+ * Discovery on all incomplete `TestItem`s are triggered, if needed.
+ *
+ * @param testItem The `TestItem` to discover.
+ * @returns A promise that never rejects and will be resolved eventually when the test cases on
+ * `TestItem` are discovered successfully.
+ */
+ discover(testItem: TestItem): Promise<void> {
+ return new Promise(resolve => {
+ this.queue.push([testItem, resolve]);
+ void this.discoverFromQueue();
+ });
+ }
+
+ /**
+ * Performs a concurrency-safe batched discovery attempt across all `TestItem`s in `this.queue` in
+ * the background.
+ *
+ * Any errors will result in a error toast notification, allowing `discoverFromQueue()` to be
+ * reattempted.
+ *
+ * @returns A promise that resolves when discovery is attempted for each item in `this.queue`.
+ */
+ private async discoverFromQueue(): Promise<void> {
+ if (this.isDiscoveringFromQueue) {
+ return;
+ }
+
+ // Setting `this.isDiscoveringFromQueue = true`
+ // ** synchronously after `this.queue.push()` in `discover()` **
+ // helps us no-op/avoid duplicate `discoverFromQueue()` background tasks.
+ this.isDiscoveringFromQueue = true;
+
+ // Critical section.
+ const failedDiscoveries: [TestItem, () => void][] = [];
+ for (const [testItem, discoveredCallback] of this.queue) {
+ try {
+ await discoverTestCases(this.controller, this.fx, testItem);
+ discoveredCallback();
+ } catch {
+ failedDiscoveries.push([testItem, discoveredCallback]);
+ }
+ }
+ this.queue = failedDiscoveries;
+
+ // Setting `this.isDiscoveringFromQueue = false`
+ // ** synchronously after the for-of loop completes **
+ // allows `discover()` to trigger another meaningful `discoverFromQueue()` in the background
+ // whenever the test item added by `this.queue.push()` won't be picked up by the above for-of
+ // loop.
+ this.isDiscoveringFromQueue = false;
+
+ // All of the following code must be safe to run concurrently with another `discoverFromQueue()`
+ // background task.
+ if (failedDiscoveries.length) {
+ const presentErrorToast = async () => {
+ const choice = await vscode.window.showErrorMessage(
+ 'Failed to discover test cases.',
+ 'Retry',
+ 'Show Logs',
+ 'Dismiss',
+ );
+
+ switch (choice) {
+ case 'Retry':
+ void this.discoverFromQueue();
+ break;
+ case 'Show Logs':
+ logger.show();
+ // Give the user a way to Retry testcase discovery.
+ void presentErrorToast();
+ break;
+ }
+ };
+ void presentErrorToast();
+ }
+ }
+}
+
+/**
* register the test controller and related commands
*/
export function setUpTestController(ctx: vscode.ExtensionContext, setup: Setup) {
const controller = vscode.tests.createTestController('fuchsiaTests', 'Fuchsia Tests');
ctx.subscriptions.push(controller);
+ const testcaseDiscoveryQueue = new TestcaseDiscoveryQueue(controller, setup.fx);
+
controller.resolveHandler = async (testItem?: TestItem) => {
if (!testItem) {
await discoverTests(controller);
} else {
- await discoverTestCases(controller, setup.fx, testItem);
+ await testcaseDiscoveryQueue.discover(testItem);
}
};
@@ -189,37 +301,53 @@
*/
async function discoverTestCases(controller: TestController, fx: Fx, testItem: TestItem) {
const args = ['test', '--logpath=-', '--list', testItem.id];
+ let error: boolean = false;
+ logger.info(`Running command: ${JSON.stringify(['fx', ...args])}`, 'discoverTestCases');
const process = fx.runJsonStreaming(args, (object: any) => {
- const payload = object['payload'];
- if (typeof payload !== 'object') {
+ const testCaseNames = object?.payload?.enumerate_test_cases?.test_case_names;
+ // Skip irrelevant JSON results.
+ if (!testCaseNames) {
return;
}
- const testCases = payload['enumerate_test_cases'];
- if (typeof testCases !== 'object') {
- return;
- }
- const testCaseNames = testCases['test_case_names'];
- if (typeof testCaseNames !== 'object') {
- logger.error(`Invalid test_case_names field: ${testCaseNames}`);
+ if (!Array.isArray(testCaseNames)) {
+ logger.error(
+ `Invalid test_case_names field: ${JSON.stringify(testCaseNames)}`,
+ 'discoverTestCases',
+ );
+ error = true;
return;
}
for (const testCaseName of testCaseNames) {
if (typeof testCaseName !== 'string') {
- logger.error(`Invalid test_case_names arrray entry: ${testCaseName}`);
+ logger.error(
+ `Invalid test_case_names array entry: ${JSON.stringify(testCaseName)}`,
+ 'discoverTestCases',
+ );
+ error = true;
return;
}
if (testItem.children.get(testCaseName)) {
- return;
+ continue;
}
const testCaseItem = controller.createTestItem(testCaseName, testCaseName, testItem.uri);
- if (testItem.tags.filter(tag => tag.id === 'FuchsiaTest').length > 0) {
+ if (testItem.tags.some(tag => tag.id === 'FuchsiaTest')) {
testCaseItem.tags = [new vscode.TestTag('FuchsiaTest')];
}
testItem.children.add(testCaseItem);
}
});
- await process.exitCode;
+ const exitCode = await process.exitCode;
+ if (exitCode !== 0) {
+ logger.error(`fx exited with code ${exitCode}`, 'discoverTestCases');
+ error = true;
+ }
+
+ if (error) {
+ throw new Error(
+ 'Test case discovery failed due to a non-zero exit code or JSON parsing error.'
+ );
+ }
}
/**
