src/test/suite/test_controller/discovery.test.ts

// Copyright 2025 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import * as assert from 'assert';
import * as vscode from 'vscode';
import { describe, it } from 'mocha';
import {createSandbox, SinonStub} from 'sinon';

import {Fx} from '../../../fx';
import {
  cacheTestCases,
  discoverTestCasesLogic,
  fxListTestCases,
  queryTestCaseCache,
  TestcaseDiscoveryQueue,
} from '../../../test_controller/discovery';

import {macrotask, MockedCommandInvocation, setupMockedCommand, StubbedSpawn} from '../utils';
import {Logger, initLogger} from '../../../logger';
import {Ffx} from '../../../ffx';

describe('Test Controller Discovery', function() {
  const sandbox = createSandbox();
  const TEST_CWD = '/path/to/workspace';
  let controller: vscode.TestController;
  let fx: Fx;

  this.beforeEach(function() {
    controller = vscode.tests.createTestController('TestControllerDiscovery', 'TestControllerDiscovery');
    const log = vscode.window.createOutputChannel('test_controller.test', { log: true });
    initLogger(log);
    const ffx = new Ffx(TEST_CWD);
    fx = new Fx(TEST_CWD, ffx);
  });

  this.afterEach(function() {
    controller.dispose();
    sandbox.restore();
  });

  describe('#fxListTestCases', function() {
    let stubbedSpawn: StubbedSpawn;

    this.beforeEach(function() {
      stubbedSpawn = new StubbedSpawn(sandbox);
    });

    const mockCommands = (...commandInvocations: MockedCommandInvocation[]) => setupMockedCommand(
      stubbedSpawn,
      TEST_CWD,
      commandInvocations,
    );

    it('resolves if no tests are discovered', async function() {
      const TEST_URL = 'fuchsia-pkg://a_repo/test_pkg#meta/foo_test_component.cm';
      const {actualInvocations, expectedInvocations} = mockCommands(
        {
          args: ['fx', `--invoker=Extension: ${vscode.env.appName}`, 'test', '--logpath=-', '--list', TEST_URL],
          output: {},
        },
      );

      const testItem = controller.createTestItem(TEST_URL, 'foo');

      const result = await fxListTestCases(controller, fx, testItem, false);

      assert.strictEqual(result.length, 0);
      assert.deepStrictEqual(actualInvocations, expectedInvocations);
    });

    it('populates a TestItem', async function() {
      const TEST_URL = 'fuchsia-pkg://a_repo/test_pkg#meta/bar_test_component.cm';
      const {actualInvocations, expectedInvocations} = mockCommands(
        {
          args: ['fx', `--invoker=Extension: ${vscode.env.appName}`, 'test', '--logpath=-', '--list', TEST_URL],
          output: {
            payload: {
              // eslint-disable-next-line @typescript-eslint/naming-convention
              enumerate_test_cases: {
                // eslint-disable-next-line @typescript-eslint/naming-convention
                test_case_names: [
                  'BarTest.testA',
                  'BarTest.testB',
                  'BarTest.testC',
                ]
              }
            }
          },
        },
      );

      const testItem = controller.createTestItem(TEST_URL, 'bar');

      const result = await fxListTestCases(controller, fx, testItem, false);

      assert.strictEqual(result.length, 3);
      assert.ok(result.find(item => item.id === 'BarTest.testA'));
      assert.ok(result.find(item => item.id === 'BarTest.testB'));
      assert.ok(result.find(item => item.id === 'BarTest.testC'));
      assert.deepStrictEqual(actualInvocations, expectedInvocations);
    });

    it('specifies --no-build', async function() {
      const TEST_URL = 'fuchsia-pkg://a_repo/test_pkg#meta/bar_test_component.cm';
      const {actualInvocations, expectedInvocations} = mockCommands(
        {
          args: ['fx', `--invoker=Extension: ${vscode.env.appName}`, 'test', '--logpath=-', '--list', TEST_URL, '--no-build'],
          output: {
            payload: {
              // eslint-disable-next-line @typescript-eslint/naming-convention
              enumerate_test_cases: {
                // eslint-disable-next-line @typescript-eslint/naming-convention
                test_case_names: [
                  'BarTest.testA',
                ]
              }
            }
          },
        },
      );

      const testItem = controller.createTestItem(TEST_URL, 'bar');

      const result = await fxListTestCases(controller, fx, testItem, true);

      assert.strictEqual(result.length, 1);
      assert.strictEqual(result[0].id, 'BarTest.testA');
      assert.deepStrictEqual(actualInvocations, expectedInvocations);
    });

    it('attaches the FuchsiaTest tag', async function() {
      const TEST_URL = 'fuchsia-pkg://a_repo/test_pkg#meta/bar_test_component.cm';
      const {actualInvocations, expectedInvocations} = mockCommands(
        {
          args: ['fx', `--invoker=Extension: ${vscode.env.appName}`, 'test', '--logpath=-', '--list', TEST_URL],
          output: {
            payload: {
              // eslint-disable-next-line @typescript-eslint/naming-convention
              enumerate_test_cases: {
                // eslint-disable-next-line @typescript-eslint/naming-convention
                test_case_names: [
                  'BarTest.testA',
                ]
              }
            }
          },
        },
      );

      const testItem = controller.createTestItem(TEST_URL, 'bar');
      testItem.tags = [new vscode.TestTag('FuchsiaTest')];

      const result = await fxListTestCases(controller, fx, testItem, false);

      assert.strictEqual(result.length, 1);
      assert.strictEqual(result[0].id, 'BarTest.testA');
      assert.strictEqual(result[0].tags.length, 1);
      assert.strictEqual(result[0].tags[0].id, 'FuchsiaTest');
      assert.deepStrictEqual(actualInvocations, expectedInvocations);
    });

    it('does not attach the FuchsiaTest tag', async function() {
      const TEST_URL = 'fuchsia-pkg://a_repo/test_pkg#meta/bar_test_component.cm';
      const {actualInvocations, expectedInvocations} = mockCommands(
        {
          args: ['fx', `--invoker=Extension: ${vscode.env.appName}`, 'test', '--logpath=-', '--list', TEST_URL],
          output: {
            payload: {
              // eslint-disable-next-line @typescript-eslint/naming-convention
              enumerate_test_cases: {
                // eslint-disable-next-line @typescript-eslint/naming-convention
                test_case_names: [
                  'BarTest.testA',
                ]
              }
            }
          },
        },
      );

      const testItem = controller.createTestItem(TEST_URL, 'bar');
      testItem.tags = [new vscode.TestTag('NonFuchsiaTest')];

      const result = await fxListTestCases(controller, fx, testItem, false);

      assert.strictEqual(result.length, 1);
      assert.strictEqual(result[0].id, 'BarTest.testA');
      assert.strictEqual(result[0].tags.length, 0);
      assert.deepStrictEqual(actualInvocations, expectedInvocations);
    });

    it('throws if fx test errors', async function() {
      const TEST_URL = 'fuchsia-pkg://a_repo/test_pkg#meta/foo_test_component.cm';
      const {actualInvocations, expectedInvocations} = mockCommands(
        {
          args: ['fx', `--invoker=Extension: ${vscode.env.appName}`, 'test', '--logpath=-', '--list', TEST_URL],
          output: new Error('{}'),
        },
      );

      const testItem = controller.createTestItem(TEST_URL, 'foo');

      await assert.rejects(fxListTestCases(controller, fx, testItem, false));
      assert.deepStrictEqual(actualInvocations, expectedInvocations);
    });

    it('ignores irrelevant JSON objects', async function() {
      const TEST_URL = 'fuchsia-pkg://a_repo/test_pkg#meta/foo_test_component.cm';
      const MOCK_FX_JSON_OUTPUTS = [
        undefined,
        true,
        {},
        {
          payload: -1,
        },
        {
          payload: {},
        },
        {
          payload: {
            // eslint-disable-next-line @typescript-eslint/naming-convention
            enumerate_test_cases: 'invalid',
          },
        },
        {
          payload: {
            // eslint-disable-next-line @typescript-eslint/naming-convention
            enumerate_test_cases: [],
          },
        },
        {
          payload: {
            // eslint-disable-next-line @typescript-eslint/naming-convention
            enumerate_test_cases: {
              // eslint-disable-next-line @typescript-eslint/naming-convention
              test_case_names: [],
            },
          },
        },
        {
          payload: {
            // eslint-disable-next-line @typescript-eslint/naming-convention
            enumerate_test_cases: {
              // eslint-disable-next-line @typescript-eslint/naming-convention
              test_case_names: ['FooTest.testA'],
            },
          },
        },
      ];
      const {actualInvocations, expectedInvocations} = mockCommands(
        {
          args: ['fx', `--invoker=Extension: ${vscode.env.appName}`, 'test', '--logpath=-', '--list', TEST_URL],
          output: MOCK_FX_JSON_OUTPUTS.map(obj => JSON.stringify(obj)).join('\n'),
        },
      );

      const testItem = controller.createTestItem(TEST_URL, 'foo');

      const result = await fxListTestCases(controller, fx, testItem, false);

      assert.strictEqual(result.length, 1);
      assert.ok(result.find(item => item.id === 'FooTest.testA'));
      assert.deepStrictEqual(actualInvocations, expectedInvocations);
    });

    it('deduplicates test cases', async function() {
      const TEST_URL = 'fuchsia-pkg://a_repo/test_pkg#meta/foo_test_component.cm';
      const {actualInvocations, expectedInvocations} = mockCommands(
        {
          args: ['fx', `--invoker=Extension: ${vscode.env.appName}`, 'test', '--logpath=-', '--list', TEST_URL],
          output: JSON.stringify({
            payload: {
              // eslint-disable-next-line @typescript-eslint/naming-convention
              enumerate_test_cases: {
                // eslint-disable-next-line @typescript-eslint/naming-convention
                test_case_names: ['FooTest.testA', 'FooTest.testB'],
              },
            },
          }) + '\n' + JSON.stringify({
            payload: {
              // eslint-disable-next-line @typescript-eslint/naming-convention
              enumerate_test_cases: {
                // eslint-disable-next-line @typescript-eslint/naming-convention
                test_case_names: ['FooTest.testB', 'FooTest.testC'],
              },
            },
          }),
        },
      );

      const testItem = controller.createTestItem(TEST_URL, 'foo');

      const result = await fxListTestCases(controller, fx, testItem, false);

      assert.strictEqual(result.length, 3);
      assert.ok(result.find(item => item.id === 'FooTest.testA'));
      assert.ok(result.find(item => item.id === 'FooTest.testB'));
      assert.ok(result.find(item => item.id === 'FooTest.testC'));
      assert.deepStrictEqual(actualInvocations, expectedInvocations);
    });

    it('throws if test_case_names is the wrong type', async function() {
      const TEST_URL = 'fuchsia-pkg://a_repo/test_pkg#meta/foo_test_component.cm';
      const {actualInvocations, expectedInvocations} = mockCommands(
        {
          args: ['fx', `--invoker=Extension: ${vscode.env.appName}`, 'test', '--logpath=-', '--list', TEST_URL],
          output: {
            payload: {
              // eslint-disable-next-line @typescript-eslint/naming-convention
              enumerate_test_cases: {
                // eslint-disable-next-line @typescript-eslint/naming-convention
                test_case_names: {
                  fooTest: TEST_URL
                }
              }
            }
          },
        },
      );

      const testItem = controller.createTestItem(TEST_URL, 'foo');

      await assert.rejects(fxListTestCases(controller, fx, testItem, false));
      assert.deepStrictEqual(actualInvocations, expectedInvocations);
    });

    it('throws if test_case_names entries are the wrong type', async function() {
      const TEST_URL = 'fuchsia-pkg://a_repo/test_pkg#meta/foo_test_component.cm';
      const {actualInvocations, expectedInvocations} = mockCommands(
        {
          args: ['fx', `--invoker=Extension: ${vscode.env.appName}`, 'test', '--logpath=-', '--list', TEST_URL],
          output: {
            payload: {
              // eslint-disable-next-line @typescript-eslint/naming-convention
              enumerate_test_cases: {
                // eslint-disable-next-line @typescript-eslint/naming-convention
                test_case_names: [
                  123,
                ]
              }
            }
          },
        },
      );

      const testItem = controller.createTestItem(TEST_URL, 'foo');

      await assert.rejects(fxListTestCases(controller, fx, testItem, false));
      assert.deepStrictEqual(actualInvocations, expectedInvocations);
    });
  });

  describe('TestcaseDiscoveryQueue', function() {
    describe('#discover', function() {
      it('populates independent TestItem discovery requests', async function() {
        const fooTestItem = controller.createTestItem('foo', 'foo');
        const barTestItem = controller.createTestItem('bar', 'bar');

        // eslint-disable-next-line require-await
        const discoverFn = sandbox.stub().callsFake(async (_controller, _fx, testItem) => {
          testItem.children.add(
            controller.createTestItem(`${testItem.id}.testCase`, `${testItem.id}.testCase`)
          );
        });

        const queue = TestcaseDiscoveryQueue.createForTesting(controller, fx, discoverFn);

        await queue.discover(fooTestItem);
        await queue.discover(barTestItem);

        assert.strictEqual(fooTestItem.children.size, 1);
        assert.ok(fooTestItem.children.get('foo.testCase'));
        assert.strictEqual(barTestItem.children.size, 1);
        assert.ok(barTestItem.children.get('bar.testCase'));
        assert.ok(discoverFn.calledTwice);
        assert.ok(discoverFn.firstCall.calledWith(controller, fx, fooTestItem));
        assert.ok(discoverFn.secondCall.calledWith(controller, fx, barTestItem));
      });

      it('performs TestItem discovery requests in parallel', async function() {
        const fooTestItem = controller.createTestItem('foo', 'foo');
        const barTestItem = controller.createTestItem('bar', 'bar');

        const discoverFn = sandbox.stub().callsFake(async (_controller, _fx, testItem) => {
          await new Promise(resolve => setTimeout(resolve, 50));
          testItem.children.add(
            controller.createTestItem(`${testItem.id}.testCase`, `${testItem.id}.testCase`)
          );
        });

        const queue = TestcaseDiscoveryQueue.createForTesting(controller, fx, discoverFn);

        let finishedDiscoveries = 0;
        void queue.discover(fooTestItem).then(() => finishedDiscoveries++);
        void queue.discover(barTestItem).then(() => finishedDiscoveries++);

        // Both discoveries should start in the background immediately.
        await macrotask();
        assert.ok(discoverFn.calledTwice);
        assert.strictEqual(finishedDiscoveries, 0);

        // Wait for both discoveries to finish.
        await new Promise(resolve => setTimeout(resolve, 50));
        assert.strictEqual(finishedDiscoveries, 2);

        assert.strictEqual(fooTestItem.children.size, 1);
        assert.ok(fooTestItem.children.get('foo.testCase'));
        assert.strictEqual(barTestItem.children.size, 1);
        assert.ok(barTestItem.children.get('bar.testCase'));
        assert.ok(discoverFn.firstCall.calledWith(controller, fx, fooTestItem));
        assert.ok(discoverFn.secondCall.calledWith(controller, fx, barTestItem));
      });

      it('populates varied concurrent TestItem discovery requests', async function() {
        const MOCK_DISCOVERY_DURATION = 10;
        const discoverFn = sandbox.stub().callsFake(
          (_controller, _fx, testItem: vscode.TestItem) => new Promise<void>(resolve => {
            setTimeout(() => {
              const child = controller.createTestItem(
                `${testItem.id}.testCase`,
                `${testItem.id}.testCase`,
              );
              testItem.children.add(child);
              resolve();
            }, MOCK_DISCOVERY_DURATION);
          })
        );

        const queue = TestcaseDiscoveryQueue.createForTesting(controller, fx, discoverFn);
        const initialTestItem = controller.createTestItem('initial', 'initial');
        const synchronousTestItem = controller.createTestItem('synchronous', 'synchronous');
        const microtaskTestItem = controller.createTestItem('microtask', 'microtask');
        const macrotaskTestItem = controller.createTestItem('macrotask', 'macrotask');
        const timedTestItem = controller.createTestItem('timed', 'timed');

        const discovered = {
          initial: false,
          synchronous: false,
          microtask: false,
          macrotask: false,
          timed: false,
        };

        // Test synchronous concurrent discovery requests.
        const initialPromise = queue.discover(initialTestItem)
          .then(() => discovered.initial = true);
        const synchronousPromise = queue.discover(synchronousTestItem)
          .then(() => discovered.synchronous = true);

        // Wait for the `initial` discovery to complete; keep the `synchronous` one in the queue.
        await initialPromise;
        assert.deepStrictEqual(
          discovered,
          {initial: true, synchronous: false, microtask: false, macrotask: false, timed: false},
        );
        assert.strictEqual(initialTestItem.children.size, 1);
        assert.ok(initialTestItem.children.get('initial.testCase'));

        // Test concurrent discovery request after 1 microtask.
        await Promise.resolve();
        const microtaskPromise = queue.discover(microtaskTestItem)
          .then(() => discovered.microtask = true);

        // Wait for the `synchronous` discovery to complete; keep the `microtask` one in the queue.
        await synchronousPromise;
        assert.deepStrictEqual(
          discovered,
          {initial: true, synchronous: true, microtask: false, macrotask: false, timed: false},
        );
        assert.strictEqual(synchronousTestItem.children.size, 1);
        assert.ok(synchronousTestItem.children.get('synchronous.testCase'));

        // Test concurrent discovery request after 1 macrotask.
        await macrotask();
        const macrotaskPromise = queue.discover(macrotaskTestItem)
          .then(() => discovered.macrotask = true);

        // Wait for the `microtask` discovery to complete; keep the `macrotask` one in the queue.
        await microtaskPromise;
        assert.deepStrictEqual(
          discovered,
          {initial: true, synchronous: true, microtask: true, macrotask: false, timed: false},
        );
        assert.strictEqual(microtaskTestItem.children.size, 1);
        assert.ok(microtaskTestItem.children.get('microtask.testCase'));

        // Test concurrent discovery request halfway through command execution.
        await new Promise(resolve => setTimeout(resolve, MOCK_DISCOVERY_DURATION / 2));
        const timedPromise = queue.discover(timedTestItem).then(() => discovered.timed = true);

        // Wait for the `macrotask` discovery to complete; keep the `timed` one in the queue.
        await macrotaskPromise;
        assert.deepStrictEqual(
          discovered,
          {initial: true, synchronous: true, microtask: true, macrotask: true, timed: false},
        );
        assert.strictEqual(macrotaskTestItem.children.size, 1);
        assert.ok(macrotaskTestItem.children.get('macrotask.testCase'));

        // Wait for the `timed` discovery to complete; the discovery queue should be empty now.
        await timedPromise;
        assert.deepStrictEqual(
          discovered,
          {initial: true, synchronous: true, microtask: true, macrotask: true, timed: true},
        );
        assert.strictEqual(timedTestItem.children.size, 1);
        assert.ok(timedTestItem.children.get('timed.testCase'));

        // Check whether `TestcaseDiscoveryQueue` makes any extra unnecessary `fx` invocations.
        await new Promise(resolve => setTimeout(resolve, MOCK_DISCOVERY_DURATION * 2));
        assert.strictEqual(discoverFn.callCount, 5);
      });

      it('issues a notification without resolving if discovery fails', async function() {
        const discoverFn = sandbox.stub().rejects(new Error('Discovery failed'));
        const stubShowErrorMessage = sandbox.stub(vscode.window, 'showErrorMessage')
          .resolves('Dismiss' as any);

        const queue = TestcaseDiscoveryQueue.createForTesting(controller, fx, discoverFn);
        const testItem = controller.createTestItem('id', 'label');

        let discoverDidResolve = false;
        void queue.discover(testItem).then(() => {
          discoverDidResolve = true;
        });

        // Give time for the promise to reject and the error message to be shown.
        await macrotask();

        assert.ok(discoverFn.calledOnce);
        assert.ok(stubShowErrorMessage.calledOnce);
        assert.strictEqual(discoverDidResolve, false);
        assert.strictEqual(testItem.children.size, 0);
      });

      it('shows error logs if the user clicks show logs', async function() {
        const CHOICES = ['Show Logs'];
        const stubShowErrorMessage = sandbox.stub(vscode.window, 'showErrorMessage')
          .callsFake(() => new Promise(resolve => resolve(CHOICES.shift() as any)));
        const spyLoggerShow = sandbox.spy(Logger.prototype, 'show');

        const discoverFn = sandbox.stub().rejects(new Error('Discovery failed'));
        const queue = TestcaseDiscoveryQueue.createForTesting(controller, fx, discoverFn);
        const testItem = controller.createTestItem('id', 'label');

        void queue.discover(testItem);

        // Give time for the promise to reject and the error message to be shown.
        await macrotask();

        assert.strictEqual(stubShowErrorMessage.callCount, 2);
        assert.ok(spyLoggerShow.calledOnce);
        assert.strictEqual(testItem.children.size, 0);
      });

      it('reattempts a discovery and fails again', async function() {
        const CHOICES = ['Retry'];
        const stubShowErrorMessage = sandbox.stub(vscode.window, 'showErrorMessage')
          .callsFake(() => new Promise(resolve => resolve(CHOICES.shift() as any)));

        const discoverFn = sandbox.stub().rejects(new Error('Discovery failed'));
        const queue = TestcaseDiscoveryQueue.createForTesting(controller, fx, discoverFn);
        const testItem = controller.createTestItem('id', 'label');

        let discoverDidResolve = false;
        void queue.discover(testItem).then(() => {
          discoverDidResolve = true;
        });

        // Wait for both discovery failures to complete.
        await macrotask();

        assert.strictEqual(stubShowErrorMessage.callCount, 2);
        assert.strictEqual(discoverDidResolve, false);
        assert.strictEqual(testItem.children.size, 0);
        assert.strictEqual(discoverFn.callCount, 2);
      });

      it('issues one notification when multiple TestItems fail in a batch', async function() {
        const spyShowErrorMessage = sandbox.spy(vscode.window, 'showErrorMessage');

        const discoverFn = sandbox.stub().rejects(new Error('Discovery failed'));
        const queue = TestcaseDiscoveryQueue.createForTesting(controller, fx, discoverFn);
        const fooTestItem = controller.createTestItem('foo', 'foo');
        const barTestItem = controller.createTestItem('bar', 'bar');

        let discoverDidResolve = false;
        void queue.discover(fooTestItem).then(() => {
          discoverDidResolve = true;
        });
        void queue.discover(barTestItem).then(() => {
          discoverDidResolve = true;
        });

        // Wait for both discoveries (run sequentially) to complete and propagate up to
        // `queue.discover()`.
        await macrotask();

        assert.ok(spyShowErrorMessage.calledOnce);
        assert.strictEqual(discoverDidResolve, false);
        assert.strictEqual(fooTestItem.children.size, 0);
        assert.strictEqual(barTestItem.children.size, 0);
        assert.strictEqual(discoverFn.callCount, 2);
      });

      it('batches previous failures onto future discoveries', async function() {
        const CHOICES = [
          undefined, // User clicks the (x) button on the notification.
          'Retry',
          'Dismiss',
        ];
        const stubShowErrorMessage = sandbox.stub(vscode.window, 'showErrorMessage')
          .callsFake(() => new Promise(resolve => resolve(CHOICES.shift() as any)));

        const discoverFn = sandbox.stub().rejects(new Error('Discovery failed'));
        const queue = TestcaseDiscoveryQueue.createForTesting(controller, fx, discoverFn);
        const fooTestItem = controller.createTestItem('foo', 'foo');
        const barTestItem = controller.createTestItem('bar', 'bar');

        let fooDidResolve = false;
        void queue.discover(fooTestItem).then(() => fooDidResolve = true);

        // Wait for the first discovery (foo) to fail.
        // This shows an error message notification, which the user clicks (x) out of.
        // `fooTestItem` remains in the queue as a failed item.
        await macrotask();
        assert.strictEqual(stubShowErrorMessage.callCount, 1);
        assert.strictEqual(discoverFn.callCount, 1);

        let barDidResolve = false;
        void queue.discover(barTestItem).then(() => barDidResolve = true);

        // Wait for the second discovery batch (containing the failed `foo` and new `bar`) and the
        // third discovery batches (foo and bar again) to fail.
        // The User clicks 'Retry' on the second discovery batch's error message, and 'Dismiss' on
        // the third batch's error message.
        await macrotask();
        assert.strictEqual(stubShowErrorMessage.callCount, 3);
        assert.strictEqual(discoverFn.callCount, 5);

        // Final state check
        assert.strictEqual(fooDidResolve, false);
        assert.strictEqual(barDidResolve, false);
        assert.strictEqual(fooTestItem.children.size, 0);
        assert.strictEqual(barTestItem.children.size, 0);
      });

      it('reattempts a discovery and succeeds', async function() {
        const CHOICES = ['Retry'];
        const stubShowErrorMessage = sandbox.stub(vscode.window, 'showErrorMessage')
          .callsFake(() => new Promise(resolve => resolve(CHOICES.shift() as any)));

        const discoverFn = sandbox.stub()
          .onFirstCall().rejects(new Error('Discovery failed'))
          // eslint-disable-next-line require-await
          .onSecondCall().callsFake(async (_controller, _fx, testItem: vscode.TestItem) => {
            const child = controller.createTestItem('TestCase.testA', 'TestCase.testA');
            testItem.children.add(child);
          });
        const queue = TestcaseDiscoveryQueue.createForTesting(controller, fx, discoverFn);
        const testItem = controller.createTestItem('id', 'label');

        await queue.discover(testItem);

        assert.ok(stubShowErrorMessage.calledOnce);
        assert.strictEqual(discoverFn.callCount, 2);
        assert.strictEqual(testItem.children.size, 1);
        assert.ok(testItem.children.get('TestCase.testA'));
      });

      it('only retries failing discoveries when mixed with successful discoveries', async function() {
        const CHOICES = ['Retry'];
        const stubShowErrorMessage = sandbox.stub(vscode.window, 'showErrorMessage')
          .callsFake(() => new Promise(resolve => resolve(CHOICES.shift() as any)));

        const fooTestItem = controller.createTestItem('foo', 'foo');
        const barTestItem = controller.createTestItem('bar', 'bar');

        const discoverFn = sandbox.stub()
          .onFirstCall().rejects(new Error('Discovery failed'))
          // eslint-disable-next-line require-await
          .callsFake(async (_controller, _fx, testItem: vscode.TestItem) => {
            const child = controller.createTestItem(testItem.id, testItem.id);
            testItem.children.add(child);
          });

        const queue = TestcaseDiscoveryQueue.createForTesting(controller, fx, discoverFn);

        const fooPromise = queue.discover(fooTestItem);
        await queue.discover(barTestItem);
        await fooPromise;

        assert.ok(stubShowErrorMessage.calledOnce);
        assert.strictEqual(discoverFn.callCount, 3);

        // Verify call order
        assert.ok(discoverFn.getCall(0).calledWith(controller, fx, fooTestItem));
        assert.ok(discoverFn.getCall(1).calledWith(controller, fx, barTestItem));
        assert.ok(discoverFn.getCall(2).calledWith(controller, fx, fooTestItem));

        // Verify test items are populated
        assert.strictEqual(fooTestItem.children.size, 1);
        assert.ok(fooTestItem.children.get('foo'));
        assert.strictEqual(barTestItem.children.size, 1);
        assert.ok(barTestItem.children.get('bar'));
      });

      it('batches failing discoveries onto future discoveries when mixed with successful discoveries', async function() {
        const spyShowErrorMessage = sandbox.spy(vscode.window, 'showErrorMessage');

        const fooTestItem = controller.createTestItem('foo', 'foo');
        const barTestItem = controller.createTestItem('bar', 'bar');
        const bazTestItem = controller.createTestItem('baz', 'baz');

        const discoverFn = sandbox.stub()
          .onFirstCall().rejects(new Error('Discovery failed'))
          // eslint-disable-next-line require-await
          .callsFake(async (_controller, _fx, testItem: vscode.TestItem) => {
            const child = controller.createTestItem(testItem.id, testItem.id);
            testItem.children.add(child);
          });

        const queue = TestcaseDiscoveryQueue.createForTesting(controller, fx, discoverFn);

        let isFooResolved = false;

        // Discover `foo` and `bar` in the same batch; `foo` should fail while `bar` passes.
        void queue.discover(fooTestItem).then(() => isFooResolved = true);
        await queue.discover(barTestItem);
        assert.strictEqual(isFooResolved, false);
        assert.strictEqual(spyShowErrorMessage.callCount, 1);
        assert.strictEqual(barTestItem.children.size, 1);
        assert.ok(barTestItem.children.get('bar'));
        assert.strictEqual(discoverFn.callCount, 2);

        // Now, trigger a new discovery for `baz`. This will start a new discovery batch that also
        // includes the previously failed `foo` item from the queue, effectively retrying it.
        await queue.discover(bazTestItem);
        assert.strictEqual(isFooResolved, true);
        assert.strictEqual(spyShowErrorMessage.callCount, 1);
        assert.strictEqual(fooTestItem.children.size, 1);
        assert.ok(fooTestItem.children.get('foo'));
        assert.strictEqual(bazTestItem.children.size, 1);
        assert.ok(bazTestItem.children.get('baz'));
        assert.strictEqual(discoverFn.callCount, 4);

        // Verify call order
        assert.ok(discoverFn.getCall(0).calledWith(controller, fx, fooTestItem));
        assert.ok(discoverFn.getCall(1).calledWith(controller, fx, barTestItem));
        assert.ok(discoverFn.getCall(2).calledWith(controller, fx, fooTestItem));
        assert.ok(discoverFn.getCall(3).calledWith(controller, fx, bazTestItem));
      });

      it('deduplicates 2 retry buttons rapidly clicked', async function() {
        const pendingErrorMessageToasts: ((choice: string | undefined) => void)[] = [];
        const stubShowErrorMessage = sandbox.stub(vscode.window, 'showErrorMessage')
          .callsFake(() => new Promise(resolve => pendingErrorMessageToasts.push(resolve as any)));

        const discoverFn = sandbox.stub().rejects(new Error('Discovery failed'));
        const queue = TestcaseDiscoveryQueue.createForTesting(controller, fx, discoverFn);
        const fooTestItem = controller.createTestItem('foo', 'foo');
        const barTestItem = controller.createTestItem('bar', 'bar');

        const resolved = {
          foo: false,
          bar: false,
        };

        // Batch 1: Discover `foo`, which fails.
        void queue.discover(fooTestItem).then(() => resolved.foo = true);
        await macrotask();
        assert.strictEqual(discoverFn.callCount, 1);
        assert.strictEqual(stubShowErrorMessage.callCount, 1);

        // Batch 2: Discover `bar`, which also retries the failed `foo`. Both fail.
        void queue.discover(barTestItem).then(() => resolved.bar = true);
        await macrotask();
        assert.strictEqual(discoverFn.callCount, 3); // foo (initial) + foo (retry) + bar
        assert.strictEqual(stubShowErrorMessage.callCount, 2);

        // User clicks 'Retry' on both notifications simultaneously.
        pendingErrorMessageToasts.forEach(makeSelection => makeSelection('Retry'));

        // Batch 3: A single, deduplicated retry batch for `foo` and `bar` runs and fails.
        await macrotask();
        assert.strictEqual(discoverFn.callCount, 5); // + foo (retry) + bar (retry)
        assert.strictEqual(stubShowErrorMessage.callCount, 3);

        // Final state check
        assert.deepStrictEqual(resolved, {foo: false, bar: false});
        assert.strictEqual(fooTestItem.children.size, 0);
        assert.strictEqual(barTestItem.children.size, 0);
      });
    });
  });

  describe('Test Case Caching', function() {
    let memento: vscode.Memento;

    this.beforeEach(function() {
      const storage = new Map<string, any>();
      memento = {
        keys: () => [...storage.keys()],
        get: <T>(key: string, defaultValue?: T): T | undefined =>
          storage.has(key) ? storage.get(key) : defaultValue,
        // eslint-disable-next-line require-await
        update: async (key: string, value: any): Promise<void> => void storage.set(key, value),
      };
    });

    it('saves and restores TestItems from the cache', async function() {
      const TEST_ID = 'fuchsia-pkg://a_repo/test_pkg#meta/foo_test_component.cm';
      const TEST_URI = vscode.Uri.parse('file:///path/to/workspace/foo_test.cc');
      const testCases: vscode.TestItem[] = [
        controller.createTestItem('FooTest.testA', 'FooTest.testA', TEST_URI),
        controller.createTestItem('FooTest.testB', 'FooTest.testB', TEST_URI),
      ];
      testCases[0].tags = [new vscode.TestTag('FuchsiaTest')];

      await cacheTestCases(memento, TEST_ID, testCases);
      const restoredTestCases = queryTestCaseCache(memento, controller, TEST_ID);

      assert.strictEqual(restoredTestCases?.length, 2);

      const restoredTestA = restoredTestCases?.find(item => item.id === 'FooTest.testA');
      assert.ok(restoredTestA);
      assert.strictEqual(restoredTestA.label, 'FooTest.testA');
      assert.strictEqual(restoredTestA.uri?.toString(), TEST_URI.toString());
      assert.strictEqual(restoredTestA.tags.length, 1);
      assert.strictEqual(restoredTestA.tags[0].id, 'FuchsiaTest');

      const restoredTestB = restoredTestCases?.find(item => item.id === 'FooTest.testB');
      assert.ok(restoredTestB);
      assert.strictEqual(restoredTestB.label, 'FooTest.testB');
      assert.strictEqual(restoredTestB.uri?.toString(), TEST_URI.toString());
      assert.strictEqual(restoredTestB.tags.length, 0);
    });

    it('returns undefined for a cache miss', function() {
      const restoredTestCases = queryTestCaseCache(memento, controller, 'non-existent-id');
      assert.deepStrictEqual(restoredTestCases, undefined);
    });

    it('handles test items without URIs or tags', async function() {
      const TEST_ID = 'test-id-no-uri';
      const testCases: vscode.TestItem[] = [
        controller.createTestItem('NoUri.test', 'NoUri.test'),
      ];

      await cacheTestCases(memento, TEST_ID, testCases);
      const restoredTestCases = queryTestCaseCache(memento, controller, TEST_ID);

      assert.strictEqual(restoredTestCases?.length, 1);
      const restoredTest = restoredTestCases[0];
      assert.strictEqual(restoredTest.id, 'NoUri.test');
      assert.strictEqual(restoredTest.label, 'NoUri.test');
      assert.strictEqual(restoredTest.uri, undefined);
      assert.strictEqual(restoredTest.tags.length, 0);
    });
  });

  describe('#discoverTestCasesLogic', function() {
    let queryCacheStub: SinonStub;
    let listTestCasesNoBuildStub: SinonStub;
    let listTestCasesWithBuildStub: SinonStub;
    let updateChildrenStub: SinonStub;
    let cacheResultStub: SinonStub;

    this.beforeEach(function() {
      queryCacheStub = sandbox.stub();
      listTestCasesNoBuildStub = sandbox.stub();
      listTestCasesWithBuildStub = sandbox.stub();
      updateChildrenStub = sandbox.stub();
      cacheResultStub = sandbox.stub();
    });

    it('should populate children from cache first, then no-build, then with-build', async function() {
      const cachedChildren = [controller.createTestItem('cached', 'Testcase (cached)')];
      const noBuildChildren = [controller.createTestItem('no-build', 'Testcase (no-build)')];
      const withBuildChildren = [controller.createTestItem('with-build', 'Testcase (with-build)')];

      queryCacheStub.returns(cachedChildren);
      listTestCasesNoBuildStub.resolves(noBuildChildren);
      listTestCasesWithBuildStub.resolves(withBuildChildren);
      cacheResultStub.resolves();

      await discoverTestCasesLogic(
        queryCacheStub,
        listTestCasesNoBuildStub,
        listTestCasesWithBuildStub,
        updateChildrenStub,
        cacheResultStub
      );

      // Verify that the cache query, cached children update, no-build discovery, and with-build
      // discovery are all stated synchronously.
      assert.ok(queryCacheStub.calledOnce);
      assert.ok(updateChildrenStub.calledWith(cachedChildren));
      assert.ok(listTestCasesNoBuildStub.calledOnce);
      assert.ok(listTestCasesWithBuildStub.calledOnce);

      // Wait for both outstanding listTestCases discovery promises to resolve.
      await macrotask();

      // Verify children are populated in the correct order.
      assert.strictEqual(updateChildrenStub.callCount, 3);
      assert.ok(updateChildrenStub.getCall(0).calledWith(cachedChildren));
      assert.ok(updateChildrenStub.getCall(1).calledWith(noBuildChildren));
      assert.ok(updateChildrenStub.getCall(2).calledWith(withBuildChildren));

      // Verify cache is updated with the no-build results, then with the with-build results.
      assert.strictEqual(cacheResultStub.callCount, 2);
      assert.ok(cacheResultStub.getCall(0).calledWith(noBuildChildren));
      assert.ok(cacheResultStub.getCall(1).calledWith(withBuildChildren));
    });

    it('should not block on no-build results', async function() {
      const cachedChildren = [controller.createTestItem('cached', 'Testcase (cached)')];
      const withBuildChildren = [controller.createTestItem('with-build', 'Testcase (with-build)')];

      queryCacheStub.returns(cachedChildren);
      listTestCasesNoBuildStub.returns(new Promise(() => {})); // Never finishes.
      listTestCasesWithBuildStub.resolves(withBuildChildren); // Finishes quickly.
      cacheResultStub.resolves();

      await discoverTestCasesLogic(
        queryCacheStub,
        listTestCasesNoBuildStub,
        listTestCasesWithBuildStub,
        updateChildrenStub,
        cacheResultStub
      );

      // Verify that the cache query, cached children update, no-build discovery, and with-build
      // discovery are all stated synchronously.
      assert.ok(queryCacheStub.calledOnce);
      assert.ok(updateChildrenStub.calledWith(cachedChildren));
      assert.ok(listTestCasesNoBuildStub.calledOnce);
      assert.ok(listTestCasesWithBuildStub.calledOnce);

      // Wait for withBuildChildren to resolve.
      await macrotask();

      // Verify children are populated in the correct order.
      assert.strictEqual(updateChildrenStub.callCount, 2);
      assert.ok(updateChildrenStub.getCall(0).calledWith(cachedChildren));
      assert.ok(updateChildrenStub.getCall(1).calledWith(withBuildChildren));

      // Verify cache is updated only with the with-build results.
      assert.strictEqual(cacheResultStub.callCount, 1);
      assert.ok(cacheResultStub.calledWith(withBuildChildren));
    });

    it('should reject and update with no-build results if with-build fails', async function() {
      const cachedChildren = [controller.createTestItem('cached', 'Testcase (cached)')];
      const noBuildChildren = [controller.createTestItem('no-build', 'Testcase (no-build)')];

      queryCacheStub.returns(cachedChildren);
      listTestCasesNoBuildStub.resolves(noBuildChildren);
      listTestCasesWithBuildStub.rejects(new Error('Build failed'));
      cacheResultStub.resolves();

      // discoverTestCasesLogic should reject, which helps TestcaseDiscoveryQueue present an error
      // notification.
      await assert.rejects(discoverTestCasesLogic(
        queryCacheStub,
        listTestCasesNoBuildStub,
        listTestCasesWithBuildStub,
        updateChildrenStub,
        cacheResultStub
      ));


      // Verify that the cache query, cached children update, no-build discovery, and with-build
      // discovery are all stated synchronously.
      assert.ok(queryCacheStub.calledOnce);
      assert.ok(updateChildrenStub.calledWith(cachedChildren));
      assert.ok(listTestCasesNoBuildStub.calledOnce);
      assert.ok(listTestCasesWithBuildStub.calledOnce);

      // Wait for noBuildChildren to resolve.
      await macrotask();

      // Verify children are populated in the correct order.
      assert.strictEqual(updateChildrenStub.callCount, 2);
      assert.ok(updateChildrenStub.getCall(0).calledWith(cachedChildren));
      assert.ok(updateChildrenStub.getCall(1).calledWith(noBuildChildren));

      // Verify cache is updated only with the no-build results.
      assert.strictEqual(cacheResultStub.callCount, 1);
      assert.ok(cacheResultStub.calledWith(noBuildChildren));
    });

    it('should not update with no-build results if with-build has already updated', async function() {
      const cachedChildren = [controller.createTestItem('cached', 'Testcase (cached)')];
      const noBuildChildren = [controller.createTestItem('no-build', 'Testcase (no-build)')];
      const withBuildChildren = [controller.createTestItem('with-build', 'Testcase (with-build)')];

      queryCacheStub.returns(cachedChildren);
      listTestCasesNoBuildStub.returns(new Promise(
        resolve => setTimeout(() => resolve(noBuildChildren), 50)
      )); // Finishes slower.
      listTestCasesWithBuildStub.resolves(withBuildChildren); // Finishes quicker.
      cacheResultStub.resolves();

      await discoverTestCasesLogic(
        queryCacheStub,
        listTestCasesNoBuildStub,
        listTestCasesWithBuildStub,
        updateChildrenStub,
        cacheResultStub
      );

      // Verify that the cache query, cached children update, no-build discovery, and with-build
      // discovery are all stated synchronously.
      assert.ok(queryCacheStub.calledOnce);
      assert.ok(updateChildrenStub.calledWith(cachedChildren));
      assert.ok(listTestCasesNoBuildStub.calledOnce);
      assert.ok(listTestCasesWithBuildStub.calledOnce);

      // Wait for both promises to resolve.
      await macrotask();
      await new Promise(resolve => setTimeout(resolve, 50));

      // Verify `noBuildChildren` is not populated into the list of children.
      assert.strictEqual(updateChildrenStub.callCount, 2);
      assert.ok(updateChildrenStub.getCall(0).calledWith(cachedChildren));
      assert.ok(updateChildrenStub.getCall(1).calledWith(withBuildChildren));

      // Verify cache is updated only with the with-build results.
      assert.strictEqual(cacheResultStub.callCount, 1);
      assert.ok(cacheResultStub.calledWith(withBuildChildren));
    });

    it('should populate from no-build, then with-build when cache is unavailable', async function() {
      const noBuildChildren = [controller.createTestItem('no-build', 'Testcase (no-build)')];
      const withBuildChildren = [controller.createTestItem('with-build', 'Testcase (with-build)')];

      queryCacheStub.returns(undefined); // Cache is unavailable
      listTestCasesNoBuildStub.resolves(noBuildChildren);
      listTestCasesWithBuildStub.resolves(withBuildChildren);
      cacheResultStub.resolves();

      await discoverTestCasesLogic(
        queryCacheStub,
        listTestCasesNoBuildStub,
        listTestCasesWithBuildStub,
        updateChildrenStub,
        cacheResultStub
      );


      // Verify that the cache query, and both discoveries are all stated synchronously.
      assert.ok(queryCacheStub.calledOnce);
      assert.ok(listTestCasesNoBuildStub.calledOnce);
      assert.ok(listTestCasesWithBuildStub.calledOnce);

      // Wait for both promises to resolve.
      await macrotask();

      // Verify children are populated in the correct order.
      assert.strictEqual(updateChildrenStub.callCount, 2);
      assert.ok(updateChildrenStub.getCall(0).calledWith(noBuildChildren));
      assert.ok(updateChildrenStub.getCall(1).calledWith(withBuildChildren));

      // Verify cache is updated with the no-build results, then with the with-build results.
      assert.strictEqual(cacheResultStub.callCount, 2);
      assert.ok(cacheResultStub.getCall(0).calledWith(noBuildChildren));
      assert.ok(cacheResultStub.getCall(1).calledWith(withBuildChildren));
    });

    it('should handle when no test cases are available from any source', async function() {
      queryCacheStub.returns(undefined);
      listTestCasesNoBuildStub.rejects(new Error('failed to connect to target device'));
      listTestCasesWithBuildStub.rejects(new Error('failed to connect to target device'));
      cacheResultStub.resolves();

      await assert.rejects(discoverTestCasesLogic(
        queryCacheStub,
        listTestCasesNoBuildStub,
        listTestCasesWithBuildStub,
        updateChildrenStub,
        cacheResultStub
      ));

      // Verify that the cache query, no-build discovery, and with-build discovery are all called.
      assert.ok(queryCacheStub.calledOnce);
      assert.ok(listTestCasesNoBuildStub.calledOnce);
      assert.ok(listTestCasesWithBuildStub.calledOnce);

      // Wait for promises to resolve.
      await macrotask();

      // Verify children aren't updated.
      assert.strictEqual(updateChildrenStub.callCount, 0);

      // Verify cache isn't updated.
      assert.strictEqual(cacheResultStub.callCount, 0);
    });

    it('resolves when only built testcase results are available', async function() {
      const withBuildChildren = [controller.createTestItem('with-build', 'Testcase (with-build)')];
      queryCacheStub.returns(undefined);
      listTestCasesNoBuildStub.rejects(new Error('test not built yet'));
      listTestCasesWithBuildStub.resolves(withBuildChildren);
      cacheResultStub.resolves();

      await discoverTestCasesLogic(
        queryCacheStub,
        listTestCasesNoBuildStub,
        listTestCasesWithBuildStub,
        updateChildrenStub,
        cacheResultStub
      );

      // Verify that the cache query, no-build discovery, and with-build discovery are all called.
      assert.ok(queryCacheStub.calledOnce);
      assert.ok(listTestCasesNoBuildStub.calledOnce);
      assert.ok(listTestCasesWithBuildStub.calledOnce);

      // Wait for promises to resolve.
      await macrotask();

      // Verify children are only populated with the with-build results.
      assert.strictEqual(updateChildrenStub.callCount, 1);
      assert.ok(updateChildrenStub.calledWith(withBuildChildren));

      // Verify cache is only updated with the with-build results.
      assert.strictEqual(cacheResultStub.callCount, 1);
      assert.ok(cacheResultStub.calledWith(withBuildChildren));
    });
  });
});