// Copyright 2019 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

use assert_matches::assert_matches;
use diagnostics_data::Severity;
use fidl_fuchsia_test_manager::{LaunchError, RunBuilderMarker};
use regex::Regex;
use run_test_suite_lib::{output, Outcome, RunTestSuiteError, TestParams};
use std::convert::TryInto;
use std::ops::Deref;
use std::str::from_utf8;
use std::sync::Arc;
use test_list::TestTag;
use test_output_directory::{
    self as directory,
    testing::{ExpectedDirectory, ExpectedSuite, ExpectedTestCase, ExpectedTestRun},
};
use test_util::assert_geq;

/// split and sort output as output can come in any order.
/// `output` is of type vec<u8> and `expected_output` is a string.
macro_rules! assert_output {
    ($output:expr, $expected_output:expr) => {
        let mut expected_output = $expected_output.split("\n").collect::<Vec<_>>();

        // no need to check for lines starting with "[stdout - " and "[stderr - ". We just want to
        // make sure that we are printing important details.
        let mut output = from_utf8(&$output)
            .expect("we should not get utf8 error.")
            .split("\n")
            .filter(|x| {
                let is_stdout = x.starts_with("[stdout - ");
                let is_stderr = x.starts_with("[stderr - ");
                let is_debug_data_warn = is_debug_data_warning(x);
                !(is_stdout || is_stderr || is_debug_data_warn)
            })
            .map(sanitize_log_for_comparison)
            .collect::<Vec<_>>();

        expected_output.sort();
        output.sort();

        assert_eq!(output, expected_output);
    };
}

// TODO(fxbug.dev/61180): once debug data routing is fixed, this log should be gone and this
// function can be deleted.
fn is_debug_data_warning(log: impl AsRef<str>) -> bool {
    log.as_ref().contains("No capability available at path /svc/fuchsia.debugdata.Publisher")
}

fn sanitize_log_for_comparison(log: impl AsRef<str>) -> String {
    let log_timestamp_re = Regex::new(r"^\[\d+.\d+\]\[\d+\]\[\d+\]").unwrap();
    log_timestamp_re.replace_all(log.as_ref(), "[TIMESTAMP][PID][TID]").to_string()
}

fn new_test_params(test_url: &str) -> TestParams {
    TestParams {
        test_url: test_url.to_string(),
        timeout_seconds: None,
        test_filters: None,
        also_run_disabled_tests: false,
        parallel: None,
        test_args: vec![],
        max_severity_logs: None,
        tags: vec![TestTag { key: "internal".to_string(), value: "true".to_string() }],
    }
}

fn new_run_params() -> run_test_suite_lib::RunParams {
    run_test_suite_lib::RunParams {
        timeout_behavior: run_test_suite_lib::TimeoutBehavior::TerminateRemaining,
        stop_after_failures: None,
    }
}

type TestShellReporter = run_test_suite_lib::output::ShellReporter<Vec<u8>>;
type TestMuxReporter = run_test_suite_lib::output::MultiplexedReporter<
    TestShellReporter,
    run_test_suite_lib::output::DirectoryReporter,
>;
type TestMuxMuxReporter = run_test_suite_lib::output::MultiplexedReporter<
    TestMuxReporter,
    run_test_suite_lib::output::ShellReporter<std::io::BufWriter<std::fs::File>>,
>;
type TestOutputView = run_test_suite_lib::output::ShellWriterView<Vec<u8>>;

fn create_shell_reporter() -> (TestShellReporter, TestOutputView) {
    run_test_suite_lib::output::ShellReporter::new_expose_writer_for_test()
}

fn create_dir_reporter() -> (run_test_suite_lib::output::DirectoryReporter, tempfile::TempDir) {
    let tmp_dir = tempfile::tempdir().unwrap();
    let dir_reporter =
        run_test_suite_lib::output::DirectoryReporter::new(tmp_dir.path().to_path_buf()).unwrap();
    (dir_reporter, tmp_dir)
}

fn create_shell_and_dir_reporter() -> (TestMuxReporter, TestOutputView, tempfile::TempDir) {
    let (shell_reporter, output) = create_shell_reporter();
    let (dir_reporter, tmp_dir) = create_dir_reporter();
    (
        run_test_suite_lib::output::MultiplexedReporter::new(shell_reporter, dir_reporter),
        output,
        tmp_dir,
    )
}

/// Runs a test with a reporter that saves in the directory output format and shell output.
/// This also automatically saves the shell output to a file in custom artifacts.
async fn run_with_reporter<F, Fut>(case_name: &str, test_fn: F)
where
    F: FnOnce(TestMuxMuxReporter, TestOutputView, tempfile::TempDir) -> Fut,
    Fut: futures::future::Future<Output = ()>,
{
    let file_for_shell_reporter =
        std::fs::File::create(format!("custom_artifacts/{}_shell.txt", case_name)).unwrap();
    let file_reporter = run_test_suite_lib::output::ShellReporter::new(std::io::BufWriter::new(
        file_for_shell_reporter,
    ));
    let (mux_reporter, output, tmpdir) = create_shell_and_dir_reporter();
    let mux_mux_reporter =
        run_test_suite_lib::output::MultiplexedReporter::new(mux_reporter, file_reporter);
    test_fn(mux_mux_reporter, output, tmpdir).await;
}

/// run specified test once. Returns suite result and output to shell.
async fn run_test_once(
    reporter: impl 'static + run_test_suite_lib::output::Reporter + Send + Sync,
    test_params: TestParams,
    min_log_severity: Option<Severity>,
) -> Result<Outcome, RunTestSuiteError> {
    let run_reporter = run_test_suite_lib::output::RunReporter::new(reporter);
    Ok(run_test_suite_lib::run_tests_and_get_outcome(
        fuchsia_component::client::connect_to_protocol::<RunBuilderMarker>()
            .expect("connecting to RunBuilderProxy"),
        vec![test_params],
        new_run_params(),
        min_log_severity,
        run_reporter,
        futures::future::pending(),
    )
    .await)
}

#[fixture::fixture(run_with_reporter)]
#[fuchsia::test]
async fn launch_and_test_no_clean_exit(
    reporter: TestMuxMuxReporter,
    output: TestOutputView,
    output_dir: tempfile::TempDir,
) {
    let outcome = run_test_once(
        reporter,
        new_test_params(
            "fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/no-onfinished-after-test-example.cm",
            ),
        None,
    )
    .await
    .expect("Running test should not fail");

    let expected_output = "Running test 'fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/no-onfinished-after-test-example.cm'
[RUNNING]	Example.Test1
log1 for Example.Test1
log2 for Example.Test1
log3 for Example.Test1
[PASSED]	Example.Test1
[RUNNING]	Example.Test2
log1 for Example.Test2
log2 for Example.Test2
log3 for Example.Test2
[PASSED]	Example.Test2
[RUNNING]	Example.Test3
log1 for Example.Test3
log2 for Example.Test3
log3 for Example.Test3
[PASSED]	Example.Test3

3 out of 3 tests passed...
fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/no-onfinished-after-test-example.cm completed with result: INCONCLUSIVE
";
    assert_output!(output.lock().as_slice(), expected_output);

    assert_eq!(outcome, Outcome::Inconclusive);

    directory::testing::assert_run_result(
        output_dir.path(),
        &ExpectedTestRun::new(directory::Outcome::Inconclusive)
            .with_suite(
                ExpectedSuite::new(
                    "fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/no-onfinished-after-test-example.cm",
                    directory::Outcome::Inconclusive,
                )
                .with_tag(TestTag::new("internal", "true"))
                .with_case(ExpectedTestCase::new("Example.Test1", directory::Outcome::Passed))
                .with_case(ExpectedTestCase::new("Example.Test2", directory::Outcome::Passed))
                .with_case(ExpectedTestCase::new("Example.Test3", directory::Outcome::Passed))
            )
    );
}

#[fixture::fixture(run_with_reporter)]
#[fuchsia::test]
async fn launch_and_test_passing_v2_test(
    reporter: TestMuxMuxReporter,
    output: TestOutputView,
    output_dir: tempfile::TempDir,
) {
    let reporter = output::RunReporter::new(reporter);
    let outcome = run_test_suite_lib::run_tests_and_get_outcome(
        fuchsia_component::client::connect_to_protocol::<RunBuilderMarker>()
                    .expect("connecting to RunBuilderProxy"),
        vec![new_test_params(
            "fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/passing-test-example.cm",
        )],
        new_run_params(),
        None,
        reporter,
        futures::future::pending(),
    )
    .await;

    let expected_output = "Running test 'fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/passing-test-example.cm'
[RUNNING]	Example.Test1
log1 for Example.Test1
log2 for Example.Test1
log3 for Example.Test1
[PASSED]	Example.Test1
[RUNNING]	Example.Test2
log1 for Example.Test2
log2 for Example.Test2
log3 for Example.Test2
[PASSED]	Example.Test2
[RUNNING]	Example.Test3
log1 for Example.Test3
log2 for Example.Test3
log3 for Example.Test3
[PASSED]	Example.Test3

3 out of 3 tests passed...
fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/passing-test-example.cm completed with result: PASSED
";
    assert_output!(output.lock().as_ref(), expected_output);

    assert_eq!(outcome, Outcome::Passed);

    directory::testing::assert_run_result(
        output_dir.path(),
        &ExpectedTestRun::new(directory::Outcome::Passed)
            .with_suite(
                ExpectedSuite::new(
                    "fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/passing-test-example.cm",
                    directory::Outcome::Passed,
                )
                .with_tag(TestTag::new("internal", "true"))
                .with_artifact(directory::ArtifactType::Syslog, "syslog.txt".into(), "")
                .with_case(ExpectedTestCase::new("Example.Test1", directory::Outcome::Passed).with_artifact(
                    directory::ArtifactType::Stdout,
                    "stdout.txt".into(),
                    "log1 for Example.Test1\nlog2 for Example.Test1\nlog3 for Example.Test1\n",
                ))
                .with_case(ExpectedTestCase::new("Example.Test2", directory::Outcome::Passed).with_artifact(
                    directory::ArtifactType::Stdout,
                    "stdout.txt".into(),
                    "log1 for Example.Test2\nlog2 for Example.Test2\nlog3 for Example.Test2\n",
                ))
                .with_case(
                    ExpectedTestCase::new("Example.Test3", directory::Outcome::Passed).with_artifact(
                        directory::ArtifactType::Stdout,
                        "stdout.txt".into(),
                        "log1 for Example.Test3\nlog2 for Example.Test3\nlog3 for Example.Test3\n",
                    ),
                )
            )
    );
}

#[fixture::fixture(run_with_reporter)]
#[fuchsia::test]
async fn launch_and_test_stderr_test(
    reporter: TestMuxMuxReporter,
    output: TestOutputView,
    output_dir: tempfile::TempDir,
) {
    let reporter = output::RunReporter::new(reporter);
    let outcome = run_test_suite_lib::run_tests_and_get_outcome(
        fuchsia_component::client::connect_to_protocol::<RunBuilderMarker>()
            .expect("connecting to RunBuilderProxy"),
        vec![new_test_params(
            "fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/test-with-stderr.cm",
        )],
        new_run_params(),
        None,
        reporter,
        futures::future::pending(),
    )
    .await;

    let expected_output = "Running test 'fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/test-with-stderr.cm'
[RUNNING]	Example.Test1
log1 for Example.Test1
log2 for Example.Test1
log3 for Example.Test1
stderr msg1 for Example.Test1
stderr msg2 for Example.Test1
[PASSED]	Example.Test1
[RUNNING]	Example.Test2
log1 for Example.Test2
log2 for Example.Test2
log3 for Example.Test2
[PASSED]	Example.Test2
[RUNNING]	Example.Test3
log1 for Example.Test3
log2 for Example.Test3
log3 for Example.Test3
stderr msg1 for Example.Test3
stderr msg2 for Example.Test3
[PASSED]	Example.Test3

3 out of 3 tests passed...
fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/test-with-stderr.cm completed with result: PASSED
";
    assert_output!(output.lock().as_ref(), expected_output);

    assert_eq!(outcome, Outcome::Passed);
    directory::testing::assert_run_result(
        output_dir.path(),
        &ExpectedTestRun::new(directory::Outcome::Passed)
            .with_suite(
                ExpectedSuite::new(
                    "fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/test-with-stderr.cm",
                    directory::Outcome::Passed,
                )
                .with_tag(TestTag::new("internal", "true"))
                .with_artifact(directory::ArtifactType::Syslog, "syslog.txt".into(), "")
                .with_case(
                    ExpectedTestCase::new("Example.Test1", directory::Outcome::Passed)
                        .with_artifact(
                            directory::ArtifactType::Stdout,
                            "stdout.txt".into(),
                            "log1 for Example.Test1\nlog2 for Example.Test1\nlog3 for Example.Test1\n",
                        )
                        .with_artifact(
                            directory::ArtifactType::Stderr,
                            "stderr.txt".into(),
                            "stderr msg1 for Example.Test1\nstderr msg2 for Example.Test1\n",
                        ),
                )
                .with_case(ExpectedTestCase::new("Example.Test2", directory::Outcome::Passed).with_artifact(
                    directory::ArtifactType::Stdout,
                    "stdout.txt".into(),
                    "log1 for Example.Test2\nlog2 for Example.Test2\nlog3 for Example.Test2\n",
                ))
                .with_case(
                    ExpectedTestCase::new("Example.Test3", directory::Outcome::Passed)
                        .with_artifact(
                            directory::ArtifactType::Stdout,
                            "stdout.txt".into(),
                            "log1 for Example.Test3\nlog2 for Example.Test3\nlog3 for Example.Test3\n",
                        )
                        .with_artifact(
                            directory::ArtifactType::Stderr,
                            "stderr.txt".into(),
                            "stderr msg1 for Example.Test3\nstderr msg2 for Example.Test3\n",
                        ),
                )
            )
    );
}

#[fixture::fixture(run_with_reporter)]
#[fuchsia::test]
async fn launch_and_test_passing_v2_test_multiple_times(
    reporter: TestMuxMuxReporter,
    _: TestOutputView,
    output_dir: tempfile::TempDir,
) {
    let reporter = output::RunReporter::new(reporter);
    let outcome = run_test_suite_lib::run_tests_and_get_outcome(
        fuchsia_component::client::connect_to_protocol::<RunBuilderMarker>()
                    .expect("connecting to RunBuilderProxy"),
            vec![new_test_params(
                "fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/passing-test-example.cm",
                ); 10],
                new_run_params(),
            None,
            reporter,
            futures::future::pending(),
        )
    .await;
    assert_eq!(outcome, Outcome::Passed);

    let expected_test_suite = ExpectedSuite::new(
        "fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/passing-test-example.cm",
        directory::Outcome::Passed,
    )
    .with_tag(TestTag::new("internal", "true"))
    .with_case(ExpectedTestCase::new("Example.Test1", directory::Outcome::Passed))
    .with_case(ExpectedTestCase::new("Example.Test2", directory::Outcome::Passed))
    .with_case(ExpectedTestCase::new("Example.Test3", directory::Outcome::Passed));

    let suite_results =
        directory::TestRunResult::from_dir(output_dir.path()).expect("parse dir").suites;

    assert_eq!(suite_results.len(), 10);
    for suite_result in suite_results {
        directory::testing::assert_suite_result(
            output_dir.path(),
            &suite_result,
            &expected_test_suite,
        );
    }
}

#[fixture::fixture(run_with_reporter)]
#[fuchsia::test]
async fn launch_and_test_multiple_passing_tests(
    reporter: TestMuxMuxReporter,
    _: TestOutputView,
    output_dir: tempfile::TempDir,
) {
    let reporter = output::RunReporter::new(reporter);
    let outcome = run_test_suite_lib::run_tests_and_get_outcome(
        fuchsia_component::client::connect_to_protocol::<RunBuilderMarker>()
                    .expect("connecting to RunBuilderProxy"),
            vec![new_test_params(
                    "fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/passing-test-example.cm",
                ),
                new_test_params(
                    "fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/test-with-stderr.cm",
                )
            ],
            new_run_params(),
            None,
            reporter,
            futures::future::pending(),
        )
    .await;
    assert_eq!(outcome, Outcome::Passed);

    let expected_test_run = ExpectedTestRun::new(directory::Outcome::Passed)
        .with_suite(
            ExpectedSuite::new(
                "fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/passing-test-example.cm",
                directory::Outcome::Passed,
            )
            .with_tag(TestTag::new("internal", "true"))
            .with_case(ExpectedTestCase::new("Example.Test1", directory::Outcome::Passed))
            .with_case(ExpectedTestCase::new("Example.Test2", directory::Outcome::Passed))
            .with_case(ExpectedTestCase::new("Example.Test3", directory::Outcome::Passed)),
        )
        .with_suite(
            ExpectedSuite::new(
                "fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/test-with-stderr.cm",
                directory::Outcome::Passed,
            )
            .with_tag(TestTag::new("internal", "true"))
            .with_case(ExpectedTestCase::new("Example.Test1", directory::Outcome::Passed))
            .with_case(ExpectedTestCase::new("Example.Test2", directory::Outcome::Passed))
            .with_case(ExpectedTestCase::new("Example.Test3", directory::Outcome::Passed))
        );

    directory::testing::assert_run_result(output_dir.path(), &expected_test_run);
}

#[fixture::fixture(run_with_reporter)]
#[fuchsia::test]
async fn launch_and_test_with_filter(
    reporter: TestMuxMuxReporter,
    output: TestOutputView,
    output_dir: tempfile::TempDir,
) {
    let mut test_params = new_test_params(
        "fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/passing-test-example.cm",
    );

    test_params.test_filters = Some(vec!["*Test3".to_string()]);
    let outcome =
        run_test_once(reporter, test_params, None).await.expect("Running test should not fail");

    let expected_output = "Running test 'fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/passing-test-example.cm'
[RUNNING]	Example.Test3
log1 for Example.Test3
log2 for Example.Test3
log3 for Example.Test3
[PASSED]	Example.Test3

1 out of 1 tests passed...
fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/passing-test-example.cm completed with result: PASSED
";
    assert_output!(output.lock().as_slice(), expected_output);

    assert_eq!(outcome, Outcome::Passed);

    directory::testing::assert_run_result(
        output_dir.path(),
        &ExpectedTestRun::new(directory::Outcome::Passed)
            .with_suite(
                ExpectedSuite::new(
                    "fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/passing-test-example.cm",
                    directory::Outcome::Passed
                )
                .with_tag(TestTag::new("internal", "true"))
                .with_case(ExpectedTestCase::new("Example.Test3", directory::Outcome::Passed))
            )
    );
}

#[fixture::fixture(run_with_reporter)]
#[fuchsia::test]
async fn launch_and_test_with_multiple_filter(
    reporter: TestMuxMuxReporter,
    output: TestOutputView,
    output_dir: tempfile::TempDir,
) {
    let mut test_params = new_test_params(
        "fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/passing-test-example.cm",
    );

    test_params.test_filters = Some(vec!["*Test3".to_string(), "*Test1".to_string()]);
    let outcome =
        run_test_once(reporter, test_params, None).await.expect("Running test should not fail");

    let expected_output = "Running test 'fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/passing-test-example.cm'
[RUNNING]	Example.Test1
log1 for Example.Test1
log2 for Example.Test1
log3 for Example.Test1
[PASSED]	Example.Test1
[RUNNING]	Example.Test3
log1 for Example.Test3
log2 for Example.Test3
log3 for Example.Test3
[PASSED]	Example.Test3

2 out of 2 tests passed...
fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/passing-test-example.cm completed with result: PASSED
";
    assert_output!(output.lock().as_slice(), expected_output);

    assert_eq!(outcome, Outcome::Passed);
    directory::testing::assert_run_result(
        output_dir.path(),
        &ExpectedTestRun::new(directory::Outcome::Passed)
            .with_suite(
                ExpectedSuite::new(
                    "fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/passing-test-example.cm",
                    directory::Outcome::Passed
                )
                .with_tag(TestTag::new("internal", "true"))
                .with_case(ExpectedTestCase::new("Example.Test1", directory::Outcome::Passed))
                .with_case(ExpectedTestCase::new("Example.Test3", directory::Outcome::Passed))
            )
    );
}

#[fuchsia::test]
async fn launch_with_filter_no_matching_cases() {
    let mut test_params = new_test_params(
        "fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/passing-test-example.cm",
    );

    test_params.test_filters = Some(vec!["matches-nothing".to_string()]);
    let outcome = run_test_once(output::NoopReporter, test_params, None).await.unwrap();

    match outcome {
        Outcome::Error { origin } => {
            assert_matches!(
                origin.as_ref(),
                RunTestSuiteError::Launch(LaunchError::NoMatchingCases)
            );
            assert!(!origin.is_internal_error());
        }
        _ => panic!("Expected error but got {:?}", outcome),
    }
}

#[fixture::fixture(run_with_reporter)]
#[fuchsia::test]
async fn launch_and_test_empty_test(
    reporter: TestMuxMuxReporter,
    _: TestOutputView,
    output_dir: tempfile::TempDir,
) {
    let outcome = run_test_once(
        reporter,
        new_test_params(
            "fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/no-test-example.cm",
        ),
        None,
    )
    .await
    .unwrap();

    assert_eq!(outcome, Outcome::Passed);

    directory::testing::assert_run_result(
        output_dir.path(),
        &ExpectedTestRun::new(directory::Outcome::Passed)
            .with_suite(
                ExpectedSuite::new(
                    "fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/no-test-example.cm",
                    directory::Outcome::Passed,
                )
                .with_tag(TestTag::new("internal", "true"))
            )
    );
}

#[fixture::fixture(run_with_reporter)]
#[fuchsia::test]
async fn launch_and_test_huge_test(
    reporter: TestMuxMuxReporter,
    _: TestOutputView,
    output_dir: tempfile::TempDir,
) {
    let outcome = run_test_once(
        reporter,
        new_test_params(
            "fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/huge-test-example.cm",
        ),
        None,
    )
    .await
    .unwrap();

    assert_eq!(outcome, Outcome::Passed);

    let mut expected_test_suite = ExpectedSuite::new(
        "fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/huge-test-example.cm",
        directory::Outcome::Passed,
    )
    .with_tag(TestTag::new("internal", "true"));
    for i in 1..1001 {
        expected_test_suite = expected_test_suite.with_case(ExpectedTestCase::new(
            format!("FooTest{:?}", i),
            directory::Outcome::Passed,
        ));
    }

    directory::testing::assert_run_result(
        output_dir.path(),
        &ExpectedTestRun::new(directory::Outcome::Passed).with_suite(expected_test_suite),
    );
}

#[fuchsia::test]
async fn launch_and_test_logspam_test() {
    let (reporter, _output, output_dir) = create_shell_and_dir_reporter();
    let outcome = run_test_once(
        reporter,
        new_test_params(
            "fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/logspam_test.cm",
        ),
        None,
    )
    .await
    .expect("Running test should not fail");
    assert_eq!(outcome, Outcome::Passed);

    directory::testing::assert_run_result(
        output_dir.path(),
        &ExpectedTestRun::new(directory::Outcome::Passed).with_suite(
            ExpectedSuite::new(
                "fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/logspam_test.cm",
                directory::Outcome::Passed,
            )
            .with_case(ExpectedTestCase::new("spam_logs", directory::Outcome::Passed))
            .with_tag(TestTag::new("internal", "true"))
            .with_matching_artifact(
                directory::ArtifactType::Syslog,
                "syslog.txt".into(),
                move |contents| {
                    // The test should produce many more lines, but logs could be dropped due to
                    // timeouts.
                    assert_geq!(contents.lines().count(), 1000);
                },
            ),
        ),
    );
}

#[fixture::fixture(run_with_reporter)]
#[fuchsia::test]
async fn launch_and_test_disabled_test_exclude_disabled(
    reporter: TestMuxMuxReporter,
    output: TestOutputView,
    output_dir: tempfile::TempDir,
) {
    let outcome = run_test_once(
        reporter,
            new_test_params(
                "fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/disabled-test-example.cm",
            ),
            None,
        )
    .await
    .expect("Running test should not fail");

    let expected_output = "Running test 'fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/disabled-test-example.cm'
[RUNNING]	Example.Test1
log1 for Example.Test1
log2 for Example.Test1
log3 for Example.Test1
[PASSED]	Example.Test1
[RUNNING]	Example.Test2
[SKIPPED]	Example.Test2
[RUNNING]	Example.Test3
[SKIPPED]	Example.Test3

1 out of 3 tests passed...
fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/disabled-test-example.cm completed with result: PASSED
";
    assert_output!(output.lock().as_slice(), expected_output);

    assert_eq!(outcome, Outcome::Passed);

    directory::testing::assert_run_result(
        output_dir.path(),
        &ExpectedTestRun::new(directory::Outcome::Passed)
            .with_suite(
                ExpectedSuite::new(
                    "fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/disabled-test-example.cm",
                    directory::Outcome::Passed
                )
                .with_tag(TestTag::new("internal", "true"))
                .with_case(ExpectedTestCase::new("Example.Test1", directory::Outcome::Passed))
                .with_case(ExpectedTestCase::new("Example.Test2", directory::Outcome::Skipped))
                .with_case(ExpectedTestCase::new("Example.Test3", directory::Outcome::Skipped))
            )
    );
}

#[fixture::fixture(run_with_reporter)]
#[fuchsia::test]
async fn launch_and_test_disabled_test_include_disabled(
    reporter: TestMuxMuxReporter,
    output: TestOutputView,
    output_dir: tempfile::TempDir,
) {
    let mut test_params = new_test_params(
        "fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/disabled-test-example.cm",
    );
    test_params.also_run_disabled_tests = true;
    let outcome =
        run_test_once(reporter, test_params, None).await.expect("Running test should not fail");

    let expected_output = "Running test 'fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/disabled-test-example.cm'
[RUNNING]	Example.Test1
log1 for Example.Test1
log2 for Example.Test1
log3 for Example.Test1
[PASSED]	Example.Test1
[RUNNING]	Example.Test2
log1 for Example.Test2
log2 for Example.Test2
log3 for Example.Test2
[PASSED]	Example.Test2
[RUNNING]	Example.Test3
log1 for Example.Test3
log2 for Example.Test3
log3 for Example.Test3
[FAILED]	Example.Test3

Failed tests: Example.Test3
2 out of 3 tests passed...
fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/disabled-test-example.cm completed with result: FAILED
";
    assert_output!(output.lock().as_slice(), expected_output);

    assert_eq!(outcome, Outcome::Failed);

    directory::testing::assert_run_result(
        output_dir.path(),
        &ExpectedTestRun::new(directory::Outcome::Failed)
            .with_suite(
                ExpectedSuite::new(
                    "fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/disabled-test-example.cm",
                    directory::Outcome::Failed
                )
                .with_tag(TestTag::new("internal", "true"))
                .with_case(ExpectedTestCase::new("Example.Test1", directory::Outcome::Passed))
                .with_case(ExpectedTestCase::new("Example.Test2", directory::Outcome::Passed))
                .with_case(ExpectedTestCase::new("Example.Test3", directory::Outcome::Failed))
            )
    );
}

#[fixture::fixture(run_with_reporter)]
#[fuchsia::test]
async fn launch_and_test_failing_test(
    reporter: TestMuxMuxReporter,
    output: TestOutputView,
    output_dir: tempfile::TempDir,
) {
    let outcome = run_test_once(
            reporter,
            new_test_params(
                "fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/failing-test-example.cm",
            ),
            None,
        )
    .await
    .expect("Running test should not fail");

    let expected_output = "Running test 'fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/failing-test-example.cm'
[RUNNING]	Example.Test1
log1 for Example.Test1
log2 for Example.Test1
log3 for Example.Test1
[PASSED]	Example.Test1
[RUNNING]	Example.Test2
log1 for Example.Test2
log2 for Example.Test2
log3 for Example.Test2
[FAILED]	Example.Test2
[RUNNING]	Example.Test3
log1 for Example.Test3
log2 for Example.Test3
log3 for Example.Test3
[PASSED]	Example.Test3

Failed tests: Example.Test2
2 out of 3 tests passed...
fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/failing-test-example.cm completed with result: FAILED
";

    assert_output!(output.lock().as_slice(), expected_output);

    assert_eq!(outcome, Outcome::Failed);

    directory::testing::assert_run_result(
        output_dir.path(),
        &ExpectedTestRun::new(directory::Outcome::Failed)
            .with_suite(
                ExpectedSuite::new(
                    "fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/failing-test-example.cm",
                    directory::Outcome::Failed
                )
                .with_tag(TestTag::new("internal", "true"))
                .with_case(ExpectedTestCase::new("Example.Test1", directory::Outcome::Passed))
                .with_case(ExpectedTestCase::new("Example.Test2", directory::Outcome::Failed))
                .with_case(ExpectedTestCase::new("Example.Test3", directory::Outcome::Passed))
            )
    );
}

#[fixture::fixture(run_with_reporter)]
#[fuchsia::test]
async fn launch_and_test_failing_v2_test_multiple_times(
    reporter: TestMuxMuxReporter,
    _: TestOutputView,
    output_dir: tempfile::TempDir,
) {
    let reporter = output::RunReporter::new(reporter);
    let outcome = run_test_suite_lib::run_tests_and_get_outcome(
        fuchsia_component::client::connect_to_protocol::<RunBuilderMarker>()
                    .expect("connecting to RunBuilderProxy"),
        vec![new_test_params(
                "fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/failing-test-example.cm",
                ); 10],
                new_run_params(),
                None, reporter, futures::future::pending(),
        )
    .await;

    assert_eq!(outcome, Outcome::Failed);

    let suite_results =
        directory::TestRunResult::from_dir(output_dir.path()).expect("parse dir").suites;

    let expected_suite = ExpectedSuite::new(
        "fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/failing-test-example.cm",
        directory::Outcome::Failed,
    )
    .with_tag(TestTag::new("internal", "true"))
    .with_case(ExpectedTestCase::new("Example.Test1", directory::Outcome::Passed))
    .with_case(ExpectedTestCase::new("Example.Test2", directory::Outcome::Failed))
    .with_case(ExpectedTestCase::new("Example.Test3", directory::Outcome::Passed));

    for suite_result in suite_results {
        directory::testing::assert_suite_result(output_dir.path(), &suite_result, &expected_suite);
    }
}

#[fixture::fixture(run_with_reporter)]
#[fuchsia::test]
async fn launch_and_test_incomplete_test(
    reporter: TestMuxMuxReporter,
    _: TestOutputView,
    output_dir: tempfile::TempDir,
) {
    let outcome = run_test_once(
            reporter,
            new_test_params(
                "fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/incomplete-test-example.cm",
            ),
            None,
        )
    .await
    .expect("Running test should not fail");

    assert_eq!(outcome, Outcome::DidNotFinish);

    directory::testing::assert_run_result(
        output_dir.path(),
        &ExpectedTestRun::new(directory::Outcome::Inconclusive)
            .with_suite(
                ExpectedSuite::new(
                    "fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/incomplete-test-example.cm",
                    directory::Outcome::Inconclusive
                )
                .with_tag(TestTag::new("internal", "true"))
                .with_case(ExpectedTestCase::new("Example.Test1", directory::Outcome::Error))
                .with_case(ExpectedTestCase::new("Example.Test2", directory::Outcome::Passed))
                .with_case(ExpectedTestCase::new("Example.Test3", directory::Outcome::Error))
            )
    );
}

#[fixture::fixture(run_with_reporter)]
#[fuchsia::test]
async fn launch_and_test_invalid_test(
    reporter: TestMuxMuxReporter,
    _: TestOutputView,
    output_dir: tempfile::TempDir,
) {
    let outcome = run_test_once(
        reporter,
            new_test_params(
                "fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/invalid-test-example.cm",
            ),
            None,
        )
    .await
    .expect("Running test should not fail");

    assert_eq!(outcome, Outcome::DidNotFinish);

    directory::testing::assert_run_result(
        output_dir.path(),
        &ExpectedTestRun::new(directory::Outcome::Inconclusive)
        .with_suite(
            ExpectedSuite::new(
                "fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/invalid-test-example.cm",
                directory::Outcome::Inconclusive
            )
            .with_tag(TestTag::new("internal", "true"))
            .with_case(ExpectedTestCase::new("Example.Test1", directory::Outcome::Error))
            .with_case(ExpectedTestCase::new("Example.Test2", directory::Outcome::Passed))
            .with_case(ExpectedTestCase::new("Example.Test3", directory::Outcome::Error))
        )
    );
}

// This test also acts an example on how to right a v2 test.
// This will launch a echo_realm which will inject echo_server, launch v2 test which will
// then test that server out and return back results.
#[fixture::fixture(run_with_reporter)]
#[fuchsia::test]
async fn launch_and_run_echo_test(
    reporter: TestMuxMuxReporter,
    output: TestOutputView,
    _: tempfile::TempDir,
) {
    let outcome = run_test_once(
        reporter,
        new_test_params(
            "fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/echo_test_realm.cm",
        ),
        None,
    )
    .await
    .expect("Running test should not fail");

    let expected_output = "Running test 'fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/echo_test_realm.cm'
[RUNNING]	EchoTest
[PASSED]	EchoTest

1 out of 1 tests passed...
fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/echo_test_realm.cm completed with result: PASSED
";
    assert_output!(output.lock().as_slice(), expected_output);

    assert_eq!(outcome, Outcome::Passed);
}

/// Time to wait for tests that verify timeout behavior.
/// If this is too short, the test may not complete launching.
const TIMEOUT_SECONDS: Option<std::num::NonZeroU32> = std::num::NonZeroU32::new(3);

#[fixture::fixture(run_with_reporter)]
#[fuchsia::test]
async fn test_timeout(reporter: TestMuxMuxReporter, output: TestOutputView, _: tempfile::TempDir) {
    let mut test_params = new_test_params(
        "fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/long_running_test.cm",
    );
    test_params.timeout_seconds = TIMEOUT_SECONDS;
    let outcome =
        run_test_once(reporter, test_params, None).await.expect("Running test should not fail");
    let expected_output = "Running test 'fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/long_running_test.cm'
[RUNNING]	long_running
[TIMED_OUT]	long_running

Failed tests: long_running
0 out of 1 tests passed...
fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/long_running_test.cm completed with result: TIMED_OUT
";
    assert_output!(output.lock().as_slice(), expected_output);
    assert_eq!(outcome, Outcome::Timedout);
}

#[fixture::fixture(run_with_reporter)]
#[fuchsia::test]
// when a test times out, we should not run it again.
async fn test_timeout_multiple_times(
    reporter: TestMuxMuxReporter,
    output: TestOutputView,
    output_dir: tempfile::TempDir,
) {
    let reporter = output::RunReporter::new(reporter);
    let mut test_params = new_test_params(
        "fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/long_running_test.cm",
    );
    test_params.timeout_seconds = TIMEOUT_SECONDS;
    let outcome = run_test_suite_lib::run_tests_and_get_outcome(
        fuchsia_component::client::connect_to_protocol::<RunBuilderMarker>()
            .expect("connecting to RunBuilderProxy"),
        vec![test_params; 10],
        new_run_params(),
        None,
        reporter,
        futures::future::pending(),
    )
    .await;
    assert_eq!(outcome, Outcome::Timedout);

    let expected_output = "Running test 'fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/long_running_test.cm'
[RUNNING]	long_running
[TIMED_OUT]	long_running

Failed tests: long_running
0 out of 1 tests passed...
fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/long_running_test.cm completed with result: TIMED_OUT
";
    assert_output!(output.lock().as_ref(), expected_output);

    let run_result = directory::TestRunResult::from_dir(output_dir.path()).expect("parse dir");
    assert_eq!(
        run_result.common.deref().outcome,
        directory::MaybeUnknown::Known(directory::Outcome::Timedout)
    );

    let (timed_out_suites, not_started_suites): (Vec<_>, Vec<_>) =
        run_result.suites.into_iter().partition(|suite_result| {
            suite_result.common.deref().outcome
                == directory::MaybeUnknown::Known(directory::Outcome::Timedout)
        });

    assert_eq!(timed_out_suites.len(), 1);
    directory::testing::assert_suite_result(
        output_dir.path(),
        &timed_out_suites[0],
        &ExpectedSuite::new(
            "fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/long_running_test.cm",
            directory::Outcome::Timedout,
        )
        .with_tag(TestTag::new("internal", "true"))
        .with_case(ExpectedTestCase::new("long_running", directory::Outcome::Timedout)),
    );

    assert_eq!(not_started_suites.len(), 9);
    for suite_result in not_started_suites {
        directory::testing::assert_suite_result(
            output_dir.path(),
            &suite_result,
            &ExpectedSuite::new(
                "fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/long_running_test.cm",
                directory::Outcome::NotStarted
            )
            .with_tag(TestTag::new("internal", "true"))
        );
    }
}

#[fixture::fixture(run_with_reporter)]
#[fuchsia::test]
async fn test_continue_on_timeout(
    reporter: TestMuxMuxReporter,
    _: TestOutputView,
    output_dir: tempfile::TempDir,
) {
    let reporter = output::RunReporter::new(reporter);
    let mut long_test_params = new_test_params(
        "fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/long_running_test.cm",
    );
    long_test_params.timeout_seconds = TIMEOUT_SECONDS;

    let short_test_params = new_test_params(
        "fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/passing-test-example.cm",
    );

    let mut test_params = vec![long_test_params];
    for _ in 0..10 {
        test_params.push(short_test_params.clone());
    }

    let outcome = run_test_suite_lib::run_tests_and_get_outcome(
        fuchsia_component::client::connect_to_protocol::<RunBuilderMarker>()
            .expect("connecting to RunBuilderProxy"),
        test_params,
        run_test_suite_lib::RunParams {
            timeout_behavior: run_test_suite_lib::TimeoutBehavior::Continue,
            stop_after_failures: None,
        },
        None,
        reporter,
        futures::future::pending(),
    )
    .await;
    assert_eq!(outcome, Outcome::Timedout);

    let run_result = directory::TestRunResult::from_dir(output_dir.path()).expect("parse dir");
    assert_eq!(
        run_result.common.deref().outcome,
        directory::MaybeUnknown::Known(directory::Outcome::Timedout)
    );

    let (timed_out_suites, passing_suites): (Vec<_>, Vec<_>) =
        run_result.suites.into_iter().partition(|suite_result| {
            suite_result.common.deref().outcome
                == directory::MaybeUnknown::Known(directory::Outcome::Timedout)
        });

    assert_eq!(timed_out_suites.len(), 1);
    directory::testing::assert_suite_result(
        output_dir.path(),
        &timed_out_suites[0],
        &ExpectedSuite::new(
            "fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/long_running_test.cm",
            directory::Outcome::Timedout,
        )
        .with_tag(TestTag::new("internal", "true"))
        .with_case(ExpectedTestCase::new("long_running", directory::Outcome::Timedout)),
    );

    assert_eq!(passing_suites.len(), 10);
    for suite_result in passing_suites {
        directory::testing::assert_suite_result(
            output_dir.path(),
            &suite_result,
            &ExpectedSuite::new(
                "fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/passing-test-example.cm",
                directory::Outcome::Passed
            )
            .with_tag(TestTag::new("internal", "true"))
            .with_case(ExpectedTestCase::new("Example.Test1", directory::Outcome::Passed))
            .with_case(ExpectedTestCase::new("Example.Test2", directory::Outcome::Passed))
            .with_case(ExpectedTestCase::new("Example.Test3", directory::Outcome::Passed)),
        );
    }
}

#[fixture::fixture(run_with_reporter)]
#[fuchsia::test]
async fn test_stop_after_n_failures(
    reporter: TestMuxMuxReporter,
    _: TestOutputView,
    output_dir: tempfile::TempDir,
) {
    let reporter = output::RunReporter::new(reporter);
    let outcome = run_test_suite_lib::run_tests_and_get_outcome(
        fuchsia_component::client::connect_to_protocol::<RunBuilderMarker>()
                    .expect("connecting to RunBuilderProxy"),
        vec![new_test_params(
                "fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/failing-test-example.cm",
                ); 10],
                run_test_suite_lib::RunParams {
                    timeout_behavior: run_test_suite_lib::TimeoutBehavior::Continue,
                    stop_after_failures: Some(5u32.try_into().unwrap()),
                },
                None, reporter, futures::future::pending(),
        )
    .await;
    assert_eq!(outcome, Outcome::Failed);

    let run_result = directory::TestRunResult::from_dir(output_dir.path()).expect("parse dir");
    assert_eq!(
        run_result.common.deref().outcome,
        directory::MaybeUnknown::Known(directory::Outcome::Failed)
    );

    let (failed_suites, not_started_suites): (Vec<_>, Vec<_>) =
        run_result.suites.into_iter().partition(|suite_result| {
            suite_result.common.deref().outcome
                == directory::MaybeUnknown::Known(directory::Outcome::Failed)
        });

    assert_eq!(failed_suites.len(), 5);
    assert_eq!(not_started_suites.len(), 5);
    for failed_suite in failed_suites {
        directory::testing::assert_suite_result(
            output_dir.path(),
            &failed_suite,
            &ExpectedSuite::new(
                "fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/failing-test-example.cm",
                directory::Outcome::Failed
            )
            .with_tag(TestTag::new("internal", "true"))
            .with_case(ExpectedTestCase::new("Example.Test1", directory::Outcome::Passed))
            .with_case(ExpectedTestCase::new("Example.Test2", directory::Outcome::Failed))
            .with_case(ExpectedTestCase::new("Example.Test3", directory::Outcome::Passed))
        );
    }

    for not_started_suite in not_started_suites {
        directory::testing::assert_suite_result(
            output_dir.path(),
            &not_started_suite,
            &ExpectedSuite::new(
                "fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/failing-test-example.cm",
                directory::Outcome::NotStarted
            )
            .with_tag(TestTag::new("internal", "true"))
        );
    }
}

#[fixture::fixture(run_with_reporter)]
#[fuchsia::test]
async fn test_passes_with_large_timeout(
    reporter: TestMuxMuxReporter,
    output: TestOutputView,
    _: tempfile::TempDir,
) {
    let mut test_params = new_test_params(
        "fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/echo_test_realm.cm",
    );
    test_params.timeout_seconds = std::num::NonZeroU32::new(600);
    let outcome =
        run_test_once(reporter, test_params, None).await.expect("Running test should not fail");

    let expected_output = "Running test 'fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/echo_test_realm.cm'
[RUNNING]	EchoTest
[PASSED]	EchoTest

1 out of 1 tests passed...
fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/echo_test_realm.cm completed with result: PASSED
";
    assert_output!(output.lock().as_slice(), expected_output);

    assert_eq!(outcome, Outcome::Passed);
}

#[fixture::fixture(run_with_reporter)]
#[fuchsia::test]
async fn test_logging_component(
    reporter: TestMuxMuxReporter,
    output: TestOutputView,
    output_dir: tempfile::TempDir,
) {
    let mut test_params = new_test_params(
        "fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/logging_test.cm",
    );
    test_params.timeout_seconds = std::num::NonZeroU32::new(600);
    let outcome =
        run_test_once(reporter, test_params, None).await.expect("Running test should not fail");

    let expected_logs =
        "[TIMESTAMP][PID][TID][<root>][log_and_exit,logging_test] DEBUG: Logging initialized
[TIMESTAMP][PID][TID][<root>][log_and_exit,logging_test] DEBUG: my debug message
[TIMESTAMP][PID][TID][<root>][log_and_exit,logging_test] INFO: my info message
[TIMESTAMP][PID][TID][<root>][log_and_exit,logging_test] WARN: my warn message
";

    let expected_output = format!("Running test 'fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/logging_test.cm'
[RUNNING]	log_and_exit
{}[PASSED]	log_and_exit

1 out of 1 tests passed...
fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/logging_test.cm completed with result: PASSED
", expected_logs);
    assert_output!(output.lock().as_slice(), expected_output.as_str());
    assert_eq!(outcome, Outcome::Passed);

    directory::testing::assert_run_result(
        output_dir.path(),
        &ExpectedTestRun::new(directory::Outcome::Passed).with_suite(
            ExpectedSuite::new(
                "fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/logging_test.cm",
                directory::Outcome::Passed,
            )
            .with_tag(TestTag::new("internal", "true"))
            .with_matching_artifact(
                directory::ArtifactType::Syslog,
                "syslog.txt".into(),
                move |contents| {
                    assert_output!(contents.as_bytes(), expected_logs);
                },
            )
            .with_case(ExpectedTestCase::new("log_and_exit", directory::Outcome::Passed)),
        ),
    );
}

#[fixture::fixture(run_with_reporter)]
#[fuchsia::test]
async fn test_logging_component_min_severity(
    reporter: TestMuxMuxReporter,
    output: TestOutputView,
    _: tempfile::TempDir,
) {
    let mut test_params = new_test_params(
        "fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/logging_test.cm",
    );
    test_params.timeout_seconds = std::num::NonZeroU32::new(600);
    let outcome = run_test_once(reporter, test_params, Some(Severity::Info))
        .await
        .expect("Running test should not fail");

    let expected_output = "Running test 'fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/logging_test.cm'
[RUNNING]	log_and_exit
[TIMESTAMP][PID][TID][<root>][log_and_exit,logging_test] INFO: my info message\n\
[TIMESTAMP][PID][TID][<root>][log_and_exit,logging_test] WARN: my warn message\n\
[PASSED]	log_and_exit

1 out of 1 tests passed...
fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/logging_test.cm completed with result: PASSED
";
    assert_output!(output.lock().as_slice(), expected_output);
    assert_eq!(outcome, Outcome::Passed);
}

#[fixture::fixture(run_with_reporter)]
#[fuchsia::test]
async fn test_stdout_and_log_ansi(
    reporter: TestMuxMuxReporter,
    output: TestOutputView,
    _: tempfile::TempDir,
) {
    let mut test_params = new_test_params(
        "fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/stdout_ansi_test.cm",
    );
    test_params.timeout_seconds = std::num::NonZeroU32::new(600);
    let outcome = run_test_once(reporter, test_params, Some(Severity::Info))
        .await
        .expect("Running test should not fail");

    let expected_output = "Running test 'fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/stdout_ansi_test.cm'
[RUNNING]	log_ansi_test
[TIMESTAMP][PID][TID][<root>][log_ansi_test] INFO: \u{1b}[31mred log\u{1b}[0m
[PASSED]	log_ansi_test
[RUNNING]	stdout_ansi_test
\u{1b}[31mred stdout\u{1b}[0m
[PASSED]	stdout_ansi_test

2 out of 2 tests passed...
fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/stdout_ansi_test.cm completed with result: PASSED
";
    assert_output!(output.lock().as_slice(), expected_output);
    assert_eq!(outcome, Outcome::Passed);
}

#[fixture::fixture(run_with_reporter)]
#[fuchsia::test]
async fn test_stdout_and_log_filter_ansi(
    reporter: TestMuxMuxReporter,
    output: TestOutputView,
    _: tempfile::TempDir,
) {
    let reporter = output::RunReporter::new_ansi_filtered(reporter);
    let mut test_params = new_test_params(
        "fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/stdout_ansi_test.cm",
    );
    test_params.timeout_seconds = std::num::NonZeroU32::new(600);

    let outcome = run_test_suite_lib::run_tests_and_get_outcome(
        fuchsia_component::client::connect_to_protocol::<RunBuilderMarker>()
            .expect("connecting to RunBuilderProxy"),
        vec![test_params],
        new_run_params(),
        Some(Severity::Info),
        reporter,
        futures::future::pending(),
    )
    .await;

    let expected_output = "Running test 'fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/stdout_ansi_test.cm'
[RUNNING]	log_ansi_test
[TIMESTAMP][PID][TID][<root>][log_ansi_test] INFO: red log
[PASSED]	log_ansi_test
[RUNNING]	stdout_ansi_test
red stdout
[PASSED]	stdout_ansi_test

2 out of 2 tests passed...
fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/stdout_ansi_test.cm completed with result: PASSED
";
    assert_output!(output.lock().as_ref(), expected_output);
    assert_eq!(outcome, Outcome::Passed);
}

#[fuchsia_async::run_singlethreaded(test)]
async fn test_logging_component_max_severity_info() {
    test_max_severity(Severity::Info).await;
}

#[fuchsia_async::run_singlethreaded(test)]
async fn test_logging_component_max_severity_warn() {
    test_max_severity(Severity::Warn).await;
}

#[fuchsia_async::run_singlethreaded(test)]
async fn test_logging_component_max_severity_error() {
    test_max_severity(Severity::Error).await;
}

async fn test_max_severity(max_severity: Severity) {
    let mut test_params = new_test_params(
        "fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/error_logging_test.cm",
    );
    test_params.timeout_seconds = std::num::NonZeroU32::new(600);
    test_params.max_severity_logs = Some(max_severity);
    let (reporter, output) = create_shell_reporter();
    let outcome =
        run_test_once(reporter, test_params, None).await.expect("Running test should not fail");

    let expected_output_prefix = "Running test 'fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/error_logging_test.cm'
[RUNNING]	log_and_exit
[TIMESTAMP][PID][TID][<root>][log_and_exit,error_logging_test] DEBUG: Logging initialized
[TIMESTAMP][PID][TID][<root>][log_and_exit,error_logging_test] INFO: my info message
[TIMESTAMP][PID][TID][<root>][log_and_exit,error_logging_test] WARN: my warn message
[TIMESTAMP][PID][TID][<root>][log_and_exit,error_logging_test] ERROR: [../../src/sys/run_test_suite/tests/error_logging_test.rs(12)] my error message
[PASSED]	log_and_exit

1 out of 1 tests passed...

";
    let (expected_outcome, expected_output_postfix) = match max_severity {
        Severity::Info => (
                Outcome::Failed,
                "Test fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/error_logging_test.cm produced unexpected high-severity logs:
----------------xxxxx----------------
[TIMESTAMP][PID][TID][<root>][log_and_exit,error_logging_test] WARN: my warn message
[TIMESTAMP][PID][TID][<root>][log_and_exit,error_logging_test] ERROR: [../../src/sys/run_test_suite/tests/error_logging_test.rs(12)] my error message

----------------xxxxx----------------
Failing this test. See: https://fuchsia.dev/fuchsia-src/development/diagnostics/test_and_logs#restricting_log_severity

fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/error_logging_test.cm completed with result: FAILED
"
        ),
        Severity::Warn => (
            Outcome::Failed,
            "Test fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/error_logging_test.cm produced unexpected high-severity logs:
----------------xxxxx----------------
[TIMESTAMP][PID][TID][<root>][log_and_exit,error_logging_test] ERROR: [../../src/sys/run_test_suite/tests/error_logging_test.rs(12)] my error message

----------------xxxxx----------------
Failing this test. See: https://fuchsia.dev/fuchsia-src/development/diagnostics/test_and_logs#restricting_log_severity

fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/error_logging_test.cm completed with result: FAILED
"
        ),
        Severity::Error => (
            Outcome::Passed,
            "fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/error_logging_test.cm completed with result: PASSED"
        ),
        _ => panic!("unexpected severity")
    };
    let expected_output = format!("{}{}", expected_output_prefix, expected_output_postfix);
    assert_output!(output.lock().as_slice(), expected_output);
    assert_eq!(outcome, expected_outcome);
}

#[fuchsia::test]
async fn test_does_not_resolve() {
    let test_params = new_test_params(
        "fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/nonexistant_test.cm",
    );
    let log_opts = None;
    let outcome = run_test_once(output::NoopReporter, test_params, log_opts).await.unwrap();
    let origin_error = match outcome {
        Outcome::Error { origin } => origin,
        other => panic!("Expected an error outcome but got {:?}", other),
    };
    assert!(!origin_error.is_internal_error());
    assert_matches!(
        Arc::try_unwrap(origin_error).unwrap(),
        RunTestSuiteError::Launch(LaunchError::InstanceCannotResolve)
    );
}

#[fixture::fixture(run_with_reporter)]
#[fuchsia::test]
async fn test_stdout_to_directory(
    reporter: TestMuxMuxReporter,
    _: TestOutputView,
    output_dir: tempfile::TempDir,
) {
    let reporter = output::RunReporter::new(reporter);
    let mut test_params = new_test_params(
        "fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/stdout_ansi_test.cm",
    );
    test_params.timeout_seconds = std::num::NonZeroU32::new(600);

    let outcome = run_test_suite_lib::run_tests_and_get_outcome(
        fuchsia_component::client::connect_to_protocol::<RunBuilderMarker>()
            .expect("connecting to RunBuilderProxy"),
        vec![test_params],
        new_run_params(),
        None,
        reporter,
        futures::future::pending(),
    )
    .await;

    assert_eq!(outcome, Outcome::Passed);

    let expected_test_run = ExpectedTestRun::new(directory::Outcome::Passed)
        .with_any_start_time()
        .with_no_run_duration()
        .with_suite(
            ExpectedSuite::new(
                "fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/stdout_ansi_test.cm",
                directory::Outcome::Passed,
            )
            .with_tag(TestTag::new("internal", "true"))
            .with_matching_artifact(directory::ArtifactType::Syslog, "syslog.txt".into(), |logs| {
                assert_output!(
                    logs.as_bytes(),
                    "\n[TIMESTAMP][PID][TID][<root>][log_ansi_test] INFO: \u{1b}[31mred log\u{1b}[0m"
                );
            })
            .with_case(
                ExpectedTestCase::new("stdout_ansi_test", directory::Outcome::Passed)
                    .with_artifact(
                        directory::ArtifactType::Stdout,
                        "stdout.txt".into(),
                        "\u{1b}[31mred stdout\u{1b}[0m\n",
                    )
                    .with_no_run_duration()
                    .with_any_start_time(),
            )
            .with_case(
                ExpectedTestCase::new("log_ansi_test", directory::Outcome::Passed)
                    .with_no_run_duration()
                    .with_any_start_time(),
            )
            .with_any_run_duration()
            .with_any_start_time()
        );
    directory::testing::assert_run_result(output_dir.path(), &expected_test_run);
}

#[fixture::fixture(run_with_reporter)]
#[fuchsia::test]
async fn test_syslog_to_directory(
    reporter: TestMuxMuxReporter,
    _: TestOutputView,
    output_dir: tempfile::TempDir,
) {
    let reporter = output::RunReporter::new(reporter);
    let mut test_params = new_test_params(
        "fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/error_logging_test.cm",
    );
    test_params.timeout_seconds = std::num::NonZeroU32::new(600);
    test_params.max_severity_logs = Some(Severity::Warn);

    let outcome = run_test_suite_lib::run_tests_and_get_outcome(
        fuchsia_component::client::connect_to_protocol::<RunBuilderMarker>()
            .expect("connecting to RunBuilderProxy"),
        vec![test_params],
        new_run_params(),
        None,
        reporter,
        futures::future::pending(),
    )
    .await;

    assert_eq!(outcome, Outcome::Failed);

    const EXPECTED_SYSLOG: &str =  "[TIMESTAMP][PID][TID][<root>][log_and_exit,error_logging_test] DEBUG: Logging initialized\n\
[TIMESTAMP][PID][TID][<root>][log_and_exit,error_logging_test] INFO: my info message\n\
[TIMESTAMP][PID][TID][<root>][log_and_exit,error_logging_test] WARN: my warn message\n\
[TIMESTAMP][PID][TID][<root>][log_and_exit,error_logging_test] ERROR: [../../src/sys/run_test_suite/tests/error_logging_test.rs(12)] my error message\n\
";
    let expected_test_run = ExpectedTestRun::new(directory::Outcome::Failed)
        .with_suite(
            ExpectedSuite::new(
                "fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/error_logging_test.cm",
                directory::Outcome::Failed,
            )
            .with_tag(TestTag::new("internal", "true"))
            .with_case(
                ExpectedTestCase::new("log_and_exit", directory::Outcome::Passed)
                    .with_any_start_time()
                    .with_no_run_duration(),
            )
            .with_matching_artifact(directory::ArtifactType::Syslog, "syslog.txt".into(), |actual| {
                assert_output!(actual.as_bytes(), EXPECTED_SYSLOG);
            })
            .with_matching_artifact(directory::ArtifactType::RestrictedLog, "restricted_logs.txt".into(), |actual| {
                assert!(actual.contains("ERROR: [../../src/sys/run_test_suite/tests/error_logging_test.rs(12)] my error message"))
            })
            .with_any_start_time()
            .with_any_run_duration()
        );

    directory::testing::assert_run_result(output_dir.path(), &expected_test_run);
}

#[fixture::fixture(run_with_reporter)]
#[fuchsia::test]
async fn test_custom_artifacts_to_directory(
    reporter: TestMuxMuxReporter,
    _: TestOutputView,
    output_dir: tempfile::TempDir,
) {
    let reporter = output::RunReporter::new(reporter);
    let test_params = new_test_params(
        "fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/custom_artifact_user.cm",
    );

    let outcome = run_test_suite_lib::run_tests_and_get_outcome(
        fuchsia_component::client::connect_to_protocol::<RunBuilderMarker>()
            .expect("connecting to RunBuilderProxy"),
        vec![test_params],
        new_run_params(),
        None,
        reporter,
        futures::future::pending(),
    )
    .await;

    assert_eq!(outcome, Outcome::Passed);

    let expected_test_run = ExpectedTestRun::new(directory::Outcome::Passed)
        .with_suite(
            ExpectedSuite::new(
                "fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/custom_artifact_user.cm",
                directory::Outcome::Passed,
            )
            .with_tag(TestTag::new("internal", "true"))
            .with_case(
                ExpectedTestCase::new("use_artifact", directory::Outcome::Passed)
                    .with_any_start_time()
                    .with_no_run_duration(),
            )
            .with_artifact(directory::ArtifactType::Syslog, "syslog.txt".into(), "")
            .with_directory_artifact(
                directory::ArtifactMetadata {
                    artifact_type: directory::ArtifactType::Custom.into(),
                    component_moniker: Some(".".to_string()),
                },
                Option::<&str>::None,
                ExpectedDirectory::new().with_file("artifact.txt", "Hello, world!"),
            )
            .with_any_start_time()
            .with_any_run_duration()
        );

    directory::testing::assert_run_result(output_dir.path(), &expected_test_run);
}

#[fixture::fixture(run_with_reporter)]
#[fuchsia::test]
async fn test_terminate_signal(
    reporter: TestMuxMuxReporter,
    _: TestOutputView,
    output_dir: tempfile::TempDir,
) {
    let reporter = output::RunReporter::new(reporter);
    let test_params = new_test_params(
        "fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/long_running_test.cm",
    );

    let outcome = run_test_suite_lib::run_tests_and_get_outcome(
        fuchsia_component::client::connect_to_protocol::<RunBuilderMarker>()
            .expect("connecting to RunBuilderProxy"),
        vec![test_params],
        new_run_params(),
        None,
        reporter,
        futures::future::ready(()),
    )
    .await;

    assert_eq!(outcome, Outcome::Cancelled);

    let run_result = directory::TestRunResult::from_dir(output_dir.path()).expect("parse dir");
    assert_eq!(
        run_result.common.deref().outcome,
        directory::MaybeUnknown::Known(directory::Outcome::Inconclusive)
    );

    // Based on the exact timing, it's possible some test cases were reported, or the suite never
    // started, so manually assert only on the fields that shouldn't vary.
    assert_eq!(run_result.suites.len(), 1);
    let directory::SuiteResult { common, .. } = run_result.suites.into_iter().next().unwrap();
    assert_matches!(
        common.deref().outcome,
        directory::MaybeUnknown::Known(directory::Outcome::Inconclusive)
            | directory::MaybeUnknown::Known(directory::Outcome::NotStarted)
    );
    assert_eq!(
        common.deref().name,
        "fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/long_running_test.cm"
    );

    // TODO(satsukiu): add a `Reporter` implementation for tests that signals on events and use it
    // to make more sophisticated tests. Since we need to make assertions on the directory reporter
    // simultaneously, we'll need to support writing to multiple reporters at once as well.
}

#[fixture::fixture(run_with_reporter)]
#[fuchsia::test]
async fn test_terminate_signal_multiple_suites(
    reporter: TestMuxMuxReporter,
    _: TestOutputView,
    output_dir: tempfile::TempDir,
) {
    let reporter = output::RunReporter::new(reporter);
    let test_params = new_test_params(
        "fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/long_running_test.cm",
    );

    let outcome = run_test_suite_lib::run_tests_and_get_outcome(
        fuchsia_component::client::connect_to_protocol::<RunBuilderMarker>()
            .expect("connecting to RunBuilderProxy"),
        vec![test_params; 10],
        new_run_params(),
        None,
        reporter,
        futures::future::ready(()),
    )
    .await;

    assert_eq!(outcome, Outcome::Cancelled);

    let run_result = directory::TestRunResult::from_dir(output_dir.path()).expect("parse dir");
    assert_eq!(
        run_result.common.deref().outcome,
        directory::MaybeUnknown::Known(directory::Outcome::Inconclusive)
    );

    // There should be at most one test that started and is inconclusive. Remaining tests should
    // be NOT_STARTED. Note this assumes that test manager is running tests sequentially, this test
    // could start flaking when this changes.
    let (inconclusive_suite_results, other_suite_results): (Vec<_>, Vec<_>) =
        run_result.suites.into_iter().partition(|suite| {
            suite.common.deref().outcome
                == directory::MaybeUnknown::Known(directory::Outcome::Inconclusive)
        });
    assert!(inconclusive_suite_results.len() <= 1);
    assert!(other_suite_results.len() >= 9);

    other_suite_results.into_iter().for_each(|suite_result| {
        directory::testing::assert_suite_result(
            output_dir.path(),
            &suite_result,
            &ExpectedSuite::new(
                "fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/long_running_test.cm",
                directory::Outcome::NotStarted,
            )
            .with_tag(TestTag::new("internal", "true")),
        );
    });
}

async fn wait_until_contains(output: &TestOutputView, contents: &str) {
    loop {
        if String::from_utf8(output.lock().clone()).unwrap().contains(contents) {
            break;
        }
        fuchsia_async::Timer::new(std::time::Duration::from_millis(250)).await;
    }
}

#[fixture::fixture(run_with_reporter)]
#[fuchsia::test]
async fn test_collect_stream_artifacts_from_hung_test(
    reporter: TestMuxMuxReporter,
    output: TestOutputView,
    _: tempfile::TempDir,
) {
    // This test verifies that artifacts that need to be shown to a developer in realtime are
    // streamed, even if the test hangs. For example, this test would fail submission of a change
    // that made log collection occur once, after the suite completed.
    const SUITE_NAME: &'static str =
        "fuchsia-pkg://fuchsia.com/run_test_suite_integration_tests#meta/log_then_hang_test.cm";
    let reporter = output::RunReporter::new(reporter);
    let cancel_event = async_utils::event::Event::new();
    let cancel_waiter = cancel_event.wait();

    let run_fut = run_test_suite_lib::run_tests_and_get_outcome(
        fuchsia_component::client::connect_to_protocol::<RunBuilderMarker>()
            .expect("connecting to RunBuilderProxy"),
        vec![new_test_params(SUITE_NAME)],
        new_run_params(),
        None,
        reporter,
        cancel_waiter,
    );
    let observer_fut = async move {
        // check that test started, and we can observe streamed artifacts before run completes
        futures::future::join4(
            wait_until_contains(&output, "[RUNNING]\tlog_then_hang"),
            wait_until_contains(&output, "stdout from hanging test"),
            wait_until_contains(&output, "stderr from hanging test"),
            wait_until_contains(&output, "syslog from hanging test"),
        )
        .await;

        // Verify that the test hasn't finished.
        let contents = String::from_utf8(output.lock().clone()).unwrap();
        assert!(!contents.contains("cancelled before completion"));

        // Cancel the test, then verify test stops
        cancel_event.signal();
        wait_until_contains(&output, "cancelled before completion").await;
        let contents = String::from_utf8(output.lock().clone()).unwrap();
        assert!(contents.contains("cancelled before completion"));
    };
    let (outcome, ()) = futures::future::join(run_fut, observer_fut).await;
    assert_matches!(outcome, Outcome::Cancelled);
}