src/developer/ffx/plugins/test/src/pilot.rs

// Copyright 2024 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

use anyhow::{Result, anyhow};
use serde::Deserialize;
use std::collections::HashMap;
use std::fs;
use std::io::BufReader;
use std::path::Path;
use test_pilot_lib::test_output::{
    OutputDirectory, Summary, SummaryArtifact, SummaryCase, SummaryCommonProperties,
    SummaryOutcome, SummaryOutcomeResult,
};

const OLD_RUN_SUMMARY_FILE_NAME: &str = "run_summary.json";

/// Converts an output directory generated by run_test_suite_lib into a form compatible with
/// test_pilot.
pub fn convert_output_for_test_pilot(output_directory: &Path) -> Result<()> {
    let output_directory = OutputDirectory::new(output_directory);

    let run_summary_path = output_directory.join(OLD_RUN_SUMMARY_FILE_NAME);

    let file = fs::File::open(&run_summary_path)?;
    let mut reader = BufReader::new(file);
    let run_summary_envelope: RunSummaryEnvelope = serde_json::from_reader(&mut reader)?;

    let run_summary = &run_summary_envelope.data;

    // We only support one suite per test run.
    if run_summary.suites.len() != 1 {
        return Err(anyhow!(
            "{0} contains {1} suite runs, expected 1",
            OLD_RUN_SUMMARY_FILE_NAME,
            run_summary.suites.len(),
        ));
    }

    let suite = &run_summary.suites[0];

    let mut new_summary = Summary {
        common: SummaryCommonProperties {
            duration: suite.entity_common.duration_milliseconds,
            outcome: SummaryOutcome {
                result: summary_result_from(&suite.entity_common.outcome),
                ..Default::default()
            },
            ..Default::default()
        },
        ..Default::default()
    };

    // Create the test directory.
    let new_test_dir_path = output_directory.test_subdir();
    fs::create_dir_all(new_test_dir_path.clone())?;

    // Handle suite artifacts.
    let new_test_dir_rel_path = output_directory
        .make_relative(&new_test_dir_path)
        .expect("test dir path is in output_directory");

    if suite.entity_common.artifact_dir.is_empty() {
        if !suite.entity_common.artifacts.is_empty() {
            return Err(anyhow!("suite has artifacts but no artifact directory"));
        }
    } else {
        let old_artifact_dir_path =
            output_directory.join(suite.entity_common.artifact_dir.as_str());

        for (name, metadata) in &suite.entity_common.artifacts {
            let mut new_file_rel_path = new_test_dir_rel_path.to_path_buf();
            new_file_rel_path.push(name);

            new_summary.common.artifacts.insert(
                new_file_rel_path,
                SummaryArtifact { artifact_type: metadata.artifact_type.clone() },
            );

            // Move the artifact to <root>/test.
            let mut old_file_path = old_artifact_dir_path.clone();
            old_file_path.push(name);

            let mut new_file_path = new_test_dir_path.clone();
            new_file_path.push(name);

            fs::rename(old_file_path, new_file_path)?;
        }

        if old_artifact_dir_path.exists() {
            fs::remove_dir_all(old_artifact_dir_path)?;
        }
    }

    let mut case_outcomes_merged =
        SummaryOutcome { result: SummaryOutcomeResult::Passed, detail: None };

    // Handle the cases.
    for old_case in &suite.cases {
        let mut case = SummaryCase {
            common: SummaryCommonProperties {
                duration: old_case.entity_common.duration_milliseconds,
                outcome: SummaryOutcome {
                    result: summary_result_from(&old_case.entity_common.outcome),
                    ..Default::default()
                },
                ..Default::default()
            },
        };

        case_outcomes_merged.merge_case_outcome(case.common.outcome.clone());

        if old_case.entity_common.artifact_dir.is_empty() {
            if !old_case.entity_common.artifacts.is_empty() {
                return Err(anyhow!(
                    "case {} has artifacts but no artifact directory",
                    old_case.name.as_str()
                ));
            }
        } else {
            let old_artifact_dir_path =
                output_directory.join(old_case.entity_common.artifact_dir.as_str());

            let mut new_artifact_dir_path = new_test_dir_path.clone();
            new_artifact_dir_path.push(old_case.name.as_str());

            if !old_case.entity_common.artifacts.is_empty() {
                // Create the new artifact dir at <root>/test/<case name>.
                fs::create_dir(new_artifact_dir_path.clone())?;
            }

            for (name, metadata) in &old_case.entity_common.artifacts {
                let mut new_file_rel_path = new_test_dir_rel_path.clone();
                new_file_rel_path.push(old_case.name.as_str());
                new_file_rel_path.push(name);

                case.common.artifacts.insert(
                    new_file_rel_path,
                    SummaryArtifact { artifact_type: metadata.artifact_type.clone() },
                );

                // Move the artifact to <root>/target/<case name>.
                let mut old_file_path = old_artifact_dir_path.clone();
                old_file_path.push(name);

                let mut new_file_path = new_artifact_dir_path.clone();
                new_file_path.push(name);

                fs::rename(old_file_path, new_file_path)?;
            }

            if old_artifact_dir_path.exists() {
                fs::remove_dir_all(old_artifact_dir_path)?;
            }
        }

        new_summary.cases.insert(old_case.name.clone(), case);
    }

    if case_outcomes_merged != new_summary.common.outcome {
        eprintln!(
            "WARN: merged case outcome {:?} differs from original suite outcome {:?}, using merged outcome",
            case_outcomes_merged, new_summary.common.outcome
        );

        new_summary.common.outcome = case_outcomes_merged;
    }

    fs::write(
        output_directory.test_summary().to_str().unwrap(),
        serde_json::to_string_pretty(&new_summary).unwrap(),
    )?;

    fs::remove_file(run_summary_path)?;

    Ok(())
}

fn summary_result_from(old_result: &str) -> SummaryOutcomeResult {
    match old_result {
        "SKIPPED" => SummaryOutcomeResult::Skipped,
        "PASSED" => SummaryOutcomeResult::Passed,
        "FAILED" => SummaryOutcomeResult::Failed,
        "TIMED_OUT" => SummaryOutcomeResult::TimedOut,
        "ERROR" => SummaryOutcomeResult::Error,
        "FINISHED" => SummaryOutcomeResult::Passed,
        "DID_NOT_FINISH" => SummaryOutcomeResult::Canceled,
        "STOPPED" => SummaryOutcomeResult::Canceled,
        "INTERNAL_ERROR" => SummaryOutcomeResult::Error,
        _ => {
            eprintln!("UNRECOGNIZED LEGACY RESULT {}", old_result);
            SummaryOutcomeResult::Error
        }
    }
}

/// Top level of the legacy run_summary.json. Identifies the schema.
#[derive(Deserialize, Debug, Default)]
struct RunSummaryEnvelope {
    data: RunSummary,

    #[serde(rename = "schema_id")]
    _schema_id: String,
}

/// The body of the summary. The `entity_common` bit refers to the run.
#[derive(Deserialize, Debug, Default)]
struct RunSummary {
    #[serde(flatten)]
    #[serde(rename = "entity_common")]
    _entity_common: RunSummaryEntityCommon,

    suites: Vec<RunSummarySuiteEntry>,
}

/// Properties of the run, a suite, and a case.
#[derive(Deserialize, Debug, Default)]
struct RunSummaryEntityCommon {
    #[serde(default)]
    artifacts: HashMap<String, RunSummaryArtifactMetadata>,

    #[serde(default)]
    artifact_dir: String,

    #[serde(default)]
    duration_milliseconds: i64,

    // This is the name actually used in summary files, as opposed to the name in the schema, which
    // is start_time_milliseconds.
    #[serde(default)]
    #[serde(rename = "start_time")]
    _start_time: i64,

    outcome: String,
}

/// Describes an artifact.
#[derive(Deserialize, Debug, Default)]
struct RunSummaryArtifactMetadata {
    artifact_type: String,

    #[serde(default)]
    #[serde(rename = "component_moniker")]
    _component_moniker: String,
}

/// Describes a case.
#[derive(Deserialize, Debug, Default)]
struct RunSummaryCaseEntry {
    #[serde(flatten)]
    entity_common: RunSummaryEntityCommon,

    name: String,
}

/// Describes a suite.
#[derive(Deserialize, Debug, Default)]
struct RunSummarySuiteEntry {
    #[serde(flatten)]
    entity_common: RunSummaryEntityCommon,

    cases: Vec<RunSummaryCaseEntry>,

    #[serde(rename = "name")]
    _name: String,
}

#[cfg(test)]
mod tests {
    use super::*;
    use serde_json::json;
    use std::fs;
    use tempfile::tempdir;

    #[test]
    fn test_convert_output() {
        // Setup
        let tmp_dir = tempdir().unwrap();
        let output_dir = tmp_dir.path();

        // Create old run_summary.json
        let run_summary_content = json!({
            "schema_id": "schema_id_string",
            "data": {
                "outcome": "FAILED",
                "suites": [
                    {
                        "name": "suite_name",
                        "outcome": "FAILED",
                        "duration_milliseconds": 123,
                        "artifact_dir": "suite_artifacts",
                        "artifacts": {
                            "stdout.txt": {
                                "artifact_type": "stdout"
                            }
                        },
                        "cases": [
                            {
                                "name": "case1",
                                "outcome": "PASSED",
                                "duration_milliseconds": 100,
                                "artifact_dir": "case1_artifacts",
                                "artifacts": {
                                    "screenshot.png": {
                                        "artifact_type": "screenshot"
                                    }
                                }
                            },
                            {
                                "name": "case2",
                                "outcome": "FAILED",
                                "duration_milliseconds": 23,
                                "artifacts": {},
                                "artifact_dir": "case2_artifacts"
                            }
                        ]
                    }
                ]
            }
        });

        let run_summary_path = output_dir.join(OLD_RUN_SUMMARY_FILE_NAME);
        fs::write(&run_summary_path, serde_json::to_string(&run_summary_content).unwrap()).unwrap();

        // Create artifact directories and files
        let suite_artifact_dir = output_dir.join("suite_artifacts");
        fs::create_dir(&suite_artifact_dir).unwrap();
        fs::write(suite_artifact_dir.join("stdout.txt"), "suite stdout").unwrap();

        let case1_artifact_dir = output_dir.join("case1_artifacts");
        fs::create_dir(&case1_artifact_dir).unwrap();
        fs::write(case1_artifact_dir.join("screenshot.png"), "fake png").unwrap();

        let case2_artifact_dir = output_dir.join("case2_artifacts");
        fs::create_dir(&case2_artifact_dir).unwrap();

        // Run conversion
        let result = convert_output_for_test_pilot(output_dir);
        assert!(result.is_ok());

        // Assertions
        // 1. old run_summary.json is gone
        assert!(!run_summary_path.exists());

        // 2. new summary.json exists and is correct
        let new_summary_path = output_dir.join("test/output_summary.json");
        assert!(new_summary_path.exists());

        let new_summary_content = fs::read_to_string(&new_summary_path).unwrap();
        let new_summary: Summary = serde_json::from_str(&new_summary_content).unwrap();

        assert_eq!(new_summary.common.duration, 123);
        assert_eq!(new_summary.common.outcome.result, SummaryOutcomeResult::Failed);
        assert_eq!(new_summary.common.artifacts.len(), 1);
        assert!(new_summary.common.artifacts.contains_key(Path::new("test/stdout.txt")));

        assert_eq!(new_summary.cases.len(), 2);

        let case1 = new_summary.cases.get("case1").unwrap();
        assert_eq!(case1.common.duration, 100);
        assert_eq!(case1.common.outcome.result, SummaryOutcomeResult::Passed);
        assert_eq!(case1.common.artifacts.len(), 1);
        assert!(case1.common.artifacts.contains_key(Path::new("test/case1/screenshot.png")));

        let case2 = new_summary.cases.get("case2").unwrap();
        assert_eq!(case2.common.duration, 23);
        assert_eq!(case2.common.outcome.result, SummaryOutcomeResult::Failed);
        assert!(case2.common.artifacts.is_empty());

        // 3. artifacts are moved
        let new_suite_artifact_path = output_dir.join("test/stdout.txt");
        assert!(new_suite_artifact_path.exists());
        assert_eq!(fs::read_to_string(new_suite_artifact_path).unwrap(), "suite stdout");

        let new_case1_artifact_path = output_dir.join("test/case1/screenshot.png");
        assert!(new_case1_artifact_path.exists());
        assert_eq!(fs::read_to_string(new_case1_artifact_path).unwrap(), "fake png");

        // 4. old artifact directories are gone
        assert!(!suite_artifact_dir.exists());
        assert!(!case1_artifact_dir.exists());
        assert!(!case2_artifact_dir.exists()); // This one was empty, should be removed.
    }

    #[test]
    fn test_convert_no_cases_no_suite_artifacts() {
        let tmp_dir = tempdir().unwrap();
        let output_dir = tmp_dir.path();

        let run_summary_content = json!({
            "schema_id": "schema_id_string",
            "data": {
                "outcome": "PASSED",
                "suites": [
                    {
                        "name": "suite_name",
                        "outcome": "PASSED",
                        "duration_milliseconds": 42,
                        "cases": []
                    }
                ]
            }
        });

        let run_summary_path = output_dir.join(OLD_RUN_SUMMARY_FILE_NAME);
        fs::write(&run_summary_path, serde_json::to_string(&run_summary_content).unwrap()).unwrap();

        let result = convert_output_for_test_pilot(output_dir);
        assert!(result.is_ok());

        assert!(!run_summary_path.exists());

        let new_summary_path = output_dir.join("test/output_summary.json");
        assert!(new_summary_path.exists());

        let new_summary: Summary =
            serde_json::from_str(&fs::read_to_string(new_summary_path).unwrap()).unwrap();

        assert_eq!(new_summary.common.duration, 42);
        assert_eq!(new_summary.common.outcome.result, SummaryOutcomeResult::Passed);
        assert!(new_summary.common.artifacts.is_empty());
        assert!(new_summary.cases.is_empty());

        assert!(output_dir.exists());
    }

    #[test]
    fn test_convert_multiple_suites_is_error() {
        let tmp_dir = tempdir().unwrap();
        let output_dir = tmp_dir.path();

        let run_summary_content = json!({
            "schema_id": "schema_id_string",
            "data": {
                "outcome": "PASSED",
                "suites": [
                    { "name": "suite1", "outcome": "PASSED", "cases": [] },
                    { "name": "suite2", "outcome": "PASSED", "cases": [] }
                ]
            }
        });

        let run_summary_path = output_dir.join(OLD_RUN_SUMMARY_FILE_NAME);
        fs::write(&run_summary_path, serde_json::to_string(&run_summary_content).unwrap()).unwrap();

        let result = convert_output_for_test_pilot(output_dir);
        assert!(result.is_err());
        assert_eq!(
            result.unwrap_err().to_string(),
            format!("{0} contains 2 suite runs, expected 1", OLD_RUN_SUMMARY_FILE_NAME)
        );
    }

    #[test]
    fn test_summary_result_from() {
        assert_eq!(summary_result_from("SKIPPED"), SummaryOutcomeResult::Skipped);
        assert_eq!(summary_result_from("PASSED"), SummaryOutcomeResult::Passed);
        assert_eq!(summary_result_from("FAILED"), SummaryOutcomeResult::Failed);
        assert_eq!(summary_result_from("TIMED_OUT"), SummaryOutcomeResult::TimedOut);
        assert_eq!(summary_result_from("ERROR"), SummaryOutcomeResult::Error);
        assert_eq!(summary_result_from("FINISHED"), SummaryOutcomeResult::Passed);
        assert_eq!(summary_result_from("DID_NOT_FINISH"), SummaryOutcomeResult::Canceled);
        assert_eq!(summary_result_from("STOPPED"), SummaryOutcomeResult::Canceled);
        assert_eq!(summary_result_from("INTERNAL_ERROR"), SummaryOutcomeResult::Error);
        assert_eq!(summary_result_from("UNRECOGNIZED"), SummaryOutcomeResult::Error);
    }
}