recipes/fuchsia/determinism_check.py

# Copyright 2021 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""determinism_check.py - Runs two builds with the same checkout and compares
                          their output to validate the build produces
                          deterministic blobs.
"""

from PB.recipes.fuchsia.fuchsia.determinism_check import InputProperties

DEPS = [
    "fuchsia/build",
    "fuchsia/buildbucket_util",
    "fuchsia/checkout",
    "recipe_engine/file",
    "recipe_engine/path",
    "recipe_engine/properties",
    "recipe_engine/step",
]

PROPERTIES = InputProperties


def RunSteps(api, props):
    checkout = api.checkout.fuchsia_with_options(
        manifest=props.manifest,
        remote=props.remote,
    )

    with api.step.nest("build 1"):
        b1_blobs = run_build(api, checkout, props)

    with api.step.nest("build 2"):
        b2_blobs = run_build(api, checkout, props)

    with api.step.nest("compare blobs"):
        compare_blobs(api, b1_blobs, b2_blobs)


def run_build(api, checkout, props):
    """Performs a single build of fuchsia and retrieves blobs.json."""
    build_results = api.build.with_options(
        checkout, fint_params_path=props.fint_params_path
    )

    new_build_dir = api.path.mkdtemp()
    api.file.move(
        "move outdir to %s" % new_build_dir, build_results.build_dir, new_build_dir
    )
    blobs_path = api.path.join(
        new_build_dir, api.path.basename(build_results.build_dir), "blobs.json"
    )
    return api.file.read_json("reading blobs", blobs_path)


def compare_blobs(api, b1_blobs, b2_blobs):
    """Compares blob hashes between the two builds."""
    b1_blob_map = {blob["source_path"]: blob["merkle"] for blob in b1_blobs}
    b2_blob_map = {blob["source_path"]: blob["merkle"] for blob in b2_blobs}

    diffs = {}
    for blob in b1_blobs:
        path = blob["source_path"]
        b1_merkle = blob["merkle"]
        b2_merkle = b2_blob_map.get(path, "not found")

        if b1_merkle != b2_merkle:
            diffs[path] = {
                "b1_merkle": b1_merkle,
                "b2_merkle": b2_merkle,
            }

    for blob in b2_blobs:
        path = blob["source_path"]
        b2_merkle = blob["merkle"]
        b1_merkle = b1_blob_map.get(path, "not found")

        if b1_merkle != b2_merkle and not diffs.get(path):
            diffs[path] = {
                "b1_merkle": b1_merkle,
                "b2_merkle": b2_merkle,
            }

    if diffs:
        diffs_path = api.path.join(api.path.mkdtemp(), "diffs.json")
        api.file.write_json("write diffs.json", diffs_path, diffs)
        raise api.step.StepFailure(
            "blob hashes were different on %d paths, see diffs.json" % len(diffs)
        )


def GenTests(api):
    properties = {
        "manifest": "minimal",
        "project": "integration",
        "remote": "https://fuchsia.googlesource.com/manifest",
        "fint_params_path": "specs/core.fint.textproto",
    }

    # Test case for deterministic builds.
    yield (
        api.buildbucket_util.test("deterministic_build", tryjob=False)
        + api.step_data(
            "build 1.reading blobs",
            api.file.read_json([{"source_path": "/foo/bar", "merkle": "hash"}]),
        )
        + api.step_data(
            "build 2.reading blobs",
            api.file.read_json([{"source_path": "/foo/bar", "merkle": "hash"}]),
        )
        + api.properties(**properties)
    )

    # Test case where builds generate different numbers of blobs.
    yield (
        api.buildbucket_util.test(
            "different_number_of_blobs", tryjob=False, status="failure"
        )
        + api.step_data("build 1.reading blobs", api.file.read_json([]))
        + api.step_data(
            "build 2.reading blobs",
            api.file.read_json([{"source_path": "/foo/bar", "merkle": "hash"}]),
        )
        + api.properties(**properties)
    )

    # Test case where build artifacts differ.
    yield (
        api.buildbucket_util.test("different_hashes", tryjob=False, status="failure")
        + api.step_data(
            "build 1.reading blobs",
            api.file.read_json([{"source_path": "/foo/bar", "merkle": "hash"}]),
        )
        + api.step_data(
            "build 2.reading blobs",
            api.file.read_json([{"source_path": "/foo/bar", "merkle": "hash2"}]),
        )
        + api.properties(**properties)
    )