recipes/contrib/upload_yaml_bigquery.py

# Copyright 2022 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""
Recipe to upload a YAML file to a bigquery table.

Table should have this schema:
  field     type      Comment
  timestamp TIMESTAMP Commit timestamp
  revision  STRING    Commit hash
  data      STRING    Parsed data
"""
from recipe_engine.recipe_api import Property
from PB.recipes.fuchsia.contrib.upload_yaml_bigquery import InputProperties

DEPS = [
    "fuchsia/bqupload",
    "fuchsia/buildbucket_util",
    "fuchsia/git",
    "fuchsia/git_checkout",
    "fuchsia/yaml",
    "recipe_engine/context",
    "recipe_engine/json",
    "recipe_engine/properties",
    "recipe_engine/raw_io",
]

PROPERTIES = InputProperties


def RunSteps(api, props):
    git_path, revision = api.git_checkout(props.repo)
    with api.context(cwd=git_path):
        timestamp = api.git(
            "commit timetsamp",
            "log",
            revision,
            "-1",
            "--format=%cI",
            stdout=api.raw_io.output_text(),
        ).stdout.rstrip()
        data_dict = api.yaml.read_file(
            "read " + props.yaml_path, props.yaml_path, test_data={"foo": "bar"}
        )
    data_str = api.json.dumps(data_dict, indent=2)
    rows = [{"timestamp": timestamp, "revision": revision, "data": data_str}]
    api.bqupload.insert(
        "upload to bigquery",
        project=props.bq_project,
        dataset=props.bq_dataset,
        table=props.bq_table,
        rows=rows,
    )


def GenTests(api):
    repo = "https://fuchsia.googlesource.com/infra/salt"
    yaml_path = "pillar/generated/crimson.sls"
    bq_project = "fuchsia-infra-monitoring-dev"
    bq_dataset = "salt_master_monitoring"
    bq_table = "crimson"
    yield (
        api.buildbucket_util.test("success", git_repo=repo)
        + api.properties(
            repo=repo,
            yaml_path=yaml_path,
            bq_project=bq_project,
            bq_dataset=bq_dataset,
            bq_table=bq_table,
        )
    )