blob: 7a1ceebfe038984189c6ccde69a7708a52f20697 [file] [log] [blame]
#!/usr/bin/env python3
# Copyright 2021 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Hacky script to convert a recipe's dict-style properties to proto properties.
Rationale: protos are the more modern and preferred way of declaring properties.
They're easier to read and understand and are less of a one-off than dict
Usage: `scripts/ recipes/`
Creates a `recipes/foo.proto` file and updates the recipe to import the proto
instead of declaring dict-style properties. The .proto file generation is
best-effort, and generated protos should be inspected for correctness before
committing (` test train` will catch most issues).
- Default variable values are lost since protos don't provide a way to declare
default values. `RunSteps()` must be manually updated to set default property
- Does not handle all possible property types. If it encounters a property whose
`kind` it doesn't know how to handle, it will just use the `kind` as the proto
field type, which is generally invalid and will result in an uncompilable
proto until someone manually corrects the field type.
- Does *not* update variable names or function signatures within the recipe –
manual changes will be required to update the signature of RunSteps().
- Does not work for recipe module properties.
- May mess up formatting. Make sure to run `black` before committing.
- Uses structs for all sub-messages instead of custom message types, which are
preferable due to stronger schema guarantees. It's recommended that you
replace structs with explicit sub-messages in most cases.
- No automatic wrapping of proto field comments. Field comments should be
manually wrapped to 80 columns.
import ast
import dataclasses
import datetime
import os
import re
import sys
class ProtoField:
# `Help` string for the property.
docstring: str
# The type that should be used for the proto field.
proto_type: str
# The name of the property.
name: str
# The number that should be used for the proto field corresponding to this
# property.
num: int
# Whether the field is repeated or not.
repeated: bool
def main(filename):
with open(filename) as f:
contents =
tree = ast.parse(contents)
properties = None
props_assignment = None
for item in tree.body:
if isinstance(item, ast.Assign) and item.targets[0].id == "PROPERTIES":
if not isinstance(item.value, ast.Dict):
print("Recipe already converted to proto properties")
props_assignment = item
properties = item.value
if not properties:
print("Recipe has no properties")
imports = set()
fields = []
for i, (key, val) in enumerate(zip(properties.keys, properties.values)):
prop_name = key.s
kwargs = {kw.arg: kw.value for kw in val.keywords}
kind = ast.unparse(kwargs["kind"])
if "dict" in kind.lower():
repeated = False
list_match = re.match(r"^List\((\w+)\)$", kind)
if list_match:
repeated = True
kind =
proto_type = { # Best effort conversion from Python type to proto type.
"str": "string",
"bool": "bool",
"int": "int32",
"dict": "google.protobuf.Struct",
"Dict()": "google.protobuf.Struct",
}.get(kind, "FIXME(%s)" % kind)
docstring = kwargs.get("help", "").s
if docstring and not docstring.endswith("."):
docstring += "."
default = kwargs.get("default")
if not default:
has_truthy_default = False
elif isinstance(default, ast.Name):
has_truthy_default = != "None"
elif isinstance(default, ast.Constant):
has_truthy_default = bool(default.value)
elif isinstance(default, (ast.Tuple, ast.List)):
has_truthy_default = bool(default.elts)
elif isinstance(default, ast.Dict):
has_truthy_default = bool(default.keys)
has_truthy_default = True
if has_truthy_default:
"WARNING: property %s has default value: %s"
% (prop_name, ast.unparse(default))
num=i + 1,
lines = []
for field in fields:
field_lines = []
if field.docstring:
field_lines.append("// %s" % field.docstring)
"%s%s %s = %s;"
% (
"repeated " if field.repeated else "",
lines.append("\n".join(" " + l for l in field_lines))
message_contents = "\n\n".join(lines)
# Remove "recipes" from the beginning to get the name of the recipe as used
# by the recipe engine. E.g. "recipes/contrib/" -> "".
recipe_path_parts = filename.split(os.path.sep)[1:]
recipe_path_parts[-1] = recipe_path_parts[-1].split(".")[0]
recipe_name = ".".join(recipe_path_parts)
import_block = ""
if imports:
import_lines = ['import "%s";' % imp for imp in sorted(imports)]
import_block = "\n%s\n" % "\n".join(import_lines)
proto_contents = """\
// Copyright %d The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
syntax = "proto3";
package recipes.fuchsia.%s;
message InputProperties {
""" % (,
with open(filename.replace(".py", ".proto"), "w") as f:
update_recipe(tree, filename, recipe_name, contents, props_assignment.lineno)
def update_recipe(tree, filename, recipe_name, contents, props_line):
lines = contents.splitlines()
for i in range(props_line, len(lines)):
if lines[i].endswith("}"):
end_line = i + 1
# Not sure why we need to subtract one here...
lines[props_line - 1 : end_line] = ["PROPERTIES = InputProperties"]
import_lineno = 0
for item in tree.body:
if not isinstance(item, (ast.Expr, ast.Import, ast.ImportFrom)):
import_lineno = item.lineno
import_lineno, "from import InputProperties" % recipe_name
# Remove imports of types needed for declaring dict-style properties.
lines = [
for l in lines
if not re.match(r"from recipe_engine.(recipe_api|config) import", l)
with open(filename, "w") as f:
f.write("".join(l + "\n" for l in lines))
if __name__ == "__main__":