| #!/usr/bin/env python3 |
| # Copyright 2021 The Fuchsia Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """Hacky script to convert a recipe's dict-style properties to proto properties. |
| |
| Rationale: protos are the more modern and preferred way of declaring properties. |
| They're easier to read and understand and are less of a one-off than dict |
| properties. |
| |
| Usage: `scripts/props_to_proto.py recipes/foo.py` |
| |
| Creates a `recipes/foo.proto` file and updates the recipe to import the proto |
| instead of declaring dict-style properties. The .proto file generation is |
| best-effort, and generated protos should be inspected for correctness before |
| committing (`recipes.py test train` will catch most issues). |
| |
| Limitations: |
| - Default variable values are lost since protos don't provide a way to declare |
| default values. `RunSteps()` must be manually updated to set default property |
| values. |
| - Does not handle all possible property types. If it encounters a property whose |
| `kind` it doesn't know how to handle, it will just use the `kind` as the proto |
| field type, which is generally invalid and will result in an uncompilable |
| proto until someone manually corrects the field type. |
| - Does *not* update variable names or function signatures within the recipe – |
| manual changes will be required to update the signature of RunSteps(). |
| - Does not work for recipe module properties. |
| - May mess up formatting. Make sure to run `black` before committing. |
| - Uses structs for all sub-messages instead of custom message types, which are |
| preferable due to stronger schema guarantees. It's recommended that you |
| replace structs with explicit sub-messages in most cases. |
| - No automatic wrapping of proto field comments. Field comments should be |
| manually wrapped to 80 columns. |
| """ |
| |
| import ast |
| import dataclasses |
| import datetime |
| import os |
| import re |
| import sys |
| |
| |
| @dataclasses.dataclass |
| class ProtoField: |
| # `Help` string for the property. |
| docstring: str |
| # The type that should be used for the proto field. |
| proto_type: str |
| # The name of the property. |
| name: str |
| # The number that should be used for the proto field corresponding to this |
| # property. |
| num: int |
| # Whether the field is repeated or not. |
| repeated: bool |
| |
| |
| def main(filename): |
| with open(filename) as f: |
| contents = f.read() |
| tree = ast.parse(contents) |
| |
| properties = None |
| props_assignment = None |
| for item in tree.body: |
| if isinstance(item, ast.Assign) and item.targets[0].id == "PROPERTIES": |
| if not isinstance(item.value, ast.Dict): |
| print("Recipe already converted to proto properties") |
| return |
| props_assignment = item |
| properties = item.value |
| break |
| |
| if not properties: |
| print("Recipe has no properties") |
| return |
| |
| imports = set() |
| |
| fields = [] |
| for i, (key, val) in enumerate(zip(properties.keys, properties.values)): |
| prop_name = key.s |
| kwargs = {kw.arg: kw.value for kw in val.keywords} |
| |
| kind = ast.unparse(kwargs["kind"]) |
| if "dict" in kind.lower(): |
| imports.add("google/protobuf/struct.proto") |
| |
| repeated = False |
| list_match = re.match(r"^List\((\w+)\)$", kind) |
| if list_match: |
| repeated = True |
| kind = list_match.group(1) |
| proto_type = { # Best effort conversion from Python type to proto type. |
| "str": "string", |
| "bool": "bool", |
| "int": "int32", |
| "dict": "google.protobuf.Struct", |
| "Dict()": "google.protobuf.Struct", |
| }.get(kind, f"FIXME({kind})") |
| |
| docstring = kwargs.get("help", "").s |
| if docstring and not docstring.endswith("."): |
| docstring += "." |
| default = kwargs.get("default") |
| if not default: |
| has_truthy_default = False |
| elif isinstance(default, ast.Name): |
| has_truthy_default = default.id != "None" |
| elif isinstance(default, ast.Constant): |
| has_truthy_default = bool(default.value) |
| elif isinstance(default, (ast.Tuple, ast.List)): |
| has_truthy_default = bool(default.elts) |
| elif isinstance(default, ast.Dict): |
| has_truthy_default = bool(default.keys) |
| else: |
| has_truthy_default = True |
| |
| if has_truthy_default: |
| # pylint: disable=no-member |
| print( |
| f"WARNING: property {prop_name} has default value: {ast.unparse(default)}" |
| ) |
| fields.append( |
| ProtoField( |
| docstring=docstring, |
| proto_type=proto_type, |
| name=prop_name, |
| num=i + 1, |
| repeated=repeated, |
| ) |
| ) |
| |
| lines = [] |
| for field in fields: |
| field_lines = [] |
| if field.docstring: |
| field_lines.append(f"// {field.docstring}") |
| field_lines.append( |
| f"{'repeated ' if field.repeated else ''}{field.proto_type} {field.name} = {field.num};" |
| ) |
| lines.append("\n".join(" " + l for l in field_lines)) |
| |
| message_contents = "\n\n".join(lines) |
| |
| # Remove "recipes" from the beginning to get the name of the recipe as used |
| # by the recipe engine. E.g. "recipes/contrib/foo.py" -> "contrib.foo". |
| recipe_path_parts = filename.split(os.path.sep)[1:] |
| recipe_path_parts[-1] = recipe_path_parts[-1].split(".")[0] |
| recipe_name = ".".join(recipe_path_parts) |
| |
| import_block = "" |
| if imports: |
| import_lines = [f'import "{imp}";' for imp in sorted(imports)] |
| import_block = "\n%s\n" % "\n".join(import_lines) |
| |
| proto_contents = """\ |
| // Copyright %d The Fuchsia Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style license that can be |
| // found in the LICENSE file. |
| |
| syntax = "proto3"; |
| |
| package recipes.fuchsia.%s; |
| %s |
| message InputProperties { |
| %s |
| } |
| """ % ( |
| datetime.datetime.now().year, |
| recipe_name, |
| import_block, |
| message_contents, |
| ) |
| |
| with open(filename.replace(".py", ".proto"), "w") as f: |
| f.write(proto_contents) |
| |
| update_recipe(tree, filename, recipe_name, contents, props_assignment.lineno) |
| |
| |
| def update_recipe(tree, filename, recipe_name, contents, props_line): |
| lines = contents.splitlines() |
| |
| for i in range(props_line, len(lines)): |
| if lines[i].endswith("}"): |
| end_line = i + 1 |
| break |
| # Not sure why we need to subtract one here... |
| lines[props_line - 1 : end_line] = ["PROPERTIES = InputProperties"] |
| |
| import_lineno = 0 |
| for item in tree.body: |
| if not isinstance(item, (ast.Expr, ast.Import, ast.ImportFrom)): |
| break |
| |
| import_lineno = item.lineno |
| |
| lines.insert( |
| import_lineno, f"from PB.recipes.fuchsia.{recipe_name} import InputProperties" |
| ) |
| |
| # Remove imports of types needed for declaring dict-style properties. |
| lines = [ |
| l |
| for l in lines |
| if not re.match(r"from recipe_engine.(recipe_api|config) import", l) |
| ] |
| |
| with open(filename, "w") as f: |
| f.write("".join(l + "\n" for l in lines)) |
| |
| |
| if __name__ == "__main__": |
| main(sys.argv[1]) |