scripts/props_to_proto.py - infra/recipes.git - Git at Google

 #!/usr/bin/env python3
 # Copyright 2021 The Fuchsia Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.

 """Hacky script to convert a recipe's dict-style properties to proto properties.

 Rationale: protos are the more modern and preferred way of declaring properties.
 They're easier to read and understand and are less of a one-off than dict
 properties.

 Usage: `scripts/props_to_proto.py recipes/foo.py`

 Creates a `recipes/foo.proto` file and updates the recipe to import the proto
 instead of declaring dict-style properties. The .proto file generation is
 best-effort, and generated protos should be inspected for correctness before
 committing (`recipes.py test train` will catch most issues).

 Limitations:
 - Default variable values are lost since protos don't provide a way to declare
   default values. `RunSteps()` must be manually updated to set default property
   values.
 - Does not handle all possible property types. If it encounters a property whose
   `kind` it doesn't know how to handle, it will just use the `kind` as the proto
   field type, which is generally invalid and will result in an uncompilable
   proto until someone manually corrects the field type.
 - Does *not* update variable names or function signatures within the recipe –
   manual changes will be required to update the signature of RunSteps().
 - Does not work for recipe module properties.
 - May mess up formatting. Make sure to run `black` before committing.
 - Uses structs for all sub-messages instead of custom message types, which are
   preferable due to stronger schema guarantees. It's recommended that you
   replace structs with explicit sub-messages in most cases.
 - No automatic wrapping of proto field comments. Field comments should be
   manually wrapped to 80 columns.
 """

 import ast
 import dataclasses
 import datetime
 import os
 import re
 import sys


 @dataclasses.dataclass
 class ProtoField:
     # `Help` string for the property.
     docstring: str
     # The type that should be used for the proto field.
     proto_type: str
     # The name of the property.
     name: str
     # The number that should be used for the proto field corresponding to this
     # property.
     num: int
     # Whether the field is repeated or not.
     repeated: bool


 def main(filename):
     with open(filename) as f:
         contents = f.read()
         tree = ast.parse(contents)

     properties = None
     props_assignment = None
     for item in tree.body:
         if isinstance(item, ast.Assign) and item.targets[0].id == "PROPERTIES":
             if not isinstance(item.value, ast.Dict):
                 print("Recipe already converted to proto properties")
                 return
             props_assignment = item
             properties = item.value
             break

     if not properties:
         print("Recipe has no properties")
         return

     imports = set()

     fields = []
     for i, (key, val) in enumerate(zip(properties.keys, properties.values)):
         prop_name = key.s
         kwargs = {kw.arg: kw.value for kw in val.keywords}

         kind = ast.unparse(kwargs["kind"])
         if "dict" in kind.lower():
             imports.add("google/protobuf/struct.proto")

         repeated = False
         list_match = re.match(r"^List\((\w+)\)$", kind)
         if list_match:
             repeated = True
             kind = list_match.group(1)
         proto_type = {  # Best effort conversion from Python type to proto type.
             "str": "string",
             "bool": "bool",
             "int": "int32",
             "dict": "google.protobuf.Struct",
             "Dict()": "google.protobuf.Struct",
         }.get(kind, "FIXME(%s)" % kind)

         docstring = kwargs.get("help", "").s
         if docstring and not docstring.endswith("."):
             docstring += "."
         default = kwargs.get("default")
         if not default:
             has_truthy_default = False
         elif isinstance(default, ast.Name):
             has_truthy_default = default.id != "None"
         elif isinstance(default, ast.Constant):
             has_truthy_default = bool(default.value)
         elif isinstance(default, (ast.Tuple, ast.List)):
             has_truthy_default = bool(default.elts)
         elif isinstance(default, ast.Dict):
             has_truthy_default = bool(default.keys)
         else:
             has_truthy_default = True

         if has_truthy_default:
             print(
                 "WARNING: property %s has default value: %s"
                 % (prop_name, ast.unparse(default))
             )
         fields.append(
             ProtoField(
                 docstring=docstring,
                 proto_type=proto_type,
                 name=prop_name,
                 num=i + 1,
                 repeated=repeated,
             )
         )

     lines = []
     for field in fields:
         field_lines = []
         if field.docstring:
             field_lines.append("// %s" % field.docstring)
         field_lines.append(
             "%s%s %s = %s;"
             % (
                 "repeated " if field.repeated else "",
                 field.proto_type,
                 field.name,
                 field.num,
             )
         )
         lines.append("\n".join("  " + l for l in field_lines))

     message_contents = "\n\n".join(lines)

     # Remove "recipes" from the beginning to get the name of the recipe as used
     # by the recipe engine. E.g.  "recipes/contrib/foo.py" -> "contrib.foo".
     recipe_path_parts = filename.split(os.path.sep)[1:]
     recipe_path_parts[-1] = recipe_path_parts[-1].split(".")[0]
     recipe_name = ".".join(recipe_path_parts)

     import_block = ""
     if imports:
         import_lines = ['import "%s";' % imp for imp in sorted(imports)]
         import_block = "\n%s\n" % "\n".join(import_lines)

     proto_contents = """\
 // Copyright %d The Fuchsia Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.

 syntax = "proto3";

 package recipes.fuchsia.%s;
 %s
 message InputProperties {
 %s
 }
 """ % (
         datetime.datetime.now().year,
         recipe_name,
         import_block,
         message_contents,
     )

     with open(filename.replace(".py", ".proto"), "w") as f:
         f.write(proto_contents)

     update_recipe(tree, filename, recipe_name, contents, props_assignment.lineno)


 def update_recipe(tree, filename, recipe_name, contents, props_line):
     lines = contents.splitlines()

     for i in range(props_line, len(lines)):
         if lines[i].endswith("}"):
             end_line = i + 1
             break
     # Not sure why we need to subtract one here...
     lines[props_line - 1 : end_line] = ["PROPERTIES = InputProperties"]

     import_lineno = 0
     for item in tree.body:
         if not isinstance(item, (ast.Expr, ast.Import, ast.ImportFrom)):
             break

         import_lineno = item.lineno

     lines.insert(
         import_lineno, "from PB.recipes.fuchsia.%s import InputProperties" % recipe_name
     )

     # Remove imports of types needed for declaring dict-style properties.
     lines = [
         l
         for l in lines
         if not re.match(r"from recipe_engine.(recipe_api|config) import", l)
     ]

     with open(filename, "w") as f:
         f.write("".join(l + "\n" for l in lines))


 if __name__ == "__main__":
     main(sys.argv[1])
	#!/usr/bin/env python3
	# Copyright 2021 The Fuchsia Authors. All rights reserved.
	# Use of this source code is governed by a BSD-style license that can be
	# found in the LICENSE file.

	"""Hacky script to convert a recipe's dict-style properties to proto properties.

	Rationale: protos are the more modern and preferred way of declaring properties.
	They're easier to read and understand and are less of a one-off than dict
	properties.

	Usage: `scripts/props_to_proto.py recipes/foo.py`

	Creates a `recipes/foo.proto` file and updates the recipe to import the proto
	instead of declaring dict-style properties. The .proto file generation is
	best-effort, and generated protos should be inspected for correctness before
	committing (`recipes.py test train` will catch most issues).

	Limitations:
	- Default variable values are lost since protos don't provide a way to declare
	default values. `RunSteps()` must be manually updated to set default property
	values.
	- Does not handle all possible property types. If it encounters a property whose
	`kind` it doesn't know how to handle, it will just use the `kind` as the proto
	field type, which is generally invalid and will result in an uncompilable
	proto until someone manually corrects the field type.
	- Does not update variable names or function signatures within the recipe –
	manual changes will be required to update the signature of RunSteps().
	- Does not work for recipe module properties.
	- May mess up formatting. Make sure to run `black` before committing.
	- Uses structs for all sub-messages instead of custom message types, which are
	preferable due to stronger schema guarantees. It's recommended that you
	replace structs with explicit sub-messages in most cases.
	- No automatic wrapping of proto field comments. Field comments should be
	manually wrapped to 80 columns.
	"""

	import ast
	import dataclasses
	import datetime
	import os
	import re
	import sys


	@dataclasses.dataclass
	class ProtoField:
	# `Help` string for the property.
	docstring: str
	# The type that should be used for the proto field.
	proto_type: str
	# The name of the property.
	name: str
	# The number that should be used for the proto field corresponding to this
	# property.
	num: int
	# Whether the field is repeated or not.
	repeated: bool


	def main(filename):
	with open(filename) as f:
	contents = f.read()
	tree = ast.parse(contents)

	properties = None
	props_assignment = None
	for item in tree.body:
	if isinstance(item, ast.Assign) and item.targets[0].id == "PROPERTIES":
	if not isinstance(item.value, ast.Dict):
	print("Recipe already converted to proto properties")
	return
	props_assignment = item
	properties = item.value
	break

	if not properties:
	print("Recipe has no properties")
	return

	imports = set()

	fields = []
	for i, (key, val) in enumerate(zip(properties.keys, properties.values)):
	prop_name = key.s
	kwargs = {kw.arg: kw.value for kw in val.keywords}

	kind = ast.unparse(kwargs["kind"])
	if "dict" in kind.lower():
	imports.add("google/protobuf/struct.proto")

	repeated = False
	list_match = re.match(r"^List\((\w+)\)$", kind)
	if list_match:
	repeated = True
	kind = list_match.group(1)
	proto_type = { # Best effort conversion from Python type to proto type.
	"str": "string",
	"bool": "bool",
	"int": "int32",
	"dict": "google.protobuf.Struct",
	"Dict()": "google.protobuf.Struct",
	}.get(kind, "FIXME(%s)" % kind)

	docstring = kwargs.get("help", "").s
	if docstring and not docstring.endswith("."):
	docstring += "."
	default = kwargs.get("default")
	if not default:
	has_truthy_default = False
	elif isinstance(default, ast.Name):
	has_truthy_default = default.id != "None"
	elif isinstance(default, ast.Constant):
	has_truthy_default = bool(default.value)
	elif isinstance(default, (ast.Tuple, ast.List)):
	has_truthy_default = bool(default.elts)
	elif isinstance(default, ast.Dict):
	has_truthy_default = bool(default.keys)
	else:
	has_truthy_default = True

	if has_truthy_default:
	print(
	"WARNING: property %s has default value: %s"
	% (prop_name, ast.unparse(default))
	)
	fields.append(
	ProtoField(
	docstring=docstring,
	proto_type=proto_type,
	name=prop_name,
	num=i + 1,
	repeated=repeated,
	)
	)

	lines = []
	for field in fields:
	field_lines = []
	if field.docstring:
	field_lines.append("// %s" % field.docstring)
	field_lines.append(
	"%s%s %s = %s;"
	% (
	"repeated " if field.repeated else "",
	field.proto_type,
	field.name,
	field.num,
	)
	)
	lines.append("\n".join(" " + l for l in field_lines))

	message_contents = "\n\n".join(lines)

	# Remove "recipes" from the beginning to get the name of the recipe as used
	# by the recipe engine. E.g. "recipes/contrib/foo.py" -> "contrib.foo".
	recipe_path_parts = filename.split(os.path.sep)[1:]
	recipe_path_parts[-1] = recipe_path_parts[-1].split(".")[0]
	recipe_name = ".".join(recipe_path_parts)

	import_block = ""
	if imports:
	import_lines = ['import "%s";' % imp for imp in sorted(imports)]
	import_block = "\n%s\n" % "\n".join(import_lines)

	proto_contents = """\
	// Copyright %d The Fuchsia Authors. All rights reserved.
	// Use of this source code is governed by a BSD-style license that can be
	// found in the LICENSE file.

	syntax = "proto3";

	package recipes.fuchsia.%s;
	%s
	message InputProperties {
	%s
	}
	""" % (
	datetime.datetime.now().year,
	recipe_name,
	import_block,
	message_contents,
	)

	with open(filename.replace(".py", ".proto"), "w") as f:
	f.write(proto_contents)

	update_recipe(tree, filename, recipe_name, contents, props_assignment.lineno)


	def update_recipe(tree, filename, recipe_name, contents, props_line):
	lines = contents.splitlines()

	for i in range(props_line, len(lines)):
	if lines[i].endswith("}"):
	end_line = i + 1
	break
	# Not sure why we need to subtract one here...
	lines[props_line - 1 : end_line] = ["PROPERTIES = InputProperties"]

	import_lineno = 0
	for item in tree.body:
	if not isinstance(item, (ast.Expr, ast.Import, ast.ImportFrom)):
	break

	import_lineno = item.lineno

	lines.insert(
	import_lineno, "from PB.recipes.fuchsia.%s import InputProperties" % recipe_name
	)

	# Remove imports of types needed for declaring dict-style properties.
	lines = [
	l
	for l in lines
	if not re.match(r"from recipe_engine.(recipe_api\|config) import", l)
	]

	with open(filename, "w") as f:
	f.write("".join(l + "\n" for l in lines))


	if __name__ == "__main__":
	main(sys.argv[1])