gerrit-policy/gerrit-policy.py - infra/infra - Git at Google

 #!/usr/bin/env python3
 """Checks and enforces Gerrit configuration policies.

 Run with "help" for usage.
 """

 # TODO(dbort): Add more policy checks, refactoring as we go.
 # Possible checks:
 # - Projects are either public or hidden/read-only
 # - No rules.pl files exist outside of All-Projects
 # - rules.pl is in sync across all hosts
 # - Upstream sync configs are in place for all third_party repos
 # - All upstreams point to gob-mirrored repos
 # TODO(dbort): Rewrite in Go if it seems worth the trouble.

 import collections
 import json
 import logging
 import subprocess
 import sys
 import textwrap


 class GerritHost(object):
     """Represents a remote Gerrit Git-on-Borg host."""

     def __init__(self, gob_name):
         self.__url = 'https://{}-review.googlesource.com/a'.format(gob_name)

     @property
     def url(self):
         return self.__url

     def __GetJson(self, query):
         """Performs a GET query against the host and returns parsed JSON.

         Args:
             query: The HTTP query string; e.g., '/projects/?t'.
         Returns:
             Parsed JSON from the body of the HTTP response.
         """

         url = '{}/{}'.format(self.__url, query.lstrip('/'))

         # check=True raises subprocess.CalledProcessError on failure.
         args = ['gob-curl', '--nointeractive', url]
         logging.debug('Run: %s', ' '.join(args))
         p = subprocess.run(args, stdout=subprocess.PIPE, check=True)
         stdout = p.stdout.decode('utf-8')

         # Remove the first line, which contains the )]}' guard.
         _, body = stdout.split('\n', 1)

         return json.loads(body)

     def GetProjects(self):
         """Returns a dict that maps project names to ProjectInfo entries.

         https://gerrit-review.googlesource.com/Documentation/rest-api-projects.html#project-info
         """
         # 't' requests "tree view", which includes the parent projects.
         return self.__GetJson('/projects/?t')


 class ThirdPartyParents(object):
     # XParent fields:
     #     name: Gerrit project name
     #     id: Gerrit project id
     #     has: Project's current parent ID
     #     needs: Project's required parent ID, according to policy
     ValidParent = collections.namedtuple('ValidParent', ('name', 'id', 'has'))
     ChangeParent = collections.namedtuple('ChangeParent',
                                           ('name', 'id', 'has', 'needs'))
     UnknownParent = collections.namedtuple('UnknownParent',
                                            ('name', 'id', 'has'))
     # Contains sequences of ValidParent/ChangeParent/UnknownParent.
     Checks = collections.namedtuple('Checks', ('valid', 'change', 'unknown'))

     def __init__(self, projects):
         # Only hold onto //third_party projects.
         self.__projects = {
             name: project
             for name, project in projects.items()
             if name.startswith('third_party/')
         }

     def __Validate(self):
         valid = []
         change = []
         unknown = []
         for name, info in self.__projects.items():
             pid = info.get('id', '')
             if not pid:
                 raise KeyError('No id for project "{}"'.format(name))
             parent = info.get('parent', '')
             entry = {
                 'name': name,
                 'id': pid,
                 'has': parent,
             }

             if parent in ('Third-Party-Projects',
                           'Third-Party-Commit-Queue-Projects'):
                 valid.append(self.ValidParent(**entry))
                 continue
             elif parent == 'Public-Projects':
                 entry['needs'] = 'Third-Party-Projects'
                 change.append(self.ChangeParent(**entry))
             elif parent in ('Commit-Queue', 'gerrit/commit-queue-projects'):
                 entry['needs'] = 'Third-Party-Commit-Queue-Projects'
                 change.append(self.ChangeParent(**entry))
             else:
                 # TODO(dbort): Figure out what to do with All-Projects.
                 # If they're supposed to be visible, they should at
                 # least be Public-Projects. Look at the hidden and
                 # read-only states. That should be a separate policy
                 # even for non-third_party repos.
                 unknown.append(self.UnknownParent(**entry))
         return self.Checks(
             valid=tuple(sorted(valid)),
             change=tuple(sorted(change)),
             unknown=tuple(sorted(unknown)))

     def Check(self):
         checks = self.__Validate()
         for p in checks.valid:
             print(p)
         for p in checks.change:
             print(p)
         for p in checks.unknown:
             print(p)
         print("%d total //third_party projects:" %
               (len(checks.valid) + len(checks.change) + len(checks.unknown)))
         print("  %d with valid parents" % len(checks.valid))
         print("  %d need to change parents" % len(checks.change))
         print("  %d with unrecognized parents" % len(checks.unknown))
         if checks.unknown:
             print("    [%s]" % ', '.join(
                 sorted(set([p.has for p in checks.unknown]))))
         # TODO(dbort): Return failure if change/unknown are non-empty

     def Enforce(self, gerrit_host, out_script):
         """Creates a shell script to enforce the policy.

         Args:
             gerrit_host: The host that the script should modify.
             out_script: The file-like object to write a bash script to.
         """
         checks = self.__Validate()
         header = textwrap.dedent(r"""
             # Generated by gerrit-policy.py:%(gen)s

             set -o nounset
             set -o errexit
             set -o pipefail

             ### TODO: Set to a non-empty value
             readonly BUG_ID=""

             ### TODO: Set DRY_RUN=0 to enable this script
             readonly DRY_RUN=1

             ### No edits required below this line ###

             readonly HOST_URL='%(host_url)s'

             if [[ -z "${BUG_ID}" ]]; then
               echo "ERROR: Must set BUG_ID by editing this file ($0)"
               exit 1
             fi

             # run [<args>...]
             # Prints the args, and runs them if DRY_RUN == 0
             run() {
               if (( DRY_RUN )); then
                 echo -n "DRY_RUN:"
                 printf " %%q" "$@"
                 echo ""
               else
                 echo -n "RUNNING:"
                 printf " %%q" "$@"
                 echo ""
                 "$@"
                 return $?
               fi
             }

             # change-parent <project-name> <new-parent>
             change-parent() {
               local project="$1"
               local new_parent="$2"
               run gob-curl \
                   -d '{"parent": "'${new_parent}'", "commit_message": "Set parent to '${new_parent}'\n\nBug: '${BUG_ID}'"}' \
                   -H "Content-Type: application/json" \
                   -X PUT \
                   "${HOST_URL}/projects/${project}/parent"
             }
             """ % {
             'gen': self.__class__.__name__,
             'host_url': gerrit_host.url
         })
         lines = [
             '#!/bin/bash',
             header,
         ]

         if checks.valid:
             lines.extend([
                 '# Nothing to do for %d valid projects' % len(checks.valid),
                 '',
             ])
         lines.append('# %d projects to change' % len(checks.change))
         for p in checks.change:
             lines.append('change-parent \'%s\' \'%s\'' % (p.id, p.needs))
         lines.append(
             '\n# %d projects with unhandled status' % len(checks.unknown))
         lines.append('\necho DONE')

         out_script.write('\n'.join(lines))


 def main(argv):
     def Usage(argv):
         print(textwrap.dedent("""\
             Usage: {argv0} <command> <gob-host>

             command:
               check: Print repo policy violations but do not modify anything
               enforce: Generate a script to fix policy violations

             gob-host: "fuchsia", "cobalt-analytics", etc.
             """.format(argv0=argv[0])))

     if len(argv) < 3:
         Usage(argv)
         sys.exit(1)
     if argv[1] in ('help', '-help', '--help', '-h'):
         Usage(argv)
         sys.exit(0)

     host = GerritHost(argv[2])
     projects = host.GetProjects()

     tpp = ThirdPartyParents(projects)
     if argv[1] == 'check':
         tpp.Check()
     elif argv[1] == 'enforce':
         # TODO(dbort): Add a flag to allow specifying the bug ID to be injected
         #     into the generated script
         # TODO(dbort): Make out_script a flag
         out_script = '/tmp/policy.sh'
         with open(out_script, 'w') as f:
             tpp.Enforce(host, f)
         logging.info('Enforcement script written to %s', out_script)


 if __name__ == '__main__':
     logging.basicConfig(level=logging.DEBUG)
     main(sys.argv)
	#!/usr/bin/env python3
	"""Checks and enforces Gerrit configuration policies.

	Run with "help" for usage.
	"""

	# TODO(dbort): Add more policy checks, refactoring as we go.
	# Possible checks:
	# - Projects are either public or hidden/read-only
	# - No rules.pl files exist outside of All-Projects
	# - rules.pl is in sync across all hosts
	# - Upstream sync configs are in place for all third_party repos
	# - All upstreams point to gob-mirrored repos
	# TODO(dbort): Rewrite in Go if it seems worth the trouble.

	import collections
	import json
	import logging
	import subprocess
	import sys
	import textwrap


	class GerritHost(object):
	"""Represents a remote Gerrit Git-on-Borg host."""

	def __init__(self, gob_name):
	self.__url = 'https://{}-review.googlesource.com/a'.format(gob_name)

	@property
	def url(self):
	return self.__url

	def __GetJson(self, query):
	"""Performs a GET query against the host and returns parsed JSON.

	Args:
	query: The HTTP query string; e.g., '/projects/?t'.
	Returns:
	Parsed JSON from the body of the HTTP response.
	"""

	url = '{}/{}'.format(self.__url, query.lstrip('/'))

	# check=True raises subprocess.CalledProcessError on failure.
	args = ['gob-curl', '--nointeractive', url]
	logging.debug('Run: %s', ' '.join(args))
	p = subprocess.run(args, stdout=subprocess.PIPE, check=True)
	stdout = p.stdout.decode('utf-8')

	# Remove the first line, which contains the )]}' guard.
	_, body = stdout.split('\n', 1)

	return json.loads(body)

	def GetProjects(self):
	"""Returns a dict that maps project names to ProjectInfo entries.

	https://gerrit-review.googlesource.com/Documentation/rest-api-projects.html#project-info
	"""
	# 't' requests "tree view", which includes the parent projects.
	return self.__GetJson('/projects/?t')


	class ThirdPartyParents(object):
	# XParent fields:
	# name: Gerrit project name
	# id: Gerrit project id
	# has: Project's current parent ID
	# needs: Project's required parent ID, according to policy
	ValidParent = collections.namedtuple('ValidParent', ('name', 'id', 'has'))
	ChangeParent = collections.namedtuple('ChangeParent',
	('name', 'id', 'has', 'needs'))
	UnknownParent = collections.namedtuple('UnknownParent',
	('name', 'id', 'has'))
	# Contains sequences of ValidParent/ChangeParent/UnknownParent.
	Checks = collections.namedtuple('Checks', ('valid', 'change', 'unknown'))

	def __init__(self, projects):
	# Only hold onto //third_party projects.
	self.__projects = {
	name: project
	for name, project in projects.items()
	if name.startswith('third_party/')
	}

	def __Validate(self):
	valid = []
	change = []
	unknown = []
	for name, info in self.__projects.items():
	pid = info.get('id', '')
	if not pid:
	raise KeyError('No id for project "{}"'.format(name))
	parent = info.get('parent', '')
	entry = {
	'name': name,
	'id': pid,
	'has': parent,
	}

	if parent in ('Third-Party-Projects',
	'Third-Party-Commit-Queue-Projects'):
	valid.append(self.ValidParent(**entry))
	continue
	elif parent == 'Public-Projects':
	entry['needs'] = 'Third-Party-Projects'
	change.append(self.ChangeParent(**entry))
	elif parent in ('Commit-Queue', 'gerrit/commit-queue-projects'):
	entry['needs'] = 'Third-Party-Commit-Queue-Projects'
	change.append(self.ChangeParent(**entry))
	else:
	# TODO(dbort): Figure out what to do with All-Projects.
	# If they're supposed to be visible, they should at
	# least be Public-Projects. Look at the hidden and
	# read-only states. That should be a separate policy
	# even for non-third_party repos.
	unknown.append(self.UnknownParent(**entry))
	return self.Checks(
	valid=tuple(sorted(valid)),
	change=tuple(sorted(change)),
	unknown=tuple(sorted(unknown)))

	def Check(self):
	checks = self.__Validate()
	for p in checks.valid:
	print(p)
	for p in checks.change:
	print(p)
	for p in checks.unknown:
	print(p)
	print("%d total //third_party projects:" %
	(len(checks.valid) + len(checks.change) + len(checks.unknown)))
	print(" %d with valid parents" % len(checks.valid))
	print(" %d need to change parents" % len(checks.change))
	print(" %d with unrecognized parents" % len(checks.unknown))
	if checks.unknown:
	print(" [%s]" % ', '.join(
	sorted(set([p.has for p in checks.unknown]))))
	# TODO(dbort): Return failure if change/unknown are non-empty

	def Enforce(self, gerrit_host, out_script):
	"""Creates a shell script to enforce the policy.

	Args:
	gerrit_host: The host that the script should modify.
	out_script: The file-like object to write a bash script to.
	"""
	checks = self.__Validate()
	header = textwrap.dedent(r"""
	# Generated by gerrit-policy.py:%(gen)s

	set -o nounset
	set -o errexit
	set -o pipefail

	### TODO: Set to a non-empty value
	readonly BUG_ID=""

	### TODO: Set DRY_RUN=0 to enable this script
	readonly DRY_RUN=1

	### No edits required below this line ###

	readonly HOST_URL='%(host_url)s'

	if [[ -z "${BUG_ID}" ]]; then
	echo "ERROR: Must set BUG_ID by editing this file ($0)"
	exit 1
	fi

	# run [<args>...]
	# Prints the args, and runs them if DRY_RUN == 0
	run() {
	if (( DRY_RUN )); then
	echo -n "DRY_RUN:"
	printf " %%q" "$@"
	echo ""
	else
	echo -n "RUNNING:"
	printf " %%q" "$@"
	echo ""
	"$@"
	return $?
	fi
	}

	# change-parent <project-name> <new-parent>
	change-parent() {
	local project="$1"
	local new_parent="$2"
	run gob-curl \
	-d '{"parent": "'${new_parent}'", "commit_message": "Set parent to '${new_parent}'\n\nBug: '${BUG_ID}'"}' \
	-H "Content-Type: application/json" \
	-X PUT \
	"${HOST_URL}/projects/${project}/parent"
	}
	""" % {
	'gen': self.__class__.__name__,
	'host_url': gerrit_host.url
	})
	lines = [
	'#!/bin/bash',
	header,
	]

	if checks.valid:
	lines.extend([
	'# Nothing to do for %d valid projects' % len(checks.valid),
	'',
	])
	lines.append('# %d projects to change' % len(checks.change))
	for p in checks.change:
	lines.append('change-parent \'%s\' \'%s\'' % (p.id, p.needs))
	lines.append(
	'\n# %d projects with unhandled status' % len(checks.unknown))
	lines.append('\necho DONE')

	out_script.write('\n'.join(lines))


	def main(argv):
	def Usage(argv):
	print(textwrap.dedent("""\
	Usage: {argv0} <command> <gob-host>

	command:
	check: Print repo policy violations but do not modify anything
	enforce: Generate a script to fix policy violations

	gob-host: "fuchsia", "cobalt-analytics", etc.
	""".format(argv0=argv[0])))

	if len(argv) < 3:
	Usage(argv)
	sys.exit(1)
	if argv[1] in ('help', '-help', '--help', '-h'):
	Usage(argv)
	sys.exit(0)

	host = GerritHost(argv[2])
	projects = host.GetProjects()

	tpp = ThirdPartyParents(projects)
	if argv[1] == 'check':
	tpp.Check()
	elif argv[1] == 'enforce':
	# TODO(dbort): Add a flag to allow specifying the bug ID to be injected
	# into the generated script
	# TODO(dbort): Make out_script a flag
	out_script = '/tmp/policy.sh'
	with open(out_script, 'w') as f:
	tpp.Enforce(host, f)
	logging.info('Enforcement script written to %s', out_script)


	if __name__ == '__main__':
	logging.basicConfig(level=logging.DEBUG)
	main(sys.argv)