| #!/usr/bin/env python |
| # Copyright 2015 gRPC authors. |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| """Detect new flakes and create issues for them""" |
| |
| from __future__ import absolute_import |
| from __future__ import division |
| from __future__ import print_function |
| |
| import datetime |
| import json |
| import logging |
| import os |
| import pprint |
| import sys |
| import urllib |
| import urllib2 |
| from collections import namedtuple |
| |
| gcp_utils_dir = os.path.abspath( |
| os.path.join(os.path.dirname(__file__), '../gcp/utils')) |
| sys.path.append(gcp_utils_dir) |
| |
| import big_query_utils |
| |
| GH_ISSUE_CREATION_URL = 'https://api.github.com/repos/grpc/grpc/issues' |
| GH_ISSUE_SEARCH_URL = 'https://api.github.com/search/issues' |
| KOKORO_BASE_URL = 'https://kokoro2.corp.google.com/job/' |
| |
| |
| def gh(url, data=None): |
| request = urllib2.Request(url, data=data) |
| assert TOKEN |
| request.add_header('Authorization', 'token {}'.format(TOKEN)) |
| if data: |
| request.add_header('Content-type', 'application/json') |
| response = urllib2.urlopen(request) |
| if 200 <= response.getcode() < 300: |
| return json.loads(response.read()) |
| else: |
| raise ValueError('Error ({}) accessing {}'.format( |
| response.getcode(), response.geturl())) |
| |
| |
| def search_gh_issues(search_term, status='open'): |
| params = ' '.join((search_term, 'is:issue', 'is:open', 'repo:grpc/grpc')) |
| qargs = urllib.urlencode({'q': params}) |
| url = '?'.join((GH_ISSUE_SEARCH_URL, qargs)) |
| response = gh(url) |
| return response |
| |
| |
| def create_gh_issue(title, body, labels, assignees=[]): |
| params = {'title': title, 'body': body, 'labels': labels} |
| if assignees: |
| params['assignees'] = assignees |
| data = json.dumps(params) |
| response = gh(GH_ISSUE_CREATION_URL, data) |
| issue_url = response['html_url'] |
| print('Created issue {} for {}'.format(issue_url, title)) |
| |
| |
| def build_kokoro_url(job_name, build_id): |
| job_path = '{}/{}'.format('/job/'.join(job_name.split('/')), build_id) |
| return KOKORO_BASE_URL + job_path |
| |
| |
| def create_issues(new_flakes, always_create): |
| for test_name, results_row in new_flakes.items(): |
| poll_strategy, job_name, build_id, timestamp = results_row |
| # TODO(dgq): the Kokoro URL has a limited lifetime. The permanent and ideal |
| # URL would be the sponge one, but there's currently no easy way to retrieve |
| # it. |
| url = build_kokoro_url(job_name, build_id) |
| title = 'New Failure: ' + test_name |
| body = '- Test: {}\n- Poll Strategy: {}\n- URL: {}'.format( |
| test_name, poll_strategy, url) |
| labels = ['infra/New Failure'] |
| if always_create: |
| proceed = True |
| else: |
| preexisting_issues = search_gh_issues(test_name) |
| if preexisting_issues['total_count'] > 0: |
| print('\nFound {} issues for "{}":'.format( |
| preexisting_issues['total_count'], test_name)) |
| for issue in preexisting_issues['items']: |
| print('\t"{}" ; URL: {}'.format(issue['title'], |
| issue['html_url'])) |
| else: |
| print( |
| '\nNo preexisting issues found for "{}"'.format(test_name)) |
| proceed = raw_input( |
| 'Create issue for:\nTitle: {}\nBody: {}\n[Y/n] '.format( |
| title, body)) in ('y', 'Y', '') |
| if proceed: |
| assignees_str = raw_input( |
| 'Asignees? (comma-separated, leave blank for unassigned): ') |
| assignees = [ |
| assignee.strip() for assignee in assignees_str.split(',') |
| ] |
| create_gh_issue(title, body, labels, assignees) |
| |
| |
| def print_table(table, format): |
| first_time = True |
| for test_name, results_row in table.items(): |
| poll_strategy, job_name, build_id, timestamp = results_row |
| full_kokoro_url = build_kokoro_url(job_name, build_id) |
| if format == 'human': |
| print("\t- Test: {}, Polling: {}, Timestamp: {}, url: {}".format( |
| test_name, poll_strategy, timestamp, full_kokoro_url)) |
| else: |
| assert (format == 'csv') |
| if first_time: |
| print('test,timestamp,url') |
| first_time = False |
| print("{},{},{}".format(test_name, timestamp, full_kokoro_url)) |
| |
| |
| Row = namedtuple('Row', ['poll_strategy', 'job_name', 'build_id', 'timestamp']) |
| |
| |
| def get_new_failures(dates): |
| bq = big_query_utils.create_big_query() |
| this_script_path = os.path.join(os.path.dirname(__file__)) |
| sql_script = os.path.join(this_script_path, 'sql/new_failures_24h.sql') |
| with open(sql_script) as query_file: |
| query = query_file.read().format( |
| calibration_begin=dates['calibration']['begin'], |
| calibration_end=dates['calibration']['end'], |
| reporting_begin=dates['reporting']['begin'], |
| reporting_end=dates['reporting']['end']) |
| logging.debug("Query:\n%s", query) |
| query_job = big_query_utils.sync_query_job(bq, 'grpc-testing', query) |
| page = bq.jobs().getQueryResults( |
| pageToken=None, **query_job['jobReference']).execute(num_retries=3) |
| rows = page.get('rows') |
| if rows: |
| return { |
| row['f'][0]['v']: Row( |
| poll_strategy=row['f'][1]['v'], |
| job_name=row['f'][2]['v'], |
| build_id=row['f'][3]['v'], |
| timestamp=row['f'][4]['v']) |
| for row in rows |
| } |
| else: |
| return {} |
| |
| |
| def parse_isodate(date_str): |
| return datetime.datetime.strptime(date_str, "%Y-%m-%d").date() |
| |
| |
| def get_new_flakes(args): |
| """The from_date_str argument marks the beginning of the "calibration", used |
| to establish the set of pre-existing flakes, which extends over |
| "calibration_days". After the calibration period, "reporting_days" is the |
| length of time during which new flakes will be reported. |
| |
| from |
| date |
| |--------------------|---------------| |
| ^____________________^_______________^ |
| calibration reporting |
| days days |
| """ |
| dates = process_date_args(args) |
| new_failures = get_new_failures(dates) |
| logging.info('|new failures| = %d', len(new_failures)) |
| return new_failures |
| |
| |
| def build_args_parser(): |
| import argparse, datetime |
| parser = argparse.ArgumentParser() |
| today = datetime.date.today() |
| a_week_ago = today - datetime.timedelta(days=7) |
| parser.add_argument( |
| '--calibration_days', |
| type=int, |
| default=7, |
| help='How many days to consider for pre-existing flakes.') |
| parser.add_argument( |
| '--reporting_days', |
| type=int, |
| default=1, |
| help='How many days to consider for the detection of new flakes.') |
| parser.add_argument( |
| '--count_only', |
| dest='count_only', |
| action='store_true', |
| help='Display only number of new flakes.') |
| parser.set_defaults(count_only=False) |
| parser.add_argument( |
| '--create_issues', |
| dest='create_issues', |
| action='store_true', |
| help='Create issues for all new flakes.') |
| parser.set_defaults(create_issues=False) |
| parser.add_argument( |
| '--always_create_issues', |
| dest='always_create_issues', |
| action='store_true', |
| help='Always create issues for all new flakes. Otherwise,' |
| ' interactively prompt for every issue.') |
| parser.set_defaults(always_create_issues=False) |
| parser.add_argument( |
| '--token', |
| type=str, |
| default='', |
| help='GitHub token to use its API with a higher rate limit') |
| parser.add_argument( |
| '--format', |
| type=str, |
| choices=['human', 'csv'], |
| default='human', |
| help='Output format: are you a human or a machine?') |
| parser.add_argument( |
| '--loglevel', |
| type=str, |
| choices=['INFO', 'DEBUG', 'WARNING', 'ERROR', 'CRITICAL'], |
| default='WARNING', |
| help='Logging level.') |
| return parser |
| |
| |
| def process_date_args(args): |
| calibration_begin = ( |
| datetime.date.today() - datetime.timedelta(days=args.calibration_days) - |
| datetime.timedelta(days=args.reporting_days)) |
| calibration_end = calibration_begin + datetime.timedelta( |
| days=args.calibration_days) |
| reporting_begin = calibration_end |
| reporting_end = reporting_begin + datetime.timedelta( |
| days=args.reporting_days) |
| return { |
| 'calibration': { |
| 'begin': calibration_begin, |
| 'end': calibration_end |
| }, |
| 'reporting': { |
| 'begin': reporting_begin, |
| 'end': reporting_end |
| } |
| } |
| |
| |
| def main(): |
| global TOKEN |
| args_parser = build_args_parser() |
| args = args_parser.parse_args() |
| if args.create_issues and not args.token: |
| raise ValueError( |
| 'Missing --token argument, needed to create GitHub issues') |
| TOKEN = args.token |
| |
| logging_level = getattr(logging, args.loglevel) |
| logging.basicConfig(format='%(asctime)s %(message)s', level=logging_level) |
| new_flakes = get_new_flakes(args) |
| |
| dates = process_date_args(args) |
| |
| dates_info_string = 'from {} until {} (calibrated from {} until {})'.format( |
| dates['reporting']['begin'].isoformat(), |
| dates['reporting']['end'].isoformat(), |
| dates['calibration']['begin'].isoformat(), |
| dates['calibration']['end'].isoformat()) |
| |
| if args.format == 'human': |
| if args.count_only: |
| print(len(new_flakes), dates_info_string) |
| elif new_flakes: |
| found_msg = 'Found {} new flakes {}'.format( |
| len(new_flakes), dates_info_string) |
| print(found_msg) |
| print('*' * len(found_msg)) |
| print_table(new_flakes, 'human') |
| if args.create_issues: |
| create_issues(new_flakes, args.always_create_issues) |
| else: |
| print('No new flakes found '.format(len(new_flakes)), |
| dates_info_string) |
| elif args.format == 'csv': |
| if args.count_only: |
| print('from_date,to_date,count') |
| print('{},{},{}'.format(dates['reporting']['begin'].isoformat(), |
| dates['reporting']['end'].isoformat(), |
| len(new_flakes))) |
| else: |
| print_table(new_flakes, 'csv') |
| else: |
| raise ValueError('Invalid argument for --format: {}'.format( |
| args.format)) |
| |
| |
| if __name__ == '__main__': |
| main() |
