| # Copyright 2019 The Fuchsia Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| from recipe_engine import recipe_api |
| |
| import os |
| |
| FORMATTING_MESSAGE = """File not formatted properly. |
| Run the following to format: |
| |
| """ |
| |
| |
| class TriciumAnalyzeApi(recipe_api.RecipeApi): |
| """API for running analyses on Tricium.""" |
| |
| def __init__(self, analyses, compdb_filtered_paths, *args, **kwargs): |
| super(TriciumAnalyzeApi, self).__init__(*args, **kwargs) |
| self._enabled = [a.lower() for a in analyses] |
| self._compdb_filtered_paths = compdb_filtered_paths |
| |
| self._ext_to_analysis = { |
| '.c': [self._ClangFormat, self._ClangTidy], |
| '.cc': [self._ClangFormat, self._ClangTidy], |
| '.cpp': [self._ClangFormat, self._ClangTidy], |
| '.dart': [self._DartFmt], |
| '.h': [self._ClangFormat, self._ClangTidy], |
| '.hh': [self._ClangFormat, self._ClangTidy], |
| '.hpp': [self._ClangFormat, self._ClangTidy], |
| '.fidl': [self._FidlFormat, self._FidlLint], |
| '.gn': [self._GNFormat], |
| '.gni': [self._GNFormat], |
| '.go': [self._GoFmt, self._GoVet], |
| '.py': [self._Yapf], |
| '.rs': [self._RustFmt], |
| '.star': [self._Yapf], |
| '.ts': [self._ClangFormat], |
| } |
| |
| # Track which analyses have been run, so that it can skip building/preprocessing. |
| self._analyses_run = [] |
| self.__gn_results = None |
| |
| # The paths to these tools may be set directly by the recipe. |
| self.go = None |
| self.gofmt = None |
| self.yapf = None |
| |
| self.checkout = None |
| # Whether to suggest the use of the fx tool. |
| # The tool only works properly when run in fuchsia.git or one of its sub-directories. |
| self.suggest_fx = True |
| |
| @property |
| def _gn_results(self): |
| if not self.__gn_results: |
| assert self.checkout |
| # If any build-based analyzers are desired on topaz, experiences, etc., |
| # we should add these build config parameters to properties. |
| self.__gn_results = self.m.build.gen( |
| checkout_root=self.checkout.root_dir, |
| fuchsia_build_dir=self.checkout.root_dir.join('out', 'default'), |
| target='x64', |
| build_type='release', |
| board='boards/x64.gni', |
| product='products/core.gni', |
| packages=['//bundles:kitchen_sink'], |
| export_compdb=True, |
| ) |
| return self.__gn_results |
| |
| def __call__(self, filename): |
| """Run the relevant language's analyses over a file and post Tricium comments |
| if errors are found. |
| |
| Args: |
| filename (Path): Path to file. |
| """ |
| with self.m.step.nest('analyze %s' % filename): |
| _, ext = self.m.path.splitext(filename) |
| if ext not in self._ext_to_analysis: |
| return |
| |
| for analysis in self._ext_to_analysis[ext]: |
| if analysis.__name__.lstrip('_').lower() in self._enabled: |
| analysis(filename) |
| |
| def _diff_format(self, |
| category, |
| filename, |
| cmd_format='fx format-code --files=%s'): |
| diff_result = self.m.git( |
| 'diff', |
| '--name-only', |
| name='check file formatting', |
| stdout=self.m.raw_io.output()).stdout.strip().split('\n') |
| |
| # Ideally we'd have a generic way to support self.suggest_fx == False in this |
| # function. However today there's only one analyzer that actually needs this, |
| # and restructuring the code around this use case would add complexity on net. |
| # If we start supporting this for many analyzers we should reconsider, perhaps |
| # by having a class per analyzer rather than just a function. |
| for filename in diff_result: |
| if filename: # ignore whitespace-only lines |
| self.m.tricium.add_comment( |
| 'Format/%s' % category, |
| '%s%s' % (FORMATTING_MESSAGE, cmd_format % filename), filename) |
| self.m.git('reset', '--hard', 'HEAD', name='reset') |
| |
| def _FidlFormat(self, filename): |
| assert self.checkout |
| # Fidl test files often purposefully formatted in unrecommended ways |
| # so they should be skipped. |
| if str(filename).endswith('.test.fidl'): |
| return |
| |
| with self.m.step.nest('fidl-format'): |
| # Both fidl-lint and fidl-format build the same thing. |
| if 'FidlLint' not in self._analyses_run and 'FidlFormat' not in self._analyses_run: |
| # Build the tool. |
| self.m.build.ninja( |
| gn_results=self._gn_results, |
| zircon_targets=['tools'], |
| build_canonical_zircon_targets=False, |
| build_fuchsia=False, |
| ) |
| |
| self.m.step('run', [ |
| self._gn_results.tool('fidl-format'), |
| '-i', |
| filename, |
| ]) |
| self._analyses_run.append('FidlFormat') |
| self._diff_format('FidlFormat', filename) |
| |
| def _GoFmt(self, filename): |
| with self.m.step.nest('gofmt'): |
| if not self.gofmt: |
| self.gofmt = self._gn_results.tool('gofmt') |
| self.m.step('run', [self.gofmt, '-w', '-s', filename]) |
| self._analyses_run.append('GoFmt') |
| self._diff_format('GoFmt', filename) |
| |
| def _GNFormat(self, filename): |
| with self.m.step.nest('gn format'): |
| self.m.step('run', [self._gn_results.tool('gn'), 'format', filename]) |
| self._analyses_run.append('GNFormat') |
| self._diff_format('GNFormat', filename) |
| |
| def _RustFmt(self, filename): |
| with self.m.step.nest('rustfmt'): |
| self.m.step('run', [ |
| self._gn_results.tool('rustfmt'), |
| filename, |
| ]) |
| self._analyses_run.append('RustFmt') |
| self._diff_format('RustFmt', filename) |
| |
| def _Yapf(self, filename): |
| with self.m.step.nest('yapf'): |
| if not self.yapf: |
| self.yapf = self._gn_results.tool('yapf') |
| self.m.step('run', [self.yapf, '--in-place', filename]) |
| self._analyses_run.append('YAPF') |
| |
| if self.suggest_fx: |
| self._diff_format('YAPF', filename) |
| else: |
| self._diff_format('YAPF', filename, cmd_format='yapf --in-place %s') |
| |
| def _DartFmt(self, filename): |
| with self.m.step.nest('dartfmt'): |
| self.m.step('run', [self._gn_results.tool('dartfmt'), '-w', filename]) |
| self._analyses_run.append('DartFmt') |
| self._diff_format('DartFmt', filename) |
| |
| def _ClangFormat(self, filename): |
| # If clang-format has been run already, it needn't be run again as it runs on the whole diff. |
| if 'ClangFormat' in self._analyses_run: |
| return |
| |
| with self.m.step.nest('clang-format'): |
| paths = self.m.git( |
| 'diff', |
| '-U0', |
| '--no-color', |
| 'HEAD^', |
| name='get changed files', |
| stdout=self.m.raw_io.output()) |
| |
| self.m.python( |
| name='clang-format-diff.py', |
| script=self._gn_results.tool('clang-format-diff'), |
| args=[ |
| '-p1', '-i', '-style=file', '-sort-includes', '-binary', |
| self._gn_results.tool('clang-format') |
| ], |
| stdin=self.m.raw_io.input_text(data=paths.stdout)) |
| |
| self._analyses_run.append('ClangFormat') |
| self._diff_format('ClangFormat', filename) |
| |
| def _capitalize_msg(self, message): |
| if not message or message[0].isupper(): |
| return message |
| return message[0].upper() + message[1:] |
| |
| def _FidlLint(self, filename): |
| assert self.checkout |
| # Fidl test files are often purposefully use syntax that does not follow |
| # linting rules so they should be skipped. |
| if str(filename).endswith('.test.fidl'): |
| return |
| |
| with self.m.step.nest('fidl-lint'): |
| # Both fidl-lint and fidl-format build the same thing. |
| if 'FidlLint' not in self._analyses_run and 'FidlFormat' not in self._analyses_run: |
| # Build the tool. |
| self.m.build.ninja( |
| gn_results=self._gn_results, |
| zircon_targets=['tools'], |
| build_canonical_zircon_targets=False, |
| build_fuchsia=False, |
| ) |
| |
| results = self.m.step( |
| 'run', [ |
| self._gn_results.tool('fidl-lint'), |
| '--format=json', |
| filename, |
| ], |
| ok_ret=(0, 1), |
| stdout=self.m.json.output()).stdout |
| |
| for result in results: |
| capitalized_msg = self._capitalize_msg(result['message']) + '.' |
| capitalized_desc = '' |
| for suggestion in result.get('suggestions', ()): |
| if 'description' in suggestion: |
| capitalized_desc += self._capitalize_msg( |
| suggestion['description']) + '. ' |
| if capitalized_desc: |
| capitalized_msg = capitalized_msg + ' ' + capitalized_desc[:-1] |
| self.m.tricium.add_comment( |
| 'Lint/FidlLint', |
| capitalized_msg, |
| # All file paths reported to tricium should be relative to the root of the git repo. |
| # The caller ensures that cwd is the root of the git repo. |
| os.path.relpath(result['path'], |
| self.m.path.abspath(self.m.context.cwd)), |
| start_line=result['start_line'], |
| start_char=result['start_char'], |
| end_line=result['end_line'], |
| end_char=result['end_char']) |
| |
| self._analyses_run.append('FidlLint') |
| |
| def _GoVet(self, filename): |
| with self.m.step.nest('go vet') as step: |
| cwd = self.m.context.cwd |
| with self.m.context(cwd=cwd.join(self.m.path.dirname(filename))): |
| if not self.go: |
| self.go = self._gn_results.tool('go') |
| stderr = self.m.step( |
| 'run', [self.go, 'vet', '-json'], |
| stderr=self.m.raw_io.output()).stderr |
| |
| stderr_lines = stderr.splitlines() |
| step.presentation.logs['stderr'] = stderr_lines |
| # Unfortunately `go vet -json` does not output only valid JSON, so |
| # we have to parse the output manually. |
| # Look at the test cases in examples/ for the expected output format. |
| result = None |
| result_lines = [] |
| for line in stderr_lines: |
| if result_lines: |
| result_lines.append(line) |
| # Ends the JSON object |
| if line == '}': |
| result = self.m.json.loads('\n'.join(result_lines)) |
| break |
| # Empty JSON object |
| elif line == '{}': |
| result = {} |
| break |
| # Start new non-empty JSON object |
| elif line == '{': |
| assert not result_lines |
| result_lines.append(line) |
| |
| assert result is not None |
| |
| for package in result: |
| for error in result[package]: |
| for warning in result[package][error]: |
| pos = warning['posn'].split(':') |
| self.m.tricium.add_comment( |
| 'Lint/GoVet', |
| warning['message'], |
| # All file paths reported to tricium should be relative to the root of the git repo. |
| # The caller ensures that cwd is the root of the git repo. |
| os.path.relpath(pos[0], self.m.path.abspath(cwd)), |
| start_line=int(pos[1]), |
| start_char=int(pos[2]), |
| end_line=int(pos[1]), |
| end_char=int(pos[2])) |
| self._analyses_run.append('GoVet') |
| |
| def _ClangTidy(self, filename): |
| del filename # unused |
| # If clang-tidy has been run already, it needn't be run again as it runs on the whole diff. |
| if 'ClangTidy' in self._analyses_run: |
| return |
| |
| assert self.checkout |
| |
| with self.m.step.nest('clang-tidy'): |
| self.m.build.ninja( |
| gn_results=self._gn_results, |
| zircon_targets=['tools'], |
| build_canonical_zircon_targets=False, |
| build_generated_sources=True, |
| ) |
| |
| compile_commands = self._gn_results.filtered_compdb( |
| self._compdb_filtered_paths) |
| |
| clang_tidy = self._gn_results.tool('clang-tidy') |
| clang_tidy_diff = self._gn_results.tool('clang-tidy-diff') |
| warnings_file = self.m.path['cleanup'].join('clang_tidy_fixes.yaml') |
| |
| with self.m.context(cwd=self.checkout.root_dir): |
| diff = self.m.git( |
| 'diff', |
| '-U0', |
| '--no-color', |
| 'HEAD^', |
| name='get changed files', |
| stdout=self.m.raw_io.output()) |
| clang_tidy_args = [ |
| '-p1', |
| '-path', |
| compile_commands, |
| '-export-fixes', |
| warnings_file, |
| '-clang-tidy-binary', |
| clang_tidy, |
| ] |
| |
| step_result = self.m.python( |
| name='clang-tidy-diff.py', |
| script=clang_tidy_diff, |
| args=clang_tidy_args, |
| stdin=self.m.raw_io.input_text(data=diff.stdout), |
| # This script may return 1 if there are compile |
| # errors -- that's okay, since this is a linter |
| # check. We'll log them below. |
| ok_ret='any', |
| venv=self.resource('clang-tidy-diff.vpython')) |
| |
| assert (step_result.retcode in (0, 1)) |
| if step_result.retcode == 1: |
| self.m.step.active_result.presentation.status = 'WARNING' |
| errors = self._parse_warnings(warnings_file) |
| |
| self.m.path.mock_add_paths('[START_DIR]/path/to/file.cpp') |
| # We iterate through all produced error sets... |
| for check in errors: |
| # ...and for each check, iterate through all the errors it produced... |
| for err in errors[check]: |
| # ...and extract the information from that error for a comment. |
| error_filepath = self.m.path.abspath( |
| self._gn_results.fuchsia_build_dir.join( |
| err['DiagnosticMessage']['FilePath'])) |
| if not self.m.path.exists( |
| error_filepath) or err['DiagnosticMessage']['FilePath'] == '': |
| continue # pragma: no cover |
| |
| # Extract the line and character for this warning. |
| sline, schar = self._get_line_from_offset( |
| error_filepath, err['DiagnosticMessage']['FileOffset']) |
| |
| # Add the comment to Tricium. |
| self.m.tricium.add_comment( |
| 'Lint/ClangTidy', |
| '%s: %s' % |
| (err['DiagnosticName'], err['DiagnosticMessage']['Message']), |
| # All file paths reported to tricium should be relative to the root of the git repo. |
| # The caller ensures that cwd is the root of the git repo. |
| os.path.relpath( |
| str(err['DiagnosticMessage']['FilePath']), |
| self.m.path.abspath(self.m.context.cwd)), |
| start_line=sline, |
| start_char=schar, |
| end_line=sline, |
| end_char=schar, |
| ) |
| |
| self._analyses_run.append('ClangTidy') |
| |
| def _parse_warnings(self, warnings_file): |
| """Parse all warnings output by clang-tidy. |
| |
| Clang-Tidy issues warnings as follows: |
| - DiagnosticName: 'check name' |
| Message: 'error message' |
| FileOffset: <offset (int)> |
| FilePath: 'file path' |
| Replacements: |
| - FilePath: 'replacement file path' |
| Offset: <replacement start offset (int)> |
| Length: <replacement length (int)> |
| ReplacementText: 'replacement text' |
| |
| Args: |
| raw_warnings (str): YAML-encoded warnings as output by the clang-tidy binary |
| |
| Returns: |
| A dict of parsed warnings by check. |
| Schema: |
| { |
| '<check name>': [ |
| { |
| 'DiagnosticName': 'check name' |
| 'Message': 'error message', |
| 'StartLine': <error start line (int)>, |
| 'StartChar': <error start char (int)>, |
| 'Replacements': [ |
| { |
| 'File': 'replacement file path', |
| 'StartLine': <replacement start line (int)>, |
| 'StartChar': <replacement start char (int)>, |
| 'EndLine': <replacement end line (int)>, |
| 'EndChar': <replacement end char (int)>, |
| 'Text': 'replacement text' |
| }, |
| ... |
| ] |
| }, |
| ... |
| ], |
| '<other check name>': [ ... ] |
| } |
| """ |
| self.m.path.mock_add_paths(warnings_file) |
| if not self.m.path.exists(warnings_file): |
| return {} # pragma: no cover |
| parsed_results = self.m.python( |
| 'load yaml', |
| self.resource('parse_yaml.py'), |
| args=[warnings_file], |
| stdout=self.m.json.output()).stdout |
| if not parsed_results: |
| return {} |
| all_warnings = {} |
| for warning in parsed_results['Diagnostics']: |
| if warning['DiagnosticName'] not in all_warnings: |
| all_warnings[warning['DiagnosticName']] = [] |
| all_warnings[warning['DiagnosticName']].append(warning) |
| return all_warnings |
| |
| def _get_line_from_offset(self, f, offset): |
| """Get the file line and char number from a file offset. |
| |
| Clang-Tidy emits warnings that mark the location of the error by the char |
| offset from the beginning of the file. This converts that number into a line |
| and char position. |
| |
| Args: |
| filename (str): Path to file. |
| offset (int): Offset to convert. |
| """ |
| file_data = self.m.file.read_text( |
| 'read %s' % f, f, test_data="""test |
| d |
| newlineoutput""") |
| line = 1 |
| char = 0 |
| for i, c in enumerate(file_data): |
| if c == '\n': |
| line += 1 |
| char = 0 |
| else: |
| char += 1 |
| if i + 1 == offset: |
| return line, char |
| return 0, 0 |
