| # Copyright 2018 The Fuchsia Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| """Recipe for generating docs for upload to Firebase.""" |
| |
| from recipe_engine.config import Enum, List |
| from recipe_engine.recipe_api import Property |
| |
| DEPS = [ |
| 'fuchsia/auto_roller', |
| 'fuchsia/build', |
| 'fuchsia/checkout', |
| 'fuchsia/fuchsia', |
| 'fuchsia/git', |
| 'fuchsia/jiri', |
| 'recipe_engine/buildbucket', |
| 'recipe_engine/cipd', |
| 'recipe_engine/context', |
| 'recipe_engine/json', |
| 'recipe_engine/file', |
| 'recipe_engine/path', |
| 'recipe_engine/properties', |
| 'recipe_engine/python', |
| 'recipe_engine/raw_io', |
| 'recipe_engine/step', |
| ] |
| |
| TARGETS = ['arm64', 'x64'] |
| BUILD_TYPES = ['debug', 'release', 'thinlto', 'lto'] |
| GENERATORS = ['dartdoc', 'rustdoc', 'fidldoc', 'clangdoc'] |
| FIDLDOC_COMMIT_MESSAGE = '[fidldoc] Updating fidldocs' |
| |
| REFERENCE_DOCS_REPOSITORY = 'https://fuchsia.googlesource.com/reference-docs' |
| API_DOCS_RESOURCES_REPOSITORY = 'https://fuchsia.googlesource.com/api-docs-resources' |
| |
| PROPERTIES = { |
| 'dry_run': |
| Property( |
| kind=bool, |
| help='Whether to upload docs to firebase.', |
| default=False), |
| # TODO(juliehockett): Replace `basestring` with Enum(*GENERATORS) once crbug/903469 is resolved. |
| 'generators': |
| Property( |
| kind=List(basestring), help='Packages to build', |
| default=GENERATORS), |
| 'manifest': |
| Property(kind=str, help='Jiri manifest to use', default='topaz/topaz'), |
| 'remote': |
| Property( |
| kind=str, |
| help='Remote manifest repository', |
| default='https://fuchsia.googlesource.com/integration'), |
| 'target': |
| Property(kind=Enum(*TARGETS), help='Target to build', default='x64'), |
| 'build_type': |
| Property( |
| kind=Enum(*BUILD_TYPES), help='The build type', default='release'), |
| 'packages': |
| Property( |
| kind=List(basestring), |
| help='Packages to build', |
| default=['//topaz/bundles:buildbot', '//bundles:kitchen_sink']), |
| 'output_name': |
| Property( |
| kind=str, |
| help='Name of top-level output reference-docs directory', |
| default='all'), |
| } |
| |
| DARTDOC_PUBSPEC = """name: Fuchsia |
| homepage: https://fuchsia-docs.firebaseapp.com/dart |
| description: API documentation for fuchsia |
| dependencies: |
| """ |
| |
| |
| def gen_dartdoc(api, out_dir, docs_dir, cipd_dir): |
| """Generate dartdoc output. |
| |
| Dartdoc runs on a single package, but has the capability to generate docs for all |
| dependencies. Thus, to generate Dart documentation for Fuchsia, we first generate |
| a 'fake' package that lists the libraries we want documented. We then run `pub` |
| over that new package to fetch the dependencies, and finally `dartdoc` to generate |
| documentation for it all. |
| |
| Args: |
| out_dir (Path) - The output directory for generated files. |
| docs_dir (Path) - The output directory for documentation. |
| cipd_dir (Path) - The cipd directory. |
| """ |
| dart_packages_path = api.path['start_dir'].join('topaz', 'public', 'dart') |
| api.path.mock_add_paths(dart_packages_path) |
| # If either dartdoc or dart packages path doesn't exist, we didn't checkout topaz |
| # and so we won't generate dart docs on this run. |
| if not api.path.exists(dart_packages_path): |
| return # pragma: no cover |
| |
| # Make a temporary docs dir to be pushed to firebase. |
| api.file.ensure_directory('create lib dir', out_dir.join('lib')) |
| |
| # Build .packages and lib.dart importing all packages. |
| dart_imports_content = 'library Fuchsia;\n' |
| dart_pubspec_content = DARTDOC_PUBSPEC |
| |
| # Gather documentable dart packages. |
| dart_packages = [ |
| api.path.basename(p) for p in api.file.listdir( |
| 'list dart packages', |
| dart_packages_path, |
| test_data=('fuchsia', 'topaz', 'other')) |
| ] |
| |
| api.path.mock_add_paths(dart_packages_path.join('fuchsia', 'lib')) |
| api.path.mock_add_paths(dart_packages_path.join('fuchsia', 'pubspec.yaml')) |
| api.path.mock_add_paths(dart_packages_path.join('topaz', 'lib')) |
| |
| for package in dart_packages: |
| if not api.path.exists(dart_packages_path.join(package, 'lib')): |
| continue |
| |
| pubspec_path = dart_packages_path.join(package, 'pubspec.yaml') |
| if not api.path.exists(pubspec_path): |
| continue |
| |
| pubspec = api.python( |
| 'load %s pubspec.yaml' % package, |
| api.resource('parse_yaml.py'), |
| args=[pubspec_path], |
| stdout=api.json.output()).stdout |
| |
| if not pubspec or pubspec['name'] != package: |
| continue # pragma: no cover |
| |
| dart_pubspec_content += ' %s:\n path: %s/\n' % ( |
| package, dart_packages_path.join(package)) |
| package_imports = [ |
| api.path.basename(i) |
| for i in api.file.listdir('list %s packages' % package, |
| dart_packages_path.join(package, 'lib')) |
| if api.path.basename(i).endswith('.dart') |
| ] |
| for i in package_imports: |
| dart_imports_content += 'import \'package:%s/%s\';\n' % (package, i) |
| |
| # Build package pubspec.yaml depending on all desired source packages. |
| api.file.write_text('write pubspec.yaml', out_dir.join('pubspec.yaml'), |
| dart_pubspec_content) |
| api.file.write_text('write lib.dart', out_dir.join('lib', 'lib.dart'), |
| dart_imports_content) |
| |
| # Run pub over this package to fetch deps. |
| with api.context(cwd=out_dir): |
| api.step('pub', [cipd_dir.join('dart-sdk', 'bin', 'pub'), 'get']) |
| |
| # Run dartdoc over this package. |
| with api.context(cwd=out_dir): |
| api.step('dartdoc', [ |
| cipd_dir.join('dart-sdk', 'bin', 'dartdoc'), |
| '--auto-include-dependencies', |
| '--exclude-packages', |
| 'Dart', |
| '--output', |
| docs_dir.join('public', 'dart'), |
| ]) |
| |
| |
| def gen_rustdoc(api, docs_dir, build_dir, gn_results): |
| """Generate rust output. |
| |
| The rustdoc script runs on GN targets. We find the Rust GN targets by finding |
| all targets that generate a Cargo.toml file, and use those. |
| |
| Args: |
| docs_dir (Path) - The output directory for documentation. |
| build_dir (Path) - The build directory. |
| gn_results (Object) - Result of a `gn gen` invocation in the fuchsia build. |
| """ |
| step_result = api.step( |
| 'gn desc', [ |
| gn_results.tool('gn'), |
| 'desc', |
| build_dir, |
| '//*', |
| 'outputs', |
| '--type=action', |
| '--format=json', |
| ], |
| stdout=api.json.output()) |
| |
| skipped = [] |
| test_cargo_path = build_dir.join('target', 'Cargo.toml') |
| api.path.mock_add_paths(test_cargo_path) |
| for target in step_result.stdout: |
| if not 'outputs' in step_result.stdout[target]: |
| continue |
| outputs = step_result.stdout[target]['outputs'] |
| # If the target doesn't output a Cargo.toml file, it's not a rust target. |
| if not outputs or not outputs[0].endswith('Cargo.toml'): |
| continue |
| |
| output = api.path['start_dir'].join(*outputs[0].split('/')) |
| if not api.path.exists(output): |
| skipped.append(target) |
| continue |
| |
| with api.context(env={ |
| 'FUCHSIA_DIR': api.path['start_dir'], |
| 'FUCHSIA_BUILD_DIR': build_dir, |
| }): |
| api.step('rustdoc %s' % target, [ |
| api.path['start_dir'].join('tools', 'devshell', 'contrib', 'lib', |
| 'rust', 'rustdoc.py'), |
| output, |
| '--no-deps', |
| '--out-dir', |
| build_dir, |
| ]) |
| |
| with api.context( |
| env={ |
| 'RUSTC': gn_results.tool('rustc'), |
| 'RUSTDOC': gn_results.tool('rustdoc') |
| }, |
| cwd=api.path['start_dir'].join('third_party', 'rust_crates')): |
| api.step('cargo doc third_party/rust_crates', |
| [gn_results.tool('cargo'), 'doc', '--target=x86_64-fuchsia']) |
| |
| # Move the output to the docs directory. |
| step_result = api.step('move output to docs', [ |
| 'mv', |
| api.path['start_dir'].join('out', 'cargo_target', 'x86_64-fuchsia', |
| 'doc'), |
| docs_dir.join('public', 'rust'), |
| ]) |
| |
| step_result.presentation.logs['skipped'] = skipped |
| |
| |
| def gen_fidldoc(api, build_dir, output_name, dry_run): |
| """Generate fidl output. |
| |
| The fidldoc tool runs on the all_fidl_json.txt file. Pushes the resulting |
| docs to the given docs repository. |
| |
| Args: |
| docs_dir (Path) - The output directory for documentation (repository). |
| build_dir (Path) - The build directory. |
| output_name (str) - Output name of the directory. |
| dry_run (str) - Run but don't push. |
| """ |
| out_dir = api.path['start_dir'].join('fidldoc_out') |
| all_fidl_json_txt = build_dir.join('all_fidl_json.txt') |
| all_fidl_json = api.file.read_text('read all_fidl_json.txt', |
| all_fidl_json_txt).splitlines() |
| |
| with api.context(cwd=build_dir): |
| # Cannot use tool_paths.json here, since it maps to host_x64, which |
| # doesn't work for fidldoc because of the accompanying fidldoc.config.json |
| # that is only in host-tools. |
| fidldoc_path = build_dir.join('host-tools/fidldoc') |
| api.step('run fidldoc', [ |
| fidldoc_path, '--verbose', '--path', '/reference/fidl/', '--out', |
| out_dir |
| ] + all_fidl_json) |
| |
| # Push resulting docs to the reference-docs repository. |
| |
| # The fidldocs get checked into their own repository. Checking it out here reduces the likelihood |
| # of a race with another bot trying to commit in between this bot's checkout and commit steps. |
| fidldoc_docs_dir = api.path['start_dir'].join('reference-docs') |
| with api.step.nest('checkout fidldoc'): |
| api.git.checkout(REFERENCE_DOCS_REPOSITORY, path=fidldoc_docs_dir) |
| |
| with api.context(cwd=fidldoc_docs_dir): |
| # Clear the repository and move results in. |
| api.file.rmtree( |
| name='Clear reference-docs/out', |
| source=fidldoc_docs_dir.join(output_name)) |
| api.file.move( |
| name='Move docs to reference-docs/out', |
| source=out_dir, |
| dest=fidldoc_docs_dir.join(output_name, 'fidl')) |
| # Add all modified files to git cache |
| api.git('add', '-A') |
| # Only commit and push if there's a diff, otherwise commit fails. |
| if api.git('diff', '--cached', '--exit-code', ok_ret='any').retcode: |
| api.git.commit(FIDLDOC_COMMIT_MESSAGE) |
| if not dry_run: |
| api.git.push(ref='HEAD:master') |
| |
| |
| def gen_clang_doc(api, docs_dir, build_dir, gn_results): |
| """Generate clang-doc output. |
| |
| clang-doc runs on the translation units specified in the compilation database |
| file. This file will be generated by gn_results after filtering unwanted |
| directories or files. clang-doc will output documentation files directly in |
| docs_dir. |
| |
| Args: |
| docs_dir (Path) - The output directory for documentation. |
| build_dir (Path) - The build directory. |
| gn_results (Object) - Result of a `gn gen` invocation in the fuchsia build. |
| """ |
| with api.step.nest('filter compile commands'): |
| white_list_dirs = ['third_party'] |
| compile_commands = gn_results.filtered_compdb(white_list_dirs) |
| |
| with api.context(cwd=build_dir): |
| api.step('run clang-doc', [ |
| gn_results.tool('clang-doc'), '--output', |
| docs_dir.join('public', 'cpp'), '--public', '--format=html', |
| compile_commands |
| ]) |
| |
| |
| def RunSteps(api, dry_run, manifest, remote, target, build_type, packages, |
| generators, output_name): |
| cipd_dir = api.path['start_dir'].join('cipd') |
| node_modules_dir = cipd_dir.join('node_modules') |
| with api.step.nest('ensure_packages'): |
| with api.context(infra_steps=True): |
| pkgs = api.cipd.EnsureFile() |
| pkgs.add_package('infra/nodejs/nodejs/${platform}', 'latest') |
| if 'dartdoc' in generators: |
| pkgs.add_package('dart/dart-sdk/${platform}', 'dev') |
| api.cipd.ensure(cipd_dir, pkgs) |
| |
| # firebase-tools expects to live in the node_modules subdir of where nodejs is installed. |
| pkgs = api.cipd.EnsureFile() |
| pkgs.add_package('infra/npm/firebase-tools', 'latest') |
| api.cipd.ensure(node_modules_dir, pkgs) |
| |
| resources_dir = api.path['start_dir'].join('api-docs-resources') |
| with api.step.nest('checkout api docs'): |
| api.git.checkout(API_DOCS_RESOURCES_REPOSITORY, path=resources_dir) |
| |
| checkout_root = api.path['start_dir'] |
| with api.step.nest('checkout fuchsia'): |
| checkout = api.checkout.fuchsia_with_options( |
| path=checkout_root, |
| manifest=manifest, |
| remote=remote, |
| build=api.buildbucket.build, |
| ) |
| |
| gn_results = api.build.gen( |
| checkout_root=checkout.root_dir, |
| fuchsia_build_dir=api.path['start_dir'].join('out', 'default'), |
| target=target, |
| build_type=build_type, |
| board='boards/%s.gni' % target, |
| product='products/core.gni', |
| packages=packages + ['//tools/fidl/fidldoc'], |
| export_compdb=True, |
| ) |
| |
| api.build.ninja( |
| checkout_root=checkout.root_dir, |
| gn_results=gn_results, |
| ) |
| |
| out_dir = api.path['start_dir'].join('docs_out') |
| docs_dir = api.path['start_dir'].join('firebase') |
| |
| api.file.rmtree('remove old docs', docs_dir) |
| api.file.copytree('copy resources', resources_dir, docs_dir) |
| |
| if 'clangdoc' in generators: |
| with api.step.nest('clangdoc'): |
| gen_clang_doc(api, docs_dir, api.path['start_dir'], gn_results) |
| if 'dartdoc' in generators: |
| with api.step.nest('dartdoc'): |
| gen_dartdoc(api, out_dir, docs_dir, cipd_dir) |
| if 'fidldoc' in generators: |
| with api.step.nest('fidldoc'): |
| gen_fidldoc(api, gn_results.fuchsia_build_dir, output_name, dry_run) |
| if 'rustdoc' in generators: |
| with api.step.nest('rustdoc'): |
| gen_rustdoc(api, docs_dir, gn_results.fuchsia_build_dir, gn_results) |
| |
| # Only deploy if running the default 'all' one, to avoid deploying partial docs to firebase. |
| if not dry_run and output_name == 'all': |
| with api.context(cwd=docs_dir, env={'PATH': cipd_dir.join('bin')}): |
| api.step('firebase deploy', [ |
| node_modules_dir.join('.bin', 'firebase'), |
| 'deploy', |
| '--only', |
| 'hosting', |
| '--debug', |
| ]) |
| |
| |
| def GenTests(api): |
| yield ( |
| api.test('firebase_docs') + |
| api.buildbucket.ci_build( |
| git_repo='https://fuchsia.googlesource.com/topaz',) + |
| api.step_data( |
| 'dartdoc.load fuchsia pubspec.yaml', |
| stdout=api.json.output({'name': 'fuchsia'})) + |
| api.step_data( |
| 'dartdoc.list fuchsia packages', |
| api.file.listdir(['fuchsia.dart'])) + |
| api.step_data( |
| 'rustdoc.gn desc', |
| stdout=api.json.output({ |
| '//topaz/target:target_cargo': { |
| 'outputs': ['//out/default/target/Cargo.toml'] |
| }, |
| '//topaz/not_target:not_target': { |
| 'outputs': |
| ['//out/default/not_target.h', '//out/x64/not_target.cc'] |
| }, |
| '//topaz/target:missing_outputs': { |
| 'outputs': ['//out/default/missing/Cargo.toml'] |
| }, |
| '//topaz/not_target:no_outputs': {} |
| })) + |
| api.step_data('fidldoc.read all_fidl_json.txt', |
| api.raw_io.output('foo.fidl\nbar.fidl\n')) + |
| api.step_data('fidldoc.git diff', retcode=1)) # yapf: disable |