recipes/contrib/ffmpeg.py - infra/recipes.git - Git at Google

 # Copyright 2018 The Fuchsia Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 """Recipe for building FFmpeg and uploading it and required source files."""

 from PB.recipes.fuchsia.contrib.ffmpeg import InputProperties

 DEPS = [
     "fuchsia/artifacts",
     "fuchsia/build",
     "fuchsia/buildbucket_util",
     "fuchsia/cas_util",
     "fuchsia/checkout",
     "fuchsia/cipd_util",
     "recipe_engine/context",
     "recipe_engine/file",
     "recipe_engine/path",
     "recipe_engine/properties",
     "recipe_engine/step",
 ]

 PROPERTIES = InputProperties

 # Patterns of source code to include in the archive that this recipe produces.
 # All relative to third_party/ffmpeg/src.
 SRC_SOURCE_PATTERN = "lib*/*.h"

 # Patterns of source code to include in the archive that this recipe produces.
 # All relative to build/secondary/third_party/ffmpeg.
 SECONDARY_SOURCE_PATTERN = "config/**/*"


 def RunSteps(api, props):
     checkout = api.checkout.fuchsia_with_options(
         manifest=props.manifest,
         remote=props.remote,
     )

     staging_dir = api.path.mkdtemp("ffmpeg")
     pkg_dir = staging_dir.join("pkg")
     debug_dir = staging_dir.join("debug")
     prebuilt_variants = set()

     # Build and archive for all targets before uploading any to avoid an
     # incomplete upload.
     for idx, build in enumerate(props.builds):
         api.artifacts.gcs_bucket = props.artifact_gcs_bucket
         with api.step.nest(f"{build.target}-{build.profile}"):
             # Create a unique namespace per-build since we may build and upload
             # more than once.
             api.artifacts.namespace = f"{api.buildbucket_util.id}_{int(idx)}"
             build_results = api.build.with_options(
                 checkout=checkout,
                 fint_params_path=build.fint_params_path,
                 artifact_gcs_bucket=props.artifact_gcs_bucket,
                 upload_namespace=api.artifacts.namespace,
             )
             if props.artifact_gcs_bucket:
                 build_results.upload_artifacts()

             with api.context(infra_steps=True):
                 shared_build_dir = build_results.build_dir.join(
                     f"{build.target}-novariant-shared"
                 )
                 lib_dir = pkg_dir.join(build.profile, build.target, "lib")
                 api.file.ensure_directory("create lib dir", lib_dir)
                 api.file.copy(
                     "copy libffmpeg.so", shared_build_dir.join("libffmpeg.so"), lib_dir
                 )
                 build_id = api.file.read_text(
                     "read stamp file",
                     shared_build_dir.join("libffmpeg.so.build-id.stamp"),
                     test_data="f01cc36fed7d1a4f",
                 )
                 build_id_dir = debug_dir.join(build_id[:2])
                 api.file.ensure_directory("create debug dir", build_id_dir)
                 api.file.copy(
                     "copy libffmpeg.so (debug)",
                     shared_build_dir.join("lib.unstripped", "libffmpeg.so"),
                     build_id_dir.join(build_id[2:] + ".debug"),
                 )

                 for variant in build_results.set_metadata.variants:
                     prebuilt_variants.add(variant)
                     with api.step.nest(variant):
                         shared_variant_build_dir = build_results.build_dir.join(
                             f"{build.target}-{variant}-shared"
                         )
                         lib_variant_dir = lib_dir.join(variant)
                         api.file.ensure_directory("create lib dir", lib_variant_dir)
                         api.file.copy(
                             "copy libffmpeg.so",
                             shared_variant_build_dir.join("libffmpeg.so"),
                             lib_variant_dir,
                         )
                         build_id = api.file.read_text(
                             "read stamp file",
                             shared_variant_build_dir.join(
                                 "libffmpeg.so.build-id.stamp"
                             ),
                             test_data="f01cc36fed7d1a4f",
                         )
                         build_id_dir = debug_dir.join(build_id[:2])
                         api.file.ensure_directory("create debug dir", build_id_dir)
                         api.file.copy(
                             "copy libffmpeg.so (debug)",
                             shared_variant_build_dir.join(
                                 "lib.unstripped", "libffmpeg.so"
                             ),
                             build_id_dir.join(build_id[2:] + ".debug"),
                         )

     ffmpeg_src_dir = checkout.root_dir.join("third_party", "ffmpeg", "src")
     include_dir = pkg_dir.join("pkg", "ffmpeg", "include")

     # Copy include files from third_party/ffmpeg/src (Chromium ffmpeg repo).
     with api.step.nest("copy headers"):
         paths = api.file.glob_paths(
             "glob src paths",
             ffmpeg_src_dir,
             SRC_SOURCE_PATTERN,
             test_data=["libavcodec/avcodec.h"],
         )
         # Avoid ensuring directories more than once. Step overhead is about 125ms,
         # and some directories are ensured > 500 times.
         ensured = set()
         for srcpath in paths:
             if api.path.isdir(srcpath):  # pragma: no cover
                 continue
             relpath = api.path.relpath(srcpath, ffmpeg_src_dir)
             dstpath = include_dir.join(relpath)
             dirname = api.path.dirname(dstpath)
             if not dirname in ensured:
                 api.file.ensure_directory(f"ensure {dirname}", dirname)
                 ensured.add(dirname)
             api.file.copy(f"copy {relpath}", srcpath, dstpath)

     ffmpeg_secondary_dir = checkout.root_dir.join(
         "build", "secondary", "third_party", "ffmpeg"
     )
     include_fuchsia_dir = pkg_dir.join("pkg", "ffmpeg", "include", "fuchsia")

     # Copy include files from build/secondary/third_party/ffmpeg (Fuchsia ffmpeg repo).
     # We copy to ffmpeg/include/fuchsia (rather than ffmpeg/include) to match the previous
     # layout.
     with api.step.nest("copy secondary headers"):
         paths = api.file.glob_paths(
             "glob src files",
             ffmpeg_secondary_dir,
             SECONDARY_SOURCE_PATTERN,
             test_data=["config/default/x64/config.h"],
         )
         # Avoid ensuring directories more than once. Step overhead is about 125ms,
         # and some directories are ensured > 500 times.
         ensured = set()
         for srcpath in paths:
             if api.path.isdir(srcpath):  # pragma: no cover
                 continue
             relpath = api.path.relpath(srcpath, ffmpeg_secondary_dir)
             dstpath = include_fuchsia_dir.join(relpath)
             if not dirname in ensured:
                 dirname = api.path.dirname(dstpath)
                 api.file.ensure_directory(f"ensure {dirname}", dirname)
             api.file.copy(f"copy {relpath}", srcpath, dstpath)

     docs_dir = pkg_dir.join("pkg", "ffmpeg", "docs")
     api.file.ensure_directory("create docs dir", docs_dir)

     # Copy CREDITS.fuchsia and LICENSE.md from build/secondary/third_party/ffmpeg. If the Fuchsia
     # ffmpeg repo moves to third_party/ffmpeg, this must be changed to third_party/ffmpeg.
     api.file.copy(
         "copy credits",
         ffmpeg_secondary_dir.join("CREDITS.fuchsia"),
         docs_dir.join("CREDITS.fuchsia"),
     )
     api.file.copy(
         "copy license",
         ffmpeg_secondary_dir.join("LICENSE.md"),
         docs_dir.join("LICENSE.md"),
     )

     # Create prebuilt_variants.json that includes a list of prebuilt variants.
     prebuilt_variants_dest = pkg_dir.join("prebuilt_variants.json")
     api.file.write_json(
         "create prebuilt_variants.json",
         prebuilt_variants_dest,
         {"prebuilt_variants": sorted(prebuilt_variants)},
     )

     # Upload files to CAS for debugging.
     api.cas_util.upload(pkg_dir, step_name="archive pkg")
     api.cas_util.upload(debug_dir, step_name="archive debug")

     if api.buildbucket_util.is_dev_or_try:
         return

     with api.context(infra_steps=True):
         project = checkout.project("integration")

     # Upload files to CIPD.
     api.cipd_util.upload_package(
         "fuchsia/lib/ffmpeg/fuchsia",
         pkg_dir,
         search_tag={"git_revision": project["revision"]},
         repository=project["remote"],
     )
     api.cipd_util.upload_package(
         "fuchsia/lib/ffmpeg/fuchsia-debug-symbols",
         debug_dir,
         search_tag={"git_revision": project["revision"]},
         repository=project["remote"],
     )


 def GenTests(api):
     def properties():
         return api.properties(
             manifest="third_party/ffmpeg",
             remote="https://fuchsia.googlesource.com/integration",
             builds=[
                 {
                     "profile": "default",
                     "target": "arm64",
                     "fint_params_path": "fint_params/ffmpeg-arm64.textproto",
                 },
                 {
                     "profile": "max",
                     "target": "arm64",
                     "fint_params_path": "fint_params/ffmpeg-arm64.textproto",
                 },
                 {
                     "profile": "default",
                     "target": "x64",
                     "fint_params_path": "fint_params/ffmpeg-x64.textproto",
                 },
                 {
                     "profile": "max",
                     "target": "x64",
                     "fint_params_path": "fint_params/ffmpeg-x64.textproto",
                 },
             ],
             artifact_gcs_bucket="fuchsia-artifacts",
         )

     yield (
         api.buildbucket_util.test("default", repo="third_party/ffmpeg") + properties()
     )
     yield (
         api.buildbucket_util.test("cq", tryjob=True, repo="third_party/ffmpeg")
         + properties()
     )
	# Copyright 2018 The Fuchsia Authors. All rights reserved.
	# Use of this source code is governed by a BSD-style license that can be
	# found in the LICENSE file.
	"""Recipe for building FFmpeg and uploading it and required source files."""

	from PB.recipes.fuchsia.contrib.ffmpeg import InputProperties

	DEPS = [
	"fuchsia/artifacts",
	"fuchsia/build",
	"fuchsia/buildbucket_util",
	"fuchsia/cas_util",
	"fuchsia/checkout",
	"fuchsia/cipd_util",
	"recipe_engine/context",
	"recipe_engine/file",
	"recipe_engine/path",
	"recipe_engine/properties",
	"recipe_engine/step",
	]

	PROPERTIES = InputProperties

	# Patterns of source code to include in the archive that this recipe produces.
	# All relative to third_party/ffmpeg/src.
	SRC_SOURCE_PATTERN = "lib/.h"

	# Patterns of source code to include in the archive that this recipe produces.
	# All relative to build/secondary/third_party/ffmpeg.
	SECONDARY_SOURCE_PATTERN = "config/*/"


	def RunSteps(api, props):
	checkout = api.checkout.fuchsia_with_options(
	manifest=props.manifest,
	remote=props.remote,
	)

	staging_dir = api.path.mkdtemp("ffmpeg")
	pkg_dir = staging_dir.join("pkg")
	debug_dir = staging_dir.join("debug")
	prebuilt_variants = set()

	# Build and archive for all targets before uploading any to avoid an
	# incomplete upload.
	for idx, build in enumerate(props.builds):
	api.artifacts.gcs_bucket = props.artifact_gcs_bucket
	with api.step.nest(f"{build.target}-{build.profile}"):
	# Create a unique namespace per-build since we may build and upload
	# more than once.
	api.artifacts.namespace = f"{api.buildbucket_util.id}_{int(idx)}"
	build_results = api.build.with_options(
	checkout=checkout,
	fint_params_path=build.fint_params_path,
	artifact_gcs_bucket=props.artifact_gcs_bucket,
	upload_namespace=api.artifacts.namespace,
	)
	if props.artifact_gcs_bucket:
	build_results.upload_artifacts()

	with api.context(infra_steps=True):
	shared_build_dir = build_results.build_dir.join(
	f"{build.target}-novariant-shared"
	)
	lib_dir = pkg_dir.join(build.profile, build.target, "lib")
	api.file.ensure_directory("create lib dir", lib_dir)
	api.file.copy(
	"copy libffmpeg.so", shared_build_dir.join("libffmpeg.so"), lib_dir
	)
	build_id = api.file.read_text(
	"read stamp file",
	shared_build_dir.join("libffmpeg.so.build-id.stamp"),
	test_data="f01cc36fed7d1a4f",
	)
	build_id_dir = debug_dir.join(build_id[:2])
	api.file.ensure_directory("create debug dir", build_id_dir)
	api.file.copy(
	"copy libffmpeg.so (debug)",
	shared_build_dir.join("lib.unstripped", "libffmpeg.so"),
	build_id_dir.join(build_id[2:] + ".debug"),
	)

	for variant in build_results.set_metadata.variants:
	prebuilt_variants.add(variant)
	with api.step.nest(variant):
	shared_variant_build_dir = build_results.build_dir.join(
	f"{build.target}-{variant}-shared"
	)
	lib_variant_dir = lib_dir.join(variant)
	api.file.ensure_directory("create lib dir", lib_variant_dir)
	api.file.copy(
	"copy libffmpeg.so",
	shared_variant_build_dir.join("libffmpeg.so"),
	lib_variant_dir,
	)
	build_id = api.file.read_text(
	"read stamp file",
	shared_variant_build_dir.join(
	"libffmpeg.so.build-id.stamp"
	),
	test_data="f01cc36fed7d1a4f",
	)
	build_id_dir = debug_dir.join(build_id[:2])
	api.file.ensure_directory("create debug dir", build_id_dir)
	api.file.copy(
	"copy libffmpeg.so (debug)",
	shared_variant_build_dir.join(
	"lib.unstripped", "libffmpeg.so"
	),
	build_id_dir.join(build_id[2:] + ".debug"),
	)

	ffmpeg_src_dir = checkout.root_dir.join("third_party", "ffmpeg", "src")
	include_dir = pkg_dir.join("pkg", "ffmpeg", "include")

	# Copy include files from third_party/ffmpeg/src (Chromium ffmpeg repo).
	with api.step.nest("copy headers"):
	paths = api.file.glob_paths(
	"glob src paths",
	ffmpeg_src_dir,
	SRC_SOURCE_PATTERN,
	test_data=["libavcodec/avcodec.h"],
	)
	# Avoid ensuring directories more than once. Step overhead is about 125ms,
	# and some directories are ensured > 500 times.
	ensured = set()
	for srcpath in paths:
	if api.path.isdir(srcpath): # pragma: no cover
	continue
	relpath = api.path.relpath(srcpath, ffmpeg_src_dir)
	dstpath = include_dir.join(relpath)
	dirname = api.path.dirname(dstpath)
	if not dirname in ensured:
	api.file.ensure_directory(f"ensure {dirname}", dirname)
	ensured.add(dirname)
	api.file.copy(f"copy {relpath}", srcpath, dstpath)

	ffmpeg_secondary_dir = checkout.root_dir.join(
	"build", "secondary", "third_party", "ffmpeg"
	)
	include_fuchsia_dir = pkg_dir.join("pkg", "ffmpeg", "include", "fuchsia")

	# Copy include files from build/secondary/third_party/ffmpeg (Fuchsia ffmpeg repo).
	# We copy to ffmpeg/include/fuchsia (rather than ffmpeg/include) to match the previous
	# layout.
	with api.step.nest("copy secondary headers"):
	paths = api.file.glob_paths(
	"glob src files",
	ffmpeg_secondary_dir,
	SECONDARY_SOURCE_PATTERN,
	test_data=["config/default/x64/config.h"],
	)
	# Avoid ensuring directories more than once. Step overhead is about 125ms,
	# and some directories are ensured > 500 times.
	ensured = set()
	for srcpath in paths:
	if api.path.isdir(srcpath): # pragma: no cover
	continue
	relpath = api.path.relpath(srcpath, ffmpeg_secondary_dir)
	dstpath = include_fuchsia_dir.join(relpath)
	if not dirname in ensured:
	dirname = api.path.dirname(dstpath)
	api.file.ensure_directory(f"ensure {dirname}", dirname)
	api.file.copy(f"copy {relpath}", srcpath, dstpath)

	docs_dir = pkg_dir.join("pkg", "ffmpeg", "docs")
	api.file.ensure_directory("create docs dir", docs_dir)

	# Copy CREDITS.fuchsia and LICENSE.md from build/secondary/third_party/ffmpeg. If the Fuchsia
	# ffmpeg repo moves to third_party/ffmpeg, this must be changed to third_party/ffmpeg.
	api.file.copy(
	"copy credits",
	ffmpeg_secondary_dir.join("CREDITS.fuchsia"),
	docs_dir.join("CREDITS.fuchsia"),
	)
	api.file.copy(
	"copy license",
	ffmpeg_secondary_dir.join("LICENSE.md"),
	docs_dir.join("LICENSE.md"),
	)

	# Create prebuilt_variants.json that includes a list of prebuilt variants.
	prebuilt_variants_dest = pkg_dir.join("prebuilt_variants.json")
	api.file.write_json(
	"create prebuilt_variants.json",
	prebuilt_variants_dest,
	{"prebuilt_variants": sorted(prebuilt_variants)},
	)

	# Upload files to CAS for debugging.
	api.cas_util.upload(pkg_dir, step_name="archive pkg")
	api.cas_util.upload(debug_dir, step_name="archive debug")

	if api.buildbucket_util.is_dev_or_try:
	return

	with api.context(infra_steps=True):
	project = checkout.project("integration")

	# Upload files to CIPD.
	api.cipd_util.upload_package(
	"fuchsia/lib/ffmpeg/fuchsia",
	pkg_dir,
	search_tag={"git_revision": project["revision"]},
	repository=project["remote"],
	)
	api.cipd_util.upload_package(
	"fuchsia/lib/ffmpeg/fuchsia-debug-symbols",
	debug_dir,
	search_tag={"git_revision": project["revision"]},
	repository=project["remote"],
	)


	def GenTests(api):
	def properties():
	return api.properties(
	manifest="third_party/ffmpeg",
	remote="https://fuchsia.googlesource.com/integration",
	builds=[
	{
	"profile": "default",
	"target": "arm64",
	"fint_params_path": "fint_params/ffmpeg-arm64.textproto",
	},
	{
	"profile": "max",
	"target": "arm64",
	"fint_params_path": "fint_params/ffmpeg-arm64.textproto",
	},
	{
	"profile": "default",
	"target": "x64",
	"fint_params_path": "fint_params/ffmpeg-x64.textproto",
	},
	{
	"profile": "max",
	"target": "x64",
	"fint_params_path": "fint_params/ffmpeg-x64.textproto",
	},
	],
	artifact_gcs_bucket="fuchsia-artifacts",
	)

	yield (
	api.buildbucket_util.test("default", repo="third_party/ffmpeg") + properties()
	)
	yield (
	api.buildbucket_util.test("cq", tryjob=True, repo="third_party/ffmpeg")
	+ properties()
	)