blob: 77f3ee702ce4559c953534fdc7ab1bfdb3301fb8 [file]
#!/usr/bin/env python3
# Copyright 2022 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Generate a content hash file from one or more source repository content.
By default, scan all files in the source paths (file or directory) and hash
their content.
- For symlinks, the link value is used as input (not the target file). If the
link points to another symlink, the whole chain of link values is used.
- For regular files, its sha1 digest is used as input.
- For git directories (which have a .git entry), use the HEAD commit as input
to speed things dramatically. NOTE: This ignores changes to the index,
during development.
- If --cipd-name=NAME is set, and <source_path>/.versions/NAME.cipd_version
exists, its content will be used as input.
- Otherwise, for directories, all files in it are found recursively,
and used as hash input independently.
"""
import argparse
import hashlib
import os
import sys
import typing as T
from pathlib import Path
from build.bazel.scripts import get_git_head_commit as gghc
_HASH = "sha1"
def _depfile_quote(p: Path | str) -> str:
"""Quote a Path value for depfile output."""
return str(p).replace("\\", "\\\\").replace(" ", "\\ ")
def _next_symlink_hop(link_path: Path, target: Path) -> Path:
"""Return the absolute path of the entry a symlink points to.
All directory components of the result are resolved, so that a relative
target containing '..' is applied to the real directory containing the
link, not to a lexical path that may go through symlinked directories.
The final component is left unresolved so callers can inspect it to see
whether it is itself a symlink (i.e. the next hop in a symlink chain).
Args:
link_path: Path to a symlink.
target: The value of the symlink, as returned by readlink().
"""
# Note that os.path.join() ignores the first argument when target is
# absolute.
joined = os.path.join(os.path.dirname(link_path), target)
head, tail = os.path.split(joined)
if tail in ("", os.curdir, os.pardir):
# The target ends with a directory-only component (e.g. 'foo/' or
# '..'), which cannot be a symlink itself, so resolve it entirely.
return Path(os.path.realpath(joined))
return Path(os.path.realpath(head)) / tail
class FileState(object):
"""State object used to hash one or more source paths.
Usage is:
- Create instance
- Call hash_source_path() as many times as possible.
- Use content_hash property to get final result.
- Use sorted_input_files to get list of input files.
"""
def __init__(
self,
cipd_names: T.Sequence[str] = [],
exclude_suffixes: T.Sequence[str] = [],
git_binary: Path = Path("git"),
) -> None:
"""Create new instance.
Args:
cipd_names: A sequence of cipd names for prebuilt directories.
exclude_suffixes: A sequence of filename suffixes to exclude from hashing.
git_binary: Path to the git binary to use for .git repositories.
"""
self._cipd_names = cipd_names
self._exclude_suffixes = tuple(exclude_suffixes)
self._git_binary = git_binary
self._input_files: set[Path] = set()
self._sorted_input_files: list[str] | None = None
self._hstate = hashlib.new(_HASH)
def hash_source_path(self, source_path: Path) -> None:
"""Process and hash a given source file, updating internal state."""
self._hstate.update(self.process_source_path(source_path).encode())
def find_directory_files(self, source_path: Path) -> set[Path]:
source_path.is_dir(), f"Input source path is not a directory: {source_path}"
if self._cipd_names:
for cipd_name in self._cipd_names:
clang_version_file = (
source_path / ".versions" / f"{cipd_name}.cipd_version"
)
if clang_version_file.exists():
return set([clang_version_file])
# Find all files in direcrory.
dir_files: set[Path] = set()
for dirpath, dirnames, filenames in os.walk(source_path):
for filename in filenames:
if filename.endswith(self._exclude_suffixes):
continue
file_path = Path(os.path.join(dirpath, filename))
dir_files.add(file_path)
return dir_files
def process_source_path(self, source_path: Path) -> str:
"""Process a given source file, and return a string descriptor for it.
The first letter of the result corresponds to the type of the source path.
This function is useful for unit-testing the implementation and verify
that different types of source paths are handled correctly. Apart from
that, consider this as an implementation detail.
"""
if not source_path.exists():
raise ValueError(f"Path does not exist: {source_path}")
if source_path.is_dir() and (source_path / ".git").exists():
head_commit = gghc.get_git_head_commit(
source_path, self._git_binary
)
self._input_files.update(gghc.find_git_head_inputs(source_path))
return "G" + head_commit
if source_path.is_symlink():
# Follow the whole symlink chain, so that every intermediate link
# is an input (a change to any of them must trigger a rehash) and
# every link value is part of the descriptor. Only inspecting the
# first hop would miss changes to the target of a chained link.
#
# Intermediate links are recorded relative to the current directory
# (like source paths rebased on the build directory) unless
# source_path is absolute. The chain is finite because
# source_path.exists() is checked above, which fails on loops.
link_path = source_path
targets = []
while link_path.is_symlink():
self._input_files.add(link_path)
target = link_path.readlink()
targets.append(str(target))
next_hop = _next_symlink_hop(link_path, target)
if source_path.is_absolute():
link_path = next_hop
else:
link_path = Path(os.path.relpath(next_hop))
return "S" + "->".join(targets)
if source_path.is_file():
self._input_files.add(source_path)
with source_path.open("rb") as f:
digest = hashlib.file_digest(f, _HASH)
return "F" + digest.hexdigest()
assert source_path.is_dir(), f"Unexpected file type for {source_path}"
if self._cipd_names:
for cipd_name in self._cipd_names:
clang_version_file = (
source_path / ".versions" / f"{cipd_name}.cipd_version"
)
if clang_version_file.exists():
return self.process_source_path(clang_version_file)
# Get the list of files relative to the source directory.
dir_files: list[str] = [
os.path.relpath(f, source_path)
for f in self.find_directory_files(source_path)
]
# Process them recursively to build a directory description text
# where each line looks like: <file> <type><digest>
dir_content = "D\n"
for dir_file in sorted(dir_files):
file_hash = self.process_source_path(source_path / dir_file)
dir_content += f" {dir_file} {file_hash}\n"
return dir_content
@property
def content_hash(self) -> str:
"""Return final content hash as hexadecimal string."""
return self._hstate.hexdigest()
@property
def sorted_input_files(self) -> list[str]:
"""Return the list of input files used by this instance."""
if self._sorted_input_files is None:
self._sorted_input_files = sorted(
[str(p) for p in self._input_files]
)
return self._sorted_input_files
def get_input_file_paths(self) -> set[Path]:
"""Return the set of input file Path values used by this instance."""
return self._input_files
def main() -> int:
parser = argparse.ArgumentParser(
description=__doc__, formatter_class=argparse.RawTextHelpFormatter
)
parser.add_argument(
"--cipd-name",
action="append",
default=[],
help="Provide name for optional CIPD version file. Can be used multiple times.",
)
parser.add_argument(
"--exclude-suffix",
action="append",
default=[],
help='Exclude directory entries with given suffix (e.g. ".pyc").\n'
+ "Can be used multiple times.",
)
parser.add_argument(
"--git-binary",
type=Path,
default=Path("git"),
help="Specify git binary to use for git repositories.",
)
parser.add_argument(
"source_path",
type=Path,
nargs="+",
help="Source file or directory path.",
)
parser.add_argument(
"--output", type=Path, help="Optional output file path."
)
parser.add_argument(
"--depfile", type=Path, help="Optional Ninja depfile output file path."
)
parser.add_argument(
"--inputs-list",
type=Path,
help="Write list of inputs to file, one path per line.",
)
args = parser.parse_args()
if args.depfile and not args.output:
parser.error("--depfile option requires --output.")
fstate = FileState(args.cipd_name, args.exclude_suffix, args.git_binary)
for source_path in args.source_path:
fstate.hash_source_path(source_path)
if args.output:
# Do not modify existing output if it has the same content.
current_content = "~~~"
if args.output.exists():
current_content = args.output.read_text()
if current_content != fstate.content_hash:
args.output.write_text(fstate.content_hash)
else:
print(fstate.content_hash)
if args.inputs_list:
args.inputs_list.write_text("\n".join(fstate.sorted_input_files) + "\n")
if args.depfile:
args.depfile.write_text(
"%s: \\\n %s\n"
% (
args.output,
" \\\n ".join(
_depfile_quote(f) for f in fstate.sorted_input_files
),
)
)
return 0
if __name__ == "__main__":
sys.exit(main())