| // Package digest contains functions to simplify handling content digests. |
| package digest |
| |
| import ( |
| "crypto" |
| "encoding/hex" |
| "fmt" |
| "io" |
| "os" |
| "regexp" |
| "strconv" |
| "strings" |
| "sync" |
| |
| "github.com/golang/protobuf/proto" |
| |
| repb "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" |
| ) |
| |
| var ( |
| // hexStringRegex doesn't contain the size because that's checked separately. |
| hexStringRegex = regexp.MustCompile("^[a-f0-9]+$") |
| |
| // The digest function used. |
| HashFn = crypto.SHA256 |
| |
| // Empty is the digest of the empty blob. |
| Empty = NewFromBlob([]byte{}) |
| |
| // copyBufs is a pool of 32KiB []byte slices, used to compute hashes. |
| copyBufs = sync.Pool{ |
| New: func() interface{} { |
| buf := make([]byte, 32*1024) |
| return &buf |
| }, |
| } |
| ) |
| |
| // Digest is a Go type to mirror the repb.Digest message. |
| type Digest struct { |
| Hash string |
| Size int64 |
| } |
| |
| // GetDigestFunction returns the digest function used by the client. |
| func GetDigestFunction() repb.DigestFunction_Value { |
| // TODO: replace this line after all clients support Go 1.15 where .String() is defined: |
| // name := strings.ReplaceAll(HashFn.String(), "-", "") |
| name := "SHA256" |
| if val, ok := repb.DigestFunction_Value_value[name]; ok { |
| return repb.DigestFunction_Value(val) |
| } |
| return repb.DigestFunction_UNKNOWN |
| } |
| |
| // ToProto converts a Digest into a repb.Digest. No validation is performed! |
| func (d Digest) ToProto() *repb.Digest { |
| return &repb.Digest{Hash: d.Hash, SizeBytes: d.Size} |
| } |
| |
| // String returns a hash in a canonical form of hash/size. |
| func (d Digest) String() string { |
| return fmt.Sprintf("%s/%d", d.Hash, d.Size) |
| } |
| |
| // IsEmpty returns true iff digest is of an empty blob. |
| func (d Digest) IsEmpty() bool { |
| return d.Size == 0 && d.Hash == Empty.Hash |
| } |
| |
| // Validate returns nil if a digest appears to be valid, or a descriptive error |
| // if it is not. All functions accepting digests directly from clients should |
| // call this function, whether it's via an RPC call or by reading a serialized |
| // proto message that contains digests that was uploaded directly from the |
| // client. |
| func (d Digest) Validate() error { |
| length := len(d.Hash) |
| if length != HashFn.Size()*2 { |
| return fmt.Errorf("valid hash length is %d, got length %d (%s)", HashFn.Size()*2, length, d.Hash) |
| } |
| if !hexStringRegex.MatchString(d.Hash) { |
| return fmt.Errorf("hash is not a lowercase hex string (%s)", d.Hash) |
| } |
| if d.Size < 0 { |
| return fmt.Errorf("expected non-negative size, got %d", d.Size) |
| } |
| return nil |
| } |
| |
| // New creates a new digest from a string and size. It does some basic |
| // validation, which makes it marginally superior to constructing a Digest |
| // yourself. It returns an error if the hash/size are invalid. |
| func New(hash string, size int64) (Digest, error) { |
| d := Digest{Hash: hash, Size: size} |
| return d, d.Validate() |
| } |
| |
| // NewFromBlob takes a blob (in the form of a byte array) and returns the |
| // Digest for that blob. Changing this function will lead to cache |
| // invalidations (execution cache and potentially others). |
| // This cannot return an error, since the result is valid by definition. |
| func NewFromBlob(blob []byte) Digest { |
| h := HashFn.New() |
| h.Write(blob) |
| arr := h.Sum(nil) |
| return Digest{Hash: hex.EncodeToString(arr[:]), Size: int64(len(blob))} |
| } |
| |
| // NewFromMessage calculates the digest of a protobuf in SHA-256 mode. |
| // It returns an error if the proto marshalling failed. |
| func NewFromMessage(msg proto.Message) (Digest, error) { |
| blob, err := proto.Marshal(msg) |
| if err != nil { |
| return Empty, err |
| } |
| return NewFromBlob(blob), nil |
| } |
| |
| // NewFromProto converts a proto digest to a Digest. |
| // It returns an error if the hash/size are invalid. |
| func NewFromProto(dg *repb.Digest) (Digest, error) { |
| d := NewFromProtoUnvalidated(dg) |
| return d, d.Validate() |
| } |
| |
| // NewFromProtoUnvalidated converts a proto digest to a Digest, skipping validation. |
| func NewFromProtoUnvalidated(dg *repb.Digest) Digest { |
| return Digest{Hash: dg.Hash, Size: dg.SizeBytes} |
| } |
| |
| // NewFromString returns a digest from a canonical digest string. |
| // It returns an error if the hash/size are invalid. |
| func NewFromString(s string) (Digest, error) { |
| pair := strings.Split(s, "/") |
| if len(pair) != 2 { |
| return Empty, fmt.Errorf("expected digest in the form hash/size, got %s", s) |
| } |
| size, err := strconv.ParseInt(pair[1], 10, 64) |
| if err != nil { |
| return Empty, fmt.Errorf("invalid size in digest %s: %s", s, err) |
| } |
| return New(pair[0], size) |
| } |
| |
| // NewFromFile computes a file digest from a path. |
| // It returns an error if there was a problem accessing the file. |
| func NewFromFile(path string) (Digest, error) { |
| f, err := os.Open(path) |
| if err != nil { |
| return Empty, err |
| } |
| defer f.Close() |
| return NewFromReader(f) |
| } |
| |
| // NewFromReader computes a file digest from a reader. |
| // It returns an error if there was a problem reading the file. |
| func NewFromReader(r io.Reader) (Digest, error) { |
| h := HashFn.New() |
| buf := copyBufs.Get().(*[]byte) |
| defer copyBufs.Put(buf) |
| size, err := io.CopyBuffer(h, r, *buf) |
| if err != nil { |
| return Empty, err |
| } |
| return Digest{ |
| Hash: hex.EncodeToString(h.Sum(nil)), |
| Size: size, |
| }, nil |
| } |
| |
| // CheckCapabilities returns an error if the digest function is not supported |
| // by the server. |
| func CheckCapabilities(caps *repb.ServerCapabilities) error { |
| fn := GetDigestFunction() |
| |
| if caps.ExecutionCapabilities != nil { |
| if serverFn := caps.ExecutionCapabilities.DigestFunction; serverFn != fn { |
| return fmt.Errorf("server requires %v, client uses %v", serverFn, fn) |
| } |
| } |
| |
| if caps.CacheCapabilities != nil { |
| cc := caps.CacheCapabilities |
| found := false |
| for _, serverFn := range cc.DigestFunctions { |
| if serverFn == fn { |
| found = true |
| break |
| } |
| } |
| if !found { |
| return fmt.Errorf("server requires one of %v, client uses %v", cc.DigestFunctions, fn) |
| } |
| } |
| |
| return nil |
| } |
| |
| // TestNew is like New but also pads your hash with zeros if it is shorter than the required length, |
| // and panics on error rather than returning the error. |
| // ONLY USE FOR TESTS. |
| func TestNew(hash string, size int64) Digest { |
| hashLen := HashFn.Size() * 2 |
| if len(hash) < hashLen { |
| hash = strings.Repeat("0", hashLen-len(hash)) + hash |
| } |
| |
| digest, err := New(hash, size) |
| if err != nil { |
| panic(err.Error()) |
| } |
| return digest |
| } |
| |
| // TestNewFromMessage is only suitable for testing and panics on error. |
| // ONLY USE FOR TESTS. |
| func TestNewFromMessage(msg proto.Message) Digest { |
| digest, err := NewFromMessage(msg) |
| if err != nil { |
| panic(err.Error()) |
| } |
| return digest |
| } |