| // Copyright 2020 Google LLC |
| // |
| // Licensed under the Apache License, Version 2.0 (the "License"); |
| // you may not use this file except in compliance with the License. |
| // You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, software |
| // distributed under the License is distributed on an "AS IS" BASIS, |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| // See the License for the specific language governing permissions and |
| // limitations under the License. |
| |
| package storage |
| |
| import ( |
| "crypto" |
| "crypto/rand" |
| "crypto/rsa" |
| "crypto/sha256" |
| "encoding/base64" |
| "encoding/json" |
| "errors" |
| "fmt" |
| "net/url" |
| "strings" |
| "time" |
| ) |
| |
| // PostPolicyV4Options are used to construct a signed post policy. |
| // Please see https://cloud.google.com/storage/docs/xml-api/post-object |
| // for reference about the fields. |
| type PostPolicyV4Options struct { |
| // GoogleAccessID represents the authorizer of the signed post policy generation. |
| // It is typically the Google service account client email address from |
| // the Google Developers Console in the form of "xxx@developer.gserviceaccount.com". |
| // Required. |
| GoogleAccessID string |
| |
| // PrivateKey is the Google service account private key. It is obtainable |
| // from the Google Developers Console. |
| // At https://console.developers.google.com/project/<your-project-id>/apiui/credential, |
| // create a service account client ID or reuse one of your existing service account |
| // credentials. Click on the "Generate new P12 key" to generate and download |
| // a new private key. Once you download the P12 file, use the following command |
| // to convert it into a PEM file. |
| // |
| // $ openssl pkcs12 -in key.p12 -passin pass:notasecret -out key.pem -nodes |
| // |
| // Provide the contents of the PEM file as a byte slice. |
| // Exactly one of PrivateKey or SignBytes must be non-nil. |
| PrivateKey []byte |
| |
| // SignBytes is a function for implementing custom signing. |
| // |
| // Deprecated: Use SignRawBytes. If both SignBytes and SignRawBytes are defined, |
| // SignBytes will be ignored. |
| // This SignBytes function expects the bytes it receives to be hashed, while |
| // SignRawBytes accepts the raw bytes without hashing, allowing more flexibility. |
| // Add the following to the top of your signing function to hash the bytes |
| // to use SignRawBytes instead: |
| // shaSum := sha256.Sum256(bytes) |
| // bytes = shaSum[:] |
| // |
| SignBytes func(hashBytes []byte) (signature []byte, err error) |
| |
| // SignRawBytes is a function for implementing custom signing. For example, if |
| // your application is running on Google App Engine, you can use |
| // appengine's internal signing function: |
| // ctx := appengine.NewContext(request) |
| // acc, _ := appengine.ServiceAccount(ctx) |
| // &PostPolicyV4Options{ |
| // GoogleAccessID: acc, |
| // SignRawBytes: func(b []byte) ([]byte, error) { |
| // _, signedBytes, err := appengine.SignBytes(ctx, b) |
| // return signedBytes, err |
| // }, |
| // // etc. |
| // }) |
| // |
| // SignRawBytes is equivalent to the SignBytes field on SignedURLOptions; |
| // that is, you may use the same signing function for the two. |
| // |
| // Exactly one of PrivateKey or SignRawBytes must be non-nil. |
| SignRawBytes func(bytes []byte) (signature []byte, err error) |
| |
| // Expires is the expiration time on the signed post policy. |
| // It must be a time in the future. |
| // Required. |
| Expires time.Time |
| |
| // Style provides options for the type of URL to use. Options are |
| // PathStyle (default), BucketBoundHostname, and VirtualHostedStyle. See |
| // https://cloud.google.com/storage/docs/request-endpoints for details. |
| // Optional. |
| Style URLStyle |
| |
| // Insecure when set indicates that the generated URL's scheme |
| // will use "http" instead of "https" (default). |
| // Optional. |
| Insecure bool |
| |
| // Fields specifies the attributes of a PostPolicyV4 request. |
| // When Fields is non-nil, its attributes must match those that will |
| // passed into field Conditions. |
| // Optional. |
| Fields *PolicyV4Fields |
| |
| // The conditions that the uploaded file will be expected to conform to. |
| // When used, the failure of an upload to satisfy a condition will result in |
| // a 4XX status code, back with the message describing the problem. |
| // Optional. |
| Conditions []PostPolicyV4Condition |
| |
| // Hostname sets the host of the signed post policy. This field overrides |
| // any endpoint set on a storage Client or through STORAGE_EMULATOR_HOST. |
| // Only compatible with PathStyle URLStyle. |
| // Optional. |
| Hostname string |
| |
| shouldHashSignBytes bool |
| } |
| |
| func (opts *PostPolicyV4Options) clone() *PostPolicyV4Options { |
| return &PostPolicyV4Options{ |
| GoogleAccessID: opts.GoogleAccessID, |
| PrivateKey: opts.PrivateKey, |
| SignBytes: opts.SignBytes, |
| SignRawBytes: opts.SignRawBytes, |
| Expires: opts.Expires, |
| Style: opts.Style, |
| Insecure: opts.Insecure, |
| Fields: opts.Fields, |
| Conditions: opts.Conditions, |
| shouldHashSignBytes: opts.shouldHashSignBytes, |
| Hostname: opts.Hostname, |
| } |
| } |
| |
| // PolicyV4Fields describes the attributes for a PostPolicyV4 request. |
| type PolicyV4Fields struct { |
| // ACL specifies the access control permissions for the object. |
| // Optional. |
| ACL string |
| // CacheControl specifies the caching directives for the object. |
| // Optional. |
| CacheControl string |
| // ContentType specifies the media type of the object. |
| // Optional. |
| ContentType string |
| // ContentDisposition specifies how the file will be served back to requesters. |
| // Optional. |
| ContentDisposition string |
| // ContentEncoding specifies the decompressive transcoding that the object. |
| // This field is complementary to ContentType in that the file could be |
| // compressed but ContentType specifies the file's original media type. |
| // Optional. |
| ContentEncoding string |
| // Metadata specifies custom metadata for the object. |
| // If any key doesn't begin with "x-goog-meta-", an error will be returned. |
| // Optional. |
| Metadata map[string]string |
| // StatusCodeOnSuccess when set, specifies the status code that Cloud Storage |
| // will serve back on successful upload of the object. |
| // Optional. |
| StatusCodeOnSuccess int |
| // RedirectToURLOnSuccess when set, specifies the URL that Cloud Storage |
| // will serve back on successful upload of the object. |
| // Optional. |
| RedirectToURLOnSuccess string |
| } |
| |
| // PostPolicyV4 describes the URL and respective form fields for a generated PostPolicyV4 request. |
| type PostPolicyV4 struct { |
| // URL is the generated URL that the file upload will be made to. |
| URL string |
| // Fields specifies the generated key-values that the file uploader |
| // must include in their multipart upload form. |
| Fields map[string]string |
| } |
| |
| // PostPolicyV4Condition describes the constraints that the subsequent |
| // object upload's multipart form fields will be expected to conform to. |
| type PostPolicyV4Condition interface { |
| isEmpty() bool |
| json.Marshaler |
| } |
| |
| type startsWith struct { |
| key, value string |
| } |
| |
| func (sw *startsWith) MarshalJSON() ([]byte, error) { |
| return json.Marshal([]string{"starts-with", sw.key, sw.value}) |
| } |
| func (sw *startsWith) isEmpty() bool { |
| return sw.value == "" |
| } |
| |
| // ConditionStartsWith checks that an attributes starts with value. |
| // An empty value will cause this condition to be ignored. |
| func ConditionStartsWith(key, value string) PostPolicyV4Condition { |
| return &startsWith{key, value} |
| } |
| |
| type contentLengthRangeCondition struct { |
| start, end uint64 |
| } |
| |
| func (clr *contentLengthRangeCondition) MarshalJSON() ([]byte, error) { |
| return json.Marshal([]interface{}{"content-length-range", clr.start, clr.end}) |
| } |
| func (clr *contentLengthRangeCondition) isEmpty() bool { |
| return clr.start == 0 && clr.end == 0 |
| } |
| |
| type singleValueCondition struct { |
| name, value string |
| } |
| |
| func (svc *singleValueCondition) MarshalJSON() ([]byte, error) { |
| return json.Marshal(map[string]string{svc.name: svc.value}) |
| } |
| func (svc *singleValueCondition) isEmpty() bool { |
| return svc.value == "" |
| } |
| |
| // ConditionContentLengthRange constraints the limits that the |
| // multipart upload's range header will be expected to be within. |
| func ConditionContentLengthRange(start, end uint64) PostPolicyV4Condition { |
| return &contentLengthRangeCondition{start, end} |
| } |
| |
| func conditionRedirectToURLOnSuccess(redirectURL string) PostPolicyV4Condition { |
| return &singleValueCondition{"success_action_redirect", redirectURL} |
| } |
| |
| func conditionStatusCodeOnSuccess(statusCode int) PostPolicyV4Condition { |
| svc := &singleValueCondition{name: "success_action_status"} |
| if statusCode > 0 { |
| svc.value = fmt.Sprintf("%d", statusCode) |
| } |
| return svc |
| } |
| |
| // GenerateSignedPostPolicyV4 generates a PostPolicyV4 value from bucket, object and opts. |
| // The generated URL and fields will then allow an unauthenticated client to perform multipart uploads. |
| // If initializing a Storage Client, instead use the Bucket.GenerateSignedPostPolicyV4 |
| // method which uses the Client's credentials to handle authentication. |
| func GenerateSignedPostPolicyV4(bucket, object string, opts *PostPolicyV4Options) (*PostPolicyV4, error) { |
| if bucket == "" { |
| return nil, errors.New("storage: bucket must be non-empty") |
| } |
| if object == "" { |
| return nil, errors.New("storage: object must be non-empty") |
| } |
| now := utcNow() |
| if err := validatePostPolicyV4Options(opts, now); err != nil { |
| return nil, err |
| } |
| |
| var signingFn func(hashedBytes []byte) ([]byte, error) |
| switch { |
| case opts.SignRawBytes != nil: |
| signingFn = opts.SignRawBytes |
| case opts.shouldHashSignBytes: |
| signingFn = opts.SignBytes |
| case len(opts.PrivateKey) != 0: |
| parsedRSAPrivKey, err := parseKey(opts.PrivateKey) |
| if err != nil { |
| return nil, err |
| } |
| signingFn = func(b []byte) ([]byte, error) { |
| sum := sha256.Sum256(b) |
| return rsa.SignPKCS1v15(rand.Reader, parsedRSAPrivKey, crypto.SHA256, sum[:]) |
| } |
| |
| default: |
| return nil, errors.New("storage: exactly one of PrivateKey or SignRawBytes must be set") |
| } |
| |
| var descFields PolicyV4Fields |
| if opts.Fields != nil { |
| descFields = *opts.Fields |
| } |
| |
| if err := validateMetadata(descFields.Metadata); err != nil { |
| return nil, err |
| } |
| |
| // Build the policy. |
| conds := make([]PostPolicyV4Condition, len(opts.Conditions)) |
| copy(conds, opts.Conditions) |
| conds = append(conds, |
| // These are ordered lexicographically. Technically the order doesn't matter |
| // for creating the policy, but we use this order to match the |
| // cross-language conformance tests for this feature. |
| &singleValueCondition{"acl", descFields.ACL}, |
| &singleValueCondition{"cache-control", descFields.CacheControl}, |
| &singleValueCondition{"content-disposition", descFields.ContentDisposition}, |
| &singleValueCondition{"content-encoding", descFields.ContentEncoding}, |
| &singleValueCondition{"content-type", descFields.ContentType}, |
| conditionRedirectToURLOnSuccess(descFields.RedirectToURLOnSuccess), |
| conditionStatusCodeOnSuccess(descFields.StatusCodeOnSuccess), |
| ) |
| |
| YYYYMMDD := now.Format(yearMonthDay) |
| policyFields := map[string]string{ |
| "key": object, |
| "x-goog-date": now.Format(iso8601), |
| "x-goog-credential": opts.GoogleAccessID + "/" + YYYYMMDD + "/auto/storage/goog4_request", |
| "x-goog-algorithm": "GOOG4-RSA-SHA256", |
| "acl": descFields.ACL, |
| "cache-control": descFields.CacheControl, |
| "content-disposition": descFields.ContentDisposition, |
| "content-encoding": descFields.ContentEncoding, |
| "content-type": descFields.ContentType, |
| "success_action_redirect": descFields.RedirectToURLOnSuccess, |
| } |
| for key, value := range descFields.Metadata { |
| conds = append(conds, &singleValueCondition{key, value}) |
| policyFields[key] = value |
| } |
| |
| // Following from the order expected by the conformance test cases, |
| // hence manually inserting these fields in a specific order. |
| conds = append(conds, |
| &singleValueCondition{"bucket", bucket}, |
| &singleValueCondition{"key", object}, |
| &singleValueCondition{"x-goog-date", now.Format(iso8601)}, |
| &singleValueCondition{ |
| name: "x-goog-credential", |
| value: opts.GoogleAccessID + "/" + YYYYMMDD + "/auto/storage/goog4_request", |
| }, |
| &singleValueCondition{"x-goog-algorithm", "GOOG4-RSA-SHA256"}, |
| ) |
| |
| nonEmptyConds := make([]PostPolicyV4Condition, 0, len(opts.Conditions)) |
| for _, cond := range conds { |
| if cond == nil || !cond.isEmpty() { |
| nonEmptyConds = append(nonEmptyConds, cond) |
| } |
| } |
| condsAsJSON, err := json.Marshal(map[string]interface{}{ |
| "conditions": nonEmptyConds, |
| "expiration": opts.Expires.Format(time.RFC3339), |
| }) |
| if err != nil { |
| return nil, fmt.Errorf("storage: PostPolicyV4 JSON serialization failed: %w", err) |
| } |
| |
| b64Policy := base64.StdEncoding.EncodeToString(condsAsJSON) |
| var signature []byte |
| var signErr error |
| |
| if opts.shouldHashSignBytes { |
| // SignBytes expects hashed bytes as input instead of raw bytes, so we hash them |
| shaSum := sha256.Sum256([]byte(b64Policy)) |
| signature, signErr = signingFn(shaSum[:]) |
| } else { |
| signature, signErr = signingFn([]byte(b64Policy)) |
| } |
| if signErr != nil { |
| return nil, signErr |
| } |
| |
| policyFields["policy"] = b64Policy |
| policyFields["x-goog-signature"] = fmt.Sprintf("%x", signature) |
| |
| // Construct the URL. |
| scheme := "https" |
| if opts.Insecure { |
| scheme = "http" |
| } |
| path := opts.Style.path(bucket, "") + "/" |
| u := &url.URL{ |
| Path: path, |
| RawPath: pathEncodeV4(path), |
| Host: opts.Style.host(opts.Hostname, bucket), |
| Scheme: scheme, |
| } |
| |
| if descFields.StatusCodeOnSuccess > 0 { |
| policyFields["success_action_status"] = fmt.Sprintf("%d", descFields.StatusCodeOnSuccess) |
| } |
| |
| // Clear out fields with blanks values. |
| for key, value := range policyFields { |
| if value == "" { |
| delete(policyFields, key) |
| } |
| } |
| pp4 := &PostPolicyV4{ |
| Fields: policyFields, |
| URL: u.String(), |
| } |
| return pp4, nil |
| } |
| |
| // validatePostPolicyV4Options checks that: |
| // * GoogleAccessID is set |
| // * either PrivateKey or SignRawBytes/SignBytes is set, but not both |
| // * the deadline set in Expires is not in the past |
| // * if Style is not set, it'll use PathStyle |
| // * sets shouldHashSignBytes to true if opts.SignBytes should be used |
| func validatePostPolicyV4Options(opts *PostPolicyV4Options, now time.Time) error { |
| if opts == nil || opts.GoogleAccessID == "" { |
| return errors.New("storage: missing required GoogleAccessID") |
| } |
| if privBlank, signBlank := len(opts.PrivateKey) == 0, opts.SignBytes == nil && opts.SignRawBytes == nil; privBlank == signBlank { |
| return errors.New("storage: exactly one of PrivateKey or SignRawBytes must be set") |
| } |
| if opts.Expires.Before(now) { |
| return errors.New("storage: expecting Expires to be in the future") |
| } |
| if opts.Style == nil { |
| opts.Style = PathStyle() |
| } |
| if opts.SignRawBytes == nil && opts.SignBytes != nil { |
| opts.shouldHashSignBytes = true |
| } |
| return nil |
| } |
| |
| // validateMetadata ensures that all keys passed in have a prefix of "x-goog-meta-", |
| // otherwise it will return an error. |
| func validateMetadata(hdrs map[string]string) (err error) { |
| if len(hdrs) == 0 { |
| return nil |
| } |
| |
| badKeys := make([]string, 0, len(hdrs)) |
| for key := range hdrs { |
| if !strings.HasPrefix(key, "x-goog-meta-") { |
| badKeys = append(badKeys, key) |
| } |
| } |
| if len(badKeys) != 0 { |
| err = errors.New("storage: expected metadata to begin with x-goog-meta-, got " + strings.Join(badKeys, ", ")) |
| } |
| return |
| } |