| // Copyright 2016 Google Inc. All Rights Reserved. |
| // |
| // Licensed under the Apache License, Version 2.0 (the "License"); |
| // you may not use this file except in compliance with the License. |
| // You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, software |
| // distributed under the License is distributed on an "AS IS" BASIS, |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| // See the License for the specific language governing permissions and |
| // limitations under the License. |
| |
| package jsonclient |
| |
| import ( |
| "bytes" |
| "context" |
| "crypto" |
| "encoding/json" |
| "errors" |
| "fmt" |
| "io/ioutil" |
| "log" |
| "math/rand" |
| "net/http" |
| "net/url" |
| "strconv" |
| "strings" |
| "time" |
| |
| ct "github.com/google/certificate-transparency-go" |
| "github.com/google/certificate-transparency-go/x509" |
| "golang.org/x/net/context/ctxhttp" |
| ) |
| |
| const maxJitter = 250 * time.Millisecond |
| |
| type backoffer interface { |
| // set adjusts/increases the current backoff interval (typically on retryable failure); |
| // if the optional parameter is provided, this will be used as the interval if it is greater |
| // than the currently set interval. Returns the current wait period so that it can be |
| // logged along with any error message. |
| set(*time.Duration) time.Duration |
| // decreaseMultiplier reduces the current backoff multiplier, typically on success. |
| decreaseMultiplier() |
| // until returns the time until which the client should wait before making a request, |
| // it may be in the past in which case it should be ignored. |
| until() time.Time |
| } |
| |
| // JSONClient provides common functionality for interacting with a JSON server |
| // that uses cryptographic signatures. |
| type JSONClient struct { |
| uri string // the base URI of the server. e.g. https://ct.googleapis/pilot |
| httpClient *http.Client // used to interact with the server via HTTP |
| Verifier *ct.SignatureVerifier // nil for no verification (e.g. no public key available) |
| logger Logger // interface to use for logging warnings and errors |
| backoff backoffer // object used to store and calculate backoff information |
| } |
| |
| // Logger is a simple logging interface used to log internal errors and warnings |
| type Logger interface { |
| // Printf formats and logs a message |
| Printf(string, ...interface{}) |
| } |
| |
| // Options are the options for creating a new JSONClient. |
| type Options struct { |
| // Interface to use for logging warnings and errors, if nil the |
| // standard library log package will be used. |
| Logger Logger |
| // PEM format public key to use for signature verification. |
| PublicKey string |
| // DER format public key to use for signature verification. |
| PublicKeyDER []byte |
| } |
| |
| // ParsePublicKey parses and returns the public key contained in opts. |
| // If both opts.PublicKey and opts.PublicKeyDER are set, PublicKeyDER is used. |
| // If neither is set, nil will be returned. |
| func (opts *Options) ParsePublicKey() (crypto.PublicKey, error) { |
| if len(opts.PublicKeyDER) > 0 { |
| return x509.ParsePKIXPublicKey(opts.PublicKeyDER) |
| } |
| |
| if opts.PublicKey != "" { |
| pubkey, _ /* keyhash */, rest, err := ct.PublicKeyFromPEM([]byte(opts.PublicKey)) |
| if err != nil { |
| return nil, err |
| } |
| if len(rest) > 0 { |
| return nil, errors.New("extra data found after PEM key decoded") |
| } |
| return pubkey, nil |
| } |
| |
| return nil, nil |
| } |
| |
| type basicLogger struct{} |
| |
| func (bl *basicLogger) Printf(msg string, args ...interface{}) { |
| log.Printf(msg, args...) |
| } |
| |
| // New constructs a new JSONClient instance, for the given base URI, using the |
| // given http.Client object (if provided) and the Options object. |
| // If opts does not specify a public key, signatures will not be verified. |
| func New(uri string, hc *http.Client, opts Options) (*JSONClient, error) { |
| pubkey, err := opts.ParsePublicKey() |
| if err != nil { |
| return nil, fmt.Errorf("invalid public key: %v", err) |
| } |
| |
| var verifier *ct.SignatureVerifier |
| if pubkey != nil { |
| var err error |
| verifier, err = ct.NewSignatureVerifier(pubkey) |
| if err != nil { |
| return nil, err |
| } |
| } |
| |
| if hc == nil { |
| hc = new(http.Client) |
| } |
| logger := opts.Logger |
| if logger == nil { |
| logger = &basicLogger{} |
| } |
| return &JSONClient{ |
| uri: strings.TrimRight(uri, "/"), |
| httpClient: hc, |
| Verifier: verifier, |
| logger: logger, |
| backoff: &backoff{}, |
| }, nil |
| } |
| |
| // BaseURI returns the base URI that the JSONClient makes queries to. |
| func (c *JSONClient) BaseURI() string { |
| return c.uri |
| } |
| |
| // GetAndParse makes a HTTP GET call to the given path, and attempta to parse |
| // the response as a JSON representation of the rsp structure. Returns the |
| // http.Response, the body of the response, and an error. Note that the |
| // returned http.Response can be non-nil even when an error is returned, |
| // in particular when the HTTP status is not OK or when the JSON parsing fails. |
| func (c *JSONClient) GetAndParse(ctx context.Context, path string, params map[string]string, rsp interface{}) (*http.Response, []byte, error) { |
| if ctx == nil { |
| return nil, nil, errors.New("context.Context required") |
| } |
| // Build a GET request with URL-encoded parameters. |
| vals := url.Values{} |
| for k, v := range params { |
| vals.Add(k, v) |
| } |
| fullURI := fmt.Sprintf("%s%s?%s", c.uri, path, vals.Encode()) |
| httpReq, err := http.NewRequest(http.MethodGet, fullURI, nil) |
| if err != nil { |
| return nil, nil, err |
| } |
| |
| httpRsp, err := ctxhttp.Do(ctx, c.httpClient, httpReq) |
| if err != nil { |
| return nil, nil, err |
| } |
| |
| // Read everything now so http.Client can reuse the connection. |
| body, err := ioutil.ReadAll(httpRsp.Body) |
| httpRsp.Body.Close() |
| if err != nil { |
| return httpRsp, body, fmt.Errorf("failed to read response body: %v", err) |
| } |
| |
| if httpRsp.StatusCode != http.StatusOK { |
| return httpRsp, body, fmt.Errorf("got HTTP Status %q", httpRsp.Status) |
| } |
| |
| if err := json.NewDecoder(bytes.NewReader(body)).Decode(rsp); err != nil { |
| return httpRsp, body, err |
| } |
| |
| return httpRsp, body, nil |
| } |
| |
| // PostAndParse makes a HTTP POST call to the given path, including the request |
| // parameters, and attempts to parse the response as a JSON representation of |
| // the rsp structure. Returns the http.Response, the body of the response, and |
| // an error. Note that the returned http.Response can be non-nil even when an |
| // error is returned, in particular when the HTTP status is not OK or when the |
| // JSON parsing fails. |
| func (c *JSONClient) PostAndParse(ctx context.Context, path string, req, rsp interface{}) (*http.Response, []byte, error) { |
| if ctx == nil { |
| return nil, nil, errors.New("context.Context required") |
| } |
| // Build a POST request with JSON body. |
| postBody, err := json.Marshal(req) |
| if err != nil { |
| return nil, nil, err |
| } |
| fullURI := fmt.Sprintf("%s%s", c.uri, path) |
| httpReq, err := http.NewRequest(http.MethodPost, fullURI, bytes.NewReader(postBody)) |
| if err != nil { |
| return nil, nil, err |
| } |
| httpReq.Header.Set("Content-Type", "application/json") |
| |
| httpRsp, err := ctxhttp.Do(ctx, c.httpClient, httpReq) |
| |
| // Read all of the body, if there is one, so that the http.Client can do Keep-Alive. |
| var body []byte |
| if httpRsp != nil { |
| body, err = ioutil.ReadAll(httpRsp.Body) |
| httpRsp.Body.Close() |
| } |
| if err != nil { |
| return httpRsp, body, err |
| } |
| |
| if httpRsp.StatusCode == http.StatusOK { |
| if err = json.Unmarshal(body, &rsp); err != nil { |
| return httpRsp, body, err |
| } |
| } |
| return httpRsp, body, nil |
| } |
| |
| // waitForBackoff blocks until the defined backoff interval or context has expired, if the returned |
| // not before time is in the past it returns immediately. |
| func (c *JSONClient) waitForBackoff(ctx context.Context) error { |
| dur := time.Until(c.backoff.until().Add(time.Millisecond * time.Duration(rand.Intn(int(maxJitter.Seconds()*1000))))) |
| if dur < 0 { |
| dur = 0 |
| } |
| backoffTimer := time.NewTimer(dur) |
| select { |
| case <-ctx.Done(): |
| return ctx.Err() |
| case <-backoffTimer.C: |
| } |
| return nil |
| } |
| |
| // PostAndParseWithRetry makes a HTTP POST call, but retries (with backoff) on |
| // retriable errors; the caller should set a deadline on the provided context |
| // to prevent infinite retries. Return values are as for PostAndParse. |
| func (c *JSONClient) PostAndParseWithRetry(ctx context.Context, path string, req, rsp interface{}) (*http.Response, []byte, error) { |
| if ctx == nil { |
| return nil, nil, errors.New("context.Context required") |
| } |
| for { |
| httpRsp, body, err := c.PostAndParse(ctx, path, req, rsp) |
| if err != nil { |
| // Don't retry context errors. |
| if err == context.Canceled || err == context.DeadlineExceeded { |
| return nil, nil, err |
| } |
| wait := c.backoff.set(nil) |
| c.logger.Printf("Request failed, backing-off for %s: %s", wait, err) |
| } else { |
| switch { |
| case httpRsp.StatusCode == http.StatusOK: |
| return httpRsp, body, nil |
| case httpRsp.StatusCode == http.StatusRequestTimeout: |
| // Request timeout, retry immediately |
| c.logger.Printf("Request timed out, retrying immediately") |
| case httpRsp.StatusCode == http.StatusServiceUnavailable: |
| var backoff *time.Duration |
| // Retry-After may be either a number of seconds as a int or a RFC 1123 |
| // date string (RFC 7231 Section 7.1.3) |
| if retryAfter := httpRsp.Header.Get("Retry-After"); retryAfter != "" { |
| if seconds, err := strconv.Atoi(retryAfter); err == nil { |
| b := time.Duration(seconds) * time.Second |
| backoff = &b |
| } else if date, err := time.Parse(time.RFC1123, retryAfter); err == nil { |
| b := date.Sub(time.Now()) |
| backoff = &b |
| } |
| } |
| wait := c.backoff.set(backoff) |
| c.logger.Printf("Request failed, backing-off for %s: got HTTP status %s", wait, httpRsp.Status) |
| default: |
| return httpRsp, body, fmt.Errorf("got HTTP Status %q", httpRsp.Status) |
| } |
| } |
| if err := c.waitForBackoff(ctx); err != nil { |
| return nil, nil, err |
| } |
| } |
| } |