credentials/xds/xds_client_test.go - third_party/github.com/grpc/grpc-go - Git at Google

 /*
  *
  * Copyright 2020 gRPC authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
  *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  *
  */

 package xds

 import (
 	"context"
 	"crypto/tls"
 	"crypto/x509"
 	"errors"
 	"fmt"
 	"io/ioutil"
 	"net"
 	"strings"
 	"testing"
 	"time"

 	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/credentials/tls/certprovider"
 	"google.golang.org/grpc/internal"
 	xdsinternal "google.golang.org/grpc/internal/credentials/xds"
 	"google.golang.org/grpc/internal/grpctest"
 	"google.golang.org/grpc/internal/testutils"
 	"google.golang.org/grpc/resolver"
 	"google.golang.org/grpc/testdata"
 )

 const (
 	defaultTestTimeout      = 10 * time.Second
 	defaultTestShortTimeout = 10 * time.Millisecond
 	defaultTestCertSAN      = "*.test.example.com"
 	authority               = "authority"
 )

 type s struct {
 	grpctest.Tester
 }

 func Test(t *testing.T) {
 	grpctest.RunSubTests(t, s{})
 }

 // Helper function to create a real TLS client credentials which is used as
 // fallback credentials from multiple tests.
 func makeFallbackClientCreds(t *testing.T) credentials.TransportCredentials {
 	creds, err := credentials.NewClientTLSFromFile(testdata.Path("x509/server_ca_cert.pem"), "x.test.example.com")
 	if err != nil {
 		t.Fatal(err)
 	}
 	return creds
 }

 // testServer is a no-op server which listens on a local TCP port for incoming
 // connections, and performs a manual TLS handshake on the received raw
 // connection using a user specified handshake function. It then makes the
 // result of the handshake operation available through a channel for tests to
 // inspect. Tests should stop the testServer as part of their cleanup.
 type testServer struct {
 	lis           net.Listener
 	address       string             // Listening address of the test server.
 	handshakeFunc testHandshakeFunc  // Test specified handshake function.
 	hsResult      *testutils.Channel // Channel to deliver handshake results.
 }

 // handshakeResult wraps the result of the handshake operation on the test
 // server. It consists of TLS connection state and an error, if the handshake
 // failed. This result is delivered on the `hsResult` channel on the testServer.
 type handshakeResult struct {
 	connState tls.ConnectionState
 	err       error
 }

 // Configurable handshake function for the testServer. Tests can set this to
 // simulate different conditions like handshake success, failure, timeout etc.
 type testHandshakeFunc func(net.Conn) handshakeResult

 // newTestServerWithHandshakeFunc starts a new testServer which listens for
 // connections on a local TCP port, and uses the provided custom handshake
 // function to perform TLS handshake.
 func newTestServerWithHandshakeFunc(f testHandshakeFunc) *testServer {
 	ts := &testServer{
 		handshakeFunc: f,
 		hsResult:      testutils.NewChannel(),
 	}
 	ts.start()
 	return ts
 }

 // starts actually starts listening on a local TCP port, and spawns a goroutine
 // to handle new connections.
 func (ts *testServer) start() error {
 	lis, err := net.Listen("tcp", "localhost:0")
 	if err != nil {
 		return err
 	}
 	ts.lis = lis
 	ts.address = lis.Addr().String()
 	go ts.handleConn()
 	return nil
 }

 // handleconn accepts a new raw connection, and invokes the test provided
 // handshake function to perform TLS handshake, and returns the result on the
 // `hsResult` channel.
 func (ts *testServer) handleConn() {
 	for {
 		rawConn, err := ts.lis.Accept()
 		if err != nil {
 			// Once the listeners closed, Accept() will return with an error.
 			return
 		}
 		hsr := ts.handshakeFunc(rawConn)
 		ts.hsResult.Send(hsr)
 	}
 }

 // stop closes the associated listener which causes the connection handling
 // goroutine to exit.
 func (ts *testServer) stop() {
 	ts.lis.Close()
 }

 // A handshake function which simulates a successful handshake without client
 // authentication (server does not request for client certificate during the
 // handshake here).
 func testServerTLSHandshake(rawConn net.Conn) handshakeResult {
 	cert, err := tls.LoadX509KeyPair(testdata.Path("x509/server1_cert.pem"), testdata.Path("x509/server1_key.pem"))
 	if err != nil {
 		return handshakeResult{err: err}
 	}
 	cfg := &tls.Config{Certificates: []tls.Certificate{cert}}
 	conn := tls.Server(rawConn, cfg)
 	if err := conn.Handshake(); err != nil {
 		return handshakeResult{err: err}
 	}
 	return handshakeResult{connState: conn.ConnectionState()}
 }

 // A handshake function which simulates a successful handshake with mutual
 // authentication.
 func testServerMutualTLSHandshake(rawConn net.Conn) handshakeResult {
 	cert, err := tls.LoadX509KeyPair(testdata.Path("x509/server1_cert.pem"), testdata.Path("x509/server1_key.pem"))
 	if err != nil {
 		return handshakeResult{err: err}
 	}
 	pemData, err := ioutil.ReadFile(testdata.Path("x509/client_ca_cert.pem"))
 	if err != nil {
 		return handshakeResult{err: err}
 	}
 	roots := x509.NewCertPool()
 	roots.AppendCertsFromPEM(pemData)
 	cfg := &tls.Config{
 		Certificates: []tls.Certificate{cert},
 		ClientCAs:    roots,
 	}
 	conn := tls.Server(rawConn, cfg)
 	if err := conn.Handshake(); err != nil {
 		return handshakeResult{err: err}
 	}
 	return handshakeResult{connState: conn.ConnectionState()}
 }

 // fakeProvider is an implementation of the certprovider.Provider interface
 // which returns the configured key material and error in calls to
 // KeyMaterial().
 type fakeProvider struct {
 	km  *certprovider.KeyMaterial
 	err error
 }

 func (f *fakeProvider) KeyMaterial(ctx context.Context) (*certprovider.KeyMaterial, error) {
 	return f.km, f.err
 }

 func (f *fakeProvider) Close() {}

 // makeIdentityProvider creates a new instance of the fakeProvider returning the
 // identity key material specified in the provider file paths.
 func makeIdentityProvider(t *testing.T, certPath, keyPath string) certprovider.Provider {
 	t.Helper()
 	cert, err := tls.LoadX509KeyPair(testdata.Path(certPath), testdata.Path(keyPath))
 	if err != nil {
 		t.Fatal(err)
 	}
 	return &fakeProvider{km: &certprovider.KeyMaterial{Certs: []tls.Certificate{cert}}}
 }

 // makeRootProvider creates a new instance of the fakeProvider returning the
 // root key material specified in the provider file paths.
 func makeRootProvider(t *testing.T, caPath string) *fakeProvider {
 	pemData, err := ioutil.ReadFile(testdata.Path(caPath))
 	if err != nil {
 		t.Fatal(err)
 	}
 	roots := x509.NewCertPool()
 	roots.AppendCertsFromPEM(pemData)
 	return &fakeProvider{km: &certprovider.KeyMaterial{Roots: roots}}
 }

 // newTestContextWithHandshakeInfo returns a copy of parent with HandshakeInfo
 // context value added to it.
 func newTestContextWithHandshakeInfo(parent context.Context, root, identity certprovider.Provider, sans ...string) context.Context {
 	// Creating the HandshakeInfo and adding it to the attributes is very
 	// similar to what the CDS balancer would do when it intercepts calls to
 	// NewSubConn().
 	info := xdsinternal.NewHandshakeInfo(root, identity, sans...)
 	addr := xdsinternal.SetHandshakeInfo(resolver.Address{}, info)

 	// Moving the attributes from the resolver.Address to the context passed to
 	// the handshaker is done in the transport layer. Since we directly call the
 	// handshaker in these tests, we need to do the same here.
 	contextWithHandshakeInfo := internal.NewClientHandshakeInfoContext.(func(context.Context, credentials.ClientHandshakeInfo) context.Context)
 	return contextWithHandshakeInfo(parent, credentials.ClientHandshakeInfo{Attributes: addr.Attributes})
 }

 // compareAuthInfo compares the AuthInfo received on the client side after a
 // successful handshake with the authInfo available on the testServer.
 func compareAuthInfo(ctx context.Context, ts *testServer, ai credentials.AuthInfo) error {
 	if ai.AuthType() != "tls" {
 		return fmt.Errorf("ClientHandshake returned authType %q, want %q", ai.AuthType(), "tls")
 	}
 	info, ok := ai.(credentials.TLSInfo)
 	if !ok {
 		return fmt.Errorf("ClientHandshake returned authInfo of type %T, want %T", ai, credentials.TLSInfo{})
 	}
 	gotState := info.State

 	// Read the handshake result from the testServer which contains the TLS
 	// connection state and compare it with the one received on the client-side.
 	val, err := ts.hsResult.Receive(ctx)
 	if err != nil {
 		return fmt.Errorf("testServer failed to return handshake result: %v", err)
 	}
 	hsr := val.(handshakeResult)
 	if hsr.err != nil {
 		return fmt.Errorf("testServer handshake failure: %v", hsr.err)
 	}
 	// AuthInfo contains a variety of information. We only verify a subset here.
 	// This is the same subset which is verified in TLS credentials tests.
 	if err := compareConnState(gotState, hsr.connState); err != nil {
 		return err
 	}
 	return nil
 }

 func compareConnState(got, want tls.ConnectionState) error {
 	switch {
 	case got.Version != want.Version:
 		return fmt.Errorf("TLS.ConnectionState got Version: %v, want: %v", got.Version, want.Version)
 	case got.HandshakeComplete != want.HandshakeComplete:
 		return fmt.Errorf("TLS.ConnectionState got HandshakeComplete: %v, want: %v", got.HandshakeComplete, want.HandshakeComplete)
 	case got.CipherSuite != want.CipherSuite:
 		return fmt.Errorf("TLS.ConnectionState got CipherSuite: %v, want: %v", got.CipherSuite, want.CipherSuite)
 	case got.NegotiatedProtocol != want.NegotiatedProtocol:
 		return fmt.Errorf("TLS.ConnectionState got NegotiatedProtocol: %v, want: %v", got.NegotiatedProtocol, want.NegotiatedProtocol)
 	}
 	return nil
 }

 // TestClientCredsWithoutFallback verifies that the call to
 // NewClientCredentials() fails when no fallback is specified.
 func (s) TestClientCredsWithoutFallback(t *testing.T) {
 	if _, err := NewClientCredentials(ClientOptions{}); err == nil {
 		t.Fatal("NewClientCredentials() succeeded without specifying fallback")
 	}
 }

 // TestClientCredsInvalidHandshakeInfo verifies scenarios where the passed in
 // HandshakeInfo is invalid because it does not contain the expected certificate
 // providers.
 func (s) TestClientCredsInvalidHandshakeInfo(t *testing.T) {
 	opts := ClientOptions{FallbackCreds: makeFallbackClientCreds(t)}
 	creds, err := NewClientCredentials(opts)
 	if err != nil {
 		t.Fatalf("NewClientCredentials(%v) failed: %v", opts, err)
 	}

 	pCtx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer cancel()
 	ctx := newTestContextWithHandshakeInfo(pCtx, nil, &fakeProvider{})
 	if _, _, err := creds.ClientHandshake(ctx, authority, nil); err == nil {
 		t.Fatal("ClientHandshake succeeded without root certificate provider in HandshakeInfo")
 	}
 }

 // TestClientCredsProviderFailure verifies the cases where an expected
 // certificate provider is missing in the HandshakeInfo value in the context.
 func (s) TestClientCredsProviderFailure(t *testing.T) {
 	opts := ClientOptions{FallbackCreds: makeFallbackClientCreds(t)}
 	creds, err := NewClientCredentials(opts)
 	if err != nil {
 		t.Fatalf("NewClientCredentials(%v) failed: %v", opts, err)
 	}

 	tests := []struct {
 		desc             string
 		rootProvider     certprovider.Provider
 		identityProvider certprovider.Provider
 		wantErr          string
 	}{
 		{
 			desc:         "erroring root provider",
 			rootProvider: &fakeProvider{err: errors.New("root provider error")},
 			wantErr:      "root provider error",
 		},
 		{
 			desc:             "erroring identity provider",
 			rootProvider:     &fakeProvider{km: &certprovider.KeyMaterial{}},
 			identityProvider: &fakeProvider{err: errors.New("identity provider error")},
 			wantErr:          "identity provider error",
 		},
 	}
 	for _, test := range tests {
 		t.Run(test.desc, func(t *testing.T) {
 			ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 			defer cancel()
 			ctx = newTestContextWithHandshakeInfo(ctx, test.rootProvider, test.identityProvider)
 			if _, _, err := creds.ClientHandshake(ctx, authority, nil); err == nil || !strings.Contains(err.Error(), test.wantErr) {
 				t.Fatalf("ClientHandshake() returned error: %q, wantErr: %q", err, test.wantErr)
 			}
 		})
 	}
 }

 // TestClientCredsSuccess verifies successful client handshake cases.
 func (s) TestClientCredsSuccess(t *testing.T) {
 	tests := []struct {
 		desc             string
 		handshakeFunc    testHandshakeFunc
 		handshakeInfoCtx func(ctx context.Context) context.Context
 	}{
 		{
 			desc:          "fallback",
 			handshakeFunc: testServerTLSHandshake,
 			handshakeInfoCtx: func(ctx context.Context) context.Context {
 				// Since we don't add a HandshakeInfo to the context, the
 				// ClientHandshake() method will delegate to the fallback.
 				return ctx
 			},
 		},
 		{
 			desc:          "TLS",
 			handshakeFunc: testServerTLSHandshake,
 			handshakeInfoCtx: func(ctx context.Context) context.Context {
 				return newTestContextWithHandshakeInfo(ctx, makeRootProvider(t, "x509/server_ca_cert.pem"), nil, defaultTestCertSAN)
 			},
 		},
 		{
 			desc:          "mTLS",
 			handshakeFunc: testServerMutualTLSHandshake,
 			handshakeInfoCtx: func(ctx context.Context) context.Context {
 				return newTestContextWithHandshakeInfo(ctx, makeRootProvider(t, "x509/server_ca_cert.pem"), makeIdentityProvider(t, "x509/server1_cert.pem", "x509/server1_key.pem"), defaultTestCertSAN)
 			},
 		},
 		{
 			desc:          "mTLS with no acceptedSANs specified",
 			handshakeFunc: testServerMutualTLSHandshake,
 			handshakeInfoCtx: func(ctx context.Context) context.Context {
 				return newTestContextWithHandshakeInfo(ctx, makeRootProvider(t, "x509/server_ca_cert.pem"), makeIdentityProvider(t, "x509/server1_cert.pem", "x509/server1_key.pem"))
 			},
 		},
 	}

 	for _, test := range tests {
 		t.Run(test.desc, func(t *testing.T) {
 			ts := newTestServerWithHandshakeFunc(test.handshakeFunc)
 			defer ts.stop()

 			opts := ClientOptions{FallbackCreds: makeFallbackClientCreds(t)}
 			creds, err := NewClientCredentials(opts)
 			if err != nil {
 				t.Fatalf("NewClientCredentials(%v) failed: %v", opts, err)
 			}

 			conn, err := net.Dial("tcp", ts.address)
 			if err != nil {
 				t.Fatalf("net.Dial(%s) failed: %v", ts.address, err)
 			}
 			defer conn.Close()

 			ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 			defer cancel()
 			_, ai, err := creds.ClientHandshake(test.handshakeInfoCtx(ctx), authority, conn)
 			if err != nil {
 				t.Fatalf("ClientHandshake() returned failed: %q", err)
 			}
 			if err := compareAuthInfo(ctx, ts, ai); err != nil {
 				t.Fatal(err)
 			}
 		})
 	}
 }

 func (s) TestClientCredsHandshakeTimeout(t *testing.T) {
 	clientDone := make(chan struct{})
 	// A handshake function which simulates a handshake timeout from the
 	// server-side by simply blocking on the client-side handshake to timeout
 	// and not writing any handshake data.
 	hErr := errors.New("server handshake error")
 	ts := newTestServerWithHandshakeFunc(func(rawConn net.Conn) handshakeResult {
 		<-clientDone
 		return handshakeResult{err: hErr}
 	})
 	defer ts.stop()

 	opts := ClientOptions{FallbackCreds: makeFallbackClientCreds(t)}
 	creds, err := NewClientCredentials(opts)
 	if err != nil {
 		t.Fatalf("NewClientCredentials(%v) failed: %v", opts, err)
 	}

 	conn, err := net.Dial("tcp", ts.address)
 	if err != nil {
 		t.Fatalf("net.Dial(%s) failed: %v", ts.address, err)
 	}
 	defer conn.Close()

 	sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
 	defer sCancel()
 	ctx := newTestContextWithHandshakeInfo(sCtx, makeRootProvider(t, "x509/server_ca_cert.pem"), nil, defaultTestCertSAN)
 	if _, _, err := creds.ClientHandshake(ctx, authority, conn); err == nil {
 		t.Fatal("ClientHandshake() succeeded when expected to timeout")
 	}
 	close(clientDone)

 	// Read the handshake result from the testServer and make sure the expected
 	// error is returned.
 	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer cancel()
 	val, err := ts.hsResult.Receive(ctx)
 	if err != nil {
 		t.Fatalf("testServer failed to return handshake result: %v", err)
 	}
 	hsr := val.(handshakeResult)
 	if hsr.err != hErr {
 		t.Fatalf("testServer handshake returned error: %v, want: %v", hsr.err, hErr)
 	}
 }

 // TestClientCredsHandshakeFailure verifies different handshake failure cases.
 func (s) TestClientCredsHandshakeFailure(t *testing.T) {
 	tests := []struct {
 		desc          string
 		handshakeFunc testHandshakeFunc
 		rootProvider  certprovider.Provider
 		san           string
 		wantErr       string
 	}{
 		{
 			desc:          "cert validation failure",
 			handshakeFunc: testServerTLSHandshake,
 			rootProvider:  makeRootProvider(t, "x509/client_ca_cert.pem"),
 			san:           defaultTestCertSAN,
 			wantErr:       "x509: certificate signed by unknown authority",
 		},
 		{
 			desc:          "SAN mismatch",
 			handshakeFunc: testServerTLSHandshake,
 			rootProvider:  makeRootProvider(t, "x509/server_ca_cert.pem"),
 			san:           "bad-san",
 			wantErr:       "does not match any of the accepted SANs",
 		},
 	}

 	for _, test := range tests {
 		t.Run(test.desc, func(t *testing.T) {
 			ts := newTestServerWithHandshakeFunc(test.handshakeFunc)
 			defer ts.stop()

 			opts := ClientOptions{FallbackCreds: makeFallbackClientCreds(t)}
 			creds, err := NewClientCredentials(opts)
 			if err != nil {
 				t.Fatalf("NewClientCredentials(%v) failed: %v", opts, err)
 			}

 			conn, err := net.Dial("tcp", ts.address)
 			if err != nil {
 				t.Fatalf("net.Dial(%s) failed: %v", ts.address, err)
 			}
 			defer conn.Close()

 			ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 			defer cancel()
 			ctx = newTestContextWithHandshakeInfo(ctx, test.rootProvider, nil, test.san)
 			if _, _, err := creds.ClientHandshake(ctx, authority, conn); err == nil || !strings.Contains(err.Error(), test.wantErr) {
 				t.Fatalf("ClientHandshake() returned %q, wantErr %q", err, test.wantErr)
 			}
 		})
 	}
 }

 // TestClientCredsProviderSwitch verifies the case where the first attempt of
 // ClientHandshake fails because of a handshake failure. Then we update the
 // certificate provider and the second attempt succeeds. This is an
 // approximation of the flow of events when the control plane specifies new
 // security config which results in new certificate providers being used.
 func (s) TestClientCredsProviderSwitch(t *testing.T) {
 	ts := newTestServerWithHandshakeFunc(testServerTLSHandshake)
 	defer ts.stop()

 	opts := ClientOptions{FallbackCreds: makeFallbackClientCreds(t)}
 	creds, err := NewClientCredentials(opts)
 	if err != nil {
 		t.Fatalf("NewClientCredentials(%v) failed: %v", opts, err)
 	}

 	conn, err := net.Dial("tcp", ts.address)
 	if err != nil {
 		t.Fatalf("net.Dial(%s) failed: %v", ts.address, err)
 	}
 	defer conn.Close()

 	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer cancel()
 	// Create a root provider which will fail the handshake because it does not
 	// use the correct trust roots.
 	root1 := makeRootProvider(t, "x509/client_ca_cert.pem")
 	handshakeInfo := xdsinternal.NewHandshakeInfo(root1, nil, defaultTestCertSAN)

 	// We need to repeat most of what newTestContextWithHandshakeInfo() does
 	// here because we need access to the underlying HandshakeInfo so that we
 	// can update it before the next call to ClientHandshake().
 	addr := xdsinternal.SetHandshakeInfo(resolver.Address{}, handshakeInfo)
 	contextWithHandshakeInfo := internal.NewClientHandshakeInfoContext.(func(context.Context, credentials.ClientHandshakeInfo) context.Context)
 	ctx = contextWithHandshakeInfo(ctx, credentials.ClientHandshakeInfo{Attributes: addr.Attributes})
 	if _, _, err := creds.ClientHandshake(ctx, authority, conn); err == nil {
 		t.Fatal("ClientHandshake() succeeded when expected to fail")
 	}
 	// Drain the result channel on the test server so that we can inspect the
 	// result for the next handshake.
 	_, err = ts.hsResult.Receive(ctx)
 	if err != nil {
 		t.Errorf("testServer failed to return handshake result: %v", err)
 	}

 	conn, err = net.Dial("tcp", ts.address)
 	if err != nil {
 		t.Fatalf("net.Dial(%s) failed: %v", ts.address, err)
 	}
 	defer conn.Close()

 	// Create a new root provider which uses the correct trust roots. And update
 	// the HandshakeInfo with the new provider.
 	root2 := makeRootProvider(t, "x509/server_ca_cert.pem")
 	handshakeInfo.SetRootCertProvider(root2)
 	_, ai, err := creds.ClientHandshake(ctx, authority, conn)
 	if err != nil {
 		t.Fatalf("ClientHandshake() returned failed: %q", err)
 	}
 	if err := compareAuthInfo(ctx, ts, ai); err != nil {
 		t.Fatal(err)
 	}
 }

 // TestClientClone verifies the Clone() method on client credentials.
 func (s) TestClientClone(t *testing.T) {
 	opts := ClientOptions{FallbackCreds: makeFallbackClientCreds(t)}
 	orig, err := NewClientCredentials(opts)
 	if err != nil {
 		t.Fatalf("NewClientCredentials(%v) failed: %v", opts, err)
 	}

 	// The credsImpl does not have any exported fields, and it does not make
 	// sense to use any cmp options to look deep into. So, all we make sure here
 	// is that the cloned object points to a different location in memory.
 	if clone := orig.Clone(); clone == orig {
 		t.Fatal("return value from Clone() doesn't point to new credentials instance")
 	}
 }
	/*
	*
	* Copyright 2020 gRPC authors.
	*
	* Licensed under the Apache License, Version 2.0 (the "License");
	* you may not use this file except in compliance with the License.
	* You may obtain a copy of the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS,
	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	* See the License for the specific language governing permissions and
	* limitations under the License.
	*
	*/

	package xds

	import (
	"context"
	"crypto/tls"
	"crypto/x509"
	"errors"
	"fmt"
	"io/ioutil"
	"net"
	"strings"
	"testing"
	"time"

	"google.golang.org/grpc/credentials"
	"google.golang.org/grpc/credentials/tls/certprovider"
	"google.golang.org/grpc/internal"
	xdsinternal "google.golang.org/grpc/internal/credentials/xds"
	"google.golang.org/grpc/internal/grpctest"
	"google.golang.org/grpc/internal/testutils"
	"google.golang.org/grpc/resolver"
	"google.golang.org/grpc/testdata"
	)

	const (
	defaultTestTimeout = 10 * time.Second
	defaultTestShortTimeout = 10 * time.Millisecond
	defaultTestCertSAN = "*.test.example.com"
	authority = "authority"
	)

	type s struct {
	grpctest.Tester
	}

	func Test(t *testing.T) {
	grpctest.RunSubTests(t, s{})
	}

	// Helper function to create a real TLS client credentials which is used as
	// fallback credentials from multiple tests.
	func makeFallbackClientCreds(t *testing.T) credentials.TransportCredentials {
	creds, err := credentials.NewClientTLSFromFile(testdata.Path("x509/server_ca_cert.pem"), "x.test.example.com")
	if err != nil {
	t.Fatal(err)
	}
	return creds
	}

	// testServer is a no-op server which listens on a local TCP port for incoming
	// connections, and performs a manual TLS handshake on the received raw
	// connection using a user specified handshake function. It then makes the
	// result of the handshake operation available through a channel for tests to
	// inspect. Tests should stop the testServer as part of their cleanup.
	type testServer struct {
	lis net.Listener
	address string // Listening address of the test server.
	handshakeFunc testHandshakeFunc // Test specified handshake function.
	hsResult *testutils.Channel // Channel to deliver handshake results.
	}

	// handshakeResult wraps the result of the handshake operation on the test
	// server. It consists of TLS connection state and an error, if the handshake
	// failed. This result is delivered on the `hsResult` channel on the testServer.
	type handshakeResult struct {
	connState tls.ConnectionState
	err error
	}

	// Configurable handshake function for the testServer. Tests can set this to
	// simulate different conditions like handshake success, failure, timeout etc.
	type testHandshakeFunc func(net.Conn) handshakeResult

	// newTestServerWithHandshakeFunc starts a new testServer which listens for
	// connections on a local TCP port, and uses the provided custom handshake
	// function to perform TLS handshake.
	func newTestServerWithHandshakeFunc(f testHandshakeFunc) *testServer {
	ts := &testServer{
	handshakeFunc: f,
	hsResult: testutils.NewChannel(),
	}
	ts.start()
	return ts
	}

	// starts actually starts listening on a local TCP port, and spawns a goroutine
	// to handle new connections.
	func (ts *testServer) start() error {
	lis, err := net.Listen("tcp", "localhost:0")
	if err != nil {
	return err
	}
	ts.lis = lis
	ts.address = lis.Addr().String()
	go ts.handleConn()
	return nil
	}

	// handleconn accepts a new raw connection, and invokes the test provided
	// handshake function to perform TLS handshake, and returns the result on the
	// `hsResult` channel.
	func (ts *testServer) handleConn() {
	for {
	rawConn, err := ts.lis.Accept()
	if err != nil {
	// Once the listeners closed, Accept() will return with an error.
	return
	}
	hsr := ts.handshakeFunc(rawConn)
	ts.hsResult.Send(hsr)
	}
	}

	// stop closes the associated listener which causes the connection handling
	// goroutine to exit.
	func (ts *testServer) stop() {
	ts.lis.Close()
	}

	// A handshake function which simulates a successful handshake without client
	// authentication (server does not request for client certificate during the
	// handshake here).
	func testServerTLSHandshake(rawConn net.Conn) handshakeResult {
	cert, err := tls.LoadX509KeyPair(testdata.Path("x509/server1_cert.pem"), testdata.Path("x509/server1_key.pem"))
	if err != nil {
	return handshakeResult{err: err}
	}
	cfg := &tls.Config{Certificates: []tls.Certificate{cert}}
	conn := tls.Server(rawConn, cfg)
	if err := conn.Handshake(); err != nil {
	return handshakeResult{err: err}
	}
	return handshakeResult{connState: conn.ConnectionState()}
	}

	// A handshake function which simulates a successful handshake with mutual
	// authentication.
	func testServerMutualTLSHandshake(rawConn net.Conn) handshakeResult {
	cert, err := tls.LoadX509KeyPair(testdata.Path("x509/server1_cert.pem"), testdata.Path("x509/server1_key.pem"))
	if err != nil {
	return handshakeResult{err: err}
	}
	pemData, err := ioutil.ReadFile(testdata.Path("x509/client_ca_cert.pem"))
	if err != nil {
	return handshakeResult{err: err}
	}
	roots := x509.NewCertPool()
	roots.AppendCertsFromPEM(pemData)
	cfg := &tls.Config{
	Certificates: []tls.Certificate{cert},
	ClientCAs: roots,
	}
	conn := tls.Server(rawConn, cfg)
	if err := conn.Handshake(); err != nil {
	return handshakeResult{err: err}
	}
	return handshakeResult{connState: conn.ConnectionState()}
	}

	// fakeProvider is an implementation of the certprovider.Provider interface
	// which returns the configured key material and error in calls to
	// KeyMaterial().
	type fakeProvider struct {
	km *certprovider.KeyMaterial
	err error
	}

	func (f fakeProvider) KeyMaterial(ctx context.Context) (certprovider.KeyMaterial, error) {
	return f.km, f.err
	}

	func (f *fakeProvider) Close() {}

	// makeIdentityProvider creates a new instance of the fakeProvider returning the
	// identity key material specified in the provider file paths.
	func makeIdentityProvider(t *testing.T, certPath, keyPath string) certprovider.Provider {
	t.Helper()
	cert, err := tls.LoadX509KeyPair(testdata.Path(certPath), testdata.Path(keyPath))
	if err != nil {
	t.Fatal(err)
	}
	return &fakeProvider{km: &certprovider.KeyMaterial{Certs: []tls.Certificate{cert}}}
	}

	// makeRootProvider creates a new instance of the fakeProvider returning the
	// root key material specified in the provider file paths.
	func makeRootProvider(t testing.T, caPath string) fakeProvider {
	pemData, err := ioutil.ReadFile(testdata.Path(caPath))
	if err != nil {
	t.Fatal(err)
	}
	roots := x509.NewCertPool()
	roots.AppendCertsFromPEM(pemData)
	return &fakeProvider{km: &certprovider.KeyMaterial{Roots: roots}}
	}

	// newTestContextWithHandshakeInfo returns a copy of parent with HandshakeInfo
	// context value added to it.
	func newTestContextWithHandshakeInfo(parent context.Context, root, identity certprovider.Provider, sans ...string) context.Context {
	// Creating the HandshakeInfo and adding it to the attributes is very
	// similar to what the CDS balancer would do when it intercepts calls to
	// NewSubConn().
	info := xdsinternal.NewHandshakeInfo(root, identity, sans...)
	addr := xdsinternal.SetHandshakeInfo(resolver.Address{}, info)

	// Moving the attributes from the resolver.Address to the context passed to
	// the handshaker is done in the transport layer. Since we directly call the
	// handshaker in these tests, we need to do the same here.
	contextWithHandshakeInfo := internal.NewClientHandshakeInfoContext.(func(context.Context, credentials.ClientHandshakeInfo) context.Context)
	return contextWithHandshakeInfo(parent, credentials.ClientHandshakeInfo{Attributes: addr.Attributes})
	}

	// compareAuthInfo compares the AuthInfo received on the client side after a
	// successful handshake with the authInfo available on the testServer.
	func compareAuthInfo(ctx context.Context, ts *testServer, ai credentials.AuthInfo) error {
	if ai.AuthType() != "tls" {
	return fmt.Errorf("ClientHandshake returned authType %q, want %q", ai.AuthType(), "tls")
	}
	info, ok := ai.(credentials.TLSInfo)
	if !ok {
	return fmt.Errorf("ClientHandshake returned authInfo of type %T, want %T", ai, credentials.TLSInfo{})
	}
	gotState := info.State

	// Read the handshake result from the testServer which contains the TLS
	// connection state and compare it with the one received on the client-side.
	val, err := ts.hsResult.Receive(ctx)
	if err != nil {
	return fmt.Errorf("testServer failed to return handshake result: %v", err)
	}
	hsr := val.(handshakeResult)
	if hsr.err != nil {
	return fmt.Errorf("testServer handshake failure: %v", hsr.err)
	}
	// AuthInfo contains a variety of information. We only verify a subset here.
	// This is the same subset which is verified in TLS credentials tests.
	if err := compareConnState(gotState, hsr.connState); err != nil {
	return err
	}
	return nil
	}

	func compareConnState(got, want tls.ConnectionState) error {
	switch {
	case got.Version != want.Version:
	return fmt.Errorf("TLS.ConnectionState got Version: %v, want: %v", got.Version, want.Version)
	case got.HandshakeComplete != want.HandshakeComplete:
	return fmt.Errorf("TLS.ConnectionState got HandshakeComplete: %v, want: %v", got.HandshakeComplete, want.HandshakeComplete)
	case got.CipherSuite != want.CipherSuite:
	return fmt.Errorf("TLS.ConnectionState got CipherSuite: %v, want: %v", got.CipherSuite, want.CipherSuite)
	case got.NegotiatedProtocol != want.NegotiatedProtocol:
	return fmt.Errorf("TLS.ConnectionState got NegotiatedProtocol: %v, want: %v", got.NegotiatedProtocol, want.NegotiatedProtocol)
	}
	return nil
	}

	// TestClientCredsWithoutFallback verifies that the call to
	// NewClientCredentials() fails when no fallback is specified.
	func (s) TestClientCredsWithoutFallback(t *testing.T) {
	if _, err := NewClientCredentials(ClientOptions{}); err == nil {
	t.Fatal("NewClientCredentials() succeeded without specifying fallback")
	}
	}

	// TestClientCredsInvalidHandshakeInfo verifies scenarios where the passed in
	// HandshakeInfo is invalid because it does not contain the expected certificate
	// providers.
	func (s) TestClientCredsInvalidHandshakeInfo(t *testing.T) {
	opts := ClientOptions{FallbackCreds: makeFallbackClientCreds(t)}
	creds, err := NewClientCredentials(opts)
	if err != nil {
	t.Fatalf("NewClientCredentials(%v) failed: %v", opts, err)
	}

	pCtx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
	defer cancel()
	ctx := newTestContextWithHandshakeInfo(pCtx, nil, &fakeProvider{})
	if _, _, err := creds.ClientHandshake(ctx, authority, nil); err == nil {
	t.Fatal("ClientHandshake succeeded without root certificate provider in HandshakeInfo")
	}
	}

	// TestClientCredsProviderFailure verifies the cases where an expected
	// certificate provider is missing in the HandshakeInfo value in the context.
	func (s) TestClientCredsProviderFailure(t *testing.T) {
	opts := ClientOptions{FallbackCreds: makeFallbackClientCreds(t)}
	creds, err := NewClientCredentials(opts)
	if err != nil {
	t.Fatalf("NewClientCredentials(%v) failed: %v", opts, err)
	}

	tests := []struct {
	desc string
	rootProvider certprovider.Provider
	identityProvider certprovider.Provider
	wantErr string
	}{
	{
	desc: "erroring root provider",
	rootProvider: &fakeProvider{err: errors.New("root provider error")},
	wantErr: "root provider error",
	},
	{
	desc: "erroring identity provider",
	rootProvider: &fakeProvider{km: &certprovider.KeyMaterial{}},
	identityProvider: &fakeProvider{err: errors.New("identity provider error")},
	wantErr: "identity provider error",
	},
	}
	for _, test := range tests {
	t.Run(test.desc, func(t *testing.T) {
	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
	defer cancel()
	ctx = newTestContextWithHandshakeInfo(ctx, test.rootProvider, test.identityProvider)
	if _, _, err := creds.ClientHandshake(ctx, authority, nil); err == nil \|\| !strings.Contains(err.Error(), test.wantErr) {
	t.Fatalf("ClientHandshake() returned error: %q, wantErr: %q", err, test.wantErr)
	}
	})
	}
	}

	// TestClientCredsSuccess verifies successful client handshake cases.
	func (s) TestClientCredsSuccess(t *testing.T) {
	tests := []struct {
	desc string
	handshakeFunc testHandshakeFunc
	handshakeInfoCtx func(ctx context.Context) context.Context
	}{
	{
	desc: "fallback",
	handshakeFunc: testServerTLSHandshake,
	handshakeInfoCtx: func(ctx context.Context) context.Context {
	// Since we don't add a HandshakeInfo to the context, the
	// ClientHandshake() method will delegate to the fallback.
	return ctx
	},
	},
	{
	desc: "TLS",
	handshakeFunc: testServerTLSHandshake,
	handshakeInfoCtx: func(ctx context.Context) context.Context {
	return newTestContextWithHandshakeInfo(ctx, makeRootProvider(t, "x509/server_ca_cert.pem"), nil, defaultTestCertSAN)
	},
	},
	{
	desc: "mTLS",
	handshakeFunc: testServerMutualTLSHandshake,
	handshakeInfoCtx: func(ctx context.Context) context.Context {
	return newTestContextWithHandshakeInfo(ctx, makeRootProvider(t, "x509/server_ca_cert.pem"), makeIdentityProvider(t, "x509/server1_cert.pem", "x509/server1_key.pem"), defaultTestCertSAN)
	},
	},
	{
	desc: "mTLS with no acceptedSANs specified",
	handshakeFunc: testServerMutualTLSHandshake,
	handshakeInfoCtx: func(ctx context.Context) context.Context {
	return newTestContextWithHandshakeInfo(ctx, makeRootProvider(t, "x509/server_ca_cert.pem"), makeIdentityProvider(t, "x509/server1_cert.pem", "x509/server1_key.pem"))
	},
	},
	}

	for _, test := range tests {
	t.Run(test.desc, func(t *testing.T) {
	ts := newTestServerWithHandshakeFunc(test.handshakeFunc)
	defer ts.stop()

	opts := ClientOptions{FallbackCreds: makeFallbackClientCreds(t)}
	creds, err := NewClientCredentials(opts)
	if err != nil {
	t.Fatalf("NewClientCredentials(%v) failed: %v", opts, err)
	}

	conn, err := net.Dial("tcp", ts.address)
	if err != nil {
	t.Fatalf("net.Dial(%s) failed: %v", ts.address, err)
	}
	defer conn.Close()

	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
	defer cancel()
	_, ai, err := creds.ClientHandshake(test.handshakeInfoCtx(ctx), authority, conn)
	if err != nil {
	t.Fatalf("ClientHandshake() returned failed: %q", err)
	}
	if err := compareAuthInfo(ctx, ts, ai); err != nil {
	t.Fatal(err)
	}
	})
	}
	}

	func (s) TestClientCredsHandshakeTimeout(t *testing.T) {
	clientDone := make(chan struct{})
	// A handshake function which simulates a handshake timeout from the
	// server-side by simply blocking on the client-side handshake to timeout
	// and not writing any handshake data.
	hErr := errors.New("server handshake error")
	ts := newTestServerWithHandshakeFunc(func(rawConn net.Conn) handshakeResult {
	<-clientDone
	return handshakeResult{err: hErr}
	})
	defer ts.stop()

	opts := ClientOptions{FallbackCreds: makeFallbackClientCreds(t)}
	creds, err := NewClientCredentials(opts)
	if err != nil {
	t.Fatalf("NewClientCredentials(%v) failed: %v", opts, err)
	}

	conn, err := net.Dial("tcp", ts.address)
	if err != nil {
	t.Fatalf("net.Dial(%s) failed: %v", ts.address, err)
	}
	defer conn.Close()

	sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
	defer sCancel()
	ctx := newTestContextWithHandshakeInfo(sCtx, makeRootProvider(t, "x509/server_ca_cert.pem"), nil, defaultTestCertSAN)
	if _, _, err := creds.ClientHandshake(ctx, authority, conn); err == nil {
	t.Fatal("ClientHandshake() succeeded when expected to timeout")
	}
	close(clientDone)

	// Read the handshake result from the testServer and make sure the expected
	// error is returned.
	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
	defer cancel()
	val, err := ts.hsResult.Receive(ctx)
	if err != nil {
	t.Fatalf("testServer failed to return handshake result: %v", err)
	}
	hsr := val.(handshakeResult)
	if hsr.err != hErr {
	t.Fatalf("testServer handshake returned error: %v, want: %v", hsr.err, hErr)
	}
	}

	// TestClientCredsHandshakeFailure verifies different handshake failure cases.
	func (s) TestClientCredsHandshakeFailure(t *testing.T) {
	tests := []struct {
	desc string
	handshakeFunc testHandshakeFunc
	rootProvider certprovider.Provider
	san string
	wantErr string
	}{
	{
	desc: "cert validation failure",
	handshakeFunc: testServerTLSHandshake,
	rootProvider: makeRootProvider(t, "x509/client_ca_cert.pem"),
	san: defaultTestCertSAN,
	wantErr: "x509: certificate signed by unknown authority",
	},
	{
	desc: "SAN mismatch",
	handshakeFunc: testServerTLSHandshake,
	rootProvider: makeRootProvider(t, "x509/server_ca_cert.pem"),
	san: "bad-san",
	wantErr: "does not match any of the accepted SANs",
	},
	}

	for _, test := range tests {
	t.Run(test.desc, func(t *testing.T) {
	ts := newTestServerWithHandshakeFunc(test.handshakeFunc)
	defer ts.stop()

	opts := ClientOptions{FallbackCreds: makeFallbackClientCreds(t)}
	creds, err := NewClientCredentials(opts)
	if err != nil {
	t.Fatalf("NewClientCredentials(%v) failed: %v", opts, err)
	}

	conn, err := net.Dial("tcp", ts.address)
	if err != nil {
	t.Fatalf("net.Dial(%s) failed: %v", ts.address, err)
	}
	defer conn.Close()

	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
	defer cancel()
	ctx = newTestContextWithHandshakeInfo(ctx, test.rootProvider, nil, test.san)
	if _, _, err := creds.ClientHandshake(ctx, authority, conn); err == nil \|\| !strings.Contains(err.Error(), test.wantErr) {
	t.Fatalf("ClientHandshake() returned %q, wantErr %q", err, test.wantErr)
	}
	})
	}
	}

	// TestClientCredsProviderSwitch verifies the case where the first attempt of
	// ClientHandshake fails because of a handshake failure. Then we update the
	// certificate provider and the second attempt succeeds. This is an
	// approximation of the flow of events when the control plane specifies new
	// security config which results in new certificate providers being used.
	func (s) TestClientCredsProviderSwitch(t *testing.T) {
	ts := newTestServerWithHandshakeFunc(testServerTLSHandshake)
	defer ts.stop()

	opts := ClientOptions{FallbackCreds: makeFallbackClientCreds(t)}
	creds, err := NewClientCredentials(opts)
	if err != nil {
	t.Fatalf("NewClientCredentials(%v) failed: %v", opts, err)
	}

	conn, err := net.Dial("tcp", ts.address)
	if err != nil {
	t.Fatalf("net.Dial(%s) failed: %v", ts.address, err)
	}
	defer conn.Close()

	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
	defer cancel()
	// Create a root provider which will fail the handshake because it does not
	// use the correct trust roots.
	root1 := makeRootProvider(t, "x509/client_ca_cert.pem")
	handshakeInfo := xdsinternal.NewHandshakeInfo(root1, nil, defaultTestCertSAN)

	// We need to repeat most of what newTestContextWithHandshakeInfo() does
	// here because we need access to the underlying HandshakeInfo so that we
	// can update it before the next call to ClientHandshake().
	addr := xdsinternal.SetHandshakeInfo(resolver.Address{}, handshakeInfo)
	contextWithHandshakeInfo := internal.NewClientHandshakeInfoContext.(func(context.Context, credentials.ClientHandshakeInfo) context.Context)
	ctx = contextWithHandshakeInfo(ctx, credentials.ClientHandshakeInfo{Attributes: addr.Attributes})
	if _, _, err := creds.ClientHandshake(ctx, authority, conn); err == nil {
	t.Fatal("ClientHandshake() succeeded when expected to fail")
	}
	// Drain the result channel on the test server so that we can inspect the
	// result for the next handshake.
	_, err = ts.hsResult.Receive(ctx)
	if err != nil {
	t.Errorf("testServer failed to return handshake result: %v", err)
	}

	conn, err = net.Dial("tcp", ts.address)
	if err != nil {
	t.Fatalf("net.Dial(%s) failed: %v", ts.address, err)
	}
	defer conn.Close()

	// Create a new root provider which uses the correct trust roots. And update
	// the HandshakeInfo with the new provider.
	root2 := makeRootProvider(t, "x509/server_ca_cert.pem")
	handshakeInfo.SetRootCertProvider(root2)
	_, ai, err := creds.ClientHandshake(ctx, authority, conn)
	if err != nil {
	t.Fatalf("ClientHandshake() returned failed: %q", err)
	}
	if err := compareAuthInfo(ctx, ts, ai); err != nil {
	t.Fatal(err)
	}
	}

	// TestClientClone verifies the Clone() method on client credentials.
	func (s) TestClientClone(t *testing.T) {
	opts := ClientOptions{FallbackCreds: makeFallbackClientCreds(t)}
	orig, err := NewClientCredentials(opts)
	if err != nil {
	t.Fatalf("NewClientCredentials(%v) failed: %v", opts, err)
	}

	// The credsImpl does not have any exported fields, and it does not make
	// sense to use any cmp options to look deep into. So, all we make sure here
	// is that the cloned object points to a different location in memory.
	if clone := orig.Clone(); clone == orig {
	t.Fatal("return value from Clone() doesn't point to new credentials instance")
	}
	}