credentials/tls/certprovider/pemfile/watcher_test.go - third_party/github.com/grpc/grpc-go - Git at Google

 // +build go1.12

 /*
  *
  * Copyright 2020 gRPC authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
  *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  *
  */

 package pemfile

 import (
 	"context"
 	"fmt"
 	"io/ioutil"
 	"math/big"
 	"os"
 	"path"
 	"testing"
 	"time"

 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"

 	"google.golang.org/grpc/credentials/tls/certprovider"
 	"google.golang.org/grpc/internal/grpctest"
 	"google.golang.org/grpc/internal/testutils"
 	"google.golang.org/grpc/testdata"
 )

 const (
 	// These are the names of files inside temporary directories, which the
 	// plugin is asked to watch.
 	certFile = "cert.pem"
 	keyFile  = "key.pem"
 	rootFile = "ca.pem"

 	defaultTestRefreshDuration = 100 * time.Millisecond
 	defaultTestTimeout         = 5 * time.Second
 )

 type s struct {
 	grpctest.Tester
 }

 func Test(t *testing.T) {
 	grpctest.RunSubTests(t, s{})
 }

 func compareKeyMaterial(got, want *certprovider.KeyMaterial) error {
 	// x509.Certificate type defines an Equal() method, but does not check for
 	// nil. This has been fixed in
 	// https://github.com/golang/go/commit/89865f8ba64ccb27f439cce6daaa37c9aa38f351,
 	// but this is only available starting go1.14.
 	// TODO(easwars): Remove this check once we remove support for go1.13.
 	if (got.Certs == nil && want.Certs != nil) || (want.Certs == nil && got.Certs != nil) {
 		return fmt.Errorf("keyMaterial certs = %+v, want %+v", got, want)
 	}
 	if !cmp.Equal(got.Certs, want.Certs, cmp.AllowUnexported(big.Int{})) {
 		return fmt.Errorf("keyMaterial certs = %+v, want %+v", got, want)
 	}
 	// x509.CertPool contains only unexported fields some of which contain other
 	// unexported fields. So usage of cmp.AllowUnexported() or
 	// cmpopts.IgnoreUnexported() does not help us much here. Also, the standard
 	// library does not provide a way to compare CertPool values. Comparing the
 	// subjects field of the certs in the CertPool seems like a reasonable
 	// approach.
 	if gotR, wantR := got.Roots.Subjects(), want.Roots.Subjects(); !cmp.Equal(gotR, wantR, cmpopts.EquateEmpty()) {
 		return fmt.Errorf("keyMaterial roots = %v, want %v", gotR, wantR)
 	}
 	return nil
 }

 // TestNewProvider tests the NewProvider() function with different inputs.
 func (s) TestNewProvider(t *testing.T) {
 	tests := []struct {
 		desc      string
 		options   Options
 		wantError bool
 	}{
 		{
 			desc:      "No credential files specified",
 			options:   Options{},
 			wantError: true,
 		},
 		{
 			desc: "Only identity cert is specified",
 			options: Options{
 				CertFile: testdata.Path("x509/client1_cert.pem"),
 			},
 			wantError: true,
 		},
 		{
 			desc: "Only identity key is specified",
 			options: Options{
 				KeyFile: testdata.Path("x509/client1_key.pem"),
 			},
 			wantError: true,
 		},
 		{
 			desc: "Identity cert/key pair is specified",
 			options: Options{
 				KeyFile:  testdata.Path("x509/client1_key.pem"),
 				CertFile: testdata.Path("x509/client1_cert.pem"),
 			},
 		},
 		{
 			desc: "Only root certs are specified",
 			options: Options{
 				RootFile: testdata.Path("x509/client_ca_cert.pem"),
 			},
 		},
 		{
 			desc: "Everything is specified",
 			options: Options{
 				KeyFile:  testdata.Path("x509/client1_key.pem"),
 				CertFile: testdata.Path("x509/client1_cert.pem"),
 				RootFile: testdata.Path("x509/client_ca_cert.pem"),
 			},
 			wantError: false,
 		},
 	}
 	for _, test := range tests {
 		t.Run(test.desc, func(t *testing.T) {
 			provider, err := NewProvider(test.options)
 			if (err != nil) != test.wantError {
 				t.Fatalf("NewProvider(%v) = %v, want %v", test.options, err, test.wantError)
 			}
 			if err != nil {
 				return
 			}
 			provider.Close()
 		})
 	}
 }

 // wrappedDistributor wraps a distributor and pushes on a channel whenever new
 // key material is pushed to the distributor.
 type wrappedDistributor struct {
 	*certprovider.Distributor
 	distCh *testutils.Channel
 }

 func newWrappedDistributor(distCh *testutils.Channel) *wrappedDistributor {
 	return &wrappedDistributor{
 		distCh:      distCh,
 		Distributor: certprovider.NewDistributor(),
 	}
 }

 func (wd *wrappedDistributor) Set(km *certprovider.KeyMaterial, err error) {
 	wd.Distributor.Set(km, err)
 	wd.distCh.Send(nil)
 }

 func createTmpFile(t *testing.T, src, dst string) {
 	t.Helper()

 	data, err := ioutil.ReadFile(src)
 	if err != nil {
 		t.Fatalf("ioutil.ReadFile(%q) failed: %v", src, err)
 	}
 	if err := ioutil.WriteFile(dst, data, os.ModePerm); err != nil {
 		t.Fatalf("ioutil.WriteFile(%q) failed: %v", dst, err)
 	}
 	t.Logf("Wrote file at: %s", dst)
 	t.Logf("%s", string(data))
 }

 // createTempDirWithFiles creates a temporary directory under the system default
 // tempDir with the given dirSuffix. It also reads from certSrc, keySrc and
 // rootSrc files are creates appropriate files under the newly create tempDir.
 // Returns the name of the created tempDir.
 func createTmpDirWithFiles(t *testing.T, dirSuffix, certSrc, keySrc, rootSrc string) string {
 	t.Helper()

 	// Create a temp directory. Passing an empty string for the first argument
 	// uses the system temp directory.
 	dir, err := ioutil.TempDir("", dirSuffix)
 	if err != nil {
 		t.Fatalf("ioutil.TempDir() failed: %v", err)
 	}
 	t.Logf("Using tmpdir: %s", dir)

 	createTmpFile(t, testdata.Path(certSrc), path.Join(dir, certFile))
 	createTmpFile(t, testdata.Path(keySrc), path.Join(dir, keyFile))
 	createTmpFile(t, testdata.Path(rootSrc), path.Join(dir, rootFile))
 	return dir
 }

 // initializeProvider performs setup steps common to all tests (except the one
 // which uses symlinks).
 func initializeProvider(t *testing.T, testName string) (string, certprovider.Provider, *testutils.Channel, func()) {
 	t.Helper()

 	// Override the newDistributor to one which pushes on a channel that we
 	// can block on.
 	origDistributorFunc := newDistributor
 	distCh := testutils.NewChannel()
 	d := newWrappedDistributor(distCh)
 	newDistributor = func() distributor { return d }

 	// Create a new provider to watch the files in tmpdir.
 	dir := createTmpDirWithFiles(t, testName+"*", "x509/client1_cert.pem", "x509/client1_key.pem", "x509/client_ca_cert.pem")
 	opts := Options{
 		CertFile:        path.Join(dir, certFile),
 		KeyFile:         path.Join(dir, keyFile),
 		RootFile:        path.Join(dir, rootFile),
 		RefreshDuration: defaultTestRefreshDuration,
 	}
 	prov, err := NewProvider(opts)
 	if err != nil {
 		t.Fatalf("NewProvider(%+v) failed: %v", opts, err)
 	}

 	// Make sure the provider picks up the files and pushes the key material on
 	// to the distributors.
 	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer cancel()
 	for i := 0; i < 2; i++ {
 		// Since we have root and identity certs, we need to make sure the
 		// update is pushed on both of them.
 		if _, err := distCh.Receive(ctx); err != nil {
 			t.Fatalf("timeout waiting for provider to read files and push key material to distributor: %v", err)
 		}
 	}

 	return dir, prov, distCh, func() {
 		newDistributor = origDistributorFunc
 		prov.Close()
 	}
 }

 // TestProvider_NoUpdate tests the case where a file watcher plugin is created
 // successfully, and the underlying files do not change. Verifies that the
 // plugin does not push new updates to the distributor in this case.
 func (s) TestProvider_NoUpdate(t *testing.T) {
 	_, prov, distCh, cancel := initializeProvider(t, "no_update")
 	defer cancel()

 	// Make sure the provider is healthy and returns key material.
 	ctx, cc := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer cc()
 	if _, err := prov.KeyMaterial(ctx); err != nil {
 		t.Fatalf("provider.KeyMaterial() failed: %v", err)
 	}

 	// Files haven't change. Make sure no updates are pushed by the provider.
 	sCtx, sc := context.WithTimeout(context.Background(), 2*defaultTestRefreshDuration)
 	defer sc()
 	if _, err := distCh.Receive(sCtx); err == nil {
 		t.Fatal("new key material pushed to distributor when underlying files did not change")
 	}
 }

 // TestProvider_UpdateSuccess tests the case where a file watcher plugin is
 // created successfully and the underlying files change. Verifies that the
 // changes are picked up by the provider.
 func (s) TestProvider_UpdateSuccess(t *testing.T) {
 	dir, prov, distCh, cancel := initializeProvider(t, "update_success")
 	defer cancel()

 	// Make sure the provider is healthy and returns key material.
 	ctx, cc := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer cc()
 	km1, err := prov.KeyMaterial(ctx)
 	if err != nil {
 		t.Fatalf("provider.KeyMaterial() failed: %v", err)
 	}

 	// Change only the root file.
 	createTmpFile(t, testdata.Path("x509/server_ca_cert.pem"), path.Join(dir, rootFile))
 	if _, err := distCh.Receive(ctx); err != nil {
 		t.Fatal("timeout waiting for new key material to be pushed to the distributor")
 	}

 	// Make sure update is picked up.
 	km2, err := prov.KeyMaterial(ctx)
 	if err != nil {
 		t.Fatalf("provider.KeyMaterial() failed: %v", err)
 	}
 	if err := compareKeyMaterial(km1, km2); err == nil {
 		t.Fatal("expected provider to return new key material after update to underlying file")
 	}

 	// Change only cert/key files.
 	createTmpFile(t, testdata.Path("x509/client2_cert.pem"), path.Join(dir, certFile))
 	createTmpFile(t, testdata.Path("x509/client2_key.pem"), path.Join(dir, keyFile))
 	if _, err := distCh.Receive(ctx); err != nil {
 		t.Fatal("timeout waiting for new key material to be pushed to the distributor")
 	}

 	// Make sure update is picked up.
 	km3, err := prov.KeyMaterial(ctx)
 	if err != nil {
 		t.Fatalf("provider.KeyMaterial() failed: %v", err)
 	}
 	if err := compareKeyMaterial(km2, km3); err == nil {
 		t.Fatal("expected provider to return new key material after update to underlying file")
 	}
 }

 // TestProvider_UpdateSuccessWithSymlink tests the case where a file watcher
 // plugin is created successfully to watch files through a symlink and the
 // symlink is updates to point to new files. Verifies that the changes are
 // picked up by the provider.
 func (s) TestProvider_UpdateSuccessWithSymlink(t *testing.T) {
 	// Override the newDistributor to one which pushes on a channel that we
 	// can block on.
 	origDistributorFunc := newDistributor
 	distCh := testutils.NewChannel()
 	d := newWrappedDistributor(distCh)
 	newDistributor = func() distributor { return d }
 	defer func() { newDistributor = origDistributorFunc }()

 	// Create two tempDirs with different files.
 	dir1 := createTmpDirWithFiles(t, "update_with_symlink1_*", "x509/client1_cert.pem", "x509/client1_key.pem", "x509/client_ca_cert.pem")
 	dir2 := createTmpDirWithFiles(t, "update_with_symlink2_*", "x509/server1_cert.pem", "x509/server1_key.pem", "x509/server_ca_cert.pem")

 	// Create a symlink under a new tempdir, and make it point to dir1.
 	tmpdir, err := ioutil.TempDir("", "test_symlink_*")
 	if err != nil {
 		t.Fatalf("ioutil.TempDir() failed: %v", err)
 	}
 	symLinkName := path.Join(tmpdir, "test_symlink")
 	if err := os.Symlink(dir1, symLinkName); err != nil {
 		t.Fatalf("failed to create symlink to %q: %v", dir1, err)
 	}

 	// Create a provider which watches the files pointed to by the symlink.
 	opts := Options{
 		CertFile:        path.Join(symLinkName, certFile),
 		KeyFile:         path.Join(symLinkName, keyFile),
 		RootFile:        path.Join(symLinkName, rootFile),
 		RefreshDuration: defaultTestRefreshDuration,
 	}
 	prov, err := NewProvider(opts)
 	if err != nil {
 		t.Fatalf("NewProvider(%+v) failed: %v", opts, err)
 	}
 	defer prov.Close()

 	// Make sure the provider picks up the files and pushes the key material on
 	// to the distributors.
 	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer cancel()
 	for i := 0; i < 2; i++ {
 		// Since we have root and identity certs, we need to make sure the
 		// update is pushed on both of them.
 		if _, err := distCh.Receive(ctx); err != nil {
 			t.Fatalf("timeout waiting for provider to read files and push key material to distributor: %v", err)
 		}
 	}
 	km1, err := prov.KeyMaterial(ctx)
 	if err != nil {
 		t.Fatalf("provider.KeyMaterial() failed: %v", err)
 	}

 	// Update the symlink to point to dir2.
 	symLinkTmpName := path.Join(tmpdir, "test_symlink.tmp")
 	if err := os.Symlink(dir2, symLinkTmpName); err != nil {
 		t.Fatalf("failed to create symlink to %q: %v", dir2, err)
 	}
 	if err := os.Rename(symLinkTmpName, symLinkName); err != nil {
 		t.Fatalf("failed to update symlink: %v", err)
 	}

 	// Make sure the provider picks up the new files and pushes the key material
 	// on to the distributors.
 	for i := 0; i < 2; i++ {
 		// Since we have root and identity certs, we need to make sure the
 		// update is pushed on both of them.
 		if _, err := distCh.Receive(ctx); err != nil {
 			t.Fatalf("timeout waiting for provider to read files and push key material to distributor: %v", err)
 		}
 	}
 	km2, err := prov.KeyMaterial(ctx)
 	if err != nil {
 		t.Fatalf("provider.KeyMaterial() failed: %v", err)
 	}

 	if err := compareKeyMaterial(km1, km2); err == nil {
 		t.Fatal("expected provider to return new key material after symlink update")
 	}
 }

 // TestProvider_UpdateFailure_ThenSuccess tests the case where updating cert/key
 // files fail. Verifies that the failed update does not push anything on the
 // distributor. Then the update succeeds, and the test verifies that the key
 // material is updated.
 func (s) TestProvider_UpdateFailure_ThenSuccess(t *testing.T) {
 	dir, prov, distCh, cancel := initializeProvider(t, "update_failure")
 	defer cancel()

 	// Make sure the provider is healthy and returns key material.
 	ctx, cc := context.WithTimeout(context.Background(), defaultTestTimeout)
 	defer cc()
 	km1, err := prov.KeyMaterial(ctx)
 	if err != nil {
 		t.Fatalf("provider.KeyMaterial() failed: %v", err)
 	}

 	// Update only the cert file. The key file is left unchanged. This should
 	// lead to these two files being not compatible with each other. This
 	// simulates the case where the watching goroutine might catch the files in
 	// the midst of an update.
 	createTmpFile(t, testdata.Path("x509/server1_cert.pem"), path.Join(dir, certFile))

 	// Since the last update left the files in an incompatible state, the update
 	// should not be picked up by our provider.
 	sCtx, sc := context.WithTimeout(context.Background(), 2*defaultTestRefreshDuration)
 	defer sc()
 	if _, err := distCh.Receive(sCtx); err == nil {
 		t.Fatal("new key material pushed to distributor when underlying files did not change")
 	}

 	// The provider should return key material corresponding to the old state.
 	km2, err := prov.KeyMaterial(ctx)
 	if err != nil {
 		t.Fatalf("provider.KeyMaterial() failed: %v", err)
 	}
 	if err := compareKeyMaterial(km1, km2); err != nil {
 		t.Fatalf("expected provider to not update key material: %v", err)
 	}

 	// Update the key file to match the cert file.
 	createTmpFile(t, testdata.Path("x509/server1_key.pem"), path.Join(dir, keyFile))

 	// Make sure update is picked up.
 	if _, err := distCh.Receive(ctx); err != nil {
 		t.Fatal("timeout waiting for new key material to be pushed to the distributor")
 	}
 	km3, err := prov.KeyMaterial(ctx)
 	if err != nil {
 		t.Fatalf("provider.KeyMaterial() failed: %v", err)
 	}
 	if err := compareKeyMaterial(km2, km3); err == nil {
 		t.Fatal("expected provider to return new key material after update to underlying file")
 	}
 }
	// +build go1.12

	/*
	*
	* Copyright 2020 gRPC authors.
	*
	* Licensed under the Apache License, Version 2.0 (the "License");
	* you may not use this file except in compliance with the License.
	* You may obtain a copy of the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS,
	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	* See the License for the specific language governing permissions and
	* limitations under the License.
	*
	*/

	package pemfile

	import (
	"context"
	"fmt"
	"io/ioutil"
	"math/big"
	"os"
	"path"
	"testing"
	"time"

	"github.com/google/go-cmp/cmp"
	"github.com/google/go-cmp/cmp/cmpopts"

	"google.golang.org/grpc/credentials/tls/certprovider"
	"google.golang.org/grpc/internal/grpctest"
	"google.golang.org/grpc/internal/testutils"
	"google.golang.org/grpc/testdata"
	)

	const (
	// These are the names of files inside temporary directories, which the
	// plugin is asked to watch.
	certFile = "cert.pem"
	keyFile = "key.pem"
	rootFile = "ca.pem"

	defaultTestRefreshDuration = 100 * time.Millisecond
	defaultTestTimeout = 5 * time.Second
	)

	type s struct {
	grpctest.Tester
	}

	func Test(t *testing.T) {
	grpctest.RunSubTests(t, s{})
	}

	func compareKeyMaterial(got, want *certprovider.KeyMaterial) error {
	// x509.Certificate type defines an Equal() method, but does not check for
	// nil. This has been fixed in
	// https://github.com/golang/go/commit/89865f8ba64ccb27f439cce6daaa37c9aa38f351,
	// but this is only available starting go1.14.
	// TODO(easwars): Remove this check once we remove support for go1.13.
	if (got.Certs == nil && want.Certs != nil) \|\| (want.Certs == nil && got.Certs != nil) {
	return fmt.Errorf("keyMaterial certs = %+v, want %+v", got, want)
	}
	if !cmp.Equal(got.Certs, want.Certs, cmp.AllowUnexported(big.Int{})) {
	return fmt.Errorf("keyMaterial certs = %+v, want %+v", got, want)
	}
	// x509.CertPool contains only unexported fields some of which contain other
	// unexported fields. So usage of cmp.AllowUnexported() or
	// cmpopts.IgnoreUnexported() does not help us much here. Also, the standard
	// library does not provide a way to compare CertPool values. Comparing the
	// subjects field of the certs in the CertPool seems like a reasonable
	// approach.
	if gotR, wantR := got.Roots.Subjects(), want.Roots.Subjects(); !cmp.Equal(gotR, wantR, cmpopts.EquateEmpty()) {
	return fmt.Errorf("keyMaterial roots = %v, want %v", gotR, wantR)
	}
	return nil
	}

	// TestNewProvider tests the NewProvider() function with different inputs.
	func (s) TestNewProvider(t *testing.T) {
	tests := []struct {
	desc string
	options Options
	wantError bool
	}{
	{
	desc: "No credential files specified",
	options: Options{},
	wantError: true,
	},
	{
	desc: "Only identity cert is specified",
	options: Options{
	CertFile: testdata.Path("x509/client1_cert.pem"),
	},
	wantError: true,
	},
	{
	desc: "Only identity key is specified",
	options: Options{
	KeyFile: testdata.Path("x509/client1_key.pem"),
	},
	wantError: true,
	},
	{
	desc: "Identity cert/key pair is specified",
	options: Options{
	KeyFile: testdata.Path("x509/client1_key.pem"),
	CertFile: testdata.Path("x509/client1_cert.pem"),
	},
	},
	{
	desc: "Only root certs are specified",
	options: Options{
	RootFile: testdata.Path("x509/client_ca_cert.pem"),
	},
	},
	{
	desc: "Everything is specified",
	options: Options{
	KeyFile: testdata.Path("x509/client1_key.pem"),
	CertFile: testdata.Path("x509/client1_cert.pem"),
	RootFile: testdata.Path("x509/client_ca_cert.pem"),
	},
	wantError: false,
	},
	}
	for _, test := range tests {
	t.Run(test.desc, func(t *testing.T) {
	provider, err := NewProvider(test.options)
	if (err != nil) != test.wantError {
	t.Fatalf("NewProvider(%v) = %v, want %v", test.options, err, test.wantError)
	}
	if err != nil {
	return
	}
	provider.Close()
	})
	}
	}

	// wrappedDistributor wraps a distributor and pushes on a channel whenever new
	// key material is pushed to the distributor.
	type wrappedDistributor struct {
	*certprovider.Distributor
	distCh *testutils.Channel
	}

	func newWrappedDistributor(distCh testutils.Channel) wrappedDistributor {
	return &wrappedDistributor{
	distCh: distCh,
	Distributor: certprovider.NewDistributor(),
	}
	}

	func (wd wrappedDistributor) Set(km certprovider.KeyMaterial, err error) {
	wd.Distributor.Set(km, err)
	wd.distCh.Send(nil)
	}

	func createTmpFile(t *testing.T, src, dst string) {
	t.Helper()

	data, err := ioutil.ReadFile(src)
	if err != nil {
	t.Fatalf("ioutil.ReadFile(%q) failed: %v", src, err)
	}
	if err := ioutil.WriteFile(dst, data, os.ModePerm); err != nil {
	t.Fatalf("ioutil.WriteFile(%q) failed: %v", dst, err)
	}
	t.Logf("Wrote file at: %s", dst)
	t.Logf("%s", string(data))
	}

	// createTempDirWithFiles creates a temporary directory under the system default
	// tempDir with the given dirSuffix. It also reads from certSrc, keySrc and
	// rootSrc files are creates appropriate files under the newly create tempDir.
	// Returns the name of the created tempDir.
	func createTmpDirWithFiles(t *testing.T, dirSuffix, certSrc, keySrc, rootSrc string) string {
	t.Helper()

	// Create a temp directory. Passing an empty string for the first argument
	// uses the system temp directory.
	dir, err := ioutil.TempDir("", dirSuffix)
	if err != nil {
	t.Fatalf("ioutil.TempDir() failed: %v", err)
	}
	t.Logf("Using tmpdir: %s", dir)

	createTmpFile(t, testdata.Path(certSrc), path.Join(dir, certFile))
	createTmpFile(t, testdata.Path(keySrc), path.Join(dir, keyFile))
	createTmpFile(t, testdata.Path(rootSrc), path.Join(dir, rootFile))
	return dir
	}

	// initializeProvider performs setup steps common to all tests (except the one
	// which uses symlinks).
	func initializeProvider(t testing.T, testName string) (string, certprovider.Provider, testutils.Channel, func()) {
	t.Helper()

	// Override the newDistributor to one which pushes on a channel that we
	// can block on.
	origDistributorFunc := newDistributor
	distCh := testutils.NewChannel()
	d := newWrappedDistributor(distCh)
	newDistributor = func() distributor { return d }

	// Create a new provider to watch the files in tmpdir.
	dir := createTmpDirWithFiles(t, testName+"*", "x509/client1_cert.pem", "x509/client1_key.pem", "x509/client_ca_cert.pem")
	opts := Options{
	CertFile: path.Join(dir, certFile),
	KeyFile: path.Join(dir, keyFile),
	RootFile: path.Join(dir, rootFile),
	RefreshDuration: defaultTestRefreshDuration,
	}
	prov, err := NewProvider(opts)
	if err != nil {
	t.Fatalf("NewProvider(%+v) failed: %v", opts, err)
	}

	// Make sure the provider picks up the files and pushes the key material on
	// to the distributors.
	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
	defer cancel()
	for i := 0; i < 2; i++ {
	// Since we have root and identity certs, we need to make sure the
	// update is pushed on both of them.
	if _, err := distCh.Receive(ctx); err != nil {
	t.Fatalf("timeout waiting for provider to read files and push key material to distributor: %v", err)
	}
	}

	return dir, prov, distCh, func() {
	newDistributor = origDistributorFunc
	prov.Close()
	}
	}

	// TestProvider_NoUpdate tests the case where a file watcher plugin is created
	// successfully, and the underlying files do not change. Verifies that the
	// plugin does not push new updates to the distributor in this case.
	func (s) TestProvider_NoUpdate(t *testing.T) {
	_, prov, distCh, cancel := initializeProvider(t, "no_update")
	defer cancel()

	// Make sure the provider is healthy and returns key material.
	ctx, cc := context.WithTimeout(context.Background(), defaultTestTimeout)
	defer cc()
	if _, err := prov.KeyMaterial(ctx); err != nil {
	t.Fatalf("provider.KeyMaterial() failed: %v", err)
	}

	// Files haven't change. Make sure no updates are pushed by the provider.
	sCtx, sc := context.WithTimeout(context.Background(), 2*defaultTestRefreshDuration)
	defer sc()
	if _, err := distCh.Receive(sCtx); err == nil {
	t.Fatal("new key material pushed to distributor when underlying files did not change")
	}
	}

	// TestProvider_UpdateSuccess tests the case where a file watcher plugin is
	// created successfully and the underlying files change. Verifies that the
	// changes are picked up by the provider.
	func (s) TestProvider_UpdateSuccess(t *testing.T) {
	dir, prov, distCh, cancel := initializeProvider(t, "update_success")
	defer cancel()

	// Make sure the provider is healthy and returns key material.
	ctx, cc := context.WithTimeout(context.Background(), defaultTestTimeout)
	defer cc()
	km1, err := prov.KeyMaterial(ctx)
	if err != nil {
	t.Fatalf("provider.KeyMaterial() failed: %v", err)
	}

	// Change only the root file.
	createTmpFile(t, testdata.Path("x509/server_ca_cert.pem"), path.Join(dir, rootFile))
	if _, err := distCh.Receive(ctx); err != nil {
	t.Fatal("timeout waiting for new key material to be pushed to the distributor")
	}

	// Make sure update is picked up.
	km2, err := prov.KeyMaterial(ctx)
	if err != nil {
	t.Fatalf("provider.KeyMaterial() failed: %v", err)
	}
	if err := compareKeyMaterial(km1, km2); err == nil {
	t.Fatal("expected provider to return new key material after update to underlying file")
	}

	// Change only cert/key files.
	createTmpFile(t, testdata.Path("x509/client2_cert.pem"), path.Join(dir, certFile))
	createTmpFile(t, testdata.Path("x509/client2_key.pem"), path.Join(dir, keyFile))
	if _, err := distCh.Receive(ctx); err != nil {
	t.Fatal("timeout waiting for new key material to be pushed to the distributor")
	}

	// Make sure update is picked up.
	km3, err := prov.KeyMaterial(ctx)
	if err != nil {
	t.Fatalf("provider.KeyMaterial() failed: %v", err)
	}
	if err := compareKeyMaterial(km2, km3); err == nil {
	t.Fatal("expected provider to return new key material after update to underlying file")
	}
	}

	// TestProvider_UpdateSuccessWithSymlink tests the case where a file watcher
	// plugin is created successfully to watch files through a symlink and the
	// symlink is updates to point to new files. Verifies that the changes are
	// picked up by the provider.
	func (s) TestProvider_UpdateSuccessWithSymlink(t *testing.T) {
	// Override the newDistributor to one which pushes on a channel that we
	// can block on.
	origDistributorFunc := newDistributor
	distCh := testutils.NewChannel()
	d := newWrappedDistributor(distCh)
	newDistributor = func() distributor { return d }
	defer func() { newDistributor = origDistributorFunc }()

	// Create two tempDirs with different files.
	dir1 := createTmpDirWithFiles(t, "update_with_symlink1_*", "x509/client1_cert.pem", "x509/client1_key.pem", "x509/client_ca_cert.pem")
	dir2 := createTmpDirWithFiles(t, "update_with_symlink2_*", "x509/server1_cert.pem", "x509/server1_key.pem", "x509/server_ca_cert.pem")

	// Create a symlink under a new tempdir, and make it point to dir1.
	tmpdir, err := ioutil.TempDir("", "test_symlink_*")
	if err != nil {
	t.Fatalf("ioutil.TempDir() failed: %v", err)
	}
	symLinkName := path.Join(tmpdir, "test_symlink")
	if err := os.Symlink(dir1, symLinkName); err != nil {
	t.Fatalf("failed to create symlink to %q: %v", dir1, err)
	}

	// Create a provider which watches the files pointed to by the symlink.
	opts := Options{
	CertFile: path.Join(symLinkName, certFile),
	KeyFile: path.Join(symLinkName, keyFile),
	RootFile: path.Join(symLinkName, rootFile),
	RefreshDuration: defaultTestRefreshDuration,
	}
	prov, err := NewProvider(opts)
	if err != nil {
	t.Fatalf("NewProvider(%+v) failed: %v", opts, err)
	}
	defer prov.Close()

	// Make sure the provider picks up the files and pushes the key material on
	// to the distributors.
	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
	defer cancel()
	for i := 0; i < 2; i++ {
	// Since we have root and identity certs, we need to make sure the
	// update is pushed on both of them.
	if _, err := distCh.Receive(ctx); err != nil {
	t.Fatalf("timeout waiting for provider to read files and push key material to distributor: %v", err)
	}
	}
	km1, err := prov.KeyMaterial(ctx)
	if err != nil {
	t.Fatalf("provider.KeyMaterial() failed: %v", err)
	}

	// Update the symlink to point to dir2.
	symLinkTmpName := path.Join(tmpdir, "test_symlink.tmp")
	if err := os.Symlink(dir2, symLinkTmpName); err != nil {
	t.Fatalf("failed to create symlink to %q: %v", dir2, err)
	}
	if err := os.Rename(symLinkTmpName, symLinkName); err != nil {
	t.Fatalf("failed to update symlink: %v", err)
	}

	// Make sure the provider picks up the new files and pushes the key material
	// on to the distributors.
	for i := 0; i < 2; i++ {
	// Since we have root and identity certs, we need to make sure the
	// update is pushed on both of them.
	if _, err := distCh.Receive(ctx); err != nil {
	t.Fatalf("timeout waiting for provider to read files and push key material to distributor: %v", err)
	}
	}
	km2, err := prov.KeyMaterial(ctx)
	if err != nil {
	t.Fatalf("provider.KeyMaterial() failed: %v", err)
	}

	if err := compareKeyMaterial(km1, km2); err == nil {
	t.Fatal("expected provider to return new key material after symlink update")
	}
	}

	// TestProvider_UpdateFailure_ThenSuccess tests the case where updating cert/key
	// files fail. Verifies that the failed update does not push anything on the
	// distributor. Then the update succeeds, and the test verifies that the key
	// material is updated.
	func (s) TestProvider_UpdateFailure_ThenSuccess(t *testing.T) {
	dir, prov, distCh, cancel := initializeProvider(t, "update_failure")
	defer cancel()

	// Make sure the provider is healthy and returns key material.
	ctx, cc := context.WithTimeout(context.Background(), defaultTestTimeout)
	defer cc()
	km1, err := prov.KeyMaterial(ctx)
	if err != nil {
	t.Fatalf("provider.KeyMaterial() failed: %v", err)
	}

	// Update only the cert file. The key file is left unchanged. This should
	// lead to these two files being not compatible with each other. This
	// simulates the case where the watching goroutine might catch the files in
	// the midst of an update.
	createTmpFile(t, testdata.Path("x509/server1_cert.pem"), path.Join(dir, certFile))

	// Since the last update left the files in an incompatible state, the update
	// should not be picked up by our provider.
	sCtx, sc := context.WithTimeout(context.Background(), 2*defaultTestRefreshDuration)
	defer sc()
	if _, err := distCh.Receive(sCtx); err == nil {
	t.Fatal("new key material pushed to distributor when underlying files did not change")
	}

	// The provider should return key material corresponding to the old state.
	km2, err := prov.KeyMaterial(ctx)
	if err != nil {
	t.Fatalf("provider.KeyMaterial() failed: %v", err)
	}
	if err := compareKeyMaterial(km1, km2); err != nil {
	t.Fatalf("expected provider to not update key material: %v", err)
	}

	// Update the key file to match the cert file.
	createTmpFile(t, testdata.Path("x509/server1_key.pem"), path.Join(dir, keyFile))

	// Make sure update is picked up.
	if _, err := distCh.Receive(ctx); err != nil {
	t.Fatal("timeout waiting for new key material to be pushed to the distributor")
	}
	km3, err := prov.KeyMaterial(ctx)
	if err != nil {
	t.Fatalf("provider.KeyMaterial() failed: %v", err)
	}
	if err := compareKeyMaterial(km2, km3); err == nil {
	t.Fatal("expected provider to return new key material after update to underlying file")
	}
	}