xds/internal/xdsclient/authority_test.go

/*
 *
 * Copyright 2021 gRPC authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package xdsclient

import (
	"testing"
	"time"

	"github.com/google/go-cmp/cmp"
	"github.com/google/go-cmp/cmp/cmpopts"
	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials/insecure"
	"google.golang.org/grpc/internal/testutils"
	xdstestutils "google.golang.org/grpc/xds/internal/testutils"
	"google.golang.org/grpc/xds/internal/xdsclient/bootstrap"
	"google.golang.org/grpc/xds/internal/xdsclient/xdsresource"
	"google.golang.org/grpc/xds/internal/xdsclient/xdsresource/version"
	"google.golang.org/protobuf/testing/protocmp"
)

var (
	serverConfigs = []*bootstrap.ServerConfig{
		{
			ServerURI:    testXDSServer + "0",
			Creds:        grpc.WithTransportCredentials(insecure.NewCredentials()),
			CredsType:    "creds-0",
			TransportAPI: version.TransportV2,
			NodeProto:    xdstestutils.EmptyNodeProtoV2,
		},
		{
			ServerURI:    testXDSServer + "1",
			Creds:        grpc.WithTransportCredentials(insecure.NewCredentials()),
			CredsType:    "creds-1",
			TransportAPI: version.TransportV3,
			NodeProto:    xdstestutils.EmptyNodeProtoV3,
		},
		{
			ServerURI:    testXDSServer + "2",
			Creds:        grpc.WithTransportCredentials(insecure.NewCredentials()),
			CredsType:    "creds-2",
			TransportAPI: version.TransportV2,
			NodeProto:    xdstestutils.EmptyNodeProtoV2,
		},
	}

	serverConfigCmpOptions = cmp.Options{
		cmpopts.IgnoreFields(bootstrap.ServerConfig{}, "Creds"),
		protocmp.Transform(),
	}
)

// watchAndFetchNewController starts a CDS watch on the client for the given
// resourceName, and tries to receive a new controller from the ctrlCh.
//
// It returns false if there's no controller in the ctrlCh.
func watchAndFetchNewController(t *testing.T, client *clientImpl, resourceName string, ctrlCh *testutils.Channel) (*testController, bool, func()) {
	updateCh := testutils.NewChannel()
	cancelWatch := client.WatchCluster(resourceName, func(update xdsresource.ClusterUpdate, err error) {
		updateCh.Send(xdsresource.ClusterUpdateErrTuple{Update: update, Err: err})
	})

	// Clear the item in the watch channel, otherwise the next watch will block.
	authority := xdsresource.ParseName(resourceName).Authority
	var config *bootstrap.ServerConfig
	if authority == "" {
		config = client.config.XDSServer
	} else {
		authConfig, ok := client.config.Authorities[authority]
		if !ok {
			t.Fatalf("failed to find authority %q", authority)
		}
		config = authConfig.XDSServer
	}
	a := client.authorities[config.String()]
	if a == nil {
		t.Fatalf("authority for %q is not created", authority)
	}
	ctrlTemp := a.controller.(*testController)
	// Clear the channel so the next watch on this controller can proceed.
	ctrlTemp.addWatches[xdsresource.ClusterResource].ReceiveOrFail()

	cancelWatchRet := func() {
		cancelWatch()
		ctrlTemp.removeWatches[xdsresource.ClusterResource].ReceiveOrFail()
	}

	// Try to receive a new controller.
	c, ok := ctrlCh.ReceiveOrFail()
	if !ok {
		return nil, false, cancelWatchRet
	}
	ctrl := c.(*testController)
	return ctrl, true, cancelWatchRet
}

// TestAuthorityDefaultAuthority covers that a watch for an old style resource
// name (one without authority) builds a controller using the top level server
// config.
func (s) TestAuthorityDefaultAuthority(t *testing.T) {
	overrideFedEnvVar(t)
	ctrlCh := overrideNewController(t)

	client, err := newWithConfig(&bootstrap.Config{
		XDSServer:   serverConfigs[0],
		Authorities: map[string]*bootstrap.Authority{testAuthority: {XDSServer: serverConfigs[1]}},
	}, defaultWatchExpiryTimeout, defaultIdleAuthorityDeleteTimeout)
	if err != nil {
		t.Fatalf("failed to create client: %v", err)
	}
	t.Cleanup(client.Close)

	ctrl, ok, _ := watchAndFetchNewController(t, client, testCDSName, ctrlCh)
	if !ok {
		t.Fatalf("want a new controller to be built, got none")
	}
	// Want the default server config.
	wantConfig := serverConfigs[0]
	if diff := cmp.Diff(ctrl.config, wantConfig, serverConfigCmpOptions); diff != "" {
		t.Fatalf("controller is built with unexpected config, diff (-got +want): %v", diff)
	}
}

// TestAuthorityNoneDefaultAuthority covers that a watch with a new style
// resource name creates a controller with the corresponding server config.
func (s) TestAuthorityNoneDefaultAuthority(t *testing.T) {
	overrideFedEnvVar(t)
	ctrlCh := overrideNewController(t)

	client, err := newWithConfig(&bootstrap.Config{
		XDSServer:   serverConfigs[0],
		Authorities: map[string]*bootstrap.Authority{testAuthority: {XDSServer: serverConfigs[1]}},
	}, defaultWatchExpiryTimeout, defaultIdleAuthorityDeleteTimeout)
	if err != nil {
		t.Fatalf("failed to create client: %v", err)
	}
	t.Cleanup(client.Close)

	resourceName := xdstestutils.BuildResourceName(xdsresource.ClusterResource, testAuthority, testCDSName, nil)
	ctrl, ok, _ := watchAndFetchNewController(t, client, resourceName, ctrlCh)
	if !ok {
		t.Fatalf("want a new controller to be built, got none")
	}
	// Want the server config for this authority.
	wantConfig := serverConfigs[1]
	if diff := cmp.Diff(ctrl.config, wantConfig, serverConfigCmpOptions); diff != "" {
		t.Fatalf("controller is built with unexpected config, diff (-got +want): %v", diff)
	}
}

// TestAuthorityShare covers that
// - watch with the same authority name doesn't create new authority
// - watch with different authority name but same authority config doesn't
//   create new authority
func (s) TestAuthorityShare(t *testing.T) {
	overrideFedEnvVar(t)
	ctrlCh := overrideNewController(t)

	client, err := newWithConfig(&bootstrap.Config{
		XDSServer: serverConfigs[0],
		Authorities: map[string]*bootstrap.Authority{
			testAuthority:  {XDSServer: serverConfigs[1]},
			testAuthority2: {XDSServer: serverConfigs[1]}, // Another authority name, but with the same config.
		},
	}, defaultWatchExpiryTimeout, defaultIdleAuthorityDeleteTimeout)
	if err != nil {
		t.Fatalf("failed to create client: %v", err)
	}
	t.Cleanup(client.Close)

	resourceName := xdstestutils.BuildResourceName(xdsresource.ClusterResource, testAuthority, testCDSName, nil)
	ctrl1, ok1, _ := watchAndFetchNewController(t, client, resourceName, ctrlCh)
	if !ok1 {
		t.Fatalf("want a new controller to be built, got none")
	}
	// Want the server config for this authority.
	wantConfig := serverConfigs[1]
	if diff := cmp.Diff(ctrl1.config, wantConfig, serverConfigCmpOptions); diff != "" {
		t.Fatalf("controller is built with unexpected config, diff (-got +want): %v", diff)
	}

	// Call the watch with the same authority name. This shouldn't create a new
	// controller.
	resourceNameSameAuthority := xdstestutils.BuildResourceName(xdsresource.ClusterResource, testAuthority, testCDSName+"1", nil)
	ctrl2, ok2, _ := watchAndFetchNewController(t, client, resourceNameSameAuthority, ctrlCh)
	if ok2 {
		t.Fatalf("an unexpected controller is built with config: %v", ctrl2.config)
	}

	// Call the watch with a different authority name, but the same server
	// config. This shouldn't create a new controller.
	resourceNameSameConfig := xdstestutils.BuildResourceName(xdsresource.ClusterResource, testAuthority2, testCDSName+"1", nil)
	if ctrl, ok, _ := watchAndFetchNewController(t, client, resourceNameSameConfig, ctrlCh); ok {
		t.Fatalf("an unexpected controller is built with config: %v", ctrl.config)
	}
}

// TestAuthorityIdle covers that
// - authorities are put in a timeout cache when the last watch is canceled
// - idle authorities are not immediately closed. They will be closed after a
//   timeout.
func (s) TestAuthorityIdleTimeout(t *testing.T) {
	overrideFedEnvVar(t)
	ctrlCh := overrideNewController(t)

	const idleTimeout = 50 * time.Millisecond

	client, err := newWithConfig(&bootstrap.Config{
		XDSServer: serverConfigs[0],
		Authorities: map[string]*bootstrap.Authority{
			testAuthority: {XDSServer: serverConfigs[1]},
		},
	}, defaultWatchExpiryTimeout, idleTimeout)
	if err != nil {
		t.Fatalf("failed to create client: %v", err)
	}
	t.Cleanup(client.Close)

	resourceName := xdstestutils.BuildResourceName(xdsresource.ClusterResource, testAuthority, testCDSName, nil)
	ctrl1, ok1, cancelWatch1 := watchAndFetchNewController(t, client, resourceName, ctrlCh)
	if !ok1 {
		t.Fatalf("want a new controller to be built, got none")
	}

	var cancelWatch2 func()
	// Call the watch with the same authority name. This shouldn't create a new
	// controller.
	resourceNameSameAuthority := xdstestutils.BuildResourceName(xdsresource.ClusterResource, testAuthority, testCDSName+"1", nil)
	ctrl2, ok2, cancelWatch2 := watchAndFetchNewController(t, client, resourceNameSameAuthority, ctrlCh)
	if ok2 {
		t.Fatalf("an unexpected controller is built with config: %v", ctrl2.config)
	}

	cancelWatch1()
	if ctrl1.done.HasFired() {
		t.Fatalf("controller is closed immediately when the watch is canceled, wanted to be put in the idle cache")
	}

	// Cancel the second watch, should put controller in the idle cache.
	cancelWatch2()
	if ctrl1.done.HasFired() {
		t.Fatalf("controller is closed when the second watch is closed")
	}

	time.Sleep(idleTimeout * 2)
	if !ctrl1.done.HasFired() {
		t.Fatalf("controller is not closed after idle timeout")
	}
}

// TestAuthorityClientClose covers that the authorities in use and in idle cache
// are all closed when the client is closed.
func (s) TestAuthorityClientClose(t *testing.T) {
	overrideFedEnvVar(t)
	ctrlCh := overrideNewController(t)

	client, err := newWithConfig(&bootstrap.Config{
		XDSServer: serverConfigs[0],
		Authorities: map[string]*bootstrap.Authority{
			testAuthority: {XDSServer: serverConfigs[1]},
		},
	}, defaultWatchExpiryTimeout, defaultIdleAuthorityDeleteTimeout)
	if err != nil {
		t.Fatalf("failed to create client: %v", err)
	}
	t.Cleanup(client.Close)

	resourceName := testCDSName
	ctrl1, ok1, cancelWatch1 := watchAndFetchNewController(t, client, resourceName, ctrlCh)
	if !ok1 {
		t.Fatalf("want a new controller to be built, got none")
	}

	resourceNameWithAuthority := xdstestutils.BuildResourceName(xdsresource.ClusterResource, testAuthority, testCDSName, nil)
	ctrl2, ok2, _ := watchAndFetchNewController(t, client, resourceNameWithAuthority, ctrlCh)
	if !ok2 {
		t.Fatalf("want a new controller to be built, got none")
	}

	cancelWatch1()
	if ctrl1.done.HasFired() {
		t.Fatalf("controller is closed immediately when the watch is canceled, wanted to be put in the idle cache")
	}

	// Close the client while watch2 is not canceled. ctrl1 is in the idle
	// cache, ctrl2 is in use. Both should be closed.
	client.Close()

	if !ctrl1.done.HasFired() {
		t.Fatalf("controller in idle cache is not closed after client is closed")
	}
	if !ctrl2.done.HasFired() {
		t.Fatalf("controller in use is not closed after client is closed")
	}
}

// TestAuthorityRevive covers that the authorities in the idle cache is revived
// when a new watch is started on this authority.
func (s) TestAuthorityRevive(t *testing.T) {
	overrideFedEnvVar(t)
	ctrlCh := overrideNewController(t)

	const idleTimeout = 50 * time.Millisecond

	client, err := newWithConfig(&bootstrap.Config{
		XDSServer: serverConfigs[0],
		Authorities: map[string]*bootstrap.Authority{
			testAuthority: {XDSServer: serverConfigs[1]},
		},
	}, defaultWatchExpiryTimeout, idleTimeout)
	if err != nil {
		t.Fatalf("failed to create client: %v", err)
	}
	t.Cleanup(client.Close)

	// Start a watch on the authority, and cancel it. This puts the authority in
	// the idle cache.
	resourceName := xdstestutils.BuildResourceName(xdsresource.ClusterResource, testAuthority, testCDSName, nil)
	ctrl1, ok1, cancelWatch1 := watchAndFetchNewController(t, client, resourceName, ctrlCh)
	if !ok1 {
		t.Fatalf("want a new controller to be built, got none")
	}
	cancelWatch1()

	// Start another watch on this authority, it should retrieve the authority
	// from the cache, instead of creating a new one.
	resourceNameWithAuthority := xdstestutils.BuildResourceName(xdsresource.ClusterResource, testAuthority, testCDSName+"1", nil)
	ctrl2, ok2, _ := watchAndFetchNewController(t, client, resourceNameWithAuthority, ctrlCh)
	if ok2 {
		t.Fatalf("an unexpected controller is built with config: %v", ctrl2.config)
	}

	// Wait for double the idle timeout, the controller shouldn't be closed,
	// since it was revived.
	time.Sleep(idleTimeout * 2)
	if ctrl1.done.HasFired() {
		t.Fatalf("controller that was revived is closed after timeout, want not closed")
	}
}