blob: f2d81968f9ec4bc06b1edf97698a02bdd00fb6af [file] [log] [blame]
Menghan Li4bbdf232017-10-02 09:22:57 -07001/*
2 *
3 * Copyright 2017 gRPC authors.
4 *
5 * Licensed under the Apache License, Version 2.0 (the "License");
6 * you may not use this file except in compliance with the License.
7 * You may obtain a copy of the License at
8 *
9 * http://www.apache.org/licenses/LICENSE-2.0
10 *
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
16 *
17 */
18
19package grpc
20
21import (
Menghan Lia3535372017-10-19 11:32:06 -070022 "fmt"
Menghan Li4bbdf232017-10-02 09:22:57 -070023 "strings"
Doug Fawleyed563a02019-10-04 12:59:43 -070024 "sync"
25 "time"
Menghan Li4bbdf232017-10-02 09:22:57 -070026
Doug Fawleyed563a02019-10-04 12:59:43 -070027 "google.golang.org/grpc/balancer"
Easwar Swaminathan88bf0702019-11-04 08:41:14 -080028 "google.golang.org/grpc/credentials"
lyuxuanacd14292018-09-12 11:15:32 -070029 "google.golang.org/grpc/internal/channelz"
Doug Fawleyed563a02019-10-04 12:59:43 -070030 "google.golang.org/grpc/internal/grpcsync"
Menghan Li4bbdf232017-10-02 09:22:57 -070031 "google.golang.org/grpc/resolver"
Doug Fawleyed563a02019-10-04 12:59:43 -070032 "google.golang.org/grpc/serviceconfig"
Menghan Li4bbdf232017-10-02 09:22:57 -070033)
34
35// ccResolverWrapper is a wrapper on top of cc for resolvers.
Ning Xie5f3da2a2019-12-31 02:01:47 +080036// It implements resolver.ClientConn interface.
Menghan Li4bbdf232017-10-02 09:22:57 -070037type ccResolverWrapper struct {
Easwar Swaminathan027cd622019-10-14 15:13:33 -070038 cc *ClientConn
39 resolverMu sync.Mutex
40 resolver resolver.Resolver
41 done *grpcsync.Event
42 curState resolver.State
Doug Fawleyed563a02019-10-04 12:59:43 -070043
Easwar Swaminathan027cd622019-10-14 15:13:33 -070044 pollingMu sync.Mutex
45 polling chan struct{}
Menghan Li4bbdf232017-10-02 09:22:57 -070046}
47
Doug Fawleycd74fa22020-01-15 08:54:42 -080048// newCCResolverWrapper uses the resolver.Builder to build a Resolver and
49// returns a ccResolverWrapper object which wraps the newly built resolver.
50func newCCResolverWrapper(cc *ClientConn, rb resolver.Builder) (*ccResolverWrapper, error) {
Doug Fawleyed563a02019-10-04 12:59:43 -070051 ccr := &ccResolverWrapper{
52 cc: cc,
53 done: grpcsync.NewEvent(),
54 }
Menghan Li4bbdf232017-10-02 09:22:57 -070055
Easwar Swaminathan88bf0702019-11-04 08:41:14 -080056 var credsClone credentials.TransportCredentials
57 if creds := cc.dopts.copts.TransportCredentials; creds != nil {
58 credsClone = creds.Clone()
59 }
Doug Fawley2cb07fc2019-11-12 15:23:46 -080060 rbo := resolver.BuildOptions{
Easwar Swaminathan88bf0702019-11-04 08:41:14 -080061 DisableServiceConfig: cc.dopts.disableServiceConfig,
62 DialCreds: credsClone,
63 CredsBundle: cc.dopts.copts.CredsBundle,
64 Dialer: cc.dopts.copts.Dialer,
65 }
66
Menghan Li4bbdf232017-10-02 09:22:57 -070067 var err error
Easwar Swaminathan027cd622019-10-14 15:13:33 -070068 // We need to hold the lock here while we assign to the ccr.resolver field
69 // to guard against a data race caused by the following code path,
70 // rb.Build-->ccr.ReportError-->ccr.poll-->ccr.resolveNow, would end up
71 // accessing ccr.resolver which is being assigned here.
72 ccr.resolverMu.Lock()
Easwar Swaminathane3baa762019-12-13 10:55:51 -080073 defer ccr.resolverMu.Unlock()
Easwar Swaminathan88bf0702019-11-04 08:41:14 -080074 ccr.resolver, err = rb.Build(cc.parsedTarget, ccr, rbo)
Menghan Li4bbdf232017-10-02 09:22:57 -070075 if err != nil {
76 return nil, err
77 }
Menghan Li4bbdf232017-10-02 09:22:57 -070078 return ccr, nil
79}
80
Doug Fawley2cb07fc2019-11-12 15:23:46 -080081func (ccr *ccResolverWrapper) resolveNow(o resolver.ResolveNowOptions) {
Easwar Swaminathan027cd622019-10-14 15:13:33 -070082 ccr.resolverMu.Lock()
Doug Fawleyed563a02019-10-04 12:59:43 -070083 if !ccr.done.HasFired() {
84 ccr.resolver.ResolveNow(o)
85 }
Easwar Swaminathan027cd622019-10-14 15:13:33 -070086 ccr.resolverMu.Unlock()
Menghan Li1e1a47f2017-11-28 13:16:53 -080087}
88
Menghan Li4bbdf232017-10-02 09:22:57 -070089func (ccr *ccResolverWrapper) close() {
Easwar Swaminathan027cd622019-10-14 15:13:33 -070090 ccr.resolverMu.Lock()
Menghan Li4bbdf232017-10-02 09:22:57 -070091 ccr.resolver.Close()
Doug Fawleyed563a02019-10-04 12:59:43 -070092 ccr.done.Fire()
Easwar Swaminathan027cd622019-10-14 15:13:33 -070093 ccr.resolverMu.Unlock()
Doug Fawley3910b872019-03-22 10:48:55 -070094}
95
Doug Fawleyed563a02019-10-04 12:59:43 -070096// poll begins or ends asynchronous polling of the resolver based on whether
97// err is ErrBadResolverState.
98func (ccr *ccResolverWrapper) poll(err error) {
Easwar Swaminathan027cd622019-10-14 15:13:33 -070099 ccr.pollingMu.Lock()
100 defer ccr.pollingMu.Unlock()
Doug Fawleyed563a02019-10-04 12:59:43 -0700101 if err != balancer.ErrBadResolverState {
102 // stop polling
103 if ccr.polling != nil {
104 close(ccr.polling)
105 ccr.polling = nil
106 }
107 return
108 }
109 if ccr.polling != nil {
110 // already polling
111 return
112 }
113 p := make(chan struct{})
114 ccr.polling = p
115 go func() {
116 for i := 0; ; i++ {
Doug Fawley2cb07fc2019-11-12 15:23:46 -0800117 ccr.resolveNow(resolver.ResolveNowOptions{})
Easwar Swaminathan027cd622019-10-14 15:13:33 -0700118 t := time.NewTimer(ccr.cc.dopts.resolveNowBackoff(i))
Doug Fawleyed563a02019-10-04 12:59:43 -0700119 select {
120 case <-p:
121 t.Stop()
122 return
123 case <-ccr.done.Done():
124 // Resolver has been closed.
125 t.Stop()
126 return
127 case <-t.C:
128 select {
129 case <-p:
130 return
131 default:
132 }
133 // Timer expired; re-resolve.
134 }
135 }
136 }()
Doug Fawley3910b872019-03-22 10:48:55 -0700137}
138
139func (ccr *ccResolverWrapper) UpdateState(s resolver.State) {
Doug Fawleyed563a02019-10-04 12:59:43 -0700140 if ccr.done.HasFired() {
Doug Fawley3910b872019-03-22 10:48:55 -0700141 return
142 }
Garrett Gutierrez506b7732020-06-26 12:04:47 -0700143 channelz.Infof(logger, ccr.cc.channelzID, "ccResolverWrapper: sending update to cc: %v", s)
Doug Fawley3910b872019-03-22 10:48:55 -0700144 if channelz.IsOn() {
145 ccr.addChannelzTraceEvent(s)
146 }
Doug Fawley3910b872019-03-22 10:48:55 -0700147 ccr.curState = s
Doug Fawleyed563a02019-10-04 12:59:43 -0700148 ccr.poll(ccr.cc.updateResolverState(ccr.curState, nil))
149}
150
151func (ccr *ccResolverWrapper) ReportError(err error) {
152 if ccr.done.HasFired() {
153 return
154 }
Garrett Gutierrez506b7732020-06-26 12:04:47 -0700155 channelz.Warningf(logger, ccr.cc.channelzID, "ccResolverWrapper: reporting error to cc: %v", err)
Doug Fawleyed563a02019-10-04 12:59:43 -0700156 ccr.poll(ccr.cc.updateResolverState(resolver.State{}, err))
Menghan Li4bbdf232017-10-02 09:22:57 -0700157}
158
lyuxuanea5e6da2019-04-03 10:50:28 -0700159// NewAddress is called by the resolver implementation to send addresses to gRPC.
Menghan Li4bbdf232017-10-02 09:22:57 -0700160func (ccr *ccResolverWrapper) NewAddress(addrs []resolver.Address) {
Doug Fawleyed563a02019-10-04 12:59:43 -0700161 if ccr.done.HasFired() {
lyuxuaneb55fa52018-11-09 15:31:07 -0800162 return
Menghan Li4bbdf232017-10-02 09:22:57 -0700163 }
Garrett Gutierrez506b7732020-06-26 12:04:47 -0700164 channelz.Infof(logger, ccr.cc.channelzID, "ccResolverWrapper: sending new addresses to cc: %v", addrs)
lyuxuaneb55fa52018-11-09 15:31:07 -0800165 if channelz.IsOn() {
Doug Fawley3910b872019-03-22 10:48:55 -0700166 ccr.addChannelzTraceEvent(resolver.State{Addresses: addrs, ServiceConfig: ccr.curState.ServiceConfig})
lyuxuaneb55fa52018-11-09 15:31:07 -0800167 }
Doug Fawley3910b872019-03-22 10:48:55 -0700168 ccr.curState.Addresses = addrs
Doug Fawleyed563a02019-10-04 12:59:43 -0700169 ccr.poll(ccr.cc.updateResolverState(ccr.curState, nil))
Menghan Li4bbdf232017-10-02 09:22:57 -0700170}
171
lyuxuanea5e6da2019-04-03 10:50:28 -0700172// NewServiceConfig is called by the resolver implementation to send service
Gyuho Lee6f352462018-06-08 15:37:41 -0700173// configs to gRPC.
Menghan Li4bbdf232017-10-02 09:22:57 -0700174func (ccr *ccResolverWrapper) NewServiceConfig(sc string) {
Doug Fawleyed563a02019-10-04 12:59:43 -0700175 if ccr.done.HasFired() {
lyuxuaneb55fa52018-11-09 15:31:07 -0800176 return
Menghan Li4bbdf232017-10-02 09:22:57 -0700177 }
Garrett Gutierrez506b7732020-06-26 12:04:47 -0700178 channelz.Infof(logger, ccr.cc.channelzID, "ccResolverWrapper: got new service config: %v", sc)
Doug Fawleycb47f382019-11-22 13:38:04 -0800179 if ccr.cc.dopts.disableServiceConfig {
Garrett Gutierrez506b7732020-06-26 12:04:47 -0700180 channelz.Info(logger, ccr.cc.channelzID, "Service config lookups disabled; ignoring config")
Doug Fawleycb47f382019-11-22 13:38:04 -0800181 return
182 }
Doug Fawleyed563a02019-10-04 12:59:43 -0700183 scpr := parseServiceConfig(sc)
184 if scpr.Err != nil {
Garrett Gutierrez506b7732020-06-26 12:04:47 -0700185 channelz.Warningf(logger, ccr.cc.channelzID, "ccResolverWrapper: error parsing service config: %v", scpr.Err)
Doug Fawleyed563a02019-10-04 12:59:43 -0700186 ccr.poll(balancer.ErrBadResolverState)
Doug Fawleyd40a9952019-05-30 09:12:58 -0700187 return
Doug Fawley3910b872019-03-22 10:48:55 -0700188 }
Doug Fawleyd40a9952019-05-30 09:12:58 -0700189 if channelz.IsOn() {
Doug Fawleyed563a02019-10-04 12:59:43 -0700190 ccr.addChannelzTraceEvent(resolver.State{Addresses: ccr.curState.Addresses, ServiceConfig: scpr})
Doug Fawleyd40a9952019-05-30 09:12:58 -0700191 }
Doug Fawleyed563a02019-10-04 12:59:43 -0700192 ccr.curState.ServiceConfig = scpr
193 ccr.poll(ccr.cc.updateResolverState(ccr.curState, nil))
194}
195
196func (ccr *ccResolverWrapper) ParseServiceConfig(scJSON string) *serviceconfig.ParseResult {
197 return parseServiceConfig(scJSON)
Menghan Li4bbdf232017-10-02 09:22:57 -0700198}
lyuxuanacd14292018-09-12 11:15:32 -0700199
Doug Fawley3910b872019-03-22 10:48:55 -0700200func (ccr *ccResolverWrapper) addChannelzTraceEvent(s resolver.State) {
Doug Fawley3910b872019-03-22 10:48:55 -0700201 var updates []string
Doug Fawleyed563a02019-10-04 12:59:43 -0700202 var oldSC, newSC *ServiceConfig
203 var oldOK, newOK bool
204 if ccr.curState.ServiceConfig != nil {
205 oldSC, oldOK = ccr.curState.ServiceConfig.Config.(*ServiceConfig)
206 }
207 if s.ServiceConfig != nil {
208 newSC, newOK = s.ServiceConfig.Config.(*ServiceConfig)
209 }
Doug Fawleyd40a9952019-05-30 09:12:58 -0700210 if oldOK != newOK || (oldOK && newOK && oldSC.rawJSONString != newSC.rawJSONString) {
Doug Fawley3910b872019-03-22 10:48:55 -0700211 updates = append(updates, "service config updated")
212 }
213 if len(ccr.curState.Addresses) > 0 && len(s.Addresses) == 0 {
214 updates = append(updates, "resolver returned an empty address list")
215 } else if len(ccr.curState.Addresses) == 0 && len(s.Addresses) > 0 {
216 updates = append(updates, "resolver returned new addresses")
217 }
Garrett Gutierrez506b7732020-06-26 12:04:47 -0700218 channelz.AddTraceEvent(logger, ccr.cc.channelzID, 0, &channelz.TraceEventDesc{
Doug Fawley3910b872019-03-22 10:48:55 -0700219 Desc: fmt.Sprintf("Resolver state updated: %+v (%v)", s, strings.Join(updates, "; ")),
George Kechagias7464f752020-10-07 18:50:23 +0300220 Severity: channelz.CtInfo,
Doug Fawley3910b872019-03-22 10:48:55 -0700221 })
lyuxuanacd14292018-09-12 11:15:32 -0700222}