#20033

Committed 29 Oct 2025 04:43PM UTC coverage: 88.533% (-0.03%) from 88.561%

Build # #20033

Build Type

push

github

Committed by

web-flow

Commit Message

xds,googleapis: Allow wrapping NameResolver to inject XdsClient (#12450)

Since there is no longer a single global XdsClient, it makes more sense
to allow things like the c2p name resolver to inject its own bootstrap
even if there is one defined in an environment variable.
GoogleCloudToProdNameResolver can now pass an XdsClient instance to
XdsNameResolver, and SharedXdsClientPoolProvider allows
GoogleCloudToProdNameResolver to choose the bootstrap for that one
specific target.

Since XdsNameResolver is no longer in control of the XdsClient pool the
XdsClient instance is now passed to ClusterImplLb. A channel will now
only access the global XdsClient pool exactly once: in the name
resolver.

BootstrapInfo is purposefully being shared across channels, as we really
want to share things like credentials which can have significant memory
use and may have caches which reduce I/O when shared. That is why
SharedXdsClientPoolProvider receives BootstrapInfo instead of
Map<String,?>.

Verifying BootstrapInfo.server() is not empty was moved from
SharedXdsClientPoolProvider to GrpcBootstrapperImpl so avoid
getOrCreate() throwing an exception in only that one case. It might make
sense to move that to BootstrapperImpl, but that will need more
investigation.

A lot of tests needed updating because XdsClientPoolProvider is no
longer responsible for parsing the bootstrap, so we now need bootstraps
even if XdsClientPoolProvider will ignore it.

This also fixes a bug in GoogleCloudToProdNameResolver where it would
initialize the delegate even when it failed to create the bootstrap.
That would certainly cause all RPCs on the channel to fail because of
the missing bootstrap and it defeated the point of `succeeded == false`
and `refresh()` which was supposed to retry contacting the metadata
server.

The server tests were enhanced to give a useful error when
server.start() throws an exception, as otherwise the real error is lost.

b/442819521

Run Details

34966 of 39495 relevant lines covered (88.53%)

0.89 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

92.54

/../xds/src/main/java/io/grpc/xds/SharedXdsClientPoolProvider.java

/*
 * Copyright 2020 The gRPC Authors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package io.grpc.xds;

import static com.google.common.base.Preconditions.checkNotNull;

import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableList;
import com.google.errorprone.annotations.concurrent.GuardedBy;
import io.grpc.CallCredentials;
import io.grpc.MetricRecorder;
import io.grpc.internal.ExponentialBackoffPolicy;
import io.grpc.internal.GrpcUtil;
import io.grpc.internal.ObjectPool;
import io.grpc.internal.SharedResourceHolder;
import io.grpc.internal.TimeProvider;
import io.grpc.xds.client.Bootstrapper;
import io.grpc.xds.client.Bootstrapper.BootstrapInfo;
import io.grpc.xds.client.XdsClient;
import io.grpc.xds.client.XdsClientImpl;
import io.grpc.xds.client.XdsInitializationException;
import io.grpc.xds.internal.security.TlsContextManagerImpl;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.atomic.AtomicReference;
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.annotation.Nullable;
import javax.annotation.concurrent.ThreadSafe;

/**
 * The global factory for creating a singleton {@link XdsClient} instance to be used by all gRPC
 * clients in the process.
 */
@ThreadSafe
final class SharedXdsClientPoolProvider implements XdsClientPoolFactory {
  private static final boolean LOG_XDS_NODE_ID = Boolean.parseBoolean(
      System.getenv("GRPC_LOG_XDS_NODE_ID"));
  private static final Logger log = Logger.getLogger(XdsClientImpl.class.getName());
  private static final ExponentialBackoffPolicy.Provider BACKOFF_POLICY_PROVIDER =
      new ExponentialBackoffPolicy.Provider();

  private final Bootstrapper bootstrapper;
  private final Object lock = new Object();
  private final AtomicReference<Map<String, ?>> bootstrapOverride = new AtomicReference<>();
  private final Map<String, ObjectPool<XdsClient>> targetToXdsClientMap = new ConcurrentHashMap<>();

  SharedXdsClientPoolProvider() {
    this(new GrpcBootstrapperImpl());
  }

  @VisibleForTesting
  SharedXdsClientPoolProvider(Bootstrapper bootstrapper) {
    this.bootstrapper = checkNotNull(bootstrapper, "bootstrapper");
  }

  static SharedXdsClientPoolProvider getDefaultProvider() {
    return SharedXdsClientPoolProviderHolder.instance;
  }

  @Deprecated
  public void setBootstrapOverride(Map<String, ?> bootstrap) {
    bootstrapOverride.set(bootstrap);
  }

  @Override
  @Nullable
  public ObjectPool<XdsClient> get(String target) {
    return targetToXdsClientMap.get(target);
  }

  @Deprecated
  public ObjectPool<XdsClient> getOrCreate(
      String target, MetricRecorder metricRecorder, CallCredentials transportCallCredentials)
      throws XdsInitializationException {
    BootstrapInfo bootstrapInfo;
    Map<String, ?> rawBootstrap = bootstrapOverride.get();
    if (rawBootstrap != null) {
      bootstrapInfo = bootstrapper.bootstrap(rawBootstrap);
    } else {
      bootstrapInfo = bootstrapper.bootstrap();
    }
    return getOrCreate(target, bootstrapInfo, metricRecorder, transportCallCredentials);
  }

  @Override
  public ObjectPool<XdsClient> getOrCreate(
      String target, BootstrapInfo bootstrapInfo, MetricRecorder metricRecorder) {
    return getOrCreate(target, bootstrapInfo, metricRecorder, null);
  }

  public ObjectPool<XdsClient> getOrCreate(
      String target,
      BootstrapInfo bootstrapInfo,
      MetricRecorder metricRecorder,
      CallCredentials transportCallCredentials) {
    ObjectPool<XdsClient> ref = targetToXdsClientMap.get(target);
    if (ref == null) {
      synchronized (lock) {
        ref = targetToXdsClientMap.get(target);
        if (ref == null) {
          ref =
              new RefCountedXdsClientObjectPool(
                  bootstrapInfo, target, metricRecorder, transportCallCredentials);
          targetToXdsClientMap.put(target, ref);
        }
      }
    }
    return ref;
  }

  @Override
  public ImmutableList<String> getTargets() {
    return ImmutableList.copyOf(targetToXdsClientMap.keySet());
  }

  private static class SharedXdsClientPoolProviderHolder {
    private static final SharedXdsClientPoolProvider instance = new SharedXdsClientPoolProvider();
  }

  @ThreadSafe
  @VisibleForTesting
  class RefCountedXdsClientObjectPool implements ObjectPool<XdsClient> {

    private final BootstrapInfo bootstrapInfo;
    private final String target; // The target associated with the xDS client.
    private final MetricRecorder metricRecorder;
    private final CallCredentials transportCallCredentials;
    private final Object lock = new Object();
    @GuardedBy("lock")
    private ScheduledExecutorService scheduler;
    @GuardedBy("lock")
    private XdsClient xdsClient;
    @GuardedBy("lock")
    private int refCount;
    @GuardedBy("lock")
    private XdsClientMetricReporterImpl metricReporter;

    @VisibleForTesting
    RefCountedXdsClientObjectPool(
        BootstrapInfo bootstrapInfo, String target, MetricRecorder metricRecorder) {
      this(bootstrapInfo, target, metricRecorder, null);
    }

    @VisibleForTesting
    RefCountedXdsClientObjectPool(
        BootstrapInfo bootstrapInfo,
        String target,
        MetricRecorder metricRecorder,
        CallCredentials transportCallCredentials) {
      this.bootstrapInfo = checkNotNull(bootstrapInfo, "bootstrapInfo");
      this.target = target;
      this.metricRecorder = checkNotNull(metricRecorder, "metricRecorder");
      this.transportCallCredentials = transportCallCredentials;
    }

    @Override
    public XdsClient getObject() {
      synchronized (lock) {
        if (refCount == 0) {
          if (LOG_XDS_NODE_ID) {
            log.log(Level.INFO, "xDS node ID: {0}", bootstrapInfo.node().getId());
          }
          scheduler = SharedResourceHolder.get(GrpcUtil.TIMER_SERVICE);
          metricReporter = new XdsClientMetricReporterImpl(metricRecorder, target);
          GrpcXdsTransportFactory xdsTransportFactory =
              new GrpcXdsTransportFactory(transportCallCredentials);
          xdsClient =
              new XdsClientImpl(
                  xdsTransportFactory,
                  bootstrapInfo,
                  scheduler,
                  BACKOFF_POLICY_PROVIDER,
                  GrpcUtil.STOPWATCH_SUPPLIER,
                  TimeProvider.SYSTEM_TIME_PROVIDER,
                  MessagePrinter.INSTANCE,
                  new TlsContextManagerImpl(bootstrapInfo),
                  metricReporter);
          metricReporter.setXdsClient(xdsClient);
        }
        refCount++;
        return xdsClient;
      }
    }

    @Override
    public XdsClient returnObject(Object object) {
      synchronized (lock) {
        refCount--;
        if (refCount == 0) {
          xdsClient.shutdown();
          xdsClient = null;
          metricReporter.close();
          metricReporter = null;
          targetToXdsClientMap.remove(target);
          scheduler = SharedResourceHolder.release(GrpcUtil.TIMER_SERVICE, scheduler);
        }
        return null;
      }
    }

    @VisibleForTesting
    @Nullable
    XdsClient getXdsClientForTest() {
      synchronized (lock) {
        return xdsClient;
      }
    }

    public String getTarget() {
      return target;
    }
  }

}

1	/*
2	* Copyright 2020 The gRPC Authors
3	*
4	* Licensed under the Apache License, Version 2.0 (the "License");
5	* you may not use this file except in compliance with the License.
6	* You may obtain a copy of the License at
7	*
8	* http://www.apache.org/licenses/LICENSE-2.0
9	*
10	* Unless required by applicable law or agreed to in writing, software
11	* distributed under the License is distributed on an "AS IS" BASIS,
12	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13	* See the License for the specific language governing permissions and
14	* limitations under the License.
15	*/
16
17	package io.grpc.xds;
18
19	import static com.google.common.base.Preconditions.checkNotNull;
20
21	import com.google.common.annotations.VisibleForTesting;
22	import com.google.common.collect.ImmutableList;
23	import com.google.errorprone.annotations.concurrent.GuardedBy;
24	import io.grpc.CallCredentials;
25	import io.grpc.MetricRecorder;
26	import io.grpc.internal.ExponentialBackoffPolicy;
27	import io.grpc.internal.GrpcUtil;
28	import io.grpc.internal.ObjectPool;
29	import io.grpc.internal.SharedResourceHolder;
30	import io.grpc.internal.TimeProvider;
31	import io.grpc.xds.client.Bootstrapper;
32	import io.grpc.xds.client.Bootstrapper.BootstrapInfo;
33	import io.grpc.xds.client.XdsClient;
34	import io.grpc.xds.client.XdsClientImpl;
35	import io.grpc.xds.client.XdsInitializationException;
36	import io.grpc.xds.internal.security.TlsContextManagerImpl;
37	import java.util.Map;
38	import java.util.concurrent.ConcurrentHashMap;
39	import java.util.concurrent.ScheduledExecutorService;
40	import java.util.concurrent.atomic.AtomicReference;
41	import java.util.logging.Level;
42	import java.util.logging.Logger;
43	import javax.annotation.Nullable;
44	import javax.annotation.concurrent.ThreadSafe;
45
46	/**
47	* The global factory for creating a singleton {@link XdsClient} instance to be used by all gRPC
48	* clients in the process.
49	*/
50	@ThreadSafe
51	final class SharedXdsClientPoolProvider implements XdsClientPoolFactory {
52	private static final boolean LOG_XDS_NODE_ID = Boolean.parseBoolean(	1✔
53	System.getenv("GRPC_LOG_XDS_NODE_ID"));	1✔
54	private static final Logger log = Logger.getLogger(XdsClientImpl.class.getName());	1✔
55	private static final ExponentialBackoffPolicy.Provider BACKOFF_POLICY_PROVIDER =	1✔
56	new ExponentialBackoffPolicy.Provider();
57
58	private final Bootstrapper bootstrapper;
59	private final Object lock = new Object();	1✔
60	private final AtomicReference<Map<String, ?>> bootstrapOverride = new AtomicReference<>();	1✔
61	private final Map<String, ObjectPool<XdsClient>> targetToXdsClientMap = new ConcurrentHashMap<>();	1✔
62
63	SharedXdsClientPoolProvider() {
64	this(new GrpcBootstrapperImpl());	1✔
65	}	1✔
66
67	@VisibleForTesting
68	SharedXdsClientPoolProvider(Bootstrapper bootstrapper) {	1✔
69	this.bootstrapper = checkNotNull(bootstrapper, "bootstrapper");	1✔
70	}	1✔
71
72	static SharedXdsClientPoolProvider getDefaultProvider() {
73	return SharedXdsClientPoolProviderHolder.instance;	1✔
74	}
75
76	@Deprecated
77	public void setBootstrapOverride(Map<String, ?> bootstrap) {
78	bootstrapOverride.set(bootstrap);	×
79	}	×
80
81	@Override
82	@Nullable
83	public ObjectPool<XdsClient> get(String target) {
84	return targetToXdsClientMap.get(target);	1✔
85	}
86
87	@Deprecated
88	public ObjectPool<XdsClient> getOrCreate(
89	String target, MetricRecorder metricRecorder, CallCredentials transportCallCredentials)
90	throws XdsInitializationException {
91	BootstrapInfo bootstrapInfo;
92	Map<String, ?> rawBootstrap = bootstrapOverride.get();	1✔
93	if (rawBootstrap != null) {	1✔
94	bootstrapInfo = bootstrapper.bootstrap(rawBootstrap);	×
95	} else {
96	bootstrapInfo = bootstrapper.bootstrap();	1✔
97	}
98	return getOrCreate(target, bootstrapInfo, metricRecorder, transportCallCredentials);	1✔
99	}
100
101	@Override
102	public ObjectPool<XdsClient> getOrCreate(
103	String target, BootstrapInfo bootstrapInfo, MetricRecorder metricRecorder) {
104	return getOrCreate(target, bootstrapInfo, metricRecorder, null);	1✔
105	}
106
107	public ObjectPool<XdsClient> getOrCreate(
108	String target,
109	BootstrapInfo bootstrapInfo,
110	MetricRecorder metricRecorder,
111	CallCredentials transportCallCredentials) {
112	ObjectPool<XdsClient> ref = targetToXdsClientMap.get(target);	1✔
113	if (ref == null) {	1✔
114	synchronized (lock) {	1✔
115	ref = targetToXdsClientMap.get(target);	1✔
116	if (ref == null) {	1✔
117	ref =	1✔
118	new RefCountedXdsClientObjectPool(
119	bootstrapInfo, target, metricRecorder, transportCallCredentials);
120	targetToXdsClientMap.put(target, ref);	1✔
121	}
122	}	1✔
123	}
124	return ref;	1✔
125	}
126
127	@Override
128	public ImmutableList<String> getTargets() {
129	return ImmutableList.copyOf(targetToXdsClientMap.keySet());	1✔
130	}
131
132	private static class SharedXdsClientPoolProviderHolder {
133	private static final SharedXdsClientPoolProvider instance = new SharedXdsClientPoolProvider();	1✔
134	}
135
136	@ThreadSafe
137	@VisibleForTesting
138	class RefCountedXdsClientObjectPool implements ObjectPool<XdsClient> {
139
140	private final BootstrapInfo bootstrapInfo;
141	private final String target; // The target associated with the xDS client.
142	private final MetricRecorder metricRecorder;
143	private final CallCredentials transportCallCredentials;
144	private final Object lock = new Object();	1✔
145	@GuardedBy("lock")
146	private ScheduledExecutorService scheduler;
147	@GuardedBy("lock")
148	private XdsClient xdsClient;
149	@GuardedBy("lock")
150	private int refCount;
151	@GuardedBy("lock")
152	private XdsClientMetricReporterImpl metricReporter;
153
154	@VisibleForTesting
155	RefCountedXdsClientObjectPool(
156	BootstrapInfo bootstrapInfo, String target, MetricRecorder metricRecorder) {
157	this(bootstrapInfo, target, metricRecorder, null);	1✔
158	}	1✔
159
160	@VisibleForTesting
161	RefCountedXdsClientObjectPool(
162	BootstrapInfo bootstrapInfo,
163	String target,
164	MetricRecorder metricRecorder,
165	CallCredentials transportCallCredentials) {	1✔
166	this.bootstrapInfo = checkNotNull(bootstrapInfo, "bootstrapInfo");	1✔
167	this.target = target;	1✔
168	this.metricRecorder = checkNotNull(metricRecorder, "metricRecorder");	1✔
169	this.transportCallCredentials = transportCallCredentials;	1✔
170	}	1✔
171
172	@Override
173	public XdsClient getObject() {
174	synchronized (lock) {	1✔
175	if (refCount == 0) {	1✔
176	if (LOG_XDS_NODE_ID) {	1✔
177	log.log(Level.INFO, "xDS node ID: {0}", bootstrapInfo.node().getId());	×
178	}
179	scheduler = SharedResourceHolder.get(GrpcUtil.TIMER_SERVICE);	1✔
180	metricReporter = new XdsClientMetricReporterImpl(metricRecorder, target);	1✔
181	GrpcXdsTransportFactory xdsTransportFactory =	1✔
182	new GrpcXdsTransportFactory(transportCallCredentials);
183	xdsClient =	1✔
184	new XdsClientImpl(
185	xdsTransportFactory,
186	bootstrapInfo,
187	scheduler,
188	BACKOFF_POLICY_PROVIDER,	1✔
189	GrpcUtil.STOPWATCH_SUPPLIER,
190	TimeProvider.SYSTEM_TIME_PROVIDER,
191	MessagePrinter.INSTANCE,
192	new TlsContextManagerImpl(bootstrapInfo),
193	metricReporter);
194	metricReporter.setXdsClient(xdsClient);	1✔
195	}
196	refCount++;	1✔
197	return xdsClient;	1✔
198	}
199	}
200
201	@Override
202	public XdsClient returnObject(Object object) {
203	synchronized (lock) {	1✔
204	refCount--;	1✔
205	if (refCount == 0) {	1✔
206	xdsClient.shutdown();	1✔
207	xdsClient = null;	1✔
208	metricReporter.close();	1✔
209	metricReporter = null;	1✔
210	targetToXdsClientMap.remove(target);	1✔
211	scheduler = SharedResourceHolder.release(GrpcUtil.TIMER_SERVICE, scheduler);	1✔
212	}
213	return null;	1✔
214	}
215	}
216
217	@VisibleForTesting
218	@Nullable
219	XdsClient getXdsClientForTest() {
220	synchronized (lock) {	1✔
221	return xdsClient;	1✔
222	}
223	}
224
225	public String getTarget() {
226	return target;	×
227	}
228	}
229
230	}

grpc / grpc-java / #20033

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous