#20432

Committed 01 Sep 2026 01:58PM UTC coverage: 89.218% (+0.008%) from 89.21%

Build # #20432

Build Type

push

github

Committed by

web-flow

Commit Message

fix(pick_first): ignore state updates from shut down or replaced subchannels (#12968)

In legacy PickFirstLoadBalancer, when handleNameResolutionError() or
shutdown() is called, subchannel.shutdown() is invoked. Because
ManagedChannelImpl delays subchannel shutdown by 5 seconds
(SUBCHANNEL_SHUTDOWN_DELAY_SECONDS), the old subchannel may complete a
connection attempt during this window and fire READY state updates.
Without a check verifying if the calling subchannel is still the current
active subchannel, PickFirstLoadBalancer publishes a READY picker for
the obsolete subchannel right before its 5-second delayed shutdown task
fires and kills the transport. This leaves the channel permanently stuck
in READY with a dead subchannel picker.

Partially addresses #12958

Coverage Stats

38642 of 43312 relevant lines covered (89.22%)

0.89 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

98.82

/../core/src/main/java/io/grpc/internal/PickFirstLoadBalancer.java

/*
 * Copyright 2015 The gRPC Authors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package io.grpc.internal;

import static com.google.common.base.Preconditions.checkNotNull;
import static io.grpc.ConnectivityState.CONNECTING;
import static io.grpc.ConnectivityState.IDLE;
import static io.grpc.ConnectivityState.SHUTDOWN;
import static io.grpc.ConnectivityState.TRANSIENT_FAILURE;

import io.grpc.ConnectivityState;
import io.grpc.ConnectivityStateInfo;
import io.grpc.EquivalentAddressGroup;
import io.grpc.LoadBalancer;
import io.grpc.Status;
import java.util.List;
import java.util.Random;
import java.util.concurrent.atomic.AtomicBoolean;
import javax.annotation.Nullable;

/**
 * A {@link LoadBalancer} that provides no load-balancing over the addresses from the {@link
 * io.grpc.NameResolver}.  The channel's default behavior is used, which is walking down the address
 * list and sticking to the first that works.
 */
final class PickFirstLoadBalancer extends LoadBalancer {
  private static final PickResult CONNECTING_RESULT =
      PickResult.withNoResult("connecting", "pick_first: attempting to connect");
  private final Helper helper;
  private Subchannel subchannel;
  private ConnectivityState currentState = IDLE;

  PickFirstLoadBalancer(Helper helper) {
    this.helper = checkNotNull(helper, "helper");
  }

  @Override
  public Status acceptResolvedAddresses(ResolvedAddresses resolvedAddresses) {
    List<EquivalentAddressGroup> servers = resolvedAddresses.getAddresses();
    if (servers.isEmpty()) {
      Status unavailableStatus = Status.UNAVAILABLE.withDescription(
              "NameResolver returned no usable address. addrs=" + resolvedAddresses.getAddresses()
                      + ", attrs=" + resolvedAddresses.getAttributes());
      handleNameResolutionError(unavailableStatus);
      return unavailableStatus;
    }

    // We can optionally be configured to shuffle the address list. This can help better distribute
    // the load.
    if (resolvedAddresses.getLoadBalancingPolicyConfig() instanceof PickFirstLoadBalancerConfig) {
      PickFirstLoadBalancerConfig config
          = (PickFirstLoadBalancerConfig) resolvedAddresses.getLoadBalancingPolicyConfig();
      if (config.shuffleAddressList != null && config.shuffleAddressList) {
        servers = PickFirstLeafLoadBalancer.shuffle(
            servers, config.randomSeed != null ? new Random(config.randomSeed) : new Random());
      }
    }

    if (subchannel == null) {
      final Subchannel subchannel = helper.createSubchannel(
          CreateSubchannelArgs.newBuilder()
              .setAddresses(servers)
              .build());
      subchannel.start(new SubchannelStateListener() {
          @Override
          public void onSubchannelState(ConnectivityStateInfo stateInfo) {
            processSubchannelState(subchannel, stateInfo);
          }
        });
      this.subchannel = subchannel;

      // The channel state does not get updated when doing name resolving today, so for the moment
      // let LB report CONNECTION and call subchannel.requestConnection() immediately.
      updateBalancingState(CONNECTING, new FixedResultPicker(connectingResult()));
      subchannel.requestConnection();
    } else {
      subchannel.updateAddresses(servers);
    }

    return Status.OK;
  }

  @Override
  public void handleNameResolutionError(Status error) {
    if (subchannel != null) {
      subchannel.shutdown();
      subchannel = null;
    }

    // NB(lukaszx0) Whether we should propagate the error unconditionally is arguable. It's fine
    // for time being.
    updateBalancingState(TRANSIENT_FAILURE, new FixedResultPicker(PickResult.withError(error)));
  }

  private void processSubchannelState(Subchannel subchannel, ConnectivityStateInfo stateInfo) {
    if (subchannel != this.subchannel) {
      return;
    }
    ConnectivityState newState = stateInfo.getState();
    if (newState == SHUTDOWN) {
      return;
    }
    if (newState == TRANSIENT_FAILURE || newState == IDLE) {
      helper.refreshNameResolution();
    }

    // If we are transitioning from a TRANSIENT_FAILURE to CONNECTING or IDLE we ignore this state
    // transition and still keep the LB in TRANSIENT_FAILURE state. This is referred to as "sticky
    // transient failure". Only a subchannel state change to READY will get the LB out of
    // TRANSIENT_FAILURE. If the state is IDLE we additionally request a new connection so that we
    // keep retrying for a connection.
    if (currentState == TRANSIENT_FAILURE) {
      if (newState == CONNECTING) {
        return;
      } else if (newState == IDLE) {
        requestConnection();
        return;
      }
    }

    SubchannelPicker picker;
    switch (newState) {
      case IDLE:
        picker = new RequestConnectionPicker();
        break;
      case CONNECTING:
        // It's safe to use RequestConnectionPicker here, so when coming from IDLE we could leave
        // the current picker in-place. But ignoring the potential optimization is simpler.
        picker = new FixedResultPicker(connectingResult());
        break;
      case READY:
        picker = new FixedResultPicker(PickResult.withSubchannel(subchannel));
        break;
      case TRANSIENT_FAILURE:
        picker = new FixedResultPicker(PickResult.withError(stateInfo.getStatus()));
        break;
      default:
        throw new IllegalArgumentException("Unsupported state:" + newState);
    }

    updateBalancingState(newState, picker);
  }

  private void updateBalancingState(ConnectivityState state, SubchannelPicker picker) {
    currentState = state;
    helper.updateBalancingState(state, picker);
  }

  @Override
  public void shutdown() {
    if (subchannel != null) {
      subchannel.shutdown();
      subchannel = null;
    }
  }

  @Override
  public void requestConnection() {
    if (subchannel != null) {
      subchannel.requestConnection();
    }
  }

  private PickResult connectingResult() {
    return CONNECTING_RESULT;
  }

  /** Picker that requests connection during the first pick, and returns noResult. */
  private final class RequestConnectionPicker extends SubchannelPicker {
    private final AtomicBoolean connectionRequested = new AtomicBoolean(false);

    @Override
    public PickResult pickSubchannel(PickSubchannelArgs args) {
      if (connectionRequested.compareAndSet(false, true)) {
        helper.getSynchronizationContext().execute(PickFirstLoadBalancer.this::requestConnection);
      }
      return PickResult.withNoResult(
          "connecting", "pick_first: requesting connection");
    }
  }

  public static final class PickFirstLoadBalancerConfig {

    @Nullable
    public final Boolean shuffleAddressList;

    // For testing purposes only, not meant to be parsed from a real config.
    @Nullable final Long randomSeed;

    public PickFirstLoadBalancerConfig(@Nullable Boolean shuffleAddressList) {
      this(shuffleAddressList, null);
    }

    PickFirstLoadBalancerConfig(@Nullable Boolean shuffleAddressList, @Nullable Long randomSeed) {
      this.shuffleAddressList = shuffleAddressList;
      this.randomSeed = randomSeed;
    }
  }
}

1	/*
2	* Copyright 2015 The gRPC Authors
3	*
4	* Licensed under the Apache License, Version 2.0 (the "License");
5	* you may not use this file except in compliance with the License.
6	* You may obtain a copy of the License at
7	*
8	* http://www.apache.org/licenses/LICENSE-2.0
9	*
10	* Unless required by applicable law or agreed to in writing, software
11	* distributed under the License is distributed on an "AS IS" BASIS,
12	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13	* See the License for the specific language governing permissions and
14	* limitations under the License.
15	*/
16
17	package io.grpc.internal;
18
19	import static com.google.common.base.Preconditions.checkNotNull;
20	import static io.grpc.ConnectivityState.CONNECTING;
21	import static io.grpc.ConnectivityState.IDLE;
22	import static io.grpc.ConnectivityState.SHUTDOWN;
23	import static io.grpc.ConnectivityState.TRANSIENT_FAILURE;
24
25	import io.grpc.ConnectivityState;
26	import io.grpc.ConnectivityStateInfo;
27	import io.grpc.EquivalentAddressGroup;
28	import io.grpc.LoadBalancer;
29	import io.grpc.Status;
30	import java.util.List;
31	import java.util.Random;
32	import java.util.concurrent.atomic.AtomicBoolean;
33	import javax.annotation.Nullable;
34
35	/**
36	* A {@link LoadBalancer} that provides no load-balancing over the addresses from the {@link
37	* io.grpc.NameResolver}. The channel's default behavior is used, which is walking down the address
38	* list and sticking to the first that works.
39	*/
40	final class PickFirstLoadBalancer extends LoadBalancer {
41	private static final PickResult CONNECTING_RESULT =	1 ✔
42	PickResult.withNoResult("connecting", "pick_first: attempting to connect");	1 ✔
43	private final Helper helper;
44	private Subchannel subchannel;
45	private ConnectivityState currentState = IDLE;	1 ✔
46
47	PickFirstLoadBalancer(Helper helper) {	1 ✔
48	this.helper = checkNotNull(helper, "helper");	1 ✔
49	}	1 ✔
50
51	@Override
52	public Status acceptResolvedAddresses(ResolvedAddresses resolvedAddresses) {
53	List<EquivalentAddressGroup> servers = resolvedAddresses.getAddresses();	1 ✔
54	if (servers.isEmpty()) {	1 ✔
55	Status unavailableStatus = Status.UNAVAILABLE.withDescription(	1 ✔
56	"NameResolver returned no usable address. addrs=" + resolvedAddresses.getAddresses()	1 ✔
57	+ ", attrs=" + resolvedAddresses.getAttributes());	1 ✔
58	handleNameResolutionError(unavailableStatus);	1 ✔
59	return unavailableStatus;	1 ✔
60	}
61
62	// We can optionally be configured to shuffle the address list. This can help better distribute
63	// the load.
64	if (resolvedAddresses.getLoadBalancingPolicyConfig() instanceof PickFirstLoadBalancerConfig) {	1 ✔
65	PickFirstLoadBalancerConfig config	1 ✔
66	= (PickFirstLoadBalancerConfig) resolvedAddresses.getLoadBalancingPolicyConfig();	1 ✔
67	if (config.shuffleAddressList != null && config.shuffleAddressList) {	1 ✔
68	servers = PickFirstLeafLoadBalancer.shuffle(	1 ✔
69	servers, config.randomSeed != null ? new Random(config.randomSeed) : new Random());	1 ✔
70	}
71	}
72
73	if (subchannel == null) {	1 ✔
74	final Subchannel subchannel = helper.createSubchannel(	1 ✔
75	CreateSubchannelArgs.newBuilder()	1 ✔
76	.setAddresses(servers)	1 ✔
77	.build());	1 ✔
78	subchannel.start(new SubchannelStateListener() {	1 ✔
79	@Override
80	public void onSubchannelState(ConnectivityStateInfo stateInfo) {
81	processSubchannelState(subchannel, stateInfo);	1 ✔
82	}	1 ✔
83	});
84	this.subchannel = subchannel;	1 ✔
85
86	// The channel state does not get updated when doing name resolving today, so for the moment
87	// let LB report CONNECTION and call subchannel.requestConnection() immediately.
88	updateBalancingState(CONNECTING, new FixedResultPicker(connectingResult()));	1 ✔
89	subchannel.requestConnection();	1 ✔
90	} else {	1 ✔
91	subchannel.updateAddresses(servers);	1 ✔
92	}
93
94	return Status.OK;	1 ✔
95	}
96
97	@Override
98	public void handleNameResolutionError(Status error) {
99	if (subchannel != null) {	1 ✔
100	subchannel.shutdown();	1 ✔
101	subchannel = null;	1 ✔
102	}
103
104	// NB(lukaszx0) Whether we should propagate the error unconditionally is arguable. It's fine
105	// for time being.
106	updateBalancingState(TRANSIENT_FAILURE, new FixedResultPicker(PickResult.withError(error)));	1 ✔
107	}	1 ✔
108
109	private void processSubchannelState(Subchannel subchannel, ConnectivityStateInfo stateInfo) {
110	if (subchannel != this.subchannel) {	1 ✔
111	return;	1 ✔
112	}
113	ConnectivityState newState = stateInfo.getState();	1 ✔
114	if (newState == SHUTDOWN) {	1 ✔
115	return;	1 ✔
116	}
117	if (newState == TRANSIENT_FAILURE \|\| newState == IDLE) {	1 ✔
118	helper.refreshNameResolution();	1 ✔
119	}
120
121	// If we are transitioning from a TRANSIENT_FAILURE to CONNECTING or IDLE we ignore this state
122	// transition and still keep the LB in TRANSIENT_FAILURE state. This is referred to as "sticky
123	// transient failure". Only a subchannel state change to READY will get the LB out of
124	// TRANSIENT_FAILURE. If the state is IDLE we additionally request a new connection so that we
125	// keep retrying for a connection.
126	if (currentState == TRANSIENT_FAILURE) {	1 ✔
127	if (newState == CONNECTING) {	1 ✔
128	return;	1 ✔
129	} else if (newState == IDLE) {	1 ✔
130	requestConnection();	1 ✔
131	return;	1 ✔
132	}
133	}
134
135	SubchannelPicker picker;
136	switch (newState) {	1 ✔
137	case IDLE:
138	picker = new RequestConnectionPicker();	1 ✔
139	break;	1 ✔
140	case CONNECTING:
141	// It's safe to use RequestConnectionPicker here, so when coming from IDLE we could leave
142	// the current picker in-place. But ignoring the potential optimization is simpler.
143	picker = new FixedResultPicker(connectingResult());	1 ✔
144	break;	1 ✔
145	case READY:
146	picker = new FixedResultPicker(PickResult.withSubchannel(subchannel));	1 ✔
147	break;	1 ✔
148	case TRANSIENT_FAILURE:
149	picker = new FixedResultPicker(PickResult.withError(stateInfo.getStatus()));	1 ✔
150	break;	1 ✔
151	default:
152	throw new IllegalArgumentException("Unsupported state:" + newState);	×
153	}
154
155	updateBalancingState(newState, picker);	1 ✔
156	}	1 ✔
157
158	private void updateBalancingState(ConnectivityState state, SubchannelPicker picker) {
159	currentState = state;	1 ✔
160	helper.updateBalancingState(state, picker);	1 ✔
161	}	1 ✔
162
163	@Override
164	public void shutdown() {
165	if (subchannel != null) {	1 ✔
166	subchannel.shutdown();	1 ✔
167	subchannel = null;	1 ✔
168	}
169	}	1 ✔
170
171	@Override
172	public void requestConnection() {
173	if (subchannel != null) {	1 ✔
174	subchannel.requestConnection();	1 ✔
175	}
176	}	1 ✔
177
178	private PickResult connectingResult() {
179	return CONNECTING_RESULT;	1 ✔
180	}
181
182	/** Picker that requests connection during the first pick, and returns noResult. */
183	private final class RequestConnectionPicker extends SubchannelPicker {	1 ✔
184	private final AtomicBoolean connectionRequested = new AtomicBoolean(false);	1 ✔
185
186	@Override
187	public PickResult pickSubchannel(PickSubchannelArgs args) {
188	if (connectionRequested.compareAndSet(false, true)) {	1 ✔
189	helper.getSynchronizationContext().execute(PickFirstLoadBalancer.this::requestConnection);	1 ✔
190	}
191	return PickResult.withNoResult(	1 ✔
192	"connecting", "pick_first: requesting connection");
193	}
194	}
195
196	public static final class PickFirstLoadBalancerConfig {
197
198	@Nullable
199	public final Boolean shuffleAddressList;
200
201	// For testing purposes only, not meant to be parsed from a real config.
202	@Nullable final Long randomSeed;
203
204	public PickFirstLoadBalancerConfig(@Nullable Boolean shuffleAddressList) {
205	this(shuffleAddressList, null);	1 ✔
206	}	1 ✔
207
208	PickFirstLoadBalancerConfig(@Nullable Boolean shuffleAddressList, @Nullable Long randomSeed) {	1 ✔
209	this.shuffleAddressList = shuffleAddressList;	1 ✔
210	this.randomSeed = randomSeed;	1 ✔
211	}	1 ✔
212	}
213	}

grpc / grpc-java / #20432

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous