com.google.truth
truth
diff --git a/grpc-gcp-java/src/main/java/com/google/cloud/grpc/GcpChannelPrimeController.java b/grpc-gcp-java/src/main/java/com/google/cloud/grpc/GcpChannelPrimeController.java
new file mode 100644
index 000000000000..c2ecdb6b5a77
--- /dev/null
+++ b/grpc-gcp-java/src/main/java/com/google/cloud/grpc/GcpChannelPrimeController.java
@@ -0,0 +1,481 @@
+/*
+ * Copyright 2026 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.grpc;
+
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static java.util.concurrent.TimeUnit.NANOSECONDS;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.util.concurrent.FutureCallback;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.errorprone.annotations.concurrent.GuardedBy;
+import io.grpc.ManagedChannel;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeoutException;
+import java.util.function.LongUnaryOperator;
+import javax.annotation.Nullable;
+
+/**
+ * Primes channels built by dynamic scale-up before they are published to the pool: each channel
+ * gets a bounded number of {@link GcpChannelPrimer#prime} attempts, each under a timeout and with
+ * backoff between them, and joins the pool only once an attempt succeeds.
+ *
+ * All state is guarded by the pool's own monitor, passed in as {@code lock}, so publishing a
+ * primed channel and shutting the pool down cannot race each other.
+ */
+final class GcpChannelPrimeController {
+
+ /** The pool-side operations priming needs but cannot own. */
+ interface Pool {
+ /** Whether the pool has started shutting down. Called with the pool lock held. */
+ boolean isShuttingDown();
+
+ /**
+ * Whether the pool already holds its maximum number of channels. Called with the pool lock
+ * held.
+ */
+ boolean isFull();
+
+ /** Publishes a successfully primed channel. Called with the pool lock held. */
+ void addPrimedChannel(ManagedChannel channel);
+
+ /** Records a channel rejected because priming failed. Called without the pool lock. */
+ void reportPrimeFailure(Throwable failure);
+
+ /**
+ * Reports a prime abandoned without retry because its {@code prime()} call was still blocked at
+ * the timeout: the channel is closed, but the call keeps occupying a primer thread and a
+ * scale-up slot until it returns. Called without the pool lock, before {@link
+ * #reportPrimeFailure} for the same channel.
+ */
+ void reportPrimeAbandoned();
+ }
+
+ static final class PendingPrime {
+ private final ManagedChannel channel;
+ private int attempt = -1;
+ // The attempt is in its invocation phase: prime() has been dispatched and its returned future,
+ // if any, has not been recorded yet.
+ private boolean invoking;
+ // The primer thread is inside primer.prime() right now. Set by that thread under the lock as
+ // the last step before the call, once it has confirmed the attempt is still live, and cleared
+ // without the lock as soon as the call returns, so the timeout can tell a call that has
+ // already returned (recorded under the lock only later) from one that is still blocked.
+ private volatile boolean insidePrime;
+ private boolean finished;
+ @Nullable private ListenableFuture future;
+ @Nullable private ScheduledFuture timeoutTask;
+ @Nullable private ScheduledFuture retryTask;
+
+ private PendingPrime(ManagedChannel channel) {
+ this.channel = channel;
+ }
+ }
+
+ // Primer invocations run here so a primer that blocks inside prime() occupies a thread of its
+ // own instead of starving the shared scheduler that drives scale-up, draining, and timeouts.
+ private static final ExecutorService SHARED_PRIME_SERVICE =
+ Executors.newCachedThreadPool(GcpThreadFactory.newThreadFactory("gcp-mc-prime-%d"));
+
+ // The owning pool instance: every mutation below is made while synchronized on it.
+ private final Object lock;
+ private final Pool pool;
+ private final GcpChannelPrimer primer;
+ private final long timeoutNanos;
+ private final int maxAttempts;
+ private final ScheduledExecutorService scheduler;
+
+ @GuardedBy("lock")
+ private final Set pendingPrimes = new HashSet<>();
+
+ // Primes whose prime() call was still blocked at its timeout. Each one still holds a primer
+ // thread and its closed delegate until the call returns, so it keeps occupying a scale-up slot:
+ // a primer that never returns can pin at most maxSize slots, never one thread per event.
+ @GuardedBy("lock")
+ private final Set abandonedPrimes = new HashSet<>();
+
+ /**
+ * @param lock the pool instance whose monitor guards this controller's state
+ * @param pool the pool operations priming needs
+ * @param scheduler runs timeouts, retries, and future callbacks
+ */
+ GcpChannelPrimeController(
+ Object lock,
+ Pool pool,
+ GcpChannelPrimer primer,
+ long timeoutNanos,
+ int maxAttempts,
+ ScheduledExecutorService scheduler) {
+ this.lock = lock;
+ this.pool = pool;
+ this.primer = primer;
+ this.timeoutNanos = timeoutNanos;
+ this.maxAttempts = maxAttempts;
+ this.scheduler = scheduler;
+ }
+
+ /**
+ * The number of scale-up slots priming still occupies: channels being primed plus abandoned
+ * primes whose blocked prime() call has not returned yet.
+ */
+ @GuardedBy("lock")
+ int primingCount() {
+ return pendingPrimes.size() + abandonedPrimes.size();
+ }
+
+ @GuardedBy("lock")
+ int inFlightCount() {
+ return pendingPrimes.size();
+ }
+
+ @GuardedBy("lock")
+ int abandonedCount() {
+ return abandonedPrimes.size();
+ }
+
+ /** Starts bounded asynchronous priming without holding the scale-up worker. */
+ void startPrime(ManagedChannel channel) {
+ PendingPrime pendingPrime = new PendingPrime(channel);
+ synchronized (lock) {
+ if (pool.isShuttingDown()) {
+ channel.shutdownNow();
+ return;
+ }
+ pendingPrimes.add(pendingPrime);
+ }
+ submitPrimeAttempt(pendingPrime, 0);
+ }
+
+ /**
+ * Finishes every pending prime and releases the abandoned slots as part of pool shutdown. The
+ * returned primes are still running; pass them to {@link #cancel} once the pool lock is released.
+ */
+ @GuardedBy("lock")
+ List detachAll() {
+ List primesToCancel = new ArrayList<>(pendingPrimes);
+ primesToCancel.forEach(this::finishPendingPrime);
+ // Abandoned delegates are already closed; no further scale-up can consult their slots.
+ abandonedPrimes.clear();
+ return primesToCancel;
+ }
+
+ /** Stops the work of primes returned by {@link #detachAll} and closes their channels. */
+ void cancel(List pendingPrimes, boolean force) {
+ for (PendingPrime pendingPrime : pendingPrimes) {
+ if (pendingPrime.future != null) {
+ pendingPrime.future.cancel(true);
+ }
+ if (pendingPrime.timeoutTask != null) {
+ pendingPrime.timeoutTask.cancel(false);
+ }
+ if (pendingPrime.retryTask != null) {
+ pendingPrime.retryTask.cancel(false);
+ }
+ if (force) {
+ pendingPrime.channel.shutdownNow();
+ } else {
+ pendingPrime.channel.shutdown();
+ }
+ }
+ }
+
+ private void submitPrimeAttempt(PendingPrime pendingPrime, int attempt) {
+ try {
+ SHARED_PRIME_SERVICE.execute(() -> startPrimeAttempt(pendingPrime, attempt));
+ } catch (RejectedExecutionException failure) {
+ rejectPendingPrime(pendingPrime, failure);
+ }
+ }
+
+ private void startPrimeAttempt(PendingPrime pendingPrime, int attempt) {
+ Throwable schedulingFailure = null;
+ synchronized (lock) {
+ if (pendingPrime.finished) {
+ return;
+ }
+ if (pool.isShuttingDown()) {
+ finishPendingPrime(pendingPrime);
+ pendingPrime.channel.shutdownNow();
+ return;
+ }
+ pendingPrime.attempt = attempt;
+ pendingPrime.invoking = true;
+ pendingPrime.future = null;
+ pendingPrime.retryTask = null;
+ // The timeout is armed before prime() runs so it bounds the whole attempt, including a
+ // primer that blocks instead of returning a future.
+ try {
+ pendingPrime.timeoutTask =
+ scheduler.schedule(
+ () ->
+ finishPrimeFailure(
+ pendingPrime,
+ attempt,
+ null,
+ new TimeoutException("Channel priming timed out"),
+ true),
+ timeoutNanos,
+ NANOSECONDS);
+ } catch (RejectedExecutionException failure) {
+ schedulingFailure = failure;
+ }
+ }
+ if (schedulingFailure != null) {
+ finishPrimeFailure(pendingPrime, attempt, null, schedulingFailure, false);
+ return;
+ }
+
+ // The lock was released between arming the timeout and getting here, so the timeout (or a
+ // shutdown) may already have finished this attempt and released its slot. Such a prime() call
+ // would run untracked, so confirm the attempt is still live and mark it as inside prime()
+ // under the same lock hold: a timeout that fires from now on sees insidePrime and abandons.
+ synchronized (lock) {
+ if (!isCurrentPrimeAttempt(pendingPrime, attempt, null)) {
+ // Whoever finished the attempt owns the channel and the outcome.
+ return;
+ }
+ pendingPrime.insidePrime = true;
+ }
+
+ ListenableFuture future;
+ try {
+ try {
+ future = primer.prime(pendingPrime.channel);
+ } finally {
+ pendingPrime.insidePrime = false;
+ }
+ if (future == null) {
+ throw new NullPointerException("Channel primer returned null");
+ }
+ } catch (Throwable failure) {
+ synchronized (lock) {
+ abandonedPrimes.remove(pendingPrime);
+ }
+ finishPrimeFailure(pendingPrime, attempt, null, failure, false);
+ return;
+ }
+
+ boolean cancelFuture = false;
+ boolean closeChannel = false;
+ synchronized (lock) {
+ abandonedPrimes.remove(pendingPrime);
+ if (pendingPrime.finished
+ || pendingPrime.attempt != attempt
+ || !pendingPrime.invoking
+ || pool.isShuttingDown()) {
+ // The attempt timed out or the pool shut down while prime() was still running.
+ cancelFuture = true;
+ closeChannel = pendingPrime.finished || pool.isShuttingDown();
+ } else {
+ pendingPrime.invoking = false;
+ pendingPrime.future = future;
+ }
+ }
+ if (cancelFuture) {
+ future.cancel(true);
+ if (closeChannel) {
+ pendingPrime.channel.shutdownNow();
+ }
+ return;
+ }
+
+ Futures.addCallback(
+ future,
+ new FutureCallback() {
+ @Override
+ public void onSuccess(@Nullable Void unused) {
+ finishPrimeSuccess(pendingPrime, attempt, future);
+ }
+
+ @Override
+ public void onFailure(Throwable failure) {
+ finishPrimeFailure(pendingPrime, attempt, future, failure, false);
+ }
+ },
+ scheduler);
+ }
+
+ private void finishPrimeSuccess(
+ PendingPrime pendingPrime, int attempt, ListenableFuture future) {
+ boolean surplus;
+ synchronized (lock) {
+ if (!isCurrentPrimeAttempt(pendingPrime, attempt, future)) {
+ return;
+ }
+ cancelPrimeTimeout(pendingPrime);
+ pendingPrime.future = null;
+ finishPendingPrime(pendingPrime);
+ surplus = pool.isShuttingDown() || pool.isFull();
+ if (!surplus) {
+ pool.addPrimedChannel(pendingPrime.channel);
+ }
+ }
+ if (surplus) {
+ pendingPrime.channel.shutdownNow();
+ }
+ }
+
+ /**
+ * Records a failed attempt. {@code future} identifies the attempt when the failure came from the
+ * primer's future; {@code null} matches whichever phase the attempt is in, which is what the
+ * timeout, a thrown {@code prime()}, and a scheduling rejection need. {@code timedOut} cancels a
+ * still-running future and, when the primer is still blocked inside {@code prime()}, ends the
+ * prime without a retry: nothing can stop that call and a fresh attempt would overlap it. A
+ * timeout that fires before the primer thread entered {@code prime()} is an ordinary timed-out
+ * attempt as well: the thread then sees the attempt is no longer live and never calls {@code
+ * prime()} for it. A timeout that fires after {@code prime()} returned but before its future was
+ * recorded is likewise ordinary: the worker cancels the late future and the attempt is retried.
+ */
+ private void finishPrimeFailure(
+ PendingPrime pendingPrime,
+ int attempt,
+ @Nullable ListenableFuture future,
+ Throwable failure,
+ boolean timedOut) {
+ boolean finalFailure = false;
+ boolean publishFailure = false;
+ boolean reportAbandoned = false;
+ ListenableFuture futureToCancel = null;
+ synchronized (lock) {
+ if (!isCurrentPrimeAttempt(pendingPrime, attempt, future)) {
+ return;
+ }
+ boolean shuttingDown = pool.isShuttingDown();
+ boolean blockedInPrimer = timedOut && pendingPrime.invoking && pendingPrime.insidePrime;
+ futureToCancel = timedOut ? pendingPrime.future : null;
+ pendingPrime.invoking = false;
+ pendingPrime.future = null;
+ cancelPrimeTimeout(pendingPrime);
+ if (blockedInPrimer || shuttingDown || attempt + 1 >= maxAttempts) {
+ finishPendingPrime(pendingPrime);
+ if (blockedInPrimer && !shuttingDown) {
+ abandonedPrimes.add(pendingPrime);
+ reportAbandoned = true;
+ }
+ finalFailure = true;
+ publishFailure = !shuttingDown;
+ } else {
+ long delayMillis = backoffMillis(attempt);
+ try {
+ pendingPrime.retryTask =
+ scheduler.schedule(
+ () -> submitPrimeAttempt(pendingPrime, attempt + 1), delayMillis, MILLISECONDS);
+ } catch (RejectedExecutionException rejected) {
+ failure.addSuppressed(rejected);
+ finishPendingPrime(pendingPrime);
+ finalFailure = true;
+ publishFailure = true;
+ }
+ }
+ }
+ if (futureToCancel != null) {
+ futureToCancel.cancel(true);
+ }
+ if (finalFailure) {
+ pendingPrime.channel.shutdownNow();
+ if (reportAbandoned) {
+ pool.reportPrimeAbandoned();
+ }
+ if (publishFailure) {
+ pool.reportPrimeFailure(failure);
+ }
+ }
+ }
+
+ private void rejectPendingPrime(PendingPrime pendingPrime, Throwable failure) {
+ boolean publishFailure;
+ synchronized (lock) {
+ if (pendingPrime.finished) {
+ return;
+ }
+ finishPendingPrime(pendingPrime);
+ publishFailure = !pool.isShuttingDown();
+ }
+ pendingPrime.channel.shutdownNow();
+ if (publishFailure) {
+ pool.reportPrimeFailure(failure);
+ }
+ }
+
+ @GuardedBy("lock")
+ private boolean isCurrentPrimeAttempt(
+ PendingPrime pendingPrime, int attempt, @Nullable ListenableFuture future) {
+ if (pendingPrime.finished || pendingPrime.attempt != attempt) {
+ return false;
+ }
+ if (future == null) {
+ // Any live phase of this attempt: still inside prime(), or waiting on its future.
+ return pendingPrime.invoking || pendingPrime.future != null;
+ }
+ return pendingPrime.future == future;
+ }
+
+ @GuardedBy("lock")
+ private void cancelPrimeTimeout(PendingPrime pendingPrime) {
+ if (pendingPrime.timeoutTask != null) {
+ pendingPrime.timeoutTask.cancel(false);
+ pendingPrime.timeoutTask = null;
+ }
+ }
+
+ @GuardedBy("lock")
+ private void finishPendingPrime(PendingPrime pendingPrime) {
+ if (!pendingPrime.finished) {
+ pendingPrime.finished = true;
+ pendingPrimes.remove(pendingPrime);
+ }
+ }
+
+ private static final long MAX_BACKOFF_MILLIS = 5000L;
+
+ /**
+ * Exponential backoff base for the given retry attempt, capped at {@link #MAX_BACKOFF_MILLIS}.
+ */
+ @VisibleForTesting
+ static long baseBackoffMillis(int attempt) {
+ return Math.min(100L << Math.min(attempt, 12), MAX_BACKOFF_MILLIS);
+ }
+
+ /**
+ * Retry delay with equal jitter: half the exponential base plus a random share of the other half,
+ * so concurrent prime retries across channels do not fire in lockstep.
+ */
+ static long backoffMillis(int attempt) {
+ return backoffMillis(attempt, bound -> ThreadLocalRandom.current().nextLong(bound));
+ }
+
+ /**
+ * @param random returns a uniformly distributed value in {@code [0, bound)} for a positive bound
+ */
+ @VisibleForTesting
+ static long backoffMillis(int attempt, LongUnaryOperator random) {
+ long base = baseBackoffMillis(attempt);
+ long half = base / 2;
+ long jitter = half > 0 ? random.applyAsLong(half) : 0L;
+ return Math.min(half + jitter, MAX_BACKOFF_MILLIS);
+ }
+}
diff --git a/grpc-gcp-java/src/main/java/com/google/cloud/grpc/GcpChannelPrimer.java b/grpc-gcp-java/src/main/java/com/google/cloud/grpc/GcpChannelPrimer.java
new file mode 100644
index 000000000000..8d907216f7c0
--- /dev/null
+++ b/grpc-gcp-java/src/main/java/com/google/cloud/grpc/GcpChannelPrimer.java
@@ -0,0 +1,72 @@
+/*
+ * Copyright 2026 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.grpc;
+
+import com.google.common.util.concurrent.ListenableFuture;
+import io.grpc.ManagedChannel;
+
+/**
+ * Primes delegate channels built by dynamic scale-up before they are published to the pool. The
+ * initial pool and non-dynamic growth are not primed.
+ */
+@FunctionalInterface
+public interface GcpChannelPrimer {
+
+ /**
+ * Issues a cheap, read-only end-to-end RPC on {@code channel} so its transport, TLS session, and
+ * server-side connection are established before the channel serves live traffic. Scale-up batches
+ * invoke this method concurrently across channels and publish each channel as soon as its own
+ * future succeeds. For example, a Cloud Spanner implementation can execute {@code SELECT 1}.
+ * Return a failed future to reject the attempt; the pool retries up to the configured attempt
+ * count and closes the channel when they are exhausted.
+ *
+ * Implementations must observe the following:
+ *
+ *
+ * - Return promptly and do the work inside the future. Do not block in this method (no
+ * blocking stub, {@code ResultSet.next()}, {@code Future.get()}, or waiting on other
+ * threads). The configured prime timeout bounds each attempt as a whole: a future still
+ * pending at the timeout is cancelled and the attempt retried, while a call that is still
+ * blocked inside this method at the timeout fails the channel with no retry, because
+ * nothing can stop it. Such a call keeps occupying its primer thread and a scale-up slot
+ * until it returns.
+ *
- Set an explicit, short deadline on the RPC, for example {@code
+ * stub.withDeadlineAfter(5, SECONDS)}. Do not rely on the default deadline: many Google
+ * Cloud APIs default to very long ones, such as one hour for Cloud Spanner's {@code
+ * ExecuteStreamingSql}, so an unresponsive backend would leave the call lingering on the
+ * server long after the pool has given up on it.
+ *
- Propagate cancellation to the gRPC call. The pool calls {@code
+ * future.cancel(true)} on the returned future when the attempt times out or the pool shuts
+ * down. Cancelling the future must cancel the underlying {@link io.grpc.ClientCall} or
+ * {@link io.grpc.Context.CancellableContext} so the stream and its server-side resources
+ * are released promptly.
+ *
- Expect the channel to be shut down under a running call. The pool may call {@code
+ * shutdownNow()} on {@code channel} while this method is still running: when the call is
+ * still blocked at the timeout, or when the pool shuts down. A channel-shutdown status from
+ * an RPC on {@code channel} is then expected and is not a backend outage.
+ *
- Tolerate concurrent calls for one channel. A retry may overlap work from a
+ * cancelled future if that work does not promptly honor cancellation, so the implementation
+ * must be safe for multiple invocations on the same channel at once.
+ *
- Keep the RPC trivial and side-effect free. It should be strictly read-only and
+ * need minimal server work, such as {@code SELECT 1} or a lightweight ping.
+ *
+ *
+ * @param channel the newly built, not yet published delegate channel to prime
+ * @return a future that succeeds when the priming RPC succeeds, or fails to reject the attempt
+ */
+ ListenableFuture prime(ManagedChannel channel);
+}
diff --git a/grpc-gcp-java/src/main/java/com/google/cloud/grpc/GcpClientCall.java b/grpc-gcp-java/src/main/java/com/google/cloud/grpc/GcpClientCall.java
index 517e716da8ac..6c9b0851b165 100644
--- a/grpc-gcp-java/src/main/java/com/google/cloud/grpc/GcpClientCall.java
+++ b/grpc-gcp-java/src/main/java/com/google/cloud/grpc/GcpClientCall.java
@@ -18,6 +18,7 @@
import com.google.cloud.grpc.proto.AffinityConfig;
import com.google.common.base.MoreObjects;
+import com.google.common.base.Preconditions;
import io.grpc.Attributes;
import io.grpc.CallOptions;
import io.grpc.ClientCall;
@@ -30,7 +31,7 @@
import java.util.Collections;
import java.util.List;
import java.util.Queue;
-import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
import javax.annotation.Nullable;
import javax.annotation.concurrent.GuardedBy;
@@ -52,7 +53,8 @@ public class GcpClientCall extends ClientCall {
private ClientCall delegateCall = null;
private List keys = null;
private boolean received = false;
- private final AtomicBoolean decremented = new AtomicBoolean(false);
+ // 0 = not counted, 1 = counted, 2 = finished.
+ private final AtomicInteger countState = new AtomicInteger();
@GuardedBy("this")
private final Queue calls = new ArrayDeque<>();
@@ -123,13 +125,23 @@ public void sendMessage(ReqT message) {
delegateChannelRef = delegateChannel.getChannelRef(key);
}
delegateChannelRef.activeStreamsCountIncr();
+ Preconditions.checkState(countState.compareAndSet(0, 1));
// Create the client call and do the previous operations.
- delegateCall = delegateChannelRef.getChannel().newCall(methodDescriptor, callOptions);
- for (Runnable call : calls) {
- call.run();
+ CallOptions callOptionsWithChannelId =
+ callOptions.withOption(GcpManagedChannel.CHANNEL_ID_KEY, delegateChannelRef.getId());
+ try {
+ delegateCall =
+ delegateChannelRef.getChannel().newCall(methodDescriptor, callOptionsWithChannelId);
+ for (Runnable call : calls) {
+ call.run();
+ }
+ } catch (RuntimeException | Error failure) {
+ finishCount(Status.fromThrowable(failure), true);
+ throw failure;
+ } finally {
+ calls.clear();
}
- calls.clear();
started = true;
}
}
@@ -162,12 +174,16 @@ public String toString() {
}
private void checkedCancel(@Nullable String message, @Nullable Throwable cause) {
- if (!decremented.getAndSet(true)) {
- delegateChannelRef.activeStreamsCountDecr(startNanos, Status.CANCELLED, true);
- }
+ finishCount(Status.CANCELLED, true);
delegateCall.cancel(message, cause);
}
+ private void finishCount(Status status, boolean cancelled) {
+ if (countState.compareAndSet(1, 2)) {
+ delegateChannelRef.activeStreamsCountDecr(startNanos, status, cancelled);
+ }
+ }
+
private void checkSendMessage(Runnable call) {
synchronized (this) {
if (started) {
@@ -185,9 +201,7 @@ private Listener getListener(final Listener responseListener) {
// Decrement the stream number by one when the call is closed.
@Override
public void onClose(Status status, Metadata trailers) {
- if (!decremented.getAndSet(true)) {
- delegateChannelRef.activeStreamsCountDecr(startNanos, status, false);
- }
+ finishCount(status, false);
// If the operation completed successfully, bind/unbind the affinity key.
if (keys != null && status.getCode() == Status.Code.OK) {
if (affinity.getCommand() == AffinityConfig.Command.UNBIND) {
@@ -228,9 +242,10 @@ public static class SimpleGcpClientCall extends ForwardingClientCal
private final ClientCall delegateCall;
@Nullable private final String affinityKey;
private final boolean unbindOnComplete;
- private long startNanos = 0;
+ private long startNanos;
- private final AtomicBoolean decremented = new AtomicBoolean(false);
+ // 0 = not counted, 1 = counted, 2 = finished.
+ private final AtomicInteger countState = new AtomicInteger();
protected SimpleGcpClientCall(
GcpManagedChannel delegateChannel,
@@ -244,8 +259,16 @@ protected SimpleGcpClientCall(
// Set the actual channel ID in callOptions so downstream interceptors can access it.
CallOptions callOptionsWithChannelId =
callOptions.withOption(GcpManagedChannel.CHANNEL_ID_KEY, channelRef.getId());
- this.delegateCall =
- channelRef.getChannel().newCall(methodDescriptor, callOptionsWithChannelId);
+ startNanos = System.nanoTime();
+ channelRef.activeStreamsCountIncr();
+ Preconditions.checkState(countState.compareAndSet(0, 1));
+ try {
+ this.delegateCall =
+ channelRef.getChannel().newCall(methodDescriptor, callOptionsWithChannelId);
+ } catch (RuntimeException | Error failure) {
+ finishCount(Status.fromThrowable(failure), true);
+ throw failure;
+ }
}
@Override
@@ -255,16 +278,12 @@ protected ClientCall delegate() {
@Override
public void start(Listener responseListener, Metadata headers) {
- startNanos = System.nanoTime();
-
Listener listener =
new ForwardingClientCallListener.SimpleForwardingClientCallListener(
responseListener) {
@Override
public void onClose(Status status, Metadata trailers) {
- if (!decremented.getAndSet(true)) {
- channelRef.activeStreamsCountDecr(startNanos, status, false);
- }
+ finishCount(status, false);
// Unbind the affinity key when the caller explicitly requests it
// (e.g., on terminal RPCs like Commit or Rollback) to prevent
// unbounded growth of the affinity map.
@@ -281,20 +300,28 @@ public void onMessage(RespT message) {
}
};
- channelRef.activeStreamsCountIncr();
- delegateCall.start(listener, headers);
+ try {
+ delegateCall.start(listener, headers);
+ } catch (RuntimeException | Error failure) {
+ finishCount(Status.fromThrowable(failure), true);
+ throw failure;
+ }
}
@Override
public void cancel(String message, Throwable cause) {
- if (!decremented.getAndSet(true)) {
- channelRef.activeStreamsCountDecr(startNanos, Status.CANCELLED, true);
- }
+ finishCount(Status.CANCELLED, true);
// Always unbind on cancel — the transaction is being abandoned.
if (affinityKey != null) {
delegateChannel.unbind(Collections.singletonList(affinityKey));
}
delegateCall.cancel(message, cause);
}
+
+ private void finishCount(Status status, boolean cancelled) {
+ if (countState.compareAndSet(1, 2)) {
+ channelRef.activeStreamsCountDecr(startNanos, status, cancelled);
+ }
+ }
}
}
diff --git a/grpc-gcp-java/src/main/java/com/google/cloud/grpc/GcpManagedChannel.java b/grpc-gcp-java/src/main/java/com/google/cloud/grpc/GcpManagedChannel.java
index da3a3ce3cdf3..5e8bab2a1e33 100644
--- a/grpc-gcp-java/src/main/java/com/google/cloud/grpc/GcpManagedChannel.java
+++ b/grpc-gcp-java/src/main/java/com/google/cloud/grpc/GcpManagedChannel.java
@@ -56,24 +56,27 @@
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
-import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.LongSummaryStatistics;
import java.util.Map;
-import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.Consumer;
+import java.util.function.IntUnaryOperator;
+import java.util.function.LongUnaryOperator;
import java.util.function.Supplier;
import java.util.logging.Level;
import java.util.logging.Logger;
@@ -82,9 +85,15 @@
/** A channel management factory that implements grpc.Channel APIs. */
public class GcpManagedChannel extends ManagedChannel {
+
private static final Logger logger = Logger.getLogger(GcpManagedChannel.class.getName());
static final AtomicInteger channelPoolIndex = new AtomicInteger();
+ @FunctionalInterface
+ interface NanoClock {
+ long get();
+ }
+
// Counter for tracking channel ids.
final AtomicInteger nextChannelId = new AtomicInteger();
static final int DEFAULT_MAX_CHANNEL = 10;
@@ -111,7 +120,12 @@ public class GcpManagedChannel extends ManagedChannel {
public static final CallOptions.Key CHANNEL_AFFINITY_REF_KEY =
CallOptions.Key.create("GcpChannelAffinityRef");
- /** Opaque sticky channel reference for callers that should not depend on {@link ChannelRef}. */
+ /**
+ * Opaque caller-owned channel reference for transaction-lifetime stickiness.
+ *
+ * The reference remains on a draining channel until its delegate shuts down. Call {@link
+ * #useDifferentChannelOnNextCall()} to move the next RPC to another active channel.
+ */
public static final class ChannelAffinityRef {
private static final int USE_DIFFERENT_CHANNEL_ON_NEXT_CALL_MASK = 1 << 31;
private static final int CHANNEL_ID_MASK = ~USE_DIFFERENT_CHANNEL_ON_NEXT_CALL_MASK;
@@ -127,6 +141,11 @@ public void useDifferentChannelOnNextCall() {
state.getAndUpdate(value -> value | USE_DIFFERENT_CHANNEL_ON_NEXT_CALL_MASK);
}
+ @VisibleForTesting
+ void setChannelIdForTest(int channelId) {
+ state.set(stateFromChannelId(channelId));
+ }
+
private static int channelIdFromState(int state) {
int encodedChannelId = state & CHANNEL_ID_MASK;
return encodedChannelId == 0 ? NO_CHANNEL_ID : encodedChannelId - 1;
@@ -156,6 +175,15 @@ private static int stateFromChannelId(int channelId) {
private int minRpcPerChannel = 0;
private int maxRpcPerChannel = 0;
private Duration scaleDownInterval = Duration.ZERO;
+ private Duration scaleUpCooldown = Duration.ofSeconds(10);
+ private int scaleDownConsecutiveLowLoadChecks = 3;
+ private int maxScaleUpPercent = 30;
+ private int maxScaleDownChannels = 2;
+ private Duration drainIdleGrace = Duration.ofMinutes(1);
+ private int errorPenaltyStep = 5;
+ private Duration errorPenaltyDuration = Duration.ofSeconds(5);
+ private long errorPenaltyDurationNanos = Duration.ofSeconds(5).toNanos();
+ @Nullable private GcpChannelPrimeController channelPrimeController;
private boolean isDynamicScalingEnabled = false;
private int maxConcurrentStreamsLowWatermark = DEFAULT_MAX_STREAM;
private GcpManagedChannelOptions.ChannelPickStrategy channelPickStrategy =
@@ -177,7 +205,19 @@ private static int stateFromChannelId(int channelId) {
private final Map channelIdToChannelRef = new ConcurrentHashMap<>();
// A set of channels that we removed from the pool and wait for their RPCs to be completed before
// we can shut them down.
- final Set removedChannelRefs = new HashSet<>();
+ final Set removedChannelRefs = ConcurrentHashMap.newKeySet();
+
+ @GuardedBy("this")
+ private final Map> drainTasks = new HashMap<>();
+
+ // One-slot scale-up signal. At most one worker mutates pool size at a time.
+ private final AtomicBoolean scaleUpSignalPending = new AtomicBoolean();
+ private final AtomicBoolean scaleUpWorkerRunning = new AtomicBoolean();
+ private final AtomicLong totalErrorPenaltyLoad = new AtomicLong();
+
+ private volatile long lastScaleUpNanos = Long.MIN_VALUE;
+ private int consecutiveLowLoadChecks;
+ private volatile boolean shuttingDown;
private final ExecutorService stateNotificationExecutor =
Executors.newCachedThreadPool(
@@ -224,6 +264,7 @@ private static int stateFromChannelId(int channelId) {
private ScheduledFuture cleanupTask;
private ScheduledFuture scaleDownTask;
private ScheduledFuture logMetricsTask;
+ private ScheduledExecutorService drainScheduler = SHARED_BACKGROUND_SERVICE;
// Metrics counters.
private final AtomicInteger readyChannels = new AtomicInteger();
@@ -242,7 +283,6 @@ private static int stateFromChannelId(int channelId) {
private AtomicInteger maxActiveStreams = new AtomicInteger();
private AtomicInteger minTotalActiveStreams = new AtomicInteger();
private AtomicInteger maxTotalActiveStreams = new AtomicInteger();
- private AtomicInteger maxTotalActiveStreamsForScaleDown = new AtomicInteger();
private long minOkCalls = 0;
private long maxOkCalls = 0;
private final AtomicLong totalOkCalls = new AtomicLong();
@@ -265,15 +305,99 @@ private static int stateFromChannelId(int channelId) {
private AtomicLong maxUnresponsiveDrops = new AtomicLong();
private AtomicLong scaleUpCount = new AtomicLong();
private AtomicLong scaleDownCount = new AtomicLong();
+ private final AtomicLong scaleUpPrimeFailures = new AtomicLong();
// Clock supplier for nanoTime, injectable for testing.
- private Supplier nanoClock = System::nanoTime;
+ private NanoClock nanoClock = System::nanoTime;
+ private IntUnaryOperator candidateIndexPicker =
+ bound -> ThreadLocalRandom.current().nextInt(bound);
+ @Nullable private volatile Consumer pickerValidationHookForTest;
+ @Nullable private volatile Runnable inactiveMappingRemovalHookForTest;
@VisibleForTesting
- void setNanoClock(Supplier nanoClock) {
+ void setNanoClock(NanoClock nanoClock) {
this.nanoClock = nanoClock;
}
+ @VisibleForTesting
+ void setPickerValidationHookForTest(Consumer hook) {
+ pickerValidationHookForTest = hook;
+ }
+
+ @VisibleForTesting
+ void setCandidateIndexPickerForTest(IntUnaryOperator candidateIndexPicker) {
+ this.candidateIndexPicker = candidateIndexPicker;
+ }
+
+ @VisibleForTesting
+ void setInactiveMappingRemovalHookForTest(Runnable hook) {
+ inactiveMappingRemovalHookForTest = hook;
+ }
+
+ @VisibleForTesting
+ void setDrainSchedulerForTest(ScheduledExecutorService drainScheduler) {
+ this.drainScheduler = drainScheduler;
+ }
+
+ private boolean validatePickedChannel(ChannelRef channelRef) {
+ Consumer hook = pickerValidationHookForTest;
+ if (hook != null) {
+ pickerValidationHookForTest = null;
+ hook.accept(channelRef);
+ }
+ return channelRef.isActive();
+ }
+
+ @VisibleForTesting
+ Map> fallbackMapForTest() {
+ return fallbackMap;
+ }
+
+ @VisibleForTesting
+ int readyChannelCountForTest() {
+ return readyChannels.get();
+ }
+
+ @VisibleForTesting
+ long scaleUpPrimeFailuresForTest() {
+ return scaleUpPrimeFailures.get();
+ }
+
+ @VisibleForTesting
+ boolean scaleUpWorkerRunningForTest() {
+ return scaleUpWorkerRunning.get();
+ }
+
+ @VisibleForTesting
+ synchronized int inFlightPrimeCountForTest() {
+ return channelPrimeController == null ? 0 : channelPrimeController.inFlightCount();
+ }
+
+ @VisibleForTesting
+ synchronized int abandonedPrimeCountForTest() {
+ return channelPrimeController == null ? 0 : channelPrimeController.abandonedCount();
+ }
+
+ @VisibleForTesting
+ static long primeBackoffMillisForTest(int attempt) {
+ return GcpChannelPrimeController.backoffMillis(attempt);
+ }
+
+ @VisibleForTesting
+ static long primeBackoffMillisForTest(int attempt, LongUnaryOperator random) {
+ return GcpChannelPrimeController.backoffMillis(attempt, random);
+ }
+
+ @VisibleForTesting
+ static long primeBaseBackoffMillisForTest(int attempt) {
+ return GcpChannelPrimeController.baseBackoffMillis(attempt);
+ }
+
+ @VisibleForTesting
+ synchronized int drainTaskCountForTest() {
+ return drainTasks.size();
+ }
+
private static ScheduledThreadPoolExecutor createSharedBackgroundService() {
ScheduledThreadPoolExecutor executor =
new ScheduledThreadPoolExecutor(
@@ -349,8 +473,9 @@ public GcpManagedChannel(
}
}
- private void cleanupAffinityKeys() {
- final long cutoff = System.nanoTime() - affinityKeyLifetime.toNanos();
+ @VisibleForTesting
+ void cleanupAffinityKeys() {
+ final long cutoff = nanoClock.get() - affinityKeyLifetime.toNanos();
affinityKeyLastUsed.forEach(
(String key, Long time) -> {
if (time < cutoff) {
@@ -359,75 +484,265 @@ private void cleanupAffinityKeys() {
});
}
- private synchronized void checkScaleDown() {
- if (!isDynamicScalingEnabled) {
- return;
- }
-
- // Use and reset maxTotalActiveStreamsForScaleDown.
- int maxTotalActiveStreamsCount =
- maxTotalActiveStreamsForScaleDown.getAndSet(totalActiveStreams.get());
- // Number of channels to support maximum seen (since last check) concurrent streams
- // with lowest desired utilization (minRpcPerChannel).
- int desiredSize =
- maxTotalActiveStreamsCount / minRpcPerChannel
- + ((maxTotalActiveStreamsCount % minRpcPerChannel == 0) ? 0 : 1);
+ /**
+ * Evaluates instantaneous active load; consecutive checks provide the low-load history rather
+ * than retaining a maximum observed between checks.
+ */
+ @VisibleForTesting
+ void checkScaleDown() {
+ List removedChannels;
+ synchronized (this) {
+ if (!isDynamicScalingEnabled || shuttingDown) {
+ return;
+ }
- int scaleDownTo = Math.max(minSize, desiredSize);
- // Remove those extra channels that are the oldest.
- removeOldestChannels(channelRefs.size() - scaleDownTo);
+ int channelCount = channelRefs.size();
+ if (channelCount <= minSize) {
+ consecutiveLowLoadChecks = 0;
+ return;
+ }
+ long activeLoad = activeLoad(channelRefs);
+ if (activeLoad > (long) minRpcPerChannel * channelCount) {
+ consecutiveLowLoadChecks = 0;
+ return;
+ }
+ if (++consecutiveLowLoadChecks < scaleDownConsecutiveLowLoadChecks) {
+ return;
+ }
+ consecutiveLowLoadChecks = 0;
- // Shutdown removed channels where all RPCs are completed.
- List completedChRefs =
- removedChannelRefs.stream()
- .filter(chRef -> (chRef.getActiveStreamsCount() == 0))
+ int desiredSize = Math.max(minSize, ceilDiv(activeLoad, targetRpcPerChannel()));
+ int removeCount = Math.min(maxScaleDownChannels, Math.max(0, channelCount - desiredSize));
+ removedChannels = removeChannels(removeCount);
+ }
+ for (ChannelRef channelRef : removedChannels) {
+ channelRef.clearErrorPenalty();
+ }
+ List keysToUnbind =
+ affinityKeyToChannelRef.entrySet().stream()
+ .filter(entry -> removedChannels.contains(entry.getValue()))
+ .map(Map.Entry::getKey)
.collect(Collectors.toList());
- removedChannelRefs.removeAll(completedChRefs);
- for (ChannelRef channelRef : completedChRefs) {
- channelRef.getChannel().shutdown();
- // Remove channel from broken channels map.
- fallbackMap.remove(channelRef.getId());
- channelIdToChannelRef.remove(channelRef.getId());
+ for (String key : keysToUnbind) {
+ ChannelRef mappedChannel = affinityKeyToChannelRef.get(key);
+ if (removedChannels.contains(mappedChannel)) {
+ unbindInactiveMapping(key, mappedChannel);
+ }
+ }
+ for (ChannelRef channelRef : removedChannels) {
+ scheduleDrain(channelRef);
}
}
- private void removeOldestChannels(int num) {
+ @GuardedBy("this")
+ private List removeChannels(int num) {
if (num <= 0) {
- return;
+ return Collections.emptyList();
}
- // Select longest connected channels (or disconnected channels).
+ // Drain least-loaded channels first, preferring fewer affinity bindings, then older channels.
final List channelsToRemove =
channelRefs.stream()
- .sorted(Comparator.comparing(ChannelRef::getConnectedSinceNanos))
+ .sorted(
+ Comparator.comparingInt(ChannelRef::getActiveStreamsCount)
+ .thenComparingInt(ChannelRef::getAffinityCount)
+ .thenComparingLong(ChannelRef::getCreatedNanos))
.limit(num)
.collect(Collectors.toList());
- // Remove from active channels.
+ for (ChannelRef channelRef : channelsToRemove) {
+ // Stop new picks before publishing the shorter active list.
+ channelRef.deactivateAndAccountReadiness();
+ }
channelRefs.removeAll(channelsToRemove);
for (ChannelRef channelRef : channelsToRemove) {
- channelRef.resetAffinityCount();
- channelRef.deactivate();
- if (channelRef.getState() == ConnectivityState.READY) {
- decReadyChannels(false);
+ removedChannelRefs.add(channelRef);
+ }
+ minChannels.accumulateAndGet(getNumberOfChannels(), Math::min);
+ scaleDownCount.addAndGet(channelsToRemove.size());
+ executeStateChangeCallbacks();
+ return channelsToRemove;
+ }
+
+ /** Drain task bookkeeping is guarded by the pool monitor. */
+ @VisibleForTesting
+ void scheduleDrain(ChannelRef channelRef) {
+ boolean closeChannel = false;
+ @Nullable DrainTask inlineTask = null;
+ synchronized (this) {
+ if (channelRef.isActive() || channelRef.getActiveStreamsCount() != 0 || shuttingDown) {
+ return;
+ }
+ long elapsed = Math.max(0, nanoClock.get() - channelRef.getLastActivityNanos());
+ long delay = Math.max(0, drainIdleGrace.toNanos() - elapsed);
+ DrainTask drainTask = new DrainTask(channelRef);
+ ScheduledFuture task;
+ try {
+ task = drainScheduler.schedule(drainTask, delay, NANOSECONDS);
+ drainTask.future = task;
+ } catch (RejectedExecutionException e) {
+ logger.fine(log("Drain task rejected: %s", e.getMessage()));
+ closeChannel = removeDrainedChannel(channelRef);
+ task = null;
+ }
+ if (task != null) {
+ ScheduledFuture previous = drainTasks.put(channelRef, task);
+ if (previous != null) {
+ previous.cancel(false);
+ }
+ if (drainTask.ranBeforeFutureAssignment) {
+ inlineTask = drainTask;
+ }
}
}
+ if (inlineTask != null) {
+ finishDrain(channelRef, inlineTask);
+ } else if (closeChannel) {
+ channelRef.getChannel().shutdown();
+ }
+ }
+
+ @VisibleForTesting
+ void finishDrain(ChannelRef channelRef) {
+ finishDrain(channelRef, null);
+ }
+
+ private void finishDrain(ChannelRef channelRef, @Nullable DrainTask runningTask) {
+ boolean closeChannel = false;
+ boolean reschedule = false;
+ synchronized (this) {
+ ScheduledFuture drainTask = drainTasks.get(channelRef);
+ if (runningTask != null && drainTask != runningTask.future) {
+ return;
+ }
+ drainTasks.remove(channelRef);
+ if (drainTask != null) {
+ drainTask.cancel(false);
+ }
+ if (channelRef.isActive()
+ || channelRef.getActiveStreamsCount() != 0
+ || !removedChannelRefs.contains(channelRef)
+ || shuttingDown) {
+ return;
+ }
+ long elapsed = Math.max(0, nanoClock.get() - channelRef.getLastActivityNanos());
+ if (elapsed < drainIdleGrace.toNanos()) {
+ reschedule = true;
+ } else {
+ closeChannel = removeDrainedChannel(channelRef);
+ }
+ }
+ if (reschedule) {
+ scheduleDrain(channelRef);
+ } else if (closeChannel) {
+ channelRef.getChannel().shutdown();
+ }
+ }
- // Remove affinity keys mapping for the channels.
- affinityKeyToChannelRef
- .keySet()
- .removeIf(key -> channelsToRemove.contains(affinityKeyToChannelRef.get(key)));
+ private final class DrainTask implements Runnable {
+ private final ChannelRef channelRef;
- // Keep them aside to wait for all RPCs to complete.
- removedChannelRefs.addAll(channelsToRemove);
+ @GuardedBy("GcpManagedChannel.this")
+ private boolean ranBeforeFutureAssignment;
- // Track minimum number of channels for metrics.
- minChannels.accumulateAndGet(getNumberOfChannels(), Math::min);
- scaleDownCount.addAndGet(channelsToRemove.size());
+ @GuardedBy("GcpManagedChannel.this")
+ @Nullable
+ private ScheduledFuture future;
- // Removing a channel may change channel pool state.
- executeStateChangeCallbacks();
+ private DrainTask(ChannelRef channelRef) {
+ this.channelRef = channelRef;
+ }
+
+ @Override
+ public void run() {
+ synchronized (GcpManagedChannel.this) {
+ if (future == null) {
+ ranBeforeFutureAssignment = true;
+ return;
+ }
+ }
+ finishDrain(channelRef, this);
+ }
+ }
+
+ @GuardedBy("this")
+ private boolean removeDrainedChannel(ChannelRef channelRef) {
+ if (!removedChannelRefs.remove(channelRef)) {
+ return false;
+ }
+ fallbackMap.remove(channelRef.getId());
+ channelIdToChannelRef.remove(channelRef.getId(), channelRef);
+ return true;
+ }
+
+ private static int ceilDiv(long numerator, int denominator) {
+ if (numerator <= 0) {
+ return 0;
+ }
+ return (int) Math.min(Integer.MAX_VALUE, 1 + ((numerator - 1) / denominator));
+ }
+
+ private static int ceilMultiplyDivide(int factor, long numerator, long denominator) {
+ if (factor <= 0 || numerator <= 0 || denominator <= 0) {
+ return 0;
+ }
+ if (numerator >= denominator) {
+ return factor;
+ }
+ if (numerator <= Long.MAX_VALUE / factor) {
+ long product = factor * numerator;
+ return (int) (product / denominator + (product % denominator == 0 ? 0 : 1));
+ }
+
+ // Overflow-safe binary long division for unusually large configured durations or penalties.
+ long quotient = 0;
+ long remainder = 0;
+ for (int bit = Integer.highestOneBit(factor); bit != 0; bit >>>= 1) {
+ quotient <<= 1;
+ long denominatorMinusRemainder = denominator - remainder;
+ if (remainder >= denominatorMinusRemainder) {
+ remainder -= denominatorMinusRemainder;
+ quotient++;
+ } else {
+ remainder += remainder;
+ }
+ if ((factor & bit) != 0) {
+ long denominatorMinusNumerator = denominator - numerator;
+ if (remainder >= denominatorMinusNumerator) {
+ remainder -= denominatorMinusNumerator;
+ quotient++;
+ } else {
+ remainder += numerator;
+ }
+ }
+ }
+ return (int) (quotient + (remainder == 0 ? 0 : 1));
+ }
+
+ private int targetRpcPerChannel() {
+ return Math.max(1, (minRpcPerChannel + maxRpcPerChannel) / 2);
+ }
+
+ private long activeLoad(List refs) {
+ long load = 0;
+ for (ChannelRef channelRef : refs) {
+ if (channelRef.isActive()) {
+ load += channelRef.getActiveStreamsCount();
+ }
+ }
+ return load;
+ }
+
+ private long pickerLoad(List refs, long now) {
+ // Expiry accounting is swept before dynamicUpscale acquires the pool monitor.
+ long load = 0;
+ for (ChannelRef channelRef : refs) {
+ if (channelRef.isActive()) {
+ load += channelRef.getPickerLoad(now);
+ }
+ }
+ return load;
}
private Supplier log(Supplier messageSupplier) {
@@ -458,6 +773,15 @@ private void initOptions() {
minRpcPerChannel = poolOptions.getMinRpcPerChannel();
maxRpcPerChannel = poolOptions.getMaxRpcPerChannel();
scaleDownInterval = poolOptions.getScaleDownInterval();
+ scaleUpCooldown = poolOptions.getScaleUpCooldown();
+ scaleDownConsecutiveLowLoadChecks = poolOptions.getScaleDownConsecutiveLowLoadChecks();
+ maxScaleUpPercent = poolOptions.getMaxScaleUpPercent();
+ maxScaleDownChannels = poolOptions.getMaxScaleDownChannels();
+ drainIdleGrace = poolOptions.getDrainIdleGrace();
+ errorPenaltyStep = poolOptions.getErrorPenaltyStep();
+ errorPenaltyDuration = poolOptions.getErrorPenaltyDuration();
+ errorPenaltyDurationNanos = errorPenaltyDuration.toNanos();
+ channelPrimeController = createChannelPrimeController(poolOptions);
isDynamicScalingEnabled =
minRpcPerChannel > 0 && maxRpcPerChannel > 0 && !scaleDownInterval.isZero();
channelPickStrategy = poolOptions.getChannelPickStrategy();
@@ -465,6 +789,56 @@ private void initOptions() {
initMetrics();
}
+ @Nullable
+ private GcpChannelPrimeController createChannelPrimeController(
+ GcpManagedChannelOptions.GcpChannelPoolOptions poolOptions) {
+ GcpChannelPrimer primer = poolOptions.getChannelPrimer();
+ if (primer == null) {
+ return null;
+ }
+ GcpChannelPrimeController.Pool pool =
+ new GcpChannelPrimeController.Pool() {
+ @Override
+ public boolean isShuttingDown() {
+ return shuttingDown;
+ }
+
+ @Override
+ public boolean isFull() {
+ return channelRefs.size() >= maxSize;
+ }
+
+ @Override
+ public void addPrimedChannel(ManagedChannel channel) {
+ addBuiltChannel(channel);
+ scaleUpCount.incrementAndGet();
+ }
+
+ @Override
+ public void reportPrimeFailure(Throwable failure) {
+ scaleUpPrimeFailures.incrementAndGet();
+ logger.log(Level.WARNING, log("Scaled-up channel priming failed"), failure);
+ }
+
+ @Override
+ public void reportPrimeAbandoned() {
+ logger.warning(
+ log(
+ "Channel priming abandoned without retry: prime() was still blocked at the"
+ + " timeout and nothing can stop it. The channel is closed and the blocked"
+ + " call keeps occupying a primer thread and a scale-up slot until it"
+ + " returns."));
+ }
+ };
+ return new GcpChannelPrimeController(
+ this,
+ pool,
+ primer,
+ poolOptions.getChannelPrimeTimeout().toNanos(),
+ poolOptions.getChannelPrimeMaxAttempts(),
+ SHARED_BACKGROUND_SERVICE);
+ }
+
private synchronized void initCleanupTask(Duration cleanupInterval) {
if (cleanupInterval.isZero()) {
return;
@@ -484,7 +858,13 @@ private synchronized void initScaleDownChecker(Duration scaleDownInterval) {
scaleDownTask =
SHARED_BACKGROUND_SERVICE.scheduleAtFixedRate(
- this::checkScaleDown,
+ () -> {
+ try {
+ checkScaleDown();
+ } catch (Throwable failure) {
+ logger.log(Level.WARNING, log("Scale-down check failed"), failure);
+ }
+ },
scaleDownInterval.toMillis(),
scaleDownInterval.toMillis(),
MILLISECONDS);
@@ -770,6 +1150,13 @@ private void initMetrics() {
this,
GcpManagedChannel::reportScaleUp,
GcpManagedChannel::reportScaleDown);
+
+ createDerivedLongCumulativeTimeSeries(
+ GcpMetricsConstants.METRIC_SCALE_UP_PRIME_FAILURES,
+ "The number of scaled-up channels rejected because priming failed or timed out.",
+ GcpMetricsConstants.COUNT,
+ this,
+ GcpManagedChannel::reportScaleUpPrimeFailures);
}
private void setupOtelCommonAttributes(GcpMetricsOptions metricsOptions) {
@@ -1025,6 +1412,14 @@ private void initOtelMetrics(Meter meter) {
m.record(reportScaleUp(), withDirection(GcpMetricsConstants.DIRECTION_UP));
m.record(reportScaleDown(), withDirection(GcpMetricsConstants.DIRECTION_DOWN));
});
+
+ meter
+ .gaugeBuilder(metricPrefix + GcpMetricsConstants.METRIC_SCALE_UP_PRIME_FAILURES)
+ .ofLongs()
+ .setDescription(
+ "The number of scaled-up channels rejected because priming failed or timed out.")
+ .setUnit(GcpMetricsConstants.COUNT)
+ .buildWithCallback(m -> m.record(reportScaleUpPrimeFailures(), otelCommonAttributes));
}
private void logGauge(String key, long value) {
@@ -1053,6 +1448,7 @@ void logMetrics() {
reportMaxAllowedChannels();
reportScaleUp();
reportScaleDown();
+ reportScaleUpPrimeFailures();
reportNumChannelDisconnect();
reportNumChannelConnect();
reportMinReadinessTime();
@@ -1417,6 +1813,12 @@ private long reportScaleDown() {
return value;
}
+ private long reportScaleUpPrimeFailures() {
+ long value = scaleUpPrimeFailures.get();
+ logCumulative(GcpMetricsConstants.METRIC_SCALE_UP_PRIME_FAILURES, value);
+ return value;
+ }
+
private void incReadyChannels(boolean connected) {
if (connected) {
numChannelConnect.incrementAndGet();
@@ -1478,9 +1880,11 @@ public void notifyWhenStateChanged(ConnectivityState source, Runnable callback)
private class ChannelStateMonitor implements Runnable {
private final ChannelRef channelRef;
private final ManagedChannel channel;
- private ConnectivityState currentState;
+ private volatile ConnectivityState currentState;
private long connectingStartNanos;
- private long connectedSinceNanos;
+
+ @GuardedBy("channelRef")
+ private boolean readyAccounted;
private ChannelStateMonitor(ManagedChannel channel, ChannelRef channelRef) {
this.channelRef = channelRef;
@@ -1488,23 +1892,32 @@ private ChannelStateMonitor(ManagedChannel channel, ChannelRef channelRef) {
run();
}
- public long getConnectedSinceNanos() {
- return connectedSinceNanos;
- }
-
public ConnectivityState getCurrentState() {
return currentState;
}
+ private boolean accountReadyIfNeeded() {
+ if (currentState == ConnectivityState.READY && !readyAccounted) {
+ readyAccounted = true;
+ return true;
+ }
+ return false;
+ }
+
+ private boolean unaccountReadyIfNeeded() {
+ if (readyAccounted) {
+ readyAccounted = false;
+ return true;
+ }
+ return false;
+ }
+
@Override
public void run() {
if (channel == null) {
return;
}
- // Is the channel in the pool?
- boolean isActive = channelRefs.contains(this.channelRef);
-
// Keep minSize channels always connected.
boolean requestConnection =
channelRefs.size() < minSize
@@ -1515,35 +1928,47 @@ public void run() {
.anyMatch(id -> (id == channelRef.getId()));
ConnectivityState newState = channel.getState(requestConnection);
- if (logger.isLoggable(Level.FINER)) {
- logger.finer(
- log(
- "Channel %d state change detected: %s -> %s",
- channelRef.getId(), currentState, newState));
- }
- if (newState == ConnectivityState.READY && currentState != ConnectivityState.READY) {
- connectedSinceNanos = System.nanoTime();
- if (isActive) {
- incReadyChannels(true);
- if (connectingStartNanos > 0) {
- saveReadinessTime(System.nanoTime() - connectingStartNanos);
+ boolean isActive;
+ boolean incrementReady = false;
+ boolean decrementReady = false;
+ long readinessNanos = 0;
+ synchronized (channelRef) {
+ isActive = channelRef.isActive() && channelRefs.contains(channelRef);
+ if (logger.isLoggable(Level.FINER)) {
+ logger.finer(
+ log(
+ "Channel %d state change detected: %s -> %s",
+ channelRef.getId(), currentState, newState));
+ }
+ if (newState == ConnectivityState.READY && currentState != ConnectivityState.READY) {
+ if (isActive && !readyAccounted) {
+ readyAccounted = true;
+ incrementReady = true;
+ if (connectingStartNanos > 0) {
+ readinessNanos = nanoClock.get() - connectingStartNanos;
+ }
}
+ connectingStartNanos = 0;
+ }
+ if (newState != ConnectivityState.READY && readyAccounted) {
+ readyAccounted = false;
+ decrementReady = true;
+ }
+ if (newState == ConnectivityState.CONNECTING
+ && currentState != ConnectivityState.CONNECTING) {
+ connectingStartNanos = nanoClock.get();
}
- connectingStartNanos = 0;
+ currentState = newState;
}
- if (isActive
- && newState != ConnectivityState.READY
- && currentState == ConnectivityState.READY) {
+
+ if (incrementReady) {
+ incReadyChannels(true);
+ if (readinessNanos > 0) {
+ saveReadinessTime(readinessNanos);
+ }
+ } else if (decrementReady) {
decReadyChannels(true);
}
- if (newState == ConnectivityState.CONNECTING
- && currentState != ConnectivityState.CONNECTING) {
- connectingStartNanos = System.nanoTime();
- }
- if (newState != ConnectivityState.READY) {
- connectedSinceNanos = 0;
- }
- currentState = newState;
processChannelStateChange(channelRef.getId(), newState);
if (isActive) {
@@ -1573,6 +1998,11 @@ void processChannelStateChange(int channelId, ConnectivityState state) {
if (!fallbackEnabled) {
return;
}
+ ChannelRef channelRef = channelIdToChannelRef.get(channelId);
+ if (channelRef == null || !channelRef.isActive()) {
+ fallbackMap.remove(channelId);
+ return;
+ }
if (state == ConnectivityState.READY || state == ConnectivityState.IDLE) {
// Ready
fallbackMap.remove(channelId);
@@ -1599,11 +2029,19 @@ public int getStreamsLowWatermark() {
}
public int getMinActiveStreams() {
- return channelRefs.stream().mapToInt(ChannelRef::getActiveStreamsCount).min().orElse(0);
+ return channelRefs.stream()
+ .filter(ChannelRef::isActive)
+ .mapToInt(ChannelRef::getActiveStreamsCount)
+ .min()
+ .orElse(0);
}
public int getMaxActiveStreams() {
- return channelRefs.stream().mapToInt(ChannelRef::getActiveStreamsCount).max().orElse(0);
+ return channelRefs.stream()
+ .filter(ChannelRef::isActive)
+ .mapToInt(ChannelRef::getActiveStreamsCount)
+ .max()
+ .orElse(0);
}
/**
@@ -1637,37 +2075,60 @@ protected ChannelRef getChannelRefForBind() {
* @return {@link ChannelRef}
*/
protected synchronized ChannelRef getChannelRefRoundRobin() {
+ ChannelRef first = createFirstChannel();
+ if (first != null) {
+ return first;
+ }
if (!isDynamicScalingEnabled && channelRefs.size() < maxSize) {
return createNewChannel();
}
- maybeDynamicUpscale();
- bindingIndex++;
- if (bindingIndex >= channelRefs.size()) {
- bindingIndex = 0;
+ Object[] snapshot = channelRefs.toArray();
+ if (snapshot.length > 0) {
+ for (int attempts = 0; attempts < snapshot.length; attempts++) {
+ bindingIndex = (bindingIndex + 1) % snapshot.length;
+ ChannelRef candidate = (ChannelRef) snapshot[bindingIndex];
+ if (candidate.isActive()) {
+ return candidate;
+ }
+ }
}
- return channelRefs.get(bindingIndex);
+ return pickFromCandidates(channelRefs);
}
/**
* Pick a {@link ChannelRef} (and create a new one if necessary). If notReadyFallbackEnabled is
* true in the {@link GcpResiliencyOptions} then instead of a channel in a non-READY state another
- * channel in the READY state and having fewer than maximum allowed number of active streams will
- * be provided if available. Subsequent calls with the same affinity key will provide the same
+ * channel in the READY state and having picker load below the maximum allowed threshold will be
+ * provided if available. Subsequent calls with the same affinity key will provide the same
* fallback channel as long as the fallback channel is in the READY state.
*
* @param key affinity key. If it is specified, pick the ChannelRef bound with the affinity key.
- * Otherwise pick the one with the smallest number of streams.
+ * Otherwise pick using the lowest picker load: active streams plus active error penalty.
*/
protected ChannelRef getChannelRef(@Nullable String key) {
- maybeDynamicUpscale();
if (key == null || key.isEmpty()) {
return pickLeastBusyChannel(/* forFallback= */ false);
}
- ChannelRef mappedChannel = affinityKeyToChannelRef.get(key);
- affinityKeyLastUsed.put(key, System.nanoTime());
+ ChannelRef mappedChannel;
+ while (true) {
+ mappedChannel = affinityKeyToChannelRef.get(key);
+ if (mappedChannel == null) {
+ break;
+ }
+ long lastUsed = nanoClock.get();
+ affinityKeyLastUsed.merge(key, lastUsed, Long::max);
+ if (affinityKeyToChannelRef.get(key) == mappedChannel) {
+ break;
+ }
+ affinityKeyLastUsed.remove(key, lastUsed);
+ }
+ while (mappedChannel != null && !mappedChannel.isActive()) {
+ mappedChannel =
+ unbindInactiveMapping(key, mappedChannel) ? null : affinityKeyToChannelRef.get(key);
+ }
if (mappedChannel == null) {
ChannelRef channelRef = pickLeastBusyChannel(/* forFallback= */ false);
- bind(channelRef, Collections.singletonList(key));
+ channelRef = bind(channelRef, Collections.singletonList(key));
return channelRef;
}
if (!fallbackEnabled) {
@@ -1682,17 +2143,20 @@ protected ChannelRef getChannelRef(@Nullable String key) {
// Channel is not ready. Look up if the affinity key mapped to another channel.
Integer channelId = tempMap.get(key);
if (channelId != null && !fallbackMap.containsKey(channelId)) {
- // Fallback channel is ready.
- if (logger.isLoggable(Level.FINEST)) {
- logger.finest(log("Using fallback channel: %d -> %d", mappedChannel.getId(), channelId));
+ ChannelRef fallbackChannel = channelIdToChannelRef.get(channelId);
+ if (fallbackChannel != null && fallbackChannel.isActive()) {
+ if (logger.isLoggable(Level.FINEST)) {
+ logger.finest(log("Using fallback channel: %d -> %d", mappedChannel.getId(), channelId));
+ }
+ fallbacksSucceeded.incrementAndGet();
+ return fallbackChannel;
}
- fallbacksSucceeded.incrementAndGet();
- return channelRefs.get(channelId);
+ tempMap.remove(key, channelId);
}
// No temp mapping for this key or fallback channel is also broken.
ChannelRef channelRef = pickLeastBusyChannel(/* forFallback= */ true);
if (!fallbackMap.containsKey(channelRef.getId())
- && channelRef.getActiveStreamsCount() < DEFAULT_MAX_STREAM) {
+ && channelRef.getActiveStreamsCount() < maxConcurrentStreamsLowWatermark) {
// Got a ready and not an overloaded channel.
if (channelRef.getId() != mappedChannel.getId()) {
if (logger.isLoggable(Level.FINEST)) {
@@ -1710,16 +2174,20 @@ protected ChannelRef getChannelRef(@Nullable String key) {
fallbacksFailed.incrementAndGet();
if (channelId != null) {
// Stick with previous mapping if fallback has failed.
- return channelRefs.get(channelId);
+ ChannelRef fallbackChannel = channelIdToChannelRef.get(channelId);
+ if (fallbackChannel != null && fallbackChannel.isActive()) {
+ return fallbackChannel;
+ }
}
return mappedChannel;
}
/**
- * Pick a {@link ChannelRef} using a caller-owned reference instead of grpc-gcp's affinity map.
+ * Picks a {@link ChannelRef} using a caller-owned reference instead of grpc-gcp's affinity map. A
+ * reference remains sticky while its delegate is open, including while the channel drains, and
+ * re-resolves after delegate shutdown or an explicit request to use a different channel.
*/
protected ChannelRef getChannelRefByAffinityRef(ChannelAffinityRef affinityRef) {
- maybeDynamicUpscale();
// Retry if another thread updates the caller-owned affinity ref while we are picking a channel.
while (true) {
int state = affinityRef.state.get();
@@ -1730,7 +2198,7 @@ protected ChannelRef getChannelRefByAffinityRef(ChannelAffinityRef affinityRef)
channelId == ChannelAffinityRef.NO_CHANNEL_ID
? null
: channelIdToChannelRef.get(channelId);
- if (!useDifferentChannel && channelRef != null && channelRef.isActive()) {
+ if (!useDifferentChannel && channelRef != null && !channelRef.getChannel().isShutdown()) {
return channelRef;
}
@@ -1747,7 +2215,7 @@ protected ChannelRef getChannelRefByAffinityRef(ChannelAffinityRef affinityRef)
private ChannelRef pickLeastBusyChannelDifferentFrom(@Nullable ChannelRef excludedChannelRef) {
ChannelRef channelRef = pickLeastBusyChannel(/* forFallback= */ false);
- if (excludedChannelRef == null || channelRefs.size() <= 1) {
+ if (excludedChannelRef == null) {
return channelRef;
}
if (channelRef != excludedChannelRef && channelRef.isActive()) {
@@ -1755,11 +2223,12 @@ private ChannelRef pickLeastBusyChannelDifferentFrom(@Nullable ChannelRef exclud
}
ChannelRef leastBusyChannelRef = null;
int leastBusyStreams = Integer.MAX_VALUE;
+ long now = nanoClock.get();
for (ChannelRef candidate : channelRefs) {
if (candidate == excludedChannelRef || !candidate.isActive()) {
continue;
}
- int streams = candidate.getActiveStreamsCount();
+ int streams = candidate.getPickerLoad(now);
if (leastBusyChannelRef == null || streams < leastBusyStreams) {
leastBusyChannelRef = candidate;
leastBusyStreams = streams;
@@ -1769,42 +2238,25 @@ private ChannelRef pickLeastBusyChannelDifferentFrom(@Nullable ChannelRef exclud
}
// Create a new channel and add it to channelRefs.
- // If we have a ready channel not in the pool that we wait for completing its RPCs,
- // then re-use that channel instead.
@VisibleForTesting
- ChannelRef createNewChannel() {
- Optional reusedChannelRef = pickChannelForReuse();
- if (reusedChannelRef.isPresent()) {
- ChannelRef chRef = reusedChannelRef.get();
- channelRefs.add(chRef);
- removedChannelRefs.remove(chRef);
- channelIdToChannelRef.put(chRef.getId(), chRef);
- chRef.activate();
- logger.finer(log("Channel %d reused.", chRef.getId()));
- incReadyChannels(false);
- maxChannels.accumulateAndGet(getNumberOfChannels(), Math::max);
- return chRef;
- }
-
+ synchronized ChannelRef createNewChannel() {
ChannelRef channelRef = new ChannelRef(delegateChannelBuilder.build());
channelRefs.add(channelRef);
channelIdToChannelRef.put(channelRef.getId(), channelRef);
+ channelRef.activateAndAccountReadiness();
logger.finer(log("Channel %d created.", channelRef.getId()));
maxChannels.accumulateAndGet(getNumberOfChannels(), Math::max);
return channelRef;
}
- private Optional pickChannelForReuse() {
- // Pick the most recently connected, if any.
- Optional chRef =
- removedChannelRefs.stream().max(Comparator.comparing(ChannelRef::getConnectedSinceNanos));
-
- // Make sure it is ready, because connectedSinceNanos may be 0.
- if (chRef.isPresent() && chRef.get().getState() != ConnectivityState.READY) {
- return Optional.empty();
- }
-
- return chRef;
+ @GuardedBy("this")
+ private ChannelRef addBuiltChannel(ManagedChannel channel) {
+ ChannelRef channelRef = new ChannelRef(channel);
+ channelRefs.add(channelRef);
+ channelIdToChannelRef.put(channelRef.getId(), channelRef);
+ channelRef.activateAndAccountReadiness();
+ maxChannels.accumulateAndGet(getNumberOfChannels(), Math::max);
+ return channelRef;
}
// Returns first newly created channel or null if there are already some channels in the pool.
@@ -1814,7 +2266,7 @@ private ChannelRef createFirstChannel() {
return null;
}
synchronized (this) {
- if (channelRefs.isEmpty()) {
+ if (channelRefs.isEmpty() && !shuttingDown) {
return createNewChannel();
}
}
@@ -1836,25 +2288,130 @@ private ChannelRef tryCreateNewChannel() {
return null;
}
- private void maybeDynamicUpscale() {
- if (!isDynamicScalingEnabled || channelRefs.size() >= maxSize) {
+ private void maybeSignalScaleUp(ChannelRef selectedChannel) {
+ int activeChannels = channelRefs.size();
+ if (!selectedChannel.isActive()
+ || !isDynamicScalingEnabled
+ || shuttingDown
+ || activeChannels == 0
+ || activeChannels >= maxSize) {
+ return;
+ }
+ long now = nanoClock.get();
+ int selectedLoad = selectedChannel.getPickerLoad(now);
+ long totalLoad = (long) totalActiveStreams.get() + totalErrorPenaltyLoad.get();
+ // A pool at the average cap has no spare capacity, including when penalty represents loss.
+ if (selectedLoad <= maxRpcPerChannel && totalLoad < (long) activeChannels * maxRpcPerChannel) {
+ return;
+ }
+ signalScaleUp();
+ }
+
+ private void signalScaleUp() {
+ scaleUpSignalPending.set(true);
+ if (!scaleUpWorkerRunning.compareAndSet(false, true)) {
return;
}
+ try {
+ SHARED_BACKGROUND_SERVICE.execute(this::runScaleUpWorker);
+ } catch (RejectedExecutionException e) {
+ scaleUpWorkerRunning.set(false);
+ logger.fine(log("Scale-up task rejected: %s", e.getMessage()));
+ }
+ }
- if ((totalActiveStreams.get() / channelRefs.size()) >= maxRpcPerChannel) {
- dynamicUpscale();
+ private void runScaleUpWorker() {
+ try {
+ do {
+ scaleUpSignalPending.set(false);
+ try {
+ dynamicUpscale();
+ } catch (Throwable failure) {
+ logger.log(Level.WARNING, log("Scale-up failed"), failure);
+ }
+ } while (scaleUpSignalPending.get() && !shuttingDown);
+ } finally {
+ scaleUpWorkerRunning.set(false);
+ // Close the race where a signal arrives between the final test and clearing running.
+ if (scaleUpSignalPending.get() && !shuttingDown) {
+ signalScaleUp();
+ }
}
}
- private synchronized void dynamicUpscale() {
- if (!isDynamicScalingEnabled || channelRefs.size() >= maxSize) {
+ private void dynamicUpscale() {
+ long now = nanoClock.get();
+ for (ChannelRef channelRef : channelRefs) {
+ channelRef.currentErrorPenalty(now);
+ }
+ final int channelsToBuild;
+ synchronized (this) {
+ // Channels still priming are capacity already on its way: they count toward the size cap
+ // and the desired size, so overlapping scale-ups cannot build the same shortfall twice.
+ // Abandoned primes keep their slot until their blocked prime() call returns.
+ int priming = channelPrimeController == null ? 0 : channelPrimeController.primingCount();
+ if (!isDynamicScalingEnabled || shuttingDown || channelRefs.size() + priming >= maxSize) {
+ return;
+ }
+ if (lastScaleUpNanos != Long.MIN_VALUE
+ && now - lastScaleUpNanos < scaleUpCooldown.toNanos()) {
+ return;
+ }
+ List activeChannels =
+ channelRefs.stream().filter(ChannelRef::isActive).collect(Collectors.toList());
+ int active = activeChannels.size();
+ if (active == 0) {
+ return;
+ }
+ int desired = ceilDiv(pickerLoad(activeChannels, now), targetRpcPerChannel());
+ int add = desired - active - priming;
+ // Small pools may add two channels per event before percentage growth dominates.
+ int percentCap = Math.max(2, ceilDiv((long) active * maxScaleUpPercent, 100));
+ add = Math.min(add, percentCap);
+ add = Math.min(add, maxSize - channelRefs.size() - priming);
+ if (add <= 0) {
+ return;
+ }
+ channelsToBuild = add;
+ // Claim cooldown before delegate construction begins.
+ lastScaleUpNanos = now;
+ }
+
+ List builtChannels = new ArrayList<>(channelsToBuild);
+ try {
+ for (int i = 0; i < channelsToBuild; i++) {
+ builtChannels.add(delegateChannelBuilder.build());
+ }
+ } catch (Throwable failure) {
+ for (ManagedChannel channel : builtChannels) {
+ try {
+ channel.shutdownNow();
+ } catch (Throwable shutdownFailure) {
+ failure.addSuppressed(shutdownFailure);
+ }
+ }
+ throw failure;
+ }
+
+ if (channelPrimeController != null) {
+ builtChannels.forEach(channelPrimeController::startPrime);
return;
}
- if ((totalActiveStreams.get() / channelRefs.size()) >= maxRpcPerChannel) {
- createNewChannel();
- scaleUpCount.incrementAndGet();
+ int added = 0;
+ List surplus = new ArrayList<>();
+ synchronized (this) {
+ for (ManagedChannel channel : builtChannels) {
+ if (shuttingDown || channelRefs.size() >= maxSize) {
+ surplus.add(channel);
+ } else {
+ addBuiltChannel(channel);
+ added++;
+ }
+ }
}
+ surplus.forEach(ManagedChannel::shutdownNow);
+ scaleUpCount.addAndGet(added);
}
// This is pre-dynamic scaling functionality where we only scale up when the minimum number of
@@ -1872,20 +2429,23 @@ private boolean shouldScaleUp(int minStreams) {
/**
* Pick a {@link ChannelRef} (and create a new one if necessary). If notReadyFallbackEnabled is
* true in the {@link GcpResiliencyOptions} then instead of a channel in a non-READY state another
- * channel in the READY state and having fewer than maximum allowed number of active streams will
- * be provided if available.
+ * channel in the READY state and having picker load below the maximum allowed threshold will be
+ * provided if available.
*/
private ChannelRef pickLeastBusyChannel(boolean forFallback) {
- ChannelRef first = createFirstChannel();
- if (first != null) {
- return first;
- }
-
- if (!fallbackEnabled) {
- return pickLeastBusyNoFallback();
+ // Retries cover post-snapshot deactivation, not draining density.
+ for (int attempt = 0; attempt < 3; attempt++) {
+ ChannelRef first = createFirstChannel();
+ if (first != null) {
+ return first;
+ }
+ ChannelRef picked =
+ fallbackEnabled ? pickLeastBusyWithFallback(forFallback) : pickLeastBusyNoFallback();
+ if (validatePickedChannel(picked)) {
+ return picked;
+ }
}
-
- return pickLeastBusyWithFallback(forFallback);
+ return leastLoadedActiveChannel(channelRefs);
}
/**
@@ -1893,36 +2453,26 @@ private ChannelRef pickLeastBusyChannel(boolean forFallback) {
* GcpManagedChannelOptions.ChannelPickStrategy}.
*/
private ChannelRef pickLeastBusyNoFallback() {
- ChannelRef channelCandidate;
- int minStreams;
-
- if (channelPickStrategy == GcpManagedChannelOptions.ChannelPickStrategy.POWER_OF_TWO) {
- channelCandidate = pickFromCandidates(channelRefs);
+ long now = nanoClock.get();
+ ChannelRef channelCandidate = pickFromCandidates(channelRefs, now);
+ if (!isDynamicScalingEnabled && channelRefs.size() < maxSize) {
// With power-of-two, streams distribute approximately (not exactly) evenly.
// Use max streams for scale-up: if ANY channel hits the watermark, it's overloaded now
// and we should add capacity before other channels follow. This preserves the original
// per-channel watermark semantics (with LINEAR_SCAN, min == max so it didn't matter).
// Global min would delay scale-up; sampled min would be noisy.
- minStreams = getMaxActiveStreams();
- } else {
- channelCandidate = channelRefs.get(0);
- minStreams = channelCandidate.getActiveStreamsCount();
- for (ChannelRef channelRef : channelRefs) {
- int cnt = channelRef.getActiveStreamsCount();
- if (cnt < minStreams) {
- minStreams = cnt;
- channelCandidate = channelRef;
+ int streams =
+ channelPickStrategy == GcpManagedChannelOptions.ChannelPickStrategy.POWER_OF_TWO
+ ? getMaxActiveStreams()
+ : channelCandidate.getPickerLoad(now);
+ if (streams >= maxConcurrentStreamsLowWatermark) {
+ ChannelRef newChannel = tryCreateNewChannel();
+ if (newChannel != null) {
+ scaleUpCount.incrementAndGet();
+ return newChannel;
}
}
}
-
- if (shouldScaleUp(minStreams)) {
- ChannelRef newChannel = tryCreateNewChannel();
- if (newChannel != null) {
- scaleUpCount.incrementAndGet();
- return newChannel;
- }
- }
return channelCandidate;
}
@@ -1933,17 +2483,21 @@ private ChannelRef pickLeastBusyNoFallback() {
private ChannelRef pickLeastBusyWithFallback(boolean forFallback) {
// Full scan to collect eligible ("ready") channels not in fallbackMap and under max streams.
List readyCandidates = new ArrayList<>();
- ChannelRef overallCandidate = channelRefs.get(0);
- int overallMinStreams = overallCandidate.getActiveStreamsCount();
+ ChannelRef overallCandidate = null;
+ int overallMinStreams = Integer.MAX_VALUE;
int readyMaxStreams = 0;
+ long now = nanoClock.get();
for (ChannelRef channelRef : channelRefs) {
- int cnt = channelRef.getActiveStreamsCount();
- if (cnt < overallMinStreams) {
+ if (!channelRef.isActive()) {
+ continue;
+ }
+ int cnt = channelRef.getPickerLoad(now);
+ if (overallCandidate == null || cnt < overallMinStreams) {
overallMinStreams = cnt;
overallCandidate = channelRef;
}
- if (!fallbackMap.containsKey(channelRef.getId()) && cnt < DEFAULT_MAX_STREAM) {
+ if (!fallbackMap.containsKey(channelRef.getId()) && cnt < maxConcurrentStreamsLowWatermark) {
readyCandidates.add(channelRef);
if (cnt > readyMaxStreams) {
readyMaxStreams = cnt;
@@ -1951,6 +2505,10 @@ private ChannelRef pickLeastBusyWithFallback(boolean forFallback) {
}
}
+ if (overallCandidate == null) {
+ return leastLoadedActiveChannel(channelRefs, now);
+ }
+
// For scale-up, use maxStreams among ready channels (consistent with non-fallback path).
int scaleUpStreams = readyCandidates.isEmpty() ? Integer.MAX_VALUE : readyMaxStreams;
if (shouldScaleUp(scaleUpStreams)) {
@@ -1969,7 +2527,7 @@ private ChannelRef pickLeastBusyWithFallback(boolean forFallback) {
if (!readyCandidates.isEmpty()) {
// Apply power-of-two among eligible channels to avoid thundering herd.
- ChannelRef readyCandidate = pickFromCandidates(readyCandidates);
+ ChannelRef readyCandidate = pickFromCandidates(readyCandidates, now);
if (!forFallback && readyCandidate.getId() != overallCandidate.getId()) {
if (logger.isLoggable(Level.FINEST)) {
logger.finest(
@@ -1994,43 +2552,62 @@ private ChannelRef pickLeastBusyWithFallback(boolean forFallback) {
/**
* Picks a channel from the given candidate list using the configured strategy.
*
- * For {@code POWER_OF_TWO}: samples two distinct random candidates and picks the less busy
- * one. On tie, prefers the channel with more recent activity (warmer) to preserve connection
- * warmth under low traffic.
+ *
For {@code POWER_OF_TWO}: samples twice with replacement and picks the less busy sample. The
+ * first sample wins ties. Draining or inactive samples are retried up to twice the candidate
+ * count before a full active-channel scan.
*
- *
For {@code LINEAR_SCAN}: deterministic scan picking the first least-busy channel.
+ *
For {@code LINEAR_SCAN}: deterministic scan picking the first least-busy active channel.
*/
- private ChannelRef pickFromCandidates(List candidates) {
- if (candidates.size() == 1) {
- return candidates.get(0);
- }
+ @VisibleForTesting
+ ChannelRef pickFromCandidates(List candidates) {
+ return pickFromCandidates(candidates, nanoClock.get());
+ }
+
+ private ChannelRef pickFromCandidates(List candidates, long now) {
+ Object[] snapshot = candidates.toArray();
+ int size = snapshot.length;
if (channelPickStrategy == GcpManagedChannelOptions.ChannelPickStrategy.POWER_OF_TWO) {
- ThreadLocalRandom random = ThreadLocalRandom.current();
- int i = random.nextInt(candidates.size());
- int j = random.nextInt(candidates.size() - 1);
- if (j >= i) {
- j++;
- }
- ChannelRef a = candidates.get(i);
- ChannelRef b = candidates.get(j);
- int aStreams = a.getActiveStreamsCount();
- int bStreams = b.getActiveStreamsCount();
- if (aStreams < bStreams) return a;
- if (bStreams < aStreams) return b;
- // Tie: prefer the warmer channel (more recent activity).
- return a.lastResponseNanos >= b.lastResponseNanos ? a : b;
- }
- // LINEAR_SCAN: pick the least busy.
- ChannelRef best = candidates.get(0);
- int bestStreams = best.getActiveStreamsCount();
- for (int k = 1; k < candidates.size(); k++) {
- int cnt = candidates.get(k).getActiveStreamsCount();
- if (cnt < bestStreams) {
- bestStreams = cnt;
- best = candidates.get(k);
- }
- }
- return best;
+ for (int attempt = 0; attempt < 2 * size; attempt++) {
+ ChannelRef first = (ChannelRef) snapshot[candidateIndexPicker.applyAsInt(size)];
+ ChannelRef second = (ChannelRef) snapshot[candidateIndexPicker.applyAsInt(size)];
+ if (!first.isActive() || !second.isActive()) {
+ continue;
+ }
+ ChannelRef picked = pickLessBusy(first, second, now);
+ if (picked.isActive()) {
+ return picked;
+ }
+ }
+ }
+ return leastLoadedActiveChannel(candidates, now);
+ }
+
+ private ChannelRef leastLoadedActiveChannel(List candidates) {
+ return leastLoadedActiveChannel(candidates, nanoClock.get());
+ }
+
+ private ChannelRef leastLoadedActiveChannel(List candidates, long now) {
+ ChannelRef best = null;
+ int bestLoad = Integer.MAX_VALUE;
+ for (ChannelRef candidate : candidates) {
+ if (!candidate.isActive()) {
+ continue;
+ }
+ int candidateLoad = candidate.getPickerLoad(now);
+ if (best == null || candidateLoad < bestLoad) {
+ best = candidate;
+ bestLoad = candidateLoad;
+ }
+ }
+ if (best != null) {
+ return best;
+ }
+ throw Status.UNAVAILABLE.withDescription("No available channels").asRuntimeException();
+ }
+
+ @VisibleForTesting
+ ChannelRef pickLessBusy(ChannelRef first, ChannelRef second, long now) {
+ return first.getPickerLoad(now) <= second.getPickerLoad(now) ? first : second;
}
@Override
@@ -2050,6 +2627,9 @@ public String authority() {
* If method-affinity is specified, we will use the GcpClientCall to fetch the affinitykey and
* bind/unbind the channel, otherwise we just need the SimpleGcpClientCall to keep track of the
* number of streams in each channel.
+ *
+ *
A returned simple call reserves one unit of pool load immediately. If never started, callers
+ * must invoke {@link ClientCall#cancel(String, Throwable)} to release that reservation.
*/
@Override
public ClientCall newCall(
@@ -2096,7 +2676,9 @@ private String keyFromOptsCtx(CallOptions callOptions) {
return key;
}
- private synchronized void cancelBackgroundTasks() {
+ private synchronized List cancelBackgroundTasks() {
+ shuttingDown = true;
+ scaleUpSignalPending.set(false);
if (cleanupTask != null) {
cleanupTask.cancel(false);
cleanupTask = null;
@@ -2109,22 +2691,32 @@ private synchronized void cancelBackgroundTasks() {
logMetricsTask.cancel(false);
logMetricsTask = null;
}
+ drainTasks.values().forEach(task -> task.cancel(false));
+ drainTasks.clear();
+ return channelPrimeController == null
+ ? Collections.emptyList()
+ : channelPrimeController.detachAll();
}
@Override
public ManagedChannel shutdownNow() {
logger.finer(log("Shutdown now started."));
- for (ChannelRef channelRef : channelRefs) {
+ List primesToCancel = cancelBackgroundTasks();
+ if (channelPrimeController != null) {
+ channelPrimeController.cancel(primesToCancel, true);
+ }
+ List activeSnapshot = new ArrayList<>(channelRefs);
+ List removedSnapshot = new ArrayList<>(removedChannelRefs);
+ for (ChannelRef channelRef : activeSnapshot) {
if (!channelRef.getChannel().isTerminated()) {
channelRef.getChannel().shutdownNow();
}
}
- for (ChannelRef channelRef : removedChannelRefs) {
+ for (ChannelRef channelRef : removedSnapshot) {
if (!channelRef.getChannel().isTerminated()) {
channelRef.getChannel().shutdownNow();
}
}
- cancelBackgroundTasks();
if (!stateNotificationExecutor.isTerminated()) {
stateNotificationExecutor.shutdownNow();
}
@@ -2134,13 +2726,18 @@ public ManagedChannel shutdownNow() {
@Override
public ManagedChannel shutdown() {
logger.finer(log("Shutdown started."));
- for (ChannelRef channelRef : channelRefs) {
+ List primesToCancel = cancelBackgroundTasks();
+ if (channelPrimeController != null) {
+ channelPrimeController.cancel(primesToCancel, false);
+ }
+ List activeSnapshot = new ArrayList<>(channelRefs);
+ List removedSnapshot = new ArrayList<>(removedChannelRefs);
+ for (ChannelRef channelRef : activeSnapshot) {
channelRef.getChannel().shutdown();
}
- for (ChannelRef channelRef : removedChannelRefs) {
+ for (ChannelRef channelRef : removedSnapshot) {
channelRef.getChannel().shutdown();
}
- cancelBackgroundTasks();
stateNotificationExecutor.shutdown();
return this;
}
@@ -2252,9 +2849,13 @@ public ConnectivityState getState(boolean requestConnection) {
* One channel can be mapped to more than one keys. But one key can only be mapped to one
* channel.
*/
- protected void bind(ChannelRef channelRef, List affinityKeys) {
+ protected synchronized ChannelRef bind(ChannelRef channelRef, List affinityKeys) {
if (channelRef == null || affinityKeys == null) {
- return;
+ return channelRef;
+ }
+ if (!channelRef.isActive()) {
+ channelRef = pickLeastBusyChannel(/* forFallback= */ false);
+ // Deactivation also holds the pool monitor, so this pick stays active through binding.
}
if (logger.isLoggable(Level.FINEST)) {
logger.finest(
@@ -2266,13 +2867,28 @@ protected void bind(ChannelRef channelRef, List affinityKeys) {
while (affinityKeyToChannelRef.putIfAbsent(affinityKey, channelRef) != null) {
unbind(Collections.singletonList(affinityKey));
}
- affinityKeyLastUsed.put(affinityKey, System.nanoTime());
+ affinityKeyLastUsed.merge(affinityKey, nanoClock.get(), Long::max);
channelRef.affinityCountIncr();
}
+ return channelRef;
+ }
+
+ private synchronized boolean unbindInactiveMapping(String affinityKey, ChannelRef mappedChannel) {
+ Runnable hook = inactiveMappingRemovalHookForTest;
+ if (hook != null) {
+ inactiveMappingRemovalHookForTest = null;
+ hook.run();
+ }
+ if (affinityKeyToChannelRef.remove(affinityKey, mappedChannel)) {
+ affinityKeyLastUsed.remove(affinityKey);
+ mappedChannel.affinityCountDecr();
+ return true;
+ }
+ return false;
}
/** Unbind channel with affinity key. */
- protected void unbind(List affinityKeys) {
+ protected synchronized void unbind(List affinityKeys) {
if (affinityKeys == null) {
return;
}
@@ -2408,7 +3024,13 @@ protected class ChannelRef {
// activeStreamsCount are mutated from the GcpClientCall concurrently using the
// `activeStreamsCountIncr()` and `activeStreamsCountDecr()` methods.
private final AtomicInteger activeStreamsCount;
- private long lastResponseNanos = nanoClock.get();
+ private final long createdNanos = nanoClock.get();
+ private volatile long lastActivityNanos = createdNanos;
+ private long lastResponseNanos = createdNanos;
+
+ private volatile int errorPenaltyLoad;
+ private volatile long errorPenaltyExpiresAtNanos;
+
private final AtomicInteger deadlineExceededCount = new AtomicInteger();
private final AtomicLong okCalls = new AtomicLong();
private final AtomicLong errCalls = new AtomicLong();
@@ -2427,8 +3049,12 @@ protected ChannelRef(ManagedChannel channel, int affinityCount, int activeStream
channelStateMonitor = new ChannelStateMonitor(channel, this);
}
- protected long getConnectedSinceNanos() {
- return channelStateMonitor.getConnectedSinceNanos();
+ protected long getCreatedNanos() {
+ return createdNanos;
+ }
+
+ protected long getLastActivityNanos() {
+ return lastActivityNanos;
}
protected ConnectivityState getState() {
@@ -2447,12 +3073,37 @@ protected boolean isActive() {
return active;
}
- private void activate() {
- active = true;
+ private void activateAndAccountReadiness() {
+ boolean incrementReady;
+ synchronized (this) {
+ active = true;
+ incrementReady = channelStateMonitor.accountReadyIfNeeded();
+ }
+ if (incrementReady) {
+ incReadyChannels(false);
+ }
+ }
+
+ private void deactivateAndAccountReadiness() {
+ boolean decrementReady;
+ synchronized (this) {
+ decrementReady = channelStateMonitor.unaccountReadyIfNeeded();
+ active = false;
+ }
+ if (decrementReady) {
+ decReadyChannels(false);
+ }
+ }
+
+ @VisibleForTesting
+ void deactivateForTest() {
+ deactivateAndAccountReadiness();
}
- private void deactivate() {
- active = false;
+ @VisibleForTesting
+ void setActiveStreamsForTest(int streams) {
+ int previous = activeStreamsCount.getAndSet(streams);
+ totalActiveStreams.addAndGet(streams - previous);
}
protected void affinityCountIncr() {
@@ -2472,14 +3123,16 @@ protected void resetAffinityCount() {
}
protected void activeStreamsCountIncr() {
+ lastActivityNanos = nanoClock.get();
int actStreams = activeStreamsCount.incrementAndGet();
maxActiveStreams.accumulateAndGet(actStreams, Math::max);
int totalActStreams = totalActiveStreams.incrementAndGet();
maxTotalActiveStreams.accumulateAndGet(totalActStreams, Math::max);
- maxTotalActiveStreamsForScaleDown.accumulateAndGet(totalActStreams, Math::max);
+ maybeSignalScaleUp(this);
}
protected void activeStreamsCountDecr(long startNanos, Status status, boolean fromClientSide) {
+ lastActivityNanos = nanoClock.get();
int actStreams = activeStreamsCount.decrementAndGet();
minActiveStreams.accumulateAndGet(actStreams, Math::min);
int totalActStreams = totalActiveStreams.decrementAndGet();
@@ -2494,6 +3147,10 @@ protected void activeStreamsCountDecr(long startNanos, Status status, boolean fr
if (unresponsiveDetectionEnabled) {
detectUnresponsiveConnection(startNanos, status, fromClientSide);
}
+ applyErrorPenalty(status);
+ if (actStreams == 0 && !isActive()) {
+ scheduleDrain(this);
+ }
}
protected void messageReceived() {
@@ -2509,6 +3166,119 @@ protected int getActiveStreamsCount() {
return activeStreamsCount.get();
}
+ protected int getPickerLoad() {
+ return (int)
+ Math.min(Integer.MAX_VALUE, (long) getActiveStreamsCount() + currentErrorPenalty());
+ }
+
+ @VisibleForTesting
+ int currentErrorPenalty() {
+ int penalty = errorPenaltyLoad;
+ if (penalty == 0) {
+ return 0;
+ }
+ return currentErrorPenalty(nanoClock.get());
+ }
+
+ private int currentErrorPenalty(long now) {
+ int penalty = errorPenaltyLoad;
+ long expiresAtNanos = errorPenaltyExpiresAtNanos;
+ if (penalty == 0 || expiresAtNanos == 0) {
+ return 0;
+ }
+ if (now - expiresAtNanos < 0) {
+ return decayedErrorPenalty(penalty, now, expiresAtNanos);
+ }
+ int expiredPenalty;
+ synchronized (this) {
+ penalty = errorPenaltyLoad;
+ expiresAtNanos = errorPenaltyExpiresAtNanos;
+ if (penalty == 0 || expiresAtNanos == 0) {
+ return 0;
+ }
+ if (now - expiresAtNanos < 0) {
+ return decayedErrorPenalty(penalty, now, expiresAtNanos);
+ }
+ errorPenaltyLoad = 0;
+ errorPenaltyExpiresAtNanos = 0;
+ expiredPenalty = penalty;
+ }
+ totalErrorPenaltyLoad.addAndGet(-expiredPenalty);
+ return 0;
+ }
+
+ private int getPickerLoad(long now) {
+ int penalty = errorPenaltyLoad;
+ long expiresAtNanos = errorPenaltyExpiresAtNanos;
+ if (penalty != 0) {
+ penalty = decayedErrorPenalty(penalty, now, expiresAtNanos);
+ }
+ return (int) Math.min(Integer.MAX_VALUE, (long) getActiveStreamsCount() + penalty);
+ }
+
+ private int decayedErrorPenalty(int penalty, long now, long expiresAtNanos) {
+ if (expiresAtNanos == 0) {
+ return 0;
+ }
+ long remainingNanos = expiresAtNanos - now;
+ if (remainingNanos <= 0) {
+ return 0;
+ }
+ // Picker steering decays smoothly. Aggregate accounting deliberately retains the full
+ // contribution until expiry or clear, making scale-up load a conservative upper bound while
+ // preserving one atomic net delta per aggregate transition.
+ return ceilMultiplyDivide(
+ penalty, Math.min(remainingNanos, errorPenaltyDurationNanos), errorPenaltyDurationNanos);
+ }
+
+ private void applyErrorPenalty(Status status) {
+ if (!isDynamicScalingEnabled
+ || !active
+ || errorPenaltyStep == 0
+ || (status.getCode() != Code.UNAVAILABLE
+ && status.getCode() != Code.RESOURCE_EXHAUSTED)) {
+ return;
+ }
+ long now = nanoClock.get();
+ long addedPenalty;
+ synchronized (this) {
+ if (!active) {
+ return;
+ }
+ int previousContribution = errorPenaltyLoad;
+ long previousExpiry = errorPenaltyExpiresAtNanos;
+ int current =
+ previousContribution != 0 && previousExpiry != 0 && now - previousExpiry < 0
+ ? previousContribution
+ : 0;
+ int next = (int) Math.min(maxRpcPerChannel, (long) current + errorPenaltyStep);
+ long nextExpiry = now + errorPenaltyDurationNanos;
+ // Zero is reserved as the cleared expiry sentinel.
+ errorPenaltyExpiresAtNanos = nextExpiry == 0 ? 1 : nextExpiry;
+ errorPenaltyLoad = next;
+ addedPenalty = (long) next - previousContribution;
+ }
+ if (addedPenalty != 0) {
+ totalErrorPenaltyLoad.addAndGet(addedPenalty);
+ }
+ if (addedPenalty > 0) {
+ maybeSignalScaleUp(this);
+ }
+ }
+
+ private void clearErrorPenalty() {
+ int clearedPenalty;
+ synchronized (this) {
+ clearedPenalty = errorPenaltyLoad;
+ if (clearedPenalty == 0) {
+ return;
+ }
+ errorPenaltyLoad = 0;
+ errorPenaltyExpiresAtNanos = 0;
+ }
+ totalErrorPenaltyLoad.addAndGet(-clearedPenalty);
+ }
+
protected long getAndResetOkCalls() {
return okCalls.getAndSet(0);
}
diff --git a/grpc-gcp-java/src/main/java/com/google/cloud/grpc/GcpManagedChannelOptions.java b/grpc-gcp-java/src/main/java/com/google/cloud/grpc/GcpManagedChannelOptions.java
index 94a180b6a6e7..ccb232785b8c 100644
--- a/grpc-gcp-java/src/main/java/com/google/cloud/grpc/GcpManagedChannelOptions.java
+++ b/grpc-gcp-java/src/main/java/com/google/cloud/grpc/GcpManagedChannelOptions.java
@@ -39,8 +39,9 @@ public class GcpManagedChannelOptions {
*/
public enum ChannelPickStrategy {
/**
- * Scans all channels and picks the one with the fewest active streams. Ties are broken by
- * iteration order (lowest index wins). This is the legacy behavior.
+ * Scans all channels and picks the one with the lowest picker load: active streams plus active
+ * error penalty. Ties are broken by iteration order (lowest index wins). This is the legacy
+ * behavior.
*
* This strategy finds the global minimum but is susceptible to the thundering herd problem:
* under burst traffic, all concurrent callers observe the same minimum and pile onto the same
@@ -49,13 +50,13 @@ public enum ChannelPickStrategy {
LINEAR_SCAN,
/**
- * Picks two channels at random and returns the one with fewer active streams. Ties are broken
- * by preferring the more recently active channel (warmth-preserving).
+ * Samples two channels at random with replacement and returns the one with lower picker load.
+ * The first sample wins ties, with no channel-warmth preference. Inactive or draining samples
+ * are retried.
*
- *
This is the default strategy. It avoids the thundering herd problem while keeping warm
- * channels preferred under low traffic. The trade-off is that it may not always find the global
- * minimum, but in practice the difference is negligible because stream counts are inherently
- * racy.
+ *
This is the default strategy. It avoids the thundering herd problem. The trade-off is that
+ * it may not always find the global minimum, but in practice the difference is negligible
+ * because picker load is inherently racy.
*/
POWER_OF_TWO,
}
@@ -212,6 +213,23 @@ public static class GcpChannelPoolOptions {
private final int maxRpcPerChannel;
// How often to check for a possibility to scale down.
private final Duration scaleDownInterval;
+ // Minimum interval between successful scale-up actions.
+ private final Duration scaleUpCooldown;
+ // Consecutive low-load checks required before scaling down.
+ private final int scaleDownConsecutiveLowLoadChecks;
+ // Maximum percentage growth in one scale-up action.
+ private final int maxScaleUpPercent;
+ // Maximum channels removed in one scale-down check.
+ private final int maxScaleDownChannels;
+ private final Duration drainIdleGrace;
+ private final int errorPenaltyStep;
+ private final Duration errorPenaltyDuration;
+ // Optional hook that warms a scaled-up channel before publication.
+ @Nullable private final GcpChannelPrimer channelPrimer;
+ // Maximum time to wait for one channel-primer future.
+ private final Duration channelPrimeTimeout;
+ // Maximum number of channel-primer attempts before rejecting a channel.
+ private final int channelPrimeMaxAttempts;
// Use round-robin channel selection for affinity binding calls.
private final boolean useRoundRobinOnBind;
@@ -229,6 +247,16 @@ public GcpChannelPoolOptions(Builder builder) {
minRpcPerChannel = builder.minRpcPerChannel;
maxRpcPerChannel = builder.maxRpcPerChannel;
scaleDownInterval = builder.scaleDownInterval;
+ scaleUpCooldown = builder.scaleUpCooldown;
+ scaleDownConsecutiveLowLoadChecks = builder.scaleDownConsecutiveLowLoadChecks;
+ maxScaleUpPercent = builder.maxScaleUpPercent;
+ maxScaleDownChannels = builder.maxScaleDownChannels;
+ drainIdleGrace = builder.drainIdleGrace;
+ errorPenaltyStep = builder.errorPenaltyStep;
+ errorPenaltyDuration = builder.errorPenaltyDuration;
+ channelPrimer = builder.channelPrimer;
+ channelPrimeTimeout = builder.channelPrimeTimeout;
+ channelPrimeMaxAttempts = builder.channelPrimeMaxAttempts;
concurrentStreamsLowWatermark = builder.concurrentStreamsLowWatermark;
useRoundRobinOnBind = builder.useRoundRobinOnBind;
affinityKeyLifetime = builder.affinityKeyLifetime;
@@ -260,6 +288,47 @@ public Duration getScaleDownInterval() {
return scaleDownInterval;
}
+ public Duration getScaleUpCooldown() {
+ return scaleUpCooldown;
+ }
+
+ public int getScaleDownConsecutiveLowLoadChecks() {
+ return scaleDownConsecutiveLowLoadChecks;
+ }
+
+ public int getMaxScaleUpPercent() {
+ return maxScaleUpPercent;
+ }
+
+ public int getMaxScaleDownChannels() {
+ return maxScaleDownChannels;
+ }
+
+ public Duration getDrainIdleGrace() {
+ return drainIdleGrace;
+ }
+
+ public int getErrorPenaltyStep() {
+ return errorPenaltyStep;
+ }
+
+ public Duration getErrorPenaltyDuration() {
+ return errorPenaltyDuration;
+ }
+
+ @Nullable
+ public GcpChannelPrimer getChannelPrimer() {
+ return channelPrimer;
+ }
+
+ public Duration getChannelPrimeTimeout() {
+ return channelPrimeTimeout;
+ }
+
+ public int getChannelPrimeMaxAttempts() {
+ return channelPrimeMaxAttempts;
+ }
+
public int getConcurrentStreamsLowWatermark() {
return concurrentStreamsLowWatermark;
}
@@ -293,8 +362,35 @@ public static GcpChannelPoolOptions.Builder newBuilder(GcpChannelPoolOptions opt
@Override
public String toString() {
return String.format(
- "{maxSize: %d, minSize: %d, concurrentStreamsLowWatermark: %d, useRoundRobinOnBind: %s}",
- getMaxSize(), getMinSize(), getConcurrentStreamsLowWatermark(), isUseRoundRobinOnBind());
+ "{maxSize: %d, minSize: %d, initSize: %d, minRpcPerChannel: %d, "
+ + "maxRpcPerChannel: %d, scaleDownInterval: %s, scaleUpCooldown: %s, "
+ + "scaleDownConsecutiveLowLoadChecks: %d, maxScaleUpPercent: %d, "
+ + "maxScaleDownChannels: %d, drainIdleGrace: %s, errorPenaltyStep: %d, "
+ + "errorPenaltyDuration: %s, concurrentStreamsLowWatermark: %d, "
+ + "useRoundRobinOnBind: %s, affinityKeyLifetime: %s, cleanupInterval: %s, "
+ + "channelPickStrategy: %s, channelPrimer: %s, channelPrimeTimeout: %s, "
+ + "channelPrimeMaxAttempts: %d}",
+ getMaxSize(),
+ getMinSize(),
+ getInitSize(),
+ getMinRpcPerChannel(),
+ getMaxRpcPerChannel(),
+ getScaleDownInterval(),
+ getScaleUpCooldown(),
+ getScaleDownConsecutiveLowLoadChecks(),
+ getMaxScaleUpPercent(),
+ getMaxScaleDownChannels(),
+ getDrainIdleGrace(),
+ getErrorPenaltyStep(),
+ getErrorPenaltyDuration(),
+ getConcurrentStreamsLowWatermark(),
+ isUseRoundRobinOnBind(),
+ getAffinityKeyLifetime(),
+ getCleanupInterval(),
+ getChannelPickStrategy(),
+ getChannelPrimer(),
+ getChannelPrimeTimeout(),
+ getChannelPrimeMaxAttempts());
}
public static class Builder {
@@ -304,6 +400,16 @@ public static class Builder {
private int minRpcPerChannel = 0;
private int maxRpcPerChannel = 0;
private Duration scaleDownInterval = Duration.ZERO;
+ private Duration scaleUpCooldown = Duration.ofSeconds(10);
+ private int scaleDownConsecutiveLowLoadChecks = 3;
+ private int maxScaleUpPercent = 30;
+ private int maxScaleDownChannels = 2;
+ private Duration drainIdleGrace = Duration.ofMinutes(1);
+ private int errorPenaltyStep = 5;
+ private Duration errorPenaltyDuration = Duration.ofSeconds(5);
+ @Nullable private GcpChannelPrimer channelPrimer;
+ private Duration channelPrimeTimeout = Duration.ofSeconds(10);
+ private int channelPrimeMaxAttempts = 3;
private int concurrentStreamsLowWatermark = GcpManagedChannel.DEFAULT_MAX_STREAM;
private boolean useRoundRobinOnBind = false;
private Duration affinityKeyLifetime = Duration.ZERO;
@@ -323,6 +429,16 @@ public Builder(GcpChannelPoolOptions options) {
this.minRpcPerChannel = options.getMinRpcPerChannel();
this.maxRpcPerChannel = options.getMaxRpcPerChannel();
this.scaleDownInterval = options.getScaleDownInterval();
+ this.scaleUpCooldown = options.getScaleUpCooldown();
+ this.scaleDownConsecutiveLowLoadChecks = options.getScaleDownConsecutiveLowLoadChecks();
+ this.maxScaleUpPercent = options.getMaxScaleUpPercent();
+ this.maxScaleDownChannels = options.getMaxScaleDownChannels();
+ this.drainIdleGrace = options.getDrainIdleGrace();
+ this.errorPenaltyStep = options.getErrorPenaltyStep();
+ this.errorPenaltyDuration = options.getErrorPenaltyDuration();
+ this.channelPrimer = options.getChannelPrimer();
+ this.channelPrimeTimeout = options.getChannelPrimeTimeout();
+ this.channelPrimeMaxAttempts = options.getChannelPrimeMaxAttempts();
this.concurrentStreamsLowWatermark = options.getConcurrentStreamsLowWatermark();
this.useRoundRobinOnBind = options.isUseRoundRobinOnBind();
this.affinityKeyLifetime = options.getAffinityKeyLifetime();
@@ -375,23 +491,14 @@ public Builder setInitSize(int initSize) {
/**
* Enables dynamic scaling functionality.
*
- *
When the average number of concurrent calls per channel reaches maxRpcPerChannel
- * the pool will create and add a new channel unless already at max size.
- *
- *
Every scaleDownInterval a check for downscaling is performed. Based on the
- * maximum total concurrent calls observed since the last check, the desired number of
- * channels is calculated as:
- *
- *
(max_total_concurrent_calls / minRpcPerChannel) rounded up.
+ *
After a call is counted, load above maxRpcPerChannel on its selected
+ * channel or across the pool average signals a background scale-up worker.
*
- *
If the calculated desired number of channels is lower than the current number of
- * channels, the pool will be downscaled to the desired number or min size (whichever is
- * greater).
- *
- *
When downscaling, channels with the oldest connections are selected. Then the selected
- * channels are removed from the pool but are not instructed to shutdown until all calls are
- * completed. In a case when the pool is scaling up and there is a ready channel awaiting
- * calls completion, the channel will be re-used instead of creating a new channel.
+ *
Every scaleDownInterval, after consecutive low-load checks, the
+ * least-loaded channels are marked draining and removed from selection, bounded by
+ * maxScaleDownChannels; fewer affinity bindings and then older allocations break load ties.
+ * Draining channels receive no new picks or affinity binds and close after their in-flight
+ * calls reach zero and drainIdleGrace elapses. Scale-up always creates new channels.
*
* @param minRpcPerChannel minimum desired average concurrent calls per channel.
* @param maxRpcPerChannel maximum desired average concurrent calls per channel.
@@ -403,6 +510,9 @@ public Builder setDynamicScaling(
minRpcPerChannel > 0, "Minimum RPCs per channel must be positive.");
Preconditions.checkArgument(
maxRpcPerChannel > 0, "Maximum RPCs per channel must be positive.");
+ Preconditions.checkArgument(
+ minRpcPerChannel <= maxRpcPerChannel,
+ "Minimum RPCs per channel must not exceed maximum RPCs per channel.");
Preconditions.checkArgument(
!scaleDownInterval.isNegative() && !scaleDownInterval.isZero(),
"Scale down interval must be positive.");
@@ -424,6 +534,128 @@ public Builder disableDynamicScaling() {
return this;
}
+ /**
+ * Sets the minimum interval between successful scale-up operations. Zero uses the 10-second
+ * default.
+ */
+ public Builder setScaleUpCooldown(Duration scaleUpCooldown) {
+ Preconditions.checkNotNull(scaleUpCooldown, "Scale up cooldown must not be null.");
+ Preconditions.checkArgument(
+ !scaleUpCooldown.isNegative(), "Scale up cooldown must not be negative.");
+ this.scaleUpCooldown = scaleUpCooldown.isZero() ? Duration.ofSeconds(10) : scaleUpCooldown;
+ return this;
+ }
+
+ public Builder setScaleDownConsecutiveLowLoadChecks(int checks) {
+ Preconditions.checkArgument(checks > 0, "Scale down checks must be positive.");
+ this.scaleDownConsecutiveLowLoadChecks = checks;
+ return this;
+ }
+
+ /**
+ * Sets the maximum percentage of active channels added by one scale-up operation. The
+ * percentage cap has a two-channel floor before desired-size and maximum-size clamps.
+ */
+ public Builder setMaxScaleUpPercent(int percent) {
+ Preconditions.checkArgument(
+ percent > 0 && percent <= 100, "Scale up percent must be in (0, 100].");
+ this.maxScaleUpPercent = percent;
+ return this;
+ }
+
+ public Builder setMaxScaleDownChannels(int channels) {
+ Preconditions.checkArgument(channels > 0, "Scale down channel limit must be positive.");
+ this.maxScaleDownChannels = channels;
+ return this;
+ }
+
+ /**
+ * Sets how long an idle draining channel remains open for sticky affinity references and
+ * in-flight calls before its delegate closes. Defaults to one minute.
+ */
+ public Builder setDrainIdleGrace(Duration drainIdleGrace) {
+ Preconditions.checkNotNull(drainIdleGrace, "Drain idle grace must not be null.");
+ Preconditions.checkArgument(
+ !drainIdleGrace.isNegative(), "Drain idle grace must not be negative.");
+ this.drainIdleGrace = drainIdleGrace;
+ return this;
+ }
+
+ /**
+ * Sets the load penalty added after each retryable channel error. A value of 0 disables error
+ * penalties. Each step accumulates on the stored undecayed penalty, capped at {@code
+ * maxRpcPerChannel}. Must not be negative. Defaults to 5.
+ */
+ public Builder setErrorPenaltyStep(int errorPenaltyStep) {
+ Preconditions.checkArgument(
+ errorPenaltyStep >= 0, "Error penalty step must not be negative.");
+ this.errorPenaltyStep = errorPenaltyStep;
+ return this;
+ }
+
+ /**
+ * Sets how long an applied penalty takes to decay linearly to zero. Each new retryable error
+ * resets the decay timer to a full window. Must be positive. Defaults to 5 seconds.
+ */
+ public Builder setErrorPenaltyDuration(Duration errorPenaltyDuration) {
+ Preconditions.checkNotNull(
+ errorPenaltyDuration, "Error penalty duration must not be null.");
+ Preconditions.checkArgument(
+ !errorPenaltyDuration.isNegative() && !errorPenaltyDuration.isZero(),
+ "Error penalty duration must be positive.");
+ try {
+ errorPenaltyDuration.toNanos();
+ } catch (ArithmeticException failure) {
+ throw new IllegalArgumentException(
+ "Error penalty duration must fit in nanoseconds.", failure);
+ }
+ this.errorPenaltyDuration = errorPenaltyDuration;
+ return this;
+ }
+
+ /**
+ * Sets the optional hook that primes newly built scale-up channels concurrently. Only
+ * channels added by dynamic scale-up are primed; the initial pool and non-dynamic growth are
+ * published as soon as they are built. Each scale-up channel is published individually when
+ * its own primer future succeeds. A {@code null} primer disables priming and preserves the
+ * existing scale-up path.
+ */
+ public Builder setChannelPrimer(@Nullable GcpChannelPrimer channelPrimer) {
+ this.channelPrimer = channelPrimer;
+ return this;
+ }
+
+ /**
+ * Sets the maximum time allowed for each channel-primer attempt, including the synchronous
+ * call to the primer. Zero uses the 10-second default.
+ */
+ public Builder setChannelPrimeTimeout(Duration channelPrimeTimeout) {
+ Preconditions.checkNotNull(channelPrimeTimeout, "Channel prime timeout must not be null.");
+ Preconditions.checkArgument(
+ !channelPrimeTimeout.isNegative(), "Channel prime timeout must not be negative.");
+ Duration timeout =
+ channelPrimeTimeout.isZero() ? Duration.ofSeconds(10) : channelPrimeTimeout;
+ try {
+ timeout.toNanos();
+ } catch (ArithmeticException failure) {
+ throw new IllegalArgumentException(
+ "Channel prime timeout must fit in nanoseconds.", failure);
+ }
+ this.channelPrimeTimeout = timeout;
+ return this;
+ }
+
+ /**
+ * Sets the maximum number of attempts to prime one scaled-up channel. Zero uses the default
+ * of 3. Retry backoff is exponential from 100 ms and capped at 5 s.
+ */
+ public Builder setChannelPrimeMaxAttempts(int channelPrimeMaxAttempts) {
+ Preconditions.checkArgument(
+ channelPrimeMaxAttempts >= 0, "Channel prime max attempts must not be negative.");
+ this.channelPrimeMaxAttempts = channelPrimeMaxAttempts == 0 ? 3 : channelPrimeMaxAttempts;
+ return this;
+ }
+
/**
* Sets the concurrent streams low watermark. If every channel in the pool has at least this
* amount of concurrent streams then a new channel will be created in the pool unless the pool
@@ -482,11 +714,11 @@ public Builder setCleanupInterval(Duration cleanupInterval) {
* Sets the strategy for picking the least busy channel from the pool.
*
*
Defaults to {@link ChannelPickStrategy#POWER_OF_TWO} which avoids the thundering herd
- * problem by randomly sampling two channels and picking the less busy one, with ties broken
- * by channel warmth (most recently active).
+ * problem by sampling two channels with replacement and picking the less busy one. The first
+ * sample wins ties, with no channel-warmth preference. Inactive samples are retried.
*
*
Use {@link ChannelPickStrategy#LINEAR_SCAN} to restore the legacy behavior of scanning
- * all channels and always picking the one with the fewest active streams.
+ * all channels and always picking the one with the lowest picker load.
*
* @param strategy the channel pick strategy to use.
*/
diff --git a/grpc-gcp-java/src/main/java/com/google/cloud/grpc/GcpMetricsConstants.java b/grpc-gcp-java/src/main/java/com/google/cloud/grpc/GcpMetricsConstants.java
index 94db0da5c354..262559ba41f7 100644
--- a/grpc-gcp-java/src/main/java/com/google/cloud/grpc/GcpMetricsConstants.java
+++ b/grpc-gcp-java/src/main/java/com/google/cloud/grpc/GcpMetricsConstants.java
@@ -77,4 +77,5 @@ class GcpMetricsConstants {
public static String METRIC_ENDPOINT_SWITCH = "endpoint_switch";
public static String METRIC_CURRENT_ENDPOINT = "current_endpoint";
public static String METRIC_CHANNEL_POOL_SCALING = "channel_pool_scaling";
+ public static String METRIC_SCALE_UP_PRIME_FAILURES = "scale_up_prime_failures";
}
diff --git a/grpc-gcp-java/src/test/java/com/google/cloud/grpc/GcpChannelPrimeControllerTest.java b/grpc-gcp-java/src/test/java/com/google/cloud/grpc/GcpChannelPrimeControllerTest.java
new file mode 100644
index 000000000000..0c36681efbf2
--- /dev/null
+++ b/grpc-gcp-java/src/test/java/com/google/cloud/grpc/GcpChannelPrimeControllerTest.java
@@ -0,0 +1,480 @@
+/*
+ * Copyright 2026 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.grpc;
+
+import static com.google.common.truth.Truth.assertThat;
+import static org.awaitility.Awaitility.await;
+
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.SettableFuture;
+import io.grpc.ManagedChannel;
+import java.time.Duration;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+import javax.annotation.Nullable;
+import org.junit.After;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/**
+ * Direct {@link GcpChannelPrimeController} tests for the interleavings a pool-level test cannot
+ * force: they hand the controller a scheduler whose timers never fire on their own, so a test can
+ * run the timeout of an attempt at a moment of its choosing.
+ */
+@RunWith(JUnit4.class)
+public final class GcpChannelPrimeControllerTest {
+ private final ExecutorService stateExecutor = Executors.newSingleThreadExecutor();
+ private final RecordingScheduler scheduler = new RecordingScheduler();
+ private final Object lock = new Object();
+ private final FakePool pool = new FakePool(lock);
+
+ @After
+ public void tearDown() {
+ scheduler.shutdownNow();
+ stateExecutor.shutdownNow();
+ }
+
+ @Test
+ public void poolCallbacksObserveTheLockContract() throws Exception {
+ AtomicInteger primeCalls = new AtomicInteger();
+ GcpChannelPrimer primer =
+ channel ->
+ primeCalls.incrementAndGet() == 1
+ ? Futures.immediateFailedFuture(new IllegalStateException("prime failed"))
+ : Futures.immediateVoidFuture();
+ GcpChannelPrimeController controller = newController(primer, 1);
+
+ controller.startPrime(newChannel());
+ awaitCondition(() -> pool.failures.size() == 1);
+ controller.startPrime(newChannel());
+ awaitCondition(() -> pool.published.size() == 1);
+
+ assertThat(pool.lockViolations).isEmpty();
+ assertThat(pool.abandonReports.get()).isEqualTo(0);
+ synchronized (lock) {
+ assertThat(controller.inFlightCount()).isEqualTo(0);
+ }
+ }
+
+ @Test
+ public void timeoutAfterPrimeReturnsCancelsTheLateFutureAndRetries() throws Exception {
+ CountDownLatch primerEntered = new CountDownLatch(1);
+ CountDownLatch releasePrimer = new CountDownLatch(1);
+ SettableFuture lateFuture = SettableFuture.create();
+ AtomicInteger primeCalls = new AtomicInteger();
+ AtomicReference primerThread = new AtomicReference<>();
+ GcpChannelPrimer primer =
+ channel -> {
+ if (primeCalls.incrementAndGet() > 1) {
+ return Futures.immediateVoidFuture();
+ }
+ primerThread.set(Thread.currentThread());
+ primerEntered.countDown();
+ awaitUninterruptibly(releasePrimer);
+ return lateFuture;
+ };
+ GcpChannelPrimeController controller = newController(primer, 3);
+ FakeChannel channel = newChannel();
+
+ controller.startPrime(channel);
+ assertThat(primerEntered.await(5, TimeUnit.SECONDS)).isTrue();
+ assertThat(scheduler.tasks).hasSize(1);
+ Runnable timeout = scheduler.tasks.get(0);
+
+ try {
+ synchronized (lock) {
+ // prime() returns while this thread holds the pool lock, so the primer thread parks on
+ // the monitor before it can record the returned future.
+ releasePrimer.countDown();
+ awaitCondition(() -> primerThread.get().getState() == Thread.State.BLOCKED);
+ // The timeout wins that race. prime() has returned, so this is an ordinary timed-out
+ // attempt: it must not be abandoned without a retry.
+ timeout.run();
+ assertThat(controller.abandonedCount()).isEqualTo(0);
+ assertThat(controller.inFlightCount()).isEqualTo(1);
+ }
+ } finally {
+ releasePrimer.countDown();
+ }
+
+ // The primer thread then sees the attempt has moved on and cancels the late future.
+ awaitCondition(lateFuture::isCancelled);
+ assertThat(pool.abandonReports.get()).isEqualTo(0);
+ assertThat(pool.failures).isEmpty();
+ assertThat(channel.isShutdown()).isFalse();
+ awaitCondition(() -> scheduler.tasks.size() == 2);
+ Runnable retry = scheduler.tasks.get(1);
+
+ retry.run();
+
+ awaitCondition(() -> pool.published.size() == 1);
+ assertThat(pool.published.get(0)).isSameInstanceAs(channel);
+ assertThat(primeCalls.get()).isEqualTo(2);
+ assertThat(pool.lockViolations).isEmpty();
+ }
+
+ @Test
+ public void timeoutBeforePrimeEntryRefusesTheExpiredAttempt() throws Exception {
+ CountDownLatch timeoutArmed = new CountDownLatch(1);
+ AtomicInteger primeCalls = new AtomicInteger();
+ AtomicReference primerThread = new AtomicReference<>();
+ GcpChannelPrimer primer =
+ channel -> {
+ primeCalls.incrementAndGet();
+ return Futures.immediateVoidFuture();
+ };
+ GcpChannelPrimeController controller = newController(primer, 1);
+ FakeChannel channel = newChannel();
+ // Arming the timeout is the primer thread's last step under the pool lock before it enters
+ // prime(). Park it there with the lock released, exactly as if it had been descheduled between
+ // dropping the lock and entering prime(), until the test lets it go.
+ scheduler.onSchedule =
+ () -> {
+ primerThread.set(Thread.currentThread());
+ timeoutArmed.countDown();
+ synchronized (lock) {
+ while (!scheduler.releaseParkedThread) {
+ awaitUninterruptibly(lock);
+ }
+ }
+ };
+
+ controller.startPrime(channel);
+ assertThat(timeoutArmed.await(5, TimeUnit.SECONDS)).isTrue();
+ assertThat(scheduler.tasks).hasSize(1);
+ // The timeout wins that race: it runs to completion, releasing the slot and closing the
+ // channel, before the primer thread gets to move on.
+ scheduler.execute(scheduler.tasks.get(0));
+ awaitCondition(() -> pool.failures.size() == 1);
+ assertThat(channel.isShutdown()).isTrue();
+ synchronized (lock) {
+ assertThat(controller.inFlightCount()).isEqualTo(0);
+ assertThat(controller.abandonedCount()).isEqualTo(0);
+ scheduler.releaseParkedThread = true;
+ lock.notifyAll();
+ }
+
+ // The primer thread then finds its attempt expired and returns to the pool without ever
+ // calling prime(): idle pool threads wait on a timed poll, unlike a parked or blocked primer.
+ awaitCondition(
+ () -> {
+ Thread.State state = primerThread.get().getState();
+ return state == Thread.State.TIMED_WAITING || state == Thread.State.TERMINATED;
+ });
+ assertThat(primeCalls.get()).isEqualTo(0);
+ assertThat(pool.failures).hasSize(1);
+ assertThat(pool.abandonReports.get()).isEqualTo(0);
+ assertThat(pool.published).isEmpty();
+ assertThat(scheduler.tasks).hasSize(1);
+ synchronized (lock) {
+ assertThat(controller.primingCount()).isEqualTo(0);
+ }
+ assertThat(pool.lockViolations).isEmpty();
+ }
+
+ @Test
+ public void timeoutWhileBlockedInPrimeAbandonsAndReportsOutsideTheLock() throws Exception {
+ CountDownLatch primerEntered = new CountDownLatch(1);
+ CountDownLatch releasePrimer = new CountDownLatch(1);
+ SettableFuture lateFuture = SettableFuture.create();
+ GcpChannelPrimer primer =
+ channel -> {
+ primerEntered.countDown();
+ awaitUninterruptibly(releasePrimer);
+ return lateFuture;
+ };
+ GcpChannelPrimeController controller = newController(primer, 3);
+ FakeChannel channel = newChannel();
+
+ controller.startPrime(channel);
+ assertThat(primerEntered.await(5, TimeUnit.SECONDS)).isTrue();
+ try {
+ scheduler.tasks.get(0).run();
+
+ assertThat(pool.abandonReports.get()).isEqualTo(1);
+ assertThat(pool.failures).hasSize(1);
+ assertThat(channel.isShutdown()).isTrue();
+ assertThat(scheduler.tasks).hasSize(1);
+ synchronized (lock) {
+ assertThat(controller.abandonedCount()).isEqualTo(1);
+ assertThat(controller.inFlightCount()).isEqualTo(0);
+ }
+ } finally {
+ releasePrimer.countDown();
+ }
+ awaitCondition(lateFuture::isCancelled);
+ synchronized (lock) {
+ assertThat(controller.abandonedCount()).isEqualTo(0);
+ }
+ assertThat(pool.lockViolations).isEmpty();
+ }
+
+ @Test
+ public void timeoutWhileBlockedInPrimeDuringShutdownDoesNotReportAbandoning() throws Exception {
+ CountDownLatch primerEntered = new CountDownLatch(1);
+ CountDownLatch releasePrimer = new CountDownLatch(1);
+ SettableFuture lateFuture = SettableFuture.create();
+ GcpChannelPrimer primer =
+ channel -> {
+ primerEntered.countDown();
+ awaitUninterruptibly(releasePrimer);
+ return lateFuture;
+ };
+ GcpChannelPrimeController controller = newController(primer, 3);
+ FakeChannel channel = newChannel();
+
+ controller.startPrime(channel);
+ assertThat(primerEntered.await(5, TimeUnit.SECONDS)).isTrue();
+ try {
+ pool.shuttingDown = true;
+ scheduler.tasks.get(0).run();
+
+ assertThat(pool.abandonReports.get()).isEqualTo(0);
+ assertThat(pool.failures).isEmpty();
+ assertThat(channel.isShutdown()).isTrue();
+ synchronized (lock) {
+ assertThat(controller.abandonedCount()).isEqualTo(0);
+ assertThat(controller.inFlightCount()).isEqualTo(0);
+ }
+ } finally {
+ releasePrimer.countDown();
+ }
+ awaitCondition(lateFuture::isCancelled);
+ assertThat(pool.lockViolations).isEmpty();
+ }
+
+ private GcpChannelPrimeController newController(GcpChannelPrimer primer, int maxAttempts) {
+ return new GcpChannelPrimeController(
+ lock, pool, primer, TimeUnit.SECONDS.toNanos(5), maxAttempts, scheduler);
+ }
+
+ private FakeChannel newChannel() {
+ return new FakeChannel(stateExecutor);
+ }
+
+ private static void awaitCondition(Callable condition) {
+ await().atMost(Duration.ofSeconds(5)).until(condition);
+ }
+
+ private static void awaitUninterruptibly(CountDownLatch latch) {
+ boolean interrupted = false;
+ while (true) {
+ try {
+ latch.await();
+ break;
+ } catch (InterruptedException failure) {
+ interrupted = true;
+ }
+ }
+ if (interrupted) {
+ Thread.currentThread().interrupt();
+ }
+ }
+
+ /**
+ * One {@link Object#wait()} on {@code monitor}, which the caller must hold; may wake spuriously.
+ */
+ private static void awaitUninterruptibly(Object monitor) {
+ try {
+ monitor.wait();
+ } catch (InterruptedException failure) {
+ Thread.currentThread().interrupt();
+ }
+ }
+
+ private static final class FakeChannel extends GcpManagedChannelTest.FakeManagedChannel {
+ FakeChannel(ExecutorService exec) {
+ super(exec);
+ }
+ }
+
+ /** Records every pool callback together with whether it honored its lock contract. */
+ private static final class FakePool implements GcpChannelPrimeController.Pool {
+ private final Object lock;
+ volatile boolean shuttingDown;
+ volatile boolean full;
+ final List published = new CopyOnWriteArrayList<>();
+ final List failures = new CopyOnWriteArrayList<>();
+ final AtomicInteger abandonReports = new AtomicInteger();
+ final List lockViolations = new CopyOnWriteArrayList<>();
+
+ FakePool(Object lock) {
+ this.lock = lock;
+ }
+
+ @Override
+ public boolean isShuttingDown() {
+ expectLock("isShuttingDown", true);
+ return shuttingDown;
+ }
+
+ @Override
+ public boolean isFull() {
+ expectLock("isFull", true);
+ return full;
+ }
+
+ @Override
+ public void addPrimedChannel(ManagedChannel channel) {
+ expectLock("addPrimedChannel", true);
+ published.add(channel);
+ }
+
+ @Override
+ public void reportPrimeFailure(Throwable failure) {
+ expectLock("reportPrimeFailure", false);
+ failures.add(failure);
+ }
+
+ @Override
+ public void reportPrimeAbandoned() {
+ expectLock("reportPrimeAbandoned", false);
+ abandonReports.incrementAndGet();
+ }
+
+ private void expectLock(String method, boolean held) {
+ if (Thread.holdsLock(lock) != held) {
+ lockViolations.add(
+ method + (held ? " called without the pool lock" : " called under the pool lock"));
+ }
+ }
+ }
+
+ /**
+ * Runs immediate work on a real thread but only records delayed tasks, leaving it to the test to
+ * run a timeout or retry at the moment under test.
+ */
+ private static final class RecordingScheduler implements ScheduledExecutorService {
+ private final ScheduledExecutorService delegate = Executors.newSingleThreadScheduledExecutor();
+ final List tasks = new CopyOnWriteArrayList<>();
+ // Runs on the scheduling thread once a delayed task is recorded, so a test can park that
+ // thread at the moment it arms a timeout.
+ @Nullable volatile Runnable onSchedule;
+ // Guarded by the test's pool lock: set under it and announced with notifyAll().
+ boolean releaseParkedThread;
+
+ @Override
+ public ScheduledFuture schedule(Runnable command, long delay, TimeUnit unit) {
+ tasks.add(command);
+ Runnable hook = onSchedule;
+ if (hook != null) {
+ hook.run();
+ }
+ return delegate.schedule(() -> {}, 365, TimeUnit.DAYS);
+ }
+
+ @Override
+ public ScheduledFuture schedule(Callable callable, long delay, TimeUnit unit) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public ScheduledFuture scheduleAtFixedRate(
+ Runnable command, long initialDelay, long period, TimeUnit unit) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public ScheduledFuture scheduleWithFixedDelay(
+ Runnable command, long initialDelay, long delay, TimeUnit unit) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void execute(Runnable command) {
+ delegate.execute(command);
+ }
+
+ @Override
+ public void shutdown() {
+ delegate.shutdown();
+ }
+
+ @Override
+ public List shutdownNow() {
+ return delegate.shutdownNow();
+ }
+
+ @Override
+ public boolean isShutdown() {
+ return delegate.isShutdown();
+ }
+
+ @Override
+ public boolean isTerminated() {
+ return delegate.isTerminated();
+ }
+
+ @Override
+ public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException {
+ return delegate.awaitTermination(timeout, unit);
+ }
+
+ @Override
+ public java.util.concurrent.Future submit(Callable task) {
+ return delegate.submit(task);
+ }
+
+ @Override
+ public java.util.concurrent.Future submit(Runnable task, T result) {
+ return delegate.submit(task, result);
+ }
+
+ @Override
+ public java.util.concurrent.Future submit(Runnable task) {
+ return delegate.submit(task);
+ }
+
+ @Override
+ public List> invokeAll(
+ java.util.Collection> tasks) throws InterruptedException {
+ return delegate.invokeAll(tasks);
+ }
+
+ @Override
+ public List> invokeAll(
+ java.util.Collection> tasks, long timeout, TimeUnit unit)
+ throws InterruptedException {
+ return delegate.invokeAll(tasks, timeout, unit);
+ }
+
+ @Override
+ public T invokeAny(java.util.Collection> tasks)
+ throws InterruptedException, java.util.concurrent.ExecutionException {
+ return delegate.invokeAny(tasks);
+ }
+
+ @Override
+ public T invokeAny(
+ java.util.Collection> tasks, long timeout, TimeUnit unit)
+ throws InterruptedException,
+ java.util.concurrent.ExecutionException,
+ java.util.concurrent.TimeoutException {
+ return delegate.invokeAny(tasks, timeout, unit);
+ }
+ }
+}
diff --git a/grpc-gcp-java/src/test/java/com/google/cloud/grpc/GcpClientCallTest.java b/grpc-gcp-java/src/test/java/com/google/cloud/grpc/GcpClientCallTest.java
index ce113e95c457..1524c571ab4b 100644
--- a/grpc-gcp-java/src/test/java/com/google/cloud/grpc/GcpClientCallTest.java
+++ b/grpc-gcp-java/src/test/java/com/google/cloud/grpc/GcpClientCallTest.java
@@ -17,12 +17,15 @@
package com.google.cloud.grpc;
import static com.google.common.truth.Truth.assertThat;
+import static org.junit.Assert.assertThrows;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyBoolean;
import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
+import com.google.cloud.grpc.proto.AffinityConfig;
import io.grpc.CallOptions;
import io.grpc.ClientCall;
import io.grpc.ConnectivityState;
@@ -162,4 +165,104 @@ public void simpleCallUnbindsAffinityKeyOnCancel() {
assertThat(channelRef.getAffinityCount()).isEqualTo(0);
verify(delegateCall).cancel("cancelled", null);
}
+
+ @SuppressWarnings("unchecked")
+ @Test
+ public void simpleCallCountsExactlyOnceForWholeLifetime() {
+ GcpClientCall.SimpleGcpClientCall call =
+ new GcpClientCall.SimpleGcpClientCall<>(
+ gcpChannel, channelRef, METHOD_DESCRIPTOR, CallOptions.DEFAULT);
+
+ assertThat(channelRef.getActiveStreamsCount()).isEqualTo(1);
+ call.start(new ClientCall.Listener() {}, new Metadata());
+ assertThat(channelRef.getActiveStreamsCount()).isEqualTo(1);
+
+ ArgumentCaptor> listenerCaptor =
+ (ArgumentCaptor>)
+ (ArgumentCaptor) ArgumentCaptor.forClass(ClientCall.Listener.class);
+ verify(delegateCall).start(listenerCaptor.capture(), any(Metadata.class));
+ listenerCaptor.getValue().onClose(Status.OK, new Metadata());
+ call.cancel("late cancel", null);
+
+ assertThat(channelRef.getActiveStreamsCount()).isEqualTo(0);
+ }
+
+ @Test
+ public void simpleCallCancelledBeforeStartNeverDecrementsBelowZero() {
+ GcpClientCall.SimpleGcpClientCall call =
+ new GcpClientCall.SimpleGcpClientCall<>(
+ gcpChannel, channelRef, METHOD_DESCRIPTOR, CallOptions.DEFAULT);
+
+ assertThat(channelRef.getActiveStreamsCount()).isEqualTo(1);
+ call.cancel("before start", null);
+ assertThat(channelRef.getActiveStreamsCount()).isEqualTo(0);
+ call.start(new ClientCall.Listener() {}, new Metadata());
+
+ assertThat(channelRef.getActiveStreamsCount()).isEqualTo(0);
+ }
+
+ @SuppressWarnings("unchecked")
+ @Test
+ public void affinityCallCountsExactlyOnceAfterSelection() {
+ gcpChannel.channelRefs.add(channelRef);
+ GcpClientCall call =
+ new GcpClientCall<>(
+ gcpChannel,
+ METHOD_DESCRIPTOR,
+ CallOptions.DEFAULT,
+ AffinityConfig.newBuilder()
+ .setCommand(AffinityConfig.Command.BOUND)
+ .setAffinityKey("name")
+ .build());
+ call.start(new ClientCall.Listener() {}, new Metadata());
+ call.sendMessage("request");
+
+ assertThat(channelRef.getActiveStreamsCount()).isEqualTo(1);
+ ArgumentCaptor> listenerCaptor =
+ (ArgumentCaptor>)
+ (ArgumentCaptor) ArgumentCaptor.forClass(ClientCall.Listener.class);
+ verify(delegateCall).start(listenerCaptor.capture(), any(Metadata.class));
+ listenerCaptor.getValue().onClose(Status.OK, new Metadata());
+ call.cancel("late cancel", null);
+
+ assertThat(channelRef.getActiveStreamsCount()).isEqualTo(0);
+ }
+
+ @Test
+ public void gcpClientCall_exceptionInQueuedStart_doesNotLeakActiveStreamCount() {
+ gcpChannel.channelRefs.add(channelRef);
+ doThrow(new IllegalStateException("start failed"))
+ .when(delegateCall)
+ .start(any(), any(Metadata.class));
+ GcpClientCall call =
+ new GcpClientCall<>(
+ gcpChannel,
+ METHOD_DESCRIPTOR,
+ CallOptions.DEFAULT,
+ AffinityConfig.newBuilder()
+ .setCommand(AffinityConfig.Command.BOUND)
+ .setAffinityKey("name")
+ .build());
+ call.start(new ClientCall.Listener() {}, new Metadata());
+
+ assertThrows(IllegalStateException.class, () -> call.sendMessage("request"));
+
+ assertThat(channelRef.getActiveStreamsCount()).isEqualTo(0);
+ }
+
+ @Test
+ public void simpleGcpClientCall_exceptionInStart_doesNotLeakActiveStreamCount() {
+ doThrow(new IllegalStateException("start failed"))
+ .when(delegateCall)
+ .start(any(), any(Metadata.class));
+ GcpClientCall.SimpleGcpClientCall call =
+ new GcpClientCall.SimpleGcpClientCall<>(
+ gcpChannel, channelRef, METHOD_DESCRIPTOR, CallOptions.DEFAULT);
+
+ assertThrows(
+ IllegalStateException.class,
+ () -> call.start(new ClientCall.Listener() {}, new Metadata()));
+
+ assertThat(channelRef.getActiveStreamsCount()).isEqualTo(0);
+ }
}
diff --git a/grpc-gcp-java/src/test/java/com/google/cloud/grpc/GcpManagedChannelDrainingTest.java b/grpc-gcp-java/src/test/java/com/google/cloud/grpc/GcpManagedChannelDrainingTest.java
new file mode 100644
index 000000000000..571da13dd377
--- /dev/null
+++ b/grpc-gcp-java/src/test/java/com/google/cloud/grpc/GcpManagedChannelDrainingTest.java
@@ -0,0 +1,347 @@
+/*
+ * Copyright 2026 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.grpc;
+
+import static com.google.common.truth.Truth.assertThat;
+import static org.awaitility.Awaitility.await;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import com.google.cloud.grpc.GcpManagedChannel.ChannelRef;
+import com.google.cloud.grpc.GcpManagedChannelOptions.GcpChannelPoolOptions;
+import com.google.common.util.concurrent.MoreExecutors;
+import io.grpc.ManagedChannel;
+import io.grpc.Status;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Supplier;
+import org.junit.After;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+@RunWith(JUnit4.class)
+public final class GcpManagedChannelDrainingTest {
+ private final ExecutorService executor = MoreExecutors.newDirectExecutorService();
+ private GcpManagedChannel pool;
+
+ @After
+ public void tearDown() {
+ if (pool != null) {
+ pool.shutdownNow();
+ }
+ executor.shutdownNow();
+ }
+
+ @Test
+ public void idleDrainedChannelWaitsForGraceBeforeClose() {
+ AtomicLong clock = new AtomicLong(System.nanoTime());
+ pool = newPool(Duration.ofMinutes(1));
+ clock.set(System.nanoTime());
+ pool.setNanoClock(clock::get);
+
+ scaleDown();
+ ChannelRef draining = pool.removedChannelRefs.iterator().next();
+ pool.finishDrain(draining);
+ assertThat(draining.getChannel().isShutdown()).isFalse();
+
+ clock.addAndGet(Duration.ofMinutes(1).plusNanos(1).toNanos());
+ pool.finishDrain(draining);
+ assertThat(draining.getChannel().isShutdown()).isTrue();
+ }
+
+ @Test
+ public void activeDrainedChannelClosesAfterFinalStream() {
+ pool = newPool(Duration.ZERO);
+ ChannelRef victim = pool.channelRefs.get(0);
+ victim.activeStreamsCountIncr();
+ for (int i = 1; i < pool.channelRefs.size(); i++) {
+ pool.channelRefs.get(i).setActiveStreamsForTest(2);
+ }
+
+ scaleDown();
+ assertThat(pool.removedChannelRefs).contains(victim);
+ assertThat(victim.getChannel().isShutdown()).isFalse();
+
+ victim.activeStreamsCountDecr(System.nanoTime(), Status.OK, false);
+ await().atMost(Duration.ofSeconds(5)).until(() -> victim.getChannel().isShutdown());
+ }
+
+ @Test
+ public void scaledUpPoolBuildsFreshChannelWhileRemovedChannelDrains() {
+ pool = newPool(Duration.ofMinutes(1));
+ ChannelRef victim = pool.channelRefs.get(0);
+ ((GcpManagedChannelTest.FakeManagedChannel) victim.getChannel())
+ .setState(io.grpc.ConnectivityState.READY);
+
+ scaleDown();
+ assertThat(pool.removedChannelRefs).contains(victim);
+
+ ChannelRef fresh = pool.createNewChannel();
+ pool.finishDrain(victim);
+
+ assertThat(fresh).isNotSameInstanceAs(victim);
+ assertThat(pool.channelRefs).contains(fresh);
+ assertThat(pool.channelRefs).doesNotContain(victim);
+ assertThat(pool.removedChannelRefs).contains(victim);
+ assertThat(victim.isActive()).isFalse();
+ assertThat(victim.getChannel().isShutdown()).isFalse();
+ }
+
+ @Test
+ public void scaleDownPrefersChannelWithFewerAffinitiesWhenLoadsTie() {
+ pool = newPool(Duration.ofMinutes(1));
+ ChannelRef affiliated = pool.channelRefs.get(0);
+ pool.bind(affiliated, Collections.singletonList("session"));
+
+ scaleDown();
+
+ assertThat(pool.channelRefs).contains(affiliated);
+ assertThat(pool.removedChannelRefs).doesNotContain(affiliated);
+ assertThat(affiliated.getAffinityCount()).isEqualTo(1);
+ }
+
+ @Test
+ public void finishDrainShutsDelegateDownOutsidePoolMonitor() {
+ AtomicLong clock = new AtomicLong(System.nanoTime());
+ AtomicReference poolReference = new AtomicReference<>();
+ pool =
+ newPool(
+ Duration.ofMinutes(1), () -> new LockCheckingManagedChannel(executor, poolReference));
+ poolReference.set(pool);
+ clock.set(System.nanoTime());
+ pool.setNanoClock(clock::get);
+
+ scaleDown();
+ ChannelRef draining = pool.removedChannelRefs.iterator().next();
+ LockCheckingManagedChannel delegate = (LockCheckingManagedChannel) draining.getChannel();
+ clock.addAndGet(Duration.ofMinutes(1).plusNanos(1).toNanos());
+
+ pool.finishDrain(draining);
+
+ assertThat(delegate.isShutdown()).isTrue();
+ assertThat(delegate.shutdownWithPoolMonitorHeld.get()).isFalse();
+ }
+
+ @Test
+ public void rejectedDrainScheduleShutsDelegatesDownOutsidePoolMonitor() {
+ AtomicReference poolReference = new AtomicReference<>();
+ List delegates = new CopyOnWriteArrayList<>();
+ pool =
+ newPool(
+ Duration.ofMinutes(1),
+ () -> {
+ LockCheckingManagedChannel delegate =
+ new LockCheckingManagedChannel(executor, poolReference);
+ delegates.add(delegate);
+ return delegate;
+ });
+ poolReference.set(pool);
+ ScheduledThreadPoolExecutor rejectingScheduler = new ScheduledThreadPoolExecutor(1);
+ rejectingScheduler.shutdown();
+ pool.setDrainSchedulerForTest(rejectingScheduler);
+
+ scaleDown();
+
+ int shutdownCount = 0;
+ for (LockCheckingManagedChannel delegate : delegates) {
+ if (delegate.isShutdown()) {
+ shutdownCount++;
+ assertThat(delegate.shutdownWithPoolMonitorHeld.get()).isFalse();
+ }
+ }
+ assertThat(shutdownCount).isEqualTo(2);
+ assertThat(pool.removedChannelRefs).isEmpty();
+ }
+
+ @Test
+ public void staleDrainTaskCannotActOnBehalfOfNewerTask() {
+ ScheduledExecutorService scheduler = mock(ScheduledExecutorService.class);
+ List tasks = new ArrayList<>();
+ when(scheduler.schedule(any(Runnable.class), anyLong(), any(TimeUnit.class)))
+ .thenAnswer(
+ invocation -> {
+ tasks.add(invocation.getArgument(0));
+ return mock(ScheduledFuture.class);
+ });
+ pool = newPool(Duration.ZERO);
+ pool.setDrainSchedulerForTest(scheduler);
+ scaleDown();
+ assertThat(tasks).hasSize(2);
+ ChannelRef draining = pool.removedChannelRefs.iterator().next();
+
+ pool.scheduleDrain(draining);
+ assertThat(tasks).hasSize(3);
+ assertThat(pool.drainTaskCountForTest()).isEqualTo(2);
+
+ tasks.get(0).run();
+ tasks.get(1).run();
+
+ assertThat(draining.getChannel().isShutdown()).isFalse();
+ assertThat(pool.removedChannelRefs).contains(draining);
+ assertThat(pool.drainTaskCountForTest()).isEqualTo(1);
+
+ tasks.get(2).run();
+
+ assertThat(draining.getChannel().isShutdown()).isTrue();
+ assertThat(pool.removedChannelRefs).doesNotContain(draining);
+ assertThat(pool.drainTaskCountForTest()).isEqualTo(0);
+ }
+
+ @Test
+ public void inlineDrainTaskFinishesAfterItsFutureIsPublished() {
+ ScheduledExecutorService scheduler = mock(ScheduledExecutorService.class);
+ when(scheduler.schedule(any(Runnable.class), anyLong(), any(TimeUnit.class)))
+ .thenAnswer(
+ invocation -> {
+ invocation.getArgument(0).run();
+ return mock(ScheduledFuture.class);
+ });
+ pool = newPool(Duration.ZERO);
+ pool.setDrainSchedulerForTest(scheduler);
+
+ scaleDown();
+
+ assertThat(pool.removedChannelRefs).isEmpty();
+ assertThat(pool.drainTaskCountForTest()).isEqualTo(0);
+ }
+
+ @Test
+ public void affinityKeyRebindsAwayFromDrainingChannel() {
+ pool = newPool(Duration.ofMinutes(1));
+ ChannelRef victim = pool.channelRefs.get(0);
+ pool.bind(victim, Collections.singletonList("session"));
+ for (int i = 1; i < pool.channelRefs.size(); i++) {
+ pool.channelRefs.get(i).setActiveStreamsForTest(1);
+ }
+
+ scaleDown();
+
+ assertThat(pool.getChannelRef("session")).isNotSameInstanceAs(victim);
+ assertThat(pool.affinityKeyToChannelRef.get("session").isActive()).isTrue();
+ }
+
+ @Test
+ public void powerOfTwoUsesCandidateRetryBoundBeforeFullScan() {
+ pool = newPool(Duration.ofMinutes(1));
+ List candidates = new CopyOnWriteArrayList<>(pool.channelRefs);
+ int initialSize = candidates.size();
+ ChannelRef firstInactive = candidates.get(0);
+ ChannelRef secondInactive = candidates.get(1);
+ ChannelRef fallback = candidates.get(2);
+ firstInactive.deactivateForTest();
+ secondInactive.deactivateForTest();
+ AtomicBoolean shrunk = new AtomicBoolean();
+ AtomicInteger nextSample = new AtomicInteger();
+ pool.setCandidateIndexPickerForTest(
+ bound -> {
+ if (shrunk.compareAndSet(false, true)) {
+ candidates.remove(firstInactive);
+ candidates.remove(secondInactive);
+ }
+ nextSample.incrementAndGet();
+ return 0;
+ });
+
+ assertThat(pool.pickFromCandidates(candidates)).isSameInstanceAs(fallback);
+ assertThat(nextSample.get()).isEqualTo(4 * initialSize);
+ }
+
+ @Test
+ public void pickerRetriesWhenChannelDeactivatesBeforeValidation() {
+ pool = newPool(Duration.ofMinutes(1));
+ AtomicReference deactivated = new AtomicReference<>();
+ pool.setPickerValidationHookForTest(
+ candidate -> {
+ deactivated.set(candidate);
+ candidate.deactivateForTest();
+ pool.channelRefs.remove(candidate);
+ pool.removedChannelRefs.add(candidate);
+ });
+
+ ChannelRef picked = pool.getChannelRef(null);
+ picked.activeStreamsCountIncr();
+
+ assertThat(picked).isNotSameInstanceAs(deactivated.get());
+ assertThat(pool.channelRefs).doesNotContain(deactivated.get());
+ assertThat(deactivated.get().getActiveStreamsCount()).isEqualTo(0);
+ assertThat(picked.getActiveStreamsCount()).isEqualTo(1);
+ }
+
+ private void scaleDown() {
+ pool.checkScaleDown();
+ pool.checkScaleDown();
+ pool.checkScaleDown();
+ }
+
+ private GcpManagedChannel newPool(Duration drainIdleGrace) {
+ return newPool(drainIdleGrace, () -> new GcpManagedChannelTest.FakeManagedChannel(executor));
+ }
+
+ private GcpManagedChannel newPool(
+ Duration drainIdleGrace, Supplier channelFactory) {
+ GcpChannelPoolOptions options =
+ GcpChannelPoolOptions.newBuilder()
+ .setInitSize(4)
+ .setMinSize(2)
+ .setMaxSize(4)
+ .setDynamicScaling(10, 20, Duration.ofMinutes(1))
+ .setScaleDownConsecutiveLowLoadChecks(3)
+ .setMaxScaleDownChannels(2)
+ .setDrainIdleGrace(drainIdleGrace)
+ .build();
+ return (GcpManagedChannel)
+ GcpManagedChannelBuilder.forDelegateBuilder(
+ new GcpManagedChannelTest.FakeManagedChannelBuilder(channelFactory))
+ .withOptions(
+ GcpManagedChannelOptions.newBuilder().withChannelPoolOptions(options).build())
+ .build();
+ }
+
+ private static final class LockCheckingManagedChannel
+ extends GcpManagedChannelTest.FakeManagedChannel {
+ private final AtomicReference poolReference;
+ private final AtomicBoolean shutdownWithPoolMonitorHeld = new AtomicBoolean();
+
+ private LockCheckingManagedChannel(
+ ExecutorService executor, AtomicReference poolReference) {
+ super(executor);
+ this.poolReference = poolReference;
+ }
+
+ @Override
+ public ManagedChannel shutdown() {
+ GcpManagedChannel currentPool = poolReference.get();
+ shutdownWithPoolMonitorHeld.set(currentPool != null && Thread.holdsLock(currentPool));
+ return super.shutdown();
+ }
+ }
+}
diff --git a/grpc-gcp-java/src/test/java/com/google/cloud/grpc/GcpManagedChannelErrorPenaltyTest.java b/grpc-gcp-java/src/test/java/com/google/cloud/grpc/GcpManagedChannelErrorPenaltyTest.java
new file mode 100644
index 000000000000..6f8867b6f4f5
--- /dev/null
+++ b/grpc-gcp-java/src/test/java/com/google/cloud/grpc/GcpManagedChannelErrorPenaltyTest.java
@@ -0,0 +1,569 @@
+/*
+ * Copyright 2026 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.grpc;
+
+import static com.google.common.truth.Truth.assertThat;
+import static org.awaitility.Awaitility.await;
+import static org.junit.Assert.assertThrows;
+
+import com.google.cloud.grpc.GcpManagedChannel.ChannelRef;
+import com.google.cloud.grpc.GcpManagedChannelOptions.GcpChannelPoolOptions;
+import com.google.common.util.concurrent.MoreExecutors;
+import io.grpc.Status;
+import java.lang.reflect.Field;
+import java.lang.reflect.Method;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+import org.junit.After;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+@RunWith(JUnit4.class)
+public final class GcpManagedChannelErrorPenaltyTest {
+ private final ExecutorService executor = MoreExecutors.newDirectExecutorService();
+ private GcpManagedChannel pool;
+
+ @After
+ public void tearDown() {
+ if (pool != null) {
+ pool.shutdownNow();
+ }
+ executor.shutdownNow();
+ }
+
+ @Test
+ public void retryableErrorsAddBoundedDecayingPickerLoad() {
+ AtomicLong clock = new AtomicLong(1);
+ pool = newPool(2, 2, 2, 20, 10, Duration.ofSeconds(10));
+ pool.setNanoClock(clock::get);
+ ChannelRef penalized = pool.channelRefs.get(0);
+ ChannelRef healthy = pool.channelRefs.get(1);
+ healthy.setActiveStreamsForTest(12);
+
+ completeWithError(penalized, clock.get(), Status.UNAVAILABLE);
+ completeWithError(penalized, clock.get(), Status.RESOURCE_EXHAUSTED);
+ assertThat(penalized.currentErrorPenalty()).isEqualTo(20);
+ assertThat(totalErrorPenaltyLoad()).isEqualTo(20);
+
+ AtomicInteger sample = new AtomicInteger();
+ pool.setCandidateIndexPickerForTest(ignored -> sample.getAndIncrement() % 2);
+ assertThat(pool.pickFromCandidates(pool.channelRefs)).isSameInstanceAs(healthy);
+
+ int previousPenalty = penalized.currentErrorPenalty();
+ clock.addAndGet(Duration.ofSeconds(2).toNanos());
+ assertThat(penalized.currentErrorPenalty()).isEqualTo(16);
+ assertThat(penalized.currentErrorPenalty()).isAtMost(previousPenalty);
+ previousPenalty = penalized.currentErrorPenalty();
+
+ clock.addAndGet(Duration.ofSeconds(3).toNanos());
+ assertThat(penalized.currentErrorPenalty()).isEqualTo(10);
+ assertThat(penalized.currentErrorPenalty()).isAtMost(previousPenalty);
+ assertThat(totalErrorPenaltyLoad()).isEqualTo(20);
+
+ sample.set(0);
+ assertThat(pool.pickFromCandidates(pool.channelRefs)).isSameInstanceAs(penalized);
+ pool.setCandidateIndexPickerForTest(ignored -> 1);
+ assertThat(pool.pickFromCandidates(pool.channelRefs)).isSameInstanceAs(healthy);
+
+ previousPenalty = penalized.currentErrorPenalty();
+ clock.addAndGet(Duration.ofSeconds(4).toNanos());
+ assertThat(penalized.currentErrorPenalty()).isEqualTo(2);
+ assertThat(penalized.currentErrorPenalty()).isAtMost(previousPenalty);
+ assertThat(totalErrorPenaltyLoad()).isEqualTo(20);
+
+ clock.addAndGet(Duration.ofSeconds(1).toNanos());
+ assertThat(penalized.currentErrorPenalty()).isEqualTo(0);
+ assertThat(totalErrorPenaltyLoad()).isEqualTo(0);
+ }
+
+ @Test
+ public void linearDecayHandlesMultiplicationOverflow() {
+ AtomicLong clock = new AtomicLong(1);
+ pool = newPool(2, Integer.MAX_VALUE, Integer.MAX_VALUE, Duration.ofNanos(Long.MAX_VALUE));
+ pool.setNanoClock(clock::get);
+ ChannelRef channel = pool.channelRefs.get(0);
+ completeWithError(channel, clock.get(), Status.UNAVAILABLE);
+
+ clock.addAndGet(Long.MAX_VALUE / 2);
+
+ assertThat(channel.currentErrorPenalty()).isEqualTo(1_073_741_824);
+ assertThat(totalErrorPenaltyLoad()).isEqualTo(Integer.MAX_VALUE);
+
+ clock.addAndGet(Long.MAX_VALUE - Long.MAX_VALUE / 2 - 1);
+ assertThat(channel.currentErrorPenalty()).isEqualTo(1);
+ assertThat(totalErrorPenaltyLoad()).isEqualTo(Integer.MAX_VALUE);
+
+ clock.incrementAndGet();
+ assertThat(channel.currentErrorPenalty()).isEqualTo(0);
+ assertThat(totalErrorPenaltyLoad()).isEqualTo(0);
+ }
+
+ @Test
+ public void retryableErrorDuringDecayRestoresAccumulatedPenalty() {
+ AtomicLong clock = new AtomicLong(1);
+ pool = newPool(2, 10, 5, Duration.ofSeconds(10));
+ pool.setNanoClock(clock::get);
+ ChannelRef channel = pool.channelRefs.get(0);
+ completeWithError(channel, clock.get(), Status.UNAVAILABLE);
+ clock.addAndGet(Duration.ofSeconds(5).toNanos());
+ assertThat(channel.currentErrorPenalty()).isEqualTo(3);
+
+ completeWithError(channel, clock.get(), Status.UNAVAILABLE);
+
+ assertThat(channel.currentErrorPenalty()).isEqualTo(10);
+ assertThat(totalErrorPenaltyLoad()).isEqualTo(10);
+ }
+
+ @Test
+ public void penaltyStepSaturatesWithoutIntegerOverflow() {
+ AtomicLong clock = new AtomicLong(1);
+ pool = newPool(2, Integer.MAX_VALUE, Integer.MAX_VALUE, Duration.ofSeconds(5));
+ pool.setNanoClock(clock::get);
+ ChannelRef channel = pool.channelRefs.get(0);
+
+ completeWithError(channel, clock.get(), Status.UNAVAILABLE);
+ completeWithError(channel, clock.get(), Status.RESOURCE_EXHAUSTED);
+
+ assertThat(channel.currentErrorPenalty()).isEqualTo(Integer.MAX_VALUE);
+ assertThat(totalErrorPenaltyLoad()).isEqualTo(Integer.MAX_VALUE);
+ }
+
+ @Test
+ public void pickerLoadSaturatesWithoutIntegerOverflow() {
+ AtomicLong clock = new AtomicLong(1);
+ pool = newPool(2, Integer.MAX_VALUE, Integer.MAX_VALUE, Duration.ofSeconds(5));
+ pool.setNanoClock(clock::get);
+ ChannelRef channel = pool.channelRefs.get(0);
+ completeWithError(channel, clock.get(), Status.UNAVAILABLE);
+ channel.setActiveStreamsForTest(1);
+
+ assertThat(channel.getPickerLoad()).isEqualTo(Integer.MAX_VALUE);
+ }
+
+ @Test
+ public void aggregatePenaltyUsesLongWithoutOverflow() {
+ AtomicLong clock = new AtomicLong(1);
+ pool = newPool(2, Integer.MAX_VALUE, Integer.MAX_VALUE, Duration.ofSeconds(5));
+ pool.setNanoClock(clock::get);
+
+ completeWithError(pool.channelRefs.get(0), clock.get(), Status.UNAVAILABLE);
+ completeWithError(pool.channelRefs.get(1), clock.get(), Status.UNAVAILABLE);
+
+ assertThat(totalErrorPenaltyLoad()).isEqualTo(2L * Integer.MAX_VALUE);
+ }
+
+ @Test
+ public void expiredPenaltyReplacementUsesOneNetAggregateDelta() {
+ AtomicLong clock = new AtomicLong(1);
+ pool = newPool(2, 10, 5, Duration.ofSeconds(5));
+ pool.setNanoClock(clock::get);
+ ChannelRef channel = pool.channelRefs.get(0);
+ completeWithError(channel, clock.get(), Status.UNAVAILABLE);
+ completeWithError(channel, clock.get(), Status.UNAVAILABLE);
+ assertThat(totalErrorPenaltyLoad()).isEqualTo(10);
+
+ clock.addAndGet(Duration.ofSeconds(6).toNanos());
+ completeWithError(channel, clock.get(), Status.UNAVAILABLE);
+
+ assertThat(channel.currentErrorPenalty()).isEqualTo(5);
+ assertThat(totalErrorPenaltyLoad()).isEqualTo(5);
+ }
+
+ @Test
+ public void concurrentExpirySubtractsAggregateExactlyOnce() throws Exception {
+ AtomicLong clock = new AtomicLong(1);
+ pool = newPool();
+ pool.setNanoClock(clock::get);
+ ChannelRef channel = pool.channelRefs.get(0);
+ completeWithError(channel, clock.get(), Status.UNAVAILABLE);
+ clock.addAndGet(Duration.ofSeconds(6).toNanos());
+ CountDownLatch start = new CountDownLatch(1);
+ ExecutorService readers = Executors.newFixedThreadPool(8);
+ try {
+ List> results = new ArrayList<>();
+ for (int i = 0; i < 100; i++) {
+ results.add(
+ readers.submit(
+ () -> {
+ start.await();
+ return channel.currentErrorPenalty();
+ }));
+ }
+ start.countDown();
+ for (Future result : results) {
+ assertThat(result.get()).isEqualTo(0);
+ }
+ assertThat(totalErrorPenaltyLoad()).isEqualTo(0);
+ } finally {
+ start.countDown();
+ readers.shutdownNow();
+ }
+ }
+
+ @Test
+ public void nonRetryableErrorsAddNoPenalty() {
+ AtomicLong clock = new AtomicLong(1);
+ pool = newPool();
+ pool.setNanoClock(clock::get);
+ ChannelRef channel = pool.channelRefs.get(0);
+
+ completeWithError(channel, clock.get(), Status.OK);
+ completeWithError(channel, clock.get(), Status.INTERNAL);
+ completeWithError(channel, clock.get(), Status.DEADLINE_EXCEEDED);
+ completeWithError(channel, clock.get(), Status.CANCELLED);
+
+ assertThat(channel.currentErrorPenalty()).isEqualTo(0);
+ assertThat(totalErrorPenaltyLoad()).isEqualTo(0);
+ }
+
+ @Test
+ public void negativeAndZeroDurationRejected() {
+ assertThrows(
+ IllegalArgumentException.class,
+ () -> GcpChannelPoolOptions.newBuilder().setErrorPenaltyDuration(Duration.ofSeconds(-1)));
+ assertThrows(
+ IllegalArgumentException.class,
+ () -> GcpChannelPoolOptions.newBuilder().setErrorPenaltyDuration(Duration.ZERO));
+ }
+
+ @Test
+ public void penaltyContributesToScaleUpSignal() {
+ AtomicLong clock = new AtomicLong(1);
+ pool = newPool(2, 2, 3, 2, 2, Duration.ofSeconds(5));
+ pool.setNanoClock(clock::get);
+ ChannelRef channel = pool.channelRefs.get(0);
+ channel.activeStreamsCountIncr();
+ channel.activeStreamsCountIncr();
+
+ channel.activeStreamsCountDecr(clock.get(), Status.UNAVAILABLE, false);
+
+ await().atMost(Duration.ofSeconds(5)).until(() -> pool.getNumberOfChannels() == 3);
+ }
+
+ @Test
+ public void fullyPenalizedIdlePoolTriggersOneBoundedScaleUp() {
+ AtomicLong clock = new AtomicLong(1);
+ pool = newPool(2, 2, 6, 2, 2, Duration.ofMinutes(1));
+ pool.setNanoClock(clock::get);
+ ChannelRef first = pool.channelRefs.get(0);
+ ChannelRef second = pool.channelRefs.get(1);
+
+ completeWithError(first, clock.get(), Status.UNAVAILABLE);
+ completeWithError(second, clock.get(), Status.UNAVAILABLE);
+
+ await().atMost(Duration.ofSeconds(5)).until(() -> pool.getNumberOfChannels() == 4);
+ assertThat(first.getActiveStreamsCount()).isEqualTo(0);
+ assertThat(second.getActiveStreamsCount()).isEqualTo(0);
+ assertThat(totalErrorPenaltyLoad()).isEqualTo(4);
+
+ clock.addAndGet(Duration.ofSeconds(30).toNanos());
+ assertThat(first.currentErrorPenalty()).isEqualTo(1);
+ assertThat(second.currentErrorPenalty()).isEqualTo(1);
+ first.activeStreamsCountIncr();
+ first.activeStreamsCountDecr(clock.get(), Status.OK, false);
+ await()
+ .during(Duration.ofMillis(200))
+ .atMost(Duration.ofSeconds(2))
+ .until(() -> pool.getNumberOfChannels() == 4);
+
+ clock.addAndGet(Duration.ofSeconds(30).toNanos());
+ assertThat(first.currentErrorPenalty()).isEqualTo(0);
+ assertThat(second.currentErrorPenalty()).isEqualTo(0);
+ assertThat(totalErrorPenaltyLoad()).isEqualTo(0);
+ second.activeStreamsCountIncr();
+ second.activeStreamsCountDecr(clock.get(), Status.OK, false);
+ await()
+ .during(Duration.ofMillis(200))
+ .atMost(Duration.ofSeconds(2))
+ .until(() -> pool.getNumberOfChannels() == 4);
+ }
+
+ @Test
+ public void scaleDownClearsRemovedChannelPenalty() {
+ AtomicLong clock = new AtomicLong(1);
+ pool = newPool(1, 10, 5, Duration.ofSeconds(5));
+ pool.setNanoClock(clock::get);
+ ChannelRef removed = pool.channelRefs.get(0);
+ completeWithError(removed, clock.get(), Status.UNAVAILABLE);
+ assertThat(totalErrorPenaltyLoad()).isEqualTo(5);
+
+ pool.checkScaleDown();
+ pool.checkScaleDown();
+ pool.checkScaleDown();
+
+ assertThat(removed.isActive()).isFalse();
+ assertThat(removed.currentErrorPenalty()).isEqualTo(0);
+ assertThat(totalErrorPenaltyLoad()).isEqualTo(0);
+ }
+
+ @Test
+ public void rpcCompletionAfterScaleDownDoesNotLeakPenalty() throws Exception {
+ AtomicLong clock = new AtomicLong(1);
+ pool = newPool(1, 10, 5, Duration.ofSeconds(5));
+ pool.setNanoClock(clock::get);
+ ChannelRef victim = pool.channelRefs.get(0);
+ victim.activeStreamsCountIncr();
+ CountDownLatch completionStarted = new CountDownLatch(1);
+ AtomicReference completionThread = new AtomicReference<>();
+ ExecutorService completionExecutor =
+ Executors.newSingleThreadExecutor(
+ command -> {
+ Thread thread = new Thread(command, "penalty-completion");
+ completionThread.set(thread);
+ return thread;
+ });
+ try {
+ Future completion;
+ synchronized (victim) {
+ completion =
+ completionExecutor.submit(
+ () -> {
+ completionStarted.countDown();
+ victim.activeStreamsCountDecr(clock.get(), Status.UNAVAILABLE, false);
+ });
+ await().atMost(Duration.ofSeconds(5)).until(() -> completionStarted.getCount() == 0);
+ await()
+ .atMost(Duration.ofSeconds(5))
+ .until(
+ () ->
+ completionThread.get() != null
+ && completionThread.get().getState() == Thread.State.BLOCKED);
+
+ pool.checkScaleDown();
+ pool.checkScaleDown();
+ pool.checkScaleDown();
+
+ assertThat(victim.isActive()).isFalse();
+ assertThat(pool.channelRefs).doesNotContain(victim);
+ assertThat(totalErrorPenaltyLoad()).isEqualTo(0);
+ }
+
+ completion.get(5, TimeUnit.SECONDS);
+ assertThat(victim.currentErrorPenalty()).isEqualTo(0);
+ assertThat(totalErrorPenaltyLoad()).isEqualTo(0);
+
+ clock.addAndGet(Duration.ofHours(1).toNanos());
+ pool.checkScaleDown();
+ assertThat(totalErrorPenaltyLoad()).isEqualTo(0);
+ } finally {
+ completionExecutor.shutdownNow();
+ }
+ }
+
+ @Test
+ public void leastLoadedActiveChannelReadsClockOncePerScan() throws Exception {
+ AtomicLong clockReads = new AtomicLong();
+ pool = newPool(4, 2, 4, 10, 5, Duration.ofSeconds(5));
+ pool.setNanoClock(
+ () -> {
+ clockReads.incrementAndGet();
+ return 1_000;
+ });
+ ChannelRef penalized = pool.channelRefs.get(0);
+ completeWithError(penalized, 1_000, Status.UNAVAILABLE);
+ for (ChannelRef channelRef : pool.channelRefs) {
+ if (channelRef != penalized) {
+ channelRef.setActiveStreamsForTest(10);
+ }
+ }
+ clockReads.set(0);
+ Method leastLoaded =
+ GcpManagedChannel.class.getDeclaredMethod("leastLoadedActiveChannel", List.class);
+ leastLoaded.setAccessible(true);
+
+ leastLoaded.invoke(pool, pool.channelRefs);
+
+ assertThat(clockReads.get()).isEqualTo(1);
+ }
+
+ @Test
+ public void powerOfTwoReadsClockOncePerPick() {
+ AtomicLong clockReads = new AtomicLong();
+ pool = newPool();
+ pool.setNanoClock(
+ () -> {
+ clockReads.incrementAndGet();
+ return 1_000;
+ });
+ completeWithError(pool.channelRefs.get(0), 1_000, Status.UNAVAILABLE);
+ completeWithError(pool.channelRefs.get(1), 1_000, Status.UNAVAILABLE);
+ AtomicInteger sample = new AtomicInteger();
+ pool.setCandidateIndexPickerForTest(ignored -> sample.getAndIncrement() % 2);
+ clockReads.set(0);
+
+ pool.pickFromCandidates(pool.channelRefs);
+
+ assertThat(clockReads.get()).isEqualTo(1);
+ }
+
+ @Test
+ public void pickerAndScaleUpReadsDoNotClearPenaltyUnderPoolMonitor() throws Exception {
+ AtomicLong clock = new AtomicLong(1);
+ pool = newPool(2, 2, 3, 10, 5, Duration.ofSeconds(5));
+ pool.setNanoClock(clock::get);
+ ChannelRef channel = pool.channelRefs.get(0);
+ ChannelRef other = pool.channelRefs.get(1);
+ completeWithError(channel, clock.get(), Status.UNAVAILABLE);
+ clock.addAndGet(Duration.ofSeconds(6).toNanos());
+ channel.deactivateForTest();
+ other.deactivateForTest();
+ Field active = ChannelRef.class.getDeclaredField("active");
+ active.setAccessible(true);
+ AtomicInteger sample = new AtomicInteger();
+ pool.setCandidateIndexPickerForTest(
+ ignored -> {
+ int index = sample.getAndIncrement();
+ if (index == 0) {
+ try {
+ active.setBoolean(channel, true);
+ active.setBoolean(other, true);
+ } catch (IllegalAccessException failure) {
+ throw new AssertionError(failure);
+ }
+ }
+ return index % 2;
+ });
+ Method pickerLoad =
+ GcpManagedChannel.class.getDeclaredMethod("pickerLoad", List.class, long.class);
+ pickerLoad.setAccessible(true);
+
+ synchronized (pool) {
+ assertThat((long) pickerLoad.invoke(pool, pool.channelRefs, clock.get())).isEqualTo(0);
+ assertThat(totalErrorPenaltyLoad()).isEqualTo(5);
+ pool.getChannelRefRoundRobin();
+ assertThat(totalErrorPenaltyLoad()).isEqualTo(5);
+ channel.activeStreamsCountIncr();
+ assertThat(totalErrorPenaltyLoad()).isEqualTo(5);
+ channel.activeStreamsCountDecr(clock.get(), Status.OK, false);
+ }
+
+ assertThat(channel.currentErrorPenalty()).isEqualTo(0);
+ assertThat(totalErrorPenaltyLoad()).isEqualTo(0);
+ }
+
+ @Test
+ public void clearedExpirySentinelWinsWithNegativeClock() throws Exception {
+ pool = newPool();
+ pool.setNanoClock(() -> -1);
+ ChannelRef channel = pool.channelRefs.get(0);
+ Field penaltyLoad = ChannelRef.class.getDeclaredField("errorPenaltyLoad");
+ penaltyLoad.setAccessible(true);
+ penaltyLoad.setInt(channel, 5);
+ Field expiresAt = ChannelRef.class.getDeclaredField("errorPenaltyExpiresAtNanos");
+ expiresAt.setAccessible(true);
+ expiresAt.setLong(channel, 0);
+
+ assertThat(channel.currentErrorPenalty()).isEqualTo(0);
+ assertThat(channel.getPickerLoad()).isEqualTo(0);
+ assertThat(totalErrorPenaltyLoad()).isEqualTo(0);
+ }
+
+ @Test
+ public void appliedPenaltyAvoidsClearedExpirySentinel() {
+ long durationNanos = Duration.ofSeconds(5).toNanos();
+ AtomicLong clock = new AtomicLong(-durationNanos);
+ pool = newPool();
+ pool.setNanoClock(clock::get);
+ ChannelRef channel = pool.channelRefs.get(0);
+
+ completeWithError(channel, clock.get(), Status.UNAVAILABLE);
+
+ assertThat(channel.currentErrorPenalty()).isEqualTo(5);
+ assertThat(channel.getPickerLoad()).isEqualTo(5);
+ assertThat(totalErrorPenaltyLoad()).isEqualTo(5);
+
+ clock.set(0);
+ assertThat(channel.currentErrorPenalty()).isEqualTo(1);
+ assertThat(totalErrorPenaltyLoad()).isEqualTo(5);
+
+ clock.set(1);
+ assertThat(channel.currentErrorPenalty()).isEqualTo(0);
+ assertThat(totalErrorPenaltyLoad()).isEqualTo(0);
+ }
+
+ @Test
+ public void zeroPenaltyStepDisablesPenalties() {
+ AtomicLong clock = new AtomicLong(1);
+ pool = newPool(2, 10, 0, Duration.ofSeconds(5));
+ pool.setNanoClock(clock::get);
+ ChannelRef channel = pool.channelRefs.get(0);
+
+ completeWithError(channel, clock.get(), Status.UNAVAILABLE);
+
+ assertThat(channel.currentErrorPenalty()).isEqualTo(0);
+ assertThat(totalErrorPenaltyLoad()).isEqualTo(0);
+ }
+
+ private GcpManagedChannel newPool() {
+ return newPool(2, 10, 5, Duration.ofSeconds(5));
+ }
+
+ private GcpManagedChannel newPool(
+ int minimum, int maximumRpc, int penaltyStep, Duration penaltyDuration) {
+ return newPool(2, minimum, 2, maximumRpc, penaltyStep, penaltyDuration);
+ }
+
+ private GcpManagedChannel newPool(
+ int initial,
+ int minimum,
+ int maximum,
+ int maximumRpc,
+ int penaltyStep,
+ Duration penaltyDuration) {
+ GcpChannelPoolOptions options =
+ GcpChannelPoolOptions.newBuilder()
+ .setInitSize(initial)
+ .setMinSize(minimum)
+ .setMaxSize(maximum)
+ .setDynamicScaling(1, maximumRpc, Duration.ofMinutes(1))
+ .setErrorPenaltyStep(penaltyStep)
+ .setErrorPenaltyDuration(penaltyDuration)
+ .build();
+ return (GcpManagedChannel)
+ GcpManagedChannelBuilder.forDelegateBuilder(
+ new GcpManagedChannelTest.FakeManagedChannelBuilder(
+ () -> new GcpManagedChannelTest.FakeManagedChannel(executor)))
+ .withOptions(
+ GcpManagedChannelOptions.newBuilder().withChannelPoolOptions(options).build())
+ .build();
+ }
+
+ private static void completeWithError(ChannelRef channel, long now, Status status) {
+ channel.activeStreamsCountIncr();
+ channel.activeStreamsCountDecr(now, status, false);
+ }
+
+ private long totalErrorPenaltyLoad() {
+ try {
+ Field field = GcpManagedChannel.class.getDeclaredField("totalErrorPenaltyLoad");
+ field.setAccessible(true);
+ return ((AtomicLong) field.get(pool)).get();
+ } catch (ReflectiveOperationException failure) {
+ throw new AssertionError(failure);
+ }
+ }
+}
diff --git a/grpc-gcp-java/src/test/java/com/google/cloud/grpc/GcpManagedChannelHotChannelReproducerTest.java b/grpc-gcp-java/src/test/java/com/google/cloud/grpc/GcpManagedChannelHotChannelReproducerTest.java
new file mode 100644
index 000000000000..79eae75919fb
--- /dev/null
+++ b/grpc-gcp-java/src/test/java/com/google/cloud/grpc/GcpManagedChannelHotChannelReproducerTest.java
@@ -0,0 +1,1141 @@
+/*
+ * Copyright 2026 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.grpc;
+
+import static com.google.common.truth.Truth.assertThat;
+import static com.google.common.truth.Truth.assertWithMessage;
+import static org.awaitility.Awaitility.await;
+import static org.junit.Assume.assumeTrue;
+
+import com.google.cloud.grpc.GcpManagedChannel.ChannelAffinityRef;
+import com.google.cloud.grpc.GcpManagedChannel.ChannelRef;
+import com.google.cloud.grpc.GcpManagedChannelOptions.GcpChannelPoolOptions;
+import com.google.cloud.grpc.proto.AffinityConfig;
+import com.google.cloud.grpc.proto.ApiConfig;
+import com.google.cloud.grpc.proto.MethodConfig;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.spanner.v1.CreateSessionRequest;
+import com.google.spanner.v1.ExecuteSqlRequest;
+import com.google.spanner.v1.ResultSet;
+import com.google.spanner.v1.Session;
+import io.grpc.CallOptions;
+import io.grpc.ClientCall;
+import io.grpc.ClientInterceptor;
+import io.grpc.CompressorRegistry;
+import io.grpc.ConnectivityState;
+import io.grpc.DecompressorRegistry;
+import io.grpc.ManagedChannel;
+import io.grpc.ManagedChannelBuilder;
+import io.grpc.Metadata;
+import io.grpc.MethodDescriptor;
+import io.grpc.NameResolver.Factory;
+import io.grpc.Status;
+import io.grpc.protobuf.ProtoUtils;
+import io.grpc.stub.ClientCalls;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Executor;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/**
+ * Dynamic-pool per-channel request-skew tests.
+ *
+ * Deterministic regressions always run. The load-shaped reproducer is opt-in with {@code
+ * -Dhotchannel.load=true}; configure its seeds with {@code -Dhotchannel.seeds=1103,7,42,99,123}.
+ */
+@RunWith(JUnit4.class)
+public final class GcpManagedChannelHotChannelReproducerTest {
+ private static final int MIN_SIZE = 2;
+ private static final int INITIAL_SIZE = 4;
+ private static final int MAX_SIZE = 48;
+ private static final int MIN_RPC_PER_CHANNEL = 15;
+ private static final int MAX_RPC_PER_CHANNEL = 25;
+ private static final int DEFAULT_CALLERS = 300;
+ private static final int SESSION_COUNT = 100;
+ private static final Duration TEST_SCALE_DOWN_INTERVAL = Duration.ofMillis(120);
+ private static final CallOptions.Key TEST_LATENCY_MILLIS =
+ CallOptions.Key.create("hotchannel-test-latency-millis");
+ private static final MethodDescriptor CREATE_SESSION_METHOD =
+ MethodDescriptor.newBuilder()
+ .setType(MethodDescriptor.MethodType.UNARY)
+ .setFullMethodName("google.spanner.v1.Spanner/CreateSession")
+ .setRequestMarshaller(ProtoUtils.marshaller(CreateSessionRequest.getDefaultInstance()))
+ .setResponseMarshaller(ProtoUtils.marshaller(Session.getDefaultInstance()))
+ .build();
+ private static final MethodDescriptor EXECUTE_SQL_METHOD =
+ MethodDescriptor.newBuilder()
+ .setType(MethodDescriptor.MethodType.UNARY)
+ .setFullMethodName("google.spanner.v1.Spanner/ExecuteSql")
+ .setRequestMarshaller(ProtoUtils.marshaller(ExecuteSqlRequest.getDefaultInstance()))
+ .setResponseMarshaller(ProtoUtils.marshaller(ResultSet.getDefaultInstance()))
+ .build();
+ private static final CallOptions.Key CHANNEL_AFFINITY_REF_KEY =
+ GcpManagedChannel.CHANNEL_AFFINITY_REF_KEY;
+
+ @Test
+ public void dynamicPoolDoesNotDevelopOneHotChannel() throws Exception {
+ assumeTrue("load-shaped reproducer disabled", Boolean.getBoolean("hotchannel.load"));
+ Variant variant =
+ Variant.valueOf(
+ System.getProperty("hotchannel.variant", "REF_6120").toUpperCase(Locale.ROOT));
+ LoadShape loadShape =
+ LoadShape.valueOf(
+ System.getProperty("hotchannel.loadShape", "BURSTY").toUpperCase(Locale.ROOT));
+ String[] seeds = System.getProperty("hotchannel.seeds", "1103").split(",");
+ boolean observeOnly = Boolean.getBoolean("hotchannel.observeOnly");
+
+ for (String seedValue : seeds) {
+ long seed = Long.parseLong(seedValue.trim());
+ RunResult result = runScenario(seed, variant, loadShape);
+ System.out.println(result.format());
+ if (!observeOnly) {
+ assertWithMessage("one-hot-channel skew: %s", result.format())
+ .that(result.hotToMedian)
+ .isLessThan(3.0);
+ }
+ }
+ }
+
+ @Test
+ public void warmChannelDoesNotBiasPowerOfTwoTies() throws Exception {
+ ScheduledExecutorService responses = Executors.newSingleThreadScheduledExecutor();
+ GcpManagedChannel pool = null;
+ try {
+ pool = fixedPool(new RecordingChannelBuilder(responses, 1), 2, 2, false);
+ ChannelRef warm = pool.channelRefs.get(1);
+ warm.messageReceived();
+
+ int warmPicks = 0;
+ for (int i = 0; i < 200; i++) {
+ if (pool.getChannelRef(null) == warm) {
+ warmPicks++;
+ }
+ }
+
+ assertThat(warmPicks).isGreaterThan(60);
+ assertThat(warmPicks).isLessThan(140);
+ } finally {
+ if (pool != null) {
+ pool.shutdownNow();
+ }
+ responses.shutdownNow();
+ }
+ }
+
+ @Test
+ public void selectedCallsReserveLoadBeforeStart() throws Exception {
+ ScheduledExecutorService responses = Executors.newSingleThreadScheduledExecutor();
+ GcpManagedChannel pool = null;
+ try {
+ RecordingChannelBuilder delegateBuilder = new RecordingChannelBuilder(responses, 2);
+ pool = fixedPool(delegateBuilder, 2, 2, false);
+ pool.channelRefs.get(1).messageReceived();
+
+ for (int i = 0; i < 200; i++) {
+ ChannelAffinityRef affinityRef = new ChannelAffinityRef();
+ pool.newCall(
+ EXECUTE_SQL_METHOD,
+ CallOptions.DEFAULT.withOption(CHANNEL_AFFINITY_REF_KEY, affinityRef));
+ }
+
+ long first = delegateBuilder.stats.get(0).selected.get();
+ long second = delegateBuilder.stats.get(1).selected.get();
+ assertThat(first).isGreaterThan(60);
+ assertThat(second).isGreaterThan(60);
+ assertThat(pool.channelRefs.get(0).getActiveStreamsCount()).isEqualTo((int) first);
+ assertThat(pool.channelRefs.get(1).getActiveStreamsCount()).isEqualTo((int) second);
+ } finally {
+ if (pool != null) {
+ pool.shutdownNow();
+ }
+ responses.shutdownNow();
+ }
+ }
+
+ @Test
+ public void affinityReferencesRedistributeAfterDrainingChannelsShutdown() throws Exception {
+ ScheduledExecutorService responses = Executors.newSingleThreadScheduledExecutor();
+ ExecutorService executor = Executors.newFixedThreadPool(100);
+ GcpManagedChannel pool = null;
+ try {
+ pool = fixedPool(new RecordingChannelBuilder(responses, 3), 2, 4, true);
+ List affinityRefs = new ArrayList<>();
+ List originalIds = new ArrayList<>();
+ for (int i = 0; i < 200; i++) {
+ int channelId = pool.channelRefs.get(i % 4).getId();
+ ChannelAffinityRef affinityRef = new ChannelAffinityRef();
+ affinityRef.setChannelIdForTest(channelId);
+ affinityRefs.add(affinityRef);
+ originalIds.add(channelId);
+ }
+
+ invokeScaleDownCheck(pool, 3);
+ assertThat(pool.channelRefs).hasSize(2);
+ Set removedIds = new HashSet<>();
+ for (ChannelRef removed : pool.removedChannelRefs) {
+ removedIds.add(removed.getId());
+ }
+ List orphaned = new ArrayList<>();
+ for (int i = 0; i < affinityRefs.size(); i++) {
+ if (removedIds.contains(originalIds.get(i))) {
+ orphaned.add(affinityRefs.get(i));
+ }
+ }
+ assertThat(orphaned).hasSize(100);
+ for (int i = 0; i < affinityRefs.size(); i++) {
+ if (removedIds.contains(originalIds.get(i))) {
+ assertThat(pool.getChannelRefByAffinityRef(affinityRefs.get(i)).getId())
+ .isEqualTo(originalIds.get(i));
+ }
+ }
+ for (ChannelRef removed : pool.removedChannelRefs) {
+ removed.getChannel().shutdownNow();
+ }
+
+ ChannelRef first = pool.channelRefs.get(0);
+ ChannelRef second = pool.channelRefs.get(1);
+ second.messageReceived();
+ CountDownLatch start = new CountDownLatch(1);
+ List> resolutions = new ArrayList<>();
+ GcpManagedChannel resolvingPool = pool;
+ for (ChannelAffinityRef affinityRef : orphaned) {
+ resolutions.add(
+ executor.submit(
+ () -> {
+ start.await();
+ return resolvingPool.getChannelRefByAffinityRef(affinityRef);
+ }));
+ }
+ start.countDown();
+ int firstPicks = 0;
+ int secondPicks = 0;
+ for (Future resolution : resolutions) {
+ ChannelRef resolved = resolution.get(5, TimeUnit.SECONDS);
+ if (resolved == first) {
+ firstPicks++;
+ } else if (resolved == second) {
+ secondPicks++;
+ }
+ }
+ assertThat(firstPicks).isGreaterThan(25);
+ assertThat(secondPicks).isGreaterThan(25);
+ } finally {
+ if (pool != null) {
+ pool.shutdownNow();
+ }
+ executor.shutdownNow();
+ responses.shutdownNow();
+ }
+ }
+
+ @Test
+ public void scaleDownMarksAtMostTwoOfFortyEightLiveReferencesPerCheck() throws Exception {
+ ScheduledExecutorService responses = Executors.newSingleThreadScheduledExecutor();
+ GcpManagedChannel pool = null;
+ try {
+ pool = fixedPool(new RecordingChannelBuilder(responses, 4), 2, 48, true);
+ List handles = new ArrayList<>();
+ for (ChannelRef channelRef : pool.channelRefs) {
+ channelRef.activeStreamsCountIncr();
+ ChannelAffinityRef handle = new ChannelAffinityRef();
+ handle.setChannelIdForTest(channelRef.getId());
+ handles.add(handle);
+ }
+ for (ChannelRef channelRef : pool.channelRefs) {
+ channelRef.activeStreamsCountDecr(System.nanoTime(), Status.OK, false);
+ }
+
+ invokeScaleDownCheck(pool, 2);
+ assertThat(pool.channelRefs).hasSize(48);
+ invokeScaleDownCheck(pool, 1);
+ assertThat(pool.channelRefs).hasSize(46);
+ assertThat(pool.removedChannelRefs).hasSize(2);
+ for (ChannelRef removed : pool.removedChannelRefs) {
+ ChannelRef resolved = pool.getChannelRefByAffinityRef(handles.get(removed.getId()));
+ assertThat(resolved).isSameInstanceAs(removed);
+ assertThat(resolved.isActive()).isFalse();
+ assertThat(resolved.getChannel().isShutdown()).isFalse();
+ }
+
+ invokeScaleDownCheck(pool, 3);
+ assertThat(pool.channelRefs).hasSize(44);
+ assertThat(pool.removedChannelRefs).hasSize(4);
+ } finally {
+ if (pool != null) {
+ pool.shutdownNow();
+ }
+ responses.shutdownNow();
+ }
+ }
+
+ private static GcpManagedChannel fixedPool(
+ RecordingChannelBuilder delegateBuilder, int minimum, int initial, boolean dynamic) {
+ GcpChannelPoolOptions.Builder options =
+ GcpChannelPoolOptions.newBuilder()
+ .setMinSize(minimum)
+ .setInitSize(initial)
+ .setMaxSize(initial);
+ if (dynamic) {
+ options
+ .setDynamicScaling(10, 20, Duration.ofMinutes(1))
+ .setScaleDownConsecutiveLowLoadChecks(3)
+ .setMaxScaleDownChannels(2);
+ }
+ return (GcpManagedChannel)
+ GcpManagedChannelBuilder.forDelegateBuilder(delegateBuilder)
+ .withOptions(
+ GcpManagedChannelOptions.newBuilder()
+ .withChannelPoolOptions(options.build())
+ .build())
+ .build();
+ }
+
+ private static void invokeScaleDownCheck(GcpManagedChannel pool, int times) {
+ for (int i = 0; i < times; i++) {
+ pool.checkScaleDown();
+ }
+ }
+
+ private RunResult runScenario(long seed, Variant variant, LoadShape loadShape) throws Exception {
+ int callers = Integer.getInteger("hotchannel.callers", DEFAULT_CALLERS);
+ if (callers < 20) {
+ throw new IllegalArgumentException("hotchannel.callers must be at least 20");
+ }
+ boolean staticPool = Boolean.getBoolean("hotchannel.staticPool");
+ boolean noScaleDown = Boolean.getBoolean("hotchannel.noScaleDown");
+ Duration phaseInterval =
+ Duration.ofMillis(
+ Long.getLong("hotchannel.scaleDownMillis", TEST_SCALE_DOWN_INTERVAL.toMillis()));
+ Duration scaleDownInterval = noScaleDown ? Duration.ofHours(1) : phaseInterval;
+
+ ScheduledExecutorService responses = Executors.newScheduledThreadPool(32);
+ ExecutorService callersExecutor = Executors.newFixedThreadPool(callers);
+ GcpManagedChannel pool = null;
+ try {
+ RecordingChannelBuilder delegateBuilder = new RecordingChannelBuilder(responses, seed);
+ GcpChannelPoolOptions.Builder poolOptions =
+ GcpChannelPoolOptions.newBuilder()
+ .setMaxSize(MAX_SIZE)
+ .setMinSize(staticPool ? MAX_SIZE : MIN_SIZE)
+ .setInitSize(staticPool ? MAX_SIZE : INITIAL_SIZE)
+ .setAffinityKeyLifetime(Duration.ofMinutes(10))
+ .setCleanupInterval(Duration.ofMinutes(1));
+ if (staticPool) {
+ poolOptions.disableDynamicScaling();
+ } else {
+ poolOptions.setDynamicScaling(MIN_RPC_PER_CHANNEL, MAX_RPC_PER_CHANNEL, scaleDownInterval);
+ compressBranchSpecificScaleUpTimer(poolOptions);
+ }
+
+ pool =
+ (GcpManagedChannel)
+ GcpManagedChannelBuilder.forDelegateBuilder(delegateBuilder)
+ .withApiConfig(spannerAffinityConfig())
+ .withOptions(
+ GcpManagedChannelOptions.newBuilder()
+ .withChannelPoolOptions(poolOptions.build())
+ .build())
+ .build();
+
+ // Reach the customer-sized pool before creating the session pool. Each caller owns one
+ // transaction and fans out five RPCs on its transaction affinity handle.
+ runWave(pool, callersExecutor, variant, Collections.emptyList(), seed, callers, 5, 0);
+
+ List sessionNames = createSessionsInBatches(pool);
+
+ if (!staticPool && loadShape == LoadShape.BURSTY) {
+ runScaleCycles(pool, callersExecutor, variant, sessionNames, seed, callers, phaseInterval);
+ } else if (loadShape == LoadShape.CONSTANT) {
+ runConstantLoad(pool, callersExecutor, variant, sessionNames, seed, callers);
+ }
+
+ // Final ramp restores a full active pool. Reset delegate counters so prior channel history
+ // does not manufacture skew in the measured hold period.
+ runWave(pool, callersExecutor, variant, sessionNames, seed + 10_000, callers, 5, 10_000);
+ delegateBuilder.resetMeasurements();
+
+ if (variant == Variant.REF_6120 && loadShape == LoadShape.BURSTY) {
+ runTransactionsAcrossScaleDown(
+ pool,
+ callersExecutor,
+ delegateBuilder,
+ sessionNames,
+ seed + 15_000,
+ callers,
+ phaseInterval);
+ }
+
+ int measureWaves = Integer.getInteger("hotchannel.measureWaves", 8);
+ for (int wave = 0; wave < measureWaves; wave++) {
+ runWave(
+ pool,
+ callersExecutor,
+ variant,
+ sessionNames,
+ seed + 20_000,
+ callers,
+ 5,
+ 20_000 + wave * callers);
+ }
+
+ Set activeIds = new HashSet<>();
+ for (ChannelRef channelRef : pool.channelRefs) {
+ activeIds.add(channelRef.getId());
+ }
+ return delegateBuilder.snapshot(
+ seed, variant, loadShape, pool.getNumberOfChannels(), activeIds);
+ } finally {
+ if (pool != null) {
+ pool.shutdownNow();
+ }
+ callersExecutor.shutdownNow();
+ responses.shutdownNow();
+ callersExecutor.awaitTermination(10, TimeUnit.SECONDS);
+ responses.awaitTermination(10, TimeUnit.SECONDS);
+ }
+ }
+
+ private static void compressBranchSpecificScaleUpTimer(GcpChannelPoolOptions.Builder builder) {
+ builder.setScaleUpCooldown(Duration.ofNanos(1));
+ }
+
+ private static void runTransactionsAcrossScaleDown(
+ GcpManagedChannel pool,
+ ExecutorService executor,
+ RecordingChannelBuilder delegateBuilder,
+ List sessions,
+ long seed,
+ int callers,
+ Duration scaleDownInterval)
+ throws Exception {
+ List transactions = new ArrayList<>(callers);
+ for (int transaction = 0; transaction < callers; transaction++) {
+ String session = sessions.get(Math.floorMod(mix(seed ^ transaction), sessions.size()));
+ transactions.add(new TransactionContext(transaction, session, new ChannelAffinityRef()));
+ }
+
+ CountDownLatch firstStart = new CountDownLatch(1);
+ List>> firstRpcStarts = new ArrayList<>(callers);
+ for (TransactionContext transaction : transactions) {
+ firstRpcStarts.add(
+ executor.submit(
+ () -> {
+ firstStart.await();
+ CallOptions options =
+ CallOptions.DEFAULT.withOption(
+ CHANNEL_AFFINITY_REF_KEY, transaction.affinityRef);
+ if (transaction.id < 20) {
+ options = options.withOption(TEST_LATENCY_MILLIS, 900L);
+ }
+ return executeSql(pool, options, transaction.session);
+ }));
+ }
+ firstStart.countDown();
+ List> firstRpcs = new ArrayList<>(callers);
+ for (Future> start : firstRpcStarts) {
+ firstRpcs.add(start.get(10, TimeUnit.SECONDS));
+ }
+ for (int transaction = 20; transaction < callers; transaction++) {
+ firstRpcs.get(transaction).get(10, TimeUnit.SECONDS);
+ }
+
+ // Twenty slow streams remain while a scaled 120 ms interval stands in for Spanner's three
+ // minutes. Mainline removes almost the entire pool in one check. Caller-owned refs still point
+ // at those now-inactive channels.
+ await()
+ .atMost(scaleDownInterval.multipliedBy(5).plusSeconds(1))
+ .until(() -> pool.getNumberOfChannels() < MAX_SIZE);
+
+ // Select every remaining call before starting any of them. This exposes the real newCall/start
+ // accounting gap: channel selection happens before SimpleGcpClientCall publishes its stream.
+ Set candidateIds = new HashSet<>();
+ for (ChannelRef channelRef : pool.channelRefs) {
+ candidateIds.add(channelRef.getId());
+ }
+ delegateBuilder.resetSelections();
+ List>>> prepared =
+ new ArrayList<>(callers);
+ for (TransactionContext transaction : transactions) {
+ prepared.add(
+ executor.submit(
+ () -> {
+ int remainingRpcCount = 1 + Math.floorMod(mix(seed + transaction.id * 17L), 4);
+ List> calls =
+ new ArrayList<>(remainingRpcCount);
+ CallOptions options =
+ CallOptions.DEFAULT.withOption(
+ CHANNEL_AFFINITY_REF_KEY, transaction.affinityRef);
+ for (int rpc = 0; rpc < remainingRpcCount; rpc++) {
+ calls.add(pool.newCall(EXECUTE_SQL_METHOD, options));
+ }
+ return calls;
+ }));
+ }
+
+ List>> preparedCalls = new ArrayList<>(callers);
+ for (Future>> calls : prepared) {
+ preparedCalls.add(calls.get(10, TimeUnit.SECONDS));
+ }
+ System.out.println(delegateBuilder.formatSelections(seed, candidateIds));
+ CountDownLatch remainingStart = new CountDownLatch(1);
+ List> remainingTransactions = new ArrayList<>(callers);
+ for (int transaction = 0; transaction < callers; transaction++) {
+ TransactionContext context = transactions.get(transaction);
+ List> calls = preparedCalls.get(transaction);
+ remainingTransactions.add(
+ executor.submit(
+ () -> {
+ remainingStart.await();
+ List> results = new ArrayList<>(calls.size());
+ for (ClientCall call : calls) {
+ results.add(
+ ClientCalls.futureUnaryCall(
+ call,
+ ExecuteSqlRequest.newBuilder()
+ .setSession(context.session)
+ .setSql("SELECT 1")
+ .build()));
+ }
+ for (ListenableFuture result : results) {
+ result.get(10, TimeUnit.SECONDS);
+ }
+ return null;
+ }));
+ }
+ remainingStart.countDown();
+ for (Future transaction : remainingTransactions) {
+ transaction.get(30, TimeUnit.SECONDS);
+ }
+ for (int transaction = 0; transaction < 20; transaction++) {
+ firstRpcs.get(transaction).get(10, TimeUnit.SECONDS);
+ }
+ }
+
+ private static ListenableFuture executeSql(
+ GcpManagedChannel pool, CallOptions callOptions, String session) {
+ return ClientCalls.futureUnaryCall(
+ pool.newCall(EXECUTE_SQL_METHOD, callOptions),
+ ExecuteSqlRequest.newBuilder().setSession(session).setSql("SELECT 1").build());
+ }
+
+ private static void runScaleCycles(
+ GcpManagedChannel pool,
+ ExecutorService executor,
+ Variant variant,
+ List sessions,
+ long seed,
+ int callers,
+ Duration interval)
+ throws Exception {
+ int[] ramp = {callers / 8, callers / 3, (callers * 2) / 3, callers};
+ for (int cycle = 0; cycle < 3; cycle++) {
+ for (int waveSize : ramp) {
+ runWave(
+ pool,
+ executor,
+ variant,
+ sessions,
+ seed + cycle * 1_000,
+ waveSize,
+ 0,
+ cycle * 10_000 + waveSize);
+ }
+ for (int hold = 0; hold < 3; hold++) {
+ runWave(
+ pool,
+ executor,
+ variant,
+ sessions,
+ seed + cycle * 1_000,
+ callers,
+ 0,
+ cycle * 10_000 + 1_000 + hold * callers);
+ }
+ int sizeBeforeScaleDown = pool.getNumberOfChannels();
+ await()
+ .atMost(interval.multipliedBy(5).plusSeconds(1))
+ .until(
+ () ->
+ pool.getNumberOfChannels() < sizeBeforeScaleDown
+ || pool.getNumberOfChannels() == MIN_SIZE);
+ }
+ }
+
+ private static void runConstantLoad(
+ GcpManagedChannel pool,
+ ExecutorService executor,
+ Variant variant,
+ List sessions,
+ long seed,
+ int callers)
+ throws Exception {
+ for (int wave = 0; wave < 20; wave++) {
+ runWave(pool, executor, variant, sessions, seed + 5_000, callers, 0, 5_000 + wave * callers);
+ }
+ }
+
+ private static List createSessionsInBatches(GcpManagedChannel pool) throws Exception {
+ List sessions = new ArrayList<>();
+ for (int batch = 0; batch < SESSION_COUNT / 10; batch++) {
+ List> futures = new ArrayList<>();
+ for (int item = 0; item < 10; item++) {
+ futures.add(
+ ClientCalls.futureUnaryCall(
+ pool.newCall(CREATE_SESSION_METHOD, CallOptions.DEFAULT),
+ CreateSessionRequest.newBuilder()
+ .setDatabase("projects/p/instances/i/databases/d")
+ .build()));
+ }
+ for (ListenableFuture future : futures) {
+ sessions.add(future.get(10, TimeUnit.SECONDS).getName());
+ }
+ }
+ return sessions;
+ }
+
+ private static void runWave(
+ GcpManagedChannel pool,
+ ExecutorService executor,
+ Variant variant,
+ List sessions,
+ long seed,
+ int transactionCount,
+ int fixedRpcCount,
+ int transactionOffset)
+ throws Exception {
+ CountDownLatch start = new CountDownLatch(1);
+ List> transactions = new ArrayList<>(transactionCount);
+ for (int transaction = 0; transaction < transactionCount; transaction++) {
+ final int transactionId = transactionOffset + transaction;
+ transactions.add(
+ executor.submit(
+ () -> {
+ start.await();
+ int rpcCount =
+ fixedRpcCount > 0
+ ? fixedRpcCount
+ : 1 + Math.floorMod(mix(seed + transactionId), 5);
+ ChannelAffinityRef affinityRef = new ChannelAffinityRef();
+ String session =
+ sessions.isEmpty()
+ ? "projects/p/instances/i/databases/d/sessions/bootstrap"
+ : sessions.get(Math.floorMod(mix(seed ^ transactionId), sessions.size()));
+ List> calls = new ArrayList<>(rpcCount);
+ for (int rpc = 0; rpc < rpcCount; rpc++) {
+ CallOptions callOptions = CallOptions.DEFAULT;
+ if (variant == Variant.REF_6120) {
+ callOptions = callOptions.withOption(CHANNEL_AFFINITY_REF_KEY, affinityRef);
+ } else {
+ int route = Math.floorMod(mix(seed + transactionId * 31L), 100);
+ if (route < 9) {
+ callOptions =
+ callOptions.withOption(
+ GcpManagedChannel.AFFINITY_KEY,
+ "projects/p/instances/i/databases/d/sessions/multiplexed");
+ } else if (route >= 29) {
+ callOptions =
+ callOptions.withOption(GcpManagedChannel.DISABLE_AFFINITY_KEY, true);
+ }
+ }
+ calls.add(
+ ClientCalls.futureUnaryCall(
+ pool.newCall(EXECUTE_SQL_METHOD, callOptions),
+ ExecuteSqlRequest.newBuilder()
+ .setSession(session)
+ .setSql("SELECT 1")
+ .build()));
+ }
+ for (ListenableFuture call : calls) {
+ call.get(10, TimeUnit.SECONDS);
+ }
+ return null;
+ }));
+ }
+ start.countDown();
+ for (Future transaction : transactions) {
+ transaction.get(30, TimeUnit.SECONDS);
+ }
+ }
+
+ private static ApiConfig spannerAffinityConfig() {
+ AffinityConfig bind =
+ AffinityConfig.newBuilder()
+ .setCommand(AffinityConfig.Command.BIND)
+ .setAffinityKey("name")
+ .build();
+ AffinityConfig bound =
+ AffinityConfig.newBuilder()
+ .setCommand(AffinityConfig.Command.BOUND)
+ .setAffinityKey("session")
+ .build();
+ return ApiConfig.newBuilder()
+ .addMethod(
+ MethodConfig.newBuilder()
+ .addName(CREATE_SESSION_METHOD.getFullMethodName())
+ .setAffinity(bind))
+ .addMethod(
+ MethodConfig.newBuilder()
+ .addName(EXECUTE_SQL_METHOD.getFullMethodName())
+ .setAffinity(bound))
+ .build();
+ }
+
+ private static int mix(long value) {
+ value = (value ^ (value >>> 33)) * 0xff51afd7ed558ccdL;
+ value = (value ^ (value >>> 33)) * 0xc4ceb9fe1a85ec53L;
+ return (int) (value ^ (value >>> 33));
+ }
+
+ private enum Variant {
+ REF_6120,
+ KEY_6117
+ }
+
+ private enum LoadShape {
+ BURSTY,
+ CONSTANT
+ }
+
+ private static final class RecordingChannelBuilder
+ extends ManagedChannelBuilder {
+ private final ScheduledExecutorService responses;
+ private final long seed;
+ private final AtomicInteger nextId = new AtomicInteger();
+ private final AtomicInteger nextSession = new AtomicInteger();
+ private final AtomicLong nextCall = new AtomicLong();
+ private final Map stats = new ConcurrentHashMap<>();
+
+ private RecordingChannelBuilder(ScheduledExecutorService responses, long seed) {
+ this.responses = responses;
+ this.seed = seed;
+ }
+
+ @Override
+ public ManagedChannel build() {
+ int id = nextId.getAndIncrement();
+ ChannelStats channelStats = new ChannelStats(id);
+ stats.put(id, channelStats);
+ return new RecordingManagedChannel(id, channelStats, responses, nextSession, nextCall, seed);
+ }
+
+ private void resetMeasurements() {
+ for (ChannelStats channelStats : stats.values()) {
+ channelStats.completed.set(0);
+ channelStats.maxActive.set(channelStats.active.get());
+ }
+ }
+
+ private void resetSelections() {
+ for (ChannelStats channelStats : stats.values()) {
+ channelStats.selected.set(0);
+ }
+ }
+
+ private String formatSelections(long runSeed, Set candidateIds) {
+ List candidates = new ArrayList<>();
+ for (int id : candidateIds) {
+ candidates.add(stats.get(id));
+ }
+ candidates.sort(Comparator.comparingInt(candidate -> candidate.id));
+ ChannelStats hot =
+ Collections.max(
+ candidates, Comparator.comparingLong(candidate -> candidate.selected.get()));
+ long[] selected =
+ candidates.stream().mapToLong(candidate -> candidate.selected.get()).sorted().toArray();
+ double median =
+ selected.length % 2 == 0
+ ? (selected[selected.length / 2 - 1] + selected[selected.length / 2]) / 2.0
+ : selected[selected.length / 2];
+ StringBuilder distribution = new StringBuilder();
+ for (ChannelStats candidate : candidates) {
+ if (distribution.length() > 0) {
+ distribution.append(',');
+ }
+ distribution.append(candidate.id).append('=').append(candidate.selected.get());
+ }
+ return String.format(
+ Locale.ROOT,
+ "HOTCHANNEL_EARLIEST seed=%d poolAfterDip=%d hotId=%d hotSelected=%d median=%.1f "
+ + "hotToMedian=%.3f preparedDistribution=[%s]",
+ runSeed,
+ candidateIds.size(),
+ hot.id,
+ hot.selected.get(),
+ median,
+ hot.selected.get() / median,
+ distribution);
+ }
+
+ private RunResult snapshot(
+ long runSeed,
+ Variant variant,
+ LoadShape loadShape,
+ int activePoolSize,
+ Set activeIds) {
+ List snapshots = new ArrayList<>();
+ for (int id : activeIds) {
+ ChannelStats channelStats = stats.get(id);
+ snapshots.add(
+ new ChannelStatsSnapshot(
+ id, channelStats.completed.get(), channelStats.maxActive.get()));
+ }
+ snapshots.sort(Comparator.comparingInt(snapshot -> snapshot.id));
+ return new RunResult(runSeed, variant, loadShape, activePoolSize, nextId.get(), snapshots);
+ }
+
+ @Override
+ public RecordingChannelBuilder directExecutor() {
+ return this;
+ }
+
+ @Override
+ public RecordingChannelBuilder executor(Executor executor) {
+ return this;
+ }
+
+ @Override
+ public RecordingChannelBuilder intercept(List interceptors) {
+ return this;
+ }
+
+ @Override
+ public RecordingChannelBuilder intercept(ClientInterceptor... interceptors) {
+ return this;
+ }
+
+ @Override
+ public RecordingChannelBuilder userAgent(String userAgent) {
+ return this;
+ }
+
+ @Override
+ public RecordingChannelBuilder overrideAuthority(String authority) {
+ return this;
+ }
+
+ @Override
+ public RecordingChannelBuilder nameResolverFactory(Factory resolverFactory) {
+ return this;
+ }
+
+ @Override
+ public RecordingChannelBuilder decompressorRegistry(DecompressorRegistry registry) {
+ return this;
+ }
+
+ @Override
+ public RecordingChannelBuilder compressorRegistry(CompressorRegistry registry) {
+ return this;
+ }
+
+ @Override
+ public RecordingChannelBuilder idleTimeout(long value, TimeUnit unit) {
+ return this;
+ }
+ }
+
+ private static final class RecordingManagedChannel extends ManagedChannel {
+ private final int id;
+ private final ChannelStats stats;
+ private final ScheduledExecutorService responses;
+ private final AtomicInteger nextSession;
+ private final AtomicLong nextCall;
+ private final long seed;
+ private final AtomicBoolean shutdown = new AtomicBoolean();
+
+ private RecordingManagedChannel(
+ int id,
+ ChannelStats stats,
+ ScheduledExecutorService responses,
+ AtomicInteger nextSession,
+ AtomicLong nextCall,
+ long seed) {
+ this.id = id;
+ this.stats = stats;
+ this.responses = responses;
+ this.nextSession = nextSession;
+ this.nextCall = nextCall;
+ this.seed = seed;
+ }
+
+ @Override
+ public ClientCall newCall(
+ MethodDescriptor method, CallOptions callOptions) {
+ stats.selected.incrementAndGet();
+ return new RecordingClientCall<>(
+ id, stats, responses, nextSession, nextCall, seed, method, callOptions);
+ }
+
+ @Override
+ public ConnectivityState getState(boolean requestConnection) {
+ return shutdown.get() ? ConnectivityState.SHUTDOWN : ConnectivityState.READY;
+ }
+
+ @Override
+ public void notifyWhenStateChanged(ConnectivityState source, Runnable callback) {}
+
+ @Override
+ public ManagedChannel shutdown() {
+ shutdown.set(true);
+ return this;
+ }
+
+ @Override
+ public boolean isShutdown() {
+ return shutdown.get();
+ }
+
+ @Override
+ public boolean isTerminated() {
+ return shutdown.get();
+ }
+
+ @Override
+ public ManagedChannel shutdownNow() {
+ return shutdown();
+ }
+
+ @Override
+ public boolean awaitTermination(long timeout, TimeUnit unit) {
+ return shutdown.get();
+ }
+
+ @Override
+ public String authority() {
+ return "in-process-channel-" + id;
+ }
+
+ @Override
+ public void enterIdle() {}
+ }
+
+ private static final class RecordingClientCall extends ClientCall {
+ private final int channelId;
+ private final ChannelStats stats;
+ private final ScheduledExecutorService responses;
+ private final AtomicInteger nextSession;
+ private final AtomicLong nextCall;
+ private final long seed;
+ private final MethodDescriptor method;
+ private final Long latencyMillisOverride;
+ private final AtomicBoolean completed = new AtomicBoolean();
+ private Listener listener;
+
+ private RecordingClientCall(
+ int channelId,
+ ChannelStats stats,
+ ScheduledExecutorService responses,
+ AtomicInteger nextSession,
+ AtomicLong nextCall,
+ long seed,
+ MethodDescriptor method,
+ CallOptions callOptions) {
+ this.channelId = channelId;
+ this.stats = stats;
+ this.responses = responses;
+ this.nextSession = nextSession;
+ this.nextCall = nextCall;
+ this.seed = seed;
+ this.method = method;
+ this.latencyMillisOverride = callOptions.getOption(TEST_LATENCY_MILLIS);
+ }
+
+ @Override
+ public void start(Listener listener, Metadata headers) {
+ this.listener = listener;
+ int active = stats.active.incrementAndGet();
+ stats.maxActive.accumulateAndGet(active, Math::max);
+ }
+
+ @Override
+ public void request(int numMessages) {}
+
+ @Override
+ public void cancel(String message, Throwable cause) {
+ finish(Status.CANCELLED, null);
+ }
+
+ @Override
+ public void halfClose() {
+ long callId = nextCall.getAndIncrement();
+ long latencyMillis =
+ latencyMillisOverride == null
+ ? 20 + Math.floorMod(mix(seed ^ callId), 11)
+ : latencyMillisOverride;
+ responses.schedule(
+ () -> finish(Status.OK, responseForMethod()), latencyMillis, TimeUnit.MILLISECONDS);
+ }
+
+ @Override
+ public void sendMessage(ReqT message) {}
+
+ @Override
+ public boolean isReady() {
+ return true;
+ }
+
+ @SuppressWarnings("unchecked")
+ private RespT responseForMethod() {
+ if (method.getFullMethodName().equals(CREATE_SESSION_METHOD.getFullMethodName())) {
+ return (RespT)
+ Session.newBuilder()
+ .setName(
+ "projects/p/instances/i/databases/d/sessions/session-"
+ + nextSession.incrementAndGet())
+ .build();
+ }
+ return (RespT) ResultSet.getDefaultInstance();
+ }
+
+ private void finish(Status status, RespT response) {
+ if (!completed.compareAndSet(false, true)) {
+ return;
+ }
+ if (response != null) {
+ listener.onMessage(response);
+ }
+ listener.onClose(status, new Metadata());
+ if (status.isOk()) {
+ stats.completed.incrementAndGet();
+ }
+ stats.active.decrementAndGet();
+ }
+ }
+
+ private static final class ChannelStats {
+ private final int id;
+ private final AtomicLong completed = new AtomicLong();
+ private final AtomicLong selected = new AtomicLong();
+ private final AtomicInteger active = new AtomicInteger();
+ private final AtomicInteger maxActive = new AtomicInteger();
+
+ private ChannelStats(int id) {
+ this.id = id;
+ }
+ }
+
+ private static final class TransactionContext {
+ private final int id;
+ private final String session;
+ private final ChannelAffinityRef affinityRef;
+
+ private TransactionContext(int id, String session, ChannelAffinityRef affinityRef) {
+ this.id = id;
+ this.session = session;
+ this.affinityRef = affinityRef;
+ }
+ }
+
+ private static final class ChannelStatsSnapshot {
+ private final int id;
+ private final long completed;
+ private final int maxActive;
+
+ private ChannelStatsSnapshot(int id, long completed, int maxActive) {
+ this.id = id;
+ this.completed = completed;
+ this.maxActive = maxActive;
+ }
+ }
+
+ private static final class RunResult {
+ private final long seed;
+ private final Variant variant;
+ private final LoadShape loadShape;
+ private final int activePoolSize;
+ private final int createdChannels;
+ private final List channels;
+ private final ChannelStatsSnapshot hot;
+ private final double median;
+ private final double hotToMedian;
+
+ private RunResult(
+ long seed,
+ Variant variant,
+ LoadShape loadShape,
+ int activePoolSize,
+ int createdChannels,
+ List channels) {
+ this.seed = seed;
+ this.variant = variant;
+ this.loadShape = loadShape;
+ this.activePoolSize = activePoolSize;
+ this.createdChannels = createdChannels;
+ this.channels = channels;
+ this.hot =
+ Collections.max(channels, Comparator.comparingLong(snapshot -> snapshot.completed));
+ long[] counts =
+ channels.stream().mapToLong(snapshot -> snapshot.completed).sorted().toArray();
+ this.median =
+ counts.length % 2 == 0
+ ? (counts[counts.length / 2 - 1] + counts[counts.length / 2]) / 2.0
+ : counts[counts.length / 2];
+ this.hotToMedian = hot.completed / median;
+ }
+
+ private String format() {
+ StringBuilder distribution = new StringBuilder();
+ for (ChannelStatsSnapshot channel : channels) {
+ if (distribution.length() > 0) {
+ distribution.append(',');
+ }
+ distribution
+ .append(channel.id)
+ .append('=')
+ .append(channel.completed)
+ .append('/')
+ .append(channel.maxActive);
+ }
+ return String.format(
+ Locale.ROOT,
+ "HOTCHANNEL_RESULT seed=%d variant=%s load=%s pool=%d created=%d hotId=%d "
+ + "hotOrder=%d hotCalls=%d hotMaxActive=%d median=%.1f hotToMedian=%.3f "
+ + "distribution[id=calls/maxActive]=[%s]",
+ seed,
+ variant,
+ loadShape,
+ activePoolSize,
+ createdChannels,
+ hot.id,
+ hot.id + 1,
+ hot.completed,
+ hot.maxActive,
+ median,
+ hotToMedian,
+ distribution);
+ }
+ }
+}
diff --git a/grpc-gcp-java/src/test/java/com/google/cloud/grpc/GcpManagedChannelOptionsTest.java b/grpc-gcp-java/src/test/java/com/google/cloud/grpc/GcpManagedChannelOptionsTest.java
index 7dc407ff1381..0bea6abb1561 100644
--- a/grpc-gcp-java/src/test/java/com/google/cloud/grpc/GcpManagedChannelOptionsTest.java
+++ b/grpc-gcp-java/src/test/java/com/google/cloud/grpc/GcpManagedChannelOptionsTest.java
@@ -20,11 +20,14 @@
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertThrows;
import static org.junit.Assert.assertTrue;
import com.google.cloud.grpc.GcpManagedChannelOptions.GcpChannelPoolOptions;
import com.google.cloud.grpc.GcpManagedChannelOptions.GcpMetricsOptions;
import com.google.cloud.grpc.GcpManagedChannelOptions.GcpResiliencyOptions;
+import com.google.common.util.concurrent.Futures;
+import io.grpc.ManagedChannel;
import io.opencensus.metrics.LabelKey;
import io.opencensus.metrics.LabelValue;
import io.opencensus.metrics.MetricRegistry;
@@ -208,6 +211,132 @@ public void testAffinityKeysCleanupZeroByDefault() {
assertThat(channelPoolOptions.getCleanupInterval()).isEqualTo(Duration.ZERO);
}
+ @Test
+ public void testDynamicScalingKnobsHaveDefaultsAndSurviveCopy() {
+ GcpChannelPoolOptions defaults = GcpChannelPoolOptions.newBuilder().build();
+
+ assertThat(defaults.getScaleUpCooldown()).isEqualTo(Duration.ofSeconds(10));
+ assertThat(defaults.getMaxScaleUpPercent()).isEqualTo(30);
+ assertThat(defaults.getErrorPenaltyStep()).isEqualTo(5);
+ assertThat(defaults.getErrorPenaltyDuration()).isEqualTo(Duration.ofSeconds(5));
+ assertThat(defaults.getChannelPrimer()).isNull();
+ assertThat(defaults.getChannelPrimeTimeout()).isEqualTo(Duration.ofSeconds(10));
+ assertThat(defaults.getChannelPrimeMaxAttempts()).isEqualTo(3);
+
+ GcpChannelPoolOptions copiedDefaults = GcpChannelPoolOptions.newBuilder(defaults).build();
+ assertThat(copiedDefaults.getErrorPenaltyStep()).isEqualTo(5);
+ assertThat(copiedDefaults.getErrorPenaltyDuration()).isEqualTo(Duration.ofSeconds(5));
+ assertThat(copiedDefaults.toString()).contains("errorPenaltyStep: 5");
+ GcpChannelPoolOptions zeroValues =
+ GcpChannelPoolOptions.newBuilder()
+ .setChannelPrimeTimeout(Duration.ZERO)
+ .setChannelPrimeMaxAttempts(0)
+ .build();
+ assertThat(zeroValues.getChannelPrimeTimeout()).isEqualTo(Duration.ofSeconds(10));
+ assertThat(zeroValues.getChannelPrimeMaxAttempts()).isEqualTo(3);
+ GcpChannelPoolOptions copiedZeroValues = GcpChannelPoolOptions.newBuilder(zeroValues).build();
+ assertThat(copiedZeroValues.getChannelPrimeTimeout()).isEqualTo(Duration.ofSeconds(10));
+ assertThat(copiedZeroValues.getChannelPrimeMaxAttempts()).isEqualTo(3);
+ assertThat(copiedZeroValues.toString()).contains("channelPrimeTimeout: PT10S");
+ assertThat(copiedZeroValues.toString()).contains("channelPrimeMaxAttempts: 3");
+
+ GcpChannelPoolOptions configured =
+ GcpChannelPoolOptions.newBuilder(defaults)
+ .setScaleUpCooldown(Duration.ofSeconds(1))
+ .setMaxScaleUpPercent(40)
+ .setErrorPenaltyStep(2)
+ .setErrorPenaltyDuration(Duration.ofSeconds(3))
+ .build();
+ GcpChannelPoolOptions copied = GcpChannelPoolOptions.newBuilder(configured).build();
+
+ assertThat(copied.getScaleUpCooldown()).isEqualTo(Duration.ofSeconds(1));
+ assertThat(copied.getMaxScaleUpPercent()).isEqualTo(40);
+ assertThat(copied.getErrorPenaltyStep()).isEqualTo(2);
+ assertThat(copied.getErrorPenaltyDuration()).isEqualTo(Duration.ofSeconds(3));
+
+ GcpChannelPoolOptions disabled =
+ GcpChannelPoolOptions.newBuilder().setErrorPenaltyStep(0).build();
+ assertThat(disabled.getErrorPenaltyStep()).isEqualTo(0);
+ }
+
+ @Test
+ public void dynamicScalingDefaultKnobsRejectNegativeAndDefaultZeroCooldown() {
+ assertThrows(
+ IllegalArgumentException.class,
+ () -> GcpChannelPoolOptions.newBuilder().setScaleUpCooldown(Duration.ofNanos(-1)));
+ assertThat(
+ GcpChannelPoolOptions.newBuilder()
+ .setScaleUpCooldown(Duration.ZERO)
+ .build()
+ .getScaleUpCooldown())
+ .isEqualTo(Duration.ofSeconds(10));
+ assertThrows(
+ IllegalArgumentException.class,
+ () -> GcpChannelPoolOptions.newBuilder().setErrorPenaltyStep(-1));
+ assertThrows(
+ IllegalArgumentException.class,
+ () ->
+ GcpChannelPoolOptions.newBuilder()
+ .setErrorPenaltyDuration(Duration.ofSeconds(Long.MAX_VALUE)));
+ assertThrows(
+ IllegalArgumentException.class,
+ () -> GcpChannelPoolOptions.newBuilder().setChannelPrimeTimeout(Duration.ofNanos(-1)));
+ assertThrows(
+ IllegalArgumentException.class,
+ () ->
+ GcpChannelPoolOptions.newBuilder()
+ .setChannelPrimeTimeout(Duration.ofSeconds(Long.MAX_VALUE)));
+ assertThrows(
+ IllegalArgumentException.class,
+ () -> GcpChannelPoolOptions.newBuilder().setChannelPrimeMaxAttempts(-1));
+ }
+
+ @Test
+ public void channelPoolOptionsToStringIncludesEveryKnob() {
+ String options = GcpChannelPoolOptions.newBuilder().build().toString();
+
+ assertThat(options).contains("maxSize:");
+ assertThat(options).contains("minSize:");
+ assertThat(options).contains("initSize:");
+ assertThat(options).contains("minRpcPerChannel:");
+ assertThat(options).contains("maxRpcPerChannel:");
+ assertThat(options).contains("scaleDownInterval:");
+ assertThat(options).contains("scaleUpCooldown:");
+ assertThat(options).contains("scaleDownConsecutiveLowLoadChecks:");
+ assertThat(options).contains("maxScaleUpPercent:");
+ assertThat(options).contains("maxScaleDownChannels:");
+ assertThat(options).contains("drainIdleGrace:");
+ assertThat(options).contains("errorPenaltyStep:");
+ assertThat(options).contains("errorPenaltyDuration:");
+ assertThat(options).contains("concurrentStreamsLowWatermark:");
+ assertThat(options).contains("useRoundRobinOnBind:");
+ assertThat(options).contains("affinityKeyLifetime:");
+ assertThat(options).contains("cleanupInterval:");
+ assertThat(options).contains("channelPickStrategy:");
+ assertThat(options).contains("channelPrimer:");
+ assertThat(options).contains("channelPrimeTimeout:");
+ assertThat(options).contains("channelPrimeMaxAttempts:");
+ }
+
+ @Test
+ public void channelPrimerOptionsSurviveCopy() {
+ GcpChannelPrimer primer = (ManagedChannel channel) -> Futures.immediateVoidFuture();
+ GcpChannelPoolOptions configured =
+ GcpChannelPoolOptions.newBuilder()
+ .setChannelPrimer(primer)
+ .setChannelPrimeTimeout(Duration.ofSeconds(7))
+ .setChannelPrimeMaxAttempts(2)
+ .build();
+
+ GcpChannelPoolOptions copied = GcpChannelPoolOptions.newBuilder(configured).build();
+
+ assertThat(copied.getChannelPrimer()).isSameInstanceAs(primer);
+ assertThat(copied.getChannelPrimeTimeout()).isEqualTo(Duration.ofSeconds(7));
+ assertThat(copied.getChannelPrimeMaxAttempts()).isEqualTo(2);
+ assertThat(copied.toString()).contains("channelPrimeTimeout: PT7S");
+ assertThat(copied.toString()).contains("channelPrimeMaxAttempts: 2");
+ }
+
@Test
public void testCleanupDefault() {
GcpManagedChannelOptions opts =
diff --git a/grpc-gcp-java/src/test/java/com/google/cloud/grpc/GcpManagedChannelOtelMetricsTest.java b/grpc-gcp-java/src/test/java/com/google/cloud/grpc/GcpManagedChannelOtelMetricsTest.java
index c7eb432ce281..87b8cd33f7f3 100644
--- a/grpc-gcp-java/src/test/java/com/google/cloud/grpc/GcpManagedChannelOtelMetricsTest.java
+++ b/grpc-gcp-java/src/test/java/com/google/cloud/grpc/GcpManagedChannelOtelMetricsTest.java
@@ -98,5 +98,11 @@ public void emitsOtelMetricsWhenMeterProvided() {
names.stream()
.anyMatch(
n -> n.equals("test/grpc-gcp/" + GcpMetricsConstants.METRIC_MAX_READY_CHANNELS)));
+ assertTrue(
+ names.stream()
+ .anyMatch(
+ n ->
+ n.equals(
+ "test/grpc-gcp/" + GcpMetricsConstants.METRIC_SCALE_UP_PRIME_FAILURES)));
}
}
diff --git a/grpc-gcp-java/src/test/java/com/google/cloud/grpc/GcpManagedChannelPrimerTest.java b/grpc-gcp-java/src/test/java/com/google/cloud/grpc/GcpManagedChannelPrimerTest.java
new file mode 100644
index 000000000000..9d36abf17371
--- /dev/null
+++ b/grpc-gcp-java/src/test/java/com/google/cloud/grpc/GcpManagedChannelPrimerTest.java
@@ -0,0 +1,760 @@
+/*
+ * Copyright 2026 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.grpc;
+
+import static com.google.common.truth.Truth.assertThat;
+import static org.awaitility.Awaitility.await;
+
+import com.google.cloud.grpc.GcpManagedChannel.ChannelRef;
+import com.google.cloud.grpc.GcpManagedChannelOptions.GcpChannelPoolOptions;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.SettableFuture;
+import io.grpc.ConnectivityState;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.logging.Handler;
+import java.util.logging.Level;
+import java.util.logging.LogRecord;
+import java.util.logging.Logger;
+import org.junit.After;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Channel-primer behavior tests. */
+@RunWith(JUnit4.class)
+public final class GcpManagedChannelPrimerTest {
+ private final ExecutorService stateExecutor = Executors.newSingleThreadExecutor();
+ private GcpManagedChannel pool;
+
+ @After
+ public void tearDown() {
+ if (pool != null) {
+ pool.shutdownNow();
+ }
+ stateExecutor.shutdownNow();
+ }
+
+ @Test
+ public void successfulPrimerDelaysPublicationAndRunsOnBackgroundWorker() throws Exception {
+ SettableFuture primeFuture = SettableFuture.create();
+ AtomicReference primerThread = new AtomicReference<>();
+ AtomicReference primingChannel =
+ new AtomicReference<>();
+ GcpChannelPrimer primer =
+ channel -> {
+ primerThread.set(Thread.currentThread().getName());
+ primingChannel.set((GcpManagedChannelTest.FakeManagedChannel) channel);
+ return primeFuture;
+ };
+ pool = newPrimedPool(primer, Duration.ofSeconds(5), builder());
+ ChannelRef hot = pool.channelRefs.get(0);
+ hot.setActiveStreamsForTest(6);
+
+ hot.activeStreamsCountIncr();
+
+ awaitCondition(() -> primingChannel.get() != null);
+ awaitCondition(() -> !pool.scaleUpWorkerRunningForTest());
+ assertThat(pool.inFlightPrimeCountForTest()).isEqualTo(1);
+ assertThat(pool.getNumberOfChannels()).isEqualTo(2);
+ assertThat(primerThread.get()).startsWith("gcp-mc-prime-");
+ primeFuture.set(null);
+ awaitCondition(() -> pool.getNumberOfChannels() == 3);
+ }
+
+ @Test
+ public void scaleUpStartsAllChannelPrimersConcurrently() throws Exception {
+ CountDownLatch allPrimersStarted = new CountDownLatch(3);
+ List> primeFutures = new CopyOnWriteArrayList<>();
+ GcpChannelPrimer primer =
+ channel -> {
+ SettableFuture future = SettableFuture.create();
+ primeFutures.add(future);
+ allPrimersStarted.countDown();
+ return future;
+ };
+ pool = newPrimedPool(10, 13, primer, Duration.ofSeconds(5), 1, builder());
+ ChannelRef hot = pool.channelRefs.get(0);
+ hot.setActiveStreamsForTest(99);
+
+ hot.activeStreamsCountIncr();
+
+ assertThat(allPrimersStarted.await(5, TimeUnit.SECONDS)).isTrue();
+ assertThat(pool.inFlightPrimeCountForTest()).isEqualTo(3);
+ assertThat(pool.getNumberOfChannels()).isEqualTo(10);
+ primeFutures.forEach(future -> future.set(null));
+ awaitCondition(() -> pool.inFlightPrimeCountForTest() == 0);
+ awaitCondition(() -> pool.getNumberOfChannels() == 13);
+ assertThat(pool.getNumberOfChannels()).isEqualTo(13);
+ }
+
+ @Test
+ public void secondScaleUpCountsChannelsStillPriming() throws Exception {
+ AtomicInteger builtChannels = new AtomicInteger();
+ List> primeFutures = new CopyOnWriteArrayList<>();
+ GcpManagedChannelTest.FakeManagedChannelBuilder delegate =
+ new GcpManagedChannelTest.FakeManagedChannelBuilder(
+ () -> {
+ builtChannels.incrementAndGet();
+ return new GcpManagedChannelTest.FakeManagedChannel(stateExecutor);
+ });
+ GcpChannelPrimer primer =
+ channel -> {
+ SettableFuture future = SettableFuture.create();
+ primeFutures.add(future);
+ return future;
+ };
+ pool = newPrimedPool(2, 4, primer, Duration.ofSeconds(5), 1, delegate);
+ AtomicLong clock = new AtomicLong(1);
+ pool.setNanoClock(clock::get);
+ ChannelRef hot = pool.channelRefs.get(0);
+ hot.setActiveStreamsForTest(6);
+
+ hot.activeStreamsCountIncr();
+ awaitCondition(() -> primeFutures.size() == 2);
+ awaitCondition(() -> !pool.scaleUpWorkerRunningForTest());
+ assertThat(pool.inFlightPrimeCountForTest()).isEqualTo(2);
+
+ clock.incrementAndGet();
+ hot.activeStreamsCountIncr();
+ awaitCondition(() -> !pool.scaleUpWorkerRunningForTest());
+
+ assertThat(builtChannels.get()).isEqualTo(4);
+ assertThat(primeFutures).hasSize(2);
+ assertThat(pool.getNumberOfChannels() + pool.inFlightPrimeCountForTest()).isEqualTo(4);
+ primeFutures.forEach(future -> future.set(null));
+ awaitCondition(() -> pool.getNumberOfChannels() == 4);
+ }
+
+ @Test
+ public void primedChannelIsPublishedBeforeRestOfBatchCompletes() throws Exception {
+ CountDownLatch allPrimersStarted = new CountDownLatch(3);
+ List> primeFutures = new CopyOnWriteArrayList<>();
+ Map, GcpManagedChannelTest.FakeManagedChannel> primingChannels =
+ new java.util.concurrent.ConcurrentHashMap<>();
+ GcpChannelPrimer primer =
+ channel -> {
+ SettableFuture future = SettableFuture.create();
+ primingChannels.put(future, (GcpManagedChannelTest.FakeManagedChannel) channel);
+ primeFutures.add(future);
+ allPrimersStarted.countDown();
+ return future;
+ };
+ pool = newPrimedPool(10, 13, primer, Duration.ofSeconds(5), 1, builder());
+ ChannelRef hot = pool.channelRefs.get(0);
+ hot.setActiveStreamsForTest(99);
+
+ hot.activeStreamsCountIncr();
+
+ assertThat(allPrimersStarted.await(5, TimeUnit.SECONDS)).isTrue();
+ SettableFuture firstFuture = primeFutures.get(0);
+ GcpManagedChannelTest.FakeManagedChannel firstChannel = primingChannels.get(firstFuture);
+ firstFuture.set(null);
+ awaitCondition(() -> pool.getNumberOfChannels() == 11);
+ assertThat(
+ pool.channelRefs.stream()
+ .anyMatch(channelRef -> channelRef.getChannel() == firstChannel))
+ .isTrue();
+ assertThat(pool.inFlightPrimeCountForTest()).isEqualTo(2);
+ assertThat(primeFutures.get(1).isDone()).isFalse();
+ assertThat(primeFutures.get(2).isDone()).isFalse();
+ primeFutures.get(1).set(null);
+ primeFutures.get(2).set(null);
+ awaitCondition(() -> pool.inFlightPrimeCountForTest() == 0);
+ awaitCondition(() -> pool.getNumberOfChannels() == 13);
+ }
+
+ @Test
+ public void failedPrimeDoesNotDelayOtherChannels() throws Exception {
+ CountDownLatch allPrimersStarted = new CountDownLatch(3);
+ List> primeFutures = new CopyOnWriteArrayList<>();
+ GcpChannelPrimer primer =
+ channel -> {
+ SettableFuture future = SettableFuture.create();
+ primeFutures.add(future);
+ allPrimersStarted.countDown();
+ return future;
+ };
+ pool = newPrimedPool(10, 13, primer, Duration.ofSeconds(5), 1, builder());
+ ChannelRef hot = pool.channelRefs.get(0);
+ hot.setActiveStreamsForTest(99);
+
+ hot.activeStreamsCountIncr();
+
+ assertThat(allPrimersStarted.await(5, TimeUnit.SECONDS)).isTrue();
+ primeFutures.get(0).setException(new IllegalStateException("prime failed"));
+ primeFutures.get(1).set(null);
+ primeFutures.get(2).set(null);
+ awaitCondition(() -> pool.scaleUpPrimeFailuresForTest() == 1);
+ awaitCondition(() -> pool.inFlightPrimeCountForTest() == 0);
+ awaitCondition(() -> pool.getNumberOfChannels() == 12);
+ assertThat(pool.scaleUpPrimeFailuresForTest()).isEqualTo(1);
+ }
+
+ @Test
+ public void timedOutPrimeDoesNotDelayOtherChannels() throws Exception {
+ CountDownLatch allPrimersStarted = new CountDownLatch(3);
+ List> primeFutures = new CopyOnWriteArrayList<>();
+ GcpChannelPrimer primer =
+ channel -> {
+ SettableFuture future = SettableFuture.create();
+ primeFutures.add(future);
+ allPrimersStarted.countDown();
+ return future;
+ };
+ pool = newPrimedPool(10, 13, primer, Duration.ofSeconds(1), 1, builder());
+ ChannelRef hot = pool.channelRefs.get(0);
+ hot.setActiveStreamsForTest(99);
+
+ hot.activeStreamsCountIncr();
+
+ assertThat(allPrimersStarted.await(5, TimeUnit.SECONDS)).isTrue();
+ primeFutures.get(1).set(null);
+ primeFutures.get(2).set(null);
+ awaitCondition(() -> pool.getNumberOfChannels() == 12);
+ assertThat(primeFutures.get(0).isDone()).isFalse();
+ assertThat(pool.inFlightPrimeCountForTest()).isEqualTo(1);
+ awaitCondition(() -> pool.scaleUpPrimeFailuresForTest() == 1);
+ awaitCondition(() -> pool.inFlightPrimeCountForTest() == 0);
+ }
+
+ @Test
+ public void shutdownClosesEveryUnpublishedPrimingChannel() throws Exception {
+ CountDownLatch allPrimersStarted = new CountDownLatch(3);
+ List primingChannels = new CopyOnWriteArrayList<>();
+ List> primeFutures = new CopyOnWriteArrayList<>();
+ GcpChannelPrimer primer =
+ channel -> {
+ primingChannels.add((GcpManagedChannelTest.FakeManagedChannel) channel);
+ SettableFuture future = SettableFuture.create();
+ primeFutures.add(future);
+ allPrimersStarted.countDown();
+ return future;
+ };
+ pool = newPrimedPool(10, 13, primer, Duration.ofSeconds(5), 1, builder());
+ ChannelRef hot = pool.channelRefs.get(0);
+ hot.setActiveStreamsForTest(99);
+ hot.activeStreamsCountIncr();
+ assertThat(allPrimersStarted.await(5, TimeUnit.SECONDS)).isTrue();
+
+ pool.shutdownNow();
+
+ awaitCondition(() -> pool.inFlightPrimeCountForTest() == 0);
+ assertThat(primingChannels).hasSize(3);
+ assertThat(primingChannels.stream().allMatch(channel -> channel.isShutdown())).isTrue();
+ assertThat(primeFutures.stream().allMatch(Future::isCancelled)).isTrue();
+ assertThat(pool.getNumberOfChannels()).isEqualTo(10);
+ }
+
+ @Test
+ public void failedPrimerRejectsChannelAndLaterScaleUpStillWorks() throws Exception {
+ AtomicInteger primeCalls = new AtomicInteger();
+ AtomicReference rejected = new AtomicReference<>();
+ GcpChannelPrimer primer =
+ channel -> {
+ if (primeCalls.incrementAndGet() == 1) {
+ rejected.set((GcpManagedChannelTest.FakeManagedChannel) channel);
+ return Futures.immediateFailedFuture(new IllegalStateException("prime failed"));
+ }
+ return Futures.immediateVoidFuture();
+ };
+ pool = newPrimedPool(primer, Duration.ofSeconds(5), 1, builder());
+ AtomicLong clock = new AtomicLong(1);
+ pool.setNanoClock(clock::get);
+ ChannelRef hot = pool.channelRefs.get(0);
+ hot.setActiveStreamsForTest(6);
+
+ hot.activeStreamsCountIncr();
+
+ awaitCondition(() -> pool.scaleUpPrimeFailuresForTest() == 1);
+ assertThat(pool.getNumberOfChannels()).isEqualTo(2);
+ awaitCondition(() -> rejected.get() != null && rejected.get().isShutdown());
+ awaitCondition(() -> !pool.scaleUpWorkerRunningForTest());
+ clock.incrementAndGet();
+ hot.activeStreamsCountIncr();
+ awaitCondition(() -> pool.getNumberOfChannels() == 3);
+ assertThat(primeCalls.get()).isEqualTo(2);
+ }
+
+ @Test
+ public void primerTimeoutRejectsAndClosesChannel() throws Exception {
+ SettableFuture neverCompletes = SettableFuture.create();
+ AtomicReference rejected = new AtomicReference<>();
+ GcpChannelPrimer primer =
+ channel -> {
+ rejected.set((GcpManagedChannelTest.FakeManagedChannel) channel);
+ return neverCompletes;
+ };
+ pool = newPrimedPool(primer, Duration.ofMillis(20), 1, builder());
+ ChannelRef hot = pool.channelRefs.get(0);
+ hot.setActiveStreamsForTest(6);
+
+ hot.activeStreamsCountIncr();
+
+ awaitCondition(() -> pool.scaleUpPrimeFailuresForTest() == 1);
+ assertThat(pool.getNumberOfChannels()).isEqualTo(2);
+ awaitCondition(() -> rejected.get() != null && rejected.get().isShutdown());
+ }
+
+ @Test
+ public void primerTimeoutIncludesBlockingInvocation() throws Exception {
+ CountDownLatch primerEntered = new CountDownLatch(1);
+ CountDownLatch releasePrimer = new CountDownLatch(1);
+ SettableFuture lateFuture = SettableFuture.create();
+ AtomicInteger primeCalls = new AtomicInteger();
+ AtomicReference primerThread = new AtomicReference<>();
+ AtomicReference rejected = new AtomicReference<>();
+ AbandonWarnings warnings = new AbandonWarnings();
+ GcpChannelPrimer primer =
+ channel -> {
+ if (primeCalls.incrementAndGet() > 1) {
+ return Futures.immediateVoidFuture();
+ }
+ primerThread.set(Thread.currentThread().getName());
+ rejected.set((GcpManagedChannelTest.FakeManagedChannel) channel);
+ primerEntered.countDown();
+ awaitUninterruptibly(releasePrimer);
+ return lateFuture;
+ };
+ try {
+ pool = newPrimedPool(primer, Duration.ofMillis(20), 3, builder());
+ AtomicLong clock = new AtomicLong(1);
+ pool.setNanoClock(clock::get);
+ ChannelRef hot = pool.channelRefs.get(0);
+ hot.setActiveStreamsForTest(6);
+
+ hot.activeStreamsCountIncr();
+
+ assertThat(primerEntered.await(5, TimeUnit.SECONDS)).isTrue();
+ // A primer blocked inside prime() fails at the timeout and is not retried: nothing can stop
+ // the running call and a fresh attempt would overlap it on the same channel.
+ awaitCondition(() -> pool.scaleUpPrimeFailuresForTest() == 1);
+ awaitCondition(() -> pool.inFlightPrimeCountForTest() == 0);
+ assertThat(pool.getNumberOfChannels()).isEqualTo(2);
+ awaitCondition(() -> rejected.get() != null && rejected.get().isShutdown());
+ assertThat(primeCalls.get()).isEqualTo(1);
+ // The blocked call holds a primer thread, never one of the shared scheduler's threads.
+ assertThat(primerThread.get()).startsWith("gcp-mc-prime-");
+ awaitCondition(() -> !pool.scaleUpWorkerRunningForTest());
+ // While the call is still blocked it keeps its scale-up slot, so the pool (2 of 3) does not
+ // build another delegate and hand it to the same stuck primer.
+ assertThat(pool.abandonedPrimeCountForTest()).isEqualTo(1);
+ // Abandoning is invisible to the application, so it is also logged as a warning, attributed
+ // to the pool like every other prime warning.
+ assertThat(warnings.messages()).hasSize(1);
+ assertThat(warnings.messages().get(0)).startsWith("pool-");
+ assertThat(warnings.messages().get(0)).contains("still blocked at the timeout");
+ clock.incrementAndGet();
+ hot.activeStreamsCountIncr();
+ awaitCondition(() -> !pool.scaleUpWorkerRunningForTest());
+ assertThat(primeCalls.get()).isEqualTo(1);
+ assertThat(pool.inFlightPrimeCountForTest()).isEqualTo(0);
+ releasePrimer.countDown();
+
+ // Once the blocked call returns, its late future is cancelled without publishing, the slot
+ // is released, and the next scale-up primes a fresh delegate.
+ awaitCondition(lateFuture::isCancelled);
+ awaitCondition(() -> pool.abandonedPrimeCountForTest() == 0);
+ assertThat(pool.getNumberOfChannels()).isEqualTo(2);
+ assertThat(pool.inFlightPrimeCountForTest()).isEqualTo(0);
+ clock.incrementAndGet();
+ hot.activeStreamsCountIncr();
+ awaitCondition(() -> pool.getNumberOfChannels() == 3);
+ assertThat(primeCalls.get()).isEqualTo(2);
+ assertThat(pool.scaleUpPrimeFailuresForTest()).isEqualTo(1);
+ assertThat(warnings.messages()).hasSize(1);
+ } finally {
+ releasePrimer.countDown();
+ warnings.close();
+ }
+ }
+
+ @Test
+ public void shutdownDuringBlockedPrimeDoesNotWarnAboutAbandoning() throws Exception {
+ CountDownLatch primerEntered = new CountDownLatch(1);
+ CountDownLatch releasePrimer = new CountDownLatch(1);
+ AtomicReference primingChannel =
+ new AtomicReference<>();
+ AbandonWarnings warnings = new AbandonWarnings();
+ GcpChannelPrimer primer =
+ channel -> {
+ primingChannel.set((GcpManagedChannelTest.FakeManagedChannel) channel);
+ primerEntered.countDown();
+ awaitUninterruptibly(releasePrimer);
+ return Futures.immediateVoidFuture();
+ };
+ try {
+ pool = newPrimedPool(primer, Duration.ofSeconds(5), 3, builder());
+ ChannelRef hot = pool.channelRefs.get(0);
+ hot.setActiveStreamsForTest(6);
+ hot.activeStreamsCountIncr();
+ assertThat(primerEntered.await(5, TimeUnit.SECONDS)).isTrue();
+
+ pool.shutdownNow();
+
+ awaitCondition(() -> pool.inFlightPrimeCountForTest() == 0);
+ assertThat(primingChannel.get().isShutdown()).isTrue();
+ releasePrimer.countDown();
+ awaitCondition(() -> !pool.scaleUpWorkerRunningForTest());
+ // Shutdown closes the priming channel without abandoning it: no slot stays occupied and
+ // nothing is logged about a blocked call.
+ assertThat(pool.abandonedPrimeCountForTest()).isEqualTo(0);
+ assertThat(pool.scaleUpPrimeFailuresForTest()).isEqualTo(0);
+ assertThat(warnings.messages()).isEmpty();
+ assertThat(pool.getNumberOfChannels()).isEqualTo(2);
+ } finally {
+ releasePrimer.countDown();
+ warnings.close();
+ }
+ }
+
+ @Test
+ public void primedChannelIsPublishedUnderPoolMonitor() throws Exception {
+ SettableFuture primeFuture = SettableFuture.create();
+ AtomicReference primingChannel =
+ new AtomicReference<>();
+ GcpChannelPrimer primer =
+ channel -> {
+ primingChannel.set((GcpManagedChannelTest.FakeManagedChannel) channel);
+ return primeFuture;
+ };
+ pool = newPrimedPool(primer, Duration.ofSeconds(5), builder());
+ ChannelRef hot = pool.channelRefs.get(0);
+ hot.setActiveStreamsForTest(6);
+
+ hot.activeStreamsCountIncr();
+ awaitCondition(() -> primingChannel.get() != null);
+ awaitCondition(() -> !pool.scaleUpWorkerRunningForTest());
+
+ // The controller guards its state with the pool's own monitor, so while this thread holds
+ // it a completed prime cannot be published or even leave the in-flight set.
+ synchronized (pool) {
+ primeFuture.set(null);
+ Thread.sleep(200);
+ assertThat(pool.inFlightPrimeCountForTest()).isEqualTo(1);
+ assertThat(pool.getNumberOfChannels()).isEqualTo(2);
+ }
+ awaitCondition(() -> pool.getNumberOfChannels() == 3);
+ assertThat(pool.inFlightPrimeCountForTest()).isEqualTo(0);
+ }
+
+ @Test
+ public void throwingAndNullPrimersRejectChannels() throws Exception {
+ AtomicInteger primeCalls = new AtomicInteger();
+ GcpChannelPrimer primer =
+ channel -> {
+ if (primeCalls.incrementAndGet() == 1) {
+ throw new IllegalStateException("prime threw");
+ }
+ return null;
+ };
+ pool = newPrimedPool(primer, Duration.ofSeconds(5), 1, builder());
+ AtomicLong clock = new AtomicLong(1);
+ pool.setNanoClock(clock::get);
+ ChannelRef hot = pool.channelRefs.get(0);
+ hot.setActiveStreamsForTest(6);
+
+ hot.activeStreamsCountIncr();
+ awaitCondition(() -> pool.scaleUpPrimeFailuresForTest() == 1);
+ awaitCondition(() -> !pool.scaleUpWorkerRunningForTest());
+ clock.incrementAndGet();
+ hot.activeStreamsCountIncr();
+
+ awaitCondition(() -> pool.scaleUpPrimeFailuresForTest() == 2);
+ assertThat(primeCalls.get()).isEqualTo(2);
+ assertThat(pool.inFlightPrimeCountForTest()).isEqualTo(0);
+ assertThat(pool.getNumberOfChannels()).isEqualTo(2);
+ }
+
+ @Test
+ public void successfulPrimeIsDiscardedWhenPoolAlreadyAtMaximum() throws Exception {
+ SettableFuture primeFuture = SettableFuture.create();
+ AtomicReference primingChannel =
+ new AtomicReference<>();
+ GcpChannelPrimer primer =
+ channel -> {
+ primingChannel.set((GcpManagedChannelTest.FakeManagedChannel) channel);
+ return primeFuture;
+ };
+ pool = newPrimedPool(primer, Duration.ofSeconds(5), 1, builder());
+ ChannelRef hot = pool.channelRefs.get(0);
+ hot.setActiveStreamsForTest(6);
+
+ hot.activeStreamsCountIncr();
+ awaitCondition(() -> primingChannel.get() != null);
+ pool.createNewChannel();
+ assertThat(pool.getNumberOfChannels()).isEqualTo(3);
+ primeFuture.set(null);
+
+ awaitCondition(() -> pool.inFlightPrimeCountForTest() == 0);
+ awaitCondition(() -> primingChannel.get().isShutdown());
+ assertThat(pool.getNumberOfChannels()).isEqualTo(3);
+ }
+
+ @Test
+ public void primerRetriesUntilSuccess() throws Exception {
+ AtomicInteger primeCalls = new AtomicInteger();
+ GcpChannelPrimer primer =
+ channel ->
+ primeCalls.incrementAndGet() < 3
+ ? Futures.immediateFailedFuture(new IllegalStateException("prime failed"))
+ : Futures.immediateVoidFuture();
+ pool = newPrimedPool(primer, Duration.ofSeconds(5), 3, builder());
+ ChannelRef hot = pool.channelRefs.get(0);
+ hot.setActiveStreamsForTest(6);
+
+ hot.activeStreamsCountIncr();
+
+ awaitCondition(() -> pool.getNumberOfChannels() == 3);
+ assertThat(primeCalls.get()).isEqualTo(3);
+ assertThat(pool.scaleUpPrimeFailuresForTest()).isEqualTo(0);
+ }
+
+ @Test
+ public void primerRetriesExhaustedRejectsChannel() throws Exception {
+ AtomicInteger primeCalls = new AtomicInteger();
+ AtomicReference rejected = new AtomicReference<>();
+ AbandonWarnings warnings = new AbandonWarnings();
+ GcpChannelPrimer primer =
+ channel -> {
+ rejected.set((GcpManagedChannelTest.FakeManagedChannel) channel);
+ primeCalls.incrementAndGet();
+ return Futures.immediateFailedFuture(new IllegalStateException("prime failed"));
+ };
+ try {
+ pool = newPrimedPool(primer, Duration.ofSeconds(5), 3, builder());
+ ChannelRef hot = pool.channelRefs.get(0);
+ hot.setActiveStreamsForTest(6);
+
+ hot.activeStreamsCountIncr();
+
+ awaitCondition(() -> pool.scaleUpPrimeFailuresForTest() == 1);
+ assertThat(primeCalls.get()).isEqualTo(3);
+ assertThat(pool.getNumberOfChannels()).isEqualTo(2);
+ awaitCondition(() -> rejected.get() != null && rejected.get().isShutdown());
+ // Exhausting the attempts is an ordinary failure: the slot is released and nothing warns
+ // about a blocked call.
+ assertThat(pool.abandonedPrimeCountForTest()).isEqualTo(0);
+ assertThat(warnings.messages()).isEmpty();
+ } finally {
+ warnings.close();
+ }
+ }
+
+ @Test
+ public void primerBackoffIsCappedForManyAttempts() {
+ List bases = new CopyOnWriteArrayList<>();
+ List minDelays = new CopyOnWriteArrayList<>();
+ List maxDelays = new CopyOnWriteArrayList<>();
+
+ for (int attempt = 0; attempt < 49; attempt++) {
+ bases.add(GcpManagedChannel.primeBaseBackoffMillisForTest(attempt));
+ minDelays.add(GcpManagedChannel.primeBackoffMillisForTest(attempt, bound -> 0L));
+ maxDelays.add(GcpManagedChannel.primeBackoffMillisForTest(attempt, bound -> bound - 1));
+ }
+
+ assertThat(bases).hasSize(49);
+ assertThat(bases.subList(0, 6))
+ .containsExactly(100L, 200L, 400L, 800L, 1_600L, 3_200L)
+ .inOrder();
+ assertThat(bases.subList(6, 49)).containsExactlyElementsIn(Collections.nCopies(43, 5_000L));
+ assertThat(bases.stream().mapToLong(Long::longValue).sum()).isEqualTo(221_300L);
+ assertThat(minDelays.subList(6, 49)).containsExactlyElementsIn(Collections.nCopies(43, 2_500L));
+ assertThat(maxDelays.subList(6, 49)).containsExactlyElementsIn(Collections.nCopies(43, 4_999L));
+ assertThat(maxDelays.stream().mapToLong(Long::longValue).max().orElse(0)).isAtMost(5_000L);
+ }
+
+ @Test
+ public void primerBackoffAppliesEqualJitter() {
+ for (int attempt = 0; attempt < 20; attempt++) {
+ long base = GcpManagedChannel.primeBaseBackoffMillisForTest(attempt);
+ long half = base / 2;
+ assertThat(GcpManagedChannel.primeBackoffMillisForTest(attempt, bound -> 0L)).isEqualTo(half);
+ assertThat(GcpManagedChannel.primeBackoffMillisForTest(attempt, bound -> 7L))
+ .isEqualTo(half + 7L);
+ assertThat(GcpManagedChannel.primeBackoffMillisForTest(attempt, bound -> bound - 1))
+ .isEqualTo(base - 1);
+ for (int sample = 0; sample < 50; sample++) {
+ long delay = GcpManagedChannel.primeBackoffMillisForTest(attempt);
+ assertThat(delay).isAtLeast(half);
+ assertThat(delay).isLessThan(base);
+ }
+ }
+
+ Set distinct = new HashSet<>();
+ for (int sample = 0; sample < 200; sample++) {
+ distinct.add(GcpManagedChannel.primeBackoffMillisForTest(10));
+ }
+ assertThat(distinct.size()).isGreaterThan(1);
+ }
+
+ @Test
+ public void scaledChannelAfterDrainRunsPrimer() throws Exception {
+ AtomicInteger primeCalls = new AtomicInteger();
+ GcpChannelPrimer primer =
+ channel -> {
+ primeCalls.incrementAndGet();
+ return Futures.immediateVoidFuture();
+ };
+ pool = newPrimedPool(primer, Duration.ofSeconds(5), builder());
+ assertThat(primeCalls.get()).isEqualTo(0);
+ for (ChannelRef ref : pool.channelRefs) {
+ ((GcpManagedChannelTest.FakeManagedChannel) ref.getChannel())
+ .setState(ConnectivityState.READY);
+ }
+ awaitCondition(() -> pool.readyChannelCountForTest() == 2);
+ pool.checkScaleDown();
+ assertThat(pool.getNumberOfChannels()).isEqualTo(1);
+
+ ChannelRef active = pool.channelRefs.get(0);
+ active.setActiveStreamsForTest(3);
+ active.activeStreamsCountIncr();
+
+ awaitCondition(() -> pool.getNumberOfChannels() == 2);
+ assertThat(primeCalls.get()).isEqualTo(1);
+ }
+
+ private GcpManagedChannelTest.FakeManagedChannelBuilder builder() {
+ return new GcpManagedChannelTest.FakeManagedChannelBuilder(
+ () -> new GcpManagedChannelTest.FakeManagedChannel(stateExecutor));
+ }
+
+ private GcpManagedChannel newPrimedPool(
+ int initial,
+ int maximum,
+ GcpChannelPrimer primer,
+ Duration primeTimeout,
+ int primeMaxAttempts,
+ GcpManagedChannelTest.FakeManagedChannelBuilder delegate) {
+ GcpChannelPoolOptions poolOptions =
+ GcpChannelPoolOptions.newBuilder()
+ .setInitSize(initial)
+ .setMinSize(1)
+ .setMaxSize(maximum)
+ .setDynamicScaling(1, 3, Duration.ofSeconds(30))
+ .setScaleUpCooldown(Duration.ofNanos(1))
+ .setScaleDownConsecutiveLowLoadChecks(1)
+ .setMaxScaleUpPercent(30)
+ .setMaxScaleDownChannels(2)
+ .setDrainIdleGrace(Duration.ofMinutes(1))
+ .setChannelPrimer(primer)
+ .setChannelPrimeTimeout(primeTimeout)
+ .setChannelPrimeMaxAttempts(primeMaxAttempts)
+ .build();
+ return (GcpManagedChannel)
+ GcpManagedChannelBuilder.forDelegateBuilder(delegate)
+ .withOptions(
+ GcpManagedChannelOptions.newBuilder().withChannelPoolOptions(poolOptions).build())
+ .build();
+ }
+
+ private GcpManagedChannel newPrimedPool(
+ GcpChannelPrimer primer,
+ Duration primeTimeout,
+ GcpManagedChannelTest.FakeManagedChannelBuilder delegate) {
+ return newPrimedPool(primer, primeTimeout, 3, delegate);
+ }
+
+ private GcpManagedChannel newPrimedPool(
+ GcpChannelPrimer primer,
+ Duration primeTimeout,
+ int primeMaxAttempts,
+ GcpManagedChannelTest.FakeManagedChannelBuilder delegate) {
+ GcpChannelPoolOptions poolOptions =
+ GcpChannelPoolOptions.newBuilder()
+ .setInitSize(2)
+ .setMinSize(1)
+ .setMaxSize(3)
+ .setDynamicScaling(1, 3, Duration.ofSeconds(30))
+ .setScaleUpCooldown(Duration.ofNanos(1))
+ .setScaleDownConsecutiveLowLoadChecks(1)
+ .setMaxScaleUpPercent(30)
+ .setMaxScaleDownChannels(2)
+ .setDrainIdleGrace(Duration.ofMinutes(1))
+ .setChannelPrimer(primer)
+ .setChannelPrimeTimeout(primeTimeout)
+ .setChannelPrimeMaxAttempts(primeMaxAttempts)
+ .build();
+ return (GcpManagedChannel)
+ GcpManagedChannelBuilder.forDelegateBuilder(delegate)
+ .withOptions(
+ GcpManagedChannelOptions.newBuilder().withChannelPoolOptions(poolOptions).build())
+ .build();
+ }
+
+ private static void awaitCondition(java.util.concurrent.Callable condition) {
+ await().atMost(Duration.ofSeconds(5)).until(condition);
+ }
+
+ private static void awaitUninterruptibly(CountDownLatch latch) {
+ boolean interrupted = false;
+ while (true) {
+ try {
+ latch.await();
+ break;
+ } catch (InterruptedException failure) {
+ interrupted = true;
+ }
+ }
+ if (interrupted) {
+ Thread.currentThread().interrupt();
+ }
+ }
+
+ /** Captures the pool's abandoned-prime warnings for as long as it is open. */
+ private static final class AbandonWarnings extends Handler {
+ private static final Logger poolLogger = Logger.getLogger(GcpManagedChannel.class.getName());
+ private final List messages = new CopyOnWriteArrayList<>();
+
+ AbandonWarnings() {
+ poolLogger.addHandler(this);
+ }
+
+ List messages() {
+ return messages;
+ }
+
+ @Override
+ public void publish(LogRecord record) {
+ if (record.getLevel().intValue() >= Level.WARNING.intValue()
+ && record.getMessage().contains("priming abandoned")) {
+ messages.add(record.getMessage());
+ }
+ }
+
+ @Override
+ public void flush() {}
+
+ @Override
+ public void close() {
+ poolLogger.removeHandler(this);
+ }
+ }
+}
diff --git a/grpc-gcp-java/src/test/java/com/google/cloud/grpc/GcpManagedChannelScaleUpWorkerTest.java b/grpc-gcp-java/src/test/java/com/google/cloud/grpc/GcpManagedChannelScaleUpWorkerTest.java
new file mode 100644
index 000000000000..f801e59b070a
--- /dev/null
+++ b/grpc-gcp-java/src/test/java/com/google/cloud/grpc/GcpManagedChannelScaleUpWorkerTest.java
@@ -0,0 +1,414 @@
+/*
+ * Copyright 2026 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.grpc;
+
+import static com.google.common.truth.Truth.assertThat;
+import static org.awaitility.Awaitility.await;
+
+import com.google.cloud.grpc.GcpManagedChannel.ChannelRef;
+import com.google.cloud.grpc.GcpManagedChannelOptions.GcpChannelPoolOptions;
+import com.google.common.util.concurrent.MoreExecutors;
+import java.time.Duration;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+import org.junit.After;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+@RunWith(JUnit4.class)
+public final class GcpManagedChannelScaleUpWorkerTest {
+ private final ExecutorService executor = MoreExecutors.newDirectExecutorService();
+ private GcpManagedChannel pool;
+
+ @After
+ public void tearDown() {
+ if (pool != null) {
+ pool.shutdownNow();
+ }
+ executor.shutdownNow();
+ }
+
+ @Test
+ public void scaleUpAddsAtMostThirtyPercent() {
+ pool = newPool(10, 30);
+ ChannelRef hot = pool.channelRefs.get(0);
+ hot.setActiveStreamsForTest(99);
+
+ hot.activeStreamsCountIncr();
+
+ await().atMost(Duration.ofSeconds(5)).until(() -> pool.getNumberOfChannels() == 13);
+ }
+
+ @Test
+ public void scaleUpUsesTwoChannelFloorForSmallPool() {
+ pool = newPool(1, 5);
+ ChannelRef hot = pool.channelRefs.get(0);
+ hot.setActiveStreamsForTest(9);
+
+ hot.activeStreamsCountIncr();
+
+ await().atMost(Duration.ofSeconds(5)).until(() -> pool.getNumberOfChannels() == 3);
+ }
+
+ @Test
+ public void hottestChannelCanSignalBeforePoolAverageCrossesLimit() {
+ pool = newPool(4, 8);
+ pool.channelRefs.get(0).setActiveStreamsForTest(5);
+ pool.channelRefs.get(1).setActiveStreamsForTest(3);
+ pool.channelRefs.get(2).setActiveStreamsForTest(2);
+ pool.channelRefs.get(3).setActiveStreamsForTest(2);
+
+ pool.channelRefs.get(0).activeStreamsCountIncr();
+
+ await().atMost(Duration.ofSeconds(5)).until(() -> pool.getNumberOfChannels() == 5);
+ }
+
+ @Test
+ public void poolAverageCanSignalWhenSelectedChannelIsBelowMaximum() {
+ pool = newPool(2, 4);
+ ChannelRef selected = pool.channelRefs.get(0);
+ pool.channelRefs.get(1).setActiveStreamsForTest(12);
+
+ selected.activeStreamsCountIncr();
+
+ await().atMost(Duration.ofSeconds(5)).until(() -> pool.getNumberOfChannels() == 4);
+ }
+
+ @Test
+ public void scaleUpBuildsOffCallerThread() {
+ AtomicInteger builds = new AtomicInteger();
+ AtomicReference scaleUpThread = new AtomicReference<>();
+ GcpManagedChannelTest.FakeManagedChannelBuilder delegate =
+ new GcpManagedChannelTest.FakeManagedChannelBuilder(
+ () -> {
+ if (builds.incrementAndGet() > 2) {
+ scaleUpThread.set(Thread.currentThread().getName());
+ }
+ return new GcpManagedChannelTest.FakeManagedChannel(executor);
+ });
+ pool = newPool(2, 4, delegate);
+ String callerThread = Thread.currentThread().getName();
+
+ for (int i = 0; i < 7; i++) {
+ pool.channelRefs.get(0).activeStreamsCountIncr();
+ }
+
+ await().atMost(Duration.ofSeconds(5)).until(() -> scaleUpThread.get() != null);
+ assertThat(scaleUpThread.get()).isNotEqualTo(callerThread);
+ }
+
+ @Test
+ public void burstSignalsCoalesceWhileWorkerIsBusy() throws Exception {
+ AtomicInteger builds = new AtomicInteger();
+ CountDownLatch buildStarted = new CountDownLatch(1);
+ CountDownLatch releaseBuild = new CountDownLatch(1);
+ GcpManagedChannelTest.FakeManagedChannelBuilder delegate =
+ new GcpManagedChannelTest.FakeManagedChannelBuilder(
+ () -> {
+ if (builds.incrementAndGet() == 3) {
+ buildStarted.countDown();
+ try {
+ releaseBuild.await(5, TimeUnit.SECONDS);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ return new GcpManagedChannelTest.FakeManagedChannel(executor);
+ });
+ pool = newPool(2, 10, delegate);
+ AtomicLong clock = new AtomicLong(1);
+ pool.setNanoClock(clock::get);
+ ChannelRef hot = pool.channelRefs.get(0);
+ for (int i = 0; i < 7; i++) {
+ hot.activeStreamsCountIncr();
+ }
+ assertThat(buildStarted.await(5, TimeUnit.SECONDS)).isTrue();
+
+ for (int i = 7; i < 30; i++) {
+ hot.activeStreamsCountIncr();
+ }
+ clock.incrementAndGet();
+ releaseBuild.countDown();
+
+ await().atMost(Duration.ofSeconds(5)).until(() -> pool.getNumberOfChannels() == 5);
+ await()
+ .during(Duration.ofMillis(100))
+ .atMost(Duration.ofSeconds(1))
+ .until(() -> pool.getNumberOfChannels() == 5);
+ }
+
+ @Test
+ public void buildExceptionClosesPartialBatchAndFutureSignalStillScales() {
+ AtomicInteger builds = new AtomicInteger();
+ AtomicLong clock = new AtomicLong(1);
+ AtomicReference partialBuild =
+ new AtomicReference<>();
+ GcpManagedChannelTest.FakeManagedChannelBuilder delegate =
+ new GcpManagedChannelTest.FakeManagedChannelBuilder(
+ () -> {
+ int build = builds.incrementAndGet();
+ if (build == 3) {
+ GcpManagedChannelTest.FakeManagedChannel channel =
+ new GcpManagedChannelTest.FakeManagedChannel(executor);
+ partialBuild.set(channel);
+ return channel;
+ }
+ if (build == 4) {
+ throw new AssertionError("build failed");
+ }
+ return new GcpManagedChannelTest.FakeManagedChannel(executor);
+ });
+ pool = newPool(2, 6, delegate);
+ pool.setNanoClock(clock::get);
+ ChannelRef hot = pool.channelRefs.get(0);
+ hot.setActiveStreamsForTest(9);
+
+ hot.activeStreamsCountIncr();
+
+ await().atMost(Duration.ofSeconds(5)).until(() -> builds.get() == 4);
+ await().atMost(Duration.ofSeconds(5)).until(() -> partialBuild.get().isShutdown());
+ clock.incrementAndGet();
+ hot.activeStreamsCountIncr();
+ await().atMost(Duration.ofSeconds(5)).until(() -> pool.getNumberOfChannels() == 4);
+ }
+
+ @Test
+ public void scaleUpCooldownIsHonored() {
+ AtomicLong clock = new AtomicLong(100);
+ pool = newPool(2, 6, Duration.ofNanos(10));
+ pool.setNanoClock(clock::get);
+ ChannelRef hot = pool.channelRefs.get(0);
+ hot.setActiveStreamsForTest(20);
+
+ hot.activeStreamsCountIncr();
+ await().atMost(Duration.ofSeconds(5)).until(() -> pool.getNumberOfChannels() == 4);
+
+ hot.activeStreamsCountIncr();
+ await()
+ .during(Duration.ofMillis(100))
+ .atMost(Duration.ofSeconds(1))
+ .until(() -> pool.getNumberOfChannels() == 4);
+
+ clock.addAndGet(11);
+ hot.activeStreamsCountIncr();
+ await().atMost(Duration.ofSeconds(5)).until(() -> pool.getNumberOfChannels() == 6);
+ }
+
+ @Test
+ public void scaleUpClampsToMaxSize() {
+ pool = newPool(4, 5);
+ ChannelRef hot = pool.channelRefs.get(0);
+ hot.setActiveStreamsForTest(30);
+
+ hot.activeStreamsCountIncr();
+
+ await().atMost(Duration.ofSeconds(5)).until(() -> pool.getNumberOfChannels() == 5);
+ }
+
+ @Test
+ public void scaleUpCountsOnlyActiveChannels() {
+ pool = newPool(2, 5);
+ ChannelRef hot = pool.channelRefs.get(0);
+ pool.channelRefs.get(1).deactivateForTest();
+ hot.setActiveStreamsForTest(6);
+
+ hot.activeStreamsCountIncr();
+
+ await().atMost(Duration.ofSeconds(1)).until(() -> pool.getNumberOfChannels() == 4);
+ }
+
+ @Test
+ public void shutdownDuringScaleUpClosesUnpublishedChannel() throws Exception {
+ AtomicInteger builds = new AtomicInteger();
+ CountDownLatch buildStarted = new CountDownLatch(1);
+ CountDownLatch releaseBuild = new CountDownLatch(1);
+ AtomicReference unpublished = new AtomicReference<>();
+ GcpManagedChannelTest.FakeManagedChannelBuilder delegate =
+ new GcpManagedChannelTest.FakeManagedChannelBuilder(
+ () -> {
+ if (builds.incrementAndGet() == 3) {
+ GcpManagedChannelTest.FakeManagedChannel channel =
+ new GcpManagedChannelTest.FakeManagedChannel(executor);
+ unpublished.set(channel);
+ buildStarted.countDown();
+ try {
+ releaseBuild.await(5, TimeUnit.SECONDS);
+ } catch (InterruptedException failure) {
+ Thread.currentThread().interrupt();
+ }
+ return channel;
+ }
+ return new GcpManagedChannelTest.FakeManagedChannel(executor);
+ });
+ pool = newPool(2, 3, delegate);
+ ChannelRef hot = pool.channelRefs.get(0);
+ hot.setActiveStreamsForTest(9);
+ hot.activeStreamsCountIncr();
+ assertThat(buildStarted.await(5, TimeUnit.SECONDS)).isTrue();
+
+ pool.shutdownNow();
+ releaseBuild.countDown();
+
+ await().atMost(Duration.ofSeconds(5)).until(() -> unpublished.get().isShutdown());
+ assertThat(pool.getNumberOfChannels()).isEqualTo(2);
+ }
+
+ @Test
+ public void shutdownDuringScaleUpClosesChannelPublishingUnderPoolMonitor() throws Exception {
+ AtomicInteger builds = new AtomicInteger();
+ CountDownLatch publishing = new CountDownLatch(1);
+ CountDownLatch releasePublish = new CountDownLatch(1);
+ AtomicReference publishingChannel =
+ new AtomicReference<>();
+ GcpManagedChannelTest.FakeManagedChannelBuilder delegate =
+ new GcpManagedChannelTest.FakeManagedChannelBuilder(
+ () -> {
+ if (builds.incrementAndGet() == 3) {
+ GcpManagedChannelTest.FakeManagedChannel channel =
+ new GcpManagedChannelTest.FakeManagedChannel(executor) {
+ @Override
+ public io.grpc.ConnectivityState getState(boolean requestConnection) {
+ publishing.countDown();
+ try {
+ releasePublish.await(5, TimeUnit.SECONDS);
+ } catch (InterruptedException failure) {
+ Thread.currentThread().interrupt();
+ }
+ return super.getState(requestConnection);
+ }
+ };
+ publishingChannel.set(channel);
+ return channel;
+ }
+ return new GcpManagedChannelTest.FakeManagedChannel(executor);
+ });
+ pool = newPool(2, 3, delegate);
+ ChannelRef hot = pool.channelRefs.get(0);
+ hot.setActiveStreamsForTest(9);
+ hot.activeStreamsCountIncr();
+ assertThat(publishing.await(5, TimeUnit.SECONDS)).isTrue();
+
+ AtomicReference shutdownThread = new AtomicReference<>();
+ ExecutorService shutdownExecutor =
+ Executors.newSingleThreadExecutor(
+ command -> {
+ Thread thread = new Thread(command);
+ shutdownThread.set(thread);
+ return thread;
+ });
+ try {
+ Future shutdown = shutdownExecutor.submit(pool::shutdown);
+ await()
+ .atMost(Duration.ofSeconds(5))
+ .until(
+ () ->
+ shutdownThread.get() != null
+ && shutdownThread.get().getState() == Thread.State.BLOCKED);
+ releasePublish.countDown();
+ shutdown.get(5, TimeUnit.SECONDS);
+ await().atMost(Duration.ofSeconds(5)).until(() -> publishingChannel.get().isShutdown());
+ assertThat(pool.channelRefs).hasSize(3);
+ assertThat(
+ pool.channelRefs.stream()
+ .allMatch(channelRef -> channelRef.getChannel().isShutdown()))
+ .isTrue();
+ } finally {
+ releasePublish.countDown();
+ shutdownExecutor.shutdownNow();
+ }
+ }
+
+ @Test
+ public void removedChannelReadersTolerateConcurrentMutation() throws Exception {
+ pool = newPool(2, 3);
+ ChannelRef removed = pool.channelRefs.get(0);
+ ExecutorService mutator = Executors.newSingleThreadExecutor();
+ AtomicReference failure = new AtomicReference<>();
+ CountDownLatch started = new CountDownLatch(1);
+ Future mutation =
+ mutator.submit(
+ () -> {
+ started.countDown();
+ try {
+ for (int i = 0; i < 200; i++) {
+ pool.removedChannelRefs.add(removed);
+ pool.removedChannelRefs.remove(removed);
+ }
+ } catch (Throwable throwable) {
+ failure.set(throwable);
+ }
+ });
+ try {
+ assertThat(started.await(1, TimeUnit.SECONDS)).isTrue();
+ for (int i = 0; i < 200; i++) {
+ pool.isShutdown();
+ pool.isTerminated();
+ pool.awaitTermination(0, TimeUnit.NANOSECONDS);
+ }
+ await().atMost(Duration.ofSeconds(1)).until(mutation::isDone);
+ assertThat(failure.get()).isNull();
+ } finally {
+ mutator.shutdownNow();
+ }
+ }
+
+ private GcpManagedChannel newPool(int initial, int maximum) {
+ return newPool(initial, maximum, Duration.ofNanos(1));
+ }
+
+ private GcpManagedChannel newPool(int initial, int maximum, Duration scaleUpCooldown) {
+ return newPool(
+ initial,
+ maximum,
+ new GcpManagedChannelTest.FakeManagedChannelBuilder(
+ () -> new GcpManagedChannelTest.FakeManagedChannel(executor)),
+ scaleUpCooldown);
+ }
+
+ private GcpManagedChannel newPool(
+ int initial, int maximum, GcpManagedChannelTest.FakeManagedChannelBuilder delegate) {
+ return newPool(initial, maximum, delegate, Duration.ofNanos(1));
+ }
+
+ private GcpManagedChannel newPool(
+ int initial,
+ int maximum,
+ GcpManagedChannelTest.FakeManagedChannelBuilder delegate,
+ Duration scaleUpCooldown) {
+ GcpChannelPoolOptions options =
+ GcpChannelPoolOptions.newBuilder()
+ .setInitSize(initial)
+ .setMinSize(initial)
+ .setMaxSize(maximum)
+ .setDynamicScaling(2, 5, Duration.ofMinutes(1))
+ .setScaleUpCooldown(scaleUpCooldown)
+ .setMaxScaleUpPercent(30)
+ .build();
+ return (GcpManagedChannel)
+ GcpManagedChannelBuilder.forDelegateBuilder(delegate)
+ .withOptions(
+ GcpManagedChannelOptions.newBuilder().withChannelPoolOptions(options).build())
+ .build();
+ }
+}
diff --git a/grpc-gcp-java/src/test/java/com/google/cloud/grpc/GcpManagedChannelSkewFixTest.java b/grpc-gcp-java/src/test/java/com/google/cloud/grpc/GcpManagedChannelSkewFixTest.java
new file mode 100644
index 000000000000..8991101b7005
--- /dev/null
+++ b/grpc-gcp-java/src/test/java/com/google/cloud/grpc/GcpManagedChannelSkewFixTest.java
@@ -0,0 +1,258 @@
+/*
+ * Copyright 2026 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.grpc;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import com.google.cloud.grpc.GcpManagedChannel.ChannelAffinityRef;
+import com.google.cloud.grpc.GcpManagedChannel.ChannelRef;
+import com.google.cloud.grpc.GcpManagedChannelOptions.ChannelPickStrategy;
+import com.google.cloud.grpc.GcpManagedChannelOptions.GcpChannelPoolOptions;
+import com.google.cloud.grpc.GcpManagedChannelOptions.GcpResiliencyOptions;
+import com.google.cloud.grpc.proto.ApiConfig;
+import com.google.common.util.concurrent.MoreExecutors;
+import java.time.Duration;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.junit.After;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+@RunWith(JUnit4.class)
+public final class GcpManagedChannelSkewFixTest {
+ private final ExecutorService executor = MoreExecutors.newDirectExecutorService();
+ private GcpManagedChannel pool;
+
+ @After
+ public void tearDown() {
+ if (pool != null) {
+ pool.shutdownNow();
+ }
+ executor.shutdownNow();
+ }
+
+ @Test
+ public void scaleDownRequiresThreeConsecutiveLowLoadChecks() {
+ pool = newPool(6, 1);
+ reserveOneStreamPerChannel();
+
+ pool.checkScaleDown();
+ pool.checkScaleDown();
+ assertThat(pool.getNumberOfChannels()).isEqualTo(6);
+
+ pool.checkScaleDown();
+ assertThat(pool.getNumberOfChannels()).isEqualTo(4);
+ }
+
+ @Test
+ public void scaleDownRemovesAtMostTwoChannelsPerCheck() {
+ pool = newPool(48, 2);
+ reserveOneStreamPerChannel();
+
+ pool.checkScaleDown();
+ pool.checkScaleDown();
+ pool.checkScaleDown();
+
+ assertThat(pool.getNumberOfChannels()).isEqualTo(46);
+ assertThat(pool.removedChannelRefs).hasSize(2);
+ }
+
+ @Test
+ public void powerOfTwoSamplesWithReplacementAndKeepsFirstTie() {
+ pool = newPool(2, 2);
+ ChannelRef first = pool.channelRefs.get(0);
+ ChannelRef second = pool.channelRefs.get(1);
+ AtomicInteger sample = new AtomicInteger();
+ int[] indexes = {0, 1, 1, 0};
+ pool.setCandidateIndexPickerForTest(ignored -> indexes[sample.getAndIncrement()]);
+
+ assertThat(pool.pickFromCandidates(pool.channelRefs)).isSameInstanceAs(first);
+ assertThat(pool.pickFromCandidates(pool.channelRefs)).isSameInstanceAs(second);
+ }
+
+ @Test
+ public void pickFromCandidates_concurrentScaleDownShrink_doesNotThrowIndexOutOfBounds() {
+ pool = newPool(2, 1);
+ AtomicBoolean shrunk = new AtomicBoolean();
+ pool.setCandidateIndexPickerForTest(
+ size -> {
+ if (shrunk.compareAndSet(false, true)) {
+ pool.channelRefs.remove(size - 1);
+ }
+ return size - 1;
+ });
+
+ assertThat(pool.pickFromCandidates(pool.channelRefs)).isNotNull();
+ }
+
+ @Test
+ public void pickLeastBusyNoFallback_linearScan_skipsInactiveChannels() {
+ pool = newPool(2, 2, ChannelPickStrategy.LINEAR_SCAN);
+ ChannelRef inactive = pool.channelRefs.get(0);
+ ChannelRef active = pool.channelRefs.get(1);
+ inactive.deactivateForTest();
+
+ assertThat(pool.getChannelRef(null)).isSameInstanceAs(active);
+ }
+
+ @Test
+ public void roundRobin_skipsInactiveChannels() {
+ pool = newPool(2, 2);
+ ChannelRef active = pool.channelRefs.get(0);
+ pool.channelRefs.get(1).deactivateForTest();
+
+ assertThat(pool.getChannelRefRoundRobin()).isSameInstanceAs(active);
+ }
+
+ @Test
+ public void roundRobin_dynamicEmptyPool_createsFirstChannel() {
+ GcpChannelPoolOptions options =
+ GcpChannelPoolOptions.newBuilder()
+ .setInitSize(0)
+ .setMinSize(0)
+ .setMaxSize(2)
+ .setDynamicScaling(10, 20, Duration.ofMinutes(1))
+ .build();
+ pool =
+ (GcpManagedChannel)
+ GcpManagedChannelBuilder.forDelegateBuilder(
+ new GcpManagedChannelTest.FakeManagedChannelBuilder(
+ () -> new GcpManagedChannelTest.FakeManagedChannel(executor)))
+ .withOptions(
+ GcpManagedChannelOptions.newBuilder().withChannelPoolOptions(options).build())
+ .build();
+
+ assertThat(pool.channelRefs).isEmpty();
+ assertThat(pool.getChannelRefRoundRobin()).isNotNull();
+ assertThat(pool.channelRefs).hasSize(1);
+ }
+
+ @Test
+ public void fallbackPicker_skipsInactiveChannels() {
+ pool = newPoolWithFallback();
+ ChannelRef inactive = pool.channelRefs.get(0);
+ ChannelRef active = pool.channelRefs.get(1);
+ inactive.deactivateForTest();
+ pool.fallbackMapForTest().put(active.getId(), new java.util.concurrent.ConcurrentHashMap<>());
+
+ assertThat(pool.getChannelRef(null)).isSameInstanceAs(active);
+ }
+
+ @Test
+ public void activeStreamExtrema_skipInactiveChannels() {
+ pool = newPool(2, 2);
+ ChannelRef inactive = pool.channelRefs.get(0);
+ ChannelRef active = pool.channelRefs.get(1);
+ inactive.setActiveStreamsForTest(99);
+ active.setActiveStreamsForTest(3);
+ inactive.deactivateForTest();
+
+ assertThat(pool.getMinActiveStreams()).isEqualTo(3);
+ assertThat(pool.getMaxActiveStreams()).isEqualTo(3);
+ }
+
+ @Test
+ public void dynamicPowerOfTwoPickDoesNotReadLegacyWatermarkLoad() {
+ GcpChannelPoolOptions options =
+ GcpChannelPoolOptions.newBuilder()
+ .setInitSize(2)
+ .setMinSize(2)
+ .setMaxSize(4)
+ .setDynamicScaling(10, 20, Duration.ofMinutes(1))
+ .setChannelPickStrategy(ChannelPickStrategy.POWER_OF_TWO)
+ .build();
+ pool =
+ new GcpManagedChannel(
+ new GcpManagedChannelTest.FakeManagedChannelBuilder(
+ () -> new GcpManagedChannelTest.FakeManagedChannel(executor)),
+ ApiConfig.getDefaultInstance(),
+ GcpManagedChannelOptions.newBuilder().withChannelPoolOptions(options).build()) {
+ @Override
+ public int getMaxActiveStreams() {
+ throw new AssertionError("legacy watermark load must be lazy");
+ }
+ };
+
+ assertThat(pool.getChannelRef(null)).isNotNull();
+ }
+
+ @Test
+ public void affinityReferenceStaysStickyUntilDelegateShutdown() {
+ pool = newPool(4, 2);
+ reserveOneStreamPerChannel();
+ ChannelRef victim = pool.channelRefs.get(0);
+ ChannelAffinityRef handle = new ChannelAffinityRef();
+ handle.setChannelIdForTest(victim.getId());
+
+ pool.checkScaleDown();
+ pool.checkScaleDown();
+ pool.checkScaleDown();
+ assertThat(pool.getChannelRefByAffinityRef(handle)).isSameInstanceAs(victim);
+
+ victim.getChannel().shutdownNow();
+ assertThat(pool.getChannelRefByAffinityRef(handle)).isNotSameInstanceAs(victim);
+ }
+
+ private GcpManagedChannel newPool(int initial, int minimum) {
+ return newPool(initial, minimum, ChannelPickStrategy.POWER_OF_TWO);
+ }
+
+ private GcpManagedChannel newPool(
+ int initial, int minimum, ChannelPickStrategy channelPickStrategy) {
+ GcpChannelPoolOptions options =
+ GcpChannelPoolOptions.newBuilder()
+ .setInitSize(initial)
+ .setMinSize(minimum)
+ .setMaxSize(initial)
+ .setDynamicScaling(10, 20, Duration.ofMinutes(1))
+ .setScaleDownConsecutiveLowLoadChecks(3)
+ .setMaxScaleDownChannels(2)
+ .setChannelPickStrategy(channelPickStrategy)
+ .build();
+ return (GcpManagedChannel)
+ GcpManagedChannelBuilder.forDelegateBuilder(
+ new GcpManagedChannelTest.FakeManagedChannelBuilder(
+ () -> new GcpManagedChannelTest.FakeManagedChannel(executor)))
+ .withOptions(
+ GcpManagedChannelOptions.newBuilder().withChannelPoolOptions(options).build())
+ .build();
+ }
+
+ private GcpManagedChannel newPoolWithFallback() {
+ GcpChannelPoolOptions options =
+ GcpChannelPoolOptions.newBuilder().setInitSize(2).setMinSize(2).setMaxSize(2).build();
+ return (GcpManagedChannel)
+ GcpManagedChannelBuilder.forDelegateBuilder(
+ new GcpManagedChannelTest.FakeManagedChannelBuilder(
+ () -> new GcpManagedChannelTest.FakeManagedChannel(executor)))
+ .withOptions(
+ GcpManagedChannelOptions.newBuilder()
+ .withChannelPoolOptions(options)
+ .withResiliencyOptions(
+ GcpResiliencyOptions.newBuilder().setNotReadyFallback(true).build())
+ .build())
+ .build();
+ }
+
+ private void reserveOneStreamPerChannel() {
+ for (ChannelRef channelRef : pool.channelRefs) {
+ channelRef.activeStreamsCountIncr();
+ }
+ }
+}
diff --git a/grpc-gcp-java/src/test/java/com/google/cloud/grpc/GcpManagedChannelTest.java b/grpc-gcp-java/src/test/java/com/google/cloud/grpc/GcpManagedChannelTest.java
index c765efac7675..6d8567e85ebf 100644
--- a/grpc-gcp-java/src/test/java/com/google/cloud/grpc/GcpManagedChannelTest.java
+++ b/grpc-gcp-java/src/test/java/com/google/cloud/grpc/GcpManagedChannelTest.java
@@ -19,6 +19,7 @@
import static com.google.cloud.grpc.GcpManagedChannel.getKeysFromMessage;
import static com.google.common.truth.Truth.assertThat;
import static com.google.common.truth.Truth.assertWithMessage;
+import static org.awaitility.Awaitility.await;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
@@ -54,21 +55,19 @@
import io.opencensus.metrics.LabelValue;
import java.io.File;
import java.io.InputStream;
-import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.net.URL;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
-import java.util.Set;
+import java.util.concurrent.CountDownLatch;
import java.util.concurrent.Executor;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
@@ -79,7 +78,6 @@
import java.util.logging.Level;
import java.util.logging.LogRecord;
import java.util.logging.Logger;
-import java.util.stream.Collectors;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -102,6 +100,15 @@ public final class GcpManagedChannelTest {
private final List logRecords = new LinkedList<>();
+ @Test
+ public void ceilDivReturnsZeroForNonPositiveNumerator() throws Exception {
+ Method ceilDiv = GcpManagedChannel.class.getDeclaredMethod("ceilDiv", long.class, int.class);
+ ceilDiv.setAccessible(true);
+
+ assertThat(ceilDiv.invoke(null, -1L, 3)).isEqualTo(0);
+ assertThat(ceilDiv.invoke(null, 0L, 3)).isEqualTo(0);
+ }
+
private String lastLogMessage() {
return lastLogMessage(1);
}
@@ -280,7 +287,9 @@ public void testGetChannelRefInitializationWithMinSize() throws InterruptedExcep
GcpManagedChannelBuilder.forDelegateBuilder(builder).withOptions(options).build();
// Should have 2 channels since the beginning.
assertThat(gcpChannel.channelRefs.size()).isEqualTo(2);
- TimeUnit.MILLISECONDS.sleep(50);
+ await()
+ .atMost(Duration.ofSeconds(1))
+ .until(() -> gcpChannel.getState(false) != ConnectivityState.IDLE);
// The connection establishment must have been started on these two channels.
assertThat(gcpChannel.getState(false))
.isAnyOf(
@@ -350,7 +359,7 @@ public void testChannelAffinityRefInvalidChannelIdPicksAvailableChannel() throws
gcpChannel = createPoolWithFakeReadyChannels(executorService, 2);
ChannelAffinityRef affinityRef = new ChannelAffinityRef();
- setChannelAffinityRefState(affinityRef, 1000);
+ affinityRef.setChannelIdForTest(999);
ChannelRef selected = gcpChannel.getChannelRefByAffinityRef(affinityRef);
ChannelRef next = gcpChannel.getChannelRefByAffinityRef(affinityRef);
@@ -365,7 +374,122 @@ public void testChannelAffinityRefInvalidChannelIdPicksAvailableChannel() throws
}
@Test
- public void testChannelAffinityRefRemovedChannelPicksAvailableChannel() throws Exception {
+ public void fallbackUsesChannelIdMapAfterPoolHasIndexGap() {
+ resetGcpChannel();
+ ExecutorService executorService = Executors.newSingleThreadExecutor();
+ try {
+ List channels = new ArrayList<>();
+ for (int i = 0; i < 3; i++) {
+ FakeManagedChannel channel = new FakeManagedChannel(executorService);
+ channel.setState(ConnectivityState.READY);
+ channels.add(channel);
+ }
+ gcpChannel =
+ (GcpManagedChannel)
+ GcpManagedChannelBuilder.forDelegateBuilder(new FakeManagedChannelBuilder(channels))
+ .withOptions(
+ GcpManagedChannelOptions.newBuilder()
+ .withChannelPoolOptions(
+ GcpChannelPoolOptions.newBuilder()
+ .setMinSize(3)
+ .setMaxSize(3)
+ .build())
+ .withResiliencyOptions(
+ GcpResiliencyOptions.newBuilder().setNotReadyFallback(true).build())
+ .build())
+ .build();
+ ChannelRef removed = gcpChannel.channelRefs.get(0);
+ ChannelRef mapped = gcpChannel.channelRefs.get(1);
+ ChannelRef fallback = gcpChannel.channelRefs.get(2);
+ String key = "session";
+ gcpChannel.bind(mapped, Collections.singletonList(key));
+ gcpChannel.processChannelStateChange(mapped.getId(), ConnectivityState.TRANSIENT_FAILURE);
+ gcpChannel.fallbackMapForTest().get(mapped.getId()).put(key, fallback.getId());
+ gcpChannel.channelRefs.remove(removed);
+
+ assertThat(gcpChannel.getChannelRef(key)).isSameInstanceAs(fallback);
+ } finally {
+ gcpChannel.shutdownNow();
+ executorService.shutdownNow();
+ }
+ }
+
+ @Test
+ public void readyAccountingRemainsExactWhenFreshChannelReplacesReadyDrainingChannel() {
+ resetGcpChannel();
+ ExecutorService executorService = Executors.newSingleThreadExecutor();
+ try {
+ gcpChannel =
+ new GcpManagedChannel(
+ new FakeManagedChannelBuilder(
+ () -> {
+ FakeManagedChannel channel = new FakeManagedChannel(executorService);
+ channel.setState(ConnectivityState.READY);
+ return channel;
+ }),
+ ApiConfig.getDefaultInstance(),
+ GcpManagedChannelOptions.newBuilder()
+ .withChannelPoolOptions(
+ GcpChannelPoolOptions.newBuilder()
+ .setInitSize(2)
+ .setMinSize(2)
+ .setMaxSize(3)
+ .build())
+ .build());
+ assertThat(gcpChannel.readyChannelCountForTest()).isEqualTo(2);
+
+ ChannelRef draining = gcpChannel.channelRefs.get(0);
+ gcpChannel.channelRefs.remove(draining);
+ draining.deactivateForTest();
+ gcpChannel.removedChannelRefs.add(draining);
+ assertThat(gcpChannel.readyChannelCountForTest()).isEqualTo(1);
+
+ ChannelRef fresh = gcpChannel.createNewChannel();
+ assertThat(fresh).isNotSameInstanceAs(draining);
+ assertThat(gcpChannel.channelRefs).contains(fresh);
+ assertThat(gcpChannel.channelRefs).doesNotContain(draining);
+ assertThat(gcpChannel.removedChannelRefs).contains(draining);
+ assertThat(gcpChannel.readyChannelCountForTest()).isEqualTo(2);
+ fresh.deactivateForTest();
+ fresh.deactivateForTest();
+ assertThat(gcpChannel.readyChannelCountForTest()).isEqualTo(1);
+ } finally {
+ gcpChannel.shutdownNow();
+ executorService.shutdownNow();
+ }
+ }
+
+ @Test
+ public void readyRemovedChannelIsNeverReused() {
+ resetGcpChannel();
+ ExecutorService executorService = MoreExecutors.newDirectExecutorService();
+ try {
+ gcpChannel =
+ new GcpManagedChannel(
+ new FakeManagedChannelBuilder(() -> new FakeManagedChannel(executorService)),
+ ApiConfig.getDefaultInstance(),
+ GcpManagedChannelOptions.newBuilder().build());
+ ChannelRef firstDraining = gcpChannel.new ChannelRef(new FakeManagedChannel(executorService));
+ ChannelRef secondDraining =
+ gcpChannel.new ChannelRef(new FakeManagedChannel(executorService));
+ firstDraining.deactivateForTest();
+ secondDraining.deactivateForTest();
+ gcpChannel.removedChannelRefs.add(firstDraining);
+ gcpChannel.removedChannelRefs.add(secondDraining);
+
+ ChannelRef fresh = gcpChannel.createNewChannel();
+ assertThat(fresh).isNotSameInstanceAs(firstDraining);
+ assertThat(fresh).isNotSameInstanceAs(secondDraining);
+ assertThat(gcpChannel.channelRefs).contains(fresh);
+ assertThat(gcpChannel.removedChannelRefs).containsExactly(firstDraining, secondDraining);
+ } finally {
+ gcpChannel.shutdownNow();
+ executorService.shutdownNow();
+ }
+ }
+
+ @Test
+ public void testChannelAffinityRefRemovedOpenChannelStaysStickyUntilShutdown() throws Exception {
resetGcpChannel();
ExecutorService executorService = Executors.newSingleThreadExecutor();
try {
@@ -374,18 +498,189 @@ public void testChannelAffinityRefRemovedChannelPicksAvailableChannel() throws E
ChannelRef removed = gcpChannel.getChannelRefByAffinityRef(affinityRef);
gcpChannel.channelRefs.remove(removed);
- deactivateChannelRef(removed);
+ removed.deactivateForTest();
ChannelRef selected = gcpChannel.getChannelRefByAffinityRef(affinityRef);
+ assertThat(selected).isSameInstanceAs(removed);
+ assertThat(selected.isActive()).isFalse();
+
+ removed.getChannel().shutdownNow();
+ ChannelRef afterShutdown = gcpChannel.getChannelRefByAffinityRef(affinityRef);
ChannelRef next = gcpChannel.getChannelRefByAffinityRef(affinityRef);
+ assertThat(afterShutdown).isNotSameInstanceAs(removed);
+ assertThat(afterShutdown).isIn(gcpChannel.channelRefs);
+ assertThat(afterShutdown.isActive()).isTrue();
+ assertThat(next).isSameInstanceAs(afterShutdown);
+ } finally {
+ gcpChannel.shutdownNow();
+ executorService.shutdownNow();
+ }
+ }
- assertThat(selected).isNotSameInstanceAs(removed);
- assertThat(selected).isIn(gcpChannel.channelRefs);
+ @Test
+ public void inactiveAffinityRemovalUsesConcurrentRebind() {
+ resetGcpChannel();
+ ExecutorService channelExecutor = MoreExecutors.newDirectExecutorService();
+ try {
+ gcpChannel = createPoolWithFakeReadyChannels(channelExecutor, 3);
+ ChannelRef inactive = gcpChannel.channelRefs.get(0);
+ ChannelRef rebound = gcpChannel.channelRefs.get(1);
+ ChannelRef alternative = gcpChannel.channelRefs.get(2);
+ String key = "session";
+ gcpChannel.bind(inactive, Collections.singletonList(key));
+ inactive.deactivateForTest();
+ rebound.setActiveStreamsForTest(10);
+ gcpChannel.setCandidateIndexPickerForTest(bound -> 2);
+ gcpChannel.setInactiveMappingRemovalHookForTest(
+ () -> gcpChannel.bind(rebound, Collections.singletonList(key)));
+
+ ChannelRef selected = gcpChannel.getChannelRef(key);
+
+ assertThat(selected).isSameInstanceAs(rebound);
+ assertThat(selected).isNotSameInstanceAs(alternative);
+ assertThat(gcpChannel.affinityKeyToChannelRef).containsEntry(key, rebound);
+ assertThat(gcpChannel.affinityKeyLastUsed).containsKey(key);
+ } finally {
+ gcpChannel.shutdownNow();
+ channelExecutor.shutdownNow();
+ }
+ }
+
+ @Test
+ public void inactiveAffinityRemovalRetriesWhenConcurrentRebindBecomesInactive() {
+ resetGcpChannel();
+ ExecutorService channelExecutor = MoreExecutors.newDirectExecutorService();
+ try {
+ gcpChannel = createPoolWithFakeReadyChannels(channelExecutor, 3);
+ ChannelRef inactive = gcpChannel.channelRefs.get(0);
+ ChannelRef rebound = gcpChannel.channelRefs.get(1);
+ ChannelRef alternative = gcpChannel.channelRefs.get(2);
+ String key = "session";
+ gcpChannel.bind(inactive, Collections.singletonList(key));
+ inactive.deactivateForTest();
+ gcpChannel.setCandidateIndexPickerForTest(bound -> 2);
+ gcpChannel.setInactiveMappingRemovalHookForTest(
+ () -> {
+ gcpChannel.bind(rebound, Collections.singletonList(key));
+ rebound.deactivateForTest();
+ });
+
+ ChannelRef selected = gcpChannel.getChannelRef(key);
+
+ assertThat(selected).isSameInstanceAs(alternative);
assertThat(selected.isActive()).isTrue();
- assertThat(next).isSameInstanceAs(selected);
+ assertThat(gcpChannel.affinityKeyToChannelRef).containsEntry(key, alternative);
+ assertThat(gcpChannel.affinityKeyLastUsed).containsKey(key);
} finally {
gcpChannel.shutdownNow();
- executorService.shutdownNow();
+ channelExecutor.shutdownNow();
+ }
+ }
+
+ @Test
+ public void bindReturnsRedirectedActiveChannel() throws Exception {
+ resetGcpChannel();
+ ExecutorService channelExecutor = MoreExecutors.newDirectExecutorService();
+ try {
+ gcpChannel = createPoolWithFakeReadyChannels(channelExecutor, 2);
+ ChannelRef inactive = gcpChannel.channelRefs.get(0);
+ ChannelRef active = gcpChannel.channelRefs.get(1);
+ inactive.deactivateForTest();
+ gcpChannel.setCandidateIndexPickerForTest(bound -> 1);
+ Method bind = GcpManagedChannel.class.getDeclaredMethod("bind", ChannelRef.class, List.class);
+
+ Object bound = bind.invoke(gcpChannel, inactive, Collections.singletonList("session"));
+
+ assertThat(bound).isSameInstanceAs(active);
+ assertThat(gcpChannel.affinityKeyToChannelRef).containsEntry("session", active);
+ } finally {
+ gcpChannel.shutdownNow();
+ channelExecutor.shutdownNow();
+ }
+ }
+
+ @Test
+ public void affinityTouchRetriesWhenMappingIsUnboundConcurrently() throws Exception {
+ resetGcpChannel();
+ ExecutorService channelExecutor = MoreExecutors.newDirectExecutorService();
+ ExecutorService caller = Executors.newSingleThreadExecutor();
+ CountDownLatch mappingRead = new CountDownLatch(1);
+ CountDownLatch releaseTouch = new CountDownLatch(1);
+ AtomicBoolean firstClockRead = new AtomicBoolean(true);
+ try {
+ gcpChannel = createPoolWithFakeReadyChannels(channelExecutor, 2);
+ String key = "session";
+ gcpChannel.bind(gcpChannel.channelRefs.get(0), Collections.singletonList(key));
+ gcpChannel.setNanoClock(
+ () -> {
+ if (firstClockRead.compareAndSet(true, false)) {
+ mappingRead.countDown();
+ try {
+ releaseTouch.await(5, TimeUnit.SECONDS);
+ } catch (InterruptedException failure) {
+ Thread.currentThread().interrupt();
+ throw new AssertionError(failure);
+ }
+ }
+ return 42L;
+ });
+
+ Future resolution = caller.submit(() -> gcpChannel.getChannelRef(key));
+ assertThat(mappingRead.await(5, TimeUnit.SECONDS)).isTrue();
+ gcpChannel.unbind(Collections.singletonList(key));
+ releaseTouch.countDown();
+ ChannelRef selected = resolution.get(5, TimeUnit.SECONDS);
+
+ assertThat(gcpChannel.affinityKeyToChannelRef).containsEntry(key, selected);
+ assertThat(gcpChannel.affinityKeyLastUsed).containsKey(key);
+ } finally {
+ releaseTouch.countDown();
+ caller.shutdownNow();
+ gcpChannel.shutdownNow();
+ channelExecutor.shutdownNow();
+ }
+ }
+
+ @Test
+ public void affinityTouchDoesNotOverwriteNewerConcurrentRebindTimestamp() throws Exception {
+ resetGcpChannel();
+ ExecutorService channelExecutor = MoreExecutors.newDirectExecutorService();
+ ExecutorService caller = Executors.newSingleThreadExecutor();
+ CountDownLatch lookupClockRead = new CountDownLatch(1);
+ CountDownLatch releaseLookup = new CountDownLatch(1);
+ AtomicBoolean firstClockRead = new AtomicBoolean(true);
+ try {
+ gcpChannel = createPoolWithFakeReadyChannels(channelExecutor, 1);
+ String key = "session";
+ ChannelRef channelRef = gcpChannel.channelRefs.get(0);
+ gcpChannel.bind(channelRef, Collections.singletonList(key));
+ gcpChannel.setNanoClock(
+ () -> {
+ if (firstClockRead.compareAndSet(true, false)) {
+ lookupClockRead.countDown();
+ try {
+ releaseLookup.await(5, TimeUnit.SECONDS);
+ } catch (InterruptedException failure) {
+ Thread.currentThread().interrupt();
+ throw new AssertionError(failure);
+ }
+ return 10L;
+ }
+ return 20L;
+ });
+
+ Future resolution = caller.submit(() -> gcpChannel.getChannelRef(key));
+ assertThat(lookupClockRead.await(5, TimeUnit.SECONDS)).isTrue();
+ gcpChannel.bind(channelRef, Collections.singletonList(key));
+ releaseLookup.countDown();
+
+ assertThat(resolution.get(5, TimeUnit.SECONDS)).isSameInstanceAs(channelRef);
+ assertThat(gcpChannel.affinityKeyLastUsed).containsEntry(key, 20L);
+ } finally {
+ releaseLookup.countDown();
+ caller.shutdownNow();
+ gcpChannel.shutdownNow();
+ channelExecutor.shutdownNow();
}
}
@@ -410,19 +705,6 @@ private GcpManagedChannel createPoolWithFakeReadyChannels(
.build();
}
- private void setChannelAffinityRefState(ChannelAffinityRef affinityRef, int state)
- throws Exception {
- Field stateField = ChannelAffinityRef.class.getDeclaredField("state");
- stateField.setAccessible(true);
- ((AtomicInteger) stateField.get(affinityRef)).set(state);
- }
-
- private void deactivateChannelRef(ChannelRef channelRef) throws Exception {
- Method deactivate = ChannelRef.class.getDeclaredMethod("deactivate");
- deactivate.setAccessible(true);
- deactivate.invoke(channelRef);
- }
-
@Test
public void testGetChannelRefPickUpSmallest() {
// This test verifies deterministic smallest-stream selection (LINEAR_SCAN behavior).
@@ -536,9 +818,8 @@ public void testPickLeastBusyStillPrefersLessBusyChannels() {
busyPicks++;
}
}
- // Power-of-two guarantees distinct indices, so channel 0 (50 streams) is always
- // paired with an idle channel (0 streams) and can never win.
- assertEquals(0, busyPicks);
+ // Sampling is with replacement, so the busy channel wins only when sampled twice.
+ assertThat(busyPicks).isLessThan(10);
}
/**
@@ -549,8 +830,8 @@ public void testPickLeastBusyStillPrefersLessBusyChannels() {
public void testPickLeastBusyWithDynamicScaleUp() throws InterruptedException {
final int minSize = 2;
final int maxSize = 6;
- final int minRpcPerChannel = 2;
- final int maxRpcPerChannel = 5;
+ final int minRpcPerChannel = 5;
+ final int maxRpcPerChannel = 7;
final Duration scaleDownInterval = Duration.ofMillis(50);
final ExecutorService executorService = Executors.newSingleThreadExecutor();
@@ -587,6 +868,7 @@ public void testPickLeastBusyWithDynamicScaleUp() throws InterruptedException {
// One more call triggers scale-up.
pool.getChannelRef(null).activeStreamsCountIncr();
+ await().atMost(Duration.ofSeconds(5)).until(() -> pool.getNumberOfChannels() > minSize);
assertThat(pool.getNumberOfChannels()).isEqualTo(minSize + 1);
// Mark the new channel as READY.
@@ -639,10 +921,7 @@ public void testPickLeastBusySingleChannel() {
}
}
- /**
- * With only 2 channels, power-of-two degenerates to comparing both — should always pick the less
- * busy one.
- */
+ /** With only 2 channels, sampling with replacement strongly prefers the less busy one. */
@Test
public void testPickLeastBusyTwoChannels() {
resetGcpChannel();
@@ -651,11 +930,14 @@ public void testPickLeastBusyTwoChannels() {
gcpChannel.channelRefs.add(gcpChannel.new ChannelRef(ch0, 0, 10));
gcpChannel.channelRefs.add(gcpChannel.new ChannelRef(ch1, 1, 3));
- // With 2 channels, both are always selected, so the one with fewer streams always wins.
+ int lessBusyPicks = 0;
for (int i = 0; i < 100; i++) {
ChannelRef picked = gcpChannel.getChannelRef(null);
- assertThat(picked).isEqualTo(gcpChannel.channelRefs.get(1));
+ if (picked == gcpChannel.channelRefs.get(1)) {
+ lessBusyPicks++;
+ }
}
+ assertThat(lessBusyPicks).isGreaterThan(60);
}
/**
@@ -692,13 +974,9 @@ public void testLinearScanStrategyAlwaysPicksFirstOnTie() {
}
}
- /**
- * Verifies that under low traffic with POWER_OF_TWO, the warm channel (most recently active) is
- * preferred when stream counts are tied. This preserves connection warmth without the thundering
- * herd problem.
- */
+ /** Verifies that POWER_OF_TWO does not add a warmth bias when stream counts are tied. */
@Test
- public void testPowerOfTwoPrefersWarmChannelOnTie() throws Exception {
+ public void testPowerOfTwoDoesNotPreferWarmChannelOnTie() throws Exception {
resetGcpChannel();
// Use a fake clock to deterministically control lastResponseNanos.
final AtomicLong fakeNanos = new AtomicLong(1_000_000_000L);
@@ -716,8 +994,7 @@ public void testPowerOfTwoPrefersWarmChannelOnTie() throws Exception {
ChannelRef warmChannel = gcpChannel.channelRefs.get(5);
warmChannel.messageReceived();
- // Pick many times. The warm channel should be picked more often than average because
- // whenever it appears in a random pair with another 0-stream channel, it wins the tie.
+ // Repeated picks keep the first sample on ties, so warmth does not bias selection.
int warmPicks = 0;
final int numPicks = 1000;
for (int i = 0; i < numPicks; i++) {
@@ -727,11 +1004,8 @@ public void testPowerOfTwoPrefersWarmChannelOnTie() throws Exception {
}
}
- // Without warmth bias, channel 5 would get ~10% (100/1000) picks.
- // With warmth bias, it should get significantly more because it wins every tie.
- // P(channel 5 in sample of 2) = 1 - (9/10)*(8/9) -- wait, it's 1-(9/10)^2 = 19%.
- // It wins tie with any other cold channel, so ~19% of picks. Allow some variance.
- assertThat(warmPicks).isGreaterThan(numPicks * 14 / 100);
+ assertThat(warmPicks).isAtLeast(numPicks * 5 / 100);
+ assertThat(warmPicks).isLessThan(numPicks * 15 / 100);
}
private void assertFallbacksMetric(
@@ -833,26 +1107,23 @@ public void testGetChannelRefWithFallback() {
assertEquals(2, chRef.getId());
assertEquals(3, pool.getNumberOfChannels());
- // Now we reached max pool size. Let's bring channel 2 to the low watermark and channel 1 to the
- // low watermark + 1 streams.
- for (int i = 0; i < lowWatermark; i++) {
+ // Now we reached max pool size. Bring channel 2 just below the configured watermark and
+ // channel 1 above it.
+ for (int i = 0; i < lowWatermark - 1; i++) {
pool.channelRefs.get(2).activeStreamsCountIncr();
}
pool.channelRefs.get(1).activeStreamsCountIncr();
- // As we reached max size and cannot create new channels and having ready channels with low
- // watermark and low watermark + 1 streams, the best channel for the next channel request with
- // the fallback enabled is the channel 2 with low watermark streams because it's the least busy
- // ready channel.
+ // Channel 2 remains eligible because its load is below the configured watermark.
assertEquals(lowWatermark + 1, pool.channelRefs.get(1).getActiveStreamsCount());
- assertEquals(lowWatermark, pool.channelRefs.get(2).getActiveStreamsCount());
+ assertEquals(lowWatermark - 1, pool.channelRefs.get(2).getActiveStreamsCount());
chRef = pool.getChannelRef(null);
assertEquals(2, chRef.getId());
assertEquals(3, pool.getNumberOfChannels());
- // This was the third fallback from non-ready channel 0 to the channel 2.
- assertFallbacksMetric(fakeRegistry, 3, 0);
+ // Both creating and subsequently selecting channel 2 are successful fallbacks.
+ assertFallbacksMetric(fakeRegistry, 4, 0);
// Let's bring channel 1 to max streams and mark channel 2 as not ready.
- for (int i = 0; i < MAX_STREAM - lowWatermark; i++) {
+ for (int i = 0; i < MAX_STREAM - (lowWatermark - 1); i++) {
pool.channelRefs.get(2).activeStreamsCountIncr();
}
pool.processChannelStateChange(1, ConnectivityState.CONNECTING);
@@ -871,7 +1142,7 @@ public void testGetChannelRefWithFallback() {
assertThat(logRecords.size()).isEqualTo(++logCount);
assertThat(lastLogMessage()).isEqualTo(poolIndex + ": Failed to find fallback for channel 0");
assertThat(lastLogLevel()).isEqualTo(Level.FINEST);
- assertFallbacksMetric(fakeRegistry, 3, 1);
+ assertFallbacksMetric(fakeRegistry, 4, 1);
// Let's have an affinity key and bind it to channel 0.
final String key = "ABC";
@@ -886,9 +1157,12 @@ public void testGetChannelRefWithFallback() {
assertThat(logRecords.size()).isEqualTo(++logCount);
assertThat(lastLogMessage()).isEqualTo(poolIndex + ": Failed to find fallback for channel 0");
assertThat(lastLogLevel()).isEqualTo(Level.FINEST);
- assertFallbacksMetric(fakeRegistry, 3, 2);
+ assertFallbacksMetric(fakeRegistry, 4, 2);
- // Let's return channel 1 to a ready state.
+ // Return channel 1 below the configured watermark and to a ready state.
+ while (pool.channelRefs.get(1).getActiveStreamsCount() >= lowWatermark) {
+ pool.channelRefs.get(1).activeStreamsCountDecr(System.nanoTime(), Status.OK, false);
+ }
pool.processChannelStateChange(1, ConnectivityState.READY);
logCount = logRecords.size();
// Now we have a fallback candidate.
@@ -898,7 +1172,7 @@ public void testGetChannelRefWithFallback() {
assertThat(logRecords.size()).isEqualTo(++logCount);
assertThat(lastLogMessage()).isEqualTo(poolIndex + ": Setting fallback channel: 0 -> 1");
assertThat(lastLogLevel()).isEqualTo(Level.FINEST);
- assertFallbacksMetric(fakeRegistry, 4, 2);
+ assertFallbacksMetric(fakeRegistry, 5, 2);
// Let's briefly bring channel 2 to ready state.
pool.processChannelStateChange(2, ConnectivityState.READY);
@@ -912,7 +1186,7 @@ public void testGetChannelRefWithFallback() {
assertThat(logRecords.size()).isEqualTo(++logCount);
assertThat(lastLogMessage()).isEqualTo(poolIndex + ": Using fallback channel: 0 -> 1");
assertThat(lastLogLevel()).isEqualTo(Level.FINEST);
- assertFallbacksMetric(fakeRegistry, 5, 2);
+ assertFallbacksMetric(fakeRegistry, 6, 2);
pool.processChannelStateChange(2, ConnectivityState.CONNECTING);
// Let's bring channel 1 back to connecting state.
@@ -926,7 +1200,7 @@ public void testGetChannelRefWithFallback() {
assertThat(logRecords.size()).isEqualTo(++logCount);
assertThat(lastLogMessage()).isEqualTo(poolIndex + ": Failed to find fallback for channel 0");
assertThat(lastLogLevel()).isEqualTo(Level.FINEST);
- assertFallbacksMetric(fakeRegistry, 5, 3);
+ assertFallbacksMetric(fakeRegistry, 6, 3);
// Finally, we bring both channel 1 and channel 0 to the ready state and we should get the
// original channel 0 for the key without any fallbacks happening.
@@ -936,7 +1210,7 @@ public void testGetChannelRefWithFallback() {
chRef = pool.getChannelRef(key);
assertEquals(0, chRef.getId());
assertThat(logRecords.size()).isEqualTo(logCount);
- assertFallbacksMetric(fakeRegistry, 5, 3);
+ assertFallbacksMetric(fakeRegistry, 6, 3);
}
@Test
@@ -1027,15 +1301,15 @@ public void testUsingKeyWithoutBinding() {
final String key = "non-binded-key";
ChannelRef channelRef = gcpChannel.getChannelRef(key);
- // Should bind on the fly to the least busy channel, which is 2.
- assertThat(channelRef.getId()).isEqualTo(2);
+ // Power-of-two binds on the fly to its sampled winner.
+ int boundChannelId = channelRef.getId();
+ assertThat(gcpChannel.affinityKeyToChannelRef.get(key)).isSameInstanceAs(channelRef);
cf1.activeStreamsCountDecr(System.nanoTime(), Status.OK, true);
cf1.activeStreamsCountDecr(System.nanoTime(), Status.OK, true);
channelRef = gcpChannel.getChannelRef(key);
- // Even after channel 1 now has less active streams (3) the channel 2 is still mapped for the
- // same key.
- assertThat(channelRef.getId()).isEqualTo(2);
+ // A load change does not move the existing binding.
+ assertThat(channelRef.getId()).isEqualTo(boundChannelId);
}
@Test
@@ -1210,7 +1484,9 @@ public void testMetrics() {
}
MetricsRecord record = fakeRegistry.pollRecord();
- assertThat(record.getMetrics().size()).isEqualTo(28);
+ assertThat(record.getMetrics().size()).isEqualTo(29);
+ assertThat(record.getMetrics())
+ .containsKey(prefix + GcpMetricsConstants.METRIC_SCALE_UP_PRIME_FAILURES);
// Initial log messages count.
int logCount = logRecords.size();
@@ -1341,6 +1617,8 @@ public void testLogMetrics() throws Exception {
.build())
.build())
.build();
+ AtomicLong nanoClock = new AtomicLong(System.nanoTime());
+ pool.setNanoClock(nanoClock::get);
try {
final int currentIndex = GcpManagedChannel.channelPoolIndex.get();
@@ -1352,7 +1630,7 @@ public void testLogMetrics() throws Exception {
// Simulate channel connecting.
channels.get(i).setState(ConnectivityState.CONNECTING);
waitForStateCallbacks(executorService);
- TimeUnit.MILLISECONDS.sleep(10);
+ nanoClock.addAndGet(Duration.ofMillis(10).toNanos());
// For the last one...
if (i == streams.length - 1) {
@@ -1364,12 +1642,12 @@ public void testLogMetrics() throws Exception {
channels.get(j).setState(ConnectivityState.CONNECTING);
}
waitForStateCallbacks(executorService);
- TimeUnit.MILLISECONDS.sleep(100);
+ nanoClock.addAndGet(Duration.ofMillis(110).toNanos());
// And this will be a failed fallback (no ready channels).
pool.getChannelRef(null);
// Simulate unresponsive connection.
- long startNanos = System.nanoTime();
+ long startNanos = nanoClock.get();
final Status deStatus = Status.fromCode(Code.DEADLINE_EXCEEDED);
ref.activeStreamsCountIncr();
ref.activeStreamsCountDecr(startNanos, deStatus, false);
@@ -1377,12 +1655,12 @@ public void testLogMetrics() throws Exception {
ref.activeStreamsCountDecr(startNanos, deStatus, false);
// Simulate unresponsive connection with more dropped calls.
- startNanos = System.nanoTime();
+ startNanos = nanoClock.get();
ref.activeStreamsCountIncr();
ref.activeStreamsCountDecr(startNanos, deStatus, false);
ref.activeStreamsCountIncr();
ref.activeStreamsCountDecr(startNanos, deStatus, false);
- TimeUnit.MILLISECONDS.sleep(110);
+ nanoClock.addAndGet(Duration.ofMillis(110).toNanos());
ref.activeStreamsCountIncr();
ref.activeStreamsCountDecr(startNanos, deStatus, false);
}
@@ -1494,8 +1772,9 @@ public void testLogMetrics() throws Exception {
assertThat(messages).contains(poolIndex + ": stat: max_unresponsive_dropped_calls = 3");
assertThat(messages).contains(poolIndex + ": stat: channel_pool_scaling_up = 0");
assertThat(messages).contains(poolIndex + ": stat: channel_pool_scaling_down = 0");
+ assertThat(messages).contains(poolIndex + ": stat: scale_up_prime_failures = 0");
- assertThat(logRecords.size()).isEqualTo(39);
+ assertThat(logRecords.size()).isEqualTo(40);
logRecords.forEach(
logRecord ->
assertWithMessage(logRecord.getMessage())
@@ -1549,8 +1828,9 @@ public void testLogMetrics() throws Exception {
assertThat(messages).contains(poolIndex + ": stat: max_unresponsive_dropped_calls = 0");
assertThat(messages).contains(poolIndex + ": stat: channel_pool_scaling_up = 0");
assertThat(messages).contains(poolIndex + ": stat: channel_pool_scaling_down = 0");
+ assertThat(messages).contains(poolIndex + ": stat: scale_up_prime_failures = 0");
- assertThat(logRecords.size()).isEqualTo(39);
+ assertThat(logRecords.size()).isEqualTo(40);
} finally {
pool.shutdownNow();
@@ -1581,6 +1861,8 @@ public void testUnresponsiveDetection() throws InterruptedException {
GcpMetricsOptions.newBuilder().withMetricRegistry(fakeRegistry).build())
.build())
.build();
+ AtomicLong nanoClock = new AtomicLong(System.nanoTime());
+ pool.setNanoClock(nanoClock::get);
int currentIndex = GcpManagedChannel.channelPoolIndex.get();
String poolIndex = String.format("pool-%d", currentIndex);
final AtomicInteger idleCounter = new AtomicInteger();
@@ -1588,10 +1870,10 @@ public void testUnresponsiveDetection() throws InterruptedException {
ChannelRef chRef = pool.new ChannelRef(channel);
assertEquals(0, idleCounter.get());
- TimeUnit.MILLISECONDS.sleep(105);
+ nanoClock.addAndGet(Duration.ofMillis(105).toNanos());
// Report 3 deadline exceeded errors after 100 ms.
- long startNanos = System.nanoTime();
+ long startNanos = nanoClock.get();
final Status deStatus = Status.fromCode(Code.DEADLINE_EXCEEDED);
chRef.activeStreamsCountDecr(startNanos, deStatus, false);
assertEquals(0, idleCounter.get());
@@ -1667,34 +1949,36 @@ public void testUnresponsiveDetection() throws InterruptedException {
+ " = 1\\d\\d");
// Any message from the server must reset the dropped requests count and timestamp.
- TimeUnit.MILLISECONDS.sleep(105);
- startNanos = System.nanoTime();
+ nanoClock.addAndGet(Duration.ofMillis(105).toNanos());
+ startNanos = nanoClock.get();
chRef.activeStreamsCountDecr(startNanos, deStatus, false);
assertEquals(1, idleCounter.get());
chRef.activeStreamsCountDecr(startNanos, deStatus, false);
assertEquals(1, idleCounter.get());
// A message received from the server.
+ nanoClock.incrementAndGet();
chRef.messageReceived();
chRef.activeStreamsCountDecr(startNanos, deStatus, false);
// No idle increment expected because dropped requests count and timestamp were reset.
assertEquals(1, idleCounter.get());
// Any non-deadline exceeded response must reset the dropped requests count and timestamp.
- TimeUnit.MILLISECONDS.sleep(105);
- startNanos = System.nanoTime();
+ nanoClock.addAndGet(Duration.ofMillis(105).toNanos());
+ startNanos = nanoClock.get();
chRef.activeStreamsCountDecr(startNanos, deStatus, false);
assertEquals(1, idleCounter.get());
chRef.activeStreamsCountDecr(startNanos, deStatus, false);
assertEquals(1, idleCounter.get());
// Response with UNAVAILABLE status received from the server.
final Status unavailableStatus = Status.fromCode(Code.UNAVAILABLE);
+ nanoClock.incrementAndGet();
chRef.activeStreamsCountDecr(startNanos, unavailableStatus, false);
chRef.activeStreamsCountDecr(startNanos, deStatus, false);
// No idle increment expected because dropped requests count and timestamp were reset.
assertEquals(1, idleCounter.get());
// Even if dropped requests count is reached, it must also respect 100 ms configured.
- startNanos = System.nanoTime();
+ startNanos = nanoClock.get();
chRef.activeStreamsCountDecr(startNanos, deStatus, false);
assertEquals(1, idleCounter.get());
chRef.activeStreamsCountDecr(startNanos, deStatus, false);
@@ -1703,7 +1987,7 @@ public void testUnresponsiveDetection() throws InterruptedException {
// Even it's third deadline exceeded no idle increment is expected because 100ms has not pass.
assertEquals(1, idleCounter.get());
- TimeUnit.MILLISECONDS.sleep(105);
+ nanoClock.addAndGet(Duration.ofMillis(105).toNanos());
// Any subsequent deadline exceeded after 100ms must trigger the reconnection.
chRef.activeStreamsCountDecr(startNanos, deStatus, false);
assertEquals(2, idleCounter.get());
@@ -1751,9 +2035,7 @@ public void testStateNotifications() throws InterruptedException {
gcpChannel.notifyWhenStateChanged(
ConnectivityState.SHUTDOWN, () -> immediateCallbackCalled.set(true));
- TimeUnit.MILLISECONDS.sleep(2);
-
- assertThat(immediateCallbackCalled.get()).isTrue();
+ await().atMost(Duration.ofSeconds(1)).untilTrue(immediateCallbackCalled);
// Subscribe for notification when leaving IDLE state.
final AtomicReference newState = new AtomicReference<>();
@@ -1777,10 +2059,12 @@ public void run() {
// Make sure it was IDLE;
assertThat(currentState).isEqualTo(ConnectivityState.IDLE);
- TimeUnit.MILLISECONDS.sleep(25);
-
- assertThat(newState.get())
- .isAnyOf(ConnectivityState.CONNECTING, ConnectivityState.TRANSIENT_FAILURE);
+ await()
+ .atMost(Duration.ofSeconds(1))
+ .untilAsserted(
+ () ->
+ assertThat(newState.get())
+ .isAnyOf(ConnectivityState.CONNECTING, ConnectivityState.TRANSIENT_FAILURE));
}
@Test
@@ -1935,6 +2219,8 @@ public void testAffinityKeysCleanup() throws InterruptedException {
.build())
.build())
.build();
+ AtomicLong nanoClock = new AtomicLong(System.nanoTime());
+ pool.setNanoClock(nanoClock::get);
final String liveKey = "live-key";
ChannelRef ch0 = pool.getChannelRef(liveKey);
@@ -1962,15 +2248,18 @@ public void testAffinityKeysCleanup() throws InterruptedException {
assertThat(pool.getChannelRef(expKey).getId()).isEqualTo(2);
// Halfway through affinity lifetime we use the live key again.
- TimeUnit.MILLISECONDS.sleep(100);
+ nanoClock.addAndGet(Duration.ofMillis(100).toNanos());
ch0 = pool.getChannelRef(liveKey);
// Make sure affinity still works.
assertThat(ch0.getId()).isEqualTo(0);
// Wait the remaining time and check that there is still affinity for the live key
// but no affinity for the expired key.
+ nanoClock.addAndGet(Duration.ofMillis(150).toNanos());
- TimeUnit.MILLISECONDS.sleep(150);
+ await()
+ .atMost(Duration.ofSeconds(1))
+ .until(() -> !pool.affinityKeyToChannelRef.containsKey(expKey));
assertThat(pool.affinityKeyToChannelRef.keySet().size()).isEqualTo(1);
assertThat(pool.affinityKeyToChannelRef.get(liveKey)).isEqualTo(ch0);
@@ -1987,443 +2276,88 @@ public void testAffinityKeysCleanup() throws InterruptedException {
@Test
public void testDynamicChannelPool() throws InterruptedException {
-
- final int minSize = 2;
- final int maxSize = 4;
- final int minRpcPerChannel = 2;
- final int maxRpcPerChannel = 5;
- final Duration scaleDownInterval = Duration.ofMillis(50);
- // Must catch 2 check scale down invocations + some time to avoid race with channel movement.
- final long intervalWaitMs = 2 * scaleDownInterval.toMillis() + 10;
- final ExecutorService executorService = Executors.newSingleThreadExecutor();
-
- FakeManagedChannelBuilder fmcb =
- new FakeManagedChannelBuilder(() -> new FakeManagedChannel(executorService));
-
- // Creating a pool with dynamic sizing and LINEAR_SCAN for deterministic assertions.
- final GcpManagedChannel pool =
- (GcpManagedChannel)
- GcpManagedChannelBuilder.forDelegateBuilder(fmcb)
- .withOptions(
- GcpManagedChannelOptions.newBuilder()
- .withChannelPoolOptions(
- GcpChannelPoolOptions.newBuilder()
- .setMinSize(minSize)
- .setMaxSize(maxSize)
- .setDynamicScaling(
- minRpcPerChannel, maxRpcPerChannel, scaleDownInterval)
- .setChannelPickStrategy(
- GcpManagedChannelOptions.ChannelPickStrategy.LINEAR_SCAN)
- .build())
- .build())
- .build();
-
- // Starts with minSize.
- assertThat(pool.getNumberOfChannels()).isEqualTo(minSize);
-
- // Mark connected in random order.
- List shuffled = new ArrayList<>(pool.channelRefs);
- Collections.shuffle(shuffled);
- for (ChannelRef channelRef : shuffled) {
- ((FakeManagedChannel) channelRef.getChannel()).setState(ConnectivityState.READY);
- }
-
- long startTime = System.nanoTime();
-
- // Simulate starting 10 calls which should be within the limit (2 channels x 5
- // maxRpcPerChannel).
- for (int i = 0; i < minSize * maxRpcPerChannel; i++) {
- pool.getChannelRef(null).activeStreamsCountIncr();
- }
-
- // As we are still within threshold of maxRpcPerChannel the pool must not scale yet.
- assertThat(pool.getNumberOfChannels()).isEqualTo(minSize);
-
- // Adding 11th call should trigger scaling up immediately.
- pool.getChannelRef(null).activeStreamsCountIncr();
- assertThat(pool.getNumberOfChannels()).isEqualTo(minSize + 1);
-
- // Mark newly created channel connected.
- ((FakeManagedChannel) pool.channelRefs.get(minSize).getChannel())
- .setState(ConnectivityState.READY);
-
- // Continue adding calls to verify the pool respects the maxSize value.
- for (int i = 0; i < maxSize * maxRpcPerChannel - minSize * maxRpcPerChannel; i++) {
- pool.getChannelRef(null).activeStreamsCountIncr();
- }
-
- // Now we have 21 calls in-flight which should bring us to 5 channels because
- // of maxRpcPerChannel is 5, but the max size of the pool is 4, so there should be 4 channels.
- assertThat(pool.getNumberOfChannels()).isEqualTo(maxSize);
-
- // Threshold for scaling down is minRpcPerChannel * number of channels. 2 * 3 in our case.
- // Going down 21 -> 7.
- for (ChannelRef channelRef : pool.channelRefs) {
- for (int i = 0; i < maxRpcPerChannel - minRpcPerChannel; i++) {
- channelRef.activeStreamsCountDecr(startTime, Status.OK, false);
- }
- }
- for (int i = 0; i < minRpcPerChannel; i++) {
- pool.channelRefs.get(i).activeStreamsCountDecr(startTime, Status.OK, false);
- }
-
- // Should not downscale yet.
- assertThat(pool.getNumberOfChannels()).isEqualTo(maxSize);
-
- // Should not downscale even after scale down check is passed.
- TimeUnit.MILLISECONDS.sleep(intervalWaitMs);
- assertThat(pool.getNumberOfChannels()).isEqualTo(maxSize);
-
- // Set all except last channel ready.
- for (int i = 0; i < pool.getNumberOfChannels(); i++) {
- if (i == pool.getNumberOfChannels() - 1) {
- continue;
+ ExecutorService executorService = Executors.newSingleThreadExecutor();
+ GcpManagedChannel pool = null;
+ try {
+ pool =
+ (GcpManagedChannel)
+ GcpManagedChannelBuilder.forDelegateBuilder(
+ new FakeManagedChannelBuilder(() -> new FakeManagedChannel(executorService)))
+ .withOptions(
+ GcpManagedChannelOptions.newBuilder()
+ .withChannelPoolOptions(
+ GcpChannelPoolOptions.newBuilder()
+ .setInitSize(2)
+ .setMinSize(2)
+ .setMaxSize(4)
+ .setDynamicScaling(2, 5, Duration.ofMillis(20))
+ .setScaleUpCooldown(Duration.ofNanos(1))
+ .setScaleDownConsecutiveLowLoadChecks(1)
+ .setDrainIdleGrace(Duration.ZERO)
+ .build())
+ .build())
+ .build();
+ GcpManagedChannel monitoredPool = pool;
+
+ ChannelRef hot = pool.channelRefs.get(0);
+ for (int i = 0; i < 7; i++) {
+ hot.activeStreamsCountIncr();
}
- ((FakeManagedChannel) pool.channelRefs.get(i).getChannel()).setState(ConnectivityState.READY);
- }
-
- // Remember not connected channel or oldest connected channel. In our case the last one (not
- // connected yet).
- final ChannelRef disconnectedRef =
- pool.channelRefs.stream()
- .min(
- Comparator.comparing(
- (GcpManagedChannel.ChannelRef chRef) -> chRef.getConnectedSinceNanos()))
- .get();
-
- // Removing one more stream should trigger scale down after the interval.
- pool.channelRefs.get(0).activeStreamsCountDecr(startTime, Status.OK, false);
- TimeUnit.MILLISECONDS.sleep(intervalWaitMs);
- assertThat(pool.getNumberOfChannels()).isEqualTo(maxSize - 1);
-
- // Make sure the oldest connected channel is removed.
- assertThat(pool.channelRefs.stream().anyMatch((chRef) -> (chRef == disconnectedRef))).isFalse();
-
- Set prevChannels = new HashSet<>(pool.channelRefs);
-
- // Scale up again to make sure not connected channels are not reused.
- for (int i = 0; i < 2 * maxRpcPerChannel + disconnectedRef.getActiveStreamsCount(); i++) {
- pool.getChannelRef(null).activeStreamsCountIncr();
- }
-
- assertThat(pool.getNumberOfChannels()).isEqualTo(maxSize);
-
- // Find newly created channel.
- ChannelRef newChannel =
- pool.channelRefs.stream().filter(chRef -> !prevChannels.contains(chRef)).findFirst().get();
- // Mark ready.
- ((FakeManagedChannel) newChannel.getChannel()).setState(ConnectivityState.READY);
-
- // Make sure disconnectedRef is not reused.
- assertThat(newChannel == disconnectedRef).isFalse();
-
- // Make sure previously removed channel is not shutted down as it still has a couple of calls.
- assertThat(disconnectedRef.getState()).isNotEqualTo(ConnectivityState.SHUTDOWN);
-
- // Cancel the calls and make sure the channel shutdown.
- while (disconnectedRef.getActiveStreamsCount() > 0) {
- disconnectedRef.activeStreamsCountDecr(startTime, Status.CANCELLED, true);
- }
- TimeUnit.MILLISECONDS.sleep(intervalWaitMs);
- assertThat(disconnectedRef.getChannel().getState(false)).isEqualTo(ConnectivityState.SHUTDOWN);
+ await().atMost(Duration.ofSeconds(5)).until(() -> monitoredPool.getNumberOfChannels() == 3);
+ assertThat(pool.getNumberOfChannels()).isEqualTo(3);
- // Find the oldest connected channel.
- ChannelRef oldestConnected =
- pool.channelRefs.stream()
- .sorted(
- Comparator.comparing(
- (GcpManagedChannel.ChannelRef chRef) -> chRef.getConnectedSinceNanos()))
- .findFirst()
- .get();
-
- // Scale down. Desired state: minRpcPerChannel on every channel, then closing minRpcPerChannel
- // streams cycling through channels.
- for (ChannelRef channelRef : pool.channelRefs) {
- while (channelRef.getActiveStreamsCount() != minRpcPerChannel) {
- if (channelRef.getActiveStreamsCount() > minRpcPerChannel) {
- channelRef.activeStreamsCountDecr(startTime, Status.OK, false);
- } else {
- channelRef.activeStreamsCountIncr();
- }
+ while (hot.getActiveStreamsCount() > 0) {
+ hot.activeStreamsCountDecr(System.nanoTime(), Status.OK, false);
}
- }
- for (int i = 0; i < minRpcPerChannel; i++) {
- pool.channelRefs.get(i).activeStreamsCountDecr(startTime, Status.OK, false);
- }
- // Remember its streams count.
- int oldestStreamsCount = oldestConnected.getActiveStreamsCount();
-
- // Should scale down after interval.
- TimeUnit.MILLISECONDS.sleep(intervalWaitMs);
- assertThat(pool.getNumberOfChannels()).isEqualTo(maxSize - 1);
-
- // Make sure it is removed.
- assertThat(pool.channelRefs.stream().anyMatch(chRef -> chRef == oldestConnected)).isFalse();
-
- // The active streams should still be there.
- assertThat(oldestConnected.getActiveStreamsCount()).isEqualTo(oldestStreamsCount);
-
- // The removed oldest connected channel must still be ready.
- assertThat(oldestConnected.getState()).isEqualTo(ConnectivityState.READY);
-
- // Scale up.
- for (int i = 0; i < 2 * maxRpcPerChannel + oldestConnected.getActiveStreamsCount(); i++) {
- pool.getChannelRef(null).activeStreamsCountIncr();
- }
- assertThat(pool.getNumberOfChannels()).isEqualTo(maxSize);
-
- // Make sure it is reused.
- assertThat(pool.channelRefs.stream().anyMatch(chRef -> chRef == oldestConnected)).isTrue();
-
- // Remember maxSize-minSize oldest connected channels.
- List oldestConnectedChannels =
- pool.channelRefs.stream()
- .sorted(
- Comparator.comparing(
- (GcpManagedChannel.ChannelRef chRef) -> chRef.getConnectedSinceNanos()))
- .collect(Collectors.toList())
- .subList(0, maxSize - minSize);
-
- // Remove all streams so that channel pool downscales to minSize.
- for (ChannelRef channelRef : pool.channelRefs) {
- while (channelRef.getActiveStreamsCount() > 0) {
- channelRef.activeStreamsCountDecr(startTime, Status.OK, false);
+ await().atMost(Duration.ofSeconds(5)).until(() -> monitoredPool.getNumberOfChannels() == 2);
+ assertThat(pool.getNumberOfChannels()).isEqualTo(2);
+ } finally {
+ if (pool != null) {
+ pool.shutdownNow();
}
+ executorService.shutdownNow();
}
-
- // Make sure channel pool scaled down to minSize after the interval.
- TimeUnit.MILLISECONDS.sleep(intervalWaitMs);
- assertThat(pool.getNumberOfChannels()).isEqualTo(minSize);
-
- // Make sure the oldest connected channels were removed.
- assertThat(pool.channelRefs.stream().anyMatch(chRef -> oldestConnectedChannels.contains(chRef)))
- .isFalse();
-
- // Make sure the removed channels are shutted down.
- assertThat(
- oldestConnectedChannels.stream()
- .allMatch(chRef -> chRef.getState() == ConnectivityState.SHUTDOWN))
- .isTrue();
-
- pool.shutdown();
}
@Test
public void testDynamicChannelPoolWithAffinity() throws InterruptedException {
- final String keyFormat = "abc-%d";
- final int minSize = 2;
- final int maxSize = 4;
- final int minRpcPerChannel = 2;
- final int maxRpcPerChannel = 5;
- final Duration scaleDownInterval = Duration.ofMillis(50);
- // Must catch 2 check scale down invocations + some time to avoid race with channel movement.
- final long intervalWaitMs = 2 * scaleDownInterval.toMillis() + 10;
- final ExecutorService executorService = Executors.newSingleThreadExecutor();
-
- FakeManagedChannelBuilder fmcb =
- new FakeManagedChannelBuilder(() -> new FakeManagedChannel(executorService));
-
- // Creating a pool with dynamic sizing and LINEAR_SCAN for deterministic assertions.
- final GcpManagedChannel pool =
- (GcpManagedChannel)
- GcpManagedChannelBuilder.forDelegateBuilder(fmcb)
- .withOptions(
- GcpManagedChannelOptions.newBuilder()
- .withChannelPoolOptions(
- GcpChannelPoolOptions.newBuilder()
- .setMinSize(minSize)
- .setMaxSize(maxSize)
- .setDynamicScaling(
- minRpcPerChannel, maxRpcPerChannel, scaleDownInterval)
- .setChannelPickStrategy(
- GcpManagedChannelOptions.ChannelPickStrategy.LINEAR_SCAN)
- .build())
- .build())
- .build();
-
- // Starts with minSize.
- assertThat(pool.getNumberOfChannels()).isEqualTo(minSize);
-
- // Mark connected in random order.
- List shuffled = new ArrayList<>(pool.channelRefs);
- Collections.shuffle(shuffled);
- for (ChannelRef channelRef : shuffled) {
- ((FakeManagedChannel) channelRef.getChannel()).setState(ConnectivityState.READY);
- }
-
- long startTime = System.nanoTime();
- int keyIndex = 0;
-
- // Simulate starting 10 calls which should be within the limit (2 channels x 5
- // maxRpcPerChannel).
- for (int i = 0; i < minSize * maxRpcPerChannel; i++) {
- pool.getChannelRef(String.format(keyFormat, keyIndex++)).activeStreamsCountIncr();
- }
-
- // As we are still within threshold of maxRpcPerChannel the pool must not scale yet.
- assertThat(pool.getNumberOfChannels()).isEqualTo(minSize);
-
- // Adding 11th call should trigger scaling up immediately.
- pool.getChannelRef(String.format(keyFormat, keyIndex++)).activeStreamsCountIncr();
- assertThat(pool.getNumberOfChannels()).isEqualTo(minSize + 1);
-
- // Mark newly created channel connected.
- ((FakeManagedChannel) pool.channelRefs.get(minSize).getChannel())
- .setState(ConnectivityState.READY);
-
- // Continue adding calls to verify the pool respects the maxSize value.
- for (int i = 0; i < maxSize * maxRpcPerChannel - minSize * maxRpcPerChannel; i++) {
- pool.getChannelRef(String.format(keyFormat, keyIndex++)).activeStreamsCountIncr();
- }
-
- // Now we have 21 calls in-flight which should bring us to 5 channels because
- // of maxRpcPerChannel is 5, but the max size of the pool is 4, so there should be 4 channels.
- assertThat(pool.getNumberOfChannels()).isEqualTo(maxSize);
-
- // Threshold for scaling down is minRpcPerChannel * number of channels. 2 * 3 in our case.
- // Going down 21 -> 7.
- int totalStreamCount =
- pool.channelRefs.stream().mapToInt(ChannelRef::getActiveStreamsCount).sum();
- while (totalStreamCount > 7) {
- for (ChannelRef channelRef : pool.channelRefs) {
- if (channelRef.getActiveStreamsCount() > 0 && totalStreamCount > 7) {
- channelRef.activeStreamsCountDecr(startTime, Status.OK, false);
- totalStreamCount--;
- }
- }
- }
-
- // Should not downscale yet.
- assertThat(pool.getNumberOfChannels()).isEqualTo(maxSize);
-
- // Should not downscale even after scale down check is passed.
- TimeUnit.MILLISECONDS.sleep(intervalWaitMs);
- assertThat(pool.getNumberOfChannels()).isEqualTo(maxSize);
-
- // Remember not connected channel or oldest connected channel. In our case the last one (not
- // connected yet).
- final ChannelRef disconnectedRef =
- pool.channelRefs.stream()
- .min(
- Comparator.comparing(
- (GcpManagedChannel.ChannelRef chRef) -> chRef.getConnectedSinceNanos()))
- .get();
-
- // Removing one more stream should trigger scale down after the interval.
- pool.channelRefs.get(0).activeStreamsCountDecr(startTime, Status.OK, false);
- TimeUnit.MILLISECONDS.sleep(intervalWaitMs);
- assertThat(pool.getNumberOfChannels()).isEqualTo(maxSize - 1);
-
- // Make sure the oldest connected channel is removed.
- assertThat(pool.channelRefs.stream().anyMatch((chRef) -> (chRef == disconnectedRef))).isFalse();
-
- Set prevChannels = new HashSet<>(pool.channelRefs);
-
- // Scale up again to make sure not connected channels are not reused.
- for (int i = 0; i < 2 * maxRpcPerChannel + disconnectedRef.getActiveStreamsCount(); i++) {
- pool.getChannelRef(String.format(keyFormat, keyIndex++)).activeStreamsCountIncr();
- }
-
- assertThat(pool.getNumberOfChannels()).isEqualTo(maxSize);
-
- // Find newly created channel.
- ChannelRef newChannel =
- pool.channelRefs.stream().filter(chRef -> !prevChannels.contains(chRef)).findFirst().get();
- // Mark ready.
- ((FakeManagedChannel) newChannel.getChannel()).setState(ConnectivityState.READY);
-
- // Make sure disconnectedRef is not reused.
- assertThat(newChannel == disconnectedRef).isFalse();
-
- // Make sure previously removed channel is not shutted down as it still has a couple of calls.
- assertThat(disconnectedRef.getState()).isNotEqualTo(ConnectivityState.SHUTDOWN);
-
- // Cancel the calls and make sure the channel shutdown.
- while (disconnectedRef.getActiveStreamsCount() > 0) {
- disconnectedRef.activeStreamsCountDecr(startTime, Status.CANCELLED, true);
- }
- TimeUnit.MILLISECONDS.sleep(intervalWaitMs);
- assertThat(disconnectedRef.getChannel().getState(false)).isEqualTo(ConnectivityState.SHUTDOWN);
-
- // Find the oldest connected channel.
- ChannelRef oldestConnected =
- pool.channelRefs.stream()
- .sorted(
- Comparator.comparing(
- (GcpManagedChannel.ChannelRef chRef) -> chRef.getConnectedSinceNanos()))
- .findFirst()
- .get();
-
- // Scale down. Desired state: minRpcPerChannel on every channel, then closing minRpcPerChannel
- // streams cycling through channels.
- for (ChannelRef channelRef : pool.channelRefs) {
- while (channelRef.getActiveStreamsCount() != minRpcPerChannel) {
- if (channelRef.getActiveStreamsCount() > minRpcPerChannel) {
- channelRef.activeStreamsCountDecr(startTime, Status.OK, false);
- } else {
- channelRef.activeStreamsCountIncr();
- }
- }
- }
- for (int i = 0; i < minRpcPerChannel; i++) {
- pool.channelRefs.get(i).activeStreamsCountDecr(startTime, Status.OK, false);
- }
- // Remember its streams count.
- int oldestStreamsCount = oldestConnected.getActiveStreamsCount();
-
- // Should scale down after interval.
- TimeUnit.MILLISECONDS.sleep(intervalWaitMs);
- assertThat(pool.getNumberOfChannels()).isEqualTo(maxSize - 1);
-
- // Make sure it is removed.
- assertThat(pool.channelRefs.stream().anyMatch(chRef -> chRef == oldestConnected)).isFalse();
-
- // The active streams should still be there.
- assertThat(oldestConnected.getActiveStreamsCount()).isEqualTo(oldestStreamsCount);
-
- // The removed oldest connected channel must still be ready.
- assertThat(oldestConnected.getState()).isEqualTo(ConnectivityState.READY);
-
- // Scale up.
- for (int i = 0; i < 2 * maxRpcPerChannel + oldestConnected.getActiveStreamsCount(); i++) {
- pool.getChannelRef(String.format(keyFormat, keyIndex++)).activeStreamsCountIncr();
- }
- assertThat(pool.getNumberOfChannels()).isEqualTo(maxSize);
-
- // Make sure it is reused.
- assertThat(pool.channelRefs.stream().anyMatch(chRef -> chRef == oldestConnected)).isTrue();
-
- // Remember maxSize-minSize oldest connected channels.
- List oldestConnectedChannels =
- pool.channelRefs.stream()
- .sorted(
- Comparator.comparing(
- (GcpManagedChannel.ChannelRef chRef) -> chRef.getConnectedSinceNanos()))
- .collect(Collectors.toList())
- .subList(0, maxSize - minSize);
+ ExecutorService executorService = Executors.newSingleThreadExecutor();
+ GcpManagedChannel pool = null;
+ try {
+ pool =
+ (GcpManagedChannel)
+ GcpManagedChannelBuilder.forDelegateBuilder(
+ new FakeManagedChannelBuilder(() -> new FakeManagedChannel(executorService)))
+ .withOptions(
+ GcpManagedChannelOptions.newBuilder()
+ .withChannelPoolOptions(
+ GcpChannelPoolOptions.newBuilder()
+ .setInitSize(2)
+ .setMinSize(1)
+ .setMaxSize(2)
+ .setDynamicScaling(1, 3, Duration.ofMillis(20))
+ .setScaleDownConsecutiveLowLoadChecks(1)
+ .setDrainIdleGrace(Duration.ofSeconds(5))
+ .build())
+ .build())
+ .build();
+ GcpManagedChannel monitoredPool = pool;
+
+ ChannelRef victim = pool.channelRefs.get(0);
+ pool.bind(victim, Collections.singletonList("session"));
+ pool.channelRefs.get(1).activeStreamsCountIncr();
+ await().atMost(Duration.ofSeconds(5)).until(() -> monitoredPool.getNumberOfChannels() == 1);
- // Remove all streams so that channel pool downscales to minSize.
- for (ChannelRef channelRef : pool.channelRefs) {
- while (channelRef.getActiveStreamsCount() > 0) {
- channelRef.activeStreamsCountDecr(startTime, Status.OK, false);
+ assertThat(pool.getNumberOfChannels()).isEqualTo(1);
+ assertThat(pool.affinityKeyToChannelRef).doesNotContainKey("session");
+ assertThat(pool.affinityKeyLastUsed).doesNotContainKey("session");
+ assertThat(victim.getAffinityCount()).isEqualTo(0);
+ } finally {
+ if (pool != null) {
+ pool.shutdownNow();
}
+ executorService.shutdownNow();
}
-
- // Make sure channel pool scaled down to minSize after the interval.
- TimeUnit.MILLISECONDS.sleep(intervalWaitMs);
- assertThat(pool.getNumberOfChannels()).isEqualTo(minSize);
-
- // Make sure the oldest connected channels were removed.
- assertThat(pool.channelRefs.stream().anyMatch(chRef -> oldestConnectedChannels.contains(chRef)))
- .isFalse();
-
- // Make sure the removed channels are shutted down.
- assertThat(
- oldestConnectedChannels.stream()
- .allMatch(chRef -> chRef.getState() == ConnectivityState.SHUTDOWN))
- .isTrue();
-
- pool.shutdown();
}
static class FakeManagedChannelBuilder extends ManagedChannelBuilder {
@@ -2563,13 +2497,6 @@ public ManagedChannel shutdownNow() {
@Override
public boolean awaitTermination(long timeout, TimeUnit unit) {
- if (this.state == ConnectivityState.SHUTDOWN) {
- return true;
- }
- try {
- unit.sleep(timeout);
- } catch (InterruptedException e) {
- }
return this.state == ConnectivityState.SHUTDOWN;
}
diff --git a/grpc-gcp-java/src/test/java/com/google/cloud/grpc/fallback/GcpFallbackChannelTest.java b/grpc-gcp-java/src/test/java/com/google/cloud/grpc/fallback/GcpFallbackChannelTest.java
index 210c482d588f..79ad995121f7 100644
--- a/grpc-gcp-java/src/test/java/com/google/cloud/grpc/fallback/GcpFallbackChannelTest.java
+++ b/grpc-gcp-java/src/test/java/com/google/cloud/grpc/fallback/GcpFallbackChannelTest.java
@@ -75,11 +75,10 @@
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
-import java.util.ArrayList;
import java.util.Collection;
-import java.util.Collections;
import java.util.EnumSet;
import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
@@ -352,7 +351,7 @@ private void simulateCanceledCall(boolean expectFallbackRouting) {
}
private class TestMetricExporter implements MetricExporter {
- public final List exportedMetrics = Collections.synchronizedList(new ArrayList<>());
+ public final List exportedMetrics = new CopyOnWriteArrayList<>();
@Override
public CompletableResultCode export(@Nonnull Collection metrics) {
diff --git a/java-spanner/benchmarks/pom.xml b/java-spanner/benchmarks/pom.xml
index e394422ef478..99c9fe6cd0f5 100644
--- a/java-spanner/benchmarks/pom.xml
+++ b/java-spanner/benchmarks/pom.xml
@@ -24,7 +24,7 @@
com.google.cloud
google-cloud-spanner-parent
- 6.121.0
+ 6.121.1
diff --git a/java-spanner/google-cloud-spanner-bom/pom.xml b/java-spanner/google-cloud-spanner-bom/pom.xml
index 92e2a4d755e5..0178af3eedfa 100644
--- a/java-spanner/google-cloud-spanner-bom/pom.xml
+++ b/java-spanner/google-cloud-spanner-bom/pom.xml
@@ -3,7 +3,7 @@
4.0.0
com.google.cloud
google-cloud-spanner-bom
- 6.121.0
+ 6.121.1
pom
com.google.cloud
@@ -53,43 +53,43 @@
com.google.cloud
google-cloud-spanner
- 6.121.0
+ 6.121.1
com.google.cloud
google-cloud-spanner
test-jar
- 6.121.0
+ 6.121.1
com.google.api.grpc
grpc-google-cloud-spanner-v1
- 6.121.0
+ 6.121.1
com.google.api.grpc
grpc-google-cloud-spanner-admin-instance-v1
- 6.121.0
+ 6.121.1
com.google.api.grpc
grpc-google-cloud-spanner-admin-database-v1
- 6.121.0
+ 6.121.1
com.google.api.grpc
proto-google-cloud-spanner-admin-instance-v1
- 6.121.0
+ 6.121.1
com.google.api.grpc
proto-google-cloud-spanner-v1
- 6.121.0
+ 6.121.1
com.google.api.grpc
proto-google-cloud-spanner-admin-database-v1
- 6.121.0
+ 6.121.1
diff --git a/java-spanner/google-cloud-spanner-executor/pom.xml b/java-spanner/google-cloud-spanner-executor/pom.xml
index 2a8eb6fe9ca5..794046a8cf1a 100644
--- a/java-spanner/google-cloud-spanner-executor/pom.xml
+++ b/java-spanner/google-cloud-spanner-executor/pom.xml
@@ -5,14 +5,14 @@
4.0.0
com.google.cloud
google-cloud-spanner-executor
- 6.121.0
+ 6.121.1
jar
Google Cloud Spanner Executor
com.google.cloud
google-cloud-spanner-parent
- 6.121.0
+ 6.121.1
diff --git a/java-spanner/google-cloud-spanner-executor/src/main/java/com/google/cloud/spanner/executor/v1/stub/Version.java b/java-spanner/google-cloud-spanner-executor/src/main/java/com/google/cloud/spanner/executor/v1/stub/Version.java
index fe9b12b060d8..48427624f49f 100644
--- a/java-spanner/google-cloud-spanner-executor/src/main/java/com/google/cloud/spanner/executor/v1/stub/Version.java
+++ b/java-spanner/google-cloud-spanner-executor/src/main/java/com/google/cloud/spanner/executor/v1/stub/Version.java
@@ -21,7 +21,7 @@
@InternalApi("For internal use only")
final class Version {
// {x-version-update-start:google-cloud-spanner:current}
- static final String VERSION = "6.121.0";
+ static final String VERSION = "6.121.1";
// {x-version-update-end}
}
diff --git a/java-spanner/google-cloud-spanner/pom.xml b/java-spanner/google-cloud-spanner/pom.xml
index 75ad83fdd24a..dd8003054e0b 100644
--- a/java-spanner/google-cloud-spanner/pom.xml
+++ b/java-spanner/google-cloud-spanner/pom.xml
@@ -3,7 +3,7 @@
4.0.0
com.google.cloud
google-cloud-spanner
- 6.121.0
+ 6.121.1
jar
Google Cloud Spanner
https://github.com/googleapis/google-cloud-java
@@ -11,7 +11,7 @@
com.google.cloud
google-cloud-spanner-parent
- 6.121.0
+ 6.121.1
google-cloud-spanner
diff --git a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/MultiplexedSessionDatabaseClient.java b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/MultiplexedSessionDatabaseClient.java
index ece6d862f87b..afe498219139 100644
--- a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/MultiplexedSessionDatabaseClient.java
+++ b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/MultiplexedSessionDatabaseClient.java
@@ -28,8 +28,10 @@
import com.google.cloud.spanner.Options.UpdateOption;
import com.google.cloud.spanner.SessionClient.SessionConsumer;
import com.google.cloud.spanner.SpannerException.ResourceNotFoundException;
+import com.google.cloud.spanner.spi.v1.SpannerRpc.ChannelPrimeSessionSource;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
+import com.google.common.util.concurrent.Futures;
import com.google.spanner.v1.BatchWriteResponse;
import java.time.Clock;
import java.time.Duration;
@@ -38,6 +40,7 @@
import java.util.EnumSet;
import java.util.HashMap;
import java.util.Map;
+import java.util.concurrent.CancellationException;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
@@ -48,12 +51,14 @@
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
+import javax.annotation.Nullable;
/**
* {@link DatabaseClient} implementation that uses a single multiplexed session to execute
* transactions.
*/
-final class MultiplexedSessionDatabaseClient extends AbstractMultiplexedSessionDatabaseClient {
+final class MultiplexedSessionDatabaseClient extends AbstractMultiplexedSessionDatabaseClient
+ implements ChannelPrimeSessionSource {
/**
* The maximum number of attempts that the client will try to execute CreateSession for the
* initial multiplexed session. This value is only used for the very first multiplexed session
@@ -186,7 +191,7 @@ private SharedChannelUsage(int numChannels) {
*/
private final AtomicInteger numCurrentSingleUseTransactions = new AtomicInteger();
- private boolean isClosed;
+ private volatile boolean isClosed;
/** The duration before we try to replace the multiplexed session. The default is 7 days. */
private final Duration sessionExpirationDuration;
@@ -253,15 +258,23 @@ private SharedChannelUsage(int numChannels) {
final SettableApiFuture initialSessionReferenceFuture =
SettableApiFuture.create();
this.multiplexedSessionReference = new AtomicReference<>(initialSessionReferenceFuture);
+ spanner.getRpc().registerChannelPrimeSessionSource(this);
- Duration waitDuration =
- sessionClient.getSpanner().getOptions().getSessionPoolOptions().getWaitForMinSessions();
- int initialAttempts =
- waitDuration == null || waitDuration.isZero() ? MAX_INITIAL_CREATE_SESSION_ATTEMPTS : 1;
- asyncCreateMultiplexedSession(initialSessionReferenceFuture, initialAttempts);
- maybeWaitForSessionCreation(
- sessionClient.getSpanner().getOptions().getSessionPoolOptions(),
- initialSessionReferenceFuture);
+ try {
+ Duration waitDuration =
+ sessionClient.getSpanner().getOptions().getSessionPoolOptions().getWaitForMinSessions();
+ int initialAttempts =
+ waitDuration == null || waitDuration.isZero() ? MAX_INITIAL_CREATE_SESSION_ATTEMPTS : 1;
+ asyncCreateMultiplexedSession(initialSessionReferenceFuture, initialAttempts);
+ maybeWaitForSessionCreation(
+ sessionClient.getSpanner().getOptions().getSessionPoolOptions(),
+ initialSessionReferenceFuture);
+ } catch (Throwable t) {
+ // The caller never receives this client, so it will never be closed; close() therefore undoes
+ // the registrations.
+ close();
+ throw t;
+ }
}
private void asyncCreateMultiplexedSession(
@@ -270,10 +283,19 @@ private void asyncCreateMultiplexedSession(
new SessionConsumer() {
@Override
public void onSessionReady(SessionImpl session) {
+ synchronized (MultiplexedSessionDatabaseClient.this) {
+ if (isClosed) {
+ // The client was closed while the session was being created. Ignore the session:
+ // it must neither be handed to waiters of a closed client nor keep a maintainer
+ // running for a client that no longer exists.
+ sessionReferenceFuture.setException(newClosedException());
+ return;
+ }
+ // only start the maintainer if we actually managed to create a session in the first
+ // place. Starting it under the lock guarantees that a concurrent close() stops it.
+ maintainer.start();
+ }
sessionReferenceFuture.set(session.getSessionReference());
- // only start the maintainer if we actually managed to create a session in the first
- // place.
- maintainer.start();
if (sessionClient
.getSpanner()
.getOptions()
@@ -296,9 +318,11 @@ public void onSessionCreateFailure(Throwable t, int createFailureForSessionCount
(ResourceNotFoundException) spannerException);
}
// Set the exception to trigger an error for all waiters.
- // Then retry the session creation if the error is (potentially) transient.
+ // Then retry the session creation if the error is (potentially) transient and the
+ // client has not been closed in the meantime.
sessionReferenceFuture.setException(t);
if (remainingAttempts > 1
+ && !isClientClosed()
&& RETRYABLE_ERROR_CODES.contains(spannerException.getErrorCode())) {
final SettableApiFuture