Skip to content

test(storage): fast open failure and retry behaviour #14865

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
Dhriti07 wants to merge 1 commit into
base: fast-open/03-validate-fast-open-config
Choose a base branch
from
+175 −0
Draft

test(storage): fast open failure and retry behaviour #14865

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
175 changes: 175 additions & 0 deletions ...orage/google-cloud-storage/src/test/java/com/google/cloud/storage/ITFastOpenFakeTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,28 +20,40 @@
import static com.google.cloud.storage.ITObjectReadSessionFakeTest.getReadRange;
import static com.google.cloud.storage.ITObjectReadSessionFakeTest.read;
import static com.google.cloud.storage.PackagePrivateMethodWorkarounds.maybeGetStorageDataClient;
import static com.google.cloud.storage.TestUtils.GRPC_STATUS_DETAILS_KEY;
import static com.google.cloud.storage.TestUtils.xxd;
import static com.google.common.truth.Truth.assertThat;
import static java.util.concurrent.TimeUnit.SECONDS;
import static java.util.stream.Collectors.toList;
import static org.junit.Assert.assertThrows;

import com.google.api.core.ApiFuture;
import com.google.api.gax.grpc.GrpcCallContext;
import com.google.api.gax.rpc.OutOfRangeException;
import com.google.cloud.storage.ITObjectReadSessionFakeTest.FakeStorage;
import com.google.cloud.storage.it.ChecksummedTestContent;
import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.protobuf.Any;
import com.google.storage.v2.BidiReadObjectError;
import com.google.storage.v2.BidiReadObjectRequest;
import com.google.storage.v2.BidiReadObjectResponse;
import com.google.storage.v2.BidiReadObjectSpec;
import com.google.storage.v2.BucketName;
import com.google.storage.v2.Object;
import com.google.storage.v2.ObjectRangeData;
import com.google.storage.v2.ReadRangeError;
import io.grpc.Metadata;
import io.grpc.Status;
import io.grpc.stub.StreamObserver;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Consumer;
import org.junit.Test;

Expand Down Expand Up @@ -236,6 +248,169 @@ public void readSession_withASeekableChannelConfig_isRejected() throws Exception
assertThat(requestsReceived).isEmpty();
}

@Test
public void open_whenTheServerFailsTheStreamWithOutOfRange_failsWithoutRetrying()
throws Exception {
FakeStorage server =
FakeStorage.of(
ImmutableMap.of(
OPEN_WITH_READ_OF_10_TO_20,
client -> {
requestsReceived.add(OPEN_WITH_READ_OF_10_TO_20);
client.onError(Status.OUT_OF_RANGE.asRuntimeException());
}));

try (FakeServer fakeServer = FakeServer.of(server);
Storage storage = fakeServer.getGrpcStorageOptions().getService()) {
StorageDataClient dataClient = maybeGetStorageDataClient(storage);

ApiFuture<ObjectReadSession> open =
dataClient.readSession(OPEN, CTX, ImmutableList.of(READ_10_TO_20));

ExecutionException e = assertThrows(ExecutionException.class, () -> open.get(5, SECONDS));
// the error the server returned is somewhere in the chain of causes
assertThat(
Throwables.getCausalChain(e).stream()
.<Class<?>>map(Throwable::getClass)
.collect(toList()))
.contains(OutOfRangeException.class);
Comment on lines +272 to +276

@gemini-code-assist gemini-code-assist Bot Oct 30, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

Using Throwable::getClass with .contains(OutOfRangeException.class) performs an exact class equality check. If a subclass of OutOfRangeException is thrown, this assertion will fail. Using instanceof (via anyMatch) is more robust as it correctly handles subclasses, is more idiomatic, and avoids the overhead of collecting the stream to a list. Additionally, this allows removing the unused import static java.util.stream.Collectors.toList;.

      assertThat(Throwables.getCausalChain(e).stream().anyMatch(t -> t instanceof OutOfRangeException))
          .isTrue();

}

assertThat(requestsReceived).containsExactly(OPEN_WITH_READ_OF_10_TO_20);
}

@Test
public void open_whenTheServerRejectsOnlyTheFastOpenRange_reopensWithoutItAndReadAsFails()
throws Exception {
// OUT_OF_RANGE that names the failing read, as the server reports a bad range on a bidi read
BidiReadObjectError rangeError =
BidiReadObjectError.newBuilder()
.addReadRangeErrors(
ReadRangeError.newBuilder()
.setReadId(1)
.setStatus(
com.google.rpc.Status.newBuilder()
.setCode(com.google.rpc.Code.OUT_OF_RANGE_VALUE)))
.build();
Metadata trailers = new Metadata();
trailers.put(
GRPC_STATUS_DETAILS_KEY,
com.google.rpc.Status.newBuilder()
.setCode(com.google.rpc.Code.OUT_OF_RANGE_VALUE)
.addDetails(Any.pack(rangeError))
.build());
FakeStorage server =
FakeStorage.of(
ImmutableMap.of(
OPEN_WITH_READ_OF_10_TO_20,
client -> {
requestsReceived.add(OPEN_WITH_READ_OF_10_TO_20);
client.onError(Status.OUT_OF_RANGE.asRuntimeException(trailers));
},
OPEN,
client -> {
requestsReceived.add(OPEN);
client.onNext(metadataOnly());
}));

try (FakeServer fakeServer = FakeServer.of(server);
Storage storage = fakeServer.getGrpcStorageOptions().getService()) {
StorageDataClient dataClient = maybeGetStorageDataClient(storage);

try (ObjectReadSession session =
dataClient.readSession(OPEN, CTX, ImmutableList.of(READ_10_TO_20)).get(10, SECONDS)) {
ApiFuture<byte[]> read = session.readAs(READ_10_TO_20);

ExecutionException e = assertThrows(ExecutionException.class, () -> read.get(5, SECONDS));
assertThat(
Throwables.getCausalChain(e).stream()
.<Class<?>>map(Throwable::getClass)
.collect(toList()))
.contains(OutOfRangeException.class);
Comment on lines +325 to +329

@gemini-code-assist gemini-code-assist Bot Oct 30, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

Using Throwable::getClass with .contains(OutOfRangeException.class) performs an exact class equality check. If a subclass of OutOfRangeException is thrown, this assertion will fail. Using instanceof (via anyMatch) is more robust as it correctly handles subclasses, is more idiomatic, and avoids the overhead of collecting the stream to a list. Additionally, this allows removing the unused import static java.util.stream.Collectors.toList;.

        assertThat(Throwables.getCausalChain(e).stream().anyMatch(t -> t instanceof OutOfRangeException))
            .isTrue();

}
}

assertThat(requestsReceived).containsExactly(OPEN_WITH_READ_OF_10_TO_20, OPEN).inOrder();
}

@Test
public void open_whenTheConnectionDropsBeforeAnyReply_retriesWithTheSameRange() throws Exception {
AtomicInteger attempts = new AtomicInteger();
FakeStorage server =
FakeStorage.of(
ImmutableMap.of(
OPEN_WITH_READ_OF_10_TO_20,
client -> {
requestsReceived.add(OPEN_WITH_READ_OF_10_TO_20);
if (attempts.getAndIncrement() == 0) {
client.onError(Status.UNAVAILABLE.asRuntimeException());
} else {
client.onNext(metadataAndBytes(1, 10, 10));
}
}));

try (FakeServer fakeServer = FakeServer.of(server);
Storage storage = fakeServer.getGrpcStorageOptions().getService()) {
StorageDataClient dataClient = maybeGetStorageDataClient(storage);

try (ObjectReadSession session =
dataClient.readSession(OPEN, CTX, ImmutableList.of(READ_10_TO_20)).get(10, SECONDS)) {
byte[] actual = session.readAs(READ_10_TO_20).get(5, SECONDS);
assertThat(xxd(actual)).isEqualTo(xxd(slice(10, 10)));
}
}

assertThat(requestsReceived)
.containsExactly(OPEN_WITH_READ_OF_10_TO_20, OPEN_WITH_READ_OF_10_TO_20)
.inOrder();
}

@Test
public void open_whenTheConnectionDropsHalfway_fetchesOnlyTheRestOfTheRange() throws Exception {
// the first reply has bytes 10-15 only, it is not the end of the range
BidiReadObjectResponse bytes10To15 = metadataAndBytes(1, 10, 5);
BidiReadObjectResponse firstReply =
bytes10To15.toBuilder()
.setObjectDataRanges(
0, bytes10To15.getObjectDataRanges(0).toBuilder().setRangeEnd(false))
.build();
// after the first reply the session knows the object's generation, so a re-open pins it
BidiReadObjectRequest reopenForBytes15To20 =
OPEN.toBuilder()
.setReadObjectSpec(OPEN.getReadObjectSpec().toBuilder().setGeneration(1))
.addReadRanges(getReadRange(1, 15, 5))
.build();
FakeStorage server =
FakeStorage.of(
ImmutableMap.of(
OPEN_WITH_READ_OF_10_TO_20,
client -> {
requestsReceived.add(OPEN_WITH_READ_OF_10_TO_20);
client.onNext(firstReply);
client.onError(Status.UNAVAILABLE.asRuntimeException());
},
reopenForBytes15To20,
client -> {
requestsReceived.add(reopenForBytes15To20);
client.onNext(metadataAndBytes(1, 15, 5));
}));

try (FakeServer fakeServer = FakeServer.of(server);
Storage storage = fakeServer.getGrpcStorageOptions().getService()) {
StorageDataClient dataClient = maybeGetStorageDataClient(storage);

try (ObjectReadSession session =
dataClient.readSession(OPEN, CTX, ImmutableList.of(READ_10_TO_20)).get(10, SECONDS)) {
byte[] actual = session.readAs(READ_10_TO_20).get(10, SECONDS);
assertThat(xxd(actual)).isEqualTo(xxd(slice(10, 10)));
}
}

assertThat(requestsReceived)
.containsExactly(OPEN_WITH_READ_OF_10_TO_20, reopenForBytes15To20)
.inOrder();
}

/**
* A fake server that answers each request in {@code responses} and adds it to {@link
* #requestsReceived}. Any other request is answered with an error.
Expand Down
Loading