diff --git a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/AsyncTransactionManagerImpl.java b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/AsyncTransactionManagerImpl.java index c394ad09fe26..2377925eacef 100644 --- a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/AsyncTransactionManagerImpl.java +++ b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/AsyncTransactionManagerImpl.java @@ -165,6 +165,9 @@ public void onFailure(Throwable t) { txnState = TransactionState.ABORTED; } else { txnState = TransactionState.COMMIT_FAILED; + if (TransactionRunnerImpl.shouldRollbackOnCommitFailure(t)) { + TransactionRunnerImpl.rollbackBestEffort(txn, t); + } if (span != null) { span.setStatus(t); span.end(); diff --git a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/TransactionManagerImpl.java b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/TransactionManagerImpl.java index 469376c52edf..97ea93cc8372 100644 --- a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/TransactionManagerImpl.java +++ b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/TransactionManagerImpl.java @@ -92,6 +92,9 @@ public void commit() { throw e1; } catch (SpannerException e2) { txnState = TransactionState.COMMIT_FAILED; + if (TransactionRunnerImpl.shouldRollbackOnCommitFailure(e2)) { + TransactionRunnerImpl.rollbackBestEffort(txn, e2); + } throw e2; } finally { // At this point, if the TransactionState is not ABORTED, then the transaction has reached an diff --git a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/TransactionRunnerImpl.java b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/TransactionRunnerImpl.java index d6353bfcdb86..0c0a1fabfe27 100644 --- a/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/TransactionRunnerImpl.java +++ b/java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/TransactionRunnerImpl.java @@ -1404,6 +1404,9 @@ private T runInternal(final TransactionCallable txCallable) { "Transaction Attempt Failed in Commit", ImmutableMap.of( "Attempt", attempt.longValue(), "Status", e.getErrorCode().toString())); + if (shouldRollbackOnCommitFailure(e)) { + rollbackBestEffort(txn, e); + } throw e; } }; @@ -1428,4 +1431,26 @@ public void invalidate() { @Override public void close() {} + + static boolean shouldRollbackOnCommitFailure(Throwable t) { + SpannerException e = SpannerExceptionFactory.asSpannerException(t); + switch (e.getErrorCode()) { + case DEADLINE_EXCEEDED: + case CANCELLED: + case UNAVAILABLE: + return true; + case RESOURCE_EXHAUSTED: + return e.getRetryDelayInMillis() > -1L; + default: + return false; + } + } + + static void rollbackBestEffort(TransactionContextImpl txn, Throwable commitError) { + try { + ApiFuture unused = txn.rollbackAsync(); + } catch (RuntimeException rollbackError) { + commitError.addSuppressed(rollbackError); + } + } } diff --git a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/AsyncTransactionManagerImplTest.java b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/AsyncTransactionManagerImplTest.java index dd13c39abc8b..917ef385af08 100644 --- a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/AsyncTransactionManagerImplTest.java +++ b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/AsyncTransactionManagerImplTest.java @@ -16,16 +16,23 @@ package com.google.cloud.spanner; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import com.google.api.core.ApiFutures; import com.google.cloud.Timestamp; +import com.google.cloud.spanner.TransactionManager.TransactionState; +import com.google.protobuf.Empty; import io.opentelemetry.api.trace.Span; import io.opentelemetry.context.Scope; +import java.util.concurrent.ExecutionException; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mock; @@ -56,4 +63,115 @@ public void testCommitReturnsCommitStats() { verify(transaction).commitAsync(); } } + + @Test + public void testCommitFailsWithDeadlineExceededRollsBack() { + Span oTspan = mock(Span.class); + ISpan span = new OpenTelemetrySpan(oTspan); + when(oTspan.makeCurrent()).thenReturn(mock(Scope.class)); + when(session.newTransaction(eq(Options.fromTransactionOptions()), any())) + .thenReturn(transaction); + when(transaction.ensureTxnAsync()).thenReturn(ApiFutures.immediateFuture(null)); + when(transaction.commitAsync()) + .thenReturn( + ApiFutures.immediateFailedFuture( + SpannerExceptionFactory.newSpannerException( + ErrorCode.DEADLINE_EXCEEDED, "commit deadline exceeded"))); + when(transaction.rollbackAsync()) + .thenReturn(ApiFutures.immediateFuture(Empty.getDefaultInstance())); + try (AsyncTransactionManagerImpl manager = new AsyncTransactionManagerImpl(session, span)) { + manager.beginAsync(); + assertThrows(ExecutionException.class, () -> manager.commitAsync().get()); + assertEquals(TransactionState.COMMIT_FAILED, manager.getState()); + verify(transaction, times(1)).rollbackAsync(); + } + } + + @Test + public void testCommitFailsWithCancelledRollsBack() { + Span oTspan = mock(Span.class); + ISpan span = new OpenTelemetrySpan(oTspan); + when(oTspan.makeCurrent()).thenReturn(mock(Scope.class)); + when(session.newTransaction(eq(Options.fromTransactionOptions()), any())) + .thenReturn(transaction); + when(transaction.ensureTxnAsync()).thenReturn(ApiFutures.immediateFuture(null)); + when(transaction.commitAsync()) + .thenReturn( + ApiFutures.immediateFailedFuture( + SpannerExceptionFactory.newSpannerException( + ErrorCode.CANCELLED, "commit cancelled"))); + when(transaction.rollbackAsync()) + .thenReturn(ApiFutures.immediateFuture(Empty.getDefaultInstance())); + try (AsyncTransactionManagerImpl manager = new AsyncTransactionManagerImpl(session, span)) { + manager.beginAsync(); + assertThrows(ExecutionException.class, () -> manager.commitAsync().get()); + assertEquals(TransactionState.COMMIT_FAILED, manager.getState()); + verify(transaction, times(1)).rollbackAsync(); + } + } + + @Test + public void testCommitFailsWithUnavailableRollsBack() { + Span oTspan = mock(Span.class); + ISpan span = new OpenTelemetrySpan(oTspan); + when(oTspan.makeCurrent()).thenReturn(mock(Scope.class)); + when(session.newTransaction(eq(Options.fromTransactionOptions()), any())) + .thenReturn(transaction); + when(transaction.ensureTxnAsync()).thenReturn(ApiFutures.immediateFuture(null)); + when(transaction.commitAsync()) + .thenReturn( + ApiFutures.immediateFailedFuture( + SpannerExceptionFactory.newSpannerException( + ErrorCode.UNAVAILABLE, "commit unavailable"))); + when(transaction.rollbackAsync()) + .thenReturn(ApiFutures.immediateFuture(Empty.getDefaultInstance())); + try (AsyncTransactionManagerImpl manager = new AsyncTransactionManagerImpl(session, span)) { + manager.beginAsync(); + assertThrows(ExecutionException.class, () -> manager.commitAsync().get()); + assertEquals(TransactionState.COMMIT_FAILED, manager.getState()); + verify(transaction, times(1)).rollbackAsync(); + } + } + + @Test + public void testCommitFailsWithResourceExhaustedWithRetryInfoRollsBack() { + Span oTspan = mock(Span.class); + ISpan span = new OpenTelemetrySpan(oTspan); + when(oTspan.makeCurrent()).thenReturn(mock(Scope.class)); + when(session.newTransaction(eq(Options.fromTransactionOptions()), any())) + .thenReturn(transaction); + when(transaction.ensureTxnAsync()).thenReturn(ApiFutures.immediateFuture(null)); + when(transaction.commitAsync()) + .thenReturn( + ApiFutures.immediateFailedFuture( + TransactionRunnerImplTest.resourceExhaustedWithRetryInfo())); + when(transaction.rollbackAsync()) + .thenReturn(ApiFutures.immediateFuture(Empty.getDefaultInstance())); + try (AsyncTransactionManagerImpl manager = new AsyncTransactionManagerImpl(session, span)) { + manager.beginAsync(); + assertThrows(ExecutionException.class, () -> manager.commitAsync().get()); + assertEquals(TransactionState.COMMIT_FAILED, manager.getState()); + verify(transaction, times(1)).rollbackAsync(); + } + } + + @Test + public void testCommitFailsWithNonAbortDoesNotRollBack() { + Span oTspan = mock(Span.class); + ISpan span = new OpenTelemetrySpan(oTspan); + when(oTspan.makeCurrent()).thenReturn(mock(Scope.class)); + when(session.newTransaction(eq(Options.fromTransactionOptions()), any())) + .thenReturn(transaction); + when(transaction.ensureTxnAsync()).thenReturn(ApiFutures.immediateFuture(null)); + when(transaction.commitAsync()) + .thenReturn( + ApiFutures.immediateFailedFuture( + SpannerExceptionFactory.newSpannerException(ErrorCode.UNKNOWN, "unknown error"))); + try (AsyncTransactionManagerImpl manager = new AsyncTransactionManagerImpl(session, span)) { + manager.beginAsync(); + assertThrows(ExecutionException.class, () -> manager.commitAsync().get()); + assertEquals(TransactionState.COMMIT_FAILED, manager.getState()); + } + verify(transaction, never()).rollbackAsync(); + } } diff --git a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/OrphanedTransactionTest.java b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/OrphanedTransactionTest.java index 2e0a72086ed2..6c896add5c9c 100644 --- a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/OrphanedTransactionTest.java +++ b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/OrphanedTransactionTest.java @@ -16,21 +16,26 @@ package com.google.cloud.spanner; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThrows; import com.google.api.core.ApiFuture; import com.google.cloud.NoCredentials; +import com.google.cloud.spanner.AsyncTransactionManager.CommitTimestampFuture; import com.google.cloud.spanner.AsyncTransactionManager.TransactionContextFuture; import com.google.cloud.spanner.MockSpannerServiceImpl.SimulatedExecutionTime; import com.google.cloud.spanner.MockSpannerServiceImpl.StatementResult; import com.google.cloud.spanner.connection.AbstractMockServerTest; import com.google.cloud.spanner.connection.RandomResultSetGenerator; import com.google.common.base.Function; +import com.google.common.util.concurrent.MoreExecutors; import com.google.spanner.v1.ExecuteSqlRequest; import com.google.spanner.v1.RollbackRequest; import io.grpc.ManagedChannelBuilder; import io.grpc.Status; import java.util.Objects; +import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import org.junit.BeforeClass; @@ -144,4 +149,111 @@ public void testOrphanedTransactionWithFailedFirstQuery() throws Exception { executor.shutdown(); } } + + private void assertRollbackRequestReceived() throws InterruptedException { + long deadline = System.currentTimeMillis() + 200L; + while (mockSpanner.countRequestsOfType(RollbackRequest.class) == 0 + && System.currentTimeMillis() < deadline) { + Thread.sleep(10L); + } + assertEquals( + "Expected RollbackRequest after Commit DEADLINE_EXCEEDED", + 1, + mockSpanner.countRequestsOfType(RollbackRequest.class)); + } + + @Test + public void testTransactionRunnerCommitDeadlineExceededSendsRollback() throws Exception { + mockSpanner.clearRequests(); + mockSpanner.setCommitExecutionTime( + SimulatedExecutionTime.ofException( + Status.DEADLINE_EXCEEDED + .withDescription("commit deadline exceeded") + .asRuntimeException())); + try (Spanner spanner = createSpanner()) { + DatabaseClient client = + spanner.getDatabaseClient( + DatabaseId.of("fake-project", "fake-instance", "fake-database")); + SpannerException e = + assertThrows( + SpannerException.class, + () -> + client + .readWriteTransaction() + .run( + txn -> { + try (ResultSet rs = txn.executeQuery(STATEMENT)) { + while (rs.next()) {} + } + return null; + })); + assertEquals(ErrorCode.DEADLINE_EXCEEDED, e.getErrorCode()); + assertRollbackRequestReceived(); + } finally { + mockSpanner.removeAllExecutionTimes(); + } + } + + @Test + public void testTransactionManagerCommitDeadlineExceededSendsRollback() throws Exception { + mockSpanner.clearRequests(); + mockSpanner.setCommitExecutionTime( + SimulatedExecutionTime.ofException( + Status.DEADLINE_EXCEEDED + .withDescription("commit deadline exceeded") + .asRuntimeException())); + try (Spanner spanner = createSpanner()) { + DatabaseClient client = + spanner.getDatabaseClient( + DatabaseId.of("fake-project", "fake-instance", "fake-database")); + try (TransactionManager manager = client.transactionManager()) { + TransactionContext txn = manager.begin(); + try (ResultSet rs = txn.executeQuery(STATEMENT)) { + while (rs.next()) {} + } + SpannerException e = assertThrows(SpannerException.class, manager::commit); + assertEquals(ErrorCode.DEADLINE_EXCEEDED, e.getErrorCode()); + } + assertRollbackRequestReceived(); + } finally { + mockSpanner.removeAllExecutionTimes(); + } + } + + @Test + public void testAsyncTransactionManagerCommitDeadlineExceededSendsRollback() throws Exception { + mockSpanner.clearRequests(); + mockSpanner.setCommitExecutionTime( + SimulatedExecutionTime.ofException( + Status.DEADLINE_EXCEEDED + .withDescription("commit deadline exceeded") + .asRuntimeException())); + try (Spanner spanner = createSpanner()) { + DatabaseClient client = + spanner.getDatabaseClient( + DatabaseId.of("fake-project", "fake-instance", "fake-database")); + try (AsyncTransactionManager manager = client.transactionManagerAsync()) { + TransactionContextFuture context = manager.beginAsync(); + CommitTimestampFuture commitFuture = + context + .then( + (txn, input) -> { + try (AsyncResultSet rs = txn.executeQueryAsync(STATEMENT)) { + return rs.toListAsync( + row -> Objects.requireNonNull(row).getValue(0).getAsString(), + MoreExecutors.directExecutor()); + } + }, + MoreExecutors.directExecutor()) + .commitAsync(); + ExecutionException e = assertThrows(ExecutionException.class, commitFuture::get); + assertEquals( + ErrorCode.DEADLINE_EXCEEDED, + SpannerExceptionFactory.asSpannerException(e.getCause()).getErrorCode()); + } + assertRollbackRequestReceived(); + } finally { + mockSpanner.removeAllExecutionTimes(); + } + } } diff --git a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/TransactionManagerImplTest.java b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/TransactionManagerImplTest.java index 547f6b70a22d..2a8c6db211b4 100644 --- a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/TransactionManagerImplTest.java +++ b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/TransactionManagerImplTest.java @@ -179,12 +179,68 @@ public void resetAfterErrorFails() { doThrow(SpannerExceptionFactory.newSpannerException(ErrorCode.UNKNOWN, "")).when(txn).commit(); SpannerException e = assertThrows(SpannerException.class, () -> manager.commit()); assertEquals(ErrorCode.UNKNOWN, e.getErrorCode()); + verify(txn, Mockito.never()).rollback(); + verify(txn, Mockito.never()).rollbackAsync(); IllegalStateException illegal = assertThrows(IllegalStateException.class, () -> manager.resetForRetry()); assertNotNull(illegal.getMessage()); } + @Test + public void commitFailsWithDeadlineExceededRollsBack() { + when(session.newTransaction(eq(Options.fromTransactionOptions()), any())).thenReturn(txn); + manager.begin(); + doThrow( + SpannerExceptionFactory.newSpannerException( + ErrorCode.DEADLINE_EXCEEDED, "commit deadline exceeded")) + .when(txn) + .commit(); + SpannerException e = assertThrows(SpannerException.class, () -> manager.commit()); + assertEquals(ErrorCode.DEADLINE_EXCEEDED, e.getErrorCode()); + assertEquals(TransactionState.COMMIT_FAILED, manager.getState()); + verify(txn, times(1)).rollbackAsync(); + } + + @Test + public void commitFailsWithCancelledRollsBack() { + when(session.newTransaction(eq(Options.fromTransactionOptions()), any())).thenReturn(txn); + manager.begin(); + doThrow(SpannerExceptionFactory.newSpannerException(ErrorCode.CANCELLED, "commit cancelled")) + .when(txn) + .commit(); + SpannerException e = assertThrows(SpannerException.class, () -> manager.commit()); + assertEquals(ErrorCode.CANCELLED, e.getErrorCode()); + assertEquals(TransactionState.COMMIT_FAILED, manager.getState()); + verify(txn, times(1)).rollbackAsync(); + } + + @Test + public void commitFailsWithUnavailableRollsBack() { + when(session.newTransaction(eq(Options.fromTransactionOptions()), any())).thenReturn(txn); + manager.begin(); + doThrow( + SpannerExceptionFactory.newSpannerException( + ErrorCode.UNAVAILABLE, "commit unavailable")) + .when(txn) + .commit(); + SpannerException e = assertThrows(SpannerException.class, () -> manager.commit()); + assertEquals(ErrorCode.UNAVAILABLE, e.getErrorCode()); + assertEquals(TransactionState.COMMIT_FAILED, manager.getState()); + verify(txn, times(1)).rollbackAsync(); + } + + @Test + public void commitFailsWithResourceExhaustedWithRetryInfoRollsBack() { + when(session.newTransaction(eq(Options.fromTransactionOptions()), any())).thenReturn(txn); + manager.begin(); + doThrow(TransactionRunnerImplTest.resourceExhaustedWithRetryInfo()).when(txn).commit(); + SpannerException e = assertThrows(SpannerException.class, () -> manager.commit()); + assertEquals(ErrorCode.RESOURCE_EXHAUSTED, e.getErrorCode()); + assertEquals(TransactionState.COMMIT_FAILED, manager.getState()); + verify(txn, times(1)).rollbackAsync(); + } + @Test public void rollbackAfterCommitFails() { when(session.newTransaction(eq(Options.fromTransactionOptions()), any())).thenReturn(txn); diff --git a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/TransactionRunnerImplTest.java b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/TransactionRunnerImplTest.java index 1dd2418aa05a..5fbcac89025d 100644 --- a/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/TransactionRunnerImplTest.java +++ b/java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/TransactionRunnerImplTest.java @@ -322,6 +322,107 @@ public void commitFailsWithNonAbort() { assertEquals(1, numCalls.get()); verify(txn, never()).ensureTxn(); verify(txn, times(1)).commit(); + verify(txn, never()).rollback(); + verify(txn, never()).rollbackAsync(); + } + + @Test + public void commitFailsWithDeadlineExceededRollsBack() { + final SpannerException error = + SpannerExceptionFactory.newSpannerException( + ErrorCode.DEADLINE_EXCEEDED, "commit deadline exceeded"); + doThrow(error).when(txn).commit(); + final AtomicInteger numCalls = new AtomicInteger(0); + SpannerException e = + assertThrows( + SpannerException.class, + () -> transactionRunner.run(transaction -> numCalls.incrementAndGet())); + assertEquals(ErrorCode.DEADLINE_EXCEEDED, e.getErrorCode()); + assertEquals(1, numCalls.get()); + verify(txn, times(1)).commit(); + verify(txn, times(1)).rollbackAsync(); + } + + @Test + public void commitFailsWithCancelledRollsBack() { + final SpannerException error = + SpannerExceptionFactory.newSpannerException(ErrorCode.CANCELLED, "commit cancelled"); + doThrow(error).when(txn).commit(); + final AtomicInteger numCalls = new AtomicInteger(0); + SpannerException e = + assertThrows( + SpannerException.class, + () -> transactionRunner.run(transaction -> numCalls.incrementAndGet())); + assertEquals(ErrorCode.CANCELLED, e.getErrorCode()); + assertEquals(1, numCalls.get()); + verify(txn, times(1)).commit(); + verify(txn, times(1)).rollbackAsync(); + } + + @Test + public void commitFailsWithUnavailableRollsBack() { + final SpannerException error = + SpannerExceptionFactory.newSpannerException(ErrorCode.UNAVAILABLE, "commit unavailable"); + doThrow(error).when(txn).commit(); + final AtomicInteger numCalls = new AtomicInteger(0); + SpannerException e = + assertThrows( + SpannerException.class, + () -> transactionRunner.run(transaction -> numCalls.incrementAndGet())); + assertEquals(ErrorCode.UNAVAILABLE, e.getErrorCode()); + assertEquals(1, numCalls.get()); + verify(txn, times(1)).commit(); + verify(txn, times(1)).rollbackAsync(); + } + + @Test + public void commitFailsWithResourceExhaustedWithRetryInfoRollsBack() { + final SpannerException error = resourceExhaustedWithRetryInfo(); + doThrow(error).when(txn).commit(); + final AtomicInteger numCalls = new AtomicInteger(0); + SpannerException e = + assertThrows( + SpannerException.class, + () -> transactionRunner.run(transaction -> numCalls.incrementAndGet())); + assertEquals(ErrorCode.RESOURCE_EXHAUSTED, e.getErrorCode()); + assertEquals(1, numCalls.get()); + verify(txn, times(1)).commit(); + verify(txn, times(1)).rollbackAsync(); + } + + @Test + public void commitFailsWithResourceExhaustedWithoutRetryInfoDoesNotRollBack() { + final SpannerException error = + SpannerExceptionFactory.newSpannerException( + ErrorCode.RESOURCE_EXHAUSTED, "mutation limit exceeded"); + doThrow(error).when(txn).commit(); + final AtomicInteger numCalls = new AtomicInteger(0); + SpannerException e = + assertThrows( + SpannerException.class, + () -> transactionRunner.run(transaction -> numCalls.incrementAndGet())); + assertEquals(ErrorCode.RESOURCE_EXHAUSTED, e.getErrorCode()); + assertEquals(1, numCalls.get()); + verify(txn, times(1)).commit(); + verify(txn, never()).rollback(); + verify(txn, never()).rollbackAsync(); + } + + @Test + public void commitFailsWithDeadlineExceededWhenRollbackThrowsPreservesOriginalException() { + final SpannerException commitError = + SpannerExceptionFactory.newSpannerException( + ErrorCode.DEADLINE_EXCEEDED, "commit deadline exceeded"); + final RuntimeException rollbackError = new RuntimeException("rollback failed"); + doThrow(commitError).when(txn).commit(); + doThrow(rollbackError).when(txn).rollbackAsync(); + SpannerException e = + assertThrows(SpannerException.class, () -> transactionRunner.run(transaction -> 1)); + assertEquals(ErrorCode.DEADLINE_EXCEEDED, e.getErrorCode()); + assertEquals(1, e.getSuppressed().length); + assertEquals(rollbackError, e.getSuppressed()[0]); + verify(txn, times(1)).commit(); + verify(txn, times(1)).rollbackAsync(); } @Test @@ -465,7 +566,15 @@ private SpannerException abortedWithRetryInfo() { ErrorCode.ABORTED, "test", new StatusRuntimeException(status, createRetryTrailers())); } - private Metadata createRetryTrailers() { + static SpannerException resourceExhaustedWithRetryInfo() { + Status status = Status.fromCodeValue(Status.Code.RESOURCE_EXHAUSTED.value()); + return SpannerExceptionFactory.newSpannerException( + ErrorCode.RESOURCE_EXHAUSTED, + "test", + new StatusRuntimeException(status, createRetryTrailers())); + } + + private static Metadata createRetryTrailers() { Metadata.Key key = ProtoUtils.keyForProto(RetryInfo.getDefaultInstance()); Metadata trailers = new Metadata(); RetryInfo retryInfo =