diff --git a/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/ApiaryUnbufferedReadableByteChannel.java b/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/ApiaryUnbufferedReadableByteChannel.java index 25757921ba54..a0bbadf67076 100644 --- a/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/ApiaryUnbufferedReadableByteChannel.java +++ b/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/ApiaryUnbufferedReadableByteChannel.java @@ -258,6 +258,9 @@ private ScatteringByteChannel open() { if (shouldValidate && expectedCrc32cBase64 != null) { this.hashingInputStream = new HashingInputStream(Hashing.crc32c(), content); content = this.hashingInputStream; + } else { + // e.g. a resumed (partial) read: don't validate a partial hash against the full object + this.hashingInputStream = null; } ReadableByteChannel rbc = Channels.newChannel(content); diff --git a/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpStorageRpcHasherHelper.java b/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpStorageRpcHasherHelper.java index 7d1bf13ff629..062564d103ac 100644 --- a/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpStorageRpcHasherHelper.java +++ b/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpStorageRpcHasherHelper.java @@ -25,6 +25,7 @@ import java.io.IOException; import java.io.OutputStream; import java.nio.ByteBuffer; +import java.util.Locale; import java.util.Map; import java.util.function.Supplier; @@ -106,19 +107,25 @@ private static boolean isFullObjectResponse(HttpResponse response) { } private boolean isTranscoded(HttpResponse response) { + return isDecompressedByServer(response) || isDecompressedByClient(response); + } + + /** GCS stored the object gzip-compressed but served it decompressed. */ + private static boolean isDecompressedByServer(HttpResponse response) { com.google.api.client.http.HttpHeaders headers = response.getHeaders(); String storedEncoding = HttpClientContext.firstHeaderValue(headers, "x-goog-stored-content-encoding"); - String storedLength = - HttpClientContext.firstHeaderValue(headers, "x-goog-stored-content-length"); - return storedEncoding != null || storedLength != null || isDecompressedByClient(response); + return isGzip(storedEncoding) && !isGzip(headers.getContentEncoding()); + } + + private static boolean isGzip(String encoding) { + return encoding != null && encoding.trim().toLowerCase(Locale.ROOT).contains("gzip"); } private boolean isDecompressedByClient(HttpResponse response) { boolean returnRaw = response.getRequest().getResponseReturnRawInputStream(); if (!returnRaw) { - String encoding = response.getHeaders().getContentEncoding(); - return encoding != null && encoding.contains("gzip"); + return isGzip(response.getHeaders().getContentEncoding()); } return false; } diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/ApiaryUnbufferedReadableByteChannelTest.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/ApiaryUnbufferedReadableByteChannelTest.java index ac0a7c70fd73..2b4304f6234e 100644 --- a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/ApiaryUnbufferedReadableByteChannelTest.java +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/ApiaryUnbufferedReadableByteChannelTest.java @@ -22,9 +22,13 @@ import static org.junit.Assert.assertTrue; import com.google.api.client.http.HttpTransport; +import com.google.api.client.http.LowLevelHttpRequest; +import com.google.api.client.http.LowLevelHttpResponse; import com.google.api.client.json.gson.GsonFactory; +import com.google.api.client.testing.http.MockLowLevelHttpRequest; import com.google.api.client.testing.http.MockLowLevelHttpResponse; import com.google.api.core.SettableApiFuture; +import com.google.api.gax.retrying.BasicResultRetryAlgorithm; import com.google.api.services.storage.Storage; import com.google.api.services.storage.model.StorageObject; import com.google.cloud.storage.ApiaryUnbufferedReadableByteChannel.ApiaryReadRequest; @@ -33,14 +37,20 @@ import com.google.cloud.storage.UnbufferedReadableByteChannelSession.UnbufferedReadableByteChannel; import com.google.cloud.storage.spi.v1.AuditingHttpTransport; import com.google.common.collect.ImmutableMap; +import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.InputStream; +import java.io.SequenceInputStream; import java.nio.ByteBuffer; import java.nio.channels.Channels; import java.nio.channels.WritableByteChannel; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; +import java.util.Locale; import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Handler; import java.util.logging.LogRecord; import java.util.logging.Logger; @@ -228,6 +238,127 @@ public void testRead_partialRangeNoCrc32cValidation() throws IOException { } } + @Test + public void testRead_gcsHeaders_mismatchedCrc32c_throwsChecksumMismatch() throws IOException { + // GCS returns x-goog-stored-content-* on every download, not only for transcoded objects. + Map extraHeaders = + ImmutableMap.of( + "x-goog-stored-content-encoding", + "identity", + "x-goog-stored-content-length", + String.valueOf(CONTENT_BYTES.length)); + Storage storageClient = + createMockStorageClient(200, "crc32c=" + WRONG_CRC32C_BASE64, extraHeaders); + + StorageObject from = new StorageObject().setBucket("bucket").setName("blob"); + ApiaryReadRequest apiaryReadRequest = + new ApiaryReadRequest(from, ImmutableMap.of(), ByteRangeSpec.nullRange()); + + SettableApiFuture resultFuture = SettableApiFuture.create(); + try (ApiaryUnbufferedReadableByteChannel channel = + new ApiaryUnbufferedReadableByteChannel( + apiaryReadRequest, + storageClient, + resultFuture, + RetrierWithAlg.attemptOnce(), + Retrying.neverRetry(), + Hasher.defaultHasher()); ) { + + ByteBuffer buf = ByteBuffer.allocate(4096); + Hasher.ChecksumMismatchException expected = + assertThrows( + Hasher.ChecksumMismatchException.class, + () -> { + while (channel.read(new ByteBuffer[] {buf}, 0, 1) != -1) { + buf.clear(); + } + }); + + assertTrue(expected.getMessage().contains("Mismatch checksum value")); + } + } + + @Test + public void testRead_resumedAfterBrokenStream_doesNotReportFalseChecksumMismatch() + throws IOException { + int splitAt = 6; + int length = CONTENT_BYTES.length; + AtomicInteger requestCount = new AtomicInteger(); + HttpTransport transport = + new HttpTransport() { + @Override + protected LowLevelHttpRequest buildRequest(String method, String url) { + return new MockLowLevelHttpRequest() { + @Override + public LowLevelHttpResponse execute() { + MockLowLevelHttpResponse response = + new MockLowLevelHttpResponse() + .addHeader("x-goog-hash", "crc32c=" + CORRECT_CRC32C_BASE64) + .addHeader("x-goog-stored-content-encoding", "identity") + .addHeader("x-goog-stored-content-length", String.valueOf(length)) + .addHeader("x-goog-generation", "12345"); + if (requestCount.getAndIncrement() == 0) { + // Full object response whose stream breaks part way through. + InputStream broken = + new SequenceInputStream( + new ByteArrayInputStream(Arrays.copyOf(CONTENT_BYTES, splitAt)), + new InputStream() { + @Override + public int read() throws IOException { + throw new BrokenStreamException(); + } + }); + return response.setStatusCode(200).setContent(broken); + } + // Resumed read of the rest of the object. + return response + .setStatusCode(206) + .addHeader( + "Content-Range", + String.format(Locale.US, "bytes %d-%d/%d", splitAt, length - 1, length)) + .setContent(Arrays.copyOfRange(CONTENT_BYTES, splitAt, length)); + } + }; + } + }; + Storage storageClient = + new Storage.Builder(transport, GsonFactory.getDefaultInstance(), null) + .setApplicationName("test") + .build(); + BasicResultRetryAlgorithm retryBrokenStream = + new BasicResultRetryAlgorithm() { + @Override + public boolean shouldRetry(Throwable previousThrowable, Object previousResponse) { + return previousThrowable instanceof BrokenStreamException; + } + }; + + StorageObject from = new StorageObject().setBucket("bucket").setName("blob"); + ApiaryReadRequest apiaryReadRequest = + new ApiaryReadRequest(from, ImmutableMap.of(), ByteRangeSpec.nullRange()); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + try (ApiaryUnbufferedReadableByteChannel channel = + new ApiaryUnbufferedReadableByteChannel( + apiaryReadRequest, + storageClient, + SettableApiFuture.create(), + RetrierWithAlg.attemptOnce(), + retryBrokenStream, + Hasher.defaultHasher())) { + ByteBuffer buf = ByteBuffer.allocate(4096); + while (channel.read(new ByteBuffer[] {buf}, 0, 1) != -1) { + buf.flip(); + out.write(buf.array(), 0, buf.limit()); + buf.clear(); + } + } + + assertThat(requestCount.get()).isEqualTo(2); + assertArrayEquals(CONTENT_BYTES, out.toByteArray()); + } + + private static final class BrokenStreamException extends IOException {} + @Test public void logsWarning_whenReceivingMoreBytesThanRequested() throws IOException { byte[] content = "0123456789extra_bytes".getBytes(); diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/HttpStorageRpcHasherHelperTest.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/HttpStorageRpcHasherHelperTest.java index 6a8777d5ba92..a2bddfa9bced 100644 --- a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/HttpStorageRpcHasherHelperTest.java +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/HttpStorageRpcHasherHelperTest.java @@ -153,12 +153,14 @@ public void testShouldValidate_transcoded_storedEncoding() throws IOException { } @Test - public void testShouldValidate_transcoded_storedLength() throws IOException { + public void testShouldValidate_normalDownload_realGcsHeaders() throws IOException { + // GCS sends these headers on every download, including uncompressed objects (#14773). Map headers = new HashMap<>(); + headers.put("x-goog-stored-content-encoding", "identity"); headers.put("x-goog-stored-content-length", "123"); HttpResponse response = createHttpResponse(200, headers, true); - assertFalse(HttpStorageRpcHasherHelper.INSTANCE.shouldValidate(response)); + assertTrue(HttpStorageRpcHasherHelper.INSTANCE.shouldValidate(response)); } @Test @@ -169,4 +171,17 @@ public void testShouldValidate_decompressedByClient() throws IOException { assertFalse(HttpStorageRpcHasherHelper.INSTANCE.shouldValidate(response)); } + + @Test + public void testShouldValidate_decompressedByClient_uppercaseContentEncoding() + throws IOException { + // GCS echoes the object's Content-Encoding metadata as-is, and the HTTP client matches it + // case-insensitively when deciding to decompress. + Map headers = new HashMap<>(); + headers.put("x-goog-stored-content-encoding", "GZIP"); + headers.put("Content-Encoding", "GZIP"); + HttpResponse response = createHttpResponse(200, headers, false); + + assertFalse(HttpStorageRpcHasherHelper.INSTANCE.shouldValidate(response)); + } } diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/spi/v1/HttpStorageRpcChecksumTest.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/spi/v1/HttpStorageRpcChecksumTest.java index 540e14eb045e..5b70391a7412 100644 --- a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/spi/v1/HttpStorageRpcChecksumTest.java +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/spi/v1/HttpStorageRpcChecksumTest.java @@ -299,6 +299,44 @@ public void testRead_rangeSuffixFullLength_failedCrc32cValidation() throws Excep assertThat(ex.getCause().getMessage()).contains("Mismatch checksum value"); } + @Test + public void testReadAllBytes_gcsHeaders_failedCrc32cValidation() throws Exception { + // GCS returns x-goog-stored-content-* on every download, not only for transcoded objects. + MockLowLevelHttpResponse response = + new MockLowLevelHttpResponse() + .setContentType("application/octet-stream") + .addHeader("x-goog-hash", "crc32c=" + BAD_CRC32C_BASE64) + .addHeader("x-goog-stored-content-encoding", "identity") + .addHeader("x-goog-stored-content-length", String.valueOf(CONTENT.length)) + .setContent(new String(CONTENT, StandardCharsets.UTF_8)) + .setStatusCode(200); + + try (Storage storage = createMockStorage(response)) { + StorageException ex = + assertThrows(StorageException.class, () -> storage.readAllBytes(BlobId.of(BUCKET, BLOB))); + assertThat(ex.getMessage()).contains("Mismatch checksum value"); + } + } + + @Test + public void testReadAllBytes_storedGzipServedDecompressed_skipsCrc32cValidation() + throws Exception { + // The object is stored gzip-compressed but GCS served it decompressed, so the served bytes + // cannot match the stored object's checksum and validation must be skipped. + MockLowLevelHttpResponse response = + new MockLowLevelHttpResponse() + .setContentType("application/octet-stream") + .addHeader("x-goog-hash", "crc32c=" + BAD_CRC32C_BASE64) + .addHeader("x-goog-stored-content-encoding", "gzip") + .addHeader("x-goog-stored-content-length", "42") + .setContent(new String(CONTENT, StandardCharsets.UTF_8)) + .setStatusCode(200); + + try (Storage storage = createMockStorage(response)) { + assertThat(storage.readAllBytes(BlobId.of(BUCKET, BLOB))).isEqualTo(CONTENT); + } + } + private HttpStorageRpc createMockRpc(MockLowLevelHttpResponse response) { AuditingHttpTransport transport = new AuditingHttpTransport(response); TransportOptions transportOptions =