fix(storage): validate CRC32C on HTTP downloads that were not decompressed

java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/ApiaryUnbufferedReadableByteChannel.java

@@ -258,6 +258,9 @@ private ScatteringByteChannel open() {
       if (shouldValidate && expectedCrc32cBase64 != null) {
         this.hashingInputStream = new HashingInputStream(Hashing.crc32c(), content);
         content = this.hashingInputStream;
+      } else {
+        // e.g. a resumed (partial) read: don't validate a partial hash against the full object
+        this.hashingInputStream = null;
       }
 
       ReadableByteChannel rbc = Channels.newChannel(content);

java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpStorageRpcHasherHelper.java

@@ -25,6 +25,7 @@
 import java.io.IOException;
 import java.io.OutputStream;
 import java.nio.ByteBuffer;
+import java.util.Locale;
 import java.util.Map;
 import java.util.function.Supplier;
 
@@ -106,19 +107,25 @@ private static boolean isFullObjectResponse(HttpResponse response) {
   }
 
   private boolean isTranscoded(HttpResponse response) {
+    return isDecompressedByServer(response) || isDecompressedByClient(response);
+  }
+
+  /** GCS stored the object gzip-compressed but served it decompressed. */
+  private static boolean isDecompressedByServer(HttpResponse response) {
     com.google.api.client.http.HttpHeaders headers = response.getHeaders();
     String storedEncoding =
         HttpClientContext.firstHeaderValue(headers, "x-goog-stored-content-encoding");
-    String storedLength =
-        HttpClientContext.firstHeaderValue(headers, "x-goog-stored-content-length");
-    return storedEncoding != null || storedLength != null || isDecompressedByClient(response);
+    return isGzip(storedEncoding) && !isGzip(headers.getContentEncoding());
+  }
+
+  private static boolean isGzip(String encoding) {
+    return encoding != null && encoding.trim().toLowerCase(Locale.ROOT).contains("gzip");
   }
 
   private boolean isDecompressedByClient(HttpResponse response) {
     boolean returnRaw = response.getRequest().getResponseReturnRawInputStream();
     if (!returnRaw) {
-      String encoding = response.getHeaders().getContentEncoding();
-      return encoding != null && encoding.contains("gzip");
+      return isGzip(response.getHeaders().getContentEncoding());
     }
     return false;
   }

java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/ApiaryUnbufferedReadableByteChannelTest.java

@@ -22,9 +22,13 @@
 import static org.junit.Assert.assertTrue;
 
 import com.google.api.client.http.HttpTransport;
+import com.google.api.client.http.LowLevelHttpRequest;
+import com.google.api.client.http.LowLevelHttpResponse;
 import com.google.api.client.json.gson.GsonFactory;
+import com.google.api.client.testing.http.MockLowLevelHttpRequest;
 import com.google.api.client.testing.http.MockLowLevelHttpResponse;
 import com.google.api.core.SettableApiFuture;
+import com.google.api.gax.retrying.BasicResultRetryAlgorithm;
 import com.google.api.services.storage.Storage;
 import com.google.api.services.storage.model.StorageObject;
 import com.google.cloud.storage.ApiaryUnbufferedReadableByteChannel.ApiaryReadRequest;
@@ -33,14 +37,20 @@
 import com.google.cloud.storage.UnbufferedReadableByteChannelSession.UnbufferedReadableByteChannel;
 import com.google.cloud.storage.spi.v1.AuditingHttpTransport;
 import com.google.common.collect.ImmutableMap;
+import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
+import java.io.InputStream;
+import java.io.SequenceInputStream;
 import java.nio.ByteBuffer;
 import java.nio.channels.Channels;
 import java.nio.channels.WritableByteChannel;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.logging.Handler;
 import java.util.logging.LogRecord;
 import java.util.logging.Logger;
@@ -228,6 +238,127 @@ public void testRead_partialRangeNoCrc32cValidation() throws IOException {
     }
   }
 
+  @Test
+  public void testRead_gcsHeaders_mismatchedCrc32c_throwsChecksumMismatch() throws IOException {
+    // GCS returns x-goog-stored-content-* on every download, not only for transcoded objects.
+    Map<String, String> extraHeaders =
+        ImmutableMap.of(
+            "x-goog-stored-content-encoding",
+            "identity",
+            "x-goog-stored-content-length",
+            String.valueOf(CONTENT_BYTES.length));
+    Storage storageClient =
+        createMockStorageClient(200, "crc32c=" + WRONG_CRC32C_BASE64, extraHeaders);
+
+    StorageObject from = new StorageObject().setBucket("bucket").setName("blob");
+    ApiaryReadRequest apiaryReadRequest =
+        new ApiaryReadRequest(from, ImmutableMap.of(), ByteRangeSpec.nullRange());
+
+    SettableApiFuture<StorageObject> resultFuture = SettableApiFuture.create();
+    try (ApiaryUnbufferedReadableByteChannel channel =
+        new ApiaryUnbufferedReadableByteChannel(
+            apiaryReadRequest,
+            storageClient,
+            resultFuture,
+            RetrierWithAlg.attemptOnce(),
+            Retrying.neverRetry(),
+            Hasher.defaultHasher()); ) {
+
+      ByteBuffer buf = ByteBuffer.allocate(4096);
+      Hasher.ChecksumMismatchException expected =
+          assertThrows(
+              Hasher.ChecksumMismatchException.class,
+              () -> {
+                while (channel.read(new ByteBuffer[] {buf}, 0, 1) != -1) {
+                  buf.clear();
+                }
+              });
+
+      assertTrue(expected.getMessage().contains("Mismatch checksum value"));
+    }
+  }
+
+  @Test
+  public void testRead_resumedAfterBrokenStream_doesNotReportFalseChecksumMismatch()
+      throws IOException {
+    int splitAt = 6;
+    int length = CONTENT_BYTES.length;
+    AtomicInteger requestCount = new AtomicInteger();
+    HttpTransport transport =
+        new HttpTransport() {
+          @Override
+          protected LowLevelHttpRequest buildRequest(String method, String url) {
+            return new MockLowLevelHttpRequest() {
+              @Override
+              public LowLevelHttpResponse execute() {
+                MockLowLevelHttpResponse response =
+                    new MockLowLevelHttpResponse()
+                        .addHeader("x-goog-hash", "crc32c=" + CORRECT_CRC32C_BASE64)
+                        .addHeader("x-goog-stored-content-encoding", "identity")
+                        .addHeader("x-goog-stored-content-length", String.valueOf(length))
+                        .addHeader("x-goog-generation", "12345");
+                if (requestCount.getAndIncrement() == 0) {
+                  // Full object response whose stream breaks part way through.
+                  InputStream broken =
+                      new SequenceInputStream(
+                          new ByteArrayInputStream(Arrays.copyOf(CONTENT_BYTES, splitAt)),
+                          new InputStream() {
+                            @Override
+                            public int read() throws IOException {
+                              throw new BrokenStreamException();
+                            }
+                          });
+                  return response.setStatusCode(200).setContent(broken);
+                }
+                // Resumed read of the rest of the object.
+                return response
+                    .setStatusCode(206)
+                    .addHeader(
+                        "Content-Range",
+                        String.format(Locale.US, "bytes %d-%d/%d", splitAt, length - 1, length))
+                    .setContent(Arrays.copyOfRange(CONTENT_BYTES, splitAt, length));
+              }
+            };
+          }
+        };
+    Storage storageClient =
+        new Storage.Builder(transport, GsonFactory.getDefaultInstance(), null)
+            .setApplicationName("test")
+            .build();
+    BasicResultRetryAlgorithm<Object> retryBrokenStream =
+        new BasicResultRetryAlgorithm<Object>() {
+          @Override
+          public boolean shouldRetry(Throwable previousThrowable, Object previousResponse) {
+            return previousThrowable instanceof BrokenStreamException;
+          }
+        };
+
+    StorageObject from = new StorageObject().setBucket("bucket").setName("blob");
+    ApiaryReadRequest apiaryReadRequest =
+        new ApiaryReadRequest(from, ImmutableMap.of(), ByteRangeSpec.nullRange());
+    ByteArrayOutputStream out = new ByteArrayOutputStream();
+    try (ApiaryUnbufferedReadableByteChannel channel =
+        new ApiaryUnbufferedReadableByteChannel(
+            apiaryReadRequest,
+            storageClient,
+            SettableApiFuture.create(),
+            RetrierWithAlg.attemptOnce(),
+            retryBrokenStream,
+            Hasher.defaultHasher())) {
+      ByteBuffer buf = ByteBuffer.allocate(4096);
+      while (channel.read(new ByteBuffer[] {buf}, 0, 1) != -1) {
+        buf.flip();
+        out.write(buf.array(), 0, buf.limit());
+        buf.clear();
+      }
+    }
+
+    assertThat(requestCount.get()).isEqualTo(2);
+    assertArrayEquals(CONTENT_BYTES, out.toByteArray());
+  }
+
+  private static final class BrokenStreamException extends IOException {}
+
   @Test
   public void logsWarning_whenReceivingMoreBytesThanRequested() throws IOException {
     byte[] content = "0123456789extra_bytes".getBytes();

java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/HttpStorageRpcHasherHelperTest.java

@@ -153,12 +153,14 @@ public void testShouldValidate_transcoded_storedEncoding() throws IOException {
   }
 
   @Test
-  public void testShouldValidate_transcoded_storedLength() throws IOException {
+  public void testShouldValidate_normalDownload_realGcsHeaders() throws IOException {
+    // GCS sends these headers on every download, including uncompressed objects (#14773).
     Map<String, String> headers = new HashMap<>();
+    headers.put("x-goog-stored-content-encoding", "identity");
     headers.put("x-goog-stored-content-length", "123");
     HttpResponse response = createHttpResponse(200, headers, true);
 
-    assertFalse(HttpStorageRpcHasherHelper.INSTANCE.shouldValidate(response));
+    assertTrue(HttpStorageRpcHasherHelper.INSTANCE.shouldValidate(response));
   }
 
   @Test
@@ -169,4 +171,17 @@ public void testShouldValidate_decompressedByClient() throws IOException {
 
     assertFalse(HttpStorageRpcHasherHelper.INSTANCE.shouldValidate(response));
   }
+
+  @Test
+  public void testShouldValidate_decompressedByClient_uppercaseContentEncoding()
+      throws IOException {
+    // GCS echoes the object's Content-Encoding metadata as-is, and the HTTP client matches it
+    // case-insensitively when deciding to decompress.
+    Map<String, String> headers = new HashMap<>();
+    headers.put("x-goog-stored-content-encoding", "GZIP");
+    headers.put("Content-Encoding", "GZIP");
+    HttpResponse response = createHttpResponse(200, headers, false);
+
+    assertFalse(HttpStorageRpcHasherHelper.INSTANCE.shouldValidate(response));
+  }
 }

java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/spi/v1/HttpStorageRpcChecksumTest.java

@@ -299,6 +299,44 @@ public void testRead_rangeSuffixFullLength_failedCrc32cValidation() throws Excep
     assertThat(ex.getCause().getMessage()).contains("Mismatch checksum value");
   }
 
+  @Test
+  public void testReadAllBytes_gcsHeaders_failedCrc32cValidation() throws Exception {
+    // GCS returns x-goog-stored-content-* on every download, not only for transcoded objects.
+    MockLowLevelHttpResponse response =
+        new MockLowLevelHttpResponse()
+            .setContentType("application/octet-stream")
+            .addHeader("x-goog-hash", "crc32c=" + BAD_CRC32C_BASE64)
+            .addHeader("x-goog-stored-content-encoding", "identity")
+            .addHeader("x-goog-stored-content-length", String.valueOf(CONTENT.length))
+            .setContent(new String(CONTENT, StandardCharsets.UTF_8))
+            .setStatusCode(200);
+
+    try (Storage storage = createMockStorage(response)) {
+      StorageException ex =
+          assertThrows(StorageException.class, () -> storage.readAllBytes(BlobId.of(BUCKET, BLOB)));
+      assertThat(ex.getMessage()).contains("Mismatch checksum value");
+    }
+  }
+
+  @Test
+  public void testReadAllBytes_storedGzipServedDecompressed_skipsCrc32cValidation()
+      throws Exception {
+    // The object is stored gzip-compressed but GCS served it decompressed, so the served bytes
+    // cannot match the stored object's checksum and validation must be skipped.
+    MockLowLevelHttpResponse response =
+        new MockLowLevelHttpResponse()
+            .setContentType("application/octet-stream")
+            .addHeader("x-goog-hash", "crc32c=" + BAD_CRC32C_BASE64)
+            .addHeader("x-goog-stored-content-encoding", "gzip")
+            .addHeader("x-goog-stored-content-length", "42")
+            .setContent(new String(CONTENT, StandardCharsets.UTF_8))
+            .setStatusCode(200);
+
+    try (Storage storage = createMockStorage(response)) {
+      assertThat(storage.readAllBytes(BlobId.of(BUCKET, BLOB))).isEqualTo(CONTENT);
+    }
+  }
+
   private HttpStorageRpc createMockRpc(MockLowLevelHttpResponse response) {
     AuditingHttpTransport transport = new AuditingHttpTransport(response);
     TransportOptions transportOptions =