HDDS-16362. GetObjectAttributes ObjectParts should return Part entries for FSO buckets

hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java

@@ -1907,6 +1907,72 @@ public void testGetObjectAttributesMultipartObjectParts(@TempDir Path tempDir) t
     assertTrue(secondPage.objectParts().parts().isEmpty());
   }
 
+  /**
+   * Directory (FSO layout) buckets return per-part {@code Part} elements even when no additional
+   * checksum was stored at upload time, matching AWS S3 directory-bucket behavior.
+   */
+  @Test
+  public void testGetObjectAttrFsoMpuNoChecksum(@TempDir Path tempDir)
+      throws Exception {
+    final String bucketName = uniqueObjectName();
+    final String keyName = getKeyName();
+    final int partSize = (int) (5 * MB);
+
+    createFsoBucket(bucketName);
+    try {
+      File multipartUploadFile =
+          Files.createFile(tempDir.resolve("fso-get-object-attributes-mpu.txt")).toFile();
+      createFile(multipartUploadFile, (int) (15 * MB));
+      multipartUpload(bucketName, keyName, multipartUploadFile, partSize, new HashMap<>(),
+          Collections.emptyList());
+
+      GetObjectAttributesResponse attributesResponse = s3Client.getObjectAttributes(
+          GetObjectAttributesRequest.builder()
+              .bucket(bucketName)
+              .key(keyName)
+              .objectAttributes(ObjectAttributes.OBJECT_PARTS, ObjectAttributes.OBJECT_SIZE)
+              .build());
+
+      assertNotNull(attributesResponse.objectParts());
+      assertEquals(3, attributesResponse.objectParts().totalPartsCount());
+      assertFalse(attributesResponse.objectParts().isTruncated());
+      assertEquals(multipartUploadFile.length(), attributesResponse.objectSize());
+      assertEquals(3, attributesResponse.objectParts().parts().size());
+      for (int i = 0; i < 3; i++) {
+        assertEquals(i + 1, attributesResponse.objectParts().parts().get(i).partNumber());
+        assertEquals((long) partSize, attributesResponse.objectParts().parts().get(i).size());
+      }
+
+      GetObjectAttributesResponse firstPage = s3Client.getObjectAttributes(
+          GetObjectAttributesRequest.builder()
+              .bucket(bucketName)
+              .key(keyName)
+              .objectAttributes(ObjectAttributes.OBJECT_PARTS)
+              .maxParts(2)
+              .partNumberMarker(0)
+              .build());
+      assertTrue(firstPage.objectParts().isTruncated());
+      assertEquals(2, firstPage.objectParts().parts().size());
+      assertEquals(1, firstPage.objectParts().parts().get(0).partNumber());
+      assertEquals(2, firstPage.objectParts().parts().get(1).partNumber());
+
+      GetObjectAttributesResponse secondPage = s3Client.getObjectAttributes(
+          GetObjectAttributesRequest.builder()
+              .bucket(bucketName)
+              .key(keyName)
+              .objectAttributes(ObjectAttributes.OBJECT_PARTS)
+              .maxParts(2)
+              .partNumberMarker(2)
+              .build());
+      assertFalse(secondPage.objectParts().isTruncated());
+      assertEquals(1, secondPage.objectParts().parts().size());
+      assertEquals(3, secondPage.objectParts().parts().get(0).partNumber());
+    } finally {
+      s3Client.deleteObject(b -> b.bucket(bucketName).key(keyName));
+      deleteFsoBucket(bucketName);
+    }
+  }
+
   @Test
   public void testGetObjectAttributesNonContiguousMultipartObjectParts() throws Exception {
     final String bucketName = getBucketName();
@@ -3080,6 +3146,22 @@ private String getKeyName(String ignored) {
     return uniqueObjectName();
   }
 
+  private void createFsoBucket(String bucketName) throws Exception {
+    try (OzoneClient ozoneClient = cluster.newClient()) {
+      OzoneVolume volume = ozoneClient.getObjectStore().getS3Volume();
+      volume.createBucket(bucketName, BucketArgs.newBuilder()
+          .setBucketLayout(BucketLayout.FILE_SYSTEM_OPTIMIZED)
+          .build());
+    }
+  }
+
+  private void deleteFsoBucket(String bucketName) throws Exception {
+    try (OzoneClient ozoneClient = cluster.newClient()) {
+      OzoneVolume volume = ozoneClient.getObjectStore().getS3Volume();
+      volume.deleteBucket(bucketName);
+    }
+  }
+
   private String multipartUpload(String bucketName, String key, File file, int partSize,
                                  Map<String, String> userMetadata, List<Tag> tags) throws Exception {
     String uploadId = initiateMultipartUpload(bucketName, key, userMetadata, tags);
@@ -4228,21 +4310,5 @@ public void testListBucketsIncludesFSOBuckets() throws Exception {
         deleteFsoBucket(fsoBucketName);
       }
     }
-
-    private void createFsoBucket(String bucketName) throws Exception {
-      try (OzoneClient ozoneClient = cluster.newClient()) {
-        OzoneVolume volume = ozoneClient.getObjectStore().getS3Volume();
-        volume.createBucket(bucketName, BucketArgs.newBuilder()
-            .setBucketLayout(BucketLayout.FILE_SYSTEM_OPTIMIZED)
-            .build());
-      }
-    }
-
-    private void deleteFsoBucket(String bucketName) throws Exception {
-      try (OzoneClient ozoneClient = cluster.newClient()) {
-        OzoneVolume volume = ozoneClient.getObjectStore().getS3Volume();
-        volume.deleteBucket(bucketName);
-      }
-    }
   }
 }

hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectAttributesHandler.java

@@ -57,10 +57,10 @@
  * <p>Returns selected metadata about an object without transferring the object body.
  * Supported attributes: {@code ETag}, {@code ObjectSize}, {@code StorageClass}, {@code ObjectParts}.
  *
- * <p>The {@code Checksum} attribute is not yet supported because Ozone does not store
- * non-MD5 checksum algorithms in key metadata. For general-purpose buckets, {@code Part}
- * elements under {@code ObjectParts} are omitted unless an additional checksum is stored
- * on the object, matching AWS S3 behavior. Object versioning ({@code versionId}) and
+ * <p>For general-purpose buckets, individual {@code Part} elements under {@code ObjectParts}
+ * are omitted unless an additional checksum is stored on the object, matching AWS S3 behavior.
+ * For FSO layout buckets (Ozone directory buckets), {@code Part} elements are always returned,
+ * matching AWS S3 directory-bucket behavior. Object versioning ({@code versionId}) and
  * SSE-C encryption headers are also not supported and are silently ignored.
  *
  * <p>See https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObjectAttributes.html
@@ -101,6 +101,15 @@ static boolean hasStoredAdditionalChecksum(OzoneKey key) {
     return false;
   }
 
+  /**
+   * Whether {@code ObjectParts} should include per-part {@code Part} elements in the response.
+   * Directory (FSO layout) buckets always include them; general-purpose buckets include them
+   * only when an additional checksum was stored at upload time.
+   */
+  static boolean shouldIncludePartElements(OzoneKey key, boolean directoryBucketLayout) {
+    return directoryBucketLayout || hasStoredAdditionalChecksum(key);
+  }
+
   @Override
   Response handleGetRequest(ObjectRequestContext context, String keyPath)
       throws IOException, OS3Exception {
@@ -137,8 +146,12 @@ Response handleGetRequest(ObjectRequestContext context, String keyPath)
         throw ex;
       }
 
+      boolean directoryBucketLayout =
+          context.getBucket().getBucketLayout().isFileSystemOptimized();
+
       GetObjectAttributesResponse response =
-          buildResponse(keyPath, key, requestedAttributes, completedPartSizes);
+          buildResponse(keyPath, key, requestedAttributes, completedPartSizes,
+              directoryBucketLayout);
 
       Response.ResponseBuilder rb = Response.ok(response, MediaType.APPLICATION_XML_TYPE);
       ObjectEndpoint.addLastModifiedDate(rb, key);
@@ -177,7 +190,8 @@ private Set<String> parseAttributesHeader(String keyPath) throws OS3Exception {
   }
 
   private GetObjectAttributesResponse buildResponse(String keyPath, OzoneKey key,
-      Set<String> requested, NavigableMap<Integer, Long> completedPartSizes)
+      Set<String> requested, NavigableMap<Integer, Long> completedPartSizes,
+      boolean directoryBucketLayout)
       throws IOException, OS3Exception {
     GetObjectAttributesResponse resp = new GetObjectAttributesResponse();
 
@@ -205,7 +219,7 @@ private GetObjectAttributesResponse buildResponse(String keyPath, OzoneKey key,
         String partsCountStr = extractPartsCount(eTag);
         if (partsCountStr != null && completedPartSizes != null) {
           resp.setObjectParts(buildObjectParts(keyPath, Integer.parseInt(partsCountStr),
-              completedPartSizes, key));
+              completedPartSizes, key, directoryBucketLayout));
         }
       }
     }
@@ -228,10 +242,12 @@ private GetObjectAttributesResponse buildResponse(String keyPath, OzoneKey key,
    *
    * <p>For general-purpose buckets, individual {@code Part} elements are returned only when the
    * object has a stored AWS additional checksum in key metadata; otherwise only {@code ObjectParts}
-   * summary and pagination fields are returned.
+   * summary and pagination fields are returned. For FSO layout (directory) buckets, {@code Part}
+   * elements are always returned.
    */
   private GetObjectAttributesResponse.ObjectParts buildObjectParts(String keyPath,
-      int totalPartsCount, NavigableMap<Integer, Long> partSizes, OzoneKey key)
+      int totalPartsCount, NavigableMap<Integer, Long> partSizes, OzoneKey key,
+      boolean directoryBucketLayout)
       throws OS3Exception {
     int maxParts = parseMaxPartsHeader(keyPath);
     int marker = parsePartNumberMarkerHeader(keyPath);
@@ -252,9 +268,7 @@ private GetObjectAttributesResponse.ObjectParts buildObjectParts(String keyPath,
 
     Iterator<Map.Entry<Integer, Long>> partIterator =
         partSizes.tailMap(marker, false).entrySet().iterator();
-    // TODO: For FSO (directory) buckets, always include Part entries per AWS
-    // directory-bucket GetObjectAttributes behavior, regardless of checksum metadata.
-    boolean includePartEntries = hasStoredAdditionalChecksum(key);
+    boolean includePartEntries = shouldIncludePartElements(key, directoryBucketLayout);
     Integer lastPartReturned = null;
     int partsOnPage = 0;
     while (partIterator.hasNext() && partsOnPage < maxParts) {

hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectAttributesGet.java

@@ -42,10 +42,14 @@
 import javax.ws.rs.core.HttpHeaders;
 import javax.ws.rs.core.Response;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.client.BucketArgs;
 import org.apache.hadoop.ozone.client.OzoneBucket;
 import org.apache.hadoop.ozone.client.OzoneBucketStub;
 import org.apache.hadoop.ozone.client.OzoneClient;
 import org.apache.hadoop.ozone.client.OzoneClientStub;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.om.helpers.BucketLayout;
 import org.apache.hadoop.ozone.s3.endpoint.CompleteMultipartUploadRequest.Part;
 import org.apache.hadoop.ozone.s3.exception.OS3Exception;
 import org.apache.hadoop.ozone.s3.util.S3Consts;
@@ -60,6 +64,7 @@ public class TestObjectAttributesGet {
 
   private static final String CONTENT = "0123456789";
   private static final String BUCKET_NAME = "b1";
+  private static final String FSO_BUCKET_NAME = "fso-b1";
   private static final String KEY_NAME = "key1";
   private ObjectEndpoint rest;
   private OzoneBucket bucket;
@@ -299,13 +304,52 @@ public void testGetObjectAttributesNonContiguousMultipartParts() throws IOExcept
     assertEquals(partThreeContent.length(), paginatedParts.getParts().get(0).getSize());
   }
 
-  private void completeMultipartUploadWithParts(String key, String... partContents)
+  @Test
+  public void testGetObjectAttrFsoMPUPartsWithoutChecksum()
       throws IOException, OS3Exception {
-    String uploadID = initiateMultipartUpload(rest, BUCKET_NAME, key);
+    OzoneClient client = rest.getClient();
+    String volumeName = rest.getOzoneConfiguration().get(OzoneConfigKeys.OZONE_S3_VOLUME_NAME,
+        OzoneConfigKeys.OZONE_S3_VOLUME_NAME_DEFAULT);
+    OzoneVolume volume = client.getObjectStore().getVolume(volumeName);
+    volume.createBucket(FSO_BUCKET_NAME, BucketArgs.newBuilder()
+        .setBucketLayout(BucketLayout.FILE_SYSTEM_OPTIMIZED)
+        .build());
+
+    final String key = "fso-mpu-key";
+    completeMultipartUploadWithPartsInBucket(FSO_BUCKET_NAME, key, "part-one", "part-two");
+
+    Response response = getObjectAttributes(rest, FSO_BUCKET_NAME, key, "ObjectParts");
+
+    assertEquals(HTTP_OK, response.getStatus());
+    GetObjectAttributesResponse.ObjectParts objectParts =
+        ((GetObjectAttributesResponse) response.getEntity()).getObjectParts();
+    assertNotNull(objectParts);
+    assertEquals(2, objectParts.getPartsCount().intValue());
+    assertEquals(2, objectParts.getParts().size());
+    assertEquals(1, objectParts.getParts().get(0).getPartNumber());
+    assertEquals("part-one".length(), objectParts.getParts().get(0).getSize());
+    assertEquals(2, objectParts.getParts().get(1).getPartNumber());
+    assertEquals("part-two".length(), objectParts.getParts().get(1).getSize());
+  }
+
+  @Test
+  public void testShouldIncludePartElements() {
+    assertFalse(ObjectAttributesHandler.shouldIncludePartElements(null, false));
+    assertTrue(ObjectAttributesHandler.shouldIncludePartElements(null, true));
+  }
+
+  private void completeMultipartUploadWithPartsInBucket(String bucketName, String key,
+      String... partContents) throws IOException, OS3Exception {
+    String uploadID = initiateMultipartUpload(rest, bucketName, key);
     List<Part> partsList = new ArrayList<>();
     for (int i = 0; i < partContents.length; i++) {
-      partsList.add(uploadPart(rest, BUCKET_NAME, key, i + 1, uploadID, partContents[i]));
+      partsList.add(uploadPart(rest, bucketName, key, i + 1, uploadID, partContents[i]));
     }
-    completeMultipartUpload(rest, BUCKET_NAME, key, uploadID, partsList);
+    completeMultipartUpload(rest, bucketName, key, uploadID, partsList);
+  }
+
+  private void completeMultipartUploadWithParts(String key, String... partContents)
+      throws IOException, OS3Exception {
+    completeMultipartUploadWithPartsInBucket(BUCKET_NAME, key, partContents);
   }
 }