From 15e797305818e3e90f437a98d06e0e7a837cf384 Mon Sep 17 00:00:00 2001 From: Gargi Jaiswal Date: Tue, 15 Sep 2026 11:17:32 +0530 Subject: [PATCH 1/2] GetObjectAttributes ObjectParts should return Part entries for FSO buckets --- .../s3/awssdk/v2/AbstractS3SDKV2Tests.java | 97 ++++++++++++++++--- .../s3/endpoint/ObjectAttributesHandler.java | 38 +++++--- .../s3/endpoint/TestObjectAttributesGet.java | 52 +++++++++- 3 files changed, 155 insertions(+), 32 deletions(-) diff --git a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java index bfcf5b741bc3..ad7a6c17209c 100644 --- a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java +++ b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java @@ -1907,6 +1907,71 @@ public void testGetObjectAttributesMultipartObjectParts(@TempDir Path tempDir) t assertTrue(secondPage.objectParts().parts().isEmpty()); } + /** + * Directory (FSO layout) buckets return per-part {@code Part} elements even when no additional + * checksum was stored at upload time, matching AWS S3 directory-bucket behavior. + */ + @Test + public void testGetObjectAttrFsoMpuNoChecksum(@TempDir Path tempDir) + throws Exception { + final String bucketName = uniqueObjectName(); + final String keyName = getKeyName(); + final int partSize = (int) (5 * MB); + + createFsoBucket(bucketName); + try { + File multipartUploadFile = + Files.createFile(tempDir.resolve("fso-get-object-attributes-mpu.txt")).toFile(); + createFile(multipartUploadFile, (int) (15 * MB)); + multipartUpload(bucketName, keyName, multipartUploadFile, partSize, new HashMap<>(), + Collections.emptyList()); + + GetObjectAttributesResponse attributesResponse = s3Client.getObjectAttributes( + GetObjectAttributesRequest.builder() + .bucket(bucketName) + .key(keyName) + .objectAttributes(ObjectAttributes.OBJECT_PARTS, ObjectAttributes.OBJECT_SIZE) + .build()); + + assertNotNull(attributesResponse.objectParts()); + assertEquals(3, attributesResponse.objectParts().totalPartsCount()); + assertFalse(attributesResponse.objectParts().isTruncated()); + assertEquals(multipartUploadFile.length(), attributesResponse.objectSize()); + assertEquals(3, attributesResponse.objectParts().parts().size()); + for (int i = 0; i < 3; i++) { + assertEquals(i + 1, attributesResponse.objectParts().parts().get(i).partNumber()); + assertEquals((long) partSize, attributesResponse.objectParts().parts().get(i).size()); + } + + GetObjectAttributesResponse firstPage = s3Client.getObjectAttributes( + GetObjectAttributesRequest.builder() + .bucket(bucketName) + .key(keyName) + .objectAttributes(ObjectAttributes.OBJECT_PARTS) + .maxParts(2) + .partNumberMarker(0) + .build()); + assertTrue(firstPage.objectParts().isTruncated()); + assertEquals(2, firstPage.objectParts().parts().size()); + assertEquals(1, firstPage.objectParts().parts().get(0).partNumber()); + assertEquals(2, firstPage.objectParts().parts().get(1).partNumber()); + + GetObjectAttributesResponse secondPage = s3Client.getObjectAttributes( + GetObjectAttributesRequest.builder() + .bucket(bucketName) + .key(keyName) + .objectAttributes(ObjectAttributes.OBJECT_PARTS) + .maxParts(2) + .partNumberMarker(2) + .build()); + assertFalse(secondPage.objectParts().isTruncated()); + assertEquals(1, secondPage.objectParts().parts().size()); + assertEquals(3, secondPage.objectParts().parts().get(0).partNumber()); + } finally { + deleteFsoBucket(bucketName); + } + } + @Test public void testGetObjectAttributesNonContiguousMultipartObjectParts() throws Exception { final String bucketName = getBucketName(); @@ -3080,6 +3145,22 @@ private String getKeyName(String ignored) { return uniqueObjectName(); } + private void createFsoBucket(String bucketName) throws Exception { + try (OzoneClient ozoneClient = cluster.newClient()) { + OzoneVolume volume = ozoneClient.getObjectStore().getS3Volume(); + volume.createBucket(bucketName, BucketArgs.newBuilder() + .setBucketLayout(BucketLayout.FILE_SYSTEM_OPTIMIZED) + .build()); + } + } + + private void deleteFsoBucket(String bucketName) throws Exception { + try (OzoneClient ozoneClient = cluster.newClient()) { + OzoneVolume volume = ozoneClient.getObjectStore().getS3Volume(); + volume.deleteBucket(bucketName); + } + } + private String multipartUpload(String bucketName, String key, File file, int partSize, Map userMetadata, List tags) throws Exception { String uploadId = initiateMultipartUpload(bucketName, key, userMetadata, tags); @@ -4228,21 +4309,5 @@ public void testListBucketsIncludesFSOBuckets() throws Exception { deleteFsoBucket(fsoBucketName); } } - - private void createFsoBucket(String bucketName) throws Exception { - try (OzoneClient ozoneClient = cluster.newClient()) { - OzoneVolume volume = ozoneClient.getObjectStore().getS3Volume(); - volume.createBucket(bucketName, BucketArgs.newBuilder() - .setBucketLayout(BucketLayout.FILE_SYSTEM_OPTIMIZED) - .build()); - } - } - - private void deleteFsoBucket(String bucketName) throws Exception { - try (OzoneClient ozoneClient = cluster.newClient()) { - OzoneVolume volume = ozoneClient.getObjectStore().getS3Volume(); - volume.deleteBucket(bucketName); - } - } } } diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectAttributesHandler.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectAttributesHandler.java index 1a1319bc13ae..38242177fa4d 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectAttributesHandler.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectAttributesHandler.java @@ -57,10 +57,10 @@ *

Returns selected metadata about an object without transferring the object body. * Supported attributes: {@code ETag}, {@code ObjectSize}, {@code StorageClass}, {@code ObjectParts}. * - *

The {@code Checksum} attribute is not yet supported because Ozone does not store - * non-MD5 checksum algorithms in key metadata. For general-purpose buckets, {@code Part} - * elements under {@code ObjectParts} are omitted unless an additional checksum is stored - * on the object, matching AWS S3 behavior. Object versioning ({@code versionId}) and + *

For general-purpose buckets, individual {@code Part} elements under {@code ObjectParts} + * are omitted unless an additional checksum is stored on the object, matching AWS S3 behavior. + * For FSO layout buckets (Ozone directory buckets), {@code Part} elements are always returned, + * matching AWS S3 directory-bucket behavior. Object versioning ({@code versionId}) and * SSE-C encryption headers are also not supported and are silently ignored. * *

See https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObjectAttributes.html @@ -101,6 +101,15 @@ static boolean hasStoredAdditionalChecksum(OzoneKey key) { return false; } + /** + * Whether {@code ObjectParts} should include per-part {@code Part} elements in the response. + * Directory (FSO layout) buckets always include them; general-purpose buckets include them + * only when an additional checksum was stored at upload time. + */ + static boolean shouldIncludePartElements(OzoneKey key, boolean directoryBucketLayout) { + return directoryBucketLayout || hasStoredAdditionalChecksum(key); + } + @Override Response handleGetRequest(ObjectRequestContext context, String keyPath) throws IOException, OS3Exception { @@ -137,8 +146,12 @@ Response handleGetRequest(ObjectRequestContext context, String keyPath) throw ex; } + boolean directoryBucketLayout = + context.getBucket().getBucketLayout().isFileSystemOptimized(); + GetObjectAttributesResponse response = - buildResponse(keyPath, key, requestedAttributes, completedPartSizes); + buildResponse(keyPath, key, requestedAttributes, completedPartSizes, + directoryBucketLayout); Response.ResponseBuilder rb = Response.ok(response, MediaType.APPLICATION_XML_TYPE); ObjectEndpoint.addLastModifiedDate(rb, key); @@ -177,7 +190,8 @@ private Set parseAttributesHeader(String keyPath) throws OS3Exception { } private GetObjectAttributesResponse buildResponse(String keyPath, OzoneKey key, - Set requested, NavigableMap completedPartSizes) + Set requested, NavigableMap completedPartSizes, + boolean directoryBucketLayout) throws IOException, OS3Exception { GetObjectAttributesResponse resp = new GetObjectAttributesResponse(); @@ -205,7 +219,7 @@ private GetObjectAttributesResponse buildResponse(String keyPath, OzoneKey key, String partsCountStr = extractPartsCount(eTag); if (partsCountStr != null && completedPartSizes != null) { resp.setObjectParts(buildObjectParts(keyPath, Integer.parseInt(partsCountStr), - completedPartSizes, key)); + completedPartSizes, key, directoryBucketLayout)); } } } @@ -228,10 +242,12 @@ private GetObjectAttributesResponse buildResponse(String keyPath, OzoneKey key, * *

For general-purpose buckets, individual {@code Part} elements are returned only when the * object has a stored AWS additional checksum in key metadata; otherwise only {@code ObjectParts} - * summary and pagination fields are returned. + * summary and pagination fields are returned. For FSO layout (directory) buckets, {@code Part} + * elements are always returned. */ private GetObjectAttributesResponse.ObjectParts buildObjectParts(String keyPath, - int totalPartsCount, NavigableMap partSizes, OzoneKey key) + int totalPartsCount, NavigableMap partSizes, OzoneKey key, + boolean directoryBucketLayout) throws OS3Exception { int maxParts = parseMaxPartsHeader(keyPath); int marker = parsePartNumberMarkerHeader(keyPath); @@ -252,9 +268,7 @@ private GetObjectAttributesResponse.ObjectParts buildObjectParts(String keyPath, Iterator> partIterator = partSizes.tailMap(marker, false).entrySet().iterator(); - // TODO: For FSO (directory) buckets, always include Part entries per AWS - // directory-bucket GetObjectAttributes behavior, regardless of checksum metadata. - boolean includePartEntries = hasStoredAdditionalChecksum(key); + boolean includePartEntries = shouldIncludePartElements(key, directoryBucketLayout); Integer lastPartReturned = null; int partsOnPage = 0; while (partIterator.hasNext() && partsOnPage < maxParts) { diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectAttributesGet.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectAttributesGet.java index 6c39ac89e1a0..0af6e4b105be 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectAttributesGet.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectAttributesGet.java @@ -42,10 +42,14 @@ import javax.ws.rs.core.HttpHeaders; import javax.ws.rs.core.Response; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.ozone.OzoneConfigKeys; +import org.apache.hadoop.ozone.client.BucketArgs; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneBucketStub; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneClientStub; +import org.apache.hadoop.ozone.client.OzoneVolume; +import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.s3.endpoint.CompleteMultipartUploadRequest.Part; import org.apache.hadoop.ozone.s3.exception.OS3Exception; import org.apache.hadoop.ozone.s3.util.S3Consts; @@ -60,6 +64,7 @@ public class TestObjectAttributesGet { private static final String CONTENT = "0123456789"; private static final String BUCKET_NAME = "b1"; + private static final String FSO_BUCKET_NAME = "fso-b1"; private static final String KEY_NAME = "key1"; private ObjectEndpoint rest; private OzoneBucket bucket; @@ -299,13 +304,52 @@ public void testGetObjectAttributesNonContiguousMultipartParts() throws IOExcept assertEquals(partThreeContent.length(), paginatedParts.getParts().get(0).getSize()); } - private void completeMultipartUploadWithParts(String key, String... partContents) + @Test + public void testGetObjectAttrFsoMPUPartsWithoutChecksum() throws IOException, OS3Exception { - String uploadID = initiateMultipartUpload(rest, BUCKET_NAME, key); + OzoneClient client = rest.getClient(); + String volumeName = rest.getOzoneConfiguration().get(OzoneConfigKeys.OZONE_S3_VOLUME_NAME, + OzoneConfigKeys.OZONE_S3_VOLUME_NAME_DEFAULT); + OzoneVolume volume = client.getObjectStore().getVolume(volumeName); + volume.createBucket(FSO_BUCKET_NAME, BucketArgs.newBuilder() + .setBucketLayout(BucketLayout.FILE_SYSTEM_OPTIMIZED) + .build()); + + final String key = "fso-mpu-key"; + completeMultipartUploadWithPartsInBucket(FSO_BUCKET_NAME, key, "part-one", "part-two"); + + Response response = getObjectAttributes(rest, FSO_BUCKET_NAME, key, "ObjectParts"); + + assertEquals(HTTP_OK, response.getStatus()); + GetObjectAttributesResponse.ObjectParts objectParts = + ((GetObjectAttributesResponse) response.getEntity()).getObjectParts(); + assertNotNull(objectParts); + assertEquals(2, objectParts.getPartsCount().intValue()); + assertEquals(2, objectParts.getParts().size()); + assertEquals(1, objectParts.getParts().get(0).getPartNumber()); + assertEquals("part-one".length(), objectParts.getParts().get(0).getSize()); + assertEquals(2, objectParts.getParts().get(1).getPartNumber()); + assertEquals("part-two".length(), objectParts.getParts().get(1).getSize()); + } + + @Test + public void testShouldIncludePartElements() { + assertFalse(ObjectAttributesHandler.shouldIncludePartElements(null, false)); + assertTrue(ObjectAttributesHandler.shouldIncludePartElements(null, true)); + } + + private void completeMultipartUploadWithPartsInBucket(String bucketName, String key, + String... partContents) throws IOException, OS3Exception { + String uploadID = initiateMultipartUpload(rest, bucketName, key); List partsList = new ArrayList<>(); for (int i = 0; i < partContents.length; i++) { - partsList.add(uploadPart(rest, BUCKET_NAME, key, i + 1, uploadID, partContents[i])); + partsList.add(uploadPart(rest, bucketName, key, i + 1, uploadID, partContents[i])); } - completeMultipartUpload(rest, BUCKET_NAME, key, uploadID, partsList); + completeMultipartUpload(rest, bucketName, key, uploadID, partsList); + } + + private void completeMultipartUploadWithParts(String key, String... partContents) + throws IOException, OS3Exception { + completeMultipartUploadWithPartsInBucket(BUCKET_NAME, key, partContents); } } From 2b86393089bcac6622ac21d3e734c313a2000810 Mon Sep 17 00:00:00 2001 From: Gargi Jaiswal Date: Tue, 15 Sep 2026 12:38:36 +0530 Subject: [PATCH 2/2] fix integration test failure --- .../apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java | 1 + 1 file changed, 1 insertion(+) diff --git a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java index ad7a6c17209c..1cf08264772b 100644 --- a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java +++ b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java @@ -1968,6 +1968,7 @@ public void testGetObjectAttrFsoMpuNoChecksum(@TempDir Path tempDir) assertEquals(1, secondPage.objectParts().parts().size()); assertEquals(3, secondPage.objectParts().parts().get(0).partNumber()); } finally { + s3Client.deleteObject(b -> b.bucket(bucketName).key(keyName)); deleteFsoBucket(bucketName); } }