From d5d8dc66af2407ccd4ca45ce788629e6436f7dba Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Fri, 17 Jul 2026 13:09:40 -0700 Subject: [PATCH 1/8] Avoid redundant getBucketInfo RPC on OFS getFileStatus. Cache validated bucket layouts per volume/bucket and call GetFileStatus directly after cache warm-up, preserving layout checks and headOp behavior. Co-authored-by: Cursor Change-Id: Icddbc99fee0cf0e78e52919283f93d29a1986a1c --- .../AbstractRootedOzoneFileSystemTest.java | 50 ++++++++++++++ .../BasicRootedOzoneClientAdapterImpl.java | 67 +++++++++++++++---- 2 files changed, 105 insertions(+), 12 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java index e479cbb661eb..7ab33a90ee5d 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java @@ -1847,6 +1847,56 @@ void testGetFileStatus() throws Exception { fs.delete(volume, false); } + @Test + void testGetFileStatusUsesSingleOmRpcAfterCacheWarm() throws Exception { + Path path = new Path(bucketPath, "single-rpc-stat-test"); + fs.mkdirs(path); + // Warm the bucket layout cache; ignore metrics from the first stat. + fs.getFileStatus(path); + + long getFileStatusBefore = getOMMetrics().getNumGetFileStatus(); + long bucketInfoBefore = getOMMetrics().getNumBucketInfos(); + + FileStatus status = fs.getFileStatus(path); + assertTrue(status.isDirectory()); + + assertEquals(getFileStatusBefore + 1, getOMMetrics().getNumGetFileStatus()); + assertEquals(bucketInfoBefore, getOMMetrics().getNumBucketInfos()); + } + + @Test + void testGetFileStatusOnBucketRoot() throws Exception { + FileStatus status = fs.getFileStatus(bucketPath); + assertTrue(status.isDirectory()); + } + + @Test + void testGetFileStatusOnObjectStoreBucketRejectsInvalidLayout() + throws Exception { + OzoneBucket obsBucket = + TestDataUtil.createVolumeAndBucket(client, BucketLayout.OBJECT_STORE); + Path obsBucketPath = new Path( + new Path(OZONE_URI_DELIMITER + obsBucket.getVolumeName()), + obsBucket.getName()); + try { + IllegalArgumentException exception = assertThrows( + IllegalArgumentException.class, () -> fs.getFileStatus(obsBucketPath)); + assertThat(exception.getMessage()) + .contains(BucketLayout.OBJECT_STORE.name()); + } finally { + objectStore.deleteVolume(obsBucket.getVolumeName()); + } + } + + @Test + void testGetFileStatusMissingFile() throws Exception { + Path missingFile = new Path(bucketPath, "missing-file-" + + RandomStringUtils.secure().nextAlphanumeric(5)); + FileNotFoundException exception = assertThrows(FileNotFoundException.class, + () -> fs.getFileStatus(missingFile)); + assertThat(exception.getMessage()).contains("No such file or directory"); + } + @Test void testUnbuffer() throws IOException { String testKeyName = "testKey2"; diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java index 50842949af2c..85a01fced0cc 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java @@ -41,6 +41,7 @@ import java.util.Iterator; import java.util.List; import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; @@ -132,6 +133,11 @@ public class BasicRootedOzoneClientAdapterImpl private BucketLayout defaultOFSBucketLayout; private final OzoneConfiguration config; private final OzoneClientConfig clientConfig; + /** + * Cache of successfully validated bucket layouts for read paths. + */ + private final ConcurrentHashMap validatedBucketLayouts = + new ConcurrentHashMap<>(); /** * Create new OzoneClientAdapter implementation. @@ -274,6 +280,47 @@ OzoneBucket getBucket(OFSPath ofsPath, boolean createIfNotExist) createIfNotExist); } + private static String bucketLayoutCacheKey(String volumeStr, String bucketStr) { + return volumeStr + OZONE_URI_DELIMITER + bucketStr; + } + + private BucketLayout validateBucketLayoutForBucket(OzoneBucket bucket) + throws IOException { + BucketLayout resolvedBucketLayout = + OzoneClientUtils.resolveLinkBucketLayout(bucket, objectStore, + new HashSet<>()); + OzoneFSUtils.validateBucketLayout(bucket.getName(), resolvedBucketLayout); + return resolvedBucketLayout; + } + + private void cacheValidatedBucketLayout(String volumeStr, String bucketStr, + BucketLayout layout) { + validatedBucketLayouts.putIfAbsent( + bucketLayoutCacheKey(volumeStr, bucketStr), layout); + } + + private void validateAndCacheBucketLayout(String volumeStr, String bucketStr, + OzoneBucket bucket) throws IOException { + BucketLayout layout = validateBucketLayoutForBucket(bucket); + cacheValidatedBucketLayout(volumeStr, bucketStr, layout); + } + + private void resolveAndValidateBucketLayout(String volumeStr, String bucketStr) + throws IOException { + OzoneBucket bucket = proxy.getBucketDetails(volumeStr, bucketStr); + validateAndCacheBucketLayout(volumeStr, bucketStr, bucket); + } + + private void ensureBucketLayoutValid(OFSPath ofsPath) throws IOException { + String volumeStr = ofsPath.getVolumeName(); + String bucketStr = ofsPath.getBucketName(); + if (validatedBucketLayouts.containsKey( + bucketLayoutCacheKey(volumeStr, bucketStr))) { + return; + } + resolveAndValidateBucketLayout(volumeStr, bucketStr); + } + /** * Get OzoneBucket object to operate in. * Optionally create volume and bucket if not found. @@ -297,13 +344,7 @@ private OzoneBucket getBucket(String volumeStr, String bucketStr, OzoneBucket bucket; try { bucket = proxy.getBucketDetails(volumeStr, bucketStr); - - // resolve the bucket layout in case of Link Bucket - BucketLayout resolvedBucketLayout = - OzoneClientUtils.resolveLinkBucketLayout(bucket, objectStore, - new HashSet<>()); - - OzoneFSUtils.validateBucketLayout(bucket.getName(), resolvedBucketLayout); + validateAndCacheBucketLayout(volumeStr, bucketStr, bucket); } catch (OMException ex) { if (createIfNotExist) { // getBucketDetails can throw VOLUME_NOT_FOUND when the parent volume @@ -345,6 +386,7 @@ private OzoneBucket getBucket(String volumeStr, String bucketStr, } // Try get bucket again bucket = proxy.getBucketDetails(volumeStr, bucketStr); + validateAndCacheBucketLayout(volumeStr, bucketStr, bucket); } else { throw ex; } @@ -696,16 +738,17 @@ private FileStatusAdapter getFileStatusForKeyOrSnapshot( boolean headOp) throws IOException { String key = ofsPath.getKeyName(); try { - OzoneBucket bucket = getBucket(ofsPath, false); if (ofsPath.isSnapshotPath()) { + OzoneBucket bucket = getBucket(ofsPath, false); OzoneVolume volume = objectStore.getVolume(ofsPath.getVolumeName()); return getFileStatusAdapterWithSnapshotIndicator( volume, bucket, uri); - } else { - OzoneFileStatus status = bucket.getFileStatus(key, headOp); - return toFileStatusAdapter(status, userName, uri, qualifiedPath, - ofsPath.getNonKeyPath()); } + ensureBucketLayoutValid(ofsPath); + OzoneFileStatus status = proxy.getOzoneFileStatus( + ofsPath.getVolumeName(), ofsPath.getBucketName(), key, headOp); + return toFileStatusAdapter(status, userName, uri, qualifiedPath, + ofsPath.getNonKeyPath()); } catch (OMException e) { if (e.getResult() == OMException.ResultCodes.FILE_NOT_FOUND) { throw new FileNotFoundException(key + ": No such file or directory!"); From 9a326e5a233e06b85a06172c2bbb5f22b04667fe Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Fri, 17 Jul 2026 13:57:53 -0700 Subject: [PATCH 2/8] Fix OBS bucket cleanup in getFileStatus layout test. Delete the bucket before removing the volume to avoid VOLUME_NOT_EMPTY. Co-authored-by: Cursor Change-Id: Iddfefe48d716e77cb58501514c919a44194cdae7 --- .../hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java index 7ab33a90ee5d..2874c08e6f5d 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java @@ -1884,6 +1884,8 @@ void testGetFileStatusOnObjectStoreBucketRejectsInvalidLayout() assertThat(exception.getMessage()) .contains(BucketLayout.OBJECT_STORE.name()); } finally { + objectStore.getVolume(obsBucket.getVolumeName()) + .deleteBucket(obsBucket.getName()); objectStore.deleteVolume(obsBucket.getVolumeName()); } } From 729623ccbec53be0f0091be0075867195a570f7d Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Fri, 17 Jul 2026 14:32:50 -0700 Subject: [PATCH 3/8] Retrigger CI after test cleanup fix. Co-authored-by: Cursor Change-Id: Ifc177a61594da562cc3cbc13b6f622fada55971d From 47a477d4c662d0223cd13f5629018444ba3dc8ec Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Fri, 17 Jul 2026 15:09:13 -0700 Subject: [PATCH 4/8] Update headOp unit tests for direct getOzoneFileStatus path. Mock ClientProtocol and warm the layout cache after getFileStatus stopped calling OzoneBucket.getFileStatus for non-snapshot paths. Co-authored-by: Cursor Change-Id: I34e369396f682c096fa23b0f5df7b8b165feab6f --- ...stBasicRootedOzoneClientAdapterHeadOp.java | 56 ++++++++++++++----- 1 file changed, 41 insertions(+), 15 deletions(-) diff --git a/hadoop-ozone/ozonefs-common/src/test/java/org/apache/hadoop/fs/ozone/TestBasicRootedOzoneClientAdapterHeadOp.java b/hadoop-ozone/ozonefs-common/src/test/java/org/apache/hadoop/fs/ozone/TestBasicRootedOzoneClientAdapterHeadOp.java index 8f67adef1346..a5da06266940 100644 --- a/hadoop-ozone/ozonefs-common/src/test/java/org/apache/hadoop/fs/ozone/TestBasicRootedOzoneClientAdapterHeadOp.java +++ b/hadoop-ozone/ozonefs-common/src/test/java/org/apache/hadoop/fs/ozone/TestBasicRootedOzoneClientAdapterHeadOp.java @@ -36,6 +36,7 @@ import java.net.URI; import java.time.Instant; import java.util.Collections; +import java.util.concurrent.ConcurrentHashMap; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdds.client.RatisReplicationConfig; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; @@ -43,7 +44,9 @@ import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneVolume; +import org.apache.hadoop.ozone.client.protocol.ClientProtocol; import org.apache.hadoop.ozone.om.exceptions.OMException; +import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; import org.junit.jupiter.api.BeforeEach; @@ -62,11 +65,13 @@ public class TestBasicRootedOzoneClientAdapterHeadOp { private BasicRootedOzoneClientAdapterImpl adapter; private OzoneBucket bucket; + private ClientProtocol proxy; @BeforeEach public void setUp() throws Exception { adapter = mock(BasicRootedOzoneClientAdapterImpl.class, CALLS_REAL_METHODS); bucket = mock(OzoneBucket.class); + proxy = mock(ClientProtocol.class); doReturn(bucket).when(adapter).getBucket(any(OFSPath.class), eq(false)); // Inject a mock object store so the volume/snapshot dispatch branches can @@ -77,10 +82,29 @@ public void setUp() throws Exception { when(volume.getCreationTime()).thenReturn(Instant.EPOCH); ObjectStore objectStore = mock(ObjectStore.class); when(objectStore.getVolume(anyString())).thenReturn(volume); - Field field = - BasicRootedOzoneClientAdapterImpl.class.getDeclaredField("objectStore"); + setField(adapter, "objectStore", objectStore); + setField(adapter, "proxy", proxy); + warmLayoutCache("vol", "bucket"); + } + + private static void setField(Object target, String name, Object value) + throws Exception { + Field field = BasicRootedOzoneClientAdapterImpl.class.getDeclaredField(name); field.setAccessible(true); - field.set(adapter, objectStore); + field.set(target, value); + } + + @SuppressWarnings("unchecked") + private void warmLayoutCache(String volumeName, String bucketName) + throws Exception { + Field cacheField = + BasicRootedOzoneClientAdapterImpl.class.getDeclaredField( + "validatedBucketLayouts"); + cacheField.setAccessible(true); + ConcurrentHashMap cache = + new ConcurrentHashMap<>(); + cache.put(volumeName + "/" + bucketName, BucketLayout.FILE_SYSTEM_OPTIMIZED); + cacheField.set(adapter, cache); } private static OzoneFileStatus fileStatus(boolean isDir) { @@ -101,25 +125,27 @@ private static OzoneFileStatus fileStatus(boolean isDir) { @Test public void keyPathThreadsHeadOp() throws IOException { - when(bucket.getFileStatus(anyString(), anyBoolean())) - .thenReturn(fileStatus(false)); + when(proxy.getOzoneFileStatus(eq("vol"), eq("bucket"), anyString(), + anyBoolean())).thenReturn(fileStatus(false)); assertFalse(adapter.getFileStatus("/vol/bucket/key", URI_OFS, WORKING_DIR, "user", true).isDir()); ArgumentCaptor headOp = ArgumentCaptor.forClass(Boolean.class); - verify(bucket).getFileStatus(anyString(), headOp.capture()); + verify(proxy).getOzoneFileStatus(eq("vol"), eq("bucket"), anyString(), + headOp.capture()); assertTrue(headOp.getValue()); } @Test public void fourArgOverloadDoesNotUseHeadOp() throws IOException { - when(bucket.getFileStatus(anyString(), anyBoolean())) - .thenReturn(fileStatus(true)); + when(proxy.getOzoneFileStatus(eq("vol"), eq("bucket"), anyString(), + anyBoolean())).thenReturn(fileStatus(true)); assertTrue(adapter.getFileStatus("/vol/bucket/key", URI_OFS, WORKING_DIR, "user").isDir()); - verify(bucket).getFileStatus(anyString(), eq(false)); + verify(proxy).getOzoneFileStatus(eq("vol"), eq("bucket"), anyString(), + eq(false)); } @Test @@ -130,8 +156,8 @@ public void rootPathReturnsDirectory() throws IOException { @Test public void fileNotFoundMappedToFileNotFoundException() throws IOException { - when(bucket.getFileStatus(anyString(), anyBoolean())) - .thenThrow(new OMException("missing", + when(proxy.getOzoneFileStatus(eq("vol"), eq("bucket"), anyString(), + anyBoolean())).thenThrow(new OMException("missing", OMException.ResultCodes.FILE_NOT_FOUND)); assertThrows(FileNotFoundException.class, () -> adapter.getFileStatus("/vol/bucket/key", URI_OFS, WORKING_DIR, @@ -140,8 +166,8 @@ public void fileNotFoundMappedToFileNotFoundException() throws IOException { @Test public void otherOMExceptionPropagates() throws IOException { - when(bucket.getFileStatus(anyString(), anyBoolean())) - .thenThrow(new OMException("boom", + when(proxy.getOzoneFileStatus(eq("vol"), eq("bucket"), anyString(), + anyBoolean())).thenThrow(new OMException("boom", OMException.ResultCodes.INTERNAL_ERROR)); assertThrows(OMException.class, () -> adapter.getFileStatus("/vol/bucket/key", URI_OFS, WORKING_DIR, @@ -150,8 +176,8 @@ public void otherOMExceptionPropagates() throws IOException { @Test public void bucketNotFoundMappedToFileNotFoundException() throws IOException { - when(bucket.getFileStatus(anyString(), anyBoolean())) - .thenThrow(new OMException("no bucket", + when(proxy.getOzoneFileStatus(eq("vol"), eq("bucket"), anyString(), + anyBoolean())).thenThrow(new OMException("no bucket", OMException.ResultCodes.BUCKET_NOT_FOUND)); assertThrows(FileNotFoundException.class, () -> adapter.getFileStatus("/vol/bucket/key", URI_OFS, WORKING_DIR, From 0d9fa4864ace469e73dbb3f839f8ad978b76634e Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Thu, 23 Jul 2026 13:57:37 -0700 Subject: [PATCH 5/8] HDDS-15925. Skip redundant InfoBucket RPC on OFS getFileStatus. Call proxy.getOzoneFileStatus directly on non-snapshot paths and drop the client-side layout cache. Document accepted OBJECT_STORE stat behavior change. Co-authored-by: Cursor Change-Id: I5c9df439cb636c7a0a2f0660f6bedce41b2b4419 --- .../AbstractRootedOzoneFileSystemTest.java | 70 +++++++---------- .../BasicRootedOzoneClientAdapterImpl.java | 76 ++++++------------- ...stBasicRootedOzoneClientAdapterHeadOp.java | 60 +++++---------- 3 files changed, 69 insertions(+), 137 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java index 2874c08e6f5d..240c5f551f7e 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java @@ -1835,6 +1835,33 @@ void testBucketDefaultsShouldBeInheritedToFileForEC() assertEquals(ReplicationType.EC.name(), key.getReplicationConfig().getReplicationType().name()); } + /** + * HDDS-15925: rooted OFS getFileStatus on a key path must not issue InfoBucket + * before GetFileStatus. + */ + @Test + void testGetFileStatusUsesSingleOmRpc() throws Exception { + String keyName = "single-rpc-" + RandomStringUtils.secure().nextAlphabetic(5); + Path filePath = new Path(bucketPath, keyName); + ContractTestUtils.touch(fs, filePath); + + OMMetrics metrics = getOMMetrics(); + long bucketInfosBefore = metrics.getNumBucketInfos(); + long getFileStatusBefore = metrics.getNumGetFileStatus(); + + FileStatus status = fs.getFileStatus(filePath); + assertTrue(status.isFile()); + + assertEquals(bucketInfosBefore, metrics.getNumBucketInfos(), + "getFileStatus must not trigger InfoBucket"); + assertEquals(getFileStatusBefore + 1, metrics.getNumGetFileStatus()); + + long getFileStatusAfterFirst = metrics.getNumGetFileStatus(); + fs.getFileStatus(filePath); + assertEquals(bucketInfosBefore, metrics.getNumBucketInfos()); + assertEquals(getFileStatusAfterFirst + 1, metrics.getNumGetFileStatus()); + } + @Test void testGetFileStatus() throws Exception { String volumeNameLocal = getRandomNonExistVolumeName(); @@ -1847,49 +1874,6 @@ void testGetFileStatus() throws Exception { fs.delete(volume, false); } - @Test - void testGetFileStatusUsesSingleOmRpcAfterCacheWarm() throws Exception { - Path path = new Path(bucketPath, "single-rpc-stat-test"); - fs.mkdirs(path); - // Warm the bucket layout cache; ignore metrics from the first stat. - fs.getFileStatus(path); - - long getFileStatusBefore = getOMMetrics().getNumGetFileStatus(); - long bucketInfoBefore = getOMMetrics().getNumBucketInfos(); - - FileStatus status = fs.getFileStatus(path); - assertTrue(status.isDirectory()); - - assertEquals(getFileStatusBefore + 1, getOMMetrics().getNumGetFileStatus()); - assertEquals(bucketInfoBefore, getOMMetrics().getNumBucketInfos()); - } - - @Test - void testGetFileStatusOnBucketRoot() throws Exception { - FileStatus status = fs.getFileStatus(bucketPath); - assertTrue(status.isDirectory()); - } - - @Test - void testGetFileStatusOnObjectStoreBucketRejectsInvalidLayout() - throws Exception { - OzoneBucket obsBucket = - TestDataUtil.createVolumeAndBucket(client, BucketLayout.OBJECT_STORE); - Path obsBucketPath = new Path( - new Path(OZONE_URI_DELIMITER + obsBucket.getVolumeName()), - obsBucket.getName()); - try { - IllegalArgumentException exception = assertThrows( - IllegalArgumentException.class, () -> fs.getFileStatus(obsBucketPath)); - assertThat(exception.getMessage()) - .contains(BucketLayout.OBJECT_STORE.name()); - } finally { - objectStore.getVolume(obsBucket.getVolumeName()) - .deleteBucket(obsBucket.getName()); - objectStore.deleteVolume(obsBucket.getVolumeName()); - } - } - @Test void testGetFileStatusMissingFile() throws Exception { Path missingFile = new Path(bucketPath, "missing-file-" + diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java index 85a01fced0cc..2ac50651ec72 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java @@ -41,7 +41,6 @@ import java.util.Iterator; import java.util.List; import java.util.Objects; -import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; @@ -133,11 +132,6 @@ public class BasicRootedOzoneClientAdapterImpl private BucketLayout defaultOFSBucketLayout; private final OzoneConfiguration config; private final OzoneClientConfig clientConfig; - /** - * Cache of successfully validated bucket layouts for read paths. - */ - private final ConcurrentHashMap validatedBucketLayouts = - new ConcurrentHashMap<>(); /** * Create new OzoneClientAdapter implementation. @@ -280,47 +274,6 @@ OzoneBucket getBucket(OFSPath ofsPath, boolean createIfNotExist) createIfNotExist); } - private static String bucketLayoutCacheKey(String volumeStr, String bucketStr) { - return volumeStr + OZONE_URI_DELIMITER + bucketStr; - } - - private BucketLayout validateBucketLayoutForBucket(OzoneBucket bucket) - throws IOException { - BucketLayout resolvedBucketLayout = - OzoneClientUtils.resolveLinkBucketLayout(bucket, objectStore, - new HashSet<>()); - OzoneFSUtils.validateBucketLayout(bucket.getName(), resolvedBucketLayout); - return resolvedBucketLayout; - } - - private void cacheValidatedBucketLayout(String volumeStr, String bucketStr, - BucketLayout layout) { - validatedBucketLayouts.putIfAbsent( - bucketLayoutCacheKey(volumeStr, bucketStr), layout); - } - - private void validateAndCacheBucketLayout(String volumeStr, String bucketStr, - OzoneBucket bucket) throws IOException { - BucketLayout layout = validateBucketLayoutForBucket(bucket); - cacheValidatedBucketLayout(volumeStr, bucketStr, layout); - } - - private void resolveAndValidateBucketLayout(String volumeStr, String bucketStr) - throws IOException { - OzoneBucket bucket = proxy.getBucketDetails(volumeStr, bucketStr); - validateAndCacheBucketLayout(volumeStr, bucketStr, bucket); - } - - private void ensureBucketLayoutValid(OFSPath ofsPath) throws IOException { - String volumeStr = ofsPath.getVolumeName(); - String bucketStr = ofsPath.getBucketName(); - if (validatedBucketLayouts.containsKey( - bucketLayoutCacheKey(volumeStr, bucketStr))) { - return; - } - resolveAndValidateBucketLayout(volumeStr, bucketStr); - } - /** * Get OzoneBucket object to operate in. * Optionally create volume and bucket if not found. @@ -344,7 +297,13 @@ private OzoneBucket getBucket(String volumeStr, String bucketStr, OzoneBucket bucket; try { bucket = proxy.getBucketDetails(volumeStr, bucketStr); - validateAndCacheBucketLayout(volumeStr, bucketStr, bucket); + + // resolve the bucket layout in case of Link Bucket + BucketLayout resolvedBucketLayout = + OzoneClientUtils.resolveLinkBucketLayout(bucket, objectStore, + new HashSet<>()); + + OzoneFSUtils.validateBucketLayout(bucket.getName(), resolvedBucketLayout); } catch (OMException ex) { if (createIfNotExist) { // getBucketDetails can throw VOLUME_NOT_FOUND when the parent volume @@ -386,7 +345,12 @@ private OzoneBucket getBucket(String volumeStr, String bucketStr, } // Try get bucket again bucket = proxy.getBucketDetails(volumeStr, bucketStr); - validateAndCacheBucketLayout(volumeStr, bucketStr, bucket); + + BucketLayout resolvedBucketLayout = + OzoneClientUtils.resolveLinkBucketLayout(bucket, objectStore, + new HashSet<>()); + + OzoneFSUtils.validateBucketLayout(bucket.getName(), resolvedBucketLayout); } else { throw ex; } @@ -732,6 +696,10 @@ public FileStatusAdapter getFileStatus(String path, URI uri, * Return FileStatusAdapter based on OFSPath being a * valid bucket path or valid snapshot path. * Throws exception in case of failure. + * + *

Non-snapshot paths call OM GetFileStatus directly (HDDS-15925) without a + * prior InfoBucket RPC. OBJECT_STORE buckets are not rejected on this path; + * mutating OFS operations still validate layout via {@link #getBucket(OFSPath, boolean)}. */ private FileStatusAdapter getFileStatusForKeyOrSnapshot( OFSPath ofsPath, URI uri, Path qualifiedPath, String userName, @@ -743,12 +711,12 @@ private FileStatusAdapter getFileStatusForKeyOrSnapshot( OzoneVolume volume = objectStore.getVolume(ofsPath.getVolumeName()); return getFileStatusAdapterWithSnapshotIndicator( volume, bucket, uri); + } else { + OzoneFileStatus status = proxy.getOzoneFileStatus(ofsPath.getVolumeName(), + ofsPath.getBucketName(), key, headOp); + return toFileStatusAdapter(status, userName, uri, qualifiedPath, + ofsPath.getNonKeyPath()); } - ensureBucketLayoutValid(ofsPath); - OzoneFileStatus status = proxy.getOzoneFileStatus( - ofsPath.getVolumeName(), ofsPath.getBucketName(), key, headOp); - return toFileStatusAdapter(status, userName, uri, qualifiedPath, - ofsPath.getNonKeyPath()); } catch (OMException e) { if (e.getResult() == OMException.ResultCodes.FILE_NOT_FOUND) { throw new FileNotFoundException(key + ": No such file or directory!"); diff --git a/hadoop-ozone/ozonefs-common/src/test/java/org/apache/hadoop/fs/ozone/TestBasicRootedOzoneClientAdapterHeadOp.java b/hadoop-ozone/ozonefs-common/src/test/java/org/apache/hadoop/fs/ozone/TestBasicRootedOzoneClientAdapterHeadOp.java index a5da06266940..7b5f20e30192 100644 --- a/hadoop-ozone/ozonefs-common/src/test/java/org/apache/hadoop/fs/ozone/TestBasicRootedOzoneClientAdapterHeadOp.java +++ b/hadoop-ozone/ozonefs-common/src/test/java/org/apache/hadoop/fs/ozone/TestBasicRootedOzoneClientAdapterHeadOp.java @@ -36,7 +36,6 @@ import java.net.URI; import java.time.Instant; import java.util.Collections; -import java.util.concurrent.ConcurrentHashMap; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdds.client.RatisReplicationConfig; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; @@ -46,7 +45,6 @@ import org.apache.hadoop.ozone.client.OzoneVolume; import org.apache.hadoop.ozone.client.protocol.ClientProtocol; import org.apache.hadoop.ozone.om.exceptions.OMException; -import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; import org.junit.jupiter.api.BeforeEach; @@ -74,37 +72,21 @@ public void setUp() throws Exception { proxy = mock(ClientProtocol.class); doReturn(bucket).when(adapter).getBucket(any(OFSPath.class), eq(false)); - // Inject a mock object store so the volume/snapshot dispatch branches can - // run without a live OM connection. + Field proxyField = + BasicRootedOzoneClientAdapterImpl.class.getDeclaredField("proxy"); + proxyField.setAccessible(true); + proxyField.set(adapter, proxy); + OzoneVolume volume = mock(OzoneVolume.class); when(volume.getName()).thenReturn("vol"); when(volume.getOwner()).thenReturn("user"); when(volume.getCreationTime()).thenReturn(Instant.EPOCH); ObjectStore objectStore = mock(ObjectStore.class); when(objectStore.getVolume(anyString())).thenReturn(volume); - setField(adapter, "objectStore", objectStore); - setField(adapter, "proxy", proxy); - warmLayoutCache("vol", "bucket"); - } - - private static void setField(Object target, String name, Object value) - throws Exception { - Field field = BasicRootedOzoneClientAdapterImpl.class.getDeclaredField(name); + Field field = + BasicRootedOzoneClientAdapterImpl.class.getDeclaredField("objectStore"); field.setAccessible(true); - field.set(target, value); - } - - @SuppressWarnings("unchecked") - private void warmLayoutCache(String volumeName, String bucketName) - throws Exception { - Field cacheField = - BasicRootedOzoneClientAdapterImpl.class.getDeclaredField( - "validatedBucketLayouts"); - cacheField.setAccessible(true); - ConcurrentHashMap cache = - new ConcurrentHashMap<>(); - cache.put(volumeName + "/" + bucketName, BucketLayout.FILE_SYSTEM_OPTIMIZED); - cacheField.set(adapter, cache); + field.set(adapter, objectStore); } private static OzoneFileStatus fileStatus(boolean isDir) { @@ -125,27 +107,26 @@ private static OzoneFileStatus fileStatus(boolean isDir) { @Test public void keyPathThreadsHeadOp() throws IOException { - when(proxy.getOzoneFileStatus(eq("vol"), eq("bucket"), anyString(), - anyBoolean())).thenReturn(fileStatus(false)); + when(proxy.getOzoneFileStatus(anyString(), anyString(), anyString(), anyBoolean())) + .thenReturn(fileStatus(false)); assertFalse(adapter.getFileStatus("/vol/bucket/key", URI_OFS, WORKING_DIR, "user", true).isDir()); ArgumentCaptor headOp = ArgumentCaptor.forClass(Boolean.class); - verify(proxy).getOzoneFileStatus(eq("vol"), eq("bucket"), anyString(), + verify(proxy).getOzoneFileStatus(eq("vol"), eq("bucket"), eq("key"), headOp.capture()); assertTrue(headOp.getValue()); } @Test public void fourArgOverloadDoesNotUseHeadOp() throws IOException { - when(proxy.getOzoneFileStatus(eq("vol"), eq("bucket"), anyString(), - anyBoolean())).thenReturn(fileStatus(true)); + when(proxy.getOzoneFileStatus(anyString(), anyString(), anyString(), anyBoolean())) + .thenReturn(fileStatus(true)); assertTrue(adapter.getFileStatus("/vol/bucket/key", URI_OFS, WORKING_DIR, "user").isDir()); - verify(proxy).getOzoneFileStatus(eq("vol"), eq("bucket"), anyString(), - eq(false)); + verify(proxy).getOzoneFileStatus(eq("vol"), eq("bucket"), eq("key"), eq(false)); } @Test @@ -156,8 +137,8 @@ public void rootPathReturnsDirectory() throws IOException { @Test public void fileNotFoundMappedToFileNotFoundException() throws IOException { - when(proxy.getOzoneFileStatus(eq("vol"), eq("bucket"), anyString(), - anyBoolean())).thenThrow(new OMException("missing", + when(proxy.getOzoneFileStatus(anyString(), anyString(), anyString(), anyBoolean())) + .thenThrow(new OMException("missing", OMException.ResultCodes.FILE_NOT_FOUND)); assertThrows(FileNotFoundException.class, () -> adapter.getFileStatus("/vol/bucket/key", URI_OFS, WORKING_DIR, @@ -166,8 +147,8 @@ public void fileNotFoundMappedToFileNotFoundException() throws IOException { @Test public void otherOMExceptionPropagates() throws IOException { - when(proxy.getOzoneFileStatus(eq("vol"), eq("bucket"), anyString(), - anyBoolean())).thenThrow(new OMException("boom", + when(proxy.getOzoneFileStatus(anyString(), anyString(), anyString(), anyBoolean())) + .thenThrow(new OMException("boom", OMException.ResultCodes.INTERNAL_ERROR)); assertThrows(OMException.class, () -> adapter.getFileStatus("/vol/bucket/key", URI_OFS, WORKING_DIR, @@ -176,8 +157,8 @@ public void otherOMExceptionPropagates() throws IOException { @Test public void bucketNotFoundMappedToFileNotFoundException() throws IOException { - when(proxy.getOzoneFileStatus(eq("vol"), eq("bucket"), anyString(), - anyBoolean())).thenThrow(new OMException("no bucket", + when(proxy.getOzoneFileStatus(anyString(), anyString(), anyString(), anyBoolean())) + .thenThrow(new OMException("no bucket", OMException.ResultCodes.BUCKET_NOT_FOUND)); assertThrows(FileNotFoundException.class, () -> adapter.getFileStatus("/vol/bucket/key", URI_OFS, WORKING_DIR, @@ -195,7 +176,6 @@ public void snapshotIndicatorPathReturnsDirectory() throws IOException { when(bucket.getVolumeName()).thenReturn("vol"); when(bucket.getName()).thenReturn("bucket"); when(bucket.getCreationTime()).thenReturn(Instant.EPOCH); - // keyName == ".snapshot" is the snapshot indicator path. assertTrue(adapter.getFileStatus("/vol/bucket/.snapshot", URI_OFS, WORKING_DIR, "user", true).isDir()); } From d811a2861b43409293bfebc7c134f55ca34f0caa Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Thu, 23 Jul 2026 14:41:26 -0700 Subject: [PATCH 6/8] HDDS-15925. Expect FileNotFoundException for missing bucket getFileStatus. After skipping InfoBucket, OM BUCKET_NOT_FOUND is mapped in the adapter as today for GetFileStatus paths. Use try/finally so volume cleanup runs on assertion. Co-authored-by: Cursor Change-Id: I2408951221c741ae24b823c69847ac905d1bd9f8 --- .../fs/ozone/AbstractRootedOzoneFileSystemTest.java | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java index 240c5f551f7e..f5cce61646ba 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java @@ -1868,10 +1868,13 @@ void testGetFileStatus() throws Exception { String bucketNameLocal = RandomStringUtils.secure().nextNumeric(5); Path volume = new Path("/" + volumeNameLocal); fs.mkdirs(volume); - assertThrows(OMException.class, - () -> fs.getFileStatus(new Path(volume, bucketNameLocal))); - // Cleanup - fs.delete(volume, false); + try { + FileNotFoundException exception = assertThrows(FileNotFoundException.class, + () -> fs.getFileStatus(new Path(volume, bucketNameLocal))); + assertThat(exception.getMessage()).contains("Bucket doesn't exist"); + } finally { + fs.delete(volume, false); + } } @Test From 1fb858e52efcdcb6db02f5e38a2fcd66128e1a88 Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Thu, 23 Jul 2026 15:53:28 -0700 Subject: [PATCH 7/8] HDDS-15925. Update OBS smoketest for getFileStatus bypass. Rooted OFS getFileStatus on keys no longer rejects OBJECT_STORE layout; ls on an OBS key succeeds while mutating operations still fail as before. Co-authored-by: Cursor Change-Id: Icfacbc5e8d61b32a9c548a7ed95188dee3cc2925 --- .../dist/src/main/smoketest/ozonefs/ozonefs-obs.robot | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/dist/src/main/smoketest/ozonefs/ozonefs-obs.robot b/hadoop-ozone/dist/src/main/smoketest/ozonefs/ozonefs-obs.robot index 57f015162608..6be04a002525 100644 --- a/hadoop-ozone/dist/src/main/smoketest/ozonefs/ozonefs-obs.robot +++ b/hadoop-ozone/dist/src/main/smoketest/ozonefs/ozonefs-obs.robot @@ -51,10 +51,10 @@ Verify ls fails on OBS bucket Create key in OBS bucket Execute ozone sh key put /${volume}/${bucket}/testfile NOTICE.txt -Verify ls fails on OBS bucket key +Verify ls succeeds on OBS bucket key ${url} = Format FS URL ${SCHEME} ${volume} ${bucket} testfile - ${result} = Execute and checkrc ozone fs -ls ${url} 255 - Should contain ${result} Bucket: ${bucket} has layout: OBJECT_STORE, which does not support file system semantics. Bucket Layout must be FILE_SYSTEM_OPTIMIZED or LEGACY. + ${result} = Execute and checkrc ozone fs -ls ${url} 0 + Should contain ${result} testfile Verify rm fails on OBS bucket ${url} = Format FS URL ${SCHEME} ${volume} ${bucket} testfile From 79f3f997196087d57b8505493bea62557cb32f72 Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Wed, 9 Sep 2026 09:21:35 +0800 Subject: [PATCH 8/8] HDDS-15925. Reject OBS buckets in OM GetFileStatus for OFS compat. Rooted OFS can call GetFileStatus without InfoBucket; validate OBJECT_STORE layout in OmMetadataReader so stat paths keep the prior rejection behavior. Co-authored-by: Cursor Change-Id: I125b8aed50398bff1745b849254ca948a97317e8 --- .../main/smoketest/ozonefs/ozonefs-obs.robot | 6 +- .../AbstractRootedOzoneFileSystemTest.java | 27 ++++++-- .../apache/hadoop/ozone/om/TestOmAcls.java | 4 +- .../ozone/om/snapshot/OmSnapshotTests.java | 2 + .../hadoop/ozone/om/OmMetadataReader.java | 5 ++ .../hadoop/ozone/om/TestOMMetadataReader.java | 68 +++++++++++++++++++ .../BasicRootedOzoneClientAdapterImpl.java | 8 ++- 7 files changed, 110 insertions(+), 10 deletions(-) diff --git a/hadoop-ozone/dist/src/main/smoketest/ozonefs/ozonefs-obs.robot b/hadoop-ozone/dist/src/main/smoketest/ozonefs/ozonefs-obs.robot index 6be04a002525..57f015162608 100644 --- a/hadoop-ozone/dist/src/main/smoketest/ozonefs/ozonefs-obs.robot +++ b/hadoop-ozone/dist/src/main/smoketest/ozonefs/ozonefs-obs.robot @@ -51,10 +51,10 @@ Verify ls fails on OBS bucket Create key in OBS bucket Execute ozone sh key put /${volume}/${bucket}/testfile NOTICE.txt -Verify ls succeeds on OBS bucket key +Verify ls fails on OBS bucket key ${url} = Format FS URL ${SCHEME} ${volume} ${bucket} testfile - ${result} = Execute and checkrc ozone fs -ls ${url} 0 - Should contain ${result} testfile + ${result} = Execute and checkrc ozone fs -ls ${url} 255 + Should contain ${result} Bucket: ${bucket} has layout: OBJECT_STORE, which does not support file system semantics. Bucket Layout must be FILE_SYSTEM_OPTIMIZED or LEGACY. Verify rm fails on OBS bucket ${url} = Format FS URL ${SCHEME} ${volume} ${bucket} testfile diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java index f5cce61646ba..52a44e85297c 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java @@ -1835,10 +1835,6 @@ void testBucketDefaultsShouldBeInheritedToFileForEC() assertEquals(ReplicationType.EC.name(), key.getReplicationConfig().getReplicationType().name()); } - /** - * HDDS-15925: rooted OFS getFileStatus on a key path must not issue InfoBucket - * before GetFileStatus. - */ @Test void testGetFileStatusUsesSingleOmRpc() throws Exception { String keyName = "single-rpc-" + RandomStringUtils.secure().nextAlphabetic(5); @@ -1862,6 +1858,29 @@ void testGetFileStatusUsesSingleOmRpc() throws Exception { assertEquals(getFileStatusAfterFirst + 1, metrics.getNumGetFileStatus()); } + @Test + void testGetFileStatusRejectsObsBucket() throws Exception { + OzoneBucket obsBucket = + TestDataUtil.createVolumeAndBucket(client, BucketLayout.OBJECT_STORE); + Path obsBucketPath = new Path( + new Path(OZONE_URI_DELIMITER, obsBucket.getVolumeName()), + obsBucket.getName()); + String keyName = "obs-key-" + RandomStringUtils.secure().nextAlphabetic(5); + TestDataUtil.createKey(obsBucket, keyName, + "data".getBytes(StandardCharsets.UTF_8)); + Path keyPath = new Path(obsBucketPath, keyName); + + OMMetrics metrics = getOMMetrics(); + long bucketInfosBefore = metrics.getNumBucketInfos(); + + IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, + () -> fs.getFileStatus(keyPath)); + assertThat(exception.getMessage()).contains(obsBucket.getName()); + assertThat(exception.getMessage()).contains("OBJECT_STORE"); + assertEquals(bucketInfosBefore, metrics.getNumBucketInfos(), + "getFileStatus must not trigger InfoBucket"); + } + @Test void testGetFileStatus() throws Exception { String volumeNameLocal = getRandomNonExistVolumeName(); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmAcls.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmAcls.java index 8faf7d973cff..c61ca25a8b4e 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmAcls.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmAcls.java @@ -46,6 +46,7 @@ import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes; +import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer; import org.apache.hadoop.ozone.security.acl.IOzoneObj; import org.apache.hadoop.ozone.security.acl.OzoneObj; @@ -193,7 +194,8 @@ public void testReadKeyPermissionDenied() throws Exception { @Test public void testGetFileStatusPermissionDenied() throws Exception { - OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(client); + OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(client, + BucketLayout.FILE_SYSTEM_OPTIMIZED); TestDataUtil.createKey(bucket, "testKey", "testcontent".getBytes(StandardCharsets.UTF_8)); authorizer.keyAclAllow = false; diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/OmSnapshotTests.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/OmSnapshotTests.java index 44131d81c88f..f3223bbbbfe3 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/OmSnapshotTests.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/OmSnapshotTests.java @@ -65,6 +65,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; +import static org.junit.jupiter.api.Assumptions.assumeFalse; import static org.junit.jupiter.api.Assumptions.assumeTrue; import com.google.common.collect.Lists; @@ -525,6 +526,7 @@ public void checkKey() throws Exception { KeyInfoWithVolumeContext fileInfo = writeClient.getKeyInfo(keyArgs, false); assertEquals(fileInfo.getKeyInfo().getKeyName(), snapshotKeyPrefix + key1); + assumeFalse(bucketLayout.equals(BucketLayout.OBJECT_STORE)); OzoneFileStatus ozoneFileStatus = writeClient.getFileStatus(keyArgs); assertEquals(ozoneFileStatus.getKeyInfo().getKeyName(), snapshotKeyPrefix + key1); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java index 64f46089c066..3c5db0eb56aa 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java @@ -52,6 +52,7 @@ import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatusLight; import org.apache.hadoop.ozone.om.helpers.S3VolumeContext; @@ -277,6 +278,10 @@ public OzoneFileStatus getFileStatus(OmKeyArgs args) throws IOException { args = bucket.update(args); try { + if (bucket.bucketLayout() != null) { + OzoneFSUtils.validateBucketLayout(bucket.requestedBucket(), + bucket.bucketLayout()); + } if (isAclEnabled) { checkAcls(getResourceType(args), StoreType.OZONE, ACLType.READ, bucket, args.getKeyName()); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOMMetadataReader.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOMMetadataReader.java index 903b0720943d..4cb0a9871e9d 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOMMetadataReader.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOMMetadataReader.java @@ -18,12 +18,22 @@ package org.apache.hadoop.ozone.om; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import io.grpc.Context; import org.apache.hadoop.ipc_.Server; +import org.apache.hadoop.ozone.audit.AuditLogger; +import org.apache.hadoop.ozone.om.helpers.BucketLayout; +import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; +import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; import org.junit.jupiter.api.Test; import org.mockito.MockedStatic; @@ -69,4 +79,62 @@ public void testGetClientAddress() { } } + @Test + public void getFileStatusRejectsObjectStoreLayout() throws Exception { + OzoneManager ozoneManager = mock(OzoneManager.class); + KeyManager keyManager = mock(KeyManager.class); + when(ozoneManager.getAclsEnabled()).thenReturn(false); + when(ozoneManager.getBucketManager()).thenReturn(mock(BucketManager.class)); + when(ozoneManager.getVolumeManager()).thenReturn(mock(VolumeManager.class)); + when(ozoneManager.getPerfMetrics()).thenReturn(mock(OMPerformanceMetrics.class)); + when(ozoneManager.resolveBucketLink(any(OmKeyArgs.class))) + .thenReturn(new ResolvedBucket("vol", "obs-bucket", "vol", "obs-bucket", + "owner", BucketLayout.OBJECT_STORE)); + + OmMetadataReader reader = new OmMetadataReader(keyManager, + mock(PrefixManager.class), ozoneManager, mock(org.slf4j.Logger.class), + mock(AuditLogger.class), mock(OmMetadataReaderMetrics.class), null); + + OmKeyArgs keyArgs = new OmKeyArgs.Builder() + .setVolumeName("vol") + .setBucketName("obs-bucket") + .setKeyName("key1") + .build(); + + IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, + () -> reader.getFileStatus(keyArgs)); + assertTrue(exception.getMessage().contains("obs-bucket")); + assertTrue(exception.getMessage().contains("OBJECT_STORE")); + verify(keyManager, never()).getFileStatus(any(), anyString()); + } + + @Test + public void getFileStatusAllowsLegacyLayout() throws Exception { + OzoneManager ozoneManager = mock(OzoneManager.class); + KeyManager keyManager = mock(KeyManager.class); + when(ozoneManager.getAclsEnabled()).thenReturn(false); + when(ozoneManager.getBucketManager()).thenReturn(mock(BucketManager.class)); + when(ozoneManager.getVolumeManager()).thenReturn(mock(VolumeManager.class)); + when(ozoneManager.getPerfMetrics()).thenReturn(mock(OMPerformanceMetrics.class)); + when(ozoneManager.resolveBucketLink(any(OmKeyArgs.class))) + .thenReturn(new ResolvedBucket("vol", "legacy-bucket", "vol", + "legacy-bucket", "owner", BucketLayout.LEGACY)); + OzoneFileStatus expectedStatus = new OzoneFileStatus(); + when(keyManager.getFileStatus(any(OmKeyArgs.class), anyString())) + .thenReturn(expectedStatus); + + OmMetadataReader reader = new OmMetadataReader(keyManager, + mock(PrefixManager.class), ozoneManager, mock(org.slf4j.Logger.class), + mock(AuditLogger.class), mock(OmMetadataReaderMetrics.class), null); + + OmKeyArgs keyArgs = new OmKeyArgs.Builder() + .setVolumeName("vol") + .setBucketName("legacy-bucket") + .setKeyName("key1") + .build(); + + OzoneFileStatus status = reader.getFileStatus(keyArgs); + assertEquals(expectedStatus, status); + verify(keyManager).getFileStatus(any(OmKeyArgs.class), anyString()); + } } diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java index 2ac50651ec72..2f585e41a549 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java @@ -698,8 +698,8 @@ public FileStatusAdapter getFileStatus(String path, URI uri, * Throws exception in case of failure. * *

Non-snapshot paths call OM GetFileStatus directly (HDDS-15925) without a - * prior InfoBucket RPC. OBJECT_STORE buckets are not rejected on this path; - * mutating OFS operations still validate layout via {@link #getBucket(OFSPath, boolean)}. + * prior InfoBucket RPC. OBJECT_STORE buckets are rejected by OM GetFileStatus. + * Mutating OFS operations still validate layout via {@link #getBucket(OFSPath, boolean)}. */ private FileStatusAdapter getFileStatusForKeyOrSnapshot( OFSPath ofsPath, URI uri, Path qualifiedPath, String userName, @@ -723,6 +723,10 @@ private FileStatusAdapter getFileStatusForKeyOrSnapshot( } else if (e.getResult() == OMException.ResultCodes.BUCKET_NOT_FOUND) { throw new FileNotFoundException(key + ": Bucket doesn't exist!"); } + String message = e.getMessage(); + if (message != null && message.contains("does not support file system semantics")) { + throw new IllegalArgumentException(message); + } throw e; } }