From 8831f08906b2bc6be37230d4e1d50b0ba6c04ebd Mon Sep 17 00:00:00 2001 From: Siyao Meng <50227127+smengcl@users.noreply.github.com> Date: Mon, 14 Sep 2026 08:43:12 -0700 Subject: [PATCH] HDDS-16423. Fix intermittent failure in TestContainerReconciliationWithMockDatanodes#testContainerReconciliationFailureContainerScan The container replicas and the OnDemandContainerScanner are created once in @BeforeAll and shared by every test method. MockDatanode.scanContainer asserted that OnDemandContainerScanner.scanContainerWithoutGap(...) returns a present Future, but that method returns an empty Optional when the container is already registered as in progress in containerRescheduleCheckSet. KeyValueHandler.reconcileContainer schedules a fire-and-forget scan via scanContainerWithoutGap in a finally block, so every reconciliation test leaves one scan in flight. Those tests wait via waitForExpectedScanCount(...), which polls the numContainersScanned metric that is incremented inside the scan, before the executor thread runs removeContainerFromScheduledContainers. When the next test's synchronous scanContainer ran before that removal, no scan was scheduled, the Optional was empty, and the assertion failed intermittently. Make scanContainer wait until scanContainerWithoutGap returns a scheduled Future before blocking on it. The single-thread scan executor is guaranteed to drain the prior scan, so this is deterministic by construction and keeps the per-call scan count at exactly one. Co-Authored-By: Claude Opus 4.8 (1M context) --- ...tainerReconciliationWithMockDatanodes.java | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestContainerReconciliationWithMockDatanodes.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestContainerReconciliationWithMockDatanodes.java index a97a3418cc87..6c52ecb1219f 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestContainerReconciliationWithMockDatanodes.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestContainerReconciliationWithMockDatanodes.java @@ -58,6 +58,7 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -556,9 +557,21 @@ public KeyValueContainer getContainer(long containerID) { * Triggers a synchronous scan of the container. This method will block until the scan completes. */ public void scanContainer(long containerID) { - Optional> scanFuture = onDemandScanner.scanContainerWithoutGap(containerSet.getContainer(containerID), - TEST_SCAN); - assertTrue(scanFuture.isPresent()); + // A previously triggered on-demand scan may still be registered as in progress, for example the + // fire-and-forget scan that reconciliation schedules in a finally block. While it is, the scanner + // returns an empty Optional instead of scheduling a new one. Wait for the prior scan to drain so + // this synchronous scan is actually scheduled. + AtomicReference> scanFuture = new AtomicReference<>(); + try { + GenericTestUtils.waitFor(() -> { + Optional> future = + onDemandScanner.scanContainerWithoutGap(containerSet.getContainer(containerID), TEST_SCAN); + future.ifPresent(scanFuture::set); + return future.isPresent(); + }, 100, 10_000); + } catch (InterruptedException | TimeoutException e) { + fail("On demand container scan was not scheduled", e); + } try { scanFuture.get().get();