From 17b3cd30900bcc227cb256e6e246c74c8d40e811 Mon Sep 17 00:00:00 2001 From: Roy Dahan Date: Mon, 24 Aug 2026 19:52:37 +0300 Subject: [PATCH] Fix flaky PeersV2NodeRefreshIT: retry on Simulacron port-bind race PeersV2NodeRefreshIT builds its own Server with withMultipleNodesPerIp(true), which always binds its nodes starting at 127.0.0.1:49152 -- the first port of the OS ephemeral/dynamic port range on both Linux and macOS. That range is also handed out by the kernel as the *source* port for any outbound socket on the machine, including other integration tests' concurrent driver connections, so a bind race is expected periodically: java.lang.RuntimeException: com.datastax.oss.simulacron.server.BindNodeException: Failed to bind NodeSpec{id=0, name='node1', address=null} to /127.0.0.1:49152 This was previously reported in #951 and closed as fixed by #942/#943, but those PRs only addressed the unrelated TupleTest flake bundled in the same issue -- the Simulacron port race was never actually fixed, it just happened not to reproduce on the CI re-run. Ideally this would be avoided by binding to a fixed, non-ephemeral port via Server.Builder#withAddressResolver(...), but decompiling the vendored com.scylladb.oss.simulacron:simulacron-native-server:0.14.0.0 jar shows that build() unconditionally replaces the resolver with a fresh `new NodePerPortResolver()` (hardcoded to port 49152) whenever withMultipleNodesPerIp(true) is set, silently discarding whatever was passed to withAddressResolver(...). This is a deviation from upstream datastax/simulacron, where that override is respected. Since the starting port can't be changed through the public API, setup() now retries the register-and-bind attempt (fresh Server each time) with a short backoff when the failure is a BindNodeException, giving the OS a chance to move its ephemeral allocator off of 49152 before the next attempt. Verified locally by reproducing the exact reported stack trace: holding 127.0.0.1:49152 with an external socket makes the original code fail immediately with the same BindNodeException, while the patched code retries and passes once the port frees up. The unmodified (no collision) case is unaffected in both timing and behavior. Co-Authored-By: Claude Sonnet 5 --- .../oss/driver/core/PeersV2NodeRefreshIT.java | 53 +++++++++++++++++-- 1 file changed, 50 insertions(+), 3 deletions(-) diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/core/PeersV2NodeRefreshIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/core/PeersV2NodeRefreshIT.java index 9feb7bb965f..adc29bb44a5 100644 --- a/integration-tests/src/test/java/com/datastax/oss/driver/core/PeersV2NodeRefreshIT.java +++ b/integration-tests/src/test/java/com/datastax/oss/driver/core/PeersV2NodeRefreshIT.java @@ -26,6 +26,7 @@ import com.datastax.oss.protocol.internal.request.Query; import com.datastax.oss.simulacron.common.cluster.ClusterSpec; import com.datastax.oss.simulacron.common.cluster.QueryLog; +import com.datastax.oss.simulacron.server.BindNodeException; import com.datastax.oss.simulacron.server.BoundCluster; import com.datastax.oss.simulacron.server.Server; import java.util.concurrent.ExecutionException; @@ -36,13 +37,59 @@ /** Test for JAVA-2654. */ public class PeersV2NodeRefreshIT { + // Simulacron's "multiple nodes per IP" mode (needed here to emulate several nodes sharing a + // single address, as system.peers_v2 distinguishes nodes by (peer, peer_port)) always binds + // nodes to consecutive ports on a loopback address starting at port 49152 -- the first port of + // the OS ephemeral/dynamic port range on both Linux (/proc/sys/net/ipv4/ip_local_port_range, + // typically 32768-60999) and macOS (49152-65535). That range is also what the kernel hands out + // as the *source* port for any outbound client socket opened by any process on the machine, + // including other integration tests' driver connections running concurrently in this same + // build. When the kernel happens to allocate 49152 as an ephemeral source port at the exact + // moment this test tries to bind it as a *listening* socket, Simulacron fails with + // BindNodeException ("Failed to bind ... to /127.0.0.1:49152"). This is a real, + // previously-reported flake (see https://github.com/scylladb/java-driver/issues/951, which was + // closed after an unrelated fix for a different flaky test happened to land around the same + // time as a CI re-run that passed -- the actual Simulacron port race was never fixed). + // + // Ideally this test would sidestep the collision entirely by binding to a fixed, non-ephemeral + // starting port via Server.Builder#withAddressResolver(...). However, the vendored + // com.scylladb.oss.simulacron:simulacron-native-server:0.14.0.0 unconditionally re-creates a + // brand-new `new NodePerPortResolver()` (hardcoded to port 49152) inside build() whenever + // withMultipleNodesPerIp(true) is set, silently discarding any resolver configured via + // withAddressResolver(...) -- confirmed by decompiling the actual jar (this is a deviation + // from the upstream datastax/simulacron behavior, where withAddressResolver called after + // withMultipleNodesPerIp is honored). Since the starting port can't be changed through the + // public API, the only effective mitigation available here is to retry the whole + // register-and-bind attempt with a fresh Server/resolver and a short backoff, which gives the + // OS a chance to move its ephemeral allocator away from port 49152 before the next attempt. + private static final int MAX_BIND_ATTEMPTS = 5; + private static Server peersV2Server; private static BoundCluster cluster; @BeforeClass - public static void setup() { - peersV2Server = Server.builder().withMultipleNodesPerIp(true).build(); - cluster = peersV2Server.register(ClusterSpec.builder().withNodes(2)); + public static void setup() throws InterruptedException { + RuntimeException lastFailure = null; + for (int attempt = 1; attempt <= MAX_BIND_ATTEMPTS; attempt++) { + Server server = Server.builder().withMultipleNodesPerIp(true).build(); + try { + cluster = server.register(ClusterSpec.builder().withNodes(2)); + peersV2Server = server; + return; + } catch (RuntimeException e) { + // Always tear down the server we just created, whether or not we're going to retry, to + // avoid leaking its event loop threads. + server.close(); + boolean isBindRace = e.getCause() instanceof BindNodeException; + if (!isBindRace || attempt == MAX_BIND_ATTEMPTS) { + throw e; + } + lastFailure = e; + Thread.sleep(200L * attempt); + } + } + // Unreachable, but keeps the compiler happy about a definite return/throw. + throw lastFailure; } @AfterClass