diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/core/PeersV2NodeRefreshIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/core/PeersV2NodeRefreshIT.java index 9feb7bb965f..adc29bb44a5 100644 --- a/integration-tests/src/test/java/com/datastax/oss/driver/core/PeersV2NodeRefreshIT.java +++ b/integration-tests/src/test/java/com/datastax/oss/driver/core/PeersV2NodeRefreshIT.java @@ -26,6 +26,7 @@ import com.datastax.oss.protocol.internal.request.Query; import com.datastax.oss.simulacron.common.cluster.ClusterSpec; import com.datastax.oss.simulacron.common.cluster.QueryLog; +import com.datastax.oss.simulacron.server.BindNodeException; import com.datastax.oss.simulacron.server.BoundCluster; import com.datastax.oss.simulacron.server.Server; import java.util.concurrent.ExecutionException; @@ -36,13 +37,59 @@ /** Test for JAVA-2654. */ public class PeersV2NodeRefreshIT { + // Simulacron's "multiple nodes per IP" mode (needed here to emulate several nodes sharing a + // single address, as system.peers_v2 distinguishes nodes by (peer, peer_port)) always binds + // nodes to consecutive ports on a loopback address starting at port 49152 -- the first port of + // the OS ephemeral/dynamic port range on both Linux (/proc/sys/net/ipv4/ip_local_port_range, + // typically 32768-60999) and macOS (49152-65535). That range is also what the kernel hands out + // as the *source* port for any outbound client socket opened by any process on the machine, + // including other integration tests' driver connections running concurrently in this same + // build. When the kernel happens to allocate 49152 as an ephemeral source port at the exact + // moment this test tries to bind it as a *listening* socket, Simulacron fails with + // BindNodeException ("Failed to bind ... to /127.0.0.1:49152"). This is a real, + // previously-reported flake (see https://github.com/scylladb/java-driver/issues/951, which was + // closed after an unrelated fix for a different flaky test happened to land around the same + // time as a CI re-run that passed -- the actual Simulacron port race was never fixed). + // + // Ideally this test would sidestep the collision entirely by binding to a fixed, non-ephemeral + // starting port via Server.Builder#withAddressResolver(...). However, the vendored + // com.scylladb.oss.simulacron:simulacron-native-server:0.14.0.0 unconditionally re-creates a + // brand-new `new NodePerPortResolver()` (hardcoded to port 49152) inside build() whenever + // withMultipleNodesPerIp(true) is set, silently discarding any resolver configured via + // withAddressResolver(...) -- confirmed by decompiling the actual jar (this is a deviation + // from the upstream datastax/simulacron behavior, where withAddressResolver called after + // withMultipleNodesPerIp is honored). Since the starting port can't be changed through the + // public API, the only effective mitigation available here is to retry the whole + // register-and-bind attempt with a fresh Server/resolver and a short backoff, which gives the + // OS a chance to move its ephemeral allocator away from port 49152 before the next attempt. + private static final int MAX_BIND_ATTEMPTS = 5; + private static Server peersV2Server; private static BoundCluster cluster; @BeforeClass - public static void setup() { - peersV2Server = Server.builder().withMultipleNodesPerIp(true).build(); - cluster = peersV2Server.register(ClusterSpec.builder().withNodes(2)); + public static void setup() throws InterruptedException { + RuntimeException lastFailure = null; + for (int attempt = 1; attempt <= MAX_BIND_ATTEMPTS; attempt++) { + Server server = Server.builder().withMultipleNodesPerIp(true).build(); + try { + cluster = server.register(ClusterSpec.builder().withNodes(2)); + peersV2Server = server; + return; + } catch (RuntimeException e) { + // Always tear down the server we just created, whether or not we're going to retry, to + // avoid leaking its event loop threads. + server.close(); + boolean isBindRace = e.getCause() instanceof BindNodeException; + if (!isBindRace || attempt == MAX_BIND_ATTEMPTS) { + throw e; + } + lastFailure = e; + Thread.sleep(200L * attempt); + } + } + // Unreachable, but keeps the compiler happy about a definite return/throw. + throw lastFailure; } @AfterClass