Skip to content
Closed
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import com.datastax.oss.protocol.internal.request.Query;
import com.datastax.oss.simulacron.common.cluster.ClusterSpec;
import com.datastax.oss.simulacron.common.cluster.QueryLog;
import com.datastax.oss.simulacron.server.BindNodeException;
import com.datastax.oss.simulacron.server.BoundCluster;
import com.datastax.oss.simulacron.server.Server;
import java.util.concurrent.ExecutionException;
Expand All @@ -36,13 +37,59 @@
/** Test for JAVA-2654. */
public class PeersV2NodeRefreshIT {

// Simulacron's "multiple nodes per IP" mode (needed here to emulate several nodes sharing a
// single address, as system.peers_v2 distinguishes nodes by (peer, peer_port)) always binds
// nodes to consecutive ports on a loopback address starting at port 49152 -- the first port of
// the OS ephemeral/dynamic port range on both Linux (/proc/sys/net/ipv4/ip_local_port_range,
// typically 32768-60999) and macOS (49152-65535). That range is also what the kernel hands out
// as the *source* port for any outbound client socket opened by any process on the machine,
// including other integration tests' driver connections running concurrently in this same
// build. When the kernel happens to allocate 49152 as an ephemeral source port at the exact
// moment this test tries to bind it as a *listening* socket, Simulacron fails with
// BindNodeException ("Failed to bind ... to /127.0.0.1:49152"). This is a real,
// previously-reported flake (see https://github.com/scylladb/java-driver/issues/951, which was
// closed after an unrelated fix for a different flaky test happened to land around the same
// time as a CI re-run that passed -- the actual Simulacron port race was never fixed).
//
// Ideally this test would sidestep the collision entirely by binding to a fixed, non-ephemeral
// starting port via Server.Builder#withAddressResolver(...). However, the vendored
// com.scylladb.oss.simulacron:simulacron-native-server:0.14.0.0 unconditionally re-creates a
// brand-new `new NodePerPortResolver()` (hardcoded to port 49152) inside build() whenever
// withMultipleNodesPerIp(true) is set, silently discarding any resolver configured via
// withAddressResolver(...) -- confirmed by decompiling the actual jar (this is a deviation
// from the upstream datastax/simulacron behavior, where withAddressResolver called after
// withMultipleNodesPerIp is honored). Since the starting port can't be changed through the
// public API, the only effective mitigation available here is to retry the whole
// register-and-bind attempt with a fresh Server/resolver and a short backoff, which gives the
// OS a chance to move its ephemeral allocator away from port 49152 before the next attempt.
private static final int MAX_BIND_ATTEMPTS = 5;

private static Server peersV2Server;
private static BoundCluster cluster;

@BeforeClass
public static void setup() {
peersV2Server = Server.builder().withMultipleNodesPerIp(true).build();
cluster = peersV2Server.register(ClusterSpec.builder().withNodes(2));
public static void setup() throws InterruptedException {
RuntimeException lastFailure = null;
for (int attempt = 1; attempt <= MAX_BIND_ATTEMPTS; attempt++) {
Server server = Server.builder().withMultipleNodesPerIp(true).build();
try {
cluster = server.register(ClusterSpec.builder().withNodes(2));
peersV2Server = server;
return;
} catch (RuntimeException e) {
// Always tear down the server we just created, whether or not we're going to retry, to
// avoid leaking its event loop threads.
server.close();
boolean isBindRace = e.getCause() instanceof BindNodeException;
if (!isBindRace || attempt == MAX_BIND_ATTEMPTS) {
throw e;
}
lastFailure = e;
Thread.sleep(200L * attempt);
}
}
// Unreachable, but keeps the compiler happy about a definite return/throw.
throw lastFailure;
}

@AfterClass
Expand Down
Loading