From 5736ddfaecea8e87d5defba8469ec1c646f50fb8 Mon Sep 17 00:00:00 2001 From: tintinhamans <5984296+tintinhamans@users.noreply.github.com> Date: Fri, 25 Sep 2026 23:19:51 +0200 Subject: [PATCH 1/5] feat(lobby): Make the mesh check window tunable and wait out forming connections --- GenOnlineService/LobbyManager.cs | 67 ++++++++++++++++++-------- GenOnlineService/MatchmakingManager.cs | 2 +- GenOnlineService/appsettings.json | 6 ++- 3 files changed, 54 insertions(+), 21 deletions(-) diff --git a/GenOnlineService/LobbyManager.cs b/GenOnlineService/LobbyManager.cs index b08e86f..4ce0419 100644 --- a/GenOnlineService/LobbyManager.cs +++ b/GenOnlineService/LobbyManager.cs @@ -62,6 +62,30 @@ internal static bool ShouldRetry(bool meshComplete, bool hasLegacyResponse, int } } + // Core:full_mesh_check_* in appsettings.json, read on use + internal static class FullMeshCheckSettings + { + // how long an attempt waits for every connection before judging it + internal static int AttemptWindowMS => Get("full_mesh_check_attempt_window_ms", 8000); + + // how often an incomplete attempt asks members for a fresh snapshot + internal static int SnapshotIntervalMS => Get("full_mesh_check_snapshot_interval_ms", 1000); + + // time given to re-signalled connections before the next attempt + internal static int RetryDelayMS => Get("full_mesh_check_retry_delay_ms", 3000); + + internal static int MaxAttempts => Get("full_mesh_check_max_attempts", 2); + + // upper bound for a whole check, used as the clients' setup timeout + internal static int MaxDurationMS => (AttemptWindowMS * MaxAttempts) + (RetryDelayMS * (MaxAttempts - 1)); + + private static int Get(string key, int defaultValue) + { + int value = Program.g_Config?.GetSection("Core").GetValue(key, defaultValue) ?? defaultValue; + return value > 0 ? value : defaultValue; + } + } + public class Lobby { public Int64 LobbyID { get; private set; } = -1; @@ -81,13 +105,6 @@ public class Lobby [JsonIgnore] public Int64 TimeStartFullMeshChecks { get; private set; } = -1; - private const int MSToWaitForFullMeshChecks = 5000; - private const int MaxFullMeshCheckAttempts = 2; - private const int MSBeforeFullMeshCheckRetry = 3000; - public const int MaxFullMeshConnectivityCheckDurationMS = - (MSToWaitForFullMeshChecks * MaxFullMeshCheckAttempts) - + (MSBeforeFullMeshCheckRetry * (MaxFullMeshCheckAttempts - 1)); - private readonly object m_FullMeshCheckLock = new(); [JsonIgnore] @@ -102,6 +119,9 @@ public class Lobby [JsonIgnore] private Int64 m_TimeToRetryFullMeshChecks = -1; + [JsonIgnore] + private Int64 m_TimeNextFullMeshSnapshotRequest = -1; + [JsonIgnore] private bool m_bCurrentAttemptHasLegacyResponse = false; @@ -207,8 +227,9 @@ private void BeginFullMeshConnectivityCheckAttempt() PendingFullMeshConnectivityChecks = true; FullMeshConnectivityChecks = new(); m_bCurrentAttemptHasLegacyResponse = false; - TimeStartFullMeshChecks = Environment.TickCount64; - } + TimeStartFullMeshChecks = Environment.TickCount64; + m_TimeNextFullMeshSnapshotRequest = TimeStartFullMeshChecks + FullMeshCheckSettings.SnapshotIntervalMS; + } public void SendFullMeshConnectivityCheckRequestToMembers() { @@ -334,14 +355,9 @@ private void ProcessPendingFullMeshConnectivityChecksInternal() int totalMapEntriesExpected = GetNumberOfHumans(); //int numConnectionsExpectedPerUser = totalMapEntriesExpected - 1; // minus self - // must have a connectivity map for each lobby member - bDoneChecks = FullMeshConnectivityChecks.Count == totalMapEntriesExpected; - - // did we timeout? - if (!bDoneChecks && (Environment.TickCount64 - TimeStartFullMeshChecks) >= MSToWaitForFullMeshChecks) - { - bDoneChecks = true; - } + // must have a connectivity map for each lobby member, or the attempt window must have closed + bool bWindowElapsed = (Environment.TickCount64 - TimeStartFullMeshChecks) >= FullMeshCheckSettings.AttemptWindowMS; + bDoneChecks = bWindowElapsed || FullMeshConnectivityChecks.Count == totalMapEntriesExpected; List lstMissingConnections = new(); @@ -411,15 +427,28 @@ private void ProcessPendingFullMeshConnectivityChecksInternal() bool bMeshComplete = bDisableMeshCheck || lstMissingConnections.Count == 0; + // members report a snapshot, so a gap before the window closes may be a connection still forming: + // keep asking for fresh snapshots rather than re-signalling it early + if (!bMeshComplete && !bWindowElapsed) + { + if (Environment.TickCount64 >= m_TimeNextFullMeshSnapshotRequest) + { + m_TimeNextFullMeshSnapshotRequest = Environment.TickCount64 + FullMeshCheckSettings.SnapshotIntervalMS; + SendFullMeshConnectivityCheckRequestToMembers(); + } + + return; + } + if (FullMeshCheckProtocol.ShouldRetry( bMeshComplete, m_bCurrentAttemptHasLegacyResponse, FullMeshCheckAttempt, - MaxFullMeshCheckAttempts)) + FullMeshCheckSettings.MaxAttempts)) { ++FullMeshCheckAttempt; RestartSignallingForMissingConnections(lstMissingConnections); - m_TimeToRetryFullMeshChecks = Environment.TickCount64 + MSBeforeFullMeshCheckRetry; + m_TimeToRetryFullMeshChecks = Environment.TickCount64 + FullMeshCheckSettings.RetryDelayMS; return; } diff --git a/GenOnlineService/MatchmakingManager.cs b/GenOnlineService/MatchmakingManager.cs index bc8cd92..6ee4bcb 100644 --- a/GenOnlineService/MatchmakingManager.cs +++ b/GenOnlineService/MatchmakingManager.cs @@ -911,7 +911,7 @@ private async Task TriggerFullMeshConnectivityChecks(Lobby lobby) m_bWaitingOnMeshConnectivityChecks = true; } - QueueSetupProgress(Lobby.MaxFullMeshConnectivityCheckDurationMS + c_SetupClientTimeoutMarginMSec); + QueueSetupProgress(FullMeshCheckSettings.MaxDurationMS + c_SetupClientTimeoutMarginMSec); lobby.SendFullMeshConnectivityCheckRequestToMembers(); diff --git a/GenOnlineService/appsettings.json b/GenOnlineService/appsettings.json index d193ca6..29dc3bd 100644 --- a/GenOnlineService/appsettings.json +++ b/GenOnlineService/appsettings.json @@ -34,7 +34,11 @@ "cert_pem_path": null, "cert_key_path": null, "cert_reload_interval_seconds": 30, - "disable_full_mesh_check": false + "disable_full_mesh_check": false, + "full_mesh_check_attempt_window_ms": 8000, + "full_mesh_check_snapshot_interval_ms": 1000, + "full_mesh_check_retry_delay_ms": 3000, + "full_mesh_check_max_attempts": 2 }, "TURN": { "key": null, From 8ff30e6c44ccd18bc7b31a0612f740667b318077 Mon Sep 17 00:00:00 2001 From: tintinhamans <5984296+tintinhamans@users.noreply.github.com> Date: Fri, 25 Sep 2026 23:33:08 +0200 Subject: [PATCH 2/5] feat(lobby): Log each mesh check attempt's outcome and timing --- GenOnlineService/LobbyManager.cs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/GenOnlineService/LobbyManager.cs b/GenOnlineService/LobbyManager.cs index 4ce0419..4803009 100644 --- a/GenOnlineService/LobbyManager.cs +++ b/GenOnlineService/LobbyManager.cs @@ -231,6 +231,21 @@ private void BeginFullMeshConnectivityCheckAttempt() m_TimeNextFullMeshSnapshotRequest = TimeStartFullMeshChecks + FullMeshCheckSettings.SnapshotIntervalMS; } + // one line per judged attempt, so the window can be tuned from real connect times + private void LogFullMeshCheckAttempt(bool bMeshComplete, List lstMissingConnections) + { + Int64 elapsedMS = Environment.TickCount64 - TimeStartFullMeshChecks; + string strMissing = string.Join(", ", lstMissingConnections + .Select(c => (Math.Min(c.source_user_id, c.target_user_id), Math.Max(c.source_user_id, c.target_user_id))) + .Distinct() + .Select(p => $"{p.Item1}<->{p.Item2}")); + + Console.WriteLine("[Lobby {0}] Mesh check {1} attempt {2}/{3}: {4} after {5} ms with {6} humans{7}", + LobbyID, FullMeshCheckID, FullMeshCheckAttempt, FullMeshCheckSettings.MaxAttempts, + bMeshComplete ? "complete" : "incomplete", elapsedMS, GetNumberOfHumans(), + bMeshComplete ? "" : $", missing {strMissing}"); + } + public void SendFullMeshConnectivityCheckRequestToMembers() { WebSocketMessage_FullMeshConnectivityCheckRequest startCommand = new WebSocketMessage_FullMeshConnectivityCheckRequest(); @@ -440,6 +455,8 @@ private void ProcessPendingFullMeshConnectivityChecksInternal() return; } + LogFullMeshCheckAttempt(bMeshComplete, lstMissingConnections); + if (FullMeshCheckProtocol.ShouldRetry( bMeshComplete, m_bCurrentAttemptHasLegacyResponse, From 46ab7dec9b2a0987499c0e8c56c4d7039eabbc52 Mon Sep 17 00:00:00 2001 From: tintinhamans <5984296+tintinhamans@users.noreply.github.com> Date: Fri, 25 Sep 2026 23:33:38 +0200 Subject: [PATCH 3/5] feat(lobby): Hold off re-signalling mesh pairs that are still connecting --- GenOnlineService/Constants.cs | 3 +++ GenOnlineService/LobbyManager.cs | 25 +++++++++++++++++++++++-- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/GenOnlineService/Constants.cs b/GenOnlineService/Constants.cs index a860c70..3c3faac 100644 --- a/GenOnlineService/Constants.cs +++ b/GenOnlineService/Constants.cs @@ -2997,6 +2997,9 @@ public class WebSocketMessage_FullMeshConnectivityCheckResponseFromUser : WebSoc public Int64 mesh_check_id { get; set; } public int attempt { get; set; } public List connectivity_map { get; set; } = new(); + + // peers the member is still negotiating with; absent from older clients + public List connecting_map { get; set; } = new(); } public class WebSocketMessage_Social_NewFriendRequest : WebSocketMessage diff --git a/GenOnlineService/LobbyManager.cs b/GenOnlineService/LobbyManager.cs index 4803009..f0b6771 100644 --- a/GenOnlineService/LobbyManager.cs +++ b/GenOnlineService/LobbyManager.cs @@ -122,6 +122,10 @@ public class Lobby [JsonIgnore] private Int64 m_TimeNextFullMeshSnapshotRequest = -1; + // per member, the peers they reported as still connecting in the current attempt + [JsonIgnore] + private Dictionary> m_FullMeshConnecting = new(); + [JsonIgnore] private bool m_bCurrentAttemptHasLegacyResponse = false; @@ -226,11 +230,18 @@ private void BeginFullMeshConnectivityCheckAttempt() { PendingFullMeshConnectivityChecks = true; FullMeshConnectivityChecks = new(); + m_FullMeshConnecting = new(); m_bCurrentAttemptHasLegacyResponse = false; TimeStartFullMeshChecks = Environment.TickCount64; m_TimeNextFullMeshSnapshotRequest = TimeStartFullMeshChecks + FullMeshCheckSettings.SnapshotIntervalMS; } + private bool IsFullMeshPairStillConnecting(Int64 userA, Int64 userB) + { + return (m_FullMeshConnecting.TryGetValue(userA, out HashSet? fromA) && fromA.Contains(userB)) + || (m_FullMeshConnecting.TryGetValue(userB, out HashSet? fromB) && fromB.Contains(userA)); + } + // one line per judged attempt, so the window can be tuned from real connect times private void LogFullMeshCheckAttempt(bool bMeshComplete, List lstMissingConnections) { @@ -239,11 +250,16 @@ private void LogFullMeshCheckAttempt(bool bMeshComplete, List (Math.Min(c.source_user_id, c.target_user_id), Math.Max(c.source_user_id, c.target_user_id))) .Distinct() .Select(p => $"{p.Item1}<->{p.Item2}")); + string strConnecting = string.Join(", ", lstMissingConnections + .Where(c => IsFullMeshPairStillConnecting(c.source_user_id, c.target_user_id)) + .Select(c => (Math.Min(c.source_user_id, c.target_user_id), Math.Max(c.source_user_id, c.target_user_id))) + .Distinct() + .Select(p => $"{p.Item1}<->{p.Item2}")); Console.WriteLine("[Lobby {0}] Mesh check {1} attempt {2}/{3}: {4} after {5} ms with {6} humans{7}", LobbyID, FullMeshCheckID, FullMeshCheckAttempt, FullMeshCheckSettings.MaxAttempts, bMeshComplete ? "complete" : "incomplete", elapsedMS, GetNumberOfHumans(), - bMeshComplete ? "" : $", missing {strMissing}"); + bMeshComplete ? "" : $", missing {strMissing}{(strConnecting.Length > 0 ? $" (still connecting {strConnecting})" : "")}"); } public void SendFullMeshConnectivityCheckRequestToMembers() @@ -326,6 +342,7 @@ public Task StoreFullMeshConnectivityResponse(Int64 sourceUser, WebSocketMessage { m_bCurrentAttemptHasLegacyResponse |= bLegacyResponse; FullMeshConnectivityChecks[sourceUser] = new ConcurrentList(response.connectivity_map); + m_FullMeshConnecting[sourceUser] = new HashSet(response.connecting_map); } ProcessPendingFullMeshConnectivityChecksInternal(); @@ -464,7 +481,11 @@ private void ProcessPendingFullMeshConnectivityChecksInternal() FullMeshCheckSettings.MaxAttempts)) { ++FullMeshCheckAttempt; - RestartSignallingForMissingConnections(lstMissingConnections); + + // a pair still negotiating would be torn down by a re-signal; the next attempt re-checks it + RestartSignallingForMissingConnections(lstMissingConnections + .Where(c => !IsFullMeshPairStillConnecting(c.source_user_id, c.target_user_id)) + .ToList()); m_TimeToRetryFullMeshChecks = Environment.TickCount64 + FullMeshCheckSettings.RetryDelayMS; return; } From 19e8743f2e330b6c7bae202f5371f5ffcbb66887 Mon Sep 17 00:00:00 2001 From: tintinhamans <5984296+tintinhamans@users.noreply.github.com> Date: Fri, 25 Sep 2026 23:42:36 +0200 Subject: [PATCH 4/5] fix(matchmaking): Tell clients the start countdown length instead of letting them guess --- GenOnlineService/Constants.cs | 3 +++ GenOnlineService/MatchmakingManager.cs | 11 ++++++++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/GenOnlineService/Constants.cs b/GenOnlineService/Constants.cs index 3c3faac..678e078 100644 --- a/GenOnlineService/Constants.cs +++ b/GenOnlineService/Constants.cs @@ -3191,6 +3191,9 @@ public class WebSocketMessage_MatchmakerStartGame : WebSocketMessage public class WebSocketMessage_MatchmakerSetupProgress : WebSocketMessage { public int timeout_ms { get; set; } + + // start countdown length, 0 while the match is still being set up + public int countdown_ms { get; set; } } } diff --git a/GenOnlineService/MatchmakingManager.cs b/GenOnlineService/MatchmakingManager.cs index 6ee4bcb..3769961 100644 --- a/GenOnlineService/MatchmakingManager.cs +++ b/GenOnlineService/MatchmakingManager.cs @@ -862,6 +862,9 @@ public Int64 GetLobbyID() private const int c_GameStartCountdownMSec = 5000; private const int c_SetupClientTimeoutMarginMSec = 2000; + // older clients treat any timeout_ms below this as the start countdown + private const int c_LegacyClientCountdownThresholdMSec = 10000; + private async Task StartGameAfterSuccessfulMeshCheck(Lobby lobby) { Console.WriteLine("START GAME"); @@ -884,11 +887,12 @@ private async Task StartGameAfterSuccessfulMeshCheck(Lobby lobby) MatchmakingManager.DestroyBucket(this); } - private void QueueSetupProgress(int timeoutMSec) + private void QueueSetupProgress(int timeoutMSec, int countdownMSec = 0) { WebSocketMessage_MatchmakerSetupProgress setupProgress = new WebSocketMessage_MatchmakerSetupProgress(); setupProgress.msg_id = (int)EWebSocketMessageID.MATCHMAKING_ACTION_SETUP_PROGRESS; setupProgress.timeout_ms = timeoutMSec; + setupProgress.countdown_ms = countdownMSec; byte[] setupProgressJSON = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(setupProgress)); foreach (MatchmakingBucketMember member in m_lstMembers) @@ -911,7 +915,8 @@ private async Task TriggerFullMeshConnectivityChecks(Lobby lobby) m_bWaitingOnMeshConnectivityChecks = true; } - QueueSetupProgress(FullMeshCheckSettings.MaxDurationMS + c_SetupClientTimeoutMarginMSec); + // kept at or above the legacy threshold so short tuned checks don't show a start countdown on older clients + QueueSetupProgress(Math.Max(FullMeshCheckSettings.MaxDurationMS + c_SetupClientTimeoutMarginMSec, c_LegacyClientCountdownThresholdMSec)); lobby.SendFullMeshConnectivityCheckRequestToMembers(); @@ -1078,7 +1083,7 @@ public async Task Tick() if (bStartCountdown) { - QueueSetupProgress(c_GameStartCountdownMSec + c_SetupClientTimeoutMarginMSec); + QueueSetupProgress(c_GameStartCountdownMSec + c_SetupClientTimeoutMarginMSec, c_GameStartCountdownMSec); foreach (MatchmakingBucketMember member in m_lstMembers) { UserSession? memberSession = member.GetAssociatedSession(); From 72275700a9949e676b38010d1ff1d966a96fe4bb Mon Sep 17 00:00:00 2001 From: tintinhamans <5984296+tintinhamans@users.noreply.github.com> Date: Fri, 25 Sep 2026 23:48:50 +0200 Subject: [PATCH 5/5] fix(lobby): Judge mesh checks on current members and re-poll members who haven't replied --- GenOnlineService/LobbyManager.cs | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/GenOnlineService/LobbyManager.cs b/GenOnlineService/LobbyManager.cs index f0b6771..4dea5c5 100644 --- a/GenOnlineService/LobbyManager.cs +++ b/GenOnlineService/LobbyManager.cs @@ -384,12 +384,11 @@ private void ProcessPendingFullMeshConnectivityChecksInternal() { bool bDoneChecks = false; - int totalMapEntriesExpected = GetNumberOfHumans(); - //int numConnectionsExpectedPerUser = totalMapEntriesExpected - 1; // minus self - // must have a connectivity map for each lobby member, or the attempt window must have closed + // judged as soon as anyone replies: members who haven't replied yet count as missing, and gaps before + // the window closes are re-polled below, which also recovers replies lost to a reconnect bool bWindowElapsed = (Environment.TickCount64 - TimeStartFullMeshChecks) >= FullMeshCheckSettings.AttemptWindowMS; - bDoneChecks = bWindowElapsed || FullMeshConnectivityChecks.Count == totalMapEntriesExpected; + bDoneChecks = bWindowElapsed || !FullMeshConnectivityChecks.IsEmpty; List lstMissingConnections = new(); @@ -398,6 +397,12 @@ private void ProcessPendingFullMeshConnectivityChecksInternal() // now verify each user has provided data for all other users foreach (var userMap in FullMeshConnectivityChecks) { + // a member who left mid-check no longer counts + if (GetMemberFromUserID(userMap.Key)?.IsHuman() != true) + { + continue; + } + // foreach member in the lobby, check they are in userMap.Value foreach (LobbyMember member in Members) {