diff --git a/GenOnlineService/Constants.cs b/GenOnlineService/Constants.cs index a860c70..678e078 100644 --- a/GenOnlineService/Constants.cs +++ b/GenOnlineService/Constants.cs @@ -2997,6 +2997,9 @@ public class WebSocketMessage_FullMeshConnectivityCheckResponseFromUser : WebSoc public Int64 mesh_check_id { get; set; } public int attempt { get; set; } public List connectivity_map { get; set; } = new(); + + // peers the member is still negotiating with; absent from older clients + public List connecting_map { get; set; } = new(); } public class WebSocketMessage_Social_NewFriendRequest : WebSocketMessage @@ -3188,6 +3191,9 @@ public class WebSocketMessage_MatchmakerStartGame : WebSocketMessage public class WebSocketMessage_MatchmakerSetupProgress : WebSocketMessage { public int timeout_ms { get; set; } + + // start countdown length, 0 while the match is still being set up + public int countdown_ms { get; set; } } } diff --git a/GenOnlineService/LobbyManager.cs b/GenOnlineService/LobbyManager.cs index b08e86f..4dea5c5 100644 --- a/GenOnlineService/LobbyManager.cs +++ b/GenOnlineService/LobbyManager.cs @@ -62,6 +62,30 @@ internal static bool ShouldRetry(bool meshComplete, bool hasLegacyResponse, int } } + // Core:full_mesh_check_* in appsettings.json, read on use + internal static class FullMeshCheckSettings + { + // how long an attempt waits for every connection before judging it + internal static int AttemptWindowMS => Get("full_mesh_check_attempt_window_ms", 8000); + + // how often an incomplete attempt asks members for a fresh snapshot + internal static int SnapshotIntervalMS => Get("full_mesh_check_snapshot_interval_ms", 1000); + + // time given to re-signalled connections before the next attempt + internal static int RetryDelayMS => Get("full_mesh_check_retry_delay_ms", 3000); + + internal static int MaxAttempts => Get("full_mesh_check_max_attempts", 2); + + // upper bound for a whole check, used as the clients' setup timeout + internal static int MaxDurationMS => (AttemptWindowMS * MaxAttempts) + (RetryDelayMS * (MaxAttempts - 1)); + + private static int Get(string key, int defaultValue) + { + int value = Program.g_Config?.GetSection("Core").GetValue(key, defaultValue) ?? defaultValue; + return value > 0 ? value : defaultValue; + } + } + public class Lobby { public Int64 LobbyID { get; private set; } = -1; @@ -81,13 +105,6 @@ public class Lobby [JsonIgnore] public Int64 TimeStartFullMeshChecks { get; private set; } = -1; - private const int MSToWaitForFullMeshChecks = 5000; - private const int MaxFullMeshCheckAttempts = 2; - private const int MSBeforeFullMeshCheckRetry = 3000; - public const int MaxFullMeshConnectivityCheckDurationMS = - (MSToWaitForFullMeshChecks * MaxFullMeshCheckAttempts) - + (MSBeforeFullMeshCheckRetry * (MaxFullMeshCheckAttempts - 1)); - private readonly object m_FullMeshCheckLock = new(); [JsonIgnore] @@ -102,6 +119,13 @@ public class Lobby [JsonIgnore] private Int64 m_TimeToRetryFullMeshChecks = -1; + [JsonIgnore] + private Int64 m_TimeNextFullMeshSnapshotRequest = -1; + + // per member, the peers they reported as still connecting in the current attempt + [JsonIgnore] + private Dictionary> m_FullMeshConnecting = new(); + [JsonIgnore] private bool m_bCurrentAttemptHasLegacyResponse = false; @@ -206,10 +230,38 @@ private void BeginFullMeshConnectivityCheckAttempt() { PendingFullMeshConnectivityChecks = true; FullMeshConnectivityChecks = new(); + m_FullMeshConnecting = new(); m_bCurrentAttemptHasLegacyResponse = false; - TimeStartFullMeshChecks = Environment.TickCount64; - } + TimeStartFullMeshChecks = Environment.TickCount64; + m_TimeNextFullMeshSnapshotRequest = TimeStartFullMeshChecks + FullMeshCheckSettings.SnapshotIntervalMS; + } + private bool IsFullMeshPairStillConnecting(Int64 userA, Int64 userB) + { + return (m_FullMeshConnecting.TryGetValue(userA, out HashSet? fromA) && fromA.Contains(userB)) + || (m_FullMeshConnecting.TryGetValue(userB, out HashSet? fromB) && fromB.Contains(userA)); + } + + // one line per judged attempt, so the window can be tuned from real connect times + private void LogFullMeshCheckAttempt(bool bMeshComplete, List lstMissingConnections) + { + Int64 elapsedMS = Environment.TickCount64 - TimeStartFullMeshChecks; + string strMissing = string.Join(", ", lstMissingConnections + .Select(c => (Math.Min(c.source_user_id, c.target_user_id), Math.Max(c.source_user_id, c.target_user_id))) + .Distinct() + .Select(p => $"{p.Item1}<->{p.Item2}")); + string strConnecting = string.Join(", ", lstMissingConnections + .Where(c => IsFullMeshPairStillConnecting(c.source_user_id, c.target_user_id)) + .Select(c => (Math.Min(c.source_user_id, c.target_user_id), Math.Max(c.source_user_id, c.target_user_id))) + .Distinct() + .Select(p => $"{p.Item1}<->{p.Item2}")); + + Console.WriteLine("[Lobby {0}] Mesh check {1} attempt {2}/{3}: {4} after {5} ms with {6} humans{7}", + LobbyID, FullMeshCheckID, FullMeshCheckAttempt, FullMeshCheckSettings.MaxAttempts, + bMeshComplete ? "complete" : "incomplete", elapsedMS, GetNumberOfHumans(), + bMeshComplete ? "" : $", missing {strMissing}{(strConnecting.Length > 0 ? $" (still connecting {strConnecting})" : "")}"); + } + public void SendFullMeshConnectivityCheckRequestToMembers() { WebSocketMessage_FullMeshConnectivityCheckRequest startCommand = new WebSocketMessage_FullMeshConnectivityCheckRequest(); @@ -290,6 +342,7 @@ public Task StoreFullMeshConnectivityResponse(Int64 sourceUser, WebSocketMessage { m_bCurrentAttemptHasLegacyResponse |= bLegacyResponse; FullMeshConnectivityChecks[sourceUser] = new ConcurrentList(response.connectivity_map); + m_FullMeshConnecting[sourceUser] = new HashSet(response.connecting_map); } ProcessPendingFullMeshConnectivityChecksInternal(); @@ -331,17 +384,11 @@ private void ProcessPendingFullMeshConnectivityChecksInternal() { bool bDoneChecks = false; - int totalMapEntriesExpected = GetNumberOfHumans(); - //int numConnectionsExpectedPerUser = totalMapEntriesExpected - 1; // minus self - - // must have a connectivity map for each lobby member - bDoneChecks = FullMeshConnectivityChecks.Count == totalMapEntriesExpected; - // did we timeout? - if (!bDoneChecks && (Environment.TickCount64 - TimeStartFullMeshChecks) >= MSToWaitForFullMeshChecks) - { - bDoneChecks = true; - } + // judged as soon as anyone replies: members who haven't replied yet count as missing, and gaps before + // the window closes are re-polled below, which also recovers replies lost to a reconnect + bool bWindowElapsed = (Environment.TickCount64 - TimeStartFullMeshChecks) >= FullMeshCheckSettings.AttemptWindowMS; + bDoneChecks = bWindowElapsed || !FullMeshConnectivityChecks.IsEmpty; List lstMissingConnections = new(); @@ -350,6 +397,12 @@ private void ProcessPendingFullMeshConnectivityChecksInternal() // now verify each user has provided data for all other users foreach (var userMap in FullMeshConnectivityChecks) { + // a member who left mid-check no longer counts + if (GetMemberFromUserID(userMap.Key)?.IsHuman() != true) + { + continue; + } + // foreach member in the lobby, check they are in userMap.Value foreach (LobbyMember member in Members) { @@ -411,15 +464,34 @@ private void ProcessPendingFullMeshConnectivityChecksInternal() bool bMeshComplete = bDisableMeshCheck || lstMissingConnections.Count == 0; + // members report a snapshot, so a gap before the window closes may be a connection still forming: + // keep asking for fresh snapshots rather than re-signalling it early + if (!bMeshComplete && !bWindowElapsed) + { + if (Environment.TickCount64 >= m_TimeNextFullMeshSnapshotRequest) + { + m_TimeNextFullMeshSnapshotRequest = Environment.TickCount64 + FullMeshCheckSettings.SnapshotIntervalMS; + SendFullMeshConnectivityCheckRequestToMembers(); + } + + return; + } + + LogFullMeshCheckAttempt(bMeshComplete, lstMissingConnections); + if (FullMeshCheckProtocol.ShouldRetry( bMeshComplete, m_bCurrentAttemptHasLegacyResponse, FullMeshCheckAttempt, - MaxFullMeshCheckAttempts)) + FullMeshCheckSettings.MaxAttempts)) { ++FullMeshCheckAttempt; - RestartSignallingForMissingConnections(lstMissingConnections); - m_TimeToRetryFullMeshChecks = Environment.TickCount64 + MSBeforeFullMeshCheckRetry; + + // a pair still negotiating would be torn down by a re-signal; the next attempt re-checks it + RestartSignallingForMissingConnections(lstMissingConnections + .Where(c => !IsFullMeshPairStillConnecting(c.source_user_id, c.target_user_id)) + .ToList()); + m_TimeToRetryFullMeshChecks = Environment.TickCount64 + FullMeshCheckSettings.RetryDelayMS; return; } diff --git a/GenOnlineService/MatchmakingManager.cs b/GenOnlineService/MatchmakingManager.cs index bc8cd92..3769961 100644 --- a/GenOnlineService/MatchmakingManager.cs +++ b/GenOnlineService/MatchmakingManager.cs @@ -862,6 +862,9 @@ public Int64 GetLobbyID() private const int c_GameStartCountdownMSec = 5000; private const int c_SetupClientTimeoutMarginMSec = 2000; + // older clients treat any timeout_ms below this as the start countdown + private const int c_LegacyClientCountdownThresholdMSec = 10000; + private async Task StartGameAfterSuccessfulMeshCheck(Lobby lobby) { Console.WriteLine("START GAME"); @@ -884,11 +887,12 @@ private async Task StartGameAfterSuccessfulMeshCheck(Lobby lobby) MatchmakingManager.DestroyBucket(this); } - private void QueueSetupProgress(int timeoutMSec) + private void QueueSetupProgress(int timeoutMSec, int countdownMSec = 0) { WebSocketMessage_MatchmakerSetupProgress setupProgress = new WebSocketMessage_MatchmakerSetupProgress(); setupProgress.msg_id = (int)EWebSocketMessageID.MATCHMAKING_ACTION_SETUP_PROGRESS; setupProgress.timeout_ms = timeoutMSec; + setupProgress.countdown_ms = countdownMSec; byte[] setupProgressJSON = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(setupProgress)); foreach (MatchmakingBucketMember member in m_lstMembers) @@ -911,7 +915,8 @@ private async Task TriggerFullMeshConnectivityChecks(Lobby lobby) m_bWaitingOnMeshConnectivityChecks = true; } - QueueSetupProgress(Lobby.MaxFullMeshConnectivityCheckDurationMS + c_SetupClientTimeoutMarginMSec); + // kept at or above the legacy threshold so short tuned checks don't show a start countdown on older clients + QueueSetupProgress(Math.Max(FullMeshCheckSettings.MaxDurationMS + c_SetupClientTimeoutMarginMSec, c_LegacyClientCountdownThresholdMSec)); lobby.SendFullMeshConnectivityCheckRequestToMembers(); @@ -1078,7 +1083,7 @@ public async Task Tick() if (bStartCountdown) { - QueueSetupProgress(c_GameStartCountdownMSec + c_SetupClientTimeoutMarginMSec); + QueueSetupProgress(c_GameStartCountdownMSec + c_SetupClientTimeoutMarginMSec, c_GameStartCountdownMSec); foreach (MatchmakingBucketMember member in m_lstMembers) { UserSession? memberSession = member.GetAssociatedSession(); diff --git a/GenOnlineService/appsettings.json b/GenOnlineService/appsettings.json index d193ca6..29dc3bd 100644 --- a/GenOnlineService/appsettings.json +++ b/GenOnlineService/appsettings.json @@ -34,7 +34,11 @@ "cert_pem_path": null, "cert_key_path": null, "cert_reload_interval_seconds": 30, - "disable_full_mesh_check": false + "disable_full_mesh_check": false, + "full_mesh_check_attempt_window_ms": 8000, + "full_mesh_check_snapshot_interval_ms": 1000, + "full_mesh_check_retry_delay_ms": 3000, + "full_mesh_check_max_attempts": 2 }, "TURN": { "key": null,