From 0f8be2e45c57d3dbd7d04906da6f1981c322faa3 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Tue, 28 Jul 2026 18:37:50 -0400 Subject: [PATCH 1/3] Append out of memory diagnostics to the algorithm runtime error message --- Engine/Engine.cs | 57 +++++++++++++++++++++++++++-- Tests/Engine/EngineTests.cs | 71 +++++++++++++++++++++++++++++++++++++ 2 files changed, 126 insertions(+), 2 deletions(-) create mode 100644 Tests/Engine/EngineTests.cs diff --git a/Engine/Engine.cs b/Engine/Engine.cs index 691d8db248c9..7cc5febbe0f8 100644 --- a/Engine/Engine.cs +++ b/Engine/Engine.cs @@ -381,7 +381,7 @@ public void Run(AlgorithmNodePacket job, AlgorithmManager manager, string assemb // Algorithm runtime error: if (algorithm.RunTimeError != null) { - HandleAlgorithmError(job, algorithm.RunTimeError); + HandleAlgorithmError(job, algorithm, algorithm.RunTimeError); } // notify the LEAN manager that the algorithm has finished @@ -477,13 +477,18 @@ public void Run(AlgorithmNodePacket job, AlgorithmManager manager, string assemb /// Handle an error in the algorithm.Run method. /// /// Job we're processing + /// The algorithm instance that raised the error /// Error from algorithm stack - private void HandleAlgorithmError(AlgorithmNodePacket job, Exception err) + private void HandleAlgorithmError(AlgorithmNodePacket job, IAlgorithm algorithm, Exception err) { AlgorithmHandlers.DataFeed?.Exit(); if (AlgorithmHandlers.Results != null) { var message = $"Runtime Error: {err.Message}"; + if (IsOutOfMemoryError(err)) + { + message += GetOutOfMemoryErrorDetails(job, algorithm); + } Log.Trace("Engine.Run(): Sending runtime error to user..."); AlgorithmHandlers.Results.LogMessage(message); @@ -495,6 +500,54 @@ private void HandleAlgorithmError(AlgorithmNodePacket job, Exception err) } } + /// + /// Determines whether the given exception, or any exception in its inner exception chain, + /// is an + /// + /// The exception to inspect + internal static bool IsOutOfMemoryError(Exception err) + { + for (var exception = err; exception != null; exception = exception.InnerException) + { + if (exception is OutOfMemoryException) + { + return true; + } + } + return false; + } + + /// + /// Builds a summary of the algorithm state along with the most common causes of out of memory errors, + /// to be appended to the runtime error message to help diagnose the failure + /// + /// Job we're processing + /// The algorithm instance that ran out of memory + internal static string GetOutOfMemoryErrorDetails(AlgorithmNodePacket job, IAlgorithm algorithm) + { + var details = Invariant($" The algorithm exhausted its {job.RamAllocation}MB of RAM."); + if (algorithm != null) + { + try + { + details += Invariant($" At the time of the error it had {algorithm.Securities.Count} securities, {algorithm.SubscriptionManager.Count} data subscriptions and {algorithm.UniverseManager.Count} universes."); + } + catch (Exception exception) + { + // best effort: don't let diagnostics collection replace the original error + Log.Error(exception); + } + } + return details + + " Common causes include: a universe that selects too many securities or too many data subscriptions for the allocated RAM;" + + " collections that grow without bound, such as lists or dictionaries accumulating data points, rolling windows or" + + " consolidators with very large periods, or keeping references to every history request result;" + + " and indicators or consolidators registered for many securities and never removed." + + " Consider reducing the universe size or number of subscriptions, using coarser data resolutions," + + " bounding cached collections and history lookback periods, unsubscribing/removing what is no longer needed," + + " or increasing the RAM allocation."; + } + /// /// Load the history provider from the Composer /// diff --git a/Tests/Engine/EngineTests.cs b/Tests/Engine/EngineTests.cs new file mode 100644 index 000000000000..738a70d4c97d --- /dev/null +++ b/Tests/Engine/EngineTests.cs @@ -0,0 +1,71 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +using System; +using NUnit.Framework; +using QuantConnect.Algorithm; +using QuantConnect.Packets; +using QuantConnect.Tests.Engine.DataFeeds; +using LeanEngine = QuantConnect.Lean.Engine.Engine; + +namespace QuantConnect.Tests.Engine +{ + [TestFixture] + public class EngineTests + { + [Test] + public void DetectsOutOfMemoryExceptionsInTheInnerExceptionChain() + { + Assert.IsTrue(LeanEngine.IsOutOfMemoryError(new OutOfMemoryException())); + Assert.IsTrue(LeanEngine.IsOutOfMemoryError(new Exception("wrapper", new OutOfMemoryException()))); + Assert.IsTrue(LeanEngine.IsOutOfMemoryError( + new Exception("outer", new InvalidOperationException("inner", new OutOfMemoryException())))); + + Assert.IsFalse(LeanEngine.IsOutOfMemoryError(new Exception("not oom"))); + Assert.IsFalse(LeanEngine.IsOutOfMemoryError(new Exception("outer", new InvalidOperationException("inner")))); + } + + [Test] + public void OutOfMemoryErrorDetailsIncludeRamAllocationAndAlgorithmState() + { + var algorithm = new QCAlgorithm(); + algorithm.SubscriptionManager.SetDataManager(new DataManagerStub(algorithm)); + algorithm.AddEquity("SPY"); + + var job = new BacktestNodePacket { Controls = new Controls { RamAllocation = 512 } }; + + var details = LeanEngine.GetOutOfMemoryErrorDetails(job, algorithm); + + StringAssert.Contains("512MB of RAM", details); + StringAssert.Contains($"{algorithm.Securities.Count} securities", details); + StringAssert.Contains($"{algorithm.SubscriptionManager.Count} data subscriptions", details); + StringAssert.Contains($"{algorithm.UniverseManager.Count} universes", details); + // guidance on the common causes is always present + StringAssert.Contains("Common causes", details); + } + + [Test] + public void OutOfMemoryErrorDetailsAreProducedWithoutAnAlgorithmInstance() + { + var job = new BacktestNodePacket { Controls = new Controls { RamAllocation = 512 } }; + + var details = LeanEngine.GetOutOfMemoryErrorDetails(job, null); + + StringAssert.Contains("512MB of RAM", details); + StringAssert.Contains("Common causes", details); + } + } +} From 93aad16db574a6b3a51b5721c5b811970574e710 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Wed, 29 Jul 2026 09:17:38 -0400 Subject: [PATCH 2/3] Merge out of memory detection and diagnostics into a single method and shorten the message --- Engine/Engine.cs | 59 ++++++++++++++---------------- Tests/Engine/EngineTests.cs | 71 ------------------------------------- 2 files changed, 26 insertions(+), 104 deletions(-) delete mode 100644 Tests/Engine/EngineTests.cs diff --git a/Engine/Engine.cs b/Engine/Engine.cs index 7cc5febbe0f8..968969febd96 100644 --- a/Engine/Engine.cs +++ b/Engine/Engine.cs @@ -485,9 +485,9 @@ private void HandleAlgorithmError(AlgorithmNodePacket job, IAlgorithm algorithm, if (AlgorithmHandlers.Results != null) { var message = $"Runtime Error: {err.Message}"; - if (IsOutOfMemoryError(err)) + if (TryGetOutOfMemoryErrorDetails(job, algorithm, err, out var outOfMemoryDetails)) { - message += GetOutOfMemoryErrorDetails(job, algorithm); + message += outOfMemoryDetails; } Log.Trace("Engine.Run(): Sending runtime error to user..."); AlgorithmHandlers.Results.LogMessage(message); @@ -501,51 +501,44 @@ private void HandleAlgorithmError(AlgorithmNodePacket job, IAlgorithm algorithm, } /// - /// Determines whether the given exception, or any exception in its inner exception chain, - /// is an + /// Checks whether the given exception, or any exception in its inner exception chain, is an , + /// producing details about the algorithm state and the common causes to be appended to the runtime error message /// - /// The exception to inspect - internal static bool IsOutOfMemoryError(Exception err) + /// Job we're processing + /// The algorithm instance that raised the error + /// Error from algorithm stack + /// The out of memory details, null if the error is not an out of memory error + /// True if the error is an out of memory error + private static bool TryGetOutOfMemoryErrorDetails(AlgorithmNodePacket job, IAlgorithm algorithm, Exception err, out string details) { - for (var exception = err; exception != null; exception = exception.InnerException) + details = null; + var exception = err; + while (exception != null && exception is not OutOfMemoryException) { - if (exception is OutOfMemoryException) - { - return true; - } + exception = exception.InnerException; + } + if (exception == null) + { + return false; } - return false; - } - /// - /// Builds a summary of the algorithm state along with the most common causes of out of memory errors, - /// to be appended to the runtime error message to help diagnose the failure - /// - /// Job we're processing - /// The algorithm instance that ran out of memory - internal static string GetOutOfMemoryErrorDetails(AlgorithmNodePacket job, IAlgorithm algorithm) - { - var details = Invariant($" The algorithm exhausted its {job.RamAllocation}MB of RAM."); + details = Invariant($" The algorithm exhausted its {job.RamAllocation}MB of RAM."); if (algorithm != null) { try { - details += Invariant($" At the time of the error it had {algorithm.Securities.Count} securities, {algorithm.SubscriptionManager.Count} data subscriptions and {algorithm.UniverseManager.Count} universes."); + details += Invariant($" State: {algorithm.Securities.Count} securities, {algorithm.SubscriptionManager.Count} subscriptions, {algorithm.UniverseManager.Count} universes."); } - catch (Exception exception) + catch (Exception stateException) { // best effort: don't let diagnostics collection replace the original error - Log.Error(exception); + Log.Error(stateException); } } - return details + - " Common causes include: a universe that selects too many securities or too many data subscriptions for the allocated RAM;" + - " collections that grow without bound, such as lists or dictionaries accumulating data points, rolling windows or" + - " consolidators with very large periods, or keeping references to every history request result;" + - " and indicators or consolidators registered for many securities and never removed." + - " Consider reducing the universe size or number of subscriptions, using coarser data resolutions," + - " bounding cached collections and history lookback periods, unsubscribing/removing what is no longer needed," + - " or increasing the RAM allocation."; + details += " Common causes: universe/subscription count too high for the allocated RAM; unbounded buffers such as rolling windows," + + " consolidators or accumulated history results. Fixes: reduce universe size/subscriptions, use coarser resolutions," + + " bound buffers, or increase the RAM allocation."; + return true; } /// diff --git a/Tests/Engine/EngineTests.cs b/Tests/Engine/EngineTests.cs deleted file mode 100644 index 738a70d4c97d..000000000000 --- a/Tests/Engine/EngineTests.cs +++ /dev/null @@ -1,71 +0,0 @@ -/* - * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * -*/ - -using System; -using NUnit.Framework; -using QuantConnect.Algorithm; -using QuantConnect.Packets; -using QuantConnect.Tests.Engine.DataFeeds; -using LeanEngine = QuantConnect.Lean.Engine.Engine; - -namespace QuantConnect.Tests.Engine -{ - [TestFixture] - public class EngineTests - { - [Test] - public void DetectsOutOfMemoryExceptionsInTheInnerExceptionChain() - { - Assert.IsTrue(LeanEngine.IsOutOfMemoryError(new OutOfMemoryException())); - Assert.IsTrue(LeanEngine.IsOutOfMemoryError(new Exception("wrapper", new OutOfMemoryException()))); - Assert.IsTrue(LeanEngine.IsOutOfMemoryError( - new Exception("outer", new InvalidOperationException("inner", new OutOfMemoryException())))); - - Assert.IsFalse(LeanEngine.IsOutOfMemoryError(new Exception("not oom"))); - Assert.IsFalse(LeanEngine.IsOutOfMemoryError(new Exception("outer", new InvalidOperationException("inner")))); - } - - [Test] - public void OutOfMemoryErrorDetailsIncludeRamAllocationAndAlgorithmState() - { - var algorithm = new QCAlgorithm(); - algorithm.SubscriptionManager.SetDataManager(new DataManagerStub(algorithm)); - algorithm.AddEquity("SPY"); - - var job = new BacktestNodePacket { Controls = new Controls { RamAllocation = 512 } }; - - var details = LeanEngine.GetOutOfMemoryErrorDetails(job, algorithm); - - StringAssert.Contains("512MB of RAM", details); - StringAssert.Contains($"{algorithm.Securities.Count} securities", details); - StringAssert.Contains($"{algorithm.SubscriptionManager.Count} data subscriptions", details); - StringAssert.Contains($"{algorithm.UniverseManager.Count} universes", details); - // guidance on the common causes is always present - StringAssert.Contains("Common causes", details); - } - - [Test] - public void OutOfMemoryErrorDetailsAreProducedWithoutAnAlgorithmInstance() - { - var job = new BacktestNodePacket { Controls = new Controls { RamAllocation = 512 } }; - - var details = LeanEngine.GetOutOfMemoryErrorDetails(job, null); - - StringAssert.Contains("512MB of RAM", details); - StringAssert.Contains("Common causes", details); - } - } -} From 378a771d4da1b689349aace24f03f775d43f4273 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Wed, 29 Jul 2026 09:27:09 -0400 Subject: [PATCH 3/3] Only check the top level exception for out of memory diagnostics --- Engine/Engine.cs | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/Engine/Engine.cs b/Engine/Engine.cs index 968969febd96..d207b3cadb70 100644 --- a/Engine/Engine.cs +++ b/Engine/Engine.cs @@ -501,7 +501,7 @@ private void HandleAlgorithmError(AlgorithmNodePacket job, IAlgorithm algorithm, } /// - /// Checks whether the given exception, or any exception in its inner exception chain, is an , + /// Checks whether the given exception is an , /// producing details about the algorithm state and the common causes to be appended to the runtime error message /// /// Job we're processing @@ -512,12 +512,7 @@ private void HandleAlgorithmError(AlgorithmNodePacket job, IAlgorithm algorithm, private static bool TryGetOutOfMemoryErrorDetails(AlgorithmNodePacket job, IAlgorithm algorithm, Exception err, out string details) { details = null; - var exception = err; - while (exception != null && exception is not OutOfMemoryException) - { - exception = exception.InnerException; - } - if (exception == null) + if (err is not OutOfMemoryException) { return false; } @@ -537,7 +532,7 @@ private static bool TryGetOutOfMemoryErrorDetails(AlgorithmNodePacket job, IAlgo } details += " Common causes: universe/subscription count too high for the allocated RAM; unbounded buffers such as rolling windows," + " consolidators or accumulated history results. Fixes: reduce universe size/subscriptions, use coarser resolutions," + - " bound buffers, or increase the RAM allocation."; + " or bound buffers."; return true; }