Expose the total order count and share the paging window guard across the api read methods

Api/Api.cs

@@ -42,6 +42,12 @@ namespace QuantConnect.Api
     /// </summary>
     public class Api : IApi, IDownloadProvider
     {
+        // Widest start/end window each paging endpoint documents
+        private const int MaxBacktestOrdersWindow = 100;
+        private const int MaxLiveOrdersWindow = 1000;
+        private const int MaxInsightsWindow = 100;
+        private const int MaxLogLinesWindow = 200;
+
         private readonly BlockingCollection<Lazy<HttpClient>> _clientPool;
         private string _dataFolder;
 
@@ -412,13 +418,16 @@ public Backtest ReadBacktest(int projectId, string backtestId, bool getCharts =
         /// </summary>
         /// <param name="projectId">Id of the project from which to read the orders</param>
         /// <param name="backtestId">Id of the backtest from which to read the orders</param>
-        /// <param name="start">Starting index of the orders to be fetched. Required if end > 100</param>
-        /// <param name="end">Last index of the orders to be fetched. Note that end - start must be less than 100</param>
+        /// <param name="start">Starting index of the orders to be fetched</param>
+        /// <param name="end">Last index of the orders to be fetched. Note that end - start must not exceed 100.
+        /// Defaults to a full window starting at <paramref name="start"/></param>
         /// <remarks>Will throw an <see cref="WebException"/> if there are any API errors</remarks>
-        /// <returns>The list of <see cref="Order"/></returns>
-
-        public List<ApiOrderResponse> ReadBacktestOrders(int projectId, string backtestId, int start = 0, int end = 100)
+        /// <returns>The <see cref="OrdersResponseWrapper"/> with the requested orders and the total order count</returns>
+        /// <exception cref="ArgumentException">The requested window is wider than the documented maximum</exception>
+        public OrdersResponseWrapper ReadBacktestOrders(int projectId, string backtestId, int start = 0, int end = 0)
         {
+            end = ResolveWindowEnd(start, end, MaxBacktestOrdersWindow, "orders");
+
             using var request = ApiUtils.CreateJsonPostRequest("backtests/orders/read", new
             {
                 start,
@@ -427,7 +436,7 @@ public List<ApiOrderResponse> ReadBacktestOrders(int projectId, string backtestI
                 backtestId
             });
 
-            return MakeRequestOrThrow<OrdersResponseWrapper>(request, nameof(ReadBacktestOrders)).Orders;
+            return MakeRequestOrThrow<OrdersResponseWrapper>(request, nameof(ReadBacktestOrders));
         }
 
         /// <summary>
@@ -531,26 +540,37 @@ public RestResponse UpdateBacktestTags(int projectId, string backtestId, IReadOn
         /// <param name="projectId">Id of the project from which to read the backtest</param>
         /// <param name="backtestId">Backtest id from which we want to get the insights</param>
         /// <param name="start">Starting index of the insights to be fetched</param>
-        /// <param name="end">Last index of the insights to be fetched. Note that end - start must be less than 100</param>
+        /// <param name="end">Last index of the insights to be fetched. Note that end - start must not exceed 100.
+        /// Defaults to a full window starting at <paramref name="start"/></param>
         /// <returns><see cref="InsightResponse"/></returns>
-        /// <exception cref="ArgumentException"></exception>
+        /// <exception cref="ArgumentException">The requested window is wider than the documented maximum</exception>
         public InsightResponse ReadBacktestInsights(int projectId, string backtestId, int start = 0, int end = 0)
         {
-            //var reque
-            var diff = end - start;
-            if (diff > 100)
-            {
-                throw new ArgumentException($"The difference between the start and end index of the insights must be smaller than 100, but it was {diff}.");
-            }
-            else if (end == 0)
-            {
-                end = start + 100;
-            }
+            end = ResolveWindowEnd(start, end, MaxInsightsWindow, "insights");
 
             TryJsonPost("backtests/insights/read", out InsightResponse result, new { projectId, backtestId, start, end });
             return result;
         }
 
+        /// <summary>
+        /// Gets the logs of a specific backtest
+        /// </summary>
+        /// <param name="projectId">Id of the project from which to read the backtest</param>
+        /// <param name="backtestId">Id of the backtest from which to read the logs</param>
+        /// <param name="start">Start line (inclusive) of logs to read</param>
+        /// <param name="end">End line (exclusive) of logs to read. Note that end - start must not exceed 200.
+        /// Defaults to a full window starting at <paramref name="start"/></param>
+        /// <param name="query">Optional keyword to filter the log lines, null to return every line</param>
+        /// <returns><see cref="BacktestLog"/> with the requested log lines and the total log line count</returns>
+        /// <exception cref="ArgumentException">The requested window is wider than the documented maximum</exception>
+        public BacktestLog ReadBacktestLog(int projectId, string backtestId, int start = 0, int end = 0, string query = null)
+        {
+            end = ResolveWindowEnd(start, end, MaxLogLinesWindow, "log lines");
+
+            TryJsonPost("backtests/read/log", out BacktestLog result, new { projectId, backtestId, start, end, query });
+            return result;
+        }
+
         /// <summary>
         /// Create a live algorithm.
         /// </summary>
@@ -683,21 +703,24 @@ public PortfolioResponse ReadLivePortfolio(int projectId)
         /// Returns the orders of the specified project id live algorithm.
         /// </summary>
         /// <param name="projectId">Id of the project from which to read the live orders</param>
-        /// <param name="start">Starting index of the orders to be fetched. Required if end > 100</param>
-        /// <param name="end">Last index of the orders to be fetched. Note that end - start must be less than 100</param>
+        /// <param name="algorithmId">Deploy id (algorithm id) of the live running algorithm. Optional, the API
+        /// defaults to the latest deployment of the project</param>
+        /// <param name="start">Starting index of the orders to be fetched</param>
+        /// <param name="end">Last index of the orders to be fetched. Note that end - start must not exceed 1000.
+        /// Defaults to a full window starting at <paramref name="start"/></param>
         /// <remarks>Will throw an <see cref="WebException"/> if there are any API errors</remarks>
-        /// <returns>The list of <see cref="Order"/></returns>
-
-        public List<ApiOrderResponse> ReadLiveOrders(int projectId, int start = 0, int end = 100)
+        /// <returns>The <see cref="OrdersResponseWrapper"/> with the requested orders and the total order count</returns>
+        /// <exception cref="ArgumentException">The requested window is wider than the documented maximum</exception>
+        public OrdersResponseWrapper ReadLiveOrders(int projectId, string algorithmId = null, int start = 0, int end = 0)
         {
-            using var request = ApiUtils.CreateJsonPostRequest("live/orders/read", new
-            {
-                start,
-                end,
-                projectId
-            });
+            end = ResolveWindowEnd(start, end, MaxLiveOrdersWindow, "orders");
 
-            return MakeRequestOrThrow<OrdersResponseWrapper>(request, nameof(ReadLiveOrders)).Orders;
+            object payload = string.IsNullOrEmpty(algorithmId)
+                ? new { start, end, projectId }
+                : new { start, end, projectId, algorithmId };
+            using var request = ApiUtils.CreateJsonPostRequest("live/orders/read", payload);
+
+            return MakeRequestOrThrow<OrdersResponseWrapper>(request, nameof(ReadLiveOrders));
         }
 
         /// <summary>
@@ -753,16 +776,17 @@ public RestResponse BroadcastLiveCommand(string organizationId, int? excludeProj
         /// </summary>
         /// <param name="projectId">Project Id of the live running algorithm</param>
         /// <param name="algorithmId">Algorithm Id of the live running algorithm</param>
-        /// <param name="startLine">Start line of logs to read</param>
-        /// <param name="endLine">End line of logs to read</param>
+        /// <param name="startLine">Start line (inclusive) of logs to read</param>
+        /// <param name="endLine">End line (exclusive) of logs to read. Note that endLine - startLine must not exceed 200.
+        /// Defaults to a full window starting at <paramref name="startLine"/></param>
+        /// <param name="query">Optional keyword to filter the log lines, null to return every line</param>
+        /// <param name="deploymentLogs">Whether only the logs of the given <paramref name="algorithmId"/> deployment should be returned</param>
         /// <returns><see cref="LiveLog"/> List of strings that represent the logs of the algorithm</returns>
-        public LiveLog ReadLiveLogs(int projectId, string algorithmId, int startLine, int endLine)
+        /// <exception cref="ArgumentException">The requested window is wider than the documented maximum</exception>
+        public LiveLog ReadLiveLogs(int projectId, string algorithmId, int startLine = 0, int endLine = 0, string query = null,
+            bool deploymentLogs = false)
         {
-            var logLinesNumber = endLine - startLine;
-            if (logLinesNumber > 250)
-            {
-                throw new ArgumentException($"The maximum number of log lines allowed is 250. But the number of log lines was {logLinesNumber}.");
-            }
+            endLine = ResolveWindowEnd(startLine, endLine, MaxLogLinesWindow, "log lines");
 
             TryJsonPost("live/logs/read",
                 out LiveLog result,
@@ -773,6 +797,8 @@ public LiveLog ReadLiveLogs(int projectId, string algorithmId, int startLine, in
                     algorithmId,
                     startLine,
                     endLine,
+                    deploymentLogs,
+                    query,
                 });
             return result;
         }
@@ -817,20 +843,13 @@ public ReadChartResponse ReadLiveChart(int projectId, string name, int start, in
         /// </summary>
         /// <param name="projectId">Id of the project from which to read the live algorithm</param>
         /// <param name="start">Starting index of the insights to be fetched</param>
-        /// <param name="end">Last index of the insights to be fetched. Note that end - start must be less than 100</param>
+        /// <param name="end">Last index of the insights to be fetched. Note that end - start must not exceed 100.
+        /// Defaults to a full window starting at <paramref name="start"/></param>
         /// <returns><see cref="InsightResponse"/></returns>
-        /// <exception cref="ArgumentException"></exception>
+        /// <exception cref="ArgumentException">The requested window is wider than the documented maximum</exception>
         public InsightResponse ReadLiveInsights(int projectId, int start = 0, int end = 0)
         {
-            var diff = end - start;
-            if (diff > 100)
-            {
-                throw new ArgumentException($"The difference between the start and end index of the insights must be smaller than 100, but it was {diff}.");
-            }
-            else if (end == 0)
-            {
-                end = start + 100;
-            }
+            end = ResolveWindowEnd(start, end, MaxInsightsWindow, "insights");
 
             TryJsonPost("live/insights/read", out InsightResponse result, new { projectId, start, end });
             return result;
@@ -1484,6 +1503,32 @@ protected virtual ApiConnection CreateApiConnection(int userId, string token)
             return new ApiConnection(userId, token);
         }
 
+        /// <summary>
+        /// Resolves the end index of a paging request, where an unset end means a full window from the start index
+        /// </summary>
+        /// <param name="start">Start index of the requested window</param>
+        /// <param name="end">End index of the requested window, zero to request a full window</param>
+        /// <param name="maxWindow">Widest window the endpoint documents</param>
+        /// <param name="itemsName">Name of the paged items, used in the error message</param>
+        /// <returns>The end index to send</returns>
+        /// <exception cref="ArgumentException">The requested window is wider than <paramref name="maxWindow"/></exception>
+        private static int ResolveWindowEnd(int start, int end, int maxWindow, string itemsName)
+        {
+            if (end == 0)
+            {
+                return start + maxWindow;
+            }
+
+            var window = end - start;
+            if (window > maxWindow)
+            {
+                throw new ArgumentException($"The difference between the start and end index of the {itemsName} must be " +
+                    $"smaller than or equal to {maxWindow}, but it was {window}.");
+            }
+
+            return end;
+        }
+
         /// <summary>
         /// Helper method that will execute the given api request and throw an exception if it fails
         /// </summary>

Common/Api/BacktestLog.cs

@@ -0,0 +1,35 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System.Collections.Generic;
+
+namespace QuantConnect.Api
+{
+    /// <summary>
+    /// Logs from a backtest
+    /// </summary>
+    public class BacktestLog : RestResponse
+    {
+        /// <summary>
+        /// List of log lines from the backtest
+        /// </summary>
+        public List<string> Logs { get; set; }
+
+        /// <summary>
+        /// Total number of log lines in the backtest
+        /// </summary>
+        public int Length { get; set; }
+    }
+}

Common/Api/LiveLog.cs

@@ -29,12 +29,12 @@ public class LiveLog : RestResponse
         public List<string> Logs { get; set; }
 
         /// <summary>
-        /// Total amount of rows in the logs
+        /// Total amount of rows in the logs across all the live deployments of the project
         /// </summary>
         public int Length { get; set; }
 
         /// <summary>
-        /// Amount of log rows before the current deployment
+        /// Amount of log rows before the deployment requested through the algorithm id
         /// </summary>
         public int DeploymentOffset { get; set; }
     }