From 29971c17b6df29ad89bc4e42b08c05402fd78a51 Mon Sep 17 00:00:00 2001 From: drgrieve543 Date: Fri, 9 Oct 2026 10:12:10 +1100 Subject: [PATCH 1/2] Add overload to sheet appender to take batchinformation class, add an optional callback function on each batch --- src/GoogleSheetsWrapper/BatchInformation.cs | 30 ++++ src/GoogleSheetsWrapper/SheetAppender.cs | 177 ++++++++++++-------- 2 files changed, 140 insertions(+), 67 deletions(-) create mode 100644 src/GoogleSheetsWrapper/BatchInformation.cs diff --git a/src/GoogleSheetsWrapper/BatchInformation.cs b/src/GoogleSheetsWrapper/BatchInformation.cs new file mode 100644 index 000000000..a4068ef --- /dev/null +++ b/src/GoogleSheetsWrapper/BatchInformation.cs @@ -0,0 +1,30 @@ +using System; + +namespace GoogleSheetsWrapper +{ + /// + /// Configuration for batching rows when appending data to a Google Sheet. + /// + public class BatchInformation + { + /// + /// The delay in milliseconds between batch requests. + /// + public int BatchWaitTime { get; set; } = 1000; + + /// + /// The maximum number of rows in each batch request. + /// + public int BatchSize { get; set; } = 100; + + /// + /// Indicates whether the header row should be excluded from the sheet. + /// + public bool SkipWritingHeaderRow { get; set; } + + /// + /// Called before waiting between batches. Return true to stop appending additional records. + /// + public Func ShouldCancel { get; set; } + } +} diff --git a/src/GoogleSheetsWrapper/SheetAppender.cs b/src/GoogleSheetsWrapper/SheetAppender.cs index e89ebe0..5189448 100644 --- a/src/GoogleSheetsWrapper/SheetAppender.cs +++ b/src/GoogleSheetsWrapper/SheetAppender.cs @@ -58,7 +58,26 @@ public void AppendCsv(string filePath, bool csvHasHeaderRecord, int batchWaitTim { using (var stream = new FileStream(filePath, FileMode.Open)) { - AppendCsv(stream, csvHasHeaderRecord, batchWaitTime, batchSize, skipWritingHeaderRow); + AppendCsv(stream, csvHasHeaderRecord, new BatchInformation + { + BatchWaitTime = batchWaitTime, + BatchSize = batchSize, + SkipWritingHeaderRow = skipWritingHeaderRow + }); + } + } + + /// + /// Appends a CSV file and all its rows into the current Google Sheets tab + /// + /// + /// This boolean indicates whether the CSV file has a header record row or not + /// Batch configuration for appending rows + public void AppendCsv(string filePath, bool csvHasHeaderRecord, BatchInformation batchInformation) + { + using (var stream = new FileStream(filePath, FileMode.Open)) + { + AppendCsv(stream, csvHasHeaderRecord, batchInformation); } } @@ -77,7 +96,28 @@ public void AppendCsv(Stream stream, bool csvHasHeaderRecord, int batchWaitTime HasHeaderRecord = csvHasHeaderRecord }; - AppendCsv(stream, csvConfig, batchWaitTime, batchSize, skipWritingHeaderRow); + AppendCsv(stream, csvConfig, new BatchInformation + { + BatchWaitTime = batchWaitTime, + BatchSize = batchSize, + SkipWritingHeaderRow = skipWritingHeaderRow + }); + } + + /// + /// Appends a CSV stream and all its rows into the current Google Sheets tab + /// + /// + /// This boolean indicates whether the CSV file has a header record row or not + /// Batch configuration for appending rows + public void AppendCsv(Stream stream, bool csvHasHeaderRecord, BatchInformation batchInformation) + { + var csvConfig = new CsvConfiguration(CultureInfo.InvariantCulture) + { + HasHeaderRecord = csvHasHeaderRecord + }; + + AppendCsv(stream, csvConfig, batchInformation); } /// @@ -89,12 +129,26 @@ public void AppendCsv(Stream stream, bool csvHasHeaderRecord, int batchWaitTime /// Increasing batch size may improve throughput. Default is conservative. /// This boolean indicates if you want to actually write the header row to the Google sheet public void AppendCsv(Stream stream, CsvConfiguration csvConfig, int batchWaitTime = 1000, int batchSize = 100, bool skipWritingHeaderRow = false) + { + AppendCsv(stream, csvConfig, new BatchInformation + { + BatchWaitTime = batchWaitTime, + BatchSize = batchSize, + SkipWritingHeaderRow = skipWritingHeaderRow + }); + } + + /// + /// Appends a CSV stream and all its rows into the current Google Sheets tab + /// + /// + /// + /// Batch configuration for appending rows + public void AppendCsv(Stream stream, CsvConfiguration csvConfig, BatchInformation batchInformation) { using var streamReader = new StreamReader(stream); using var csv = new CsvReader(streamReader, csvConfig); { - var batchRowLimit = batchSize; - var dataRecords = csv.GetRecords() .Select(x => (IDictionary)x) @@ -102,13 +156,9 @@ public void AppendCsv(Stream stream, CsvConfiguration csvConfig, int batchWaitTi var rowData = new List(); - var currentBatchCount = 0; - // Only write the header record if its specified to not skip writing the header row - if (csvConfig.HasHeaderRecord && !skipWritingHeaderRow) + if (csvConfig.HasHeaderRecord && !batchInformation.SkipWritingHeaderRow) { - currentBatchCount++; - var row = new RowData() { Values = new List() @@ -124,37 +174,15 @@ public void AppendCsv(Stream stream, CsvConfiguration csvConfig, int batchWaitTi rowData.Add(row); } - foreach (var record in dataRecords) - { - currentBatchCount++; - - var row = new RowData() + AppendDataRecords( + dataRecords, + record => new RowData { - Values = new List() - }; - - foreach (var property in record.Keys) - { - row.Values.Add(StringToCellData(record[property])); - } - - rowData.Add(row); - - if (currentBatchCount >= batchRowLimit) - { - AppendRows(rowData); - - rowData = new List(); - currentBatchCount = 0; - - Thread.Sleep(batchWaitTime); - } - } - - if (rowData.Count > 0) - { - AppendRows(rowData); - } + Values = record.Keys.Select(property => StringToCellData(record[property])).ToList() + }, + batchInformation, + rowData, + csvConfig.HasHeaderRecord && !batchInformation.SkipWritingHeaderRow ? 1 : 0); } } @@ -168,31 +196,41 @@ public void AppendCsv(Stream stream, CsvConfiguration csvConfig, int batchWaitTi /// This boolean indicates if you want to actually write the header row to the Google sheet public void AppendObject(IEnumerable dataRecords, int batchWaitTime = 1000, int batchSize = 100, bool skipWritingHeaderRow = false) where T : class { - var batchRowLimit = batchSize; + AppendObject(dataRecords, new BatchInformation + { + BatchWaitTime = batchWaitTime, + BatchSize = batchSize, + SkipWritingHeaderRow = skipWritingHeaderRow + }); + } + /// + /// This lets you append a list of objects to the Google Sheet with any object using reflection to determine the property types during runtime. + /// + /// + /// The list of records + /// Batch configuration for appending rows + public void AppendObject(IEnumerable dataRecords, BatchInformation batchInformation) where T : class + { var rowData = new List(); - var currentBatchCount = 0; - var properties = typeof(T).GetProperties().Where(x => !x.CustomAttributes.Any(a => a.AttributeType == typeof(CsvHelper.Configuration.Attributes.IgnoreAttribute))); // Only write the header record if its specified to not skip writing the header row - if (!skipWritingHeaderRow) + if (!batchInformation.SkipWritingHeaderRow) { - currentBatchCount++; - var row = new RowData() { Values = new List() }; foreach (var header in properties) - { - var name = header.Name; - var nameAttribute = header.GetCustomAttributes(typeof(CsvHelper.Configuration.Attributes.NameAttribute), true).FirstOrDefault(); - if (nameAttribute != null) - { - name = (nameAttribute as CsvHelper.Configuration.Attributes.NameAttribute).Names.FirstOrDefault() ?? name; + { + var name = header.Name; + var nameAttribute = header.GetCustomAttributes(typeof(CsvHelper.Configuration.Attributes.NameAttribute), true).FirstOrDefault(); + if (nameAttribute != null) + { + name = (nameAttribute as CsvHelper.Configuration.Attributes.NameAttribute).Names.FirstOrDefault() ?? name; } row.Values.Add(StringToCellData(name)); } @@ -200,31 +238,36 @@ public void AppendObject(IEnumerable dataRecords, int batchWaitTime = 1000 rowData.Add(row); } + AppendDataRecords( + dataRecords, + record => new RowData + { + Values = properties.Select(property => ObjectToCellData(property.GetValue(record))).ToList() + }, + batchInformation, + rowData, + batchInformation.SkipWritingHeaderRow ? 0 : 1); + } + + private void AppendDataRecords(IEnumerable dataRecords, Func rowFactory, BatchInformation batchInformation, List rowData, int currentBatchCount) + { foreach (var record in dataRecords) { currentBatchCount++; + rowData.Add(rowFactory(record)); - var row = new RowData() - { - Values = new List() - }; - - foreach (var property in properties) - { - var propertyValue = property.GetValue(record); - row.Values.Add(ObjectToCellData(propertyValue)); - } - - rowData.Add(row); - - if (currentBatchCount >= batchRowLimit) + if (currentBatchCount >= batchInformation.BatchSize) { AppendRows(rowData); - rowData = new List(); currentBatchCount = 0; - Thread.Sleep(batchWaitTime); + if (batchInformation.ShouldCancel?.Invoke() == true) + { + return; + } + + Thread.Sleep(batchInformation.BatchWaitTime); } } From 2024e0ce394b8821c473b05a2e6702a0b9a24c07 Mon Sep 17 00:00:00 2001 From: drgrieve543 Date: Sat, 10 Oct 2026 13:35:19 +1100 Subject: [PATCH 2/2] PR feedback fix for callback unnecessarily being called when batch size is divisible by number of records --- .../SheetAppenderTests.cs | 187 ++++++++++++++++++ src/GoogleSheetsWrapper/SheetAppender.cs | 13 +- 2 files changed, 197 insertions(+), 3 deletions(-) create mode 100644 src/GoogleSheetsWrapper.Tests/SheetAppenderTests.cs diff --git a/src/GoogleSheetsWrapper.Tests/SheetAppenderTests.cs b/src/GoogleSheetsWrapper.Tests/SheetAppenderTests.cs new file mode 100644 index 000000000..8bd1491 --- /dev/null +++ b/src/GoogleSheetsWrapper.Tests/SheetAppenderTests.cs @@ -0,0 +1,187 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Net; +using System.Net.Http; +using System.Reflection; +using System.Threading; +using System.Threading.Tasks; +using Google.Apis.Http; +using Google.Apis.Sheets.v4; +using Google.Apis.Services; +using NUnit.Framework; + +namespace GoogleSheetsWrapper.Tests +{ + public class SheetAppenderTests : IDisposable + { + private RecordingHttpMessageHandler _httpHandler; + private SheetAppender _appender; + + [SetUp] + public void SetUp() + { + _httpHandler = new RecordingHttpMessageHandler(); + var service = new SheetsService(new BaseClientService.Initializer + { + ApplicationName = "GoogleSheetsWrapper.Tests", + HttpClientFactory = new HttpClientFromMessageHandlerFactory(_ => + new HttpClientFromMessageHandlerFactory.ConfiguredHttpMessageHandler(_httpHandler, false, false)) + }); + + var sheetHelper = new SheetHelper("spreadsheet-id", "service-account@example.com", "Sheet1") + { + SheetID = 42 + }; + + typeof(SheetHelper) + .GetProperty(nameof(SheetHelper.Service), BindingFlags.Instance | BindingFlags.Public) + .SetValue(sheetHelper, service); + + _appender = new SheetAppender(sheetHelper); + } + + [Test] + public void AppendObjectAppendsHeaderAndRecordsInBatches() + { + var records = new[] + { + new TestRecord { Name = "Alice", Age = 30 }, + new TestRecord { Name = "Bob", Age = 31 }, + new TestRecord { Name = "Charlie", Age = 32 } + }; + + _appender.AppendObject(records, new BatchInformation + { + BatchSize = 3, + BatchWaitTime = 0 + }); + + Assert.That(_httpHandler.RequestBodies, Has.Count.EqualTo(2)); + Assert.That(_httpHandler.RequestBodies[0], Does.Contain("Name")); + Assert.That(_httpHandler.RequestBodies[0], Does.Contain("Alice")); + Assert.That(_httpHandler.RequestBodies[0], Does.Contain("Bob")); + Assert.That(_httpHandler.RequestBodies[1], Does.Contain("Charlie")); + } + + [Test] + public void AppendCsvAppendsHeaderAndValues() + { + using var stream = new System.IO.MemoryStream( + System.Text.Encoding.UTF8.GetBytes("Name,Age\nAlice,30\n")); + + _appender.AppendCsv(stream, true, new BatchInformation + { + BatchSize = 10, + BatchWaitTime = 0 + }); + + Assert.That(_httpHandler.RequestBodies, Has.Count.EqualTo(1)); + Assert.That(_httpHandler.RequestBodies[0], Does.Contain("Name")); + Assert.That(_httpHandler.RequestBodies[0], Does.Contain("Alice")); + Assert.That(_httpHandler.RequestBodies[0], Does.Contain("30")); + } + + [Test] + public void AppendObjectWhenCancellationCallbackReturnsTrueStopsAfterCurrentBatch() + { + var callbackCount = 0; + var records = new[] + { + new TestRecord { Name = "Alice", Age = 30 }, + new TestRecord { Name = "Bob", Age = 31 }, + new TestRecord { Name = "Charlie", Age = 32 } + }; + + _appender.AppendObject(records, new BatchInformation + { + BatchSize = 1, + BatchWaitTime = 0, + SkipWritingHeaderRow = true, + ShouldCancel = () => + { + callbackCount++; + return true; + } + }); + + Assert.That(callbackCount, Is.EqualTo(1)); + Assert.That(_httpHandler.RequestBodies, Has.Count.EqualTo(1)); + Assert.That(_httpHandler.RequestBodies[0], Does.Contain("Alice")); + Assert.That(_httpHandler.RequestBodies[0], Does.Not.Contain("Bob")); + } + + [Test] + public void AppendObjectDoesNotCallCancellationCallbackAfterFinalExactBatch() + { + var callbackCount = 0; + var records = new[] + { + new TestRecord { Name = "Alice", Age = 30 }, + new TestRecord { Name = "Bob", Age = 31 }, + new TestRecord { Name = "Charlie", Age = 32 }, + new TestRecord { Name = "Diana", Age = 33 } + }; + + _appender.AppendObject(records, new BatchInformation + { + BatchSize = 2, + BatchWaitTime = 0, + SkipWritingHeaderRow = true, + ShouldCancel = () => + { + callbackCount++; + return false; + } + }); + + Assert.That(_httpHandler.RequestBodies, Has.Count.EqualTo(2)); + Assert.That(callbackCount, Is.EqualTo(1)); + } + + private sealed class TestRecord + { + public string Name { get; set; } + + public int Age { get; set; } + } + + private sealed class RecordingHttpMessageHandler : HttpMessageHandler + { + public List RequestBodies { get; } = new List(); + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + RequestBodies.Add(ReadRequestBody(request)); + + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent("{}") + }); + } + + private static string ReadRequestBody(HttpRequestMessage request) + { + if (request.Content == null) + { + return string.Empty; + } + + using var contentStream = request.Content.ReadAsStreamAsync().GetAwaiter().GetResult(); + using var bodyStream = request.Content.Headers.ContentEncoding.Contains("gzip") + ? new GZipStream(contentStream, CompressionMode.Decompress) + : contentStream; + using var reader = new StreamReader(bodyStream); + + return reader.ReadToEnd(); + } + } + + public void Dispose() + { + _httpHandler.Dispose(); + GC.SuppressFinalize(this); + } + } +} diff --git a/src/GoogleSheetsWrapper/SheetAppender.cs b/src/GoogleSheetsWrapper/SheetAppender.cs index 5189448..a7e16b6 100644 --- a/src/GoogleSheetsWrapper/SheetAppender.cs +++ b/src/GoogleSheetsWrapper/SheetAppender.cs @@ -251,8 +251,11 @@ public void AppendObject(IEnumerable dataRecords, BatchInformation batchIn private void AppendDataRecords(IEnumerable dataRecords, Func rowFactory, BatchInformation batchInformation, List rowData, int currentBatchCount) { - foreach (var record in dataRecords) + var records = dataRecords.ToList(); + + for (var recordIndex = 0; recordIndex < records.Count; recordIndex++) { + var record = records[recordIndex]; currentBatchCount++; rowData.Add(rowFactory(record)); @@ -262,12 +265,16 @@ private void AppendDataRecords(IEnumerable dataRecords, Func r rowData = new List(); currentBatchCount = 0; - if (batchInformation.ShouldCancel?.Invoke() == true) + var hasMoreRecords = recordIndex < records.Count - 1; + if (hasMoreRecords && batchInformation.ShouldCancel?.Invoke() == true) { return; } - Thread.Sleep(batchInformation.BatchWaitTime); + if (hasMoreRecords) + { + Thread.Sleep(batchInformation.BatchWaitTime); + } } }