diff --git a/src/MiniExcel.Csv/CsvConfiguration.cs b/src/MiniExcel.Csv/CsvConfiguration.cs
index 54bbc2ef..16db3533 100644
--- a/src/MiniExcel.Csv/CsvConfiguration.cs
+++ b/src/MiniExcel.Csv/CsvConfiguration.cs
@@ -11,7 +11,24 @@ public class CsvConfiguration : MiniExcelBaseConfiguration
public char Seperator { get; set; } = ',';
public string NewLine { get; set; } = "\r\n";
public bool ReadLineBreaksWithinQuotes { get; set; } = true;
- public bool ReadEmptyStringAsNull { get; set; } = false;
+
+ ///
+ /// Empty fields will be converted to null when queried as strings, or to the default value of the corresponding mapped type.
+ ///
+ public bool ReadEmptyFieldsAsDefault { get; set; } = false;
+
+ [Obsolete("Please use the ReadEmptyFieldsAsDefault property instead")]
+ public bool ReadEmptyStringAsNull
+ {
+ get => ReadEmptyFieldsAsDefault;
+ set => ReadEmptyFieldsAsDefault = value;
+ }
+
+ ///
+ /// When set to true if the rows have different lengths they will be returned as jagged data instead of throwing an exception.
+ ///
+ public bool AllowFieldCountMismatch { get; set; } = false;
+
public bool AlwaysQuote { get; set; } = false;
public bool QuoteWhitespaces { get; set; } = true;
public Func? SplitFn { get; set; }
diff --git a/src/MiniExcel.Csv/CsvReader.cs b/src/MiniExcel.Csv/CsvReader.cs
index a53e9ee7..c295d96a 100644
--- a/src/MiniExcel.Csv/CsvReader.cs
+++ b/src/MiniExcel.Csv/CsvReader.cs
@@ -33,14 +33,14 @@ internal CsvReader(Stream stream, IMiniExcelConfiguration? configuration, bool l
using var reader = _config.StreamReaderFunc(_stream);
var firstRow = true;
- var headRows = new Dictionary();
+ var headerRow = new Dictionary();
var rowIndex = 0;
while (await reader.ReadLineAsync(
#if NET
cancellationToken
#endif
- ).ConfigureAwait(false) is { } row)
+ ).ConfigureAwait(false) is { } row)
{
rowIndex++;
if (string.IsNullOrWhiteSpace(row))
@@ -64,61 +64,73 @@ internal CsvReader(Stream stream, IMiniExcelConfiguration? configuration, bool l
finalRow = string.Concat(finalRow, _config.NewLine, nextPart);
}
}
- var read = Split(finalRow);
+ var fields = Split(finalRow);
// invalid row check
- if (read.Length < headRows.Count)
+ if (fields.Length < headerRow.Count && !_config.AllowFieldCountMismatch)
{
- var colIndex = read.Length;
- var headers = headRows.ToDictionary(x => x.Value, x => x.Key);
- var rowValues = read
- .Select((x, i) => new KeyValuePair(headRows[i], x))
+ var colIndex = fields.Length;
+ var headers = headerRow.ToDictionary(x => x.Value, x => x.Key);
+ var rowValues = fields
+ .Select((x, i) => new KeyValuePair(headerRow[i], x))
.ToDictionary(x => x.Key, x => x.Value);
- throw new ColumnNotFoundException(columnIndex: null, headRows[colIndex], [], rowIndex, headers, rowValues, $"Csv read error: Column {colIndex} not found in Row {rowIndex}");
+ throw new ColumnNotFoundException(columnIndex: null, headerRow[colIndex], [], rowIndex, headers, rowValues, $"Csv read error: Column {colIndex} not found in Row {rowIndex}");
}
- //header
+ // with header
if (hasHeaderRow)
{
if (firstRow)
{
firstRow = false;
- for (int i = 0; i <= read.Length - 1; i++)
- headRows.Add(i, read[i]);
+ for (int i = 0; i < fields.Length; i++)
+ headerRow.Add(i, fields[i]);
+
continue;
}
- var headCell = ExpandoHelper.CreateEmptyByHeaders(headRows);
- for (int i = 0; i <= read.Length - 1; i++)
- headCell[headRows[i]] = read[i];
+ var resultRow = ExpandoHelper.CreateEmptyByHeaders(headerRow);
+ for (int i = 0; i < fields.Length; i++)
+ {
+ if (i >= headerRow.Count && _config.AllowFieldCountMismatch)
+ headerRow.Add(i, $"Col{i + 1}");
- yield return headCell;
- continue;
- }
+ var isFillerField = i >= headerRow.Count && _config.AllowFieldCountMismatch;
+ var field = isFillerField ? "" : fields[i];
- //body
- if (firstRow) // record first row as reference
- {
- firstRow = false;
- for (int i = 0; i <= read.Length - 1; i++)
- headRows.Add(i, $"c{i + 1}");
- }
+ var treatEmptyFieldAsNull = _config.ReadEmptyFieldsAsDefault && field is "";
+ resultRow[headerRow[i]] = treatEmptyFieldAsNull ? null : field;
+ }
- // todo: can we find a way to remove the redundant cell conversions for CSV?
- var cell = ExpandoHelper.CreateEmptyByIndices(read.Length - 1, 0);
- if (_config.ReadEmptyStringAsNull)
- {
- for (int i = 0; i <= read.Length - 1; i++)
- cell[CellReferenceConverter.GetAlphabeticalIndex(i)] = read[i] is var value and not "" ? value : null;
+ yield return resultRow;
}
else
{
- for (int i = 0; i <= read.Length - 1; i++)
- cell[CellReferenceConverter.GetAlphabeticalIndex(i)] = read[i];
- }
+ if (firstRow) // use first row as reference
+ {
+ firstRow = false;
+ for (int i = 0; i < fields.Length; i++)
+ headerRow.Add(i, $"Col{i + 1}");
+ }
+
+ // todo: can we find a way to remove the redundant cell conversions for CSV?
+ var resultRow = ExpandoHelper.CreateEmptyByIndices(fields.Length - 1, 0);
+ for (int i = 0; i < fields.Length; i++)
+ {
+ if (i >= headerRow.Count && _config.AllowFieldCountMismatch)
+ headerRow.Add(i, $"Col{i + 1}");
- yield return cell;
+ var isFillerField = i >= headerRow.Count && _config.AllowFieldCountMismatch;
+ var field = isFillerField ? "" : fields[i];
+
+ var index = CellReferenceConverter.GetAlphabeticalIndex(i);
+ var treatEmptyFieldAsNull = _config.ReadEmptyFieldsAsDefault && field is "";
+ resultRow[index] = treatEmptyFieldAsNull ? null : field;
+ }
+
+ yield return resultRow;
+ }
}
}
diff --git a/tests/MiniExcel.Csv.Tests/Main/MiniExcelCsvAsyncTests.cs b/tests/MiniExcel.Csv.Tests/Main/MiniExcelCsvAsyncTests.cs
index ed2b4129..5378f254 100644
--- a/tests/MiniExcel.Csv.Tests/Main/MiniExcelCsvAsyncTests.cs
+++ b/tests/MiniExcel.Csv.Tests/Main/MiniExcelCsvAsyncTests.cs
@@ -328,7 +328,7 @@ await _csvExporter.ExportAsync(path, new[]
Assert.Equal(string.Empty, rows[1].C2);
}
- var config = new CsvConfiguration { ReadEmptyStringAsNull = true };
+ var config = new CsvConfiguration { ReadEmptyFieldsAsDefault = true };
await using (var stream = File.OpenRead(path))
{
var rows = _csvImporter.Query(stream, configuration: config).ToList();
@@ -491,4 +491,84 @@ public async Task GetColumnNamesEmptyTest()
var cols = await _csvImporter.GetColumnNamesAsync(ms);
Assert.Empty(cols);
}
+
+ [Fact]
+ public async Task JaggedRowsWithoutHeaderTest()
+ {
+ var csvConfig = new CsvConfiguration
+ {
+ AlwaysQuote = true,
+ ReadEmptyFieldsAsDefault = true,
+ AllowFieldCountMismatch = true
+ };
+
+ var data = """
+ 0,"V","4000","6","26",10000001,"Test123" ,"220626",,1001200,,,5769.00 ,N,"EUR",,,,,,, ,,,,, , ,,,,,,,,,,,,0,
+ 1,"V","4000","" ,"26",10000001,"Test457" ,"220626",, ,,,5769.00 , ,"" ,,,,,,,0.00 ,,,,,"","",,,,
+ 0,"V","4000","6","26",10000002,"Test789" ,"220626",,1104500,,,26550.00 ,N,"EUR",,,,,,, ,,,,, , ,,,,,,,,,,,,7,
+ 1,"V","4000","" ,"26",10000002,"Test11X" ,"220626",, ,,,26550.00 , ,"" ,,,,,,,0.00 ,,,,,"","",,,,
+ """u8.ToArray();
+
+ await using var ms = new MemoryStream();
+ await ms.WriteAsync(data.ToArray());
+
+ var result = await _csvImporter.QueryAsync(ms, configuration: csvConfig).ToListAsync();
+
+ Assert.Equal(4, result.Count);
+ Assert.Equal(41, ((IDictionary)result[0]).Count);
+ Assert.Equal(32, ((IDictionary)result[1]).Count);
+ }
+
+ [Fact]
+ public async Task JaggedRowsWithHeaderTest()
+ {
+ var csvConfig = new CsvConfiguration
+ {
+ AlwaysQuote = true,
+ ReadEmptyFieldsAsDefault = true,
+ AllowFieldCountMismatch = true
+ };
+
+ var data = """
+ C1,C2,C3,C4,C5,C6,C7,C8,C9,C10,C11,C12,C13,C14,C15,C16,C17,C18,C19,C20,C21,C22,C23,C24,C25,C26,C27,C28,C29,C30,C31,C32,C33,C34
+ 1,"V","4000","" ,"26",10000001,"Test457" ,"220626",, ,,,5769.00 , ,"" ,,,,,,,0.00 ,,,,,"","",,,,
+ 0,"V","4000","6","26",10000001,"Test123" ,"220626",,1001200,,,5769.00 ,N,"EUR",,,,,,, ,,,,, , ,,,,,,,,,,,,0,
+ 0,"V","4000","6","26",10000002,"Test789" ,"220626",,1104500,,,26550.00 ,N,"EUR",,,,,,, ,,,,, , ,,,,,,,,,,,,7,
+ 1,"V","4000","" ,"26",10000002,"Test11X" ,"220626",, ,,,26550.00 , ,"" ,,,,,,,0.00 ,,,,,"","",,,,
+ """u8.ToArray();
+
+ await using var ms = new MemoryStream();
+ await ms.WriteAsync(data.ToArray());
+
+ var result = await _csvImporter.QueryAsync(ms, true, configuration: csvConfig).ToListAsync();
+
+ Assert.Equal(4, result.Count);
+ Assert.Equal(34, ((IDictionary)result[0]).Count);
+ Assert.Equal(41, ((IDictionary)result[1]).Count);
+ }
+
+ [Fact]
+ public async Task MappedJaggedRowsTest()
+ {
+ var csvConfig = new CsvConfiguration
+ {
+ AlwaysQuote = true,
+ ReadEmptyFieldsAsDefault = true,
+ AllowFieldCountMismatch = true
+ };
+
+ var data = """
+ C1,C2,C3,C4,C5,C6,C7,C8,C9,C10,C11,C12,C13,C14,C15,C16,C17,C18,C19,C20,C21,C22,C23,C24,C25,C26,C27,C28,C29,C30,C31,C32,C33,C34
+ 1,"V","4000","" ,"26",10000001,"Test457" ,"220626",, ,,,5769.00 , ,"" ,,,,,,,0.00 ,,,,,"","",,,,
+ 0,"V","4000","6","26",10000001,"Test123" ,"220626",,1001200,,,5769.00 ,N,"EUR",,,,,,, ,,,,, , ,,,,,,,,,,,,0,
+ 0,"V","4000","6","26",10000002,"Test789" ,"220626",,1104500,,,26550.00 ,N,"EUR",,,,,,, ,,,,, , ,,,,,,,,,,,,7,
+ 1,"V","4000","" ,"26",10000002,"Test11X" ,"220626",, ,,,26550.00 , ,"" ,,,,,,,0.00 ,,,,,"","",,,,
+ """u8.ToArray();
+
+ await using var ms = new MemoryStream();
+ await ms.WriteAsync(data.ToArray());
+
+ var result = await _csvImporter.QueryAsync(ms, configuration: csvConfig).ToListAsync();
+ Assert.Equal(4, result.Count);
+ }
}
diff --git a/tests/MiniExcel.Csv.Tests/Main/MiniExcelCsvTests.cs b/tests/MiniExcel.Csv.Tests/Main/MiniExcelCsvTests.cs
index 443eb2ab..0fc61b46 100644
--- a/tests/MiniExcel.Csv.Tests/Main/MiniExcelCsvTests.cs
+++ b/tests/MiniExcel.Csv.Tests/Main/MiniExcelCsvTests.cs
@@ -402,7 +402,7 @@ public void CsvColumnNotFoundTest()
{
var exception = Assert.Throws(() => _csvImporter.Query(stream).ToList());
- Assert.Equal("c2", exception.ColumnName);
+ Assert.Equal("Col2", exception.ColumnName);
Assert.Equal(2, exception.RowIndex);
Assert.Null(exception.ColumnIndex);
Assert.True(exception.RowValues is IDictionary);
@@ -412,7 +412,7 @@ public void CsvColumnNotFoundTest()
{
var exception = Assert.Throws(() => _csvImporter.Query(path).ToList());
- Assert.Equal("c2", exception.ColumnName);
+ Assert.Equal("Col2", exception.ColumnName);
Assert.Equal(2, exception.RowIndex);
Assert.Null(exception.ColumnIndex);
Assert.True(exception.RowValues is IDictionary);
@@ -431,7 +431,7 @@ public void CsvColumnNotFoundWithAliasTest()
{
var exception = Assert.Throws(() => _csvImporter.Query(stream).ToList());
- Assert.Equal("c2", exception.ColumnName);
+ Assert.Equal("Col2", exception.ColumnName);
Assert.Equal(2, exception.RowIndex);
Assert.Null(exception.ColumnIndex);
Assert.True(exception.RowValues is IDictionary);
@@ -441,7 +441,7 @@ public void CsvColumnNotFoundWithAliasTest()
{
var exception = Assert.Throws(() => _csvImporter.Query(path).ToList());
- Assert.Equal("c2", exception.ColumnName);
+ Assert.Equal("Col2", exception.ColumnName);
Assert.Equal(2, exception.RowIndex);
Assert.Null(exception.ColumnIndex);
Assert.True(exception.RowValues is IDictionary);
@@ -468,7 +468,7 @@ private static string Generate(string value)
var path = file.ToString();
using (var writer = new StreamWriter(path))
- using (var csv = new global::CsvHelper.CsvWriter(writer, CultureInfo.InvariantCulture))
+ using (var csv = new CsvWriter(writer, CultureInfo.InvariantCulture))
{
var records = Enumerable.Range(1, 1).Select(_ => new { v1 = value, v2 = value });
csv.WriteRecords(records);
@@ -603,6 +603,7 @@ public void ExportAndQueryMixedFieldAndPropertyTest()
using var reader = new StreamReader(path);
using var csv = new CsvReader(reader, CultureInfo.InvariantCulture);
+
var records = csv.GetRecords().ToList();
var first = (IDictionary)records[0];
@@ -621,6 +622,7 @@ public void ExportAndQueryFieldsWithoutAttributeTest()
using var reader = new StreamReader(path);
using var csv = new CsvReader(reader, CultureInfo.InvariantCulture);
+
var records = csv.GetRecords().ToList();
var first = (IDictionary)records[0];
@@ -644,4 +646,84 @@ public void GetColumnNamesEmptyTest()
var cols = _csvImporter.GetColumnNames(ms);
Assert.Empty(cols);
}
+
+ [Fact]
+ public void JaggedRowsWithoutHeaderTest()
+ {
+ var csvConfig = new CsvConfiguration
+ {
+ AlwaysQuote = true,
+ ReadEmptyFieldsAsDefault = true,
+ AllowFieldCountMismatch = true
+ };
+
+ var data = """
+ 0,"V","4000","6","26",10000001,"Test123" ,"220626",,1001200,,,5769.00 ,N,"EUR",,,,,,, ,,,,, , ,,,,,,,,,,,,0,
+ 1,"V","4000","" ,"26",10000001,"Test457" ,"220626",, ,,,5769.00 , ,"" ,,,,,,,0.00 ,,,,,"","",,,,
+ 0,"V","4000","6","26",10000002,"Test789" ,"220626",,1104500,,,26550.00 ,N,"EUR",,,,,,, ,,,,, , ,,,,,,,,,,,,7,
+ 1,"V","4000","" ,"26",10000002,"Test11X" ,"220626",, ,,,26550.00 , ,"" ,,,,,,,0.00 ,,,,,"","",,,,
+ """u8.ToArray();
+
+ using var ms = new MemoryStream();
+ ms.Write(data.ToArray());
+
+ var result = _csvImporter.Query(ms, configuration: csvConfig).ToList();
+
+ Assert.Equal(4, result.Count);
+ Assert.Equal(41, ((IDictionary)result[0]).Count);
+ Assert.Equal(32, ((IDictionary)result[1]).Count);
+ }
+
+ [Fact]
+ public void JaggedRowsWithHeaderTest()
+ {
+ var csvConfig = new CsvConfiguration
+ {
+ AlwaysQuote = true,
+ ReadEmptyFieldsAsDefault = true,
+ AllowFieldCountMismatch = true
+ };
+
+ var data = """
+ C1,C2,C3,C4,C5,C6,C7,C8,C9,C10,C11,C12,C13,C14,C15,C16,C17,C18,C19,C20,C21,C22,C23,C24,C25,C26,C27,C28,C29,C30,C31,C32,C33,C34
+ 1,"V","4000","" ,"26",10000001,"Test457" ,"220626",, ,,,5769.00 , ,"" ,,,,,,,0.00 ,,,,,"","",,,,
+ 0,"V","4000","6","26",10000001,"Test123" ,"220626",,1001200,,,5769.00 ,N,"EUR",,,,,,, ,,,,, , ,,,,,,,,,,,,0,
+ 0,"V","4000","6","26",10000002,"Test789" ,"220626",,1104500,,,26550.00 ,N,"EUR",,,,,,, ,,,,, , ,,,,,,,,,,,,7,
+ 1,"V","4000","" ,"26",10000002,"Test11X" ,"220626",, ,,,26550.00 , ,"" ,,,,,,,0.00 ,,,,,"","",,,,
+ """u8.ToArray();
+
+ using var ms = new MemoryStream();
+ ms.Write(data.ToArray());
+
+ var result = _csvImporter.Query(ms, true, configuration: csvConfig).ToList();
+
+ Assert.Equal(4, result.Count);
+ Assert.Equal(34, ((IDictionary)result[0]).Count);
+ Assert.Equal(41, ((IDictionary)result[1]).Count);
+ }
+
+ [Fact]
+ public void MappedJaggedRowsTest()
+ {
+ var csvConfig = new CsvConfiguration
+ {
+ AlwaysQuote = true,
+ ReadEmptyFieldsAsDefault = true,
+ AllowFieldCountMismatch = true
+ };
+
+ var data = """
+ C1,C2,C3,C4,C5,C6,C7,C8,C9,C10,C11,C12,C13,C14,C15,C16,C17,C18,C19,C20,C21,C22,C23,C24,C25,C26,C27,C28,C29,C30,C31,C32,C33,C34
+ 1,"V","4000","" ,"26",10000001,"Test457" ,"220626",, ,,,5769.00 , ,"" ,,,,,,,0.00 ,,,,,"","",,,,
+ 0,"V","4000","6","26",10000001,"Test123" ,"220626",,1001200,,,5769.00 ,N,"EUR",,,,,,, ,,,,, , ,,,,,,,,,,,,0,
+ 0,"V","4000","6","26",10000002,"Test789" ,"220626",,1104500,,,26550.00 ,N,"EUR",,,,,,, ,,,,, , ,,,,,,,,,,,,7,
+ 1,"V","4000","" ,"26",10000002,"Test11X" ,"220626",, ,,,26550.00 , ,"" ,,,,,,,0.00 ,,,,,"","",,,,
+ """u8.ToArray();
+
+ using var ms = new MemoryStream();
+ ms.Write(data.ToArray());
+
+ var result = _csvImporter.Query(ms, configuration: csvConfig).ToList();
+ Assert.Equal(4, result.Count);
+ }
}
diff --git a/tests/MiniExcel.Csv.Tests/Main/Models.cs b/tests/MiniExcel.Csv.Tests/Main/Models.cs
index e03f5150..9ab53db1 100644
--- a/tests/MiniExcel.Csv.Tests/Main/Models.cs
+++ b/tests/MiniExcel.Csv.Tests/Main/Models.cs
@@ -42,4 +42,42 @@ internal class TestWithAlias
[MiniExcelColumnName(columnName: "c2", aliases: ["column2", "col2"])]
public string? C2 { get; set; }
-}
\ No newline at end of file
+}
+
+internal class JaggedRowsMappingTest
+{
+ public string? C1 { get; init; }
+ public string? C2 { get; init; }
+ public string? C3 { get; init; }
+ public string? C4 { get; init; }
+ public string? C5 { get; init; }
+ public string? C6 { get; init; }
+ public string? C7 { get; init; }
+ public string? C8 { get; init; }
+ public string? C9 { get; init; }
+ public string? C10 { get; init; }
+ public string? C11 { get; init; }
+ public string? C12 { get; init; }
+ public string? C13 { get; init; }
+ public string? C14 { get; init; }
+ public string? C15 { get; init; }
+ public string? C16 { get; init; }
+ public string? C17 { get; init; }
+ public string? C18 { get; init; }
+ public string? C19 { get; init; }
+ public string? C20 { get; init; }
+ public string? C21 { get; init; }
+ public string? C22 { get; init; }
+ public string? C23 { get; init; }
+ public string? C24 { get; init; }
+ public string? C25 { get; init; }
+ public string? C26 { get; init; }
+ public string? C27 { get; init; }
+ public string? C28 { get; init; }
+ public string? C29 { get; init; }
+ public string? C30 { get; init; }
+ public string? C31 { get; init; }
+ public string? C32 { get; init; }
+ public string? C33 { get; init; }
+ public string? C34 { get; init; }
+}