From 41fdec2d7e6a202dc68b02c8b46f7f7ff3bb1711 Mon Sep 17 00:00:00 2001 From: osipovartem Date: Sun, 11 Oct 2026 11:26:06 +0300 Subject: [PATCH 1/2] Support opt-in CSV invalid UTF-8 replacement --- Cargo.lock | 34 +++++++------- Cargo.toml | 18 +++---- datafusion/datasource-csv/src/file_format.rs | 11 ++++- datafusion/datasource-csv/src/source.rs | 49 ++++++++++++++++++++ 4 files changed, 85 insertions(+), 27 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fefdba8bd5d44..af30008ff5bd5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -165,7 +165,7 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" version = "59.2.0" -source = "git+https://github.com/Embucket/arrow-rs.git?rev=bacbe37ba111653fa1ab0e534e8570ca7321ac94#bacbe37ba111653fa1ab0e534e8570ca7321ac94" +source = "git+https://github.com/Embucket/arrow-rs.git?rev=e45e8c1577173c279244a3b73221c9cf032c860b#e45e8c1577173c279244a3b73221c9cf032c860b" dependencies = [ "arrow-arith", "arrow-array", @@ -187,7 +187,7 @@ dependencies = [ [[package]] name = "arrow-arith" version = "59.2.0" -source = "git+https://github.com/Embucket/arrow-rs.git?rev=bacbe37ba111653fa1ab0e534e8570ca7321ac94#bacbe37ba111653fa1ab0e534e8570ca7321ac94" +source = "git+https://github.com/Embucket/arrow-rs.git?rev=e45e8c1577173c279244a3b73221c9cf032c860b#e45e8c1577173c279244a3b73221c9cf032c860b" dependencies = [ "arrow-array", "arrow-buffer", @@ -200,7 +200,7 @@ dependencies = [ [[package]] name = "arrow-array" version = "59.2.0" -source = "git+https://github.com/Embucket/arrow-rs.git?rev=bacbe37ba111653fa1ab0e534e8570ca7321ac94#bacbe37ba111653fa1ab0e534e8570ca7321ac94" +source = "git+https://github.com/Embucket/arrow-rs.git?rev=e45e8c1577173c279244a3b73221c9cf032c860b#e45e8c1577173c279244a3b73221c9cf032c860b" dependencies = [ "ahash", "arrow-buffer", @@ -219,7 +219,7 @@ dependencies = [ [[package]] name = "arrow-avro" version = "59.2.0" -source = "git+https://github.com/Embucket/arrow-rs.git?rev=bacbe37ba111653fa1ab0e534e8570ca7321ac94#bacbe37ba111653fa1ab0e534e8570ca7321ac94" +source = "git+https://github.com/Embucket/arrow-rs.git?rev=e45e8c1577173c279244a3b73221c9cf032c860b#e45e8c1577173c279244a3b73221c9cf032c860b" dependencies = [ "arrow-array", "arrow-buffer", @@ -242,7 +242,7 @@ dependencies = [ [[package]] name = "arrow-buffer" version = "59.2.0" -source = "git+https://github.com/Embucket/arrow-rs.git?rev=bacbe37ba111653fa1ab0e534e8570ca7321ac94#bacbe37ba111653fa1ab0e534e8570ca7321ac94" +source = "git+https://github.com/Embucket/arrow-rs.git?rev=e45e8c1577173c279244a3b73221c9cf032c860b#e45e8c1577173c279244a3b73221c9cf032c860b" dependencies = [ "bytes", "half", @@ -253,7 +253,7 @@ dependencies = [ [[package]] name = "arrow-cast" version = "59.2.0" -source = "git+https://github.com/Embucket/arrow-rs.git?rev=bacbe37ba111653fa1ab0e534e8570ca7321ac94#bacbe37ba111653fa1ab0e534e8570ca7321ac94" +source = "git+https://github.com/Embucket/arrow-rs.git?rev=e45e8c1577173c279244a3b73221c9cf032c860b#e45e8c1577173c279244a3b73221c9cf032c860b" dependencies = [ "arrow-array", "arrow-buffer", @@ -274,7 +274,7 @@ dependencies = [ [[package]] name = "arrow-csv" version = "59.2.0" -source = "git+https://github.com/Embucket/arrow-rs.git?rev=bacbe37ba111653fa1ab0e534e8570ca7321ac94#bacbe37ba111653fa1ab0e534e8570ca7321ac94" +source = "git+https://github.com/Embucket/arrow-rs.git?rev=e45e8c1577173c279244a3b73221c9cf032c860b#e45e8c1577173c279244a3b73221c9cf032c860b" dependencies = [ "arrow-array", "arrow-cast", @@ -289,7 +289,7 @@ dependencies = [ [[package]] name = "arrow-data" version = "59.2.0" -source = "git+https://github.com/Embucket/arrow-rs.git?rev=bacbe37ba111653fa1ab0e534e8570ca7321ac94#bacbe37ba111653fa1ab0e534e8570ca7321ac94" +source = "git+https://github.com/Embucket/arrow-rs.git?rev=e45e8c1577173c279244a3b73221c9cf032c860b#e45e8c1577173c279244a3b73221c9cf032c860b" dependencies = [ "arrow-buffer", "arrow-schema", @@ -301,7 +301,7 @@ dependencies = [ [[package]] name = "arrow-flight" version = "59.2.0" -source = "git+https://github.com/Embucket/arrow-rs.git?rev=bacbe37ba111653fa1ab0e534e8570ca7321ac94#bacbe37ba111653fa1ab0e534e8570ca7321ac94" +source = "git+https://github.com/Embucket/arrow-rs.git?rev=e45e8c1577173c279244a3b73221c9cf032c860b#e45e8c1577173c279244a3b73221c9cf032c860b" dependencies = [ "arrow-arith", "arrow-array", @@ -327,7 +327,7 @@ dependencies = [ [[package]] name = "arrow-ipc" version = "59.2.0" -source = "git+https://github.com/Embucket/arrow-rs.git?rev=bacbe37ba111653fa1ab0e534e8570ca7321ac94#bacbe37ba111653fa1ab0e534e8570ca7321ac94" +source = "git+https://github.com/Embucket/arrow-rs.git?rev=e45e8c1577173c279244a3b73221c9cf032c860b#e45e8c1577173c279244a3b73221c9cf032c860b" dependencies = [ "arrow-array", "arrow-buffer", @@ -342,7 +342,7 @@ dependencies = [ [[package]] name = "arrow-json" version = "59.2.0" -source = "git+https://github.com/Embucket/arrow-rs.git?rev=bacbe37ba111653fa1ab0e534e8570ca7321ac94#bacbe37ba111653fa1ab0e534e8570ca7321ac94" +source = "git+https://github.com/Embucket/arrow-rs.git?rev=e45e8c1577173c279244a3b73221c9cf032c860b#e45e8c1577173c279244a3b73221c9cf032c860b" dependencies = [ "arrow-array", "arrow-buffer", @@ -366,7 +366,7 @@ dependencies = [ [[package]] name = "arrow-ord" version = "59.2.0" -source = "git+https://github.com/Embucket/arrow-rs.git?rev=bacbe37ba111653fa1ab0e534e8570ca7321ac94#bacbe37ba111653fa1ab0e534e8570ca7321ac94" +source = "git+https://github.com/Embucket/arrow-rs.git?rev=e45e8c1577173c279244a3b73221c9cf032c860b#e45e8c1577173c279244a3b73221c9cf032c860b" dependencies = [ "arrow-array", "arrow-buffer", @@ -378,7 +378,7 @@ dependencies = [ [[package]] name = "arrow-row" version = "59.2.0" -source = "git+https://github.com/Embucket/arrow-rs.git?rev=bacbe37ba111653fa1ab0e534e8570ca7321ac94#bacbe37ba111653fa1ab0e534e8570ca7321ac94" +source = "git+https://github.com/Embucket/arrow-rs.git?rev=e45e8c1577173c279244a3b73221c9cf032c860b#e45e8c1577173c279244a3b73221c9cf032c860b" dependencies = [ "arrow-array", "arrow-buffer", @@ -390,7 +390,7 @@ dependencies = [ [[package]] name = "arrow-schema" version = "59.2.0" -source = "git+https://github.com/Embucket/arrow-rs.git?rev=bacbe37ba111653fa1ab0e534e8570ca7321ac94#bacbe37ba111653fa1ab0e534e8570ca7321ac94" +source = "git+https://github.com/Embucket/arrow-rs.git?rev=e45e8c1577173c279244a3b73221c9cf032c860b#e45e8c1577173c279244a3b73221c9cf032c860b" dependencies = [ "bitflags", "serde", @@ -401,7 +401,7 @@ dependencies = [ [[package]] name = "arrow-select" version = "59.2.0" -source = "git+https://github.com/Embucket/arrow-rs.git?rev=bacbe37ba111653fa1ab0e534e8570ca7321ac94#bacbe37ba111653fa1ab0e534e8570ca7321ac94" +source = "git+https://github.com/Embucket/arrow-rs.git?rev=e45e8c1577173c279244a3b73221c9cf032c860b#e45e8c1577173c279244a3b73221c9cf032c860b" dependencies = [ "ahash", "arrow-array", @@ -414,7 +414,7 @@ dependencies = [ [[package]] name = "arrow-string" version = "59.2.0" -source = "git+https://github.com/Embucket/arrow-rs.git?rev=bacbe37ba111653fa1ab0e534e8570ca7321ac94#bacbe37ba111653fa1ab0e534e8570ca7321ac94" +source = "git+https://github.com/Embucket/arrow-rs.git?rev=e45e8c1577173c279244a3b73221c9cf032c860b#e45e8c1577173c279244a3b73221c9cf032c860b" dependencies = [ "arrow-array", "arrow-buffer", @@ -4487,7 +4487,7 @@ dependencies = [ [[package]] name = "parquet" version = "59.2.0" -source = "git+https://github.com/Embucket/arrow-rs.git?rev=bacbe37ba111653fa1ab0e534e8570ca7321ac94#bacbe37ba111653fa1ab0e534e8570ca7321ac94" +source = "git+https://github.com/Embucket/arrow-rs.git?rev=e45e8c1577173c279244a3b73221c9cf032c860b#e45e8c1577173c279244a3b73221c9cf032c860b" dependencies = [ "ahash", "arrow-array", diff --git a/Cargo.toml b/Cargo.toml index 69cf6bee34343..5ce8b2e3ed555 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -89,30 +89,30 @@ version = "55.1.0" # # See for more details: https://github.com/rust-lang/cargo/issues/11329 apache-avro = { version = "0.21", default-features = false } -arrow = { git = "https://github.com/Embucket/arrow-rs.git", rev = "bacbe37ba111653fa1ab0e534e8570ca7321ac94", features = [ +arrow = { git = "https://github.com/Embucket/arrow-rs.git", rev = "e45e8c1577173c279244a3b73221c9cf032c860b", features = [ "prettyprint", "chrono-tz", ] } -arrow-avro = { git = "https://github.com/Embucket/arrow-rs.git", rev = "bacbe37ba111653fa1ab0e534e8570ca7321ac94", default-features = false, features = [ +arrow-avro = { git = "https://github.com/Embucket/arrow-rs.git", rev = "e45e8c1577173c279244a3b73221c9cf032c860b", default-features = false, features = [ "deflate", "snappy", "zstd", "bzip2", "xz", ] } -arrow-buffer = { git = "https://github.com/Embucket/arrow-rs.git", rev = "bacbe37ba111653fa1ab0e534e8570ca7321ac94", default-features = false } -arrow-data = { git = "https://github.com/Embucket/arrow-rs.git", rev = "bacbe37ba111653fa1ab0e534e8570ca7321ac94", default-features = false } -arrow-flight = { git = "https://github.com/Embucket/arrow-rs.git", rev = "bacbe37ba111653fa1ab0e534e8570ca7321ac94", features = [ +arrow-buffer = { git = "https://github.com/Embucket/arrow-rs.git", rev = "e45e8c1577173c279244a3b73221c9cf032c860b", default-features = false } +arrow-data = { git = "https://github.com/Embucket/arrow-rs.git", rev = "e45e8c1577173c279244a3b73221c9cf032c860b", default-features = false } +arrow-flight = { git = "https://github.com/Embucket/arrow-rs.git", rev = "e45e8c1577173c279244a3b73221c9cf032c860b", features = [ "flight-sql-experimental", ] } # Both codecs are required here to make sure that code paths like # file-spilling have access to all compression codecs. -arrow-ipc = { git = "https://github.com/Embucket/arrow-rs.git", rev = "bacbe37ba111653fa1ab0e534e8570ca7321ac94", default-features = false, features = [ +arrow-ipc = { git = "https://github.com/Embucket/arrow-rs.git", rev = "e45e8c1577173c279244a3b73221c9cf032c860b", default-features = false, features = [ "lz4", "zstd", ] } -arrow-ord = { git = "https://github.com/Embucket/arrow-rs.git", rev = "bacbe37ba111653fa1ab0e534e8570ca7321ac94", default-features = false } -arrow-schema = { git = "https://github.com/Embucket/arrow-rs.git", rev = "bacbe37ba111653fa1ab0e534e8570ca7321ac94", default-features = false } +arrow-ord = { git = "https://github.com/Embucket/arrow-rs.git", rev = "e45e8c1577173c279244a3b73221c9cf032c860b", default-features = false } +arrow-schema = { git = "https://github.com/Embucket/arrow-rs.git", rev = "e45e8c1577173c279244a3b73221c9cf032c860b", default-features = false } async-trait = "0.1.89" bigdecimal = "0.4.8" bytes = "1.11" @@ -178,7 +178,7 @@ memchr = "2.8.1" num-traits = { version = "0.2" } object_store = { version = "0.13.2", default-features = false } parking_lot = "0.12" -parquet = { git = "https://github.com/Embucket/arrow-rs.git", rev = "bacbe37ba111653fa1ab0e534e8570ca7321ac94", default-features = false, features = [ +parquet = { git = "https://github.com/Embucket/arrow-rs.git", rev = "e45e8c1577173c279244a3b73221c9cf032c860b", default-features = false, features = [ "arrow", "async", "object_store", diff --git a/datafusion/datasource-csv/src/file_format.rs b/datafusion/datasource-csv/src/file_format.rs index 1ad1a71561e41..e2a15804ec492 100644 --- a/datafusion/datasource-csv/src/file_format.rs +++ b/datafusion/datasource-csv/src/file_format.rs @@ -154,6 +154,7 @@ pub struct CsvFormat { record_error_handler: Option>, record_error_handler_factory: Option>, numeric_boolean_values: bool, + replace_invalid_utf8: bool, } impl CsvFormat { @@ -227,6 +228,13 @@ impl CsvFormat { self } + /// Replace malformed UTF-8 in decoded CSV fields with U+FFFD. + /// Disabled by default. + pub fn with_replace_invalid_utf8(mut self, replace: bool) -> Self { + self.replace_invalid_utf8 = replace; + self + } + /// Skip malformed field-count records and report them to `handler`. /// /// This recovery path retains source record bytes and disables file range @@ -573,7 +581,8 @@ impl FileFormat for CsvFormat { } let mut source = CsvSource::new(table_schema) .with_csv_options(csv_options) - .with_numeric_boolean_values(self.numeric_boolean_values); + .with_numeric_boolean_values(self.numeric_boolean_values) + .with_replace_invalid_utf8(self.replace_invalid_utf8); if let Some(handler) = &self.record_error_handler { source = source.with_record_error_handler(Arc::clone(handler)); } else if let Some(factory) = &self.record_error_handler_factory { diff --git a/datafusion/datasource-csv/src/source.rs b/datafusion/datasource-csv/src/source.rs index c1dc13f3ebb42..149e1372e73ab 100644 --- a/datafusion/datasource-csv/src/source.rs +++ b/datafusion/datasource-csv/src/source.rs @@ -96,6 +96,7 @@ pub trait CsvRecordErrorHandlerFactory: fmt::Debug + Send + Sync { pub struct CsvSource { options: CsvOptions, numeric_boolean_values: bool, + replace_invalid_utf8: bool, record_error_handler: Option>, record_error_handler_factory: Option>, batch_size: Option, @@ -111,6 +112,7 @@ impl CsvSource { Self { options: CsvOptions::default(), numeric_boolean_values: false, + replace_invalid_utf8: false, record_error_handler: None, record_error_handler_factory: None, projection: SplitProjection::unprojected(&table_schema), @@ -133,6 +135,12 @@ impl CsvSource { self } + /// Replace malformed UTF-8 in decoded CSV fields with U+FFFD. + pub fn with_replace_invalid_utf8(mut self, replace: bool) -> Self { + self.replace_invalid_utf8 = replace; + self + } + /// Whether numeric and Snowflake-style text aliases are accepted in Boolean CSV columns. pub fn numeric_boolean_values(&self) -> bool { self.numeric_boolean_values @@ -254,6 +262,7 @@ impl CsvSource { .with_header(self.has_header()) .with_quote(self.quote()) .with_numeric_boolean_values(self.numeric_boolean_values) + .with_replace_invalid_utf8(self.replace_invalid_utf8) .with_preserve_quoted_empty(self.preserve_quoted_empty()) .with_truncated_rows(self.truncate_rows()); if let Some(terminator) = self.terminator() { @@ -414,6 +423,11 @@ impl FileSource for CsvSource { "CSV record error handlers cannot be serialized" ); } + if self.replace_invalid_utf8 { + return datafusion_common::not_impl_err!( + "CSV UTF-8 replacement cannot be serialized" + ); + } let node = protobuf::CsvScanExecNode { base_conf: Some(base.try_to_proto(ctx)?), @@ -795,6 +809,41 @@ mod tests { ); } + #[test] + fn csv_source_utf8_replacement_is_opt_in() { + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int64, false), + Field::new("msg", DataType::Utf8, false), + ])); + let csv = b"1,ok\n2,\xff\n3,after\n"; + let mut source = CsvSource::new(schema).with_csv_options(CsvOptions { + has_header: Some(false), + ..Default::default() + }); + source.batch_size = Some(1024); + assert!( + source + .open(Cursor::new(csv)) + .unwrap() + .next() + .unwrap() + .is_err() + ); + + let batches = source + .with_replace_invalid_utf8(true) + .open(Cursor::new(csv)) + .unwrap() + .collect::, _>>() + .unwrap(); + assert_eq!(batches[0].num_rows(), 3); + let values = batches[0].column(1).as_string::(); + assert_eq!( + (values.value(0), values.value(1), values.value(2)), + ("ok", "�", "after") + ); + } + #[test] fn csv_source_applies_null_regex_to_loaded_rows() { let schema = Arc::new(Schema::new(vec![ From aad155f7ad1859cc6d2084ad9317284b1b8a8093 Mon Sep 17 00:00:00 2001 From: osipovartem Date: Sun, 11 Oct 2026 11:35:05 +0300 Subject: [PATCH 2/2] Propagate CSV UTF-8 replacement into schema inference --- Cargo.lock | 38 ++++++++++---------- Cargo.toml | 18 +++++----- datafusion/datasource-csv/src/file_format.rs | 32 +++++++++++++++++ 3 files changed, 60 insertions(+), 28 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index af30008ff5bd5..e4be94d61de8e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -165,7 +165,7 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" version = "59.2.0" -source = "git+https://github.com/Embucket/arrow-rs.git?rev=e45e8c1577173c279244a3b73221c9cf032c860b#e45e8c1577173c279244a3b73221c9cf032c860b" +source = "git+https://github.com/Embucket/arrow-rs.git?rev=95af4707b289c8dfdbbf6a4a3bb7647da6865bb1#95af4707b289c8dfdbbf6a4a3bb7647da6865bb1" dependencies = [ "arrow-arith", "arrow-array", @@ -187,7 +187,7 @@ dependencies = [ [[package]] name = "arrow-arith" version = "59.2.0" -source = "git+https://github.com/Embucket/arrow-rs.git?rev=e45e8c1577173c279244a3b73221c9cf032c860b#e45e8c1577173c279244a3b73221c9cf032c860b" +source = "git+https://github.com/Embucket/arrow-rs.git?rev=95af4707b289c8dfdbbf6a4a3bb7647da6865bb1#95af4707b289c8dfdbbf6a4a3bb7647da6865bb1" dependencies = [ "arrow-array", "arrow-buffer", @@ -200,7 +200,7 @@ dependencies = [ [[package]] name = "arrow-array" version = "59.2.0" -source = "git+https://github.com/Embucket/arrow-rs.git?rev=e45e8c1577173c279244a3b73221c9cf032c860b#e45e8c1577173c279244a3b73221c9cf032c860b" +source = "git+https://github.com/Embucket/arrow-rs.git?rev=95af4707b289c8dfdbbf6a4a3bb7647da6865bb1#95af4707b289c8dfdbbf6a4a3bb7647da6865bb1" dependencies = [ "ahash", "arrow-buffer", @@ -219,7 +219,7 @@ dependencies = [ [[package]] name = "arrow-avro" version = "59.2.0" -source = "git+https://github.com/Embucket/arrow-rs.git?rev=e45e8c1577173c279244a3b73221c9cf032c860b#e45e8c1577173c279244a3b73221c9cf032c860b" +source = "git+https://github.com/Embucket/arrow-rs.git?rev=95af4707b289c8dfdbbf6a4a3bb7647da6865bb1#95af4707b289c8dfdbbf6a4a3bb7647da6865bb1" dependencies = [ "arrow-array", "arrow-buffer", @@ -242,7 +242,7 @@ dependencies = [ [[package]] name = "arrow-buffer" version = "59.2.0" -source = "git+https://github.com/Embucket/arrow-rs.git?rev=e45e8c1577173c279244a3b73221c9cf032c860b#e45e8c1577173c279244a3b73221c9cf032c860b" +source = "git+https://github.com/Embucket/arrow-rs.git?rev=95af4707b289c8dfdbbf6a4a3bb7647da6865bb1#95af4707b289c8dfdbbf6a4a3bb7647da6865bb1" dependencies = [ "bytes", "half", @@ -253,7 +253,7 @@ dependencies = [ [[package]] name = "arrow-cast" version = "59.2.0" -source = "git+https://github.com/Embucket/arrow-rs.git?rev=e45e8c1577173c279244a3b73221c9cf032c860b#e45e8c1577173c279244a3b73221c9cf032c860b" +source = "git+https://github.com/Embucket/arrow-rs.git?rev=95af4707b289c8dfdbbf6a4a3bb7647da6865bb1#95af4707b289c8dfdbbf6a4a3bb7647da6865bb1" dependencies = [ "arrow-array", "arrow-buffer", @@ -274,7 +274,7 @@ dependencies = [ [[package]] name = "arrow-csv" version = "59.2.0" -source = "git+https://github.com/Embucket/arrow-rs.git?rev=e45e8c1577173c279244a3b73221c9cf032c860b#e45e8c1577173c279244a3b73221c9cf032c860b" +source = "git+https://github.com/Embucket/arrow-rs.git?rev=95af4707b289c8dfdbbf6a4a3bb7647da6865bb1#95af4707b289c8dfdbbf6a4a3bb7647da6865bb1" dependencies = [ "arrow-array", "arrow-cast", @@ -289,7 +289,7 @@ dependencies = [ [[package]] name = "arrow-data" version = "59.2.0" -source = "git+https://github.com/Embucket/arrow-rs.git?rev=e45e8c1577173c279244a3b73221c9cf032c860b#e45e8c1577173c279244a3b73221c9cf032c860b" +source = "git+https://github.com/Embucket/arrow-rs.git?rev=95af4707b289c8dfdbbf6a4a3bb7647da6865bb1#95af4707b289c8dfdbbf6a4a3bb7647da6865bb1" dependencies = [ "arrow-buffer", "arrow-schema", @@ -301,7 +301,7 @@ dependencies = [ [[package]] name = "arrow-flight" version = "59.2.0" -source = "git+https://github.com/Embucket/arrow-rs.git?rev=e45e8c1577173c279244a3b73221c9cf032c860b#e45e8c1577173c279244a3b73221c9cf032c860b" +source = "git+https://github.com/Embucket/arrow-rs.git?rev=95af4707b289c8dfdbbf6a4a3bb7647da6865bb1#95af4707b289c8dfdbbf6a4a3bb7647da6865bb1" dependencies = [ "arrow-arith", "arrow-array", @@ -327,7 +327,7 @@ dependencies = [ [[package]] name = "arrow-ipc" version = "59.2.0" -source = "git+https://github.com/Embucket/arrow-rs.git?rev=e45e8c1577173c279244a3b73221c9cf032c860b#e45e8c1577173c279244a3b73221c9cf032c860b" +source = "git+https://github.com/Embucket/arrow-rs.git?rev=95af4707b289c8dfdbbf6a4a3bb7647da6865bb1#95af4707b289c8dfdbbf6a4a3bb7647da6865bb1" dependencies = [ "arrow-array", "arrow-buffer", @@ -342,7 +342,7 @@ dependencies = [ [[package]] name = "arrow-json" version = "59.2.0" -source = "git+https://github.com/Embucket/arrow-rs.git?rev=e45e8c1577173c279244a3b73221c9cf032c860b#e45e8c1577173c279244a3b73221c9cf032c860b" +source = "git+https://github.com/Embucket/arrow-rs.git?rev=95af4707b289c8dfdbbf6a4a3bb7647da6865bb1#95af4707b289c8dfdbbf6a4a3bb7647da6865bb1" dependencies = [ "arrow-array", "arrow-buffer", @@ -366,7 +366,7 @@ dependencies = [ [[package]] name = "arrow-ord" version = "59.2.0" -source = "git+https://github.com/Embucket/arrow-rs.git?rev=e45e8c1577173c279244a3b73221c9cf032c860b#e45e8c1577173c279244a3b73221c9cf032c860b" +source = "git+https://github.com/Embucket/arrow-rs.git?rev=95af4707b289c8dfdbbf6a4a3bb7647da6865bb1#95af4707b289c8dfdbbf6a4a3bb7647da6865bb1" dependencies = [ "arrow-array", "arrow-buffer", @@ -378,7 +378,7 @@ dependencies = [ [[package]] name = "arrow-row" version = "59.2.0" -source = "git+https://github.com/Embucket/arrow-rs.git?rev=e45e8c1577173c279244a3b73221c9cf032c860b#e45e8c1577173c279244a3b73221c9cf032c860b" +source = "git+https://github.com/Embucket/arrow-rs.git?rev=95af4707b289c8dfdbbf6a4a3bb7647da6865bb1#95af4707b289c8dfdbbf6a4a3bb7647da6865bb1" dependencies = [ "arrow-array", "arrow-buffer", @@ -390,7 +390,7 @@ dependencies = [ [[package]] name = "arrow-schema" version = "59.2.0" -source = "git+https://github.com/Embucket/arrow-rs.git?rev=e45e8c1577173c279244a3b73221c9cf032c860b#e45e8c1577173c279244a3b73221c9cf032c860b" +source = "git+https://github.com/Embucket/arrow-rs.git?rev=95af4707b289c8dfdbbf6a4a3bb7647da6865bb1#95af4707b289c8dfdbbf6a4a3bb7647da6865bb1" dependencies = [ "bitflags", "serde", @@ -401,7 +401,7 @@ dependencies = [ [[package]] name = "arrow-select" version = "59.2.0" -source = "git+https://github.com/Embucket/arrow-rs.git?rev=e45e8c1577173c279244a3b73221c9cf032c860b#e45e8c1577173c279244a3b73221c9cf032c860b" +source = "git+https://github.com/Embucket/arrow-rs.git?rev=95af4707b289c8dfdbbf6a4a3bb7647da6865bb1#95af4707b289c8dfdbbf6a4a3bb7647da6865bb1" dependencies = [ "ahash", "arrow-array", @@ -414,7 +414,7 @@ dependencies = [ [[package]] name = "arrow-string" version = "59.2.0" -source = "git+https://github.com/Embucket/arrow-rs.git?rev=e45e8c1577173c279244a3b73221c9cf032c860b#e45e8c1577173c279244a3b73221c9cf032c860b" +source = "git+https://github.com/Embucket/arrow-rs.git?rev=95af4707b289c8dfdbbf6a4a3bb7647da6865bb1#95af4707b289c8dfdbbf6a4a3bb7647da6865bb1" dependencies = [ "arrow-array", "arrow-buffer", @@ -4487,7 +4487,7 @@ dependencies = [ [[package]] name = "parquet" version = "59.2.0" -source = "git+https://github.com/Embucket/arrow-rs.git?rev=e45e8c1577173c279244a3b73221c9cf032c860b#e45e8c1577173c279244a3b73221c9cf032c860b" +source = "git+https://github.com/Embucket/arrow-rs.git?rev=95af4707b289c8dfdbbf6a4a3bb7647da6865bb1#95af4707b289c8dfdbbf6a4a3bb7647da6865bb1" dependencies = [ "ahash", "arrow-array", @@ -4866,7 +4866,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042" dependencies = [ "heck", - "itertools 0.14.0", + "itertools 0.13.0", "log", "multimap", "petgraph", @@ -4885,7 +4885,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.13.0", "proc-macro2", "quote", "syn 2.0.119", diff --git a/Cargo.toml b/Cargo.toml index 5ce8b2e3ed555..2d918820ade79 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -89,30 +89,30 @@ version = "55.1.0" # # See for more details: https://github.com/rust-lang/cargo/issues/11329 apache-avro = { version = "0.21", default-features = false } -arrow = { git = "https://github.com/Embucket/arrow-rs.git", rev = "e45e8c1577173c279244a3b73221c9cf032c860b", features = [ +arrow = { git = "https://github.com/Embucket/arrow-rs.git", rev = "95af4707b289c8dfdbbf6a4a3bb7647da6865bb1", features = [ "prettyprint", "chrono-tz", ] } -arrow-avro = { git = "https://github.com/Embucket/arrow-rs.git", rev = "e45e8c1577173c279244a3b73221c9cf032c860b", default-features = false, features = [ +arrow-avro = { git = "https://github.com/Embucket/arrow-rs.git", rev = "95af4707b289c8dfdbbf6a4a3bb7647da6865bb1", default-features = false, features = [ "deflate", "snappy", "zstd", "bzip2", "xz", ] } -arrow-buffer = { git = "https://github.com/Embucket/arrow-rs.git", rev = "e45e8c1577173c279244a3b73221c9cf032c860b", default-features = false } -arrow-data = { git = "https://github.com/Embucket/arrow-rs.git", rev = "e45e8c1577173c279244a3b73221c9cf032c860b", default-features = false } -arrow-flight = { git = "https://github.com/Embucket/arrow-rs.git", rev = "e45e8c1577173c279244a3b73221c9cf032c860b", features = [ +arrow-buffer = { git = "https://github.com/Embucket/arrow-rs.git", rev = "95af4707b289c8dfdbbf6a4a3bb7647da6865bb1", default-features = false } +arrow-data = { git = "https://github.com/Embucket/arrow-rs.git", rev = "95af4707b289c8dfdbbf6a4a3bb7647da6865bb1", default-features = false } +arrow-flight = { git = "https://github.com/Embucket/arrow-rs.git", rev = "95af4707b289c8dfdbbf6a4a3bb7647da6865bb1", features = [ "flight-sql-experimental", ] } # Both codecs are required here to make sure that code paths like # file-spilling have access to all compression codecs. -arrow-ipc = { git = "https://github.com/Embucket/arrow-rs.git", rev = "e45e8c1577173c279244a3b73221c9cf032c860b", default-features = false, features = [ +arrow-ipc = { git = "https://github.com/Embucket/arrow-rs.git", rev = "95af4707b289c8dfdbbf6a4a3bb7647da6865bb1", default-features = false, features = [ "lz4", "zstd", ] } -arrow-ord = { git = "https://github.com/Embucket/arrow-rs.git", rev = "e45e8c1577173c279244a3b73221c9cf032c860b", default-features = false } -arrow-schema = { git = "https://github.com/Embucket/arrow-rs.git", rev = "e45e8c1577173c279244a3b73221c9cf032c860b", default-features = false } +arrow-ord = { git = "https://github.com/Embucket/arrow-rs.git", rev = "95af4707b289c8dfdbbf6a4a3bb7647da6865bb1", default-features = false } +arrow-schema = { git = "https://github.com/Embucket/arrow-rs.git", rev = "95af4707b289c8dfdbbf6a4a3bb7647da6865bb1", default-features = false } async-trait = "0.1.89" bigdecimal = "0.4.8" bytes = "1.11" @@ -178,7 +178,7 @@ memchr = "2.8.1" num-traits = { version = "0.2" } object_store = { version = "0.13.2", default-features = false } parking_lot = "0.12" -parquet = { git = "https://github.com/Embucket/arrow-rs.git", rev = "e45e8c1577173c279244a3b73221c9cf032c860b", default-features = false, features = [ +parquet = { git = "https://github.com/Embucket/arrow-rs.git", rev = "95af4707b289c8dfdbbf6a4a3bb7647da6865bb1", default-features = false, features = [ "arrow", "async", "object_store", diff --git a/datafusion/datasource-csv/src/file_format.rs b/datafusion/datasource-csv/src/file_format.rs index e2a15804ec492..7d3fb7155717f 100644 --- a/datafusion/datasource-csv/src/file_format.rs +++ b/datafusion/datasource-csv/src/file_format.rs @@ -718,6 +718,7 @@ impl CsvFormat { ) .with_delimiter(self.options.delimiter) .with_quote(self.options.quote) + .with_replace_invalid_utf8(self.replace_invalid_utf8) .with_preserve_quoted_empty( self.options.preserve_quoted_empty.unwrap_or(false), ) @@ -1566,6 +1567,37 @@ mod tests { assert_eq!(schema.field(0).data_type(), &DataType::Float64); } + #[tokio::test] + async fn test_inference_replaces_invalid_utf8_only_when_enabled() { + for preserve_quoted_empty in [false, true] { + let csv = Bytes::from_static(b"id,msg\n1,\xff\n"); + let infer = |replace| { + CsvFormat::default() + .with_has_header(true) + .with_preserve_quoted_empty(preserve_quoted_empty) + .with_replace_invalid_utf8(replace) + }; + let strict = infer(false) + .infer_schema_from_stream_impl::<_, true, true>( + &InferenceSession::new(), + 1, + stream::iter(vec![Ok(csv.clone())]), + ) + .await; + assert!(strict.is_err()); + let (schema, records) = infer(true) + .infer_schema_from_stream_impl::<_, true, true>( + &InferenceSession::new(), + 1, + stream::iter(vec![Ok(csv)]), + ) + .await + .unwrap(); + assert_eq!(records, 1); + assert_eq!(schema.field(1).data_type(), &DataType::Utf8); + } + } + #[tokio::test] async fn test_opt_in_decimal_stream_preserves_zero_only_chunks() { let tiny = format!("0.{}1", "0".repeat(37));