diff --git a/datafusion/datasource-csv/src/file_format.rs b/datafusion/datasource-csv/src/file_format.rs index a8c27369cb9c1..1f7da7e8499e7 100644 --- a/datafusion/datasource-csv/src/file_format.rs +++ b/datafusion/datasource-csv/src/file_format.rs @@ -30,8 +30,8 @@ use arrow::error::ArrowError; use datafusion_common::config::{ConfigField, ConfigFileType, CsvOptions}; use datafusion_common::file_options::csv_writer::CsvWriterOptions; use datafusion_common::{ - DEFAULT_CSV_EXTENSION, DataFusionError, GetExt, Result, Statistics, exec_err, - not_impl_err, + DEFAULT_CSV_EXTENSION, DataFusionError, GetExt, Result, Statistics, + exec_datafusion_err, exec_err, not_impl_err, }; use datafusion_common_runtime::SpawnedTask; use datafusion_datasource::TableSchema; @@ -540,6 +540,21 @@ impl CsvFormat { let mut record_number = -1; let initial_records_to_read = records_to_read; + // Compile once rather than per chunk, and report a malformed pattern + // instead of panicking on it. + let null_regex = self + .options + .null_regex + .as_ref() + .map(|null_regex| { + Regex::new(null_regex).map_err(|e| { + exec_datafusion_err!( + "Unable to parse CSV null regex '{null_regex}': {e}" + ) + }) + }) + .transpose()?; + pin_mut!(stream); while let Some(chunk) = stream.next().await.transpose()? { @@ -557,10 +572,8 @@ impl CsvFormat { .with_quote(self.options.quote) .with_truncated_rows(self.options.truncated_rows.unwrap_or(false)); - if let Some(null_regex) = &self.options.null_regex { - let regex = Regex::new(null_regex.as_str()) - .expect("Unable to parse CSV null regex."); - format = format.with_null_regex(regex); + if let Some(regex) = &null_regex { + format = format.with_null_regex(regex.clone()); } if let Some(escape) = self.options.escape { diff --git a/datafusion/sqllogictest/test_files/csv_files.slt b/datafusion/sqllogictest/test_files/csv_files.slt index b5b6130940c40..3562383c8bd52 100644 --- a/datafusion/sqllogictest/test_files/csv_files.slt +++ b/datafusion/sqllogictest/test_files/csv_files.slt @@ -22,6 +22,14 @@ STORED AS CSV LOCATION '../core/tests/data/duplicate_header.csv' OPTIONS ('format.has_header' 'true'); +# A malformed null_regex is reported as an error rather than panicking the +# query. Schema inference compiles the pattern, so CREATE is where it surfaces. +statement error Unable to parse CSV null regex +CREATE EXTERNAL TABLE csv_with_invalid_null_regex +STORED AS CSV +LOCATION '../core/tests/data/quote.csv' +OPTIONS ('format.has_header' 'true', 'format.null_regex' '('); + # create_external_table_with_quote_escape statement ok CREATE EXTERNAL TABLE csv_with_quote (