From ee28b4fca249213ee3191a10dd23109d07430d7f Mon Sep 17 00:00:00 2001 From: osipovartem Date: Wed, 16 Sep 2026 16:56:33 +0300 Subject: [PATCH 1/3] fix(functions): preserve case conversion nullability --- datafusion/functions/src/string/lower.rs | 29 +++++++++++++++++++++--- datafusion/functions/src/string/upper.rs | 29 +++++++++++++++++++++--- 2 files changed, 52 insertions(+), 6 deletions(-) diff --git a/datafusion/functions/src/string/lower.rs b/datafusion/functions/src/string/lower.rs index 88f2c800e9e0c..18e718526f2b1 100644 --- a/datafusion/functions/src/string/lower.rs +++ b/datafusion/functions/src/string/lower.rs @@ -15,14 +15,14 @@ // specific language governing permissions and limitations // under the License. -use arrow::datatypes::DataType; +use arrow::datatypes::{DataType, Field, FieldRef}; use crate::string::common::to_lower; use datafusion_common::Result; use datafusion_common::types::logical_string; use datafusion_expr::{ - Coercion, ColumnarValue, Documentation, EncodingPreservation, ScalarFunctionArgs, - ScalarUDFImpl, Signature, TypeSignatureClass, Volatility, + Coercion, ColumnarValue, Documentation, EncodingPreservation, ReturnFieldArgs, + ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignatureClass, Volatility, }; use datafusion_macros::user_doc; @@ -80,6 +80,14 @@ impl ScalarUDFImpl for LowerFunc { Ok(arg_types[0].clone()) } + fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { + let input = &args.arg_fields[0]; + Ok( + Field::new(self.name(), input.data_type().clone(), input.is_nullable()) + .into(), + ) + } + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { to_lower(&args.args, "lower") } @@ -97,6 +105,21 @@ mod tests { use datafusion_common::config::ConfigOptions; use std::sync::Arc; + #[test] + fn preserves_input_nullability() -> Result<()> { + let func = LowerFunc::new(); + for nullable in [false, true] { + let input = Field::new("input", DataType::Utf8, nullable); + let result = func.return_field_from_args(ReturnFieldArgs { + arg_fields: &[input.into()], + scalar_arguments: &[None], + })?; + assert_eq!(result.data_type(), &DataType::Utf8); + assert_eq!(result.is_nullable(), nullable); + } + Ok(()) + } + fn invoke_lower(input: ArrayRef) -> Result { let func = LowerFunc::new(); let data_type = input.data_type().clone(); diff --git a/datafusion/functions/src/string/upper.rs b/datafusion/functions/src/string/upper.rs index 789ab2c046203..c7aa430d978fe 100644 --- a/datafusion/functions/src/string/upper.rs +++ b/datafusion/functions/src/string/upper.rs @@ -16,12 +16,12 @@ // under the License. use crate::string::common::to_upper; -use arrow::datatypes::DataType; +use arrow::datatypes::{DataType, Field, FieldRef}; use datafusion_common::Result; use datafusion_common::types::logical_string; use datafusion_expr::{ - Coercion, ColumnarValue, Documentation, EncodingPreservation, ScalarFunctionArgs, - ScalarUDFImpl, Signature, TypeSignatureClass, Volatility, + Coercion, ColumnarValue, Documentation, EncodingPreservation, ReturnFieldArgs, + ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignatureClass, Volatility, }; use datafusion_macros::user_doc; @@ -79,6 +79,14 @@ impl ScalarUDFImpl for UpperFunc { Ok(arg_types[0].clone()) } + fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { + let input = &args.arg_fields[0]; + Ok( + Field::new(self.name(), input.data_type().clone(), input.is_nullable()) + .into(), + ) + } + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { to_upper(&args.args, "upper") } @@ -96,6 +104,21 @@ mod tests { use datafusion_common::config::ConfigOptions; use std::sync::Arc; + #[test] + fn preserves_input_nullability() -> Result<()> { + let func = UpperFunc::new(); + for nullable in [false, true] { + let input = Field::new("input", DataType::Utf8, nullable); + let result = func.return_field_from_args(ReturnFieldArgs { + arg_fields: &[input.into()], + scalar_arguments: &[None], + })?; + assert_eq!(result.data_type(), &DataType::Utf8); + assert_eq!(result.is_nullable(), nullable); + } + Ok(()) + } + fn invoke_upper(input: ArrayRef) -> Result { let func = UpperFunc::new(); let data_type = input.data_type().clone(); From 62d185acd868567b5f400d514f2c248b6b191466 Mon Sep 17 00:00:00 2001 From: osipovartem Date: Wed, 16 Sep 2026 18:12:22 +0300 Subject: [PATCH 2/3] test: cover string case function nullability --- .../sqllogictest/test_files/functions.slt | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/datafusion/sqllogictest/test_files/functions.slt b/datafusion/sqllogictest/test_files/functions.slt index 4baee8ce6fbe5..e484887d836ed 100644 --- a/datafusion/sqllogictest/test_files/functions.slt +++ b/datafusion/sqllogictest/test_files/functions.slt @@ -486,6 +486,35 @@ BAR Dictionary(Int32, Utf8) statement ok DROP TABLE upper_dictionary_test +# upper and lower preserve input nullability in the planned schema +statement ok +CREATE TABLE case_nullability (a VARCHAR NOT NULL, b VARCHAR) AS VALUES ('Ab', NULL) + +statement ok +CREATE TABLE case_nullability_out AS +SELECT upper(a) AS ua, lower(a) AS la, upper(b) AS ub FROM case_nullability + +query TTT +DESCRIBE case_nullability_out +---- +ua Utf8View NO +la Utf8View NO +ub Utf8View YES + +# An outer join widens a non-nullable input before it reaches upper +query T +SELECT upper(t.a) +FROM (SELECT 1 AS k) x +LEFT JOIN case_nullability t ON false +---- +NULL + +statement ok +DROP TABLE case_nullability_out + +statement ok +DROP TABLE case_nullability + query T SELECT btrim(' foo ') ---- From be7d666848fe2998efd1e42ed09fa95c5059d580 Mon Sep 17 00:00:00 2001 From: osipovartem Date: Wed, 16 Sep 2026 22:15:44 +0300 Subject: [PATCH 3/3] test: simplify case nullability coverage --- .../sqllogictest/test_files/functions.slt | 29 ++++++++----------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/datafusion/sqllogictest/test_files/functions.slt b/datafusion/sqllogictest/test_files/functions.slt index e484887d836ed..fbabbb07fee0a 100644 --- a/datafusion/sqllogictest/test_files/functions.slt +++ b/datafusion/sqllogictest/test_files/functions.slt @@ -487,34 +487,29 @@ statement ok DROP TABLE upper_dictionary_test # upper and lower preserve input nullability in the planned schema -statement ok -CREATE TABLE case_nullability (a VARCHAR NOT NULL, b VARCHAR) AS VALUES ('Ab', NULL) - -statement ok -CREATE TABLE case_nullability_out AS -SELECT upper(a) AS ua, lower(a) AS la, upper(b) AS ub FROM case_nullability - query TTT -DESCRIBE case_nullability_out +DESCRIBE SELECT upper(a) AS ua, lower(a) AS la, upper(b) AS ub +FROM (VALUES ('Ab', CAST(NULL AS VARCHAR))) AS t(a, b) ---- -ua Utf8View NO -la Utf8View NO +ua Utf8 NO +la Utf8 NO ub Utf8View YES # An outer join widens a non-nullable input before it reaches upper +query TTT +DESCRIBE SELECT upper(t.a) AS ua +FROM (SELECT 1 AS k) x +LEFT JOIN (VALUES ('Ab')) AS t(a) ON false +---- +ua Utf8 YES + query T SELECT upper(t.a) FROM (SELECT 1 AS k) x -LEFT JOIN case_nullability t ON false +LEFT JOIN (VALUES ('Ab')) AS t(a) ON false ---- NULL -statement ok -DROP TABLE case_nullability_out - -statement ok -DROP TABLE case_nullability - query T SELECT btrim(' foo ') ----