From 422f65bf32931cb61d084a212e3b25b5a9239523 Mon Sep 17 00:00:00 2001 From: osipovartem Date: Thu, 15 Oct 2026 16:06:57 +0300 Subject: [PATCH 1/3] Preserve scalar function spans in JSON rewrites --- Cargo.toml | 11 ++--------- src/rewrite.rs | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 9 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index ad4acb8..7d45f2f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -39,12 +39,5 @@ pedantic = { level = "deny", priority = -1 } name = "main" harness = false -# [patch.crates-io] -# datafusion = { git = "https://github.com/apache/datafusion.git", rev = "d47bd599ee911eb4be018595b860341ff437ccb1" } -# datafusion-common = { git = "https://github.com/apache/datafusion.git", rev = "d47bd599ee911eb4be018595b860341ff437ccb1" } -# datafusion-expr = { git = "https://github.com/apache/datafusion.git", rev = "d47bd599ee911eb4be018595b860341ff437ccb1" } -# datafusion-execution = { git = "https://github.com/apache/datafusion.git", rev = "d47bd599ee911eb4be018595b860341ff437ccb1" } -# datafusion-sql = { git = "https://github.com/apache/datafusion.git", rev = "d47bd599ee911eb4be018595b860341ff437ccb1" } -# datafusion-proto = { git = "https://github.com/apache/datafusion.git", rev = "d47bd599ee911eb4be018595b860341ff437ccb1" } -# datafusion-datasource = { git = "https://github.com/apache/datafusion.git", rev = "d47bd599ee911eb4be018595b860341ff437ccb1" } -# datafusion-physical-expr-adapter = { git = "https://github.com/apache/datafusion.git", rev = "d47bd599ee911eb4be018595b860341ff437ccb1" } +[patch.crates-io] +datafusion = { git = "https://github.com/Embucket/datafusion.git", rev = "5654173da369eb2d3d1b6b0350919c35ee21b511" } diff --git a/src/rewrite.rs b/src/rewrite.rs index 1dea562..e969d01 100644 --- a/src/rewrite.rs +++ b/src/rewrite.rs @@ -50,6 +50,7 @@ fn optimise_json_get_cast(cast: &Cast) -> Option> { Some(Transformed::yes(Expr::ScalarFunction(ScalarFunction { func, args: scalar_func.args.clone(), + spans: scalar_func.spans.clone(), }))) } @@ -84,6 +85,7 @@ fn unnest_json_calls(func: &ScalarFunction) -> Option> { Some(Transformed::yes(Expr::ScalarFunction(ScalarFunction { func: func.func.clone(), args, + spans: func.spans.clone(), }))) } else { None @@ -190,9 +192,46 @@ impl ExprPlanner for JsonExprPlanner { Expr::ScalarFunction(ScalarFunction { func: op.into(), args: vec![expr.left, expr.right], + spans: Default::default(), }), None::<&str>, alias_name, )))) } } + +#[cfg(test)] +mod tests { + use super::*; + use datafusion::common::{Location, Span}; + use datafusion::logical_expr::{col, lit}; + + fn span() -> Span { + Span::new(Location { line: 2, column: 4 }, Location { line: 2, column: 12 }) + } + + #[test] + fn nested_json_get_rewrite_preserves_outer_span() { + let inner = ScalarFunction::new_udf(crate::udfs::json_get_udf(), vec![col("json"), lit("a")]); + let mut outer = + ScalarFunction::new_udf(crate::udfs::json_get_udf(), vec![Expr::ScalarFunction(inner), lit("b")]); + outer.spans.add_span(span()); + + let Expr::ScalarFunction(rewritten) = unnest_json_calls(&outer).unwrap().data else { + panic!("expected a scalar function") + }; + assert_eq!(rewritten.spans.first(), Some(span())); + } + + #[test] + fn json_get_cast_rewrite_preserves_call_span() { + let mut function = ScalarFunction::new_udf(crate::udfs::json_get_udf(), vec![col("json"), lit("a")]); + function.spans.add_span(span()); + let cast = Cast::new(Box::new(Expr::ScalarFunction(function)), DataType::Int64); + + let Expr::ScalarFunction(rewritten) = optimise_json_get_cast(&cast).unwrap().data else { + panic!("expected a scalar function") + }; + assert_eq!(rewritten.spans.first(), Some(span())); + } +} From a846c14f1260d708971d7d23bb05ca37741ac82d Mon Sep 17 00:00:00 2001 From: osipovartem Date: Thu, 15 Oct 2026 17:22:21 +0300 Subject: [PATCH 2/3] Align JSON span tests with final DataFusion revision --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 7d45f2f..8f66ba2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,4 +40,4 @@ name = "main" harness = false [patch.crates-io] -datafusion = { git = "https://github.com/Embucket/datafusion.git", rev = "5654173da369eb2d3d1b6b0350919c35ee21b511" } +datafusion = { git = "https://github.com/Embucket/datafusion.git", rev = "d412f43ec5e83bc4fa16c3380fff1ce0967ec01c" } From 226c1032ec36cbd52d99dc63ebf4104411c98156 Mon Sep 17 00:00:00 2001 From: osipovartem Date: Thu, 15 Oct 2026 17:40:25 +0300 Subject: [PATCH 3/3] Align JSON fork MSRV and span initializer with DataFusion 55 --- Cargo.toml | 2 +- src/rewrite.rs | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 8f66ba2..c09f632 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ license = "Apache-2.0" keywords = ["datafusion", "JSON", "SQL"] categories = ["database-implementations", "parsing"] repository = "https://github.com/datafusion-contrib/datafusion-functions-json/" -rust-version = "1.88.0" +rust-version = "1.94.0" [dependencies] datafusion = { version = "55", default-features = false, features = [ diff --git a/src/rewrite.rs b/src/rewrite.rs index e969d01..aa4e978 100644 --- a/src/rewrite.rs +++ b/src/rewrite.rs @@ -6,6 +6,7 @@ use datafusion::common::tree_node::Transformed; use datafusion::common::Column; use datafusion::common::DFSchema; use datafusion::common::Result; +use datafusion::common::Spans; use datafusion::logical_expr::expr::{Alias, Cast, Expr, ScalarFunction}; use datafusion::logical_expr::expr_rewriter::FunctionRewrite; use datafusion::logical_expr::planner::{ExprPlanner, PlannerResult, RawBinaryExpr}; @@ -192,7 +193,7 @@ impl ExprPlanner for JsonExprPlanner { Expr::ScalarFunction(ScalarFunction { func: op.into(), args: vec![expr.left, expr.right], - spans: Default::default(), + spans: Spans::default(), }), None::<&str>, alias_name,