diff --git a/crates/paimon/src/arrow/date_extract.rs b/crates/paimon/src/arrow/date_extract.rs new file mode 100644 index 000000000..102914202 --- /dev/null +++ b/crates/paimon/src/arrow/date_extract.rs @@ -0,0 +1,174 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Gregorian field extraction using Java Paimon's DATE/TIMESTAMP representation. + +use crate::common::date_time::epoch_day_to_date; +use crate::spec::DatePart; +use arrow_array::{cast::AsArray, ArrayRef, Int32Array}; +use arrow_schema::{DataType, TimeUnit}; +use std::sync::Arc; + +pub(crate) fn evaluate_date_extract(source: &ArrayRef, part: DatePart) -> crate::Result { + let values: Vec> = match source.data_type() { + DataType::Date32 => source + .as_primitive::() + .iter() + .map(|day| day.map(|day| extract(day, 0, part))) + .collect(), + DataType::Timestamp(unit, None) => { + let raw = arrow_cast::cast(source, &DataType::Int64).map_err(|e| { + crate::Error::DataInvalid { + message: format!("cannot read calendar transform input: {e}"), + source: None, + } + })?; + raw.as_primitive::() + .iter() + .map(|ticks| { + ticks.map(|ticks| { + let millis = match unit { + TimeUnit::Second => ticks.wrapping_mul(1000), + TimeUnit::Millisecond => ticks, + TimeUnit::Microsecond => ticks.div_euclid(1000), + TimeUnit::Nanosecond => ticks.div_euclid(1_000_000), + }; + // Timestamp.toLocalDateTime narrows the epoch day to an int. + // Casting after floor division is also exact when the day wraps. + extract( + millis.div_euclid(86_400_000) as i32, + millis.rem_euclid(86_400_000), + part, + ) + }) + }) + .collect() + } + other => { + return Err(crate::Error::DataInvalid { + message: format!( + "calendar transforms require DATE or TIMESTAMP without time zone, got {other}" + ), + source: None, + }) + } + }; + Ok(Arc::new(Int32Array::from(values))) +} + +fn extract(day: i32, millis: i64, part: DatePart) -> i32 { + let day = i64::from(day); + let (year, month, date) = epoch_day_to_date(day); + let weekday = (day + 3).rem_euclid(7) as i32; + match part { + DatePart::Year => year, + DatePart::Month => month as i32, + DatePart::Day => date as i32, + DatePart::Hour => (millis / 3_600_000) as i32, + DatePart::Minute => (millis / 60_000 % 60) as i32, + DatePart::Second => (millis / 1000 % 60) as i32, + DatePart::Quarter => ((month - 1) / 3 + 1) as i32, + DatePart::DayOfYear => (day - january_first(year) + 1) as i32, + DatePart::Weekday => weekday, + DatePart::IsoDayOfWeek => weekday + 1, + DatePart::DayOfWeek => (weekday + 1) % 7 + 1, + DatePart::Week | DatePart::YearOfWeek => { + // The Thursday of an ISO week always belongs to its week-based year. + let thursday = day + 3 - i64::from(weekday); + let (week_year, _, _) = epoch_day_to_date(thursday); + if part == DatePart::YearOfWeek { + week_year + } else { + ((thursday - january_first(week_year)) / 7 + 1) as i32 + } + } + } +} + +fn january_first(year: i32) -> i64 { + // March-based Gregorian eras: floor division is essential before year zero. + let year = i64::from(year) - 1; + let era = year.div_euclid(400); + let yoe = year - era * 400; + era * 146_097 + yoe * 365 + yoe / 4 - yoe / 100 + 306 - 719_468 +} + +#[cfg(test)] +mod tests { + use super::*; + use arrow_array::{ + Date32Array, TimestampMicrosecondArray, TimestampMillisecondArray, + TimestampNanosecondArray, TimestampSecondArray, + }; + + #[test] + fn calendar_parts_match_actual_java_for_all_units_and_nulls() { + let names = [ + "YEAR", + "MONTH", + "DAY", + "HOUR", + "MINUTE", + "SECOND", + "QUARTER", + "DAY_OF_YEAR", + "WEEKDAY", + "ISO_DAY_OF_WEEK", + "DAY_OF_WEEK", + "WEEK", + "YEAR_OF_WEEK", + ]; + // Actual DateExtractTransform outputs, including Timestamp's int epoch-day + // narrowing. Generated with Java DATE / timestamp(s,ms,us,ns) inputs. + let cases: Vec = serde_json::from_str(include_str!( + "../../tests/fixtures/calendar_extract_java.json" + )) + .unwrap(); + for case in cases { + let ticks = case["value"].as_i64().unwrap(); + let source: ArrayRef = match case["unit"].as_str().unwrap() { + "date" => Arc::new(Date32Array::from(vec![Some(ticks as i32), None])), + "s" => Arc::new(TimestampSecondArray::from(vec![Some(ticks), None])), + "ms" => Arc::new(TimestampMillisecondArray::from(vec![Some(ticks), None])), + "us" => Arc::new(TimestampMicrosecondArray::from(vec![Some(ticks), None])), + "ns" => Arc::new(TimestampNanosecondArray::from(vec![Some(ticks), None])), + _ => unreachable!(), + }; + for (name, expected) in names.iter().zip(case["parts"].as_array().unwrap()) { + let result = + evaluate_date_extract(&source, DatePart::from_name(name).unwrap()).unwrap(); + let values = result.as_primitive::(); + assert_eq!( + values.iter().collect::>(), + vec![Some(expected.as_i64().unwrap() as i32), None], + "{name}: {case}" + ); + } + } + } + + #[test] + fn date_extraction_rejects_noncalendar_and_local_zoned_arrays() { + for source in [ + Arc::new(arrow_array::Int32Array::from(vec![1])) as ArrayRef, + Arc::new(arrow_array::Time32MillisecondArray::from(vec![1])), + Arc::new(TimestampMillisecondArray::from(vec![1]).with_timezone("UTC")), + ] { + assert!(evaluate_date_extract(&source, DatePart::Year).is_err()); + } + } +} diff --git a/crates/paimon/src/arrow/mod.rs b/crates/paimon/src/arrow/mod.rs index 7273e1751..7a8fc94c6 100644 --- a/crates/paimon/src/arrow/mod.rs +++ b/crates/paimon/src/arrow/mod.rs @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +mod date_extract; +pub(crate) use date_extract::evaluate_date_extract; mod evolution_cast; mod evolution_decimal; mod evolution_string; diff --git a/crates/paimon/src/spec/mod.rs b/crates/paimon/src/spec/mod.rs index a108f89ea..743004c7e 100644 --- a/crates/paimon/src/spec/mod.rs +++ b/crates/paimon/src/spec/mod.rs @@ -133,7 +133,7 @@ pub(crate) use predicate::like_match; pub(crate) use predicate::row_id_leaf; pub(crate) use predicate::validate_datum_matches_type; pub use predicate::{ - field_idx_to_partition_idx, Datum, PadDirection, Predicate, PredicateBuilder, + field_idx_to_partition_idx, DatePart, Datum, PadDirection, Predicate, PredicateBuilder, PredicateOperator, Transform, TransformInput, TrimFlag, }; pub(crate) mod murmur_hash; diff --git a/crates/paimon/src/spec/predicate.rs b/crates/paimon/src/spec/predicate.rs index 79b4b25bd..16186f7b0 100644 --- a/crates/paimon/src/spec/predicate.rs +++ b/crates/paimon/src/spec/predicate.rs @@ -1198,6 +1198,8 @@ pub enum Transform { FieldRef(usize), /// `CAST`: a field cast to a type. Cast(usize, DataType), + /// Extract a Gregorian calendar field from DATE or TIMESTAMP. + DateExtract(usize, DatePart), /// String transforms over mixed literal / field inputs. Upper(Vec), /// Lowercase of the inputs. @@ -1222,6 +1224,58 @@ pub enum Transform { Overlay(Vec), } +/// Calendar fields exposed by Java DateExtractTransform subclasses. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DatePart { + /// Proleptic Gregorian year, including year zero and negative years. + Year, + /// Month numbered 1 through 12. + Month, + /// Day of month, numbered 1 through 31. + Day, + /// Hour of day, or zero for DATE. + Hour, + /// Minute of hour, or zero for DATE. + Minute, + /// Second of minute, or zero for DATE. + Second, + /// Quarter numbered 1 through 4. + Quarter, + /// Day of year, numbered 1 through 366. + DayOfYear, + /// Monday is zero; Sunday is six. + Weekday, + /// Monday is one; Sunday is seven. + IsoDayOfWeek, + /// Sunday is one; Saturday is seven. + DayOfWeek, + /// ISO week of the week-based year, numbered 1 through 53. + Week, + /// ISO week-based year, which can differ from the calendar year. + YearOfWeek, +} + +impl DatePart { + pub(crate) fn from_name(name: &str) -> Option { + Some(match name { + "YEAR" => Self::Year, + "MONTH" => Self::Month, + "DAY" => Self::Day, + "HOUR" => Self::Hour, + "MINUTE" => Self::Minute, + "SECOND" => Self::Second, + "QUARTER" => Self::Quarter, + "DAY_OF_YEAR" => Self::DayOfYear, + "WEEKDAY" => Self::Weekday, + "ISO_DAY_OF_WEEK" => Self::IsoDayOfWeek, + "DAY_OF_WEEK" => Self::DayOfWeek, + "WEEK" => Self::Week, + "YEAR_OF_WEEK" => Self::YearOfWeek, + _ => return None, + }) + } +} + /// The ends selected by Java's TRIM transform. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum TrimFlag { @@ -1268,7 +1322,9 @@ impl Transform { pub fn collect_field_indices(&self, out: &mut std::collections::HashSet) { match self { Transform::Null => {} - Transform::FieldRef(index) | Transform::Cast(index, _) => { + Transform::FieldRef(index) + | Transform::Cast(index, _) + | Transform::DateExtract(index, _) => { out.insert(*index); } _ => { @@ -1414,10 +1470,59 @@ fn parse_rest_transform(value: &serde_json::Value, fields: &[DataField]) -> Resu &[2, 3], false, )?)), + name if DatePart::from_name(name).is_some() => { + let input = value + .get("fieldRef") + .ok_or_else(|| rest_json_err("calendar transform without fieldRef"))?; + check_calendar_input_type(input)?; + let index = field_ref_index(input)?; + if !matches!(fields[index].data_type(), DataType::Date(_)) + && !matches!(fields[index].data_type(), DataType::Timestamp(t) if t.precision() <= 9) + { + return Err(rest_json_err( + "calendar transforms require DATE or TIMESTAMP without time zone", + )); + } + Ok(Transform::DateExtract( + index, + DatePart::from_name(name).unwrap(), + )) + } other => Err(rest_json_err(format!("transform `{other}`"))), } } +/// Validate Java's atomic FieldRef type spellings before binding the current schema. +fn check_calendar_input_type(input: &serde_json::Value) -> Result<()> { + static TYPE: std::sync::LazyLock = std::sync::LazyLock::new(|| { + regex::Regex::new(concat!( + r"(?i)^\s*(?:DATE(?:\s+(?:NOT\s+)?NULL)?|", + r"TIMESTAMP\s*\(\s*([0-9]+)\s*\)", + r"(?:\s*WITHOUT\s+TIME\s+ZONE(?:\s+(?:NOT\s+)?NULL)?|\s*(?:NOT\s+)?NULL)?|", + r"TIMESTAMP(?:\s+WITHOUT\s+TIME\s+ZONE)?(?:\s+(?:NOT\s+)?NULL)?)\s*$" + )) + .unwrap() + }); + let stored = input.get("type").and_then(serde_json::Value::as_str); + if stored + .and_then(|stored| TYPE.captures(stored)) + .is_some_and(|captures| { + captures.get(1).is_none_or(|precision| { + precision + .as_str() + .parse::() + .is_ok_and(|precision| precision <= 9) + }) + }) + { + Ok(()) + } else { + Err(rest_json_err( + "calendar transforms require DATE or TIMESTAMP without time zone", + )) + } +} + /// Bind mixed string/integer inputs by name to the actual table schema. fn transform_inputs( value: &serde_json::Value, @@ -2310,6 +2415,108 @@ mod tests { ] } + #[test] + fn calendar_masks_parse_and_collect_the_date_operand() { + let fields = vec![DataField::new( + 7, + "d".into(), + DataType::Date(crate::spec::DateType::new()), + )]; + for name in [ + "YEAR", + "MONTH", + "DAY", + "HOUR", + "MINUTE", + "SECOND", + "QUARTER", + "DAY_OF_YEAR", + "WEEKDAY", + "ISO_DAY_OF_WEEK", + "DAY_OF_WEEK", + "WEEK", + "YEAR_OF_WEEK", + ] { + let transform = Transform::from_rest_json( + &format!( + r#"{{"name":"{name}","fieldRef":{{"index":99,"name":"d","type":"DATE"}}}}"# + ), + &fields, + ) + .unwrap(); + let mut indices = std::collections::HashSet::new(); + transform.collect_field_indices(&mut indices); + assert_eq!(indices, std::collections::HashSet::from([0])); + } + } + + #[test] + fn calendar_masks_validate_stored_and_rebound_types() { + let parse = |stored: serde_json::Value, actual: &str| { + let fields = vec![DataField::new( + 7, + "d".into(), + serde_json::from_value(serde_json::json!(actual)).unwrap(), + )]; + Transform::from_rest_json( + &serde_json::json!({ + "name": "YEAR", + "fieldRef": {"name": "d", "index": 99, "type": stored}, + }) + .to_string(), + &fields, + ) + }; + for stored in [ + "INT", + "BIGINT", + "TIME(3)", + "TIMESTAMP(3) WITH LOCAL TIME ZONE", + "TIMESTAMP(10)", + "DATENULL", + "TIMESTAMP(3) WITHOUT TIME ZONENULL", + "TIMESTAMP(3) WITHOUT TIME ZONENOT NULL", + ] { + assert!( + parse(serde_json::json!(stored), "DATE").is_err(), + "{stored}" + ); + } + for actual in ["INT", "TIME(3)", "TIMESTAMP(3) WITH LOCAL TIME ZONE"] { + assert!( + parse(serde_json::json!("DATE"), actual).is_err(), + "{actual}" + ); + } + for stored in [ + serde_json::json!("DATE"), + serde_json::json!("DATE NOT NULL"), + serde_json::json!("TIMESTAMP"), + serde_json::json!("TIMESTAMP WITHOUT TIME ZONE"), + serde_json::json!("TIMESTAMP(3)"), + serde_json::json!("TIMESTAMP(3)NOT NULL"), + serde_json::json!("TIMESTAMP(3) WITHOUT TIME ZONE"), + serde_json::json!("TIMESTAMP(3)WITHOUT TIME ZONE"), + ] { + // Name rebinding permits DATE <-> TIMESTAMP and ignores stale indices. + for actual in ["DATE", "TIMESTAMP(6)"] { + assert!( + parse(stored.clone(), actual).is_ok(), + "{stored} -> {actual}" + ); + } + } + for stored in [ + serde_json::json!({"type": "DATE"}), + serde_json::json!({"type": "TIMESTAMP(6)"}), + serde_json::json!("TIMESTAMP(٣)"), + serde_json::json!("TIMESTAMP(３)"), + serde_json::Value::Null, + ] { + assert!(parse(stored.clone(), "DATE").is_err(), "{stored}"); + } + } + #[test] fn test_system_column_never_maps_onto_a_field_space() { let fields = test_fields(); diff --git a/crates/paimon/src/table/query_auth/rules.rs b/crates/paimon/src/table/query_auth/rules.rs index d1e71ac01..5d5ccd4f3 100644 --- a/crates/paimon/src/table/query_auth/rules.rs +++ b/crates/paimon/src/table/query_auth/rules.rs @@ -387,6 +387,9 @@ pub(crate) fn mask_batch( // Arrow strings do not enforce CHAR/VARCHAR width. apply_declared_width(&cast, to) } + Transform::DateExtract(index, part) => { + crate::arrow::evaluate_date_extract(&input(*index)?, *part)? + } Transform::Upper(_) | Transform::Lower(_) | Transform::Concat(_) @@ -417,9 +420,9 @@ fn mask_can_be_null(transform: &Transform, fields: &[DataField]) -> bool { }; match transform { Transform::Null => true, - Transform::FieldRef(index) | Transform::Cast(index, _) => { - fields[*index].data_type().is_nullable() - } + Transform::FieldRef(index) + | Transform::Cast(index, _) + | Transform::DateExtract(index, _) => fields[*index].data_type().is_nullable(), // Only a null separator makes CONCAT_WS null. Transform::ConcatWs(inputs) => inputs.first().is_some_and(nullable), _ => transform.inputs().iter().any(nullable), @@ -434,7 +437,9 @@ fn mask_output_type(transform: &Transform, fields: &[DataField]) -> Option crate::arrow::paimon_type_to_arrow(to).ok(), - Transform::Length(_) | Transform::BitLength(_) => Some(arrow_schema::DataType::Int32), + Transform::Length(_) | Transform::BitLength(_) | Transform::DateExtract(_, _) => { + Some(arrow_schema::DataType::Int32) + } _ => Some(arrow_schema::DataType::Utf8), } } @@ -655,6 +660,54 @@ mod tests { DataType::VarChar(VarCharType::string_type()) } + #[test] + fn calendar_masks_read_reordered_operands_and_preserve_int_nulls() { + let fields = vec![ + field(0, "n", DataType::Int(IntType::new())), + field(1, "d", DataType::Date(DateType::new())), + ]; + let read_fields = vec![fields[1].clone(), fields[0].clone(), fields[0].clone()]; + let source = batch( + &read_fields, + vec![ + Arc::new(Date32Array::from(vec![Some(20087), Some(-1), None])), + Arc::new(Int32Array::from(vec![0, 0, 0])), + Arc::new(Int32Array::from(vec![0, 0, 0])), + ], + ); + let mask = r#"{"name":"YEAR_OF_WEEK","fieldRef":{"name":"d","index":99,"type":"DATE"}}"#; + let rules = parse(&[], &[("n", mask)], &fields).unwrap(); + assert_eq!(mask_inputs(&rules.masks[0]), HashSet::from([1])); + let result = mask_batch(&source, &rules.masks, &fields, &read_fields).unwrap(); + for position in [1, 2] { + assert_eq!( + result + .column(position) + .as_primitive::(), + &Int32Array::from(vec![Some(2025), Some(1970), None]) + ); + } + assert_eq!(result.column(0), source.column(0)); + } + + #[test] + fn calendar_mask_types_nullability_and_cross_mask_reads_fail_closed() { + let mut fields = vec![ + field(0, "n", DataType::Int(IntType::new())), + field(1, "d", DataType::Date(DateType::new())), + field(2, "text", string()), + ]; + let mask = r#"{"name":"YEAR","fieldRef":{"name":"d","index":99,"type":"DATE NOT NULL"}}"#; + assert!(refusal(parse(&[], &[("text", mask)], &fields)).contains("produces")); + assert!(refusal(parse(&[], &[("n", mask), ("d", NULL)], &fields)) + .contains("reads 'd', which is masked too")); + // Nullability comes from the actual schema, not the stale rule metadata. + fields[0] = field(0, "n", DataType::Int(IntType::with_nullable(false))); + assert!(refusal(parse(&[], &[("n", mask)], &fields)).contains("can produce null")); + fields[1] = field(1, "d", DataType::Date(DateType::with_nullable(false))); + assert!(parse(&[], &[("n", mask)], &fields).is_ok()); + } + #[test] fn extended_string_masks_match_java_character_semantics() { let fields = vec![DataField::new(0, "value".into(), string())]; diff --git a/crates/paimon/tests/fixtures/calendar_extract_java.json b/crates/paimon/tests/fixtures/calendar_extract_java.json new file mode 100644 index 000000000..24d99dfee --- /dev/null +++ b/crates/paimon/tests/fixtures/calendar_extract_java.json @@ -0,0 +1,148 @@ +[ +{"unit":"date","value":0,"calendar":"1970-01-01","parts":[1970,1,1,0,0,0,1,1,3,4,5,1,1970]}, +{"unit":"date","value":-1,"calendar":"1969-12-31","parts":[1969,12,31,0,0,0,4,365,2,3,4,1,1970]}, +{"unit":"date","value":-719528,"calendar":"0000-01-01","parts":[0,1,1,0,0,0,1,1,5,6,7,52,-1]}, +{"unit":"date","value":-719529,"calendar":"-0001-12-31","parts":[-1,12,31,0,0,0,4,365,4,5,6,52,-1]}, +{"unit":"date","value":-719469,"calendar":"0000-02-29","parts":[0,2,29,0,0,0,1,60,1,2,3,9,0]}, +{"unit":"date","value":-719103,"calendar":"0001-03-01","parts":[1,3,1,0,0,0,1,60,3,4,5,9,1]}, +{"unit":"date","value":-2147483648,"calendar":"-5877641-06-23","parts":[-5877641,6,23,0,0,0,2,174,1,2,3,26,-5877641]}, +{"unit":"date","value":-2147483647,"calendar":"-5877641-06-24","parts":[-5877641,6,24,0,0,0,2,175,2,3,4,26,-5877641]}, +{"unit":"date","value":2147483646,"calendar":"+5881580-07-10","parts":[5881580,7,10,0,0,0,3,192,3,4,5,28,5881580]}, +{"unit":"date","value":2147483647,"calendar":"+5881580-07-11","parts":[5881580,7,11,0,0,0,3,193,4,5,6,28,5881580]}, +{"unit":"date","value":-25509,"calendar":"1900-02-28","parts":[1900,2,28,0,0,0,1,59,2,3,4,9,1900]}, +{"unit":"date","value":-25508,"calendar":"1900-03-01","parts":[1900,3,1,0,0,0,1,60,3,4,5,9,1900]}, +{"unit":"date","value":11015,"calendar":"2000-02-28","parts":[2000,2,28,0,0,0,1,59,0,1,2,9,2000]}, +{"unit":"date","value":11016,"calendar":"2000-02-29","parts":[2000,2,29,0,0,0,1,60,1,2,3,9,2000]}, +{"unit":"date","value":11017,"calendar":"2000-03-01","parts":[2000,3,1,0,0,0,1,61,2,3,4,9,2000]}, +{"unit":"date","value":16800,"calendar":"2015-12-31","parts":[2015,12,31,0,0,0,4,365,3,4,5,53,2015]}, +{"unit":"date","value":16801,"calendar":"2016-01-01","parts":[2016,1,1,0,0,0,1,1,4,5,6,53,2015]}, +{"unit":"date","value":16803,"calendar":"2016-01-03","parts":[2016,1,3,0,0,0,1,3,6,7,1,53,2015]}, +{"unit":"date","value":16804,"calendar":"2016-01-04","parts":[2016,1,4,0,0,0,1,4,0,1,2,1,2016]}, +{"unit":"date","value":18259,"calendar":"2019-12-29","parts":[2019,12,29,0,0,0,4,363,6,7,1,52,2019]}, +{"unit":"date","value":18260,"calendar":"2019-12-30","parts":[2019,12,30,0,0,0,4,364,0,1,2,1,2020]}, +{"unit":"date","value":18627,"calendar":"2020-12-31","parts":[2020,12,31,0,0,0,4,366,3,4,5,53,2020]}, +{"unit":"date","value":18628,"calendar":"2021-01-01","parts":[2021,1,1,0,0,0,1,1,4,5,6,53,2020]}, +{"unit":"date","value":18630,"calendar":"2021-01-03","parts":[2021,1,3,0,0,0,1,3,6,7,1,53,2020]}, +{"unit":"date","value":18631,"calendar":"2021-01-04","parts":[2021,1,4,0,0,0,1,4,0,1,2,1,2021]}, +{"unit":"date","value":19782,"calendar":"2024-02-29","parts":[2024,2,29,0,0,0,1,60,3,4,5,9,2024]}, +{"unit":"date","value":20086,"calendar":"2024-12-29","parts":[2024,12,29,0,0,0,4,364,6,7,1,52,2024]}, +{"unit":"date","value":20087,"calendar":"2024-12-30","parts":[2024,12,30,0,0,0,4,365,0,1,2,1,2025]}, +{"unit":"date","value":20089,"calendar":"2025-01-01","parts":[2025,1,1,0,0,0,1,1,2,3,4,1,2025]}, +{"unit":"ms","value":-9223372036854775808,"millis":-9223372036854775808,"nanos":0,"calendar":"+1705471-09-26T16:47:04.192","parts":[1705471,9,26,16,47,4,3,269,1,2,3,39,1705471]}, +{"unit":"ms","value":9223372036854775807,"millis":9223372036854775807,"nanos":0,"calendar":"-1701532-04-06T07:12:55.807","parts":[-1701532,4,6,7,12,55,2,97,4,5,6,14,-1701532]}, +{"unit":"ms","value":-1,"millis":-1,"nanos":0,"calendar":"1969-12-31T23:59:59.999","parts":[1969,12,31,23,59,59,4,365,2,3,4,1,1970]}, +{"unit":"ms","value":0,"millis":0,"nanos":0,"calendar":"1970-01-01T00:00","parts":[1970,1,1,0,0,0,1,1,3,4,5,1,1970]}, +{"unit":"ms","value":1,"millis":1,"nanos":0,"calendar":"1970-01-01T00:00:00.001","parts":[1970,1,1,0,0,0,1,1,3,4,5,1,1970]}, +{"unit":"ms","value":-1001,"millis":-1001,"nanos":0,"calendar":"1969-12-31T23:59:58.999","parts":[1969,12,31,23,59,58,4,365,2,3,4,1,1970]}, +{"unit":"ms","value":-1000,"millis":-1000,"nanos":0,"calendar":"1969-12-31T23:59:59","parts":[1969,12,31,23,59,59,4,365,2,3,4,1,1970]}, +{"unit":"ms","value":-999,"millis":-999,"nanos":0,"calendar":"1969-12-31T23:59:59.001","parts":[1969,12,31,23,59,59,4,365,2,3,4,1,1970]}, +{"unit":"ms","value":999,"millis":999,"nanos":0,"calendar":"1970-01-01T00:00:00.999","parts":[1970,1,1,0,0,0,1,1,3,4,5,1,1970]}, +{"unit":"ms","value":1000,"millis":1000,"nanos":0,"calendar":"1970-01-01T00:00:01","parts":[1970,1,1,0,0,1,1,1,3,4,5,1,1970]}, +{"unit":"ms","value":1001,"millis":1001,"nanos":0,"calendar":"1970-01-01T00:00:01.001","parts":[1970,1,1,0,0,1,1,1,3,4,5,1,1970]}, +{"unit":"ms","value":-86400001,"millis":-86400001,"nanos":0,"calendar":"1969-12-30T23:59:59.999","parts":[1969,12,30,23,59,59,4,364,1,2,3,1,1970]}, +{"unit":"ms","value":-86400000,"millis":-86400000,"nanos":0,"calendar":"1969-12-31T00:00","parts":[1969,12,31,0,0,0,4,365,2,3,4,1,1970]}, +{"unit":"ms","value":-86399999,"millis":-86399999,"nanos":0,"calendar":"1969-12-31T00:00:00.001","parts":[1969,12,31,0,0,0,4,365,2,3,4,1,1970]}, +{"unit":"ms","value":86399999,"millis":86399999,"nanos":0,"calendar":"1970-01-01T23:59:59.999","parts":[1970,1,1,23,59,59,1,1,3,4,5,1,1970]}, +{"unit":"ms","value":86400000,"millis":86400000,"nanos":0,"calendar":"1970-01-02T00:00","parts":[1970,1,2,0,0,0,1,2,4,5,6,1,1970]}, +{"unit":"ms","value":86400001,"millis":86400001,"nanos":0,"calendar":"1970-01-02T00:00:00.001","parts":[1970,1,2,0,0,0,1,2,4,5,6,1,1970]}, +{"unit":"ms","value":-185542587187200000,"millis":-185542587187200000,"nanos":0,"calendar":"-5877641-06-23T00:00","parts":[-5877641,6,23,0,0,0,2,174,1,2,3,26,-5877641]}, +{"unit":"ms","value":-185542587187200001,"millis":-185542587187200001,"nanos":0,"calendar":"+5881580-07-11T23:59:59.999","parts":[5881580,7,11,23,59,59,3,193,4,5,6,28,5881580]}, +{"unit":"ms","value":185542587100800000,"millis":185542587100800000,"nanos":0,"calendar":"+5881580-07-11T00:00","parts":[5881580,7,11,0,0,0,3,193,4,5,6,28,5881580]}, +{"unit":"ms","value":185542587187199999,"millis":185542587187199999,"nanos":0,"calendar":"+5881580-07-11T23:59:59.999","parts":[5881580,7,11,23,59,59,3,193,4,5,6,28,5881580]}, +{"unit":"ms","value":185542587187200000,"millis":185542587187200000,"nanos":0,"calendar":"-5877641-06-23T00:00","parts":[-5877641,6,23,0,0,0,2,174,1,2,3,26,-5877641]}, +{"unit":"ms","value":371085174374400000,"millis":371085174374400000,"nanos":0,"calendar":"1970-01-01T00:00","parts":[1970,1,1,0,0,0,1,1,3,4,5,1,1970]}, +{"unit":"ms","value":-371085174374400000,"millis":-371085174374400000,"nanos":0,"calendar":"1970-01-01T00:00","parts":[1970,1,1,0,0,0,1,1,3,4,5,1,1970]}, +{"unit":"us","value":-9223372036854775808,"millis":-9223372036854776,"nanos":192000,"calendar":"-290308-12-21T19:59:05.224192","parts":[-290308,12,21,19,59,5,4,356,6,7,1,51,-290308]}, +{"unit":"us","value":9223372036854775807,"millis":9223372036854775,"nanos":807000,"calendar":"+294247-01-10T04:00:54.775807","parts":[294247,1,10,4,0,54,1,10,6,7,1,1,294247]}, +{"unit":"us","value":-1,"millis":-1,"nanos":999000,"calendar":"1969-12-31T23:59:59.999999","parts":[1969,12,31,23,59,59,4,365,2,3,4,1,1970]}, +{"unit":"us","value":0,"millis":0,"nanos":0,"calendar":"1970-01-01T00:00","parts":[1970,1,1,0,0,0,1,1,3,4,5,1,1970]}, +{"unit":"us","value":1,"millis":0,"nanos":1000,"calendar":"1970-01-01T00:00:00.000001","parts":[1970,1,1,0,0,0,1,1,3,4,5,1,1970]}, +{"unit":"us","value":-1000001,"millis":-1001,"nanos":999000,"calendar":"1969-12-31T23:59:58.999999","parts":[1969,12,31,23,59,58,4,365,2,3,4,1,1970]}, +{"unit":"us","value":-1000000,"millis":-1000,"nanos":0,"calendar":"1969-12-31T23:59:59","parts":[1969,12,31,23,59,59,4,365,2,3,4,1,1970]}, +{"unit":"us","value":-999999,"millis":-1000,"nanos":1000,"calendar":"1969-12-31T23:59:59.000001","parts":[1969,12,31,23,59,59,4,365,2,3,4,1,1970]}, +{"unit":"us","value":999999,"millis":999,"nanos":999000,"calendar":"1970-01-01T00:00:00.999999","parts":[1970,1,1,0,0,0,1,1,3,4,5,1,1970]}, +{"unit":"us","value":1000000,"millis":1000,"nanos":0,"calendar":"1970-01-01T00:00:01","parts":[1970,1,1,0,0,1,1,1,3,4,5,1,1970]}, +{"unit":"us","value":1000001,"millis":1000,"nanos":1000,"calendar":"1970-01-01T00:00:01.000001","parts":[1970,1,1,0,0,1,1,1,3,4,5,1,1970]}, +{"unit":"us","value":-86400000001,"millis":-86400001,"nanos":999000,"calendar":"1969-12-30T23:59:59.999999","parts":[1969,12,30,23,59,59,4,364,1,2,3,1,1970]}, +{"unit":"us","value":-86400000000,"millis":-86400000,"nanos":0,"calendar":"1969-12-31T00:00","parts":[1969,12,31,0,0,0,4,365,2,3,4,1,1970]}, +{"unit":"us","value":-86399999999,"millis":-86400000,"nanos":1000,"calendar":"1969-12-31T00:00:00.000001","parts":[1969,12,31,0,0,0,4,365,2,3,4,1,1970]}, +{"unit":"us","value":86399999999,"millis":86399999,"nanos":999000,"calendar":"1970-01-01T23:59:59.999999","parts":[1970,1,1,23,59,59,1,1,3,4,5,1,1970]}, +{"unit":"us","value":86400000000,"millis":86400000,"nanos":0,"calendar":"1970-01-02T00:00","parts":[1970,1,2,0,0,0,1,2,4,5,6,1,1970]}, +{"unit":"us","value":86400000001,"millis":86400000,"nanos":1000,"calendar":"1970-01-02T00:00:00.000001","parts":[1970,1,2,0,0,0,1,2,4,5,6,1,1970]}, +{"unit":"ns","value":-9223372036854775808,"millis":-9223372036855,"nanos":224192,"calendar":"1677-09-21T00:12:43.145224192","parts":[1677,9,21,0,12,43,3,264,1,2,3,38,1677]}, +{"unit":"ns","value":9223372036854775807,"millis":9223372036854,"nanos":775807,"calendar":"2262-04-11T23:47:16.854775807","parts":[2262,4,11,23,47,16,2,101,4,5,6,15,2262]}, +{"unit":"ns","value":-1,"millis":-1,"nanos":999999,"calendar":"1969-12-31T23:59:59.999999999","parts":[1969,12,31,23,59,59,4,365,2,3,4,1,1970]}, +{"unit":"ns","value":0,"millis":0,"nanos":0,"calendar":"1970-01-01T00:00","parts":[1970,1,1,0,0,0,1,1,3,4,5,1,1970]}, +{"unit":"ns","value":1,"millis":0,"nanos":1,"calendar":"1970-01-01T00:00:00.000000001","parts":[1970,1,1,0,0,0,1,1,3,4,5,1,1970]}, +{"unit":"ns","value":-1000000001,"millis":-1001,"nanos":999999,"calendar":"1969-12-31T23:59:58.999999999","parts":[1969,12,31,23,59,58,4,365,2,3,4,1,1970]}, +{"unit":"ns","value":-1000000000,"millis":-1000,"nanos":0,"calendar":"1969-12-31T23:59:59","parts":[1969,12,31,23,59,59,4,365,2,3,4,1,1970]}, +{"unit":"ns","value":-999999999,"millis":-1000,"nanos":1,"calendar":"1969-12-31T23:59:59.000000001","parts":[1969,12,31,23,59,59,4,365,2,3,4,1,1970]}, +{"unit":"ns","value":999999999,"millis":999,"nanos":999999,"calendar":"1970-01-01T00:00:00.999999999","parts":[1970,1,1,0,0,0,1,1,3,4,5,1,1970]}, +{"unit":"ns","value":1000000000,"millis":1000,"nanos":0,"calendar":"1970-01-01T00:00:01","parts":[1970,1,1,0,0,1,1,1,3,4,5,1,1970]}, +{"unit":"ns","value":1000000001,"millis":1000,"nanos":1,"calendar":"1970-01-01T00:00:01.000000001","parts":[1970,1,1,0,0,1,1,1,3,4,5,1,1970]}, +{"unit":"ns","value":-86400000000001,"millis":-86400001,"nanos":999999,"calendar":"1969-12-30T23:59:59.999999999","parts":[1969,12,30,23,59,59,4,364,1,2,3,1,1970]}, +{"unit":"ns","value":-86400000000000,"millis":-86400000,"nanos":0,"calendar":"1969-12-31T00:00","parts":[1969,12,31,0,0,0,4,365,2,3,4,1,1970]}, +{"unit":"ns","value":-86399999999999,"millis":-86400000,"nanos":1,"calendar":"1969-12-31T00:00:00.000000001","parts":[1969,12,31,0,0,0,4,365,2,3,4,1,1970]}, +{"unit":"ns","value":86399999999999,"millis":86399999,"nanos":999999,"calendar":"1970-01-01T23:59:59.999999999","parts":[1970,1,1,23,59,59,1,1,3,4,5,1,1970]}, +{"unit":"ns","value":86400000000000,"millis":86400000,"nanos":0,"calendar":"1970-01-02T00:00","parts":[1970,1,2,0,0,0,1,2,4,5,6,1,1970]}, +{"unit":"ns","value":86400000000001,"millis":86400000,"nanos":1,"calendar":"1970-01-02T00:00:00.000000001","parts":[1970,1,2,0,0,0,1,2,4,5,6,1,1970]}, +{"unit":"s","value":-9223372036854775,"millis":-9223372036854775000,"nanos":0,"calendar":"+1705471-09-26T16:47:05","parts":[1705471,9,26,16,47,5,3,269,1,2,3,39,1705471]}, +{"unit":"s","value":9223372036854775,"millis":9223372036854775000,"nanos":0,"calendar":"-1701532-04-06T07:12:55","parts":[-1701532,4,6,7,12,55,2,97,4,5,6,14,-1701532]}, +{"unit":"s","value":-86401,"millis":-86401000,"nanos":0,"calendar":"1969-12-30T23:59:59","parts":[1969,12,30,23,59,59,4,364,1,2,3,1,1970]}, +{"unit":"s","value":-86400,"millis":-86400000,"nanos":0,"calendar":"1969-12-31T00:00","parts":[1969,12,31,0,0,0,4,365,2,3,4,1,1970]}, +{"unit":"s","value":-86399,"millis":-86399000,"nanos":0,"calendar":"1969-12-31T00:00:01","parts":[1969,12,31,0,0,1,4,365,2,3,4,1,1970]}, +{"unit":"s","value":-1,"millis":-1000,"nanos":0,"calendar":"1969-12-31T23:59:59","parts":[1969,12,31,23,59,59,4,365,2,3,4,1,1970]}, +{"unit":"s","value":0,"millis":0,"nanos":0,"calendar":"1970-01-01T00:00","parts":[1970,1,1,0,0,0,1,1,3,4,5,1,1970]}, +{"unit":"s","value":1,"millis":1000,"nanos":0,"calendar":"1970-01-01T00:00:01","parts":[1970,1,1,0,0,1,1,1,3,4,5,1,1970]}, +{"unit":"s","value":86399,"millis":86399000,"nanos":0,"calendar":"1970-01-01T23:59:59","parts":[1970,1,1,23,59,59,1,1,3,4,5,1,1970]}, +{"unit":"s","value":86400,"millis":86400000,"nanos":0,"calendar":"1970-01-02T00:00","parts":[1970,1,2,0,0,0,1,2,4,5,6,1,1970]}, +{"unit":"s","value":86401,"millis":86401000,"nanos":0,"calendar":"1970-01-02T00:00:01","parts":[1970,1,2,0,0,1,1,2,4,5,6,1,1970]}, +{"unit":"date","value":-719163,"calendar":"0000-12-31","parts":[0,12,31,0,0,0,4,366,6,7,1,52,0]}, +{"unit":"date","value":-719163,"calendar":"0000-12-31","parts":[0,12,31,0,0,0,4,366,6,7,1,52,0]}, +{"unit":"date","value":-719162,"calendar":"0001-01-01","parts":[1,1,1,0,0,0,1,1,0,1,2,1,1]}, +{"unit":"date","value":-719161,"calendar":"0001-01-02","parts":[1,1,2,0,0,0,1,2,1,2,3,1,1]}, +{"unit":"date","value":-719162,"calendar":"0001-01-01","parts":[1,1,1,0,0,0,1,1,0,1,2,1,1]}, +{"unit":"date","value":-719161,"calendar":"0001-01-02","parts":[1,1,2,0,0,0,1,2,1,2,3,1,1]}, +{"unit":"date","value":2932896,"calendar":"9999-12-31","parts":[9999,12,31,0,0,0,4,365,4,5,6,52,9999]}, +{"unit":"date","value":2932896,"calendar":"9999-12-31","parts":[9999,12,31,0,0,0,4,365,4,5,6,52,9999]}, +{"unit":"date","value":2932897,"calendar":"+10000-01-01","parts":[10000,1,1,0,0,0,1,1,5,6,7,52,9999]}, +{"unit":"date","value":2932898,"calendar":"+10000-01-02","parts":[10000,1,2,0,0,0,1,2,6,7,1,52,9999]}, +{"unit":"date","value":2932897,"calendar":"+10000-01-01","parts":[10000,1,1,0,0,0,1,1,5,6,7,52,9999]}, +{"unit":"date","value":2932898,"calendar":"+10000-01-02","parts":[10000,1,2,0,0,0,1,2,6,7,1,52,9999]}, +{"unit":"s","value":-62135683200,"millis":-62135683200000,"nanos":0,"calendar":"0000-12-31T00:00","parts":[0,12,31,0,0,0,4,366,6,7,1,52,0]}, +{"unit":"s","value":-62135596801,"millis":-62135596801000,"nanos":0,"calendar":"0000-12-31T23:59:59","parts":[0,12,31,23,59,59,4,366,6,7,1,52,0]}, +{"unit":"s","value":-62135596800,"millis":-62135596800000,"nanos":0,"calendar":"0001-01-01T00:00","parts":[1,1,1,0,0,0,1,1,0,1,2,1,1]}, +{"unit":"s","value":-62135596799,"millis":-62135596799000,"nanos":0,"calendar":"0001-01-01T00:00:01","parts":[1,1,1,0,0,1,1,1,0,1,2,1,1]}, +{"unit":"s","value":-62135510401,"millis":-62135510401000,"nanos":0,"calendar":"0001-01-01T23:59:59","parts":[1,1,1,23,59,59,1,1,0,1,2,1,1]}, +{"unit":"s","value":-62135510400,"millis":-62135510400000,"nanos":0,"calendar":"0001-01-02T00:00","parts":[1,1,2,0,0,0,1,2,1,2,3,1,1]}, +{"unit":"s","value":253402214400,"millis":253402214400000,"nanos":0,"calendar":"9999-12-31T00:00","parts":[9999,12,31,0,0,0,4,365,4,5,6,52,9999]}, +{"unit":"s","value":253402300799,"millis":253402300799000,"nanos":0,"calendar":"9999-12-31T23:59:59","parts":[9999,12,31,23,59,59,4,365,4,5,6,52,9999]}, +{"unit":"s","value":253402300800,"millis":253402300800000,"nanos":0,"calendar":"+10000-01-01T00:00","parts":[10000,1,1,0,0,0,1,1,5,6,7,52,9999]}, +{"unit":"s","value":253402300801,"millis":253402300801000,"nanos":0,"calendar":"+10000-01-01T00:00:01","parts":[10000,1,1,0,0,1,1,1,5,6,7,52,9999]}, +{"unit":"s","value":253402387199,"millis":253402387199000,"nanos":0,"calendar":"+10000-01-01T23:59:59","parts":[10000,1,1,23,59,59,1,1,5,6,7,52,9999]}, +{"unit":"s","value":253402387200,"millis":253402387200000,"nanos":0,"calendar":"+10000-01-02T00:00","parts":[10000,1,2,0,0,0,1,2,6,7,1,52,9999]}, +{"unit":"ms","value":-62135683200000,"millis":-62135683200000,"nanos":0,"calendar":"0000-12-31T00:00","parts":[0,12,31,0,0,0,4,366,6,7,1,52,0]}, +{"unit":"ms","value":-62135596800001,"millis":-62135596800001,"nanos":0,"calendar":"0000-12-31T23:59:59.999","parts":[0,12,31,23,59,59,4,366,6,7,1,52,0]}, +{"unit":"ms","value":-62135596800000,"millis":-62135596800000,"nanos":0,"calendar":"0001-01-01T00:00","parts":[1,1,1,0,0,0,1,1,0,1,2,1,1]}, +{"unit":"ms","value":-62135596799999,"millis":-62135596799999,"nanos":0,"calendar":"0001-01-01T00:00:00.001","parts":[1,1,1,0,0,0,1,1,0,1,2,1,1]}, +{"unit":"ms","value":-62135510400001,"millis":-62135510400001,"nanos":0,"calendar":"0001-01-01T23:59:59.999","parts":[1,1,1,23,59,59,1,1,0,1,2,1,1]}, +{"unit":"ms","value":-62135510400000,"millis":-62135510400000,"nanos":0,"calendar":"0001-01-02T00:00","parts":[1,1,2,0,0,0,1,2,1,2,3,1,1]}, +{"unit":"ms","value":253402214400000,"millis":253402214400000,"nanos":0,"calendar":"9999-12-31T00:00","parts":[9999,12,31,0,0,0,4,365,4,5,6,52,9999]}, +{"unit":"ms","value":253402300799999,"millis":253402300799999,"nanos":0,"calendar":"9999-12-31T23:59:59.999","parts":[9999,12,31,23,59,59,4,365,4,5,6,52,9999]}, +{"unit":"ms","value":253402300800000,"millis":253402300800000,"nanos":0,"calendar":"+10000-01-01T00:00","parts":[10000,1,1,0,0,0,1,1,5,6,7,52,9999]}, +{"unit":"ms","value":253402300800001,"millis":253402300800001,"nanos":0,"calendar":"+10000-01-01T00:00:00.001","parts":[10000,1,1,0,0,0,1,1,5,6,7,52,9999]}, +{"unit":"ms","value":253402387199999,"millis":253402387199999,"nanos":0,"calendar":"+10000-01-01T23:59:59.999","parts":[10000,1,1,23,59,59,1,1,5,6,7,52,9999]}, +{"unit":"ms","value":253402387200000,"millis":253402387200000,"nanos":0,"calendar":"+10000-01-02T00:00","parts":[10000,1,2,0,0,0,1,2,6,7,1,52,9999]}, +{"unit":"us","value":-62135683200000000,"millis":-62135683200000,"nanos":0,"calendar":"0000-12-31T00:00","parts":[0,12,31,0,0,0,4,366,6,7,1,52,0]}, +{"unit":"us","value":-62135596800000001,"millis":-62135596800001,"nanos":999000,"calendar":"0000-12-31T23:59:59.999999","parts":[0,12,31,23,59,59,4,366,6,7,1,52,0]}, +{"unit":"us","value":-62135596800000000,"millis":-62135596800000,"nanos":0,"calendar":"0001-01-01T00:00","parts":[1,1,1,0,0,0,1,1,0,1,2,1,1]}, +{"unit":"us","value":-62135596799999999,"millis":-62135596800000,"nanos":1000,"calendar":"0001-01-01T00:00:00.000001","parts":[1,1,1,0,0,0,1,1,0,1,2,1,1]}, +{"unit":"us","value":-62135510400000001,"millis":-62135510400001,"nanos":999000,"calendar":"0001-01-01T23:59:59.999999","parts":[1,1,1,23,59,59,1,1,0,1,2,1,1]}, +{"unit":"us","value":-62135510400000000,"millis":-62135510400000,"nanos":0,"calendar":"0001-01-02T00:00","parts":[1,1,2,0,0,0,1,2,1,2,3,1,1]}, +{"unit":"us","value":253402214400000000,"millis":253402214400000,"nanos":0,"calendar":"9999-12-31T00:00","parts":[9999,12,31,0,0,0,4,365,4,5,6,52,9999]}, +{"unit":"us","value":253402300799999999,"millis":253402300799999,"nanos":999000,"calendar":"9999-12-31T23:59:59.999999","parts":[9999,12,31,23,59,59,4,365,4,5,6,52,9999]}, +{"unit":"us","value":253402300800000000,"millis":253402300800000,"nanos":0,"calendar":"+10000-01-01T00:00","parts":[10000,1,1,0,0,0,1,1,5,6,7,52,9999]}, +{"unit":"us","value":253402300800000001,"millis":253402300800000,"nanos":1000,"calendar":"+10000-01-01T00:00:00.000001","parts":[10000,1,1,0,0,0,1,1,5,6,7,52,9999]}, +{"unit":"us","value":253402387199999999,"millis":253402387199999,"nanos":999000,"calendar":"+10000-01-01T23:59:59.999999","parts":[10000,1,1,23,59,59,1,1,5,6,7,52,9999]}, +{"unit":"us","value":253402387200000000,"millis":253402387200000,"nanos":0,"calendar":"+10000-01-02T00:00","parts":[10000,1,2,0,0,0,1,2,6,7,1,52,9999]} +] diff --git a/crates/paimon/tests/rest_catalog_test.rs b/crates/paimon/tests/rest_catalog_test.rs index 3d5991675..356e5c320 100644 --- a/crates/paimon/tests/rest_catalog_test.rs +++ b/crates/paimon/tests/rest_catalog_test.rs @@ -4447,6 +4447,78 @@ async fn test_query_auth_refuses_a_mask_that_reads_another_masked_column() { assert_refused(plan_err(&g.table, "'b' would publish the raw value of 'a'").await); } +#[cfg(not(windows))] +#[tokio::test] +async fn test_query_auth_calendar_masks_read_hidden_operands_before_filter_and_limit() { + use arrow_array::{Date32Array, TimestampMillisecondArray}; + use paimon::spec::{DateType, TimestampType}; + for (primary_key, evolution) in [(false, false), (true, false), (false, true)] { + let schema = |options: &[(&str, &str)]| { + let mut builder = Schema::builder() + .column("id", DataType::Int(IntType::new())) + .column("n", DataType::Int(IntType::new())) + .column("d", DataType::Date(DateType::new())) + .column("ts", DataType::Timestamp(TimestampType::new(3).unwrap())); + if primary_key { + builder = builder.primary_key(["id"]).option("bucket", "1"); + } + if evolution { + builder = builder + .option("data-evolution.enabled", "true") + .option("row-tracking.enabled", "true"); + } + for (key, value) in options { + builder = builder.option(*key, *value); + } + builder.build().unwrap() + }; + let source = RecordBatch::try_from_iter(vec![ + ( + "id", + Arc::new(Int32Array::from(vec![1, 2, 3])) as Arc, + ), + ("n", Arc::new(Int32Array::from(vec![0, 0, 0]))), + ( + "d", + Arc::new(Date32Array::from(vec![Some(20087), Some(-1), None])), + ), + ( + "ts", + Arc::new(TimestampMillisecondArray::from(vec![ + Some(1735516800000), + Some(-1), + None, + ])), + ), + ]) + .unwrap(); + let g = written("calendar_mask", schema, vec![source]).await; + for (name, type_) in [("d", "DATE"), ("ts", "TIMESTAMP(3)")] { + g.set_rules(&[], &[("n", serde_json::json!({ + "name": "YEAR_OF_WEEK", "fieldRef": {"name": name, "index": 99, "type": type_}, + }))]); + let mut builder = g.table.new_read_builder(); + builder.with_projection(&["n"]).unwrap(); + let batches = read_all(&builder).await.unwrap(); + let values: Vec<_> = batches + .iter() + .flat_map(|batch| column::(batch, "n").iter()) + .collect(); + assert_eq!(values, vec![Some(2025), Some(1970), None]); + assert!(batches.iter().all(|batch| batch.num_columns() == 1)); + // The raw zero in n must not prune away a matching masked value. + builder.with_filter( + PredicateBuilder::new(g.table.schema().fields()) + .equal("n", Datum::Int(1970)) + .unwrap(), + ); + builder.with_projection(&["id"]).unwrap(); + builder.with_limit(1); + assert_eq!(ids(&read_all(&builder).await.unwrap()), vec![2]); + } + } +} + #[cfg(not(windows))] #[tokio::test] async fn test_query_auth_widens_an_integer_mask() {