From b5a25a4f621ddbfd9622e667230eaa13e8ad602a Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Fri, 16 Oct 2026 12:33:52 +0800 Subject: [PATCH 1/3] feat(datafusion): complete system-table reads for Python Native --- .../python/pypaimon_rust/datafusion.pyi | 1 + bindings/python/src/table.rs | 22 +- crates/integrations/datafusion/src/lib.rs | 1 + .../datafusion/src/system_tables/branches.rs | 4 +- .../datafusion/src/system_tables/files.rs | 351 ++++++++++++++---- .../datafusion/src/system_tables/mod.rs | 39 ++ .../datafusion/src/system_tables/options.rs | 9 +- .../src/system_tables/row_string_cast.rs | 162 ++++---- .../datafusion/src/system_tables/schemas.rs | 4 +- .../datafusion/src/system_tables/snapshots.rs | 8 +- .../src/system_tables/table_indexes.rs | 55 ++- .../datafusion/src/system_tables/tags.rs | 2 +- .../datafusion/tests/system_tables.rs | 129 ++++++- crates/paimon/src/common/date_time.rs | 2 +- crates/paimon/src/common/mod.rs | 1 + crates/paimon/src/spec/binary_row.rs | 10 + crates/paimon/src/spec/schema.rs | 27 ++ crates/paimon/src/spec/snapshot.rs | 9 + 18 files changed, 664 insertions(+), 172 deletions(-) diff --git a/bindings/python/python/pypaimon_rust/datafusion.pyi b/bindings/python/python/pypaimon_rust/datafusion.pyi index 3af2afc51..1c34f332f 100644 --- a/bindings/python/python/pypaimon_rust/datafusion.pyi +++ b/bindings/python/python/pypaimon_rust/datafusion.pyi @@ -283,6 +283,7 @@ class Table: def branch(self) -> str: ... def location(self) -> str: ... def schema(self) -> TableSchema: ... + def _read_system_table(self, name: str) -> RecordBatchReader: ... def new_read_builder(self, options: Optional[Dict[str, str]] = None) -> ReadBuilder: ... def new_stream_scan(self) -> StreamTableScan: ... def new_full_text_search_builder(self) -> FullTextSearchBuilder: ... diff --git a/bindings/python/src/table.rs b/bindings/python/src/table.rs index 14e4298f0..f70d9465c 100644 --- a/bindings/python/src/table.rs +++ b/bindings/python/src/table.rs @@ -18,6 +18,7 @@ use std::collections::HashMap; use std::sync::Arc; +use futures::TryStreamExt; use paimon::catalog::{Identifier, RESTToken}; use paimon::io::FileIO; use paimon::spec::TableSchema; @@ -28,9 +29,9 @@ use pyo3::prelude::*; use pyo3::types::PyDict; use crate::context::complete_storage_options; -use crate::error::to_py_err; +use crate::error::{df_to_py_err, to_py_err}; use crate::partition::PyPartitionStat; -use crate::read::PyReadBuilder; +use crate::read::{PyReadBuilder, PyRecordBatchReader}; use crate::schema::PyTableSchema; use crate::snapshot::PySnapshot; use crate::tag::PyTag; @@ -198,6 +199,23 @@ impl PyTable { } } + /// Read metadata through the existing system-table providers. + fn _read_system_table(&self, py: Python<'_>, name: &str) -> PyResult { + let stream = py + .detach(|| { + runtime().block_on(paimon_datafusion::read_system_table( + self.inner.as_ref().clone(), + name, + )) + }) + .map_err(df_to_py_err)?; + let schema = stream.schema(); + Ok(PyRecordBatchReader::from_py_stream( + Some(schema), + Box::pin(stream.map_err(df_to_py_err)), + )) + } + fn new_full_text_search_builder(&self) -> crate::full_text_search::PyFullTextSearchBuilder { crate::full_text_search::PyFullTextSearchBuilder::new(Arc::clone(&self.inner)) } diff --git a/crates/integrations/datafusion/src/lib.rs b/crates/integrations/datafusion/src/lib.rs index cfab74c0e..a50235963 100644 --- a/crates/integrations/datafusion/src/lib.rs +++ b/crates/integrations/datafusion/src/lib.rs @@ -93,6 +93,7 @@ pub use merge_condition::compile_merge_condition; pub use physical_plan::PaimonTableScan; pub use relation_planner::PaimonRelationPlanner; pub use sql_context::{SQLContext, SQLContextBuilder}; +pub use system_tables::read_system_table; pub use table::PaimonTableProvider; pub use variant_functions::register_variant_functions; pub use vector_search::{register_vector_search, VectorSearchFunction}; diff --git a/crates/integrations/datafusion/src/system_tables/branches.rs b/crates/integrations/datafusion/src/system_tables/branches.rs index c925def71..ff1412176 100644 --- a/crates/integrations/datafusion/src/system_tables/branches.rs +++ b/crates/integrations/datafusion/src/system_tables/branches.rs @@ -20,7 +20,7 @@ use std::sync::{Arc, OnceLock}; use async_trait::async_trait; -use datafusion::arrow::array::{RecordBatch, StringArray, TimestampMillisecondArray}; +use datafusion::arrow::array::{RecordBatch, StringArray}; use datafusion::arrow::datatypes::{DataType, Field, Schema, SchemaRef, TimeUnit}; use datafusion::catalog::Session; use datafusion::datasource::memory::MemorySourceConfig; @@ -85,7 +85,7 @@ impl TableProvider for BranchesTable { schema.clone(), vec![ Arc::new(StringArray::from(names)), - Arc::new(TimestampMillisecondArray::from(create_times)), + super::local_timestamp_column(create_times)?, ], )?; diff --git a/crates/integrations/datafusion/src/system_tables/files.rs b/crates/integrations/datafusion/src/system_tables/files.rs index e9749007a..e9e575e7b 100644 --- a/crates/integrations/datafusion/src/system_tables/files.rs +++ b/crates/integrations/datafusion/src/system_tables/files.rs @@ -22,7 +22,7 @@ use std::sync::{Arc, OnceLock}; use async_trait::async_trait; use datafusion::arrow::array::{ - Int32Array, Int64Array, ListBuilder, RecordBatch, StringArray, StringBuilder, + ArrayRef, Int32Array, Int64Array, ListBuilder, RecordBatch, StringArray, StringBuilder, TimestampMillisecondArray, }; use datafusion::arrow::datatypes::{DataType as ArrowDataType, Field, Schema, SchemaRef, TimeUnit}; @@ -37,12 +37,29 @@ use paimon::table::{DataSplit, Table}; use super::row_string_cast::{ format_row_as_java_array_string, format_row_as_java_cast_string, - format_row_field_as_java_cast_string, + format_row_field_as_java_object_string, }; use crate::error::to_datafusion_error; pub(super) fn build(table: Table) -> DFResult> { - Ok(Arc::new(FilesTable { table })) + Ok(Arc::new(FilesTable { + table, + key_ranges: false, + })) +} + +pub(super) fn build_key_ranges(table: Table) -> DFResult> { + Ok(Arc::new(FilesTable { + table, + key_ranges: true, + })) +} + +fn key_ranges_schema() -> SchemaRef { + let files = files_schema(); + let mut fields = files.fields()[..10].to_vec(); + fields.push(files.field(18).clone().into()); + Arc::new(Schema::new(fields)) } fn files_schema() -> SchemaRef { @@ -70,7 +87,7 @@ fn files_schema() -> SchemaRef { ArrowDataType::Timestamp(TimeUnit::Millisecond, None), true, ), - Field::new("delete_row_count", ArrowDataType::Int64, true), + Field::new("deleteRowCount", ArrowDataType::Int64, true), Field::new("file_source", ArrowDataType::Utf8, true), Field::new("first_row_id", ArrowDataType::Int64, true), Field::new( @@ -86,12 +103,17 @@ fn files_schema() -> SchemaRef { #[derive(Debug)] pub(super) struct FilesTable { table: Table, + key_ranges: bool, } #[async_trait] impl TableProvider for FilesTable { fn schema(&self) -> SchemaRef { - files_schema() + if self.key_ranges { + key_ranges_schema() + } else { + files_schema() + } } fn table_type(&self) -> TableType { @@ -106,13 +128,23 @@ impl TableProvider for FilesTable { _limit: Option, ) -> DFResult> { let table = self.table.clone(); - let rows = - crate::runtime::await_with_runtime(async move { collect_file_rows(&table).await }) - .await - .map_err(to_datafusion_error)?; - - let schema = files_schema(); - let batch = file_rows_to_record_batch(&rows)?; + let key_ranges = self.key_ranges; + let rows = crate::runtime::await_with_runtime(async move { + collect_file_rows(&table, !key_ranges).await + }) + .await + .map_err(to_datafusion_error)?; + + let schema = self.schema(); + let batch = if key_ranges { + let mut columns = common_file_columns(&rows); + columns.push(Arc::new(Int64Array::from_iter( + rows.iter().map(|row| row.first_row_id), + ))); + RecordBatch::try_new(schema.clone(), columns)? + } else { + file_rows_to_record_batch(&rows)? + }; Ok(MemorySourceConfig::try_new_exec( &[vec![batch]], @@ -134,9 +166,7 @@ struct FileRow { file_size_in_bytes: i64, min_key: Option, max_key: Option, - null_value_counts: String, - min_value_stats: String, - max_value_stats: String, + stats: Option, min_sequence_number: Option, max_sequence_number: Option, creation_time: Option, @@ -146,7 +176,7 @@ struct FileRow { write_cols: Option>, } -async fn collect_file_rows(table: &Table) -> paimon::Result> { +async fn collect_file_rows(table: &Table, include_stats: bool) -> paimon::Result> { let scan = table .new_read_builder() .new_scan() @@ -159,7 +189,16 @@ async fn collect_file_rows(table: &Table) -> paimon::Result> { let mut rows = Vec::new(); for split in scan.splits() { - rows.extend(data_split_rows(table, split, &partition_fields, &mut schema_cache).await?); + rows.extend( + data_split_rows( + table, + split, + &partition_fields, + &mut schema_cache, + include_stats, + ) + .await?, + ); } Ok(rows) } @@ -169,6 +208,7 @@ async fn data_split_rows( split: &DataSplit, partition_fields: &[DataField], schema_cache: &mut HashMap>, + include_stats: bool, ) -> paimon::Result> { let partition = format_partition(split.partition(), partition_fields)?; let mut rows = Vec::with_capacity(split.data_files().len()); @@ -177,7 +217,11 @@ async fn data_split_rows( let key_fields = key_fields_for_schema(&file_schema); let min_key = format_key(&file.min_key, &key_fields)?; let max_key = format_key(&file.max_key, &key_fields)?; - let stats = format_value_stats(table, schema_cache, file).await?; + let stats = if include_stats { + Some(format_value_stats(table, schema_cache, file).await?) + } else { + None + }; rows.push(FileRow { partition: partition.clone(), @@ -185,7 +229,7 @@ async fn data_split_rows( file_path: file .external_path .clone() - .unwrap_or_else(|| split.data_file_path(file)), + .unwrap_or_else(|| java_file_path(&split.data_file_path(file))), file_format: data_file_format_identifier(&file.file_name)?, schema_id: file.schema_id, level: file.level, @@ -193,9 +237,7 @@ async fn data_split_rows( file_size_in_bytes: file.file_size, min_key, max_key, - null_value_counts: stats.null_value_counts, - min_value_stats: stats.min_value_stats, - max_value_stats: stats.max_value_stats, + stats, min_sequence_number: Some(file.min_sequence_number), max_sequence_number: Some(file.max_sequence_number), creation_time: file.creation_time.map(|t| t.timestamp_millis()), @@ -272,9 +314,17 @@ async fn format_value_stats( schema_cache: &mut HashMap>, file: &DataFileMeta, ) -> paimon::Result { - let table_fields = table.schema().fields(); let file_schema = schema_for_id(table, schema_cache, file.schema_id).await?; - let data_fields = file_schema.fields(); + format_file_value_stats(table.schema().fields(), &file_schema, file) +} + +fn format_file_value_stats( + table_fields: &[DataField], + file_schema: &TableSchema, + file: &DataFileMeta, +) -> paimon::Result { + let physical_fields = file_schema.data_file_fields(file.write_cols.as_deref()); + let data_fields = physical_fields.as_slice(); let field_mapping = table_to_data_field_mapping(table_fields, data_fields); let dense_mapping = dense_stats_mapping(data_fields, file.value_stats_cols.as_deref()); let min_row = decode_stats_row(file.value_stats.min_values())?; @@ -302,11 +352,21 @@ async fn format_value_stats( .unwrap_or_else(|| table_field.data_type()); lower_bounds.insert( table_field.name().to_string(), - format_stats_value(min_row.as_ref(), stats_index, value_type)?, + format_stats_value( + min_row.as_ref(), + stats_index, + value_type, + table_field.data_type(), + )?, ); upper_bounds.insert( table_field.name().to_string(), - format_stats_value(max_row.as_ref(), stats_index, value_type)?, + format_stats_value( + max_row.as_ref(), + stats_index, + value_type, + table_field.data_type(), + )?, ); } @@ -389,6 +449,7 @@ fn format_stats_value( row: Option<&BinaryRow>, stats_index: Option, data_type: &paimon::spec::DataType, + target_type: &paimon::spec::DataType, ) -> paimon::Result> { let Some(row) = row else { return Ok(None); @@ -399,7 +460,24 @@ fn format_stats_value( if stats_index >= row.arity() as usize { return Ok(None); } - format_row_field_as_java_cast_string(row, stats_index, data_type) + if data_type == target_type { + return format_row_field_as_java_object_string(row, stats_index, data_type); + } + let source = row.field_to_arrow(stats_index, data_type)?; + let target = paimon::arrow::cast_array_for_schema_evolution(&source, data_type, target_type)?; + let fields = [DataField::new(0, "value".to_string(), target_type.clone())]; + let schema = Arc::new(Schema::new(vec![Field::new( + "value", + target.data_type().clone(), + true, + )])); + let batch = + RecordBatch::try_new(schema, vec![target]).map_err(|error| paimon::Error::DataInvalid { + message: format!("Invalid evolved file statistics: {error}"), + source: Some(Box::new(error)), + })?; + let row = BinaryRow::from_arrow(&batch, 0, &[0], &fields)?; + format_row_field_as_java_object_string(&row, 0, target_type) } fn format_java_map(map: &BTreeMap, value_to_string: F) -> String @@ -419,6 +497,14 @@ where out } +fn java_file_path(path: &str) -> String { + if let Some((scheme, tail)) = path.split_once(":///") { + format!("{scheme}:/{tail}") + } else { + path.to_string() + } +} + fn data_file_format_identifier(file_name: &str) -> paimon::Result { let Some(dot) = file_name.rfind('.') else { return Err(paimon::Error::DataInvalid { @@ -455,18 +541,39 @@ fn file_source_to_string(file_source: Option) -> Option { }) } +fn common_file_columns(rows: &[FileRow]) -> Vec { + vec![ + Arc::new(StringArray::from_iter( + rows.iter().map(|r| r.partition.as_deref()), + )), + Arc::new(Int32Array::from_iter_values(rows.iter().map(|r| r.bucket))), + Arc::new(StringArray::from_iter_values( + rows.iter().map(|r| r.file_path.as_str()), + )), + Arc::new(StringArray::from_iter_values( + rows.iter().map(|r| r.file_format.as_str()), + )), + Arc::new(Int64Array::from_iter_values( + rows.iter().map(|r| r.schema_id), + )), + Arc::new(Int32Array::from_iter_values(rows.iter().map(|r| r.level))), + Arc::new(Int64Array::from_iter_values( + rows.iter().map(|r| r.record_count), + )), + Arc::new(Int64Array::from_iter_values( + rows.iter().map(|r| r.file_size_in_bytes), + )), + Arc::new(StringArray::from_iter( + rows.iter().map(|r| r.min_key.as_deref()), + )), + Arc::new(StringArray::from_iter( + rows.iter().map(|r| r.max_key.as_deref()), + )), + ] +} + fn file_rows_to_record_batch(rows: &[FileRow]) -> DFResult { let n = rows.len(); - let mut partitions = Vec::with_capacity(n); - let mut buckets = Vec::with_capacity(n); - let mut file_paths = Vec::with_capacity(n); - let mut file_formats = Vec::with_capacity(n); - let mut schema_ids = Vec::with_capacity(n); - let mut levels = Vec::with_capacity(n); - let mut record_counts = Vec::with_capacity(n); - let mut file_sizes = Vec::with_capacity(n); - let mut min_keys = Vec::with_capacity(n); - let mut max_keys = Vec::with_capacity(n); let mut null_value_counts = Vec::with_capacity(n); let mut min_value_stats = Vec::with_capacity(n); let mut max_value_stats = Vec::with_capacity(n); @@ -479,19 +586,14 @@ fn file_rows_to_record_batch(rows: &[FileRow]) -> DFResult { let mut write_cols = ListBuilder::new(StringBuilder::new()); for row in rows { - partitions.push(row.partition.clone()); - buckets.push(row.bucket); - file_paths.push(row.file_path.clone()); - file_formats.push(row.file_format.clone()); - schema_ids.push(row.schema_id); - levels.push(row.level); - record_counts.push(row.record_count); - file_sizes.push(row.file_size_in_bytes); - min_keys.push(row.min_key.clone()); - max_keys.push(row.max_key.clone()); - null_value_counts.push(row.null_value_counts.clone()); - min_value_stats.push(row.min_value_stats.clone()); - max_value_stats.push(row.max_value_stats.clone()); + let stats = row.stats.as_ref().ok_or_else(|| { + datafusion::error::DataFusionError::Internal( + "Files table requires value statistics".to_string(), + ) + })?; + null_value_counts.push(stats.null_value_counts.clone()); + min_value_stats.push(stats.min_value_stats.clone()); + max_value_stats.push(stats.max_value_stats.clone()); min_sequence_numbers.push(row.min_sequence_number); max_sequence_numbers.push(row.max_sequence_number); creation_times.push(row.creation_time); @@ -509,37 +611,136 @@ fn file_rows_to_record_batch(rows: &[FileRow]) -> DFResult { } } - Ok(RecordBatch::try_new( - files_schema(), - vec![ - Arc::new(StringArray::from(partitions)), - Arc::new(Int32Array::from(buckets)), - Arc::new(StringArray::from(file_paths)), - Arc::new(StringArray::from(file_formats)), - Arc::new(Int64Array::from(schema_ids)), - Arc::new(Int32Array::from(levels)), - Arc::new(Int64Array::from(record_counts)), - Arc::new(Int64Array::from(file_sizes)), - Arc::new(StringArray::from(min_keys)), - Arc::new(StringArray::from(max_keys)), - Arc::new(StringArray::from(null_value_counts)), - Arc::new(StringArray::from(min_value_stats)), - Arc::new(StringArray::from(max_value_stats)), - Arc::new(Int64Array::from(min_sequence_numbers)), - Arc::new(Int64Array::from(max_sequence_numbers)), - Arc::new(TimestampMillisecondArray::from(creation_times)), - Arc::new(Int64Array::from(delete_row_counts)), - Arc::new(StringArray::from(file_sources)), - Arc::new(Int64Array::from(first_row_ids)), - Arc::new(write_cols.finish()), - ], - )?) + let mut columns = common_file_columns(rows); + columns.extend(vec![ + Arc::new(StringArray::from(null_value_counts)) as ArrayRef, + Arc::new(StringArray::from(min_value_stats)), + Arc::new(StringArray::from(max_value_stats)), + Arc::new(Int64Array::from(min_sequence_numbers)), + Arc::new(Int64Array::from(max_sequence_numbers)), + Arc::new(TimestampMillisecondArray::from(creation_times)), + Arc::new(Int64Array::from(delete_row_counts)), + Arc::new(StringArray::from(file_sources)), + Arc::new(Int64Array::from(first_row_ids)), + Arc::new(write_cols.finish()), + ]); + Ok(RecordBatch::try_new(files_schema(), columns)?) } #[cfg(test)] mod tests { use super::*; + fn stats_file(columns: Option>) -> DataFileMeta { + use paimon::spec::{BinaryTableStats, DataType, Datum, IntType}; + let typ = DataType::Int(IntType::new()); + let value = Datum::Int(42); + let row = BinaryRow::from_datums(&[(Some(&value), &typ)]).to_serialized_bytes(); + DataFileMeta { + file_name: "data.parquet".to_string(), + file_size: 100, + row_count: 2, + min_key: vec![], + max_key: vec![], + key_stats: BinaryTableStats::empty(), + value_stats: BinaryTableStats::new(row.clone(), row, vec![Some(0)]), + min_sequence_number: 0, + max_sequence_number: 1, + schema_id: 0, + level: 0, + extra_files: vec![], + creation_time: None, + delete_row_count: None, + embedded_index: None, + file_source: None, + value_stats_cols: None, + external_path: None, + first_row_id: None, + write_cols: columns, + column_max_sequence_numbers: None, + } + } + + #[test] + fn file_stats_follow_physical_columns_and_schema_casts() { + use paimon::spec::{DataType, DecimalType, IntType, Schema as PaimonSchema}; + let schema = PaimonSchema::builder() + .column("a", DataType::Int(IntType::new())) + .column("b", DataType::Int(IntType::new())) + .build() + .unwrap(); + let schema = TableSchema::new(0, &schema); + let mut file = stats_file(Some(vec!["b".to_string()])); + let result = format_file_value_stats(schema.fields(), &schema, &file).unwrap(); + assert_eq!(result.min_value_stats, "{a=null, b=42}"); + assert_eq!(result.null_value_counts, "{a=2, b=0}"); + // Missing dense stats are unknown; missing physical fields are all NULL. + file.value_stats_cols = Some(vec![]); + let result = format_file_value_stats(schema.fields(), &schema, &file).unwrap(); + assert_eq!(result.min_value_stats, "{a=null, b=null}"); + assert_eq!(result.null_value_counts, "{a=2, b=null}"); + file.value_stats_cols = None; + let target = vec![DataField::new( + 1, + "renamed".to_string(), + DataType::Decimal(DecimalType::new(10, 2).unwrap()), + )]; + let result = format_file_value_stats(&target, &schema, &file).unwrap(); + assert_eq!(result.min_value_stats, "{renamed=42.00}"); + } + + #[test] + fn stats_cast_overflow_is_nullable_even_for_a_required_field() { + use paimon::spec::{DataType, DecimalType, IntType}; + let source_type = DataType::Int(IntType::new()); + let value = paimon::spec::Datum::Int(42); + let row = BinaryRow::from_datums(&[(Some(&value), &source_type)]); + let target = DataType::Decimal(DecimalType::with_nullable(false, 1, 0).unwrap()); + assert_eq!( + format_stats_value(Some(&row), Some(0), &source_type, &target).unwrap(), + None + ); + } + + #[test] + fn key_range_batch_does_not_require_value_statistics() { + let row = FileRow { + partition: Some("{}".into()), + bucket: 0, + file_path: "s3://bucket/data.parquet".into(), + file_format: "parquet".into(), + schema_id: 0, + level: 0, + record_count: 2, + file_size_in_bytes: 100, + min_key: Some("[1, a]".into()), + max_key: Some("[2, b]".into()), + stats: None, + min_sequence_number: None, + max_sequence_number: None, + creation_time: None, + delete_row_count: None, + file_source: None, + first_row_id: Some(8), + write_cols: None, + }; + let rows = [row]; + let mut columns = common_file_columns(&rows); + columns.push(Arc::new(Int64Array::from(vec![Some(8)]))); + let batch = RecordBatch::try_new(key_ranges_schema(), columns).unwrap(); + assert_eq!(batch.num_rows(), 1); + assert_eq!(batch.num_columns(), 11); + assert!(file_rows_to_record_batch(&rows).is_err()); + assert_eq!( + java_file_path("file:///tmp/data.parquet"), + "file:/tmp/data.parquet" + ); + assert_eq!( + java_file_path("s3://bucket/data.parquet"), + "s3://bucket/data.parquet" + ); + } + #[test] fn test_data_file_format_identifier() { assert_eq!( diff --git a/crates/integrations/datafusion/src/system_tables/mod.rs b/crates/integrations/datafusion/src/system_tables/mod.rs index 0a206fdf7..29539e14d 100644 --- a/crates/integrations/datafusion/src/system_tables/mod.rs +++ b/crates/integrations/datafusion/src/system_tables/mod.rs @@ -47,6 +47,22 @@ mod snapshots; mod table_indexes; mod tags; +/// Metadata instants are exposed as Java TIMESTAMP in the system default zone. +/// Reuse the core LTZ cast so DST, negative and extreme epochs use reader rules. +fn local_timestamp_column(values: Vec) -> DFResult { + use datafusion::arrow::array::{ArrayRef, TimestampMillisecondArray}; + use paimon::spec::{DataType, LocalZonedTimestampType, TimestampType}; + let array: ArrayRef = Arc::new(TimestampMillisecondArray::from(values)); + paimon::arrow::cast_array_for_schema_evolution( + &array, + &DataType::LocalZonedTimestamp( + LocalZonedTimestampType::new(3).map_err(to_datafusion_error)?, + ), + &DataType::Timestamp(TimestampType::new(3).map_err(to_datafusion_error)?), + ) + .map_err(to_datafusion_error) +} + type Builder = fn(Table) -> DFResult>; // Most system tables only need the base `Table`. `partitions` is special-cased @@ -59,6 +75,7 @@ const TABLES: &[(&str, Builder)] = &[ ("buckets", buckets::build), ("consumers", consumers::build), ("files", files::build), + ("file_key_ranges", files::build_key_ranges), ("manifests", manifests::build), ("options", options::build), ("physical_files_size", physical_files_size::build), @@ -76,6 +93,7 @@ const SYSTEM_TABLE_NAMES: &[&str] = &[ "buckets", "consumers", "files", + "file_key_ranges", "manifests", "options", "partitions", @@ -136,6 +154,27 @@ fn wrap_to_system_table(name: &str, base_table: Table) -> Option DFResult { + crate::table_loader::ensure_paimon_served(&table, table.identifier())?; + paimon::spec::CoreOptions::new(table.schema().options()) + .ensure_read_authorized() + .map_err(to_datafusion_error)?; + let provider = wrap_to_system_table(name, table).ok_or_else(|| { + DataFusionError::Plan(format!( + "System table '{name}' requires a catalog or is unknown" + )) + })??; + datafusion::prelude::SessionContext::new() + .read_table(provider)? + .execute_stream() + .await +} + pub(crate) fn provider_for_table( catalog: Arc, identifier: Identifier, diff --git a/crates/integrations/datafusion/src/system_tables/options.rs b/crates/integrations/datafusion/src/system_tables/options.rs index 04d85f87f..a32da351e 100644 --- a/crates/integrations/datafusion/src/system_tables/options.rs +++ b/crates/integrations/datafusion/src/system_tables/options.rs @@ -68,8 +68,15 @@ impl TableProvider for OptionsTable { _filters: &[Expr], _limit: Option, ) -> DFResult> { + let manager = self.table.schema_manager(); + let latest = crate::runtime::await_with_runtime(async move { manager.latest().await }) + .await + .map_err(crate::error::to_datafusion_error)? + .ok_or_else(|| { + datafusion::error::DataFusionError::Plan("Table does not exist".to_string()) + })?; // Java uses LinkedHashMap insertion order; HashMap has none — sort for stable output. - let mut entries: Vec<(&String, &String)> = self.table.schema().options().iter().collect(); + let mut entries: Vec<(&String, &String)> = latest.options().iter().collect(); entries.sort_by(|a, b| a.0.cmp(b.0)); let keys = StringArray::from_iter_values(entries.iter().map(|(k, _)| k.as_str())); diff --git a/crates/integrations/datafusion/src/system_tables/row_string_cast.rs b/crates/integrations/datafusion/src/system_tables/row_string_cast.rs index 11c1d46a5..1bcfa503a 100644 --- a/crates/integrations/datafusion/src/system_tables/row_string_cast.rs +++ b/crates/integrations/datafusion/src/system_tables/row_string_cast.rs @@ -17,7 +17,7 @@ use std::num::NonZeroI32; -use chrono::{Local, NaiveDate, NaiveDateTime, TimeZone, Timelike}; +use chrono::NaiveDate; use lexical_write_float::{format::STANDARD, Options, ToLexicalWithOptions}; use paimon::spec::{BinaryRow, DataField, DataType, Datum}; use paimon::{Error, Result}; @@ -58,13 +58,15 @@ pub(super) fn format_row_as_java_array_string( if pos > 0 { out.push_str(", "); } - out.push_str(&format_field(row, pos, field.data_type())?); + out.push_str( + &format_object_field(row, pos, field.data_type())?.unwrap_or_else(|| "null".into()), + ); } out.push(']'); Ok(out) } -pub(super) fn format_row_field_as_java_cast_string( +pub(super) fn format_row_field_as_java_object_string( row: &BinaryRow, pos: usize, data_type: &DataType, @@ -73,7 +75,26 @@ pub(super) fn format_row_field_as_java_cast_string( if row.is_null_at(pos) { return Ok(None); } - format_field(row, pos, data_type).map(Some) + format_object_field(row, pos, data_type) +} + +fn format_object_field( + row: &BinaryRow, + pos: usize, + data_type: &DataType, +) -> Result> { + let value = match row.get_datum(pos, data_type)? { + None => return Ok(None), + Some(Datum::Date(value) | Datum::Time(value)) => value.to_string(), + Some(Datum::Timestamp { millis, nanos } | Datum::LocalZonedTimestamp { millis, nanos }) => { + paimon::common::format_iso_timestamp( + millis.div_euclid(1000), + (millis.rem_euclid(1000) * 1_000_000 + i64::from(nanos)) as u32, + ) + } + Some(_) => format_field(row, pos, data_type)?, + }; + Ok(Some(value)) } fn validate_row(row: &BinaryRow, fields: &[DataField]) -> Result<()> { @@ -154,11 +175,18 @@ fn format_field(row: &BinaryRow, pos: usize, data_type: &DataType) -> Result Ok(format_decimal_plain(unscaled, scale)), - (Datum::Timestamp { millis, nanos }, DataType::Timestamp(t)) => { - format_timestamp(millis, nanos, t.precision()) - } - (Datum::LocalZonedTimestamp { millis, nanos }, DataType::LocalZonedTimestamp(t)) => { - format_local_zoned_timestamp(millis, nanos, t.precision()) + (Datum::Timestamp { .. } | Datum::LocalZonedTimestamp { .. }, _) => { + let source = row.field_to_arrow(pos, data_type)?; + let target = DataType::VarChar(paimon::spec::VarCharType::string_type()); + let strings = + paimon::arrow::cast_array_for_schema_evolution(&source, data_type, &target)?; + let strings = strings + .as_any() + .downcast_ref::() + .ok_or_else(|| { + data_invalid("Timestamp string cast returned a non-string array".into()) + })?; + Ok(strings.value(0).to_string()) } (datum, _) => Err(data_invalid(format!( "Decoded row string cast datum {datum:?} does not match type {data_type:?}" @@ -264,76 +292,6 @@ fn format_decimal_plain(unscaled: i128, scale: u32) -> String { } } -fn format_timestamp(millis: i64, nano_of_milli: i32, precision: u32) -> Result { - format_timestamp_naive(millis_to_naive_datetime(millis, nano_of_milli)?, precision) -} - -fn format_local_zoned_timestamp(millis: i64, nano_of_milli: i32, precision: u32) -> Result { - let nanos = timestamp_nanos(millis, nano_of_milli)?; - let secs = millis.div_euclid(1000); - let local = Local - .timestamp_opt(secs, nanos) - .single() - .ok_or_else(|| data_invalid(format!("Invalid local zoned timestamp millis {millis}")))?; - format_timestamp_naive(local.naive_local(), precision) -} - -fn format_timestamp_naive(dt: NaiveDateTime, precision: u32) -> Result { - let precision = usize::try_from(precision).map_err(|e| Error::DataInvalid { - message: format!("Timestamp row string cast precision {precision} is invalid"), - source: Some(Box::new(e)), - })?; - if precision > 9 { - return Err(data_invalid(format!( - "Timestamp row string cast precision {precision} is outside 0..=9" - ))); - } - - let mut out = dt.format("%Y-%m-%d %H:%M:%S").to_string(); - if precision > 0 { - let fraction = format!("{:09}", dt.nanosecond()); - out.push('.'); - out.push_str(&fraction[..precision]); - } - Ok(out) -} - -fn millis_to_naive_datetime(millis: i64, nano_of_milli: i32) -> Result { - let nanos = timestamp_nanos(millis, nano_of_milli)?; - let days = millis.div_euclid(MILLIS_PER_DAY); - let millis_of_day = millis.rem_euclid(MILLIS_PER_DAY) as u64; - let nano_of_day = millis_of_day * 1_000_000 + u64::from(nanos % 1_000_000); - let ce_days = days.checked_add(719_163).ok_or_else(|| { - data_invalid(format!( - "Timestamp row string cast millis {millis} is outside supported range" - )) - })?; - let ce_days = i32::try_from(ce_days).map_err(|e| Error::DataInvalid { - message: format!("Timestamp row string cast millis {millis} is outside supported range"), - source: Some(Box::new(e)), - })?; - let date = NaiveDate::from_num_days_from_ce_opt(ce_days).ok_or_else(|| { - data_invalid(format!( - "Timestamp row string cast millis {millis} is outside supported range" - )) - })?; - let time = chrono::NaiveTime::from_num_seconds_from_midnight_opt( - (nano_of_day / 1_000_000_000) as u32, - (nano_of_day % 1_000_000_000) as u32, - ) - .ok_or_else(|| data_invalid(format!("Invalid timestamp millis {millis}")))?; - Ok(NaiveDateTime::new(date, time)) -} - -fn timestamp_nanos(millis: i64, nano_of_milli: i32) -> Result { - if !(0..=999_999).contains(&nano_of_milli) { - return Err(data_invalid(format!( - "Timestamp nano-of-millisecond {nano_of_milli} is outside 0..=999999" - ))); - } - Ok(millis.rem_euclid(1000) as u32 * 1_000_000 + nano_of_milli as u32) -} - fn data_invalid(message: String) -> Error { Error::DataInvalid { message, @@ -344,6 +302,7 @@ fn data_invalid(message: String) -> Error { #[cfg(test)] mod tests { use super::*; + use chrono::{Local, TimeZone}; use paimon::spec::{ BigIntType, BinaryType, BlobType, BooleanType, CharType, DateType, DecimalType, DoubleType, FloatType, IntType, LocalZonedTimestampType, SmallIntType, TimeType, TimestampType, @@ -540,6 +499,51 @@ mod tests { ); } + #[test] + fn metadata_objects_use_internal_values_and_iso_timestamps() { + let timestamp_type = DataType::Timestamp(TimestampType::new(3).unwrap()); + let date_type = DataType::Date(DateType::new()); + let time_type = DataType::Time(TimeType::new(3).unwrap()); + let values = [ + (Some(Datum::Date(1)), date_type), + (Some(Datum::Time(42)), time_type), + ( + Some(Datum::Timestamp { + millis: 123, + nanos: 0, + }), + timestamp_type, + ), + ( + Some(Datum::LocalZonedTimestamp { + millis: 0, + nanos: 0, + }), + DataType::LocalZonedTimestamp(LocalZonedTimestampType::new(3).unwrap()), + ), + ]; + let datums: Vec<_> = values + .iter() + .map(|(value, typ)| (value.as_ref(), typ)) + .collect(); + let row = BinaryRow::from_datums(&datums); + let fields: Vec<_> = values + .iter() + .enumerate() + .map(|(i, (_, typ))| DataField::new(i as i32, i.to_string(), typ.clone())) + .collect(); + assert_eq!( + format_row_as_java_array_string(&row, &fields).unwrap(), + "[1, 42, 1970-01-01T00:00:00.123, 1970-01-01T00:00]" + ); + assert_eq!( + format_row_field_as_java_object_string(&row, 2, fields[2].data_type()) + .unwrap() + .as_deref(), + Some("1970-01-01T00:00:00.123") + ); + } + #[test] fn test_format_unsupported_type_returns_error() { let data_type = DataType::Blob(BlobType::new()); diff --git a/crates/integrations/datafusion/src/system_tables/schemas.rs b/crates/integrations/datafusion/src/system_tables/schemas.rs index 7575b3b02..337f2a6af 100644 --- a/crates/integrations/datafusion/src/system_tables/schemas.rs +++ b/crates/integrations/datafusion/src/system_tables/schemas.rs @@ -20,7 +20,7 @@ use std::sync::{Arc, OnceLock}; use async_trait::async_trait; -use datafusion::arrow::array::{Int64Array, RecordBatch, StringArray, TimestampMillisecondArray}; +use datafusion::arrow::array::{Int64Array, RecordBatch, StringArray}; use datafusion::arrow::datatypes::{DataType, Field, Schema, SchemaRef, TimeUnit}; use datafusion::catalog::Session; use datafusion::datasource::memory::MemorySourceConfig; @@ -117,7 +117,7 @@ impl TableProvider for SchemasTable { Arc::new(StringArray::from(primary_keys_json)), Arc::new(StringArray::from(options_json)), Arc::new(StringArray::from(comments)), - Arc::new(TimestampMillisecondArray::from(update_times)), + super::local_timestamp_column(update_times)?, ], )?; diff --git a/crates/integrations/datafusion/src/system_tables/snapshots.rs b/crates/integrations/datafusion/src/system_tables/snapshots.rs index 040c51c38..f8355b15c 100644 --- a/crates/integrations/datafusion/src/system_tables/snapshots.rs +++ b/crates/integrations/datafusion/src/system_tables/snapshots.rs @@ -20,7 +20,7 @@ use std::sync::{Arc, OnceLock}; use async_trait::async_trait; -use datafusion::arrow::array::{Int64Array, RecordBatch, StringArray, TimestampMillisecondArray}; +use datafusion::arrow::array::{Int64Array, RecordBatch, StringArray}; use datafusion::arrow::datatypes::{DataType, Field, Schema, SchemaRef, TimeUnit}; use datafusion::catalog::Session; use datafusion::datasource::memory::MemorySourceConfig; @@ -59,6 +59,7 @@ fn snapshots_schema() -> SchemaRef { Field::new("changelog_record_count", DataType::Int64, true), Field::new("watermark", DataType::Int64, true), Field::new("next_row_id", DataType::Int64, true), + Field::new("writer_version", DataType::Utf8, true), ])) }) .clone() @@ -107,6 +108,7 @@ impl TableProvider for SnapshotsTable { let mut watermarks: Vec> = Vec::with_capacity(n); let mut next_row_ids: Vec> = Vec::with_capacity(n); + let mut writer_versions = Vec::with_capacity(n); for snap in &snapshots { snapshot_ids.push(snap.id()); schema_ids.push(snap.schema_id()); @@ -122,6 +124,7 @@ impl TableProvider for SnapshotsTable { changelog_record_counts.push(snap.changelog_record_count()); watermarks.push(snap.watermark()); next_row_ids.push(snap.next_row_id()); + writer_versions.push(snap.writer_version()); } let schema = snapshots_schema(); @@ -133,7 +136,7 @@ impl TableProvider for SnapshotsTable { Arc::new(StringArray::from(commit_users)), Arc::new(Int64Array::from(commit_identifiers)), Arc::new(StringArray::from(commit_kinds)), - Arc::new(TimestampMillisecondArray::from(commit_times)), + super::local_timestamp_column(commit_times)?, Arc::new(StringArray::from(base_manifest_lists)), Arc::new(StringArray::from(delta_manifest_lists)), Arc::new(StringArray::from(changelog_manifest_lists)), @@ -142,6 +145,7 @@ impl TableProvider for SnapshotsTable { Arc::new(Int64Array::from(changelog_record_counts)), Arc::new(Int64Array::from(watermarks)), Arc::new(Int64Array::from(next_row_ids)), + Arc::new(StringArray::from(writer_versions)), ], )?; diff --git a/crates/integrations/datafusion/src/system_tables/table_indexes.rs b/crates/integrations/datafusion/src/system_tables/table_indexes.rs index c51d3e55b..b81fa825d 100644 --- a/crates/integrations/datafusion/src/system_tables/table_indexes.rs +++ b/crates/integrations/datafusion/src/system_tables/table_indexes.rs @@ -145,12 +145,7 @@ impl TableProvider for TableIndexesTable { row_range_starts.push(Some(global_meta.row_range_start)); row_range_ends.push(Some(global_meta.row_range_end)); index_field_ids.push(Some(global_meta.index_field_id)); - index_field_names.push( - fields - .iter() - .find(|field| field.id() == global_meta.index_field_id) - .map(|field| field.name().to_string()), - ); + index_field_names.push(global_index_field_names(global_meta, fields)); } else { row_range_starts.push(None); row_range_ends.push(None); @@ -212,6 +207,22 @@ fn visible_index_entries(entries: Vec) -> Vec Option { + let names: Option> = std::iter::once(&meta.index_field_id) + .chain(meta.extra_field_ids.iter().flatten()) + .map(|id| { + fields + .iter() + .find(|field| field.id() == *id) + .map(DataField::name) + }) + .collect(); + names.map(|names| names.join(",")) +} + fn format_partition(partition: &[u8], partition_fields: &[DataField]) -> DFResult { let row = BinaryRow::from_serialized_bytes(partition).map_err(to_datafusion_error)?; format_row_as_java_cast_string(&row, partition_fields).map_err(to_datafusion_error) @@ -291,6 +302,38 @@ mod tests { } } + #[test] + fn composite_index_names_preserve_order_and_require_every_field() { + use paimon::spec::{GlobalIndexMeta, IntType}; + let fields = vec![ + DataField::new(3, "a".into(), paimon::spec::DataType::Int(IntType::new())), + DataField::new( + 7, + "renamed".into(), + paimon::spec::DataType::Int(IntType::new()), + ), + ]; + let mut meta = GlobalIndexMeta { + row_range_start: 0, + row_range_end: 1, + index_field_id: 7, + extra_field_ids: Some(vec![3]), + index_meta: None, + source_meta: None, + }; + assert_eq!( + global_index_field_names(&meta, &fields).as_deref(), + Some("renamed,a") + ); + meta.extra_field_ids = Some(vec![99]); + assert_eq!(global_index_field_names(&meta, &fields), None); + meta.extra_field_ids = None; + assert_eq!( + global_index_field_names(&meta, &fields).as_deref(), + Some("renamed") + ); + } + #[test] fn visible_index_entries_skips_delete_entries() { let entries = vec![ diff --git a/crates/integrations/datafusion/src/system_tables/tags.rs b/crates/integrations/datafusion/src/system_tables/tags.rs index 433d59f17..02ee371fc 100644 --- a/crates/integrations/datafusion/src/system_tables/tags.rs +++ b/crates/integrations/datafusion/src/system_tables/tags.rs @@ -115,7 +115,7 @@ impl TableProvider for TagsTable { Arc::new(StringArray::from(tag_names)), Arc::new(Int64Array::from(snapshot_ids)), Arc::new(Int64Array::from(schema_ids)), - Arc::new(TimestampMillisecondArray::from(commit_times)), + super::local_timestamp_column(commit_times)?, Arc::new(Int64Array::from(record_counts)), Arc::new(TimestampMillisecondArray::from(create_times)), Arc::new(StringArray::from(time_retained)), diff --git a/crates/integrations/datafusion/tests/system_tables.rs b/crates/integrations/datafusion/tests/system_tables.rs index 73dc6ba36..268733b30 100644 --- a/crates/integrations/datafusion/tests/system_tables.rs +++ b/crates/integrations/datafusion/tests/system_tables.rs @@ -793,6 +793,7 @@ async fn test_snapshots_system_table() { ("changelog_record_count", DataType::Int64), ("watermark", DataType::Int64), ("next_row_id", DataType::Int64), + ("writer_version", DataType::Utf8), ]; for (i, (name, dtype)) in expected_columns.iter().enumerate() { let field = arrow_schema.field(i); @@ -1329,7 +1330,7 @@ async fn test_files_system_table() { "creation_time", DataType::Timestamp(TimeUnit::Millisecond, None), ), - ("delete_row_count", DataType::Int64), + ("deleteRowCount", DataType::Int64), ("file_source", DataType::Utf8), ("first_row_id", DataType::Int64), ( @@ -1729,3 +1730,129 @@ async fn test_buckets_system_table() { "$buckets must aggregate $files by (partition, bucket)" ); } + +#[tokio::test] +async fn test_file_key_ranges_matches_file_metadata() { + use futures::TryStreamExt; + let (ctx, catalog, _tmp) = create_context().await; + let table = catalog + .get_table(&Identifier::new("default", FIXTURE_TABLE)) + .await + .unwrap(); + let expected = run_sql(&ctx, &format!("SELECT partition, bucket, file_path, file_format, schema_id, level, record_count, file_size_in_bytes, min_key, max_key, first_row_id FROM paimon.default.{FIXTURE_TABLE}$files")).await; + let actual: Vec<_> = paimon_datafusion::read_system_table(table.clone(), "file_key_ranges") + .await + .unwrap() + .try_collect() + .await + .unwrap(); + assert_eq!(actual, expected); + assert_eq!(actual[0].num_columns(), 11); + assert!(actual[0].num_rows() > 0); + let projected = run_sql(&ctx, &format!("SELECT file_path FROM paimon.default.{FIXTURE_TABLE}$file_key_ranges WHERE record_count > 0 LIMIT 1")).await; + assert_eq!(projected[0].num_rows(), 1); + assert_eq!(projected[0].num_columns(), 1); + let empty = table.copy_with_pinned_snapshot(None); + let batches: Vec<_> = paimon_datafusion::read_system_table(empty, "file_key_ranges") + .await + .unwrap() + .try_collect() + .await + .unwrap(); + assert_eq!(batches.iter().map(RecordBatch::num_rows).sum::(), 0); +} + +#[tokio::test] +async fn test_resolved_system_table_reader_preserves_guards_and_schema() { + use futures::TryStreamExt; + let (_ctx, catalog, _tmp) = create_context().await; + let table = catalog + .get_table(&Identifier::new("default", FIXTURE_TABLE)) + .await + .unwrap(); + for name in [ + "snapshots", + "schemas", + "options", + "manifests", + "files", + "buckets", + "tags", + "branches", + "consumers", + "file_key_ranges", + "table_indexes", + ] { + let stream = paimon_datafusion::read_system_table(table.clone(), name) + .await + .unwrap(); + let schema = stream.schema(); + let batches: Vec<_> = stream.try_collect().await.unwrap(); + assert!(!schema.fields().is_empty(), "{name}"); + assert!( + batches.iter().all(|batch| batch.schema() == schema), + "{name}" + ); + } + for name in ["partitions", "unknown"] { + let error = match paimon_datafusion::read_system_table(table.clone(), name).await { + Ok(_) => panic!("{name} must require a catalog or be unknown"), + Err(error) => error.to_string(), + }; + assert!(error.contains("requires a catalog or is unknown")); + } + let denied = table.copy_with_options(std::collections::HashMap::from([( + "query-auth.enabled".to_string(), + "true".to_string(), + )])); + let error = match paimon_datafusion::read_system_table(denied, "options").await { + Ok(_) => panic!("Authorization must be checked before reading metadata"), + Err(error) => error.to_string(), + }; + assert!(error.contains("query-auth.enabled")); +} + +#[tokio::test] +async fn test_options_ignore_resolved_dynamic_overrides() { + use futures::TryStreamExt; + let (_ctx, catalog, _tmp) = create_context().await; + let table = catalog + .get_table(&Identifier::new("default", FIXTURE_TABLE)) + .await + .unwrap(); + let copy = table.copy_with_options(std::collections::HashMap::from([ + ("file.format".to_string(), "csv".to_string()), + ("metadata.runtime-only".to_string(), "private".to_string()), + ])); + let batches: Vec<_> = paimon_datafusion::read_system_table(copy, "options") + .await + .unwrap() + .try_collect() + .await + .unwrap(); + let rows: std::collections::HashMap<_, _> = batches + .iter() + .flat_map(|batch| { + let keys = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let values = batch + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + (0..batch.num_rows()) + .map(|index| { + ( + keys.value(index).to_string(), + values.value(index).to_string(), + ) + }) + .collect::>() + }) + .collect(); + assert_eq!(&rows, table.schema().options()); + assert!(!rows.contains_key("metadata.runtime-only")); +} diff --git a/crates/paimon/src/common/date_time.rs b/crates/paimon/src/common/date_time.rs index d9f800b12..ecb9fd530 100644 --- a/crates/paimon/src/common/date_time.rs +++ b/crates/paimon/src/common/date_time.rs @@ -83,7 +83,7 @@ pub(crate) fn format_epoch_timestamp(seconds: i64, nanos: u32, precision: u32) - } /// Java Timestamp.toString uses LocalDateTime's ISO form, not a SQL CAST form. -pub(crate) fn format_iso_timestamp(seconds: i64, nanos: u32) -> String { +pub fn format_iso_timestamp(seconds: i64, nanos: u32) -> String { let (year, month, day) = epoch_day_to_date(i64::from(seconds.div_euclid(86_400) as i32)); let year = if (0..=9999).contains(&year) { format!("{year:04}") diff --git a/crates/paimon/src/common/mod.rs b/crates/paimon/src/common/mod.rs index 543fb6fa9..d29489f24 100644 --- a/crates/paimon/src/common/mod.rs +++ b/crates/paimon/src/common/mod.rs @@ -18,6 +18,7 @@ //! Common utilities for Paimon. pub(crate) mod date_time; +pub use date_time::format_iso_timestamp; pub(crate) mod number; pub mod options; diff --git a/crates/paimon/src/spec/binary_row.rs b/crates/paimon/src/spec/binary_row.rs index 15b81289f..98cf25a1d 100644 --- a/crates/paimon/src/spec/binary_row.rs +++ b/crates/paimon/src/spec/binary_row.rs @@ -383,6 +383,16 @@ impl BinaryRow { Ok(Some(datum)) } + /// Decode a scalar field as a single-element Arrow array. + /// Uses the same typed decoding as partition expansion in table reads. + pub fn field_to_arrow( + &self, + index: usize, + data_type: &DataType, + ) -> crate::Result { + crate::arrow::partition::partition_array(self, index, data_type, 1) + } + /// Build a BinaryRow from selected columns of an Arrow RecordBatch at a given row. /// /// `field_indices` maps each position in the output BinaryRow to a column index diff --git a/crates/paimon/src/spec/schema.rs b/crates/paimon/src/spec/schema.rs index 69b13bbe3..8970213b1 100644 --- a/crates/paimon/src/spec/schema.rs +++ b/crates/paimon/src/spec/schema.rs @@ -99,6 +99,33 @@ impl TableSchema { &self.fields } + /// Fields physically present in a file, following Java TableSchema.dataFileSchema. + /// Explicit write columns retain their order and omit metadata columns. + pub fn data_file_fields(&self, write_cols: Option<&[String]>) -> Vec { + if let Some(columns) = write_cols { + return columns + .iter() + .filter_map(|name| self.fields.iter().find(|f| f.name() == name).cloned()) + .collect(); + } + let options = CoreOptions::new(&self.options); + if !options.data_evolution_enabled() + || !options.data_evolution_write_cols_optimization_enabled() + { + return self.fields.clone(); + } + let inline = options.blob_inline_fields(); + self.fields + .iter() + .filter(|field| { + !(field.data_type().is_blob_file_field() && !inline.contains(field.name()) + || options.vector_file_format().is_some() + && matches!(field.data_type(), DataType::Vector(_))) + }) + .cloned() + .collect() + } + /// Logical fields followed by the metadata available to table reads. /// Row tracking provides ROW_ID and SEQUENCE_NUMBER; primary-key files /// provide SEQUENCE_NUMBER independently of row tracking. Keeping this diff --git a/crates/paimon/src/spec/snapshot.rs b/crates/paimon/src/spec/snapshot.rs index 70b366dbc..1af888b02 100644 --- a/crates/paimon/src/spec/snapshot.rs +++ b/crates/paimon/src/spec/snapshot.rs @@ -119,6 +119,10 @@ pub struct Snapshot { #[builder(default = None)] #[serde(skip_serializing_if = "Option::is_none")] next_row_id: Option, + /// Engine version that produced this snapshot, absent in older metadata. + #[builder(default = None)] + #[serde(default, skip_serializing_if = "Option::is_none")] + writer_version: Option, } impl Snapshot { @@ -240,6 +244,11 @@ impl Snapshot { self.next_row_id } + /// Get the writer version, when recorded by the producer. + pub fn writer_version(&self) -> Option<&str> { + self.writer_version.as_deref() + } + /// Get the commit kind of this snapshot. #[inline] pub fn commit_kind(&self) -> &CommitKind { From 08619df2a26d5940a6df95afbe8adeae5dccca90 Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Fri, 16 Oct 2026 13:27:57 +0800 Subject: [PATCH 2/3] refactor: move native system-table reads into core --- Cargo.lock | 2 +- .../python/pypaimon_rust/datafusion.pyi | 36 +- bindings/python/src/context.rs | 6 + bindings/python/src/lib.rs | 1 + bindings/python/src/system_table.rs | 214 ++++++ bindings/python/src/table.rs | 25 +- crates/integrations/datafusion/Cargo.toml | 1 - crates/integrations/datafusion/src/lib.rs | 1 - .../datafusion/src/system_tables/consumers.rs | 190 ----- .../datafusion/src/system_tables/metadata.rs | 116 +++ .../datafusion/src/system_tables/mod.rs | 98 +-- .../datafusion/src/system_tables/options.rs | 94 --- .../datafusion/src/system_tables/schemas.rs | 134 ---- .../datafusion/src/system_tables/snapshots.rs | 158 ---- .../datafusion/tests/system_tables.rs | 70 +- crates/paimon/Cargo.toml | 1 + crates/paimon/src/table/mod.rs | 10 + .../src/table/system_table}/branches.rs | 79 +- .../src/table/system_table}/buckets.rs | 80 +- .../src/table/system_table/consumers.rs | 105 +++ .../src/table/system_table}/files.rs | 164 ++--- .../src/table/system_table}/manifests.rs | 188 ++--- crates/paimon/src/table/system_table/mod.rs | 682 ++++++++++++++++++ .../paimon/src/table/system_table/options.rs | 64 ++ .../table/system_table}/row_string_cast.rs | 16 +- .../paimon/src/table/system_table/schemas.rs | 98 +++ .../src/table/system_table/snapshots.rs | 123 ++++ .../src/table/system_table}/table_indexes.rs | 206 +++--- .../src/table/system_table}/tags.rs | 126 ++-- 29 files changed, 1856 insertions(+), 1232 deletions(-) create mode 100644 bindings/python/src/system_table.rs delete mode 100644 crates/integrations/datafusion/src/system_tables/consumers.rs create mode 100644 crates/integrations/datafusion/src/system_tables/metadata.rs delete mode 100644 crates/integrations/datafusion/src/system_tables/options.rs delete mode 100644 crates/integrations/datafusion/src/system_tables/schemas.rs delete mode 100644 crates/integrations/datafusion/src/system_tables/snapshots.rs rename crates/{integrations/datafusion/src/system_tables => paimon/src/table/system_table}/branches.rs (50%) rename crates/{integrations/datafusion/src/system_tables => paimon/src/table/system_table}/buckets.rs (78%) create mode 100644 crates/paimon/src/table/system_table/consumers.rs rename crates/{integrations/datafusion/src/system_tables => paimon/src/table/system_table}/files.rs (85%) rename crates/{integrations/datafusion/src/system_tables => paimon/src/table/system_table}/manifests.rs (58%) create mode 100644 crates/paimon/src/table/system_table/mod.rs create mode 100644 crates/paimon/src/table/system_table/options.rs rename crates/{integrations/datafusion/src/system_tables => paimon/src/table/system_table}/row_string_cast.rs (97%) create mode 100644 crates/paimon/src/table/system_table/schemas.rs create mode 100644 crates/paimon/src/table/system_table/snapshots.rs rename crates/{integrations/datafusion/src/system_tables => paimon/src/table/system_table}/table_indexes.rs (59%) rename crates/{integrations/datafusion/src/system_tables => paimon/src/table/system_table}/tags.rs (58%) diff --git a/Cargo.lock b/Cargo.lock index 0efb91445..61d3be37f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4677,6 +4677,7 @@ dependencies = [ "http", "indexmap 2.14.0", "jiff", + "lexical-write-float", "libc", "libloading 0.9.0", "log", @@ -4765,7 +4766,6 @@ dependencies = [ "datafusion", "flate2", "futures", - "lexical-write-float", "log", "paimon", "paimon-ftindex-core", diff --git a/bindings/python/python/pypaimon_rust/datafusion.pyi b/bindings/python/python/pypaimon_rust/datafusion.pyi index 1c34f332f..2c53b6d2d 100644 --- a/bindings/python/python/pypaimon_rust/datafusion.pyi +++ b/bindings/python/python/pypaimon_rust/datafusion.pyi @@ -283,7 +283,7 @@ class Table: def branch(self) -> str: ... def location(self) -> str: ... def schema(self) -> TableSchema: ... - def _read_system_table(self, name: str) -> RecordBatchReader: ... + def system_table(self, name: str) -> SystemTable: ... def new_read_builder(self, options: Optional[Dict[str, str]] = None) -> ReadBuilder: ... def new_stream_scan(self) -> StreamTableScan: ... def new_full_text_search_builder(self) -> FullTextSearchBuilder: ... @@ -864,3 +864,37 @@ class ShardTableUpdator: """Require a finished read and exactly as many update rows; repeated calls reuse messages.""" ... def close(self) -> None: ... + + +class SystemTable: + def name(self) -> str: ... + def schema(self) -> pyarrow.Schema: ... + def new_read_builder(self) -> SystemReadBuilder: ... + +class SystemReadBuilder: + def with_projection(self, columns: List[str]) -> SystemReadBuilder: ... + def with_filter(self, predicate: Dict[str, Any]) -> SystemReadBuilder: ... + def with_limit(self, limit: int) -> SystemReadBuilder: ... + def new_scan(self) -> SystemTableScan: ... + def new_read(self) -> SystemTableRead: ... + +class SystemTableScan: + def plan(self) -> SystemTablePlan: ... + +class SystemTablePlan: + def splits(self) -> List[SystemSplit]: ... + +class SystemSplit: + def __init__(self, name: str, batch: pyarrow.RecordBatch) -> None: + """Restore captured metadata with validated column types and nullability.""" + ... + def row_count(self) -> int: ... + +class SystemTableRead: + def __init__(self, name: str) -> None: + """Create a reader for captured splits without accessing REST or file IO.""" + ... + def with_projection(self, columns: List[str]) -> SystemTableRead: ... + def with_filter(self, predicate: Dict[str, Any]) -> SystemTableRead: ... + def with_limit(self, limit: int) -> SystemTableRead: ... + def read(self, splits: List[SystemSplit]) -> RecordBatchReader: ... diff --git a/bindings/python/src/context.rs b/bindings/python/src/context.rs index edd492055..c6c929567 100644 --- a/bindings/python/src/context.rs +++ b/bindings/python/src/context.rs @@ -796,6 +796,12 @@ pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> let this = PyModule::new(py, "datafusion")?; this.add_class::()?; this.add_class::()?; + this.add_class::()?; + this.add_class::()?; + this.add_class::()?; + this.add_class::()?; + this.add_class::()?; + this.add_class::()?; this.add_class::()?; this.add_class::()?; this.add_class::()?; diff --git a/bindings/python/src/lib.rs b/bindings/python/src/lib.rs index 9b92ae834..0b4508c82 100644 --- a/bindings/python/src/lib.rs +++ b/bindings/python/src/lib.rs @@ -33,6 +33,7 @@ mod predicate; mod read; mod schema; mod shard_update; +mod system_table; mod table; mod udf; mod update_assignment; diff --git a/bindings/python/src/system_table.rs b/bindings/python/src/system_table.rs new file mode 100644 index 000000000..e2c7e3965 --- /dev/null +++ b/bindings/python/src/system_table.rs @@ -0,0 +1,214 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Thin bindings for the core read-only system-table API. + +use crate::error::to_py_err; +use crate::read::PyRecordBatchReader; +use arrow::pyarrow::{FromPyArrow, ToPyArrow}; +use arrow::record_batch::RecordBatch; +use futures::TryStreamExt; +use paimon::table::{ + SystemReadBuilder, SystemSplit, SystemTable, SystemTablePlan, SystemTableRead, SystemTableScan, +}; +use paimon_datafusion::runtime::runtime; +use pyo3::prelude::*; +use pyo3::types::PyDict; + +#[pyclass(name = "SystemTable", module = "pypaimon_rust.datafusion")] +pub struct PySystemTable { + pub(crate) inner: SystemTable, +} + +#[pymethods] +impl PySystemTable { + fn name(&self) -> &str { + self.inner.name() + } + fn schema(&self, py: Python<'_>) -> PyResult> { + Ok(self.inner.schema().to_pyarrow(py)?.unbind()) + } + fn new_read_builder(&self) -> PySystemReadBuilder { + PySystemReadBuilder { + inner: self.inner.new_read_builder(), + } + } +} + +#[pyclass(name = "SystemReadBuilder", module = "pypaimon_rust.datafusion")] +pub struct PySystemReadBuilder { + inner: SystemReadBuilder, +} + +#[pymethods] +impl PySystemReadBuilder { + fn with_projection( + mut slf: PyRefMut<'_, Self>, + columns: Vec, + ) -> PyResult> { + let columns: Vec<_> = columns.iter().map(String::as_str).collect(); + slf.inner.with_projection(&columns).map_err(to_py_err)?; + Ok(slf) + } + fn with_filter<'py>( + mut slf: PyRefMut<'py, Self>, + predicate: &Bound<'_, PyDict>, + ) -> PyResult> { + // Predicates must see the full system-table schema, including unprojected columns. + let predicate = crate::predicate::dict_to_predicate( + predicate, + slf.inner.full_row_type().fields(), + true, + )?; + slf.inner.with_filter(predicate).map_err(to_py_err)?; + Ok(slf) + } + fn with_limit(mut slf: PyRefMut<'_, Self>, limit: usize) -> PyRefMut<'_, Self> { + slf.inner.with_limit(limit); + slf + } + fn new_scan(&self) -> PySystemTableScan { + PySystemTableScan { + inner: self.inner.new_scan(), + } + } + fn new_read(&self) -> PySystemTableRead { + PySystemTableRead { + inner: self.inner.new_read(), + } + } +} + +#[pyclass(name = "SystemTableScan", module = "pypaimon_rust.datafusion")] +pub struct PySystemTableScan { + inner: SystemTableScan, +} + +#[pymethods] +impl PySystemTableScan { + fn plan(&self, py: Python<'_>) -> PyResult { + Ok(PySystemTablePlan { + inner: py + .detach(|| runtime().block_on(self.inner.plan())) + .map_err(to_py_err)?, + }) + } +} + +#[pyclass(name = "SystemTablePlan", module = "pypaimon_rust.datafusion")] +pub struct PySystemTablePlan { + inner: SystemTablePlan, +} + +#[pymethods] +impl PySystemTablePlan { + fn splits(&self) -> Vec { + self.inner + .splits() + .iter() + .cloned() + .map(|inner| PySystemSplit { inner }) + .collect() + } +} + +type SystemSplitPickle = (Py, (String, Py)); + +#[pyclass(name = "SystemSplit", module = "pypaimon_rust.datafusion")] +pub struct PySystemSplit { + inner: SystemSplit, +} + +#[pymethods] +impl PySystemSplit { + fn row_count(&self) -> usize { + self.inner.row_count() + } + + #[new] + fn new(name: &str, batch: &Bound<'_, PyAny>) -> PyResult { + let batch = RecordBatch::from_pyarrow_bound(batch)?; + Ok(Self { + inner: SystemSplit::from_record_batch(name, batch).map_err(to_py_err)?, + }) + } + + fn __reduce__<'py>(slf: &Bound<'py, Self>, py: Python<'py>) -> PyResult { + let inner = &slf.borrow().inner; + Ok(( + slf.get_type().unbind().into_any(), + ( + inner.name().to_string(), + inner.record_batch().to_pyarrow(py)?.unbind(), + ), + )) + } +} + +#[pyclass(name = "SystemTableRead", module = "pypaimon_rust.datafusion")] +pub struct PySystemTableRead { + inner: SystemTableRead, +} + +#[pymethods] +impl PySystemTableRead { + #[new] + fn new(name: &str) -> PyResult { + Ok(Self { + inner: SystemTableRead::new(name).map_err(to_py_err)?, + }) + } + + fn with_projection( + mut slf: PyRefMut<'_, Self>, + columns: Vec, + ) -> PyResult> { + let columns: Vec<_> = columns.iter().map(String::as_str).collect(); + slf.inner.with_projection(&columns).map_err(to_py_err)?; + Ok(slf) + } + + fn with_filter<'py>( + mut slf: PyRefMut<'py, Self>, + predicate: &Bound<'_, PyDict>, + ) -> PyResult> { + let predicate = crate::predicate::dict_to_predicate( + predicate, + slf.inner.full_row_type().fields(), + true, + )?; + slf.inner.with_filter(predicate).map_err(to_py_err)?; + Ok(slf) + } + + fn with_limit(mut slf: PyRefMut<'_, Self>, limit: usize) -> PyRefMut<'_, Self> { + slf.inner.with_limit(limit); + slf + } + + fn read(&self, splits: &Bound<'_, PyAny>) -> PyResult { + let splits = splits + .try_iter()? + .map(|item| Ok(item?.extract::>()?.inner.clone())) + .collect::>>()?; + let stream = self.inner.read(&splits).map_err(to_py_err)?; + Ok(PyRecordBatchReader::from_py_stream( + Some(self.inner.schema()), + Box::pin(stream.map_err(to_py_err)), + )) + } +} diff --git a/bindings/python/src/table.rs b/bindings/python/src/table.rs index f70d9465c..754430126 100644 --- a/bindings/python/src/table.rs +++ b/bindings/python/src/table.rs @@ -18,7 +18,6 @@ use std::collections::HashMap; use std::sync::Arc; -use futures::TryStreamExt; use paimon::catalog::{Identifier, RESTToken}; use paimon::io::FileIO; use paimon::spec::TableSchema; @@ -29,9 +28,9 @@ use pyo3::prelude::*; use pyo3::types::PyDict; use crate::context::complete_storage_options; -use crate::error::{df_to_py_err, to_py_err}; +use crate::error::to_py_err; use crate::partition::PyPartitionStat; -use crate::read::{PyReadBuilder, PyRecordBatchReader}; +use crate::read::PyReadBuilder; use crate::schema::PyTableSchema; use crate::snapshot::PySnapshot; use crate::tag::PyTag; @@ -199,21 +198,11 @@ impl PyTable { } } - /// Read metadata through the existing system-table providers. - fn _read_system_table(&self, py: Python<'_>, name: &str) -> PyResult { - let stream = py - .detach(|| { - runtime().block_on(paimon_datafusion::read_system_table( - self.inner.as_ref().clone(), - name, - )) - }) - .map_err(df_to_py_err)?; - let schema = stream.schema(); - Ok(PyRecordBatchReader::from_py_stream( - Some(schema), - Box::pin(stream.map_err(df_to_py_err)), - )) + /// Resolve a core metadata table retaining this table's captured environment. + fn system_table(&self, name: &str) -> PyResult { + Ok(crate::system_table::PySystemTable { + inner: self.inner.system_table(name).map_err(to_py_err)?, + }) } fn new_full_text_search_builder(&self) -> crate::full_text_search::PyFullTextSearchBuilder { diff --git a/crates/integrations/datafusion/Cargo.toml b/crates/integrations/datafusion/Cargo.toml index dfca7278e..b3b33a99c 100644 --- a/crates/integrations/datafusion/Cargo.toml +++ b/crates/integrations/datafusion/Cargo.toml @@ -45,7 +45,6 @@ futures = "0.3" serde = { version = "1", features = ["derive"] } serde_json = "1" tokio = { workspace = true, features = ["rt", "rt-multi-thread", "time", "fs"] } -lexical-write-float = "1.0.6" uuid = { version = "1", features = ["v4"] } [dev-dependencies] diff --git a/crates/integrations/datafusion/src/lib.rs b/crates/integrations/datafusion/src/lib.rs index a50235963..cfab74c0e 100644 --- a/crates/integrations/datafusion/src/lib.rs +++ b/crates/integrations/datafusion/src/lib.rs @@ -93,7 +93,6 @@ pub use merge_condition::compile_merge_condition; pub use physical_plan::PaimonTableScan; pub use relation_planner::PaimonRelationPlanner; pub use sql_context::{SQLContext, SQLContextBuilder}; -pub use system_tables::read_system_table; pub use table::PaimonTableProvider; pub use variant_functions::register_variant_functions; pub use vector_search::{register_vector_search, VectorSearchFunction}; diff --git a/crates/integrations/datafusion/src/system_tables/consumers.rs b/crates/integrations/datafusion/src/system_tables/consumers.rs deleted file mode 100644 index 40c922cee..000000000 --- a/crates/integrations/datafusion/src/system_tables/consumers.rs +++ /dev/null @@ -1,190 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! Mirrors Java [ConsumersTable](https://github.com/apache/paimon/blob/release-1.3/paimon-core/src/main/java/org/apache/paimon/table/system/ConsumersTable.java). - -use std::collections::HashSet; -use std::sync::{Arc, OnceLock}; - -use async_trait::async_trait; -use datafusion::arrow::array::{Int64Array, RecordBatch, StringArray}; -use datafusion::arrow::datatypes::{DataType, Field, Schema, SchemaRef}; -use datafusion::catalog::Session; -use datafusion::common::ScalarValue; -use datafusion::datasource::memory::MemorySourceConfig; -use datafusion::datasource::{TableProvider, TableType}; -use datafusion::error::Result as DFResult; -use datafusion::logical_expr::{Expr, Operator, TableProviderFilterPushDown}; -use datafusion::physical_plan::ExecutionPlan; -use paimon::table::Table; - -use crate::error::to_datafusion_error; - -pub(super) fn build(table: Table) -> DFResult> { - Ok(Arc::new(ConsumersTable { table })) -} - -fn consumers_schema() -> SchemaRef { - static SCHEMA: OnceLock = OnceLock::new(); - SCHEMA - .get_or_init(|| { - Arc::new(Schema::new(vec![ - Field::new("consumer_id", DataType::Utf8, false), - Field::new("next_snapshot_id", DataType::Int64, false), - ])) - }) - .clone() -} - -#[derive(Debug)] -pub(super) struct ConsumersTable { - table: Table, -} - -#[async_trait] -impl TableProvider for ConsumersTable { - fn schema(&self) -> SchemaRef { - consumers_schema() - } - - fn table_type(&self) -> TableType { - TableType::View - } - - async fn scan( - &self, - _state: &dyn Session, - projection: Option<&Vec>, - filters: &[Expr], - _limit: Option, - ) -> DFResult> { - let manager = self.table.consumer_manager(); - let requested_ids = requested_consumer_ids(filters); - let consumers = crate::runtime::await_with_runtime(async move { - match requested_ids { - Some(ids) => manager.list_by_ids(&ids).await, - None => manager.list_all().await, - } - }) - .await - .map_err(to_datafusion_error)?; - - let schema = consumers_schema(); - let batch = RecordBatch::try_new( - schema.clone(), - vec![ - Arc::new(StringArray::from_iter_values( - consumers.iter().map(|(id, _)| id.as_str()), - )), - Arc::new(Int64Array::from_iter_values( - consumers.iter().map(|(_, next_snapshot)| *next_snapshot), - )), - ], - )?; - - Ok(MemorySourceConfig::try_new_exec( - &[vec![batch]], - schema, - projection.cloned(), - )?) - } - - fn supports_filters_pushdown( - &self, - filters: &[&Expr], - ) -> DFResult> { - Ok(filters - .iter() - .map(|filter| { - if consumer_ids_from_filter(filter).is_some() { - TableProviderFilterPushDown::Inexact - } else { - TableProviderFilterPushDown::Unsupported - } - }) - .collect()) - } -} - -fn requested_consumer_ids(filters: &[Expr]) -> Option> { - let ids = filters - .iter() - .filter_map(consumer_ids_from_filter) - .reduce(|mut left, right| { - left.retain(|id| right.contains(id)); - left - })?; - let mut ids = ids.into_iter().collect::>(); - ids.sort_unstable(); - Some(ids) -} - -fn consumer_ids_from_filter(filter: &Expr) -> Option> { - match filter { - Expr::BinaryExpr(binary) if binary.op == Operator::Eq => { - consumer_id_literal(binary.left.as_ref(), binary.right.as_ref()) - .or_else(|| consumer_id_literal(binary.right.as_ref(), binary.left.as_ref())) - .map(|id| HashSet::from([id])) - } - Expr::BinaryExpr(binary) if binary.op == Operator::And => { - match ( - consumer_ids_from_filter(binary.left.as_ref()), - consumer_ids_from_filter(binary.right.as_ref()), - ) { - (Some(mut left), Some(right)) => { - left.retain(|id| right.contains(id)); - Some(left) - } - (Some(ids), None) | (None, Some(ids)) => Some(ids), - (None, None) => None, - } - } - Expr::BinaryExpr(binary) if binary.op == Operator::Or => { - let mut left = consumer_ids_from_filter(binary.left.as_ref())?; - left.extend(consumer_ids_from_filter(binary.right.as_ref())?); - Some(left) - } - Expr::InList(in_list) - if !in_list.negated && is_consumer_id_column(in_list.expr.as_ref()) => - { - in_list.list.iter().map(string_literal).collect() - } - _ => None, - } -} - -fn consumer_id_literal(column: &Expr, literal: &Expr) -> Option { - is_consumer_id_column(column) - .then_some(literal) - .and_then(string_literal) -} - -fn is_consumer_id_column(expr: &Expr) -> bool { - matches!(expr, Expr::Column(column) if column.name == "consumer_id") -} - -fn string_literal(expr: &Expr) -> Option { - match expr { - Expr::Literal( - ScalarValue::Utf8(Some(value)) - | ScalarValue::LargeUtf8(Some(value)) - | ScalarValue::Utf8View(Some(value)), - _, - ) => Some(value.clone()), - _ => None, - } -} diff --git a/crates/integrations/datafusion/src/system_tables/metadata.rs b/crates/integrations/datafusion/src/system_tables/metadata.rs new file mode 100644 index 000000000..b393b5b29 --- /dev/null +++ b/crates/integrations/datafusion/src/system_tables/metadata.rs @@ -0,0 +1,116 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! DataFusion adaptation of core read-only system tables. + +use crate::error::to_datafusion_error; +use async_trait::async_trait; +use datafusion::arrow::datatypes::SchemaRef; +use datafusion::catalog::Session; +use datafusion::datasource::memory::MemorySourceConfig; +use datafusion::datasource::{TableProvider, TableType}; +use datafusion::error::Result; +use datafusion::logical_expr::{Expr, TableProviderFilterPushDown}; +use datafusion::physical_plan::ExecutionPlan; +use futures::TryStreamExt; +use paimon::table::{SystemTable, Table}; +use std::sync::Arc; + +pub(super) fn build(table: Table, name: &str) -> Result> { + Ok(Arc::new(MetadataTableProvider { + table: table.system_table(name).map_err(to_datafusion_error)?, + })) +} + +#[derive(Debug)] +pub(super) struct MetadataTableProvider { + table: SystemTable, +} + +#[async_trait] +impl TableProvider for MetadataTableProvider { + fn schema(&self) -> SchemaRef { + self.table.schema() + } + fn table_type(&self) -> TableType { + TableType::View + } + + async fn scan( + &self, + _state: &dyn Session, + projection: Option<&Vec>, + filters: &[Expr], + _limit: Option, + ) -> Result> { + let mut builder = self.table.new_read_builder(); + if let Some(indices) = projection { + let schema = self.table.schema(); + let columns: Vec<_> = indices + .iter() + .map(|&index| schema.field(index).name().as_str()) + .collect(); + builder + .with_projection(&columns) + .map_err(to_datafusion_error)?; + } + if let Some(predicate) = + crate::filter_pushdown::analyze_filters(filters, self.table.row_type().fields(), true) + .pushed_predicate + { + builder + .with_filter(predicate) + .map_err(to_datafusion_error)?; + } + let schema = builder.schema(); + let batches = crate::runtime::await_with_runtime(async move { + let plan = builder.new_scan().plan().await?; + builder + .new_read() + .read(plan.splits())? + .try_collect::>() + .await + }) + .await + .map_err(to_datafusion_error)?; + Ok(MemorySourceConfig::try_new_exec(&[batches], schema, None)?) + } + + fn supports_filters_pushdown( + &self, + filters: &[&Expr], + ) -> Result> { + let fields = self.table.row_type(); + Ok(filters + .iter() + .map(|filter| { + if crate::filter_pushdown::analyze_filters( + &[(*filter).clone()], + fields.fields(), + true, + ) + .pushed_predicate + .is_some() + { + TableProviderFilterPushDown::Inexact + } else { + TableProviderFilterPushDown::Unsupported + } + }) + .collect()) + } +} diff --git a/crates/integrations/datafusion/src/system_tables/mod.rs b/crates/integrations/datafusion/src/system_tables/mod.rs index 29539e14d..6b7d36d2c 100644 --- a/crates/integrations/datafusion/src/system_tables/mod.rs +++ b/crates/integrations/datafusion/src/system_tables/mod.rs @@ -32,36 +32,10 @@ use crate::error::to_datafusion_error; mod aggregation_fields; mod audit_log; -mod branches; -mod buckets; -mod consumers; -mod files; -mod manifests; -mod options; +mod metadata; mod partitions; mod physical_files_size; mod referenced_files_size; -mod row_string_cast; -mod schemas; -mod snapshots; -mod table_indexes; -mod tags; - -/// Metadata instants are exposed as Java TIMESTAMP in the system default zone. -/// Reuse the core LTZ cast so DST, negative and extreme epochs use reader rules. -fn local_timestamp_column(values: Vec) -> DFResult { - use datafusion::arrow::array::{ArrayRef, TimestampMillisecondArray}; - use paimon::spec::{DataType, LocalZonedTimestampType, TimestampType}; - let array: ArrayRef = Arc::new(TimestampMillisecondArray::from(values)); - paimon::arrow::cast_array_for_schema_evolution( - &array, - &DataType::LocalZonedTimestamp( - LocalZonedTimestampType::new(3).map_err(to_datafusion_error)?, - ), - &DataType::Timestamp(TimestampType::new(3).map_err(to_datafusion_error)?), - ) - .map_err(to_datafusion_error) -} type Builder = fn(Table) -> DFResult>; @@ -71,19 +45,23 @@ type Builder = fn(Table) -> DFResult>; const TABLES: &[(&str, Builder)] = &[ ("aggregation_fields", aggregation_fields::build), ("audit_log", audit_log::build), - ("branches", branches::build), - ("buckets", buckets::build), - ("consumers", consumers::build), - ("files", files::build), - ("file_key_ranges", files::build_key_ranges), - ("manifests", manifests::build), - ("options", options::build), + ("branches", |table| metadata::build(table, "branches")), + ("buckets", |table| metadata::build(table, "buckets")), + ("consumers", |table| metadata::build(table, "consumers")), + ("files", |table| metadata::build(table, "files")), + ("file_key_ranges", |table| { + metadata::build(table, "file_key_ranges") + }), + ("manifests", |table| metadata::build(table, "manifests")), + ("options", |table| metadata::build(table, "options")), ("physical_files_size", physical_files_size::build), ("referenced_files_size", referenced_files_size::build), - ("schemas", schemas::build), - ("snapshots", snapshots::build), - ("table_indexes", table_indexes::build), - ("tags", tags::build), + ("schemas", |table| metadata::build(table, "schemas")), + ("snapshots", |table| metadata::build(table, "snapshots")), + ("table_indexes", |table| { + metadata::build(table, "table_indexes") + }), + ("tags", |table| metadata::build(table, "tags")), ]; const SYSTEM_TABLE_NAMES: &[&str] = &[ @@ -131,20 +109,11 @@ pub(crate) fn is_registered(name: &str) -> bool { /// with a `$`. Keep in step with `TABLES`: one missing here goes back to /// having its time-travel clause silently dropped. pub(crate) fn is_system_table_provider(provider: &dyn TableProvider) -> bool { - provider.is::() - || provider.is::() - || provider.is::() - || provider.is::() - || provider.is::() - || provider.is::() - || provider.is::() + provider.is::() + || provider.is::() || provider.is::() || provider.is::() || provider.is::() - || provider.is::() - || provider.is::() - || provider.is::() - || provider.is::() } fn wrap_to_system_table(name: &str, base_table: Table) -> Option>> { @@ -154,27 +123,6 @@ fn wrap_to_system_table(name: &str, base_table: Table) -> Option DFResult { - crate::table_loader::ensure_paimon_served(&table, table.identifier())?; - paimon::spec::CoreOptions::new(table.schema().options()) - .ensure_read_authorized() - .map_err(to_datafusion_error)?; - let provider = wrap_to_system_table(name, table).ok_or_else(|| { - DataFusionError::Plan(format!( - "System table '{name}' requires a catalog or is unknown" - )) - })??; - datafusion::prelude::SessionContext::new() - .read_table(provider)? - .execute_stream() - .await -} - pub(crate) fn provider_for_table( catalog: Arc, identifier: Identifier, @@ -306,20 +254,14 @@ mod tests { .unwrap(); let ctx = SessionContext::new(); - for (name, builder) in [ - ("manifests", super::manifests::build as super::Builder), - ( - "table_indexes", - super::table_indexes::build as super::Builder, - ), - ] { + for name in ["manifests", "table_indexes"] { for empty in [false, true] { let view = if empty { table.copy_with_pinned_snapshot(None) } else { table.clone() }; - let provider = builder(view).unwrap(); + let provider = super::metadata::build(view, name).unwrap(); let batches = ctx.read_table(provider).unwrap().collect().await.unwrap(); let count: usize = batches.iter().map(RecordBatch::num_rows).sum(); assert_eq!(count == 0, empty, "{name}, empty={empty}"); diff --git a/crates/integrations/datafusion/src/system_tables/options.rs b/crates/integrations/datafusion/src/system_tables/options.rs deleted file mode 100644 index a32da351e..000000000 --- a/crates/integrations/datafusion/src/system_tables/options.rs +++ /dev/null @@ -1,94 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! Mirrors Java [OptionsTable](https://github.com/apache/paimon/blob/release-1.3/paimon-core/src/main/java/org/apache/paimon/table/system/OptionsTable.java). - -use std::sync::{Arc, OnceLock}; - -use async_trait::async_trait; -use datafusion::arrow::array::{RecordBatch, StringArray}; -use datafusion::arrow::datatypes::{DataType, Field, Schema, SchemaRef}; -use datafusion::catalog::Session; -use datafusion::datasource::memory::MemorySourceConfig; -use datafusion::datasource::{TableProvider, TableType}; -use datafusion::error::Result as DFResult; -use datafusion::logical_expr::Expr; -use datafusion::physical_plan::ExecutionPlan; -use paimon::table::Table; - -pub(super) fn build(table: Table) -> DFResult> { - Ok(Arc::new(OptionsTable { table })) -} - -fn options_schema() -> SchemaRef { - static SCHEMA: OnceLock = OnceLock::new(); - SCHEMA - .get_or_init(|| { - Arc::new(Schema::new(vec![ - Field::new("key", DataType::Utf8, false), - Field::new("value", DataType::Utf8, false), - ])) - }) - .clone() -} - -#[derive(Debug)] -pub(super) struct OptionsTable { - table: Table, -} - -#[async_trait] -impl TableProvider for OptionsTable { - fn schema(&self) -> SchemaRef { - options_schema() - } - - fn table_type(&self) -> TableType { - TableType::View - } - - async fn scan( - &self, - _state: &dyn Session, - projection: Option<&Vec>, - _filters: &[Expr], - _limit: Option, - ) -> DFResult> { - let manager = self.table.schema_manager(); - let latest = crate::runtime::await_with_runtime(async move { manager.latest().await }) - .await - .map_err(crate::error::to_datafusion_error)? - .ok_or_else(|| { - datafusion::error::DataFusionError::Plan("Table does not exist".to_string()) - })?; - // Java uses LinkedHashMap insertion order; HashMap has none — sort for stable output. - let mut entries: Vec<(&String, &String)> = latest.options().iter().collect(); - entries.sort_by(|a, b| a.0.cmp(b.0)); - - let keys = StringArray::from_iter_values(entries.iter().map(|(k, _)| k.as_str())); - let values = StringArray::from_iter_values(entries.iter().map(|(_, v)| v.as_str())); - - let schema = options_schema(); - let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(keys), Arc::new(values)])?; - - Ok(MemorySourceConfig::try_new_exec( - &[vec![batch]], - schema, - projection.cloned(), - )?) - } -} diff --git a/crates/integrations/datafusion/src/system_tables/schemas.rs b/crates/integrations/datafusion/src/system_tables/schemas.rs deleted file mode 100644 index 337f2a6af..000000000 --- a/crates/integrations/datafusion/src/system_tables/schemas.rs +++ /dev/null @@ -1,134 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! Mirrors Java [SchemasTable](https://github.com/apache/paimon/blob/release-1.3/paimon-core/src/main/java/org/apache/paimon/table/system/SchemasTable.java). - -use std::sync::{Arc, OnceLock}; - -use async_trait::async_trait; -use datafusion::arrow::array::{Int64Array, RecordBatch, StringArray}; -use datafusion::arrow::datatypes::{DataType, Field, Schema, SchemaRef, TimeUnit}; -use datafusion::catalog::Session; -use datafusion::datasource::memory::MemorySourceConfig; -use datafusion::datasource::{TableProvider, TableType}; -use datafusion::error::{DataFusionError, Result as DFResult}; -use datafusion::logical_expr::Expr; -use datafusion::physical_plan::ExecutionPlan; -use paimon::table::Table; -use serde::Serialize; - -use crate::error::to_datafusion_error; - -pub(super) fn build(table: Table) -> DFResult> { - Ok(Arc::new(SchemasTable { table })) -} - -fn schemas_schema() -> SchemaRef { - static SCHEMA: OnceLock = OnceLock::new(); - SCHEMA - .get_or_init(|| { - Arc::new(Schema::new(vec![ - Field::new("schema_id", DataType::Int64, false), - Field::new("fields", DataType::Utf8, false), - Field::new("partition_keys", DataType::Utf8, false), - Field::new("primary_keys", DataType::Utf8, false), - Field::new("options", DataType::Utf8, false), - Field::new("comment", DataType::Utf8, true), - Field::new( - "update_time", - DataType::Timestamp(TimeUnit::Millisecond, None), - false, - ), - ])) - }) - .clone() -} - -#[derive(Debug)] -pub(super) struct SchemasTable { - table: Table, -} - -#[async_trait] -impl TableProvider for SchemasTable { - fn schema(&self) -> SchemaRef { - schemas_schema() - } - - fn table_type(&self) -> TableType { - TableType::View - } - - async fn scan( - &self, - _state: &dyn Session, - projection: Option<&Vec>, - _filters: &[Expr], - _limit: Option, - ) -> DFResult> { - let table = self.table.clone(); - let schemas = - crate::runtime::await_with_runtime( - async move { table.schema_manager().list_all().await }, - ) - .await - .map_err(to_datafusion_error)?; - - let n = schemas.len(); - let mut schema_ids: Vec = Vec::with_capacity(n); - let mut fields_json: Vec = Vec::with_capacity(n); - let mut partition_keys_json: Vec = Vec::with_capacity(n); - let mut primary_keys_json: Vec = Vec::with_capacity(n); - let mut options_json: Vec = Vec::with_capacity(n); - let mut comments: Vec> = Vec::with_capacity(n); - let mut update_times: Vec = Vec::with_capacity(n); - - for schema in &schemas { - schema_ids.push(schema.id()); - fields_json.push(to_json(schema.fields())?); - partition_keys_json.push(to_json(schema.partition_keys())?); - primary_keys_json.push(to_json(schema.primary_keys())?); - options_json.push(to_json(schema.options())?); - comments.push(schema.comment().map(str::to_string)); - update_times.push(schema.time_millis()); - } - - let schema = schemas_schema(); - let batch = RecordBatch::try_new( - schema.clone(), - vec![ - Arc::new(Int64Array::from(schema_ids)), - Arc::new(StringArray::from(fields_json)), - Arc::new(StringArray::from(partition_keys_json)), - Arc::new(StringArray::from(primary_keys_json)), - Arc::new(StringArray::from(options_json)), - Arc::new(StringArray::from(comments)), - super::local_timestamp_column(update_times)?, - ], - )?; - - Ok(MemorySourceConfig::try_new_exec( - &[vec![batch]], - schema, - projection.cloned(), - )?) - } -} - -fn to_json(value: &T) -> DFResult { - serde_json::to_string(value).map_err(|e| DataFusionError::External(Box::new(e))) -} diff --git a/crates/integrations/datafusion/src/system_tables/snapshots.rs b/crates/integrations/datafusion/src/system_tables/snapshots.rs deleted file mode 100644 index f8355b15c..000000000 --- a/crates/integrations/datafusion/src/system_tables/snapshots.rs +++ /dev/null @@ -1,158 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! Mirrors Java [SnapshotsTable](https://github.com/apache/paimon/blob/release-1.3/paimon-core/src/main/java/org/apache/paimon/table/system/SnapshotsTable.java). - -use std::sync::{Arc, OnceLock}; - -use async_trait::async_trait; -use datafusion::arrow::array::{Int64Array, RecordBatch, StringArray}; -use datafusion::arrow::datatypes::{DataType, Field, Schema, SchemaRef, TimeUnit}; -use datafusion::catalog::Session; -use datafusion::datasource::memory::MemorySourceConfig; -use datafusion::datasource::{TableProvider, TableType}; -use datafusion::error::Result as DFResult; -use datafusion::logical_expr::Expr; -use datafusion::physical_plan::ExecutionPlan; -use paimon::table::Table; - -use crate::error::to_datafusion_error; - -pub(super) fn build(table: Table) -> DFResult> { - Ok(Arc::new(SnapshotsTable { table })) -} - -fn snapshots_schema() -> SchemaRef { - static SCHEMA: OnceLock = OnceLock::new(); - SCHEMA - .get_or_init(|| { - Arc::new(Schema::new(vec![ - Field::new("snapshot_id", DataType::Int64, false), - Field::new("schema_id", DataType::Int64, false), - Field::new("commit_user", DataType::Utf8, false), - Field::new("commit_identifier", DataType::Int64, false), - Field::new("commit_kind", DataType::Utf8, false), - Field::new( - "commit_time", - DataType::Timestamp(TimeUnit::Millisecond, None), - false, - ), - Field::new("base_manifest_list", DataType::Utf8, false), - Field::new("delta_manifest_list", DataType::Utf8, false), - Field::new("changelog_manifest_list", DataType::Utf8, true), - Field::new("total_record_count", DataType::Int64, true), - Field::new("delta_record_count", DataType::Int64, true), - Field::new("changelog_record_count", DataType::Int64, true), - Field::new("watermark", DataType::Int64, true), - Field::new("next_row_id", DataType::Int64, true), - Field::new("writer_version", DataType::Utf8, true), - ])) - }) - .clone() -} - -#[derive(Debug)] -pub(super) struct SnapshotsTable { - table: Table, -} - -#[async_trait] -impl TableProvider for SnapshotsTable { - fn schema(&self) -> SchemaRef { - snapshots_schema() - } - - fn table_type(&self) -> TableType { - TableType::View - } - - async fn scan( - &self, - _state: &dyn Session, - projection: Option<&Vec>, - _filters: &[Expr], - _limit: Option, - ) -> DFResult> { - let sm = self.table.snapshot_manager(); - let snapshots = crate::runtime::await_with_runtime(async move { sm.list_all().await }) - .await - .map_err(to_datafusion_error)?; - - let n = snapshots.len(); - let mut snapshot_ids = Vec::with_capacity(n); - let mut schema_ids = Vec::with_capacity(n); - let mut commit_users: Vec = Vec::with_capacity(n); - let mut commit_identifiers = Vec::with_capacity(n); - let mut commit_kinds: Vec = Vec::with_capacity(n); - let mut commit_times = Vec::with_capacity(n); - let mut base_manifest_lists: Vec = Vec::with_capacity(n); - let mut delta_manifest_lists: Vec = Vec::with_capacity(n); - let mut changelog_manifest_lists: Vec> = Vec::with_capacity(n); - let mut total_record_counts: Vec> = Vec::with_capacity(n); - let mut delta_record_counts: Vec> = Vec::with_capacity(n); - let mut changelog_record_counts: Vec> = Vec::with_capacity(n); - let mut watermarks: Vec> = Vec::with_capacity(n); - let mut next_row_ids: Vec> = Vec::with_capacity(n); - - let mut writer_versions = Vec::with_capacity(n); - for snap in &snapshots { - snapshot_ids.push(snap.id()); - schema_ids.push(snap.schema_id()); - commit_users.push(snap.commit_user().to_string()); - commit_identifiers.push(snap.commit_identifier()); - commit_kinds.push(snap.commit_kind().to_string()); - commit_times.push(snap.time_millis() as i64); - base_manifest_lists.push(snap.base_manifest_list().to_string()); - delta_manifest_lists.push(snap.delta_manifest_list().to_string()); - changelog_manifest_lists.push(snap.changelog_manifest_list().map(str::to_string)); - total_record_counts.push(snap.total_record_count()); - delta_record_counts.push(snap.delta_record_count()); - changelog_record_counts.push(snap.changelog_record_count()); - watermarks.push(snap.watermark()); - next_row_ids.push(snap.next_row_id()); - writer_versions.push(snap.writer_version()); - } - - let schema = snapshots_schema(); - let batch = RecordBatch::try_new( - schema.clone(), - vec![ - Arc::new(Int64Array::from(snapshot_ids)), - Arc::new(Int64Array::from(schema_ids)), - Arc::new(StringArray::from(commit_users)), - Arc::new(Int64Array::from(commit_identifiers)), - Arc::new(StringArray::from(commit_kinds)), - super::local_timestamp_column(commit_times)?, - Arc::new(StringArray::from(base_manifest_lists)), - Arc::new(StringArray::from(delta_manifest_lists)), - Arc::new(StringArray::from(changelog_manifest_lists)), - Arc::new(Int64Array::from(total_record_counts)), - Arc::new(Int64Array::from(delta_record_counts)), - Arc::new(Int64Array::from(changelog_record_counts)), - Arc::new(Int64Array::from(watermarks)), - Arc::new(Int64Array::from(next_row_ids)), - Arc::new(StringArray::from(writer_versions)), - ], - )?; - - Ok(MemorySourceConfig::try_new_exec( - &[vec![batch]], - schema, - projection.cloned(), - )?) - } -} diff --git a/crates/integrations/datafusion/tests/system_tables.rs b/crates/integrations/datafusion/tests/system_tables.rs index 268733b30..a6f15aea5 100644 --- a/crates/integrations/datafusion/tests/system_tables.rs +++ b/crates/integrations/datafusion/tests/system_tables.rs @@ -75,6 +75,15 @@ async fn run_sql(ctx: &SQLContext, sql: &str) -> Vec { .unwrap_or_else(|e| panic!("Failed to execute `{sql}`: {e}")) } +async fn read_metadata( + table: paimon::table::Table, + name: &str, +) -> paimon::Result { + let builder = table.system_table(name)?.new_read_builder(); + let plan = builder.new_scan().plan().await?; + builder.new_read().read(plan.splits()) +} + // Error text of `sql`, whether it surfaces at planning or execution. async fn query_error(ctx: &SQLContext, sql: &str) -> String { match ctx.sql(sql).await { @@ -1740,7 +1749,7 @@ async fn test_file_key_ranges_matches_file_metadata() { .await .unwrap(); let expected = run_sql(&ctx, &format!("SELECT partition, bucket, file_path, file_format, schema_id, level, record_count, file_size_in_bytes, min_key, max_key, first_row_id FROM paimon.default.{FIXTURE_TABLE}$files")).await; - let actual: Vec<_> = paimon_datafusion::read_system_table(table.clone(), "file_key_ranges") + let actual: Vec<_> = read_metadata(table.clone(), "file_key_ranges") .await .unwrap() .try_collect() @@ -1753,7 +1762,7 @@ async fn test_file_key_ranges_matches_file_metadata() { assert_eq!(projected[0].num_rows(), 1); assert_eq!(projected[0].num_columns(), 1); let empty = table.copy_with_pinned_snapshot(None); - let batches: Vec<_> = paimon_datafusion::read_system_table(empty, "file_key_ranges") + let batches: Vec<_> = read_metadata(empty, "file_key_ranges") .await .unwrap() .try_collect() @@ -1783,10 +1792,8 @@ async fn test_resolved_system_table_reader_preserves_guards_and_schema() { "file_key_ranges", "table_indexes", ] { - let stream = paimon_datafusion::read_system_table(table.clone(), name) - .await - .unwrap(); - let schema = stream.schema(); + let stream = read_metadata(table.clone(), name).await.unwrap(); + let schema = table.system_table(name).unwrap().schema(); let batches: Vec<_> = stream.try_collect().await.unwrap(); assert!(!schema.fields().is_empty(), "{name}"); assert!( @@ -1795,7 +1802,7 @@ async fn test_resolved_system_table_reader_preserves_guards_and_schema() { ); } for name in ["partitions", "unknown"] { - let error = match paimon_datafusion::read_system_table(table.clone(), name).await { + let error = match read_metadata(table.clone(), name).await { Ok(_) => panic!("{name} must require a catalog or be unknown"), Err(error) => error.to_string(), }; @@ -1805,7 +1812,7 @@ async fn test_resolved_system_table_reader_preserves_guards_and_schema() { "query-auth.enabled".to_string(), "true".to_string(), )])); - let error = match paimon_datafusion::read_system_table(denied, "options").await { + let error = match read_metadata(denied, "options").await { Ok(_) => panic!("Authorization must be checked before reading metadata"), Err(error) => error.to_string(), }; @@ -1824,7 +1831,7 @@ async fn test_options_ignore_resolved_dynamic_overrides() { ("file.format".to_string(), "csv".to_string()), ("metadata.runtime-only".to_string(), "private".to_string()), ])); - let batches: Vec<_> = paimon_datafusion::read_system_table(copy, "options") + let batches: Vec<_> = read_metadata(copy, "options") .await .unwrap() .try_collect() @@ -1856,3 +1863,48 @@ async fn test_options_ignore_resolved_dynamic_overrides() { assert_eq!(&rows, table.schema().options()); assert!(!rows.contains_key("metadata.runtime-only")); } + +#[tokio::test] +async fn test_core_system_table_read_builder_filters_before_projection() { + use futures::TryStreamExt; + use paimon::spec::{Datum, Predicate, PredicateOperator}; + let (_ctx, catalog, _tmp) = create_context().await; + let table = catalog + .get_table(&Identifier::new("default", FIXTURE_TABLE)) + .await + .unwrap(); + let system = table.system_table("snapshots").unwrap(); + let mut builder = system.new_read_builder(); + builder.with_projection(&["commit_user"]).unwrap(); + let field = system.row_type().fields()[0].clone(); + builder + .with_filter(Predicate::Leaf { + column: field.name().to_string(), + index: 0, + data_type: field.data_type().clone(), + op: PredicateOperator::Gt, + literals: vec![Datum::Long(0)], + }) + .unwrap(); + builder.with_limit(1); + let plan = builder.new_scan().plan().await.unwrap(); + let rows: Vec<_> = builder + .new_read() + .read(plan.splits()) + .unwrap() + .try_collect() + .await + .unwrap(); + assert_eq!(rows.iter().map(RecordBatch::num_rows).sum::(), 1); + assert_eq!(rows[0].schema().fields().len(), 1); + assert_eq!(rows[0].schema().field(0).name(), "commit_user"); + builder.with_limit(0); + let rows: Vec<_> = builder + .new_read() + .read(plan.splits()) + .unwrap() + .try_collect() + .await + .unwrap(); + assert!(rows.iter().all(|batch| batch.num_rows() == 0)); +} diff --git a/crates/paimon/Cargo.toml b/crates/paimon/Cargo.toml index f9aaffa83..31302b4a8 100644 --- a/crates/paimon/Cargo.toml +++ b/crates/paimon/Cargo.toml @@ -67,6 +67,7 @@ storage-jindo = ["storage-oss"] storage-oss-cpp = ["storage-oss"] [dependencies] +lexical-write-float = "1.0.6" rand = "0.8.5" url = "2.5.2" # Already in the tree via `url`; direct dep for `RESTUtil::decode_string`. diff --git a/crates/paimon/src/table/mod.rs b/crates/paimon/src/table/mod.rs index 0608b3316..94be4d07a 100644 --- a/crates/paimon/src/table/mod.rs +++ b/crates/paimon/src/table/mod.rs @@ -129,6 +129,11 @@ mod postpone_fixed_bucket_write_builder; mod postpone_retract; mod prepared_files; mod query_auth; +mod system_table; +pub use system_table::{ + SystemReadBuilder, SystemSplit, SystemTable, SystemTablePlan, SystemTableRead, SystemTableScan, +}; + mod read_builder; mod read_limit; pub mod referenced_files; @@ -544,6 +549,11 @@ impl Table { && options.partitioned_table_in_metastore() } + /// Resolve a read-only metadata table while retaining this table's environment. + pub fn system_table(&self, name: &str) -> Result { + SystemTable::new(self.clone(), name) + } + /// Create a read builder for scan/read. /// /// Reference: [pypaimon FileStoreTable.new_read_builder](https://github.com/apache/paimon/blob/release-1.3/paimon-python/pypaimon/table/file_store_table.py). diff --git a/crates/integrations/datafusion/src/system_tables/branches.rs b/crates/paimon/src/table/system_table/branches.rs similarity index 50% rename from crates/integrations/datafusion/src/system_tables/branches.rs rename to crates/paimon/src/table/system_table/branches.rs index ff1412176..3cba2ec02 100644 --- a/crates/integrations/datafusion/src/system_tables/branches.rs +++ b/crates/paimon/src/table/system_table/branches.rs @@ -19,24 +19,13 @@ use std::sync::{Arc, OnceLock}; -use async_trait::async_trait; -use datafusion::arrow::array::{RecordBatch, StringArray}; -use datafusion::arrow::datatypes::{DataType, Field, Schema, SchemaRef, TimeUnit}; -use datafusion::catalog::Session; -use datafusion::datasource::memory::MemorySourceConfig; -use datafusion::datasource::{TableProvider, TableType}; -use datafusion::error::Result as DFResult; -use datafusion::logical_expr::Expr; -use datafusion::physical_plan::ExecutionPlan; -use paimon::table::{BranchManager, Table}; +use crate::table::{BranchManager, Table}; +use arrow_array::{RecordBatch, StringArray}; +use arrow_schema::{DataType, Field, Schema, SchemaRef, TimeUnit}; -use crate::error::to_datafusion_error; +use crate::Result; -pub(super) fn build(table: Table) -> DFResult> { - Ok(Arc::new(BranchesTable { table })) -} - -fn branches_schema() -> SchemaRef { +pub(super) fn branches_schema() -> SchemaRef { static SCHEMA: OnceLock = OnceLock::new(); SCHEMA .get_or_init(|| { @@ -52,52 +41,26 @@ fn branches_schema() -> SchemaRef { .clone() } -#[derive(Debug)] -pub(super) struct BranchesTable { - table: Table, -} +pub(super) async fn read( + table: &Table, + _projection: Option<&[usize]>, + _predicate: Option<&crate::spec::Predicate>, +) -> Result { + let (names, create_times) = collect_branches(table).await?; -#[async_trait] -impl TableProvider for BranchesTable { - fn schema(&self) -> SchemaRef { - branches_schema() - } + let schema = branches_schema(); + let batch = super::record_batch( + schema.clone(), + vec![ + Arc::new(StringArray::from(names)), + super::local_timestamp_column(create_times)?, + ], + )?; - fn table_type(&self) -> TableType { - TableType::View - } - - async fn scan( - &self, - _state: &dyn Session, - projection: Option<&Vec>, - _filters: &[Expr], - _limit: Option, - ) -> DFResult> { - let table = self.table.clone(); - let (names, create_times) = - crate::runtime::await_with_runtime(async move { collect_branches(&table).await }) - .await - .map_err(to_datafusion_error)?; - - let schema = branches_schema(); - let batch = RecordBatch::try_new( - schema.clone(), - vec![ - Arc::new(StringArray::from(names)), - super::local_timestamp_column(create_times)?, - ], - )?; - - Ok(MemorySourceConfig::try_new_exec( - &[vec![batch]], - schema, - projection.cloned(), - )?) - } + Ok(batch) } -async fn collect_branches(table: &Table) -> paimon::Result<(Vec, Vec)> { +async fn collect_branches(table: &Table) -> crate::Result<(Vec, Vec)> { let file_io = table.file_io(); let bm = BranchManager::new(file_io.clone(), table.location().to_string()); let names = bm.list_all().await?; diff --git a/crates/integrations/datafusion/src/system_tables/buckets.rs b/crates/paimon/src/table/system_table/buckets.rs similarity index 78% rename from crates/integrations/datafusion/src/system_tables/buckets.rs rename to crates/paimon/src/table/system_table/buckets.rs index f34b6a803..82fb1af16 100644 --- a/crates/integrations/datafusion/src/system_tables/buckets.rs +++ b/crates/paimon/src/table/system_table/buckets.rs @@ -20,28 +20,16 @@ use std::collections::BTreeMap; use std::sync::{Arc, OnceLock}; -use async_trait::async_trait; -use datafusion::arrow::array::{ - Int32Array, Int64Array, RecordBatch, StringArray, TimestampMillisecondArray, -}; -use datafusion::arrow::datatypes::{DataType as ArrowDataType, Field, Schema, SchemaRef, TimeUnit}; -use datafusion::catalog::Session; -use datafusion::datasource::memory::MemorySourceConfig; -use datafusion::datasource::{TableProvider, TableType}; -use datafusion::error::Result as DFResult; -use datafusion::logical_expr::Expr; -use datafusion::physical_plan::ExecutionPlan; -use paimon::spec::{BinaryRow, DataField}; -use paimon::table::{DataSplit, Table}; +use crate::spec::{BinaryRow, DataField}; +use crate::table::{DataSplit, Table}; +use arrow_array::{Int32Array, Int64Array, RecordBatch, StringArray, TimestampMillisecondArray}; +use arrow_schema::{DataType as ArrowDataType, Field, Schema, SchemaRef, TimeUnit}; use super::row_string_cast::format_row_as_java_cast_string; -use crate::error::to_datafusion_error; -pub(super) fn build(table: Table) -> DFResult> { - Ok(Arc::new(BucketsTable { table })) -} +use crate::Result; -fn buckets_schema() -> SchemaRef { +pub(super) fn buckets_schema() -> SchemaRef { static SCHEMA: OnceLock = OnceLock::new(); SCHEMA .get_or_init(|| { @@ -61,40 +49,14 @@ fn buckets_schema() -> SchemaRef { .clone() } -#[derive(Debug)] -pub(super) struct BucketsTable { - table: Table, -} - -#[async_trait] -impl TableProvider for BucketsTable { - fn schema(&self) -> SchemaRef { - buckets_schema() - } - - fn table_type(&self) -> TableType { - TableType::View - } - - async fn scan( - &self, - _state: &dyn Session, - projection: Option<&Vec>, - _filters: &[Expr], - _limit: Option, - ) -> DFResult> { - let table = self.table.clone(); - let rows = - crate::runtime::await_with_runtime(async move { collect_bucket_rows(&table).await }) - .await - .map_err(to_datafusion_error)?; - let batch = bucket_rows_to_record_batch(&rows)?; - Ok(MemorySourceConfig::try_new_exec( - &[vec![batch]], - buckets_schema(), - projection.cloned(), - )?) - } +pub(super) async fn read( + table: &Table, + _projection: Option<&[usize]>, + _predicate: Option<&crate::spec::Predicate>, +) -> Result { + let rows = collect_bucket_rows(table).await?; + let batch = bucket_rows_to_record_batch(&rows)?; + Ok(batch) } /// A bucket's files aggregated: mirrors Java `BucketEntry`. @@ -112,7 +74,7 @@ struct BucketRow { agg: BucketAgg, } -async fn collect_bucket_rows(table: &Table) -> paimon::Result> { +async fn collect_bucket_rows(table: &Table) -> crate::Result> { let scan = table .new_read_builder() .new_scan() @@ -137,7 +99,7 @@ async fn collect_bucket_rows(table: &Table) -> paimon::Result> { fn aggregate_bucket_rows( splits: &[DataSplit], partition_fields: &[DataField], -) -> paimon::Result> { +) -> crate::Result> { let mut aggs: BTreeMap<(Option, i32, Vec), BucketAgg> = BTreeMap::new(); for split in splits { let partition = format_partition(split.partition(), partition_fields)?; @@ -166,7 +128,7 @@ fn aggregate_bucket_rows( .collect()) } -fn bucket_rows_to_record_batch(rows: &[BucketRow]) -> DFResult { +fn bucket_rows_to_record_batch(rows: &[BucketRow]) -> Result { let n = rows.len(); let mut partitions = Vec::with_capacity(n); let mut buckets = Vec::with_capacity(n); @@ -182,7 +144,7 @@ fn bucket_rows_to_record_batch(rows: &[BucketRow]) -> DFResult { file_counts.push(row.agg.file_count); last_update_times.push(row.agg.last_update_time); } - Ok(RecordBatch::try_new( + super::record_batch( buckets_schema(), vec![ Arc::new(StringArray::from(partitions)), @@ -192,7 +154,7 @@ fn bucket_rows_to_record_batch(rows: &[BucketRow]) -> DFResult { Arc::new(Int64Array::from(file_counts)), Arc::new(TimestampMillisecondArray::from(last_update_times)), ], - )?) + ) } /// Format `partition` as Java's cast-to-string, matching `$files`; `{}` when the @@ -200,7 +162,7 @@ fn bucket_rows_to_record_batch(rows: &[BucketRow]) -> DFResult { fn format_partition( partition: &BinaryRow, partition_fields: &[DataField], -) -> paimon::Result> { +) -> crate::Result> { if partition_fields.is_empty() { return Ok(Some("{}".to_string())); } @@ -210,7 +172,7 @@ fn format_partition( #[cfg(test)] mod tests { use super::*; - use paimon::spec::{DataType, Datum, VarCharType}; + use crate::spec::{DataType, Datum, VarCharType}; fn string_type() -> DataType { DataType::VarChar(VarCharType::new(64).unwrap()) diff --git a/crates/paimon/src/table/system_table/consumers.rs b/crates/paimon/src/table/system_table/consumers.rs new file mode 100644 index 000000000..373d0dc4c --- /dev/null +++ b/crates/paimon/src/table/system_table/consumers.rs @@ -0,0 +1,105 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Mirrors Java [ConsumersTable](https://github.com/apache/paimon/blob/release-1.3/paimon-core/src/main/java/org/apache/paimon/table/system/ConsumersTable.java). + +use std::collections::HashSet; +use std::sync::{Arc, OnceLock}; + +use crate::table::Table; +use arrow_array::{Int64Array, RecordBatch, StringArray}; +use arrow_schema::{DataType, Field, Schema, SchemaRef}; + +use crate::Result; + +pub(super) fn consumers_schema() -> SchemaRef { + static SCHEMA: OnceLock = OnceLock::new(); + SCHEMA + .get_or_init(|| { + Arc::new(Schema::new(vec![ + Field::new("consumer_id", DataType::Utf8, false), + Field::new("next_snapshot_id", DataType::Int64, false), + ])) + }) + .clone() +} + +pub(super) async fn read( + table: &Table, + _projection: Option<&[usize]>, + predicate: Option<&crate::spec::Predicate>, +) -> Result { + let manager = table.consumer_manager(); + let consumers = match predicate.and_then(consumer_ids) { + Some(ids) => { + manager + .list_by_ids(&ids.into_iter().collect::>()) + .await? + } + None => manager.list_all().await?, + }; + + let schema = consumers_schema(); + let batch = super::record_batch( + schema.clone(), + vec![ + Arc::new(StringArray::from_iter_values( + consumers.iter().map(|(id, _)| id.as_str()), + )), + Arc::new(Int64Array::from_iter_values( + consumers.iter().map(|(_, next_snapshot)| *next_snapshot), + )), + ], + )?; + + Ok(batch) +} + +fn consumer_ids(predicate: &crate::spec::Predicate) -> Option> { + use crate::spec::{Datum, Predicate, PredicateOperator}; + match predicate { + Predicate::Leaf { + column, + op, + literals, + .. + } if column == "consumer_id" + && matches!(op, PredicateOperator::Eq | PredicateOperator::In) => + { + literals + .iter() + .map(|value| { + if let Datum::String(value) = value { + Some(value.clone()) + } else { + None + } + }) + .collect() + } + Predicate::And(children) => children.iter().filter_map(consumer_ids).reduce(|mut a, b| { + a.retain(|id| b.contains(id)); + a + }), + Predicate::Or(children) => children + .iter() + .map(consumer_ids) + .collect::>>() + .map(|sets| sets.into_iter().flatten().collect()), + _ => None, + } +} diff --git a/crates/integrations/datafusion/src/system_tables/files.rs b/crates/paimon/src/table/system_table/files.rs similarity index 85% rename from crates/integrations/datafusion/src/system_tables/files.rs rename to crates/paimon/src/table/system_table/files.rs index e9e575e7b..66f7100fc 100644 --- a/crates/integrations/datafusion/src/system_tables/files.rs +++ b/crates/paimon/src/table/system_table/files.rs @@ -20,49 +20,29 @@ use std::collections::{BTreeMap, HashMap}; use std::sync::{Arc, OnceLock}; -use async_trait::async_trait; -use datafusion::arrow::array::{ - ArrayRef, Int32Array, Int64Array, ListBuilder, RecordBatch, StringArray, StringBuilder, - TimestampMillisecondArray, +use crate::spec::{BinaryRow, DataField, DataFileMeta, TableSchema}; +use crate::table::{DataSplit, Table}; +use arrow_array::builder::{ListBuilder, StringBuilder}; +use arrow_array::{ + ArrayRef, Int32Array, Int64Array, RecordBatch, StringArray, TimestampMillisecondArray, }; -use datafusion::arrow::datatypes::{DataType as ArrowDataType, Field, Schema, SchemaRef, TimeUnit}; -use datafusion::catalog::Session; -use datafusion::datasource::memory::MemorySourceConfig; -use datafusion::datasource::{TableProvider, TableType}; -use datafusion::error::Result as DFResult; -use datafusion::logical_expr::Expr; -use datafusion::physical_plan::ExecutionPlan; -use paimon::spec::{BinaryRow, DataField, DataFileMeta, TableSchema}; -use paimon::table::{DataSplit, Table}; +use arrow_schema::{DataType as ArrowDataType, Field, Schema, SchemaRef, TimeUnit}; use super::row_string_cast::{ format_row_as_java_array_string, format_row_as_java_cast_string, format_row_field_as_java_object_string, }; -use crate::error::to_datafusion_error; -pub(super) fn build(table: Table) -> DFResult> { - Ok(Arc::new(FilesTable { - table, - key_ranges: false, - })) -} - -pub(super) fn build_key_ranges(table: Table) -> DFResult> { - Ok(Arc::new(FilesTable { - table, - key_ranges: true, - })) -} +use crate::Result; -fn key_ranges_schema() -> SchemaRef { +pub(super) fn key_ranges_schema() -> SchemaRef { let files = files_schema(); let mut fields = files.fields()[..10].to_vec(); fields.push(files.field(18).clone().into()); Arc::new(Schema::new(fields)) } -fn files_schema() -> SchemaRef { +pub(super) fn files_schema() -> SchemaRef { static SCHEMA: OnceLock = OnceLock::new(); SCHEMA .get_or_init(|| { @@ -100,58 +80,30 @@ fn files_schema() -> SchemaRef { .clone() } -#[derive(Debug)] -pub(super) struct FilesTable { - table: Table, +pub(super) async fn read( + table: &Table, key_ranges: bool, -} - -#[async_trait] -impl TableProvider for FilesTable { - fn schema(&self) -> SchemaRef { - if self.key_ranges { - key_ranges_schema() - } else { - files_schema() - } - } - - fn table_type(&self) -> TableType { - TableType::View - } + _projection: Option<&[usize]>, + _predicate: Option<&crate::spec::Predicate>, +) -> Result { + let rows = collect_file_rows(table, !key_ranges).await?; - async fn scan( - &self, - _state: &dyn Session, - projection: Option<&Vec>, - _filters: &[Expr], - _limit: Option, - ) -> DFResult> { - let table = self.table.clone(); - let key_ranges = self.key_ranges; - let rows = crate::runtime::await_with_runtime(async move { - collect_file_rows(&table, !key_ranges).await - }) - .await - .map_err(to_datafusion_error)?; - - let schema = self.schema(); - let batch = if key_ranges { - let mut columns = common_file_columns(&rows); - columns.push(Arc::new(Int64Array::from_iter( - rows.iter().map(|row| row.first_row_id), - ))); - RecordBatch::try_new(schema.clone(), columns)? - } else { - file_rows_to_record_batch(&rows)? - }; + let schema = if key_ranges { + key_ranges_schema() + } else { + files_schema() + }; + let batch = if key_ranges { + let mut columns = common_file_columns(&rows); + columns.push(Arc::new(Int64Array::from_iter( + rows.iter().map(|row| row.first_row_id), + ))); + super::record_batch(schema.clone(), columns)? + } else { + file_rows_to_record_batch(&rows)? + }; - Ok(MemorySourceConfig::try_new_exec( - &[vec![batch]], - schema, - projection.cloned(), - )?) - } + Ok(batch) } #[derive(Debug)] @@ -176,7 +128,7 @@ struct FileRow { write_cols: Option>, } -async fn collect_file_rows(table: &Table, include_stats: bool) -> paimon::Result> { +async fn collect_file_rows(table: &Table, include_stats: bool) -> crate::Result> { let scan = table .new_read_builder() .new_scan() @@ -209,7 +161,7 @@ async fn data_split_rows( partition_fields: &[DataField], schema_cache: &mut HashMap>, include_stats: bool, -) -> paimon::Result> { +) -> crate::Result> { let partition = format_partition(split.partition(), partition_fields)?; let mut rows = Vec::with_capacity(split.data_files().len()); for file in split.data_files() { @@ -254,7 +206,7 @@ async fn schema_for_id( table: &Table, schema_cache: &mut HashMap>, schema_id: i64, -) -> paimon::Result> { +) -> crate::Result> { if let Some(schema) = schema_cache.get(&schema_id) { return Ok(schema.clone()); } @@ -266,7 +218,7 @@ async fn schema_for_id( fn format_partition( partition: &BinaryRow, partition_fields: &[DataField], -) -> paimon::Result> { +) -> crate::Result> { if partition_fields.is_empty() { return Ok(Some("{}".to_string())); } @@ -291,7 +243,7 @@ fn key_fields_for_schema(schema: &TableSchema) -> Vec { .collect() } -fn format_key(bytes: &[u8], key_fields: &[DataField]) -> paimon::Result> { +fn format_key(bytes: &[u8], key_fields: &[DataField]) -> crate::Result> { if bytes.is_empty() { return Ok(None); } @@ -313,7 +265,7 @@ async fn format_value_stats( table: &Table, schema_cache: &mut HashMap>, file: &DataFileMeta, -) -> paimon::Result { +) -> crate::Result { let file_schema = schema_for_id(table, schema_cache, file.schema_id).await?; format_file_value_stats(table.schema().fields(), &file_schema, file) } @@ -322,7 +274,7 @@ fn format_file_value_stats( table_fields: &[DataField], file_schema: &TableSchema, file: &DataFileMeta, -) -> paimon::Result { +) -> crate::Result { let physical_fields = file_schema.data_file_fields(file.write_cols.as_deref()); let data_fields = physical_fields.as_slice(); let field_mapping = table_to_data_field_mapping(table_fields, data_fields); @@ -437,7 +389,7 @@ fn stats_index_for_data_field( } } -fn decode_stats_row(bytes: &[u8]) -> paimon::Result> { +fn decode_stats_row(bytes: &[u8]) -> crate::Result> { if bytes.is_empty() { Ok(None) } else { @@ -448,9 +400,9 @@ fn decode_stats_row(bytes: &[u8]) -> paimon::Result> { fn format_stats_value( row: Option<&BinaryRow>, stats_index: Option, - data_type: &paimon::spec::DataType, - target_type: &paimon::spec::DataType, -) -> paimon::Result> { + data_type: &crate::spec::DataType, + target_type: &crate::spec::DataType, +) -> crate::Result> { let Some(row) = row else { return Ok(None); }; @@ -464,7 +416,7 @@ fn format_stats_value( return format_row_field_as_java_object_string(row, stats_index, data_type); } let source = row.field_to_arrow(stats_index, data_type)?; - let target = paimon::arrow::cast_array_for_schema_evolution(&source, data_type, target_type)?; + let target = crate::arrow::cast_array_for_schema_evolution(&source, data_type, target_type)?; let fields = [DataField::new(0, "value".to_string(), target_type.clone())]; let schema = Arc::new(Schema::new(vec![Field::new( "value", @@ -472,7 +424,7 @@ fn format_stats_value( true, )])); let batch = - RecordBatch::try_new(schema, vec![target]).map_err(|error| paimon::Error::DataInvalid { + super::record_batch(schema, vec![target]).map_err(|error| crate::Error::DataInvalid { message: format!("Invalid evolved file statistics: {error}"), source: Some(Box::new(error)), })?; @@ -505,9 +457,9 @@ fn java_file_path(path: &str) -> String { } } -fn data_file_format_identifier(file_name: &str) -> paimon::Result { +fn data_file_format_identifier(file_name: &str) -> crate::Result { let Some(dot) = file_name.rfind('.') else { - return Err(paimon::Error::DataInvalid { + return Err(crate::Error::DataInvalid { message: format!("{file_name} is not a legal file name."), source: None, }); @@ -516,7 +468,7 @@ fn data_file_format_identifier(file_name: &str) -> paimon::Result { let extension = &file_name[dot + 1..]; if is_hadoop_compression_extension(extension) { let Some(second_dot) = file_name[..dot].rfind('.') else { - return Err(paimon::Error::DataInvalid { + return Err(crate::Error::DataInvalid { message: format!("{file_name} is not a legal file name."), source: None, }); @@ -572,7 +524,7 @@ fn common_file_columns(rows: &[FileRow]) -> Vec { ] } -fn file_rows_to_record_batch(rows: &[FileRow]) -> DFResult { +fn file_rows_to_record_batch(rows: &[FileRow]) -> Result { let n = rows.len(); let mut null_value_counts = Vec::with_capacity(n); let mut min_value_stats = Vec::with_capacity(n); @@ -586,11 +538,13 @@ fn file_rows_to_record_batch(rows: &[FileRow]) -> DFResult { let mut write_cols = ListBuilder::new(StringBuilder::new()); for row in rows { - let stats = row.stats.as_ref().ok_or_else(|| { - datafusion::error::DataFusionError::Internal( - "Files table requires value statistics".to_string(), - ) - })?; + let stats = row + .stats + .as_ref() + .ok_or_else(|| crate::Error::DataInvalid { + message: "Files table requires value statistics".to_string(), + source: None, + })?; null_value_counts.push(stats.null_value_counts.clone()); min_value_stats.push(stats.min_value_stats.clone()); max_value_stats.push(stats.max_value_stats.clone()); @@ -624,7 +578,7 @@ fn file_rows_to_record_batch(rows: &[FileRow]) -> DFResult { Arc::new(Int64Array::from(first_row_ids)), Arc::new(write_cols.finish()), ]); - Ok(RecordBatch::try_new(files_schema(), columns)?) + super::record_batch(files_schema(), columns) } #[cfg(test)] @@ -632,7 +586,7 @@ mod tests { use super::*; fn stats_file(columns: Option>) -> DataFileMeta { - use paimon::spec::{BinaryTableStats, DataType, Datum, IntType}; + use crate::spec::{BinaryTableStats, DataType, Datum, IntType}; let typ = DataType::Int(IntType::new()); let value = Datum::Int(42); let row = BinaryRow::from_datums(&[(Some(&value), &typ)]).to_serialized_bytes(); @@ -663,7 +617,7 @@ mod tests { #[test] fn file_stats_follow_physical_columns_and_schema_casts() { - use paimon::spec::{DataType, DecimalType, IntType, Schema as PaimonSchema}; + use crate::spec::{DataType, DecimalType, IntType, Schema as PaimonSchema}; let schema = PaimonSchema::builder() .column("a", DataType::Int(IntType::new())) .column("b", DataType::Int(IntType::new())) @@ -691,9 +645,9 @@ mod tests { #[test] fn stats_cast_overflow_is_nullable_even_for_a_required_field() { - use paimon::spec::{DataType, DecimalType, IntType}; + use crate::spec::{DataType, DecimalType, IntType}; let source_type = DataType::Int(IntType::new()); - let value = paimon::spec::Datum::Int(42); + let value = crate::spec::Datum::Int(42); let row = BinaryRow::from_datums(&[(Some(&value), &source_type)]); let target = DataType::Decimal(DecimalType::with_nullable(false, 1, 0).unwrap()); assert_eq!( diff --git a/crates/integrations/datafusion/src/system_tables/manifests.rs b/crates/paimon/src/table/system_table/manifests.rs similarity index 58% rename from crates/integrations/datafusion/src/system_tables/manifests.rs rename to crates/paimon/src/table/system_table/manifests.rs index 1acf7a522..d75c20338 100644 --- a/crates/integrations/datafusion/src/system_tables/manifests.rs +++ b/crates/paimon/src/table/system_table/manifests.rs @@ -19,29 +19,19 @@ use std::sync::{Arc, OnceLock}; -use async_trait::async_trait; -use datafusion::arrow::array::{Int64Array, RecordBatch, StringArray}; -use datafusion::arrow::datatypes::{DataType, Field, Schema, SchemaRef}; -use datafusion::catalog::Session; -use datafusion::datasource::memory::MemorySourceConfig; -use datafusion::datasource::{TableProvider, TableType}; -use datafusion::error::Result as DFResult; -use datafusion::logical_expr::Expr; -use datafusion::physical_plan::ExecutionPlan; -use paimon::spec::{BinaryRow, DataField, ManifestFileMeta, ManifestList}; -use paimon::table::Table; +use crate::spec::{BinaryRow, DataField, ManifestFileMeta, ManifestList}; +use crate::table::Table; +use arrow_array::{Int64Array, RecordBatch, StringArray}; +use arrow_schema::{DataType, Field, Schema, SchemaRef}; use super::row_string_cast::format_row_as_java_cast_string; -use crate::error::to_datafusion_error; const MIN_PARTITION_STATS_INDEX: usize = 5; const MAX_PARTITION_STATS_INDEX: usize = 6; -pub(super) fn build(table: Table) -> DFResult> { - Ok(Arc::new(ManifestsTable { table })) -} +use crate::Result; -fn manifests_schema() -> SchemaRef { +pub(super) fn manifests_schema() -> SchemaRef { static SCHEMA: OnceLock = OnceLock::new(); SCHEMA .get_or_init(|| { @@ -60,108 +50,76 @@ fn manifests_schema() -> SchemaRef { .clone() } -#[derive(Debug)] -pub(super) struct ManifestsTable { - table: Table, -} - -#[async_trait] -impl TableProvider for ManifestsTable { - fn schema(&self) -> SchemaRef { - manifests_schema() +pub(super) async fn read( + table: &Table, + projection: Option<&[usize]>, + _predicate: Option<&crate::spec::Predicate>, +) -> Result { + let metas = collect_manifests(table).await?; + + let n = metas.len(); + let mut file_names: Vec = Vec::with_capacity(n); + let mut file_sizes = Vec::with_capacity(n); + let mut num_added = Vec::with_capacity(n); + let mut num_deleted = Vec::with_capacity(n); + let mut schema_ids = Vec::with_capacity(n); + let mut min_partition_stats: Vec> = Vec::with_capacity(n); + let mut max_partition_stats: Vec> = Vec::with_capacity(n); + let mut min_row_ids: Vec> = Vec::with_capacity(n); + let mut max_row_ids: Vec> = Vec::with_capacity(n); + let partition_fields = table.schema().partition_fields(); + let projected_columns = projection; + let materialize_min_partition_stats = + should_materialize_column(projected_columns, MIN_PARTITION_STATS_INDEX); + let materialize_max_partition_stats = + should_materialize_column(projected_columns, MAX_PARTITION_STATS_INDEX); + + for meta in metas { + let stats = meta.partition_stats(); + file_names.push(meta.file_name().to_string()); + file_sizes.push(meta.file_size()); + num_added.push(meta.num_added_files()); + num_deleted.push(meta.num_deleted_files()); + schema_ids.push(meta.schema_id()); + min_partition_stats.push(materialize_partition_stats_value( + materialize_min_partition_stats, + stats.min_values(), + stats.null_counts(), + &partition_fields, + )?); + max_partition_stats.push(materialize_partition_stats_value( + materialize_max_partition_stats, + stats.max_values(), + stats.null_counts(), + &partition_fields, + )?); + min_row_ids.push(meta.min_row_id()); + max_row_ids.push(meta.max_row_id()); } - fn table_type(&self) -> TableType { - TableType::View - } - - async fn scan( - &self, - _state: &dyn Session, - projection: Option<&Vec>, - _filters: &[Expr], - _limit: Option, - ) -> DFResult> { - let table = self.table.clone(); - let metas = - crate::runtime::await_with_runtime(async move { collect_manifests(&table).await }) - .await - .map_err(to_datafusion_error)?; - - let n = metas.len(); - let mut file_names: Vec = Vec::with_capacity(n); - let mut file_sizes = Vec::with_capacity(n); - let mut num_added = Vec::with_capacity(n); - let mut num_deleted = Vec::with_capacity(n); - let mut schema_ids = Vec::with_capacity(n); - let mut min_partition_stats: Vec> = Vec::with_capacity(n); - let mut max_partition_stats: Vec> = Vec::with_capacity(n); - let mut min_row_ids: Vec> = Vec::with_capacity(n); - let mut max_row_ids: Vec> = Vec::with_capacity(n); - let partition_fields = self.table.schema().partition_fields(); - let projected_columns = projection.map(Vec::as_slice); - let materialize_min_partition_stats = - should_materialize_column(projected_columns, MIN_PARTITION_STATS_INDEX); - let materialize_max_partition_stats = - should_materialize_column(projected_columns, MAX_PARTITION_STATS_INDEX); - - for meta in metas { - let stats = meta.partition_stats(); - file_names.push(meta.file_name().to_string()); - file_sizes.push(meta.file_size()); - num_added.push(meta.num_added_files()); - num_deleted.push(meta.num_deleted_files()); - schema_ids.push(meta.schema_id()); - min_partition_stats.push( - materialize_partition_stats_value( - materialize_min_partition_stats, - stats.min_values(), - stats.null_counts(), - &partition_fields, - ) - .map_err(to_datafusion_error)?, - ); - max_partition_stats.push( - materialize_partition_stats_value( - materialize_max_partition_stats, - stats.max_values(), - stats.null_counts(), - &partition_fields, - ) - .map_err(to_datafusion_error)?, - ); - min_row_ids.push(meta.min_row_id()); - max_row_ids.push(meta.max_row_id()); - } - - let schema = manifests_schema(); - let batch = RecordBatch::try_new( - schema.clone(), - vec![ - Arc::new(StringArray::from(file_names)), - Arc::new(Int64Array::from(file_sizes)), - Arc::new(Int64Array::from(num_added)), - Arc::new(Int64Array::from(num_deleted)), - Arc::new(Int64Array::from(schema_ids)), - Arc::new(StringArray::from(min_partition_stats)), - Arc::new(StringArray::from(max_partition_stats)), - Arc::new(Int64Array::from(min_row_ids)), - Arc::new(Int64Array::from(max_row_ids)), - ], - )?; - - Ok(MemorySourceConfig::try_new_exec( - &[vec![batch]], - schema, - projection.cloned(), - )?) - } + let schema = manifests_schema(); + let batch = super::record_batch( + schema.clone(), + vec![ + Arc::new(StringArray::from(file_names)), + Arc::new(Int64Array::from(file_sizes)), + Arc::new(Int64Array::from(num_added)), + Arc::new(Int64Array::from(num_deleted)), + Arc::new(Int64Array::from(schema_ids)), + Arc::new(StringArray::from(min_partition_stats)), + Arc::new(StringArray::from(max_partition_stats)), + Arc::new(Int64Array::from(min_row_ids)), + Arc::new(Int64Array::from(max_row_ids)), + ], + )?; + + Ok(batch) } -async fn collect_manifests(table: &Table) -> paimon::Result> { +async fn collect_manifests(table: &Table) -> crate::Result> { let file_io = table.file_io(); let manifest_sm = - paimon::table::SnapshotManager::new(file_io.clone(), table.location().to_string()); + crate::table::SnapshotManager::new(file_io.clone(), table.location().to_string()); let Some(snapshot) = table.resolve_read_snapshot().await? else { return Ok(Vec::new()); }; @@ -198,7 +156,7 @@ fn materialize_partition_stats_value( value_bytes: &[u8], null_counts: &[Option], partition_fields: &[DataField], -) -> paimon::Result> { +) -> crate::Result> { if materialize { format_partition_stats_value(value_bytes, null_counts, partition_fields) } else { @@ -210,7 +168,7 @@ fn format_partition_stats_value( value_bytes: &[u8], null_counts: &[Option], partition_fields: &[DataField], -) -> paimon::Result> { +) -> crate::Result> { if value_bytes.is_empty() { return if partition_fields.is_empty() || null_counts.len() == partition_fields.len() { Ok(Some(format_all_null_partition_row(partition_fields.len()))) @@ -233,7 +191,7 @@ fn format_all_null_partition_row(arity: usize) -> String { #[cfg(test)] mod tests { use super::*; - use paimon::spec::{DataType as PaimonDataType, Datum, FloatType, IntType, VarCharType}; + use crate::spec::{DataType as PaimonDataType, Datum, FloatType, IntType, VarCharType}; fn field(name: &str, data_type: PaimonDataType) -> DataField { DataField::new(0, name.to_string(), data_type) diff --git a/crates/paimon/src/table/system_table/mod.rs b/crates/paimon/src/table/system_table/mod.rs new file mode 100644 index 000000000..798198d56 --- /dev/null +++ b/crates/paimon/src/table/system_table/mod.rs @@ -0,0 +1,682 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Read-only metadata tables shared by Native bindings and query engines. + +use crate::spec::{CoreOptions, DataField, Predicate, RowType}; +use crate::table::{ArrowRecordBatchStream, Table}; +use crate::{Error, Result}; +use arrow_array::{ArrayRef, RecordBatch}; +use arrow_schema::{Schema, SchemaRef}; +use std::sync::Arc; + +mod branches; +mod buckets; +mod consumers; +mod files; +mod manifests; +mod options; +mod row_string_cast; +mod schemas; +mod snapshots; +mod table_indexes; +mod tags; + +const NAMES: &[&str] = &[ + "branches", + "buckets", + "consumers", + "files", + "file_key_ranges", + "manifests", + "options", + "schemas", + "snapshots", + "table_indexes", + "tags", +]; + +/// A read-only system-table view retaining the resolved base table environment. +#[derive(Debug, Clone)] +pub struct SystemTable { + table: Table, + name: &'static str, + fields: Vec, +} + +impl SystemTable { + pub(crate) fn new(table: Table, name: &str) -> Result { + let options = CoreOptions::new(table.schema().options()); + if options.table_type()?.requires_table_engine() { + return Err(invalid("System tables require a Paimon table")); + } + options.ensure_read_authorized()?; + let read = SystemTableRead::new(name)?; + let name = read.name; + let fields = read.fields; + Ok(Self { + table, + name, + fields, + }) + } + + /// Registered system-table suffix. + pub fn name(&self) -> &str { + self.name + } + + /// Declared Arrow output schema. + pub fn schema(&self) -> SchemaRef { + schema_for_name(self.name) + } + + /// Logical fields, preserving Java system-table field IDs. + pub fn row_type(&self) -> RowType { + RowType::with_nullable(false, self.fields.clone()) + } + + /// Configure a metadata scan and read using the ordinary read-builder shape. + pub fn new_read_builder(&self) -> SystemReadBuilder { + SystemReadBuilder { + table: self.clone(), + read: SystemTableRead::new(self.name).expect("validated system-table name"), + } + } + + async fn collect( + &self, + projection: Option<&[usize]>, + predicate: Option<&Predicate>, + ) -> Result { + match self.name { + "branches" => branches::read(&self.table, projection, predicate).await, + "buckets" => buckets::read(&self.table, projection, predicate).await, + "consumers" => consumers::read(&self.table, projection, predicate).await, + "files" | "file_key_ranges" => { + files::read( + &self.table, + self.name == "file_key_ranges", + projection, + predicate, + ) + .await + } + "manifests" => manifests::read(&self.table, projection, predicate).await, + "options" => options::read(&self.table, projection, predicate).await, + "schemas" => schemas::read(&self.table, projection, predicate).await, + "snapshots" => snapshots::read(&self.table, projection, predicate).await, + "table_indexes" => table_indexes::read(&self.table, projection, predicate).await, + "tags" => tags::read(&self.table, projection, predicate).await, + _ => unreachable!("validated system-table name"), + } + } +} + +/// Read configuration for a system table. Predicates always use the full schema. +#[derive(Debug, Clone)] +pub struct SystemReadBuilder { + table: SystemTable, + read: SystemTableRead, +} + +impl SystemReadBuilder { + /// Select output columns by name, preserving their requested order. + pub fn with_projection(&mut self, columns: &[&str]) -> Result<&mut Self> { + self.read.with_projection(columns)?; + Ok(self) + } + + /// Filter using a core Predicate, resolved against the full schema. + pub fn with_filter(&mut self, predicate: Predicate) -> Result<&mut Self> { + self.read.with_filter(predicate)?; + Ok(self) + } + + /// Limit visible rows across all splits, after filtering. + pub fn with_limit(&mut self, limit: usize) -> &mut Self { + self.read.with_limit(limit); + self + } + + /// Full logical schema used to resolve filter columns. + pub fn full_row_type(&self) -> RowType { + self.table.row_type() + } + + /// Logical output fields. + pub fn read_type(&self) -> Vec { + self.read.read_type() + } + + /// Arrow output schema, including an explicitly empty projection. + pub fn schema(&self) -> SchemaRef { + self.read.schema() + } + + /// Capture metadata for this table and the columns needed by the read. + pub fn new_scan(&self) -> SystemTableScan { + SystemTableScan { + builder: self.clone(), + } + } + + /// Build an in-memory reader without retaining the source table environment. + pub fn new_read(&self) -> SystemTableRead { + self.read.clone() + } +} + +/// A scan that captures one metadata view. Reads never reload its source files. +#[derive(Debug, Clone)] +pub struct SystemTableScan { + builder: SystemReadBuilder, +} + +impl SystemTableScan { + /// Capture the configured metadata, preserving an empty result's schema. + pub async fn plan(&self) -> Result { + let builder = &self.builder; + let mut needed = builder.read.projection.clone(); + if let Some(indices) = &mut needed { + let mut seen = std::collections::HashSet::new(); + indices.retain(|index| seen.insert(*index)); + if let Some(predicate) = &builder.read.predicate { + let mut leaves = Vec::new(); + crate::arrow::residual::collect_predicate_leaf_refs(predicate, &mut leaves); + for (column, _) in leaves { + let index = builder + .table + .schema() + .index_of(column) + .map_err(arrow_error)?; + if !indices.contains(&index) { + indices.push(index); + } + } + } + } + let batch = builder + .table + .collect(needed.as_deref(), builder.read.predicate.as_ref()) + .await?; + let batch = match needed { + Some(indices) => batch.project(&indices).map_err(arrow_error)?, + None => batch, + }; + Ok(SystemTablePlan { + splits: vec![SystemSplit { + name: builder.table.name.to_string(), + batch, + }], + }) + } +} + +/// In-memory metadata splits for one captured system-table scan. +#[derive(Debug, Clone)] +pub struct SystemTablePlan { + splits: Vec, +} + +impl SystemTablePlan { + /// Splits can be read without consulting the catalog or source metadata again. + pub fn splits(&self) -> &[SystemSplit] { + &self.splits + } +} + +/// Captured metadata rows. Arrow transport preserves the scan's selected fields. +#[derive(Debug, Clone)] +pub struct SystemSplit { + name: String, + batch: RecordBatch, +} + +impl SystemSplit { + /// Reconstruct a captured split received over an Arrow transport. + pub fn from_record_batch(name: &str, batch: RecordBatch) -> Result { + let name = NAMES + .iter() + .copied() + .find(|candidate| *candidate == name) + .ok_or_else(|| invalid("Unknown system-table split"))?; + let full = schema_for_name(name); + let mut seen = std::collections::HashSet::new(); + let mut canonical = Vec::new(); + for (index, field) in batch.schema().fields().iter().enumerate() { + let expected = full.field_with_name(field.name()).map_err(arrow_error)?; + if expected.data_type() != field.data_type() + || !seen.insert(field.name().clone()) + || !expected.is_nullable() && batch.column(index).null_count() != 0 + { + return Err(invalid("Invalid system-table split schema or nullability")); + } + canonical.push(expected.clone()); + } + let options = arrow_array::RecordBatchOptions::new().with_row_count(Some(batch.num_rows())); + let batch = RecordBatch::try_new_with_options( + Arc::new(Schema::new(canonical)), + batch.columns().to_vec(), + &options, + ) + .map_err(arrow_error)?; + Ok(Self { + name: name.to_string(), + batch, + }) + } + + /// System-table suffix identifying this split's schema. + pub fn name(&self) -> &str { + &self.name + } + /// Rows captured at scan time. + pub fn row_count(&self) -> usize { + self.batch.num_rows() + } + /// Captured batch for Arrow transport and serialization. + pub fn record_batch(&self) -> &RecordBatch { + &self.batch + } +} + +/// Exact metadata reader using the core Arrow predicate evaluator. +#[derive(Debug, Clone)] +pub struct SystemTableRead { + name: &'static str, + fields: Vec, + projection: Option>, + predicate: Option, + limit: Option, +} + +impl SystemTableRead { + /// Create a reader for captured metadata, with no catalog or file IO access. + pub fn new(name: &str) -> Result { + let name = NAMES + .iter() + .copied() + .find(|candidate| candidate.eq_ignore_ascii_case(name)) + .ok_or_else(|| { + invalid(format!( + "System table '{name}' requires a catalog or is unknown" + )) + })?; + let fields = schema_for_name(name) + .fields() + .iter() + .enumerate() + .map(|(index, field)| { + let id = if name == "snapshots" && field.name() == "writer_version" { + 15 + } else { + index as i32 + }; + Ok(DataField::new( + id, + field.name().clone(), + crate::arrow::arrow_to_paimon_type(field.data_type(), field.is_nullable())?, + )) + }) + .collect::>()?; + Ok(Self { + name, + fields, + projection: None, + predicate: None, + limit: None, + }) + } + + /// Full logical schema used to resolve filter columns. + pub fn full_row_type(&self) -> RowType { + RowType::with_nullable(false, self.fields.clone()) + } + + /// Select output columns, preserving order and repeated columns. + pub fn with_projection(&mut self, columns: &[&str]) -> Result<&mut Self> { + let schema = schema_for_name(self.name); + self.projection = Some( + columns + .iter() + .map(|name| schema.index_of(name).map_err(arrow_error)) + .collect::>()?, + ); + Ok(self) + } + + /// Filter captured rows using the full system-table schema. + pub fn with_filter(&mut self, predicate: Predicate) -> Result<&mut Self> { + self.predicate = Some(normalize_predicate(predicate, &self.fields)?); + Ok(self) + } + + /// Limit visible rows across all splits, after filtering. + pub fn with_limit(&mut self, limit: usize) -> &mut Self { + self.limit = Some(limit); + self + } + + /// Logical output fields. + pub fn read_type(&self) -> Vec { + match &self.projection { + Some(indices) => indices + .iter() + .map(|&index| self.fields[index].clone()) + .collect(), + None => self.fields.clone(), + } + } + + /// Output schema available even when no batches are emitted. + pub fn schema(&self) -> SchemaRef { + let schema = schema_for_name(self.name); + match &self.projection { + Some(indices) => Arc::new(Schema::new( + indices + .iter() + .map(|&index| schema.field(index).clone()) + .collect::>(), + )), + None => schema, + } + } + + /// Read captured splits, enforcing filter, projection and a global limit. + pub fn read(&self, splits: &[SystemSplit]) -> Result { + let builder = self.clone(); + let splits = splits.to_vec(); + for split in &splits { + if split.name != builder.name { + return Err(invalid( + "System-table split belongs to a different table type", + )); + } + let mut needed = builder.read_type(); + if let Some(predicate) = &builder.predicate { + let mut leaves = Vec::new(); + crate::arrow::residual::collect_predicate_leaf_refs(predicate, &mut leaves); + for (column, _) in leaves { + let field = builder + .fields + .iter() + .find(|field| field.name() == column) + .ok_or_else(|| invalid("Unknown predicate column"))?; + needed.push(field.clone()); + } + } + for field in needed { + split + .batch + .schema() + .index_of(field.name()) + .map_err(arrow_error)?; + } + } + Ok(Box::pin(async_stream::try_stream! { + let mut remaining = builder.limit.unwrap_or(usize::MAX); + let output = builder.read_type(); + for split in splits { + if remaining == 0 { + break; + } + let mut batch = split.batch; + if let Some(predicate) = &builder.predicate { + let mask = crate::arrow::residual::evaluate_predicates_mask( + &batch, std::slice::from_ref(predicate), &builder.fields, &builder.fields, + )?.ok_or_else(|| invalid("System-table predicate could not be evaluated exactly"))?; + batch = arrow_select::filter::filter_record_batch(&batch, &mask) + .map_err(arrow_error)?; + } + let indices = output.iter() + .map(|field| batch.schema().index_of(field.name()).map_err(arrow_error)) + .collect::>>()?; + batch = batch.project(&indices).map_err(arrow_error)?; + if batch.num_rows() > remaining { + batch = batch.slice(0, remaining); + } + remaining -= batch.num_rows(); + yield batch; + } + })) + } +} + +fn normalize_predicate(predicate: Predicate, fields: &[DataField]) -> Result { + Ok(match predicate { + Predicate::Leaf { + column, + op, + literals, + .. + } => { + let (index, field) = fields + .iter() + .enumerate() + .find(|(_, field)| field.name() == column) + .ok_or_else(|| invalid(format!("Unknown system-table filter column '{column}'")))?; + Predicate::Leaf { + column, + index, + data_type: field.data_type().clone(), + op, + literals, + } + } + Predicate::And(children) => Predicate::And( + children + .into_iter() + .map(|p| normalize_predicate(p, fields)) + .collect::>()?, + ), + Predicate::Or(children) => Predicate::Or( + children + .into_iter() + .map(|p| normalize_predicate(p, fields)) + .collect::>()?, + ), + Predicate::Not(child) => Predicate::Not(Box::new(normalize_predicate(*child, fields)?)), + other => other, + }) +} + +fn schema_for_name(name: &str) -> SchemaRef { + match name { + "branches" => branches::branches_schema(), + "buckets" => buckets::buckets_schema(), + "consumers" => consumers::consumers_schema(), + "files" => files::files_schema(), + "file_key_ranges" => files::key_ranges_schema(), + "manifests" => manifests::manifests_schema(), + "options" => options::options_schema(), + "schemas" => schemas::schemas_schema(), + "snapshots" => snapshots::snapshots_schema(), + "table_indexes" => table_indexes::table_indexes_schema(), + "tags" => tags::tags_schema(), + _ => unreachable!("validated name"), + } +} + +fn record_batch(schema: SchemaRef, columns: Vec) -> Result { + RecordBatch::try_new(schema, columns).map_err(arrow_error) +} + +fn local_timestamp_column(values: Vec) -> Result { + use crate::spec::{DataType, LocalZonedTimestampType, TimestampType}; + use arrow_array::TimestampMillisecondArray; + let values: ArrayRef = Arc::new(TimestampMillisecondArray::from(values)); + crate::arrow::cast_array_for_schema_evolution( + &values, + &DataType::LocalZonedTimestamp(LocalZonedTimestampType::new(3)?), + &DataType::Timestamp(TimestampType::new(3)?), + ) +} + +fn invalid(message: impl Into) -> Error { + Error::DataInvalid { + message: message.into(), + source: None, + } +} +fn arrow_error(error: arrow_schema::ArrowError) -> Error { + Error::DataInvalid { + message: error.to_string(), + source: Some(Box::new(error)), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::spec::{ + DataType, Datum, IntType, PredicateOperator, Schema as PaimonSchema, TableSchema, + }; + use arrow_array::{Int64Array, StringArray}; + use arrow_schema::{DataType as ArrowType, Field}; + use futures::TryStreamExt; + + fn table() -> SystemTable { + let file_io = crate::io::FileIO::from_path("memory:///") + .unwrap() + .build() + .unwrap(); + let schema = PaimonSchema::builder() + .column("id", DataType::Int(IntType::new())) + .build() + .unwrap(); + Table::from_resolved_schema( + file_io, + crate::catalog::Identifier::new("default", "metadata"), + "memory:///metadata".to_string(), + TableSchema::new(0, &schema), + "main", + ) + .unwrap() + .system_table("options") + .unwrap() + } + + fn split(keys: Vec<&str>, values: Vec<&str>) -> SystemSplit { + let batch = RecordBatch::try_new( + Arc::new(Schema::new(vec![ + Field::new("key", ArrowType::Utf8, true), + Field::new("value", ArrowType::Utf8, true), + ])), + vec![ + Arc::new(StringArray::from(keys)), + Arc::new(StringArray::from(values)), + ], + ) + .unwrap(); + SystemSplit::from_record_batch("options", batch).unwrap() + } + + #[tokio::test] + async fn filter_projection_and_global_limit_use_captured_batches() { + let table = table(); + let splits = [ + split(vec!["a", "b"], vec!["skip", "first"]), + split(vec!["b", "b"], vec!["second", "third"]), + ]; + let mut builder = table.new_read_builder(); + builder.with_projection(&["value"]).unwrap(); + builder + .with_filter(Predicate::Leaf { + column: "key".into(), + index: 999, + data_type: DataType::VarChar(crate::spec::VarCharType::string_type()), + op: PredicateOperator::Eq, + literals: vec![Datum::String("b".into())], + }) + .unwrap(); + builder.with_limit(2); + let result: Vec<_> = builder + .new_read() + .read(&splits) + .unwrap() + .try_collect() + .await + .unwrap(); + let values: Vec<_> = result + .iter() + .flat_map(|batch| { + batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap() + .iter() + .flatten() + }) + .collect(); + assert_eq!(values, ["first", "second"]); + assert_eq!(result[0].schema(), builder.schema()); + builder.with_limit(0); + assert!(builder + .new_read() + .read(&splits) + .unwrap() + .try_collect::>() + .await + .unwrap() + .is_empty()); + } + + #[tokio::test] + async fn empty_projection_preserves_row_count_and_transport() { + let mut builder = table().new_read_builder(); + builder.with_projection(&[]).unwrap(); + let batches: Vec<_> = builder + .new_read() + .read(&[split(vec!["a", "b"], vec!["1", "2"])]) + .unwrap() + .try_collect() + .await + .unwrap(); + assert_eq!(batches[0].num_columns(), 0); + assert_eq!(batches[0].num_rows(), 2); + let restored = SystemSplit::from_record_batch("options", batches[0].clone()).unwrap(); + assert_eq!(restored.row_count(), 2); + } + + #[test] + fn split_transport_checks_type_names_and_required_values() { + let schema = Arc::new(Schema::new(vec![Field::new("key", ArrowType::Utf8, true)])); + let batch = RecordBatch::try_new( + schema, + vec![Arc::new(StringArray::from(vec![Some("a"), None]))], + ) + .unwrap(); + assert!(SystemSplit::from_record_batch("options", batch).is_err()); + let schema = Arc::new(Schema::new(vec![Field::new( + "key", + ArrowType::Int64, + false, + )])); + let batch = + RecordBatch::try_new(schema, vec![Arc::new(Int64Array::from(vec![1]))]).unwrap(); + assert!(SystemSplit::from_record_batch("options", batch).is_err()); + let mut builder = table().new_read_builder(); + assert!(builder.with_projection(&["missing"]).is_err()); + let wrong = SystemSplit { + name: "snapshots".into(), + batch: split(vec!["a"], vec!["b"]).batch, + }; + assert!(builder.new_read().read(&[wrong]).is_err()); + } +} diff --git a/crates/paimon/src/table/system_table/options.rs b/crates/paimon/src/table/system_table/options.rs new file mode 100644 index 000000000..cce58793c --- /dev/null +++ b/crates/paimon/src/table/system_table/options.rs @@ -0,0 +1,64 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Mirrors Java [OptionsTable](https://github.com/apache/paimon/blob/release-1.3/paimon-core/src/main/java/org/apache/paimon/table/system/OptionsTable.java). + +use std::sync::{Arc, OnceLock}; + +use crate::table::Table; +use arrow_array::{RecordBatch, StringArray}; +use arrow_schema::{DataType, Field, Schema, SchemaRef}; + +use crate::Result; + +pub(super) fn options_schema() -> SchemaRef { + static SCHEMA: OnceLock = OnceLock::new(); + SCHEMA + .get_or_init(|| { + Arc::new(Schema::new(vec![ + Field::new("key", DataType::Utf8, false), + Field::new("value", DataType::Utf8, false), + ])) + }) + .clone() +} + +pub(super) async fn read( + table: &Table, + _projection: Option<&[usize]>, + _predicate: Option<&crate::spec::Predicate>, +) -> Result { + let manager = table.schema_manager(); + let latest = manager + .latest() + .await? + .ok_or_else(|| crate::Error::DataInvalid { + message: "Table does not exist".to_string(), + source: None, + })?; + // Java uses LinkedHashMap insertion order; HashMap has none — sort for stable output. + let mut entries: Vec<(&String, &String)> = latest.options().iter().collect(); + entries.sort_by(|a, b| a.0.cmp(b.0)); + + let keys = StringArray::from_iter_values(entries.iter().map(|(k, _)| k.as_str())); + let values = StringArray::from_iter_values(entries.iter().map(|(_, v)| v.as_str())); + + let schema = options_schema(); + let batch = super::record_batch(schema.clone(), vec![Arc::new(keys), Arc::new(values)])?; + + Ok(batch) +} diff --git a/crates/integrations/datafusion/src/system_tables/row_string_cast.rs b/crates/paimon/src/table/system_table/row_string_cast.rs similarity index 97% rename from crates/integrations/datafusion/src/system_tables/row_string_cast.rs rename to crates/paimon/src/table/system_table/row_string_cast.rs index 1bcfa503a..1101f606f 100644 --- a/crates/integrations/datafusion/src/system_tables/row_string_cast.rs +++ b/crates/paimon/src/table/system_table/row_string_cast.rs @@ -17,10 +17,10 @@ use std::num::NonZeroI32; +use crate::spec::{BinaryRow, DataField, DataType, Datum}; +use crate::{Error, Result}; use chrono::NaiveDate; use lexical_write_float::{format::STANDARD, Options, ToLexicalWithOptions}; -use paimon::spec::{BinaryRow, DataField, DataType, Datum}; -use paimon::{Error, Result}; const MILLIS_PER_DAY: i64 = 86_400_000; const JAVA_FLOAT_OPTIONS: Options = Options::builder() @@ -87,7 +87,7 @@ fn format_object_field( None => return Ok(None), Some(Datum::Date(value) | Datum::Time(value)) => value.to_string(), Some(Datum::Timestamp { millis, nanos } | Datum::LocalZonedTimestamp { millis, nanos }) => { - paimon::common::format_iso_timestamp( + crate::common::format_iso_timestamp( millis.div_euclid(1000), (millis.rem_euclid(1000) * 1_000_000 + i64::from(nanos)) as u32, ) @@ -177,12 +177,12 @@ fn format_field(row: &BinaryRow, pos: usize, data_type: &DataType) -> Result Ok(format_decimal_plain(unscaled, scale)), (Datum::Timestamp { .. } | Datum::LocalZonedTimestamp { .. }, _) => { let source = row.field_to_arrow(pos, data_type)?; - let target = DataType::VarChar(paimon::spec::VarCharType::string_type()); + let target = DataType::VarChar(crate::spec::VarCharType::string_type()); let strings = - paimon::arrow::cast_array_for_schema_evolution(&source, data_type, &target)?; + crate::arrow::cast_array_for_schema_evolution(&source, data_type, &target)?; let strings = strings .as_any() - .downcast_ref::() + .downcast_ref::() .ok_or_else(|| { data_invalid("Timestamp string cast returned a non-string array".into()) })?; @@ -302,12 +302,12 @@ fn data_invalid(message: String) -> Error { #[cfg(test)] mod tests { use super::*; - use chrono::{Local, TimeZone}; - use paimon::spec::{ + use crate::spec::{ BigIntType, BinaryType, BlobType, BooleanType, CharType, DateType, DecimalType, DoubleType, FloatType, IntType, LocalZonedTimestampType, SmallIntType, TimeType, TimestampType, TinyIntType, VarBinaryType, VarCharType, }; + use chrono::{Local, TimeZone}; fn field(id: i32, data_type: DataType) -> DataField { DataField::new(id, format!("f{id}"), data_type) diff --git a/crates/paimon/src/table/system_table/schemas.rs b/crates/paimon/src/table/system_table/schemas.rs new file mode 100644 index 000000000..638ab6a9d --- /dev/null +++ b/crates/paimon/src/table/system_table/schemas.rs @@ -0,0 +1,98 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Mirrors Java [SchemasTable](https://github.com/apache/paimon/blob/release-1.3/paimon-core/src/main/java/org/apache/paimon/table/system/SchemasTable.java). + +use std::sync::{Arc, OnceLock}; + +use crate::table::Table; +use arrow_array::{Int64Array, RecordBatch, StringArray}; +use arrow_schema::{DataType, Field, Schema, SchemaRef, TimeUnit}; +use serde::Serialize; + +use crate::Result; + +pub(super) fn schemas_schema() -> SchemaRef { + static SCHEMA: OnceLock = OnceLock::new(); + SCHEMA + .get_or_init(|| { + Arc::new(Schema::new(vec![ + Field::new("schema_id", DataType::Int64, false), + Field::new("fields", DataType::Utf8, false), + Field::new("partition_keys", DataType::Utf8, false), + Field::new("primary_keys", DataType::Utf8, false), + Field::new("options", DataType::Utf8, false), + Field::new("comment", DataType::Utf8, true), + Field::new( + "update_time", + DataType::Timestamp(TimeUnit::Millisecond, None), + false, + ), + ])) + }) + .clone() +} + +pub(super) async fn read( + table: &Table, + _projection: Option<&[usize]>, + _predicate: Option<&crate::spec::Predicate>, +) -> Result { + let schemas = table.schema_manager().list_all().await?; + + let n = schemas.len(); + let mut schema_ids: Vec = Vec::with_capacity(n); + let mut fields_json: Vec = Vec::with_capacity(n); + let mut partition_keys_json: Vec = Vec::with_capacity(n); + let mut primary_keys_json: Vec = Vec::with_capacity(n); + let mut options_json: Vec = Vec::with_capacity(n); + let mut comments: Vec> = Vec::with_capacity(n); + let mut update_times: Vec = Vec::with_capacity(n); + + for schema in &schemas { + schema_ids.push(schema.id()); + fields_json.push(to_json(schema.fields())?); + partition_keys_json.push(to_json(schema.partition_keys())?); + primary_keys_json.push(to_json(schema.primary_keys())?); + options_json.push(to_json(schema.options())?); + comments.push(schema.comment().map(str::to_string)); + update_times.push(schema.time_millis()); + } + + let schema = schemas_schema(); + let batch = super::record_batch( + schema.clone(), + vec![ + Arc::new(Int64Array::from(schema_ids)), + Arc::new(StringArray::from(fields_json)), + Arc::new(StringArray::from(partition_keys_json)), + Arc::new(StringArray::from(primary_keys_json)), + Arc::new(StringArray::from(options_json)), + Arc::new(StringArray::from(comments)), + super::local_timestamp_column(update_times)?, + ], + )?; + + Ok(batch) +} + +fn to_json(value: &T) -> Result { + serde_json::to_string(value).map_err(|e| crate::Error::DataInvalid { + message: e.to_string(), + source: Some(Box::new(e)), + }) +} diff --git a/crates/paimon/src/table/system_table/snapshots.rs b/crates/paimon/src/table/system_table/snapshots.rs new file mode 100644 index 000000000..90d85e5ef --- /dev/null +++ b/crates/paimon/src/table/system_table/snapshots.rs @@ -0,0 +1,123 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Mirrors Java [SnapshotsTable](https://github.com/apache/paimon/blob/release-1.3/paimon-core/src/main/java/org/apache/paimon/table/system/SnapshotsTable.java). + +use std::sync::{Arc, OnceLock}; + +use crate::table::Table; +use arrow_array::{Int64Array, RecordBatch, StringArray}; +use arrow_schema::{DataType, Field, Schema, SchemaRef, TimeUnit}; + +use crate::Result; + +pub(super) fn snapshots_schema() -> SchemaRef { + static SCHEMA: OnceLock = OnceLock::new(); + SCHEMA + .get_or_init(|| { + Arc::new(Schema::new(vec![ + Field::new("snapshot_id", DataType::Int64, false), + Field::new("schema_id", DataType::Int64, false), + Field::new("commit_user", DataType::Utf8, false), + Field::new("commit_identifier", DataType::Int64, false), + Field::new("commit_kind", DataType::Utf8, false), + Field::new( + "commit_time", + DataType::Timestamp(TimeUnit::Millisecond, None), + false, + ), + Field::new("base_manifest_list", DataType::Utf8, false), + Field::new("delta_manifest_list", DataType::Utf8, false), + Field::new("changelog_manifest_list", DataType::Utf8, true), + Field::new("total_record_count", DataType::Int64, true), + Field::new("delta_record_count", DataType::Int64, true), + Field::new("changelog_record_count", DataType::Int64, true), + Field::new("watermark", DataType::Int64, true), + Field::new("next_row_id", DataType::Int64, true), + Field::new("writer_version", DataType::Utf8, true), + ])) + }) + .clone() +} + +pub(super) async fn read( + table: &Table, + _projection: Option<&[usize]>, + _predicate: Option<&crate::spec::Predicate>, +) -> Result { + let sm = table.snapshot_manager(); + let snapshots = sm.list_all().await?; + + let n = snapshots.len(); + let mut snapshot_ids = Vec::with_capacity(n); + let mut schema_ids = Vec::with_capacity(n); + let mut commit_users: Vec = Vec::with_capacity(n); + let mut commit_identifiers = Vec::with_capacity(n); + let mut commit_kinds: Vec = Vec::with_capacity(n); + let mut commit_times = Vec::with_capacity(n); + let mut base_manifest_lists: Vec = Vec::with_capacity(n); + let mut delta_manifest_lists: Vec = Vec::with_capacity(n); + let mut changelog_manifest_lists: Vec> = Vec::with_capacity(n); + let mut total_record_counts: Vec> = Vec::with_capacity(n); + let mut delta_record_counts: Vec> = Vec::with_capacity(n); + let mut changelog_record_counts: Vec> = Vec::with_capacity(n); + let mut watermarks: Vec> = Vec::with_capacity(n); + let mut next_row_ids: Vec> = Vec::with_capacity(n); + + let mut writer_versions = Vec::with_capacity(n); + for snap in &snapshots { + snapshot_ids.push(snap.id()); + schema_ids.push(snap.schema_id()); + commit_users.push(snap.commit_user().to_string()); + commit_identifiers.push(snap.commit_identifier()); + commit_kinds.push(snap.commit_kind().to_string()); + commit_times.push(snap.time_millis() as i64); + base_manifest_lists.push(snap.base_manifest_list().to_string()); + delta_manifest_lists.push(snap.delta_manifest_list().to_string()); + changelog_manifest_lists.push(snap.changelog_manifest_list().map(str::to_string)); + total_record_counts.push(snap.total_record_count()); + delta_record_counts.push(snap.delta_record_count()); + changelog_record_counts.push(snap.changelog_record_count()); + watermarks.push(snap.watermark()); + next_row_ids.push(snap.next_row_id()); + writer_versions.push(snap.writer_version()); + } + + let schema = snapshots_schema(); + let batch = super::record_batch( + schema.clone(), + vec![ + Arc::new(Int64Array::from(snapshot_ids)), + Arc::new(Int64Array::from(schema_ids)), + Arc::new(StringArray::from(commit_users)), + Arc::new(Int64Array::from(commit_identifiers)), + Arc::new(StringArray::from(commit_kinds)), + super::local_timestamp_column(commit_times)?, + Arc::new(StringArray::from(base_manifest_lists)), + Arc::new(StringArray::from(delta_manifest_lists)), + Arc::new(StringArray::from(changelog_manifest_lists)), + Arc::new(Int64Array::from(total_record_counts)), + Arc::new(Int64Array::from(delta_record_counts)), + Arc::new(Int64Array::from(changelog_record_counts)), + Arc::new(Int64Array::from(watermarks)), + Arc::new(Int64Array::from(next_row_ids)), + Arc::new(StringArray::from(writer_versions)), + ], + )?; + + Ok(batch) +} diff --git a/crates/integrations/datafusion/src/system_tables/table_indexes.rs b/crates/paimon/src/table/system_table/table_indexes.rs similarity index 59% rename from crates/integrations/datafusion/src/system_tables/table_indexes.rs rename to crates/paimon/src/table/system_table/table_indexes.rs index b81fa825d..44d1d088e 100644 --- a/crates/integrations/datafusion/src/system_tables/table_indexes.rs +++ b/crates/paimon/src/table/system_table/table_indexes.rs @@ -19,31 +19,21 @@ use std::sync::{Arc, OnceLock}; -use async_trait::async_trait; -use datafusion::arrow::array::builder::{ - ArrayBuilder, Int32Builder, Int64Builder, ListBuilder, StringBuilder, StructBuilder, -}; -use datafusion::arrow::array::{ArrayRef, Int32Array, Int64Array, RecordBatch, StringArray}; -use datafusion::arrow::datatypes::{DataType, Field, Fields, Schema, SchemaRef}; -use datafusion::catalog::Session; -use datafusion::datasource::memory::MemorySourceConfig; -use datafusion::datasource::{TableProvider, TableType}; -use datafusion::error::Result as DFResult; -use datafusion::logical_expr::Expr; -use datafusion::physical_plan::ExecutionPlan; -use paimon::spec::{ +use crate::spec::{ BinaryRow, DataField, DeletionVectorMeta, FileKind, IndexManifest, IndexManifestEntry, }; -use paimon::table::Table; +use crate::table::Table; +use arrow_array::builder::{ + ArrayBuilder, Int32Builder, Int64Builder, ListBuilder, StringBuilder, StructBuilder, +}; +use arrow_array::{ArrayRef, Int32Array, Int64Array, RecordBatch, StringArray}; +use arrow_schema::{DataType, Field, Fields, Schema, SchemaRef}; use super::row_string_cast::format_row_as_java_cast_string; -use crate::error::to_datafusion_error; -pub(super) fn build(table: Table) -> DFResult> { - Ok(Arc::new(TableIndexesTable { table })) -} +use crate::Result; -fn table_indexes_schema() -> SchemaRef { +pub(super) fn table_indexes_schema() -> SchemaRef { static SCHEMA: OnceLock = OnceLock::new(); SCHEMA .get_or_init(|| { @@ -82,108 +72,82 @@ fn dv_meta_fields() -> Fields { .into() } -#[derive(Debug)] -pub(super) struct TableIndexesTable { - table: Table, -} - -#[async_trait] -impl TableProvider for TableIndexesTable { - fn schema(&self) -> SchemaRef { - table_indexes_schema() - } - - fn table_type(&self) -> TableType { - TableType::View - } - - async fn scan( - &self, - _state: &dyn Session, - projection: Option<&Vec>, - _filters: &[Expr], - _limit: Option, - ) -> DFResult> { - let table = self.table.clone(); - let entries = - crate::runtime::await_with_runtime(async move { collect_index_entries(&table).await }) - .await - .map_err(to_datafusion_error)?; - - let partition_fields = self.table.schema().partition_fields(); - let fields = self.table.schema().fields(); - let n = entries.len(); - let mut partitions: Vec> = Vec::with_capacity(n); - let mut buckets = Vec::with_capacity(n); - let mut index_types = Vec::with_capacity(n); - let mut file_names = Vec::with_capacity(n); - let mut file_sizes = Vec::with_capacity(n); - let mut row_counts = Vec::with_capacity(n); - let mut dv_ranges = dv_ranges_builder(); - let mut row_range_starts: Vec> = Vec::with_capacity(n); - let mut row_range_ends: Vec> = Vec::with_capacity(n); - let mut index_field_ids: Vec> = Vec::with_capacity(n); - let mut index_field_names: Vec> = Vec::with_capacity(n); - - for entry in &entries { - let index_file = &entry.index_file; - partitions.push(Some(format_partition(&entry.partition, &partition_fields)?)); - buckets.push(entry.bucket); - index_types.push(index_file.index_type.as_str()); - file_names.push(index_file.file_name.as_str()); - file_sizes.push(index_file.file_size); - row_counts.push(index_file.row_count); - append_dv_ranges( - &mut dv_ranges, - index_file - .deletion_vectors_ranges - .as_ref() - .map(|ranges| ranges.iter()), - ); +pub(super) async fn read( + table: &Table, + _projection: Option<&[usize]>, + _predicate: Option<&crate::spec::Predicate>, +) -> Result { + let entries = collect_index_entries(table).await?; + + let partition_fields = table.schema().partition_fields(); + let fields = table.schema().fields(); + let n = entries.len(); + let mut partitions: Vec> = Vec::with_capacity(n); + let mut buckets = Vec::with_capacity(n); + let mut index_types = Vec::with_capacity(n); + let mut file_names = Vec::with_capacity(n); + let mut file_sizes = Vec::with_capacity(n); + let mut row_counts = Vec::with_capacity(n); + let mut dv_ranges = dv_ranges_builder(); + let mut row_range_starts: Vec> = Vec::with_capacity(n); + let mut row_range_ends: Vec> = Vec::with_capacity(n); + let mut index_field_ids: Vec> = Vec::with_capacity(n); + let mut index_field_names: Vec> = Vec::with_capacity(n); + + for entry in &entries { + let index_file = &entry.index_file; + partitions.push(Some(format_partition(&entry.partition, &partition_fields)?)); + buckets.push(entry.bucket); + index_types.push(index_file.index_type.as_str()); + file_names.push(index_file.file_name.as_str()); + file_sizes.push(index_file.file_size); + row_counts.push(index_file.row_count); + append_dv_ranges( + &mut dv_ranges, + index_file + .deletion_vectors_ranges + .as_ref() + .map(|ranges| ranges.iter()), + ); - if let Some(global_meta) = &index_file.global_index_meta { - row_range_starts.push(Some(global_meta.row_range_start)); - row_range_ends.push(Some(global_meta.row_range_end)); - index_field_ids.push(Some(global_meta.index_field_id)); - index_field_names.push(global_index_field_names(global_meta, fields)); - } else { - row_range_starts.push(None); - row_range_ends.push(None); - index_field_ids.push(None); - index_field_names.push(None); - } + if let Some(global_meta) = &index_file.global_index_meta { + row_range_starts.push(Some(global_meta.row_range_start)); + row_range_ends.push(Some(global_meta.row_range_end)); + index_field_ids.push(Some(global_meta.index_field_id)); + index_field_names.push(global_index_field_names(global_meta, fields)); + } else { + row_range_starts.push(None); + row_range_ends.push(None); + index_field_ids.push(None); + index_field_names.push(None); } + } - let schema = table_indexes_schema(); - let batch = RecordBatch::try_new( - schema.clone(), - vec![ - Arc::new(StringArray::from(partitions)), - Arc::new(Int32Array::from(buckets)), - Arc::new(StringArray::from(index_types)), - Arc::new(StringArray::from(file_names)), - Arc::new(Int64Array::from(file_sizes)), - Arc::new(Int64Array::from(row_counts)), - Arc::new(dv_ranges.finish()) as ArrayRef, - Arc::new(Int64Array::from(row_range_starts)), - Arc::new(Int64Array::from(row_range_ends)), - Arc::new(Int32Array::from(index_field_ids)), - Arc::new(StringArray::from(index_field_names)), - ], - )?; + let schema = table_indexes_schema(); + let batch = super::record_batch( + schema.clone(), + vec![ + Arc::new(StringArray::from(partitions)), + Arc::new(Int32Array::from(buckets)), + Arc::new(StringArray::from(index_types)), + Arc::new(StringArray::from(file_names)), + Arc::new(Int64Array::from(file_sizes)), + Arc::new(Int64Array::from(row_counts)), + Arc::new(dv_ranges.finish()) as ArrayRef, + Arc::new(Int64Array::from(row_range_starts)), + Arc::new(Int64Array::from(row_range_ends)), + Arc::new(Int32Array::from(index_field_ids)), + Arc::new(StringArray::from(index_field_names)), + ], + )?; - Ok(MemorySourceConfig::try_new_exec( - &[vec![batch]], - schema, - projection.cloned(), - )?) - } + Ok(batch) } -async fn collect_index_entries(table: &Table) -> paimon::Result> { +async fn collect_index_entries(table: &Table) -> crate::Result> { let file_io = table.file_io(); let manifest_sm = - paimon::table::SnapshotManager::new(file_io.clone(), table.location().to_string()); + crate::table::SnapshotManager::new(file_io.clone(), table.location().to_string()); let Some(snapshot) = table.resolve_read_snapshot().await? else { return Ok(Vec::new()); }; @@ -208,7 +172,7 @@ fn visible_index_entries(entries: Vec) -> Vec Option { let names: Option> = std::iter::once(&meta.index_field_id) @@ -223,9 +187,9 @@ fn global_index_field_names( names.map(|names| names.join(",")) } -fn format_partition(partition: &[u8], partition_fields: &[DataField]) -> DFResult { - let row = BinaryRow::from_serialized_bytes(partition).map_err(to_datafusion_error)?; - format_row_as_java_cast_string(&row, partition_fields).map_err(to_datafusion_error) +fn format_partition(partition: &[u8], partition_fields: &[DataField]) -> Result { + let row = BinaryRow::from_serialized_bytes(partition)?; + format_row_as_java_cast_string(&row, partition_fields) } fn dv_ranges_builder() -> ListBuilder { @@ -282,7 +246,7 @@ where #[cfg(test)] mod tests { use super::*; - use paimon::spec::IndexFileMeta; + use crate::spec::IndexFileMeta; fn index_entry(kind: FileKind, file_name: &str) -> IndexManifestEntry { IndexManifestEntry { @@ -304,13 +268,13 @@ mod tests { #[test] fn composite_index_names_preserve_order_and_require_every_field() { - use paimon::spec::{GlobalIndexMeta, IntType}; + use crate::spec::{GlobalIndexMeta, IntType}; let fields = vec![ - DataField::new(3, "a".into(), paimon::spec::DataType::Int(IntType::new())), + DataField::new(3, "a".into(), crate::spec::DataType::Int(IntType::new())), DataField::new( 7, "renamed".into(), - paimon::spec::DataType::Int(IntType::new()), + crate::spec::DataType::Int(IntType::new()), ), ]; let mut meta = GlobalIndexMeta { diff --git a/crates/integrations/datafusion/src/system_tables/tags.rs b/crates/paimon/src/table/system_table/tags.rs similarity index 58% rename from crates/integrations/datafusion/src/system_tables/tags.rs rename to crates/paimon/src/table/system_table/tags.rs index 02ee371fc..2aacbdc34 100644 --- a/crates/integrations/datafusion/src/system_tables/tags.rs +++ b/crates/paimon/src/table/system_table/tags.rs @@ -19,24 +19,13 @@ use std::sync::{Arc, OnceLock}; -use async_trait::async_trait; -use datafusion::arrow::array::{Int64Array, RecordBatch, StringArray, TimestampMillisecondArray}; -use datafusion::arrow::datatypes::{DataType, Field, Schema, SchemaRef, TimeUnit}; -use datafusion::catalog::Session; -use datafusion::datasource::memory::MemorySourceConfig; -use datafusion::datasource::{TableProvider, TableType}; -use datafusion::error::Result as DFResult; -use datafusion::logical_expr::Expr; -use datafusion::physical_plan::ExecutionPlan; -use paimon::table::Table; +use crate::table::Table; +use arrow_array::{Int64Array, RecordBatch, StringArray, TimestampMillisecondArray}; +use arrow_schema::{DataType, Field, Schema, SchemaRef, TimeUnit}; -use crate::error::to_datafusion_error; +use crate::Result; -pub(super) fn build(table: Table) -> DFResult> { - Ok(Arc::new(TagsTable { table })) -} - -fn tags_schema() -> SchemaRef { +pub(super) fn tags_schema() -> SchemaRef { static SCHEMA: OnceLock = OnceLock::new(); SCHEMA .get_or_init(|| { @@ -61,73 +50,48 @@ fn tags_schema() -> SchemaRef { .clone() } -#[derive(Debug)] -pub(super) struct TagsTable { - table: Table, -} - -#[async_trait] -impl TableProvider for TagsTable { - fn schema(&self) -> SchemaRef { - tags_schema() - } - - fn table_type(&self) -> TableType { - TableType::View +pub(super) async fn read( + table: &Table, + _projection: Option<&[usize]>, + _predicate: Option<&crate::spec::Predicate>, +) -> Result { + let tm = table.tag_manager(); + let tags = tm.list_all_with_metadata().await?; + + let n = tags.len(); + let mut tag_names: Vec = Vec::with_capacity(n); + let mut snapshot_ids = Vec::with_capacity(n); + let mut schema_ids = Vec::with_capacity(n); + let mut commit_times = Vec::with_capacity(n); + let mut record_counts: Vec> = Vec::with_capacity(n); + let mut create_times: Vec> = Vec::with_capacity(n); + let mut time_retained: Vec> = Vec::with_capacity(n); + + for (name, snap, created, retained) in tags { + tag_names.push(name); + snapshot_ids.push(snap.id()); + schema_ids.push(snap.schema_id()); + commit_times.push(snap.time_millis() as i64); + record_counts.push(snap.total_record_count()); + create_times.push(created); + time_retained.push(retained.map(format_duration_iso8601)); } - async fn scan( - &self, - _state: &dyn Session, - projection: Option<&Vec>, - _filters: &[Expr], - _limit: Option, - ) -> DFResult> { - let tm = self.table.tag_manager(); - let tags = - crate::runtime::await_with_runtime(async move { tm.list_all_with_metadata().await }) - .await - .map_err(to_datafusion_error)?; - - let n = tags.len(); - let mut tag_names: Vec = Vec::with_capacity(n); - let mut snapshot_ids = Vec::with_capacity(n); - let mut schema_ids = Vec::with_capacity(n); - let mut commit_times = Vec::with_capacity(n); - let mut record_counts: Vec> = Vec::with_capacity(n); - let mut create_times: Vec> = Vec::with_capacity(n); - let mut time_retained: Vec> = Vec::with_capacity(n); - - for (name, snap, created, retained) in tags { - tag_names.push(name); - snapshot_ids.push(snap.id()); - schema_ids.push(snap.schema_id()); - commit_times.push(snap.time_millis() as i64); - record_counts.push(snap.total_record_count()); - create_times.push(created); - time_retained.push(retained.map(format_duration_iso8601)); - } - - let schema = tags_schema(); - let batch = RecordBatch::try_new( - schema.clone(), - vec![ - Arc::new(StringArray::from(tag_names)), - Arc::new(Int64Array::from(snapshot_ids)), - Arc::new(Int64Array::from(schema_ids)), - super::local_timestamp_column(commit_times)?, - Arc::new(Int64Array::from(record_counts)), - Arc::new(TimestampMillisecondArray::from(create_times)), - Arc::new(StringArray::from(time_retained)), - ], - )?; - - Ok(MemorySourceConfig::try_new_exec( - &[vec![batch]], - schema, - projection.cloned(), - )?) - } + let schema = tags_schema(); + let batch = super::record_batch( + schema.clone(), + vec![ + Arc::new(StringArray::from(tag_names)), + Arc::new(Int64Array::from(snapshot_ids)), + Arc::new(Int64Array::from(schema_ids)), + super::local_timestamp_column(commit_times)?, + Arc::new(Int64Array::from(record_counts)), + Arc::new(TimestampMillisecondArray::from(create_times)), + Arc::new(StringArray::from(time_retained)), + ], + )?; + + Ok(batch) } /// Render a retention in seconds the way Java's `Duration.toString()` does, so From 47c918ed2f2a7af3ad6e9c6c8020006bdf274388 Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Fri, 16 Oct 2026 13:47:11 +0800 Subject: [PATCH 3/3] refactor: simplify metadata API to SystemRead --- .../python/pypaimon_rust/datafusion.pyi | 40 +- bindings/python/src/context.rs | 7 +- bindings/python/src/system_table.rs | 172 +---- bindings/python/src/table.rs | 8 +- .../datafusion/src/system_tables/metadata.rs | 52 +- .../datafusion/tests/system_tables.rs | 64 +- crates/paimon/src/table/mod.rs | 10 +- crates/paimon/src/table/system_table/mod.rs | 613 +++++------------- 8 files changed, 261 insertions(+), 705 deletions(-) diff --git a/bindings/python/python/pypaimon_rust/datafusion.pyi b/bindings/python/python/pypaimon_rust/datafusion.pyi index 2c53b6d2d..aa2829821 100644 --- a/bindings/python/python/pypaimon_rust/datafusion.pyi +++ b/bindings/python/python/pypaimon_rust/datafusion.pyi @@ -283,7 +283,7 @@ class Table: def branch(self) -> str: ... def location(self) -> str: ... def schema(self) -> TableSchema: ... - def system_table(self, name: str) -> SystemTable: ... + def new_system_read(self, name: str) -> SystemRead: ... def new_read_builder(self, options: Optional[Dict[str, str]] = None) -> ReadBuilder: ... def new_stream_scan(self) -> StreamTableScan: ... def new_full_text_search_builder(self) -> FullTextSearchBuilder: ... @@ -866,35 +866,11 @@ class ShardTableUpdator: def close(self) -> None: ... -class SystemTable: - def name(self) -> str: ... - def schema(self) -> pyarrow.Schema: ... - def new_read_builder(self) -> SystemReadBuilder: ... - -class SystemReadBuilder: - def with_projection(self, columns: List[str]) -> SystemReadBuilder: ... - def with_filter(self, predicate: Dict[str, Any]) -> SystemReadBuilder: ... - def with_limit(self, limit: int) -> SystemReadBuilder: ... - def new_scan(self) -> SystemTableScan: ... - def new_read(self) -> SystemTableRead: ... - -class SystemTableScan: - def plan(self) -> SystemTablePlan: ... - -class SystemTablePlan: - def splits(self) -> List[SystemSplit]: ... - -class SystemSplit: - def __init__(self, name: str, batch: pyarrow.RecordBatch) -> None: - """Restore captured metadata with validated column types and nullability.""" - ... - def row_count(self) -> int: ... - -class SystemTableRead: - def __init__(self, name: str) -> None: - """Create a reader for captured splits without accessing REST or file IO.""" +class SystemRead: + def read_type(self) -> List[DataField]: ... + def with_projection(self, columns: List[str]) -> SystemRead: ... + def with_filter(self, predicate: Dict[str, Any]) -> SystemRead: ... + def with_limit(self, limit: int) -> SystemRead: ... + def read(self) -> pyarrow.RecordBatch: + """Materialize metadata once, then filter, project and limit the result.""" ... - def with_projection(self, columns: List[str]) -> SystemTableRead: ... - def with_filter(self, predicate: Dict[str, Any]) -> SystemTableRead: ... - def with_limit(self, limit: int) -> SystemTableRead: ... - def read(self, splits: List[SystemSplit]) -> RecordBatchReader: ... diff --git a/bindings/python/src/context.rs b/bindings/python/src/context.rs index c6c929567..2d9699094 100644 --- a/bindings/python/src/context.rs +++ b/bindings/python/src/context.rs @@ -796,12 +796,7 @@ pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> let this = PyModule::new(py, "datafusion")?; this.add_class::()?; this.add_class::()?; - this.add_class::()?; - this.add_class::()?; - this.add_class::()?; - this.add_class::()?; - this.add_class::()?; - this.add_class::()?; + this.add_class::()?; this.add_class::()?; this.add_class::()?; this.add_class::()?; diff --git a/bindings/python/src/system_table.rs b/bindings/python/src/system_table.rs index e2c7e3965..41d51f4a9 100644 --- a/bindings/python/src/system_table.rs +++ b/bindings/python/src/system_table.rs @@ -15,163 +15,30 @@ // specific language governing permissions and limitations // under the License. -//! Thin bindings for the core read-only system-table API. +//! Thin bindings for core metadata reads. use crate::error::to_py_err; -use crate::read::PyRecordBatchReader; -use arrow::pyarrow::{FromPyArrow, ToPyArrow}; -use arrow::record_batch::RecordBatch; -use futures::TryStreamExt; -use paimon::table::{ - SystemReadBuilder, SystemSplit, SystemTable, SystemTablePlan, SystemTableRead, SystemTableScan, -}; +use crate::schema::PyDataField; +use arrow::pyarrow::ToPyArrow; +use paimon::table::SystemRead; use paimon_datafusion::runtime::runtime; use pyo3::prelude::*; use pyo3::types::PyDict; -#[pyclass(name = "SystemTable", module = "pypaimon_rust.datafusion")] -pub struct PySystemTable { - pub(crate) inner: SystemTable, +#[pyclass(name = "SystemRead", module = "pypaimon_rust.datafusion")] +pub struct PySystemRead { + pub(crate) inner: SystemRead, } #[pymethods] -impl PySystemTable { - fn name(&self) -> &str { - self.inner.name() - } - fn schema(&self, py: Python<'_>) -> PyResult> { - Ok(self.inner.schema().to_pyarrow(py)?.unbind()) - } - fn new_read_builder(&self) -> PySystemReadBuilder { - PySystemReadBuilder { - inner: self.inner.new_read_builder(), - } - } -} - -#[pyclass(name = "SystemReadBuilder", module = "pypaimon_rust.datafusion")] -pub struct PySystemReadBuilder { - inner: SystemReadBuilder, -} - -#[pymethods] -impl PySystemReadBuilder { - fn with_projection( - mut slf: PyRefMut<'_, Self>, - columns: Vec, - ) -> PyResult> { - let columns: Vec<_> = columns.iter().map(String::as_str).collect(); - slf.inner.with_projection(&columns).map_err(to_py_err)?; - Ok(slf) - } - fn with_filter<'py>( - mut slf: PyRefMut<'py, Self>, - predicate: &Bound<'_, PyDict>, - ) -> PyResult> { - // Predicates must see the full system-table schema, including unprojected columns. - let predicate = crate::predicate::dict_to_predicate( - predicate, - slf.inner.full_row_type().fields(), - true, - )?; - slf.inner.with_filter(predicate).map_err(to_py_err)?; - Ok(slf) - } - fn with_limit(mut slf: PyRefMut<'_, Self>, limit: usize) -> PyRefMut<'_, Self> { - slf.inner.with_limit(limit); - slf - } - fn new_scan(&self) -> PySystemTableScan { - PySystemTableScan { - inner: self.inner.new_scan(), - } - } - fn new_read(&self) -> PySystemTableRead { - PySystemTableRead { - inner: self.inner.new_read(), - } - } -} - -#[pyclass(name = "SystemTableScan", module = "pypaimon_rust.datafusion")] -pub struct PySystemTableScan { - inner: SystemTableScan, -} - -#[pymethods] -impl PySystemTableScan { - fn plan(&self, py: Python<'_>) -> PyResult { - Ok(PySystemTablePlan { - inner: py - .detach(|| runtime().block_on(self.inner.plan())) - .map_err(to_py_err)?, - }) - } -} - -#[pyclass(name = "SystemTablePlan", module = "pypaimon_rust.datafusion")] -pub struct PySystemTablePlan { - inner: SystemTablePlan, -} - -#[pymethods] -impl PySystemTablePlan { - fn splits(&self) -> Vec { +impl PySystemRead { + fn read_type(&self) -> Vec { self.inner - .splits() - .iter() - .cloned() - .map(|inner| PySystemSplit { inner }) + .read_type() + .into_iter() + .map(PyDataField::new) .collect() } -} - -type SystemSplitPickle = (Py, (String, Py)); - -#[pyclass(name = "SystemSplit", module = "pypaimon_rust.datafusion")] -pub struct PySystemSplit { - inner: SystemSplit, -} - -#[pymethods] -impl PySystemSplit { - fn row_count(&self) -> usize { - self.inner.row_count() - } - - #[new] - fn new(name: &str, batch: &Bound<'_, PyAny>) -> PyResult { - let batch = RecordBatch::from_pyarrow_bound(batch)?; - Ok(Self { - inner: SystemSplit::from_record_batch(name, batch).map_err(to_py_err)?, - }) - } - - fn __reduce__<'py>(slf: &Bound<'py, Self>, py: Python<'py>) -> PyResult { - let inner = &slf.borrow().inner; - Ok(( - slf.get_type().unbind().into_any(), - ( - inner.name().to_string(), - inner.record_batch().to_pyarrow(py)?.unbind(), - ), - )) - } -} - -#[pyclass(name = "SystemTableRead", module = "pypaimon_rust.datafusion")] -pub struct PySystemTableRead { - inner: SystemTableRead, -} - -#[pymethods] -impl PySystemTableRead { - #[new] - fn new(name: &str) -> PyResult { - Ok(Self { - inner: SystemTableRead::new(name).map_err(to_py_err)?, - }) - } fn with_projection( mut slf: PyRefMut<'_, Self>, @@ -200,15 +67,10 @@ impl PySystemTableRead { slf } - fn read(&self, splits: &Bound<'_, PyAny>) -> PyResult { - let splits = splits - .try_iter()? - .map(|item| Ok(item?.extract::>()?.inner.clone())) - .collect::>>()?; - let stream = self.inner.read(&splits).map_err(to_py_err)?; - Ok(PyRecordBatchReader::from_py_stream( - Some(self.inner.schema()), - Box::pin(stream.map_err(to_py_err)), - )) + fn read(&self, py: Python<'_>) -> PyResult> { + let batch = py + .detach(|| runtime().block_on(self.inner.read())) + .map_err(to_py_err)?; + Ok(batch.to_pyarrow(py)?.unbind()) } } diff --git a/bindings/python/src/table.rs b/bindings/python/src/table.rs index 754430126..216423943 100644 --- a/bindings/python/src/table.rs +++ b/bindings/python/src/table.rs @@ -198,10 +198,10 @@ impl PyTable { } } - /// Resolve a core metadata table retaining this table's captured environment. - fn system_table(&self, name: &str) -> PyResult { - Ok(crate::system_table::PySystemTable { - inner: self.inner.system_table(name).map_err(to_py_err)?, + /// Read metadata through the core system reader. + fn new_system_read(&self, name: &str) -> PyResult { + Ok(crate::system_table::PySystemRead { + inner: self.inner.new_system_read(name).map_err(to_py_err)?, }) } diff --git a/crates/integrations/datafusion/src/system_tables/metadata.rs b/crates/integrations/datafusion/src/system_tables/metadata.rs index b393b5b29..6c635be9b 100644 --- a/crates/integrations/datafusion/src/system_tables/metadata.rs +++ b/crates/integrations/datafusion/src/system_tables/metadata.rs @@ -26,25 +26,24 @@ use datafusion::datasource::{TableProvider, TableType}; use datafusion::error::Result; use datafusion::logical_expr::{Expr, TableProviderFilterPushDown}; use datafusion::physical_plan::ExecutionPlan; -use futures::TryStreamExt; -use paimon::table::{SystemTable, Table}; +use paimon::table::{SystemRead, Table}; use std::sync::Arc; pub(super) fn build(table: Table, name: &str) -> Result> { Ok(Arc::new(MetadataTableProvider { - table: table.system_table(name).map_err(to_datafusion_error)?, + read: table.new_system_read(name).map_err(to_datafusion_error)?, })) } #[derive(Debug)] pub(super) struct MetadataTableProvider { - table: SystemTable, + read: SystemRead, } #[async_trait] impl TableProvider for MetadataTableProvider { fn schema(&self) -> SchemaRef { - self.table.schema() + self.read.schema() } fn table_type(&self) -> TableType { TableType::View @@ -57,44 +56,41 @@ impl TableProvider for MetadataTableProvider { filters: &[Expr], _limit: Option, ) -> Result> { - let mut builder = self.table.new_read_builder(); + let mut read = self.read.clone(); if let Some(indices) = projection { - let schema = self.table.schema(); + let schema = self.read.schema(); let columns: Vec<_> = indices .iter() .map(|&index| schema.field(index).name().as_str()) .collect(); - builder - .with_projection(&columns) + read.with_projection(&columns) .map_err(to_datafusion_error)?; } - if let Some(predicate) = - crate::filter_pushdown::analyze_filters(filters, self.table.row_type().fields(), true) - .pushed_predicate + if let Some(predicate) = crate::filter_pushdown::analyze_filters( + filters, + self.read.full_row_type().fields(), + true, + ) + .pushed_predicate { - builder - .with_filter(predicate) - .map_err(to_datafusion_error)?; + read.with_filter(predicate).map_err(to_datafusion_error)?; } - let schema = builder.schema(); - let batches = crate::runtime::await_with_runtime(async move { - let plan = builder.new_scan().plan().await?; - builder - .new_read() - .read(plan.splits())? - .try_collect::>() - .await - }) - .await - .map_err(to_datafusion_error)?; - Ok(MemorySourceConfig::try_new_exec(&[batches], schema, None)?) + let schema = read.schema(); + let batch = crate::runtime::await_with_runtime(read.read()) + .await + .map_err(to_datafusion_error)?; + Ok(MemorySourceConfig::try_new_exec( + &[vec![batch]], + schema, + None, + )?) } fn supports_filters_pushdown( &self, filters: &[&Expr], ) -> Result> { - let fields = self.table.row_type(); + let fields = self.read.full_row_type(); Ok(filters .iter() .map(|filter| { diff --git a/crates/integrations/datafusion/tests/system_tables.rs b/crates/integrations/datafusion/tests/system_tables.rs index a6f15aea5..48f299bcf 100644 --- a/crates/integrations/datafusion/tests/system_tables.rs +++ b/crates/integrations/datafusion/tests/system_tables.rs @@ -79,9 +79,8 @@ async fn read_metadata( table: paimon::table::Table, name: &str, ) -> paimon::Result { - let builder = table.system_table(name)?.new_read_builder(); - let plan = builder.new_scan().plan().await?; - builder.new_read().read(plan.splits()) + let batch = table.new_system_read(name)?.read().await?; + Ok(Box::pin(futures::stream::iter([Ok(batch)]))) } // Error text of `sql`, whether it surfaces at planning or execution. @@ -1793,7 +1792,7 @@ async fn test_resolved_system_table_reader_preserves_guards_and_schema() { "table_indexes", ] { let stream = read_metadata(table.clone(), name).await.unwrap(); - let schema = table.system_table(name).unwrap().schema(); + let schema = table.new_system_read(name).unwrap().schema(); let batches: Vec<_> = stream.try_collect().await.unwrap(); assert!(!schema.fields().is_empty(), "{name}"); assert!( @@ -1865,46 +1864,31 @@ async fn test_options_ignore_resolved_dynamic_overrides() { } #[tokio::test] -async fn test_core_system_table_read_builder_filters_before_projection() { - use futures::TryStreamExt; +async fn test_core_system_read_filters_before_projection() { use paimon::spec::{Datum, Predicate, PredicateOperator}; let (_ctx, catalog, _tmp) = create_context().await; let table = catalog .get_table(&Identifier::new("default", FIXTURE_TABLE)) .await .unwrap(); - let system = table.system_table("snapshots").unwrap(); - let mut builder = system.new_read_builder(); - builder.with_projection(&["commit_user"]).unwrap(); - let field = system.row_type().fields()[0].clone(); - builder - .with_filter(Predicate::Leaf { - column: field.name().to_string(), - index: 0, - data_type: field.data_type().clone(), - op: PredicateOperator::Gt, - literals: vec![Datum::Long(0)], - }) - .unwrap(); - builder.with_limit(1); - let plan = builder.new_scan().plan().await.unwrap(); - let rows: Vec<_> = builder - .new_read() - .read(plan.splits()) - .unwrap() - .try_collect() - .await - .unwrap(); - assert_eq!(rows.iter().map(RecordBatch::num_rows).sum::(), 1); - assert_eq!(rows[0].schema().fields().len(), 1); - assert_eq!(rows[0].schema().field(0).name(), "commit_user"); - builder.with_limit(0); - let rows: Vec<_> = builder - .new_read() - .read(plan.splits()) - .unwrap() - .try_collect() - .await - .unwrap(); - assert!(rows.iter().all(|batch| batch.num_rows() == 0)); + let mut read = table.new_system_read("snapshots").unwrap(); + let field = read.read_type()[0].clone(); + read.with_projection(&["commit_user"]).unwrap(); + read.with_filter(Predicate::Leaf { + column: field.name().to_string(), + index: 0, + data_type: field.data_type().clone(), + op: PredicateOperator::Gt, + literals: vec![Datum::Long(0)], + }) + .unwrap(); + read.with_limit(1); + let rows = read.read().await.unwrap(); + assert_eq!(rows.num_rows(), 1); + assert_eq!(rows.schema().fields().len(), 1); + assert_eq!(rows.schema().field(0).name(), "commit_user"); + read.with_limit(0); + let rows = read.read().await.unwrap(); + assert_eq!(rows.num_rows(), 0); + assert_eq!(rows.schema(), read.schema()); } diff --git a/crates/paimon/src/table/mod.rs b/crates/paimon/src/table/mod.rs index 94be4d07a..abc961122 100644 --- a/crates/paimon/src/table/mod.rs +++ b/crates/paimon/src/table/mod.rs @@ -130,9 +130,7 @@ mod postpone_retract; mod prepared_files; mod query_auth; mod system_table; -pub use system_table::{ - SystemReadBuilder, SystemSplit, SystemTable, SystemTablePlan, SystemTableRead, SystemTableScan, -}; +pub use system_table::SystemRead; mod read_builder; mod read_limit; @@ -549,9 +547,9 @@ impl Table { && options.partitioned_table_in_metastore() } - /// Resolve a read-only metadata table while retaining this table's environment. - pub fn system_table(&self, name: &str) -> Result { - SystemTable::new(self.clone(), name) + /// Read a small system table using this table's resolved environment. + pub fn new_system_read(&self, name: &str) -> Result { + SystemRead::new(self.clone(), name) } /// Create a read builder for scan/read. diff --git a/crates/paimon/src/table/system_table/mod.rs b/crates/paimon/src/table/system_table/mod.rs index 798198d56..b7893dbe6 100644 --- a/crates/paimon/src/table/system_table/mod.rs +++ b/crates/paimon/src/table/system_table/mod.rs @@ -18,7 +18,7 @@ //! Read-only metadata tables shared by Native bindings and query engines. use crate::spec::{CoreOptions, DataField, Predicate, RowType}; -use crate::table::{ArrowRecordBatchStream, Table}; +use crate::table::Table; use crate::{Error, Result}; use arrow_array::{ArrayRef, RecordBatch}; use arrow_schema::{Schema, SchemaRef}; @@ -50,264 +50,24 @@ const NAMES: &[&str] = &[ "tags", ]; -/// A read-only system-table view retaining the resolved base table environment. +/// Reads a small metadata table in one batch using the resolved table environment. #[derive(Debug, Clone)] -pub struct SystemTable { +pub struct SystemRead { table: Table, name: &'static str, fields: Vec, + projection: Option>, + predicate: Option, + limit: Option, } -impl SystemTable { +impl SystemRead { pub(crate) fn new(table: Table, name: &str) -> Result { let options = CoreOptions::new(table.schema().options()); if options.table_type()?.requires_table_engine() { return Err(invalid("System tables require a Paimon table")); } options.ensure_read_authorized()?; - let read = SystemTableRead::new(name)?; - let name = read.name; - let fields = read.fields; - Ok(Self { - table, - name, - fields, - }) - } - - /// Registered system-table suffix. - pub fn name(&self) -> &str { - self.name - } - - /// Declared Arrow output schema. - pub fn schema(&self) -> SchemaRef { - schema_for_name(self.name) - } - - /// Logical fields, preserving Java system-table field IDs. - pub fn row_type(&self) -> RowType { - RowType::with_nullable(false, self.fields.clone()) - } - - /// Configure a metadata scan and read using the ordinary read-builder shape. - pub fn new_read_builder(&self) -> SystemReadBuilder { - SystemReadBuilder { - table: self.clone(), - read: SystemTableRead::new(self.name).expect("validated system-table name"), - } - } - - async fn collect( - &self, - projection: Option<&[usize]>, - predicate: Option<&Predicate>, - ) -> Result { - match self.name { - "branches" => branches::read(&self.table, projection, predicate).await, - "buckets" => buckets::read(&self.table, projection, predicate).await, - "consumers" => consumers::read(&self.table, projection, predicate).await, - "files" | "file_key_ranges" => { - files::read( - &self.table, - self.name == "file_key_ranges", - projection, - predicate, - ) - .await - } - "manifests" => manifests::read(&self.table, projection, predicate).await, - "options" => options::read(&self.table, projection, predicate).await, - "schemas" => schemas::read(&self.table, projection, predicate).await, - "snapshots" => snapshots::read(&self.table, projection, predicate).await, - "table_indexes" => table_indexes::read(&self.table, projection, predicate).await, - "tags" => tags::read(&self.table, projection, predicate).await, - _ => unreachable!("validated system-table name"), - } - } -} - -/// Read configuration for a system table. Predicates always use the full schema. -#[derive(Debug, Clone)] -pub struct SystemReadBuilder { - table: SystemTable, - read: SystemTableRead, -} - -impl SystemReadBuilder { - /// Select output columns by name, preserving their requested order. - pub fn with_projection(&mut self, columns: &[&str]) -> Result<&mut Self> { - self.read.with_projection(columns)?; - Ok(self) - } - - /// Filter using a core Predicate, resolved against the full schema. - pub fn with_filter(&mut self, predicate: Predicate) -> Result<&mut Self> { - self.read.with_filter(predicate)?; - Ok(self) - } - - /// Limit visible rows across all splits, after filtering. - pub fn with_limit(&mut self, limit: usize) -> &mut Self { - self.read.with_limit(limit); - self - } - - /// Full logical schema used to resolve filter columns. - pub fn full_row_type(&self) -> RowType { - self.table.row_type() - } - - /// Logical output fields. - pub fn read_type(&self) -> Vec { - self.read.read_type() - } - - /// Arrow output schema, including an explicitly empty projection. - pub fn schema(&self) -> SchemaRef { - self.read.schema() - } - - /// Capture metadata for this table and the columns needed by the read. - pub fn new_scan(&self) -> SystemTableScan { - SystemTableScan { - builder: self.clone(), - } - } - - /// Build an in-memory reader without retaining the source table environment. - pub fn new_read(&self) -> SystemTableRead { - self.read.clone() - } -} - -/// A scan that captures one metadata view. Reads never reload its source files. -#[derive(Debug, Clone)] -pub struct SystemTableScan { - builder: SystemReadBuilder, -} - -impl SystemTableScan { - /// Capture the configured metadata, preserving an empty result's schema. - pub async fn plan(&self) -> Result { - let builder = &self.builder; - let mut needed = builder.read.projection.clone(); - if let Some(indices) = &mut needed { - let mut seen = std::collections::HashSet::new(); - indices.retain(|index| seen.insert(*index)); - if let Some(predicate) = &builder.read.predicate { - let mut leaves = Vec::new(); - crate::arrow::residual::collect_predicate_leaf_refs(predicate, &mut leaves); - for (column, _) in leaves { - let index = builder - .table - .schema() - .index_of(column) - .map_err(arrow_error)?; - if !indices.contains(&index) { - indices.push(index); - } - } - } - } - let batch = builder - .table - .collect(needed.as_deref(), builder.read.predicate.as_ref()) - .await?; - let batch = match needed { - Some(indices) => batch.project(&indices).map_err(arrow_error)?, - None => batch, - }; - Ok(SystemTablePlan { - splits: vec![SystemSplit { - name: builder.table.name.to_string(), - batch, - }], - }) - } -} - -/// In-memory metadata splits for one captured system-table scan. -#[derive(Debug, Clone)] -pub struct SystemTablePlan { - splits: Vec, -} - -impl SystemTablePlan { - /// Splits can be read without consulting the catalog or source metadata again. - pub fn splits(&self) -> &[SystemSplit] { - &self.splits - } -} - -/// Captured metadata rows. Arrow transport preserves the scan's selected fields. -#[derive(Debug, Clone)] -pub struct SystemSplit { - name: String, - batch: RecordBatch, -} - -impl SystemSplit { - /// Reconstruct a captured split received over an Arrow transport. - pub fn from_record_batch(name: &str, batch: RecordBatch) -> Result { - let name = NAMES - .iter() - .copied() - .find(|candidate| *candidate == name) - .ok_or_else(|| invalid("Unknown system-table split"))?; - let full = schema_for_name(name); - let mut seen = std::collections::HashSet::new(); - let mut canonical = Vec::new(); - for (index, field) in batch.schema().fields().iter().enumerate() { - let expected = full.field_with_name(field.name()).map_err(arrow_error)?; - if expected.data_type() != field.data_type() - || !seen.insert(field.name().clone()) - || !expected.is_nullable() && batch.column(index).null_count() != 0 - { - return Err(invalid("Invalid system-table split schema or nullability")); - } - canonical.push(expected.clone()); - } - let options = arrow_array::RecordBatchOptions::new().with_row_count(Some(batch.num_rows())); - let batch = RecordBatch::try_new_with_options( - Arc::new(Schema::new(canonical)), - batch.columns().to_vec(), - &options, - ) - .map_err(arrow_error)?; - Ok(Self { - name: name.to_string(), - batch, - }) - } - - /// System-table suffix identifying this split's schema. - pub fn name(&self) -> &str { - &self.name - } - /// Rows captured at scan time. - pub fn row_count(&self) -> usize { - self.batch.num_rows() - } - /// Captured batch for Arrow transport and serialization. - pub fn record_batch(&self) -> &RecordBatch { - &self.batch - } -} - -/// Exact metadata reader using the core Arrow predicate evaluator. -#[derive(Debug, Clone)] -pub struct SystemTableRead { - name: &'static str, - fields: Vec, - projection: Option>, - predicate: Option, - limit: Option, -} - -impl SystemTableRead { - /// Create a reader for captured metadata, with no catalog or file IO access. - pub fn new(name: &str) -> Result { let name = NAMES .iter() .copied() @@ -335,6 +95,7 @@ impl SystemTableRead { }) .collect::>()?; Ok(Self { + table, name, fields, projection: None, @@ -343,36 +104,12 @@ impl SystemTableRead { }) } - /// Full logical schema used to resolve filter columns. + /// Full logical schema used to resolve predicates, including unprojected columns. pub fn full_row_type(&self) -> RowType { RowType::with_nullable(false, self.fields.clone()) } - /// Select output columns, preserving order and repeated columns. - pub fn with_projection(&mut self, columns: &[&str]) -> Result<&mut Self> { - let schema = schema_for_name(self.name); - self.projection = Some( - columns - .iter() - .map(|name| schema.index_of(name).map_err(arrow_error)) - .collect::>()?, - ); - Ok(self) - } - - /// Filter captured rows using the full system-table schema. - pub fn with_filter(&mut self, predicate: Predicate) -> Result<&mut Self> { - self.predicate = Some(normalize_predicate(predicate, &self.fields)?); - Ok(self) - } - - /// Limit visible rows across all splits, after filtering. - pub fn with_limit(&mut self, limit: usize) -> &mut Self { - self.limit = Some(limit); - self - } - - /// Logical output fields. + /// Logical output fields, preserving the requested order and repeated columns. pub fn read_type(&self) -> Vec { match &self.projection { Some(indices) => indices @@ -383,7 +120,7 @@ impl SystemTableRead { } } - /// Output schema available even when no batches are emitted. + /// Declared output schema, including an explicitly empty projection. pub fn schema(&self) -> SchemaRef { let schema = schema_for_name(self.name); match &self.projection { @@ -397,63 +134,104 @@ impl SystemTableRead { } } - /// Read captured splits, enforcing filter, projection and a global limit. - pub fn read(&self, splits: &[SystemSplit]) -> Result { - let builder = self.clone(); - let splits = splits.to_vec(); - for split in &splits { - if split.name != builder.name { - return Err(invalid( - "System-table split belongs to a different table type", - )); - } - let mut needed = builder.read_type(); - if let Some(predicate) = &builder.predicate { - let mut leaves = Vec::new(); - crate::arrow::residual::collect_predicate_leaf_refs(predicate, &mut leaves); - for (column, _) in leaves { - let field = builder - .fields - .iter() - .find(|field| field.name() == column) - .ok_or_else(|| invalid("Unknown predicate column"))?; - needed.push(field.clone()); - } - } - for field in needed { - split - .batch - .schema() - .index_of(field.name()) + /// Select output columns by name. An empty list selects zero columns. + pub fn with_projection(&mut self, columns: &[&str]) -> Result<&mut Self> { + let schema = schema_for_name(self.name); + self.projection = Some( + columns + .iter() + .map(|name| schema.index_of(name).map_err(arrow_error)) + .collect::>()?, + ); + Ok(self) + } + + /// Filter metadata using the full schema, before output projection and limit. + pub fn with_filter(&mut self, predicate: Predicate) -> Result<&mut Self> { + self.predicate = Some(normalize_predicate(predicate, &self.fields)?); + Ok(self) + } + + /// Limit returned rows after filtering. + pub fn with_limit(&mut self, limit: usize) -> &mut Self { + self.limit = Some(limit); + self + } + + /// Materialize metadata once per call. The returned batch is independent of + /// the catalog and file IO, and retains its declared schema even when empty. + pub async fn read(&self) -> Result { + if self.limit == Some(0) { + return Ok(RecordBatch::new_empty(self.schema())); + } + let mut needed = self.projection.clone(); + if let (Some(indices), Some(predicate)) = (&mut needed, &self.predicate) { + let mut leaves = Vec::new(); + crate::arrow::residual::collect_predicate_leaf_refs(predicate, &mut leaves); + for (column, _) in leaves { + let index = schema_for_name(self.name) + .index_of(column) .map_err(arrow_error)?; + if !indices.contains(&index) { + indices.push(index); + } } } - Ok(Box::pin(async_stream::try_stream! { - let mut remaining = builder.limit.unwrap_or(usize::MAX); - let output = builder.read_type(); - for split in splits { - if remaining == 0 { - break; - } - let mut batch = split.batch; - if let Some(predicate) = &builder.predicate { - let mask = crate::arrow::residual::evaluate_predicates_mask( - &batch, std::slice::from_ref(predicate), &builder.fields, &builder.fields, - )?.ok_or_else(|| invalid("System-table predicate could not be evaluated exactly"))?; - batch = arrow_select::filter::filter_record_batch(&batch, &mask) - .map_err(arrow_error)?; - } - let indices = output.iter() - .map(|field| batch.schema().index_of(field.name()).map_err(arrow_error)) - .collect::>>()?; - batch = batch.project(&indices).map_err(arrow_error)?; - if batch.num_rows() > remaining { - batch = batch.slice(0, remaining); - } - remaining -= batch.num_rows(); - yield batch; + let mut batch = match self.name { + "branches" => { + branches::read(&self.table, needed.as_deref(), self.predicate.as_ref()).await? } - })) + "buckets" => { + buckets::read(&self.table, needed.as_deref(), self.predicate.as_ref()).await? + } + "consumers" => { + consumers::read(&self.table, needed.as_deref(), self.predicate.as_ref()).await? + } + "files" | "file_key_ranges" => { + files::read( + &self.table, + self.name == "file_key_ranges", + needed.as_deref(), + self.predicate.as_ref(), + ) + .await? + } + "manifests" => { + manifests::read(&self.table, needed.as_deref(), self.predicate.as_ref()).await? + } + "options" => { + options::read(&self.table, needed.as_deref(), self.predicate.as_ref()).await? + } + "schemas" => { + schemas::read(&self.table, needed.as_deref(), self.predicate.as_ref()).await? + } + "snapshots" => { + snapshots::read(&self.table, needed.as_deref(), self.predicate.as_ref()).await? + } + "table_indexes" => { + table_indexes::read(&self.table, needed.as_deref(), self.predicate.as_ref()).await? + } + "tags" => tags::read(&self.table, needed.as_deref(), self.predicate.as_ref()).await?, + _ => unreachable!("validated system-table name"), + }; + if let Some(predicate) = &self.predicate { + let mask = crate::arrow::residual::evaluate_predicates_mask( + &batch, + std::slice::from_ref(predicate), + &self.fields, + &self.fields, + )? + .ok_or_else(|| invalid("System-table predicate could not be evaluated exactly"))?; + batch = + arrow_select::filter::filter_record_batch(&batch, &mask).map_err(arrow_error)?; + } + if let Some(indices) = &self.projection { + batch = batch.project(indices).map_err(arrow_error)?; + } + if let Some(limit) = self.limit { + batch = batch.slice(0, limit.min(batch.num_rows())); + } + Ok(batch) } } @@ -546,137 +324,104 @@ mod tests { use crate::spec::{ DataType, Datum, IntType, PredicateOperator, Schema as PaimonSchema, TableSchema, }; - use arrow_array::{Int64Array, StringArray}; - use arrow_schema::{DataType as ArrowType, Field}; - use futures::TryStreamExt; + use arrow_array::StringArray; - fn table() -> SystemTable { + async fn table() -> Table { let file_io = crate::io::FileIO::from_path("memory:///") .unwrap() .build() .unwrap(); let schema = PaimonSchema::builder() .column("id", DataType::Int(IntType::new())) + .option("a", "skip") + .option("b", "match") + .option("c", "match") .build() .unwrap(); + let schema = TableSchema::new(0, &schema); + file_io + .new_output("memory:///metadata/schema/schema-0") + .unwrap() + .write(serde_json::to_vec(&schema).unwrap().into()) + .await + .unwrap(); Table::from_resolved_schema( file_io, crate::catalog::Identifier::new("default", "metadata"), "memory:///metadata".to_string(), - TableSchema::new(0, &schema), + schema, "main", ) .unwrap() - .system_table("options") - .unwrap() } - fn split(keys: Vec<&str>, values: Vec<&str>) -> SystemSplit { - let batch = RecordBatch::try_new( - Arc::new(Schema::new(vec![ - Field::new("key", ArrowType::Utf8, true), - Field::new("value", ArrowType::Utf8, true), - ])), - vec![ - Arc::new(StringArray::from(keys)), - Arc::new(StringArray::from(values)), - ], - ) + #[tokio::test] + async fn filter_precedes_projection_and_limit() { + let mut read = table().await.new_system_read("options").unwrap(); + read.with_projection(&["key"]).unwrap(); + read.with_filter(Predicate::Leaf { + column: "value".into(), + index: 999, + data_type: DataType::Int(IntType::new()), + op: PredicateOperator::Eq, + literals: vec![Datum::String("match".into())], + }) .unwrap(); - SystemSplit::from_record_batch("options", batch).unwrap() + read.with_limit(1); + let batch = read.read().await.unwrap(); + assert_eq!(batch.schema(), read.schema()); + assert_eq!(read.read_type()[0].name(), "key"); + assert_eq!(batch.num_rows(), 1); + assert_eq!( + batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap() + .value(0), + "b" + ); + read.with_limit(0); + let batch = read.read().await.unwrap(); + assert_eq!(batch.schema(), read.schema()); + assert_eq!(batch.num_rows(), 0); } #[tokio::test] - async fn filter_projection_and_global_limit_use_captured_batches() { - let table = table(); - let splits = [ - split(vec!["a", "b"], vec!["skip", "first"]), - split(vec!["b", "b"], vec!["second", "third"]), - ]; - let mut builder = table.new_read_builder(); - builder.with_projection(&["value"]).unwrap(); - builder - .with_filter(Predicate::Leaf { - column: "key".into(), - index: 999, - data_type: DataType::VarChar(crate::spec::VarCharType::string_type()), - op: PredicateOperator::Eq, - literals: vec![Datum::String("b".into())], - }) - .unwrap(); - builder.with_limit(2); - let result: Vec<_> = builder - .new_read() - .read(&splits) - .unwrap() - .try_collect() - .await - .unwrap(); - let values: Vec<_> = result - .iter() - .flat_map(|batch| { - batch - .column(0) - .as_any() - .downcast_ref::() - .unwrap() - .iter() - .flatten() - }) - .collect(); - assert_eq!(values, ["first", "second"]); - assert_eq!(result[0].schema(), builder.schema()); - builder.with_limit(0); - assert!(builder - .new_read() - .read(&splits) - .unwrap() - .try_collect::>() - .await - .unwrap() - .is_empty()); + async fn empty_and_repeated_projections_preserve_rows() { + let mut read = table().await.new_system_read("options").unwrap(); + read.with_projection(&[]).unwrap(); + let batch = read.read().await.unwrap(); + assert_eq!(batch.num_rows(), 3); + assert_eq!(batch.num_columns(), 0); + assert!(read.read_type().is_empty()); + read.with_projection(&["value", "key", "value"]).unwrap(); + let batch = read.read().await.unwrap(); + assert_eq!(batch.num_rows(), 3); + assert_eq!( + batch + .schema() + .fields() + .iter() + .map(|field| field.name().as_str()) + .collect::>(), + ["value", "key", "value"] + ); + assert_eq!(batch.column(0), batch.column(2)); + assert!(read.with_projection(&["missing"]).is_err()); } #[tokio::test] - async fn empty_projection_preserves_row_count_and_transport() { - let mut builder = table().new_read_builder(); - builder.with_projection(&[]).unwrap(); - let batches: Vec<_> = builder - .new_read() - .read(&[split(vec!["a", "b"], vec!["1", "2"])]) - .unwrap() - .try_collect() - .await - .unwrap(); - assert_eq!(batches[0].num_columns(), 0); - assert_eq!(batches[0].num_rows(), 2); - let restored = SystemSplit::from_record_batch("options", batches[0].clone()).unwrap(); - assert_eq!(restored.row_count(), 2); - } - - #[test] - fn split_transport_checks_type_names_and_required_values() { - let schema = Arc::new(Schema::new(vec![Field::new("key", ArrowType::Utf8, true)])); - let batch = RecordBatch::try_new( - schema, - vec![Arc::new(StringArray::from(vec![Some("a"), None]))], - ) - .unwrap(); - assert!(SystemSplit::from_record_batch("options", batch).is_err()); - let schema = Arc::new(Schema::new(vec![Field::new( - "key", - ArrowType::Int64, - false, - )])); - let batch = - RecordBatch::try_new(schema, vec![Arc::new(Int64Array::from(vec![1]))]).unwrap(); - assert!(SystemSplit::from_record_batch("options", batch).is_err()); - let mut builder = table().new_read_builder(); - assert!(builder.with_projection(&["missing"]).is_err()); - let wrong = SystemSplit { - name: "snapshots".into(), - batch: split(vec!["a"], vec!["b"]).batch, - }; - assert!(builder.new_read().read(&[wrong]).is_err()); + async fn unknown_filter_column_is_rejected() { + let mut read = table().await.new_system_read("options").unwrap(); + assert!(read + .with_filter(Predicate::Leaf { + column: "missing".into(), + index: 0, + data_type: DataType::Int(IntType::new()), + op: PredicateOperator::IsNull, + literals: vec![], + }) + .is_err()); } }