From bd550994924844ae8b5681ba15ad70142020ce1b Mon Sep 17 00:00:00 2001 From: Daniel King Date: Fri, 7 Aug 2026 16:16:19 -0400 Subject: [PATCH 1/7] Add local read-at support for file opens --- vortex-file/src/open.rs | 99 +++++++++++++++++++++++++++++- vortex-file/src/segments/source.rs | 93 +++++++++++++++++++++++++++- vortex-io/src/read_at.rs | 32 ++++++++++ vortex-io/src/runtime/handle.rs | 27 ++++++++ vortex-io/src/runtime/mod.rs | 12 ++++ vortex-io/src/runtime/single.rs | 42 +++++++++++++ 6 files changed, 303 insertions(+), 2 deletions(-) diff --git a/vortex-file/src/open.rs b/vortex-file/src/open.rs index 1bd42c7be41..63ca5163bfa 100644 --- a/vortex-file/src/open.rs +++ b/vortex-file/src/open.rs @@ -13,8 +13,8 @@ use vortex_buffer::ByteBuffer; use vortex_error::VortexError; use vortex_error::VortexExpect; use vortex_error::VortexResult; -use vortex_io::VortexReadAt; use vortex_io::session::RuntimeSessionExt; +use vortex_io::{VortexLocalReadAt, VortexReadAt}; use vortex_layout::segments::InstrumentedSegmentCache; use vortex_layout::segments::NoOpSegmentCache; use vortex_layout::segments::SegmentCache; @@ -283,6 +283,59 @@ impl VortexOpenOptions { }) } + /// Open a [`VortexFile`] using a reader whose I/O futures must stay local to one runtime + /// thread. + pub async fn open_read_local( + self, + reader: R, + ) -> VortexResult { + let segment_cache = self + .segment_cache + .clone() + .unwrap_or_else(|| Arc::new(NoOpSegmentCache)); + + let metrics_registry = self + .metrics_registry + .clone() + .unwrap_or_else(|| Arc::new(DefaultMetricsRegistry::default())); + + let footer = if let Some(footer) = self.footer { + footer + } else { + self.read_footer_local(&reader).await? + }; + + let segment_cache = Arc::new(InstrumentedSegmentCache::new( + InitialReadSegmentCache { + initial: self.initial_read_segments, + fallback: segment_cache, + }, + metrics_registry.as_ref(), + self.labels.clone(), + )); + + let metrics = RequestMetrics::new(metrics_registry.as_ref(), self.labels); + + let segment_source = Arc::new(SharedSegmentSource::new(FileSegmentSource::open_local( + Arc::clone(footer.segment_map()), + reader, + self.session.handle(), + metrics, + ))); + + let segment_source = Arc::new(SegmentCacheSourceAdapter::new( + segment_cache, + segment_source, + )); + + let file = VortexFile::new(footer, segment_source, self.session.clone()); + Ok(if self.cache_layout_reader { + file.with_caching() + } else { + file + }) + } + async fn read_footer(&self, read: &dyn VortexReadAt) -> VortexResult