From 7047e7d23123eea9213643f369e9d45325a67692 Mon Sep 17 00:00:00 2001 From: "cong.xie" Date: Mon, 17 Aug 2026 15:39:51 -0400 Subject: [PATCH] feat(search): pin Foyer and add split-range disk cache config Pin Foyer 0.22.3 so the searcher can grow a process-wide disk cache for exact split ranges, and add the disabled-by-default config surface for it. Co-authored-by: Cursor --- LICENSE-3rdparty.csv | 13 ++ config/quickwit.yaml | 16 ++ quickwit/Cargo.lock | 180 ++++++++++++++++ quickwit/Cargo.toml | 2 + quickwit/quickwit-config/src/lib.rs | 9 +- .../quickwit-config/src/node_config/mod.rs | 192 ++++++++++++++++++ .../src/node_config/serialize.rs | 1 + quickwit/quickwit-storage/Cargo.toml | 8 + 8 files changed, 417 insertions(+), 4 deletions(-) diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index b3c307aa803..76cd483b558 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -161,6 +161,7 @@ clap_builder,https://github.com/clap-rs/clap,MIT OR Apache-2.0,The clap_builder clap_lex,https://github.com/clap-rs/clap,MIT OR Apache-2.0,The clap_lex Authors cmac,https://github.com/RustCrypto/MACs,MIT OR Apache-2.0,RustCrypto Developers cmov,https://github.com/RustCrypto/utils,Apache-2.0 OR MIT,RustCrypto Developers +cmsketch,https://github.com/mrcroxx/cmsketch-rs,Apache-2.0,MrCroxx coarsetime,https://github.com/jedisct1/rust-coarsetime,BSD-2-Clause,Frank Denis cobs,https://github.com/jamesmunns/cobs.rs,MIT OR Apache-2.0,"Allen Welkie <>, James Munns " codespan-reporting,https://github.com/brendanzab/codespan,Apache-2.0,Brendan Zabarauskas @@ -186,6 +187,7 @@ constant_time_eq,https://github.com/cesarb/constant_time_eq,CC0-1.0 OR MIT-0 OR convert_case,https://github.com/rutrum/convert-case,MIT,rutrum core-foundation,https://github.com/servo/core-foundation-rs,MIT OR Apache-2.0,The Servo Project Developers core-foundation-sys,https://github.com/servo/core-foundation-rs,MIT OR Apache-2.0,The Servo Project Developers +core_affinity,https://github.com/Elzair/core_affinity_rs,MIT OR Apache-2.0,Philip Woods cpp_demangle,https://github.com/gimli-rs/cpp_demangle,MIT OR Apache-2.0,"Nick Fitzgerald , Jim Blandy , Kyle Huey " cpufeatures,https://github.com/RustCrypto/utils,MIT OR Apache-2.0,RustCrypto Developers crc,https://github.com/mrhooray/crc-rs,MIT OR Apache-2.0,"Rui Hu , Akhil Velagapudi <4@4khil.com>" @@ -306,6 +308,7 @@ event-listener-strategy,https://github.com/smol-rs/event-listener-strategy,Apach evmap,https://github.com/jonhoo/evmap,MIT OR Apache-2.0,Jon Gjengset fail,https://github.com/tikv/fail-rs,Apache-2.0,The TiKV Project Developers fancy-regex,https://github.com/fancy-regex/fancy-regex,MIT,"Raph Levien , Robin Stocker , Keith Hall " +fastant,https://github.com/fast/fastant,MIT,The fastant Authors fastdivide,https://github.com/fulmicoton/fastdivide,zlib-acknowledgement OR MIT,Paul Masurel fastrand,https://github.com/smol-rs/fastrand,Apache-2.0 OR MIT,Stjepan Glavina ff,https://github.com/zkcrypto/ff,MIT OR Apache-2.0,"Sean Bowe , Jack Grigg " @@ -323,6 +326,12 @@ foldhash,https://github.com/orlp/foldhash,Zlib,Orson Peters foreign-types-shared,https://github.com/sfackler/foreign-types,MIT OR Apache-2.0,Steven Fackler form_urlencoded,https://github.com/servo/rust-url,MIT OR Apache-2.0,The rust-url developers +foyer,https://github.com/foyer-rs/foyer,Apache-2.0,MrCroxx +foyer-common,https://github.com/foyer-rs/foyer,Apache-2.0,MrCroxx +foyer-intrusive-collections,https://github.com/foyer-rs/intrusive-rs,Apache-2.0 OR MIT,Amanieu d'Antras +foyer-memory,https://github.com/foyer-rs/foyer,Apache-2.0,MrCroxx +foyer-storage,https://github.com/foyer-rs/foyer,Apache-2.0,MrCroxx +foyer-tokio,https://github.com/foyer-rs/foyer,Apache-2.0,MrCroxx fraction,https://github.com/dnsl48/fraction,MIT OR Apache-2.0,dnsl48 fragile,https://github.com/mitsuhiko/fragile,Apache-2.0,Armin Ronacher fs4,https://github.com/al8n/fs4-rs,MIT OR Apache-2.0,"Dan Burkert , Al Liu " @@ -409,6 +418,7 @@ inout,https://github.com/RustCrypto/utils,MIT OR Apache-2.0,RustCrypto Developer instant,https://github.com/sebcrozet/instant,BSD-3-Clause,sebcrozet integer-encoding,https://github.com/dermesser/integer-encoding-rs,MIT,Lewin Bormann inventory,https://github.com/dtolnay/inventory,MIT OR Apache-2.0,David Tolnay +io-uring,https://github.com/tokio-rs/io-uring,MIT OR Apache-2.0,quininer ipcrypt-rs,https://github.com/jedisct1/rust-ipcrypt2,ISC,Frank Denis ipnet,https://github.com/krisprice/ipnet,MIT OR Apache-2.0,Kris Price ipnetwork,https://github.com/achanda/ipnetwork,MIT OR Apache-2.0,"Abhishek Chanda , Linus Färnstrand " @@ -473,6 +483,7 @@ mea,https://github.com/fast/mea,Apache-2.0,The mea Authors measure_time,https://github.com/PSeitz/rust_measure_time,MIT,Pascal Seitz memchr,https://github.com/BurntSushi/memchr,Unlicense OR MIT,"Andrew Gallant , bluss" memmap2,https://github.com/RazrFalcon/memmap2-rs,MIT OR Apache-2.0,"Dan Burkert , Yevhenii Reizner , The Contributors" +memoffset,https://github.com/Gilnaa/memoffset,MIT,Gilad Naaman metrics,https://github.com/metrics-rs/metrics,MIT,Toby Lawrence metrics-exporter-prometheus,https://github.com/metrics-rs/metrics,MIT AND Apache-2.0,Toby Lawrence metrics-opentelemetry,https://github.com/DoumanAsh/metrics-opentelemetry,BSL-1.0,The metrics-opentelemetry Authors @@ -485,6 +496,7 @@ minimal-lexical,https://github.com/Alexhuszagh/minimal-lexical,MIT OR Apache-2.0 miniserde,https://github.com/dtolnay/miniserde,MIT OR Apache-2.0,David Tolnay miniz_oxide,https://github.com/Frommi/miniz_oxide/tree/master/miniz_oxide,MIT OR Zlib OR Apache-2.0,"Frommi , oyvindln , Rich Geldreich richgel99@gmail.com" mio,https://github.com/tokio-rs/mio,MIT,"Carl Lerche , Thomas de Zeeuw , Tokio Contributors " +mixtrics,https://github.com/foyer-rs/mixtrics,Apache-2.0,MrCroxx mockall,https://github.com/asomers/mockall,MIT OR Apache-2.0,Alan Somers mockall_derive,https://github.com/asomers/mockall,MIT OR Apache-2.0,Alan Somers moka,https://github.com/moka-rs/moka,(MIT OR Apache-2.0) AND Apache-2.0,The moka Authors @@ -776,6 +788,7 @@ simple_asn1,https://github.com/acw/simple_asn1,ISC,Adam Wick siphasher,https://github.com/jedisct1/rust-siphash,MIT OR Apache-2.0,Frank Denis sketches-ddsketch,https://github.com/mheffner/rust-sketches-ddsketch,Apache-2.0,Mike Heffner slab,https://github.com/tokio-rs/slab,MIT,Carl Lerche +small_ctor,https://github.com/mitsuhiko/small-ctor,Apache-2.0,Armin Ronacher smallvec,https://github.com/servo/rust-smallvec,MIT OR Apache-2.0,The Servo Project Developers snafu,https://github.com/shepmaster/snafu,MIT OR Apache-2.0,Jake Goulding snafu-derive,https://github.com/shepmaster/snafu,MIT OR Apache-2.0,Jake Goulding diff --git a/config/quickwit.yaml b/config/quickwit.yaml index 58bdd4942ff..997c6bf7df7 100644 --- a/config/quickwit.yaml +++ b/config/quickwit.yaml @@ -210,6 +210,22 @@ indexer: # max_num_bytes: 1G # max_num_splits: 10000 # num_concurrent_downloads: 1 +# # Process-wide Foyer disk cache for exact split footer and body ranges. +# # Omitted or null disables the cache. write_policy defaults to write-on-eviction. +# split_range_disk_cache: +# path: /var/cache/quickwit/split-range-v1 +# disk_capacity: 300G +# memory_capacity: 1G +# buffer_pool_size: 512M +# submit_queue_size_threshold: 1G +# memory_eviction_policy: s3-fifo +# write_policy: write-on-eviction +# compression: lz4 +# recover_mode: quiet +# block_size: 16M +# max_entry_size: 15M +# flushers: 4 +# reclaimers: 4 # -------------------------------- Jaeger settings -------------------------------- jaeger: diff --git a/quickwit/Cargo.lock b/quickwit/Cargo.lock index 630e108c2ea..2b324c975a4 100644 --- a/quickwit/Cargo.lock +++ b/quickwit/Cargo.lock @@ -2152,6 +2152,12 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" +[[package]] +name = "cmsketch" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7ee2cfacbd29706479902b06d75ad8f1362900836aa32799eabc7e004bfd854" + [[package]] name = "coarsetime" version = "0.1.37" @@ -2407,6 +2413,17 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "core_affinity" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a034b3a7b624016c6e13f5df875747cc25f884156aad2abd12b6c46797971342" +dependencies = [ + "libc", + "num_cpus", + "winapi", +] + [[package]] name = "cpp_demangle" version = "0.4.5" @@ -4113,6 +4130,16 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "fastant" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e825441bfb2d831c47c97d05821552db8832479f44c571b97fededbf0099c07" +dependencies = [ + "small_ctor", + "web-time", +] + [[package]] name = "fastdivide" version = "0.4.2" @@ -4283,6 +4310,117 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "foyer" +version = "0.22.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0abc0b87814989efa711f9becd9f26969820e2d3905db27d10969c4bd45890" +dependencies = [ + "anyhow", + "equivalent", + "foyer-common", + "foyer-memory", + "foyer-storage", + "foyer-tokio", + "futures-util", + "mea", + "mixtrics", + "pin-project", + "serde", + "tracing", +] + +[[package]] +name = "foyer-common" +version = "0.22.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3db80d5dece93adb7ad709c84578794724a9cba342a7e566c3551c7ec626789" +dependencies = [ + "anyhow", + "bytes", + "cfg-if", + "foyer-tokio", + "mixtrics", + "parking_lot", + "pin-project", + "twox-hash", +] + +[[package]] +name = "foyer-intrusive-collections" +version = "0.10.0-dev" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e4fee46bea69e0596130e3210e65d3424e0ac1e6df3bde6636304bdf1ca4a3b" +dependencies = [ + "memoffset", +] + +[[package]] +name = "foyer-memory" +version = "0.22.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db907f40a527ca2aa2f40a5f68b32ea58aa70f050cd233518e9ffd402cfba6ce" +dependencies = [ + "anyhow", + "bitflags 2.13.0", + "cmsketch", + "equivalent", + "foyer-common", + "foyer-intrusive-collections", + "foyer-tokio", + "futures-util", + "hashbrown 0.16.1", + "itertools 0.14.0", + "mea", + "mixtrics", + "parking_lot", + "paste", + "pin-project", + "serde", + "tracing", +] + +[[package]] +name = "foyer-storage" +version = "0.22.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1983f1db3d0710e9c9d5fc116d9202dccd41a2d1e032572224f1aff5520aa958" +dependencies = [ + "allocator-api2", + "anyhow", + "bytes", + "core_affinity", + "equivalent", + "fastant", + "foyer-common", + "foyer-memory", + "foyer-tokio", + "fs4", + "futures-core", + "futures-util", + "hashbrown 0.16.1", + "io-uring", + "itertools 0.14.0", + "libc", + "lz4", + "mea", + "parking_lot", + "pin-project", + "rand 0.9.4", + "tracing", + "twox-hash", + "zstd", +] + +[[package]] +name = "foyer-tokio" +version = "0.22.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6577b05a7ffad0db555aedf00bfe52af818220fc4c1c3a7a12520896fc38627" +dependencies = [ + "tokio", +] + [[package]] name = "fraction" version = "0.15.4" @@ -5399,6 +5537,17 @@ dependencies = [ "rustversion", ] +[[package]] +name = "io-uring" +version = "0.7.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d64d8ca234d152948ceaede1f419b6a83983a5ecccaac05fb337a809c96d3aa6" +dependencies = [ + "bitflags 2.13.0", + "cfg-if", + "libc", +] + [[package]] name = "ipcrypt-rs" version = "0.9.4" @@ -6116,6 +6265,15 @@ dependencies = [ "libc", ] +[[package]] +name = "memoffset" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" +dependencies = [ + "autocfg", +] + [[package]] name = "metrics" version = "0.24.6" @@ -6260,6 +6418,16 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "mixtrics" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c46b5adfb7a3ae4996d327a5bdc90e78fec025806dd312bdbe6f07a755e0ec9" +dependencies = [ + "itertools 0.15.0", + "parking_lot", +] + [[package]] name = "mockall" version = "0.14.0" @@ -9410,13 +9578,16 @@ dependencies = [ "base64 0.22.1", "bytes", "bytesize", + "foyer", "futures", "http 1.4.2", "http-body-util", "hyper 1.10.1", "lru 0.18.0", "md5", + "metrics", "mini-moka", + "mixtrics", "mockall", "opendal", "opendal-http-transport-reqwest", @@ -11165,6 +11336,12 @@ version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" +[[package]] +name = "small_ctor" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88414a5ca1f85d82cc34471e975f0f74f6aa54c40f062efa42c0080e7f763f81" + [[package]] name = "smallvec" version = "1.15.2" @@ -12638,6 +12815,9 @@ name = "twox-hash" version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" +dependencies = [ + "rand 0.9.4", +] [[package]] name = "typeid" diff --git a/quickwit/Cargo.toml b/quickwit/Cargo.toml index 8904ed1dd3c..dad8da2d47e 100644 --- a/quickwit/Cargo.toml +++ b/quickwit/Cargo.toml @@ -136,6 +136,7 @@ fail = "0.5" flate2 = "1.1" flume = "0.12" fnv = "1" +foyer = { version = "=0.22.3", default-features = false, features = ["runtime-tokio"] } futures = "0.3" futures-util = { version = "0.3", default-features = false } glob = "0.3" @@ -180,6 +181,7 @@ metrics-exporter-prometheus = { version = "0.18", default-features = false } metrics-util = "0.20" mime_guess = "2.0" mini-moka = "0.10" +mixtrics = "=0.2.5" mockall = "0.14" mrecordlog = { git = "https://github.com/quickwit-oss/mrecordlog", rev = "3b3562ef" } new_string_template = "1.5" diff --git a/quickwit/quickwit-config/src/lib.rs b/quickwit/quickwit-config/src/lib.rs index 904e65f8180..2a27bddcecc 100644 --- a/quickwit/quickwit-config/src/lib.rs +++ b/quickwit/quickwit-config/src/lib.rs @@ -77,10 +77,11 @@ pub use crate::metastore_config::{ MetastoreBackend, MetastoreConfig, MetastoreConfigs, PostgresMetastoreConfig, }; pub use crate::node_config::{ - CacheConfig, CachePolicy, CompactorConfig, DEFAULT_QW_CONFIG_PATH, GrpcConfig, HealthConfig, - IndexerConfig, IngestApiConfig, JaegerConfig, KeepAliveConfig, LambdaConfig, - LambdaDeployConfig, MAX_GOSSIP_PROTOCOL_VERSION, NodeConfig, RestConfig, SearcherConfig, - SplitCacheLimits, StorageTimeoutPolicy, TlsConfig, + CacheConfig, CachePolicy, CompactorConfig, DEFAULT_QW_CONFIG_PATH, DiskCompression, GrpcConfig, + HealthConfig, IndexerConfig, IngestApiConfig, JaegerConfig, KeepAliveConfig, LambdaConfig, + LambdaDeployConfig, MAX_GOSSIP_PROTOCOL_VERSION, NodeConfig, RecoverMode, RestConfig, + SearcherConfig, SplitCacheLimits, SplitRangeCacheWritePolicy, SplitRangeDiskCacheConfig, + StorageTimeoutPolicy, TlsConfig, }; pub use crate::serde_utils::HumanDuration; use crate::source_config::serialize::{SourceConfigV0_7, SourceConfigV0_8, VersionedSourceConfig}; diff --git a/quickwit/quickwit-config/src/node_config/mod.rs b/quickwit/quickwit-config/src/node_config/mod.rs index 1f8dd2b81e4..0793bf442fb 100644 --- a/quickwit/quickwit-config/src/node_config/mod.rs +++ b/quickwit/quickwit-config/src/node_config/mod.rs @@ -403,6 +403,103 @@ impl SplitCacheLimits { } } +/// Admission policy for the split-range Foyer cache. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum SplitRangeCacheWritePolicy { + /// Keep admitted values in memory and persist them when memory eviction occurs. + #[default] + WriteOnEviction, + /// Persist admitted values as soon as they are inserted. + WriteOnInsertion, +} + +/// On-disk compression for the split-range Foyer cache. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum DiskCompression { + Lz4, +} + +/// Recovery mode for the split-range Foyer cache. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum RecoverMode { + Quiet, +} + +/// Disabled-by-default searcher disk cache for exact split byte ranges. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SplitRangeDiskCacheConfig { + pub path: PathBuf, + #[serde(with = "crate::serde_utils::bytesize_serde")] + pub disk_capacity: ByteSize, + #[serde(with = "crate::serde_utils::bytesize_serde")] + pub memory_capacity: ByteSize, + #[serde(with = "crate::serde_utils::bytesize_serde")] + pub buffer_pool_size: ByteSize, + #[serde(with = "crate::serde_utils::bytesize_serde")] + pub submit_queue_size_threshold: ByteSize, + pub memory_eviction_policy: CachePolicy, + #[serde(default)] + pub write_policy: SplitRangeCacheWritePolicy, + pub compression: DiskCompression, + pub recover_mode: RecoverMode, + #[serde(with = "crate::serde_utils::bytesize_serde")] + pub block_size: ByteSize, + #[serde(with = "crate::serde_utils::bytesize_serde")] + pub max_entry_size: ByteSize, + pub flushers: usize, + pub reclaimers: usize, +} + +impl SplitRangeDiskCacheConfig { + fn validate(&self) -> anyhow::Result<()> { + if self.path.as_os_str().is_empty() { + bail!("split_range_disk_cache.path must not be empty"); + } + if self.disk_capacity.as_u64() == 0 || self.memory_capacity.as_u64() == 0 { + bail!("split range disk and memory capacities must be positive"); + } + if self.buffer_pool_size.as_u64() == 0 || self.submit_queue_size_threshold.as_u64() == 0 { + bail!("split range buffer pool and submit queue sizes must be positive"); + } + if self.block_size.as_u64() == 0 || self.max_entry_size.as_u64() == 0 { + bail!("split range block and entry sizes must be positive"); + } + if self.max_entry_size >= self.block_size { + bail!("split_range_disk_cache.max_entry_size must be smaller than block_size"); + } + if self.memory_eviction_policy != CachePolicy::S3Fifo { + bail!("split_range_disk_cache.memory_eviction_policy must be s3-fifo in phase 1"); + } + if self.flushers == 0 || self.reclaimers == 0 { + bail!("split range disk cache flushers and reclaimers must be positive"); + } + Ok(()) + } + + #[cfg(any(test, feature = "testsuite"))] + pub fn for_test() -> Self { + Self { + path: PathBuf::from("/tmp/quickwit-split-range-disk-cache"), + disk_capacity: ByteSize::gb(300), + memory_capacity: ByteSize::gb(1), + buffer_pool_size: ByteSize::mb(512), + submit_queue_size_threshold: ByteSize::gb(1), + memory_eviction_policy: CachePolicy::S3Fifo, + write_policy: SplitRangeCacheWritePolicy::WriteOnEviction, + compression: DiskCompression::Lz4, + recover_mode: RecoverMode::Quiet, + block_size: ByteSize::mb(16), + max_entry_size: ByteSize::mb(15), + flushers: 4, + reclaimers: 4, + } + } +} + #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] #[serde(deny_unknown_fields, default)] pub struct SearcherConfig { @@ -437,6 +534,10 @@ pub struct SearcherConfig { // TODO document and fix if necessary. #[serde(default, skip_serializing_if = "Option::is_none")] pub split_cache: Option, + /// Process-wide Foyer disk cache for exact split footer and body ranges. + /// Omitted or `null` disables the cache. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub split_range_disk_cache: Option, #[serde(default = "SearcherConfig::default_request_timeout_secs")] request_timeout_secs: NonZeroU64, #[serde(default = "SearcherConfig::default_request_timeout_secs")] @@ -668,6 +769,7 @@ impl Default for SearcherConfig { aggregation_memory_limit: ByteSize::mb(500), aggregation_bucket_limit: 65000, split_cache: None, + split_range_disk_cache: None, request_timeout_secs: Self::default_request_timeout_secs(), leaf_request_timeout_secs: Self::default_request_timeout_secs(), storage_timeout_policy: None, @@ -712,6 +814,9 @@ impl SearcherConfig { ); } } + if let Some(split_range_disk_cache) = &self.split_range_disk_cache { + split_range_disk_cache.validate()?; + } Ok(()) } } @@ -1320,4 +1425,91 @@ mod tests { }; assert!(grpc_config.validate().is_err()); } + + #[test] + fn test_split_range_disk_cache_config_is_disabled_by_default() { + assert!(SearcherConfig::default().split_range_disk_cache.is_none()); + } + + #[test] + fn test_split_range_disk_cache_config_round_trip() { + let yaml = r#" +split_range_disk_cache: + path: /var/cache/quickwit/split-range-v1 + disk_capacity: 300G + memory_capacity: 1G + buffer_pool_size: 512M + submit_queue_size_threshold: 1G + memory_eviction_policy: s3-fifo + compression: lz4 + recover_mode: quiet + block_size: 16M + max_entry_size: 15M + flushers: 4 + reclaimers: 4 +"#; + let config: SearcherConfig = serde_yaml::from_str(yaml).unwrap(); + let disk_cache = config.split_range_disk_cache.as_ref().unwrap(); + assert_eq!( + disk_cache.path, + PathBuf::from("/var/cache/quickwit/split-range-v1") + ); + assert_eq!(disk_cache.memory_eviction_policy, CachePolicy::S3Fifo); + assert_eq!( + disk_cache.write_policy, + SplitRangeCacheWritePolicy::WriteOnEviction + ); + assert_eq!(disk_cache.block_size, ByteSize::mb(16)); + assert_eq!(disk_cache.max_entry_size, ByteSize::mb(15)); + // Round-trip the nested cache config. SearcherConfig's other ByteSize + // fields serialize as display strings and do not round-trip exactly. + assert_eq!( + serde_yaml::from_str::( + &serde_yaml::to_string(disk_cache).unwrap() + ) + .unwrap(), + *disk_cache + ); + } + + #[test] + fn test_split_range_disk_cache_accepts_write_on_insertion() { + let yaml = r#" +split_range_disk_cache: + path: /var/cache/quickwit/split-range-v1 + disk_capacity: 300G + memory_capacity: 1G + buffer_pool_size: 512M + submit_queue_size_threshold: 1G + memory_eviction_policy: s3-fifo + write_policy: write-on-insertion + compression: lz4 + recover_mode: quiet + block_size: 16M + max_entry_size: 15M + flushers: 4 + reclaimers: 4 +"#; + let config: SearcherConfig = serde_yaml::from_str(yaml).unwrap(); + assert_eq!( + config.split_range_disk_cache.unwrap().write_policy, + SplitRangeCacheWritePolicy::WriteOnInsertion + ); + } + + #[test] + fn test_split_range_disk_cache_rejects_invalid_sizes() { + let mut disk_cache = SplitRangeDiskCacheConfig::for_test(); + disk_cache.max_entry_size = ByteSize::mb(16); + let config = SearcherConfig { + split_range_disk_cache: Some(disk_cache), + ..Default::default() + }; + let error = config.validate().unwrap_err(); + assert!( + error + .to_string() + .contains("max_entry_size must be smaller than block_size") + ); + } } diff --git a/quickwit/quickwit-config/src/node_config/serialize.rs b/quickwit/quickwit-config/src/node_config/serialize.rs index 48fa2854509..14542a0c174 100644 --- a/quickwit/quickwit-config/src/node_config/serialize.rs +++ b/quickwit/quickwit-config/src/node_config/serialize.rs @@ -891,6 +891,7 @@ mod tests { max_splits_per_search: None, _max_num_concurrent_split_streams: Some(serde::de::IgnoredAny), split_cache: None, + split_range_disk_cache: None, request_timeout_secs: NonZeroU64::new(30).unwrap(), leaf_request_timeout_secs: NonZeroU64::new(30).unwrap(), storage_timeout_policy: Some(crate::StorageTimeoutPolicy { diff --git a/quickwit/quickwit-storage/Cargo.toml b/quickwit/quickwit-storage/Cargo.toml index 620e44b9ff7..fa7515dbfb2 100644 --- a/quickwit/quickwit-storage/Cargo.toml +++ b/quickwit/quickwit-storage/Cargo.toml @@ -17,12 +17,15 @@ async-trait = { workspace = true } base64 = { workspace = true } bytes = { workspace = true } bytesize = { workspace = true } +foyer = { workspace = true } futures = { workspace = true } http-body-util = { workspace = true} hyper = { workspace = true } lru = { workspace = true } md5 = { workspace = true } +metrics = { workspace = true } mini-moka = { workspace = true } +mixtrics = { workspace = true } mockall = { workspace = true, optional = true } pin-project = { workspace = true } quick_cache = { workspace = true } @@ -102,3 +105,8 @@ integration-testsuite = [ "dep:reqwest", ] testsuite = ["mockall"] + +[package.metadata.cargo-machete] +# Declared here so the Foyer pin lands with licenses. Follow-up split-range +# cache PRs use these crates; remove the ignore when they do. +ignored = ["foyer", "metrics", "mixtrics"]