feat(search): add disk cache metrics and disable fast-field RAM cache conditionally

config/quickwit.yaml

@@ -199,6 +199,8 @@ indexer:
 #   # service. Searchers require at least one `metastore_read_replica` node at
 #   # startup and do not fall back to the primary metastore.
 #   use_metastore_read_replica: false
+#   # Fast field RAM cache. Default 1G. Omitted while split_range_disk_cache
+#   # is set disables this cache; set a capacity explicitly to keep both.
 #   fast_field_cache_capacity: 1G
 #   split_footer_cache_capacity: 500M
 #   partial_request_cache_capacity: 64M
@@ -213,19 +215,18 @@ indexer:
 #   # Process-wide Foyer disk cache for exact split footer and body ranges.
 #   # Omitted or null disables the cache. write_policy defaults to write-on-eviction.
 #   split_range_disk_cache:
-#     path: /var/cache/quickwit/split-range-v1
-#     disk_capacity: 300G
-#     memory_capacity: 1G
-#     buffer_pool_size: 512M
-#     submit_queue_size_threshold: 1G
+#     path: /quickwit/qwdata/split-range-v1
+#     disk_capacity: 1500G
+#     memory_capacity: 15G
 #     memory_eviction_policy: s3-fifo
-#     write_policy: write-on-eviction
 #     compression: lz4
 #     recover_mode: quiet
-#     block_size: 16M
-#     max_entry_size: 15M
-#     flushers: 4
-#     reclaimers: 4
+#     block_size: 64M
+#     max_entry_size: 60M
+#     flushers: 24
+#     buffer_pool_size: 2G
+#     submit_queue_size_threshold: 3G
+#     reclaimers: 8
 # -------------------------------- Jaeger settings --------------------------------
 
 jaeger:

docs/configuration/node-config.md

@@ -305,11 +305,12 @@ This section contains the configuration options for a Searcher.
 | --- | --- | --- |
 | `aggregation_memory_limit` | Controls the maximum amount of memory that can be used for aggregations before aborting. This limit is per searcher node. A node may run concurrent queries, which share the limit. The first query that will hit the limit will be aborted and frees its memory. It is used to prevent excessive memory usage during the aggregation phase, which can lead to performance degradation or crashes. | `500M`|
 | `aggregation_bucket_limit` | Determines the maximum number of buckets returned to the client. | `65000` |
-| `fast_field_cache_capacity` | Fast field in memory cache capacity on a Searcher. If your filter by dates, run aggregations, range queries, or even for tracing, it might worth increasing this parameter. The [metrics](../reference/metrics.md) starting by `quickwit_cache_fastfields_cache` can help you make an informed choice when setting this value. | `1G` |
+| `fast_field_cache_capacity` | Fast field in memory cache capacity on a Searcher. If your filter by dates, run aggregations, range queries, or even for tracing, it might worth increasing this parameter. The [metrics](../reference/metrics.md) starting by `quickwit_cache_fastfields_cache` can help you make an informed choice when setting this value. Default is `1G` when `split_range_disk_cache` is unset. If `split_range_disk_cache` is set and this key is omitted, the RAM cache is disabled. Set a capacity explicitly to keep both. | `1G` |
 | `split_footer_cache_capacity` | Split footer in memory cache (it is essentially the hotcache) capacity on a Searcher.| `500M` |
 | `partial_request_cache_capacity` | Partial request in memory cache capacity on a Searcher. Cache intermediate state for a request, possibly making subsequent requests faster. It can be disabled by setting the size to `0`. | `64M` |
 | `max_num_concurrent_split_searches` | Maximum number of concurrent split search requests running on a Searcher. | `100` |
 | `split_cache` | Searcher split cache configuration options defined in the section below. Cache disabled if unspecified. | |
+| `split_range_disk_cache` | Process-wide on-disk cache for exact split footer and body byte ranges. Configuration options are defined in the section below. Cache disabled if unspecified. | |
 | `request_timeout_secs` | The time before a search request is cancelled. This should match the timeout of the stack calling into quickwit if there is one set.  | `30` |
 | `use_metastore_read_replica` | If true, routes read-only metastore requests from searchers, including DataFusion when enabled, to nodes running the `metastore_read_replica` service. Searchers require at least one `metastore_read_replica` node at startup and do not fall back to the primary metastore. | `false` |
 
@@ -323,6 +324,25 @@ This section contains the configuration options for the on-disk searcher split c
 | `max_num_splits` | Maximum number of splits allowed in the split cache.   | `10000` |
 | `num_concurrent_downloads` | Maximum number of concurrent download of splits. | `1` |
 
+### Searcher split range disk cache configuration
+
+This section contains the configuration options for the process-wide on-disk cache of exact split footer and body ranges. The cache is disabled when this section is omitted or set to `null`. If it is set and `fast_field_cache_capacity` is omitted, the long-lived fast field RAM cache is disabled; set a capacity explicitly to keep both.
+
+| Property | Description | Default value |
+| --- | --- | --- |
+| `path` | Directory used to store cache files. Created if missing. Must already sit on a usable filesystem. | |
+| `disk_capacity` | Maximum on-disk size of the cache. | |
+| `memory_capacity` | In-memory tier size in front of the disk cache. | |
+| `buffer_pool_size` | Size of the disk write buffer pool. | |
+| `submit_queue_size_threshold` | Maximum amount of data waiting to be flushed to disk. | |
+| `memory_eviction_policy` | Eviction policy for the memory tier. Currently only `s3-fifo` is accepted. | |
+| `write_policy` | When admitted values are written to disk: `write-on-eviction` or `write-on-insertion`. | `write-on-eviction` |
+| `compression` | On-disk compression. Currently only `lz4` is accepted. | |
+| `recover_mode` | How existing cache files are recovered on startup. Currently only `quiet` is accepted. | |
+| `block_size` | Disk block size. Must be larger than `max_entry_size`. | |
+| `max_entry_size` | Maximum uncompressed payload stored as one disk entry. Larger ranges stay in memory only. | |
+| `flushers` | Number of flush worker threads. Must be positive. | |
+| `reclaimers` | Number of reclaim worker threads. Must be positive. | |
 
 Example:
 
@@ -336,6 +356,19 @@ searcher:
     max_num_bytes: 1G
     max_num_splits: 10000
     num_concurrent_downloads: 1
+  split_range_disk_cache:
+    path: /quickwit/qwdata/split-range-v1
+    disk_capacity: 1500G
+    memory_capacity: 15G
+    memory_eviction_policy: s3-fifo
+    compression: lz4
+    recover_mode: quiet
+    block_size: 64M
+    max_entry_size: 60M
+    flushers: 24
+    buffer_pool_size: 2G
+    submit_queue_size_threshold: 3G
+    reclaimers: 8
 ```
 
 ## Jaeger configuration

docs/reference/metrics.md

@@ -91,3 +91,9 @@ PostgreSQL-backed metastores also expose connection pool gauges:
 | `quickwit_storage` | `object_storage_puts_total` | Number of objects uploaded. May differ from object_storage_requests_parts due to multipart upload | `counter` |
 | `quickwit_storage` | `object_storage_puts_parts` | Number of object parts uploaded | `counter` |
 | `quickwit_storage` | `object_storage_download_num_bytes` | Amount of data downloaded from an object storage | `counter` |
+| `quickwit_storage` | `split_range_disk_cache_requests_total` | Split range disk cache requests by `result` (`memory`, `disk`, `miss`, or `error`) | `counter` |
+| `quickwit_storage` | `split_range_disk_cache_requested_bytes_total` | Requested bytes by `result` | `counter` |
+| `quickwit_storage` | `split_range_disk_cache_admission_bypasses_total` | Entries kept memory-only, labeled by `reason` (`max_entry_size` or `encoded_too_large`) | `counter` |
+| `quickwit_storage` | `split_range_disk_cache_fail_open_total` | Foyer failures served from object storage | `counter` |
+
+Foyer also exports its own hybrid-cache metrics on `/metrics` when `split_range_disk_cache` is enabled, including `foyer_memory_op_total`, `foyer_memory_usage`, `foyer_memory_entries`, `foyer_storage_op_total`, and `foyer_storage_disk_io_bytes_total`, labeled by cache `name` (`split-range-v1`). Object-storage GET counters cover actual remote fetches on a cache miss.

quickwit/quickwit-config/src/node_config/mod.rs

@@ -506,11 +506,12 @@ pub struct SearcherConfig {
     pub aggregation_memory_limit: ByteSize,
     pub aggregation_bucket_limit: u32,
 
+    /// Long-lived `.fast` RAM cache. Omitted is `None` and resolved in
+    /// [`Self::resolved_fast_field_cache`].
     #[serde(alias = "fast_field_cache_capacity")]
-    #[serde(
-        deserialize_with = "CacheConfig::deserialize_with_default::<_, {ByteSize::gb(1).as_u64()}>"
-    )]
-    pub fast_field_cache: CacheConfig,
+    #[serde(default, deserialize_with = "deserialize_optional_fast_field_cache")]
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub fast_field_cache: Option<CacheConfig>,
     #[serde(alias = "split_footer_cache_capacity")]
     #[serde(deserialize_with = "CacheConfig::deserialize_with_default::<_, \
                                 {ByteSize::mb(500).as_u64()}>")]
@@ -698,6 +699,13 @@ impl CacheConfig {
     }
 }
 
+fn deserialize_optional_fast_field_cache<'de, D>(
+    deserializer: D,
+) -> Result<Option<CacheConfig>, D::Error>
+where D: Deserializer<'de> {
+    CacheConfig::deserialize_with_default::<D, { ByteSize::gb(1).as_u64() }>(deserializer).map(Some)
+}
+
 impl From<ByteSize> for CacheConfig {
     fn from(capacity: ByteSize) -> Self {
         CacheConfig::default_with_capacity(capacity)
@@ -759,7 +767,7 @@ impl StorageTimeoutPolicy {
 impl Default for SearcherConfig {
     fn default() -> Self {
         SearcherConfig {
-            fast_field_cache: CacheConfig::default_with_capacity(ByteSize::gb(1)),
+            fast_field_cache: None,
             split_footer_cache: CacheConfig::default_with_capacity(ByteSize::mb(500)),
             partial_request_cache: CacheConfig::default_with_capacity(ByteSize::mb(64)),
             predicate_cache: CacheConfig::default_with_capacity(ByteSize::mb(256)),
@@ -793,6 +801,20 @@ impl SearcherConfig {
     fn default_request_timeout_secs() -> NonZeroU64 {
         NonZeroU64::new(30).unwrap()
     }
+
+    /// Long-lived `.fast` RAM cache after applying defaults.
+    ///
+    /// An explicit config is used as-is. If omitted, Foyer disables the cache
+    /// and otherwise it is 1 GiB.
+    pub fn resolved_fast_field_cache(&self) -> CacheConfig {
+        match &self.fast_field_cache {
+            Some(cache_config) => cache_config.clone(),
+            None => match &self.split_range_disk_cache {
+                Some(_) => CacheConfig::no_cache(),
+                None => CacheConfig::default_with_capacity(ByteSize::gb(1)),
+            },
+        }
+    }
     fn validate(&self) -> anyhow::Result<()> {
         if let Some(split_cache_limits) = self.split_cache {
             if self.max_num_concurrent_split_searches
@@ -1429,6 +1451,80 @@ mod tests {
     #[test]
     fn test_split_range_disk_cache_config_is_disabled_by_default() {
         assert!(SearcherConfig::default().split_range_disk_cache.is_none());
+        assert_eq!(
+            SearcherConfig::default().resolved_fast_field_cache(),
+            CacheConfig::default_with_capacity(ByteSize::gb(1))
+        );
+    }
+
+    #[test]
+    fn test_omitted_fast_field_cache_stays_1g_without_split_range_disk_cache() {
+        let config: SearcherConfig = serde_yaml::from_str("{}").unwrap();
+        assert!(config.split_range_disk_cache.is_none());
+        assert!(config.fast_field_cache.is_none());
+        assert_eq!(
+            config.resolved_fast_field_cache(),
+            CacheConfig::default_with_capacity(ByteSize::gb(1))
+        );
+    }
+
+    #[test]
+    fn test_omitted_fast_field_cache_is_disabled_when_split_range_disk_cache_is_set() {
+        let yaml = r#"
+split_range_disk_cache:
+  path: /var/cache/quickwit/split-range-v1
+  disk_capacity: 300G
+  memory_capacity: 1G
+  buffer_pool_size: 512M
+  submit_queue_size_threshold: 1G
+  memory_eviction_policy: s3-fifo
+  compression: lz4
+  recover_mode: quiet
+  block_size: 16M
+  max_entry_size: 15M
+  flushers: 4
+  reclaimers: 4
+"#;
+        let config: SearcherConfig = serde_yaml::from_str(yaml).unwrap();
+        assert!(config.split_range_disk_cache.is_some());
+        assert!(config.fast_field_cache.is_none());
+        assert_eq!(config.resolved_fast_field_cache(), CacheConfig::no_cache());
+    }
+
+    #[test]
+    fn test_explicit_fast_field_cache_is_kept_when_split_range_disk_cache_is_set() {
+        let yaml = r#"
+fast_field_cache_capacity: 1G
+split_range_disk_cache:
+  path: /var/cache/quickwit/split-range-v1
+  disk_capacity: 300G
+  memory_capacity: 1G
+  buffer_pool_size: 512M
+  submit_queue_size_threshold: 1G
+  memory_eviction_policy: s3-fifo
+  compression: lz4
+  recover_mode: quiet
+  block_size: 16M
+  max_entry_size: 15M
+  flushers: 4
+  reclaimers: 4
+"#;
+        let config: SearcherConfig = serde_yaml::from_str(yaml).unwrap();
+        assert!(config.split_range_disk_cache.is_some());
+        assert_eq!(
+            config.resolved_fast_field_cache(),
+            CacheConfig::default_with_capacity(ByteSize::gb(1))
+        );
+    }
+
+    #[test]
+    fn test_explicit_zero_fast_field_cache_disables_without_split_range_disk_cache() {
+        let config: SearcherConfig = serde_yaml::from_str("fast_field_cache_capacity: 0").unwrap();
+        assert!(config.split_range_disk_cache.is_none());
+        assert_eq!(
+            config.resolved_fast_field_cache().capacity(),
+            ByteSize::b(0)
+        );
     }
 
     #[test]

quickwit/quickwit-config/src/node_config/serialize.rs

@@ -883,7 +883,7 @@ mod tests {
             SearcherConfig {
                 aggregation_memory_limit: ByteSize::gb(1),
                 aggregation_bucket_limit: 500_000,
-                fast_field_cache: CacheConfig::default_with_capacity(ByteSize::gb(10)),
+                fast_field_cache: Some(CacheConfig::default_with_capacity(ByteSize::gb(10))),
                 split_footer_cache: CacheConfig::default_with_capacity(ByteSize::gb(1)),
                 partial_request_cache: CacheConfig::default_with_capacity(ByteSize::mb(64)),
                 predicate_cache: CacheConfig::default_with_capacity(ByteSize::mb(256)),

quickwit/quickwit-lambda-server/src/context.rs

@@ -65,7 +65,7 @@ fn try_searcher_config_from_env() -> anyhow::Result<SearcherConfig> {
     let mut searcher_config = SearcherConfig::default();
     searcher_config.max_num_concurrent_split_searches = 20;
     searcher_config.warmup_memory_budget = warmup_memory_budget;
-    searcher_config.fast_field_cache = CacheConfig::no_cache();
+    searcher_config.fast_field_cache = Some(CacheConfig::no_cache());
     searcher_config.split_footer_cache = CacheConfig::no_cache();
     searcher_config.predicate_cache = CacheConfig::no_cache();
     searcher_config.partial_request_cache = CacheConfig::no_cache();

quickwit/quickwit-search/src/service.rs

@@ -474,8 +474,12 @@ impl SearcherContext {
             searcher_config.max_num_concurrent_split_searches,
             searcher_config.warmup_memory_budget,
         );
-        let storage_long_term_cache =
-            Arc::new(QuickwitCache::new(&searcher_config.fast_field_cache));
+        let fast_field_cache = searcher_config.resolved_fast_field_cache();
+        let storage_long_term_cache = if fast_field_cache.capacity().as_u64() == 0 {
+            Arc::new(QuickwitCache::empty())
+        } else {
+            Arc::new(QuickwitCache::new(&fast_field_cache))
+        };
         let leaf_search_cache = LeafSearchCache::new(&searcher_config.partial_request_cache);
         let predicate_cache = PredicateCacheImpl::new(&searcher_config.predicate_cache);
         let list_fields_cache = ListFieldsCache::new(&searcher_config.partial_request_cache);
@@ -507,3 +511,52 @@ impl SearcherContext {
         self.aggregation_limit.clone()
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use std::path::PathBuf;
+
+    use quickwit_config::{CacheConfig, SearcherConfig};
+    use quickwit_storage::OwnedBytes;
+
+    use super::SearcherContext;
+
+    #[tokio::test]
+    async fn test_zero_capacity_fast_field_cache_does_not_retain_entries() {
+        let mut searcher_config = SearcherConfig::default();
+        searcher_config.fast_field_cache = Some(CacheConfig::no_cache());
+        let searcher_context = SearcherContext::new_without_invoker(searcher_config, None, None);
+        let path = PathBuf::from("segment.fast");
+        searcher_context
+            .fast_fields_cache
+            .put(path.clone(), 0..3, OwnedBytes::new(&b"abc"[..]))
+            .await;
+        assert!(
+            searcher_context
+                .fast_fields_cache
+                .get(path.as_path(), 0..3)
+                .await
+                .is_none()
+        );
+    }
+
+    #[tokio::test]
+    async fn test_nonzero_fast_field_cache_retains_entries() {
+        let searcher_context =
+            SearcherContext::new_without_invoker(SearcherConfig::default(), None, None);
+        let path = PathBuf::from("segment.fast");
+        searcher_context
+            .fast_fields_cache
+            .put(path.clone(), 0..3, OwnedBytes::new(&b"abc"[..]))
+            .await;
+        assert_eq!(
+            searcher_context
+                .fast_fields_cache
+                .get(path.as_path(), 0..3)
+                .await
+                .unwrap()
+                .as_slice(),
+            b"abc"
+        );
+    }
+}

quickwit/quickwit-storage/Cargo.toml

@@ -65,6 +65,7 @@ reqwest = { workspace = true, optional = true }
 
 [dev-dependencies]
 http = { workspace = true }
+metrics-util = { workspace = true }
 mockall = { workspace = true }
 proptest = { workspace = true }
 # Match OpenDAL's internal reqwest major. `default-features = false` is
@@ -107,8 +108,3 @@ integration-testsuite = [
   "dep:reqwest",
 ]
 testsuite = ["mockall"]
-
-[package.metadata.cargo-machete]
-# Declared here so the Foyer pin lands with licenses. Follow-up split-range
-# cache PRs use these crates; remove the ignore when they do.
-ignored = ["metrics", "mixtrics"]