Skip to content
Open
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 51 additions & 8 deletions quickwit/quickwit-rest-client/src/rest_client.rs
Original file line number Diff line number Diff line change
Expand Up @@ -343,18 +343,23 @@ impl QuickwitClient {
let mut cumulated_resp = RestIngestResponse::default();
while let Some(batch) = batch_reader.next_batch().await? {
loop {
let timeout = if !batch_reader.has_next() && last_block_commit != CommitType::Auto {
// Only the last batch carries the requested commit mode; the other batches
// use the default (auto) commit.
let is_last_batch = !batch_reader.has_next();
let timeout = if is_last_batch && last_block_commit != CommitType::Auto {
self.commit_timeout
} else {
self.ingest_timeout
};
match last_block_commit {
CommitType::Auto => {}
CommitType::WaitFor => {
query_params.insert("commit", "wait_for");
}
CommitType::Force => {
query_params.insert("commit", "force");
if is_last_batch {

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Badge Preserve commit semantics across multiple ingest shards

When an index is configured or scaled to multiple ingest-v2 shards, successive HTTP batches are not ordered through one log: RoutingEntry::pick_node samples eligible ingesters afresh for each request (quickwit/quickwit-ingest/src/ingest_v2/routing_table.rs:86-105,122-143), and a force marker is appended only to the shard selected for that request (quickwit/quickwit-ingest/src/ingest_v2/mrecordlog_utils.rs:58-73); WaitFor similarly tracks only that request's shard position (quickwit/quickwit-ingest/src/ingest_v2/workbench.rs:160-166). Making every earlier batch Auto therefore lets a batch on shard A remain unpublished when the final batch on shard B is published and this call returns, despite the CLI help promising that --wait/--force waits for all documents (quickwit/quickwit-cli/src/index.rs:140-152). Keep per-batch barriers for v2 or add a source-wide final barrier before applying this optimization.

Useful? React with 👍 / 👎.

match last_block_commit {
CommitType::Auto => {}
CommitType::WaitFor => {
query_params.insert("commit", "wait_for");
}
CommitType::Force => {
query_params.insert("commit", "force");
}
}
}
let response = self
Expand Down Expand Up @@ -961,6 +966,44 @@ mod test {
assert_eq!(actual_response, mock_response);
}

#[tokio::test]
async fn test_ingest_endpoint_force_commit_only_on_last_batch() {
let mock_server = MockServer::start().await;
let server_url = Url::parse(&mock_server.uri()).unwrap();
let qw_client = QuickwitClientBuilder::new(server_url).build();
let mock_response = RestIngestResponse {
num_docs_for_processing: 1,
num_ingested_docs: Some(1),
num_rejected_docs: Some(0),
parse_failures: Some(Vec::new()),
};
Mock::given(method("POST"))
.and(path("/api/v1/my-index/ingest"))
.respond_with(ResponseTemplate::new(StatusCode::OK).set_body_json(&mock_response))
.mount(&mock_server)
.await;
// Each document is 10 bytes long, so a 12-byte limit yields one batch per document.
let ingest_source =
IngestSource::Str("{\"id\": 1}\n{\"id\": 2}\n{\"id\": 3}\n".to_string());
qw_client
.ingest("my-index", ingest_source, Some(12), None, CommitType::Force)
.await
.unwrap();
let requests = mock_server.received_requests().await.unwrap();
assert_eq!(requests.len(), 3);
let commit_params: Vec<Option<String>> = requests
.iter()
.map(|request| {
request
.url
.query_pairs()
.find(|(key, _)| key == "commit")
.map(|(_, value)| value.to_string())
})
.collect();
assert_eq!(commit_params, vec![None, None, Some("force".to_string())]);
}

#[tokio::test]
async fn test_ingest_endpoint_with_wait_for_commit() {
let mock_server = MockServer::start().await;
Expand Down