From 8ab3c48435ed2b98e1216b2fde4fcd6d071c36ef Mon Sep 17 00:00:00 2001 From: Oleksii Date: Fri, 23 Oct 2026 05:32:33 -0300 Subject: [PATCH 1/2] fix(bench): validate workloads and run functional smoke in CI --- .github/workflows/ci.yml | 7 ++ orch8-engine/benches/engine_bench.rs | 146 ++++++++++++++++++------- orch8-storage/benches/storage_bench.rs | 7 +- 3 files changed, 116 insertions(+), 44 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ff03f1e2..b5226928 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -156,6 +156,13 @@ jobs: run: cargo build --locked -p orch8-server --bin orch8-server - name: Test run: cargo test --locked --workspace + # Criterion's test mode exercises each workload without collecting + # measurements. Keep fixtures and completion assertions covered when + # evaluator contracts change; the scheduled workflow measures timings. + - name: Benchmark smoke + run: | + cargo bench --locked -p orch8-engine --bench engine_bench --profile test -- --test + cargo bench --locked -p orch8-storage --bench storage_bench --profile test -- --test dashboard: name: Dashboard diff --git a/orch8-engine/benches/engine_bench.rs b/orch8-engine/benches/engine_bench.rs index 481ba1c7..5bcbcc13 100644 --- a/orch8-engine/benches/engine_bench.rs +++ b/orch8-engine/benches/engine_bench.rs @@ -10,6 +10,7 @@ use criterion::{BatchSize, Criterion, criterion_group, criterion_main}; use serde_json::json; use tokio::runtime::Runtime; +use orch8_engine::evaluator::{EvalOutcome, TerminalReason}; use orch8_engine::handlers::StepContext; use orch8_storage::sqlite::SqliteStorage; use orch8_storage::{ @@ -27,6 +28,19 @@ fn block_on_in_place(future: F) -> F::Output { tokio::task::block_in_place(|| tokio::runtime::Handle::current().block_on(future)) } +fn completed_evaluation(outcome: EvalOutcome) -> EvalOutcome { + assert!( + matches!( + outcome, + EvalOutcome::Done { + reason: TerminalReason::Succeeded + } + ), + "benchmark must execute its workflow successfully: {outcome:?}" + ); + outcome +} + fn make_sequence() -> SequenceDefinition { SequenceDefinition { embed: None, @@ -73,6 +87,27 @@ fn make_sequence() -> SequenceDefinition { } } +fn make_flat_sequence(count: usize) -> SequenceDefinition { + let mut sequence = make_sequence(); + let BlockDefinition::Step(template) = sequence.blocks.pop().unwrap() else { + unreachable!("base benchmark sequence contains one step") + }; + sequence.blocks = (0..count) + .map(|index| { + let mut step = template.clone(); + step.id = BlockId::new(format!("s1_{index}")); + BlockDefinition::Step(step) + }) + .collect(); + sequence +} + +fn noop_handlers() -> orch8_engine::handlers::HandlerRegistry { + let mut handlers = orch8_engine::handlers::HandlerRegistry::new(); + handlers.register("noop", |_ctx| async { Ok(json!({})) }); + handlers +} + fn make_instance(seq_id: SequenceId) -> TaskInstance { let now = Utc::now(); TaskInstance { @@ -421,12 +456,12 @@ fn make_nested_blocks(depth: usize, breadth: usize) -> Vec Vec { + fn build_level(level: usize, breadth: usize, path: &str) -> Vec { if level == 0 { return (0..breadth) .map(|i| { BlockDefinition::Step(Box::new(StepDef { - id: BlockId::new(format!("s_{level}_{i}")), + id: BlockId::new(format!("s_{path}{i}")), handler: "noop".into(), params: serde_json::json!({}), delay: None, @@ -453,32 +488,39 @@ fn make_nested_blocks(depth: usize, breadth: usize) -> Vec>() + .len(), + count, + "benchmark block IDs must be unique" + ); + c.bench_function(name, |b| { + b.iter(|| orch8_engine::evaluator::flatten_blocks(black_box(&blocks))); + }); + } } // ---- evaluate() tree benchmark --------------------------------------------- @@ -491,18 +533,26 @@ fn bench_evaluate_deep_tree(c: &mut Criterion) { b.to_async(&rt).iter_batched( || { let s = block_on_in_place(SqliteStorage::in_memory()).unwrap(); - let seq = make_sequence(); + let seq = make_flat_sequence(10); + assert_eq!( + seq.blocks.len(), + 10, + "flat benchmark must contain ten steps" + ); block_on_in_place(s.create_sequence(&seq)).unwrap(); let mut inst = make_instance(seq.id); inst.state = InstanceState::Running; block_on_in_place(s.create_instance(&inst)).unwrap(); let mut nodes = Vec::new(); - for i in 0..10 { + for block in &seq.blocks { + let BlockDefinition::Step(step) = block else { + unreachable!("flat benchmark contains only steps") + }; nodes.push(ExecutionNode { id: ExecutionNodeId::new(), instance_id: inst.id, - block_id: BlockId::new(format!("s1_{i}")), + block_id: step.id.clone(), parent_id: None, block_type: BlockType::Step, branch_index: None, @@ -512,12 +562,15 @@ fn bench_evaluate_deep_tree(c: &mut Criterion) { }); } block_on_in_place(s.create_execution_nodes_batch(&nodes)).unwrap(); - (s, inst, seq) - }, - |(s, inst, seq)| async move { - let handlers = orch8_engine::handlers::HandlerRegistry::new(); let storage: Arc = Arc::new(s); - let _ = orch8_engine::evaluator::evaluate(&storage, &handlers, &inst, &seq).await; + (storage, noop_handlers(), inst, seq) + }, + |(storage, handlers, inst, seq)| async move { + completed_evaluation( + orch8_engine::evaluator::evaluate(&storage, &handlers, &inst, &seq) + .await + .unwrap(), + ); }, BatchSize::SmallInput, ); @@ -583,12 +636,15 @@ fn bench_evaluate_deep_tree(c: &mut Criterion) { let mut inst = make_instance(seq.id); inst.state = InstanceState::Running; block_on_in_place(s.create_instance(&inst)).unwrap(); - (s, inst, seq) - }, - |(s, inst, seq)| async move { - let handlers = orch8_engine::handlers::HandlerRegistry::new(); let storage: Arc = Arc::new(s); - let _ = orch8_engine::evaluator::evaluate(&storage, &handlers, &inst, &seq).await; + (storage, noop_handlers(), inst, seq) + }, + |(storage, handlers, inst, seq)| async move { + completed_evaluation( + orch8_engine::evaluator::evaluate(&storage, &handlers, &inst, &seq) + .await + .unwrap(), + ); }, BatchSize::SmallInput, ); @@ -679,14 +735,16 @@ fn bench_parallel_in_process_latency(c: &mut Criterion) { }, |(storage, instance, sequence, handlers)| async move { let storage: Arc = Arc::new(storage); - orch8_engine::evaluator::evaluate( - black_box(&storage), - black_box(&handlers), - black_box(&instance), - black_box(&sequence), + completed_evaluation( + orch8_engine::evaluator::evaluate( + black_box(&storage), + black_box(&handlers), + black_box(&instance), + black_box(&sequence), + ) + .await + .unwrap(), ) - .await - .unwrap() }, BatchSize::SmallInput, ); @@ -730,7 +788,7 @@ fn bench_tick_once(c: &mut Criterion) { externalize_output_threshold: 0, ..orch8_types::config::SchedulerConfig::default() }; - let handlers = orch8_engine::handlers::HandlerRegistry::new(); + let handlers = noop_handlers(); let cache = orch8_engine::sequence_cache::SequenceCache::new( 64, std::time::Duration::from_secs(60), @@ -745,10 +803,14 @@ fn bench_tick_once(c: &mut Criterion) { config.max_concurrent_steps as usize, )); let cache = Arc::new(cache); - let _ = orch8_engine::scheduler::tick_once( + let result = orch8_engine::scheduler::tick_once( &storage, &handlers, &semaphore, &config, &cache, &cancel, ) - .await; + .await + .unwrap(); + assert_eq!(result.instances_advanced, count); + assert_eq!(result.steps_executed, count); + assert!(!result.has_pending_work); }, BatchSize::SmallInput, ); diff --git a/orch8-storage/benches/storage_bench.rs b/orch8-storage/benches/storage_bench.rs index 3c3b9797..14195892 100644 --- a/orch8-storage/benches/storage_bench.rs +++ b/orch8-storage/benches/storage_bench.rs @@ -187,9 +187,12 @@ fn bench_block_outputs(c: &mut Criterion) { created_at: Utc::now(), }; s.save_block_output(&out).await.unwrap(); - s.get_block_output(inst_id, &BlockId::new("step_1")) + let stored = s + .get_block_output(inst_id, &BlockId::new("step_1")) .await - .unwrap(); + .unwrap() + .expect("benchmark must retrieve its saved output"); + assert_eq!(stored.output, out.output); }); }, BatchSize::SmallInput, From 89747e94529b5178563c1624e30ca11d6f76196d Mon Sep 17 00:00:00 2001 From: Oleksii Date: Fri, 23 Oct 2026 05:57:18 -0300 Subject: [PATCH 2/2] fix(security): patch Rust lockfiles and audit fuzz dependencies --- .github/workflows/ci.yml | 12 +++++++----- Cargo.lock | 4 ++-- fuzz/Cargo.lock | 29 +++++++++++++++-------------- packages/node-native/Cargo.lock | 4 ++-- packages/python-native/Cargo.lock | 4 ++-- 5 files changed, 28 insertions(+), 25 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b5226928..996d8d25 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -81,12 +81,14 @@ jobs: - name: Supply chain (licenses + advisories) run: | cargo install --locked "cargo-deny@^0.19" - cargo deny check + cargo deny --locked check # The default workspace graph omits optional features and the - # standalone Node/Python workspaces have independent lockfiles. - cargo deny --all-features check advisories - cargo deny --manifest-path packages/node-native/Cargo.toml check advisories - cargo deny --manifest-path packages/python-native/Cargo.toml check advisories + # standalone Node/Python/fuzz workspaces have independent lockfiles. + # Reject lockfile drift rather than resolving it only on the runner. + cargo deny --locked --all-features check advisories + cargo deny --locked --manifest-path packages/node-native/Cargo.toml check advisories + cargo deny --locked --manifest-path packages/python-native/Cargo.toml check advisories + cargo deny --locked --manifest-path fuzz/Cargo.toml check advisories test: name: Unit Tests diff --git a/Cargo.lock b/Cargo.lock index 4d1d09ee..f2a2c013 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1111,9 +1111,9 @@ dependencies = [ [[package]] name = "ctutils" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03bb0e1cc970d482d121d9a1744999169b69a07470b3d644a7894e53fcaf4574" +checksum = "12218662ccac37d9714f2191252553d67585b97df8c839cbe49e0bde278dc865" dependencies = [ "cmov", ] diff --git a/fuzz/Cargo.lock b/fuzz/Cargo.lock index 34d41889..15b5eb19 100644 --- a/fuzz/Cargo.lock +++ b/fuzz/Cargo.lock @@ -465,9 +465,9 @@ dependencies = [ [[package]] name = "ctutils" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03bb0e1cc970d482d121d9a1744999169b69a07470b3d644a7894e53fcaf4574" +checksum = "12218662ccac37d9714f2191252553d67585b97df8c839cbe49e0bde278dc865" dependencies = [ "cmov", ] @@ -1311,9 +1311,9 @@ dependencies = [ [[package]] name = "jsonschema" -version = "0.58.6" +version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f622c574a29fb3034a0a90645afee194dfedc75fc124228801b6e77041075bd1" +checksum = "1b320ad34c177093d21622edc9215bb608294c2333ff296672c0c8265a5ee60c" dependencies = [ "ahash", "bytecount", @@ -1339,18 +1339,18 @@ dependencies = [ [[package]] name = "jsonschema-regex" -version = "0.58.6" +version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a968fffd387b2ab005bf00c1851f5d26e90df05b6f7ad609ed0d620033cca5e7" +checksum = "1472f7be79026f79e01a9b172263311e829d7c210d57b39cb57037abc0ce0500" dependencies = [ "regex-syntax", ] [[package]] name = "jsonschema-value" -version = "0.58.6" +version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39b7670345e5ce87371339d13705dbb9276b7e3e934bbe1efa2c6734a1f65a96" +checksum = "e5a7550a7a84787e715ead0b47e01fa8c6bb7c3413630f91a8f2094c06c2078a" dependencies = [ "ahash", "bytecount", @@ -1631,6 +1631,7 @@ dependencies = [ "rand 0.10.3", "reqwest", "serde", + "serde_ignored", "serde_json", "sha2 0.11.1", "subtle", @@ -2048,9 +2049,9 @@ dependencies = [ [[package]] name = "referencing" -version = "0.58.6" +version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "558f3855529c0a7e78a48e409fa653456e55f1b167eab4109c16a255b7727e3f" +checksum = "473f4dcca28ead7ee0f8e5b5d0bbae2549f4fc4c958fb20af328961b6c174be3" dependencies = [ "ahash", "fluent-uri", @@ -2227,9 +2228,9 @@ checksum = "eec689c0bc40ff2458a5977b6619cb718087084a18e02a131c599b62d05e1a5f" [[package]] name = "rustls-webpki" -version = "0.103.15" +version = "0.103.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" +checksum = "4443b1b6d39d7c663139942cc4558262562d2c84123bb6282ef177009ca8b281" dependencies = [ "aws-lc-rs", "ring", @@ -3282,9 +3283,9 @@ dependencies = [ [[package]] name = "webpki-root-certs" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b" +checksum = "58ec80e73fd8baf57bf0d9615554c75348bda990d6d5bbeab964dd068319fecb" dependencies = [ "rustls-pki-types", ] diff --git a/packages/node-native/Cargo.lock b/packages/node-native/Cargo.lock index e7b3749a..99712add 100644 --- a/packages/node-native/Cargo.lock +++ b/packages/node-native/Cargo.lock @@ -484,9 +484,9 @@ dependencies = [ [[package]] name = "ctutils" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03bb0e1cc970d482d121d9a1744999169b69a07470b3d644a7894e53fcaf4574" +checksum = "12218662ccac37d9714f2191252553d67585b97df8c839cbe49e0bde278dc865" dependencies = [ "cmov", ] diff --git a/packages/python-native/Cargo.lock b/packages/python-native/Cargo.lock index 321a4d08..5d732bd0 100644 --- a/packages/python-native/Cargo.lock +++ b/packages/python-native/Cargo.lock @@ -469,9 +469,9 @@ dependencies = [ [[package]] name = "ctutils" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03bb0e1cc970d482d121d9a1744999169b69a07470b3d644a7894e53fcaf4574" +checksum = "12218662ccac37d9714f2191252553d67585b97df8c839cbe49e0bde278dc865" dependencies = [ "cmov", ]