Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions crates/gateway/config-ui/ui/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
"build": "node build.mjs",
"watch": "node build.mjs --watch",
"typecheck": "tsc --noEmit",
"pretest": "npm run build",
"test": "node --test \"src/**/*.test.mjs\""
},
"dependencies": {
Expand Down
4 changes: 2 additions & 2 deletions crates/gateway/config-ui/ui/src/main.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@
// and asserts the live desk's first paint: standalone with no stored
// key, the auto-boot on #app must land on the key prompt - medallion,
// title, labeled password input, and submit button - without touching
// the network. Run after `npm run build` (a debug `cargo build` also
// produces dist/).
// the network. Run with `npm test`, whose `pretest` script builds dist/
// first.
import assert from "node:assert/strict";
import { readFile } from "node:fs/promises";
import path from "node:path";
Expand Down
4 changes: 2 additions & 2 deletions crates/gateway/config-ui/ui/src/styles/tokens.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@
// (shared-ui/tokens.css) and the cascade layer order, and
// dist/index.html must link that stylesheet - a dropped import in
// main.ts or a dropped <link> would ship an unstyled UI without any
// build failure. Run after `npm run build` (a debug `cargo build` also
// produces dist/).
// build failure. Run with `npm test`, whose `pretest` script builds
// dist/ first.
import assert from "node:assert/strict";
import { readFile } from "node:fs/promises";
import path from "node:path";
Expand Down
2 changes: 1 addition & 1 deletion crates/gateway/config-ui/ui/src/test-support.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ const CONFIG_URL = "http://127.0.0.1:8081/config/";

let appModulePromise;

/** Imports the built bundle once; run `npm run build` (or a debug `cargo build`) first. */
/** Imports the built bundle once; `npm test` builds it first through the `pretest` script. */
export function loadApp() {
if (!appModulePromise) {
// Module evaluation touches `document` (the #app auto-boot probe),
Expand Down
18 changes: 12 additions & 6 deletions crates/harness-internal/runner/src/host-run.rs
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ use runvil::effect::ToolCallOrigin;
use runvil::packs::Prelude;
use runvil::tools::{ToolCatalog, ToolError, ToolErrorKind, ToolId, ToolInfo, ToolOutput};
use runvil::vfs::Access;
use runvil::{MissingService, Prompt, Requirements, UnavailablePack};
use runvil::{MissingService, Prompt, Requirements, UnavailablePack, UsablePack};
use runvil_plugin::{HostServices, Pack, PackId, PackInfo, ServiceId, ToolContext};
use serde_json::Value;

Expand Down Expand Up @@ -107,15 +107,21 @@ impl HostRunContext {
self.catalog.clone()
}

/// The run's roster: the ids of its usable Packs, with or without
/// tools, and an entry with the reason for each one that can't serve,
/// both in id order.
pub(crate) fn roster(&self) -> (Vec<PackId>, Vec<UnavailablePack>) {
/// The run's roster: each of its usable Packs, with or without tools,
/// with the vendor of its FQN, and an entry with the reason for each
/// one that can't serve, both in id order.
pub(crate) fn roster(&self) -> (Vec<UsablePack>, Vec<UnavailablePack>) {
let mut usable = Vec::new();
let mut unavailable = Vec::new();
for (name, pack) in &self.packs {
match pack {
PackState::Usable { .. } => usable.push(name.clone()),
PackState::Usable { info, .. } => {
let vendor = info
.fqn
.split_once('/')
.map_or(info.fqn, |(vendor, _)| vendor);
usable.push(UsablePack::new(name.clone(), vendor));
}
PackState::Unavailable(reason) => {
unavailable.push(UnavailablePack::new(name.clone(), reason.clone()));
}
Expand Down
32 changes: 29 additions & 3 deletions crates/harness-internal/runner/src/host-tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ use std::sync::{Arc, Mutex};

use runvil::effect::{ToolCallOrigin, ToolCaller};
use runvil::vfs::{Origin, VfsRef};
use runvil::{MissingService, Prompt, Requirements, UnavailablePack};
use runvil::{MissingService, Prompt, Requirements, UnavailablePack, UsablePack};
use runvil_plugin::{
HostServices, Pack, PackFuture, PackId, PackInfo, PackRegistry, ServiceId, ServiceKey,
ToolContext, ToolError, ToolId, ToolInfo, ToolOutput,
Expand Down Expand Up @@ -191,7 +191,33 @@ fn catalog_ids(host: &HostContext, services: HostServices, prompt: &Prompt) -> V
fn usable(host: &HostContext, prompt: &Prompt) -> Vec<String> {
let (run, _requirements) = begin(host, HostServices::new(), BTreeMap::new(), prompt);
let (usable, _unavailable) = run.roster();
usable.iter().map(ToString::to_string).collect()
usable.iter().map(|entry| entry.pack.to_string()).collect()
}

#[test]
fn the_roster_hands_each_usable_pack_with_the_vendor_the_run_picked() {
let host = host(&[
(info("runvil/kit"), offering(&["kit/a"])),
(info("acme/kit"), offering(&["kit/b"])),
(info("runvil/web"), offering(&["web/fetch"])),
(info("acme/poop"), Value::Null),
]);
let (run, _requirements) = begin(
&host,
HostServices::new(),
BTreeMap::new(),
&prompt("uses:\n - acme/kit\n"),
);
let (usable, _unavailable) = run.roster();
assert_eq!(
usable,
[
UsablePack::new(name("kit"), "acme"),
UsablePack::new(name("subagents"), "runvil"),
UsablePack::new(name("web"), "runvil"),
],
"the declared acme/kit reports acme, and every undeclared Pack reports runvil"
);
}

#[test]
Expand Down Expand Up @@ -328,7 +354,7 @@ fn the_roster_lists_the_builtin_stub_as_usable_and_each_pack_that_cannot_serve_w
let (usable, unavailable) = run.roster();
assert_eq!(
usable,
[name("subagents")],
[UsablePack::new(name("subagents"), "runvil")],
"the undeclared stub with no tools is on the roster"
);
assert_eq!(
Expand Down
4 changes: 2 additions & 2 deletions crates/harness-internal/runner/src/prepare.rs
Original file line number Diff line number Diff line change
Expand Up @@ -357,8 +357,8 @@ pub(crate) async fn prepare_noting_cancel(
})
}

/// Sets `snapshot`'s catalog, roster, and declared preludes on `ctx`, then
/// prepares it for `prompt`.
/// Sets `snapshot`'s catalog, roster with each usable Pack's vendor, and
/// declared preludes on `ctx`, then prepares it for `prompt`.
fn prepare_from_snapshot(
ctx: RunContext,
snapshot: &HostRunContext,
Expand Down
13 changes: 8 additions & 5 deletions crates/harness-internal/runner/tests/it/prepare-ready.rs
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,10 @@ const DECLARES_SLOW: &str = "---\nname: declares-slow\ndescription: d\nrunvil: 0
/// A Pack-free prompt returning how many undeclared tools the run
/// offers.
const COUNTS_OFFERED: &str = "---\nname: counts\ndescription: d\nrunvil: 0\n---\n\n\
# Title\n\n## Only\n\n```lua\nreturn #tools.extras() .. ' offered'\n```\n";
# Title\n\n## Only\n\n```lua\n\
local count = 0\n\
for _, pack in ipairs(tools.packs.extras()) do count = count + #pack.tools end\n\
return count .. ' offered'\n```\n";

/// A Pack whose tools wait on the test's [`Startup`].
struct Starting {
Expand Down Expand Up @@ -306,15 +309,15 @@ async fn an_undeclared_pack_whose_ready_fails_leaves_the_run_going_without_its_t
#[tokio::test]
async fn an_undeclared_pack_whose_ready_fails_raises_its_reason_when_the_prompt_offers_it() {
let offers_slow = "---\nname: offers-slow\ndescription: d\nrunvil: 0\n---\n\n\
# Title\n\n## Only\n\n```lua\npacks.offer('slow')\nreturn 'offered'\n```\n";
# Title\n\n## Only\n\n```lua\ntools.offer('slow')\nreturn 'offered'\n```\n";
let (host, _moves, _startup) = slow_host(Phase::Failed("the fixture server is down"));
let outcome = drive_over(offers_slow, host).await;
let RunOutcome::Failed { message, .. } = &outcome else {
panic!("offering an unavailable Pack fails the run: {outcome:?}");
};
assert!(
message.contains(
"packs.offer: the slow Pack is unavailable in this run: \
"tools.offer: the slow Pack is unavailable in this run: \
the fixture server is down"
),
"the failure names the Pack and its ready failure: {message}"
Expand Down Expand Up @@ -344,8 +347,8 @@ async fn an_undeclared_third_party_pack_that_never_gets_ready_holds_no_run_up()
async fn an_undeclared_builtin_with_no_tools_has_an_object_the_prompt_can_offer() {
let reads_stub = "---\nname: reads-stub\ndescription: d\nrunvil: 0\n---\n\n\
# Title\n\n## Only\n\n```lua\n\
local stub = packs.get('subagents')\n\
packs.offer('subagents')\n\
local stub = tools.packs.get('subagents')\n\
tools.offer('subagents')\n\
return stub.name .. ':' .. #stub.tools\n```\n";
let host = HostContext::new(registry(&[]), HostServices::new());
assert_eq!(
Expand Down
17 changes: 9 additions & 8 deletions crates/harness-internal/runner/tests/it/prepare.rs
Original file line number Diff line number Diff line change
Expand Up @@ -374,7 +374,7 @@ async fn an_undeclared_third_party_pack_is_not_in_the_run() {
let undeclared = "---\nname: undeclared\ndescription: d\nrunvil: 0\n---\n\n\
# Title\n\n## Only\n\n```lua\n\
local ok = pcall(tools.call, 'tools/echo', { value = 'hi' })\n\
return tostring(ok) .. '|' .. tostring(packs.get('tools'))\n```\n";
return tostring(ok) .. '|' .. tostring(tools.packs.get('tools'))\n```\n";
assert_eq!(
completed(drive_over(undeclared, building(TOOLS)).await),
"false|nil",
Expand Down Expand Up @@ -402,19 +402,20 @@ async fn a_bare_entry_another_vendor_answers_is_refused_naming_its_fqn() {
}

#[tokio::test]
async fn the_lists_split_the_offering_by_declared_pack_and_a_script_calls_a_tool_object() {
async fn the_packs_split_by_declaration_and_a_script_calls_an_extras_tool_object() {
let offering = "---\nname: offering\ndescription: d\nrunvil: 0\n\
uses:\n - tests/tools\n---\n\n# Title\n\n## Only\n\n```lua\n\
local required, extras = tools.required(), tools.extras()\n\
local extra = extras[1]\n\
return #required .. ':' .. required[1].id .. '|' .. #extras .. ':' .. extra.id .. '|' .. \
tools.call(extra, { value = 'hi' })\n```\n";
local declared = tools.packs.get('tools')\n\
local extras = tools.packs.extras()\n\
local extra = extras[1].tools[1]\n\
return tostring(declared.required) .. ':' .. declared.tools[1].id .. '|' .. \
extras[1].name .. ':' .. extra.id .. '|' .. tools.call(extra, { value = 'hi' })\n```\n";
let extra = echo_pack("runvil/extra", None);
let host = HostContext::new(registry(&[TOOLS, extra]), HostServices::new());
assert_eq!(
completed(drive_over(offering, host).await),
"1:tools/echo|1:extra/echo|hi",
"the declared Pack's tool is required, the undeclared one's an extra, and its \
"true:tools/echo|extra:extra/echo|hi",
"the declared Pack is required, the undeclared one an extra, and its tool's \
object reaches its Pack"
);
}
27 changes: 27 additions & 0 deletions crates/plugin-mcp/src/dynamic-tests.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
//! Tests for the Pack's two static tools.

use std::collections::BTreeMap;

use runvil_plugin::{Pack, PackId};
use tokio::runtime::Handle;

use super::Mcp;

#[tokio::test]
async fn only_the_listing_tool_describes_tools() {
let name = PackId::parse("mcp").expect("a valid Pack id");
let mcp = Mcp::new(&name, BTreeMap::new(), Handle::current())
.expect("the Pack names its tools under a valid Pack id");
let flags: Vec<(String, bool)> = mcp
.tools()
.iter()
.map(|tool| (tool.id.to_string(), tool.describes_tools))
.collect();
assert_eq!(
flags,
[
("mcp/get_dynamic_tools".to_owned(), true),
("mcp/call_dynamic_tool".to_owned(), false),
]
);
}
31 changes: 24 additions & 7 deletions crates/plugin-mcp/src/dynamic.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,8 @@ use std::collections::BTreeMap;
use rmcp::model::JsonObject;
use rmcp::service::{Peer, RoleClient};
use runvil_plugin::{
Pack, PackFuture, PackId, ToolContext, ToolError, ToolErrorKind, ToolId, ToolInfo, ToolOutput,
Pack, PackFuture, PackId, ToolCaller, ToolContext, ToolError, ToolErrorKind, ToolId, ToolInfo,
ToolOutput,
};
use serde_json::{Map, Value, json};
use tokio::runtime::Handle;
Expand Down Expand Up @@ -70,6 +71,16 @@ pub(crate) struct Mcp {
runtime: Handle,
}

/// Marks `text` by who called: the section's own script receives it
/// trusted, and every other caller, a model round included, untrusted,
/// because the text comes from the servers.
fn marked(caller: ToolCaller, text: String) -> ToolOutput {
match caller {
ToolCaller::Script => ToolOutput::trusted(text),
_ => ToolOutput::untrusted(text),
}
}

fn invalid(text: impl Into<String>) -> ToolError {
ToolError::message(text).with_kind(ToolErrorKind::InvalidArguments)
}
Expand Down Expand Up @@ -150,7 +161,8 @@ impl Mcp {
}
}
}),
);
)
.describes_tools(true);
let call = ToolInfo::new(
id(CALL)?,
CALL_DESCRIPTION,
Expand Down Expand Up @@ -265,9 +277,9 @@ impl Mcp {
})
}

/// Performs `call_dynamic_tool`. Every argument is checked before
/// anything is sent.
async fn call_tool(&self, args: Value) -> Result<ToolOutput, ToolError> {
/// Performs `call_dynamic_tool` and returns the result's text. Every
/// argument is checked before anything is sent.
async fn call_tool(&self, args: Value) -> Result<String, ToolError> {
let mut members = members(
args,
"call_dynamic_tool takes a JSON object with the strings `namespace` and `toolName` \
Expand Down Expand Up @@ -303,13 +315,18 @@ impl Pack for Mcp {
) -> PackFuture<'a, Result<ToolOutput, ToolError>> {
Box::pin(async move {
let tool = cx.tool();
let caller = cx.origin().caller;
if *tool == self.get.id {
self.list(args).map(ToolOutput::untrusted)
self.list(args).map(|text| marked(caller, text))
} else if *tool == self.call.id {
self.call_tool(args).await
self.call_tool(args).await.map(|text| marked(caller, text))
} else {
Err(ToolError::message(format!("MCP has no tool {tool}")))
}
})
}
}

#[cfg(test)]
#[path = "dynamic-tests.rs"]
mod tests;
16 changes: 12 additions & 4 deletions crates/plugin-mcp/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,14 +15,20 @@
//! Cursor's `GetDynamicTools` and `CallDynamicTool`:
//!
//! - `get_dynamic_tools` takes the optional strings `namespace` and
//! `toolName` and answers JSON text, which a script reads with
//! `json.decode`. With no arguments it lists each server's namespace,
//! `toolName` and answers JSON text. It describes tools, so a script's
//! call receives the text as the servers wrote it and reads it with
//! `json.decode`, while a model's call receives it in the untrusted
//! envelope. With no arguments it lists each server's namespace,
//! status, and tool names, in name order; with `namespace`, that
//! server's tools with their input schemas; with both, one tool.
//! - `call_dynamic_tool` takes the strings `namespace` and `toolName` and
//! an optional object `arguments`, which defaults to `{}`, and calls that
//! server's tool.
//!
//! Both tools answer a model round with untrusted text, and the section's
//! own script with trusted text, so the script receives the server's text
//! without the envelope.
//!
//! A namespace is the server's name exactly as the configuration writes
//! it, and a tool's name is the one its server lists. Both must match
//! exactly. A server's status is `ready`, `loading` while it is still
Expand Down Expand Up @@ -79,8 +85,10 @@
//! - May depend on: `runvil-plugin`, `shared-*` crates, and outside
//! libraries. `cargo test -p build-xtask` enforces the Plugin family's
//! allow-list. Only [`install`] is public.
//! - Every output is untrusted, because descriptions, schemas, and results
//! all come from the servers.
//! - Output to a model round is untrusted, because descriptions, schemas,
//! and results all come from the servers, and output to the section's
//! own script is trusted, so the script receives the server's text
//! without the envelope.
//! - No reason, log line, `Debug` output, or error names an `env` or
//! `headers` value. They may name keys.
//! - It runs on the Host's runtime, read through the contract's
Expand Down
13 changes: 5 additions & 8 deletions crates/plugin-mcp/src/result-tests.rs
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
//! Tests for how an MCP answer becomes tool output or a tool error.
//! Tests for how an MCP answer becomes text or a tool error.

use std::time::Duration;

use rmcp::ServiceError;
use rmcp::model::{CallToolResult, ServerResult};
use runvil_plugin::{OutputTrust, ToolErrorKind};
use runvil_plugin::ToolErrorKind;
use serde_json::{Value, json};

use super::{output, render, service_error};
Expand All @@ -14,9 +14,7 @@ fn result(value: Value) -> CallToolResult {
}

fn text_of(value: Value) -> String {
let output = render(&result(value)).expect("a success renders");
assert_eq!(output.trust(), OutputTrust::Untrusted);
output.text().to_owned()
render(&result(value)).expect("a success renders")
}

#[test]
Expand Down Expand Up @@ -83,7 +81,7 @@ fn content_blocks_win_over_structured_content() {
}

#[test]
fn a_result_with_nothing_is_empty_untrusted_text() {
fn a_result_with_nothing_is_empty_text() {
assert_eq!(text_of(json!({ "content": [] })), "");
}

Expand Down Expand Up @@ -165,8 +163,7 @@ fn a_failed_request_maps_through_output_and_a_good_one_renders() {
json!({ "content": [{ "type": "text", "text": "fine" }] }),
))))
.expect("renders");
assert_eq!(good.text(), "fine");
assert_eq!(good.trust(), OutputTrust::Untrusted);
assert_eq!(good, "fine");
}

#[test]
Expand Down
Loading
Loading