Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 56 additions & 0 deletions .github/workflows/ValidatePullRequest.yml
Original file line number Diff line number Diff line change
Expand Up @@ -179,6 +179,61 @@ jobs:
config: ${{ matrix.config }}
should_regen_goldens: ${{ needs.check-golden-label.outputs.should_regen_goldens }}

# Build and test the macOS HVF (Hypervisor.framework) backend on Apple silicon.
# Self-contained: guests are built and tested on the same GitHub-hosted arm64
# runner (no artifact sharing with the other build-test jobs). Requires
# macOS 26 for the 4 KiB IPA granule API.
build-test-macos-hvf:
needs: [docs-pr, update-guest-locks]
# Required because update-guest-locks is skipped on non-dependabot PRs,
# and a skipped dependency transitively skips all downstream jobs.
# See: https://github.com/actions/runner/issues/2205
if: ${{ !cancelled() && !failure() }}
timeout-minutes: 45
runs-on: macos-26

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

unfortunately I don't think github mac runners support nested so this won't work :/

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sad :(

env:
# cargo-hyperlight builds a custom target/sysroot; run everything through
# the rustup proxy with the pinned toolchain (newer rustc rejects custom
# target specs, Homebrew rustc shadows the proxy otherwise).
RUSTUP_TOOLCHAIN: "1.94"
RUSTC: ~/.cargo/bin/rustc
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
submodules: true

- uses: hyperlight-dev/ci-setup-workflow@f6bd9cc86d0737976d2128c8b8ced8edc017cbb4 # v1.9.0
with:
rust-toolchain: "1.94"
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

# crates.io cargo-hyperlight 0.1.13 does not compile on macOS (the ar
# detection fix is only on main). Pin the git rev; switch back to the
# crates.io pin (just ensure-cargo-hyperlight) once a release > 0.1.13
# ships. The Justfile's ensure-cargo-hyperlight accepts this binary
# because it reports version 0.1.13.
- name: Install cargo-hyperlight from git
run: cargo install --locked --git https://github.com/hyperlight-dev/cargo-hyperlight --rev 33384c0c4ed9dea4f0525943809fc444c41a27df cargo-hyperlight

# llvm-ar/llvm-ranlib are needed to archive the picolibc objects for C
# guests; Xcode's ar silently drops ELF members.
- name: Install LLVM
run: brew install llvm

- name: Build Rust and C guests
run: |
just build-and-move-rust-guests
just build-c-guests debug
just move-c-guests debug

# Builds, ad-hoc codesigns (com.apple.security.hypervisor) and runs the
# hyperlight-host lib + integration test binaries with the hvf feature.
# Snapshot-golden verification is a no-op here until the hvf goldens are
# published to the golden registry (see RegenSnapshotGoldens.yml).
- name: Test with the HVF backend
run: just test-hvf debug

# Run examples - needs guest artifacts, runs in parallel with build-test
run-examples:
needs:
Expand Down Expand Up @@ -255,6 +310,7 @@ jobs:
- code-checks
- check-golden-label
- build-test
- build-test-macos-hvf
- run-examples
- fuzzing
- spelling
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/dep_code_checks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ jobs:
run: just clippy-exhaustive release

- name: Verify MSRV
run: ./dev/verify-msrv.sh hyperlight-host hyperlight-guest hyperlight-guest-bin hyperlight-common
run: ./dev/verify-msrv.sh hyperlight-host hyperlight-guest hyperlight-guest-bin hyperlight-common hyperlight-hvf

- name: Check cargo features compile
run: just check
Expand Down
23 changes: 23 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ members = [
"src/hyperlight_host",
"src/hyperlight_guest_capi",
"src/hyperlight_guest_tracing",
"src/hyperlight_hvf",
"src/hyperlight_testing",
"fuzz",
"src/hyperlight_guest_bin",
Expand Down Expand Up @@ -43,6 +44,7 @@ hyperlight-guest-bin = { path = "src/hyperlight_guest_bin", version = "0.16.0",
hyperlight-guest-macro = { path = "src/hyperlight_guest_macro", version = "0.16.0", default-features = false }
hyperlight-testing = { path = "src/hyperlight_testing", default-features = false }
hyperlight-guest-tracing = { path = "src/hyperlight_guest_tracing", version = "0.16.0", default-features = false }
hyperlight-hvf = { path = "src/hyperlight_hvf", version = "0.16.0", default-features = false }
hyperlight-libc = { path = "src/hyperlight_libc", version = "0.16.0", default-features = false }
hyperlight-component-util = { path = "src/hyperlight_component_util", version = "0.16.0", default-features = false }
hyperlight-component-macro = { path = "src/hyperlight_component_macro", version = "0.16.0", default-features = false }
Expand Down
20 changes: 19 additions & 1 deletion Justfile
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ bin-suffix := if os() == "windows" { ".bat" } else { ".sh" }
nightly-toolchain := "nightly-2026-02-27"
# Pinned cargo-hyperlight version used to build the guest sysroot. Keep this in
# lockstep with the version pinned in flake.nix.
cargo-hyperlight-version := "0.1.12"
cargo-hyperlight-version := "0.1.13"

################
### cross-rs ###
Expand Down Expand Up @@ -247,6 +247,24 @@ test-isolated target=default-target features="" :
@# metrics tests
{{ cargo-cmd }} test {{ if features =="" {''} else if features=="no-default-features" {"--no-default-features" } else {"--no-default-features -F function_call_metrics," + features } }} --profile={{ if target == "debug" { "dev" } else { target } }} {{ target-triple-flag }} -p hyperlight-host --lib -- metrics::tests::test_metrics_are_emitted --exact

# Ad-hoc codesigns hyperlight test binaries with the Hypervisor.framework
# entitlement and runs the host test suite with the HVF backend (macOS/aarch64).
# Guest binaries must already be built (see build-rust-guests).
# NOTE: the binaries are executed directly, not via `cargo test` — codesigning
# modifies them, which cargo would treat as a rebuild trigger, discarding the
# signature (and unsigned binaries get HV_DENIED from Hypervisor.framework).
test-hvf target=default-target:
#!/usr/bin/env bash
set -euo pipefail
profile={{ if target == "debug" { "dev" } else { target } }}
out=$({{ cargo-cmd }} test -p hyperlight-host --no-default-features -F hvf --profile=$profile --no-run 2>&1)
bins=$(echo "$out" | sed -n 's/^ *Executable .*(\(.*\))$/\1/p')
for bin in $bins; do
codesign --sign - --entitlements {{ justfile_directory() }}/dev/hvf-entitlements.plist --force "$bin"
echo "=== $bin"
"$bin"
done

# runs integration tests
test-integration target=default-target features="":
@# run execute_on_heap test with feature "executable_heap" on (runs with off during normal tests)
Expand Down
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@

Hyperlight lets you safely run untrusted code inside hypervisor-isolated micro VMs that spin up in milliseconds, with guest function calls completing in microseconds. You embed it as a library in your Rust application, hand it a guest binary, and call functions across the VM boundary as naturally as calling a local function. To minimize startup time and memory footprint, there's no guest kernel or OS. Guests are purpose-built using the Hyperlight guest library.

- Supports [KVM](https://linux-kvm.org/page/Main_Page), [MSHV](https://github.com/rust-vmm/mshv), and [Windows Hypervisor Platform](https://docs.microsoft.com/en-us/virtualization/api/#windows-hypervisor-platform)
- Supports [KVM](https://linux-kvm.org/page/Main_Page), [MSHV](https://github.com/rust-vmm/mshv), [Windows Hypervisor Platform](https://docs.microsoft.com/en-us/virtualization/api/#windows-hypervisor-platform), and [Hypervisor.framework](https://developer.apple.com/documentation/hypervisor) (macOS on Apple silicon)
- No kernel or OS in the VM. Guests are regular ELF binaries written in `no_std` Rust or C
- Host and guest communicate through typed function calls
- Guests are sandboxed by default with no access to the host filesystem, network, etc.
Expand Down
8 changes: 8 additions & 0 deletions dev/hvf-entitlements.plist
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>com.apple.security.hypervisor</key>
<true/>
</dict>
</plist>
60 changes: 60 additions & 0 deletions docs/getting-started.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ These are the minimum requirements to use Hyperlight as a library in your own pr
- **A supported hypervisor:**
- Linux: [KVM](https://help.ubuntu.com/community/KVM/Installation) or Microsoft Hypervisor (MSHV)
- Windows: [Windows Hypervisor Platform](https://docs.microsoft.com/en-us/virtualization/api/#windows-hypervisor-platform) (WHP)
- macOS (Apple silicon): [Hypervisor.framework](https://developer.apple.com/documentation/hypervisor) (HVF)
- **[Rust](https://www.rust-lang.org/tools/install)**, installed via `rustup`.
- **Platform build tools** (provides the C linker required by Rust):
- Ubuntu/Debian: `sudo apt install build-essential`
Expand Down Expand Up @@ -93,6 +94,65 @@ Requires Windows 11 Pro/Enterprise/Education or Windows Server 2025 or later.
sudo dnf install clang
```

### macOS

Requires macOS 26 or later on Apple silicon. Hyperlight uses
[Hypervisor.framework](https://developer.apple.com/documentation/hypervisor) (HVF)
via the `hvf` cargo feature of `hyperlight-host`, which is not enabled by default:

```toml
hyperlight-host = { version = "0.16.0", features = ["hvf"] }
```

Two macOS-specific things to know:

1. **Entitlement.** Every executable that creates a VM — your application, and
Hyperlight's per-sandbox surrogate processes — must be signed with the
`com.apple.security.hypervisor` entitlement. Ad-hoc signing is sufficient for
development. Save this as `hvf-entitlements.plist`:

```xml
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>com.apple.security.hypervisor</key>
<true/>
</dict>
</plist>
```

and sign your binary after every build:

```sh
codesign --sign - --entitlements hvf-entitlements.plist --force target/debug/your-app
```

Hyperlight's surrogate binaries are extracted and ad-hoc signed automatically
at runtime; only your own executables need manual signing. Without the
entitlement, sandbox creation fails with `NoHypervisorFound`.

2. **One VM per process.** Hypervisor.framework allows only a single VM per
process, so Hyperlight runs each sandbox's VM in a small surrogate helper
process (see [Hyperlight Surrogate](./hyperlight-surrogate-development-notes.md)).
This is transparent in normal use. Setting `HYPERLIGHT_MAX_SURROGATES=0`
disables surrogates and selects the in-process backend, which limits the
process to one live sandbox at a time.

If building guest binaries on macOS:

- Use `rustup` for your Rust installation — guest builds require the toolchain
pinned in `rust-toolchain.toml`, and distributions that shadow it (e.g.
Homebrew Rust) do not work.
- Install LLVM (`brew install llvm`); Xcode's `ar` cannot archive the ELF
objects used when linking C guests.
- Until a `cargo-hyperlight` release newer than 0.1.13 is published, install it
from git (0.1.13 does not compile on macOS):

```sh
cargo install --locked --git https://github.com/hyperlight-dev/cargo-hyperlight cargo-hyperlight
```

### WSL2

Follow the Ubuntu/Debian instructions above inside your WSL2 instance. WSL2 uses KVM.
Expand Down
11 changes: 11 additions & 0 deletions docs/hyperlight-surrogate-development-notes.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,3 +9,14 @@ These surrogate processes are managed by the host via the [surrogate_process_man
> **Note:** `HYPERLIGHT_MAX_SURROGATES` is authoritative — if `HYPERLIGHT_INITIAL_SURROGATES` exceeds it, the initial count is silently clamped down to the maximum. For example, setting only `HYPERLIGHT_MAX_SURROGATES=256` limits both the initial pool and the ceiling to 256.

`hyperlight_surrogate.exe` gets built during `hyperlight-host`'s build script, gets embedded into the `hyperlight-host` Rust library via [rust-embed](https://crates.io/crates/rust-embed), and is extracted at runtime next to the executable when the surrogate process manager is initialized. The extracted filename includes a short BLAKE3 hash of the binary content (e.g., `hyperlight_surrogate_a1b2c3d4.exe`) so that multiple hyperlight versions can coexist without file-deletion races.

### HVF surrogate (macOS)

On macOS with Hypervisor.framework (HVF) the constraint is similar — only one VM per process — but the implementation is fundamentally different. HVF binds a VM to the process that created it and offers no cross-process mapping API like `WHvMapGpaRange2`, so `hvf_surrogate` is a real server process rather than an empty suspended one:

- Each sandbox's VM and vCPU are created, run, and destroyed inside a pooled `hvf_surrogate` process; the host is an IPC client (length-prefixed JSON frames over an inherited unix socket pair; the protocol is defined in the `hyperlight-hvf` crate's `proto` module).
- Guest memory is never copied: the host allocates it from named POSIX shm objects (or file-backed regions), and the surrogate maps the same objects into its own address space and then into the guest.
- The surrogate's main thread is the vCPU thread — HVF requires vCPU creation, register access, and `hv_vcpu_run` to happen on a single thread. A reader thread dispatches requests and answers `Cancel` immediately via `hv_vcpus_exit` (the only HVF call allowed from a non-owning thread); a writer thread serializes all responses.
- The extracted surrogate binary is ad-hoc codesigned with the `com.apple.security.hypervisor` entitlement at extraction time, since Hypervisor.framework refuses to create VMs from unentitled binaries.

The pool is managed by `hvf_surrogate_manager` and honors the same `HYPERLIGHT_INITIAL_SURROGATES` / `HYPERLIGHT_MAX_SURROGATES` environment variables (defaulting to 0 initial / 128 max — HVF currently allows 127 concurrent VMs system-wide, so exceeding the pool fails fast rather than surprising the guest). Setting `HYPERLIGHT_MAX_SURROGATES=0` disables surrogates and selects the direct in-process backend, which limits the process to a single live sandbox at a time.
2 changes: 1 addition & 1 deletion hack/clippy-package-features.sh
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ fi

EXCLUDED_FEATURES=("${REQUIRED_FEATURES[@]}")

if [[ -n "${TARGET_TRIPLE}" && "$TARGET_TRIPLE" =~ aarch64 ]] || [[ -z "${TARGET_TRIPLE}" && "$(uname -m)" = "aarch64" ]]; then
if [[ -n "${TARGET_TRIPLE}" && "$TARGET_TRIPLE" =~ aarch64 ]] || [[ -z "${TARGET_TRIPLE}" && "$(uname -m)" =~ ^(aarch64|arm64)$ ]]; then
EXCLUDED_FEATURES+=("crashdump" "trace_guest" "mem_profile" "hw-interrupts" "gdb" "i686-guest" "nanvix-unstable")
fi

Expand Down
8 changes: 7 additions & 1 deletion src/hyperlight_host/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -78,12 +78,17 @@ rust-embed = { version = "8.11.0", features = ["debug-embed", "include-exclude",
windows-version = "0.1"
lazy_static = "1.4.0"

[target.'cfg(unix)'.dependencies]
[target.'cfg(target_os = "linux")'.dependencies]
kvm-bindings = { version = "0.14", features = ["fam-wrappers"], optional = true }
kvm-ioctls = { version = "0.25", optional = true }
mshv-bindings = { version = "0.6", optional = true }
mshv-ioctls = { version = "0.6", optional = true}

[target.'cfg(all(target_os = "macos", target_arch = "aarch64"))'.dependencies]
applevisor-sys = { version = "1.0", optional = true }
hyperlight-hvf = { workspace = true, optional = true }
rust-embed = { version = "8.11.0", optional = true, features = ["debug-embed", "include-exclude", "interpolate-folder-path"] }

[dev-dependencies]
uuid = { version = "1.23.3", features = ["v4"] }
signal-hook-registry = "1.4.8"
Expand Down Expand Up @@ -136,6 +141,7 @@ trace_guest = ["dep:opentelemetry", "dep:tracing-opentelemetry", "dep:hyperlight
mem_profile = [ "trace_guest", "dep:framehop", "dep:fallible-iterator", "hyperlight-common/mem_profile" ]
kvm = ["dep:kvm-bindings", "dep:kvm-ioctls"]
mshv3 = ["dep:mshv-bindings", "dep:mshv-ioctls"]
hvf = ["dep:applevisor-sys", "dep:hyperlight-hvf", "dep:rust-embed"]
hw-interrupts = []
# This enables easy debug in the guest
gdb = ["dep:gdbstub", "dep:gdbstub_arch"]
Expand Down
Loading
Loading