Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -141,8 +141,9 @@ A delivered sandbox also takes four action verbs, served as subresources so the
standard `agents.x-k8s.io` schema stays untouched: `pause`, `resume`, `fork`,
`snapshot`. They are not uniformly fast — `resume` and `fork` are node-local,
while `pause`/`snapshot` cost time proportional to guest memory. Verb table,
a runnable walk-through over both surfaces, and the two consistency behaviors
callers must handle are in [docs/lifecycle.md](docs/lifecycle.md).
a runnable walk-through over both surfaces, how a sandbox's lifetime is set
and reported, and the two consistency behaviors callers must handle are in
[docs/lifecycle.md](docs/lifecycle.md).

## Use it with the e2b SDK

Expand Down
16 changes: 16 additions & 0 deletions docs/lifecycle.md
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,22 @@ prints what it did, so the output doubles as acceptance evidence:
connect 201 — restored via the mmap fast path
```

## Lifetime

Nothing is stored for a claimed sandbox, so the lease is fixed by the node at
claim time and the submitted object is the only place `Create` can hear it:

- `spec.shutdownTime` wins, rounded up to whole seconds; the
`sandbox.cocoonstack.io/ttl-seconds` annotation covers clients that cannot
set the field; neither (or `0`) asks for the node's default lease.
- A `shutdownTime` already in the past or a malformed/negative annotation is
a `400` before any warm microVM is spent.
- The node clamps the ask to its own default and maximum, so the response
carries the **granted** expiry as the `sandbox.cocoonstack.io/deadline`
annotation (RFC3339) — the submitted spec is echoed untouched. `Get`/`List`
stamp the same annotation once the owning node publishes the deadline in
its `NodeInventory`.

## Two behaviors callers must handle

- **Reads are eventually consistent.** `Create` returns as soon as the
Expand Down
2 changes: 1 addition & 1 deletion docs/scaling-design.md
Original file line number Diff line number Diff line change
Expand Up @@ -177,7 +177,7 @@ type NodeInventory struct {
metav1.TypeMeta `json:",inline"`
metav1.ObjectMeta `json:"metadata,omitempty"`
Node string `json:"node"`
Entries []InventoryEntry `json:"entries"` // {name, phase, claimRef, addr}
Entries []InventoryEntry `json:"entries"` // {name, id, phase, claimRef, addr, deadline}
}
```

Expand Down
3 changes: 3 additions & 0 deletions extensions/api/v1beta1/nodeinventory_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,9 @@ type InventoryEntry struct {
// addr is the sandbox "host:port" address, if published.
// +optional
Address string `json:"addr,omitempty"`
// deadline is the node-granted lease expiry, if published.
// +optional
Deadline *metav1.Time `json:"deadline,omitempty"`
}

// +kubebuilder:object:root=true
Expand Down
8 changes: 7 additions & 1 deletion extensions/api/v1beta1/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 3 additions & 0 deletions helm/crds/extensions.agents.x-k8s.io_nodeinventories.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,9 @@ spec:
type: string
claimRef:
type: string
deadline:
format: date-time
type: string
id:
type: string
name:
Expand Down
3 changes: 3 additions & 0 deletions k8s/crds/extensions.agents.x-k8s.io_nodeinventories.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,9 @@ spec:
type: string
claimRef:
type: string
deadline:
format: date-time
type: string
id:
type: string
name:
Expand Down
44 changes: 41 additions & 3 deletions pkg/scale/apiserver/storage.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,9 @@ package apiserver
import (
"context"
"fmt"
"math"
"strconv"
"time"

apierrors "k8s.io/apimachinery/pkg/api/errors"
apimeta "k8s.io/apimachinery/pkg/api/meta"
Expand All @@ -28,13 +31,19 @@ const (
// defined in package scale (whose store stamps it on synthesized reads); this
// alias keeps apiserver call sites writing and reading the identical key.
ClaimIDAnnotation = scale.ClaimIDAnnotation
// DeadlineAnnotation carries the node-granted lease expiry (RFC3339); the
// same scale-defined key the synthesized read path stamps from inventory.
DeadlineAnnotation = scale.DeadlineAnnotation
// AddressAnnotation carries the delivered sandbox connection address.
AddressAnnotation = "sandbox.cocoonstack.io/address"
// TokenAnnotation carries the per-sandbox ownership token so a caller can
// exec/agent into the sandbox it claimed via the L3 apiserver.
TokenAnnotation = "sandbox.cocoonstack.io/token"
// NetAnnotation selects the pool network mode on Create (default "none").
NetAnnotation = "sandbox.cocoonstack.io/net"
// TTLSecondsAnnotation bounds the claim lease in whole seconds on Create for
// clients that cannot set spec.shutdownTime (0 = the owning node's default).
TTLSecondsAnnotation = "sandbox.cocoonstack.io/ttl-seconds"
)

// The verb set an aggregated, scatter-gather resource implements: the read triad
Expand Down Expand Up @@ -122,7 +131,11 @@ func (r *sandboxREST) Create(ctx context.Context, obj runtime.Object, createVali
}

pool := poolKeyForSandbox(sb)
assignment, err := r.store.Claim(ctx, namespace, name, pool, 0)
ttlSeconds, err := ttlSecondsForSandbox(sb, time.Now())
if err != nil {
return nil, apierrors.NewBadRequest(err.Error())
}
assignment, err := r.store.Claim(ctx, namespace, name, pool, ttlSeconds)
if err != nil {
if scale.IsNoWarmCapacity(err) {
// Retryable: warm capacity refills asynchronously (the node's sandboxd
Expand Down Expand Up @@ -202,10 +215,32 @@ func poolKeyForSandbox(sb *sandboxv1beta1.Sandbox) scale.PoolKey {
return scale.PoolKeyFor(sb.Spec.PodTemplate.Spec.Containers, sb.Annotations[NetAnnotation])
}

// ttlSecondsForSandbox derives the claim lease: spec.shutdownTime wins, the
// ttl-seconds annotation is the fallback, 0 asks for the node default.
func ttlSecondsForSandbox(sb *sandboxv1beta1.Sandbox, now time.Time) (int, error) {
if t := sb.Spec.ShutdownTime; t != nil {
left := t.Sub(now)
if left <= 0 {
return 0, fmt.Errorf("spec.shutdownTime %s is not in the future", t.Format(time.RFC3339))
}
return int(math.Ceil(left.Seconds())), nil
}
raw := sb.Annotations[TTLSecondsAnnotation]
if raw == "" {
return 0, nil
}
v, err := strconv.Atoi(raw)
if err != nil || v < 0 {
return 0, fmt.Errorf("invalid %s=%q: want a non-negative integer of seconds", TTLSecondsAnnotation, raw)
}
return v, nil
}

// synthesizeClaimedSandbox builds the Sandbox object Create returns: the submitted
// spec echoed back under the request name/namespace, a fresh UID/creationTimestamp,
// the claim id + address annotations, and a Ready status pointing at the owning
// node. It is never persisted — it is the response for a node-local claim.
// the claim id + address + granted-deadline annotations, and a Ready status
// pointing at the owning node. It is never persisted — it is the response for a
// node-local claim.
func synthesizeClaimedSandbox(namespace, name string, in *sandboxv1beta1.Sandbox, a scale.Assignment) *sandboxv1beta1.Sandbox {
out := in.DeepCopy()
out.Namespace = namespace
Expand All @@ -224,6 +259,9 @@ func synthesizeClaimedSandbox(namespace, name string, in *sandboxv1beta1.Sandbox
if a.Token != "" {
out.Annotations[TokenAnnotation] = a.Token
}
if !a.Deadline.IsZero() {
out.Annotations[DeadlineAnnotation] = a.Deadline.UTC().Format(time.RFC3339)
}
out.Status = sandboxv1beta1.SandboxStatus{
NodeName: a.Node,
PodIPs: scale.AddressIPs(a.Address),
Expand Down
97 changes: 90 additions & 7 deletions pkg/scale/apiserver/storage_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,11 @@ package apiserver
import (
"context"
"testing"
"time"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/watch"
Expand All @@ -30,7 +32,7 @@ func TestDelete_ReleasesByClaimIDAnnotation(t *testing.T) {
store := &fakeStore{getSandbox: sb}
r := NewSandboxREST(store).(*sandboxREST)

obj, ok, err := r.Delete(deleteCtx(t, "ns"), "s1", nil, &metav1.DeleteOptions{})
obj, ok, err := r.Delete(nsCtx(t, "ns"), "s1", nil, &metav1.DeleteOptions{})
require.NoError(t, err)
assert.True(t, ok)
assert.NotNil(t, obj)
Expand All @@ -50,19 +52,92 @@ func TestDelete_FailsLoudWithoutClaimID(t *testing.T) {
store := &fakeStore{getSandbox: sb}
r := NewSandboxREST(store).(*sandboxREST)

_, ok, err := r.Delete(deleteCtx(t, "ns"), "s1", nil, &metav1.DeleteOptions{})
_, ok, err := r.Delete(nsCtx(t, "ns"), "s1", nil, &metav1.DeleteOptions{})
require.Error(t, err)
assert.False(t, ok)
assert.True(t, apierrors.IsInternalError(err), "expected an internal error, got %v", err)
assert.False(t, store.released, "must not release when the sandboxd claim id is unknown")
}

// fakeStore is a scale.SandboxStore stub for the Delete path: Get returns a
// preset sandbox, Release records its arguments. The read-path verbs are unused.
func TestTTLSecondsForSandbox(t *testing.T) {
now := time.Date(2026, 8, 17, 10, 0, 0, 0, time.UTC)
for name, tc := range map[string]struct {
shutdown time.Time
ttl string
want int
wantErr bool
}{
"shutdownTime": {shutdown: now.Add(90 * time.Second), want: 90},
"sub-second rounds up": {shutdown: now.Add(90*time.Second + 500*time.Millisecond), want: 91},
"annotation": {ttl: "120", want: 120},
"spec wins over annotation": {shutdown: now.Add(time.Hour), ttl: "10", want: 3600},
"explicit zero asks the node default": {ttl: "0"},
"no lifetime asks the node default": {},
"expired shutdownTime": {shutdown: now, wantErr: true},
"malformed annotation": {ttl: "banana", wantErr: true},
"negative annotation": {ttl: "-5", wantErr: true},
} {
t.Run(name, func(t *testing.T) {
sb := submittedSandbox("s", nil)
if tc.ttl != "" {
sb.Annotations = map[string]string{TTLSecondsAnnotation: tc.ttl}
}
if !tc.shutdown.IsZero() {
sb.Spec.ShutdownTime = &metav1.Time{Time: tc.shutdown}
}
got, err := ttlSecondsForSandbox(sb, now)
if tc.wantErr {
require.Error(t, err)
return
}
require.NoError(t, err)
assert.Equal(t, tc.want, got)
})
}
}

func TestCreate_TTLRidesTheClaim(t *testing.T) {
f := &fakeStore{claimAssign: scale.Assignment{SandboxName: "sb_1", Node: "n1"}}
r := NewSandboxREST(f).(*sandboxREST)

_, err := r.Create(nsCtx(t, "ns"), submittedSandbox("s1", map[string]string{TTLSecondsAnnotation: "120"}), nil, nil)
require.NoError(t, err)
assert.Equal(t, 120, f.claimTTL, "the derived lease must reach the store")
}

func TestCreate_RejectsUnusableLifetime(t *testing.T) {
f := &fakeStore{}
r := NewSandboxREST(f).(*sandboxREST)

_, err := r.Create(nsCtx(t, "ns"), submittedSandbox("s1", map[string]string{TTLSecondsAnnotation: "banana"}), nil, nil)
require.Error(t, err)
assert.True(t, apierrors.IsBadRequest(err), "expected BadRequest, got %v", err)
assert.Equal(t, 0, f.claimCalls, "no claim may be spent on a rejected request")
}

func TestCreate_ReportsGrantedDeadline(t *testing.T) {
deadline := time.Date(2026, 8, 17, 12, 0, 0, 0, time.UTC)
f := &fakeStore{claimAssign: scale.Assignment{SandboxName: "sb_1", Node: "n1", Deadline: deadline}}
r := NewSandboxREST(f).(*sandboxREST)

obj, err := r.Create(nsCtx(t, "ns"), submittedSandbox("s1", map[string]string{TTLSecondsAnnotation: "999999"}), nil, nil)
require.NoError(t, err)
out, ok := obj.(*sandboxv1beta1.Sandbox)
require.True(t, ok)
assert.Equal(t, "2026-08-17T12:00:00Z", out.Annotations[DeadlineAnnotation], "the node-granted deadline, not the caller's ask")
assert.Nil(t, out.Spec.ShutdownTime, "the submitted spec is echoed, not rewritten")
}

// fakeStore is a scale.SandboxStore stub: Get returns a preset sandbox, Claim
// and Release record their arguments. The read-path verbs are unused.
type fakeStore struct {
getSandbox *sandboxv1beta1.Sandbox
getErr error

claimCalls int
claimTTL int
claimAssign scale.Assignment

released bool
releaseNode string
releaseID string
Expand All @@ -81,8 +156,10 @@ func (f *fakeStore) Watch(context.Context, scale.ListOptions) (watch.Interface,
return watch.NewFake(), nil
}

func (f *fakeStore) Claim(context.Context, string, string, scale.PoolKey, int) (scale.Assignment, error) {
return scale.Assignment{}, nil
func (f *fakeStore) Claim(_ context.Context, _, _ string, _ scale.PoolKey, ttlSeconds int) (scale.Assignment, error) {
f.claimCalls++
f.claimTTL = ttlSeconds
return f.claimAssign, nil
}

func (f *fakeStore) Release(_ context.Context, node, id string) error {
Expand Down Expand Up @@ -120,6 +197,12 @@ func (f *fakeStore) Stats(context.Context, string, string) (scale.SandboxStats,
return scale.SandboxStats{}, nil
}

func deleteCtx(t *testing.T, ns string) context.Context {
func nsCtx(t *testing.T, ns string) context.Context {
return genericapirequest.WithNamespace(t.Context(), ns)
}

func submittedSandbox(name string, anns map[string]string) *sandboxv1beta1.Sandbox {
sb := &sandboxv1beta1.Sandbox{ObjectMeta: metav1.ObjectMeta{Name: name, Annotations: anns}}
sb.Spec.PodTemplate.Spec.Containers = []corev1.Container{{Name: "c", Image: "img"}}
return sb
}
8 changes: 7 additions & 1 deletion pkg/scale/claimgateway.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
package scale

import "context"
import (
"context"
"time"
)

// ClaimRequest identifies a node-local warm-pool claim.
type ClaimRequest struct {
Expand All @@ -23,6 +26,9 @@ type Assignment struct {
// claim. It authenticates agent/exec against the delivered VM; the L3 Create
// path surfaces it as an annotation so a caller can exec into what it claimed.
Token string
// Deadline is the node-granted lease expiry — authoritative over the requested
// TTL (node default when unasked, clamped to the node maximum); zero if unreported.
Deadline time.Time
}

// ClaimGateway is the L2 node-local fast path for warm-pool claims. A claim is
Expand Down
2 changes: 1 addition & 1 deletion pkg/scale/claimgateway_impl.go
Original file line number Diff line number Diff line change
Expand Up @@ -187,7 +187,7 @@ func (g *nodeClaimGateway) Claim(ctx context.Context, req ClaimRequest) (Assignm
return Assignment{}, fmt.Errorf("scale: sandboxd claim for %s/%s: %w", req.Namespace, req.ClaimName, err)
}

a := Assignment{SandboxName: res.ID, Node: g.node, Address: res.OwnerAddr, Token: res.Token}
a := Assignment{SandboxName: res.ID, Node: g.node, Address: res.OwnerAddr, Token: res.Token, Deadline: res.Deadline}
g.mu.Lock()
g.holdings[a.SandboxName] = delivered{id: res.ID, token: res.Token}
g.mu.Unlock()
Expand Down
5 changes: 4 additions & 1 deletion pkg/scale/claimgateway_impl_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"strings"
"sync/atomic"
"testing"
"time"

"github.com/go-logr/logr/testr"
"github.com/stretchr/testify/require"
Expand Down Expand Up @@ -227,7 +228,9 @@ func newFakeSandboxd(t *testing.T) *fakeSandboxd {
id := fmt.Sprintf("sb_%d", f.nextID.Add(1))
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(sandboxd.ClaimResult{
ID: id, Token: "tok_" + id, Deadline: "2026-07-06T00:05:00Z", OwnerAddr: "10.0.0.5:7777",
ID: id, Token: "tok_" + id,
Deadline: time.Date(2026, 7, 6, 0, 5, 0, 0, time.UTC),
OwnerAddr: "10.0.0.5:7777",
})
})
mux.HandleFunc("/v1/sandboxes/", func(w http.ResponseWriter, r *http.Request) {
Expand Down
9 changes: 5 additions & 4 deletions pkg/scale/sandboxd/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import (
"net/http"
"net/url"
"strings"
"time"
)

// ErrNodeAtCapacity is returned by Claim when sandboxd answers 429 (the node is
Expand Down Expand Up @@ -69,10 +70,10 @@ type ClaimSpec struct {

// ClaimResult is the POST /v1/claim success body.
type ClaimResult struct {
ID string `json:"id"`
Token string `json:"token"`
Deadline string `json:"deadline"`
OwnerAddr string `json:"owner_addr"`
ID string `json:"id"`
Token string `json:"token"`
Deadline time.Time `json:"deadline"`
OwnerAddr string `json:"owner_addr"`
// FromCheckpoint is the lineage edge when the claim branched from a checkpoint.
FromCheckpoint string `json:"from_checkpoint,omitempty"`
// Redirect, when non-empty on a 200, names warm peers to retry at instead of a
Expand Down
Loading