Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions .github/workflows/api-acceptance.yml
Original file line number Diff line number Diff line change
Expand Up @@ -63,9 +63,8 @@ jobs:
OAC_TEST_OFFICIAL_SDK_PYTHON: python
run: |
python services/core/tests/official_schema_test.py
# The Go Workers need an unclaimed deployment; Core claims it for its installation ID.
go test ./services/core/tests/integration -run '^(TestFunctionStateOfficialClientReadsAndLiveEvents|TestSavedReferenceRetryOfficialClient|TestAgentUpdateOfficialClient|TestAgentDeletionOfficialClient|TestSessionAgentFilterOfficialClient|TestSessionDeletionOfficialClient|TestEnvironmentInitialFailureOfficialClient|TestSelfHostedInitialCreationOfficialClient|TestSelfHostedCancellationOfficialClient)$' -count=1
python services/core/tests/official_client.py
go test ./services/core/tests/integration -run '^(TestFunctionStateOfficialClientReadsAndLiveEvents|TestSavedReferenceRetryOfficialClient|TestAgentUpdateOfficialClient|TestAgentDeletionOfficialClient|TestSessionAgentFilterOfficialClient|TestSessionDeletionOfficialClient|TestEnvironmentInitialFailureOfficialClient|TestSelfHostedInitialCreationOfficialClient|TestSelfHostedCancellationOfficialClient)$' -count=1
- uses: ./.github/actions/e2b-provider
if: inputs.container
- name: Verify the distribution's Core image
Expand Down
2 changes: 1 addition & 1 deletion CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ The [CI selection policy](docs/maintainers.md#continuous-integration) names affe
| `OAC_TEST_DATABASE_URL` | A dedicated test database. The full gate fails when it is missing. |
| `OAC_TEST_OFFICIAL_SDK_PYTHON` | The pinned official SDK interpreter |

The role needs `CREATE DATABASE`: tests of database-wide state, such as the execution lease and the provider identity, create and drop isolated `oac_*_tests` databases. Tests must not bypass the production provider-switch guard.
The role needs `CREATE DATABASE`: tests of database-wide state, such as the execution lease and the provider identity, create and drop isolated `oac_*_tests` databases. They copy them from a migrated template, the test database's name with `_template` before `_tests`, which stays beside it. Tests must not bypass the production provider-switch guard.

### Contract and schema rules

Expand Down
4 changes: 2 additions & 2 deletions apps/web/src/features/metrics/CoreMetricsPage.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@ function CoreMetricsBody({ metrics }: { metrics: CoreMetrics }) {
const databaseBuckets = useMemo(() => database.series.map((entry) => seconds(entry.start) ?? 0), [database.series]);
const count = (value: number | null) => (value === null ? MISSING : formatInteger(value, locale));
const integer = (value: number) => formatInteger(value, locale);
const slotsFull = execution.slots_in_use !== null && execution.slots_total !== null && execution.slots_in_use >= execution.slots_total;
const slotsFull = execution.slots_in_use >= execution.slots_total;
// Sandbox nodes hold their own connection to Core, as daemons do; E2B deployments have none.
const { state: fleetState, refresh: refreshFleet } = useSandboxFleet({ poll: true });
const fleet = fleetSnapshot(fleetState);
Expand All @@ -136,7 +136,7 @@ function CoreMetricsBody({ metrics }: { metrics: CoreMetrics }) {
<Kpi
label={t("core.slots")}
help={t("core.slotsHelp")}
value={execution.slots_in_use === null ? MISSING : <Figure value={<LiveNumber value={execution.slots_in_use} />} unit={execution.slots_total === null ? undefined : `/ ${integer(execution.slots_total)}`} />}
value={<Figure value={<LiveNumber value={execution.slots_in_use} />} unit={`/ ${integer(execution.slots_total)}`} />}
tone={slotsFull ? "warning" : undefined}
/>
<Kpi
Expand Down
2 changes: 1 addition & 1 deletion apps/web/src/features/overview/FleetOverview.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -189,7 +189,7 @@ function CoreGlance({ label, tone }: { label: string; tone: Tone }) {
<Fact label={t("fleet.facts.uptime")}>{started === null ? MISSING : formatDuration(Math.max(0, now - started))}</Fact>
<Fact label={t("fleet.facts.slots")}>
{count(metrics.execution.slots_in_use)}
{metrics.execution.slots_total === null ? null : <span className="kpi-unit">/ {count(metrics.execution.slots_total)}</span>}
<span className="kpi-unit">/ {count(metrics.execution.slots_total)}</span>
</Fact>
<Fact label={t("fleet.facts.queued")}>{count(metrics.execution.queued_turns)}</Fact>
<Fact label={t("fleet.facts.daemons")}>{count(metrics.execution.connected_daemons)}</Fact>
Expand Down
2 changes: 1 addition & 1 deletion apps/web/src/i18n/locales/en/core-errors.ts
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ export const coreErrors = {
"sandbox_operation_unsupported": "The selected sandbox provider does not support this operation.",
"environment_unavailable": "The Session's environment is no longer available.",
"execution_unavailable": "Execution is temporarily unavailable. Try again later.",
"runtime_history_unavailable": "Runtime history is unavailable on this Core.",
"runtime_history_unavailable": "Runtime history is temporarily unavailable. Try again later.",
"runtime_history_unsupported": "Runtime history is not supported for this Session.",
"core_metrics_unavailable": "Core metrics could not be read. Try again later.",
"file_transfer_unavailable": "The file transfer is unavailable. Try again later.",
Expand Down
1 change: 0 additions & 1 deletion apps/web/src/i18n/locales/en/metrics.ts
Original file line number Diff line number Diff line change
Expand Up @@ -310,7 +310,6 @@ export const metrics = {
reason: {
allocation_pending: "Allocation pending",
runtime_not_running: "Not running",
source_not_configured: "Source unavailable",
sample_timeout: "Sample timed out",
sample_unavailable: "Sample unavailable",
},
Expand Down
1 change: 0 additions & 1 deletion apps/web/src/i18n/locales/en/sessions.ts
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,6 @@ export const sessions = {
runtime_mode_not_observable: "Not observable",
allocation_pending: "Allocation pending",
runtime_not_running: "Not running",
source_not_configured: "Metrics source not configured",
sample_timeout: "Sample timed out",
sample_unavailable: "Sample unavailable",
},
Expand Down
2 changes: 1 addition & 1 deletion apps/web/src/i18n/locales/zh-CN/core-errors.ts
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ export const coreErrors = {
"sandbox_operation_unsupported": "所选沙箱提供商不支持此操作。",
"environment_unavailable": "此 Session 的环境已不可用。",
"execution_unavailable": "执行暂时不可用,请稍后重试。",
"runtime_history_unavailable": "此 Core 上的 Runtime 历史不可用。",
"runtime_history_unavailable": "Runtime 历史暂时不可用,请稍后重试。",
"runtime_history_unsupported": "此 Session 不支持 Runtime 历史。",
"core_metrics_unavailable": "无法读取 Core 指标,请稍后重试。",
"file_transfer_unavailable": "文件传输不可用,请稍后重试。",
Expand Down
1 change: 0 additions & 1 deletion apps/web/src/i18n/locales/zh-CN/metrics.ts
Original file line number Diff line number Diff line change
Expand Up @@ -310,7 +310,6 @@ export const metrics = {
reason: {
allocation_pending: "等待分配",
runtime_not_running: "未运行",
source_not_configured: "数据源不可用",
sample_timeout: "采样超时",
sample_unavailable: "采样不可用",
},
Expand Down
1 change: 0 additions & 1 deletion apps/web/src/i18n/locales/zh-CN/sessions.ts
Original file line number Diff line number Diff line change
Expand Up @@ -135,7 +135,6 @@ export const sessions = {
runtime_mode_not_observable: "无法观测",
allocation_pending: "等待分配",
runtime_not_running: "未运行",
source_not_configured: "未配置指标来源",
sample_timeout: "采样超时",
sample_unavailable: "采样不可用",
},
Expand Down
2 changes: 1 addition & 1 deletion contracts/agents-api/core-errors.md
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ These codes have null `param` and no `details`. [Sandbox deployment](./sandbox-d
| 500 | `internal_error` | Core could not complete the operation |
| 503 | `runtime_node_unavailable` | No sandbox node is available or has capacity |
| 503 | `execution_unavailable` | Execution is not available, such as while Core shuts down |
| 503 | `runtime_history_unavailable` | Durable Runtime history is not configured or temporarily unavailable |
| 503 | `runtime_history_unavailable` | Durable Runtime history is temporarily unavailable |
| 503 | `core_metrics_unavailable` | Core metrics could not be read |
| 503 | `file_transfer_unavailable` | Bounded content transfer is unavailable |

Expand Down
10 changes: 5 additions & 5 deletions contracts/agents-api/core-metrics.md
Original file line number Diff line number Diff line change
Expand Up @@ -25,19 +25,19 @@ title: "Core operational metrics"

## Fields

The response has `object: "core.metrics"`, `range`, `service`, `execution`, `database`, `jobs` and `process`. Every numeric value and `service.execution_owner` is nullable; each `series` always lists every complete bucket of the range.
The response has `object: "core.metrics"`, `range`, `service`, `execution`, `database`, `jobs` and `process`. Every numeric value except `execution.slots_in_use`, `execution.slots_total` and `execution.connected_daemons` is nullable, as is `service.execution_owner`; each `series` always lists every complete bucket of the range.

| Field | Meaning |
| --- | --- |
| `service.status` | `running`, or `degraded` when a measurement or job fails, the latest sample is missing or stale, or execution ownership is unknown or Core has execution slots but does not hold the execution lease. A sandbox reset is reported by the [deployment](./sandbox-deployment.md), not here |
| `service.status` | `running`, or `degraded` when a measurement or job fails, the latest sample is missing or stale, or execution ownership is unknown or Core does not hold the execution lease. A sandbox reset is reported by the [deployment](./sandbox-deployment.md), not here |
| `service.revision` | The full source commit injected at build time; null for builds without one |
| `service.started_at` | When the process initialized |
| `service.execution_owner` | Whether this process holds the execution worker's database lease |
| `execution.slots_in_use`, `execution.slots_total` | Active Session reservations of the execution worker, and its capacity: [`core.execution_concurrency`](../../docs/configuration.md#settings), 4 by default. Environment input, Turns and file work share the slots; native Harness subprocesses are not counted. Without a worker both are 0 |
| `execution.slots_in_use`, `execution.slots_total` | Active Session reservations of the execution worker, and its capacity: [`core.execution_concurrency`](../../docs/configuration.md#settings), 4 by default. Environment input, Turns and file work share the slots; native Harness subprocesses are not counted |
| `execution.queued_turns`, `execution.in_progress_turns` | Root Turns in those states, including Turns of deleted Sessions. Subagent Turns and input reserved for a preparing Environment are not counted |
| `execution.waiting_for_daemon` | Queued Turns whose Session's device is not connected; null without a gateway |
| `execution.waiting_for_daemon` | Queued Turns whose Session's device is not connected |
| `execution.oldest_queued_seconds` | Age of the oldest queued Turn, from its `created_at` |
| `execution.connected_daemons` | Runtime daemons connected to Core's gateway; null without a gateway |
| `execution.connected_daemons` | Runtime daemons connected to Core's gateway |
| `execution.queue_wait_ms` | p50 and p95 of `started_at - created_at` for Turns started in the interval, by PostgreSQL `percentile_cont` |
| `execution.interrupted` | Failed Turns with error code `execution_interrupted`, by `completed_at` in the interval |
| `execution.unavailable` | HTTP responses sent with error code `execution_unavailable`, counted once each. Other 503 codes and errors after a stream started are not counted |
Expand Down
5 changes: 0 additions & 5 deletions contracts/agents-api/core.openapi.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,6 @@ definitions:
- runtime_mode_not_observable
- allocation_pending
- runtime_not_running
- source_not_configured
- sample_timeout
- sample_unavailable
type: string
Expand Down Expand Up @@ -720,7 +719,6 @@ definitions:
properties:
connected_daemons:
type: integer
x-nullable: true
in_progress_turns:
type: integer
x-nullable: true
Expand All @@ -741,10 +739,8 @@ definitions:
type: array
slots_in_use:
type: integer
x-nullable: true
slots_total:
type: integer
x-nullable: true
unavailable:
type: integer
x-nullable: true
Expand Down Expand Up @@ -2484,7 +2480,6 @@ definitions:
- runtime_mode_not_observable
- allocation_pending
- runtime_not_running
- source_not_configured
- sample_timeout
- sample_unavailable
type: string
Expand Down
3 changes: 1 addition & 2 deletions contracts/agents-api/runtime-observability-api.md
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,6 @@ Only list rows carry `disk`: null, or `{usage_bytes, limit_bytes}` with the rule
| `unsupported` | `runtime_mode_not_observable` | `none` and `self_hosted` Sessions. |
| `unavailable` | `allocation_pending` | The managed allocation does not exist yet or is being created. |
| `unavailable` | `runtime_not_running` | The allocation is being cleaned up or is released, or the provider reports the Runtime absent, stopped or suspended. |
| `unavailable` | `source_not_configured` | This Core has no managed installation identity. |
| `unavailable` | `sample_timeout` | The provider read exceeded its deadline. |
| `unavailable` | `sample_unavailable` | The provider could not produce a current sample. |

Expand Down Expand Up @@ -233,7 +232,7 @@ Disk is not kept in history.
| 404 | `not_found_error` | A missing Project, or a Session missing from it. |
| 409 | `runtime_history_unsupported` | The Session is not `openai_hosted`. |
| 500 | `internal_error` | Inconsistent stored identity. |
| 503 | `runtime_history_unavailable` | Core collects no periodic history (it runs without the execution worker), or the read failed, timed out or produced a result outside the bounds. |
| 503 | `runtime_history_unavailable` | The read failed, timed out or produced a result outside the bounds. |

A response holds at most `max_points` buckets per array, 64 series and 10,000 coverage and series points in total. Storage error text is neither returned nor logged.

Expand Down
6 changes: 3 additions & 3 deletions contracts/agents-api/runtime-observability.md
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ The resolver (`services/core/internal/deployment/observation.go`) reads the Sess

Managed Docker, microsandbox and E2B allocations are observed. `none` and `self_hosted` Sessions are `unsupported`; Core never attributes shared host statistics to an `environment:none` Session.

Every managed allocation is read through the deployment's selected Sandbox Provider, which verifies the allocation's installation (`provider_key`) and labels or equivalent ownership data before it returns values. Before any provider read, the allocation state decides some rows: `creating` or no allocation yet gives `allocation_pending`, `cleanup_pending` or `released` gives `runtime_not_running`, and a Core without an installation identity gives `source_not_configured`. A provider read that exceeds its deadline gives `sample_timeout`, a not-running result `runtime_not_running`, and an unavailable result `sample_unavailable`. Any other error, an ownership mismatch or an invalid sample fails the read.
Every managed allocation is read through the deployment's selected Sandbox Provider, which verifies the allocation's installation (`provider_key`) and labels or equivalent ownership data before it returns values. Before any provider read, the allocation state decides some rows: `creating` or no allocation yet gives `allocation_pending`, and `cleanup_pending` or `released` gives `runtime_not_running`. A provider read that exceeds its deadline gives `sample_timeout`, a not-running result `runtime_not_running`, and an unavailable result `sample_unavailable`. Any other error, an ownership mismatch or an invalid sample fails the read.

`Observe` belongs to the [Sandbox Provider protocol](../../docs/sandbox-provider.md); `services/core/internal/runtimeobs/source.go` owns the observation types and the `Source` view of a Provider. Core loads the selected Provider and its registered kind once per page and uses that same immutable Provider for every read on that page, reading each running target with `Observe`. Without a selection the load returns typed `ErrUnavailable`, which produces `sample_unavailable` without a provider type. Other load errors follow the provider-read error rules above.

Expand Down Expand Up @@ -77,7 +77,7 @@ These durations answer different questions and stay separate:
- compute uptime: the sample's `started_at` to `observed_at`;
- busy Turn duration: `turns.started_at` to `completed_at`, or now.

CPU quietness, heartbeat age, connection state and keepalive time are not idle time.
CPU quietness, heartbeat age and connection state are not idle time.

## Retained history and optional export

Expand All @@ -95,7 +95,7 @@ The PostgreSQL store keeps only periodic `openai_hosted` records, so API reads c

The history service resolves the Project, Session and Environment before it queries; the query always carries that scope and bounded times, never provider identity. The store keeps seven days. A read covers at most 24 hours, reads at most 20,000 raw samples, starts two sampling intervals before the range to find CPU baselines, and returns at most 1,000 buckets per array, 64 series and 10,000 points in total. Results outside the requested scope, range or limits fail the read. The API's [Series](./runtime-observability-api.md#series) section describes the aggregation.

`runtimehistory.Capabilities` states the collection mode, interval, seven-day retention, minimum bucket width (30 seconds or the interval, whichever is longer), 24-hour range and point limits; the history route answers 503 unless they are valid and periodic.
`runtimehistory.Capabilities` states the sampling interval, seven-day retention, minimum bucket width (30 seconds or the interval, whichever is longer), 24-hour range and point limits; Core does not start unless they are valid.

A cleanup loop runs every minute, even without active Runtimes. Each pass has at most two seconds and deletes expired Runtime and node host rows in batches of 256 per table, at most 16 batches. Reads never return rows older than the retention.

Expand Down
2 changes: 1 addition & 1 deletion contracts/agents-api/sandbox-deployment.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
title: "Sandbox deployment"
---

The sandbox deployment selects the Sandbox Provider, the per-sandbox resources and the immutable Runtime release for Core-managed `openai_hosted` execution. PostgreSQL holds one active selection per installation; Web and the Core API write the same configuration. A node's files hold an installed copy of it plus host-specific paths and cannot override its resources or Runtime. The selection is independent of the Harness, and a deployment can stay unconfigured, with no nodes and no hosted admission.
The sandbox deployment selects the Sandbox Provider, the per-sandbox resources and the immutable Runtime release for Core-managed `openai_hosted` execution. PostgreSQL holds one active selection per installation; Web and the Core API write the same configuration. A node's files hold an installed copy of it plus host-specific paths and cannot override its resources or Runtime. The selection is independent of the Harness. A deployment can stay unconfigured, with no nodes; it then refuses hosted admission.

This contract owns the Core API routes below and their semantics. The [nodes guide](../../docs/getting-started/nodes.md) owns the operator workflow, the [machine connection API](./machine-api.md#node-routes) the routes nodes call, and the [sandbox node protocol](./node-generation-protocol.md) the node connection.

Expand Down
2 changes: 1 addition & 1 deletion contracts/agents-api/v1/runtime_observations.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ type RuntimeObservation struct {
Instance RuntimeInstance `json:"instance" binding:"required"`
LifecycleState *string `json:"lifecycle_state" extensions:"x-nullable" binding:"required" enums:"active,sleeping,transitioning,pending,stopped"`
Status string `json:"status" enums:"observed,unsupported,unavailable" binding:"required"`
Reason *string `json:"reason" extensions:"x-nullable" binding:"required" enums:"runtime_mode_not_observable,allocation_pending,runtime_not_running,source_not_configured,sample_timeout,sample_unavailable"`
Reason *string `json:"reason" extensions:"x-nullable" binding:"required" enums:"runtime_mode_not_observable,allocation_pending,runtime_not_running,sample_timeout,sample_unavailable"`
AllocationCreatedAt *int64 `json:"allocation_created_at" extensions:"x-nullable" binding:"required" minimum:"0"`
ResolvedAt int64 `json:"resolved_at" binding:"required" minimum:"0"`
ObservedAt *int64 `json:"observed_at" extensions:"x-nullable" binding:"required" minimum:"0"`
Expand Down
Loading
Loading