diff --git a/.github/workflows/api-acceptance.yml b/.github/workflows/api-acceptance.yml index 0111a8d4f..ee010ed18 100644 --- a/.github/workflows/api-acceptance.yml +++ b/.github/workflows/api-acceptance.yml @@ -63,9 +63,8 @@ jobs: OAC_TEST_OFFICIAL_SDK_PYTHON: python run: | python services/core/tests/official_schema_test.py - # The Go Workers need an unclaimed deployment; Core claims it for its installation ID. - go test ./services/core/tests/integration -run '^(TestFunctionStateOfficialClientReadsAndLiveEvents|TestSavedReferenceRetryOfficialClient|TestAgentUpdateOfficialClient|TestAgentDeletionOfficialClient|TestSessionAgentFilterOfficialClient|TestSessionDeletionOfficialClient|TestEnvironmentInitialFailureOfficialClient|TestSelfHostedInitialCreationOfficialClient|TestSelfHostedCancellationOfficialClient)$' -count=1 python services/core/tests/official_client.py + go test ./services/core/tests/integration -run '^(TestFunctionStateOfficialClientReadsAndLiveEvents|TestSavedReferenceRetryOfficialClient|TestAgentUpdateOfficialClient|TestAgentDeletionOfficialClient|TestSessionAgentFilterOfficialClient|TestSessionDeletionOfficialClient|TestEnvironmentInitialFailureOfficialClient|TestSelfHostedInitialCreationOfficialClient|TestSelfHostedCancellationOfficialClient)$' -count=1 - uses: ./.github/actions/e2b-provider if: inputs.container - name: Verify the distribution's Core image diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e5dee072e..e7c364c91 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -111,7 +111,7 @@ The [CI selection policy](docs/maintainers.md#continuous-integration) names affe | `OAC_TEST_DATABASE_URL` | A dedicated test database. The full gate fails when it is missing. | | `OAC_TEST_OFFICIAL_SDK_PYTHON` | The pinned official SDK interpreter | -The role needs `CREATE DATABASE`: tests of database-wide state, such as the execution lease and the provider identity, create and drop isolated `oac_*_tests` databases. Tests must not bypass the production provider-switch guard. +The role needs `CREATE DATABASE`: tests of database-wide state, such as the execution lease and the provider identity, create and drop isolated `oac_*_tests` databases. They copy them from a migrated template, the test database's name with `_template` before `_tests`, which stays beside it. Tests must not bypass the production provider-switch guard. ### Contract and schema rules diff --git a/apps/web/src/features/metrics/CoreMetricsPage.tsx b/apps/web/src/features/metrics/CoreMetricsPage.tsx index f6c5f64af..0d7fe6239 100644 --- a/apps/web/src/features/metrics/CoreMetricsPage.tsx +++ b/apps/web/src/features/metrics/CoreMetricsPage.tsx @@ -122,7 +122,7 @@ function CoreMetricsBody({ metrics }: { metrics: CoreMetrics }) { const databaseBuckets = useMemo(() => database.series.map((entry) => seconds(entry.start) ?? 0), [database.series]); const count = (value: number | null) => (value === null ? MISSING : formatInteger(value, locale)); const integer = (value: number) => formatInteger(value, locale); - const slotsFull = execution.slots_in_use !== null && execution.slots_total !== null && execution.slots_in_use >= execution.slots_total; + const slotsFull = execution.slots_in_use >= execution.slots_total; // Sandbox nodes hold their own connection to Core, as daemons do; E2B deployments have none. const { state: fleetState, refresh: refreshFleet } = useSandboxFleet({ poll: true }); const fleet = fleetSnapshot(fleetState); @@ -136,7 +136,7 @@ function CoreMetricsBody({ metrics }: { metrics: CoreMetrics }) { } unit={execution.slots_total === null ? undefined : `/ ${integer(execution.slots_total)}`} />} + value={
} unit={`/ ${integer(execution.slots_total)}`} />} tone={slotsFull ? "warning" : undefined} /> {started === null ? MISSING : formatDuration(Math.max(0, now - started))} {count(metrics.execution.slots_in_use)} - {metrics.execution.slots_total === null ? null : / {count(metrics.execution.slots_total)}} + / {count(metrics.execution.slots_total)} {count(metrics.execution.queued_turns)} {count(metrics.execution.connected_daemons)} diff --git a/apps/web/src/i18n/locales/en/metrics.ts b/apps/web/src/i18n/locales/en/metrics.ts index 427b768f2..2505030b6 100644 --- a/apps/web/src/i18n/locales/en/metrics.ts +++ b/apps/web/src/i18n/locales/en/metrics.ts @@ -310,7 +310,6 @@ export const metrics = { reason: { allocation_pending: "Allocation pending", runtime_not_running: "Not running", - source_not_configured: "Source unavailable", sample_timeout: "Sample timed out", sample_unavailable: "Sample unavailable", }, diff --git a/apps/web/src/i18n/locales/en/sessions.ts b/apps/web/src/i18n/locales/en/sessions.ts index fdf8078f3..389572f7a 100644 --- a/apps/web/src/i18n/locales/en/sessions.ts +++ b/apps/web/src/i18n/locales/en/sessions.ts @@ -138,7 +138,6 @@ export const sessions = { runtime_mode_not_observable: "Not observable", allocation_pending: "Allocation pending", runtime_not_running: "Not running", - source_not_configured: "Metrics source not configured", sample_timeout: "Sample timed out", sample_unavailable: "Sample unavailable", }, diff --git a/apps/web/src/i18n/locales/zh-CN/metrics.ts b/apps/web/src/i18n/locales/zh-CN/metrics.ts index 638f67afb..62463abf0 100644 --- a/apps/web/src/i18n/locales/zh-CN/metrics.ts +++ b/apps/web/src/i18n/locales/zh-CN/metrics.ts @@ -310,7 +310,6 @@ export const metrics = { reason: { allocation_pending: "等待分配", runtime_not_running: "未运行", - source_not_configured: "数据源不可用", sample_timeout: "采样超时", sample_unavailable: "采样不可用", }, diff --git a/apps/web/src/i18n/locales/zh-CN/sessions.ts b/apps/web/src/i18n/locales/zh-CN/sessions.ts index 82569748e..b6e1455da 100644 --- a/apps/web/src/i18n/locales/zh-CN/sessions.ts +++ b/apps/web/src/i18n/locales/zh-CN/sessions.ts @@ -135,7 +135,6 @@ export const sessions = { runtime_mode_not_observable: "无法观测", allocation_pending: "等待分配", runtime_not_running: "未运行", - source_not_configured: "未配置指标来源", sample_timeout: "采样超时", sample_unavailable: "采样不可用", }, diff --git a/contracts/agents-api/core-errors.md b/contracts/agents-api/core-errors.md index 33ee247ae..f9a73f846 100644 --- a/contracts/agents-api/core-errors.md +++ b/contracts/agents-api/core-errors.md @@ -95,7 +95,7 @@ These codes have null `param` and no `details`. [Sandbox deployment](./sandbox-d | 500 | `internal_error` | Core could not complete the operation | | 503 | `runtime_node_unavailable` | No sandbox node is available or has capacity | | 503 | `execution_unavailable` | Execution is not available, such as while Core shuts down | -| 503 | `runtime_history_unavailable` | Durable Runtime history is not configured or temporarily unavailable | +| 503 | `runtime_history_unavailable` | Durable Runtime history is temporarily unavailable | | 503 | `core_metrics_unavailable` | Core metrics could not be read | | 503 | `file_transfer_unavailable` | Bounded content transfer is unavailable | diff --git a/contracts/agents-api/core-metrics.md b/contracts/agents-api/core-metrics.md index def541329..5ad43d4b8 100644 --- a/contracts/agents-api/core-metrics.md +++ b/contracts/agents-api/core-metrics.md @@ -25,19 +25,19 @@ title: "Core operational metrics" ## Fields -The response has `object: "core.metrics"`, `range`, `service`, `execution`, `database`, `jobs` and `process`. Every numeric value and `service.execution_owner` is nullable; each `series` always lists every complete bucket of the range. +The response has `object: "core.metrics"`, `range`, `service`, `execution`, `database`, `jobs` and `process`. Every numeric value except `execution.slots_in_use`, `execution.slots_total` and `execution.connected_daemons` is nullable, as is `service.execution_owner`; each `series` always lists every complete bucket of the range. | Field | Meaning | | --- | --- | -| `service.status` | `running`, or `degraded` when a measurement or job fails, the latest sample is missing or stale, or execution ownership is unknown or Core has execution slots but does not hold the execution lease. A sandbox reset is reported by the [deployment](./sandbox-deployment.md), not here | +| `service.status` | `running`, or `degraded` when a measurement or job fails, the latest sample is missing or stale, or execution ownership is unknown or Core does not hold the execution lease. A sandbox reset is reported by the [deployment](./sandbox-deployment.md), not here | | `service.revision` | The full source commit injected at build time; null for builds without one | | `service.started_at` | When the process initialized | | `service.execution_owner` | Whether this process holds the execution worker's database lease | -| `execution.slots_in_use`, `execution.slots_total` | Active Session reservations of the execution worker, and its capacity: [`core.execution_concurrency`](../../docs/configuration.md#settings), 4 by default. Environment input, Turns and file work share the slots; native Harness subprocesses are not counted. Without a worker both are 0 | +| `execution.slots_in_use`, `execution.slots_total` | Active Session reservations of the execution worker, and its capacity: [`core.execution_concurrency`](../../docs/configuration.md#settings), 4 by default. Environment input, Turns and file work share the slots; native Harness subprocesses are not counted | | `execution.queued_turns`, `execution.in_progress_turns` | Root Turns in those states, including Turns of deleted Sessions. Subagent Turns and input reserved for a preparing Environment are not counted | -| `execution.waiting_for_daemon` | Queued Turns whose Session's device is not connected; null without a gateway | +| `execution.waiting_for_daemon` | Queued Turns whose Session's device is not connected | | `execution.oldest_queued_seconds` | Age of the oldest queued Turn, from its `created_at` | -| `execution.connected_daemons` | Runtime daemons connected to Core's gateway; null without a gateway | +| `execution.connected_daemons` | Runtime daemons connected to Core's gateway | | `execution.queue_wait_ms` | p50 and p95 of `started_at - created_at` for Turns started in the interval, by PostgreSQL `percentile_cont` | | `execution.interrupted` | Failed Turns with error code `execution_interrupted`, by `completed_at` in the interval | | `execution.unavailable` | HTTP responses sent with error code `execution_unavailable`, counted once each. Other 503 codes and errors after a stream started are not counted | diff --git a/contracts/agents-api/core.openapi.yaml b/contracts/agents-api/core.openapi.yaml index 90fbd2970..af9ea99b5 100644 --- a/contracts/agents-api/core.openapi.yaml +++ b/contracts/agents-api/core.openapi.yaml @@ -103,7 +103,6 @@ definitions: - runtime_mode_not_observable - allocation_pending - runtime_not_running - - source_not_configured - sample_timeout - sample_unavailable type: string @@ -725,7 +724,6 @@ definitions: properties: connected_daemons: type: integer - x-nullable: true in_progress_turns: type: integer x-nullable: true @@ -746,10 +744,8 @@ definitions: type: array slots_in_use: type: integer - x-nullable: true slots_total: type: integer - x-nullable: true unavailable: type: integer x-nullable: true @@ -2489,7 +2485,6 @@ definitions: - runtime_mode_not_observable - allocation_pending - runtime_not_running - - source_not_configured - sample_timeout - sample_unavailable type: string diff --git a/contracts/agents-api/runtime-observability-api.md b/contracts/agents-api/runtime-observability-api.md index 2e1db7019..b4bc701d1 100644 --- a/contracts/agents-api/runtime-observability-api.md +++ b/contracts/agents-api/runtime-observability-api.md @@ -145,7 +145,6 @@ Only list rows carry `disk`: null, or `{usage_bytes, limit_bytes}` with the rule | `unsupported` | `runtime_mode_not_observable` | `none` and `self_hosted` Sessions. | | `unavailable` | `allocation_pending` | The managed allocation does not exist yet or is being created. | | `unavailable` | `runtime_not_running` | The allocation is being cleaned up or is released, or the provider reports the Runtime absent, stopped or suspended. | -| `unavailable` | `source_not_configured` | This Core has no managed installation identity. | | `unavailable` | `sample_timeout` | The provider read exceeded its deadline. | | `unavailable` | `sample_unavailable` | The provider could not produce a current sample. | @@ -233,7 +232,7 @@ Disk is not kept in history. | 404 | `not_found_error` | A missing Project, or a Session missing from it. | | 409 | `runtime_history_unsupported` | The Session is not `openai_hosted`. | | 500 | `internal_error` | Inconsistent stored identity. | -| 503 | `runtime_history_unavailable` | Core collects no periodic history (it runs without the execution worker), or the read failed, timed out or produced a result outside the bounds. | +| 503 | `runtime_history_unavailable` | The read failed, timed out or produced a result outside the bounds. | A response holds at most `max_points` buckets per array, 64 series and 10,000 coverage and series points in total. Storage error text is neither returned nor logged. diff --git a/contracts/agents-api/runtime-observability.md b/contracts/agents-api/runtime-observability.md index c91351dac..36860bb48 100644 --- a/contracts/agents-api/runtime-observability.md +++ b/contracts/agents-api/runtime-observability.md @@ -20,7 +20,7 @@ The resolver (`services/core/internal/deployment/observation.go`) reads the Sess Managed Docker, microsandbox and E2B allocations are observed. `none` and `self_hosted` Sessions are `unsupported`; Core never attributes shared host statistics to an `environment:none` Session. -Every managed allocation is read through the deployment's selected Sandbox Provider, which verifies the allocation's installation (`provider_key`) and labels or equivalent ownership data before it returns values. Before any provider read, the allocation state decides some rows: `creating` or no allocation yet gives `allocation_pending`, `cleanup_pending` or `released` gives `runtime_not_running`, and a Core without an installation identity gives `source_not_configured`. A provider read that exceeds its deadline gives `sample_timeout`, a not-running result `runtime_not_running`, and an unavailable result `sample_unavailable`. Any other error, an ownership mismatch or an invalid sample fails the read. +Every managed allocation is read through the deployment's selected Sandbox Provider, which verifies the allocation's installation (`provider_key`) and labels or equivalent ownership data before it returns values. Before any provider read, the allocation state decides some rows: `creating` or no allocation yet gives `allocation_pending`, and `cleanup_pending` or `released` gives `runtime_not_running`. A provider read that exceeds its deadline gives `sample_timeout`, a not-running result `runtime_not_running`, and an unavailable result `sample_unavailable`. Any other error, an ownership mismatch or an invalid sample fails the read. `Observe` belongs to the [Sandbox Provider protocol](../../docs/sandbox-provider.md); `services/core/internal/runtimeobs/source.go` owns the observation types and the `Source` view of a Provider. Core loads the selected Provider and its registered kind once per page and uses that same immutable Provider for every read on that page, reading each running target with `Observe`. Without a selection the load returns typed `ErrUnavailable`, which produces `sample_unavailable` without a provider type. Other load errors follow the provider-read error rules above. @@ -77,7 +77,7 @@ These durations answer different questions and stay separate: - compute uptime: the sample's `started_at` to `observed_at`; - busy Turn duration: `turns.started_at` to `completed_at`, or now. -CPU quietness, heartbeat age, connection state and keepalive time are not idle time. +CPU quietness, heartbeat age and connection state are not idle time. ## Retained history and optional export @@ -95,7 +95,7 @@ The PostgreSQL store keeps only periodic `openai_hosted` records, so API reads c The history service resolves the Project, Session and Environment before it queries; the query always carries that scope and bounded times, never provider identity. The store keeps seven days. A read covers at most 24 hours, reads at most 20,000 raw samples, starts two sampling intervals before the range to find CPU baselines, and returns at most 1,000 buckets per array, 64 series and 10,000 points in total. Results outside the requested scope, range or limits fail the read. The API's [Series](./runtime-observability-api.md#series) section describes the aggregation. -`runtimehistory.Capabilities` states the collection mode, interval, seven-day retention, minimum bucket width (30 seconds or the interval, whichever is longer), 24-hour range and point limits; the history route answers 503 unless they are valid and periodic. +`runtimehistory.Capabilities` states the sampling interval, seven-day retention, minimum bucket width (30 seconds or the interval, whichever is longer), 24-hour range and point limits; Core does not start unless they are valid. A cleanup loop runs every minute, even without active Runtimes. Each pass has at most two seconds and deletes expired Runtime and node host rows in batches of 256 per table, at most 16 batches. Reads never return rows older than the retention. diff --git a/contracts/agents-api/sandbox-deployment.md b/contracts/agents-api/sandbox-deployment.md index 3e813ad98..e473a7b7e 100644 --- a/contracts/agents-api/sandbox-deployment.md +++ b/contracts/agents-api/sandbox-deployment.md @@ -2,7 +2,7 @@ title: "Sandbox deployment" --- -The sandbox deployment selects the Sandbox Provider, the per-sandbox resources and the immutable Runtime release for Core-managed `openai_hosted` execution. PostgreSQL holds one active selection per installation; Web and the Core API write the same configuration. A node's files hold an installed copy of it plus host-specific paths and cannot override its resources or Runtime. The selection is independent of the Harness, and a deployment can stay unconfigured, with no nodes and no hosted admission. +The sandbox deployment selects the Sandbox Provider, the per-sandbox resources and the immutable Runtime release for Core-managed `openai_hosted` execution. PostgreSQL holds one active selection per installation; Web and the Core API write the same configuration. A node's files hold an installed copy of it plus host-specific paths and cannot override its resources or Runtime. The selection is independent of the Harness. A deployment can stay unconfigured, with no nodes; it then refuses hosted admission. This contract owns the Core API routes below and their semantics. The [nodes guide](../../docs/getting-started/nodes.md) owns the operator workflow, the [machine connection API](./machine-api.md#node-routes) the routes nodes call, and the [sandbox node protocol](./node-generation-protocol.md) the node connection. diff --git a/contracts/agents-api/v1/runtime_observations.go b/contracts/agents-api/v1/runtime_observations.go index 7b47d8720..8e9fef69e 100644 --- a/contracts/agents-api/v1/runtime_observations.go +++ b/contracts/agents-api/v1/runtime_observations.go @@ -10,7 +10,7 @@ type RuntimeObservation struct { Instance RuntimeInstance `json:"instance" binding:"required"` LifecycleState *string `json:"lifecycle_state" extensions:"x-nullable" binding:"required" enums:"active,sleeping,transitioning,pending,stopped"` Status string `json:"status" enums:"observed,unsupported,unavailable" binding:"required"` - Reason *string `json:"reason" extensions:"x-nullable" binding:"required" enums:"runtime_mode_not_observable,allocation_pending,runtime_not_running,source_not_configured,sample_timeout,sample_unavailable"` + Reason *string `json:"reason" extensions:"x-nullable" binding:"required" enums:"runtime_mode_not_observable,allocation_pending,runtime_not_running,sample_timeout,sample_unavailable"` AllocationCreatedAt *int64 `json:"allocation_created_at" extensions:"x-nullable" binding:"required" minimum:"0"` ResolvedAt int64 `json:"resolved_at" binding:"required" minimum:"0"` ObservedAt *int64 `json:"observed_at" extensions:"x-nullable" binding:"required" minimum:"0"` diff --git a/contracts/agents-api/zh/core-errors.md b/contracts/agents-api/zh/core-errors.md index b7ba9abb3..aa2c19203 100644 --- a/contracts/agents-api/zh/core-errors.md +++ b/contracts/agents-api/zh/core-errors.md @@ -1,7 +1,7 @@ --- title: "Core 管理错误" source: contracts/agents-api/core-errors.md -source_hash: 46b6e7eb76739b9473576113aad3eee64fff3fddd429c3232ede67f671d60ed7 +source_hash: 6e917cd9928dfdeb0f5f4742b3e61d3197da5d599ba9f2b8934dd85ef23454d3 --- `/core/v1` 上的错误使用此封装结构。`message` 是安全的英文文本；`code` 和 `param` 可以为 null。客户端依据稳定的 `code` 和可选的 `param` 进行处理，对未知代码显示 `message`，绝不解析消息，也绝不自动重试被拒绝的写操作。 @@ -97,7 +97,7 @@ Web 的控制台服务器在 `/core` 路径上发生自身故障时使用此封 | 500 | `internal_error` | Core 未能完成操作 | | 503 | `runtime_node_unavailable` | 没有可用或有剩余容量的沙箱节点 | | 503 | `execution_unavailable` | 执行不可用，例如 Core 正在关闭 | -| 503 | `runtime_history_unavailable` | 持久 Runtime 历史未配置或暂时不可用 | +| 503 | `runtime_history_unavailable` | 持久 Runtime 历史暂时不可用 | | 503 | `core_metrics_unavailable` | 无法读取 Core 指标 | | 503 | `file_transfer_unavailable` | 有界内容传输不可用 | diff --git a/contracts/agents-api/zh/core-metrics.md b/contracts/agents-api/zh/core-metrics.md index 7503b8611..540750d78 100644 --- a/contracts/agents-api/zh/core-metrics.md +++ b/contracts/agents-api/zh/core-metrics.md @@ -1,7 +1,7 @@ --- title: "Core 运行指标" source: contracts/agents-api/core-metrics.md -source_hash: 08ac6802f0da2eb5138dc7706a995754a69e7c864e034cf412f75dcc4950c679 +source_hash: f94805c348f60947ac0a61de898539d982e529226c7f07bc73d2a36a3a641821 --- `GET /core/v1/metrics?range=1h|6h|24h|7d` 报告 Core 自身的健康状况：进程、执行队列与槽位、PostgreSQL 和后台任务。它要求 Core 密钥（[Core 管理 API](admin-api.md)）。 @@ -27,19 +27,19 @@ source_hash: 08ac6802f0da2eb5138dc7706a995754a69e7c864e034cf412f75dcc4950c679 ## 字段 {#fields} -响应包含 `object: "core.metrics"`、`range`、`service`、`execution`、`database`、`jobs` 和 `process`。所有数值和 `service.execution_owner` 均可为 null；每个 `series` 始终列出范围内的所有完整桶。 +响应包含 `object: "core.metrics"`、`range`、`service`、`execution`、`database`、`jobs` 和 `process`。除 `execution.slots_in_use`、`execution.slots_total` 和 `execution.connected_daemons` 外，所有数值以及 `service.execution_owner` 均可为 null；每个 `series` 始终列出范围内的所有完整桶。 | 字段 | 含义 | | --- | --- | -| `service.status` | 通常为 `running`；测量或任务失败、最新样本缺失或过期、执行所有权未知，或 Core 有执行槽位却未持有执行租约时为 `degraded`。沙箱重置由[部署](sandbox-deployment.md)报告，不在此处报告 | +| `service.status` | 通常为 `running`；测量或任务失败、最新样本缺失或过期、执行所有权未知，或 Core 未持有执行租约时为 `degraded`。沙箱重置由[部署](sandbox-deployment.md)报告，不在此处报告 | | `service.revision` | 构建时注入的完整源代码提交；未注入时为 null | | `service.started_at` | 进程初始化时间 | | `service.execution_owner` | 此进程是否持有执行 worker 的数据库租约 | -| `execution.slots_in_use`, `execution.slots_total` | 执行 worker 的活动 Session 预留数量及容量：[`core.execution_concurrency`](../../../docs/zh/configuration.md#settings)，默认为 4。Environment 输入、Turn 和文件工作共享槽位；不统计原生 Harness 子进程。没有 worker 时两者均为 0 | +| `execution.slots_in_use`, `execution.slots_total` | 执行 worker 的活动 Session 预留数量及容量：[`core.execution_concurrency`](../../../docs/zh/configuration.md#settings)，默认为 4。Environment 输入、Turn 和文件工作共享槽位；不统计原生 Harness 子进程 | | `execution.queued_turns`, `execution.in_progress_turns` | 处于相应状态的根 Turn，包括已删除 Session 的 Turn。不统计 Subagent Turn 和为准备中 Environment 预留的输入 | -| `execution.waiting_for_daemon` | Session 设备未连接的排队 Turn；无网关时为 null | +| `execution.waiting_for_daemon` | Session 设备未连接的排队 Turn | | `execution.oldest_queued_seconds` | 最早排队 Turn 自 `created_at` 起的年龄 | -| `execution.connected_daemons` | 连接到 Core 网关的 Runtime daemon 数量；无网关时为 null | +| `execution.connected_daemons` | 连接到 Core 网关的 Runtime daemon 数量 | | `execution.queue_wait_ms` | 区间内开始的 Turn 的 `started_at - created_at` 的 p50 和 p95，使用 PostgreSQL `percentile_cont` | | `execution.interrupted` | 错误代码为 `execution_interrupted` 且 `completed_at` 在区间内的失败 Turn | | `execution.unavailable` | 使用错误代码 `execution_unavailable` 发送的 HTTP 响应，每个计一次。不统计其他 503 代码或流开始后的错误 | diff --git a/contracts/agents-api/zh/runtime-observability-api.md b/contracts/agents-api/zh/runtime-observability-api.md index 984c5ab7e..660c4a7d9 100644 --- a/contracts/agents-api/zh/runtime-observability-api.md +++ b/contracts/agents-api/zh/runtime-observability-api.md @@ -1,7 +1,7 @@ --- title: "Runtime 遥测 API" source: contracts/agents-api/runtime-observability-api.md -source_hash: 6eca80ffefcaf8e26901659e5251518f84d2c6c93349085b378d46f9ac49149d +source_hash: 05ee25e01c2a8e9ce0f85c325a4a0e0e2f11eb76861e54740efa8975d9c5a999 --- Core 通过 `/core/v1` 下的只读管理员路由报告托管 Runtime 和沙箱节点所使用的信息：当前 Runtime 观测值、单个 Session 的已存储 Runtime 历史记录，以及沙箱节点的主机观测值和历史记录。读取操作绝不创建、唤醒、续期或更改计算资源，也绝不向历史记录添加样本。[Runtime observability](runtime-observability.md) 定义了 Core 如何采集和保留这些值；[Console API usage](../../../docs/zh/web/console-api-usage.md) 列出了读取这些值的 Web 页面。 @@ -147,7 +147,6 @@ Authorization: Bearer | `unsupported` | `runtime_mode_not_observable` | `none` 和 `self_hosted` Session。 | | `unavailable` | `allocation_pending` | 托管分配尚不存在或正在创建。 | | `unavailable` | `runtime_not_running` | 分配正在清理或已释放，或者提供方报告 Runtime 不存在、已停止或已暂停。 | -| `unavailable` | `source_not_configured` | 此 Core 没有托管 installation 标识。 | | `unavailable` | `sample_timeout` | 提供方读取超过其截止时间。 | | `unavailable` | `sample_unavailable` | 提供方无法生成当前样本。 | @@ -235,7 +234,7 @@ Core 先解析 Project，然后解析 Session 及其 Environment，之后才读 | 404 | `not_found_error` | Project 不存在，或 Session 不属于该 Project。 | | 409 | `runtime_history_unsupported` | Session 不是 `openai_hosted`。 | | 500 | `internal_error` | 已存储的标识不一致。 | -| 503 | `runtime_history_unavailable` | Core 未收集周期性历史（即在不使用 execution worker 的情况下运行），或读取失败、超时或产生了超出界限的结果。 | +| 503 | `runtime_history_unavailable` | 读取失败、超时或产生了超出界限的结果。 | 每个数组最多包含 `max_points` 个桶，响应最多包含 64 条 series，并且 coverage 和 series 中的点总计最多为 10,000 个。存储错误文本既不会返回，也不会记录到日志中。 diff --git a/contracts/agents-api/zh/runtime-observability.md b/contracts/agents-api/zh/runtime-observability.md index 223c463c9..0832cecf6 100644 --- a/contracts/agents-api/zh/runtime-observability.md +++ b/contracts/agents-api/zh/runtime-observability.md @@ -1,7 +1,7 @@ --- title: "运行时可观测性" source: contracts/agents-api/runtime-observability.md -source_hash: 31fa597224d654f347c7f438bea78d8e548e7588849d4346c599aac8a8f8054c +source_hash: de7eefac37eb9cc5935c28e2425efc323f1555771e48758ce9412455e891b52c --- 这是面向贡献者的契约，规定 Core 如何观测 Runtime 并保留其历史。路由和响应字段见 [Runtime telemetry API](runtime-observability-api.md)。代码位于 `services/core/internal/runtimeobs`（解析、源、采样器和导出）、`internal/runtimehistory`（历史查询和 PostgreSQL 存储）以及 `internal/runtimeobs/otlpexporter`。 @@ -22,7 +22,7 @@ none: tenant_id -> session_id (no Session-owned Runtime instance) 托管 Docker、microsandbox 和 E2B 分配均会被观测。`none` 和 `self_hosted` Session 为 `unsupported`；Core 绝不会将共享主机统计信息归属于 `environment:none` Session。 -每个托管分配都通过部署所选的 Sandbox Provider 读取；该 Provider 在返回数值前会验证分配的 installation（`provider_key`）以及分配标签或等效所有权数据。在读取任何 provider 之前，部分行的结果由分配状态决定：处于 `creating` 状态或尚无分配时得到 `allocation_pending`，处于 `cleanup_pending` 或 `released` 状态时得到 `runtime_not_running`，Core 没有 installation 标识时得到 `source_not_configured`。provider 读取超出截止时间时得到 `sample_timeout`，返回未运行结果时得到 `runtime_not_running`，返回不可用结果时得到 `sample_unavailable`。任何其他错误、所有权不匹配或无效采样都会使读取失败。 +每个托管分配都通过部署所选的 Sandbox Provider 读取；该 Provider 在返回数值前会验证分配的 installation（`provider_key`）以及分配标签或等效所有权数据。在读取任何 provider 之前，部分行的结果由分配状态决定：处于 `creating` 状态或尚无分配时得到 `allocation_pending`，处于 `cleanup_pending` 或 `released` 状态时得到 `runtime_not_running`。provider 读取超出截止时间时得到 `sample_timeout`，返回未运行结果时得到 `runtime_not_running`，返回不可用结果时得到 `sample_unavailable`。任何其他错误、所有权不匹配或无效采样都会使读取失败。 `Observe` 属于 [Sandbox Provider 协议](../../../docs/zh/sandbox-provider.md)；`services/core/internal/runtimeobs/source.go` 负责观测类型以及 Provider 的 `Source` 视图。Core 每页只加载一次所选 Provider 及其注册 kind，并在该页的每次读取中使用同一不可变 Provider，用 `Observe` 读取每个运行中的目标。没有选择时，加载返回类型化的 `ErrUnavailable`，从而生成不含 provider 类型的 `sample_unavailable`。其他加载错误遵循上述 provider 读取错误规则。 @@ -79,7 +79,7 @@ E2B 不报告累计 CPU 时间，因此 CPU 秒数保持为 null。`observed_at` - 计算运行时长：采样的 `started_at` 到 `observed_at`； - 忙碌 Turn 时长：`turns.started_at` 到 `completed_at`，或到当前时间。 -CPU 静默状态、心跳时龄、连接状态和保活时间都不是空闲时间。 +CPU 静默状态、心跳时龄和连接状态都不是空闲时间。 ## 保留的历史记录与可选导出 {#retained-history-and-optional-export} @@ -97,7 +97,7 @@ PostgreSQL 存储仅保留周期性的 `openai_hosted` 记录，因此 API 读 历史服务在查询前解析 Project、Session 和 Environment；查询始终携带该作用域和有界时间范围，但绝不携带 provider 身份。存储保留 7 天。单次读取最多覆盖 24 小时，最多读取 20,000 条原始采样，并从范围起点之前两个采样间隔处开始读取，以查找 CPU 基线；每个数组最多返回 1,000 个桶，最多返回 64 个序列，总点数最多 10,000 个。结果超出请求的作用域、时间范围或限制时，读取失败。API 的 [Series](runtime-observability-api.md#series) 部分说明了聚合方式。 -`runtimehistory.Capabilities` 声明采集模式、间隔、7 天保留期、最小桶宽度（30 秒或采样间隔，取较长者）、24 小时范围和点数限制；历史路由仅在所有这些值有效且采集模式为 `periodic` 时响应，否则返回 503。 +`runtimehistory.Capabilities` 声明采样间隔、7 天保留期、最小桶宽度（30 秒或采样间隔，取较长者）、24 小时范围和点数限制；这些值无效时 Core 不会启动。 清理循环每分钟运行一次，即使没有活跃 Runtime 也会运行。每轮最多耗时 2 秒，按每表 256 行的批次删除过期 Runtime 行和节点主机行，每张表最多 16 批。读取绝不会返回超过保留期的行。 diff --git a/contracts/agents-api/zh/sandbox-deployment.md b/contracts/agents-api/zh/sandbox-deployment.md index f14dc75eb..275cf079a 100644 --- a/contracts/agents-api/zh/sandbox-deployment.md +++ b/contracts/agents-api/zh/sandbox-deployment.md @@ -1,10 +1,10 @@ --- title: "沙箱部署" source: contracts/agents-api/sandbox-deployment.md -source_hash: f4ecc42b24dd85d2bfe3e054358aa2087331aa3110ba031bbc824cece58a54c3 +source_hash: b40a45b42e42de5e64fa3ccae4666650e6c5d8b7d53a438d61a2e6665af2168e --- -沙箱部署为 Core 管理的 `openai_hosted` 执行选择 Sandbox Provider、每个沙箱的资源以及不可变的 Runtime 发行版。PostgreSQL 为每个安装维护一个当前有效选择；Web 和 Core API 写入同一配置。节点文件保存其已安装副本和特定于主机的路径，且不能覆盖其资源或 Runtime。该选择独立于 Harness；部署可以保持未配置状态，既无节点，也不接受托管准入。 +沙箱部署为 Core 管理的 `openai_hosted` 执行选择 Sandbox Provider、每个沙箱的资源以及不可变的 Runtime 发行版。PostgreSQL 为每个安装维护一个当前有效选择；Web 和 Core API 写入同一配置。节点文件保存其已安装副本和特定于主机的路径，且不能覆盖其资源或 Runtime。该选择独立于 Harness。部署可以保持未配置状态，没有节点；此时它拒绝托管准入。 本契约负责下列 Core API 路由及其语义。[节点指南](../../../docs/zh/getting-started/nodes.md)负责操作员工作流，[机器连接 API](machine-api.md#node-routes)负责节点调用的路由，[沙箱节点协议](node-generation-protocol.md)负责节点连接。 diff --git a/docs/sandbox-provider.md b/docs/sandbox-provider.md index a3c49dc59..463f5e142 100644 --- a/docs/sandbox-provider.md +++ b/docs/sandbox-provider.md @@ -152,7 +152,7 @@ This is what Core does around every provider. Adapters implement none of it, but At startup Core claims the stable installation identity and a new owner epoch before it selects a provider. The runtime manager keeps generation-aware provider facades. Initial setup and replacement prepare and validate candidates before any database write, and a rejected candidate leaves the active configuration and workers unchanged. A backend replacement uses the deployment mutation gate: it pauses manager admission, drains old calls and loops, then repeats the resource and generation guards in the commit transaction, where the new selection, its generation and the retirement of old nodes and unused enrollment tokens commit together. After commit, Core publishes the prevalidated configuration and the shared observation and bootstrap cache under the manager mutex, with no further external work or fallible step, so a request cancelled after commit cannot discard it. An interrupted drain stays a barrier for retries. Provider I/O and draining never hold a database transaction or the manager's map mutex. -A locally unavailable provider dependency keeps hosted admission closed while the existing scan waits for repair; administrator recovery stays available, also after a restart. Database and ownership errors stay failures, and unconfigured hosted admission creates no Session state. Core derives the Runtime bootstrap and daemon WebSocket addresses from the installation public URL, never from request headers, and reads the current selection from the database, never from a startup file. +A locally unavailable provider dependency keeps hosted admission closed while the existing scan waits for repair; administrator recovery stays available, also after a restart. Database and ownership errors stay failures. An unconfigured deployment refuses hosted admission with 503 `execution_unavailable` and creates no Session state. Core derives the Runtime bootstrap and daemon WebSocket addresses from the installation public URL, never from request headers, and reads the current selection from the database, never from a startup file. Node readiness binds to the exact generation, the current connection and the owner epoch. A durable serving pin is promoted only for readiness of the then-current target, under deployment serialization, so a late report for a superseded target never acquires a pin. @@ -162,7 +162,7 @@ The allocation, its dedicated daemon credential digest and the exact Session bin With a configured provider, the Worker scans committed pending hosted Environments that have no allocation, which covers idle Session creation and recovery after an interruption between commit and bootstrap; an existing allocation never re-enters that path. The scan is bounded and serialized by the lifecycle owner and needs no caller action. An initial reservation without a Turn leaves its Session idle, and a daemon connection is never treated as native readiness. The same scan publishes authenticated connection observations with durable generations, after verifying the exact Session and device binding and a settled bootstrap. -Connected, observed compute receives service keepalives between Turns. Keepalives never revive a lapse of one hour or a cleanup request. A node allocation never expires only because its keepalive is an hour old; explicit deletion and the snapshot retention still authorize its cleanup. A stopped or missing container never authorizes discarding retained workspace or history. Disabling the provider stops new hosted admission and bootstrap but never blocks cancellation, function results or input retry outcomes of existing Sessions. +Between Turns, Core checks that connected, observed compute is still its Session's running allocation; the check changes nothing and never revives a cleanup request. Running compute never expires: explicit deletion and the snapshot retention authorize its cleanup. A stopped or missing container never authorizes discarding retained workspace or history. Disabling the provider stops new hosted admission and bootstrap but never blocks cancellation, function results or input retry outcomes of existing Sessions. Terminal cleanup atomically revokes the device's authority, records the Environment's failure or expiry, settles pending input and requests cancellation, and only then calls `Kill`; original input deadlines and retry outcomes are kept. Temporary provider outages, unknown Create results and stopped compute never prove a permanent failure. After public Session deletion Core keeps the allocation and marks it released only after owned compute and volume cleanup and proof that the original Create settled; an unknown creation keeps cleanup ownership even after an absence observation, and bounded scans continue to catch late resources without another `Create`. diff --git a/docs/zh/sandbox-provider.md b/docs/zh/sandbox-provider.md index d273df9d0..51ed3427e 100644 --- a/docs/zh/sandbox-provider.md +++ b/docs/zh/sandbox-provider.md @@ -1,7 +1,7 @@ --- title: "添加 Sandbox Provider" source: docs/sandbox-provider.md -source_hash: 305d35b08388d682347f7bce263262e65031cfe69d09cd2acf0e8dae6893ff6f +source_hash: e14ff6b3d8166a04b629723aa4bd0fa6496b74307d0b6c60ebd454247c1ca3fd --- **Sandbox Provider** 为 Core 管理的 Environment 提供 Runtime daemon 运行所需的外层计算资源，以及启动 daemon 的有界引导流程。本指南说明如何添加 Provider，并作为 Core 驱动 Provider 的参考。接口为 [`SandboxProvider`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/internal/sandbox/sandbox_provider.go)。 @@ -154,7 +154,7 @@ launcher 从[派生进程环境](configuration.md)提供 `sandbox.ProcessPaths` 启动时 Core 在选择 provider 前领取稳定 installation identity 和新的 owner epoch。runtime manager 保留了解 generation 的 provider facade。初始 setup 与 replacement 在任何数据库写入前准备并验证 candidate；candidate 被拒绝时不改变活动配置或 worker。backend replacement 使用 deployment mutation gate：暂停 manager admission，排空旧调用与循环，再在 commit transaction 中重复 resource 与 generation guard；新选择、其 generation、旧 node 和未使用 enrollment token 的退役一起提交。提交后，Core 在 manager mutex 下发布预验证配置和共享 observation、bootstrap cache，没有进一步外部工作或可失败步骤，因此提交后取消的请求不能丢弃配置。中断的 drain 保留为重试 barrier。Provider I/O 和 drain 不持有数据库事务或 manager map mutex。 -本地 provider 依赖不可用时，现有 scan 等待修复，hosted admission 保持关闭；管理员恢复仍可使用，重启后也如此。数据库和所有权错误仍是失败，未配置 hosted admission 不创建 Session 状态。Core 从 installation public URL 派生 Runtime bootstrap 和 daemon WebSocket 地址，不使用请求 header；从数据库读取当前选择，不使用 startup file。 +本地 provider 依赖不可用时，现有 scan 等待修复，hosted admission 保持关闭；管理员恢复仍可使用，重启后也如此。数据库和所有权错误仍是失败。未配置的部署以 503 `execution_unavailable` 拒绝 hosted admission，且不创建 Session 状态。Core 从 installation public URL 派生 Runtime bootstrap 和 daemon WebSocket 地址，不使用请求 header；从数据库读取当前选择，不使用 startup file。 Node readiness 绑定到精确 generation、当前连接和 owner epoch。持久 serving pin 仅在部署串行化下为当时目标的 readiness 提升，因此已被替代目标的延迟报告不获得 pin。 @@ -164,7 +164,7 @@ allocation、专用 daemon credential digest 和精确 Session binding 在 `Crea 配置 provider 后，Worker 扫描已提交且没有 allocation 的 pending hosted Environment，涵盖空闲 Session 创建以及 commit 与 bootstrap 之间中断后的恢复；已有 allocation 不重新进入此路径。scan 有界，由 lifecycle owner 串行化，不需要调用方操作。没有 Turn 的初始预约让 Session 保持空闲，daemon 连接不被当作原生 readiness。同一 scan 在验证精确 Session、device binding 和已结算 bootstrap 后，发布带持久 generation 的认证连接观测。 -已连接且已观察的计算资源在 Turn 之间接收 service keepalive。keepalive 不复活一小时的中断或 cleanup 请求。node allocation 不仅因 keepalive 已过一小时而到期；显式删除和 snapshot retention 仍授权其清理。停止或缺失 container 不授权丢弃保留工作区或历史。禁用 provider 停止新 hosted admission 与 bootstrap，但不阻止现有 Session 的取消、function result 或 input retry outcome。 +Core 在 Turn 之间检查已连接且已观察的计算资源仍是其 Session 正在运行的 allocation；该检查不做任何修改，也不复活 cleanup 请求。正在运行的计算资源不会到期：显式删除和 snapshot retention 授权其清理。停止或缺失 container 不授权丢弃保留工作区或历史。禁用 provider 停止新 hosted admission 与 bootstrap，但不阻止现有 Session 的取消、function result 或 input retry outcome。 终结清理原子撤销 device authority、记录 Environment 失败或到期、结算 pending input 并请求取消，然后才调用 `Kill`；原 input deadline 与 retry outcome 保留。临时 provider outage、未知 Create result 和停止的计算资源不证明永久失败。公开 Session 删除后 Core 保留 allocation，仅在所属 compute 与 volume 清理完成且原 Create 已结算的证明成立后标记 released；未知创建即使观察到不存在也保留 cleanup ownership，有界 scan 继续捕捉延迟资源，不再调用 `Create`。 diff --git a/packages/agents-client/src/client.ts b/packages/agents-client/src/client.ts index 2e4e8e13b..3a61a8c7d 100644 --- a/packages/agents-client/src/client.ts +++ b/packages/agents-client/src/client.ts @@ -315,7 +315,7 @@ const runtimeCPUFields = new Set(["usage_seconds_total", "capacity_cores", "usag const runtimeMemoryFields = new Set(["usage_bytes", "limit_bytes"]); const runtimeObservationReasons = new Set([ "runtime_mode_not_observable", "allocation_pending", "runtime_not_running", - "source_not_configured", "sample_timeout", "sample_unavailable", + "sample_timeout", "sample_unavailable", ]); const runtimeProviderTypePattern = /^[a-z][a-z0-9_]{0,31}$/; const runtimeLifecycleStates = new Set(["active", "sleeping", "transitioning", "pending", "stopped"]); diff --git a/packages/agents-client/src/core-metrics.test.ts b/packages/agents-client/src/core-metrics.test.ts index 9599b6365..b160cf6c8 100644 --- a/packages/agents-client/src/core-metrics.test.ts +++ b/packages/agents-client/src/core-metrics.test.ts @@ -5,7 +5,7 @@ import { CoreMetricsClient, projectCoreMetrics } from "./core-metrics"; describe("Core metrics client", () => { it("reads /core/v1/metrics without the Beta header, with a bearer only when given", async () => { - const body = { object: "core.metrics", range: { start: "2026-09-24T00:00:00Z", end: "2026-09-24T01:00:00Z", resolution_seconds: 60 }, service: { status: "running" } }; + const body = { object: "core.metrics", range: { start: "2026-09-24T00:00:00Z", end: "2026-09-24T01:00:00Z", resolution_seconds: 60 }, service: { status: "running" }, execution: { slots_in_use: 0, slots_total: 4, connected_daemons: 0 } }; const fetch = vi.fn().mockImplementation(async () => new Response(JSON.stringify(body))); const client = new CoreMetricsClient({ fetch }); expect(client).not.toBeInstanceOf(OpenAIAgentsClient); @@ -28,11 +28,11 @@ describe("Core metrics projection", () => { object: "core.metrics", range: { start: "2026-09-24T00:00:00Z", end: "2026-09-24T01:00:00Z", resolution_seconds: 60 }, service: { status: "running", revision: "b134a1b5", execution_owner: true }, - execution: { slots_in_use: 3, slots_total: 4, queue_wait_ms: { p95: 2600 }, series: [{ start: "2026-09-24T00:00:00Z", queued: 1 }] }, + execution: { slots_in_use: 3, slots_total: 4, connected_daemons: 2, queue_wait_ms: { p95: 2600 }, series: [{ start: "2026-09-24T00:00:00Z", queued: 1 }] }, jobs: [{ id: "runtime_sampler", status: "weird", processed: 12 }], }); expect(metrics.service).toMatchObject({ revision: "b134a1b5", execution_owner: true, started_at: null }); - expect(metrics.execution).toMatchObject({ slots_in_use: 3, queued_turns: null, connected_daemons: null }); + expect(metrics.execution).toMatchObject({ slots_in_use: 3, queued_turns: null, connected_daemons: 2 }); expect(metrics.execution.queue_wait_ms).toEqual({ p50: null, p95: 2600 }); expect(metrics.execution.series[0]).toMatchObject({ queued: 1, in_progress: null }); expect(metrics.database).toMatchObject({ size_bytes: null, pool: { in_use: null, idle: null, max: null }, series: [] }); @@ -46,12 +46,15 @@ describe("Core metrics projection", () => { object: "core.metrics", range: { start: "2026-09-24T00:00:00Z", end: "2026-09-24T01:00:00Z", resolution_seconds: 60 }, service: { status: "draining" }, + execution: { slots_in_use: 0, slots_total: 4, connected_daemons: 0 }, }); expect(metrics.service.status).toBe("unknown"); }); - it("rejects a response that is not Core metrics or has no resolution", () => { + it("rejects a response that is not Core metrics or lacks its resolution or execution counts", () => { expect(() => projectCoreMetrics({ object: "list" })).toThrow(); expect(() => projectCoreMetrics({ object: "core.metrics", range: { start: "", end: "" }, service: { status: "running" } })).toThrow(); + const range = { start: "2026-09-24T00:00:00Z", end: "2026-09-24T01:00:00Z", resolution_seconds: 60 }; + expect(() => projectCoreMetrics({ object: "core.metrics", range, service: { status: "running" }, execution: { slots_in_use: 0, slots_total: 4 } })).toThrow(); }); }); diff --git a/packages/agents-client/src/core-metrics.ts b/packages/agents-client/src/core-metrics.ts index 9a3bf5fca..d1b14ceeb 100644 --- a/packages/agents-client/src/core-metrics.ts +++ b/packages/agents-client/src/core-metrics.ts @@ -68,14 +68,14 @@ export interface CoreMetrics { execution_owner: boolean | null; }; execution: { - slots_in_use: number | null; - slots_total: number | null; + slots_in_use: number; + slots_total: number; queued_turns: number | null; /** Queued Turns whose Session has no connected daemon (part of queued_turns). */ waiting_for_daemon: number | null; in_progress_turns: number | null; oldest_queued_seconds: number | null; - connected_daemons: number | null; + connected_daemons: number; /** Turns failed with execution_interrupted in the range. */ interrupted: number | null; /** Requests refused with execution_unavailable in the range. */ @@ -122,6 +122,12 @@ function number(value: unknown): number | null { return typeof value === "number" && Number.isFinite(value) ? value : null; } +function count(value: unknown, path: string): number { + const result = number(value); + if (result === null) throw new AgentCoreError(`Core metrics: ${path} is missing.`, 0, "invalid_response"); + return result; +} + function text(value: unknown): string | null { return typeof value === "string" && value ? value : null; } @@ -145,7 +151,7 @@ export function projectCoreMetrics(value: unknown): CoreMetrics { if (body.object !== "core.metrics") throw new AgentCoreError("Core metrics: unexpected object type.", 0, "invalid_response"); const range = record(body.range, "range"); const service = record(body.service, "service"); - const execution = optional(body.execution); + const execution = record(body.execution, "execution"); const database = optional(body.database); const pool = optional(database.pool); const process = optional(body.process); @@ -162,13 +168,13 @@ export function projectCoreMetrics(value: unknown): CoreMetrics { execution_owner: typeof service.execution_owner === "boolean" ? service.execution_owner : null, }, execution: { - slots_in_use: number(execution.slots_in_use), - slots_total: number(execution.slots_total), + slots_in_use: count(execution.slots_in_use, "execution.slots_in_use"), + slots_total: count(execution.slots_total, "execution.slots_total"), queued_turns: number(execution.queued_turns), waiting_for_daemon: number(execution.waiting_for_daemon), in_progress_turns: number(execution.in_progress_turns), oldest_queued_seconds: number(execution.oldest_queued_seconds), - connected_daemons: number(execution.connected_daemons), + connected_daemons: count(execution.connected_daemons, "execution.connected_daemons"), interrupted: number(execution.interrupted), unavailable: number(execution.unavailable), queue_wait_ms: latency(execution.queue_wait_ms), diff --git a/packages/agents-client/src/types.ts b/packages/agents-client/src/types.ts index ceac7b2c9..9594d5b11 100644 --- a/packages/agents-client/src/types.ts +++ b/packages/agents-client/src/types.ts @@ -995,7 +995,6 @@ export type RuntimeObservationReason = | "runtime_mode_not_observable" | "allocation_pending" | "runtime_not_running" - | "source_not_configured" | "sample_timeout" | "sample_unavailable"; diff --git a/services/core/cmd/server/core_metrics.go b/services/core/cmd/server/core_metrics.go index 74b92a0cb..f8c839f0a 100644 --- a/services/core/cmd/server/core_metrics.go +++ b/services/core/cmd/server/core_metrics.go @@ -30,7 +30,7 @@ func (s *coreMetricsSource) Live() coremetrics.Live { return coremetrics.Live{Pool: coremetrics.Pool{InUse: metricPtr(int64(stat.AcquiredConns())), Idle: metricPtr(int64(stat.IdleConns())), Max: metricPtr(int64(stat.MaxConns()))}, Scheduler: coremetrics.Job{ID: "scheduler", Status: w.Scheduler.Status, LastRunAt: w.Scheduler.LastRunAt, Processed: w.Scheduler.Processed, Failed: w.Scheduler.Failed}, ExecutionOwner: w.ExecutionOwner, SlotsTotal: w.SlotsTotal, SlotsInUse: w.SlotsInUse, - ConnectedDaemons: metricPtr(int64(len(s.registry.Devices())))} + ConnectedDaemons: int64(len(s.registry.Devices()))} } func (s *coreMetricsSource) Sample(ctx context.Context) coremetrics.Sample { sample := coremetrics.Sample{Healthy: true, PoolInUse: metricPtr(int64(s.pool.Stat().AcquiredConns()))} diff --git a/services/core/cmd/server/managed_setup_test.go b/services/core/cmd/server/managed_setup_test.go index f4a022aa8..aaaf9cd02 100644 --- a/services/core/cmd/server/managed_setup_test.go +++ b/services/core/cmd/server/managed_setup_test.go @@ -107,7 +107,7 @@ func committedSetup(value *deployment.Setup) func(context.Context) (deployment.S // deployment service does. That needs no storage. func credentialService(t *testing.T) func(owner, candidate deployment.Setup) (deployment.Setup, error) { t.Helper() - rules, err := placement.NewRules(providers.Builtin(), "") + rules, err := placement.NewRules(providers.Builtin(), "https://core.example") if err != nil { t.Fatal(err) } diff --git a/services/core/cmd/server/runtime_history.go b/services/core/cmd/server/runtime_history.go index 792d04769..da05b8481 100644 --- a/services/core/cmd/server/runtime_history.go +++ b/services/core/cmd/server/runtime_history.go @@ -26,10 +26,9 @@ type runtimeHistoryExporter interface { } func runtimeHistory(ctx context.Context, units *pgunit.Pool, config processconfig.RuntimeHistory) (runtimeHistorySetup, error) { - interval := config.SampleInterval capabilities := runtimehistory.Capabilities{ - CollectionMode: runtimehistory.CollectionPeriodic, SampleInterval: interval, Retention: 7 * 24 * time.Hour, - MinimumStep: max(30*time.Second, interval), MaximumRange: 24 * time.Hour, + SampleInterval: config.SampleInterval, Retention: 7 * 24 * time.Hour, + MinimumStep: max(30*time.Second, config.SampleInterval), MaximumRange: 24 * time.Hour, MaximumPoints: 1000, MaximumSeries: 64, MaximumTotalPoints: 10000, Metrics: []runtimehistory.Metric{runtimehistory.MetricCPU, runtimehistory.MetricMemory, runtimehistory.MetricTokens}, } @@ -38,7 +37,7 @@ func runtimeHistory(ctx context.Context, units *pgunit.Pool, config processconfi return runtimeHistorySetup{}, err } options := runtimeobs.ExportOptions{QueueCapacity: config.QueueCapacity, Timeout: config.Timeout} - setup := runtimeHistorySetup{Options: []runtimeobs.ServiceOption{runtimeobs.WithExporter(backend, options)}, Reader: backend, SampleInterval: interval, Prune: backend.Prune} + setup := runtimeHistorySetup{Options: []runtimeobs.ServiceOption{runtimeobs.WithExporter(backend, options)}, Reader: backend, SampleInterval: config.SampleInterval, Prune: backend.Prune} if config.Endpoint != "" { exporter, err := otlpexporter.New(ctx, otlpexporter.Config{Endpoint: config.Endpoint, Headers: config.Headers, Insecure: config.Insecure, RequestTimeout: config.Timeout}) if err != nil { diff --git a/services/core/cmd/server/runtime_history_test.go b/services/core/cmd/server/runtime_history_test.go index c81e5b3e4..2b47b75bc 100644 --- a/services/core/cmd/server/runtime_history_test.go +++ b/services/core/cmd/server/runtime_history_test.go @@ -26,7 +26,7 @@ func TestRuntimeHistoryUsesCoreDatabaseByDefault(t *testing.T) { t.Fatal("default history requires extra deployment") } capabilities := setup.Reader.Capabilities() - if !capabilities.Durable() || capabilities.Retention != 7*24*time.Hour { + if capabilities.Validate() != nil || capabilities.Retention != 7*24*time.Hour { t.Fatalf("incorrect default capabilities: %+v", capabilities) } if setup.SampleInterval != 30*time.Second { diff --git a/services/core/internal/api/core_store_validation_test.go b/services/core/internal/api/core_store_validation_test.go index 91cb319d0..bd950b480 100644 --- a/services/core/internal/api/core_store_validation_test.go +++ b/services/core/internal/api/core_store_validation_test.go @@ -22,7 +22,7 @@ import ( func TestCoreStoreValidationFieldsAndPublicFallback(t *testing.T) { // The deployment validates these inputs before it reaches storage, so // storage without a database is enough. - rules, err := placement.NewRules(providers.Builtin(), "") + rules, err := placement.NewRules(providers.Builtin(), "https://core.example") if err != nil { t.Fatal(err) } diff --git a/services/core/internal/api/runtime_history.go b/services/core/internal/api/runtime_history.go index 5d4d0c3fe..0e39a934d 100644 --- a/services/core/internal/api/runtime_history.go +++ b/services/core/internal/api/runtime_history.go @@ -18,9 +18,8 @@ const ( runtimeHistoryRequestBudget = 15 * time.Second ) -// RuntimeHistory queries durable Runtime history. Its capabilities declare -// whether this Core collects any; one that does not answers 503 -// runtime_history_unavailable. +// RuntimeHistory queries the durable Runtime history Core samples +// periodically. type RuntimeHistory interface { Capabilities() runtimehistory.Capabilities QuerySession(context.Context, string, string, runtimehistory.Range) (runtimehistory.Response, error) @@ -29,10 +28,6 @@ type RuntimeHistory interface { // getRuntimeHistory serves the administrator per-Session history read. func (h *Handler) getRuntimeHistory(w http.ResponseWriter, r *http.Request) { capabilities := h.RuntimeHistory.Capabilities() - if capabilities.Validate() != nil || !capabilities.Durable() { - writeError(w, http.StatusServiceUnavailable, "runtime_history_unavailable", "Durable Runtime history is not configured on this service.") - return - } requested, ok := readRuntimeHistoryRange(w, r, capabilities, time.Now().UTC()) if !ok { return @@ -95,7 +90,7 @@ func readRuntimeHistoryRange(w http.ResponseWriter, r *http.Request, capabilitie } func runtimeHistoryResponse(value runtimehistory.Response, expectedTenantID, expectedSessionID string, expectedRange runtimehistory.Range) (v1.RuntimeHistory, error) { - if err := value.Validate(time.Now().UTC()); err != nil || !value.Durable() || + if err := value.Validate(time.Now().UTC()); err != nil || value.TenantID != expectedTenantID || value.SessionID != expectedSessionID || !value.Requested.Start.Equal(expectedRange.Start) || !value.Requested.End.Equal(expectedRange.End) || value.Requested.MaxPoints != expectedRange.MaxPoints { return v1.RuntimeHistory{}, runtimehistory.ErrInvalidResult diff --git a/services/core/internal/api/runtime_history_test.go b/services/core/internal/api/runtime_history_test.go index 9e526b61e..ac5b3d5c0 100644 --- a/services/core/internal/api/runtime_history_test.go +++ b/services/core/internal/api/runtime_history_test.go @@ -39,35 +39,12 @@ func historyWith(service *runtimeHistoryFixture) func(*Dependencies, *testFakes) } } -func historyCapabilities(mode runtimehistory.CollectionMode) runtimehistory.Capabilities { - value := runtimehistory.Capabilities{ - CollectionMode: mode, Retention: 7 * 24 * time.Hour, MinimumStep: 30 * time.Second, +func historyCapabilities() runtimehistory.Capabilities { + return runtimehistory.Capabilities{ + SampleInterval: 30 * time.Second, Retention: 7 * 24 * time.Hour, MinimumStep: 30 * time.Second, MaximumRange: 24 * time.Hour, MaximumPoints: 1_000, MaximumSeries: 64, MaximumTotalPoints: 10_000, Metrics: []runtimehistory.Metric{runtimehistory.MetricCPU, runtimehistory.MetricMemory, runtimehistory.MetricTokens}, } - if mode == runtimehistory.CollectionPeriodic { - value.SampleInterval = 30 * time.Second - } - return value -} - -func TestRuntimeHistoryRequiresQualifiedPeriodicCollection(t *testing.T) { - service := &runtimeHistoryFixture{capabilities: historyCapabilities(runtimehistory.CollectionOnRead)} - handler, _, _ := adminTestHandler(t, historyWith(service)) - response := runtimeObservationRequest(handler, adminSessionsPath+uuid.NewString()+"/runtime-history?start=1&end=2") - if response.Code != http.StatusServiceUnavailable || service.calls != 0 { - t.Fatalf("on-read history reached query service: %d calls=%d body=%s", response.Code, service.calls, response.Body) - } -} - -func TestRuntimeHistoryFailsClosedForMalformedCapabilities(t *testing.T) { - service := &runtimeHistoryFixture{capabilities: historyCapabilities(runtimehistory.CollectionPeriodic)} - service.capabilities.Retention = 0 - handler, _, _ := adminTestHandler(t, historyWith(service)) - response := runtimeObservationRequest(handler, adminSessionsPath+uuid.NewString()+"/runtime-history?start=1&end=2") - if response.Code != http.StatusServiceUnavailable || service.calls != 0 { - t.Fatalf("malformed capabilities reached query service: %d calls=%d body=%s", response.Code, service.calls, response.Body) - } } func TestRuntimeHistoryRouteBindsAuthenticatedSessionAndPreservesCoverage(t *testing.T) { @@ -88,7 +65,7 @@ func TestRuntimeHistoryRouteBindsAuthenticatedSessionAndPreservesCoverage(t *tes ObservationCount: 1, ObservedCount: 1, CPUContributorCount: 1, MemoryContributorCount: 1, CPUUtilizationRatio: &zeroRatio, CPUCapacityCores: &capacity, MemoryUsageBytes: &zeroMemory, MemoryLimitBytes: &limit, } - service := &runtimeHistoryFixture{capabilities: historyCapabilities(runtimehistory.CollectionPeriodic)} + service := &runtimeHistoryFixture{capabilities: historyCapabilities()} handler, _, tenant := adminTestHandler(t, historyWith(service)) scope.TenantID = tenant service.response = runtimehistory.Response{ @@ -116,7 +93,7 @@ func TestRuntimeHistoryRouteBindsAuthenticatedSessionAndPreservesCoverage(t *tes } func TestRuntimeHistoryRejectsUnsafeQueriesAndFailures(t *testing.T) { - service := &runtimeHistoryFixture{capabilities: historyCapabilities(runtimehistory.CollectionPeriodic)} + service := &runtimeHistoryFixture{capabilities: historyCapabilities()} handler, _, _ := adminTestHandler(t, historyWith(service)) sessionID := uuid.NewString() for _, query := range []string{ @@ -161,7 +138,7 @@ func TestRuntimeHistoryRejectsMismatchedServiceResponses(t *testing.T) { now := time.Now().UTC().Truncate(time.Second) start := now.Add(-time.Hour) sessionID := uuid.NewString() - service := &runtimeHistoryFixture{capabilities: historyCapabilities(runtimehistory.CollectionPeriodic)} + service := &runtimeHistoryFixture{capabilities: historyCapabilities()} handler, _, tenant := adminTestHandler(t, historyWith(service)) base := runtimehistory.Response{ Capabilities: service.capabilities, @@ -192,7 +169,7 @@ func TestRuntimeHistoryRejectsMismatchedServiceResponses(t *testing.T) { } func TestRuntimeHistoryDefaultPointBudgetRespectsCapabilities(t *testing.T) { - capabilities := historyCapabilities(runtimehistory.CollectionPeriodic) + capabilities := historyCapabilities() capabilities.MaximumPoints = 60 service := &runtimeHistoryFixture{capabilities: capabilities, err: runtimehistory.ErrUnavailable} handler, _, _ := adminTestHandler(t, historyWith(service)) @@ -207,7 +184,7 @@ func TestRuntimeHistoryExpectedCoverageUsesOverflowSafeCeiling(t *testing.T) { now := time.Now().UTC().Truncate(time.Second) start := now.Add(-time.Hour) huge := (time.Duration(1<<63-1) / time.Second) * time.Second - capabilities := historyCapabilities(runtimehistory.CollectionPeriodic) + capabilities := historyCapabilities() capabilities.SampleInterval = huge capabilities.Retention = huge capabilities.MaximumRange = time.Hour diff --git a/services/core/internal/coremetrics/service.go b/services/core/internal/coremetrics/service.go index cfb24849d..84dfc5f91 100644 --- a/services/core/internal/coremetrics/service.go +++ b/services/core/internal/coremetrics/service.go @@ -231,7 +231,7 @@ func (s *Service) Read(ctx context.Context, name string) (View, error) { view.Execution.Series[i].QueueWaitP95MS = history.Buckets[view.Execution.Series[i].Start] } } - if live.ExecutionOwner == nil || (live.SlotsTotal != nil && *live.SlotsTotal > 0 && !*live.ExecutionOwner) { + if live.ExecutionOwner == nil || !*live.ExecutionOwner { view.Service.Status = "degraded" } for _, job := range view.Jobs { diff --git a/services/core/internal/coremetrics/service_test.go b/services/core/internal/coremetrics/service_test.go index d582a2827..f901a8c62 100644 --- a/services/core/internal/coremetrics/service_test.go +++ b/services/core/internal/coremetrics/service_test.go @@ -25,7 +25,7 @@ func (f *fixtureSource) Live() Live { return f.live } func fixtureService(t *testing.T) (*Service, *fixtureSource, time.Time) { t.Helper() now := time.Date(2026, 9, 25, 12, 0, 20, 0, time.UTC) - source := &fixtureSource{history: History{Buckets: map[time.Time]*float64{}}, live: Live{ExecutionOwner: ptr(true), SlotsInUse: ptr(int64(2)), SlotsTotal: ptr(int64(4))}} + source := &fixtureSource{history: History{Buckets: map[time.Time]*float64{}}, live: Live{ExecutionOwner: ptr(true), SlotsInUse: 2, SlotsTotal: 4}} service, err := New(now.Add(-2*time.Hour), strings.Repeat("a", 40), source, Periodic{ID: "runtime_sampler"}, Periodic{ID: "history_cleanup"}, Periodic{ID: "audit_cleanup"}) if err != nil { t.Fatal(err) diff --git a/services/core/internal/coremetrics/types.go b/services/core/internal/coremetrics/types.go index f4c132e5c..9aafca860 100644 --- a/services/core/internal/coremetrics/types.go +++ b/services/core/internal/coremetrics/types.go @@ -44,13 +44,13 @@ type Pool struct { Max *int64 `json:"max" extensions:"x-nullable"` } type Execution struct { - SlotsInUse *int64 `json:"slots_in_use" extensions:"x-nullable"` - SlotsTotal *int64 `json:"slots_total" extensions:"x-nullable"` + SlotsInUse int64 `json:"slots_in_use"` + SlotsTotal int64 `json:"slots_total"` QueuedTurns *int64 `json:"queued_turns" extensions:"x-nullable"` WaitingForDaemon *int64 `json:"waiting_for_daemon" extensions:"x-nullable"` InProgressTurns *int64 `json:"in_progress_turns" extensions:"x-nullable"` OldestQueuedSeconds *float64 `json:"oldest_queued_seconds" extensions:"x-nullable"` - ConnectedDaemons *int64 `json:"connected_daemons" extensions:"x-nullable"` + ConnectedDaemons int64 `json:"connected_daemons"` Interrupted *int64 `json:"interrupted" extensions:"x-nullable"` Unavailable *int64 `json:"unavailable" extensions:"x-nullable"` QueueWaitMS Latency `json:"queue_wait_ms"` @@ -104,7 +104,7 @@ type Sample struct { Process Process } type Live struct { - SlotsInUse, SlotsTotal, ConnectedDaemons *int64 + SlotsInUse, SlotsTotal, ConnectedDaemons int64 ExecutionOwner *bool Pool Pool Scheduler Job diff --git a/services/core/internal/db/queries/runtime_allocations.sql b/services/core/internal/db/queries/runtime_allocations.sql index fc62d9deb..6902e54ab 100644 --- a/services/core/internal/db/queries/runtime_allocations.sql +++ b/services/core/internal/db/queries/runtime_allocations.sql @@ -3,14 +3,14 @@ INSERT INTO runtime_allocations (id, environment_id, device_id, provider_key, no VALUES ($1, $2, $3, $4, $5, $6) RETURNING *; -- name: GetRuntimeAllocation :one -SELECT sqlc.embed(a), e.session_id, s.tenant_id, s.deleted_at, (CASE WHEN a.compute_phase NOT IN ('disabled', 'running') THEN a.compute_retained_until IS NOT NULL AND a.compute_retained_until <= clock_timestamp() ELSE a.node_id IS NULL AND (SELECT mode FROM runtime_deployment) <> 'direct' AND a.kept_at <= clock_timestamp() - interval '1 hour' END)::boolean AS expired +SELECT sqlc.embed(a), e.session_id, s.tenant_id, s.deleted_at, (a.compute_phase NOT IN ('disabled', 'running') AND a.compute_retained_until IS NOT NULL AND a.compute_retained_until <= clock_timestamp())::boolean AS expired FROM runtime_allocations a JOIN environments e ON e.id = a.environment_id JOIN sessions s ON s.id = e.session_id WHERE s.tenant_id = $1 AND a.environment_id = $2; -- name: ListRuntimeAllocations :many -SELECT sqlc.embed(a), e.session_id, s.tenant_id, s.deleted_at, (CASE WHEN a.compute_phase NOT IN ('disabled', 'running') THEN a.compute_retained_until IS NOT NULL AND a.compute_retained_until <= clock_timestamp() ELSE a.node_id IS NULL AND (SELECT mode FROM runtime_deployment) <> 'direct' AND a.kept_at <= clock_timestamp() - interval '1 hour' END)::boolean AS expired +SELECT sqlc.embed(a), e.session_id, s.tenant_id, s.deleted_at, (a.compute_phase NOT IN ('disabled', 'running') AND a.compute_retained_until IS NOT NULL AND a.compute_retained_until <= clock_timestamp())::boolean AS expired FROM runtime_allocations a JOIN environments e ON e.id = a.environment_id JOIN sessions s ON s.id = e.session_id @@ -32,13 +32,6 @@ LIMIT $2; -- name: ObserveRuntimeRunning :one UPDATE runtime_allocations SET state = 'running', create_settled = true WHERE id = $1 AND state IN ('creating', 'running') -AND (node_id IS NOT NULL OR (SELECT mode FROM runtime_deployment) = 'direct' OR kept_at > clock_timestamp() - interval '1 hour') -RETURNING *; - --- name: KeepRuntimeAllocation :one -UPDATE runtime_allocations SET kept_at = clock_timestamp() -WHERE id = $1 AND state = 'running' -AND (node_id IS NOT NULL OR (SELECT mode FROM runtime_deployment) = 'direct' OR kept_at > clock_timestamp() - interval '1 hour') RETURNING *; -- name: RequestRuntimeCleanup :one diff --git a/services/core/internal/db/queries/runtime_lifecycle_nodes.sql b/services/core/internal/db/queries/runtime_lifecycle_nodes.sql index 334617404..a30354cbd 100644 --- a/services/core/internal/db/queries/runtime_lifecycle_nodes.sql +++ b/services/core/internal/db/queries/runtime_lifecycle_nodes.sql @@ -2,11 +2,11 @@ SELECT n.id FROM runtime_nodes n CROSS JOIN runtime_deployment d WHERE n.removed_at IS NULL AND n.installation_id=d.installation_id AND d.mode='nodes' UNION ALL -SELECT NULL::uuid AS id FROM runtime_deployment WHERE mode IN ('','direct') +SELECT NULL::uuid AS id FROM runtime_deployment WHERE mode = 'direct' ORDER BY id; -- name: ListRuntimeAllocationsForNode :many -SELECT sqlc.embed(a), e.session_id, s.tenant_id, s.deleted_at, (CASE WHEN a.compute_phase NOT IN ('disabled', 'running') THEN a.compute_retained_until IS NOT NULL AND a.compute_retained_until <= clock_timestamp() ELSE a.node_id IS NULL AND (SELECT mode FROM runtime_deployment) <> 'direct' AND a.kept_at <= clock_timestamp() - interval '1 hour' END)::boolean AS expired +SELECT sqlc.embed(a), e.session_id, s.tenant_id, s.deleted_at, (a.compute_phase NOT IN ('disabled', 'running') AND a.compute_retained_until IS NOT NULL AND a.compute_retained_until <= clock_timestamp())::boolean AS expired FROM runtime_allocations a JOIN environments e ON e.id=a.environment_id JOIN sessions s ON s.id=e.session_id diff --git a/services/core/internal/db/queries/runtime_suspension.sql b/services/core/internal/db/queries/runtime_suspension.sql index 6a1ccbeef..83b5a38ca 100644 --- a/services/core/internal/db/queries/runtime_suspension.sql +++ b/services/core/internal/db/queries/runtime_suspension.sql @@ -3,12 +3,10 @@ UPDATE runtime_allocations SET compute_phase_changed_at = CASE WHEN compute_phase = sqlc.arg(phase)::text THEN compute_phase_changed_at ELSE clock_timestamp() END, compute_phase = sqlc.arg(phase), compute_state = sqlc.arg(state)::jsonb, compute_revision = compute_revision + 1, - compute_retained_until = sqlc.narg(retained_until), - kept_at = CASE WHEN sqlc.arg(phase)::text = 'running' THEN clock_timestamp() ELSE kept_at END + compute_retained_until = sqlc.narg(retained_until) WHERE runtime_allocations.id = sqlc.arg(id) AND compute_revision = sqlc.arg(revision) AND state = 'running' AND EXISTS (SELECT 1 FROM environments e WHERE e.id = runtime_allocations.environment_id AND e.initialization = 'complete') - AND ((compute_phase IN ('disabled','running') AND (node_id IS NOT NULL OR (SELECT mode FROM runtime_deployment) = 'direct' OR kept_at > clock_timestamp() - interval '1 hour')) - OR (compute_phase NOT IN ('disabled','running') AND compute_retained_until > clock_timestamp())) + AND (compute_phase IN ('disabled','running') OR compute_retained_until > clock_timestamp()) RETURNING *; -- name: TouchRuntimeActivity :exec diff --git a/services/core/internal/db/queries/sandbox_reset.sql b/services/core/internal/db/queries/sandbox_reset.sql index 5b53b06e5..536219525 100644 --- a/services/core/internal/db/queries/sandbox_reset.sql +++ b/services/core/internal/db/queries/sandbox_reset.sql @@ -60,11 +60,10 @@ observed AS MATERIALIZED (SELECT clock_timestamp() AS as_of), held AS ( SELECT a.deployment_generation, a.node_id, s.id AS session_id, e.id AS environment_id, false AS pending, (a.state = 'cleanup_pending' OR s.deleted_at IS NOT NULL OR e.status IN ('failed', 'expired') - OR CASE WHEN a.compute_phase NOT IN ('disabled', 'running') - THEN a.compute_retained_until IS NOT NULL AND a.compute_retained_until <= observed.as_of - ELSE a.node_id IS NULL AND d.mode <> 'direct' AND a.kept_at <= observed.as_of - interval '1 hour' END) AS cleanup + OR (a.compute_phase NOT IN ('disabled', 'running') AND a.compute_retained_until IS NOT NULL + AND a.compute_retained_until <= observed.as_of)) AS cleanup FROM runtime_allocations a JOIN environments e ON e.id = a.environment_id - JOIN sessions s ON s.id = e.session_id CROSS JOIN deployment d CROSS JOIN observed + JOIN sessions s ON s.id = e.session_id CROSS JOIN observed WHERE a.state <> 'released' UNION ALL SELECT p.deployment_generation, p.node_id, s.id, e.id, true, false diff --git a/services/core/internal/db/sqlc/models.go b/services/core/internal/db/sqlc/models.go index b5936f30b..d315804dd 100644 --- a/services/core/internal/db/sqlc/models.go +++ b/services/core/internal/db/sqlc/models.go @@ -221,7 +221,6 @@ type RuntimeAllocation struct { State string `json:"state"` CreateSettled bool `json:"create_settled"` CreatedAt pgtype.Timestamptz `json:"created_at"` - KeptAt pgtype.Timestamptz `json:"kept_at"` ReleasedAt pgtype.Timestamptz `json:"released_at"` ComputePhase string `json:"compute_phase"` ComputeRevision int64 `json:"compute_revision"` diff --git a/services/core/internal/db/sqlc/runtime_allocations.sql.go b/services/core/internal/db/sqlc/runtime_allocations.sql.go index 0caae4420..4725ab1a1 100644 --- a/services/core/internal/db/sqlc/runtime_allocations.sql.go +++ b/services/core/internal/db/sqlc/runtime_allocations.sql.go @@ -13,7 +13,7 @@ import ( const createRuntimeAllocation = `-- name: CreateRuntimeAllocation :one INSERT INTO runtime_allocations (id, environment_id, device_id, provider_key, node_id, deployment_generation) -VALUES ($1, $2, $3, $4, $5, $6) RETURNING id, environment_id, device_id, provider_key, state, create_settled, created_at, kept_at, released_at, compute_phase, compute_revision, compute_state, compute_activity_at, compute_wake_requested, compute_retained_until, node_id, observation_error, compute_phase_changed_at, deployment_generation +VALUES ($1, $2, $3, $4, $5, $6) RETURNING id, environment_id, device_id, provider_key, state, create_settled, created_at, released_at, compute_phase, compute_revision, compute_state, compute_activity_at, compute_wake_requested, compute_retained_until, node_id, observation_error, compute_phase_changed_at, deployment_generation ` type CreateRuntimeAllocationParams struct { @@ -43,7 +43,6 @@ func (q *Queries) CreateRuntimeAllocation(ctx context.Context, arg CreateRuntime &i.State, &i.CreateSettled, &i.CreatedAt, - &i.KeptAt, &i.ReleasedAt, &i.ComputePhase, &i.ComputeRevision, @@ -60,7 +59,7 @@ func (q *Queries) CreateRuntimeAllocation(ctx context.Context, arg CreateRuntime } const getRuntimeAllocation = `-- name: GetRuntimeAllocation :one -SELECT a.id, a.environment_id, a.device_id, a.provider_key, a.state, a.create_settled, a.created_at, a.kept_at, a.released_at, a.compute_phase, a.compute_revision, a.compute_state, a.compute_activity_at, a.compute_wake_requested, a.compute_retained_until, a.node_id, a.observation_error, a.compute_phase_changed_at, a.deployment_generation, e.session_id, s.tenant_id, s.deleted_at, (CASE WHEN a.compute_phase NOT IN ('disabled', 'running') THEN a.compute_retained_until IS NOT NULL AND a.compute_retained_until <= clock_timestamp() ELSE a.node_id IS NULL AND (SELECT mode FROM runtime_deployment) <> 'direct' AND a.kept_at <= clock_timestamp() - interval '1 hour' END)::boolean AS expired +SELECT a.id, a.environment_id, a.device_id, a.provider_key, a.state, a.create_settled, a.created_at, a.released_at, a.compute_phase, a.compute_revision, a.compute_state, a.compute_activity_at, a.compute_wake_requested, a.compute_retained_until, a.node_id, a.observation_error, a.compute_phase_changed_at, a.deployment_generation, e.session_id, s.tenant_id, s.deleted_at, (a.compute_phase NOT IN ('disabled', 'running') AND a.compute_retained_until IS NOT NULL AND a.compute_retained_until <= clock_timestamp())::boolean AS expired FROM runtime_allocations a JOIN environments e ON e.id = a.environment_id JOIN sessions s ON s.id = e.session_id @@ -91,7 +90,6 @@ func (q *Queries) GetRuntimeAllocation(ctx context.Context, arg GetRuntimeAlloca &i.RuntimeAllocation.State, &i.RuntimeAllocation.CreateSettled, &i.RuntimeAllocation.CreatedAt, - &i.RuntimeAllocation.KeptAt, &i.RuntimeAllocation.ReleasedAt, &i.RuntimeAllocation.ComputePhase, &i.RuntimeAllocation.ComputeRevision, @@ -111,42 +109,8 @@ func (q *Queries) GetRuntimeAllocation(ctx context.Context, arg GetRuntimeAlloca return i, err } -const keepRuntimeAllocation = `-- name: KeepRuntimeAllocation :one -UPDATE runtime_allocations SET kept_at = clock_timestamp() -WHERE id = $1 AND state = 'running' -AND (node_id IS NOT NULL OR (SELECT mode FROM runtime_deployment) = 'direct' OR kept_at > clock_timestamp() - interval '1 hour') -RETURNING id, environment_id, device_id, provider_key, state, create_settled, created_at, kept_at, released_at, compute_phase, compute_revision, compute_state, compute_activity_at, compute_wake_requested, compute_retained_until, node_id, observation_error, compute_phase_changed_at, deployment_generation -` - -func (q *Queries) KeepRuntimeAllocation(ctx context.Context, id pgtype.UUID) (RuntimeAllocation, error) { - row := q.db.QueryRow(ctx, keepRuntimeAllocation, id) - var i RuntimeAllocation - err := row.Scan( - &i.ID, - &i.EnvironmentID, - &i.DeviceID, - &i.ProviderKey, - &i.State, - &i.CreateSettled, - &i.CreatedAt, - &i.KeptAt, - &i.ReleasedAt, - &i.ComputePhase, - &i.ComputeRevision, - &i.ComputeState, - &i.ComputeActivityAt, - &i.ComputeWakeRequested, - &i.ComputeRetainedUntil, - &i.NodeID, - &i.ObservationError, - &i.ComputePhaseChangedAt, - &i.DeploymentGeneration, - ) - return i, err -} - const listRuntimeAllocations = `-- name: ListRuntimeAllocations :many -SELECT a.id, a.environment_id, a.device_id, a.provider_key, a.state, a.create_settled, a.created_at, a.kept_at, a.released_at, a.compute_phase, a.compute_revision, a.compute_state, a.compute_activity_at, a.compute_wake_requested, a.compute_retained_until, a.node_id, a.observation_error, a.compute_phase_changed_at, a.deployment_generation, e.session_id, s.tenant_id, s.deleted_at, (CASE WHEN a.compute_phase NOT IN ('disabled', 'running') THEN a.compute_retained_until IS NOT NULL AND a.compute_retained_until <= clock_timestamp() ELSE a.node_id IS NULL AND (SELECT mode FROM runtime_deployment) <> 'direct' AND a.kept_at <= clock_timestamp() - interval '1 hour' END)::boolean AS expired +SELECT a.id, a.environment_id, a.device_id, a.provider_key, a.state, a.create_settled, a.created_at, a.released_at, a.compute_phase, a.compute_revision, a.compute_state, a.compute_activity_at, a.compute_wake_requested, a.compute_retained_until, a.node_id, a.observation_error, a.compute_phase_changed_at, a.deployment_generation, e.session_id, s.tenant_id, s.deleted_at, (a.compute_phase NOT IN ('disabled', 'running') AND a.compute_retained_until IS NOT NULL AND a.compute_retained_until <= clock_timestamp())::boolean AS expired FROM runtime_allocations a JOIN environments e ON e.id = a.environment_id JOIN sessions s ON s.id = e.session_id @@ -179,7 +143,6 @@ func (q *Queries) ListRuntimeAllocations(ctx context.Context, id pgtype.UUID) ([ &i.RuntimeAllocation.State, &i.RuntimeAllocation.CreateSettled, &i.RuntimeAllocation.CreatedAt, - &i.RuntimeAllocation.KeptAt, &i.RuntimeAllocation.ReleasedAt, &i.RuntimeAllocation.ComputePhase, &i.RuntimeAllocation.ComputeRevision, @@ -252,8 +215,7 @@ func (q *Queries) ListRuntimeObservationSessions(ctx context.Context, arg ListRu const observeRuntimeRunning = `-- name: ObserveRuntimeRunning :one UPDATE runtime_allocations SET state = 'running', create_settled = true WHERE id = $1 AND state IN ('creating', 'running') -AND (node_id IS NOT NULL OR (SELECT mode FROM runtime_deployment) = 'direct' OR kept_at > clock_timestamp() - interval '1 hour') -RETURNING id, environment_id, device_id, provider_key, state, create_settled, created_at, kept_at, released_at, compute_phase, compute_revision, compute_state, compute_activity_at, compute_wake_requested, compute_retained_until, node_id, observation_error, compute_phase_changed_at, deployment_generation +RETURNING id, environment_id, device_id, provider_key, state, create_settled, created_at, released_at, compute_phase, compute_revision, compute_state, compute_activity_at, compute_wake_requested, compute_retained_until, node_id, observation_error, compute_phase_changed_at, deployment_generation ` func (q *Queries) ObserveRuntimeRunning(ctx context.Context, id pgtype.UUID) (RuntimeAllocation, error) { @@ -267,7 +229,6 @@ func (q *Queries) ObserveRuntimeRunning(ctx context.Context, id pgtype.UUID) (Ru &i.State, &i.CreateSettled, &i.CreatedAt, - &i.KeptAt, &i.ReleasedAt, &i.ComputePhase, &i.ComputeRevision, @@ -285,7 +246,7 @@ func (q *Queries) ObserveRuntimeRunning(ctx context.Context, id pgtype.UUID) (Ru const releaseRuntimeAllocation = `-- name: ReleaseRuntimeAllocation :one UPDATE runtime_allocations SET state = 'released', released_at = clock_timestamp() -WHERE id = $1 AND state = 'cleanup_pending' AND create_settled RETURNING id, environment_id, device_id, provider_key, state, create_settled, created_at, kept_at, released_at, compute_phase, compute_revision, compute_state, compute_activity_at, compute_wake_requested, compute_retained_until, node_id, observation_error, compute_phase_changed_at, deployment_generation +WHERE id = $1 AND state = 'cleanup_pending' AND create_settled RETURNING id, environment_id, device_id, provider_key, state, create_settled, created_at, released_at, compute_phase, compute_revision, compute_state, compute_activity_at, compute_wake_requested, compute_retained_until, node_id, observation_error, compute_phase_changed_at, deployment_generation ` func (q *Queries) ReleaseRuntimeAllocation(ctx context.Context, id pgtype.UUID) (RuntimeAllocation, error) { @@ -299,7 +260,6 @@ func (q *Queries) ReleaseRuntimeAllocation(ctx context.Context, id pgtype.UUID) &i.State, &i.CreateSettled, &i.CreatedAt, - &i.KeptAt, &i.ReleasedAt, &i.ComputePhase, &i.ComputeRevision, @@ -317,7 +277,7 @@ func (q *Queries) ReleaseRuntimeAllocation(ctx context.Context, id pgtype.UUID) const requestRuntimeCleanup = `-- name: RequestRuntimeCleanup :one UPDATE runtime_allocations SET state = 'cleanup_pending' -WHERE id = $1 AND state <> 'released' RETURNING id, environment_id, device_id, provider_key, state, create_settled, created_at, kept_at, released_at, compute_phase, compute_revision, compute_state, compute_activity_at, compute_wake_requested, compute_retained_until, node_id, observation_error, compute_phase_changed_at, deployment_generation +WHERE id = $1 AND state <> 'released' RETURNING id, environment_id, device_id, provider_key, state, create_settled, created_at, released_at, compute_phase, compute_revision, compute_state, compute_activity_at, compute_wake_requested, compute_retained_until, node_id, observation_error, compute_phase_changed_at, deployment_generation ` func (q *Queries) RequestRuntimeCleanup(ctx context.Context, id pgtype.UUID) (RuntimeAllocation, error) { @@ -331,7 +291,6 @@ func (q *Queries) RequestRuntimeCleanup(ctx context.Context, id pgtype.UUID) (Ru &i.State, &i.CreateSettled, &i.CreatedAt, - &i.KeptAt, &i.ReleasedAt, &i.ComputePhase, &i.ComputeRevision, @@ -349,7 +308,7 @@ func (q *Queries) RequestRuntimeCleanup(ctx context.Context, id pgtype.UUID) (Ru const settleRuntimeCreation = `-- name: SettleRuntimeCreation :one UPDATE runtime_allocations SET create_settled = true -WHERE id = $1 AND state <> 'released' RETURNING id, environment_id, device_id, provider_key, state, create_settled, created_at, kept_at, released_at, compute_phase, compute_revision, compute_state, compute_activity_at, compute_wake_requested, compute_retained_until, node_id, observation_error, compute_phase_changed_at, deployment_generation +WHERE id = $1 AND state <> 'released' RETURNING id, environment_id, device_id, provider_key, state, create_settled, created_at, released_at, compute_phase, compute_revision, compute_state, compute_activity_at, compute_wake_requested, compute_retained_until, node_id, observation_error, compute_phase_changed_at, deployment_generation ` func (q *Queries) SettleRuntimeCreation(ctx context.Context, id pgtype.UUID) (RuntimeAllocation, error) { @@ -363,7 +322,6 @@ func (q *Queries) SettleRuntimeCreation(ctx context.Context, id pgtype.UUID) (Ru &i.State, &i.CreateSettled, &i.CreatedAt, - &i.KeptAt, &i.ReleasedAt, &i.ComputePhase, &i.ComputeRevision, diff --git a/services/core/internal/db/sqlc/runtime_lifecycle_nodes.sql.go b/services/core/internal/db/sqlc/runtime_lifecycle_nodes.sql.go index e4bc7dae0..c1f443347 100644 --- a/services/core/internal/db/sqlc/runtime_lifecycle_nodes.sql.go +++ b/services/core/internal/db/sqlc/runtime_lifecycle_nodes.sql.go @@ -50,7 +50,7 @@ func (q *Queries) GetRuntimeLifecyclePlacement(ctx context.Context, arg GetRunti } const listRuntimeAllocationsForNode = `-- name: ListRuntimeAllocationsForNode :many -SELECT a.id, a.environment_id, a.device_id, a.provider_key, a.state, a.create_settled, a.created_at, a.kept_at, a.released_at, a.compute_phase, a.compute_revision, a.compute_state, a.compute_activity_at, a.compute_wake_requested, a.compute_retained_until, a.node_id, a.observation_error, a.compute_phase_changed_at, a.deployment_generation, e.session_id, s.tenant_id, s.deleted_at, (CASE WHEN a.compute_phase NOT IN ('disabled', 'running') THEN a.compute_retained_until IS NOT NULL AND a.compute_retained_until <= clock_timestamp() ELSE a.node_id IS NULL AND (SELECT mode FROM runtime_deployment) <> 'direct' AND a.kept_at <= clock_timestamp() - interval '1 hour' END)::boolean AS expired +SELECT a.id, a.environment_id, a.device_id, a.provider_key, a.state, a.create_settled, a.created_at, a.released_at, a.compute_phase, a.compute_revision, a.compute_state, a.compute_activity_at, a.compute_wake_requested, a.compute_retained_until, a.node_id, a.observation_error, a.compute_phase_changed_at, a.deployment_generation, e.session_id, s.tenant_id, s.deleted_at, (a.compute_phase NOT IN ('disabled', 'running') AND a.compute_retained_until IS NOT NULL AND a.compute_retained_until <= clock_timestamp())::boolean AS expired FROM runtime_allocations a JOIN environments e ON e.id=a.environment_id JOIN sessions s ON s.id=e.session_id @@ -89,7 +89,6 @@ func (q *Queries) ListRuntimeAllocationsForNode(ctx context.Context, arg ListRun &i.RuntimeAllocation.State, &i.RuntimeAllocation.CreateSettled, &i.RuntimeAllocation.CreatedAt, - &i.RuntimeAllocation.KeptAt, &i.RuntimeAllocation.ReleasedAt, &i.RuntimeAllocation.ComputePhase, &i.RuntimeAllocation.ComputeRevision, @@ -120,7 +119,7 @@ const listRuntimeLifecycleNodes = `-- name: ListRuntimeLifecycleNodes :many SELECT n.id FROM runtime_nodes n CROSS JOIN runtime_deployment d WHERE n.removed_at IS NULL AND n.installation_id=d.installation_id AND d.mode='nodes' UNION ALL -SELECT NULL::uuid AS id FROM runtime_deployment WHERE mode IN ('','direct') +SELECT NULL::uuid AS id FROM runtime_deployment WHERE mode = 'direct' ORDER BY id ` diff --git a/services/core/internal/db/sqlc/runtime_suspension.sql.go b/services/core/internal/db/sqlc/runtime_suspension.sql.go index ed6b5bb50..8b01bf155 100644 --- a/services/core/internal/db/sqlc/runtime_suspension.sql.go +++ b/services/core/internal/db/sqlc/runtime_suspension.sql.go @@ -109,13 +109,11 @@ UPDATE runtime_allocations SET compute_phase_changed_at = CASE WHEN compute_phase = $1::text THEN compute_phase_changed_at ELSE clock_timestamp() END, compute_phase = $1, compute_state = $2::jsonb, compute_revision = compute_revision + 1, - compute_retained_until = $3, - kept_at = CASE WHEN $1::text = 'running' THEN clock_timestamp() ELSE kept_at END + compute_retained_until = $3 WHERE runtime_allocations.id = $4 AND compute_revision = $5 AND state = 'running' AND EXISTS (SELECT 1 FROM environments e WHERE e.id = runtime_allocations.environment_id AND e.initialization = 'complete') - AND ((compute_phase IN ('disabled','running') AND (node_id IS NOT NULL OR (SELECT mode FROM runtime_deployment) = 'direct' OR kept_at > clock_timestamp() - interval '1 hour')) - OR (compute_phase NOT IN ('disabled','running') AND compute_retained_until > clock_timestamp())) -RETURNING id, environment_id, device_id, provider_key, state, create_settled, created_at, kept_at, released_at, compute_phase, compute_revision, compute_state, compute_activity_at, compute_wake_requested, compute_retained_until, node_id, observation_error, compute_phase_changed_at, deployment_generation + AND (compute_phase IN ('disabled','running') OR compute_retained_until > clock_timestamp()) +RETURNING id, environment_id, device_id, provider_key, state, create_settled, created_at, released_at, compute_phase, compute_revision, compute_state, compute_activity_at, compute_wake_requested, compute_retained_until, node_id, observation_error, compute_phase_changed_at, deployment_generation ` type SetRuntimeComputeParams struct { @@ -143,7 +141,6 @@ func (q *Queries) SetRuntimeCompute(ctx context.Context, arg SetRuntimeComputePa &i.State, &i.CreateSettled, &i.CreatedAt, - &i.KeptAt, &i.ReleasedAt, &i.ComputePhase, &i.ComputeRevision, diff --git a/services/core/internal/db/sqlc/sandbox_reset.sql.go b/services/core/internal/db/sqlc/sandbox_reset.sql.go index 8929d242b..8b1d18f72 100644 --- a/services/core/internal/db/sqlc/sandbox_reset.sql.go +++ b/services/core/internal/db/sqlc/sandbox_reset.sql.go @@ -55,11 +55,10 @@ observed AS MATERIALIZED (SELECT clock_timestamp() AS as_of), held AS ( SELECT a.deployment_generation, a.node_id, s.id AS session_id, e.id AS environment_id, false AS pending, (a.state = 'cleanup_pending' OR s.deleted_at IS NOT NULL OR e.status IN ('failed', 'expired') - OR CASE WHEN a.compute_phase NOT IN ('disabled', 'running') - THEN a.compute_retained_until IS NOT NULL AND a.compute_retained_until <= observed.as_of - ELSE a.node_id IS NULL AND d.mode <> 'direct' AND a.kept_at <= observed.as_of - interval '1 hour' END) AS cleanup + OR (a.compute_phase NOT IN ('disabled', 'running') AND a.compute_retained_until IS NOT NULL + AND a.compute_retained_until <= observed.as_of)) AS cleanup FROM runtime_allocations a JOIN environments e ON e.id = a.environment_id - JOIN sessions s ON s.id = e.session_id CROSS JOIN deployment d CROSS JOIN observed + JOIN sessions s ON s.id = e.session_id CROSS JOIN observed WHERE a.state <> 'released' UNION ALL SELECT p.deployment_generation, p.node_id, s.id, e.id, true, false diff --git a/services/core/internal/deployment/allocation.go b/services/core/internal/deployment/allocation.go index c825dda4f..02324ff49 100644 --- a/services/core/internal/deployment/allocation.go +++ b/services/core/internal/deployment/allocation.go @@ -33,10 +33,10 @@ type Allocation struct { // Replayed reports that ReserveAllocation returned an existing allocation, // which never authorizes another Create. Replayed bool - // Expired reports, by the database clock, that the allocation's lease or - // retention has passed. - Expired bool - CreatedAt, KeptAt time.Time + // Expired reports, by the database clock, that the retention of the + // allocation's suspended compute has passed. + Expired bool + CreatedAt time.Time } // Key names the allocation's Environment. diff --git a/services/core/internal/deployment/allocations.go b/services/core/internal/deployment/allocations.go index 2d10c6c1d..916e2e364 100644 --- a/services/core/internal/deployment/allocations.go +++ b/services/core/internal/deployment/allocations.go @@ -99,11 +99,16 @@ func (e *ExecutionOperations) ObserveRunning(ctx context.Context, owner Allocati }) } -// KeepAllocation follows an authenticated connection and a successful -// provider observation. A keepalive never revives cleanup or an expired lease. -func (e *ExecutionOperations) KeepAllocation(ctx context.Context, owner Allocation) (Allocation, error) { - return e.change(ctx, owner, true, func(tx AllocationTx, current Allocation) (Allocation, error) { - return tx.Keep(current) +// CheckRunning returns the stored allocation while it is still the owner's +// running compute: its Session undeleted and bound to the allocation's +// device. It follows an authenticated connection and a successful provider +// observation and changes nothing. +func (e *ExecutionOperations) CheckRunning(ctx context.Context, owner Allocation) (Allocation, error) { + return e.change(ctx, owner, true, func(_ AllocationTx, current Allocation) (Allocation, error) { + if current.State != "running" { + return Allocation{}, ErrAllocationConflict + } + return current, nil }) } diff --git a/services/core/internal/deployment/allocations_test.go b/services/core/internal/deployment/allocations_test.go index 14be17800..dcbfa2a4e 100644 --- a/services/core/internal/deployment/allocations_test.go +++ b/services/core/internal/deployment/allocations_test.go @@ -81,7 +81,6 @@ type fakeAllocationTx struct { loadAllocation func() (Allocation, error) loadSessionDevice func() (SessionDevice, bool, error) settleCreation func(Allocation) (Allocation, error) - keep func(Allocation) (Allocation, error) release func(Allocation) (Allocation, error) } @@ -114,13 +113,6 @@ func (f *fakeAllocationTx) ObserveRunning(Allocation) (Allocation, error) { return Allocation{}, nil } -func (f *fakeAllocationTx) Keep(current Allocation) (Allocation, error) { - if f.keep == nil { - unexpected(f.t, "Keep") - } - return f.keep(current) -} - func (f *fakeAllocationTx) SettleCreation(current Allocation) (Allocation, error) { if f.settleCreation == nil { unexpected(f.t, "SettleCreation") @@ -316,7 +308,11 @@ func TestReserveAllocationReplaysBeforeAdmission(t *testing.T) { func TestReserveAllocationAdmitsAndTakesTheReservedNode(t *testing.T) { key := AllocationKey{TenantID: uuid.NewString(), EnvironmentID: uuid.NewString()} installation, node := uuid.NewString(), uuid.NewString() - deployment := placement.Deployment{InstallationID: installation, Mode: "nodes", Generation: 7} + specification, err := json.Marshal(testSpecification("docker")) + if err != nil { + t.Fatal(err) + } + deployment := placement.Deployment{InstallationID: installation, Provider: "docker", Mode: "nodes", Generation: 7, Specification: specification} fresh := func() *fakeReservationTx { return &fakeReservationTx{t: t, loadEnvironment: hostedEnvironment(key.EnvironmentID), findAllocation: func() (Allocation, bool, error) { return Allocation{}, false, nil }, @@ -363,10 +359,10 @@ func TestReserveAllocationAdmitsAndTakesTheReservedNode(t *testing.T) { } // A live change needs the Session undeleted and bound to the allocation's -// device; settlement continues for a deleted Session. Any other owner is a -// conflict. +// device; settlement continues for a deleted Session. Any other owner, or +// compute that no longer runs, is a conflict. func TestAllocationChangesCheckTheOwner(t *testing.T) { - owner := Allocation{ID: uuid.NewString(), DeviceID: uuid.NewString(), EnvironmentID: uuid.NewString(), TenantID: uuid.NewString(), ProviderKey: uuid.NewString()} + owner := Allocation{ID: uuid.NewString(), DeviceID: uuid.NewString(), EnvironmentID: uuid.NewString(), TenantID: uuid.NewString(), ProviderKey: uuid.NewString(), State: "running"} stored := func(current Allocation) func() (Allocation, error) { return func() (Allocation, error) { return current, nil } } @@ -377,7 +373,7 @@ func TestAllocationChangesCheckTheOwner(t *testing.T) { } deleted := owner deleted.SessionDeleted = true - if _, err := allocationOperations(t, nil, sessions.LockedSession{}, &fakeAllocationTx{t: t, loadAllocation: stored(deleted)}).KeepAllocation(t.Context(), owner); !errors.Is(err, sessions.ErrNotFound) { + if _, err := allocationOperations(t, nil, sessions.LockedSession{}, &fakeAllocationTx{t: t, loadAllocation: stored(deleted)}).CheckRunning(t.Context(), owner); !errors.Is(err, sessions.ErrNotFound) { t.Fatal("a deleted Session kept its allocation", err) } settled := &fakeAllocationTx{t: t, loadAllocation: stored(deleted), settleCreation: func(current Allocation) (Allocation, error) { @@ -390,9 +386,20 @@ func TestAllocationChangesCheckTheOwner(t *testing.T) { unbound := &fakeAllocationTx{t: t, loadAllocation: stored(owner), loadSessionDevice: func() (SessionDevice, bool, error) { return SessionDevice{ID: uuid.NewString(), EnvironmentID: owner.EnvironmentID}, true, nil }} - if _, err := allocationOperations(t, nil, sessions.LockedSession{}, unbound).KeepAllocation(t.Context(), owner); !errors.Is(err, ErrAllocationConflict) { + if _, err := allocationOperations(t, nil, sessions.LockedSession{}, unbound).CheckRunning(t.Context(), owner); !errors.Is(err, ErrAllocationConflict) { t.Fatal("a Session bound to another device kept the allocation", err) } + bound := func() (SessionDevice, bool, error) { + return SessionDevice{ID: owner.DeviceID, EnvironmentID: owner.EnvironmentID}, true, nil + } + cleanup := owner + cleanup.State = "cleanup_pending" + if _, err := allocationOperations(t, nil, sessions.LockedSession{}, &fakeAllocationTx{t: t, loadAllocation: stored(cleanup), loadSessionDevice: bound}).CheckRunning(t.Context(), owner); !errors.Is(err, ErrAllocationConflict) { + t.Fatal("cleanup kept the allocation running", err) + } + if current, err := allocationOperations(t, nil, sessions.LockedSession{}, &fakeAllocationTx{t: t, loadAllocation: stored(owner), loadSessionDevice: bound}).CheckRunning(t.Context(), owner); err != nil || current.ID != owner.ID { + t.Fatal("the owner's running allocation", current, err) + } } // SetCompute rejects an invalid phase change before it reaches storage. diff --git a/services/core/internal/deployment/execution.go b/services/core/internal/deployment/execution.go index b91abceb3..ee57e97be 100644 --- a/services/core/internal/deployment/execution.go +++ b/services/core/internal/deployment/execution.go @@ -39,18 +39,8 @@ func (e *ExecutionOperations) Claim(ctx context.Context, installationID string) if err != nil { return err } - if d.InstallationID != "" { - if d.InstallationID != id { - return ErrConflict - } - } else { - resources, err := tx.CountResources() - if err != nil { - return err - } - if resources.Allocations != 0 || resources.Pending != 0 { - return ErrConflict - } + if d.InstallationID != "" && d.InstallationID != id { + return ErrConflict } if d.Provider != "" { if _, err := e.service.specification(d); err != nil { @@ -61,21 +51,6 @@ func (e *ExecutionOperations) Claim(ctx context.Context, installationID string) }) } -// RequireUnclaimed lets an execution owner without sandbox runtimes start -// only on a deployment no installation has claimed. -func (e *ExecutionOperations) RequireUnclaimed(ctx context.Context) error { - return e.storage.WithDeployment(ctx, func(tx DeploymentTx) error { - d, err := tx.LoadDeployment() - if err != nil { - return err - } - if d.InstallationID != "" { - return ErrConflict - } - return nil - }) -} - // CheckSetup rejects a stale or reset deployment before provider preparation. // Initialize repeats the check in its committing transaction. func (e *ExecutionOperations) CheckSetup(ctx context.Context, installation string, input sandbox.Selection) error { diff --git a/services/core/internal/deployment/placement/placement.go b/services/core/internal/deployment/placement/placement.go index 2a8b0ad55..28371a52e 100644 --- a/services/core/internal/deployment/placement/placement.go +++ b/services/core/internal/deployment/placement/placement.go @@ -29,8 +29,10 @@ var ( // generation and at least one is preparing it. ErrNodesPreparing = errors.New("sandbox nodes are preparing the target generation") // ErrNodeUnavailable reports a sandbox node that is offline, unready or - // full, or no node at all. + // full, no node at all, or a deployment without a provider. ErrNodeUnavailable = errors.New("sandbox node unavailable") + // ErrPublicURLRequired rejects rules without the installation public URL. + ErrPublicURLRequired = errors.New("placement rules require the installation public URL") ) // Declarations are the provider declarations placement reads. @@ -56,6 +58,9 @@ func NewRules(declarations Declarations, publicURL string) (*Rules, error) { if declarations == nil { return nil, errors.New("placement rules require provider declarations") } + if publicURL == "" { + return nil, ErrPublicURLRequired + } return &Rules{declarations: declarations, publicURL: publicURL}, nil } @@ -66,7 +71,8 @@ func (r *Rules) PublicURL() string { return r.publicURL } // Deployment is the deployment as placement reads it, loaded under the // deployment lock. type Deployment struct { - // InstallationID is empty until Web setup claims an installation. + // InstallationID is empty until the execution owner claims the + // installation at startup. InstallationID string Provider, Mode string Generation uint64 @@ -128,19 +134,17 @@ func (r *Rules) CheckPublicOrigin(provider string) error { // CheckAdmission admits new hosted work on the deployment. installation is // the canonical installation the work was provisioned for, or empty for a new -// Session. An unclaimed deployment admits everything. +// Session. Only a claimed deployment with a provider admits hosted work. func (r *Rules) CheckAdmission(d Deployment, installation string) error { - if d.InstallationID == "" { - return nil + if d.InstallationID == "" || d.Provider == "" { + return ErrNodeUnavailable } if d.Resetting { return ErrResetAdmission } - if d.Provider != "" { - var spec sandbox.DeploymentSpec - if json.Unmarshal(d.Specification, &spec) != nil || r.declarations.ValidateSpecification(d.Provider, spec) != nil { - return fmt.Errorf("%w: sandbox creation requires a deployment specification", ErrAdmissionClosed) - } + var spec sandbox.DeploymentSpec + if json.Unmarshal(d.Specification, &spec) != nil || r.declarations.ValidateSpecification(d.Provider, spec) != nil { + return fmt.Errorf("%w: sandbox creation requires a deployment specification", ErrAdmissionClosed) } if installation != "" && installation != d.InstallationID { return fmt.Errorf("%w: sandbox installation does not match deployment", ErrAdmissionClosed) @@ -149,30 +153,24 @@ func (r *Rules) CheckAdmission(d Deployment, installation string) error { } // DecidePlacement chooses the node a new Session's hosted Environment -// reserves, or nil when the deployment places no node: in direct mode and -// without a provider. It prefers the highest ready generation, then the -// fewest active sandboxes. It places only on nodes enrolled with the public -// URL; restores still reach the others. +// reserves, or nil in direct mode, which places no node. It prefers the +// highest ready generation, then the fewest active sandboxes. It places only +// on nodes enrolled with the public URL; restores still reach the others. func (r *Rules) DecidePlacement(d Deployment, nodes []Node) (*Placement, error) { if d.Resetting { return nil, ErrResetAdmission } + if d.Provider == "" { + return nil, ErrNodeUnavailable + } // A changed installation address cannot admit guests that require a // public origin. Existing owned resources remain available for cleanup. - if d.Provider != "" { - if err := r.CheckPublicOrigin(d.Provider); err != nil { - return nil, err - } + if err := r.CheckPublicOrigin(d.Provider); err != nil { + return nil, err } if d.Mode == "direct" { return nil, nil } - if d.Provider == "" { - if d.InstallationID != "" { - return nil, ErrNodeUnavailable - } - return nil, nil - } var chosen *Node preparing := false for i := range nodes { diff --git a/services/core/internal/deployment/placement/placement_test.go b/services/core/internal/deployment/placement/placement_test.go index e60242f4d..473c343bb 100644 --- a/services/core/internal/deployment/placement/placement_test.go +++ b/services/core/internal/deployment/placement/placement_test.go @@ -46,10 +46,13 @@ func rules(t *testing.T, declarations Declarations, url string) *Rules { func generation(value uint64) *uint64 { return &value } -func TestNewRulesRequiresDeclarations(t *testing.T) { +func TestNewRulesRequiresDeclarationsAndPublicURL(t *testing.T) { if _, err := NewRules(nil, publicURL); err == nil { t.Fatal("NewRules accepted nil declarations") } + if _, err := NewRules(&fakeDeclarations{t: t}, ""); !errors.Is(err, ErrPublicURLRequired) { + t.Fatalf("NewRules without a public URL = %v", err) + } if got := rules(t, &fakeDeclarations{t: t}, publicURL).PublicURL(); got != publicURL { t.Fatalf("PublicURL = %q", got) } @@ -93,14 +96,14 @@ func TestCheckAdmission(t *testing.T) { want error message string }{ - "unclaimed admits everything": {Deployment{Resetting: true}, installation, nil, ""}, - "admitted": {valid, installation, nil, ""}, - "new Session": {valid, "", nil, ""}, - "reset": {with(func(d *Deployment) { d.Resetting = true }), installation, ErrResetAdmission, "hosted admission is paused for a sandbox reset"}, - "malformed specification": {with(func(d *Deployment) { d.Specification = json.RawMessage(`[`) }), installation, ErrAdmissionClosed, "environment is no longer available: sandbox creation requires a deployment specification"}, - "rejected specification": {with(func(d *Deployment) { d.Specification = json.RawMessage(`{"resources":{"cpus":3}}`) }), installation, ErrAdmissionClosed, "environment is no longer available: sandbox creation requires a deployment specification"}, - "no provider": {with(func(d *Deployment) { d.Provider, d.Specification = "", json.RawMessage(`[`) }), installation, nil, ""}, - "other installation": {valid, uuid.NewString(), ErrAdmissionClosed, "environment is no longer available: sandbox installation does not match deployment"}, + "unclaimed": {with(func(d *Deployment) { d.InstallationID = "" }), "", ErrNodeUnavailable, "sandbox node unavailable"}, + "no provider": {with(func(d *Deployment) { d.Provider, d.Specification = "", json.RawMessage(`[`) }), "", ErrNodeUnavailable, "sandbox node unavailable"}, + "admitted": {valid, installation, nil, ""}, + "new Session": {valid, "", nil, ""}, + "reset": {with(func(d *Deployment) { d.Resetting = true }), installation, ErrResetAdmission, "hosted admission is paused for a sandbox reset"}, + "malformed specification": {with(func(d *Deployment) { d.Specification = json.RawMessage(`[`) }), installation, ErrAdmissionClosed, "environment is no longer available: sandbox creation requires a deployment specification"}, + "rejected specification": {with(func(d *Deployment) { d.Specification = json.RawMessage(`{"resources":{"cpus":3}}`) }), installation, ErrAdmissionClosed, "environment is no longer available: sandbox creation requires a deployment specification"}, + "other installation": {valid, uuid.NewString(), ErrAdmissionClosed, "environment is no longer available: sandbox installation does not match deployment"}, } { err := rules(t, declarations, publicURL).CheckAdmission(test.d, test.installation) if !errors.Is(err, test.want) || (test.want == nil) != (err == nil) || (err != nil && err.Error() != test.message) { @@ -139,8 +142,8 @@ func TestDecidePlacement(t *testing.T) { "reset": {origin(false), publicURL, with(func(d *Deployment) { d.Resetting = true }), nil, nil, ErrResetAdmission}, "loopback public origin": {origin(true), "http://localhost:8091", nodes, []Node{ready("a", 1, 0)}, nil, ErrPublicURLUnreachable}, "direct": {origin(false), publicURL, with(func(d *Deployment) { d.Mode = "direct" }), nil, nil, nil}, - "no provider": {&fakeDeclarations{t: t}, publicURL, Deployment{}, nil, nil, nil}, - "no provider on Web": {&fakeDeclarations{t: t}, publicURL, Deployment{InstallationID: "installation"}, nil, nil, ErrNodeUnavailable}, + "unclaimed": {&fakeDeclarations{t: t}, publicURL, Deployment{}, nil, nil, ErrNodeUnavailable}, + "no provider": {&fakeDeclarations{t: t}, publicURL, Deployment{InstallationID: "installation"}, nil, nil, ErrNodeUnavailable}, "no nodes": {origin(false), publicURL, nodes, nil, nil, ErrNodeUnavailable}, "only ineligible nodes": {origin(false), publicURL, nodes, []Node{offline, unready, full, retainedFull, elsewhere, {ID: "never", Online: true, ServingReady: true, MaxActive: 1, MaxRetained: 1, CoreURL: publicURL}}, nil, ErrNodeUnavailable}, "preparing": {origin(false), publicURL, nodes, []Node{offline, preparing}, nil, ErrNodesPreparing}, diff --git a/services/core/internal/deployment/storage.go b/services/core/internal/deployment/storage.go index c1f17785a..2eb526b14 100644 --- a/services/core/internal/deployment/storage.go +++ b/services/core/internal/deployment/storage.go @@ -114,8 +114,6 @@ type AllocationTx interface { LoadRestore(current Allocation) (placement.Restore, error) // ObserveRunning records the allocation running with its creation settled. ObserveRunning(current Allocation) (Allocation, error) - // Keep renews the allocation's lease. - Keep(current Allocation) (Allocation, error) // SettleCreation records that the original Create can no longer change // resources. SettleCreation(current Allocation) (Allocation, error) diff --git a/services/core/internal/execution/deployment_provider_observations_test.go b/services/core/internal/execution/deployment_provider_observations_test.go index 440da13b2..9d769cf31 100644 --- a/services/core/internal/execution/deployment_provider_observations_test.go +++ b/services/core/internal/execution/deployment_provider_observations_test.go @@ -1,7 +1,6 @@ package execution import ( - "bytes" "context" "encoding/json" "errors" @@ -11,7 +10,6 @@ import ( v1 "github.com/MiniMax-AI/OpenAgentCore/contracts/agents-api/v1" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/adminaudit" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/identity" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/modelconfiguration" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/modelconfigurationpg" @@ -47,10 +45,7 @@ func newFinishObservationFixture(t *testing.T, maxConnections int32) finishObser t.Fatal(err) } t.Cleanup(pool.Close) - cipher, err := credentialcrypto.New(bytes.Repeat([]byte{8}, 32)) - if err != nil { - t.Fatal(err) - } + cipher := pgtest.CredentialKey(t) defaults := modelconfigurationpg.New(pgunit.NewPool(pool), cipher) service, err := modelconfiguration.NewService(defaults) if err != nil { diff --git a/services/core/internal/execution/dispatcher.go b/services/core/internal/execution/dispatcher.go index 909fb9702..6364a9ae9 100644 --- a/services/core/internal/execution/dispatcher.go +++ b/services/core/internal/execution/dispatcher.go @@ -49,7 +49,8 @@ type Dispatcher struct { Sessions *sessions.Service // SessionsReader serves the plain Session reads. It is required. SessionsReader sessions.Reader - // ManagedRuntimes is optional internal provisioning; it does not admit hosted API requests. + // ManagedRuntimes provisions hosted Environments on the sandbox deployment. + // It is required. ManagedRuntimes *RuntimeProvider // MaxConcurrentExecutions bounds work admitted by this Core execution owner. // Zero uses DefaultExecutionConcurrency. It is independent of sandbox capacity. diff --git a/services/core/internal/execution/environment_admission.go b/services/core/internal/execution/environment_admission.go index 52d1c3b05..474e0fb70 100644 --- a/services/core/internal/execution/environment_admission.go +++ b/services/core/internal/execution/environment_admission.go @@ -33,9 +33,6 @@ func (w *Worker) validateEnvironmentAdmission(ctx context.Context, engine string return sessions.ErrInvalidInput } case "openai_hosted": - if w.runtimes == nil { - return ErrExecutionUnavailable - } ready, err := w.runtimes.ensureDeployment(ctx) if err != nil { return err diff --git a/services/core/internal/execution/environment_directory.go b/services/core/internal/execution/environment_directory.go index 5d1286efb..34a533197 100644 --- a/services/core/internal/execution/environment_directory.go +++ b/services/core/internal/execution/environment_directory.go @@ -58,9 +58,6 @@ func (w *Worker) ReadEnvironmentDirectory(ctx context.Context, environment sessi func (w *Worker) runDirectoryRead(owner context.Context, request directoryReadRequest, reserved bool) (result directoryReadResult) { result.err = ErrExecutionUnavailable defer func() { - if w.runtimes == nil { - return - } touch, stop := context.WithTimeout(context.WithoutCancel(owner), 5*time.Second) defer stop() if err := w.dispatcher.Deployment.TouchActivity(touch, request.environment.TenantID, request.environment.ID); err != nil { diff --git a/services/core/internal/execution/owner_test.go b/services/core/internal/execution/owner_test.go index e7e577f9e..4f7fccfa9 100644 --- a/services/core/internal/execution/owner_test.go +++ b/services/core/internal/execution/owner_test.go @@ -65,6 +65,12 @@ func unusedSessions(t *testing.T) (*sessions.Service, sessions.Reader) { return service, struct{ sessions.Reader }{} } +// unusedRuntimes is a runtime provider for a new installation whose +// deployment no operator has configured. +func unusedRuntimes(t *testing.T) *RuntimeProvider { + return NewDeferredRuntimeProvider(uuid.NewString(), func(context.Context) (*RuntimeProvider, error) { return nil, nil }, unusedPreparation(t)) +} + // unusedObserver fails the test on any observation. The Workers it serves run // no Turn. type unusedObserver struct{ t *testing.T } @@ -118,24 +124,29 @@ func TestStartWorkerFailureClosesLeaseOnce(t *testing.T) { _, err := StartWorker(canceled, &Dispatcher{Credentials: &recordingCredentials{}, Observer: unusedObserver{t}, Deployment: deployments, DeploymentReader: deploymentReader, Sessions: service}, Owner{Lease: lease}) return err }, - "missing Session operations": func(t *testing.T, lease *closeCountingLease) error { + "missing runtime provider": func(t *testing.T, lease *closeCountingLease) error { _, deployments, deploymentReader := resetManager(t) service, reader := unusedSessions(t) _, err := StartWorker(canceled, &Dispatcher{Credentials: &recordingCredentials{}, Observer: unusedObserver{t}, Deployment: deployments, DeploymentReader: deploymentReader, Sessions: service, SessionsReader: reader}, Owner{Lease: lease}) return err }, + "missing Session operations": func(t *testing.T, lease *closeCountingLease) error { + _, deployments, deploymentReader := resetManager(t) + service, reader := unusedSessions(t) + _, err := StartWorker(canceled, &Dispatcher{Credentials: &recordingCredentials{}, Observer: unusedObserver{t}, Deployment: deployments, DeploymentReader: deploymentReader, Sessions: service, SessionsReader: reader, ManagedRuntimes: unusedRuntimes(t)}, Owner{Lease: lease}) + return err + }, "missing deployment": func(t *testing.T, lease *closeCountingLease) error { owner, deployments, deploymentReader := resetManager(t) service, reader := unusedSessions(t) - _, err := StartWorker(canceled, &Dispatcher{Credentials: &recordingCredentials{}, Observer: unusedObserver{t}, Deployment: deployments, DeploymentReader: deploymentReader, Sessions: service, SessionsReader: reader}, Owner{Lease: lease, Sessions: owner.Sessions}) + _, err := StartWorker(canceled, &Dispatcher{Credentials: &recordingCredentials{}, Observer: unusedObserver{t}, Deployment: deployments, DeploymentReader: deploymentReader, Sessions: service, SessionsReader: reader, ManagedRuntimes: unusedRuntimes(t)}, Owner{Lease: lease, Sessions: owner.Sessions}) return err }, "deployment claim": func(t *testing.T, lease *closeCountingLease) error { owner, deployments, deploymentReader := resetManager(t) lease.inner = owner.Lease - id := uuid.NewString() service, reader := unusedSessions(t) - dispatcher := &Dispatcher{Registry: runtimegateway.NewRegistry(), Credentials: &recordingCredentials{}, Observer: unusedObserver{t}, Deployment: deployments, DeploymentReader: deploymentReader, Sessions: service, SessionsReader: reader, ManagedRuntimes: NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return nil, nil }, unusedPreparation(t))} + dispatcher := &Dispatcher{Registry: runtimegateway.NewRegistry(), Credentials: &recordingCredentials{}, Observer: unusedObserver{t}, Deployment: deployments, DeploymentReader: deploymentReader, Sessions: service, SessionsReader: reader, ManagedRuntimes: unusedRuntimes(t)} _, err := StartWorker(canceled, dispatcher, Owner{Lease: lease, Deployment: owner.Deployment, Sessions: owner.Sessions}) if ping := owner.Lease.CheckOwnership(t.Context()); !errors.Is(ping, pgunit.ErrLeaseClosed) { t.Error("failed start kept the database lease", ping) @@ -160,6 +171,7 @@ func TestStartWorkerChecksDeploymentAfterItsDependencies(t *testing.T) { owner, deployments, deploymentReader := resetManager(t) credentials, observer := &recordingCredentials{}, unusedObserver{t} service, reader := unusedSessions(t) + runtimes := unusedRuntimes(t) bound := Owner{Sessions: owner.Sessions} for _, test := range []struct { name string @@ -173,8 +185,9 @@ func TestStartWorkerChecksDeploymentAfterItsDependencies(t *testing.T) { {"missing deployment reader", Dispatcher{Credentials: credentials, Observer: observer, Deployment: deployments}, bound, "execution worker requires the deployment reader"}, {"missing Session service", Dispatcher{Credentials: credentials, Observer: observer, Deployment: deployments, DeploymentReader: deploymentReader}, bound, "execution worker requires the Session service"}, {"missing Session reader", Dispatcher{Credentials: credentials, Observer: observer, Deployment: deployments, DeploymentReader: deploymentReader, Sessions: service}, bound, "execution worker requires the Session reader"}, - {"missing Session operations", Dispatcher{Credentials: credentials, Observer: observer, Deployment: deployments, DeploymentReader: deploymentReader, Sessions: service, SessionsReader: reader}, Owner{}, "execution requires the Session execution operations"}, - {"missing deployment", Dispatcher{Credentials: credentials, Observer: observer, Deployment: deployments, DeploymentReader: deploymentReader, Sessions: service, SessionsReader: reader}, bound, "execution worker requires the deployment execution operations"}, + {"missing runtime provider", Dispatcher{Credentials: credentials, Observer: observer, Deployment: deployments, DeploymentReader: deploymentReader, Sessions: service, SessionsReader: reader}, bound, "execution worker requires the sandbox runtime provider"}, + {"missing Session operations", Dispatcher{Credentials: credentials, Observer: observer, Deployment: deployments, DeploymentReader: deploymentReader, Sessions: service, SessionsReader: reader, ManagedRuntimes: runtimes}, Owner{}, "execution requires the Session execution operations"}, + {"missing deployment", Dispatcher{Credentials: credentials, Observer: observer, Deployment: deployments, DeploymentReader: deploymentReader, Sessions: service, SessionsReader: reader, ManagedRuntimes: runtimes}, bound, "execution worker requires the deployment execution operations"}, } { t.Run(test.name, func(t *testing.T) { lease := &closeCountingLease{t: t} @@ -191,10 +204,9 @@ func TestStartWorkerChecksDeploymentAfterItsDependencies(t *testing.T) { func TestWorkerRunClosesLeaseAfterDrain(t *testing.T) { owner, deployments, deploymentReader, pool := resetManagerDB(t, nil) lease := &closeCountingLease{t: t, inner: owner.Lease} - id := uuid.NewString() // The Worker's first reconciliation scans the Session work. reader, service := testSessions(t, pool, pgtest.CredentialKey(t)) - dispatcher := &Dispatcher{Registry: runtimegateway.NewRegistry(), Credentials: &recordingCredentials{}, Observer: unusedObserver{t}, Deployment: deployments, DeploymentReader: deploymentReader, Sessions: service, SessionsReader: reader, ManagedRuntimes: NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return nil, nil }, unusedPreparation(t))} + dispatcher := &Dispatcher{Registry: runtimegateway.NewRegistry(), Credentials: &recordingCredentials{}, Observer: unusedObserver{t}, Deployment: deployments, DeploymentReader: deploymentReader, Sessions: service, SessionsReader: reader, ManagedRuntimes: unusedRuntimes(t)} worker, err := StartWorker(t.Context(), dispatcher, Owner{Lease: lease, Deployment: owner.Deployment, Sessions: owner.Sessions}) if err != nil { t.Fatal(err) diff --git a/services/core/internal/execution/runtime_compute.go b/services/core/internal/execution/runtime_compute.go index c1aeab3c6..54c54e841 100644 --- a/services/core/internal/execution/runtime_compute.go +++ b/services/core/internal/execution/runtime_compute.go @@ -116,7 +116,7 @@ func (r *runtimeLifecycle) idleCompute(ctx context.Context, p sandbox.SandboxPro if err != nil { return err } - if _, err := r.deployment.KeepAllocation(ctx, owner); err != nil { + if _, err := r.deployment.CheckRunning(ctx, owner); err != nil { return err } activity, err := r.reader.Activity(ctx, owner.ID) diff --git a/services/core/internal/execution/runtime_compute_wake.go b/services/core/internal/execution/runtime_compute_wake.go index ed4223d90..0dd7f915d 100644 --- a/services/core/internal/execution/runtime_compute_wake.go +++ b/services/core/internal/execution/runtime_compute_wake.go @@ -105,9 +105,6 @@ func (r *runtimeLifecycle) cleanupCompute(ctx context.Context, p sandbox.Sandbox // waitRuntimeAwake is called only for live Environment file operations, before // entering the Worker's work queues. Persisted history/artifact reads bypass it. func (w *Worker) waitRuntimeAwake(ctx context.Context, environment sessions.Environment) error { - if w.runtimes == nil { - return nil - } key := deployment.AllocationKey{TenantID: environment.TenantID, EnvironmentID: environment.ID} owner, err := w.dispatcher.DeploymentReader.EnvironmentAllocation(ctx, key) if errors.Is(err, deployment.ErrNotFound) { diff --git a/services/core/internal/execution/runtime_lifecycle.go b/services/core/internal/execution/runtime_lifecycle.go index 5e275b09e..b2f072098 100644 --- a/services/core/internal/execution/runtime_lifecycle.go +++ b/services/core/internal/execution/runtime_lifecycle.go @@ -61,7 +61,7 @@ type runtimeLifecycle struct { func newRuntimeManager(owner Owner, deployments *deployment.Service, deploymentReader deployment.Reader, sessionReader sessions.Reader, registry *runtimegateway.Registry, config *RuntimeProvider) (*runtimeManager, error) { if config == nil { - return nil, nil + return nil, sandbox.ErrInvalid } id, err := uuid.Parse(config.InstallationID) if err != nil || id == uuid.Nil || id.String() != config.InstallationID || config.loadDeployment == nil || config.prepareDeployment == nil || registry == nil { @@ -123,9 +123,6 @@ func (r *runtimeLifecycle) lock(ctx context.Context) error { // ProvisionEnvironment is an internal bootstrap operation for an already // authorized hosted Environment. It does not enable public hosted admission. func (w *Worker) ProvisionEnvironment(ctx context.Context, tenant, environment, providerKey string) (deployment.Allocation, error) { - if w.runtimes == nil { - return deployment.Allocation{}, ErrExecutionUnavailable - } ctx, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() ctx, finish, err := w.runtimes.enter(ctx) @@ -233,9 +230,6 @@ func (r *runtimeLifecycle) provision(ctx context.Context, tenant, environment, p // observes existing allocations and bootstraps committed resources without an // allocation. It never retries an existing Create or native work. func (w *Worker) ReconcileManagedRuntimes(ctx context.Context) error { - if w.runtimes == nil { - return nil - } return w.runtimes.reconcile(ctx) } @@ -384,7 +378,7 @@ func (r *runtimeLifecycle) observe(ctx context.Context, owner deployment.Allocat if renewed.Reference != runtimeReference(owner) || renewed.State != "running" || !renewed.BootstrapComplete { return sandbox.ErrOwnership } - _, err = r.deployment.KeepAllocation(ctx, owner) + _, err = r.deployment.CheckRunning(ctx, owner) return err } diff --git a/services/core/internal/execution/worker.go b/services/core/internal/execution/worker.go index 839009390..92fabc153 100644 --- a/services/core/internal/execution/worker.go +++ b/services/core/internal/execution/worker.go @@ -65,6 +65,9 @@ func StartWorker(ctx context.Context, dispatcher *Dispatcher, owner Owner) (_ *W if dispatcher.SessionsReader == nil { return nil, errors.New("execution worker requires the Session reader") } + if dispatcher.ManagedRuntimes == nil { + return nil, errors.New("execution worker requires the sandbox runtime provider") + } owned, err := dispatcher.Bind(owner) if err != nil { return nil, err @@ -78,20 +81,15 @@ func StartWorker(ctx context.Context, dispatcher *Dispatcher, owner Owner) (_ *W if err != nil { return nil, err } - if worker.runtimes != nil { - defer func() { - if err != nil { - worker.runtimes.stop() - } - }() - err = owner.Deployment.Claim(ctx, worker.runtimes.setupInstallationID) - if err == nil { - _, err = worker.runtimes.ensureDeployment(ctx) + defer func() { + if err != nil { + worker.runtimes.stop() } - } else { - err = owner.Deployment.RequireUnclaimed(ctx) + }() + if err = owner.Deployment.Claim(ctx, worker.runtimes.setupInstallationID); err != nil { + return nil, err } - if err != nil { + if _, err = worker.runtimes.ensureDeployment(ctx); err != nil { return nil, err } if err = owned.sessionExecution.ReconcileEnvironmentConnections(ctx); err != nil { @@ -150,14 +148,10 @@ func (w *Worker) Run(ctx context.Context) (runErr error) { defer func() { w.observeWorkerStop(runErr, ctx.Err()) cancel() - if w.runtimes != nil { - w.runtimes.stop() - } + w.runtimes.stop() running.Wait() - if w.runtimes != nil { - // Drain an external provisioning caller before releasing the writer lease. - w.runtimes.drain() - } + // Drain an external provisioning caller before releasing the writer lease. + w.runtimes.drain() w.observeWorkerClosed(closeLease(ctx, w.lease)) }() active := make(map[string]bool) @@ -171,13 +165,11 @@ func (w *Worker) Run(ctx context.Context) (runErr error) { running.Add(1) go func() { defer running.Done(); preparationDone <- w.runEnvironmentInitializations(ctx) }() lifecycleDone := make(chan error, 1) - if w.runtimes != nil { - running.Add(1) - go func() { - defer running.Done() - lifecycleDone <- w.runManagedRuntimes(ctx) - }() - } + running.Add(1) + go func() { + defer running.Done() + lifecycleDone <- w.runManagedRuntimes(ctx) + }() type readCompletion struct { id string request directoryReadRequest diff --git a/services/core/internal/execution/worker_concurrency_test.go b/services/core/internal/execution/worker_concurrency_test.go index 6307f095a..d400ebdfe 100644 --- a/services/core/internal/execution/worker_concurrency_test.go +++ b/services/core/internal/execution/worker_concurrency_test.go @@ -7,7 +7,7 @@ func TestWorkerConfiguredConcurrencyReportsActualCapacity(t *testing.T) { worker := &Worker{concurrency: limit} worker.observeSlots(limit) snapshot := worker.MetricsSnapshot() - if *snapshot.SlotsTotal != int64(limit) || *snapshot.SlotsInUse != int64(limit) { + if snapshot.SlotsTotal != int64(limit) || snapshot.SlotsInUse != int64(limit) { t.Fatal("metrics do not reflect execution concurrency", snapshot) } } diff --git a/services/core/internal/execution/worker_metrics.go b/services/core/internal/execution/worker_metrics.go index c549452ca..c5833dfa1 100644 --- a/services/core/internal/execution/worker_metrics.go +++ b/services/core/internal/execution/worker_metrics.go @@ -6,11 +6,11 @@ import ( "time" ) -// WorkerMetrics contains only observations from this worker's existing operations. -// Missing pointers mean the corresponding operation has not established a value. +// WorkerMetrics contains only observations from this worker's existing +// operations. A nil ExecutionOwner means the last ownership check failed. type WorkerMetrics struct { - SlotsInUse *int64 - SlotsTotal *int64 + SlotsInUse int64 + SlotsTotal int64 ExecutionOwner *bool Scheduler WorkerJobMetrics } @@ -35,8 +35,7 @@ func (w *Worker) MetricsSnapshot() WorkerMetrics { w.metrics.mu.Lock() defer w.metrics.mu.Unlock() value := w.metrics.value - value.SlotsInUse = copyMetric(value.SlotsInUse) - value.SlotsTotal = copyMetric(value.SlotsTotal) + value.SlotsTotal = int64(w.executionConcurrency()) value.ExecutionOwner = copyMetric(value.ExecutionOwner) value.Scheduler.LastRunAt = copyMetric(value.Scheduler.LastRunAt) value.Scheduler.Processed = copyMetric(value.Scheduler.Processed) @@ -56,11 +55,9 @@ func copyMetric[T any](source *T) *T { } func (w *Worker) observeSlots(active int) { - used, total := int64(active), int64(w.executionConcurrency()) w.metrics.mu.Lock() defer w.metrics.mu.Unlock() - w.metrics.value.SlotsInUse = &used - w.metrics.value.SlotsTotal = &total + w.metrics.value.SlotsInUse = int64(active) } func (w *Worker) observeOwnership(err error) { @@ -106,6 +103,5 @@ func (w *Worker) observeWorkerClosed(err error) { owned := false w.metrics.value.ExecutionOwner = &owned } - used := int64(0) - w.metrics.value.SlotsInUse = &used + w.metrics.value.SlotsInUse = 0 } diff --git a/services/core/internal/execution/worker_metrics_test.go b/services/core/internal/execution/worker_metrics_test.go index c8dcb374f..2243862d7 100644 --- a/services/core/internal/execution/worker_metrics_test.go +++ b/services/core/internal/execution/worker_metrics_test.go @@ -14,27 +14,27 @@ import ( func TestWorkerMetricsUnknownAndDetached(t *testing.T) { worker := &Worker{} initial := worker.MetricsSnapshot() - if initial.SlotsInUse != nil || initial.SlotsTotal != nil || initial.ExecutionOwner != nil || initial.Scheduler.Status != "unknown" || initial.Scheduler.LastRunAt != nil || initial.Scheduler.Processed != nil || initial.Scheduler.Failed != nil { + if initial.SlotsInUse != 0 || initial.SlotsTotal != 4 || initial.ExecutionOwner != nil || initial.Scheduler.Status != "unknown" || initial.Scheduler.LastRunAt != nil || initial.Scheduler.Processed != nil || initial.Scheduler.Failed != nil { t.Fatalf("unobserved worker reported values: %+v", initial) } worker.observeSlots(3) worker.observeOwnership(nil) worker.observeSchedulerPoll(2, nil) first := worker.MetricsSnapshot() - if *first.SlotsInUse != 3 || *first.SlotsTotal != 4 || !*first.ExecutionOwner || *first.Scheduler.Processed != 2 || *first.Scheduler.Failed != 0 || first.Scheduler.Status != "ok" { + if first.SlotsInUse != 3 || first.SlotsTotal != 4 || !*first.ExecutionOwner || *first.Scheduler.Processed != 2 || *first.Scheduler.Failed != 0 || first.Scheduler.Status != "ok" { t.Fatalf("unexpected observation: %+v", first) } observedAt := *first.Scheduler.LastRunAt - *first.SlotsInUse, *first.SlotsTotal, *first.ExecutionOwner = 100, 100, false + *first.ExecutionOwner = false *first.Scheduler.Processed, *first.Scheduler.Failed = 100, 100 *first.Scheduler.LastRunAt = time.Time{} second := worker.MetricsSnapshot() - if *second.SlotsInUse != 3 || *second.SlotsTotal != 4 || !*second.ExecutionOwner || *second.Scheduler.Processed != 2 || *second.Scheduler.Failed != 0 || !second.Scheduler.LastRunAt.Equal(observedAt) { + if second.SlotsInUse != 3 || second.SlotsTotal != 4 || !*second.ExecutionOwner || *second.Scheduler.Processed != 2 || *second.Scheduler.Failed != 0 || !second.Scheduler.LastRunAt.Equal(observedAt) { t.Fatal("caller mutation altered the worker's observations") } worker.observeSlots(1) worker.observeSchedulerPoll(0, nil) - if *second.SlotsInUse != 3 || *second.Scheduler.Processed != 2 { + if second.SlotsInUse != 3 || *second.Scheduler.Processed != 2 { t.Fatal("new observations altered a retained snapshot") } } @@ -74,7 +74,7 @@ func TestWorkerMetricsFailuresAndClosure(t *testing.T) { worker.observeWorkerClosed(nil) worker.observeOwnership(nil) closed := worker.MetricsSnapshot() - if closed.ExecutionOwner == nil || *closed.ExecutionOwner || *closed.SlotsInUse != 0 { + if closed.ExecutionOwner == nil || *closed.ExecutionOwner || closed.SlotsInUse != 0 { t.Fatal("late ownership observation revived a closed worker") } uncertain := &Worker{} @@ -96,10 +96,9 @@ func TestWorkerMetricsConcurrentSnapshots(t *testing.T) { worker.observeOwnership(nil) worker.observeSchedulerPoll(j%5, nil) value := worker.MetricsSnapshot() - if *value.SlotsInUse < 0 || *value.SlotsInUse > *value.SlotsTotal || *value.SlotsTotal != 4 { + if value.SlotsInUse < 0 || value.SlotsInUse > value.SlotsTotal || value.SlotsTotal != 4 { t.Errorf("inconsistent slot snapshot: %+v", value) } - *value.SlotsInUse = -1 *value.ExecutionOwner = false *value.Scheduler.Processed = -1 } diff --git a/services/core/internal/persistence/postgres/agentpg/store_test.go b/services/core/internal/persistence/postgres/agentpg/store_test.go index 97e765d32..98d3c9572 100644 --- a/services/core/internal/persistence/postgres/agentpg/store_test.go +++ b/services/core/internal/persistence/postgres/agentpg/store_test.go @@ -37,9 +37,10 @@ func open(t *testing.T, pool *pgxpool.Pool, cipher *credentialcrypto.Cipher) (*a return store, service } -func testCipher(t *testing.T, seed byte) *credentialcrypto.Cipher { +// otherKey is a credential key other than pgtest.CredentialKey. +func otherKey(t *testing.T) *credentialcrypto.Cipher { t.Helper() - cipher, err := credentialcrypto.New(bytes.Repeat([]byte{seed}, 32)) + cipher, err := credentialcrypto.New(bytes.Repeat([]byte{0x5a}, 32)) if err != nil { t.Fatal(err) } @@ -311,7 +312,7 @@ func TestAgentDeleteIsTenantScoped(t *testing.T) { func TestAgentModelExecutionAtomicEncryptedSnapshot(t *testing.T) { pool := pgtest.Open(t) - c := testCipher(t, 31) + c := pgtest.CredentialKey(t) store, service := open(t, pool, c) ctx, tenant := t.Context(), uuid.NewString() provider := providerFixture(0) @@ -350,7 +351,7 @@ func TestAgentModelExecutionAtomicEncryptedSnapshot(t *testing.T) { if _, err := service.Update(ctx, agents.UpdateCommand{TenantID: tenant, AgentID: agent.ID, Configuration: []byte(`{"model":"new-model"}`)}); err != nil { t.Fatal(err) } - if _, _, err := agentpg.New(pgunit.NewPool(pool), testCipher(t, 99)).GetAgentWithModelProvider(ctx, tenant, agent.ID); err == nil || err.Error() != "agent model provider decryption failed" { + if _, _, err := agentpg.New(pgunit.NewPool(pool), otherKey(t)).GetAgentWithModelProvider(ctx, tenant, agent.ID); err == nil || err.Error() != "agent model provider decryption failed" { t.Fatal("wrong key was not a decryption failure", err) } replacement := providerFixture(1) @@ -418,7 +419,7 @@ func TestAgentModelExecutionAtomicEncryptedSnapshot(t *testing.T) { // key or a missing bundle. func TestAgentBundleSealedToAnotherAgentDoesNotOpen(t *testing.T) { pool := pgtest.Open(t) - c := testCipher(t, 32) + c := pgtest.CredentialKey(t) store, service := open(t, pool, c) ctx, tenant := t.Context(), uuid.NewString() provider := providerFixture(0) @@ -444,7 +445,7 @@ func TestAgentBundleSealedToAnotherAgentDoesNotOpen(t *testing.T) { func TestAgentModelExecutionConcurrentSnapshots(t *testing.T) { pool := pgtest.Open(t) - store, service := open(t, pool, testCipher(t, 32)) + store, service := open(t, pool, pgtest.CredentialKey(t)) ctx, tenant := t.Context(), uuid.NewString() p := providerFixture(0) agent, err := service.Create(ctx, agents.CreateCommand{TenantID: tenant, Configuration: providerConfiguration(t, p, "codex"), ModelProvider: p}) @@ -504,7 +505,7 @@ func auditContext(ctx context.Context, tenant, request, key string) context.Cont // together. An administrator delete records administrator audit only. func TestAgentWritesAuditInTheirTransaction(t *testing.T) { pool := pgtest.OpenIsolated(t, nil) - _, service := open(t, pool, testCipher(t, 91)) + _, service := open(t, pool, pgtest.CredentialKey(t)) ctx := t.Context() if _, err := pool.Exec(ctx, `CREATE FUNCTION reject_agent_audit_fixture() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.request_id = 'reject-agent-audit' THEN RAISE EXCEPTION 'forced audit insertion failure'; END IF; RETURN NEW; END $$; diff --git a/services/core/internal/persistence/postgres/deploymentpg/allocations.go b/services/core/internal/persistence/postgres/deploymentpg/allocations.go index a6ae895e1..ee99f8f4d 100644 --- a/services/core/internal/persistence/postgres/deploymentpg/allocations.go +++ b/services/core/internal/persistence/postgres/deploymentpg/allocations.go @@ -25,7 +25,7 @@ func allocation(row sqlc.RuntimeAllocation, session, tenant pgtype.UUID, deleted ComputeActivityAt: row.ComputeActivityAt.Time, ComputeWakeRequested: row.ComputeWakeRequested, ComputeRetainedUntil: timestamp(row.ComputeRetainedUntil), ID: uuidString(row.ID), EnvironmentID: uuidString(row.EnvironmentID), SessionID: uuidString(session), TenantID: uuidString(tenant), DeviceID: uuidString(row.DeviceID), ProviderKey: uuidString(row.ProviderKey), State: row.State, CreateSettled: row.CreateSettled, - SessionDeleted: deleted.Valid, Expired: expired, CreatedAt: row.CreatedAt.Time, KeptAt: row.KeptAt.Time, + SessionDeleted: deleted.Valid, Expired: expired, CreatedAt: row.CreatedAt.Time, } } @@ -271,10 +271,6 @@ func (t *allocationTx) ObserveRunning(current deployment.Allocation) (deployment return t.change(current, t.q.ObserveRuntimeRunning) } -func (t *allocationTx) Keep(current deployment.Allocation) (deployment.Allocation, error) { - return t.change(current, t.q.KeepRuntimeAllocation) -} - func (t *allocationTx) SettleCreation(current deployment.Allocation) (deployment.Allocation, error) { return t.change(current, t.q.SettleRuntimeCreation) } diff --git a/services/core/internal/persistence/postgres/deploymentpg/specification_test.go b/services/core/internal/persistence/postgres/deploymentpg/specification_test.go index 304b9121c..30531a86d 100644 --- a/services/core/internal/persistence/postgres/deploymentpg/specification_test.go +++ b/services/core/internal/persistence/postgres/deploymentpg/specification_test.go @@ -44,9 +44,6 @@ func TestSandboxSpecificationRoundTripAndClaimStays(t *testing.T) { if _, err := changes.Initialize(t.Context(), view.InstallationID, changed); !errors.Is(err, deployment.ErrConflict) { t.Fatal("initial setup silently resized a configured deployment", err) } - if err := changes.RequireUnclaimed(t.Context()); !errors.Is(err, deployment.ErrConflict) { - t.Fatal("an owner without sandbox runtimes started on a claimed deployment", err) - } after, err := f.service.View(t.Context()) if err != nil || !reflect.DeepEqual(after, view) { t.Fatal("rejected writes changed the committed specification", err) diff --git a/services/core/internal/persistence/postgres/modelconfigurationpg/fixture_test.go b/services/core/internal/persistence/postgres/modelconfigurationpg/fixture_test.go index 5e30ce5a2..32152bb75 100644 --- a/services/core/internal/persistence/postgres/modelconfigurationpg/fixture_test.go +++ b/services/core/internal/persistence/postgres/modelconfigurationpg/fixture_test.go @@ -107,7 +107,7 @@ func newObserved(t *testing.T) observed { }, } // Hosted creation needs placement rules, as cmd/server gives them. - rules, err := placement.NewRules(providers.Builtin(), "") + rules, err := placement.NewRules(providers.Builtin(), "https://core.example") if err != nil { t.Fatal(err) } diff --git a/services/core/internal/persistence/postgres/modelconfigurationpg/observation_test.go b/services/core/internal/persistence/postgres/modelconfigurationpg/observation_test.go index d46af31c2..8e788926f 100644 --- a/services/core/internal/persistence/postgres/modelconfigurationpg/observation_test.go +++ b/services/core/internal/persistence/postgres/modelconfigurationpg/observation_test.go @@ -84,7 +84,7 @@ func TestObservationExcludesOtherSourcesAndHistoricalSessions(t *testing.T) { if source == "deployment" { input.DeploymentProviderRevision = uuid.Nil } else { - input.Configuration = json.RawMessage(`{"agent":{"model":"frozen-model"},"environment":{"type":"openai_hosted"}}`) + input.Configuration = json.RawMessage(`{"agent":{"model":"frozen-model"},"environment":{"type":"self_hosted"}}`) if source == "unknown" { input.ModelProviderSource = "session" } diff --git a/services/core/internal/persistence/postgres/pgtest/pgtest.go b/services/core/internal/persistence/postgres/pgtest/pgtest.go index da2094abe..4da822865 100644 --- a/services/core/internal/persistence/postgres/pgtest/pgtest.go +++ b/services/core/internal/persistence/postgres/pgtest/pgtest.go @@ -8,11 +8,13 @@ import ( "errors" "os" "strings" + "sync" "testing" "time" "github.com/google/uuid" "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" "github.com/jackc/pgx/v5/pgxpool" "github.com/jackc/pgx/v5/stdlib" @@ -71,7 +73,7 @@ func OpenIsolated(t testing.TB, configure func(*pgxpool.Config)) *pgxpool.Pool { admin := Open(t) name := "oac_isolated_" + strings.ReplaceAll(uuid.NewString(), "-", "")[:12] + "_tests" quoted := pgx.Identifier{name}.Sanitize() - if _, err := admin.Exec(t.Context(), "CREATE DATABASE "+quoted); err != nil { + if err := copyTemplate(t.Context(), admin, quoted); err != nil { t.Fatal(err) } t.Cleanup(func() { @@ -83,12 +85,6 @@ func OpenIsolated(t testing.TB, configure func(*pgxpool.Config)) *pgxpool.Pool { }) cfg := admin.Config().Copy() cfg.ConnConfig.Database = name - connection := stdlib.RegisterConnConfig(cfg.ConnConfig) - err := migrations.Apply(t.Context(), connection) - stdlib.UnregisterConnConfig(connection) - if err != nil { - t.Fatal(err) - } if configure != nil { configure(cfg) } @@ -100,6 +96,48 @@ func OpenIsolated(t testing.TB, configure func(*pgxpool.Config)) *pgxpool.Pool { return pool } +// template is the database OpenIsolated copies: oac_*_template_tests beside +// the shared test database. It persists across runs, and each test process +// migrates it once, as Open migrates the shared database. +var template struct { + once sync.Once + name string + err error +} + +// copyTemplate creates the quoted database from the migrated template. +// Postgres refuses to copy a template while another session is connected to +// it, so migrating and copying hold one advisory lock across test processes. +func copyTemplate(ctx context.Context, admin *pgxpool.Pool, quoted string) error { + conn, err := admin.Acquire(ctx) + if err != nil { + return err + } + defer conn.Release() + if _, err = conn.Exec(ctx, "SELECT pg_advisory_lock(hashtext('oac_test_template'))"); err != nil { + return err + } + defer conn.Exec(context.Background(), "SELECT pg_advisory_unlock(hashtext('oac_test_template'))") + template.once.Do(func() { + cfg := admin.Config().Copy() + cfg.ConnConfig.Database = strings.TrimSuffix(cfg.ConnConfig.Database, "_tests") + "_template_tests" + _, err := conn.Exec(context.Background(), "CREATE DATABASE "+pgx.Identifier{cfg.ConnConfig.Database}.Sanitize()) + var exists *pgconn.PgError + if err != nil && (!errors.As(err, &exists) || exists.Code != "42P04") { + template.err = err + return + } + connection := stdlib.RegisterConnConfig(cfg.ConnConfig) + defer stdlib.UnregisterConnConfig(connection) + template.name, template.err = cfg.ConnConfig.Database, migrations.Apply(context.Background(), connection) + }) + if template.err != nil { + return template.err + } + _, err = conn.Exec(ctx, "CREATE DATABASE "+quoted+" TEMPLATE "+pgx.Identifier{template.name}.Sanitize()) + return err +} + // databaseConfig validates the driver's effective database, so query // parameters and key/value DSNs cannot redirect tests to another database. func databaseConfig(dsn string) (*pgxpool.Config, error) { diff --git a/services/core/internal/persistence/postgres/runtimehistorypg/reader_test.go b/services/core/internal/persistence/postgres/runtimehistorypg/reader_test.go index aca72a965..85bc4c916 100644 --- a/services/core/internal/persistence/postgres/runtimehistorypg/reader_test.go +++ b/services/core/internal/persistence/postgres/runtimehistorypg/reader_test.go @@ -20,7 +20,7 @@ const ( func testCapabilities() runtimehistory.Capabilities { return runtimehistory.Capabilities{ - CollectionMode: runtimehistory.CollectionPeriodic, SampleInterval: 30 * time.Second, + SampleInterval: 30 * time.Second, Retention: 7 * 24 * time.Hour, MinimumStep: 30 * time.Second, MaximumRange: 24 * time.Hour, MaximumPoints: 1_000, MaximumSeries: 64, MaximumTotalPoints: 10_000, Metrics: []runtimehistory.Metric{runtimehistory.MetricCPU, runtimehistory.MetricMemory}, diff --git a/services/core/internal/persistence/postgres/runtimehistorypg/samples_test.go b/services/core/internal/persistence/postgres/runtimehistorypg/samples_test.go index f6375b460..aae9979be 100644 --- a/services/core/internal/persistence/postgres/runtimehistorypg/samples_test.go +++ b/services/core/internal/persistence/postgres/runtimehistorypg/samples_test.go @@ -14,7 +14,7 @@ import ( ) func historyCapabilities() runtimehistory.Capabilities { - return runtimehistory.Capabilities{CollectionMode: runtimehistory.CollectionPeriodic, SampleInterval: 30 * time.Second, + return runtimehistory.Capabilities{SampleInterval: 30 * time.Second, Retention: 7 * 24 * time.Hour, MinimumStep: 30 * time.Second, MaximumRange: 24 * time.Hour, MaximumPoints: 1000, MaximumSeries: 64, MaximumTotalPoints: 10000, Metrics: []runtimehistory.Metric{runtimehistory.MetricCPU, runtimehistory.MetricMemory, runtimehistory.MetricTokens}} } diff --git a/services/core/internal/persistence/postgres/sessionpg/creation_test.go b/services/core/internal/persistence/postgres/sessionpg/creation_test.go index 58174a25a..fac2cb753 100644 --- a/services/core/internal/persistence/postgres/sessionpg/creation_test.go +++ b/services/core/internal/persistence/postgres/sessionpg/creation_test.go @@ -27,6 +27,7 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgtest" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/skillpg" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/providers" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/skills" @@ -35,13 +36,15 @@ import ( var creator = identity.Subject{Kind: "service_account", ID: "test-runner"} -var hostedConfiguration = json.RawMessage(`{"agent":{"model":"m"},"environment":{"type":"openai_hosted"}}`) +// environmentConfiguration configures a self_hosted Environment Session, +// which needs no sandbox deployment. +var environmentConfiguration = json.RawMessage(`{"agent":{"model":"m"},"environment":{"type":"self_hosted","workspace_directory":"/workspace"}}`) // creationService returns the Session store and service over pool with the // built-in placement rules. func creationService(t *testing.T, pool *pgxpool.Pool) (*Store, *sessions.Service) { t.Helper() - rules, err := placement.NewRules(providers.Builtin(), "") + rules, err := placement.NewRules(providers.Builtin(), "https://core.example") if err != nil { t.Fatal(err) } @@ -377,7 +380,7 @@ func TestCreationFreezesResourcesOnce(t *testing.T) { reference := environmentconfig.Skill{Metadata: environmentconfig.SkillMetadata{Type: "skill_reference", SkillID: skill.ID, Version: "latest"}} fileID := environmentconfig.InitialFile{Type: "file_id", Path: "/workspace/b", FileID: source.ID} input := sessions.CreateSession{ - Creator: creator, Engine: "codex", IdempotencyKey: "frozen", Configuration: hostedConfiguration, + Creator: creator, Engine: "codex", IdempotencyKey: "frozen", Configuration: environmentConfiguration, ModelProvider: &v1.ModelProviderInput{Protocol: "responses", BaseURL: "https://example.com/v1", APIKey: canary}, ModelProviderSource: v1.ModelProviderSourceSession, Initialization: environmentconfig.Setup{Skills: []environmentconfig.Skill{inline, reference}}, @@ -415,8 +418,8 @@ func TestCreationFreezesResourcesOnce(t *testing.T) { } assertFrozen() for _, reference := range []sessions.CreateSession{ - {Creator: creator, Engine: "codex", IdempotencyKey: "foreign-skill", Configuration: hostedConfiguration, Initialization: environmentconfig.Setup{Skills: []environmentconfig.Skill{reference}}}, - {Creator: creator, Engine: "codex", IdempotencyKey: "foreign-file", Configuration: hostedConfiguration, InitialFiles: []environmentconfig.InitialFile{fileID}}, + {Creator: creator, Engine: "codex", IdempotencyKey: "foreign-skill", Configuration: environmentConfiguration, Initialization: environmentconfig.Setup{Skills: []environmentconfig.Skill{reference}}}, + {Creator: creator, Engine: "codex", IdempotencyKey: "foreign-file", Configuration: environmentConfiguration, InitialFiles: []environmentconfig.InitialFile{fileID}}, } { if _, err := service.CreateSession(ctx, foreign, reference); !errors.Is(err, sessions.ErrNotFound) { t.Fatal(reference.IdempotencyKey, err) @@ -471,8 +474,17 @@ func TestHostedCreationAdmitsAndPlacesUnderTheDeploymentLock(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() tenant := uuid.NewString() + nodes, err := json.Marshal(sandbox.DeploymentSpec{Resources: sandbox.Resources{CPUs: 2, MemoryMiB: 2048}, Runtime: &sandbox.RuntimeRelease{SourceCommit: strings.Repeat("a", 40), + ImageID: "sha256:" + strings.Repeat("b", 64), ImageManifestDigest: "sha256:" + strings.Repeat("c", 64), MicrosandboxRef: "oac-runtime@sha256:" + strings.Repeat("d", 64), + RuntimeSHA256: strings.Repeat("e", 64), FirmwareSHA256: strings.Repeat("f", 64)}}) + if err != nil { + t.Fatal(err) + } + // A direct deployment admits hosted work without placing it on a node. + exec(t, pool, `UPDATE runtime_deployment SET installation_id=$1, backend_fingerprint=$2, provider_kind='e2b', mode='direct', generation=1, + specification='{"resources":{"cpus":2,"memory_mib":2048}}'`, uuid.New(), strings.Repeat("a", 64)) hosted := func(key string) sessions.CreateSession { - return sessions.CreateSession{Creator: creator, Engine: "codex", IdempotencyKey: key, Configuration: hostedConfiguration} + return sessions.CreateSession{Creator: creator, Engine: "codex", IdempotencyKey: key, Configuration: json.RawMessage(`{"agent":{"model":"m"},"environment":{"type":"openai_hosted"}}`)} } existing, err := service.CreateSession(ctx, tenant, hosted("existing")) if err != nil { @@ -493,8 +505,8 @@ func TestHostedCreationAdmitsAndPlacesUnderTheDeploymentLock(t *testing.T) { done <- err }() awaitBlocked(ctx, t, pool, holder) - if _, err := tx.Exec(ctx, `UPDATE runtime_deployment SET installation_id=$1, backend_fingerprint=$2, provider_kind='docker', mode='nodes', generation=1, - reset_clear='force', reset_requested_at=now(), reset_forced_at=now(), reset_audit='{}'`, uuid.New(), strings.Repeat("a", 64)); err != nil { + if _, err := tx.Exec(ctx, `UPDATE runtime_deployment SET provider_kind='docker', mode='nodes', generation=2, specification=$1, + reset_clear='force', reset_requested_at=now(), reset_forced_at=now(), reset_audit='{}'`, string(nodes)); err != nil { t.Fatal(err) } if err := tx.Commit(ctx); err != nil { @@ -506,7 +518,7 @@ func TestHostedCreationAdmitsAndPlacesUnderTheDeploymentLock(t *testing.T) { if retry, err := service.CreateSession(ctx, tenant, hosted("existing")); err != nil || retry.Created || retry.Session.ID != existing.Session.ID { t.Fatal("retry ran admission", retry, err) } - exec(t, pool, "UPDATE runtime_deployment SET reset_clear=NULL, reset_requested_at=NULL, reset_forced_at=NULL, reset_audit=NULL, provider_kind='', mode=''") + exec(t, pool, "UPDATE runtime_deployment SET reset_clear=NULL, reset_requested_at=NULL, reset_forced_at=NULL, reset_audit=NULL") if _, err := service.CreateSession(ctx, tenant, hosted("unplaced")); !errors.Is(err, placement.ErrNodeUnavailable) { t.Fatal("placement without a node", err) } @@ -541,7 +553,7 @@ func TestSkillFreezeSerializesWithVersionDeletion(t *testing.T) { if err := holder.QueryRow(ctx, "SELECT pg_backend_pid() FROM skills WHERE id=$1 FOR UPDATE", key).Scan(&holderPID); err != nil { t.Fatal(err) } - input := sessions.CreateSession{Creator: creator, Engine: "codex", IdempotencyKey: uuid.NewString(), Configuration: hostedConfiguration, + input := sessions.CreateSession{Creator: creator, Engine: "codex", IdempotencyKey: uuid.NewString(), Configuration: environmentConfiguration, Initialization: environmentconfig.Setup{Skills: []environmentconfig.Skill{{Metadata: environmentconfig.SkillMetadata{Type: "skill_reference", SkillID: skill.ID, Version: "2"}}}}} type outcome struct { creation sessions.Creation diff --git a/services/core/internal/persistence/postgres/skillpg/audit_test.go b/services/core/internal/persistence/postgres/skillpg/audit_test.go index de62c39d4..a9684b4f5 100644 --- a/services/core/internal/persistence/postgres/skillpg/audit_test.go +++ b/services/core/internal/persistence/postgres/skillpg/audit_test.go @@ -100,7 +100,7 @@ func snapshot(t *testing.T, pool *pgxpool.Pool, tenant string, tables ...string) // write. Comparing complete tenant rows proves the write rolled back. func TestWriteAuditTransactions(t *testing.T) { pool := pgtest.OpenIsolated(t, nil) - f := newFixture(t, pool, testCipher(t, 91)) + f := newFixture(t, pool, pgtest.CredentialKey(t)) ctx := t.Context() if _, err := pool.Exec(ctx, `CREATE FUNCTION reject_resource_audit_fixture() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.request_id = 'reject-resource-audit' THEN RAISE EXCEPTION 'forced audit insertion failure'; END IF; RETURN NEW; END $$; @@ -158,7 +158,7 @@ const rejectedAdminRequest = "reject-admin-mutation-fixture" // operation, and a failed administrator audit rolls the deletion back. func TestAdminDeleteAuditTransactions(t *testing.T) { pool := pgtest.OpenIsolated(t, nil) - f := newFixture(t, pool, testCipher(t, 94)) + f := newFixture(t, pool, pgtest.CredentialKey(t)) ctx := t.Context() if _, err := pool.Exec(ctx, `CREATE FUNCTION reject_admin_mutation_fixture() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.request_id = 'reject-admin-mutation-fixture' THEN RAISE EXCEPTION 'forced administrator audit failure'; END IF; RETURN NEW; END $$; diff --git a/services/core/internal/persistence/postgres/skillpg/skillpg_test.go b/services/core/internal/persistence/postgres/skillpg/skillpg_test.go index 8824d4534..d843546e5 100644 --- a/services/core/internal/persistence/postgres/skillpg/skillpg_test.go +++ b/services/core/internal/persistence/postgres/skillpg/skillpg_test.go @@ -39,9 +39,11 @@ func newFixture(t *testing.T, pool *pgxpool.Pool, cipher *credentialcrypto.Ciphe return fixture{pool: pool, store: store, service: service} } -func testCipher(t *testing.T, seed byte) *credentialcrypto.Cipher { +// otherKey is a credential key other than pgtest.CredentialKey, for a test that +// a replaced key cannot open content. +func otherKey(t *testing.T) *credentialcrypto.Cipher { t.Helper() - cipher, err := credentialcrypto.New(bytes.Repeat([]byte{seed}, 32)) + cipher, err := credentialcrypto.New(bytes.Repeat([]byte{0x5a}, 32)) if err != nil { t.Fatal(err) } @@ -97,7 +99,7 @@ func (f fixture) rowCounts(t *testing.T, skill uuid.UUID) (skillRows, versionRow } func TestOwnershipEncryptionAndVersions(t *testing.T) { - f := newFixture(t, pgtest.Open(t), testCipher(t, 41)) + f := newFixture(t, pgtest.Open(t), pgtest.CredentialKey(t)) ctx := t.Context() tenant, foreign := uuid.NewString(), uuid.NewString() bundle := proofArchive(t, "confidential-skill-canary") @@ -229,7 +231,7 @@ func TestOwnershipEncryptionAndVersions(t *testing.T) { // A zero page is empty; HasMore reports whether a resource follows the cursor. func TestListsAcceptLimitZero(t *testing.T) { - f := newFixture(t, pgtest.Open(t), testCipher(t, 43)) + f := newFixture(t, pgtest.Open(t), pgtest.CredentialKey(t)) ctx := t.Context() tenant, foreign := uuid.NewString(), uuid.NewString() bundle := proofArchive(t, "limit-zero") @@ -287,7 +289,7 @@ func TestListsAcceptLimitZero(t *testing.T) { func TestMetadataTracksDefaultVersion(t *testing.T) { pool := pgtest.Open(t) - f := newFixture(t, pool, testCipher(t, 74)) + f := newFixture(t, pool, otherKey(t)) // Metadata reads and default changes work under a replaced key. replaced := newFixture(t, pool, pgtest.CredentialKey(t)) ctx := t.Context() @@ -360,7 +362,7 @@ func TestMetadataTracksDefaultVersion(t *testing.T) { // Deleting the sole version deletes the Skill and all its rows in one commit. func TestSoleVersionDeletionRemovesSkill(t *testing.T) { - f := newFixture(t, pgtest.Open(t), testCipher(t, 63)) + f := newFixture(t, pgtest.Open(t), pgtest.CredentialKey(t)) ctx := t.Context() tenant, foreign := uuid.NewString(), uuid.NewString() skill := f.create(t, tenant, proofArchive(t, "sole-version")) @@ -410,7 +412,7 @@ func TestSoleVersionDeletionRemovesSkill(t *testing.T) { // commits first makes the default undeletable, and a deletion that commits // first makes the later upload miss the Skill. Neither loses acknowledged data. func TestSoleVersionDeletionSerializesWithUpload(t *testing.T) { - f := newFixture(t, pgtest.Open(t), testCipher(t, 63)) + f := newFixture(t, pgtest.Open(t), pgtest.CredentialKey(t)) ctx := t.Context() tenant := uuid.NewString() for _, uploadFirst := range []bool{true, false} { @@ -469,7 +471,7 @@ func TestSoleVersionDeletionSerializesWithUpload(t *testing.T) { // LockSkills locks each named Skill of the tenant once, and // ReadVersionForFreeze opens a version only with the key that sealed it. func TestFreezeReads(t *testing.T) { - cipher := testCipher(t, 48) + cipher := pgtest.CredentialKey(t) f := newFixture(t, pgtest.Open(t), cipher) ctx := t.Context() tenant := uuid.NewString() @@ -497,7 +499,7 @@ func TestFreezeReads(t *testing.T) { if _, err := skillpg.ReadVersionForFreeze(ctx, q, cipher, tenantID, first.ID, 2); !errors.Is(err, skills.ErrNotFound) { t.Fatal("missing version", err) } - if _, err := skillpg.ReadVersionForFreeze(ctx, q, testCipher(t, 49), tenantID, first.ID, 1); err == nil || errors.Is(err, skills.ErrNotFound) { + if _, err := skillpg.ReadVersionForFreeze(ctx, q, otherKey(t), tenantID, first.ID, 1); err == nil || errors.Is(err, skills.ErrNotFound) { t.Fatal("another key opened the version", err) } } @@ -527,7 +529,7 @@ func waitForLockWaiters(t *testing.T, pool *pgxpool.Pool, holder int32, count in // Text PostgreSQL cannot store, here a YAML-escaped U+0000 in the manifest // description, is the shared unstorable-text error and stores nothing. func TestUnstorableTextStoresNothing(t *testing.T) { - f := newFixture(t, pgtest.Open(t), testCipher(t, 45)) + f := newFixture(t, pgtest.Open(t), pgtest.CredentialKey(t)) ctx := t.Context() tenant := uuid.NewString() bad := archive(t, "proof", `"before\0after"`, "unstorable") @@ -549,7 +551,7 @@ func TestUnstorableTextStoresNothing(t *testing.T) { // A malformed tenant is invalid input, not a database error. func TestMalformedTenantIsInvalidInput(t *testing.T) { - f := newFixture(t, pgtest.Open(t), testCipher(t, 46)) + f := newFixture(t, pgtest.Open(t), pgtest.CredentialKey(t)) if _, err := f.service.CreateSkill(t.Context(), skills.CreateSkill{TenantID: "not-a-tenant", Archive: proofArchive(t, "tenant")}); !errors.Is(err, skills.ErrInvalidInput) { t.Fatal("create", err) } diff --git a/services/core/internal/persistence/postgres/vaultpg/audit_test.go b/services/core/internal/persistence/postgres/vaultpg/audit_test.go index 2c88daa5f..9a894bd5e 100644 --- a/services/core/internal/persistence/postgres/vaultpg/audit_test.go +++ b/services/core/internal/persistence/postgres/vaultpg/audit_test.go @@ -1,7 +1,6 @@ package vaultpg_test import ( - "bytes" "context" "reflect" "strings" @@ -141,7 +140,7 @@ func prepareAuditMutation(t *testing.T, service *vaults.Service, tenant, name st // table snapshots prove the rollback of ciphertext, timestamps and cascades. func TestVaultMutationsRollBackWithTheirAudit(t *testing.T) { pool := pgtest.OpenIsolated(t, nil) - service := newService(t, pool, newCipher(t, bytes.Repeat([]byte{91}, 32)), nil) + service := newService(t, pool, pgtest.CredentialKey(t), nil) rejectAudits(t, pool) secrets := []string{"audit-private-token", "audit-private-replacement"} for _, provenance := range []string{"public", "admin"} { diff --git a/services/core/internal/persistence/postgres/vaultpg/vaults_test.go b/services/core/internal/persistence/postgres/vaultpg/vaults_test.go index 6a7ce4da8..f0ccac81f 100644 --- a/services/core/internal/persistence/postgres/vaultpg/vaults_test.go +++ b/services/core/internal/persistence/postgres/vaultpg/vaults_test.go @@ -306,10 +306,10 @@ func TestVaultDeletionCascadeBindingAndRestart(t *testing.T) { func TestVaultDeletionConcurrentChildMutations(t *testing.T) { _, pool := openStore(t) tenant := uuid.NewString() - service := newService(t, pool, newCipher(t, bytes.Repeat([]byte{44}, 32)), nil) + service := newService(t, pool, pgtest.CredentialKey(t), nil) // Each operation runs on its own Store, so only PostgreSQL orders them. other := func() *vaults.Service { - return newService(t, pool, newCipher(t, bytes.Repeat([]byte{44}, 32)), nil) + return newService(t, pool, pgtest.CredentialKey(t), nil) } for range 8 { vault := createVault(t, service, tenant) diff --git a/services/core/internal/runtimehistory/service_test.go b/services/core/internal/runtimehistory/service_test.go index d56955aa7..cf0047ae6 100644 --- a/services/core/internal/runtimehistory/service_test.go +++ b/services/core/internal/runtimehistory/service_test.go @@ -77,7 +77,6 @@ func (r *fakeReader) Query(_ context.Context, query Query) (Result, error) { func capabilities() Capabilities { return Capabilities{ - CollectionMode: CollectionPeriodic, SampleInterval: 30 * time.Second, Retention: 7 * 24 * time.Hour, MinimumStep: 30 * time.Second, @@ -116,7 +115,7 @@ func TestServiceAuthorizesAndBoundsBackendQuery(t *testing.T) { if err != nil { t.Fatal(err) } - if len(reader.queries) != 1 || reader.queries[0].Scope != scope || reader.queries[0].Step != time.Minute || response.Resolution != time.Minute || !response.Durable() { + if len(reader.queries) != 1 || reader.queries[0].Scope != scope || reader.queries[0].Step != time.Minute || response.Resolution != time.Minute { t.Fatalf("unexpected bounded history query: query=%+v response=%+v", reader.queries, response) } ratio = .9 @@ -329,7 +328,7 @@ func TestServiceRejectsMalformedBackendResults(t *testing.T) { func TestServiceRejectsUnsafeCapabilities(t *testing.T) { base := capabilities() for _, mutate := range []func(*Capabilities){ - func(value *Capabilities) { value.CollectionMode = CollectionOnRead }, + func(value *Capabilities) { value.SampleInterval = 0 }, func(value *Capabilities) { value.MaximumRange = value.Retention + time.Second }, func(value *Capabilities) { value.Metrics = []Metric{MetricCPU, MetricCPU} }, } { diff --git a/services/core/internal/runtimehistory/types.go b/services/core/internal/runtimehistory/types.go index 29485ad16..f7a0a48d3 100644 --- a/services/core/internal/runtimehistory/types.go +++ b/services/core/internal/runtimehistory/types.go @@ -10,13 +10,6 @@ import ( "github.com/google/uuid" ) -type CollectionMode string - -const ( - CollectionOnRead CollectionMode = "on_read" - CollectionPeriodic CollectionMode = "periodic" -) - type Metric string const ( @@ -54,7 +47,6 @@ func validCount(value int) bool { // future public capability response. Backend names, endpoints, credentials and // tenant identity are deliberately absent. type Capabilities struct { - CollectionMode CollectionMode SampleInterval time.Duration Retention time.Duration MinimumStep time.Duration @@ -65,22 +57,9 @@ type Capabilities struct { Metrics []Metric } -func (c Capabilities) Durable() bool { - return c.CollectionMode == CollectionPeriodic && c.SampleInterval > 0 -} - func (c Capabilities) Validate() error { - switch c.CollectionMode { - case CollectionOnRead: - if c.SampleInterval != 0 { - return errors.New("on-read Runtime history cannot declare a sampling interval") - } - case CollectionPeriodic: - if c.SampleInterval <= 0 { - return errors.New("periodic Runtime history requires a sampling interval") - } - default: - return errors.New("invalid Runtime history collection mode") + if c.SampleInterval <= 0 { + return errors.New("Runtime history requires a sampling interval") } if c.Retention <= 0 || c.MinimumStep <= 0 || c.MaximumRange <= 0 || c.MaximumRange > c.Retention { return errors.New("invalid Runtime history time bounds") diff --git a/services/core/internal/runtimeobs/service.go b/services/core/internal/runtimeobs/service.go index c38db7359..26e6c15a2 100644 --- a/services/core/internal/runtimeobs/service.go +++ b/services/core/internal/runtimeobs/service.go @@ -36,11 +36,10 @@ type Service struct { // NewService reads managed Runtimes through source, which returns the Sandbox // Provider of the deployment's current selection and its registered kind, or -// ErrUnavailable while none is selected. A nil source means this Core has no -// managed deployment. +// ErrUnavailable while none is selected. func NewService(resolver TargetResolver, source func(context.Context) (Source, string, error), options ...ServiceOption) (*Service, error) { - if resolver == nil { - return nil, errors.New("Runtime observation resolver is required") + if resolver == nil || source == nil { + return nil, errors.New("Runtime observation resolver and source are required") } config := serviceOptions{} for _, option := range options { @@ -225,9 +224,6 @@ func (s *Service) resolve(ctx context.Context, tenantID, sessionID string, owner default: return Observation{}, nil, errors.New("invalid managed Runtime allocation state") } - if s.source == nil { - return Observation{Target: target, Status: StatusUnavailable, Reason: "source_not_configured", ResolvedAt: resolvedAt}, nil, nil - } return Observation{}, &sourceRead{target: target}, nil } diff --git a/services/core/internal/runtimeobs/service_test.go b/services/core/internal/runtimeobs/service_test.go index 763b4363d..40f15551b 100644 --- a/services/core/internal/runtimeobs/service_test.go +++ b/services/core/internal/runtimeobs/service_test.go @@ -126,27 +126,18 @@ func TestServiceDoesNotCallSourcesForUnsupportedModes(t *testing.T) { } } -func TestServicePreservesUnavailableAndObservedZero(t *testing.T) { +func TestServicePreservesObservedZero(t *testing.T) { target := Target{EnvironmentID: "environment", Mode: ModeManaged, Instance: Instance{AllocationID: "allocation", ProviderKey: "provider", AllocationState: "running"}} - service, err := NewService(fixedResolver{target: target}, nil) - if err != nil { - t.Fatal(err) - } - observation, err := service.ObserveSession(t.Context(), "tenant", "session") - if err != nil || observation.Status != StatusUnavailable || observation.Reason != "source_not_configured" || observation.Sample != nil { - t.Fatalf("missing source was not unavailable: %+v %v", observation, err) - } - zeroCPU := float64(0) zeroMemory := uint64(0) now := time.Date(2026, 9, 22, 1, 0, 0, 0, time.UTC) source := &fixedSource{sample: Sample{ObservedAt: now, CPUUsageSecondsTotal: &zeroCPU, MemoryUsageBytes: &zeroMemory}} - service, err = NewService(fixedResolver{target: target}, sourceOf(source)) + service, err := NewService(fixedResolver{target: target}, sourceOf(source)) if err != nil { t.Fatal(err) } service.now = func() time.Time { return now } - observation, err = service.ObserveSession(t.Context(), "tenant", "session") + observation, err := service.ObserveSession(t.Context(), "tenant", "session") if err != nil || observation.Status != StatusObserved || observation.Sample == nil || observation.Sample.CPUUsageSecondsTotal == nil || observation.Sample.MemoryUsageBytes == nil { t.Fatalf("observed zero was lost: %+v %v", observation, err) } @@ -318,13 +309,16 @@ func TestServiceExportQueueNeverBlocksOrChangesObservation(t *testing.T) { } func TestServiceIgnoresExporterFailureAndValidatesOptions(t *testing.T) { - if _, err := NewService(fixedResolver{}, nil, WithExporter(nil, ExportOptions{})); err == nil { + if _, err := NewService(fixedResolver{}, nil); err == nil { + t.Fatal("missing source was accepted") + } + if _, err := NewService(fixedResolver{}, sourceOf(&fixedSource{}), WithExporter(nil, ExportOptions{})); err == nil { t.Fatal("nil exporter was accepted") } - if _, err := NewService(fixedResolver{}, nil, WithExporter(channelExporter{}, ExportOptions{QueueCapacity: -1})); err == nil { + if _, err := NewService(fixedResolver{}, sourceOf(&fixedSource{}), WithExporter(channelExporter{}, ExportOptions{QueueCapacity: -1})); err == nil { t.Fatal("negative export queue capacity was accepted") } - if _, err := NewService(fixedResolver{}, nil, WithExporter(channelExporter{}, ExportOptions{Timeout: -time.Second})); err == nil { + if _, err := NewService(fixedResolver{}, sourceOf(&fixedSource{}), WithExporter(channelExporter{}, ExportOptions{Timeout: -time.Second})); err == nil { t.Fatal("negative export timeout was accepted") } @@ -486,7 +480,7 @@ func TestServiceExportsPeriodicSourceTimeoutAfterFinalOwnershipFence(t *testing. func TestServiceClassifiesResolverAndTerminalAllocationUnavailability(t *testing.T) { now := time.Date(2026, 9, 22, 1, 0, 0, 0, time.UTC) - service, err := NewService(fixedResolver{target: Target{SessionID: "session", EnvironmentID: "environment", Mode: ModeManaged}, err: ErrUnavailable}, nil) + service, err := NewService(fixedResolver{target: Target{SessionID: "session", EnvironmentID: "environment", Mode: ModeManaged}, err: ErrUnavailable}, sourceOf(&fixedSource{})) if err != nil { t.Fatal(err) } @@ -563,7 +557,7 @@ func TestServiceRejectsMismatchedResolvedOwnership(t *testing.T) { {TenantID: "tenant", SessionID: "session", EnvironmentID: "unexpected", Mode: ModeNone}, {TenantID: "tenant", SessionID: "session", Mode: ModeSelfHosted}, } { - service, err := NewService(fixedResolver{target: target}, nil) + service, err := NewService(fixedResolver{target: target}, sourceOf(&fixedSource{})) if err != nil { t.Fatal(err) } diff --git a/services/core/internal/sandbox/providers/configuration_flow_test.go b/services/core/internal/sandbox/providers/configuration_flow_test.go index 68624d30c..303d07e56 100644 --- a/services/core/internal/sandbox/providers/configuration_flow_test.go +++ b/services/core/internal/sandbox/providers/configuration_flow_test.go @@ -99,7 +99,7 @@ func TestAdditionalConfigurationProviderUsesCommonAPIAndStore(t *testing.T) { // registry it is built with. deployments := func() *deployment.Service { storage := deploymentpg.New(pgunit.NewPool(pool), pgtest.CredentialKey(t)) - rules, err := placement.NewRules(registry, "") + rules, err := placement.NewRules(registry, "https://core.example") if err != nil { t.Fatal(err) } diff --git a/services/core/internal/sessions/creation_test.go b/services/core/internal/sessions/creation_test.go index 5bb22c0fc..7d9cc4524 100644 --- a/services/core/internal/sessions/creation_test.go +++ b/services/core/internal/sessions/creation_test.go @@ -468,7 +468,7 @@ func TestCreateSession(t *testing.T) { calls []string }{ {"a reset closes hosted admission", rules, func(tx *fakeCreationTx) { - tx.lockDeployment = returns(placement.Deployment{InstallationID: "installation", Resetting: true}) + tx.lockDeployment = returns(placement.Deployment{InstallationID: "installation", Provider: "docker", Resetting: true}) }, placement.ErrResetAdmission, []string{"UpsertSession create", "LockDeployment"}}, {"no node places the Environment", rules, func(tx *fakeCreationTx) { tx.lockDeployment = returns(placement.Deployment{InstallationID: "installation", Provider: "docker", Specification: json.RawMessage(`{}`)}) diff --git a/services/core/migrations/000096_configured_admission.sql b/services/core/migrations/000096_configured_admission.sql new file mode 100644 index 000000000..0d2c98c4c --- /dev/null +++ b/services/core/migrations/000096_configured_admission.sql @@ -0,0 +1,20 @@ +-- +goose Up +-- Only a claimed deployment with a provider admits hosted work, so every +-- allocation runs on a node or belongs to a direct deployment, and none needs +-- a keepalive lease. +-- +goose StatementBegin +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM runtime_allocations a CROSS JOIN runtime_deployment d + WHERE a.state <> 'released' AND a.node_id IS NULL AND d.mode <> 'direct') + OR EXISTS (SELECT 1 FROM environments e JOIN sessions s ON s.id = e.session_id CROSS JOIN runtime_deployment d + WHERE d.installation_id IS NULL AND s.deleted_at IS NULL AND e.status = 'pending' + AND s.configuration->'environment'->>'type' = 'openai_hosted') THEN + RAISE EXCEPTION 'Cannot upgrade: hosted Sessions hold sandbox work that no configured sandbox deployment admitted. Delete those Sessions and release their runtime allocations, then upgrade'; + END IF; +END $$; +-- +goose StatementEnd +ALTER TABLE runtime_allocations DROP COLUMN kept_at; + +-- +goose Down +ALTER TABLE runtime_allocations ADD COLUMN kept_at timestamptz NOT NULL DEFAULT clock_timestamp(); diff --git a/services/core/tests/integration/admin_delete_audit_test.go b/services/core/tests/integration/admin_delete_audit_test.go index bceb94c40..c6543cd35 100644 --- a/services/core/tests/integration/admin_delete_audit_test.go +++ b/services/core/tests/integration/admin_delete_audit_test.go @@ -10,7 +10,6 @@ import ( "testing" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/adminaudit" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/writeaudit" "github.com/google/uuid" @@ -106,12 +105,7 @@ func assertAdminMutationAudit(t *testing.T, s *Store, tenant, request, action, k } func TestAdminDeleteResourceAuditTransactions(t *testing.T) { - _, pool := newManagedTestStore(t) - cipher, err := credentialcrypto.New(bytes.Repeat([]byte{94}, 32)) - if err != nil { - t.Fatal(err) - } - s := NewWithCredentialCipher(pool, cipher) + s, pool := newManagedTestStore(t) rejectAdminAuditInsert(t, s) tables := []string{"agents", "agent_model_execution", "sessions", "turns", "environments", "session_artifacts", "admin_audit_log", "write_audit_operations", "write_audit_owners", "pg_largeobject_metadata", "pg_largeobject"} for _, name := range []string{"session_delete", "artifact_delete"} { diff --git a/services/core/tests/integration/admin_session_archive_test.go b/services/core/tests/integration/admin_session_archive_test.go index 846c69b12..a36c7290c 100644 --- a/services/core/tests/integration/admin_session_archive_test.go +++ b/services/core/tests/integration/admin_session_archive_test.go @@ -8,7 +8,6 @@ import ( "reflect" "testing" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/files" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/filepg" @@ -39,12 +38,7 @@ func uploadSource(data []byte) func(io.Writer) (files.Upload, error) { func managedArchiveFixture(t *testing.T) (*Store, *Store, string) { t.Helper() - _, pool := newManagedTestStore(t) - cipher, err := credentialcrypto.New(bytes.Repeat([]byte{37}, 32)) - if err != nil { - t.Fatal(err) - } - s := NewWithCredentialCipher(pool, cipher) + s, _ := newManagedTestStore(t) w := executionWriter(t, s) installation := uuid.NewString() changes := deploymentExecution(t, w) diff --git a/services/core/tests/integration/admin_session_archive_worker_http_test.go b/services/core/tests/integration/admin_session_archive_worker_http_test.go index 8e42cf4e7..fb536b9e3 100644 --- a/services/core/tests/integration/admin_session_archive_worker_http_test.go +++ b/services/core/tests/integration/admin_session_archive_worker_http_test.go @@ -1,7 +1,6 @@ package integration import ( - "bytes" "context" "encoding/json" "errors" @@ -17,7 +16,6 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/adminaudit" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/api" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/execution" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" @@ -31,12 +29,7 @@ import ( // the deployment execution operations on the Worker's lease, as server startup // does. func TestAdminSessionArchiveWorkerHTTPPostgres(t *testing.T) { - _, pool := newManagedTestStore(t) - cipher, err := credentialcrypto.New(bytes.Repeat([]byte{83}, 32)) - if err != nil { - t.Fatal(err) - } - s := NewWithCredentialCipher(pool, cipher) + s, pool := newManagedTestStore(t) installation := uuid.NewString() provider := &lifecycleProvider{resources: map[string]sandbox.Info{}} deployments := deploymentService(t, s) diff --git a/services/core/tests/integration/agent_execution_defaults_http_test.go b/services/core/tests/integration/agent_execution_defaults_http_test.go index fe4fa7db8..f243f40f1 100644 --- a/services/core/tests/integration/agent_execution_defaults_http_test.go +++ b/services/core/tests/integration/agent_execution_defaults_http_test.go @@ -9,16 +9,14 @@ import ( "testing" v1 "github.com/MiniMax-AI/OpenAgentCore/contracts/agents-api/v1" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/modelconfiguration" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" "github.com/google/uuid" ) func TestAgentExecutionDefaultsPublicSnapshotAndPrecedence(t *testing.T) { - _, pool := testStore(t) - cipher, _ := credentialcrypto.New(bytes.Repeat([]byte{19}, 32)) - st := NewWithCredentialCipher(pool, cipher) + st, _ := configuredStore(t) + pool := st.pool tenant, token := uuid.NewString(), uuid.NewString() auth := newTestAuthenticator(t, []testAPIKey{{OrganizationID: "test-org", ProjectID: uuid.NewString(), SubjectKind: "service_account", SubjectID: "defaults-test", TokenSHA256: runtimedevice.HashCredential(token), TenantID: tenant}}) deployment := &v1.ModelProviderInput{Protocol: "responses", BaseURL: "https://deployment.example/v1", APIKey: "deployment-canary"} @@ -131,7 +129,7 @@ func TestAgentExecutionDefaultsPublicSnapshotAndPrecedence(t *testing.T) { t.Fatal("retry created another Session") } call("POST", "/v1/agents/sessions", body, uuid.NewString(), 404) - st = NewWithCredentialCipher(pool, cipher) + st = New(t, pool) assertSnapshot(sessionID, "model-original", "https://saved.example/v1", "saved-canary") assertSnapshot(fresh, "model-new", "https://override.example/v1", "override-canary") inline := `{"agent":{"model":"inline-model"},"environment":{"type":"openai_hosted"}}` diff --git a/services/core/tests/integration/configuration_validation_public_test.go b/services/core/tests/integration/configuration_validation_public_test.go index a04885f0d..8e29098dc 100644 --- a/services/core/tests/integration/configuration_validation_public_test.go +++ b/services/core/tests/integration/configuration_validation_public_test.go @@ -1,7 +1,6 @@ package integration import ( - "bytes" "encoding/json" "net/http" "net/http/httptest" @@ -9,7 +8,6 @@ import ( "testing" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/agents" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" "github.com/google/uuid" ) @@ -19,12 +17,7 @@ import ( // resource ownership. func TestAgentConfigurationValidationRejectsWithoutWritesPostgres(t *testing.T) { // An isolated database keeps the no-write digest independent of other tests. - _, pool := newManagedTestStore(t) - cipher, err := credentialcrypto.New(bytes.Repeat([]byte{63}, 32)) - if err != nil { - t.Fatal(err) - } - s := NewWithCredentialCipher(pool, cipher) + s, pool := newManagedTestStore(t) owner, foreign, ownerTenant := uuid.NewString(), uuid.NewString(), uuid.NewString() auth := newTestAuthenticator(t, []testAPIKey{ {OrganizationID: "test-org", ProjectID: uuid.NewString(), SubjectKind: "service_account", SubjectID: "config-owner", TokenSHA256: runtimedevice.HashCredential(owner), TenantID: ownerTenant}, diff --git a/services/core/tests/integration/deployment_model_providers_http_test.go b/services/core/tests/integration/deployment_model_providers_http_test.go index be9c35db1..64344aa6d 100644 --- a/services/core/tests/integration/deployment_model_providers_http_test.go +++ b/services/core/tests/integration/deployment_model_providers_http_test.go @@ -11,7 +11,6 @@ import ( v1 "github.com/MiniMax-AI/OpenAgentCore/contracts/agents-api/v1" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/adminaudit" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/api" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/modelconfiguration" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/auditpg" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/modelconfigurationpg" @@ -24,12 +23,8 @@ import ( // frozen only into hosted Sessions; self-hosted Sessions bring their own // provider, and a Session with no provider is rejected before any write. func TestDeploymentModelProvidersHTTP(t *testing.T) { - _, pool := testStore(t) - if _, err := pool.Exec(t.Context(), "DELETE FROM deployment_model_providers"); err != nil { - t.Fatal(err) - } - cipher, _ := credentialcrypto.New(bytes.Repeat([]byte{53}, 32)) - st := NewWithCredentialCipher(pool, cipher) + st, _ := configuredStore(t) + pool := st.pool tenant, projectKey, coreKey := uuid.NewString(), uuid.NewString(), uuid.NewString() auth := newTestAuthenticator(t, []testAPIKey{{OrganizationID: "test-org", ProjectID: uuid.NewString(), SubjectKind: "service_account", SubjectID: "defaults-http", TokenSHA256: runtimedevice.HashCredential(projectKey), TenantID: tenant}}) admin, err := api.NewDeploymentAuthenticator([]string{runtimedevice.HashCredential(coreKey)}) @@ -162,7 +157,7 @@ func TestDeploymentModelProvidersHTTP(t *testing.T) { // Simulate a default persisted when cross-protocol execution was supported. // It must remain readable, but cannot create new Sessions or be rewritten. historical := strings.Replace(codexDefault, `"protocol":"responses"`, `"protocol":"anthropic"`, 1) - encrypted, err := cipher.SealDeploymentModelProvider([]byte(historical), "codex") + encrypted, err := st.credentialCipher.SealDeploymentModelProvider([]byte(historical), "codex") if err != nil { t.Fatal(err) } diff --git a/services/core/tests/integration/device_bootstrap_binding_test.go b/services/core/tests/integration/device_bootstrap_binding_test.go index bf28d963d..4d85bc1c9 100644 --- a/services/core/tests/integration/device_bootstrap_binding_test.go +++ b/services/core/tests/integration/device_bootstrap_binding_test.go @@ -59,14 +59,14 @@ func TestDeviceCredentialCarriesPersistedAllocationNode(t *testing.T) { } func TestDeviceCredentialWithoutManagedNodeRetainsPublicRouteIdentity(t *testing.T) { - s, _ := testStore(t) + s, installation := configuredStore(t) tenant := uuid.NewString() ordinary, err := sessionService(t, s).CreateDevice(t.Context(), tenant, "ordinary", runtimedevice.HashCredential("ordinary-token")) if err != nil { t.Fatal(err) } _, environment := localEnvironment(t, s, tenant) - allocation, err := deploymentExecution(t, executionWriter(t, s)).ReserveAllocation(t.Context(), deployment.AllocationKey{TenantID: tenant, EnvironmentID: environment.ID}, uuid.NewString(), runtimedevice.HashCredential("allocation-token")) + allocation, err := deploymentExecution(t, executionWriter(t, s)).ReserveAllocation(t.Context(), deployment.AllocationKey{TenantID: tenant, EnvironmentID: environment.ID}, installation, runtimedevice.HashCredential("allocation-token")) if err != nil { t.Fatal(err) } diff --git a/services/core/tests/integration/dispatch_test.go b/services/core/tests/integration/dispatch_test.go index 2f9ab27f3..8bbd3612b 100644 --- a/services/core/tests/integration/dispatch_test.go +++ b/services/core/tests/integration/dispatch_test.go @@ -50,7 +50,16 @@ func newDispatchHarness(t *testing.T) *dispatchHarness { func newDispatchHarnessForSession(t *testing.T, configuration []byte, local bool) *dispatchHarness { t.Helper() + var snapshot struct { + Environment struct { + Type string `json:"type"` + } `json:"environment"` + } + _ = json.Unmarshal(configuration, &snapshot) s, _ := testStore(t) + if snapshot.Environment.Type == "openai_hosted" { + s, _ = configuredStore(t) + } h := &dispatchHarness{t: t, s: s, tenant: uuid.NewString(), environments: map[string]*dispatchHarness{}} ctx := context.Background() var err error @@ -60,12 +69,6 @@ func newDispatchHarnessForSession(t *testing.T, configuration []byte, local bool } secret := uuid.NewString() h.credential = secret - var snapshot struct { - Environment struct { - Type string `json:"type"` - } `json:"environment"` - } - _ = json.Unmarshal(configuration, &snapshot) if snapshot.Environment.Type == "self_hosted" { h.device, h.credential = enrollFixtureSession(t, s, h.tenant, h.session) secret = h.credential diff --git a/services/core/tests/integration/environment_executor_management_test.go b/services/core/tests/integration/environment_executor_management_test.go index a2f5a0d84..a753ee1cd 100644 --- a/services/core/tests/integration/environment_executor_management_test.go +++ b/services/core/tests/integration/environment_executor_management_test.go @@ -22,7 +22,8 @@ func projectCredentials(ctx context.Context, s *Store, project identity.Principa } func TestProjectEnvironmentExecutorManagement(t *testing.T) { - s, pool := testStore(t) + s, _ := configuredStore(t) + pool := s.pool ctx := t.Context() binding := createTestProject(t, pool) project, p := binding.Project, binding.Principal diff --git a/services/core/tests/integration/environment_file_write_semantics_public_test.go b/services/core/tests/integration/environment_file_write_semantics_public_test.go index 03398dc75..eef998402 100644 --- a/services/core/tests/integration/environment_file_write_semantics_public_test.go +++ b/services/core/tests/integration/environment_file_write_semantics_public_test.go @@ -50,7 +50,7 @@ func serveFileWrite(h *dispatchHarness, final proto.WorkspaceWriteResultPayload) func TestEnvironmentFileCreateRejectionsLeaveNoReceiptOrConsumption(t *testing.T) { h, w, environment := localWorker(t, true, false) - _, pool := testStore(t) + pool := h.s.pool if _, err := pool.Exec(t.Context(), `UPDATE environments SET status='connected' WHERE id=$1`, environment.ID); err != nil { t.Fatal(err) } diff --git a/services/core/tests/integration/environment_file_writes_test.go b/services/core/tests/integration/environment_file_writes_test.go index 276e346ad..214739949 100644 --- a/services/core/tests/integration/environment_file_writes_test.go +++ b/services/core/tests/integration/environment_file_writes_test.go @@ -24,7 +24,8 @@ type fileWriteFixture struct { func newFileWriteFixture(t *testing.T) fileWriteFixture { t.Helper() - s, pool := testStore(t) + s, _ := configuredStore(t) + pool := s.pool lease := executionWriter(t, s).lease tenant := uuid.NewString() session, env := localEnvironment(t, s, tenant) @@ -50,7 +51,7 @@ func TestEnvironmentFileWriteRetainsUnknownAcrossLeaseLoss(t *testing.T) { if _, err := f.writer.SettleEnvironmentFileWrite(ctx, f.tenant, f.env.ID, f.key, "committed"); err == nil { t.Fatal("lost writer settled an upload") } - reopened, _ := testStore(t) + reopened := New(t, f.s.pool) next := sessionExecution(t, executionWriter(t, reopened).lease) got, err := next.ReserveEnvironmentFileWrite(ctx, f.tenant, f.env.ID, f.key) if err != nil || !got.Replayed || got.State != "pending" || !got.CreatedAt.Equal(first.CreatedAt) || got.Identity != f.key { diff --git a/services/core/tests/integration/environment_initial_input_test.go b/services/core/tests/integration/environment_initial_input_test.go index da0f37d64..936be6d94 100644 --- a/services/core/tests/integration/environment_initial_input_test.go +++ b/services/core/tests/integration/environment_initial_input_test.go @@ -84,7 +84,8 @@ func TestEnvironmentInitialExpiryRollsBackWithFailureEventAndSerializesPromotion func TestEnvironmentInitialInputCreationRetainsCursorIdentityAndPromotion(t *testing.T) { for _, kind := range []string{"self_hosted", "openai_hosted"} { t.Run(kind, func(t *testing.T) { - s, pool := testStore(t) + s, _ := configuredStore(t) + pool := s.pool tenant := uuid.NewString() input := environmentInput("initial", kind, "/workspace") input.InitialInputs = []sessions.Input{messageInput("first"), messageInput("second")} @@ -126,7 +127,7 @@ func TestEnvironmentInitialInputCreationRetainsCursorIdentityAndPromotion(t *tes if err != nil || len(events) != expectedEvents || (expectedEvents > 0 && (events[0].Event.Type != "agent.session."+status || events[0].Turn != nil)) { t.Fatal("creation cursor lost initial activity", events, err) } - other, _ := testStore(t) + other := reopenStore(t, s) retry, err := createSession(t.Context(), other, tenant, input) // A retry returns the current projection; the reservation is unchanged. if err != nil || retry.Created || retry.Cursor != int64(expectedEvents) || retry.Session.ID != session.ID || activityStatus(retry.Session) != activityStatus(session) || retry.Session.LastTurn != nil { @@ -200,7 +201,8 @@ func TestEnvironmentInitialInputCreationRetainsCursorIdentityAndPromotion(t *tes func TestEnvironmentInitialInputExpiryHasNoTurnAndCannotReplay(t *testing.T) { for _, kind := range []string{"self_hosted", "openai_hosted"} { t.Run(kind, func(t *testing.T) { - s, pool := testStore(t) + s, _ := configuredStore(t) + pool := s.pool tenant := uuid.NewString() input := environmentInput("initial-expiry", kind, "/workspace") input.InitialInputs = []sessions.Input{messageInput("private initial text")} @@ -237,8 +239,9 @@ func TestEnvironmentInitialInputExpiryHasNoTurnAndCannotReplay(t *testing.T) { t.Fatal(err) } awaitRelease() + reopened := reopenStore(t, s) pool.Close() - reopened, reopenedPool := testStore(t) + reopenedPool := reopened.pool writer = executionWriter(t, reopened) if _, err := reopened.CreateSession(t.Context(), tenant, input); err != nil { t.Fatal(err) diff --git a/services/core/tests/integration/environment_initialization_test.go b/services/core/tests/integration/environment_initialization_test.go index d21de7390..77c1261d9 100644 --- a/services/core/tests/integration/environment_initialization_test.go +++ b/services/core/tests/integration/environment_initialization_test.go @@ -13,7 +13,6 @@ import ( "time" "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/execution" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" @@ -49,12 +48,7 @@ func awaitInitialization(t *testing.T, s *Store, tenant, environment, state stri func TestUserManagedPreparationUsesAuthenticatedRuntimeWithoutAllocation(t *testing.T) { for _, outcome := range []string{"completed", "failed", "unknown", "unavailable", "revoked"} { t.Run(outcome, func(t *testing.T) { - _, pool := newManagedTestStore(t) - cipher, err := credentialcrypto.New(bytes.Repeat([]byte{7}, 32)) - if err != nil { - t.Fatal(err) - } - s := NewWithCredentialCipher(pool, cipher) + s, _ := newManagedTestStore(t) principal := FixtureExecutorPrincipal(t, s, uuid.NewString()) session, err := s.CreateSession(t.Context(), principal.TenantID, sessions.CreateSession{ Creator: principal.Subject(), Engine: "codex", IdempotencyKey: uuid.NewString(), diff --git a/services/core/tests/integration/environment_installation_test.go b/services/core/tests/integration/environment_installation_test.go index e4282f866..ba25bfe3f 100644 --- a/services/core/tests/integration/environment_installation_test.go +++ b/services/core/tests/integration/environment_installation_test.go @@ -1,7 +1,6 @@ package integration import ( - "bytes" "encoding/base64" "encoding/json" "errors" @@ -10,18 +9,12 @@ import ( "testing" "time" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" "github.com/google/uuid" ) func TestEnvironmentInstallationClaimLifetimeAndRetries(t *testing.T) { - _, pool := testStore(t) - cipher, err := credentialcrypto.New(bytes.Repeat([]byte{37}, 32)) - if err != nil { - t.Fatal(err) - } - s := NewWithCredentialCipher(pool, cipher) + s, pool := testStore(t) installations := sessionService(t, s) ctx := t.Context() p := createTestProject(t, pool).Principal @@ -51,7 +44,7 @@ func TestEnvironmentInstallationClaimLifetimeAndRetries(t *testing.T) { expired.ExpiresAt = time.Now().Add(-time.Second).Unix() raw, _ = json.Marshal(expired) payload = base64.RawURLEncoding.EncodeToString(raw) - signature, _ := cipher.Fingerprint("environment-installation", payload) + signature, _ := s.credentialCipher.Fingerprint("environment-installation", payload) if _, err := installations.ValidateEnvironmentInstallation(ctx, payload+"."+signature, "build"); !errors.Is(err, sessions.ErrInstallationAuthorization) { t.Fatal("accepted expired grant", err) } diff --git a/services/core/tests/integration/environment_plugins_test.go b/services/core/tests/integration/environment_plugins_test.go index 21d207470..f3a5fb5ce 100644 --- a/services/core/tests/integration/environment_plugins_test.go +++ b/services/core/tests/integration/environment_plugins_test.go @@ -9,19 +9,13 @@ import ( "testing" "github.com/MiniMax-AI/OpenAgentCore/internal/agentplugin" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/environmentconfig" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" "github.com/google/uuid" ) func TestPluginsFrozenInSession(t *testing.T) { - _, pool := testStore(t) - cipher, err := credentialcrypto.New(bytes.Repeat([]byte{23}, 32)) - if err != nil { - t.Fatal(err) - } - s := NewWithCredentialCipher(pool, cipher) + s, _ := configuredStore(t) var archive bytes.Buffer writer := zip.NewWriter(&archive) for path, body := range map[string]string{ @@ -37,7 +31,7 @@ func TestPluginsFrozenInSession(t *testing.T) { t.Fatal(err) } } - if err = writer.Close(); err != nil { + if err := writer.Close(); err != nil { t.Fatal(err) } setup := environmentconfig.Setup{Plugins: []environmentconfig.Plugin{{Metadata: agentplugin.Metadata{Type: "inline", Name: "plugin-proof", Description: "A proof."}, Archive: archive.Bytes()}}, CapabilityDirectories: []string{"/workspace/generated"}} diff --git a/services/core/tests/integration/environment_setup_test.go b/services/core/tests/integration/environment_setup_test.go index 039131e0e..23bfe6f0a 100644 --- a/services/core/tests/integration/environment_setup_test.go +++ b/services/core/tests/integration/environment_setup_test.go @@ -8,19 +8,13 @@ import ( "testing" v1 "github.com/MiniMax-AI/OpenAgentCore/contracts/agents-api/v1" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/environmentconfig" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" "github.com/google/uuid" ) func TestEnvironmentSetupEncryptedSnapshotAndIsolation(t *testing.T) { - _, pool := testStore(t) - cipher, err := credentialcrypto.New(bytes.Repeat([]byte{5}, 32)) - if err != nil { - t.Fatal(err) - } - s := NewWithCredentialCipher(pool, cipher) + s, _ := configuredStore(t) tenant, foreign := uuid.NewString(), uuid.NewString() setup := environmentconfig.Setup{Env: map[string]string{"SECRET": "session-env-canary"}, Commands: []environmentconfig.SetupCommand{{Command: "printf session-command-canary > result"}}, Packages: v1.EnvironmentPackages{NPM: []string{"is-number@7.0.0"}}} input := sessions.CreateSession{Creator: FixtureCreator(), Engine: "codex", IdempotencyKey: uuid.NewString(), Configuration: json.RawMessage(`{"environment":{"type":"openai_hosted"}}`), Initialization: setup} diff --git a/services/core/tests/integration/environments_test.go b/services/core/tests/integration/environments_test.go index 430b829a1..7dce3a10e 100644 --- a/services/core/tests/integration/environments_test.go +++ b/services/core/tests/integration/environments_test.go @@ -26,7 +26,8 @@ func environmentInput(key, kind, directory string) sessions.CreateSession { func TestEnvironmentOwnershipPersistsAndStaysScoped(t *testing.T) { for _, kind := range []string{"self_hosted", "openai_hosted"} { t.Run(kind, func(t *testing.T) { - s, pool := testStore(t) + s, _ := configuredStore(t) + pool := s.pool ctx := context.Background() tenant, foreign := uuid.NewString(), uuid.NewString() input := environmentInput("environment", kind, "/workspace") @@ -65,8 +66,8 @@ func TestEnvironmentOwnershipPersistsAndStaysScoped(t *testing.T) { if _, err := sessionService(t, s).UpdateSessionMetadata(ctx, sessions.UpdateSessionMetadataCommand{TenantID: tenant, SessionID: session.ID, Metadata: map[string]string{"updated": "yes"}}); err != nil { t.Fatal(err) } + restarted := reopenStore(t, s) pool.Close() - restarted, _ := testStore(t) retry, err := restarted.CreateSession(ctx, tenant, input) if err != nil || retry.ID != session.ID || retry.Metadata["updated"] != "yes" { t.Fatal(retry, err) @@ -80,8 +81,9 @@ func TestEnvironmentOwnershipPersistsAndStaysScoped(t *testing.T) { } func TestEnvironmentCreationWinnerOwnsSnapshotAndIdentity(t *testing.T) { - s, pool := testStore(t) - other, _ := testStore(t) + s, _ := configuredStore(t) + pool := s.pool + other := reopenStore(t, s) ctx := context.Background() tenant := uuid.NewString() intent := json.RawMessage(`{"request":"resolved-template"}`) @@ -157,8 +159,8 @@ func TestEnvironmentCreationWinnerOwnsSnapshotAndIdentity(t *testing.T) { if err != nil { t.Fatal(err) } + restarted := reopenStore(t, s) pool.Close() - restarted, _ := testStore(t) retryInput := environmentInput("winner", "openai_hosted", "/changed-resolution") retryInput.CreationRequest = intent retryInput.InitialInputs = []sessions.Input{messageInput("initial")} diff --git a/services/core/tests/integration/file_resource_semantics_public_test.go b/services/core/tests/integration/file_resource_semantics_public_test.go index 83854d136..e297d0065 100644 --- a/services/core/tests/integration/file_resource_semantics_public_test.go +++ b/services/core/tests/integration/file_resource_semantics_public_test.go @@ -10,7 +10,6 @@ import ( "testing" "time" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" "github.com/google/uuid" ) @@ -21,10 +20,6 @@ func TestFileResourceSemanticsOfficialClientPostgres(t *testing.T) { t.Skip("pinned official Python SDK required") } _, pool := testStore(t) - cipher, err := credentialcrypto.New(bytes.Repeat([]byte{53}, 32)) - if err != nil { - t.Fatal(err) - } token, foreign := uuid.NewString(), uuid.NewString() auth := newTestAuthenticator(t, []testAPIKey{ {OrganizationID: "test-org", ProjectID: uuid.NewString(), SubjectKind: "service_account", SubjectID: "resources-owner", TokenSHA256: runtimedevice.HashCredential(token), TenantID: uuid.NewString()}, @@ -32,7 +27,7 @@ func TestFileResourceSemanticsOfficialClientPostgres(t *testing.T) { }) newServer := func() *httptest.Server { t.Helper() - s := NewWithCredentialCipher(pool, cipher) + s := New(t, pool) h, err := publicHandler(t, s, auth, "codex") if err != nil { t.Fatal(err) diff --git a/services/core/tests/integration/function_worker_test.go b/services/core/tests/integration/function_worker_test.go index 916f4f113..d15c24f79 100644 --- a/services/core/tests/integration/function_worker_test.go +++ b/services/core/tests/integration/function_worker_test.go @@ -10,7 +10,6 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto/prototest" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/execution" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/vaults" @@ -120,12 +119,7 @@ const mcpWorkerConfiguration = `{"agent":{"model":"gpt-5.5","tools":[{"type":"mc func mcpBearerWorkerConfiguration(t *testing.T, h *dispatchHarness) (string, string) { t.Helper() - _, pool := testStore(t) - cipher, err := credentialcrypto.New([]byte(strings.Repeat("k", 32))) - if err != nil { - t.Fatal(err) - } - h.s = NewWithCredentialCipher(pool, cipher) + h.s, _ = testStore(t) _, service, err := fixtureVaults(h.s) if err != nil { t.Fatal(err) diff --git a/services/core/tests/integration/hosted_initialization_failure_public_test.go b/services/core/tests/integration/hosted_initialization_failure_public_test.go index 4364a0e8b..c5805c4ed 100644 --- a/services/core/tests/integration/hosted_initialization_failure_public_test.go +++ b/services/core/tests/integration/hosted_initialization_failure_public_test.go @@ -18,7 +18,6 @@ import ( v1 "github.com/MiniMax-AI/OpenAgentCore/contracts/agents-api/v1" "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/environmentconfig" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimegateway" @@ -98,19 +97,6 @@ func (p *hostedFailureProvider) prepare(request proto.RuntimePreparePayload, _ [ return completedInitialization(request, nil) } -// hostedFailureStore returns a store whose Web deployment is claimed by the -// returned installation. -func hostedFailureStore(t *testing.T) (*Store, string) { - t.Helper() - _, pool := newManagedTestStore(t) - cipher, err := credentialcrypto.New(bytes.Repeat([]byte{7}, 32)) - if err != nil { - t.Fatal(err) - } - s := NewWithCredentialCipher(pool, cipher) - return s, webDeployment(t, s, "e2b") -} - func hostedFailureSession(t *testing.T, s *Store, tenant string, input sessions.CreateSession) (sessions.Session, sessions.Environment) { t.Helper() input.Creator, input.Engine, input.IdempotencyKey = FixtureCreator(), "codex", uuid.NewString() @@ -182,7 +168,7 @@ func TestHostedInitializationFailureRecordsSafeSessionFailure(t *testing.T) { "Failed to provision environment: Skill installation failed", []string{"configure", "skill"}}, } { t.Run(test.name, func(t *testing.T) { - s, key := hostedFailureStore(t) + s, key := configuredStore(t) tenant := uuid.NewString() session, environment := hostedFailureSession(t, s, tenant, test.input) p := &hostedFailureProvider{lifecycleProvider: lifecycleProvider{resources: map[string]sandbox.Info{}}, @@ -247,7 +233,7 @@ func TestHostedInitializationFailureRecordsSafeSessionFailure(t *testing.T) { // A pending initial input settles exactly as before; the one failed snapshot // carries both that settlement and the provisioning failure. func TestHostedInitializationFailureSettlesPendingInitialInput(t *testing.T) { - s, key := hostedFailureStore(t) + s, key := configuredStore(t) tenant := uuid.NewString() session, environment := hostedFailureSession(t, s, tenant, sessions.CreateSession{ Initialization: environmentconfig.Setup{Commands: []environmentconfig.SetupCommand{{Command: "exit 3"}}}, @@ -281,7 +267,7 @@ func TestHostedInitializationFailureSettlesPendingInitialInput(t *testing.T) { // stream ends after agent.session.failed; later input gets the observed 409; // delete succeeds; tenant B sees nothing; the canary never appears. func TestHostedInitializationFailurePublicHTTP(t *testing.T) { - s, key := hostedFailureStore(t) + s, key := configuredStore(t) tenant, token, foreign := uuid.NewString(), uuid.NewString(), uuid.NewString() session, environment := hostedFailureSession(t, s, tenant, sessions.CreateSession{ Initialization: environmentconfig.Setup{Commands: []environmentconfig.SetupCommand{{Command: "echo " + hostedFailureCanary + "; exit 3"}}}, diff --git a/services/core/tests/integration/initial_files_http_test.go b/services/core/tests/integration/initial_files_http_test.go index 25d67b80a..1ca13914a 100644 --- a/services/core/tests/integration/initial_files_http_test.go +++ b/services/core/tests/integration/initial_files_http_test.go @@ -8,18 +8,12 @@ import ( "net/http/httptest" "testing" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" "github.com/google/uuid" ) func TestInitialFilesHTTPInlineLimitsAndRetry(t *testing.T) { - _, pool := testStore(t) - cipher, err := credentialcrypto.New(bytes.Repeat([]byte{9}, 32)) - if err != nil { - t.Fatal(err) - } - s := NewWithCredentialCipher(pool, cipher) + s, _ := configuredStore(t) tenant, token := uuid.NewString(), uuid.NewString() auth := newTestAuthenticator(t, []testAPIKey{{OrganizationID: "test-org", ProjectID: uuid.NewString(), SubjectKind: "service_account", SubjectID: "test-runner", TokenSHA256: runtimedevice.HashCredential(token), TenantID: tenant}}) // Exercise HTTP parsing and durable storage without starting a Runtime. diff --git a/services/core/tests/integration/list_cursor_public_test.go b/services/core/tests/integration/list_cursor_public_test.go index 9dadd476f..2009af3e3 100644 --- a/services/core/tests/integration/list_cursor_public_test.go +++ b/services/core/tests/integration/list_cursor_public_test.go @@ -13,7 +13,6 @@ import ( v1 "github.com/MiniMax-AI/OpenAgentCore/contracts/agents-api/v1" "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/execution" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" @@ -221,12 +220,7 @@ func wireError(kind string, code, param *string, message string) string { // lookups keep their behavior (K1–K3). A foreign cursor is always byte-identical // to a missing one, and a foreign or missing parent is 404 before any cursor. func TestListCursorErrorsPostgres(t *testing.T) { - _, pool := testStore(t) - cipher, err := credentialcrypto.New(bytes.Repeat([]byte{67}, 32)) - if err != nil { - t.Fatal(err) - } - s := NewWithCredentialCipher(pool, cipher) + s, _ := configuredStore(t) owner, foreign := uuid.NewString(), uuid.NewString() ownerTenant, foreignTenant := uuid.NewString(), uuid.NewString() auth := newTestAuthenticator(t, []testAPIKey{ diff --git a/services/core/tests/integration/list_query_public_test.go b/services/core/tests/integration/list_query_public_test.go index 8a67221f8..bb15d690d 100644 --- a/services/core/tests/integration/list_query_public_test.go +++ b/services/core/tests/integration/list_query_public_test.go @@ -1,7 +1,6 @@ package integration import ( - "bytes" "context" "net/http/httptest" "os" @@ -9,9 +8,9 @@ import ( "testing" "time" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/execution" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimegateway" "github.com/google/uuid" ) @@ -20,12 +19,7 @@ func TestListQueryOfficialClientPostgres(t *testing.T) { if python == "" { t.Skip("pinned official Python SDK required") } - _, pool := testStore(t) - cipher, err := credentialcrypto.New(bytes.Repeat([]byte{72}, 32)) - if err != nil { - t.Fatal(err) - } - s := NewWithCredentialCipher(pool, cipher) + s, _ := testStore(t) token, foreign := uuid.NewString(), uuid.NewString() auth := newTestAuthenticator(t, []testAPIKey{ {OrganizationID: "test-org", ProjectID: uuid.NewString(), SubjectKind: "service_account", SubjectID: "query-owner", TokenSHA256: runtimedevice.HashCredential(token), TenantID: uuid.NewString()}, @@ -33,7 +27,7 @@ func TestListQueryOfficialClientPostgres(t *testing.T) { }) // Use real admission while leaving dispatch paused. Public cancellation retains // the queued history; this fixture does not perform native or model execution. - worker := startWorker(t, t.Context(), s, &execution.Dispatcher{}) + worker := startWorker(t, t.Context(), s, &execution.Dispatcher{Registry: runtimegateway.NewRegistry()}) t.Cleanup(func() { stopped, cancel := context.WithCancel(context.Background()) cancel() diff --git a/services/core/tests/integration/local_environment_devices_test.go b/services/core/tests/integration/local_environment_devices_test.go index 24f24c486..9a146faad 100644 --- a/services/core/tests/integration/local_environment_devices_test.go +++ b/services/core/tests/integration/local_environment_devices_test.go @@ -28,7 +28,8 @@ func localEnvironment(t *testing.T, s *Store, tenant string) (sessions.Session, } func TestEnvironmentDeviceAuthorityAndLifecycle(t *testing.T) { - s, pool := testStore(t) + s, _ := configuredStore(t) + pool := s.pool tenant, foreignTenant := uuid.NewString(), uuid.NewString() session, environment := localEnvironment(t, s, tenant) sibling, _ := localEnvironment(t, s, tenant) @@ -51,8 +52,7 @@ func TestEnvironmentDeviceAuthorityAndLifecycle(t *testing.T) { if err != nil || len(devices) != 0 { t.Fatalf("dedicated device entered general selection: %v %v", devices, err) } - reopened, _ := testStore(t) - got, err := sessionAdapter(reopened).GetSessionDevice(t.Context(), tenant, session.ID) + got, err := sessionAdapter(New(t, pool)).GetSessionDevice(t.Context(), tenant, session.ID) if err != nil || got != bound { t.Fatalf("durable exact binding: %+v %v", got, err) } @@ -72,7 +72,8 @@ func TestEnvironmentDeviceAuthorityAndLifecycle(t *testing.T) { } func TestEnvironmentDeviceProvisioningHasOneWinner(t *testing.T) { - s, pool := testStore(t) + s, _ := configuredStore(t) + pool := s.pool tenant := uuid.NewString() session, environment := localEnvironment(t, s, tenant) var wg sync.WaitGroup diff --git a/services/core/tests/integration/local_environment_worker_test.go b/services/core/tests/integration/local_environment_worker_test.go index 5d9b3214c..a3f7651e9 100644 --- a/services/core/tests/integration/local_environment_worker_test.go +++ b/services/core/tests/integration/local_environment_worker_test.go @@ -17,8 +17,7 @@ func localWorker(t *testing.T, scoped, execute bool) (*dispatchHarness, *executi t.Helper() h := newDispatchHarnessForSession(t, []byte(`{"agent":{"model":"test-model"},"environment":{"type":"openai_hosted","network":{"access":"disabled"}}}`), scoped) if scoped { - _, pool := testStore(t) - insertWorkerRuntimeAllocation(t, pool, h, "disabled") + insertWorkerRuntimeAllocation(t, h.s.pool, h, "disabled") } environment, err := sessionAdapter(h.s).GetSessionEnvironment(t.Context(), h.tenant, h.session.ID) if err != nil { diff --git a/services/core/tests/integration/managed_fixture_test.go b/services/core/tests/integration/managed_fixture_test.go index 9248f7d2b..bd7d4aca9 100644 --- a/services/core/tests/integration/managed_fixture_test.go +++ b/services/core/tests/integration/managed_fixture_test.go @@ -15,3 +15,24 @@ func newManagedTestStore(t *testing.T) (*Store, *pgxpool.Pool) { pool := pgtest.OpenIsolated(t, nil) return New(t, pool), pool } + +// configuredStore is newManagedTestStore claimed for a new installation and +// configured for E2B, as Web setup leaves an installation, so that it admits +// hosted work. It returns the installation. +func configuredStore(t *testing.T) (*Store, string) { + t.Helper() + s, _ := newManagedTestStore(t) + return s, webDeployment(t, s, "e2b") +} + +// reopenStore is the fixture on a new pool to s's database, as after a restart. +// The pool closes when the test ends. +func reopenStore(t *testing.T, s *Store) *Store { + t.Helper() + pool, err := pgxpool.NewWithConfig(t.Context(), s.pool.Config()) + if err != nil { + t.Fatal(err) + } + t.Cleanup(pool.Close) + return New(t, pool) +} diff --git a/services/core/tests/integration/mcp_credential_selection_public_test.go b/services/core/tests/integration/mcp_credential_selection_public_test.go index 407bbcafb..8c2c66e04 100644 --- a/services/core/tests/integration/mcp_credential_selection_public_test.go +++ b/services/core/tests/integration/mcp_credential_selection_public_test.go @@ -9,7 +9,6 @@ import ( "strings" "testing" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" "github.com/google/uuid" @@ -25,12 +24,7 @@ type selectionResponse struct { // M1–M8) over real HTTP and PostgreSQL, with tenants A and B. func TestMCPCredentialSelectionPublicPostgres(t *testing.T) { // An isolated database keeps the no-write digest independent of other tests. - _, pool := newManagedTestStore(t) - cipher, err := credentialcrypto.New(bytes.Repeat([]byte{67}, 32)) - if err != nil { - t.Fatal(err) - } - s := NewWithCredentialCipher(pool, cipher) + s, pool := newManagedTestStore(t) tenantA, tokenA, tokenB := uuid.NewString(), uuid.NewString(), uuid.NewString() auth := newTestAuthenticator(t, []testAPIKey{ {OrganizationID: "test-org", ProjectID: uuid.NewString(), SubjectKind: "service_account", SubjectID: "selection-a", TokenSHA256: runtimedevice.HashCredential(tokenA), TenantID: tenantA}, diff --git a/services/core/tests/integration/model_protocol_native_test.go b/services/core/tests/integration/model_protocol_native_test.go index 587074d0c..875b75cb0 100644 --- a/services/core/tests/integration/model_protocol_native_test.go +++ b/services/core/tests/integration/model_protocol_native_test.go @@ -61,7 +61,7 @@ func TestNativeModelProtocolPublicExecution(t *testing.T) { } ctx, cancel := context.WithTimeout(t.Context(), 15*time.Minute) defer cancel() - worker, err := startWorkerErr(ctx, h.s, h.d) + worker, err := startWorkerErr(t, ctx, h.s, h.d) if err != nil { t.Fatal("cannot start native execution worker") } diff --git a/services/core/tests/integration/path_id_semantics_public_test.go b/services/core/tests/integration/path_id_semantics_public_test.go index 288df95a7..46601348b 100644 --- a/services/core/tests/integration/path_id_semantics_public_test.go +++ b/services/core/tests/integration/path_id_semantics_public_test.go @@ -10,7 +10,6 @@ import ( "strings" "testing" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/skills" @@ -86,12 +85,7 @@ func databaseDigest(t *testing.T, pool *pgxpool.Pool) map[string]string { func TestMalformedPathIDsMatchMissingPostgres(t *testing.T) { // An isolated database keeps the no-write digest independent of other tests. - _, pool := newManagedTestStore(t) - cipher, err := credentialcrypto.New(bytes.Repeat([]byte{61}, 32)) - if err != nil { - t.Fatal(err) - } - s := NewWithCredentialCipher(pool, cipher) + s, pool := newManagedTestStore(t) owner, foreign := uuid.NewString(), uuid.NewString() ownerTenant := uuid.NewString() auth := newTestAuthenticator(t, []testAPIKey{ @@ -123,7 +117,7 @@ func TestMalformedPathIDsMatchMissingPostgres(t *testing.T) { t.Fatal("fixture Environment", err) } environment := hosted.Environment.ID - skill, err := SkillService(t, pool, cipher).CreateSkill(t.Context(), skills.CreateSkill{TenantID: ownerTenant, Archive: skillArchive(t, "path-skill")}) + skill, err := SkillService(t, pool, s.credentialCipher).CreateSkill(t.Context(), skills.CreateSkill{TenantID: ownerTenant, Archive: skillArchive(t, "path-skill")}) if err != nil { t.Fatal(err) } diff --git a/services/core/tests/integration/public_execution_test.go b/services/core/tests/integration/public_execution_test.go index b327607b8..e52ee9dc5 100644 --- a/services/core/tests/integration/public_execution_test.go +++ b/services/core/tests/integration/public_execution_test.go @@ -38,7 +38,7 @@ func TestExecutionWorkerAdmissionBindingAndRecovery(t *testing.T) { t.Error("worker did not stop") } }) - if second, err := startWorkerErr(ctx, h.s, h.d); err == nil { + if second, err := startWorkerErr(t, ctx, h.s, h.d); err == nil { cancel() go second.Run(ctx) t.Fatal("second service acquired database") @@ -168,7 +168,7 @@ func TestWorkerRestartReconcilesClaimedButPreservesQueuedWork(t *testing.T) { if err != nil || pending.LastTurn.Status != sessions.TurnCancelled { t.Fatal(pending, err) } - restarted, err := startWorkerErr(ctx, h.s, h.d) + restarted, err := startWorkerErr(t, ctx, h.s, h.d) if err != nil { t.Fatal("lease not released", err) } diff --git a/services/core/tests/integration/remote_mcp_test.go b/services/core/tests/integration/remote_mcp_test.go index 1bd8d3b94..c7d0ad2de 100644 --- a/services/core/tests/integration/remote_mcp_test.go +++ b/services/core/tests/integration/remote_mcp_test.go @@ -7,7 +7,6 @@ import ( "strings" "testing" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/vaults" "github.com/google/uuid" @@ -74,12 +73,7 @@ func TestSelfHostedServiceMCPRejectedWithoutWrites(t *testing.T) { func selfHostedMCPAdmissionFixture(t *testing.T) (*Store, string, vaults.Vault, vaults.Credential) { t.Helper() - _, pool := testStore(t) - cipher, err := credentialcrypto.New([]byte(strings.Repeat("k", 32))) - if err != nil { - t.Fatal(err) - } - s := NewWithCredentialCipher(pool, cipher) + s, _ := testStore(t) _, service, err := fixtureVaults(s) if err != nil { t.Fatal(err) diff --git a/services/core/tests/integration/request_body_public_test.go b/services/core/tests/integration/request_body_public_test.go index 2b6a72fe4..b4a372788 100644 --- a/services/core/tests/integration/request_body_public_test.go +++ b/services/core/tests/integration/request_body_public_test.go @@ -11,7 +11,6 @@ import ( "testing" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/api" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/execution" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" @@ -24,12 +23,7 @@ import ( // sent as text/plain, change nothing for the owner or another tenant. func TestRequestBodyGateRejectsWithoutWritesPostgres(t *testing.T) { // An isolated database keeps the no-write digest independent of other tests. - _, pool := newManagedTestStore(t) - cipher, err := credentialcrypto.New(bytes.Repeat([]byte{64}, 32)) - if err != nil { - t.Fatal(err) - } - s := NewWithCredentialCipher(pool, cipher) + s, pool := newManagedTestStore(t) owner, foreign, ownerTenant := uuid.NewString(), uuid.NewString(), uuid.NewString() auth := newTestAuthenticator(t, []testAPIKey{ {OrganizationID: "test-org", ProjectID: uuid.NewString(), SubjectKind: "service_account", SubjectID: "body-owner", TokenSHA256: runtimedevice.HashCredential(owner), TenantID: ownerTenant}, @@ -165,12 +159,7 @@ func TestRequestBodyGateRejectsWithoutWritesPostgres(t *testing.T) { // Core extension routes keep their own body handling, without the Content-Type // rule or the official body messages. func TestRequestBodyGateExcludedRoutesPostgres(t *testing.T) { - _, pool := testStore(t) - cipher, err := credentialcrypto.New(bytes.Repeat([]byte{65}, 32)) - if err != nil { - t.Fatal(err) - } - s := NewWithCredentialCipher(pool, cipher) + s, _ := testStore(t) token, tenant := uuid.NewString(), uuid.NewString() auth := newTestAuthenticator(t, []testAPIKey{{OrganizationID: "test-org", ProjectID: uuid.NewString(), SubjectKind: "service_account", SubjectID: "excluded-owner", TokenSHA256: runtimedevice.HashCredential(token), TenantID: tenant}}) h, err := publicHandler(t, s, auth, "codex") diff --git a/services/core/tests/integration/root_fixture_test.go b/services/core/tests/integration/root_fixture_test.go index e1ad7512b..a29b67530 100644 --- a/services/core/tests/integration/root_fixture_test.go +++ b/services/core/tests/integration/root_fixture_test.go @@ -33,19 +33,14 @@ type Store struct { } // defaultPlacement is the placement rules cmd/server builds on the built-in -// providers and an unset public URL. -var defaultPlacement, _ = placement.NewRules(providers.Builtin(), "") +// providers and the public URL https://core.example. +var defaultPlacement, _ = placement.NewRules(providers.Builtin(), "https://core.example") // New is the fixture on pool under the shared test credential key and // defaultPlacement. func New(t testing.TB, pool *pgxpool.Pool) *Store { - return NewWithCredentialCipher(pool, pgtest.CredentialKey(t)) -} - -// NewWithCredentialCipher is New under another credential key. -func NewWithCredentialCipher(pool *pgxpool.Pool, cipher *credentialcrypto.Cipher) *Store { pooled := pgunit.NewPool(pool) - return &Store{queries: sqlc.New(pool), pool: pool, pooled: pooled, writer: pooled, credentialCipher: cipher, placement: defaultPlacement} + return &Store{queries: sqlc.New(pool), pool: pool, pooled: pooled, writer: pooled, credentialCipher: pgtest.CredentialKey(t), placement: defaultPlacement} } // NewExecution is s with its Session transactions on lease. diff --git a/services/core/tests/integration/runtime_allocations_test.go b/services/core/tests/integration/runtime_allocations_test.go index 14ffb4994..cd9899680 100644 --- a/services/core/tests/integration/runtime_allocations_test.go +++ b/services/core/tests/integration/runtime_allocations_test.go @@ -5,7 +5,6 @@ import ( "errors" "sync" "testing" - "time" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgtest" @@ -15,9 +14,10 @@ import ( ) func TestRuntimeAllocationAtomicOwnershipAndRecovery(t *testing.T) { - s, pool := testStore(t) + s, provider := configuredStore(t) + pool := s.pool w := executionWriter(t, s) - tenant, provider := uuid.NewString(), uuid.NewString() + tenant := uuid.NewString() session, environment := localEnvironment(t, s, tenant) secret := uuid.NewString() owner, err := deploymentExecution(t, w).ReserveAllocation(t.Context(), deployment.AllocationKey{TenantID: tenant, EnvironmentID: environment.ID}, provider, runtimedevice.HashCredential(secret)) @@ -42,8 +42,7 @@ func TestRuntimeAllocationAtomicOwnershipAndRecovery(t *testing.T) { if _, err := deploymentExecution(t, w).ObserveRunning(t.Context(), owner); err == nil { t.Fatal("lost writer changed allocation") } - reopened, _ := testStore(t) - next := executionWriter(t, reopened) + next := executionWriter(t, New(t, pool)) retry, err := deploymentExecution(t, next).ReserveAllocation(t.Context(), deployment.AllocationKey{TenantID: tenant, EnvironmentID: environment.ID}, provider, runtimedevice.HashCredential(uuid.NewString())) if err != nil || !retry.Replayed || retry.ID != owner.ID || retry.DeviceID != owner.DeviceID { t.Fatalf("restart replaced unknown allocation: %+v %v", retry, err) @@ -99,9 +98,10 @@ func TestRuntimeAllocationAtomicOwnershipAndRecovery(t *testing.T) { } func TestRuntimeAllocationOneWinnerAndRollback(t *testing.T) { - s, pool := testStore(t) + s, provider := configuredStore(t) + pool := s.pool w := executionWriter(t, s) - tenant, provider := uuid.NewString(), uuid.NewString() + tenant := uuid.NewString() _, environment := localEnvironment(t, s, tenant) var wg sync.WaitGroup results := make(chan deployment.Allocation, 8) @@ -155,12 +155,12 @@ func TestRuntimeAllocationOneWinnerAndRollback(t *testing.T) { } } -func TestRuntimeAllocationExpiryAndRevocation(t *testing.T) { - s, pool := testStore(t) +func TestRuntimeAllocationCleanupRevokesAndKeepsIdentity(t *testing.T) { + s, installation := configuredStore(t) w := executionWriter(t, s) tenant := uuid.NewString() _, environment := localEnvironment(t, s, tenant) - owner, err := deploymentExecution(t, w).ReserveAllocation(t.Context(), deployment.AllocationKey{TenantID: tenant, EnvironmentID: environment.ID}, uuid.NewString(), runtimedevice.HashCredential(uuid.NewString())) + owner, err := deploymentExecution(t, w).ReserveAllocation(t.Context(), deployment.AllocationKey{TenantID: tenant, EnvironmentID: environment.ID}, installation, runtimedevice.HashCredential(uuid.NewString())) if err != nil { t.Fatal(err) } @@ -168,26 +168,23 @@ func TestRuntimeAllocationExpiryAndRevocation(t *testing.T) { if err != nil || !owner.CreateSettled { t.Fatalf("running observation: %+v %v", owner, err) } - if _, err := deploymentExecution(t, w).KeepAllocation(t.Context(), owner); err != nil { - t.Fatal(err) - } - if _, err := pool.Exec(t.Context(), "UPDATE runtime_allocations SET kept_at=clock_timestamp()-interval '61 minutes' WHERE id=$1", owner.ID); err != nil { + if _, err := deploymentExecution(t, w).CheckRunning(t.Context(), owner); err != nil { t.Fatal(err) } - if _, err := deploymentExecution(t, w).KeepAllocation(t.Context(), owner); !errors.Is(err, deployment.ErrAllocationConflict) { - t.Fatalf("expired allocation renewed: %v", err) - } if _, err := deploymentExecution(t, w).RequestCleanup(t.Context(), owner); err != nil { t.Fatal(err) } if _, ok, err := sessionAdapter(s).GetDeviceCredential(t.Context(), owner.DeviceID); err != nil || ok { t.Fatal("cleanup credential still authenticates") } + if _, err := deploymentExecution(t, w).CheckRunning(t.Context(), owner); !errors.Is(err, deployment.ErrAllocationConflict) { + t.Fatalf("cleanup kept the allocation running: %v", err) + } if _, err := deploymentExecution(t, w).ReleaseAllocation(t.Context(), owner); err != nil { t.Fatal(err) } got, err := deploymentExecution(t, w).ReserveAllocation(t.Context(), deployment.AllocationKey{TenantID: tenant, EnvironmentID: environment.ID}, owner.ProviderKey, runtimedevice.HashCredential(uuid.NewString())) - if err != nil || !got.Replayed || got.State != "released" || got.KeptAt.After(time.Now()) { + if err != nil || !got.Replayed || got.State != "released" { t.Fatalf("cleanup permitted replacement: %+v %v", got, err) } } diff --git a/services/core/tests/integration/runtime_capabilities_pending_test.go b/services/core/tests/integration/runtime_capabilities_pending_test.go index 3e3c01d2d..6f40db170 100644 --- a/services/core/tests/integration/runtime_capabilities_pending_test.go +++ b/services/core/tests/integration/runtime_capabilities_pending_test.go @@ -1,12 +1,10 @@ package integration import ( - "bytes" "encoding/json" "errors" "testing" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/environmentconfig" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" @@ -15,13 +13,7 @@ import ( ) func TestManagedCapabilitiesWaitBeforeInitializationClaim(t *testing.T) { - _, pool := newManagedTestStore(t) - cipher, err := credentialcrypto.New(bytes.Repeat([]byte{9}, 32)) - if err != nil { - t.Fatal(err) - } - s := NewWithCredentialCipher(pool, cipher) - key := webDeployment(t, s, "e2b") + s, key := configuredStore(t) tenant := uuid.NewString() session, err := s.CreateSession(t.Context(), tenant, sessions.CreateSession{ Creator: FixtureCreator(), Engine: "codex", IdempotencyKey: uuid.NewString(), diff --git a/services/core/tests/integration/runtime_configuration_cleanup_test.go b/services/core/tests/integration/runtime_configuration_cleanup_test.go index c08993908..e72ff26fe 100644 --- a/services/core/tests/integration/runtime_configuration_cleanup_test.go +++ b/services/core/tests/integration/runtime_configuration_cleanup_test.go @@ -83,8 +83,7 @@ func TestManagedRuntimeConfigurationCleanup(t *testing.T) { {name: "kill unavailable stays retained", inspectionError: sandbox.ErrInvalid, killError: sandbox.ErrComputeUnconfirmed, wantSettled: true, wantKill: true}, } { t.Run(test.name, func(t *testing.T) { - s, _ := newManagedTestStore(t) - key := webDeployment(t, s, "e2b") + s, key := configuredStore(t) p := &configurationCleanupProvider{ lifecycleProvider: lifecycleProvider{resources: map[string]sandbox.Info{}, loseCreate: test.loseCreate}, rejectCreate: test.rejectCreate, settleCreate: test.settleCreate, diff --git a/services/core/tests/integration/runtime_connection_test.go b/services/core/tests/integration/runtime_connection_test.go index 7bc66fb03..cd0cf6c0d 100644 --- a/services/core/tests/integration/runtime_connection_test.go +++ b/services/core/tests/integration/runtime_connection_test.go @@ -21,8 +21,7 @@ import ( ) func TestManagedRuntimeConnectionTracksAuthenticatedSocket(t *testing.T) { - s, _ := newManagedTestStore(t) - key := webDeployment(t, s, "e2b") + s, key := configuredStore(t) tenant, session, environment := managedSession(t, s) server := httptest.NewUnstartedServer(nil) wsURL := "ws://" + server.Listener.Addr().String() + "/api/v1/agent-daemon/ws" diff --git a/services/core/tests/integration/runtime_creation_settlement_test.go b/services/core/tests/integration/runtime_creation_settlement_test.go index 54390356d..211a0d07b 100644 --- a/services/core/tests/integration/runtime_creation_settlement_test.go +++ b/services/core/tests/integration/runtime_creation_settlement_test.go @@ -41,8 +41,7 @@ func (p *absentCreationProvider) GetInfo(_ context.Context, r sandbox.Reference) func TestManagedRuntimeConfirmedAbsentCreateReleasesAtomically(t *testing.T) { for _, cancelled := range []bool{false, true} { t.Run(map[bool]string{false: "live caller", true: "cancelled caller"}[cancelled], func(t *testing.T) { - s, _ := newManagedTestStore(t) - key := webDeployment(t, s, "e2b") + s, key := configuredStore(t) p := &absentCreationProvider{} w, _ := managedWorker(t, s, key, p) tenant, session, environment := managedSession(t, s) @@ -79,8 +78,7 @@ func TestManagedRuntimeConfirmedAbsentCreateReleasesAtomically(t *testing.T) { } } func TestManagedRuntimeForeignAbsenceCannotReleaseCreation(t *testing.T) { - s, _ := newManagedTestStore(t) - key := webDeployment(t, s, "e2b") + s, key := configuredStore(t) p := &absentCreationProvider{foreign: true} w, _ := managedWorker(t, s, key, p) tenant, _, environment := managedSession(t, s) @@ -93,8 +91,7 @@ func TestManagedRuntimeForeignAbsenceCannotReleaseCreation(t *testing.T) { } } func TestManagedRuntimeObservedSettlementAllowsOwnedCleanup(t *testing.T) { - s, _ := newManagedTestStore(t) - key := webDeployment(t, s, "e2b") + s, key := configuredStore(t) p := &absentCreationProvider{observeSettled: true} w, _ := managedWorker(t, s, key, p) tenant, session, environment := managedSession(t, s) diff --git a/services/core/tests/integration/runtime_deployment_test.go b/services/core/tests/integration/runtime_deployment_test.go index 49cd6e6ac..3133ee445 100644 --- a/services/core/tests/integration/runtime_deployment_test.go +++ b/services/core/tests/integration/runtime_deployment_test.go @@ -13,53 +13,6 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" ) -// An installation never adopts hosted work admitted before it claimed the -// deployment: a pending Session or an unreleased allocation refuses the claim. -func TestRuntimeDeploymentClaimAdoptsNoUnclaimedWork(t *testing.T) { - s, _ := newManagedTestStore(t) - changes := deploymentExecution(t, executionWriter(t, s)) - installation, tenant := uuid.NewString(), uuid.NewString() - pending, _ := localEnvironment(t, s, tenant) - if err := changes.Claim(t.Context(), installation); !errors.Is(err, deployment.ErrConflict) { - t.Fatal("pending Session adopted", err) - } - if err := sessionService(t, s).DeleteSession(t.Context(), sessions.DeleteSessionCommand{TenantID: tenant, SessionID: pending.ID}); err != nil { - t.Fatal(err) - } - session, environment := localEnvironment(t, s, tenant) - owner, err := changes.ReserveAllocation(t.Context(), deployment.AllocationKey{TenantID: tenant, EnvironmentID: environment.ID}, installation, runtimedevice.HashCredential(uuid.NewString())) - if err != nil { - t.Fatal(err) - } - if err := changes.Claim(t.Context(), installation); !errors.Is(err, deployment.ErrConflict) { - t.Fatal("unclaimed allocation adopted", err) - } - if err := sessionService(t, s).DeleteSession(t.Context(), sessions.DeleteSessionCommand{TenantID: tenant, SessionID: session.ID}); err != nil { - t.Fatal(err) - } - if _, err := changes.RequestCleanup(t.Context(), owner); err != nil { - t.Fatal(err) - } - if _, err := changes.ReleaseAllocation(t.Context(), owner); !errors.Is(err, deployment.ErrAllocationConflict) { - t.Fatal("unknown creation lost cleanup ownership", err) - } - if err := changes.Claim(t.Context(), installation); !errors.Is(err, deployment.ErrConflict) { - t.Fatal("deleted unknown allocation did not block adoption", err) - } - if _, err := changes.SettleCreation(t.Context(), owner); err != nil { - t.Fatal(err) - } - if _, err := changes.ReleaseAllocation(t.Context(), owner); err != nil { - t.Fatal(err) - } - if err := changes.Claim(t.Context(), installation); err != nil { - t.Fatal(err) - } - if err := changes.RequireUnclaimed(t.Context()); !errors.Is(err, deployment.ErrConflict) { - t.Fatal("owner without runtimes accepted a claimed deployment", err) - } -} - func TestRuntimeDeploymentResetPreservesCreationRetriesAndOtherPlacements(t *testing.T) { s, w, installation := managedArchiveFixture(t) tenant := uuid.NewString() diff --git a/services/core/tests/integration/runtime_deployment_worker_test.go b/services/core/tests/integration/runtime_deployment_worker_test.go index 6e852099f..7cc5c0e85 100644 --- a/services/core/tests/integration/runtime_deployment_worker_test.go +++ b/services/core/tests/integration/runtime_deployment_worker_test.go @@ -12,8 +12,7 @@ import ( ) func TestManagedDeploymentStartupRejectsSwitchBeforeBackendAccess(t *testing.T) { - s, _ := newManagedTestStore(t) - key := webDeployment(t, s, "e2b") + s, key := configuredStore(t) old := &lifecycleProvider{resources: map[string]sandbox.Info{}} worker, stop := managedWorker(t, s, key, old) tenant, _, environment := managedSession(t, s) @@ -23,16 +22,10 @@ func TestManagedDeploymentStartupRejectsSwitchBeforeBackendAccess(t *testing.T) } stop() replacement := &lifecycleProvider{resources: map[string]sandbox.Info{}} - start := func(runtimes *execution.RuntimeProvider) error { - _, err := startNextWorker(t.Context(), s, &execution.Dispatcher{Registry: runtimegateway.NewRegistry(), ManagedRuntimes: runtimes}) - return err - } - if err := start(webRuntimes(t, s, uuid.NewString(), replacement, nil)); !errors.Is(err, deployment.ErrConflict) { + _, err = startNextWorker(t, t.Context(), s, &execution.Dispatcher{Registry: runtimegateway.NewRegistry(), ManagedRuntimes: webRuntimes(t, s, uuid.NewString(), replacement, nil)}) + if !errors.Is(err, deployment.ErrConflict) { t.Fatal("startup switched the claimed installation", err) } - if err := start(nil); !errors.Is(err, deployment.ErrConflict) { - t.Fatal("startup without runtimes abandoned the claimed deployment", err) - } if replacement.creates != 0 || replacement.kills != 0 { t.Fatal("rejected startup touched new backend") } diff --git a/services/core/tests/integration/runtime_environment_terminal_test.go b/services/core/tests/integration/runtime_environment_terminal_test.go index 2851301a3..0dfd794ad 100644 --- a/services/core/tests/integration/runtime_environment_terminal_test.go +++ b/services/core/tests/integration/runtime_environment_terminal_test.go @@ -16,7 +16,8 @@ import ( func TestManagedEnvironmentTerminationSettlesInputAndPreservesIdentity(t *testing.T) { for _, expired := range []bool{false, true} { t.Run(map[bool]string{false: "failed", true: "expired"}[expired], func(t *testing.T) { - s, pool := testStore(t) + s, installation := configuredStore(t) + pool := s.pool tenant := uuid.NewString() input := environmentInput("initial-terminal", "openai_hosted", "/workspace") input.InitialInputs = []sessions.Input{messageInput("initial")} @@ -26,12 +27,12 @@ func TestManagedEnvironmentTerminationSettlesInputAndPreservesIdentity(t *testin } reservation := initialEnvironmentReservation(t, s, pool, tenant, session.ID) writer := executionWriter(t, s) - owner, err := deploymentExecution(t, writer).ReserveAllocation(t.Context(), deployment.AllocationKey{TenantID: tenant, EnvironmentID: session.Environment.ID}, uuid.NewString(), runtimedevice.HashCredential(uuid.NewString())) + owner, err := deploymentExecution(t, writer).ReserveAllocation(t.Context(), deployment.AllocationKey{TenantID: tenant, EnvironmentID: session.Environment.ID}, installation, runtimedevice.HashCredential(uuid.NewString())) if err != nil { t.Fatal(err) } if expired { - if _, err := pool.Exec(t.Context(), "UPDATE runtime_allocations SET kept_at=clock_timestamp()-interval '61 minutes' WHERE id=$1", owner.ID); err != nil { + if _, err := pool.Exec(t.Context(), "UPDATE runtime_allocations SET compute_phase='suspended',compute_retained_until=clock_timestamp()-interval '1 second' WHERE id=$1", owner.ID); err != nil { t.Fatal(err) } } @@ -112,7 +113,8 @@ func TestManagedEnvironmentTerminationSettlesInputAndPreservesIdentity(t *testin } func TestManagedEnvironmentFailureRollsBackWithSessionEvent(t *testing.T) { - s, pool := testStore(t) + s, installation := configuredStore(t) + pool := s.pool tenant := uuid.NewString() input := environmentInput("rollback-terminal", "openai_hosted", "/workspace") input.InitialInputs = []sessions.Input{messageInput("initial")} @@ -121,7 +123,7 @@ func TestManagedEnvironmentFailureRollsBackWithSessionEvent(t *testing.T) { t.Fatal(err) } writer := executionWriter(t, s) - owner, err := deploymentExecution(t, writer).ReserveAllocation(t.Context(), deployment.AllocationKey{TenantID: tenant, EnvironmentID: session.Environment.ID}, uuid.NewString(), runtimedevice.HashCredential(uuid.NewString())) + owner, err := deploymentExecution(t, writer).ReserveAllocation(t.Context(), deployment.AllocationKey{TenantID: tenant, EnvironmentID: session.Environment.ID}, installation, runtimedevice.HashCredential(uuid.NewString())) if err != nil { t.Fatal(err) } diff --git a/services/core/tests/integration/runtime_initialization_test.go b/services/core/tests/integration/runtime_initialization_test.go index 8acc75925..636474b84 100644 --- a/services/core/tests/integration/runtime_initialization_test.go +++ b/services/core/tests/integration/runtime_initialization_test.go @@ -16,7 +16,6 @@ import ( v1 "github.com/MiniMax-AI/OpenAgentCore/contracts/agents-api/v1" "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/environmentconfig" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" @@ -46,13 +45,8 @@ func TestEnvironmentInitializationCompletionUnknownAndRestart(t *testing.T) { setupOnly := strings.HasPrefix(mode, "setup-") mode = strings.TrimPrefix(mode, "setup-") expectedSteps := 2 - _, pool := newManagedTestStore(t) - cipher, err := credentialcrypto.New(bytes.Repeat([]byte{9}, 32)) - if err != nil { - t.Fatal(err) - } - s := NewWithCredentialCipher(pool, cipher) - key := webDeployment(t, s, "e2b") + s, key := configuredStore(t) + pool := s.pool tenant := uuid.NewString() input := sessions.CreateSession{Creator: FixtureCreator(), Engine: "codex", IdempotencyKey: uuid.NewString(), Configuration: json.RawMessage(`{"environment":{"type":"openai_hosted"}}`), InitialFiles: []environmentconfig.InitialFile{{Type: "inline", Path: "/workspace/a", Data: []byte("first")}, {Type: "inline", Path: "/workspace/b", Data: []byte("second")}}} if setupOnly { @@ -141,7 +135,7 @@ func TestEnvironmentInitializationCompletionUnknownAndRestart(t *testing.T) { } func TestManagedRuntimePreparationAllOperationsUsePeer(t *testing.T) { - s, key := hostedFailureStore(t) + s, key := configuredStore(t) var archive bytes.Buffer writer := zip.NewWriter(&archive) for path, body := range map[string]string{"proof/.codex-plugin/plugin.json": `{"name":"plugin","description":"A plugin.","skills":"./skills"}`, "proof/skills/example/SKILL.md": "---\nname: plugin-proof\ndescription: A plugin Skill.\n---\nProof."} { diff --git a/services/core/tests/integration/runtime_input_admission_test.go b/services/core/tests/integration/runtime_input_admission_test.go index 725bb0567..007522c55 100644 --- a/services/core/tests/integration/runtime_input_admission_test.go +++ b/services/core/tests/integration/runtime_input_admission_test.go @@ -14,8 +14,7 @@ import ( ) func TestManagedRuntimeResetPreservesCancelAndRetry(t *testing.T) { - s, _ := newManagedTestStore(t) - key := webDeployment(t, s, "e2b") + s, key := configuredStore(t) tenant, session, _ := managedSession(t, s) inputs := []sessions.Input{messageInput("accepted work")} accepted, err := submitInputs(t.Context(), s, tenant, session.ID, "work", inputs) diff --git a/services/core/tests/integration/runtime_lifecycle_test.go b/services/core/tests/integration/runtime_lifecycle_test.go index 8da737e4a..4f28fc6f0 100644 --- a/services/core/tests/integration/runtime_lifecycle_test.go +++ b/services/core/tests/integration/runtime_lifecycle_test.go @@ -148,8 +148,7 @@ func reconcileManagedState(t *testing.T, w *execution.Worker, s *Store, tenant, } func TestManagedRuntimeLostCreateRestartAndDeletion(t *testing.T) { - s, _ := newManagedTestStore(t) - key := webDeployment(t, s, "e2b") + s, key := configuredStore(t) tenant, session, env := managedSession(t, s) p := &lifecycleProvider{resources: map[string]sandbox.Info{}, loseCreate: true} w, stop := managedWorker(t, s, key, p) @@ -187,8 +186,7 @@ func TestManagedRuntimeLostCreateRestartAndDeletion(t *testing.T) { } func TestManagedRuntimeUnknownCreationRetainsCleanup(t *testing.T) { - s, _ := newManagedTestStore(t) - key := webDeployment(t, s, "e2b") + s, key := configuredStore(t) tenant, session, env := managedSession(t, s) p := &lifecycleProvider{resources: map[string]sandbox.Info{}, loseCreate: true, absent: true} w, _ := managedWorker(t, s, key, p) @@ -217,8 +215,7 @@ func TestManagedRuntimeUnknownCreationRetainsCleanup(t *testing.T) { } func TestManagedRuntimeStoppedComputeDoesNotRequestCleanup(t *testing.T) { - s, _ := newManagedTestStore(t) - key := webDeployment(t, s, "e2b") + s, key := configuredStore(t) tenant, _, env := managedSession(t, s) p := &lifecycleProvider{resources: map[string]sandbox.Info{}} w, _ := managedWorker(t, s, key, p) diff --git a/services/core/tests/integration/runtime_node_lifecycle_fixture_test.go b/services/core/tests/integration/runtime_node_lifecycle_fixture_test.go index 771c4c99f..055ac6383 100644 --- a/services/core/tests/integration/runtime_node_lifecycle_fixture_test.go +++ b/services/core/tests/integration/runtime_node_lifecycle_fixture_test.go @@ -1,7 +1,6 @@ package integration import ( - "bytes" "context" "encoding/json" "errors" @@ -14,7 +13,6 @@ import ( "time" "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/environmentconfig" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/execution" @@ -90,12 +88,7 @@ type nodeIsolationFixture struct { func newNodeIsolationFixture(t *testing.T, mode string) *nodeIsolationFixture { t.Helper() - _, pool := newManagedTestStore(t) - cipher, err := credentialcrypto.New(bytes.Repeat([]byte{8}, 32)) - if err != nil { - t.Fatal(err) - } - s := NewWithCredentialCipher(pool, cipher) + s, pool := newManagedTestStore(t) registry := runtimegateway.NewRegistry() cp := &fakeCheckpointProvider{lifecycleProvider: lifecycleProvider{resources: map[string]sandbox.Info{}}, computes: map[string]sandbox.ComputeState{}, snapshots: map[string]sandbox.SnapshotIdentity{}, bootstraps: map[string]sandbox.Bootstrap{}, peers: map[string]*websocket.Conn{}, registry: registry} p := &nodeIsolationProvider{fakeCheckpointProvider: cp, blocked: map[string]bool{}, mode: mode, entered: make(chan struct{})} @@ -141,7 +134,7 @@ func (f *nodeIsolationFixture) enroll(id string) { if err != nil { f.t.Fatal(err) } - _, err = f.nodes.Enroll(f.t.Context(), token, deployment.Enrollment{DeploymentGeneration: 1, SpecificationDigest: SandboxDeploymentTestSpec("microsandbox").Digest("microsandbox"), NodeID: id, Credential: strings.Repeat("x", 64), Name: id, Provider: "microsandbox", BackendFingerprint: strings.Repeat("b", 64)}) + _, err = f.nodes.Enroll(f.t.Context(), token, deployment.Enrollment{DeploymentGeneration: 1, SpecificationDigest: SandboxDeploymentTestSpec("microsandbox").Digest("microsandbox"), NodeID: id, Credential: strings.Repeat("x", 64), Name: id, Provider: "microsandbox", BackendFingerprint: strings.Repeat("b", 64), CoreURL: f.store.placement.PublicURL()}) if err != nil { f.t.Fatal(err) } diff --git a/services/core/tests/integration/runtime_nodes_test.go b/services/core/tests/integration/runtime_nodes_test.go index c00d88ceb..3ab2694c2 100644 --- a/services/core/tests/integration/runtime_nodes_test.go +++ b/services/core/tests/integration/runtime_nodes_test.go @@ -352,9 +352,6 @@ func TestRuntimeNodesLongOfflineRetainsExactAllocation(t *testing.T) { if err != nil { t.Fatal(err) } - if _, err := s.pool.Exec(t.Context(), "UPDATE runtime_allocations SET kept_at=clock_timestamp()-interval '2 days' WHERE id=$1", owner.ID); err != nil { - t.Fatal(err) - } if _, err := s.pool.Exec(t.Context(), "UPDATE runtime_nodes SET connection_id=NULL WHERE id=$1", d.NodeID); err != nil { t.Fatal(err) } @@ -370,8 +367,8 @@ func TestRuntimeNodesLongOfflineRetainsExactAllocation(t *testing.T) { if err != nil || resumed.ID != owner.ID || resumed.DeviceID != owner.DeviceID || resumed.NodeID != owner.NodeID { t.Fatal("reconnect changed instance", resumed, err) } - if _, err := deploymentExecution(t, w).KeepAllocation(t.Context(), resumed); err != nil { - t.Fatal("offline observation lease could not renew", err) + if _, err := deploymentExecution(t, w).CheckRunning(t.Context(), resumed); err != nil { + t.Fatal("reconnected allocation stopped running", err) } if _, err := s.pool.Exec(t.Context(), "UPDATE runtime_allocations SET compute_phase='suspended',compute_state=$2::jsonb,compute_retained_until=clock_timestamp()+interval '1 day' WHERE id=$1", owner.ID, json.RawMessage(`{"snapshot":{"id":"same-snapshot"}}`)); err != nil { t.Fatal(err) diff --git a/services/core/tests/integration/runtime_observation_scan_test.go b/services/core/tests/integration/runtime_observation_scan_test.go index b535ccf30..4a3a6be2d 100644 --- a/services/core/tests/integration/runtime_observation_scan_test.go +++ b/services/core/tests/integration/runtime_observation_scan_test.go @@ -8,7 +8,7 @@ import ( ) func TestRuntimeObservationScanIsDeploymentWideBoundedAndExcludesDeleted(t *testing.T) { - s, _ := newManagedTestStore(t) + s, _ := configuredStore(t) var expected []string for range 5 { _, session, _ := managedSession(t, s) diff --git a/services/core/tests/integration/runtime_pending_test.go b/services/core/tests/integration/runtime_pending_test.go index 06af06ef7..3978522ba 100644 --- a/services/core/tests/integration/runtime_pending_test.go +++ b/services/core/tests/integration/runtime_pending_test.go @@ -14,8 +14,7 @@ import ( ) func TestManagedRuntimeAutomaticBootstrapRecoversCommittedSessions(t *testing.T) { - s, _ := newManagedTestStore(t) - key := webDeployment(t, s, "e2b") + s, key := configuredStore(t) tenant, idle, idleEnvironment := managedSession(t, s) initial, err := s.CreateSession(t.Context(), tenant, sessions.CreateSession{Creator: FixtureCreator(), Engine: "codex", IdempotencyKey: uuid.NewString(), Configuration: json.RawMessage(`{"agent":{"model":"test"},"environment":{"type":"openai_hosted"}}`), InitialInputs: []sessions.Input{messageInput("hello")}}) if err != nil { diff --git a/services/core/tests/integration/runtime_scan_test.go b/services/core/tests/integration/runtime_scan_test.go index 5f88de644..943db3246 100644 --- a/services/core/tests/integration/runtime_scan_test.go +++ b/services/core/tests/integration/runtime_scan_test.go @@ -26,9 +26,8 @@ func (p *scanProvider) GetInfo(ctx context.Context, ref sandbox.Reference) (sand func TestManagedRuntimeScanWrapServicesNextPage(t *testing.T) { for _, count := range []int{0, 1, 31, 32, 33, 65} { t.Run(fmt.Sprint(count), func(t *testing.T) { - s, _ := newManagedTestStore(t) + s, key := configuredStore(t) p := &scanProvider{lifecycleProvider: lifecycleProvider{resources: map[string]sandbox.Info{}}} - key := webDeployment(t, s, "e2b") w, _ := managedWorker(t, s, key, p) var ids []string for range count { @@ -65,9 +64,8 @@ func TestManagedRuntimeScanWrapServicesNextPage(t *testing.T) { } func TestManagedRuntimeScanEmptyAfterCleanupAndCanceledCall(t *testing.T) { - s, _ := newManagedTestStore(t) + s, key := configuredStore(t) p := &scanProvider{lifecycleProvider: lifecycleProvider{resources: map[string]sandbox.Info{}}} - key := webDeployment(t, s, "e2b") w, _ := managedWorker(t, s, key, p) tenant, session, env := managedSession(t, s) owner, err := w.ProvisionEnvironment(t.Context(), tenant, env.ID, key) diff --git a/services/core/tests/integration/runtime_suspension_test.go b/services/core/tests/integration/runtime_suspension_test.go index 6ca4cd340..1d5eabad8 100644 --- a/services/core/tests/integration/runtime_suspension_test.go +++ b/services/core/tests/integration/runtime_suspension_test.go @@ -17,11 +17,12 @@ import ( func runtimeSuspensionFixture(t *testing.T) (*Store, *Store, *pgxpool.Pool, deployment.Allocation) { t.Helper() - s, pool := testStore(t) + s, installation := configuredStore(t) + pool := s.pool w := executionWriter(t, s) tenant := uuid.NewString() _, environment := localEnvironment(t, s, tenant) - owner, err := deploymentExecution(t, w).ReserveAllocation(t.Context(), deployment.AllocationKey{TenantID: tenant, EnvironmentID: environment.ID}, uuid.NewString(), runtimedevice.HashCredential(uuid.NewString())) + owner, err := deploymentExecution(t, w).ReserveAllocation(t.Context(), deployment.AllocationKey{TenantID: tenant, EnvironmentID: environment.ID}, installation, runtimedevice.HashCredential(uuid.NewString())) if err != nil { t.Fatal(err) } @@ -178,7 +179,7 @@ func TestRuntimeSuspensionWakeDoesNotLoseNewerWork(t *testing.T) { if err != nil { t.Fatal(err) } - if _, err := deploymentExecution(t, w).KeepAllocation(t.Context(), owner); err != nil { + if _, err := deploymentExecution(t, w).CheckRunning(t.Context(), owner); err != nil { t.Fatal(err) } if _, err := sessionAdapter(s).GetSession(t.Context(), owner.TenantID, owner.SessionID); err != nil { @@ -232,10 +233,9 @@ func TestRuntimeSuspensionRetentionAndDeletedSession(t *testing.T) { for _, phase := range []string{"quiescing", "suspending", "suspended"} { owner = runtimeSuspensionStep(t, w, owner, phase, &until) } - runtimeSuspensionSQL(t, pool, `UPDATE runtime_allocations SET kept_at=clock_timestamp()-interval '2 hours' WHERE id=$1`, owner.ID) retained, err := deploymentStore(s).EnvironmentAllocation(t.Context(), deployment.AllocationKey{TenantID: owner.TenantID, EnvironmentID: owner.EnvironmentID}) if err != nil || retained.Expired { - t.Fatal("suspended snapshot expired by disconnected heartbeat", retained, err) + t.Fatal("suspended snapshot expired within its retention", retained, err) } runtimeSuspensionSQL(t, pool, `UPDATE runtime_allocations SET compute_retained_until=clock_timestamp()-interval '1 second' WHERE id=$1`, owner.ID) expired, err := deploymentStore(s).EnvironmentAllocation(t.Context(), deployment.AllocationKey{TenantID: owner.TenantID, EnvironmentID: owner.EnvironmentID}) @@ -312,14 +312,9 @@ func TestRuntimeSuspensionWakeRemainsUntilRunning(t *testing.T) { } } -func TestRuntimeSuspensionExpiredRunningAndLostWriterAreFenced(t *testing.T) { +func TestRuntimeSuspensionLostWriterIsFenced(t *testing.T) { _, w, pool, owner := runtimeSuspensionFixture(t) runtimeSuspensionCompleted(t, pool, owner) - until := time.Now().Add(time.Hour) - runtimeSuspensionSQL(t, pool, `UPDATE runtime_allocations SET kept_at=clock_timestamp()-interval '2 hours' WHERE id=$1`, owner.ID) - if _, err := deploymentExecution(t, w).SetCompute(t.Context(), owner, "quiescing", json.RawMessage(`{}`), &until, time.Nanosecond); !errors.Is(err, deployment.ErrAllocationConflict) { - t.Fatal("expired running allocation entered checkpoint", err) - } if err := w.lease.Close(t.Context()); err != nil { t.Fatal(err) } diff --git a/services/core/tests/integration/runtime_worker_recovery_test.go b/services/core/tests/integration/runtime_worker_recovery_test.go index 0a8b08e51..7cb9c7f68 100644 --- a/services/core/tests/integration/runtime_worker_recovery_test.go +++ b/services/core/tests/integration/runtime_worker_recovery_test.go @@ -26,9 +26,8 @@ func runtimeWorkerHarness(t *testing.T) (*dispatchHarness, *pgxpool.Pool) { t.Helper() h := newDispatchHarnessForSession(t, []byte(`{"agent":{"model":"test-model"},"environment":{"type":"openai_hosted","network":{"access":"enabled"}}}`), true) enableWorkerEnvironment(t, h) - _, pool := testStore(t) - insertWorkerRuntimeAllocation(t, pool, h, "waking") - return h, pool + insertWorkerRuntimeAllocation(t, h.s.pool, h, "waking") + return h, h.s.pool } func TestPreparedDispatchKeepsPendingReservationAfterComputeConflict(t *testing.T) { @@ -76,9 +75,12 @@ func TestWorkerWaitsForComputeAndSurvivesPromotionConflict(t *testing.T) { if err != nil { t.Fatal(err) } - // This enters the same binding gate as scheduled input and returns without - // a native preparation, despite the socket already advertising capabilities. - if _, err := worker.ReadEnvironmentDirectory(t.Context(), environment, ""); !errors.Is(err, execution.ErrExecutionUnavailable) { + // The read waits for the compute to wake and gives up at its deadline + // without a native preparation, despite the socket already advertising + // capabilities. + read, stop := context.WithTimeout(t.Context(), 500*time.Millisecond) + defer stop() + if _, err := worker.ReadEnvironmentDirectory(read, environment, ""); !errors.Is(err, execution.ErrExecutionUnavailable) { t.Fatal("waking connection was treated as work-ready", err) } select { @@ -143,7 +145,7 @@ func TestWorkerRestartPreservesQueuedTurnWhileComputeWakes(t *testing.T) { if _, err := pool.Exec(t.Context(), `INSERT INTO turns(id,session_id,status) VALUES($1,$2,'queued')`, turn, h.session.ID); err != nil { t.Fatal(err) } - worker, err := startWorkerErr(t.Context(), h.s, h.d) + worker, err := startWorkerErr(t, t.Context(), h.s, h.d) if err != nil { t.Fatal("queued wake blocked Core startup", err) } diff --git a/services/core/tests/integration/sandbox_deployment_resources_test.go b/services/core/tests/integration/sandbox_deployment_resources_test.go index f4588ac29..b22495635 100644 --- a/services/core/tests/integration/sandbox_deployment_resources_test.go +++ b/services/core/tests/integration/sandbox_deployment_resources_test.go @@ -1,22 +1,15 @@ package integration import ( - "bytes" "testing" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" "github.com/google/uuid" ) func TestSandboxDeploymentMutationViewsIncludeActualResources(t *testing.T) { - _, pool := newManagedTestStore(t) - cipher, err := credentialcrypto.New(bytes.Repeat([]byte{9}, 32)) - if err != nil { - t.Fatal(err) - } - s := NewWithCredentialCipher(pool, cipher) + s, _ := newManagedTestStore(t) w := executionWriter(t, s) changes := deploymentExecution(t, w) installation := uuid.NewString() diff --git a/services/core/tests/integration/sandbox_deployment_switch_test.go b/services/core/tests/integration/sandbox_deployment_switch_test.go index 37b73464b..e1f96a4e4 100644 --- a/services/core/tests/integration/sandbox_deployment_switch_test.go +++ b/services/core/tests/integration/sandbox_deployment_switch_test.go @@ -14,7 +14,6 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/e2b" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" @@ -32,12 +31,7 @@ func enrollmentTokenDigest(token string) string { } func TestSandboxResetClearsCustomE2BEndpoint(t *testing.T) { - _, pool := newManagedTestStore(t) - cipher, err := credentialcrypto.New(bytes.Repeat([]byte{7}, 32)) - if err != nil { - t.Fatal(err) - } - s := NewWithCredentialCipher(pool, cipher) + s, pool := newManagedTestStore(t) w := executionWriter(t, s) changes := deploymentExecution(t, w) installation := uuid.NewString() @@ -73,12 +67,7 @@ func TestSandboxResetClearsCustomE2BEndpoint(t *testing.T) { } func TestSandboxDirectDeploymentOwnershipAndCleanSwitch(t *testing.T) { - _, pool := newManagedTestStore(t) - cipher, err := credentialcrypto.New(bytes.Repeat([]byte{4}, 32)) - if err != nil { - t.Fatal(err) - } - s := NewWithCredentialCipher(pool, cipher) + s, pool := newManagedTestStore(t) w := executionWriter(t, s) changes := deploymentExecution(t, w) id := uuid.NewString() @@ -123,13 +112,6 @@ func TestSandboxDirectDeploymentOwnershipAndCleanSwitch(t *testing.T) { if err != nil || !ok || credential.RuntimeAllocationID != owner.ID || credential.RuntimeNodeID != "" { t.Fatal("direct bootstrap lost managed identity", err) } - if _, err := pool.Exec(t.Context(), "UPDATE runtime_allocations SET kept_at=clock_timestamp()-interval '2 hours' WHERE id=$1", owner.ID); err != nil { - t.Fatal(err) - } - observed, err := deploymentStore(w).EnvironmentAllocation(t.Context(), deployment.AllocationKey{TenantID: tenant, EnvironmentID: environment}) - if err != nil || observed.Expired { - t.Fatal("cloud inherited legacy node-less expiry", err) - } if err := deploymentExecution(t, w).StartReset(SandboxResetTestContext(t.Context()), id, deployment.ResetRequest{Clear: "auto", ExpectedGeneration: 1}); err != nil { t.Fatal(err) } @@ -162,9 +144,7 @@ func TestSandboxDirectDeploymentOwnershipAndCleanSwitch(t *testing.T) { } func TestSandboxSwitchRetiresNodesAndEnrollment(t *testing.T) { - _, pool := newManagedTestStore(t) - cipher, _ := credentialcrypto.New(bytes.Repeat([]byte{5}, 32)) - s := NewWithCredentialCipher(pool, cipher) + s, pool := newManagedTestStore(t) w := executionWriter(t, s) changes := deploymentExecution(t, w) nodes := deploymentService(t, s) @@ -179,7 +159,7 @@ func TestSandboxSwitchRetiresNodesAndEnrollment(t *testing.T) { if err != nil { t.Fatal(err) } - node := deployment.Enrollment{DeploymentGeneration: 1, SpecificationDigest: SandboxDeploymentTestSpec("docker").Digest("docker"), NodeID: uuid.NewString(), Name: "Machine", Provider: "docker", Credential: strings.Repeat("c", 64), BackendFingerprint: strings.Repeat("b", 64)} + node := deployment.Enrollment{DeploymentGeneration: 1, SpecificationDigest: SandboxDeploymentTestSpec("docker").Digest("docker"), NodeID: uuid.NewString(), Name: "Machine", Provider: "docker", Credential: strings.Repeat("c", 64), BackendFingerprint: strings.Repeat("b", 64), CoreURL: s.placement.PublicURL()} if _, err := nodes.Enroll(t.Context(), token, node); err != nil { t.Fatal(err) } @@ -237,9 +217,7 @@ func TestSandboxSwitchRetiresNodesAndEnrollment(t *testing.T) { } func TestSandboxResetSerializesFreshDirectSessions(t *testing.T) { - _, pool := newManagedTestStore(t) - cipher, _ := credentialcrypto.New(bytes.Repeat([]byte{6}, 32)) - s := NewWithCredentialCipher(pool, cipher) + s, _ := newManagedTestStore(t) w := executionWriter(t, s) changes := deploymentExecution(t, w) id := uuid.NewString() @@ -281,9 +259,7 @@ func TestSandboxResetSerializesFreshDirectSessions(t *testing.T) { } func TestSandboxSwitchPreservesReleasedAllocationAndItemHistory(t *testing.T) { - _, pool := newManagedTestStore(t) - cipher, _ := credentialcrypto.New(bytes.Repeat([]byte{8}, 32)) - s := NewWithCredentialCipher(pool, cipher) + s, pool := newManagedTestStore(t) w := executionWriter(t, s) changes := deploymentExecution(t, w) installation := uuid.NewString() @@ -366,9 +342,7 @@ func TestSandboxSwitchPreservesReleasedAllocationAndItemHistory(t *testing.T) { // node reconnects to drain resources; fresh admission and node configuration // stay closed until an administrator replaces the selection. func TestUnspecifiedNodeDeploymentRejectedWithoutMutation(t *testing.T) { - _, pool := newManagedTestStore(t) - cipher, _ := credentialcrypto.New(bytes.Repeat([]byte{7}, 32)) - s := NewWithCredentialCipher(pool, cipher) + s, pool := newManagedTestStore(t) w := executionWriter(t, s) changes := deploymentExecution(t, w) nodes := deploymentService(t, s) @@ -385,7 +359,7 @@ func TestUnspecifiedNodeDeploymentRejectedWithoutMutation(t *testing.T) { if err != nil { t.Fatal(err) } - node := deployment.Enrollment{DeploymentGeneration: 1, SpecificationDigest: spec.Digest("docker"), NodeID: uuid.NewString(), Name: "Legacy", Provider: "docker", Credential: strings.Repeat("l", 64), BackendFingerprint: strings.Repeat("b", 64)} + node := deployment.Enrollment{DeploymentGeneration: 1, SpecificationDigest: spec.Digest("docker"), NodeID: uuid.NewString(), Name: "Legacy", Provider: "docker", Credential: strings.Repeat("l", 64), BackendFingerprint: strings.Repeat("b", 64), CoreURL: s.placement.PublicURL()} if _, err := nodes.Enroll(t.Context(), token, node); err != nil { t.Fatal(err) } diff --git a/services/core/tests/integration/sandbox_deployment_switch_worker_test.go b/services/core/tests/integration/sandbox_deployment_switch_worker_test.go index af2c98e9f..212751f8c 100644 --- a/services/core/tests/integration/sandbox_deployment_switch_worker_test.go +++ b/services/core/tests/integration/sandbox_deployment_switch_worker_test.go @@ -1,7 +1,6 @@ package integration import ( - "bytes" "context" "errors" "reflect" @@ -13,7 +12,6 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/e2b" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/execution" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimegateway" @@ -22,9 +20,7 @@ import ( ) func TestSandboxWorkerSwitchesAndRecoversFailedActivation(t *testing.T) { - _, pool := newManagedTestStore(t) - cipher, _ := credentialcrypto.New(bytes.Repeat([]byte{7}, 32)) - s := NewWithCredentialCipher(pool, cipher) + s, pool := newManagedTestStore(t) deployments := deploymentService(t, s) id := uuid.NewString() p := &lifecycleProvider{resources: map[string]sandbox.Info{}} @@ -75,7 +71,7 @@ func TestSandboxWorkerSwitchesAndRecoversFailedActivation(t *testing.T) { if err != nil { t.Fatal(err) } - node := deployment.Enrollment{NodeID: uuid.NewString(), Name: "retained candidate fixture", Provider: "docker", Credential: strings.Repeat("n", 64), BackendFingerprint: strings.Repeat("b", 64), DeploymentGeneration: 1, SpecificationDigest: SandboxDeploymentTestSpec("docker").Digest("docker")} + node := deployment.Enrollment{NodeID: uuid.NewString(), Name: "retained candidate fixture", Provider: "docker", Credential: strings.Repeat("n", 64), BackendFingerprint: strings.Repeat("b", 64), DeploymentGeneration: 1, SpecificationDigest: SandboxDeploymentTestSpec("docker").Digest("docker"), CoreURL: s.placement.PublicURL()} if _, err := deployments.Enroll(t.Context(), enrollment, node); err != nil { t.Fatal(err) } diff --git a/services/core/tests/integration/sandbox_deployment_view_test.go b/services/core/tests/integration/sandbox_deployment_view_test.go index 0e696c890..ca1ed70b7 100644 --- a/services/core/tests/integration/sandbox_deployment_view_test.go +++ b/services/core/tests/integration/sandbox_deployment_view_test.go @@ -8,19 +8,13 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/e2b" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" "github.com/google/uuid" ) func TestSandboxDeploymentViewRecordsTemplateBuildAndSuspension(t *testing.T) { - _, pool := newManagedTestStore(t) - cipher, err := credentialcrypto.New(bytes.Repeat([]byte{5}, 32)) - if err != nil { - t.Fatal(err) - } - s := NewWithCredentialCipher(pool, cipher) + s, pool := newManagedTestStore(t) w := executionWriter(t, s) changes := deploymentExecution(t, w) id := uuid.NewString() diff --git a/services/core/tests/integration/sandbox_deployment_worker_test.go b/services/core/tests/integration/sandbox_deployment_worker_test.go index dcae13238..7e76740a7 100644 --- a/services/core/tests/integration/sandbox_deployment_worker_test.go +++ b/services/core/tests/integration/sandbox_deployment_worker_test.go @@ -58,7 +58,7 @@ func TestSandboxDeploymentWorkerActivatesWithoutRestart(t *testing.T) { t.Fatal(err) } nodeID := uuid.NewString() - if _, err := deployments.Enroll(t.Context(), token, deployment.Enrollment{DeploymentGeneration: 1, SpecificationDigest: SandboxDeploymentTestSpec("docker").Digest("docker"), NodeID: nodeID, Name: "Remote", Provider: "docker", Credential: strings.Repeat("x", 64), BackendFingerprint: strings.Repeat("b", 64)}); err != nil { + if _, err := deployments.Enroll(t.Context(), token, deployment.Enrollment{DeploymentGeneration: 1, SpecificationDigest: SandboxDeploymentTestSpec("docker").Digest("docker"), NodeID: nodeID, Name: "Remote", Provider: "docker", Credential: strings.Repeat("x", 64), BackendFingerprint: strings.Repeat("b", 64), CoreURL: s.placement.PublicURL()}); err != nil { t.Fatal(err) } connect := func() { diff --git a/services/core/tests/integration/sandbox_specification_lifecycle_test.go b/services/core/tests/integration/sandbox_specification_lifecycle_test.go index 314e90758..6c27b7e03 100644 --- a/services/core/tests/integration/sandbox_specification_lifecycle_test.go +++ b/services/core/tests/integration/sandbox_specification_lifecycle_test.go @@ -11,7 +11,6 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment/placement" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" "github.com/google/uuid" @@ -19,12 +18,7 @@ import ( func webSpecificationFixture(t *testing.T, provider string) (*Store, *Store, deployment.View, sandbox.Selection) { t.Helper() - _, pool := newManagedTestStore(t) - cipher, err := credentialcrypto.New(bytes.Repeat([]byte{13}, 32)) - if err != nil { - t.Fatal(err) - } - s := NewWithCredentialCipher(pool, cipher) + s, _ := newManagedTestStore(t) w := executionWriter(t, s) id := uuid.NewString() changes := deploymentExecution(t, w) @@ -80,7 +74,7 @@ func TestSandboxSpecificationBootstrapReadDoesNotConsumeEnrollment(t *testing.T) if _, err := nodes.NodeConfiguration(t.Context(), "", "invalid-token", 0); !errors.Is(err, deployment.ErrNodeCredential) { t.Fatal("unauthenticated configuration read", err) } - node := deployment.Enrollment{NodeID: uuid.NewString(), Name: "bootstrap", Credential: strings.Repeat("n", 64), Provider: "docker", BackendFingerprint: strings.Repeat("b", 64), DeploymentGeneration: view.Generation, SpecificationDigest: view.SpecificationDigest} + node := deployment.Enrollment{NodeID: uuid.NewString(), Name: "bootstrap", Credential: strings.Repeat("n", 64), Provider: "docker", BackendFingerprint: strings.Repeat("b", 64), DeploymentGeneration: view.Generation, SpecificationDigest: view.SpecificationDigest, CoreURL: s.placement.PublicURL()} for _, change := range []func(*deployment.Enrollment){ func(n *deployment.Enrollment) { n.DeploymentGeneration++ }, func(n *deployment.Enrollment) { n.SpecificationDigest = strings.Repeat("c", 64) }, diff --git a/services/core/tests/integration/session_artifacts_public_test.go b/services/core/tests/integration/session_artifacts_public_test.go index 4b72e8863..77306864a 100644 --- a/services/core/tests/integration/session_artifacts_public_test.go +++ b/services/core/tests/integration/session_artifacts_public_test.go @@ -89,7 +89,7 @@ func artifactHTTPServer(t *testing.T, s *Store) (server *httptest.Server, owner, // environment_id filter matches nothing like another Environment's ID (HE-56), // without weakening tenant or Session scoping. func TestSessionArtifactListEnvelopeAndEnvironmentFilterPostgres(t *testing.T) { - s, _ := testStore(t) + s, _ := configuredStore(t) sessionService, err := newSessionService(s) if err != nil { t.Fatal(err) @@ -213,7 +213,7 @@ func TestSessionArtifactsOfficialClientPostgres(t *testing.T) { if python == "" { t.Skip("pinned official Python SDK required") } - s, _ := testStore(t) + s, _ := configuredStore(t) sessionStore := sessionAdapter(s) sessionService, err := newSessionService(s) if err != nil { diff --git a/services/core/tests/integration/session_artifacts_test.go b/services/core/tests/integration/session_artifacts_test.go index a998c8da7..9cb471cc6 100644 --- a/services/core/tests/integration/session_artifacts_test.go +++ b/services/core/tests/integration/session_artifacts_test.go @@ -71,7 +71,8 @@ func TestSessionArtifactsPublishVersionScopeAndLifetime(t *testing.T) { } func testSessionArtifactsPublishVersionScopeAndLifetime(t *testing.T, kind string) { - s, pool := testStore(t) + s, _ := configuredStore(t) + pool := s.pool tenant, session, environment, turn := artifactTurn(t, s, kind) before := largeObjectCount(t, pool) data := bytes.Repeat([]byte("immutable\x00"), 100000) @@ -187,7 +188,8 @@ func testSessionArtifactsPublishVersionScopeAndLifetime(t *testing.T, kind strin func TestSessionArtifactsDiscardTerminalPrivateCapture(t *testing.T) { for _, status := range []string{sessions.TurnFailed, sessions.TurnCancelled} { t.Run(status, func(t *testing.T) { - s, pool := testStore(t) + s, _ := configuredStore(t) + pool := s.pool tenant, session, environment, turn := artifactTurn(t, s, "openai_hosted") before := largeObjectCount(t, pool) body := artifactArchive(t, map[string][]byte{"outputs/a": []byte("private")}) @@ -211,7 +213,8 @@ func TestSessionArtifactsDiscardTerminalPrivateCapture(t *testing.T) { func TestSessionArtifactTransferDoesNotBlockDeletionOrCancellation(t *testing.T) { for _, operation := range []string{"delete", "cancel"} { t.Run(operation, func(t *testing.T) { - s, pool := testStore(t) + s, _ := configuredStore(t) + pool := s.pool tenant, session, environment, turn := artifactTurn(t, s, "openai_hosted") before := largeObjectCount(t, pool) reader, writer := io.Pipe() @@ -312,7 +315,8 @@ func publishedPaths(published map[string]sessions.Artifact) []string { // Later Turns publish a path only when it is new, its bytes differ from the // newest remaining Artifact for that path, or no Artifact remains for it (HE-52). func TestSessionArtifactsRepublishOnlyNewChangedOrDeletedPaths(t *testing.T) { - s, pool := testStore(t) + s, _ := configuredStore(t) + pool := s.pool tenant, session, environment, first := artifactTurn(t, s, "openai_hosted") before := largeObjectCount(t, pool) turnNumber := 1 @@ -432,7 +436,7 @@ func TestSessionArtifactsRepublishOnlyNewChangedOrDeletedPaths(t *testing.T) { // Publication time can come from the Runtime's reported completion and invert // the order of Turns; the newest version for a path still follows Turn order. func TestSessionArtifactsNewestVersionFollowsTurnOrder(t *testing.T) { - s, _ := testStore(t) + s, _ := configuredStore(t) tenant := uuid.NewString() created, err := s.CreateSession(t.Context(), tenant, environmentInput("artifact-order", "openai_hosted", "/workspace")) if err != nil { @@ -476,7 +480,8 @@ func TestSessionArtifactsNewestVersionFollowsTurnOrder(t *testing.T) { // A deletion that holds the Session lock while Turn completion waits for it is // seen by the completion transaction, which then republishes the path. func TestSessionArtifactsCompletionWaitsForConcurrentDeletion(t *testing.T) { - s, pool := testStore(t) + s, _ := configuredStore(t) + pool := s.pool tenant, session, environment, first := artifactTurn(t, s, "openai_hosted") before := largeObjectCount(t, pool) stageArtifactOutputs(t, s, tenant, session, environment, first, map[string]string{"a.txt": "alpha"}) diff --git a/services/core/tests/integration/session_creation_identity_test.go b/services/core/tests/integration/session_creation_identity_test.go index e919ccf68..159c8e7f9 100644 --- a/services/core/tests/integration/session_creation_identity_test.go +++ b/services/core/tests/integration/session_creation_identity_test.go @@ -1,13 +1,11 @@ package integration import ( - "bytes" "encoding/json" "testing" v1 "github.com/MiniMax-AI/OpenAgentCore/contracts/agents-api/v1" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/adminaudit" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/modelconfiguration" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/modelconfigurationpg" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" @@ -21,11 +19,7 @@ import ( // an older revision never updates the current default's observations. func TestSessionCreationKeepsItsResolvedDeploymentRevision(t *testing.T) { s, pool := newManagedTestStore(t) - cipher, err := credentialcrypto.New(bytes.Repeat([]byte{73}, 32)) - if err != nil { - t.Fatal(err) - } - defaults := modelconfigurationpg.New(pgunit.NewPool(pool), cipher) + defaults := modelconfigurationpg.New(pgunit.NewPool(pool), s.credentialCipher) service, err := modelconfiguration.NewService(defaults) if err != nil { t.Fatal(err) diff --git a/services/core/tests/integration/session_deletion_lifecycle_public_test.go b/services/core/tests/integration/session_deletion_lifecycle_public_test.go index 112f33ff9..6749a89dc 100644 --- a/services/core/tests/integration/session_deletion_lifecycle_public_test.go +++ b/services/core/tests/integration/session_deletion_lifecycle_public_test.go @@ -23,7 +23,7 @@ const deletionAgent = `"agent":{"id":"agent_deletion","model":"fixture","tools": // missing and malformed identifiers keep one not-found response. func TestSessionDeletionLifecyclePostgres(t *testing.T) { // An isolated database keeps the no-write digest independent of other tests. - s, _ := newManagedTestStore(t) + s, _ := configuredStore(t) audit := auditpg.New(pgunit.NewPool(s.pool)) ctx := t.Context() tenant, owner, foreign := uuid.NewString(), uuid.NewString(), uuid.NewString() diff --git a/services/core/tests/integration/session_diagnostics_test.go b/services/core/tests/integration/session_diagnostics_test.go index 4c3996829..6a21d8671 100644 --- a/services/core/tests/integration/session_diagnostics_test.go +++ b/services/core/tests/integration/session_diagnostics_test.go @@ -198,7 +198,8 @@ func TestDiagnosticTimingBoundOrderAndIsolation(t *testing.T) { } func TestDiagnosticProvisioningDetailAtomicAndPrivate(t *testing.T) { - s, pool := testStore(t) + s, installation := configuredStore(t) + pool := s.pool tenant := uuid.NewString() input := environmentInput("safe-detail", "openai_hosted", "/workspace") input.InitialInputs = []sessions.Input{messageInput("initial")} @@ -207,7 +208,7 @@ func TestDiagnosticProvisioningDetailAtomicAndPrivate(t *testing.T) { t.Fatal(err) } writer := executionWriter(t, s) - owner, err := deploymentExecution(t, writer).ReserveAllocation(t.Context(), deployment.AllocationKey{TenantID: tenant, EnvironmentID: session.Environment.ID}, uuid.NewString(), runtimedevice.HashCredential(uuid.NewString())) + owner, err := deploymentExecution(t, writer).ReserveAllocation(t.Context(), deployment.AllocationKey{TenantID: tenant, EnvironmentID: session.Environment.ID}, installation, runtimedevice.HashCredential(uuid.NewString())) if err != nil { t.Fatal(err) } diff --git a/services/core/tests/integration/session_execution_configuration_test.go b/services/core/tests/integration/session_execution_configuration_test.go index 0043ac053..4816546f2 100644 --- a/services/core/tests/integration/session_execution_configuration_test.go +++ b/services/core/tests/integration/session_execution_configuration_test.go @@ -10,7 +10,6 @@ import ( "time" v1 "github.com/MiniMax-AI/OpenAgentCore/contracts/agents-api/v1" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/environmentconfig" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" @@ -32,12 +31,8 @@ func executionProjectionInput(source string) sessions.CreateSession { } func TestSessionExecutionConfigurationFrozenAcrossCreationPathsAndRetry(t *testing.T) { - _, pool := testStore(t) - cipher, err := credentialcrypto.New(bytes.Repeat([]byte{41}, 32)) - if err != nil { - t.Fatal(err) - } - s := NewWithCredentialCipher(pool, cipher) + s, _ := configuredStore(t) + pool := s.pool for _, stream := range []bool{false, true} { for _, source := range []string{"session", "agent", "deployment"} { t.Run(source+map[bool]string{false: "/ordinary", true: "/stream"}[stream], func(t *testing.T) { @@ -160,12 +155,8 @@ func TestSessionExecutionConfigurationHistoricalProvenance(t *testing.T) { } func TestSessionExecutionConfigurationRollbackAndValidation(t *testing.T) { - _, pool := testStore(t) - cipher, err := credentialcrypto.New(bytes.Repeat([]byte{42}, 32)) - if err != nil { - t.Fatal(err) - } - s := NewWithCredentialCipher(pool, cipher) + s, _ := configuredStore(t) + pool := s.pool tenant := uuid.NewString() for _, kind := range []string{"model", "harness", "source", "provider", "provider_mismatch", "post_projection_failure"} { input := executionProjectionInput("session") @@ -201,7 +192,7 @@ func TestSessionExecutionConfigurationRollbackAndValidation(t *testing.T) { } func TestSessionExecutionConfigurationConcurrentRetryKeepsWinner(t *testing.T) { - s, _ := testStore(t) + s, _ := configuredStore(t) tenant := uuid.NewString() input := executionProjectionInput("session") var wg sync.WaitGroup @@ -246,7 +237,8 @@ func TestSessionExecutionConfigurationConcurrentRetryKeepsWinner(t *testing.T) { } func TestSessionExecutionConfigurationSurvivesSuspendResume(t *testing.T) { - s, pool := testStore(t) + s, installation := configuredStore(t) + pool := s.pool w := executionWriter(t, s) tenant := uuid.NewString() session, err := s.CreateSession(t.Context(), tenant, executionProjectionInput("agent")) @@ -261,7 +253,7 @@ func TestSessionExecutionConfigurationSurvivesSuspendResume(t *testing.T) { if err != nil { t.Fatal(err) } - owner, err := deploymentExecution(t, w).ReserveAllocation(t.Context(), deployment.AllocationKey{TenantID: tenant, EnvironmentID: environment.ID}, uuid.NewString(), runtimedevice.HashCredential(uuid.NewString())) + owner, err := deploymentExecution(t, w).ReserveAllocation(t.Context(), deployment.AllocationKey{TenantID: tenant, EnvironmentID: environment.ID}, installation, runtimedevice.HashCredential(uuid.NewString())) if err != nil { t.Fatal(err) } diff --git a/services/core/tests/integration/session_initial_public_test.go b/services/core/tests/integration/session_initial_public_test.go index 3b8162891..3d4ab16bb 100644 --- a/services/core/tests/integration/session_initial_public_test.go +++ b/services/core/tests/integration/session_initial_public_test.go @@ -9,6 +9,7 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/execution" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimegateway" "github.com/google/uuid" ) @@ -21,7 +22,7 @@ func TestInitialSessionInputOfficialClient(t *testing.T) { token, foreign := uuid.NewString(), uuid.NewString() auth := newTestAuthenticator(t, []testAPIKey{{OrganizationID: "test-org", ProjectID: uuid.NewString(), SubjectKind: "service_account", SubjectID: "test-runner", TokenSHA256: runtimedevice.HashCredential(token), TenantID: uuid.NewString()}, {OrganizationID: "test-org", ProjectID: uuid.NewString(), SubjectKind: "service_account", SubjectID: "test-runner", TokenSHA256: runtimedevice.HashCredential(foreign), TenantID: uuid.NewString()}}) // Exercise real worker admission with dispatch paused for deterministic reads. - worker := startWorker(t, t.Context(), s, &execution.Dispatcher{}) + worker := startWorker(t, t.Context(), s, &execution.Dispatcher{Registry: runtimegateway.NewRegistry()}) t.Cleanup(func() { stopped, cancel := context.WithCancel(context.Background()) cancel() diff --git a/services/core/tests/integration/session_model_execution_http_test.go b/services/core/tests/integration/session_model_execution_http_test.go index 3300e84a7..14ac1d8b5 100644 --- a/services/core/tests/integration/session_model_execution_http_test.go +++ b/services/core/tests/integration/session_model_execution_http_test.go @@ -1,21 +1,17 @@ package integration import ( - "bytes" "encoding/json" "net/http/httptest" "strings" "testing" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" "github.com/google/uuid" ) func TestModelExecutionHTTPWriteOnlyAndStrictAdmission(t *testing.T) { - _, pool := testStore(t) - cipher, _ := credentialcrypto.New(bytes.Repeat([]byte{6}, 32)) - st := NewWithCredentialCipher(pool, cipher) + st, _ := configuredStore(t) tenant, token := uuid.NewString(), uuid.NewString() auth := newTestAuthenticator(t, []testAPIKey{{OrganizationID: "test-org", ProjectID: uuid.NewString(), SubjectKind: "service_account", SubjectID: "catalog-test", TokenSHA256: runtimedevice.HashCredential(token), TenantID: tenant}}) handler, err := publicHandler(t, st, auth, "codex") diff --git a/services/core/tests/integration/session_model_options_test.go b/services/core/tests/integration/session_model_options_test.go index 553df763c..7ba034b94 100644 --- a/services/core/tests/integration/session_model_options_test.go +++ b/services/core/tests/integration/session_model_options_test.go @@ -1,23 +1,17 @@ package integration import ( - "bytes" "encoding/json" "testing" v1 "github.com/MiniMax-AI/OpenAgentCore/contracts/agents-api/v1" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" "github.com/google/uuid" ) func TestSessionModelExecutionStoresOnlyProviderBundle(t *testing.T) { - _, pool := testStore(t) - cipher, err := credentialcrypto.New(bytes.Repeat([]byte{33}, 32)) - if err != nil { - t.Fatal(err) - } - st := NewWithCredentialCipher(pool, cipher) + st, _ := configuredStore(t) + pool := st.pool ctx, tenant := t.Context(), uuid.NewString() provider := &v1.ModelProviderInput{Protocol: "responses", BaseURL: "https://example.com/v1", APIKey: "provider-key-canary"} input := sessions.CreateSession{Creator: FixtureCreator(), Engine: "codex", IdempotencyKey: uuid.NewString(), Configuration: []byte(`{"agent":{"model":"actual-model"},"environment":{"type":"openai_hosted"}}`), ModelProvider: provider} @@ -40,14 +34,14 @@ func TestSessionModelExecutionStoresOnlyProviderBundle(t *testing.T) { if err != nil { t.Fatal(err) } - ciphertext, err := cipher.SealModelExecution(historical, tenant, session.ID) + ciphertext, err := st.credentialCipher.SealModelExecution(historical, tenant, session.ID) if err != nil { t.Fatal(err) } if _, err := pool.Exec(ctx, "UPDATE session_model_execution SET encrypted_config=$2 WHERE session_id=$1", session.ID, ciphertext); err != nil { t.Fatal(err) } - if _, err := sessionAdapter(NewWithCredentialCipher(pool, cipher)).SessionModelExecution(ctx, tenant, session.ID); err == nil { + if _, err := sessionAdapter(st).SessionModelExecution(ctx, tenant, session.ID); err == nil { t.Fatal("retired native options accepted in provider bundle") } } diff --git a/services/core/tests/integration/session_reference_retry_public_test.go b/services/core/tests/integration/session_reference_retry_public_test.go index e3b066edf..6570098f2 100644 --- a/services/core/tests/integration/session_reference_retry_public_test.go +++ b/services/core/tests/integration/session_reference_retry_public_test.go @@ -12,6 +12,7 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/execution" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimegateway" "github.com/google/uuid" ) @@ -23,7 +24,7 @@ func TestSavedReferenceRetryOfficialClient(t *testing.T) { s, _ := testStore(t) tenant, token, foreign := uuid.NewString(), uuid.NewString(), uuid.NewString() auth := newTestAuthenticator(t, []testAPIKey{{OrganizationID: "test-org", ProjectID: uuid.NewString(), SubjectKind: "service_account", SubjectID: "test-runner", TokenSHA256: runtimedevice.HashCredential(token), TenantID: tenant}, {OrganizationID: "test-org", ProjectID: uuid.NewString(), SubjectKind: "service_account", SubjectID: "test-runner", TokenSHA256: runtimedevice.HashCredential(foreign), TenantID: uuid.NewString()}}) - worker := startWorker(t, t.Context(), s, &execution.Dispatcher{}) + worker := startWorker(t, t.Context(), s, &execution.Dispatcher{Registry: runtimegateway.NewRegistry()}) t.Cleanup(func() { ctx, cancel := context.WithCancel(context.Background()) cancel() diff --git a/services/core/tests/integration/skill_selectors_public_test.go b/services/core/tests/integration/skill_selectors_public_test.go index 7052c01b2..34122433b 100644 --- a/services/core/tests/integration/skill_selectors_public_test.go +++ b/services/core/tests/integration/skill_selectors_public_test.go @@ -10,7 +10,6 @@ import ( "testing" "time" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" "github.com/google/uuid" ) @@ -20,12 +19,7 @@ func TestSkillSelectorsOfficialClientPostgres(t *testing.T) { if python == "" { t.Skip("pinned official Python SDK required") } - _, pool := testStore(t) - cipher, err := credentialcrypto.New(bytes.Repeat([]byte{63}, 32)) - if err != nil { - t.Fatal(err) - } - s := NewWithCredentialCipher(pool, cipher) + s, pool := testStore(t) token, foreign := uuid.NewString(), uuid.NewString() auth := newTestAuthenticator(t, []testAPIKey{ {OrganizationID: "test-org", ProjectID: uuid.NewString(), SubjectKind: "service_account", SubjectID: "test-runner", TokenSHA256: runtimedevice.HashCredential(token), TenantID: uuid.NewString()}, @@ -37,7 +31,7 @@ func TestSkillSelectorsOfficialClientPostgres(t *testing.T) { } server := httptest.NewServer(h) defer server.Close() - recoveredStore := NewWithCredentialCipher(pool, cipher) + recoveredStore := New(t, pool) h, err = publicHandler(t, recoveredStore, auth, "codex") if err != nil { t.Fatal(err) diff --git a/services/core/tests/integration/skill_version_deletion_public_test.go b/services/core/tests/integration/skill_version_deletion_public_test.go index 82ea71dce..817547497 100644 --- a/services/core/tests/integration/skill_version_deletion_public_test.go +++ b/services/core/tests/integration/skill_version_deletion_public_test.go @@ -1,7 +1,6 @@ package integration import ( - "bytes" "encoding/json" "net/http" "net/http/httptest" @@ -9,7 +8,6 @@ import ( "strings" "testing" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/skills" "github.com/google/uuid" @@ -20,12 +18,7 @@ import ( // other versions remain (V2), nondefault latest deletion (V3) and tenant // isolation (V4). func TestSkillVersionDeletionHTTPPostgres(t *testing.T) { - _, pool := testStore(t) - cipher, err := credentialcrypto.New(bytes.Repeat([]byte{64}, 32)) - if err != nil { - t.Fatal(err) - } - s := NewWithCredentialCipher(pool, cipher) + s, pool := testStore(t) owner, foreign, ownerTenant := uuid.NewString(), uuid.NewString(), uuid.NewString() auth := newTestAuthenticator(t, []testAPIKey{ {OrganizationID: "test-org", ProjectID: uuid.NewString(), SubjectKind: "service_account", SubjectID: "skill-owner", TokenSHA256: runtimedevice.HashCredential(owner), TenantID: ownerTenant}, @@ -56,7 +49,7 @@ func TestSkillVersionDeletionHTTPPostgres(t *testing.T) { } missing := expect(owner, http.MethodDelete, "/v1/skills/skill_"+uuid.NewString()+"/versions/1", http.StatusNotFound) - skillService := SkillService(t, pool, cipher) + skillService := SkillService(t, pool, s.credentialCipher) sole, err := skillService.CreateSkill(t.Context(), skills.CreateSkill{TenantID: ownerTenant, Archive: skillArchive(t, "sole-http")}) if err != nil { t.Fatal(err) diff --git a/services/core/tests/integration/skill_version_deletion_test.go b/services/core/tests/integration/skill_version_deletion_test.go index 3a9b2e5a2..be27ff14d 100644 --- a/services/core/tests/integration/skill_version_deletion_test.go +++ b/services/core/tests/integration/skill_version_deletion_test.go @@ -6,7 +6,6 @@ import ( "reflect" "testing" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/environmentconfig" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/skills" @@ -16,13 +15,9 @@ import ( // Deleting a Skill's sole version deletes the Skill, but committed Session // snapshots and their idempotent retries are unchanged. func TestSoleSkillVersionDeletionKeepsFrozenSetup(t *testing.T) { - _, pool := testStore(t) - cipher, err := credentialcrypto.New(bytes.Repeat([]byte{63}, 32)) - if err != nil { - t.Fatal(err) - } - s := NewWithCredentialCipher(pool, cipher) - skillService := SkillService(t, pool, cipher) + s, _ := configuredStore(t) + pool := s.pool + skillService := SkillService(t, pool, s.credentialCipher) tenant := uuid.NewString() archive := skillArchive(t, "sole-version-frozen") skill, err := skillService.CreateSkill(t.Context(), skills.CreateSkill{TenantID: tenant, Archive: archive}) diff --git a/services/core/tests/integration/skills_public_test.go b/services/core/tests/integration/skills_public_test.go index 4aa57368b..63ee989ea 100644 --- a/services/core/tests/integration/skills_public_test.go +++ b/services/core/tests/integration/skills_public_test.go @@ -10,7 +10,6 @@ import ( "testing" "time" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" "github.com/google/uuid" ) @@ -20,12 +19,7 @@ func TestSkillsOfficialClientPostgres(t *testing.T) { if python == "" { t.Skip("pinned official Python SDK required") } - _, pool := testStore(t) - cipher, err := credentialcrypto.New(bytes.Repeat([]byte{51}, 32)) - if err != nil { - t.Fatal(err) - } - s := NewWithCredentialCipher(pool, cipher) + s, pool := testStore(t) token, foreign := uuid.NewString(), uuid.NewString() auth := newTestAuthenticator(t, []testAPIKey{ {OrganizationID: "test-org", ProjectID: uuid.NewString(), SubjectKind: "service_account", SubjectID: "test-runner", TokenSHA256: runtimedevice.HashCredential(token), TenantID: uuid.NewString()}, @@ -37,7 +31,7 @@ func TestSkillsOfficialClientPostgres(t *testing.T) { } server := httptest.NewServer(h) defer server.Close() - recoveredStore := NewWithCredentialCipher(pool, cipher) + recoveredStore := New(t, pool) h, err = publicHandler(t, recoveredStore, auth, "codex") if err != nil { t.Fatal(err) diff --git a/services/core/tests/integration/template_composition_public_test.go b/services/core/tests/integration/template_composition_public_test.go index 1a483d6fb..775744128 100644 --- a/services/core/tests/integration/template_composition_public_test.go +++ b/services/core/tests/integration/template_composition_public_test.go @@ -13,7 +13,6 @@ import ( "time" v1 "github.com/MiniMax-AI/OpenAgentCore/contracts/agents-api/v1" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/environmentconfig" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" @@ -25,12 +24,9 @@ func TestTemplateCompositionOfficialClientPostgres(t *testing.T) { if python == "" { t.Skip("pinned official Python SDK required") } - _, pool := testStore(t) - cipher, err := credentialcrypto.New(bytes.Repeat([]byte{84}, 32)) - if err != nil { - t.Fatal(err) - } - s, reopenedStore := NewWithCredentialCipher(pool, cipher), NewWithCredentialCipher(pool, cipher) + s, _ := configuredStore(t) + pool := s.pool + reopenedStore := New(t, pool) tenant, foreignTenant, token, foreign := uuid.NewString(), uuid.NewString(), uuid.NewString(), uuid.NewString() auth := newTestAuthenticator(t, []testAPIKey{ {OrganizationID: "test-org", ProjectID: uuid.NewString(), SubjectKind: "service_account", SubjectID: "composition-owner", TokenSHA256: runtimedevice.HashCredential(token), TenantID: tenant}, diff --git a/services/core/tests/integration/template_null_selection_public_test.go b/services/core/tests/integration/template_null_selection_public_test.go index f35cfecfd..d0a5b247a 100644 --- a/services/core/tests/integration/template_null_selection_public_test.go +++ b/services/core/tests/integration/template_null_selection_public_test.go @@ -15,7 +15,6 @@ import ( "time" "github.com/MiniMax-AI/OpenAgentCore/internal/agentplugin" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/environmentconfig" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" @@ -27,12 +26,9 @@ func TestTemplateNullSelectionOfficialClientPostgres(t *testing.T) { if python == "" { t.Skip("pinned official Python SDK required") } - _, pool := testStore(t) - cipher, err := credentialcrypto.New(bytes.Repeat([]byte{87}, 32)) - if err != nil { - t.Fatal(err) - } - s, reopenedStore := NewWithCredentialCipher(pool, cipher), NewWithCredentialCipher(pool, cipher) + s, _ := configuredStore(t) + pool := s.pool + reopenedStore := New(t, pool) tenant, foreignTenant, token, foreign := uuid.NewString(), uuid.NewString(), uuid.NewString(), uuid.NewString() auth := newTestAuthenticator(t, []testAPIKey{ {OrganizationID: "test-org", ProjectID: uuid.NewString(), SubjectKind: "service_account", SubjectID: "selection-owner", TokenSHA256: runtimedevice.HashCredential(token), TenantID: tenant}, diff --git a/services/core/tests/integration/unified_model_configuration_http_test.go b/services/core/tests/integration/unified_model_configuration_http_test.go index acc25954d..70d5dd009 100644 --- a/services/core/tests/integration/unified_model_configuration_http_test.go +++ b/services/core/tests/integration/unified_model_configuration_http_test.go @@ -14,7 +14,7 @@ import ( ) func TestUnifiedModelConfigurationHTTP(t *testing.T) { - st, _ := newManagedTestStore(t) + st, _ := configuredStore(t) tenant, token, coreKey := uuid.NewString(), uuid.NewString(), uuid.NewString() auth := newTestAuthenticator(t, []testAPIKey{{OrganizationID: "test-org", ProjectID: uuid.NewString(), SubjectKind: "service_account", SubjectID: "model-configuration", TokenSHA256: runtimedevice.HashCredential(token), TenantID: tenant}}) admin, err := api.NewDeploymentAuthenticator([]string{runtimedevice.HashCredential(coreKey)}) diff --git a/services/core/tests/integration/unstorable_text_public_test.go b/services/core/tests/integration/unstorable_text_public_test.go index e98651f8a..debc28c0c 100644 --- a/services/core/tests/integration/unstorable_text_public_test.go +++ b/services/core/tests/integration/unstorable_text_public_test.go @@ -10,7 +10,6 @@ import ( "strings" "testing" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" "github.com/google/uuid" ) @@ -19,12 +18,7 @@ import ( // containing it must be rejected as a client error before anything is written. func TestUnstorableTextRejectsWithoutWritesPostgres(t *testing.T) { // An isolated database keeps the no-write digest independent of other tests. - _, pool := newManagedTestStore(t) - cipher, err := credentialcrypto.New(bytes.Repeat([]byte{62}, 32)) - if err != nil { - t.Fatal(err) - } - s := NewWithCredentialCipher(pool, cipher) + s, pool := newManagedTestStore(t) token := uuid.NewString() auth := newTestAuthenticator(t, []testAPIKey{{OrganizationID: "test-org", ProjectID: uuid.NewString(), SubjectKind: "service_account", SubjectID: "nul-owner", TokenSHA256: runtimedevice.HashCredential(token), TenantID: uuid.NewString()}}) h, err := publicHandler(t, s, auth, "codex") diff --git a/services/core/tests/integration/worker_fixture_test.go b/services/core/tests/integration/worker_fixture_test.go index 790749499..590936407 100644 --- a/services/core/tests/integration/worker_fixture_test.go +++ b/services/core/tests/integration/worker_fixture_test.go @@ -3,6 +3,8 @@ package integration import ( "context" "errors" + "os" + "strings" "testing" "time" @@ -24,10 +26,11 @@ import ( // bearer tokens through the vaults service on s, records model configuration // observations through the model configuration adapter on s, runs Session use // cases and reads through the Session service and adapter on s, and reads the -// deployment through the deployment adapter on s. +// deployment through the deployment adapter on s. Without the dispatcher's +// sandbox runtimes it runs fixtureRuntimes. func startWorker(t testing.TB, ctx context.Context, s *Store, dispatcher *execution.Dispatcher) *execution.Worker { t.Helper() - worker, err := startWorkerErr(ctx, s, dispatcher) + worker, err := startWorkerErr(t, ctx, s, dispatcher) if err != nil { t.Fatal(err) } @@ -35,7 +38,7 @@ func startWorker(t testing.TB, ctx context.Context, s *Store, dispatcher *execut } // startWorkerErr is startWorker for tests that assert a startup failure. -func startWorkerErr(ctx context.Context, s *Store, dispatcher *execution.Dispatcher) (*execution.Worker, error) { +func startWorkerErr(t testing.TB, ctx context.Context, s *Store, dispatcher *execution.Dispatcher) (*execution.Worker, error) { lease, err := pgunit.AcquireLease(ctx, s.pool) if err != nil { return nil, err @@ -44,7 +47,7 @@ func startWorkerErr(ctx context.Context, s *Store, dispatcher *execution.Dispatc if err != nil { return nil, errors.Join(err, lease.Close(ctx)) } - return startOwnedWorkerErr(ctx, s, dispatcher, owner) + return startOwnedWorkerErr(t, ctx, s, dispatcher, owner) } // startOwnedWorker is startWorker on an Owner the test already holds, for tests @@ -52,14 +55,14 @@ func startWorkerErr(ctx context.Context, s *Store, dispatcher *execution.Dispatc // Run exits. func startOwnedWorker(t testing.TB, ctx context.Context, s *Store, dispatcher *execution.Dispatcher, owner execution.Owner) *execution.Worker { t.Helper() - worker, err := startOwnedWorkerErr(ctx, s, dispatcher, owner) + worker, err := startOwnedWorkerErr(t, ctx, s, dispatcher, owner) if err != nil { t.Fatal(err) } return worker } -func startOwnedWorkerErr(ctx context.Context, s *Store, dispatcher *execution.Dispatcher, owner execution.Owner) (*execution.Worker, error) { +func startOwnedWorkerErr(t testing.TB, ctx context.Context, s *Store, dispatcher *execution.Dispatcher, owner execution.Owner) (*execution.Worker, error) { _, credentials, err := fixtureVaults(s) if err != nil { return nil, errors.Join(err, owner.Lease.Close(ctx)) @@ -79,9 +82,40 @@ func startOwnedWorkerErr(ctx context.Context, s *Store, dispatcher *execution.Di owned.DeploymentReader = deploymentStore(s) owned.Sessions = service owned.SessionsReader = sessionAdapter(s) + if owned.ManagedRuntimes == nil { + owned.ManagedRuntimes = fixtureRuntimes(t, s) + } return execution.StartWorker(ctx, &owned, owner) } +// testInstallation is the installation that owns the shared test database. +// The official-client acceptance runs Core as the same installation on that +// database, so both read it from tests/testdata/installation.id. +func testInstallation(t testing.TB) string { + t.Helper() + value, err := os.ReadFile("../testdata/installation.id") + if err != nil { + t.Fatal(err) + } + return strings.TrimSpace(string(value)) +} + +// fixtureRuntimes is webRuntimes on a lifecycleProvider for the installation +// that claimed s's database, or for testInstallation before one did. On an +// unconfigured deployment it never loads a provider. +func fixtureRuntimes(t testing.TB, s *Store) *execution.RuntimeProvider { + t.Helper() + view, err := deploymentService(t, s).View(context.Background()) + if err != nil { + t.Fatal(err) + } + installation := view.InstallationID + if installation == "" { + installation = testInstallation(t) + } + return webRuntimes(t, s, installation, &lifecycleProvider{resources: map[string]sandbox.Info{}}, nil) +} + // executionOwner acquires the execution lease on s's database and builds the // execution operations on it, for tests that run them without a Worker. The // lease closes when the test ends. @@ -171,7 +205,7 @@ func unusedPreparation(t testing.TB) execution.RuntimeDeploymentPreparer { // startWebWorker starts the Worker on webRuntimes. func startWebWorker(t *testing.T, s *Store, registry *runtimegateway.Registry, installation string, p sandbox.SandboxProvider, suspension *execution.RuntimeSuspensionPolicy) *execution.Worker { t.Helper() - w, err := startNextWorker(t.Context(), s, &execution.Dispatcher{Registry: registry, ManagedRuntimes: webRuntimes(t, s, installation, p, suspension)}) + w, err := startNextWorker(t, t.Context(), s, &execution.Dispatcher{Registry: registry, ManagedRuntimes: webRuntimes(t, s, installation, p, suspension)}) if err != nil { t.Fatal(err) } @@ -181,9 +215,9 @@ func startWebWorker(t *testing.T, s *Store, registry *runtimegateway.Registry, i // startNextWorker is startWorkerErr after another owner closed its lease. A // closed lease stays held until PostgreSQL ends its backend, so startup // retries ErrLeaseHeld briefly. -func startNextWorker(ctx context.Context, s *Store, dispatcher *execution.Dispatcher) (*execution.Worker, error) { +func startNextWorker(t testing.TB, ctx context.Context, s *Store, dispatcher *execution.Dispatcher) (*execution.Worker, error) { for deadline := time.Now().Add(2 * time.Second); ; time.Sleep(20 * time.Millisecond) { - w, err := startWorkerErr(ctx, s, dispatcher) + w, err := startWorkerErr(t, ctx, s, dispatcher) if !errors.Is(err, pgunit.ErrLeaseHeld) || time.Now().After(deadline) { return w, err } diff --git a/services/core/tests/official_client.py b/services/core/tests/official_client.py index 2b9bfe0a4..f066072ee 100644 --- a/services/core/tests/official_client.py +++ b/services/core/tests/official_client.py @@ -82,10 +82,8 @@ def project_bindings(directory): credential_key.touch(mode=0o600) credential_key_values = [base64.b64encode(secrets.token_bytes(32)).decode()] credential_key.write_text(credential_key_values[0] + "\n") - # The database records the first installation ID it sees, so every run uses this one. - installation_id = Path(directory) / "installation.id" - installation_id.touch(mode=0o600) - installation_id.write_text("3f8e2c71-5b0d-4e6a-9c47-1d2a8b6f0e53\n") + # The test database belongs to the test installation, which the Go Worker fixtures also run as. + installation_id = Path(__file__).resolve().parent / "testdata/installation.id" env = dict(os.environ, OAC_DATABASE_URL=dsn, OAC_CORE_KEY_DIGESTS_FILE=str(core_key_digests), OAC_ADDR=f"127.0.0.1:{port}", OAC_DEFAULT_HARNESS="codex") env["OAC_CREDENTIAL_KEY_FILE"] = str(credential_key) env["OAC_INSTALLATION_ID_FILE"] = str(installation_id) diff --git a/services/core/tests/testdata/installation.id b/services/core/tests/testdata/installation.id new file mode 100644 index 000000000..76b1e9a1d --- /dev/null +++ b/services/core/tests/testdata/installation.id @@ -0,0 +1 @@ +3f8e2c71-5b0d-4e6a-9c47-1d2a8b6f0e53