diff --git a/.github/workflows/klaude-candidate.yml b/.github/workflows/klaude-candidate.yml index 82a9ce10e..15b602020 100644 --- a/.github/workflows/klaude-candidate.yml +++ b/.github/workflows/klaude-candidate.yml @@ -25,6 +25,7 @@ jobs: env: # Repair attempts after the initial update; the baseline is already published. MAX_REPAIRS: '3' + KLAUDE_BRANCH: klaude/auto-${{ inputs.candidate }} KLAUDE_TEST_NAME: klaud-${{ github.run_id }}-${{ inputs.candidate }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 @@ -41,11 +42,14 @@ jobs: - name: Prepare candidate context env: CANDIDATE: ${{ inputs.candidate }} + GH_TOKEN: ${{ github.token }} run: | [[ "$CANDIDATE" =~ ^[0-9a-f]{16}-[0-9a-f]{16}$ ]] cp -R "$RUNNER_TEMP/plan/$CANDIDATE" "$RUNNER_TEMP/klaude" rm -r "$RUNNER_TEMP/plan" - echo "KLAUDE_STARTED_AT=$(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "$GITHUB_ENV" + started_at="$(gh api --method GET "repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" --jq .created_at)" + [[ "$started_at" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]] + echo "KLAUDE_STARTED_AT=$started_at" >> "$GITHUB_ENV" - name: Let Klaud Cold own the candidate PR id: claude uses: anthropics/claude-code-action@d75b94d5ad426cb8546e6628b6f5f19b84e5cce1 @@ -88,7 +92,8 @@ jobs: Use configs/runners.yaml and the matrix generator for routes, all points and physical node demand. Verify the reviewed telemetry-clusters cover every actual target; never substitute a healthy sibling cluster. Run the capacity check below BEFORE edits or - branch/PR creation as well as before each dispatch. Unknown routes mean defer and stop. + branch/PR creation, each targeted dispatch and the final-sweep ready/label transition. + Unknown routes mean defer and stop. Verify upstream releases/registry for CUDA/ROCm/hardware compatibility; pin mutable tags by digest. Recheck open PRs immediately before claiming the branch; the context's pr-review is a snapshot. Before GPU use, atomically claim the context's exact @@ -102,17 +107,22 @@ jobs: The pinned image must run as shipped: no engine/serving-stack patches, source or container-file rewrites, site-packages edits, monkey-patching, or forked/rebuilt engine wheels. If one is required, reject the image as incompatible. - Keep the PR draft. After a targeted attempt passes, append one valid perf-changelog.yaml - entry at the physical tail with the PR URL; preserve all prior bytes and validate it. + Keep the PR draft through targeted attempts. After one passes, append one valid + perf-changelog.yaml entry at the physical tail with the PR URL; preserve all prior + bytes and validate it. Use uv and applicable InferenceX checks. Check every exact telemetry cluster from the current recipe/runner configuration with repeated --cluster arguments: uv run --no-project --python 3.12 --with "pydantic>=2.10,<3" python -m utils.klaude check-capacity --cluster TELEMETRY_CLUSTER_ID Exit 0 requires current telemetry, available nodes and node utilization below 20% for ALL targets. Any other result means defer and stop; do not wait for recovery or promise - automatic continuation. If capacity is the stop reason, cancel and confirm owned runs, - close any draft PR and delete its remote Klaud branch so later sweeps can retry it. - Create no placeholder PR. Never print private API responses. + automatic continuation. If this check fails before a targeted dispatch, final-sweep + transition or recovery dispatch, first record a public-safe capacity deferral and current + attempt state in any existing PR. Cancel and confirm all owned runs, then update its table + with terminal/cancelled rows and confirmed states. Finally remove any sweep label, return + the PR to draft, close it and delete its remote Klaud branch so later sweeps can retry it. + Without a PR, report the deferral in your final response; create no placeholder. Never + cancel healthy work merely because utilization rises after dispatch. Never print private API responses. Dispatch e2e-tests.yml from main with inputs.ref=exact measured SHA, generate-cli-command="test-config --config-files FILE --config-keys FAMILY", fail-fast=true. @@ -147,11 +157,14 @@ jobs: healthy work merely to finish the session. The read-only Stop hook checks owned run status; it does not handle repairs or reporting for you. Do not disable it or alter its run marker. - After a targeted attempt succeeds, commit/push the changelog entry and apply - full-sweep-enabled as the sole sweep-related label. Wait for its run-sweep.yml run - on the exact PR head to finish successfully; - this reusable PR sweep is the final validation. If it exposes a candidate failure, repair within - the remaining budget and repeat. Do not stage results, post /reuse-sweep-run, request review or merge. + After a targeted attempt succeeds, commit/push the changelog entry, mark the PR ready + with gh pr ready (this does not request review), and apply full-sweep-enabled as the sole + sweep-related label. Wait for its run-sweep.yml run on the exact PR head to finish + successfully with reusable artifacts; this is the final validation. If it fails, remove + full-sweep-enabled and return the PR to draft before any repair push, then repair within + the remaining budget and repeat. If the failure is capacity-related, run the capacity + check before recovery; use the complete deferral cleanup above only if that check fails. + Do not stage results, post /reuse-sweep-run, request review or merge. PR and final report: English/Simplified Chinese Markdown table, one row for the baseline (label its published date), every update/repair attempt and the final full sweep, including failures/cancellations. Include attempt, diff --git a/KLAUD_DEBUG.md b/KLAUD_DEBUG.md index cc209165a..5f3ed30b2 100644 --- a/KLAUD_DEBUG.md +++ b/KLAUD_DEBUG.md @@ -215,9 +215,26 @@ between the PR sweep and merge therefore does not require another GPU sweep. ### 7.2 Capacity deferrals must release the candidate claim The planner ignores closed PRs but treats every matching `klaude/auto-*` branch -as occupied. When capacity is the stop reason, cancel and confirm owned runs, -close the draft PR, and delete its remote Klaud branch so a later sweep can -select the candidate again. Closing the PR alone does not make it eligible. +as occupied. If the capacity check fails before a targeted dispatch, the final +sweep transition or a recovery dispatch, first record a public-safe deferral and +current attempt state in any existing PR. Cancel and confirm all owned runs, +update the report with their terminal states, then remove sweep labels, return +the PR to draft, close it, and delete its remote Klaud branch so a later sweep +can select the candidate again. Without a PR, report the deferral in the agent's +final response. A utilization increase after dispatch does not cancel healthy +work. Closing the PR alone does not make the candidate eligible. + +### 7.3 Final reusable sweeps require a ready PR + +`run-sweep.yml` skips PR jobs while the PR is a draft. After targeted validation, +append the changelog entry, mark the PR ready, then apply `full-sweep-enabled`. +If that sweep fails, remove the label and return the PR to draft before pushing +a repair, or each intermediate push starts another full sweep. The Klaud Stop +hook tracks a labeled final sweep by candidate branch and exact head SHA and +requires a successful run with reusable artifacts. It ignores completed +all-skipped runs from unrelated label events on that same SHA. The lookup window +starts at the parent auto-sweep's original creation time so a candidate-job rerun +still sees targeted and final sweeps created by its earlier attempt. --- diff --git a/docs/klaude.md b/docs/klaude.md index debb8ea2f..5d396670e 100644 --- a/docs/klaude.md +++ b/docs/klaude.md @@ -31,9 +31,9 @@ The `klaude-plan` artifact explicitly includes only `candidates.json`, `open-prs After checkout and context preparation, the candidate workflow hands control to Klaud Cold with `CLAUDE_PAT`, `ANTHROPIC_API_KEY` and the private API read key. Klaud Cold resolves the public observation to one live master-config family, checks the current image and existing PRs, and verifies the reviewed target IDs and their live capacity **before edits or branch/PR creation**. It then claims its branch before spending GPU time, makes a real change and opens a draft PR. Every generated PR title starts with `[Klaud Cold] ` followed by an English / Simplified Chinese description. It never @mentions GitHub users/teams or requests reviews/re-reviews; those are automatic. It rechecks open PRs immediately before claiming the branch because review is a snapshot, not a lock against new human PRs. Ambiguous, retired or already updated candidates stop without a sweep. It owns commits, pushes, dispatches, monitoring, diagnosis, repairs and bilingual PR updates within the same session. -The prompt asks Klaud Cold to measure only updated-image attempts with existing `e2e-tests.yml` on `main`, passing the exact measured SHA through `inputs.ref` and the complete generated `test-config` family command through `generate-cli-command`. It reads current `configs/*-master.yaml`, `configs/runners.yaml` and the existing matrix-generator CLI instead of maintaining another recipe catalog. Default evals, all recipe points, physical `nodes:N` labels, MTP chat templates and artifact contracts stay intact. Targeted repair attempts use no sweep label. Once a targeted attempt passes, Klaud appends the required `perf-changelog.yaml` entry at the physical tail with the PR URL, preserving all prior bytes, and applies `full-sweep-enabled` as the sole sweep-related label. It waits for the resulting `run-sweep.yml` run on the exact PR head to succeed; this final PR sweep produces artifacts eligible for staging and reuse. Klaud does not stage them, authorize reuse, request review or merge. If the full sweep exposes a candidate failure, it repairs within the remaining budget and repeats the final sweep. +The prompt asks Klaud Cold to measure only updated-image attempts with existing `e2e-tests.yml` on `main`, passing the exact measured SHA through `inputs.ref` and the complete generated `test-config` family command through `generate-cli-command`. It reads current `configs/*-master.yaml`, `configs/runners.yaml` and the existing matrix-generator CLI instead of maintaining another recipe catalog. Default evals, all recipe points, physical `nodes:N` labels, MTP chat templates and artifact contracts stay intact. Targeted repair attempts use no sweep label and keep the PR draft. Once one passes, Klaud appends the required `perf-changelog.yaml` entry at the physical tail with the PR URL, preserving all prior bytes, marks the PR ready without requesting review and applies `full-sweep-enabled` as the sole sweep-related label. It waits for the resulting `run-sweep.yml` run on the exact PR head to succeed with reusable artifacts. If the final sweep fails, Klaud removes the label and returns the PR to draft before any repair push, then repairs within the remaining budget and repeats. Klaud does not stage results, authorize reuse, request review or merge. -Before every dispatch, `check-capacity --cluster ID` checks the exact targets; repeat `--cluster` for every possible target. Exit 0 requires all targets to pass freshness, availability and below-20% utilization checks. If capacity is the stop reason, Klaud cancels and confirms its owned runs, closes any draft PR and deletes its remote Klaud branch so a later sweep can retry the candidate. It does not wait for recovery, promise automatic continuation or create a placeholder PR. The command prints no capacity details. +Before edits or branch creation, before every targeted dispatch and immediately before the final-sweep ready/label transition, `check-capacity --cluster ID` checks the exact targets; repeat `--cluster` for every possible target. Exit 0 requires all targets to pass freshness, availability and below-20% utilization checks. If that check fails before a targeted dispatch, the final-sweep transition or a recovery dispatch, Klaud first records a public-safe capacity deferral and current attempt state in any existing PR. It cancels and confirms all owned runs, then updates the table with terminal or cancelled rows and confirmed states. Finally it removes any sweep label, returns the PR to draft, closes it and deletes its remote branch so a later sweep can retry the candidate. Without an existing PR, it records the deferral in its final response and creates no placeholder. A utilization increase after dispatch does not cancel healthy running work. Klaud does not wait for recovery or promise automatic continuation. The command prints no capacity details. Klaud Cold reads the run artifacts and logs, calculates matched performance deltas and publishes evidence in its PR. A working updated image with passing selected benchmarks and default evals is a successful outcome; performance improvements are best effort, with regressions reported rather than rejected by a percentage threshold. The PR and final report must include a bilingual Markdown summary table with a dated published-baseline row and every update or repair attempt: attempt, image/SHA, run URLs, benchmark/eval results, matched throughput and latency deltas against the baseline, and diagnosis. Label per-point deltas clearly and use `N/A` with a reason for missing or incomparable measurements, including failed or cancelled attempts. Exclude invalid, duplicate and unmatched points from improvement claims and disclose limitations. A successful selected benchmark is separate from complete performance evidence and global PR approval. The public API investigation guide below directs additional reads as needed. Published public-dashboard data provide the baseline; only updated-image attempts consume GPU time. Empty aggregates and green collection jobs do not prove benchmark/eval success. Raw benchmark artifacts remain on their e2e runs. Only sanitized candidate diagnostics are uploaded afterward; arbitrary scratch files and execution transcripts are excluded. @@ -51,7 +51,7 @@ The baseline comes from the **public dashboard API at `https://inferencex.semian Dispatching a run or opening a draft is not completion. Stay in the same session with `gh run watch --interval 60`, resuming after tool timeouts, and inspect job-level state because queued workflows can contain running jobs. A failed benchmark matrix may leave eval jobs active. Diagnose the first server error rather than teardown symptoms; repair within the existing scope, budget and capacity rule. A tool denial calls for an allowed tool or command, not early success. Finish the PR table with terminal attempt results, then report the stop reason, repair count, confirmed child completion and PR URL. Never end with a promise to monitor later or cancel healthy work just to stop. -The [Claude Code Stop hook](https://code.claude.com/docs/en/hooks#stop) runs `python -m utils.klaude check-stop`. It reads every page of `e2e-tests.yml` dispatches since candidate preparation and matches `e2e Test - $KLAUDE_TEST_NAME`. Each attempt must use the exact unchanged marker `klaud--`. Unfinished matches or unavailable/incomplete listings block a normal stop and tell the same agent to continue. Other candidates' runs are ignored. No runs or all-terminal runs allow stopping. The hook performs no dispatch, cancellation, repair or model call. GitHub job limits, Claude's built-in Stop-hook loop cap, API errors, interrupts and abrupt termination can still prevent completion; this is not an external supervisor. No custom timeout or continuation budget is added. +The [Claude Code Stop hook](https://code.claude.com/docs/en/hooks#stop) runs `python -m utils.klaude check-stop`. Its lookup window starts at the parent auto-sweep's original creation time, which remains stable across candidate-job reruns. It reads every page of matching `e2e-tests.yml` dispatches and matches `e2e Test - $KLAUDE_TEST_NAME`; each attempt must use the exact unchanged marker `klaud--`. Unfinished matches or unavailable/incomplete listings block a normal stop and tell the same agent to continue. Once the candidate's open PR has `full-sweep-enabled`, the hook also follows `run-sweep.yml` by candidate branch and exact head SHA, ignoring completed all-skipped runs caused by unrelated label events. It blocks while the PR is draft, the exact run is absent or unfinished, the run fails, or a successful run lacks reusable benchmark/eval artifacts. Other candidates' runs are ignored. The hook performs no dispatch, cancellation, repair or model call. GitHub job limits, Claude's built-in Stop-hook loop cap, API errors, interrupts and abrupt termination can still prevent completion; this is not an external supervisor. No custom timeout or continuation budget is added. After the action, `diagnostics` reuses the planner's sanitizer to upload only `candidate-diagnostics.json` in `klaude-candidate-`. It retains the action outcome, allowlisted termination subtype, boolean error status, numeric duration/turn/cost metrics and fixed permission-denial categories. Unknown subtypes become `unknown`; missing or unreadable execution files are recorded as unavailable. Raw messages, results, commands, paths, errors, credentials and private telemetry are excluded. These steps use `always()` when the runner is still available; they never update the PR or resume Claude. Planner diagnostics gain the same termination fields. @@ -88,7 +88,7 @@ The app also has useful **unpublished UI readers**: `/api/unofficial-run` normal Klaud Cold is instructed to change only the selected master family's image and its already referenced, unshared srt-slurm recipe YAML images/backend compatibility settings. Model, precision, topology, speculative decoding, workload, commands, resources and recipe references stay fixed. `model.container` and any `identity.container.image` must match the master image. Shared scripts, launchers, libraries, workflow/control files and unrelated families stay unchanged. The pinned image must run as shipped: engine/serving-stack patches, source or container-file rewrites, site-packages edits, monkey-patching and forked/rebuilt wheel overlays are prohibited. If a candidate image requires one, Klaud rejects it as incompatible. Klaud Cold runs focused checks using **uv**; there is no separate patch validator after the agent. -After targeted validation passes, Klaud appends the required changelog entry without altering historical bytes. The candidate remains a draft through the final `full-sweep-enabled` run. Existing checks and human review remain in force; no automatic staging, reuse authorization, review request, merge or policy bypass is authorized. +After targeted validation passes, Klaud appends the required changelog entry without altering historical bytes, rechecks capacity, marks the candidate ready because `run-sweep.yml` skips draft PRs, then applies `full-sweep-enabled`. A failed final sweep returns the PR to draft and removes the label before repairs. For a capacity-related failure, Klaud runs the required pre-recovery capacity check and performs the complete close-and-release cleanup only when that check fails. Existing checks and human review remain in force; no automatic staging, reuse authorization, review request, merge or policy bypass is authorized. ## Workflow operation and credentials diff --git a/docs/klaude_zh.md b/docs/klaude_zh.md index 43409486f..a9adb9e78 100644 --- a/docs/klaude_zh.md +++ b/docs/klaude_zh.md @@ -31,9 +31,9 @@ PR 检查使用 `claude-opus-5`(Opus 5),关闭 fast mode(`fastMode: fals 完成 checkout 和上下文准备后,candidate 工作流将控制权交给 Klaud Cold,并提供 `CLAUDE_PAT`、`ANTHROPIC_API_KEY` 和私有 API 只读密钥。Klaud Cold 将公开观测解析到一个活动主配置族,检查当前镜像和已有 PR,并在**编辑或创建分支/PR 之前**核实检查结果中的目标 ID 和实时容量。随后在使用 GPU 前认领分支,产生实际修改并创建草稿 PR。所有生成的 PR 标题必须以 `[Klaud Cold] ` 开头,后接英文 / 简体中文描述。不得在 GitHub 上 @提及用户/团队,也不得请求 review/re-review;这些操作由自动流程处理。它在认领分支前立即重新检查开放 PR,因为检查只是快照,不能锁住后来创建的人工 PR。有歧义、已退役或已经更新的候选直接停止,不运行扫描。提交、推送、调度、监控、诊断、修复及双语 PR 更新都由同一会话完成。 -提示词要求 Klaud Cold 通过 `main` 上现有的 `e2e-tests.yml` 仅测量更新后的镜像及其修复尝试,将实际测量提交的 SHA 传入 `inputs.ref`,将完整配置族的 `test-config` 命令传入 `generate-cli-command`。它读取当前 `configs/*-master.yaml`、`configs/runners.yaml` 并使用现有矩阵生成器 CLI,不再维护另一份 recipe 目录。保留默认 eval、所有配置测试点、物理 `nodes:N` 标签、MTP chat template 和产物约定。定向修复尝试不添加 sweep 标签。定向尝试通过后,Klaud 会在 `perf-changelog.yaml` 物理末尾追加包含 PR URL 的必要条目,保留此前所有字节,并将 `full-sweep-enabled` 作为唯一与 sweep 相关的标签。它会等待针对该 PR 精确 head 的 `run-sweep.yml` 成功结束;这次最终 PR sweep 会产出可用于 staging 和复用的产物。Klaud 不会自行 staging、授权复用、请求 review 或合并。如果完整 sweep 暴露候选问题,它会在剩余修复预算内修复并重复最终 sweep。 +提示词要求 Klaud Cold 通过 `main` 上现有的 `e2e-tests.yml` 仅测量更新后的镜像及其修复尝试,将实际测量提交的 SHA 传入 `inputs.ref`,将完整配置族的 `test-config` 命令传入 `generate-cli-command`。它读取当前 `configs/*-master.yaml`、`configs/runners.yaml` 并使用现有矩阵生成器 CLI,不再维护另一份 recipe 目录。保留默认 eval、所有配置测试点、物理 `nodes:N` 标签、MTP chat template 和产物约定。定向修复尝试不添加 sweep 标签,并保持 PR 为草稿。定向尝试通过后,Klaud 会在 `perf-changelog.yaml` 物理末尾追加包含 PR URL 的必要条目,保留此前所有字节,将 PR 标记为 ready 但不请求 review,并将 `full-sweep-enabled` 作为唯一与 sweep 相关的标签。它会等待针对该 PR 精确 head 的 `run-sweep.yml` 成功结束并产生可复用产物。如果最终 sweep 失败,Klaud 会在任何修复推送之前移除该标签并将 PR 改回草稿,再在剩余修复预算内修复并重复。Klaud 不会自行 staging、授权复用、请求 review 或合并。 -每次调度前,使用 `check-capacity --cluster ID` 检查精确目标;通过重复 `--cluster` 指定每个可能的目标。退出状态 0 要求全部目标均通过新鲜度、可用性和低于 20% 利用率检查。如果因容量而停止,Klaud 会取消并确认其所属运行已经结束、关闭已有草稿 PR,并删除远程 Klaud 分支,使后续扫描可以重试该候选。它不会等待恢复、承诺自动继续或创建占位 PR。命令不打印容量详情。 +在编辑或创建分支之前、每次定向调度之前,以及最终 sweep 的 ready/标签转换之前,使用 `check-capacity --cluster ID` 检查精确目标;通过重复 `--cluster` 指定每个可能的目标。退出状态 0 要求全部目标均通过新鲜度、可用性和低于 20% 利用率检查。如果该检查在定向调度、最终 sweep 转换或恢复调度之前失败,Klaud 会先在已有 PR 中记录可公开的容量延后原因和当前尝试状态。随后取消并确认全部所属运行已经结束,再用终态或已取消行及确认后的状态更新表格。最后移除所有 sweep 标签、将 PR 改回草稿、关闭 PR,并删除远程 Klaud 分支,使后续扫描可以重试该候选。如果尚无 PR,则在最终响应中记录延后结果,不创建占位 PR。调度后利用率上升不会导致健康运行被取消。Klaud 不会等待恢复或承诺自动继续。命令不打印容量详情。 Klaud Cold 读取运行产物和日志,计算匹配性能差值并在 PR 中发布证据。更新后的镜像能够正常工作、通过所选 benchmark 和默认 eval,即视为成功;性能优化尽力而为,性能回退如实报告,不按百分比阈值拒绝更新。PR 和最终报告必须包含双语 Markdown 汇总表,分别记录带发布日期的已有基线及每次更新或修复尝试:尝试编号、镜像/SHA、运行 URL、benchmark/eval 结果、相对基线的匹配吞吐量和延迟变化,以及诊断结论。明确标注各测试点的差值;对于缺失或不可比较的测量,包括失败或取消的尝试,填写 `N/A` 并说明原因。无效、重复或未匹配的测试点不得用于声称性能提升,并应披露相关限制。所选 benchmark 成功、性能证据完整和全局 PR 审批是不同结论。下方的公开 API 调查指南说明如何按需读取补充信息。公开 dashboard 中已有的数据作为基线;只有新镜像尝试消耗 GPU 时间。空汇总文件和成功的收集作业不能证明 benchmark/eval 成功。原始 benchmark 产物保留在对应 e2e 运行中;结束后仅上传脱敏候选诊断,排除任意临时文件和执行记录。 @@ -51,7 +51,7 @@ Klaud Cold 调度 `e2e-tests.yml` 时显式设置布尔输入 `klaud-run: true` 调度运行或创建草稿不代表任务完成。使用 `gh run watch --interval 60` 留在同一会话中等待,工具超时后继续等待,并检查作业级状态,因为 queued 工作流可能包含正在运行的作业。benchmark 矩阵失败后,eval 作业仍可能继续。定位首个服务端错误而非清理阶段症状;在原有范围、预算和容量规则内修复。工具调用被拒绝时改用允许的工具或命令,不得提前报告成功。先将所有尝试的最终结果写入 PR 表格,再报告停止原因、修复次数、已确认的子运行结束状态和 PR URL。不得承诺稍后继续监控,也不得仅为结束会话而取消正常运行。 -[Claude Code Stop hook](https://code.claude.com/docs/en/hooks#stop) 运行 `python -m utils.klaude check-stop`,读取候选准备时间之后 `e2e-tests.yml` 手动调度的全部分页,并匹配 `e2e Test - $KLAUDE_TEST_NAME`。每次尝试必须使用固定标识 `klaud--`。匹配到未结束的运行,或列表不可用/不完整时,阻止正常停止并要求同一 agent 继续。其他候选的运行不受影响;没有所属运行或全部结束时允许停止。hook 不调度、不取消、不修复,也不调用模型。GitHub 作业时限、Claude 内置 Stop-hook 循环上限、API 错误、中断或异常终止仍可能导致任务未完成;它不是外部监督服务。不增加自定义超时或继续执行预算。 +[Claude Code Stop hook](https://code.claude.com/docs/en/hooks#stop) 运行 `python -m utils.klaude check-stop`。其查询窗口从父 auto-sweep 的原始创建时间开始,该时间在 candidate 作业重跑时保持不变。hook 读取匹配的 `e2e-tests.yml` 手动调度全部分页,并匹配 `e2e Test - $KLAUDE_TEST_NAME`;每次尝试必须使用固定标识 `klaud--`。匹配到未结束的运行,或列表不可用/不完整时,阻止正常停止并要求同一 agent 继续。候选的开放 PR 添加 `full-sweep-enabled` 后,hook 还会按候选分支和精确 head SHA 跟踪 `run-sweep.yml`,并忽略无关标签事件产生、已经结束且所有作业均被跳过的运行。PR 仍为草稿、精确运行不存在或未结束、运行失败,或成功运行缺少可复用 benchmark/eval 产物时,都会阻止停止。其他候选的运行不受影响。hook 不调度、不取消、不修复,也不调用模型。GitHub 作业时限、Claude 内置 Stop-hook 循环上限、API 错误、中断或异常终止仍可能导致任务未完成;它不是外部监督服务。不增加自定义超时或继续执行预算。 action 结束后,`diagnostics` 复用 planner 的脱敏逻辑,仅将 `candidate-diagnostics.json` 上传至 `klaude-candidate-`。保留 action 结果、允许列表中的终止类型、布尔错误状态、数值型耗时/轮数/成本及固定权限拒绝类别。未知终止类型记为 `unknown`;执行文件缺失或不可读时记为不可用。排除原始消息、结果、命令、路径、错误文本、凭据和私有遥测。runner 仍可用时这些步骤通过 `always()` 执行,不更新 PR,也不恢复 Claude。planner 诊断同步增加相同终止字段。 @@ -88,7 +88,7 @@ action 结束后,`diagnostics` 复用 planner 的脱敏逻辑,仅将 `candid Klaud Cold 只应修改所选主配置族的镜像,以及它已经引用且未被其他配置族共享的 srt-slurm recipe YAML 镜像/后端兼容性设置。模型、精度、拓扑、推测解码、工作负载、命令、资源和 recipe 引用保持不变。`model.container` 及存在时的 `identity.container.image` 必须与主配置镜像一致。共享脚本、launcher、库、工作流/控制文件和无关配置族保持不变。固定镜像必须原样运行:禁止对推理引擎或 serving 技术栈打补丁、改写源码或容器文件、就地修改 site-packages、monkey-patch,以及覆盖安装 fork 或重新构建的 wheel。候选镜像若依赖其中任一操作,Klaud 会将其判定为不兼容。Klaud Cold 使用 **uv** 运行针对性检查;agent 之后没有独立补丁校验器。 -定向验证通过后,Klaud 会追加必要的 changelog 条目,同时保持历史字节不变。候选 PR 在最终 `full-sweep-enabled` 运行期间仍保持草稿。现有检查和人工审核仍然有效;没有授权自动 staging、复用、请求 review、合并或绕过规则。 +定向验证通过后,Klaud 会追加必要的 changelog 条目,同时保持历史字节不变,重新检查容量,再将候选标记为 ready 并添加 `full-sweep-enabled`;`run-sweep.yml` 会跳过草稿 PR。最终 sweep 失败时,先移除标签并将 PR 改回草稿,再开始修复。对于容量相关失败,Klaud 会执行必要的恢复前容量检查,并仅在该检查失败时执行完整的关闭与释放清理。现有检查和人工审核仍然有效;没有授权自动 staging、复用、请求 review、合并或绕过规则。 ## 工作流操作与凭据 diff --git a/utils/klaude/__main__.py b/utils/klaude/__main__.py index 440b7a971..6cac1f152 100644 --- a/utils/klaude/__main__.py +++ b/utils/klaude/__main__.py @@ -143,12 +143,12 @@ def execution_diagnostics(path: Path) -> dict: def check_stop() -> dict: - """Read-only Stop hook: keep the same agent alive while its e2e runs are unfinished.""" + """Block while owned e2e work or a labeled final PR sweep is unfinished.""" try: + repository = os.environ['GITHUB_REPOSITORY'] query = urlencode({'event': 'workflow_dispatch', 'per_page': 100, 'created': '>=' + os.environ['KLAUDE_STARTED_AT']}) - pages = github_read(os.environ['GITHUB_REPOSITORY'], - 'actions/workflows/e2e-tests.yml/runs?' + query, paginate=True) + pages = github_read(repository, 'actions/workflows/e2e-tests.yml/runs?' + query, paginate=True) if not isinstance(pages, list) or not pages: raise ValueError('Missing run listing') runs = [run for page in pages for run in page['workflow_runs']] @@ -157,10 +157,51 @@ def check_stop() -> dict: title = 'e2e Test - ' + os.environ['KLAUDE_TEST_NAME'] active = [run for run in runs if run['display_title'] == title and (run['status'] != 'completed' or not run.get('conclusion'))] + if active: + return {'decision': 'block', 'reason': 'Your e2e runs are still queued or running. Continue watching their jobs, inspect results, repair within budget and update the PR table. Do not end with a promise to monitor later or cancel healthy work just to stop. For a valid stop condition, cancel only your unfinished runs and confirm completion.'} + + branch = os.environ['KLAUDE_BRANCH'] + owner = repository.split('/', 1)[0] + query = urlencode({'state': 'open', 'head': owner + ':' + branch, 'per_page': 100}) + pull_pages = github_read(repository, 'pulls?' + query, paginate=True) + pulls = [pull for page in pull_pages for pull in page] + if len(pulls) > 1: + raise ValueError('Multiple candidate pull requests') + if not pulls: + return {} + pull = pulls[0] + labels = {label['name'] for label in pull['labels']} + if 'full-sweep-enabled' not in labels: + return {} + if pull['draft']: + return {'decision': 'block', 'reason': 'The final full-sweep label is on a draft PR, so run-sweep jobs are skipped. Mark it ready with gh pr ready, then continue monitoring without requesting review.'} + + query = urlencode({'event': 'pull_request', 'branch': branch, 'per_page': 100, + 'created': '>=' + os.environ['KLAUDE_STARTED_AT']}) + sweep_pages = github_read(repository, 'actions/workflows/run-sweep.yml/runs?' + query, paginate=True) + sweep_runs = [run for page in sweep_pages for run in page['workflow_runs']] + if any(page['total_count'] > len(sweep_runs) for page in sweep_pages): + raise ValueError('Incomplete sweep run listing') + # Label churn can create an all-skipped run after the real sweep on the + # same SHA. Ignore completed no-op runs so they cannot mask validation. + exact_runs = [run for run in sweep_runs if run['head_sha'] == pull['head']['sha'] + and (run['status'] != 'completed' or run.get('conclusion') != 'skipped')] + if not exact_runs: + return {'decision': 'block', 'reason': 'No final run-sweep.yml run exists for the exact PR head. Keep full-sweep-enabled applied and wait for the labeled run to appear.'} + sweep = max(exact_runs, key=lambda run: run['created_at']) + if sweep['status'] != 'completed' or not sweep.get('conclusion'): + return {'decision': 'block', 'reason': 'The final run-sweep.yml run is still queued or running. Continue monitoring every job and do not stop before it is terminal.'} + if sweep['conclusion'] != 'success': + return {'decision': 'block', 'reason': 'The final run-sweep.yml run did not succeed. Remove full-sweep-enabled and return the PR to draft before any repair push, then diagnose, repair within budget and repeat final validation.'} + artifact_pages = github_read(repository, f'actions/runs/{sweep["id"]}/artifacts?per_page=100', paginate=True) + artifacts = [artifact for page in artifact_pages for artifact in page['artifacts']] + if any(page['total_count'] > len(artifacts) for page in artifact_pages): + raise ValueError('Incomplete artifact listing') + reusable = ('results_bmk', 'eval_results_all', 'bmk_agentic_') + if not any(not artifact['expired'] and artifact['name'].startswith(reusable) for artifact in artifacts): + return {'decision': 'block', 'reason': 'The successful final sweep has no reusable benchmark or eval artifacts. Inspect the run before stopping.'} except (KeyError, TypeError, ValueError, subprocess.SubprocessError, OSError): - return {'decision': 'block', 'reason': 'Cannot verify owned e2e runs. Inspect GitHub, resolve the read failure and finish monitoring/reporting before stopping. Do not dispatch replacements.'} - if active: - return {'decision': 'block', 'reason': 'Your e2e runs are still queued or running. Continue watching their jobs, inspect results, repair within budget and update the PR table. Do not end with a promise to monitor later or cancel healthy work just to stop. For a valid stop condition, cancel only your unfinished runs and confirm completion.'} + return {'decision': 'block', 'reason': 'Cannot verify owned e2e or final-sweep state. Inspect GitHub, resolve the read failure and finish monitoring/reporting before stopping. Do not dispatch replacements.'} return {} @@ -237,7 +278,7 @@ def main() -> int: selection.add_argument('--execution-file', type=Path, help='Claude execution log; retain numeric metrics and fixed denial categories') capacity = commands.add_parser('check-capacity', help='Exit 0 with available nodes below 20%% utilization; otherwise nonzero, without printing telemetry') capacity.add_argument('--cluster', required=True, action='append', help='Exact telemetry cluster; repeat for every possible recipe target') - commands.add_parser('check-stop', help='Claude Stop hook: block completion while this candidate has unfinished e2e runs') + commands.add_parser('check-stop', help='Claude Stop hook: block completion during owned e2e or labeled final-sweep work') diagnostics = commands.add_parser('diagnostics', help='Save sanitized Claude termination metrics and permission categories') diagnostics.add_argument('--execution-file', type=Path, required=True) diagnostics.add_argument('--output', type=Path, required=True)