From da28ede66c8b9888ba62dc61ecd800b6c3c545b4 Mon Sep 17 00:00:00 2001 From: Amp Date: Thu, 23 Jul 2026 02:13:46 +0000 Subject: [PATCH 1/4] Add retrospective orb sizing skill Agent-Signature: amp-gpt-5.6-medium on behalf of maphew Amp-Thread-ID: https://ampcode.com/threads/T-019f8cb6-a6f7-77ab-a6d7-ea71e143dc9d Co-authored-by: Matt Wilkie --- .agents/skills/analyzing-orb-sizing/SKILL.md | 74 ++++ .../analyzing-orb-sizing/scripts/analyze.py | 327 ++++++++++++++++++ .../tests/test_analyze.py | 128 +++++++ 3 files changed, 529 insertions(+) create mode 100644 .agents/skills/analyzing-orb-sizing/SKILL.md create mode 100644 .agents/skills/analyzing-orb-sizing/scripts/analyze.py create mode 100644 .agents/skills/analyzing-orb-sizing/tests/test_analyze.py diff --git a/.agents/skills/analyzing-orb-sizing/SKILL.md b/.agents/skills/analyzing-orb-sizing/SKILL.md new file mode 100644 index 0000000..f58122c --- /dev/null +++ b/.agents/skills/analyzing-orb-sizing/SKILL.md @@ -0,0 +1,74 @@ +--- +name: analyzing-orb-sizing +description: Retrospectively assesses whether an Amp orb was under-sized, over-sized, or appropriately sized from thread tool output and measured workloads. Use when reviewing orb size, resource pressure, OOM failures, build performance, or orb cost efficiency. +compatibility: Requires Python 3.9+ and the Amp CLI when analyzing a thread ID. +argument-hint: --size +--- + +# Analyzing Orb Sizing + +Assess orb sizing from evidence, not repository type alone. Run the bundled analyzer first, then use `read_thread` when semantic context is needed to decide whether observed workloads were representative. + +## Run the analyzer + +From the repository root: + +```bash +python3 .agents/skills/analyzing-orb-sizing/scripts/analyze.py \ + T-xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx --size a0.small +``` + +It also accepts an existing export, which is useful offline: + +```bash +amp threads export T-xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx > /tmp/thread.json +python3 .agents/skills/analyzing-orb-sizing/scripts/analyze.py \ + /tmp/thread.json --size a0.small --json +``` + +Always pass the size used by that thread. Project defaults are prospective and may not match an existing orb. + +## Interpret the result + +- **under-sized**: hard resource-pressure evidence such as OOM, exit 137, allocation failure, or measured usage that reaches the orb limit. Move up one size, then repeat the representative workload. +- **over-sized**: representative measured workloads fit comfortably on the next smaller size with CPU and memory headroom. Move down one size and verify there. +- **right-sized**: representative measurements use meaningful capacity without pressure and do not safely fit the next smaller size. +- **insufficient-evidence**: the transcript lacks trustworthy resource measurements or hard pressure signals. Do not interpret an uneventful thread as proof of over-sizing. + +The analyzer only treats shell/tool results as machine evidence. User and assistant prose can describe the task but must not be counted as telemetry. + +## Add evidence to a future thread + +Run the slowest representative build or test under GNU `time` inside the orb: + +```bash +/usr/bin/time -v cargo test +/usr/bin/time -v ./gradlew test +``` + +For parallel builds, also record the orb identity and capacity: + +```bash +printf 'orb-capacity cpus=%s memory_kb=%s\n' "$(nproc)" "$(awk '/MemTotal/ {print $2}' /proc/meminfo)" +``` + +Use a clean and an incremental build when both matter. One trivial command, setup/install work, model thinking time, network waits, and idle thread duration are not representative sizing measurements. + +## Apply judgment after the script + +1. Use `read_thread` to identify what the measured command did and whether it represents normal project work. +2. Separate compute delay from dependency downloads, network services, lock contention, and test sleeps. +3. Prefer multiple representative samples. A hard OOM is decisive; an over-sizing recommendation needs measured headroom. +4. Mention confidence, evidence, and missing evidence in the final recommendation. +5. Disk is 40 GB for every documented size, so changing size does not solve disk pressure. + +## Size reference + +| Size | CPUs | Memory | Hourly price | +|---|---:|---:|---:| +| `a0.tiny` | 1 | 2 GB | $0.10 | +| `a0.small` | 2 | 4 GB | $0.21 | +| `a0.medium` | 8 | 16 GB | $0.83 | +| `a0.large` | 16 | 32 GB | $1.66 | + +Treat prices as a dated reference and verify current Amp pricing before making a cost forecast. diff --git a/.agents/skills/analyzing-orb-sizing/scripts/analyze.py b/.agents/skills/analyzing-orb-sizing/scripts/analyze.py new file mode 100644 index 0000000..45baa04 --- /dev/null +++ b/.agents/skills/analyzing-orb-sizing/scripts/analyze.py @@ -0,0 +1,327 @@ +#!/usr/bin/env python3 +"""Assess Amp orb sizing from machine evidence in an exported thread.""" + +from __future__ import annotations + +import argparse +import json +import re +import subprocess +import sys +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Any + + +SIZES = { + "a0.tiny": {"cpus": 1, "memory_kb": 2 * 1024 * 1024}, + "a0.small": {"cpus": 2, "memory_kb": 4 * 1024 * 1024}, + "a0.medium": {"cpus": 8, "memory_kb": 16 * 1024 * 1024}, + "a0.large": {"cpus": 16, "memory_kb": 32 * 1024 * 1024}, +} +SIZE_NAMES = list(SIZES) + +HARD_PRESSURE_PATTERNS = { + "out-of-memory failure": re.compile( + r"\b(?:out of memory|oom(?:killed| kill)?|cannot allocate memory)\b", re.I + ), + "process killed with exit 137": re.compile( + r"(?:exit(?:ed| code| status)?\s*(?:with\s*)?137|status\s*137)", re.I + ), + "process killed by signal 9": re.compile( + r"(?:signal\s*9|sigkill|command terminated by signal 9)", re.I + ), +} +TIME_RSS = re.compile(r"Maximum resident set size \(kbytes\):\s*(\d+)", re.I) +TIME_CPU = re.compile(r"Percent of CPU this job got:\s*(\d+(?:\.\d+)?)%", re.I) +TIME_ELAPSED = re.compile( + r"Elapsed \(wall clock\) time.*?:\s*((?:\d+:)?\d+:\d+(?:\.\d+)?)", re.I +) + + +@dataclass +class Measurement: + max_rss_kb: int + cpu_percent: float | None + elapsed_seconds: float | None + + +@dataclass +class Assessment: + verdict: str + current_size: str + recommended_size: str + confidence: str + evidence: list[str] + missing_evidence: list[str] + measurements: list[Measurement] + + +@dataclass +class ShellResult: + command: str + output: str + exit_code: int | None + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Assess whether an Amp orb was under- or over-sized." + ) + parser.add_argument("source", help="Amp thread ID/URL or exported thread JSON") + parser.add_argument("--size", required=True, choices=SIZE_NAMES) + parser.add_argument("--json", action="store_true", dest="as_json") + return parser.parse_args() + + +def load_thread(source: str) -> dict[str, Any]: + path = Path(source) + if path.is_file(): + with path.open(encoding="utf-8") as handle: + return json.load(handle) + + try: + completed = subprocess.run( + ["amp", "threads", "export", source], + check=True, + capture_output=True, + text=True, + ) + except FileNotFoundError as error: + raise RuntimeError("Amp CLI not found; pass an exported JSON file instead") from error + except subprocess.CalledProcessError as error: + detail = error.stderr.strip() or "unknown Amp CLI error" + raise RuntimeError(f"could not export thread: {detail}") from error + return json.loads(completed.stdout) + + +def shell_results(thread: dict[str, Any]) -> list[ShellResult]: + shell_commands = { + block.get("id"): block.get("input", {}).get("command", "") + for message in thread.get("messages", []) + for block in message.get("content", []) + if block.get("type") == "tool_use" and block.get("name") == "shell_command" + } + results: list[ShellResult] = [] + for message in thread.get("messages", []): + for block in message.get("content", []): + if ( + block.get("type") != "tool_result" + or block.get("toolUseID") not in shell_commands + ): + continue + command = shell_commands[block.get("toolUseID")] + if not isinstance(command, str): + command = "" + run = block.get("run", {}) + result = run.get("result") + if isinstance(result, str): + results.append(ShellResult(command=command, output=result, exit_code=None)) + elif isinstance(result, dict): + output = result.get("output", "") + exit_code = result.get("exitCode") + results.append( + ShellResult( + command=command, + output=output if isinstance(output, str) else json.dumps(output), + exit_code=exit_code if isinstance(exit_code, int) else None, + ) + ) + return results + + +def parse_elapsed(value: str) -> float | None: + try: + parts = [float(part) for part in value.split(":")] + except ValueError: + return None + if len(parts) == 2: + return parts[0] * 60 + parts[1] + if len(parts) == 3: + return parts[0] * 3600 + parts[1] * 60 + parts[2] + return None + + +def measurements_from(texts: list[str]) -> list[Measurement]: + measurements: list[Measurement] = [] + for text in texts: + rss_matches = list(TIME_RSS.finditer(text)) + for index, rss_match in enumerate(rss_matches): + end = rss_matches[index + 1].start() if index + 1 < len(rss_matches) else len(text) + # GNU time usually prints RSS after CPU and elapsed, so inspect the preceding report too. + start = rss_matches[index - 1].end() if index else 0 + report = text[start:end] + cpu_matches = list(TIME_CPU.finditer(report)) + elapsed_matches = list(TIME_ELAPSED.finditer(report)) + measurements.append( + Measurement( + max_rss_kb=int(rss_match.group(1)), + cpu_percent=float(cpu_matches[-1].group(1)) if cpu_matches else None, + elapsed_seconds=( + parse_elapsed(elapsed_matches[-1].group(1)) + if elapsed_matches + else None + ), + ) + ) + return measurements + + +def assess(thread: dict[str, Any], size_name: str) -> Assessment: + results = shell_results(thread) + texts = [result.output for result in results] + failed_texts = [ + result.output + for result in results + if result.exit_code is not None and result.exit_code != 0 + ] + evidence: list[str] = [] + pressure: list[str] = [] + for label, pattern in HARD_PRESSURE_PATTERNS.items(): + if any(pattern.search(text) for text in failed_texts): + pressure.append(label) + if any(result.exit_code == 137 for result in results): + pressure.append("shell command exited with status 137") + + measurement_texts = [ + result.output + for result in results + if re.search(r"(?:^|[;&|()\s])/usr/bin/time\s+-v\b", result.command) + ] + measurements = measurements_from(measurement_texts) + current = SIZES[size_name] + current_index = SIZE_NAMES.index(size_name) + next_size = SIZE_NAMES[min(current_index + 1, len(SIZE_NAMES) - 1)] + + if pressure: + evidence.extend(pressure) + return Assessment( + verdict="under-sized", + current_size=size_name, + recommended_size=next_size, + confidence="high", + evidence=evidence, + missing_evidence=[], + measurements=measurements, + ) + + if measurements: + peak_rss = max(item.max_rss_kb for item in measurements) + memory_ratio = peak_rss / current["memory_kb"] + evidence.append( + f"peak measured RSS was {peak_rss / 1024:.0f} MiB " + f"({memory_ratio:.0%} of {size_name} memory)" + ) + + if memory_ratio >= 0.85: + return Assessment( + verdict="under-sized", + current_size=size_name, + recommended_size=next_size, + confidence="medium", + evidence=evidence, + missing_evidence=["no hard OOM signal was found"], + measurements=measurements, + ) + + cpu_samples = [m.cpu_percent for m in measurements if m.cpu_percent is not None] + representative = any( + m.elapsed_seconds is not None and m.elapsed_seconds >= 30 for m in measurements + ) + if not representative or not cpu_samples: + missing = [] + if not representative: + missing.append("no measured workload ran for at least 30 seconds") + if not cpu_samples: + missing.append("no GNU time CPU measurement was found") + return Assessment( + verdict="insufficient-evidence", + current_size=size_name, + recommended_size=size_name, + confidence="low", + evidence=evidence, + missing_evidence=missing, + measurements=measurements, + ) + + if current_index > 0: + smaller_name = SIZE_NAMES[current_index - 1] + smaller = SIZES[smaller_name] + fits_smaller_memory = peak_rss <= smaller["memory_kb"] * 0.60 + fits_smaller_cpu = max(cpu_samples) <= smaller["cpus"] * 60 + if fits_smaller_memory and fits_smaller_cpu: + evidence.append( + f"measured workload retains at least 40% CPU and memory headroom on {smaller_name}" + ) + return Assessment( + verdict="over-sized", + current_size=size_name, + recommended_size=smaller_name, + confidence="medium" if len(measurements) == 1 else "high", + evidence=evidence, + missing_evidence=( + ["only one representative measurement was found"] + if len(measurements) == 1 + else [] + ), + measurements=measurements, + ) + + return Assessment( + verdict="right-sized", + current_size=size_name, + recommended_size=size_name, + confidence="medium", + evidence=evidence, + missing_evidence=["repeat representative workloads to increase confidence"], + measurements=measurements, + ) + + return Assessment( + verdict="insufficient-evidence", + current_size=size_name, + recommended_size=size_name, + confidence="low", + evidence=[], + missing_evidence=[ + "no hard memory-pressure failure was found in tool results", + "no GNU time maximum-RSS measurement was found in tool results", + "thread exports do not contain historical CPU or memory telemetry", + ], + measurements=[], + ) + + +def print_human(assessment: Assessment) -> None: + print(f"Verdict: {assessment.verdict}") + print(f"Confidence: {assessment.confidence}") + print(f"Current size: {assessment.current_size}") + print(f"Recommended size: {assessment.recommended_size}") + if assessment.evidence: + print("Evidence:") + for item in assessment.evidence: + print(f" - {item}") + if assessment.missing_evidence: + print("Missing evidence:") + for item in assessment.missing_evidence: + print(f" - {item}") + + +def main() -> int: + args = parse_args() + try: + thread = load_thread(args.source) + assessment = assess(thread, args.size) + except (json.JSONDecodeError, OSError, RuntimeError) as error: + print(f"error: {error}", file=sys.stderr) + return 2 + + if args.as_json: + print(json.dumps(asdict(assessment), indent=2)) + else: + print_human(assessment) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.agents/skills/analyzing-orb-sizing/tests/test_analyze.py b/.agents/skills/analyzing-orb-sizing/tests/test_analyze.py new file mode 100644 index 0000000..5485676 --- /dev/null +++ b/.agents/skills/analyzing-orb-sizing/tests/test_analyze.py @@ -0,0 +1,128 @@ +import importlib.util +import sys +import unittest +from pathlib import Path + + +MODULE_PATH = Path(__file__).parents[1] / "scripts" / "analyze.py" +SPEC = importlib.util.spec_from_file_location("analyze", MODULE_PATH) +assert SPEC and SPEC.loader +analyze = importlib.util.module_from_spec(SPEC) +sys.modules["analyze"] = analyze +SPEC.loader.exec_module(analyze) + + +def thread_with_result( + result: str, exit_code: int = 0, command: str = "representative-workload" +) -> dict: + return { + "messages": [ + { + "role": "assistant", + "content": [ + { + "type": "tool_use", + "id": "shell-1", + "name": "shell_command", + "input": {"command": command}, + } + ], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "toolUseID": "shell-1", + "run": { + "status": "done", + "result": {"output": result, "exitCode": exit_code}, + }, + } + ], + } + ] + } + + +class AssessTests(unittest.TestCase): + def test_oom_is_under_sized(self) -> None: + result = analyze.assess( + thread_with_result("process exited with status 137", exit_code=137), + "a0.small", + ) + self.assertEqual(result.verdict, "under-sized") + self.assertEqual(result.recommended_size, "a0.medium") + self.assertEqual(result.confidence, "high") + + def test_prose_is_not_treated_as_evidence(self) -> None: + thread = { + "messages": [ + { + "role": "user", + "content": [{"type": "text", "text": "Did this build OOM?"}], + } + ] + } + result = analyze.assess(thread, "a0.small") + self.assertEqual(result.verdict, "insufficient-evidence") + + def test_non_shell_tool_output_is_not_treated_as_evidence(self) -> None: + thread = thread_with_result("process exited with status 137", exit_code=137) + thread["messages"][0]["content"][0]["name"] = "read_thread" + result = analyze.assess(thread, "a0.small") + self.assertEqual(result.verdict, "insufficient-evidence") + + def test_successful_discussion_of_oom_is_not_pressure(self) -> None: + result = analyze.assess( + thread_with_result("Prior analyzer said OOM and exit status 137"), + "a0.small", + ) + self.assertEqual(result.verdict, "insufficient-evidence") + + def test_absent_telemetry_does_not_imply_over_sized(self) -> None: + result = analyze.assess(thread_with_result("Finished tests successfully"), "a0.large") + self.assertEqual(result.verdict, "insufficient-evidence") + self.assertEqual(result.recommended_size, "a0.large") + + def test_trivial_measurement_is_insufficient(self) -> None: + time_output = """ +Percent of CPU this job got: 10% +Elapsed (wall clock) time (h:mm:ss or m:ss): 0:01.00 +Maximum resident set size (kbytes): 1024 +""" + result = analyze.assess( + thread_with_result(time_output, command="/usr/bin/time -v true"), + "a0.large", + ) + self.assertEqual(result.verdict, "insufficient-evidence") + + def test_representative_low_usage_recommends_smaller_size(self) -> None: + time_output = """ +Percent of CPU this job got: 85% +Elapsed (wall clock) time (h:mm:ss or m:ss): 1:10.00 +Maximum resident set size (kbytes): 1048576 +""" + result = analyze.assess( + thread_with_result(time_output, command="/usr/bin/time -v cargo test"), + "a0.medium", + ) + self.assertEqual(result.verdict, "over-sized") + self.assertEqual(result.recommended_size, "a0.small") + + def test_high_memory_measurement_recommends_larger_size(self) -> None: + time_output = """ +Percent of CPU this job got: 190% +Elapsed (wall clock) time (h:mm:ss or m:ss): 2:10.00 +Maximum resident set size (kbytes): 3700000 +""" + result = analyze.assess( + thread_with_result(time_output, command="/usr/bin/time -v cargo test"), + "a0.small", + ) + self.assertEqual(result.verdict, "under-sized") + self.assertEqual(result.recommended_size, "a0.medium") + + +if __name__ == "__main__": + unittest.main() From 9375033597c984c6d7e747e29187cd3c6e458439 Mon Sep 17 00:00:00 2001 From: Amp Date: Thu, 23 Jul 2026 02:32:16 +0000 Subject: [PATCH 2/4] Bootstrap Beads from Dolt in fresh orbs Agent-Signature: amp-gpt-5.6-medium on behalf of maphew Amp-Thread-ID: https://ampcode.com/threads/T-019f8cb6-a6f7-77ab-a6d7-ea71e143dc9d Co-authored-by: Matt Wilkie --- .agents/setup | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.agents/setup b/.agents/setup index d7e3c2d..61bdd96 100755 --- a/.agents/setup +++ b/.agents/setup @@ -24,3 +24,10 @@ if ! "$go_root/bin/go" version -m "$(command -v bd 2>/dev/null || echo /dev/null "$go_root/bin/go" install "github.com/steveyegge/beads/cmd/bd@$bd_commit" ln -sfn "$HOME/.local/bin/bd" "$HOME/.local/bin/beads" fi + +repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +git_common_dir="$(git -C "$repo_root" rev-parse --path-format=absolute --git-common-dir)" +beads_repo_root="$(dirname -- "$git_common_dir")" +if [[ ! -d "$beads_repo_root/.beads/embeddeddolt/beads/.dolt" ]]; then + "$HOME/.local/bin/bd" --directory "$repo_root" bootstrap --yes +fi From 91a3d267fbbded81aef0ea051ac7e9405444608b Mon Sep 17 00:00:00 2001 From: matt wilkie Date: Sun, 23 Aug 2026 11:05:42 -0700 Subject: [PATCH 3/4] fix: handle current Amp thread exports Parse Bash and top-level tool results while retaining legacy export support. Surface manual escalation at the largest orb size and ensure setup validates the bd binary it invokes. Agent-Signature: codex-gpt-5.6-sol-medium on behalf of maphew --- .agents/setup | 2 +- .../analyzing-orb-sizing/scripts/analyze.py | 76 ++++++++++++++++--- .../tests/test_analyze.py | 68 ++++++++++++++--- 3 files changed, 124 insertions(+), 22 deletions(-) diff --git a/.agents/setup b/.agents/setup index 61bdd96..6b62f52 100755 --- a/.agents/setup +++ b/.agents/setup @@ -19,7 +19,7 @@ mkdir -p "$HOME/.local/bin" ln -sfn "$go_root/bin/go" "$HOME/.local/bin/go" ln -sfn "$go_root/bin/gofmt" "$HOME/.local/bin/gofmt" -if ! "$go_root/bin/go" version -m "$(command -v bd 2>/dev/null || echo /dev/null)" 2>/dev/null | grep -q "${bd_commit:0:12}"; then +if ! "$go_root/bin/go" version -m "$HOME/.local/bin/bd" 2>/dev/null | grep -q "${bd_commit:0:12}"; then CGO_ENABLED=1 GOFLAGS=-tags=gms_pure_go GOBIN="$HOME/.local/bin" \ "$go_root/bin/go" install "github.com/steveyegge/beads/cmd/bd@$bd_commit" ln -sfn "$HOME/.local/bin/bd" "$HOME/.local/bin/beads" diff --git a/.agents/skills/analyzing-orb-sizing/scripts/analyze.py b/.agents/skills/analyzing-orb-sizing/scripts/analyze.py index 45baa04..307dcb7 100644 --- a/.agents/skills/analyzing-orb-sizing/scripts/analyze.py +++ b/.agents/skills/analyzing-orb-sizing/scripts/analyze.py @@ -96,12 +96,23 @@ def load_thread(source: str) -> dict[str, Any]: def shell_results(thread: dict[str, Any]) -> list[ShellResult]: - shell_commands = { - block.get("id"): block.get("input", {}).get("command", "") - for message in thread.get("messages", []) - for block in message.get("content", []) - if block.get("type") == "tool_use" and block.get("name") == "shell_command" - } + shell_commands: dict[Any, str] = {} + for message in thread.get("messages", []): + for block in message.get("content", []): + if block.get("type") != "tool_use": + continue + tool_name = block.get("name") + input_data = block.get("input", {}) + if not isinstance(input_data, dict): + continue + if tool_name == "Bash": + command = input_data.get("cmd", "") + elif tool_name == "shell_command": + command = input_data.get("command", "") + else: + continue + shell_commands[block.get("id")] = command if isinstance(command, str) else "" + results: list[ShellResult] = [] for message in thread.get("messages", []): for block in message.get("content", []): @@ -111,9 +122,38 @@ def shell_results(thread: dict[str, Any]) -> list[ShellResult]: ): continue command = shell_commands[block.get("toolUseID")] - if not isinstance(command, str): - command = "" + + # Current Amp exports expose status/output directly on tool_result. + # Retain support for the older nested run.result representation so + # previously exported threads remain analyzable. + if "output" in block: + output = block.get("output") + if isinstance(output, str): + output_text = output + elif isinstance(output, list): + output_text = "\n".join( + item.get("text", "") + for item in output + if isinstance(item, dict) + and item.get("type") == "text" + and isinstance(item.get("text"), str) + ) + else: + output_text = json.dumps(output) if output is not None else "" + status = block.get("status") + exit_code = ( + 0 + if status == "done" + else 1 if status in {"error", "cancelled"} else None + ) + results.append( + ShellResult(command=command, output=output_text, exit_code=exit_code) + ) + continue + run = block.get("run", {}) + if not isinstance(run, dict): + continue result = run.get("result") if isinstance(result, str): results.append(ShellResult(command=command, output=result, exit_code=None)) @@ -191,6 +231,7 @@ def assess(thread: dict[str, Any], size_name: str) -> Assessment: measurements = measurements_from(measurement_texts) current = SIZES[size_name] current_index = SIZE_NAMES.index(size_name) + at_largest_size = current_index == len(SIZE_NAMES) - 1 next_size = SIZE_NAMES[min(current_index + 1, len(SIZE_NAMES) - 1)] if pressure: @@ -201,7 +242,14 @@ def assess(thread: dict[str, Any], size_name: str) -> Assessment: recommended_size=next_size, confidence="high", evidence=evidence, - missing_evidence=[], + missing_evidence=( + [ + "a0.large is the largest documented size; investigate the " + "workload or request a larger orb" + ] + if at_largest_size + else [] + ), measurements=measurements, ) @@ -220,7 +268,15 @@ def assess(thread: dict[str, Any], size_name: str) -> Assessment: recommended_size=next_size, confidence="medium", evidence=evidence, - missing_evidence=["no hard OOM signal was found"], + missing_evidence=( + [ + "no hard OOM signal was found", + "a0.large is the largest documented size; investigate the " + "workload or request a larger orb", + ] + if at_largest_size + else ["no hard OOM signal was found"] + ), measurements=measurements, ) diff --git a/.agents/skills/analyzing-orb-sizing/tests/test_analyze.py b/.agents/skills/analyzing-orb-sizing/tests/test_analyze.py index 5485676..361e93a 100644 --- a/.agents/skills/analyzing-orb-sizing/tests/test_analyze.py +++ b/.agents/skills/analyzing-orb-sizing/tests/test_analyze.py @@ -13,8 +13,30 @@ def thread_with_result( - result: str, exit_code: int = 0, command: str = "representative-workload" + result: str, + exit_code: int = 0, + command: str = "representative-workload", + *, + current_schema: bool = False, + tool_name: str = "shell_command", ) -> dict: + tool_result = ( + { + "type": "tool_result", + "toolUseID": "shell-1", + "status": "done" if exit_code == 0 else "error", + "output": result, + } + if current_schema + else { + "type": "tool_result", + "toolUseID": "shell-1", + "run": { + "status": "done", + "result": {"output": result, "exitCode": exit_code}, + }, + } + ) return { "messages": [ { @@ -23,22 +45,15 @@ def thread_with_result( { "type": "tool_use", "id": "shell-1", - "name": "shell_command", - "input": {"command": command}, + "name": tool_name, + "input": {"cmd" if tool_name == "Bash" else "command": command}, } ], }, { "role": "user", "content": [ - { - "type": "tool_result", - "toolUseID": "shell-1", - "run": { - "status": "done", - "result": {"output": result, "exitCode": exit_code}, - }, - } + tool_result ], } ] @@ -46,6 +61,37 @@ def thread_with_result( class AssessTests(unittest.TestCase): + def test_pressure_at_largest_size_requests_manual_escalation(self) -> None: + result = analyze.assess( + thread_with_result("process exited with status 137", exit_code=137), + "a0.large", + ) + self.assertEqual(result.verdict, "under-sized") + self.assertEqual(result.recommended_size, "a0.large") + self.assertIn("largest documented size", result.missing_evidence[0]) + + def test_current_amp_bash_result_is_analyzed(self) -> None: + result = analyze.assess( + thread_with_result( + "process exited with status 137", + exit_code=137, + current_schema=True, + tool_name="Bash", + ), + "a0.small", + ) + self.assertEqual(result.verdict, "under-sized") + self.assertEqual(result.recommended_size, "a0.medium") + + def test_current_amp_structured_text_output_is_analyzed(self) -> None: + thread = thread_with_result("unused", current_schema=True, tool_name="Bash") + thread["messages"][1]["content"][0]["output"] = [ + {"type": "text", "text": "process exited with status 137"} + ] + thread["messages"][1]["content"][0]["status"] = "error" + result = analyze.assess(thread, "a0.small") + self.assertEqual(result.verdict, "under-sized") + def test_oom_is_under_sized(self) -> None: result = analyze.assess( thread_with_result("process exited with status 137", exit_code=137), From 61bf5f668140926a6c0ef7c9ae0504cfd6358a2c Mon Sep 17 00:00:00 2001 From: matt wilkie Date: Sun, 23 Aug 2026 11:14:23 -0700 Subject: [PATCH 4/4] fix: align orb evidence and bootstrap roots Pair each GNU time RSS sample with its own preceding CPU and elapsed values, and bootstrap Beads at the common repository root used by the linked-worktree guard. Agent-Signature: codex-gpt-5.6-sol-medium on behalf of maphew --- .agents/setup | 2 +- .../analyzing-orb-sizing/scripts/analyze.py | 3 +-- .../analyzing-orb-sizing/tests/test_analyze.py | 15 +++++++++++++++ 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/.agents/setup b/.agents/setup index 6b62f52..5aabf5a 100755 --- a/.agents/setup +++ b/.agents/setup @@ -29,5 +29,5 @@ repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" git_common_dir="$(git -C "$repo_root" rev-parse --path-format=absolute --git-common-dir)" beads_repo_root="$(dirname -- "$git_common_dir")" if [[ ! -d "$beads_repo_root/.beads/embeddeddolt/beads/.dolt" ]]; then - "$HOME/.local/bin/bd" --directory "$repo_root" bootstrap --yes + "$HOME/.local/bin/bd" --directory "$beads_repo_root" bootstrap --yes fi diff --git a/.agents/skills/analyzing-orb-sizing/scripts/analyze.py b/.agents/skills/analyzing-orb-sizing/scripts/analyze.py index 307dcb7..0f39d3f 100644 --- a/.agents/skills/analyzing-orb-sizing/scripts/analyze.py +++ b/.agents/skills/analyzing-orb-sizing/scripts/analyze.py @@ -187,10 +187,9 @@ def measurements_from(texts: list[str]) -> list[Measurement]: for text in texts: rss_matches = list(TIME_RSS.finditer(text)) for index, rss_match in enumerate(rss_matches): - end = rss_matches[index + 1].start() if index + 1 < len(rss_matches) else len(text) # GNU time usually prints RSS after CPU and elapsed, so inspect the preceding report too. start = rss_matches[index - 1].end() if index else 0 - report = text[start:end] + report = text[start : rss_match.start()] cpu_matches = list(TIME_CPU.finditer(report)) elapsed_matches = list(TIME_ELAPSED.finditer(report)) measurements.append( diff --git a/.agents/skills/analyzing-orb-sizing/tests/test_analyze.py b/.agents/skills/analyzing-orb-sizing/tests/test_analyze.py index 361e93a..0a46460 100644 --- a/.agents/skills/analyzing-orb-sizing/tests/test_analyze.py +++ b/.agents/skills/analyzing-orb-sizing/tests/test_analyze.py @@ -169,6 +169,21 @@ def test_high_memory_measurement_recommends_larger_size(self) -> None: self.assertEqual(result.verdict, "under-sized") self.assertEqual(result.recommended_size, "a0.medium") + def test_multiple_time_reports_keep_each_reports_cpu_and_elapsed(self) -> None: + time_output = """ +Percent of CPU this job got: 25% +Elapsed (wall clock) time (h:mm:ss or m:ss): 0:31.00 +Maximum resident set size (kbytes): 1000 +Percent of CPU this job got: 175% +Elapsed (wall clock) time (h:mm:ss or m:ss): 2:02.00 +Maximum resident set size (kbytes): 2000 +""" + measurements = analyze.measurements_from([time_output]) + self.assertEqual( + [(item.cpu_percent, item.elapsed_seconds) for item in measurements], + [(25.0, 31.0), (175.0, 122.0)], + ) + if __name__ == "__main__": unittest.main()