diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bc560b6..88cffb7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,14 +10,15 @@ permissions: jobs: test: + name: test runs-on: ubuntu-latest steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 - name: Python checks run: | - python3 -m py_compile runner/cli.py container/invoke.py container/evidence_safety.py container/native_observer.py container/runtime_evidence.py tests/integration/run_code_mode_observer_probe.py tests/integration/run_runtime_evidence_acceptance.py - python3 -m unittest discover -s tests -p 'test_*.py' + python3 -m compileall -q runner container tests + python3 -m unittest discover -s tests -p 'test_*.py' -v - name: Build fake Copilot transport for action smoke test run: | @@ -68,19 +69,91 @@ jobs: assert result["runtime_evidence"]["evidence_eligible"] is False PY - - name: Build OpenCode transport image + runtime-integration: + name: runtime-integration + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Decide runtime integration applicability + id: changes + shell: bash + run: | + set -euo pipefail + if [[ "${{ github.event_name }}" != "pull_request" ]]; then + echo "runtime=true" >> "$GITHUB_OUTPUT" + echo "copilot=true" >> "$GITHUB_OUTPUT" + echo "Non-PR event: run full runtime integration." + exit 0 + fi + + base="${{ github.event.pull_request.base.sha }}" + head="${{ github.event.pull_request.head.sha }}" + runtime=false + copilot=false + + while IFS= read -r path; do + [[ -z "$path" ]] && continue + printf 'changed: %s\n' "$path" + + case "$path" in + Containerfile|action.yml|bin/*|container/*|runner/*|tests/integration/*|tests/test_invoke.py|tests/test_evidence_safety.py|tests/test_native_observer.py|tests/test_runtime_evidence.py|tests/test_runtime_evidence_acceptance.py|.github/workflows/ci.yml) + runtime=true + ;; + esac + + case "$path" in + Containerfile|action.yml|bin/*|container/*|runner/*|.github/workflows/ci.yml) + copilot=true + ;; + esac + done < <(git diff --name-only "$base" "$head") + + echo "runtime=$runtime" >> "$GITHUB_OUTPUT" + echo "copilot=$copilot" >> "$GITHUB_OUTPUT" + + - name: Runtime integration not applicable + if: steps.changes.outputs.runtime != 'true' + run: echo "No runtime-relevant files changed; runtime integration is not applicable." + + - name: Prepare integration artifacts + if: steps.changes.outputs.runtime == 'true' + run: mkdir -p runtime-integration-artifacts + + - name: Build stock OpenCode 2.0.23 image + if: steps.changes.outputs.runtime == 'true' run: docker build -f Containerfile --target opencode -t opencode-eval-runner:opencode-test . - - name: Probe stock Code Mode inner observation boundary + - name: Verify pinned OpenCode runtime + if: steps.changes.outputs.runtime == 'true' run: | - python3 tests/integration/run_code_mode_observer_probe.py \ - --image opencode-eval-runner:opencode-test \ - --output /tmp/code-mode-observer-probe + version="$(docker run --rm --entrypoint opencode opencode-eval-runner:opencode-test --version)" + printf 'OpenCode version: %s\n' "$version" + test "${version#opencode v}" = "2.0.23" + docker run --rm --entrypoint opencode opencode-eval-runner:opencode-test run --help | grep -F -- 'provider/model#variant' - - name: Build Copilot transport image - run: docker build -f Containerfile --target copilot -t opencode-eval-runner:copilot-test . + - name: Run provider-free native observer probe + if: steps.changes.outputs.runtime == 'true' + run: | + docker run --rm --network none --tmpfs /tmp:rw,exec,nosuid,nodev,size=1g,mode=1777 --tmpfs /workspace:rw,nosuid,nodev,size=64m,mode=1777 --volume "$PWD:/probe-repo:ro" --entrypoint python3 opencode-eval-runner:opencode-test /probe-repo/tests/integration/run_native_observer_probe.py > runtime-integration-artifacts/native-observer-probe.json + cat runtime-integration-artifacts/native-observer-probe.json + + - name: Run stock Code Mode capability probe + if: steps.changes.outputs.runtime == 'true' + run: | + python3 tests/integration/run_code_mode_observer_probe.py --image opencode-eval-runner:opencode-test --output runtime-integration-artifacts/code-mode-observer-probe + + - name: Run provider-free runtime evidence acceptance + if: steps.changes.outputs.runtime == 'true' + run: | + python3 tests/integration/run_runtime_evidence_acceptance.py --image opencode-eval-runner:opencode-test --output runtime-integration-artifacts/runtime-evidence-acceptance - name: Smoke test OpenCode plugin activation preflight + if: steps.changes.outputs.runtime == 'true' run: | rm -rf /tmp/preflight-workspace /tmp/preflight-config-root mkdir -p /tmp/preflight-workspace @@ -92,12 +165,7 @@ jobs: } EOF - docker run --rm \ - --workdir /workspace \ - --volume /tmp/preflight-workspace:/workspace:ro \ - --volume /tmp/preflight-config-root:/seed/opencode-config:ro \ - --entrypoint python3 \ - opencode-eval-runner:opencode-test - <<'PY' + docker run --rm --workdir /workspace --volume /tmp/preflight-workspace:/workspace:ro --volume /tmp/preflight-config-root:/seed/opencode-config:ro --entrypoint python3 opencode-eval-runner:opencode-test - <<'PY' import sys sys.path.insert(0, "/opt/opencode-eval-runner") @@ -118,6 +186,7 @@ jobs: PY - name: Smoke test config plugin workspace dependency resolution + if: steps.changes.outputs.runtime == 'true' run: | rm -rf /tmp/plugin-workspace /tmp/plugin-config-root mkdir -p /tmp/plugin-workspace/node_modules/fixture-dep @@ -142,11 +211,7 @@ jobs: export default async () => ({}) EOF - docker run --rm \ - --volume /tmp/plugin-workspace:/workspace:ro \ - --volume /tmp/plugin-config-root:/seed/opencode-config:ro \ - --entrypoint python3 \ - opencode-eval-runner:opencode-test - <<'PY' + docker run --rm --volume /tmp/plugin-workspace:/workspace:ro --volume /tmp/plugin-config-root:/seed/opencode-config:ro --entrypoint python3 opencode-eval-runner:opencode-test - <<'PY' from pathlib import Path import subprocess import sys @@ -180,18 +245,29 @@ jobs: print("PASS config-root plugin dependency bridge") PY - - name: Verify pinned tools - run: | - docker run --rm --entrypoint opencode opencode-eval-runner:opencode-test --version - docker run --rm --entrypoint copilot opencode-eval-runner:copilot-test --version + - name: Build real Copilot image + if: steps.changes.outputs.runtime == 'true' && steps.changes.outputs.copilot == 'true' + run: docker build -f Containerfile --target copilot -t opencode-eval-runner:copilot-test . - - name: Verify pinned reasoning controls + - name: Verify pinned Copilot runtime + if: steps.changes.outputs.runtime == 'true' && steps.changes.outputs.copilot == 'true' run: | - docker run --rm --entrypoint opencode opencode-eval-runner:opencode-test run --help \ - | grep -F -- 'provider/model#variant' + docker run --rm --entrypoint copilot opencode-eval-runner:copilot-test --version docker run --rm --entrypoint copilot opencode-eval-runner:copilot-test --help | grep -F -- '--effort' - name: Report image sizes + if: steps.changes.outputs.runtime == 'true' run: | docker image inspect opencode-eval-runner:opencode-test --format 'OpenCode image: {{.Size}} bytes' - docker image inspect opencode-eval-runner:copilot-test --format 'Copilot image: {{.Size}} bytes' + if [[ "${{ steps.changes.outputs.copilot }}" == "true" ]]; then + docker image inspect opencode-eval-runner:copilot-test --format 'Copilot image: {{.Size}} bytes' + fi + + - name: Preserve runtime integration reports + if: always() && steps.changes.outputs.runtime == 'true' + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: runtime-integration-${{ github.event.pull_request.head.sha || github.sha }} + path: runtime-integration-artifacts/ + if-no-files-found: warn + retention-days: 14 diff --git a/.github/workflows/native-observer-integration.yml b/.github/workflows/native-observer-integration.yml deleted file mode 100644 index 9fc6bb5..0000000 --- a/.github/workflows/native-observer-integration.yml +++ /dev/null @@ -1,53 +0,0 @@ -name: Stock native observer integration - -on: - pull_request: - paths: - - 'container/invoke.py' - - 'container/native_observer.py' - - 'container/native_observer.ts' - - 'tests/integration/native_observer_probe.ts' - - 'tests/integration/run_native_observer_probe.py' - - 'tests/test_native_observer.py' - - '.github/workflows/native-observer-integration.yml' - -permissions: - contents: read - -jobs: - stock-native-observer: - runs-on: ubuntu-latest - timeout-minutes: 10 - steps: - - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 - with: - ref: ${{ github.event.pull_request.head.sha }} - persist-credentials: false - - - name: Verify probe syntax - run: python3 -m py_compile container/native_observer.py tests/integration/run_native_observer_probe.py - - - name: Build runner with stock OpenCode 2.0.23 - run: docker build -f Containerfile --target opencode -t opencode-eval-runner:native-observer-probe . - - - name: Run provider-free native observer probe - run: | - docker run --rm \ - --network none \ - --tmpfs /tmp:rw,exec,nosuid,nodev,size=1g,mode=1777 \ - --tmpfs /workspace:rw,nosuid,nodev,size=64m,mode=1777 \ - --volume "$PWD:/probe-repo:ro" \ - --entrypoint python3 \ - opencode-eval-runner:native-observer-probe \ - /probe-repo/tests/integration/run_native_observer_probe.py \ - > native-observer-probe.json - cat native-observer-probe.json - - - name: Preserve probe report - if: always() - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 - with: - name: native-observer-${{ github.event.pull_request.head.sha }} - path: native-observer-probe.json - if-no-files-found: warn - retention-days: 14 diff --git a/.github/workflows/runtime-evidence-acceptance.yml b/.github/workflows/runtime-evidence-acceptance.yml deleted file mode 100644 index 6dabe89..0000000 --- a/.github/workflows/runtime-evidence-acceptance.yml +++ /dev/null @@ -1,47 +0,0 @@ -name: Provider-free runtime evidence acceptance - -on: - pull_request: - paths: - - 'Containerfile' - - 'container/**' - - 'runner/**' - - 'tests/integration/**' - - 'tests/test_runtime_evidence_acceptance.py' - - '.github/workflows/runtime-evidence-acceptance.yml' - workflow_dispatch: - -permissions: - contents: read - -jobs: - provider-free-acceptance: - runs-on: ubuntu-latest - timeout-minutes: 15 - steps: - - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 - with: - persist-credentials: false - - - name: Check acceptance driver - run: | - python3 -m py_compile tests/integration/run_runtime_evidence_acceptance.py - python3 -m unittest discover -s tests -p 'test_runtime_evidence_acceptance.py' -v - - - name: Build stock OpenCode 2.0.23 runner image - run: docker build -f Containerfile --target opencode -t opencode-eval-runner:runtime-evidence-acceptance . - - - name: Run provider-free acceptance gate - run: | - python3 tests/integration/run_runtime_evidence_acceptance.py \ - --image opencode-eval-runner:runtime-evidence-acceptance \ - --output runtime-evidence-acceptance - - - name: Preserve sanitized acceptance report - if: always() - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 - with: - name: runtime-evidence-acceptance-${{ github.event.pull_request.head.sha || github.sha }} - path: runtime-evidence-acceptance/ - if-no-files-found: error - retention-days: 14 diff --git a/README.md b/README.md index f93709b..8b8c36c 100644 --- a/README.md +++ b/README.md @@ -237,6 +237,14 @@ The token value is not placed on the container command line. ## GitHub Actions +Pull requests expose two stable CI checks: + +- `test`: fast Python/unit checks plus the fake Copilot GitHub Action smoke test. +- `runtime-integration`: stock-runtime probes and acceptance checks when runtime-relevant files change. For documentation-only or other unrelated changes, the job still runs and succeeds with an explicit not-applicable result. + +`Publish images` remains a post-merge/tag workflow and is not a required pull-request check. + + For all 21 Action inputs, defaults, execution-mode precedence, and CLI-only capabilities, see [Invocation usage and interface reference](docs/invocation-usage.md#github-action-interface). The repository is a composite GitHub Action. It supports either a single direct invocation or setup plus a repository-owned eval harness. diff --git a/container/invoke.py b/container/invoke.py index 2df9e5b..9d45bcd 100644 --- a/container/invoke.py +++ b/container/invoke.py @@ -346,50 +346,6 @@ def extract_loaded_skills(events: list[dict[str, Any]]) -> list[str]: return found -def loaded_skills_from_export(exported: Any) -> list[str]: - found: list[str] = [] - messages = exported if isinstance(exported, list) else exported.get("messages", []) if isinstance(exported, dict) else [] - for message in messages: - if not isinstance(message, dict): - continue - info = message.get("info") - if not isinstance(info, dict) or info.get("role") != "assistant": - continue - for part in message.get("parts", []): - if not isinstance(part, dict): - continue - skill = completed_skill_from_part(part) - if skill and skill not in found: - found.append(skill) - return found - - -def assistant_from_export(exported: Any) -> tuple[str, list[str], list[dict[str, Any]]]: - parts: list[str] = [] - tools: list[str] = [] - actions: list[dict[str, Any]] = [] - messages = exported if isinstance(exported, list) else exported.get("messages", []) if isinstance(exported, dict) else [] - for message in messages: - if not isinstance(message, dict): - continue - info = message.get("info") - if not isinstance(info, dict) or info.get("role") != "assistant": - continue - for part in message.get("parts", []): - if not isinstance(part, dict): - continue - if part.get("type") == "text" and isinstance(part.get("text"), str): - value = part["text"].strip() - if value and part.get("synthetic") is not True and part.get("ignored") is not True: - parts.append(value) - if part.get("type") == "tool" and isinstance(part.get("tool"), str): - tools.append(part["tool"]) - action = tool_action(part) - if action: - actions.append(action) - return "\n\n".join(parts), list(dict.fromkeys(tools)), actions - - def prepare_opencode_env() -> dict[str, str]: env = dict(os.environ) root = Path("/tmp/runtime") diff --git a/tests/test_invoke.py b/tests/test_invoke.py index 87bfe6f..1a2cd0b 100644 --- a/tests/test_invoke.py +++ b/tests/test_invoke.py @@ -9,11 +9,9 @@ from unittest.mock import patch from container.invoke import ( - assistant_from_export, extract_actions, extract_loaded_skills, extract_tool_result_evidence, - loaded_skills_from_export, invoke_copilot, invoke_opencode, resolve_opencode_reasoning, @@ -78,67 +76,6 @@ def test_extract_loaded_skills_counts_only_completed_native_skill_calls(self): ["golang-concurrency", "architectural-design"], ) - def test_failed_exported_skill_call_is_not_reported_as_loaded(self): - exported = [ - { - "info": {"role": "assistant"}, - "parts": [ - { - "type": "tool", - "tool": "skill", - "state": { - "status": "error", - "input": {"id": "missing-skill"}, - }, - } - ], - } - ] - self.assertEqual(loaded_skills_from_export(exported), []) - - def test_session_export_preserves_tool_inputs_as_actions(self): - exported = [ - { - "info": {"role": "assistant"}, - "parts": [ - { - "type": "tool", - "tool": "skill", - "state": { - "status": "completed", - "input": {"name": "golang-concurrency"}, - "output": "omitted", - }, - }, - { - "type": "tool", - "tool": "read", - "state": { - "status": "completed", - "input": { - "filePath": "/workspace/.opencode/skills/golang-concurrency/ASSESSMENT.md" - }, - }, - }, - ], - } - ] - text, tools, actions = assistant_from_export(exported) - self.assertEqual(text, "") - self.assertEqual(tools, ["skill", "read"]) - self.assertEqual( - actions, - [ - {"tool": "skill", "args": {"name": "golang-concurrency"}}, - { - "tool": "read", - "args": { - "filePath": "/workspace/.opencode/skills/golang-concurrency/ASSESSMENT.md" - }, - }, - ], - ) - def test_v2_invocation_does_not_use_models_refresh_preflight(self): class Result: returncode = 0