diff --git a/.github/scripts/e1/PR.md b/.github/scripts/e1/PR.md new file mode 100644 index 000000000000..f0d725c549a2 --- /dev/null +++ b/.github/scripts/e1/PR.md @@ -0,0 +1,32 @@ +# E1 ColossalAI PR regression + +The `ColossalAI GPU on internal PR` workflow runs two existing ColossalAI tests: + +- `tests/test_booster/test_accelerator.py::test_accelerator`: verifies model placement on CPU and CUDA. +- `tests/test_booster/test_plugin/test_dp_plugin_base.py::test_dp_plugin_dataloader`: launches two NCCL workers through `colossalai.launch`, prepares a DPPlugin dataloader, and checks that ranks receive different data. + +This is a small initial regression set, not the complete ColossalAI suite. The previous pinned-container infrastructure smoke test remains available through `pr_gpu.py --suite smoke`; the workflow now explicitly chooses `--suite colossalai`. + +## Trigger and source + +Every same-repository PR in `hpcaitech/ColossalAI` is eligible, regardless of author, head branch, base branch or changed paths. Opening, updating, reopening or marking a PR ready triggers the workflow. Draft PRs are also tested. External fork PRs are excluded from the self-hosted GPU job. Existing PRs need a subsequent event or an explicit re-run after rollout; merging the workflow does not retroactively start them all. + +The workflow must be present in the PR merge snapshot. Publishing it to `main` covers normal PRs targeting `main`; independently maintained release branches may need the workflow backported. It does not automatically become a required merge check. + +The runner label is `colossalai-e1-h20`. A label assigns a job to a runner; the workflow and the explicit `TESTS` list in `colossalai_suite.py` select the tests. Concurrency is grouped by PR number, so pending jobs for different PRs do not replace each other. The single runner executes jobs sequentially; multiple updates within one PR may replace its older pending run. Running jobs are not automatically cancelled. + +Because the node's Git HTTPS endpoint was unreliable, the job downloads the official full source archive for `GITHUB_SHA` (the PR merge commit) and records its SHA256. Tests run from that fresh snapshot. An import-path assertion prevents accidentally testing the copy installed in the shared environment. + +## Runtime and resources + +The initial Python executable is `/mnt/beegfs/ColossalAI/wangzhijian/envs/colossalai-torch213/bin/python` (Torch 2.13/CUDA 13.0). The workflow does not install packages or modify this environment. It is a pre-existing runtime rather than an immutable CI image; dependency changes can require separate environment maintenance. Runtime versions are recorded in `colossalai/environment.json`. + +A GPU-free preflight imports the actual source and collects the selected tests. The job then selects two idle GPU UUIDs using utilization, memory and compute processes. The suite acquires the same node-local advisory lock as the E1 smoke wrapper, rechecks occupancy, and exposes only those GPUs with `CUDA_VISIBLE_DEVICES`. + +There is no reservation website access, waiting loop, or preemption. Fewer than two idle GPUs is a failure. The advisory lock coordinates E1 only; it cannot stop other users from starting work. These tests run as host processes in the trusted same-repository PR context, not in the smoke-test container. Do not enable untrusted fork code on this shared host. + +## Results and limits + +The parent bounds the suite to six minutes and terminates its process group on timeout/cancellation. The Actions job is bounded to 12 minutes. The pytest run must exit successfully and its JUnit XML must contain exactly the two selected passing test cases, with no failures, errors or skips. The infrastructure smoke marker cannot satisfy this check. + +Evidence is retained under `/mnt/beegfs/ricardoo/ci/gpu-h20-5/test-results/pr--/`: `result.json`, `test.log`, `colossalai/environment.json`, and `colossalai/junit.xml`. CPU-only preflight metadata is in the sibling `-preflight` directory. Actions logs and the job summary expose the status and tested commit. A busy-resource failure is not a passing regression run. diff --git a/.github/scripts/e1/colossalai_suite.py b/.github/scripts/e1/colossalai_suite.py new file mode 100644 index 000000000000..55303a164c89 --- /dev/null +++ b/.github/scripts/e1/colossalai_suite.py @@ -0,0 +1,123 @@ +"""Run a small, explicit set of existing ColossalAI tests from this checkout.""" + +import argparse +import json +import os +import socket +import subprocess +import sys +import xml.etree.ElementTree as ET +from pathlib import Path + +TESTS = [ + "tests/test_booster/test_accelerator.py::test_accelerator", + "tests/test_booster/test_plugin/test_dp_plugin_base.py::test_dp_plugin_dataloader", +] + + +def validate_report(path): + cases = list(ET.parse(path).getroot().iter("testcase")) + expected = {node.split("::")[1] for node in TESTS} + if len(cases) != len(TESTS) or {case.get("name") for case in cases} != expected: + raise RuntimeError("Expected exactly the two selected ColossalAI tests") + if any(case.find(tag) is not None for case in cases for tag in ("failure", "error", "skipped")): + raise RuntimeError("ColossalAI tests failed or were skipped") + return len(cases) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--collect-only", action="store_true") + parser.add_argument("--gpus", nargs=2) + args = parser.parse_args() + if socket.gethostname().split(".")[0] != "gpu-h20-5": + raise RuntimeError("This rollout targets gpu-h20-5") + root = Path(__file__).resolve().parents[3] + output = args.output.resolve() + output.mkdir(parents=True, exist_ok=True) + lock = None + if not args.collect_only: + import fcntl + + from pr_gpu import GPU_UUID + + if not args.gpus or len(set(args.gpus)) != 2 or not all(GPU_UUID.fullmatch(g) for g in args.gpus): + raise RuntimeError("Two distinct GPU UUIDs are required") + lock = open(f"/tmp/colossalai-e1-ricardoo-{os.getuid()}.lock", "a") + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + for gpu in args.gpus: + row = subprocess.check_output( + ["nvidia-smi", "-i", gpu, "--query-gpu=memory.used,utilization.gpu", "--format=csv,noheader,nounits"], + text=True, + timeout=10, + ) + memory, utilization = [value.strip() for value in row.split(",")] + processes = subprocess.check_output( + ["nvidia-smi", "-i", gpu, "--query-compute-apps=pid", "--format=csv,noheader"], + text=True, + timeout=10, + ) + if ( + not memory.isdigit() + or not utilization.isdigit() + or int(memory) > 256 + or int(utilization) + or processes.strip() + ): + raise RuntimeError(f"Selected GPU is busy or its occupancy is unknown: {gpu}") + + os.environ.update( + CUDA_VISIBLE_DEVICES="" if args.collect_only else ",".join(args.gpus), + PYTHONPATH=str(root), + PYTHONDONTWRITEBYTECODE="1", + PYTEST_DISABLE_PLUGIN_AUTOLOAD="1", + HF_HUB_OFFLINE="1", + TRANSFORMERS_OFFLINE="1", + OMP_NUM_THREADS="1", + NCCL_IB_DISABLE="1", + NCCL_SOCKET_IFNAME="lo", + GLOO_SOCKET_IFNAME="lo", + TORCH_EXTENSIONS_DIR=str(output / "torch_extensions"), + TRITON_CACHE_DIR=str(output / "triton"), + HF_HOME=str(output / "huggingface"), + ) + os.chdir(root) + sys.path.insert(0, str(root)) + import pytest + import torch + + import colossalai + + if Path(colossalai.__file__).resolve() != root / "colossalai" / "__init__.py": + raise RuntimeError("Imported ColossalAI is not from the tested source snapshot") + if not torch.__version__.startswith("2.13."): + raise RuntimeError("This initial suite requires the prevalidated Torch 2.13 environment") + metadata = { + "source": str(root), + "colossalai_import": colossalai.__file__, + "torch": torch.__version__, + "cuda": torch.version.cuda, + "tests": TESTS, + "collect_only": args.collect_only, + } + (output / "environment.json").write_text(json.dumps(metadata, indent=2) + "\n") + print(json.dumps(metadata), flush=True) + if not args.collect_only and (not torch.cuda.is_available() or torch.cuda.device_count() != 2): + raise RuntimeError("Exactly two CUDA devices must be visible") + junit = output / "junit.xml" + options = [*TESTS, "-v", "-ra", "--maxfail=1", "-p", "no:cacheprovider"] + options += ["--collect-only"] if args.collect_only else [f"--junitxml={junit}"] + code = int(pytest.main(options)) + if code: + return code + if not args.collect_only: + count = validate_report(junit) + print(json.dumps({"result": "E1_COLOSSALAI_PASS", "gpu_tested": True, "tests_passed": count}), flush=True) + if lock is not None: + lock.close() + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/scripts/e1/gpu_smoke.py b/.github/scripts/e1/gpu_smoke.py new file mode 100644 index 000000000000..bf0b1c65a1c8 --- /dev/null +++ b/.github/scripts/e1/gpu_smoke.py @@ -0,0 +1,144 @@ +"""Small, assertion-based runner qualification; this does not test ColossalAI.""" + +import argparse +import json +import os +import sys +from datetime import timedelta + +import torch +import torch.distributed as dist +from torch.nn.parallel import DistributedDataParallel + + +def require(condition, message): + if not condition: + raise RuntimeError(message) + + +def environment(): + return { + "python": sys.version.split()[0], + "torch": torch.__version__, + "cuda_build": torch.version.cuda, + "distributed_available": dist.is_available(), + "nccl_available": dist.is_available() and dist.is_nccl_available(), + } + + +def check_collectives_and_training(device, rank, local_rank): + # Exercise actual tensor computation, then verify the numerical result. + left = torch.ones((128, 128), device=device) + right = torch.full((128, 128), 2.0, device=device) + torch.testing.assert_close(left @ right, torch.full_like(left, 256.0), rtol=0, atol=0) + + reduced = torch.tensor([rank + 1.0], device=device) + dist.all_reduce(reduced, op=dist.ReduceOp.SUM) + torch.testing.assert_close(reduced, torch.tensor([3.0], device=device), rtol=0, atol=0) + + model = torch.nn.Linear(1, 1, bias=False).to(device) + with torch.no_grad(): + model.weight.zero_() + ddp = DistributedDataParallel(model, device_ids=[local_rank] if device.type == "cuda" else None) + optimizer = torch.optim.SGD(ddp.parameters(), lr=0.01) + + # Different data on each rank makes missing gradient synchronization observable. + # Global x = [1, 2, 3, 4], y = 3*x. mean(x**2) = 7.5, so grad = 15*(w-3). + inputs = torch.tensor([[1.0 + 2 * rank], [2.0 + 2 * rank]], device=device) + targets = 3 * inputs + expected_weight = 0.0 + for _ in range(3): + optimizer.zero_grad(set_to_none=True) + loss = torch.nn.functional.mse_loss(ddp(inputs), targets) + require(torch.isfinite(loss).item(), "Non-finite training loss") + loss.backward() + + expected_gradient = 15.0 * (expected_weight - 3.0) + torch.testing.assert_close( + ddp.module.weight.grad, + torch.full_like(ddp.module.weight, expected_gradient), + rtol=1e-5, + atol=1e-5, + ) + optimizer.step() + expected_weight -= 0.01 * expected_gradient + torch.testing.assert_close( + ddp.module.weight, + torch.full_like(ddp.module.weight, expected_weight), + rtol=1e-5, + atol=1e-5, + ) + + weight = ddp.module.weight.detach().contiguous() + gathered = [torch.empty_like(weight) for _ in range(2)] + dist.all_gather(gathered, weight) + torch.testing.assert_close(gathered[0], gathered[1], rtol=0, atol=0) + return float(weight.item()) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + modes = parser.add_mutually_exclusive_group() + modes.add_argument("--check", action="store_true", help="Inspect the build without initializing CUDA") + modes.add_argument("--cpu-control", action="store_true", help="Validate test math on CPU/Gloo only") + args = parser.parse_args() + + metadata = environment() + if args.check: + require(metadata["distributed_available"], "PyTorch distributed is unavailable") + require(metadata["nccl_available"], "PyTorch was not built with NCCL") + require(metadata["cuda_build"] is not None, "PyTorch was not built with CUDA") + print(json.dumps({"mode": "environment_check", "gpu_tested": False, **metadata}), flush=True) + return + + world_size = int(os.environ.get("WORLD_SIZE", "0")) + rank = int(os.environ.get("RANK", "-1")) + local_rank = int(os.environ.get("LOCAL_RANK", "-1")) + require(world_size == 2 and rank in (0, 1) and local_rank in (0, 1), "Launch exactly two torchrun workers") + + if args.cpu_control: + require(torch.cuda.device_count() == 0, "CPU control must run in a container without GPU access") + device = torch.device("cpu") + backend = "gloo" + else: + require(torch.cuda.is_available(), "CUDA unavailable: GPU test cannot be skipped or passed") + require(torch.cuda.device_count() == 2, "Expose exactly two assigned GPUs to this container") + require(metadata["nccl_available"], "NCCL unavailable") + torch.cuda.set_device(local_rank) + device = torch.device("cuda", local_rank) + backend = "nccl" + properties = torch.cuda.get_device_properties(device) + print( + json.dumps({"rank": rank, "device": str(device), "name": properties.name, "uuid": str(properties.uuid)}), + flush=True, + ) + + torch.set_num_threads(1) + dist.init_process_group(backend=backend, timeout=timedelta(seconds=45)) + try: + weight = check_collectives_and_training(device, rank, local_rank) + if device.type == "cuda": + torch.cuda.synchronize(device) + dist.barrier() + if rank == 0: + marker = "E1_CPU_CONTROL_PASS" if args.cpu_control else "E1_GPU_SMOKE_PASS" + print( + json.dumps( + { + "result": marker, + "gpu_tested": not args.cpu_control, + "backend": backend, + "world_size": world_size, + "checks": ["matmul", "all_reduce_sum", "ddp_gradients", "sgd_updates", "equal_weights"], + "final_weight": weight, + **metadata, + } + ), + flush=True, + ) + finally: + dist.destroy_process_group() + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/e1/pr_gpu.py b/.github/scripts/e1/pr_gpu.py new file mode 100644 index 000000000000..cf9e00e73063 --- /dev/null +++ b/.github/scripts/e1/pr_gpu.py @@ -0,0 +1,189 @@ +#!/usr/bin/env python3 +"""Immediate PR qualification using two currently idle GPUs on gpu-h20-5. + +For same-repository PRs from any internal branch, under manual resource +authorization. This does not contact or claim a reservation from the website. +""" + +import argparse +import json +import os +import re +import signal +import socket +import subprocess +import sys +from datetime import datetime, timezone +from pathlib import Path + +GPU_UUID = re.compile(r"GPU-[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}\Z") + + +def validate_event(event): + pr = event["pull_request"] + if ( + event["repository"]["full_name"] != "hpcaitech/ColossalAI" + or pr["head"]["repo"]["full_name"] != "hpcaitech/ColossalAI" + or pr["base"]["repo"]["full_name"] != "hpcaitech/ColossalAI" + ): + raise RuntimeError("GPU execution requires a same-repository ColossalAI PR") + return pr + + +def select_idle(inventory, processes): + busy = set() + for line in processes.splitlines(): + value = line.strip() + if value: + if not GPU_UUID.fullmatch(value): + raise RuntimeError("GPU process information is unknown") + busy.add(value) + candidates, seen = [], set() + for line in inventory.splitlines(): + fields = [part.strip() for part in line.split(",")] + if len(fields) != 4: + raise RuntimeError("Unexpected GPU inventory format") + index, gpu, memory, utilization = fields + if not GPU_UUID.fullmatch(gpu) or gpu in seen: + raise RuntimeError("Invalid or duplicate GPU UUID") + seen.add(gpu) + if not all(value.isdigit() for value in (index, memory, utilization)): + raise RuntimeError("GPU occupancy is unknown") + if gpu not in busy and int(memory) <= 256 and int(utilization) == 0: + candidates.append((int(index), gpu)) + if len(candidates) < 2: + raise RuntimeError("Fewer than two idle GPUs; immediate test cannot run") + candidates.sort() + return candidates[:2] + + +def query_gpus(): + inventory = subprocess.run( + ["nvidia-smi", "--query-gpu=index,uuid,memory.used,utilization.gpu", "--format=csv,noheader,nounits"], + check=True, + text=True, + capture_output=True, + timeout=10, + ).stdout + processes = subprocess.run( + ["nvidia-smi", "--query-compute-apps=gpu_uuid", "--format=csv,noheader"], + check=True, + text=True, + capture_output=True, + timeout=10, + ).stdout + return select_idle(inventory, processes) + + +def has_gpu_pass(log): + for line in log.splitlines(): + try: + record = json.loads(line) + except ValueError: + continue + if isinstance(record, dict) and ( + record.get("result") == "E1_GPU_SMOKE_PASS" + and record.get("gpu_tested") is True + and record.get("backend") == "nccl" + and record.get("world_size") == 2 + ): + return True + return False + + +def has_colossalai_pass(log): + for line in log.splitlines(): + try: + record = json.loads(line) + except ValueError: + continue + if isinstance(record, dict) and record.get("result") == "E1_COLOSSALAI_PASS": + if record.get("gpu_tested") is True and record.get("tests_passed") == 2: + return True + return False + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--suite", choices=("smoke", "colossalai"), default="smoke") + args = parser.parse_args() + if os.name != "posix" or socket.gethostname().split(".")[0] != "gpu-h20-5": + raise RuntimeError("This job must execute on gpu-h20-5") + if os.environ.get("GITHUB_EVENT_NAME") != "pull_request": + raise RuntimeError("This entry point requires a GitHub pull_request event") + event = json.loads(Path(os.environ["GITHUB_EVENT_PATH"]).read_text()) + pr = validate_event(event) + args.output.mkdir(parents=True, mode=0o700) + result = { + "status": "failed", + "suite": args.suite, + "gpu_tested": False, + "website_contacted": False, + "allocation_source": "manual_authorization_idle_selection", + "host": "gpu-h20-5", + "pull_request": pr["number"], + "head_sha": pr["head"]["sha"], + "checkout_sha": os.environ.get("GITHUB_SHA"), + "source_method": "github_codeload_merge_commit", + "source_archive_sha256": os.environ.get("E1_SOURCE_ARCHIVE_SHA256"), + "runner": os.environ.get("RUNNER_NAME"), + "started_at": datetime.now(timezone.utc).isoformat(), + } + child = None + + def interrupted(signum, frame): + raise RuntimeError(f"Interrupted by signal {signum}") + + signal.signal(signal.SIGTERM, interrupted) + signal.signal(signal.SIGINT, interrupted) + try: + selected = query_gpus() + result["gpu_indices"] = [index for index, gpu in selected] + result["gpu_uuids"] = [gpu for index, gpu in selected] + print("Selected idle GPUs: " + json.dumps(selected), flush=True) + command = ["bash", str(Path(__file__).with_name("run_gpu_smoke.sh")), "--run", *result["gpu_uuids"]] + if args.suite == "colossalai": + command = [ + os.environ["E1_COLOSSALAI_PYTHON"], + "-B", + str(Path(__file__).with_name("colossalai_suite.py")), + "--output", + str(args.output / "colossalai"), + "--gpus", + *result["gpu_uuids"], + ] + env = {key: value for key, value in os.environ.items() if not key.startswith("E1_BOOKING_")} + with (args.output / "test.log").open("w") as log: + child = subprocess.Popen(command, stdout=log, stderr=subprocess.STDOUT, env=env, start_new_session=True) + code = child.wait(timeout=360) + log_text = (args.output / "test.log").read_text() + print(log_text, flush=True) + passed = has_gpu_pass(log_text) if args.suite == "smoke" else has_colossalai_pass(log_text) + if code != 0 or not passed: + raise RuntimeError(f"GPU test failed (exit={code}); no successful two-GPU qualification") + result.update(status="passed", gpu_tested=True) + except (RuntimeError, OSError, ValueError, subprocess.SubprocessError) as error: + result["error"] = str(error) + finally: + signal.signal(signal.SIGTERM, signal.SIG_IGN) + signal.signal(signal.SIGINT, signal.SIG_IGN) + if child is not None and child.poll() is None: + os.killpg(child.pid, signal.SIGTERM) + try: + child.wait(timeout=25) + except subprocess.TimeoutExpired: + os.killpg(child.pid, signal.SIGKILL) + child.wait(timeout=5) + result["finished_at"] = datetime.now(timezone.utc).isoformat() + (args.output / "result.json").write_text(json.dumps(result, indent=2) + "\n") + print(json.dumps(result, indent=2), flush=True) + return 0 if result["status"] == "passed" else 1 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except (RuntimeError, OSError, ValueError, KeyError) as error: + print(f"ERROR: {error}", file=sys.stderr) + sys.exit(1) diff --git a/.github/scripts/e1/run_gpu_smoke.sh b/.github/scripts/e1/run_gpu_smoke.sh new file mode 100644 index 000000000000..3ecad4b7ddf9 --- /dev/null +++ b/.github/scripts/e1/run_gpu_smoke.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env bash +# --check and --cpu-control never expose GPUs. --run requires two explicit UUIDs. +set -euo pipefail + +usage() { + cat <<'HELP' +Usage: + bash run_gpu_smoke.sh --check + bash run_gpu_smoke.sh --cpu-control + bash run_gpu_smoke.sh --run GPU-uuid-1 GPU-uuid-2 + +Default: --check. Obtain permission for both GPUs before using --run. +This script checks occupancy; it does not create or verify a reservation. +HELP +} + +die() { printf 'ERROR: %s\n' "$*" >&2; exit 1; } +mode="${1:---check}" +case "$mode" in + --help|-h) usage; exit 0 ;; + --check|--cpu-control) [ "$#" -le 1 ] || die 'Unexpected arguments' ;; + --run) [ "$#" -eq 3 ] || die '--run requires exactly two GPU UUIDs' ;; + *) usage >&2; exit 2 ;; +esac + +[ "$(hostname -s)" = gpu-h20-5 ] || die 'This initial configuration targets gpu-h20-5' +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +image='nvcr.io/nvidia/pytorch@sha256:025d9b102b5436d4af8af58f12c6a46b7e5d16f19543b1d2cc4446bf2650b4f1' +for tool in docker timeout nvidia-smi sha256sum; do + command -v "$tool" >/dev/null || die "Missing command: $tool" +done +docker image inspect "$image" >/dev/null || die 'The pinned image must already exist locally; no automatic pull' + +gpu_args=(--runtime runc --env NVIDIA_VISIBLE_DEVICES=void) +python_args=(python /e1/gpu_smoke.py --check) +if [ "$mode" = --cpu-control ] || [ "$mode" = --run ]; then + # A static loopback endpoint avoids hostname discovery in --network none. + # Every run has a separate network namespace, so this port is not shared. + python_args=(python -m torch.distributed.run --nnodes=1 --nproc-per-node=2 --master-addr=127.0.0.1 --master-port=29500 --max-restarts=0 /e1/gpu_smoke.py) + [ "$mode" != --cpu-control ] || python_args+=(--cpu-control) +fi + +if [ "$mode" = --run ]; then + command -v flock >/dev/null || die 'Missing flock' + [ "$2" != "$3" ] || die 'GPU UUIDs must be different' + for gpu in "$2" "$3"; do + [[ "$gpu" =~ ^GPU-[[:xdigit:]]{8}-[[:xdigit:]]{4}-[[:xdigit:]]{4}-[[:xdigit:]]{4}-[[:xdigit:]]{12}$ ]] || die 'Use full GPU UUIDs, not indices or all' + done + + # Node-local advisory lock: coordinates this wrapper only, not other users' jobs. + exec 9>"/tmp/colossalai-e1-ricardoo-${UID}.lock" + flock -n 9 || die 'Another E1 GPU smoke test holds the node-local lock' + + for gpu in "$2" "$3"; do + inventory="$(nvidia-smi -i "$gpu" --query-gpu=uuid,memory.used,utilization.gpu --format=csv,noheader,nounits)" + IFS=, read -r found memory utilization <<< "$inventory" + memory="${memory//[[:space:]]/}" + utilization="${utilization//[[:space:]]/}" + [ "$found" = "$gpu" ] || die "GPU UUID mismatch: $gpu" + [[ "$memory" =~ ^[0-9]+$ && "$utilization" =~ ^[0-9]+$ ]] || die "Cannot determine occupancy: $gpu" + [ "$memory" -le 256 ] && [ "$utilization" -eq 0 ] || die "GPU appears busy: $inventory" + processes="$(nvidia-smi -i "$gpu" --query-compute-apps=pid --format=csv,noheader,nounits)" + [ -z "${processes//[[:space:]]/}" ] || die "GPU has a compute process: $gpu" + done + gpu_args=(--runtime nvidia --gpus "\"device=$2,$3\"" --env "NVIDIA_VISIBLE_DEVICES=$2,$3") +fi + +results_root='/mnt/beegfs/ricardoo/ci/gpu-h20-5/test-results' +mkdir -p "$results_root" +result_dir="$(mktemp -d "$results_root/e1-$(date -u +%Y%m%dT%H%M%SZ).XXXXXX")" +container_name="e1-smoke-$(basename "$result_dir")" +cid_file="$result_dir/container.cid" + +cleanup() { + local rc=$? + trap - EXIT + if [ -s "$cid_file" ]; then + local cid + cid="$(cat "$cid_file")" + if [[ "$cid" =~ ^[[:xdigit:]]{64}$ ]]; then + timeout 20s docker rm -f "$cid" >/dev/null 2>&1 || true + fi + fi + printf '%s\n' "$rc" > "$result_dir/exit-code.txt" + printf 'Mode: %s; exit code: %s; results: %s\n' "$mode" "$rc" "$result_dir" + exit "$rc" +} +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +{ + printf 'utc=%s\nhost=%s\nmode=%s\nimage=%s\n' "$(date -u +%FT%TZ)" "$(hostname -s)" "$mode" "$image" + if [ "$mode" = --run ]; then + printf 'assigned_gpu_uuids=%s,%s\n' "$2" "$3" + fi + if git -C "$script_dir" rev-parse HEAD >/dev/null 2>&1; then + printf 'checkout_sha=%s\n' "$(git -C "$script_dir" rev-parse HEAD)" + fi + sha256sum "$script_dir/gpu_smoke.py" "$script_dir/run_gpu_smoke.sh" + docker image inspect "$image" --format 'image_id={{.Id}}' + nvidia-smi --query-gpu=index,uuid,name,driver_version,memory.used,utilization.gpu --format=csv +} | tee "$result_dir/environment.txt" + +# The in-container deadline also bounds execution if the SSH client disappears. +# A second host deadline plus EXIT cleanup handles a stuck docker client. +timeout --signal=TERM --kill-after=20s 300s \ + docker run --rm --init --pull never \ + --name "$container_name" --cidfile "$cid_file" \ + --network none --cpus 4 --memory 8g --memory-swap 8g --pids-limit 256 \ + --shm-size 1g --ulimit memlock=-1 \ + --cap-drop ALL --security-opt no-new-privileges \ + --user "$(id -u):$(id -g)" --workdir /tmp \ + --mount "type=bind,src=$script_dir,dst=/e1,readonly" \ + --env HOME=/tmp --env USER=colossalai-ci --env LOGNAME=colossalai-ci \ + --env TORCHINDUCTOR_CACHE_DIR=/tmp/torchinductor --env TRITON_CACHE_DIR=/tmp/triton \ + --env PYTHONDONTWRITEBYTECODE=1 --env OMP_NUM_THREADS=1 \ + --env NCCL_DEBUG=WARN --env NCCL_IB_DISABLE=1 --env NCCL_SOCKET_IFNAME=lo \ + --env GLOO_SOCKET_IFNAME=lo --env TORCH_NCCL_ASYNC_ERROR_HANDLING=1 \ + "${gpu_args[@]}" --entrypoint timeout "$image" \ + --signal=TERM --kill-after=15s 240s "${python_args[@]}" \ + 2>&1 | tee "$result_dir/container.log" diff --git a/.github/scripts/e1/test_pr_gpu.py b/.github/scripts/e1/test_pr_gpu.py new file mode 100644 index 000000000000..9d7c882398b6 --- /dev/null +++ b/.github/scripts/e1/test_pr_gpu.py @@ -0,0 +1,101 @@ +import copy +import json +import tempfile +import unittest +from pathlib import Path + +from colossalai_suite import validate_report +from pr_gpu import has_colossalai_pass, has_gpu_pass, select_idle, validate_event + +GPU_A = "GPU-00000000-0000-0000-0000-000000000001" +GPU_B = "GPU-00000000-0000-0000-0000-000000000002" +GPU_C = "GPU-00000000-0000-0000-0000-000000000003" + + +class PrGpuTests(unittest.TestCase): + def test_colossalai_requires_real_success_marker(self): + self.assertFalse(has_colossalai_pass("")) + self.assertFalse(has_colossalai_pass(json.dumps({"result": "E1_GPU_SMOKE_PASS", "gpu_tested": True}))) + self.assertFalse( + has_colossalai_pass(json.dumps({"result": "E1_COLOSSALAI_PASS", "gpu_tested": True, "tests_passed": 0})) + ) + self.assertTrue( + has_colossalai_pass(json.dumps({"result": "E1_COLOSSALAI_PASS", "gpu_tested": True, "tests_passed": 2})) + ) + + def test_junit_rejects_skips_failures_missing_and_extra_tests(self): + good = '' + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "junit.xml" + path.write_text("" + good + "") + self.assertEqual(validate_report(path), 2) + for bad in ( + "", + good + '', + good.replace('name="test_accelerator"/>', 'name="test_accelerator">'), + good.replace('name="test_accelerator"/>', 'name="test_accelerator">'), + ): + path.write_text("" + bad + "") + with self.assertRaises(RuntimeError): + validate_report(path) + + def test_busy_memory_is_not_idle_even_at_zero_utilization(self): + rows = f"0,{GPU_A},26000,0\n1,{GPU_B},4,0\n2,{GPU_C},4,0" + self.assertEqual(select_idle(rows, ""), [(1, GPU_B), (2, GPU_C)]) + + def test_existing_process_disqualifies_gpu(self): + with self.assertRaises(RuntimeError): + select_idle(f"0,{GPU_A},4,0\n1,{GPU_B},4,0", GPU_A) + + def test_unknown_occupancy_fails_closed(self): + for rows, processes in [(f"0,{GPU_A},N/A,0", ""), (f"0,{GPU_A},4,0", "N/A")]: + with self.subTest(rows=rows), self.assertRaises(RuntimeError): + select_idle(rows, processes) + + def test_duplicate_uuid_rejected(self): + with self.assertRaises(RuntimeError): + select_idle(f"0,{GPU_A},4,0\n1,{GPU_A},4,0", "") + + def test_cpu_pass_or_empty_log_is_not_gpu_success(self): + self.assertFalse(has_gpu_pass("")) + self.assertFalse(has_gpu_pass(json.dumps({"result": "E1_CPU_CONTROL_PASS", "gpu_tested": False}))) + self.assertTrue( + has_gpu_pass( + json.dumps({"result": "E1_GPU_SMOKE_PASS", "gpu_tested": True, "backend": "nccl", "world_size": 2}) + ) + ) + + def test_only_authorized_internal_pr_allowed(self): + event = { + "repository": {"full_name": "hpcaitech/ColossalAI"}, + "pull_request": { + "head": {"repo": {"full_name": "hpcaitech/ColossalAI"}, "ref": "ci/e1-runner-bootstrap"}, + "base": {"ref": "main", "repo": {"full_name": "hpcaitech/ColossalAI"}}, + "user": {"login": "richardoo-707"}, + }, + } + validate_event(event) + for change in ("author", "branch", "base_branch"): + other = copy.deepcopy(event) + if change == "author": + other["pull_request"]["user"]["login"] = "another-developer" + elif change == "branch": + other["pull_request"]["head"]["ref"] = "feature/another-internal-branch" + else: + other["pull_request"]["base"]["ref"] = "release/test" + with self.subTest(allowed=change): + validate_event(other) + for change in ("fork", "repository", "base_repository"): + other = copy.deepcopy(event) + if change == "fork": + other["pull_request"]["head"]["repo"]["full_name"] = "other/ColossalAI" + elif change == "repository": + other["repository"]["full_name"] = "other/ColossalAI" + else: + other["pull_request"]["base"]["repo"]["full_name"] = "other/ColossalAI" + with self.subTest(change=change), self.assertRaises(RuntimeError): + validate_event(other) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/workflows/e1-gpu-on-pr.yml b/.github/workflows/e1-gpu-on-pr.yml new file mode 100644 index 000000000000..c42f1a7e63c9 --- /dev/null +++ b/.github/workflows/e1-gpu-on-pr.yml @@ -0,0 +1,66 @@ +name: ColossalAI GPU on internal PR + +on: + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + +permissions: + contents: read + +concurrency: + group: e1-colossalai-pr-${{ github.event.pull_request.number }} + cancel-in-progress: false + +jobs: + gpu-smoke: + if: >- + github.repository == 'hpcaitech/ColossalAI' && + github.event.pull_request.head.repo.full_name == 'hpcaitech/ColossalAI' + runs-on: colossalai-e1-h20 + timeout-minutes: 12 + env: + E1_COLOSSALAI_PYTHON: /mnt/beegfs/ColossalAI/wangzhijian/envs/colossalai-torch213/bin/python + E1_PR_RESULTS: /mnt/beegfs/ricardoo/ci/gpu-h20-5/test-results/pr-${{ github.run_id }}-${{ github.run_attempt }} + steps: + - name: Fetch full source from the exact PR merge commit + timeout-minutes: 4 + shell: bash + run: | + set -euo pipefail + [[ "$GITHUB_SHA" =~ ^[0-9a-f]{40}$ ]] + source_dir="$(mktemp -d "$RUNNER_TEMP/e1-source.XXXXXX")" + archive="$source_dir/source.tar.gz" + curl --fail --location --proto '=https' --proto-redir '=https' \ + --connect-timeout 10 --max-time 180 --retry 1 --retry-max-time 200 \ + --output "$archive" \ + "https://codeload.github.com/hpcaitech/ColossalAI/tar.gz/$GITHUB_SHA" + tar --extract --gzip --file "$archive" --directory "$source_dir" \ + --strip-components=1 --no-same-owner --no-same-permissions + archive_sha="$(sha256sum "$archive" | cut -d ' ' -f 1)" + printf 'Source commit: %s\nArchive SHA256: %s\n' "$GITHUB_SHA" "$archive_sha" + printf 'E1_SOURCE_DIR=%s\nE1_SOURCE_ARCHIVE_SHA256=%s\n' \ + "$source_dir" "$archive_sha" >> "$GITHUB_ENV" + - name: Test resource selection without using GPUs + run: python3 -B -m unittest discover -s "$E1_SOURCE_DIR/.github/scripts/e1" -p test_pr_gpu.py -v + - name: Check ColossalAI dependencies and collect selected tests without GPUs + timeout-minutes: 2 + run: >- + "$E1_COLOSSALAI_PYTHON" -B "$E1_SOURCE_DIR/.github/scripts/e1/colossalai_suite.py" + --collect-only --output "$E1_PR_RESULTS-preflight" + - name: Run ColossalAI tests on two idle GPUs + run: python3 -B "$E1_SOURCE_DIR/.github/scripts/e1/pr_gpu.py" --suite colossalai --output "$E1_PR_RESULTS" + - name: Record result + if: always() + shell: bash + run: | + { + printf 'E1 ColossalAI two-GPU regression on gpu-h20-5\n\n' + printf 'Manual resource authorization; reservation website not used.\n\n' + if test -f "$E1_PR_RESULTS/result.json"; then + printf '```json\n' + cat "$E1_PR_RESULTS/result.json" + printf '\n```\n' + else + printf 'GPU test did not produce a result; inspect the failed step.\n' + fi + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/e1-runner-bootstrap.yml b/.github/workflows/e1-runner-bootstrap.yml new file mode 100644 index 000000000000..07f78f79976b --- /dev/null +++ b/.github/workflows/e1-runner-bootstrap.yml @@ -0,0 +1,60 @@ +name: E1 Runner Bootstrap + +on: + push: + branches: + - ci/e1-runner-bootstrap + paths: + - .github/workflows/e1-runner-bootstrap.yml + workflow_dispatch: + +permissions: {} + +jobs: + runner-probe: + name: Verify runner connectivity and host access + runs-on: colossalai-e1-h20 + timeout-minutes: 5 + + steps: + - name: Verify target host and temporary workspace + shell: bash + run: | + set -euo pipefail + + test "$(hostname -s)" = "gpu-h20-5" + + printf 'Runner: %s\n' "$RUNNER_NAME" + printf 'Host: %s\n' "$(hostname -s)" + printf 'Commit: %s\n' "$GITHUB_SHA" + + probe_file="$(mktemp "${RUNNER_TEMP}/e1-probe.XXXXXX")" + trap 'rm -f -- "$probe_file"' EXIT + + printf 'e1-runner-ok\n' > "$probe_file" + test "$(cat "$probe_file")" = "e1-runner-ok" + + - name: Verify Docker access + shell: bash + run: | + set -euo pipefail + docker version --format 'Docker server: {{.Server.Version}}' + + - name: Read GPU inventory + shell: bash + run: | + set -euo pipefail + nvidia-smi \ + --query-gpu=index,name,uuid,memory.total \ + --format=csv + + - name: Write run summary + shell: bash + run: | + { + printf 'Runner: %s\n\n' "$RUNNER_NAME" + printf 'Host: %s\n\n' "$(hostname -s)" + printf 'Commit: %s\n\n' "$GITHUB_SHA" + printf 'Passed: host, workspace, Docker access and GPU inventory.\n\n' + printf 'GPU computation and two-GPU communication were not tested.\n' + } >> "$GITHUB_STEP_SUMMARY"