From a33ae415efab31c58d618badc739aebb748ed829 Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Wed, 21 Oct 2026 15:12:06 +0200 Subject: [PATCH 01/25] Add a benchmark for fn.experimental.readers.video internal_tools/video_reader_bench.py measures the experimental video reader on DALI_extra data only, with one scenario per aspect of its performance: - lanes: many distinct files, one keyframe-aligned sequence each, so a batch holds `batch` different files: decode parallelism across samples vs. num_threads. - engines: K independent readers in one pipeline: scaling with the number of concurrent NVDEC sessions. - single_stream: per-stream throughput (H.264, HEVC 8/10-bit, 512x512). - reader_depth: the reader's own prefetch_queue_depth at batch 1. - cache_key: per-switch cost between two files of the same format, cropped (800x600 coded as 800x608) or not, against a no-switch control. - startup: build time and first batch latency with many files. - cpu: the CPU backend on many VP8/VP9 files vs. num_threads. - seq_longgop, random_access, switch: no-regression checks for sequential, shuffled and mixed-resolution access. Every configuration runs in a fresh process, --repeat times, round-robin; the median and spread are reported. NVDEC utilization and memory are sampled per process with NVML, and runs are flagged when other processes use the GPU. --hash records per-sample checksums and --compare checks that two builds, or two thread counts (--ignore threads), produce identical output. Signed-off-by: Joaquin Anton Guirao --- internal_tools/video_reader_bench.py | 836 +++++++++++++++++++++++++++ 1 file changed, 836 insertions(+) create mode 100644 internal_tools/video_reader_bench.py diff --git a/internal_tools/video_reader_bench.py b/internal_tools/video_reader_bench.py new file mode 100644 index 00000000000..196c0658bf8 --- /dev/null +++ b/internal_tools/video_reader_bench.py @@ -0,0 +1,836 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Benchmark for fn.experimental.readers.video, using DALI_extra data only. + +Scenarios (each isolates one aspect of the reader): + + lanes many distinct files, one keyframe-aligned sequence each, so every + batch holds `batch` different files and no sample needs a seek: + decode parallelism across samples vs. num_threads + seq_longgop 15 H.264 clips read sequentially (long GOPs, consecutive sequences + of the same file): no-regression check for sequential access + random_access the same clips shuffled: parallelism and keyframe-seek waste + together (do not read a lanes speedup from this one alone) + single_stream one clip, batch 1: per-stream throughput, i.e. per-frame + serialization. Includes a 512x512 clip, where per-frame overheads + matter more than decode time + reader_depth batch 1, the reader's own prefetch_queue_depth (batches in flight), + on many distinct files and on one sequential long-GOP file + cache_key every sample switches between two files with the same format, either + cropped (800x600 coded as 800x608) or not (1280x720): per-switch + cost and the NVDEC decoder cache + switch clips of mixed resolutions, shuffled, short sequences: per-sample + cost under constant file switching (random access) + startup many distinct files: pipeline build time and first batch latency + engines K independent readers in one pipeline, each on its own files: + how throughput scales with concurrent NVDEC sessions (works today, + because every reader decodes on its own thread) + cpu many distinct VP8/VP9 files on the CPU backend vs. num_threads + +Methodology: + * every configuration runs in a fresh subprocess (no cache carry-over), --repeat + times, round-robin over configurations; the median and the spread are reported + * timed iterations run until both --iters and --min_time are reached; throughput + excludes the first `depth` iterations, which may have been prefetched off the clock + * Pipeline.run() without cuda_stream returns host-synchronized outputs, so iteration + times include GPU completion + * NVML metrics are per process where possible (decoder utilization, memory). A + result is flagged as contaminated if other processes use the GPU or NVDEC is busy + before the run starts + * files are read once before each scenario, so all runs see a warm page cache + * DALI_extra has no 1080p content, so absolute rates are lower than for 1080p data; + compare builds, not datasets + +--hash records a checksum of every sample (frames, label, frame_num). Compare two +--hash result files (e.g. two builds, or --ignore threads to check that the output +does not depend on num_threads) with --compare. +""" + +import argparse +import glob +import hashlib +import json +import os +import random +import shutil +import socket +import statistics +import subprocess +import sys +import tempfile +import threading +import time + +VIDEO_DIR = "db/video" + +H264_CLIPS = ["sintel/video_files/*.mp4", "sintel/labelled_videos/*/*.mp4"] +VPX_CLIPS = [ + "vp9/vp9_0.mp4", + "vp9/vp9_2.mp4", + "vp8/vp8.webm", + "cfr/test_1_vp9.mp4", + "cfr/test_2_vp9.mp4", + "vfr/test_1_vp9.mp4", + "vfr/test_2_vp9.mp4", +] +# file sets: glob patterns relative to DALI_extra/db/video +FILES = { + # 15 H.264 1280x720 clips, 122-430 frames, B-frames, 1-4 keyframes (GOP up to 250) + "h264_clips": H264_CLIPS, + # one H.264 1280x720 clip, 1253 frames, GOP 250 + "h264_long": ["sintel/sintel_trailer-720p.mp4"], + # the same clip as HEVC 8-bit (keyframe every ~9 frames) and HEVC 10-bit (GOP 250) + "hevc_long": ["hevc/sintel_trailer-720p.mp4"], + "hevc10_long": ["hevc/sintel_trailer-720p_10b.mp4"], + # one H.264 512x512 clip, 250 frames: small frames, per-frame overheads dominate + "h264_small": ["test_label.mp4"], + # H.264 clips with mixed resolutions (1280x720, 512x512, 800x600 coded as 800x608) + "h264_mixed": H264_CLIPS + + [ + "cfr_test.mp4", + "vfr_test.mp4", + "test_label.mp4", + "frame_num_timestamp/test_25fps.mp4", + "multiple_framerate/50/50fps.mp4", + "cfr/test_2.mp4", + "vfr/test_2.mp4", + ], + # two H.264 800x600 clips, coded as 800x608 (display area cropped) + "h264_cropped_pair": ["cfr/test_2.mp4", "vfr/test_2.mp4"], + # two H.264 1280x720 clips, coded size == display size + "h264_aligned_pair": ["cfr_test.mp4", "vfr_test.mp4"], + # VP8/VP9 clips (H.264 is not enabled on the CPU backend) + "vpx_clips": VPX_CLIPS, +} + + +def _cfg(files, layout="plain", device="gpu", batch=8, seq=32, threads=4, **kw): + """A benchmark configuration. Only JSON-serializable values. + + layout: "plain" - the files as they are (filenames=...) + "many:N" - N distinct symlinks cycling over the files, each with one + sequence starting at frame 0 (a keyframe) + "alternate:N" - N lines alternating between the files, each starting at + frame 0, read in order: every sample switches files + "same:N" - N lines, all of the first file, each starting at frame 0: the + no-switch control for "alternate" (same seeks, no switches) + """ + cfg = dict( + files=files, + layout=layout, + device=device, + batch=batch, + seq=seq, + threads=threads, + depth=2, # pipeline prefetch_queue_depth (DALI default) + reader_depth=1, # the reader's own prefetch_queue_depth (DALI default) + readers=1, + shuffle=False, + ) + cfg.update(kw) + return cfg + + +def scenarios(threads, readers): + t_max = max(threads) + t_pair = sorted({1, t_max}) + return { + "lanes": [ + _cfg("h264_clips", "many:256", batch=b, seq=32, threads=t) + for b in (8, 16) + for t in threads + ], + "seq_longgop": [_cfg("h264_clips", batch=16, seq=32, threads=t) for t in t_pair], + "random_access": [ + _cfg("h264_clips", batch=16, seq=32, threads=t, shuffle=True) for t in t_pair + ], + "single_stream": [ + _cfg(f, batch=1, seq=64, threads=4) + for f in ("h264_long", "hevc_long", "hevc10_long", "h264_small") + ], + "reader_depth": [ + _cfg(f, layout, batch=1, seq=32, threads=8, reader_depth=d) + for f, layout in (("h264_clips", "many:256"), ("h264_long", "plain")) + for d in (1, 2, 4, 8) + ], + # switch cost = ms_per_sample(alternate) - ms_per_sample(same), per file pair + "cache_key": [ + _cfg(f, f"{kind}:64", batch=8, seq=8, threads=t) + for f in ("h264_cropped_pair", "h264_aligned_pair") + for kind in ("alternate", "same") + for t in t_pair + ], + "switch": [_cfg("h264_mixed", batch=8, seq=8, threads=t, shuffle=True) for t in t_pair], + "startup": [ + _cfg("h264_clips", "many:256", batch=1, seq=8, threads=t, iters=0) for t in t_pair + ], + "engines": [ + _cfg("h264_clips", "many:64", batch=4, seq=32, threads=4, readers=k) for k in readers + ], + "cpu": [ + _cfg("vpx_clips", "many:128", device="cpu", batch=b, seq=16, threads=t) + for b in (1, 8) + for t in threads + ], + } + + +# ---------------------------------------------------------------------------------------- +# data + + +def resolve_files(dali_extra, name): + root = os.path.join(dali_extra, VIDEO_DIR) + files, missing = [], [] + for pattern in FILES[name]: + matches = sorted(glob.glob(os.path.join(root, pattern))) + if not matches: + missing.append(pattern) + files += matches + if missing: + raise FileNotFoundError(f"file set {name}: not found under {root}: {missing}") + return files + + +def materialize(cfg, dali_extra, workdir): + """Creates the inputs of a configuration. Returns one input spec per reader. + + A spec is either {"filenames": [...]} or {"file_list": path}. The result is fully + determined by the configuration, so repeated runs (and --hash) see the same samples. + """ + files = resolve_files(dali_extra, cfg["files"]) + kind, _, n = cfg["layout"].partition(":") + if kind == "plain": + if cfg["readers"] != 1: + raise ValueError("layout 'plain' supports a single reader") + return [{"filenames": files}] + n = int(n) + seq = cfg["seq"] + tag = "_".join(str(cfg[k]) for k in ("files", "layout", "seq", "readers")) + out_dir = os.path.join(workdir, tag.replace(":", "-")) + os.makedirs(out_dir, exist_ok=True) + lines = [] + if kind == "many": + for i in range(n * cfg["readers"]): + src = files[i % len(files)] + dst = os.path.join(out_dir, f"{i:05d}_{os.path.basename(src)}") + if not os.path.lexists(dst): + os.symlink(src, dst) + lines.append((dst, i)) + elif kind == "alternate": + lines = [(files[i % len(files)], i) for i in range(n)] + elif kind == "same": + lines = [(files[0], i) for i in range(n)] + else: + raise ValueError(f"unknown layout {cfg['layout']}") + specs = [] + per_reader = len(lines) // cfg["readers"] + for r in range(cfg["readers"]): + path = os.path.join(out_dir, f"file_list_{r}.txt") + with open(path, "w") as f: + for name, label in lines[r * per_reader : (r + 1) * per_reader]: + # frames [0, seq) (file_list_include_end=False): exactly one sequence, starting + # at frame 0, a keyframe, so no sample decodes frames it doesn't return + f.write(f"{name} {label} 0 {seq}\n") + specs.append({"file_list": path}) + return specs + + +def prewarm(paths): + """Reads files once so that every run sees a warm page cache.""" + for p in sorted(set(paths)): + with open(p, "rb") as f: + while f.read(1 << 24): + pass + + +# ---------------------------------------------------------------------------------------- +# GPU monitoring + + +def nvml_handle(device_id): + """NVML handle of the CUDA device `device_id`, honoring CUDA_VISIBLE_DEVICES. + + Assumes CUDA_DEVICE_ORDER=PCI_BUS_ID or a machine where CUDA and NVML enumerate + devices the same way. Returns (pynvml, handle) or raises. + """ + import pynvml + + pynvml.nvmlInit() + visible = os.environ.get("CUDA_VISIBLE_DEVICES") + if visible: + entry = visible.split(",")[device_id].strip() + if entry.isdigit(): + return pynvml, pynvml.nvmlDeviceGetHandleByIndex(int(entry)) + return pynvml, pynvml.nvmlDeviceGetHandleByUUID(entry) + return pynvml, pynvml.nvmlDeviceGetHandleByIndex(device_id) + + +def other_gpu_pids(nvml, handle): + procs = nvml.nvmlDeviceGetComputeRunningProcesses(handle) + procs += nvml.nvmlDeviceGetGraphicsRunningProcesses(handle) + return sorted({p.pid for p in procs if p.pid != os.getpid()}) + + +class GpuMonitor: + """Samples per-process GPU memory and collects per-process NVDEC utilization.""" + + def __init__(self, device_id, period=0.1): + self.error = None + self.period = period + self.mem = [] + self.dev_dec = [] + self._stop = threading.Event() + self._thread = threading.Thread(target=self._run, daemon=True) + try: + self.nvml, self.handle = nvml_handle(device_id) + except Exception as e: + self.nvml = None + self.error = f"NVML unavailable: {e}" + + def snapshot(self): + """Pre/post-run state: other processes, NVDEC utilization, clocks.""" + if not self.nvml: + return {} + nvml, h = self.nvml, self.handle + return dict( + other_pids=other_gpu_pids(nvml, h), + dev_nvdec_util=nvml.nvmlDeviceGetDecoderUtilization(h)[0], + clock_video_mhz=nvml.nvmlDeviceGetClockInfo(h, nvml.NVML_CLOCK_VIDEO), + clock_sm_mhz=nvml.nvmlDeviceGetClockInfo(h, nvml.NVML_CLOCK_SM), + ) + + def _run(self): + nvml, h, pid = self.nvml, self.handle, os.getpid() + while not self._stop.is_set(): + try: + for p in nvml.nvmlDeviceGetComputeRunningProcesses(h): + if p.pid == pid and p.usedGpuMemory is not None: + self.mem.append(p.usedGpuMemory) + self.dev_dec.append(nvml.nvmlDeviceGetDecoderUtilization(h)[0]) + except Exception as e: + self.error = f"NVML sampling failed: {e}" + return + time.sleep(self.period) + + def start(self): + self._t_start_us = int(time.time() * 1e6) + if self.nvml: + self._thread.start() + + def stop(self): + if not self.nvml: + return {} + self._stop.set() + self._thread.join() + res = {} + if self.mem: + res["proc_mem_max_mb"] = max(self.mem) / 2**20 + if self.dev_dec: + res["dev_nvdec_util"] = statistics.mean(self.dev_dec) + try: # per-process decoder utilization since start, where the driver supports it + samples = self.nvml.nvmlDeviceGetProcessUtilization(self.handle, self._t_start_us) + dec = [s.decUtil for s in samples if s.pid == os.getpid()] + if dec: + res["proc_nvdec_util"] = statistics.mean(dec) + except Exception: + pass + return res + + +# ---------------------------------------------------------------------------------------- +# running + + +def build_pipeline(cfg, specs, device_id, seed, frame_num=False): + import nvidia.dali.fn as fn + from nvidia.dali.pipeline import pipeline_def + + @pipeline_def( + batch_size=cfg["batch"], + num_threads=cfg["threads"], + device_id=device_id, + prefetch_queue_depth=cfg["depth"], + seed=seed, + ) + def pipe(): + outs = [] + for r, spec in enumerate(specs): + kw = dict(spec) + if "filenames" in kw: + kw["labels"] = list(range(len(kw["filenames"]))) + else: + kw["file_list_format"] = "frames" + kw["file_list_include_end"] = False # explicit: the default changed over time + if "initial_fill" in cfg: + kw["initial_fill"] = cfg["initial_fill"] + if frame_num: + kw["enable_frame_num"] = True + outs += fn.experimental.readers.video( + device=cfg["device"], + sequence_length=cfg["seq"], + random_shuffle=cfg["shuffle"], + prefetch_queue_depth=cfg["reader_depth"], + name=f"Reader{r}", + **kw, + ) + return tuple(outs) + + return pipe() + + +def warm_up_cuda(device_id): + """Initializes CUDA and the executor so build_s measures the reader, not the runtime.""" + import nvidia.dali.fn as fn + from nvidia.dali.pipeline import pipeline_def + + @pipeline_def(batch_size=1, num_threads=1, device_id=device_id) + def tiny(): + return fn.random.uniform(shape=[1]).gpu() + + p = tiny() + p.build() + p.run() + del p + + +def percentile(sorted_vals, q): + if len(sorted_vals) == 1: + return sorted_vals[0] + return statistics.quantiles(sorted_vals, n=100, method="inclusive")[q - 1] + + +def run_one(cfg, specs, args): + """Runs one configuration in this process and returns its metrics.""" + import nvidia.dali as dali + + mon = GpuMonitor(args.device_id) + res = dict(dali_version=dali.__version__) + res["pre"] = mon.snapshot() + warm_up_cuda(args.device_id) + + t0 = time.perf_counter() + pipe = build_pipeline(cfg, specs, args.device_id, args.seed) + pipe.build() + res["build_s"] = time.perf_counter() - t0 + t0 = time.perf_counter() + pipe.run() + res["first_batch_ms"] = (time.perf_counter() - t0) * 1e3 + out = pipe.run() + res["shape"] = list(out[0].shape()[0]) + + iters = cfg.get("iters", args.iters) + if iters > 0: + for _ in range(args.warmup): + pipe.run() + frames_per_iter = cfg["readers"] * cfg["batch"] * cfg["seq"] + stamps = [] + mon.start() + t_start = time.perf_counter() + while len(stamps) < iters or stamps[-1] - t_start < args.min_time: + pipe.run() + stamps.append(time.perf_counter()) + res.update(mon.stop()) + times = sorted(b - a for a, b in zip([t_start] + stamps, stamps)) + times_ms = [t * 1e3 for t in times] + # the first `depth` batches may have been produced off the clock + skip = min(cfg["depth"], len(stamps) - 1) + n = len(stamps) - skip + span = stamps[-1] - (stamps[skip - 1] if skip else t_start) + h, w = res["shape"][1:3] + res.update( + iters=len(stamps), + frames_per_s=n * frames_per_iter / span, + mpix_per_s=n * frames_per_iter * h * w / span / 1e6, + ms_per_sample=span * 1e3 / (n * cfg["readers"] * cfg["batch"]), + iter_p50_ms=statistics.median(times_ms), + iter_p90_ms=percentile(times_ms, 90), + iter_p99_ms=percentile(times_ms, 99) if len(times_ms) >= 200 else None, + iter_max_ms=times_ms[-1], + ) + if mon.error: + res["nvml_error"] = mon.error + res["post"] = mon.snapshot() + pre, post = res["pre"], res["post"] + res["contaminated"] = bool( + pre.get("other_pids") or post.get("other_pids") or pre.get("dev_nvdec_util", 0) > 5 + ) + return res + + +def hash_one(cfg, specs, args): + """Returns one checksum per sample for the first --hash_batches batches.""" + pipe = build_pipeline(cfg, specs, args.device_id, args.seed, frame_num=True) + pipe.build() + hashes = [] + for _ in range(args.hash_batches): + outs = [_as_cpu(t) for t in pipe.run()] + for r in range(cfg["readers"]): + video, label, frame_num = outs[3 * r : 3 * r + 3] + for i in range(len(video)): + h = hashlib.sha1(video.at(i).tobytes()) + h.update(label.at(i).tobytes()) + h.update(frame_num.at(i).tobytes()) + hashes.append(h.hexdigest()) + return dict(hashes=hashes) + + +def _as_cpu(tensor_list): + return tensor_list.as_cpu() if hasattr(tensor_list, "as_cpu") else tensor_list + + +def child_main(args): + job = json.loads(args._single) + cfg, specs = job["cfg"], job["specs"] + try: + res = hash_one(cfg, specs, args) if args.hash else run_one(cfg, specs, args) + except Exception as e: + lines = [ln for ln in str(e).strip().splitlines() if ln.strip()] + detail = [ln for ln in lines if "rror" in ln or "ssert" in ln or "failed" in ln] + res = dict(error=f"{type(e).__name__}: " + " | ".join((detail or lines)[-3:])) + print("RESULT " + json.dumps(res)) + + +def run_child(cfg, specs, args): + cmd = [sys.executable, os.path.abspath(__file__), "--_single"] + cmd += [json.dumps(dict(cfg=cfg, specs=specs))] + for flag in ("dali_extra", "device_id", "iters", "warmup", "min_time", "seed"): + cmd += [f"--{flag}", str(getattr(args, flag))] + cmd += ["--hash_batches", str(args.hash_batches)] + (["--hash"] if args.hash else []) + try: + p = subprocess.run(cmd, capture_output=True, text=True, timeout=args.timeout) + except subprocess.TimeoutExpired: + return dict(error=f"timeout after {args.timeout} s") + for line in reversed(p.stdout.splitlines()): + if line.startswith("RESULT "): + return json.loads(line[len("RESULT ") :]) + tail = [ln for ln in (p.stderr or p.stdout).splitlines() if ln.strip()][-15:] + detail = [ln for ln in tail if "rror" in ln or "ssert" in ln] or tail[-3:] + return dict(error=f"exit code {p.returncode}: " + " | ".join(detail[-3:])) + + +def aggregate(cfg, reps): + """Median and spread over repetitions.""" + ok = [r for r in reps if "error" not in r] + res = dict(cfg) + res["repeats"] = len(reps) + if not ok: + res["error"] = reps[-1]["error"] if reps else "no runs" + return res + res["errors"] = [r["error"] for r in reps if "error" in r] + res["contaminated"] = any(r.get("contaminated") for r in ok) + res["runs"] = ok + for key in ( + "frames_per_s", + "mpix_per_s", + "ms_per_sample", + "iter_p50_ms", + "iter_p90_ms", + "iter_p99_ms", + "iter_max_ms", + "build_s", + "first_batch_ms", + "proc_nvdec_util", + "dev_nvdec_util", + "proc_mem_max_mb", + ): + vals = [r[key] for r in ok if r.get(key) is not None] + if vals: + res[key] = statistics.median(vals) + if key in ("frames_per_s", "build_s", "first_batch_ms") and len(vals) > 1: + res[key + "_spread_pct"] = (max(vals) - min(vals)) / res[key] * 100 + res["shape"] = ok[0].get("shape") + for key in ("dali_version", "nvml_error"): + if ok[0].get(key): + res[key] = ok[0][key] + return res + + +# ---------------------------------------------------------------------------------------- +# reporting + + +COLUMNS = [ + ("scenario", "scenario", "{:<13}"), + ("files", "files", "{:<17}"), + ("layout", "layout", "{:<12}"), + ("device", "dev", "{:<3}"), + ("batch", "bs", "{:>2}"), + ("seq", "seq", "{:>3}"), + ("threads", "thr", "{:>3}"), + ("reader_depth", "rq", "{:>2}"), + ("readers", "rd", "{:>2}"), + ("shuffle", "sh", "{:<2}"), + ("frames_per_s", "frames/s", "{:>8.1f}"), + ("frames_per_s_spread_pct", "spr%", "{:>4.1f}"), + ("ms_per_sample", "ms/smp", "{:>7.2f}"), + ("iter_p50_ms", "p50 ms", "{:>8.2f}"), + ("iter_p90_ms", "p90 ms", "{:>8.2f}"), + ("iter_max_ms", "max ms", "{:>8.2f}"), + ("build_s", "build s", "{:>7.3f}"), + ("first_batch_ms", "1st ms", "{:>7.1f}"), + ("proc_nvdec_util", "dec%", "{:>4.0f}"), + ("proc_mem_max_mb", "mem MB", "{:>6.0f}"), +] + + +def _width(fmt): + return int(fmt[3:].split(".")[0].rstrip("}")) + + +def print_header(): + print(" ".join(head.rjust(_width(fmt)) for _, head, fmt in COLUMNS) + " flags") + + +def format_row(r): + cells = [] + for key, head, fmt in COLUMNS: + v = r.get(key) + if key == "shuffle": + v = "y" if v else "n" + width = _width(fmt) + try: + cells.append(fmt.format(v) if v is not None else "-".rjust(width)) + except (ValueError, TypeError): + cells.append(str(v).ljust(width)) + flags = [] + if r.get("contaminated"): + flags.append("CONTAMINATED") + if r.get("frames_per_s_spread_pct", 0) > 3: + flags.append("spread>3%") + if r.get("errors"): + flags.append(f"{len(r['errors'])} failed run(s)") + if r.get("error"): + flags.append("ERROR: " + r["error"][:160]) + return " ".join(cells) + (" " + ", ".join(flags) if flags else "") + + +def provenance(args): + info = dict( + time=time.strftime("%Y-%m-%d %H:%M:%S"), + host=socket.gethostname(), + python=sys.version.split()[0], + cpu_count=os.cpu_count(), + argv=sys.argv, + isolation="subprocess per run", + env={k: v for k, v in os.environ.items() if k.startswith("DALI_") or k.startswith("CUDA_")}, + ) + try: + import nvidia.dali as dali + + info["dali_version"] = dali.__version__ + info["dali_git_sha"] = getattr(dali, "__git_sha__", None) + info["dali_path"] = os.path.dirname(dali.__file__) + except Exception as e: + info["dali_error"] = str(e) + try: + nvml, h = nvml_handle(args.device_id) + info["gpu"] = nvml.nvmlDeviceGetName(h) + info["gpu_uuid"] = nvml.nvmlDeviceGetUUID(h) + info["driver"] = nvml.nvmlSystemGetDriverVersion() + info["other_gpu_pids"] = other_gpu_pids(nvml, h) + info["nvdec_util_at_start"] = nvml.nvmlDeviceGetDecoderUtilization(h)[0] + except Exception as e: + info["nvml_error"] = str(e) + return info + + +def compare(a_path, b_path, ignore): + """Compares per-sample hashes of matching configurations in two --hash files. + + Fields listed in `ignore` (e.g. threads, reader_depth) are left out when matching, + so a configuration in A is compared against every configuration in B that differs + only in those fields. The output must not depend on them. + """ + a, b = (json.load(open(p)) for p in (a_path, b_path)) + skip = {"hashes", "scenario", "error", "errors", "dali_version", "repeats"} | set(ignore) + + def key(r): + return json.dumps({k: v for k, v in r.items() if k not in skip}, sort_keys=True) + + def label(r): + return ", ".join(f"{k}={r[k]}" for k in sorted(r) if k not in ("hashes", "errors")) + + b_runs = {} + for r in b["results"]: + b_runs.setdefault(key(r), []).append(r) + bad = compared = 0 + for r in a["results"]: + for other in b_runs.get(key(r), []): + if "hashes" not in r or "hashes" not in other: + print(f"skip (failed run): {label(r)}") + continue + n = max(len(r["hashes"]), len(other["hashes"])) + diff = sum(x != y for x, y in zip(r["hashes"], other["hashes"])) + diff += abs(len(r["hashes"]) - len(other["hashes"])) + compared += 1 + bad += diff > 0 + vs = ", ".join(f"{k}={other[k]}" for k in ignore if other.get(k) != r.get(k)) + status = "OK " if diff == 0 else "DIFF" + print(f"{status} {diff:4d}/{n} samples {label(r)}" + (f" vs {vs}" if vs else "")) + if compared == 0: + print("nothing to compare: no matching configurations") + return 2 + print("all identical" if bad == 0 else f"{bad} comparison(s) differ") + return 1 if bad else 0 + + +# ---------------------------------------------------------------------------------------- + + +def parse_args(): + parser = argparse.ArgumentParser( + description="fn.experimental.readers.video benchmark on DALI_extra data", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=__doc__, + ) + parser.add_argument("-s", "--scenarios", default="all", help="comma separated, or all") + parser.add_argument( + "-j", "--threads", default="1,2,4,8,16", help="num_threads sweep (B200: 1,2,4,7,8,14,16)" + ) + parser.add_argument("--readers", default="1,2,4,8", help="reader counts for 'engines'") + parser.add_argument("-d", "--device_id", default=0, type=int) + parser.add_argument("--repeat", default=3, type=int, help="fresh-process runs per config") + parser.add_argument("--iters", default=20, type=int, help="minimum timed iterations") + parser.add_argument("--min_time", default=5.0, type=float, help="minimum timed seconds") + parser.add_argument("--warmup", default=3, type=int, help="untimed iterations") + parser.add_argument("--timeout", default=900, type=int, help="seconds per run") + parser.add_argument("--seed", default=42, type=int) + parser.add_argument("--dali_extra", default=os.environ.get("DALI_EXTRA_PATH", "")) + parser.add_argument("--json", help="write results (with provenance) to this file") + parser.add_argument("--quick", action="store_true", help="smoke test: tiny sweep, 1 run") + parser.add_argument("--require_idle", action="store_true", help="abort if the GPU is busy") + parser.add_argument("--hash", action="store_true", help="record per-sample checksums") + parser.add_argument("--hash_batches", default=8, type=int) + parser.add_argument("--compare", nargs=2, metavar=("A", "B"), help="compare two --hash files") + parser.add_argument( + "--ignore", + default="", + help="with --compare: comma separated config fields to ignore when matching runs, " + "e.g. threads,reader_depth to check that output does not depend on them", + ) + parser.add_argument("--_single", help=argparse.SUPPRESS) + args = parser.parse_args() + if args.compare: + return parser, args + if not args.dali_extra: + parser.error("set DALI_EXTRA_PATH or pass --dali_extra") + if args.hash and not args.json and not args._single: + parser.error("--hash needs --json to store the checksums") + if args.quick: + args.threads, args.readers, args.repeat = "1,4", "1,2", 1 + args.iters, args.min_time, args.warmup = 3, 0.0, 1 + return parser, args + + +def main(): + parser, args = parse_args() + if args.compare: + sys.exit(compare(*args.compare, [f for f in args.ignore.split(",") if f])) + if args._single: + child_main(args) + return + + threads = [int(t) for t in args.threads.split(",")] + readers = [int(r) for r in args.readers.split(",")] + all_scenarios = scenarios(threads, readers) + names = list(all_scenarios) if args.scenarios == "all" else args.scenarios.split(",") + unknown = [n for n in names if n not in all_scenarios] + if unknown: + parser.error(f"unknown scenario(s) {unknown}; choose from {list(all_scenarios)}") + if args.hash: + args.repeat = 1 + + prov = provenance(args) + print("# " + json.dumps(prov)) + if prov.get("nvml_error"): + print(f"# WARNING: NVML unavailable ({prov['nvml_error']}); GPU checks are disabled") + busy = prov.get("other_gpu_pids") or prov.get("nvdec_util_at_start", 0) > 5 + if busy: + print("# WARNING: the GPU is in use by other processes; numbers will be skewed") + if args.require_idle: + sys.exit("GPU busy, aborting (--require_idle)") + + workdir = tempfile.mkdtemp(prefix="video_reader_bench_") + results, failed = [], 0 + try: + jobs = [] + for name in names: + for cfg in all_scenarios[name]: + cfg = dict(cfg, scenario=name) + try: + specs = materialize(cfg, args.dali_extra, workdir) + except Exception as e: + jobs.append((cfg, None, f"{type(e).__name__}: {e}")) + continue + jobs.append((cfg, specs, None)) + for cfg, specs, err in jobs: + if specs: + paths = [] + for s in specs: + if "filenames" in s: + paths += s["filenames"] + else: + paths += [ln.split()[0] for ln in open(s["file_list"])] + prewarm(os.path.realpath(p) for p in paths) + reps = {i: [] for i in range(len(jobs))} + rng = random.Random(args.seed) + for rep in range(args.repeat): + order = list(range(len(jobs))) + if rep > 0: + rng.shuffle(order) # round-robin, so drift does not line up with a config + for i in order: + cfg, specs, err = jobs[i] + res = dict(error=err) if err else run_child(cfg, specs, args) + reps[i].append(res) + note = res.get("error") or ( + f"{res['frames_per_s']:.1f} frames/s" if "frames_per_s" in res else "done" + ) + print( + f"[run {rep + 1}/{args.repeat}] {cfg['scenario']} {cfg['files']} " + f"{cfg['layout']} bs={cfg['batch']} thr={cfg['threads']} " + f"rq={cfg['reader_depth']} rd={cfg['readers']}: {note}", + file=sys.stderr, + flush=True, + ) + if not args.hash: + print_header() + for i, (cfg, specs, err) in enumerate(jobs): + if args.hash: + r = dict(cfg, seed=args.seed, hash_batches=args.hash_batches, **reps[i][0]) + n = len(r.get("hashes", [])) + print( + f"{cfg['scenario']} {cfg['files']} threads={cfg['threads']}: " + f"{n} sample hashes" + (f" ERROR: {r['error']}" if r.get("error") else "") + ) + else: + r = aggregate(cfg, reps[i]) + print(format_row(r)) + failed += bool(r.get("error")) + results.append(r) + finally: + shutil.rmtree(workdir, ignore_errors=True) + + if args.json: + with open(args.json, "w") as f: + json.dump(dict(provenance=prov, results=results), f, indent=1) + print(f"wrote {args.json}") + if failed: + print(f"{failed} configuration(s) failed") + sys.exit(1) + + +if __name__ == "__main__": + main() From e8bf6c8805074f8ee738bece309db7290ad8959d Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Wed, 21 Oct 2026 15:31:54 +0200 Subject: [PATCH 02/25] Add TL1_video_reader_perf qa test Runs internal_tools/video_reader_bench.py the way TL1_decoder_perf runs hw_decoder_bench.py: pinned to the same CPUs, re-run under nsys on failure with the profile and results saved to core_artifacts. The test fails if: - any benchmark configuration fails, - the output depends on num_threads (1 vs 16) or on the reader's prefetch_queue_depth (per-sample checksums, --hash/--compare), - throughput drops by more than 10% when num_threads (lanes, cpu) or the reader's prefetch_queue_depth grow, - an absolute minimum is set and not met. The minimums are empty for now, so the values are only printed for reference until they are calibrated on the CI machines. The checks live in qa/TL1_video_reader_perf/check_results.py, which reads the benchmark's JSON output. Also make the benchmark tolerate NVML queries that older bindings (nvidia-ml-py 11.450, as used in qa) do not support with newer drivers. Signed-off-by: Joaquin Anton Guirao --- internal_tools/video_reader_bench.py | 17 ++- qa/TL1_video_reader_perf/check_results.py | 140 ++++++++++++++++++++++ qa/TL1_video_reader_perf/test.sh | 116 ++++++++++++++++++ 3 files changed, 268 insertions(+), 5 deletions(-) create mode 100644 qa/TL1_video_reader_perf/check_results.py create mode 100644 qa/TL1_video_reader_perf/test.sh diff --git a/internal_tools/video_reader_bench.py b/internal_tools/video_reader_bench.py index 196c0658bf8..bf9613dc177 100644 --- a/internal_tools/video_reader_bench.py +++ b/internal_tools/video_reader_bench.py @@ -305,12 +305,19 @@ def snapshot(self): if not self.nvml: return {} nvml, h = self.nvml, self.handle - return dict( - other_pids=other_gpu_pids(nvml, h), - dev_nvdec_util=nvml.nvmlDeviceGetDecoderUtilization(h)[0], - clock_video_mhz=nvml.nvmlDeviceGetClockInfo(h, nvml.NVML_CLOCK_VIDEO), - clock_sm_mhz=nvml.nvmlDeviceGetClockInfo(h, nvml.NVML_CLOCK_SM), + res = {} + queries = dict( + other_pids=lambda: other_gpu_pids(nvml, h), + dev_nvdec_util=lambda: nvml.nvmlDeviceGetDecoderUtilization(h)[0], + clock_video_mhz=lambda: nvml.nvmlDeviceGetClockInfo(h, nvml.NVML_CLOCK_VIDEO), + clock_sm_mhz=lambda: nvml.nvmlDeviceGetClockInfo(h, nvml.NVML_CLOCK_SM), ) + for key, query in queries.items(): + try: # older NVML bindings may lack some queries for newer drivers + res[key] = query() + except Exception as e: + self.error = f"NVML {key} failed: {e}" + return res def _run(self): nvml, h, pid = self.nvml, self.handle, os.getpid() diff --git a/qa/TL1_video_reader_perf/check_results.py b/qa/TL1_video_reader_perf/check_results.py new file mode 100644 index 00000000000..4cbc9ea4fd3 --- /dev/null +++ b/qa/TL1_video_reader_perf/check_results.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Checks the results of internal_tools/video_reader_bench.py (--json output). + +Checks: + * no configuration failed + * hardware-independent relative checks, with a tolerance: + - lanes, cpu: throughput at the highest num_threads is not lower than at the + lowest one (more threads must not make the reader slower) + - reader_depth: a deeper reader queue is not slower than depth 1 + * optional absolute minimums (frames/s) for named metrics, passed with --min: + lanes lanes, batch 16, highest num_threads + single_stream single_stream, h264_long + cpu cpu, batch 8, highest num_threads + engines engines, highest number of readers + Metrics without a minimum are printed for reference only. + +Results flagged as contaminated (other processes used the GPU) only produce warnings. +Exits with 1 if any check failed. +""" + +import argparse +import json +import sys + + +def by(results, scenario, **match): + return [ + r + for r in results + if r.get("scenario") == scenario and all(r.get(k) == v for k, v in match.items()) + ] + + +def groups(results, scenario, keys): + out = {} + for r in by(results, scenario): + out.setdefault(tuple(r.get(k) for k in keys), []).append(r) + return out + + +def named_metrics(results): + """The metrics that can have absolute minimums.""" + + def at_max(rows, key): + rows = [r for r in rows if r.get("frames_per_s") is not None] + return max(rows, key=lambda r: r[key])["frames_per_s"] if rows else None + + return { + "lanes": at_max(by(results, "lanes", batch=16), "threads"), + "single_stream": next( + (r.get("frames_per_s") for r in by(results, "single_stream", files="h264_long")), None + ), + "cpu": at_max(by(results, "cpu", batch=8), "threads"), + "engines": at_max(by(results, "engines"), "readers"), + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("results", help="--json output of video_reader_bench.py") + parser.add_argument("--tolerance", type=float, default=10.0, help="percent, relative checks") + parser.add_argument( + "--min", action="append", default=[], metavar="NAME=FPS", help="absolute minimum" + ) + args = parser.parse_args() + + data = json.load(open(args.results)) + results = data["results"] + tol = args.tolerance / 100 + failed = [] + + def check(name, ok, detail): + print(f"CHECK {name}: {'OK' if ok else 'FAIL'} ({detail})") + if not ok: + failed.append(name) + + errors = [r for r in results if r.get("error")] + check("no failed configurations", not errors, f"{len(errors)} of {len(results)} failed") + for r in errors: + print(f" {r.get('scenario')} {r.get('files')} {r.get('layout')}: {r['error']}") + + contaminated = [r for r in results if r.get("contaminated")] + if contaminated: + print(f"WARNING: {len(contaminated)} result(s) contaminated by other GPU processes") + + def relative(scenario, keys, param): + for key, rows in sorted(groups(results, scenario, keys).items(), key=str): + rows = sorted((r for r in rows if r.get("frames_per_s")), key=lambda r: r[param]) + if len(rows) < 2: + continue + base = rows[0] + for r in rows[1:]: + ok = r["frames_per_s"] >= (1 - tol) * base["frames_per_s"] + label = "/".join(str(k) for k in key) + check( + f"{scenario} {label} {param}={r[param]} vs {base[param]}", + ok, + f"{r['frames_per_s']:.1f} vs {base['frames_per_s']:.1f} frames/s, " + f"tolerance {args.tolerance:g}%", + ) + + relative("lanes", ("files", "layout", "batch"), "threads") + relative("cpu", ("files", "layout", "batch"), "threads") + relative("reader_depth", ("files", "layout"), "reader_depth") + + minimums = {} + for item in args.min: + name, _, value = item.partition("=") + if value: + minimums[name] = float(value) + for name, value in named_metrics(results).items(): + if name in minimums: + check( + f"{name} minimum", + value is not None and value >= minimums[name], + f"{value} vs min {minimums[name]:g} frames/s", + ) + else: + print(f"REFERENCE {name}: {value} frames/s (no minimum set)") + + print("all checks passed" if not failed else f"{len(failed)} check(s) failed") + sys.exit(1 if failed else 0) + + +if __name__ == "__main__": + main() diff --git a/qa/TL1_video_reader_perf/test.sh b/qa/TL1_video_reader_perf/test.sh new file mode 100644 index 00000000000..183161a5af5 --- /dev/null +++ b/qa/TL1_video_reader_perf/test.sh @@ -0,0 +1,116 @@ +#!/bin/bash -e +# used pip packages +pip_packages='numpy nvidia-ml-py==11.450.51' +target_dir=./internal_tools + +# One-time pre-step: install nsys (NVIDIA Nsight Systems) if not present +do_once() { + if command -v nsys &>/dev/null; then + echo "nsys already installed: $(nsys --version)" + return + fi + DISTRO=$(source /etc/lsb-release && echo "$DISTRIB_RELEASE" | tr -d .) + ARCH=$(dpkg --print-architecture) + apt update && apt install -y --no-install-recommends gnupg curl + echo "deb [signed-by=/usr/share/keyrings/nvidia-devtools.gpg] https://developer.download.nvidia.com/devtools/repos/ubuntu${DISTRO}/${ARCH} /" \ + | tee /etc/apt/sources.list.d/nvidia-devtools.list + curl -fsSL "https://developer.download.nvidia.com/devtools/repos/ubuntu${DISTRO}/${ARCH}/nvidia.pub" \ + | gpg --dearmor -o /usr/share/keyrings/nvidia-devtools.gpg + apt update && apt install -y nsight-systems-cli +} + +LOG_PERF="video_reader_perf.log" +JSON_PERF="video_reader_perf.json" +JSON_HASH_LOW="video_reader_hash_low.json" +JSON_HASH_HIGH="video_reader_hash_high.json" + +# Scenarios of internal_tools/video_reader_bench.py run for performance; see its --help +PERF_SCENARIOS="lanes,engines,single_stream,reader_depth,cache_key,startup,seq_longgop,cpu" +# Scenarios whose output is checked to be independent of num_threads and reader_depth +HASH_SCENARIOS="lanes,reader_depth,cache_key,cpu" +THREADS_LOW=1 +THREADS_HIGH=16 +BENCH_ARGS="--repeat 3 --min_time 3 -j ${THREADS_LOW},4,${THREADS_HIGH} --readers 1,2,4" + +function CLEAN_AND_EXIT { + rm -rf ${LOG_PERF} ${JSON_PERF} ${JSON_HASH_LOW} ${JSON_HASH_HIGH} + exit $1 +} + +# Run the benchmark; if NSYS_REP is set, wrap it with nsys profiling (child processes included). +run_bench() { + local log_file="$1" + shift + if [ -n "${NSYS_REP}" ]; then + local profile_name="${log_file%.log}.nsys-rep" + nsys profile -o "${profile_name}" --trace=cuda,nvtx,osrt "$@" | tee "${log_file}" + else + "$@" | tee "${log_file}" + fi +} + +test_body() { + if [ "$(uname -p)" == "x86_64" ]; then + # Hopper + TASKSET="taskset --cpu-list 0-127" + # Absolute minimums in frames/s (see qa/TL1_video_reader_perf/check_results.py); + # empty: reported for reference only, not checked + MIN_LANES= + MIN_SINGLE_STREAM= + MIN_CPU= + MIN_ENGINES= + else + # GraceHopper + TASKSET="taskset --cpu-list 0-71" + MIN_LANES= + MIN_SINGLE_STREAM= + MIN_CPU= + MIN_ENGINES= + fi + + unset NSYS_REP + PERF_RUN="OK" + run_bench "${LOG_PERF}" ${TASKSET} python video_reader_bench.py ${BENCH_ARGS} \ + -s ${PERF_SCENARIOS} --json ${JSON_PERF} || PERF_RUN="FAIL" + + # The output must not depend on num_threads or on the reader's prefetch_queue_depth + HASH_RESULT="OK" + for threads in ${THREADS_LOW} ${THREADS_HIGH}; do + [ ${threads} == ${THREADS_LOW} ] && json=${JSON_HASH_LOW} || json=${JSON_HASH_HIGH} + ${TASKSET} python video_reader_bench.py -s ${HASH_SCENARIOS} -j ${threads} \ + --hash --hash_batches 4 --json ${json} || HASH_RESULT="FAIL" + done + python video_reader_bench.py --compare ${JSON_HASH_LOW} ${JSON_HASH_HIGH} --ignore threads \ + || HASH_RESULT="FAIL" + python video_reader_bench.py --compare ${JSON_HASH_HIGH} ${JSON_HASH_HIGH} \ + --ignore reader_depth || HASH_RESULT="FAIL" + + CHECK_RESULT="OK" + python ${topdir}/qa/TL1_video_reader_perf/check_results.py ${JSON_PERF} --tolerance 10 \ + --min lanes=${MIN_LANES} --min single_stream=${MIN_SINGLE_STREAM} \ + --min cpu=${MIN_CPU} --min engines=${MIN_ENGINES} || CHECK_RESULT="FAIL" + + echo "PERF_RUN=${PERF_RUN}" + echo "HASH_RESULT=${HASH_RESULT}" + echo "CHECK_RESULT=${CHECK_RESULT}" + + if [[ "$PERF_RUN" == "OK" && "$HASH_RESULT" == "OK" && "$CHECK_RESULT" == "OK" ]]; then + CLEAN_AND_EXIT 0 + fi + + # On failure: re-run the performance scenarios once with nsys and save the profile + echo "Video reader checks failed; re-running the benchmark with nsys profiling..." + ARTIFACTS_DIR="${topdir}/core_artifacts" + mkdir -p "${ARTIFACTS_DIR}" + + NSYS_REP="enabled" run_bench "${LOG_PERF}" ${TASKSET} python video_reader_bench.py \ + --repeat 1 --min_time 3 -j ${THREADS_LOW},${THREADS_HIGH} --readers 1,4 \ + -s ${PERF_SCENARIOS} || true + + cp -f *.nsys-rep ${JSON_PERF} "${ARTIFACTS_DIR}/" 2>/dev/null || true + echo "nsys profile and results saved to ${ARTIFACTS_DIR}" + CLEAN_AND_EXIT 1 +} +pushd ../.. +source ./qa/test_template.sh +popd From 2d49b4eb2c8d42b55aa4a909414b9a6fe0404a7a Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Wed, 21 Oct 2026 15:56:05 +0200 Subject: [PATCH 03/25] Address review comments on the video reader benchmark Results and checks: - --compare fails (exit 1) when a configuration failed or is missing in either file, reporting each failed run once with the file it came from. A build that cannot decode a workload no longer passes as "all identical". - A configuration counts as failed if any repetition failed: the benchmark exits with 1, and check_results.py rejects results with `errors`. Measurement: - Excluded iterations cover both the pipeline queue (`depth`) and the reader's own queue (`reader_depth`), and --iters/--min_time apply to the remaining window. - Frame and pixel rates count the actual output shapes of the timed batches, so mixed-resolution sets are reported correctly. - The GPU monitor records other GPU processes during the whole run, so a process that starts and exits mid-run marks the result as contaminated. A transient NVML failure no longer disables a sampler, and every NVML error is recorded. Robustness and style: - --dali_extra is made absolute, so symlinks in the temporary directory are valid with relative paths. - Validation errors name the parameter and show the expected and actual values; malformed layouts get a descriptive error. - Warnings use warnings.warn; files are always closed; no empty except. - The cache_key docs describe alternate - same as the cost of opening another file relative to rewinding the same one, not as an isolated switch cost. Signed-off-by: Joaquin Anton Guirao --- internal_tools/video_reader_bench.py | 224 ++++++++++++++++------ qa/TL1_video_reader_perf/check_results.py | 34 ++-- 2 files changed, 184 insertions(+), 74 deletions(-) diff --git a/internal_tools/video_reader_bench.py b/internal_tools/video_reader_bench.py index bf9613dc177..9426657bd84 100644 --- a/internal_tools/video_reader_bench.py +++ b/internal_tools/video_reader_bench.py @@ -29,8 +29,11 @@ reader_depth batch 1, the reader's own prefetch_queue_depth (batches in flight), on many distinct files and on one sequential long-GOP file cache_key every sample switches between two files with the same format, either - cropped (800x600 coded as 800x608) or not (1280x720): per-switch - cost and the NVDEC decoder cache + cropped (800x600 coded as 800x608) or not (1280x720), vs. a control + that rewinds the same file to frame 0 for every sample instead: + the cost of opening another file (decoder creation or NVDEC decoder + cache reuse) relative to a flush and seek within the same file. This + is a workload difference, not an isolated switch cost switch clips of mixed resolutions, shuffled, short sequences: per-sample cost under constant file switching (random access) startup many distinct files: pipeline build time and first batch latency @@ -42,8 +45,10 @@ Methodology: * every configuration runs in a fresh subprocess (no cache carry-over), --repeat times, round-robin over configurations; the median and the spread are reported - * timed iterations run until both --iters and --min_time are reached; throughput - excludes the first `depth` iterations, which may have been prefetched off the clock + * the first `depth` + `reader_depth` timed iterations are excluded, as up to that many + batches (pipeline queue and the reader's own queue) may have been produced off the + clock; measuring continues until both --iters and --min_time are reached after them + * throughput and pixel rates count the actual output shapes of the timed batches * Pipeline.run() without cuda_stream returns host-synchronized outputs, so iteration times include GPU completion * NVML metrics are per process where possible (decoder utilization, memory). A @@ -72,6 +77,7 @@ import tempfile import threading import time +import warnings VIDEO_DIR = "db/video" @@ -125,7 +131,8 @@ def _cfg(files, layout="plain", device="gpu", batch=8, seq=32, threads=4, **kw): "alternate:N" - N lines alternating between the files, each starting at frame 0, read in order: every sample switches files "same:N" - N lines, all of the first file, each starting at frame 0: the - no-switch control for "alternate" (same seeks, no switches) + control for "alternate" (no file switches, but every sample + after the first rewinds the same decoder to frame 0) """ cfg = dict( files=files, @@ -165,7 +172,8 @@ def scenarios(threads, readers): for f, layout in (("h264_clips", "many:256"), ("h264_long", "plain")) for d in (1, 2, 4, 8) ], - # switch cost = ms_per_sample(alternate) - ms_per_sample(same), per file pair + # ms_per_sample(alternate) - ms_per_sample(same), per file pair: cost of opening the + # other file relative to rewinding the same one (see the module docstring) "cache_key": [ _cfg(f, f"{kind}:64", batch=8, seq=8, threads=t) for f in ("h264_cropped_pair", "h264_aligned_pair") @@ -192,7 +200,8 @@ def scenarios(threads, readers): def resolve_files(dali_extra, name): - root = os.path.join(dali_extra, VIDEO_DIR) + # absolute paths: they are used as symlink targets in a temporary directory + root = os.path.abspath(os.path.join(dali_extra, VIDEO_DIR)) files, missing = [], [] for pattern in FILES[name]: matches = sorted(glob.glob(os.path.join(root, pattern))) @@ -200,7 +209,10 @@ def resolve_files(dali_extra, name): missing.append(pattern) files += matches if missing: - raise FileNotFoundError(f"file set {name}: not found under {root}: {missing}") + raise FileNotFoundError( + f"File set '{name}': no files match {missing} under '{root}'. " + f"Check that --dali_extra (or DALI_EXTRA_PATH) points to a DALI_extra checkout." + ) return files @@ -214,8 +226,13 @@ def materialize(cfg, dali_extra, workdir): kind, _, n = cfg["layout"].partition(":") if kind == "plain": if cfg["readers"] != 1: - raise ValueError("layout 'plain' supports a single reader") + raise ValueError(f"layout='plain' requires readers=1; got readers={cfg['readers']}.") return [{"filenames": files}] + if kind not in ("many", "alternate", "same") or not n.isdigit() or int(n) < 1: + raise ValueError( + f"Unknown layout {cfg['layout']!r}; expected 'plain', 'many:N', 'alternate:N' " + f"or 'same:N' with a positive integer N." + ) n = int(n) seq = cfg["seq"] tag = "_".join(str(cfg[k]) for k in ("files", "layout", "seq", "readers")) @@ -234,7 +251,10 @@ def materialize(cfg, dali_extra, workdir): elif kind == "same": lines = [(files[0], i) for i in range(n)] else: - raise ValueError(f"unknown layout {cfg['layout']}") + raise ValueError( + f"Unknown layout {cfg['layout']!r}; expected 'plain', 'many:N', 'alternate:N' " + f"or 'same:N'." + ) specs = [] per_reader = len(lines) // cfg["readers"] for r in range(cfg["readers"]): @@ -285,20 +305,29 @@ def other_gpu_pids(nvml, handle): class GpuMonitor: - """Samples per-process GPU memory and collects per-process NVDEC utilization.""" + """Samples per-process GPU memory and collects per-process NVDEC utilization. + + While sampling, it also records any other process that uses the GPU, so that a + competing process that starts and exits during a run is detected. + """ def __init__(self, device_id, period=0.1): - self.error = None + self.errors = [] # NVML problems; a metric may be missing, the run is still valid self.period = period self.mem = [] self.dev_dec = [] + self.others = set() self._stop = threading.Event() self._thread = threading.Thread(target=self._run, daemon=True) try: self.nvml, self.handle = nvml_handle(device_id) except Exception as e: self.nvml = None - self.error = f"NVML unavailable: {e}" + self._error(f"NVML unavailable: {e}") + + def _error(self, msg): + if msg not in self.errors: + self.errors.append(msg) def snapshot(self): """Pre/post-run state: other processes, NVDEC utilization, clocks.""" @@ -316,20 +345,37 @@ def snapshot(self): try: # older NVML bindings may lack some queries for newer drivers res[key] = query() except Exception as e: - self.error = f"NVML {key} failed: {e}" + self._error(f"NVML {key} failed: {e}") return res def _run(self): nvml, h, pid = self.nvml, self.handle, os.getpid() + + def own_memory(): + for p in nvml.nvmlDeviceGetComputeRunningProcesses(h): + if p.pid == pid and p.usedGpuMemory is not None: + self.mem.append(p.usedGpuMemory) + + samplers = { + "own memory": own_memory, + "other processes": lambda: self.others.update(other_gpu_pids(nvml, h)), + "decoder utilization": lambda: self.dev_dec.append( + nvml.nvmlDeviceGetDecoderUtilization(h)[0] + ), + } + failures = dict.fromkeys(samplers, 0) while not self._stop.is_set(): - try: - for p in nvml.nvmlDeviceGetComputeRunningProcesses(h): - if p.pid == pid and p.usedGpuMemory is not None: - self.mem.append(p.usedGpuMemory) - self.dev_dec.append(nvml.nvmlDeviceGetDecoderUtilization(h)[0]) - except Exception as e: - self.error = f"NVML sampling failed: {e}" - return + for name, sample in list(samplers.items()): + try: + sample() + failures[name] = 0 + except Exception as e: + # e.g. a process list that changed size during the query; only give up + # on a query that keeps failing (unsupported by these bindings/driver) + failures[name] += 1 + if failures[name] >= 3: + self._error(f"NVML sampling of {name} failed: {e}") + del samplers[name] time.sleep(self.period) def start(self): @@ -347,13 +393,15 @@ def stop(self): res["proc_mem_max_mb"] = max(self.mem) / 2**20 if self.dev_dec: res["dev_nvdec_util"] = statistics.mean(self.dev_dec) - try: # per-process decoder utilization since start, where the driver supports it + try: samples = self.nvml.nvmlDeviceGetProcessUtilization(self.handle, self._t_start_us) dec = [s.decUtil for s in samples if s.pid == os.getpid()] if dec: res["proc_nvdec_util"] = statistics.mean(dec) except Exception: - pass + # per-process utilization is optional: not every driver/binding supports it, or + # there may be no samples yet; the result is still valid without it + res["proc_nvdec_util"] = None return res @@ -410,7 +458,6 @@ def tiny(): p = tiny() p.build() p.run() - del p def percentile(sorted_vals, q): @@ -442,38 +489,49 @@ def run_one(cfg, specs, args): if iters > 0: for _ in range(args.warmup): pipe.run() - frames_per_iter = cfg["readers"] * cfg["batch"] * cfg["seq"] - stamps = [] + # Up to `depth` (pipeline queue) + `reader_depth` (the reader's own queue, filled + # by its prefetch thread) batches may have been produced before the clock started. + # They are excluded, and --iters/--min_time apply to the remaining window. + skip = cfg["depth"] + cfg["reader_depth"] + stamps, frames, pixels = [], [], [] mon.start() t_start = time.perf_counter() - while len(stamps) < iters or stamps[-1] - t_start < args.min_time: - pipe.run() + while True: + out = pipe.run() stamps.append(time.perf_counter()) + # actual output sizes (mixed resolutions); shapes are host-side metadata + shapes = [s for r in range(cfg["readers"]) for s in out[2 * r].shape()] + frames.append(sum(s[0] for s in shapes)) + pixels.append(sum(s[0] * s[1] * s[2] for s in shapes)) + kept = len(stamps) - skip + if kept > 0 and kept >= iters: + t_ref = stamps[skip - 1] if skip else t_start + if stamps[-1] - t_ref >= args.min_time: + break res.update(mon.stop()) - times = sorted(b - a for a, b in zip([t_start] + stamps, stamps)) - times_ms = [t * 1e3 for t in times] - # the first `depth` batches may have been produced off the clock - skip = min(cfg["depth"], len(stamps) - 1) + t_ref = stamps[skip - 1] if skip else t_start + span = stamps[-1] - t_ref n = len(stamps) - skip - span = stamps[-1] - (stamps[skip - 1] if skip else t_start) - h, w = res["shape"][1:3] + times_ms = sorted((b - a) * 1e3 for a, b in zip([t_ref] + stamps[skip:], stamps[skip:])) res.update( - iters=len(stamps), - frames_per_s=n * frames_per_iter / span, - mpix_per_s=n * frames_per_iter * h * w / span / 1e6, + iters=n, + iters_skipped=skip, + frames_per_s=sum(frames[skip:]) / span, + mpix_per_s=sum(pixels[skip:]) / span / 1e6, ms_per_sample=span * 1e3 / (n * cfg["readers"] * cfg["batch"]), iter_p50_ms=statistics.median(times_ms), iter_p90_ms=percentile(times_ms, 90), iter_p99_ms=percentile(times_ms, 99) if len(times_ms) >= 200 else None, iter_max_ms=times_ms[-1], ) - if mon.error: - res["nvml_error"] = mon.error res["post"] = mon.snapshot() + if mon.errors: # after the last NVML query, so that every failure is recorded + res["nvml_error"] = "; ".join(mon.errors) pre, post = res["pre"], res["post"] - res["contaminated"] = bool( - pre.get("other_pids") or post.get("other_pids") or pre.get("dev_nvdec_util", 0) > 5 + res["other_pids_seen"] = sorted( + mon.others | set(pre.get("other_pids", [])) | set(post.get("other_pids", [])) ) + res["contaminated"] = bool(res["other_pids_seen"] or pre.get("dev_nvdec_util", 0) > 5) return res @@ -644,13 +702,21 @@ def provenance(args): info["dali_error"] = str(e) try: nvml, h = nvml_handle(args.device_id) - info["gpu"] = nvml.nvmlDeviceGetName(h) - info["gpu_uuid"] = nvml.nvmlDeviceGetUUID(h) - info["driver"] = nvml.nvmlSystemGetDriverVersion() - info["other_gpu_pids"] = other_gpu_pids(nvml, h) - info["nvdec_util_at_start"] = nvml.nvmlDeviceGetDecoderUtilization(h)[0] except Exception as e: info["nvml_error"] = str(e) + return info + queries = dict( + gpu=lambda: nvml.nvmlDeviceGetName(h), + gpu_uuid=lambda: nvml.nvmlDeviceGetUUID(h), + driver=lambda: nvml.nvmlSystemGetDriverVersion(), + other_gpu_pids=lambda: other_gpu_pids(nvml, h), + nvdec_util_at_start=lambda: nvml.nvmlDeviceGetDecoderUtilization(h)[0], + ) + for key, query in queries.items(): + try: # one unsupported query must not hide the others + info[key] = query() + except Exception as e: + info.setdefault("nvml_query_errors", {})[key] = str(e) return info @@ -661,24 +727,42 @@ def compare(a_path, b_path, ignore): so a configuration in A is compared against every configuration in B that differs only in those fields. The output must not depend on them. """ - a, b = (json.load(open(p)) for p in (a_path, b_path)) + with open(a_path) as fa, open(b_path) as fb: + a, b = json.load(fa), json.load(fb) skip = {"hashes", "scenario", "error", "errors", "dali_version", "repeats"} | set(ignore) def key(r): return json.dumps({k: v for k, v in r.items() if k not in skip}, sort_keys=True) def label(r): - return ", ".join(f"{k}={r[k]}" for k in sorted(r) if k not in ("hashes", "errors")) + return ", ".join(f"{k}={r[k]}" for k in sorted(r) if k not in ("hashes", "errors", "error")) b_runs = {} for r in b["results"]: b_runs.setdefault(key(r), []).append(r) + a_keys = {key(r) for r in a["results"]} bad = compared = 0 + # a configuration that failed or is missing on either side fails the comparison: a + # build that cannot decode a workload must not pass as "identical" + for r in b["results"]: + if key(r) not in a_keys: + bad += 1 + print(f"MISSING in {a_path}: {label(r)}") + failed_runs = set() # (file, index): each failed run is reported and counted once + for path, results in ((a_path, a["results"]), (b_path, b["results"])): + for i, r in enumerate(results): + if "hashes" not in r and (path, i) not in failed_runs: + failed_runs.add((path, i)) + bad += 1 + print(f"FAILED in {path}: {label(r)} ({r.get('error', 'no checksums')})") for r in a["results"]: - for other in b_runs.get(key(r), []): + matches = b_runs.get(key(r), []) + if not matches: + bad += 1 + print(f"MISSING in {b_path}: {label(r)}") + for other in matches: if "hashes" not in r or "hashes" not in other: - print(f"skip (failed run): {label(r)}") - continue + continue # reported above n = max(len(r["hashes"]), len(other["hashes"])) diff = sum(x != y for x, y in zip(r["hashes"], other["hashes"])) diff += abs(len(r["hashes"]) - len(other["hashes"])) @@ -687,10 +771,10 @@ def label(r): vs = ", ".join(f"{k}={other[k]}" for k in ignore if other.get(k) != r.get(k)) status = "OK " if diff == 0 else "DIFF" print(f"{status} {diff:4d}/{n} samples {label(r)}" + (f" vs {vs}" if vs else "")) - if compared == 0: + if compared == 0 and bad == 0: print("nothing to compare: no matching configurations") return 2 - print("all identical" if bad == 0 else f"{bad} comparison(s) differ") + print("all identical" if bad == 0 else f"{bad} comparison(s) differ, failed or are missing") return 1 if bad else 0 @@ -733,9 +817,10 @@ def parse_args(): if args.compare: return parser, args if not args.dali_extra: - parser.error("set DALI_EXTRA_PATH or pass --dali_extra") + parser.error("--dali_extra is empty; set DALI_EXTRA_PATH or pass --dali_extra.") + args.dali_extra = os.path.abspath(args.dali_extra) if args.hash and not args.json and not args._single: - parser.error("--hash needs --json to store the checksums") + parser.error("--hash requires --json to store the checksums; got no --json.") if args.quick: args.threads, args.readers, args.repeat = "1,4", "1,2", 1 args.iters, args.min_time, args.warmup = 3, 0.0, 1 @@ -756,19 +841,30 @@ def main(): names = list(all_scenarios) if args.scenarios == "all" else args.scenarios.split(",") unknown = [n for n in names if n not in all_scenarios] if unknown: - parser.error(f"unknown scenario(s) {unknown}; choose from {list(all_scenarios)}") + parser.error( + f"--scenarios got unknown scenario(s) {unknown}; expected 'all' or a subset of " + f"{list(all_scenarios)}." + ) if args.hash: args.repeat = 1 prov = provenance(args) print("# " + json.dumps(prov)) if prov.get("nvml_error"): - print(f"# WARNING: NVML unavailable ({prov['nvml_error']}); GPU checks are disabled") - busy = prov.get("other_gpu_pids") or prov.get("nvdec_util_at_start", 0) > 5 + warnings.warn(f"NVML is unavailable ({prov['nvml_error']}); GPU checks are disabled.") + busy = prov.get("other_gpu_pids") or (prov.get("nvdec_util_at_start") or 0) > 5 if busy: - print("# WARNING: the GPU is in use by other processes; numbers will be skewed") + details = [] + if prov.get("other_gpu_pids"): + details.append(f"pids {prov['other_gpu_pids']}") + if prov.get("nvdec_util_at_start") is not None: + details.append(f"NVDEC utilization {prov['nvdec_util_at_start']}%") + warnings.warn( + f"The GPU is in use by other processes ({', '.join(details)}); " + f"the results will be skewed." + ) if args.require_idle: - sys.exit("GPU busy, aborting (--require_idle)") + sys.exit("The GPU is in use by other processes and --require_idle is set; aborting.") workdir = tempfile.mkdtemp(prefix="video_reader_bench_") results, failed = [], 0 @@ -790,7 +886,8 @@ def main(): if "filenames" in s: paths += s["filenames"] else: - paths += [ln.split()[0] for ln in open(s["file_list"])] + with open(s["file_list"]) as f: + paths += [ln.split()[0] for ln in f] prewarm(os.path.realpath(p) for p in paths) reps = {i: [] for i in range(len(jobs))} rng = random.Random(args.seed) @@ -825,7 +922,8 @@ def main(): else: r = aggregate(cfg, reps[i]) print(format_row(r)) - failed += bool(r.get("error")) + # a configuration fails if any of its repetitions failed + failed += bool(r.get("error") or r.get("errors")) results.append(r) finally: shutil.rmtree(workdir, ignore_errors=True) diff --git a/qa/TL1_video_reader_perf/check_results.py b/qa/TL1_video_reader_perf/check_results.py index 4cbc9ea4fd3..a7a7196136b 100644 --- a/qa/TL1_video_reader_perf/check_results.py +++ b/qa/TL1_video_reader_perf/check_results.py @@ -35,6 +35,7 @@ import argparse import json import sys +import warnings def by(results, scenario, **match): @@ -78,8 +79,19 @@ def main(): ) args = parser.parse_args() - data = json.load(open(args.results)) - results = data["results"] + with open(args.results) as f: + results = json.load(f)["results"] + metrics = named_metrics(results) + minimums = {} + for item in args.min: + name, _, value = item.partition("=") + if name not in metrics: + parser.error(f"--min got unknown metric {name!r}; expected one of {list(metrics)}.") + if value: # an empty value means: no minimum, report only + try: + minimums[name] = float(value) + except ValueError: + parser.error(f"--min {name} expects a number of frames/s; got {value!r}.") tol = args.tolerance / 100 failed = [] @@ -88,14 +100,19 @@ def check(name, ok, detail): if not ok: failed.append(name) - errors = [r for r in results if r.get("error")] + # a configuration fails if all its repetitions failed (`error`) or only some (`errors`) + errors = [r for r in results if r.get("error") or r.get("errors")] check("no failed configurations", not errors, f"{len(errors)} of {len(results)} failed") for r in errors: - print(f" {r.get('scenario')} {r.get('files')} {r.get('layout')}: {r['error']}") + for e in [r["error"]] if r.get("error") else r["errors"]: + print(f" {r.get('scenario')} {r.get('files')} {r.get('layout')}: {e}") contaminated = [r for r in results if r.get("contaminated")] if contaminated: - print(f"WARNING: {len(contaminated)} result(s) contaminated by other GPU processes") + warnings.warn( + f"{len(contaminated)} of {len(results)} results were measured while other " + f"processes used the GPU; their numbers may be skewed." + ) def relative(scenario, keys, param): for key, rows in sorted(groups(results, scenario, keys).items(), key=str): @@ -117,12 +134,7 @@ def relative(scenario, keys, param): relative("cpu", ("files", "layout", "batch"), "threads") relative("reader_depth", ("files", "layout"), "reader_depth") - minimums = {} - for item in args.min: - name, _, value = item.partition("=") - if value: - minimums[name] = float(value) - for name, value in named_metrics(results).items(): + for name, value in metrics.items(): if name in minimums: check( f"{name} minimum", From 9f418178bea229f80d887d15ba622c9c1659ec4e Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Wed, 21 Oct 2026 17:02:29 +0200 Subject: [PATCH 04/25] Handle bytes from older nvidia-ml-py in the video reader benchmark nvidia-ml-py 11.450.51, which the TL1 tests install, returns bytes from nvmlDeviceGetName and friends and expects bytes in nvmlDeviceGetHandleByUUID. Signed-off-by: Joaquin Anton Guirao --- internal_tools/video_reader_bench.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/internal_tools/video_reader_bench.py b/internal_tools/video_reader_bench.py index 9426657bd84..d59ab50af31 100644 --- a/internal_tools/video_reader_bench.py +++ b/internal_tools/video_reader_bench.py @@ -294,7 +294,8 @@ def nvml_handle(device_id): entry = visible.split(",")[device_id].strip() if entry.isdigit(): return pynvml, pynvml.nvmlDeviceGetHandleByIndex(int(entry)) - return pynvml, pynvml.nvmlDeviceGetHandleByUUID(entry) + # older nvidia-ml-py versions take and return bytes, newer ones accept both + return pynvml, pynvml.nvmlDeviceGetHandleByUUID(entry.encode()) return pynvml, pynvml.nvmlDeviceGetHandleByIndex(device_id) @@ -714,7 +715,8 @@ def provenance(args): ) for key, query in queries.items(): try: # one unsupported query must not hide the others - info[key] = query() + value = query() + info[key] = value.decode() if isinstance(value, bytes) else value except Exception as e: info.setdefault("nvml_query_errors", {})[key] = str(e) return info From 0316837a77d60da44b6ac935dad422181932f3ac Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Thu, 22 Oct 2026 09:40:14 +0200 Subject: [PATCH 05/25] Compare the video reader with PyNvVideoCodec and torchcodec video_reader_bench.py gets --backends: besides the DALI reader, PyNvVideoCodec (SimpleDecoder) and torchcodec (FFmpeg CUDA backend, its own NVDEC backend, and CPU) decode the same samples through their own APIs, one decoder per thread of a thread pool. They run the scenarios that mean the same for them. The default stays dali only. TL1_video_reader_perf installs torch, FFmpeg, PyNvVideoCodec and a torchcodec matching the torch version on x86_64, and runs the comparison once there. The checks still apply to the DALI results only. Signed-off-by: Joaquin Anton Guirao --- internal_tools/video_reader_bench.py | 254 +++++++++++++++++++--- qa/TL1_video_reader_perf/check_results.py | 5 +- qa/TL1_video_reader_perf/test.sh | 61 +++++- 3 files changed, 287 insertions(+), 33 deletions(-) diff --git a/internal_tools/video_reader_bench.py b/internal_tools/video_reader_bench.py index d59ab50af31..d8bf5d18a37 100644 --- a/internal_tools/video_reader_bench.py +++ b/internal_tools/video_reader_bench.py @@ -61,6 +61,18 @@ --hash records a checksum of every sample (frames, label, frame_num). Compare two --hash result files (e.g. two builds, or --ignore threads to check that the output does not depend on num_threads) with --compare. + +Other libraries (--backends), for reference: + pynvvideocodec PyNvVideoCodec SimpleDecoder (NVDEC), RGB interleaved output (as DALI) + torchcodec torchcodec VideoDecoder, FFmpeg CUDA backend on GPU, FFmpeg on CPU + torchcodec_beta torchcodec VideoDecoder with its own NVDEC backend (set_cuda_backend) + They decode the same samples as the DALI reader (the same sequences of the same files, + in the same order, or shuffled with a fixed seed), each sample with frames [start, end) + of one file. A batch is decoded by a pool of `threads` threads, each with its own + decoder, kept while consecutive samples come from the same file (PyNvVideoCodec + reconfigures its decoder for a new file, torchcodec opens a new one). Batches are not + prefetched: unlike DALI, a batch is decoded when it is requested. They run only the + scenarios that mean the same for them (BACKEND_SCENARIOS), and need torch. """ import argparse @@ -135,6 +147,7 @@ def _cfg(files, layout="plain", device="gpu", batch=8, seq=32, threads=4, **kw): after the first rewinds the same decoder to frame 0) """ cfg = dict( + backend="dali", files=files, layout=layout, device=device, @@ -195,6 +208,29 @@ def scenarios(threads, readers): } +# scenarios that other backends run as well (reader_depth, startup and engines measure +# properties of the DALI reader), and the devices they decode on +_REFERENCE_SCENARIOS = { + "lanes", + "seq_longgop", + "random_access", + "single_stream", + "cache_key", + "switch", +} +BACKEND_SCENARIOS = { + "dali": (None, ("gpu", "cpu")), # None: all scenarios + "pynvvideocodec": (_REFERENCE_SCENARIOS, ("gpu",)), + "torchcodec": (_REFERENCE_SCENARIOS | {"cpu"}, ("gpu", "cpu")), + "torchcodec_beta": (_REFERENCE_SCENARIOS, ("gpu",)), +} + + +def backend_runs(backend, scenario, device): + names, devices = BACKEND_SCENARIOS[backend] + return (names is None or scenario in names) and device in devices + + # ---------------------------------------------------------------------------------------- # data @@ -461,6 +497,150 @@ def tiny(): p.run() +class DaliRunner: + """The DALI reader. run() returns the (frames, height, width) of every sample.""" + + def __init__(self, cfg, specs, args): + import nvidia.dali as dali + + self.cfg = cfg + self.version = dali.__version__ + # up to `depth` (pipeline queue) + `reader_depth` (the reader's own queue) batches + # may be produced before the clock starts + self.prefetched = cfg["depth"] + cfg["reader_depth"] + warm_up_cuda(args.device_id) + self.pipe = build_pipeline(cfg, specs, args.device_id, args.seed) + + def build(self): + self.pipe.build() + + def run(self): + out = self.pipe.run() + # shapes are host-side metadata; video outputs are every other output (video, label) + return [tuple(s[:3]) for r in range(self.cfg["readers"]) for s in out[2 * r].shape()] + + +def sample_list(specs, seq, shuffle, num_frames): + """The samples the DALI reader produces for `specs`: (path, start, end) tuples. + + Whole files are split into full sequences of `seq` frames (the reader's default step + and stride); a file list gives one sample per line. DALI shuffles with its own seed, so + a shuffled list has the same samples in a different order. + """ + (spec,) = specs + samples = [] + if "filenames" in spec: + for path in spec["filenames"]: + n = num_frames(path) + samples += [(path, s, s + seq) for s in range(0, n - seq + 1, seq)] + else: + with open(spec["file_list"]) as f: + for line in f: + path, _, start, end = line.split() + samples.append((path, int(start), int(end))) + if shuffle: + random.Random(0).shuffle(samples) + return samples + + +class ReferenceRunner: + """PyNvVideoCodec or torchcodec, called directly through their own APIs. + + Each sample of a batch is decoded by one of `threads` threads; every thread keeps its + own decoder while consecutive samples come from the same file. Outputs are what the + libraries return; torch is used only to wait for the GPU. + """ + + prefetched = 0 # a batch is decoded when it is requested + + def __init__(self, cfg, specs, args): + import itertools + from concurrent.futures import ThreadPoolExecutor + + import torch + + self.cfg, self.torch = cfg, torch + self.backend, self.gpu = cfg["backend"], cfg["device"] == "gpu" + self.device_id = args.device_id + if self.gpu: + torch.cuda.set_device(self.device_id) + torch.zeros(1, device="cuda") # CUDA context, so build_s measures the decoders + torch.cuda.synchronize() + if self.backend == "pynvvideocodec": + import PyNvVideoCodec as nvc + + self.nvc, self.version = nvc, f"PyNvVideoCodec {nvc.__version__}" + else: + import torchcodec + + self.version = f"torchcodec {torchcodec.__version__}" + self.version += f", torch {torch.__version__}" + self.local = threading.local() + samples = sample_list(specs, cfg["seq"], cfg["shuffle"], self._num_frames) + self.samples = itertools.cycle(samples) + self.pool = ThreadPoolExecutor(cfg["threads"]) if cfg["threads"] > 1 else None + + def _open(self, path): + if self.backend == "pynvvideocodec": + return self.nvc.SimpleDecoder( + path, + gpu_id=self.device_id, + use_device_memory=True, + output_color_type=self.nvc.OutputColorType.RGB, # interleaved, as DALI + ) + from torchcodec.decoders import VideoDecoder + + if not self.gpu: + return VideoDecoder(path, device="cpu") + if self.backend == "torchcodec": # the default CUDA backend, FFmpeg's NVDEC support + return VideoDecoder(path, device=f"cuda:{self.device_id}") + from torchcodec.decoders import set_cuda_backend # torchcodec >= 0.8 + + with set_cuda_backend("beta"): + return VideoDecoder(path, device=f"cuda:{self.device_id}") + + def _num_frames(self, path): + return len(self._decoder(path)) + + def _decoder(self, path): + """This thread's decoder, switched to `path`.""" + local = self.local + if getattr(local, "path", None) == path: + return local.decoder + if self.gpu: + self.torch.cuda.set_device(self.device_id) # this thread's current device + if self.backend == "pynvvideocodec" and getattr(local, "decoder", None) is not None: + local.decoder.reconfigure_decoder(path) # picks a cached NVDEC decoder if any + else: + local.decoder = self._open(path) + local.path = path + return local.decoder + + def _decode(self, sample): + path, start, end = sample + decoder = self._decoder(path) + # the libraries' own outputs, not converted or copied + if self.backend == "pynvvideocodec": + frames = decoder.get_batch_frames_by_index(list(range(start, end))) + return (len(frames),) + tuple(frames[0].shape[:2]) # frames of H, W, C + video = decoder.get_frames_in_range(start, end).data + return (video.shape[0], video.shape[2], video.shape[3]) # F, C, H, W + + def build(self): + pass + + def run(self): + batch = [next(self.samples) for _ in range(self.cfg["batch"])] + shapes = list(self.pool.map(self._decode, batch) if self.pool else map(self._decode, batch)) + if self.gpu: + self.torch.cuda.synchronize() # as Pipeline.run(): outputs are ready + return shapes + + +def make_runner(cfg, specs, args): + return (DaliRunner if cfg["backend"] == "dali" else ReferenceRunner)(cfg, specs, args) + + def percentile(sorted_vals, q): if len(sorted_vals) == 1: return sorted_vals[0] @@ -469,39 +649,33 @@ def percentile(sorted_vals, q): def run_one(cfg, specs, args): """Runs one configuration in this process and returns its metrics.""" - import nvidia.dali as dali - mon = GpuMonitor(args.device_id) - res = dict(dali_version=dali.__version__) - res["pre"] = mon.snapshot() - warm_up_cuda(args.device_id) + res = dict(pre=mon.snapshot()) - t0 = time.perf_counter() - pipe = build_pipeline(cfg, specs, args.device_id, args.seed) - pipe.build() + runner = make_runner(cfg, specs, args) # initializes CUDA before the clock starts + res["backend_version"] = runner.version + t0 = time.perf_counter() # build_s: the reader or the decoders, not the CUDA runtime + runner.build() res["build_s"] = time.perf_counter() - t0 t0 = time.perf_counter() - pipe.run() + runner.run() res["first_batch_ms"] = (time.perf_counter() - t0) * 1e3 - out = pipe.run() - res["shape"] = list(out[0].shape()[0]) + res["shape"] = list(runner.run()[0]) iters = cfg.get("iters", args.iters) if iters > 0: for _ in range(args.warmup): - pipe.run() - # Up to `depth` (pipeline queue) + `reader_depth` (the reader's own queue, filled - # by its prefetch thread) batches may have been produced before the clock started. - # They are excluded, and --iters/--min_time apply to the remaining window. - skip = cfg["depth"] + cfg["reader_depth"] + runner.run() + # Up to runner.prefetched batches (DALI: `depth` in the pipeline queue + + # `reader_depth` in the reader's own queue) may have been produced before the clock + # started. They are excluded, and --iters/--min_time apply to the remaining window. + skip = runner.prefetched stamps, frames, pixels = [], [], [] mon.start() t_start = time.perf_counter() while True: - out = pipe.run() + shapes = runner.run() # actual (frames, height, width) of every sample stamps.append(time.perf_counter()) - # actual output sizes (mixed resolutions); shapes are host-side metadata - shapes = [s for r in range(cfg["readers"]) for s in out[2 * r].shape()] frames.append(sum(s[0] for s in shapes)) pixels.append(sum(s[0] * s[1] * s[2] for s in shapes)) kept = len(stamps) - skip @@ -618,7 +792,7 @@ def aggregate(cfg, reps): if key in ("frames_per_s", "build_s", "first_batch_ms") and len(vals) > 1: res[key + "_spread_pct"] = (max(vals) - min(vals)) / res[key] * 100 res["shape"] = ok[0].get("shape") - for key in ("dali_version", "nvml_error"): + for key in ("backend_version", "nvml_error"): if ok[0].get(key): res[key] = ok[0][key] return res @@ -630,6 +804,7 @@ def aggregate(cfg, reps): COLUMNS = [ ("scenario", "scenario", "{:<13}"), + ("backend", "backend", "{:<15}"), ("files", "files", "{:<17}"), ("layout", "layout", "{:<12}"), ("device", "dev", "{:<3}"), @@ -701,6 +876,13 @@ def provenance(args): info["dali_path"] = os.path.dirname(dali.__file__) except Exception as e: info["dali_error"] = str(e) + from importlib import metadata + + for pkg in ("PyNvVideoCodec", "torchcodec", "torch"): + try: + info.setdefault("packages", {})[pkg] = metadata.version(pkg) + except metadata.PackageNotFoundError: + pass try: nvml, h = nvml_handle(args.device_id) except Exception as e: @@ -731,7 +913,7 @@ def compare(a_path, b_path, ignore): """ with open(a_path) as fa, open(b_path) as fb: a, b = json.load(fa), json.load(fb) - skip = {"hashes", "scenario", "error", "errors", "dali_version", "repeats"} | set(ignore) + skip = {"hashes", "scenario", "error", "errors", "backend_version", "repeats"} | set(ignore) def key(r): return json.dumps({k: v for k, v in r.items() if k not in skip}, sort_keys=True) @@ -794,6 +976,12 @@ def parse_args(): "-j", "--threads", default="1,2,4,8,16", help="num_threads sweep (B200: 1,2,4,7,8,14,16)" ) parser.add_argument("--readers", default="1,2,4,8", help="reader counts for 'engines'") + parser.add_argument( + "-b", + "--backends", + default="dali", + help=f"comma separated, any of {list(BACKEND_SCENARIOS)}; others for reference", + ) parser.add_argument("-d", "--device_id", default=0, type=int) parser.add_argument("--repeat", default=3, type=int, help="fresh-process runs per config") parser.add_argument("--iters", default=20, type=int, help="minimum timed iterations") @@ -823,6 +1011,15 @@ def parse_args(): args.dali_extra = os.path.abspath(args.dali_extra) if args.hash and not args.json and not args._single: parser.error("--hash requires --json to store the checksums; got no --json.") + args.backends = args.backends.split(",") + unknown = [b for b in args.backends if b not in BACKEND_SCENARIOS] + if unknown: + parser.error( + f"--backends got unknown backend(s) {unknown}; expected a subset of " + f"{list(BACKEND_SCENARIOS)}." + ) + if args.hash and args.backends != ["dali"]: + parser.error(f"--hash supports only the dali backend; got --backends {args.backends}.") if args.quick: args.threads, args.readers, args.repeat = "1,4", "1,2", 1 args.iters, args.min_time, args.warmup = 3, 0.0, 1 @@ -871,16 +1068,21 @@ def main(): workdir = tempfile.mkdtemp(prefix="video_reader_bench_") results, failed = [], 0 try: - jobs = [] + jobs, not_run = [], set() for name in names: - for cfg in all_scenarios[name]: - cfg = dict(cfg, scenario=name) + for base, backend in [(c, b) for c in all_scenarios[name] for b in args.backends]: + if not backend_runs(backend, name, base["device"]): + not_run.add((backend, name)) + continue + cfg = dict(base, scenario=name, backend=backend) try: specs = materialize(cfg, args.dali_extra, workdir) except Exception as e: jobs.append((cfg, None, f"{type(e).__name__}: {e}")) continue jobs.append((cfg, specs, None)) + for backend, name in sorted(not_run): + print(f"# not run: {name} with {backend} (DALI reader specific or unsupported)") for cfg, specs, err in jobs: if specs: paths = [] @@ -905,8 +1107,8 @@ def main(): f"{res['frames_per_s']:.1f} frames/s" if "frames_per_s" in res else "done" ) print( - f"[run {rep + 1}/{args.repeat}] {cfg['scenario']} {cfg['files']} " - f"{cfg['layout']} bs={cfg['batch']} thr={cfg['threads']} " + f"[run {rep + 1}/{args.repeat}] {cfg['scenario']} {cfg['backend']} " + f"{cfg['files']} {cfg['layout']} bs={cfg['batch']} thr={cfg['threads']} " f"rq={cfg['reader_depth']} rd={cfg['readers']}: {note}", file=sys.stderr, flush=True, diff --git a/qa/TL1_video_reader_perf/check_results.py b/qa/TL1_video_reader_perf/check_results.py index a7a7196136b..9d6005e22e0 100644 --- a/qa/TL1_video_reader_perf/check_results.py +++ b/qa/TL1_video_reader_perf/check_results.py @@ -39,10 +39,13 @@ def by(results, scenario, **match): + """DALI results only; results of other libraries (--backends) are for reference.""" return [ r for r in results - if r.get("scenario") == scenario and all(r.get(k) == v for k, v in match.items()) + if r.get("scenario") == scenario + and r.get("backend", "dali") == "dali" + and all(r.get(k) == v for k, v in match.items()) ] diff --git a/qa/TL1_video_reader_perf/test.sh b/qa/TL1_video_reader_perf/test.sh index 183161a5af5..4c6df077936 100644 --- a/qa/TL1_video_reader_perf/test.sh +++ b/qa/TL1_video_reader_perf/test.sh @@ -1,17 +1,23 @@ #!/bin/bash -e -# used pip packages -pip_packages='numpy nvidia-ml-py==11.450.51' +# used pip packages; torch (pinned per CUDA version in qa/setup_packages.py) is needed by the +# reference libraries, see install_reference_libs +pip_packages='numpy nvidia-ml-py==11.450.51 torch' target_dir=./internal_tools -# One-time pre-step: install nsys (NVIDIA Nsight Systems) if not present +# One-time pre-step: FFmpeg shared libraries for torchcodec (x86_64 only, see +# install_reference_libs), and nsys (NVIDIA Nsight Systems) do_once() { + apt update + if [ "$(uname -m)" == "x86_64" ]; then + apt install -y --no-install-recommends ffmpeg + fi if command -v nsys &>/dev/null; then echo "nsys already installed: $(nsys --version)" return fi DISTRO=$(source /etc/lsb-release && echo "$DISTRIB_RELEASE" | tr -d .) ARCH=$(dpkg --print-architecture) - apt update && apt install -y --no-install-recommends gnupg curl + apt install -y --no-install-recommends gnupg curl echo "deb [signed-by=/usr/share/keyrings/nvidia-devtools.gpg] https://developer.download.nvidia.com/devtools/repos/ubuntu${DISTRO}/${ARCH} /" \ | tee /etc/apt/sources.list.d/nvidia-devtools.list curl -fsSL "https://developer.download.nvidia.com/devtools/repos/ubuntu${DISTRO}/${ARCH}/nvidia.pub" \ @@ -23,6 +29,34 @@ LOG_PERF="video_reader_perf.log" JSON_PERF="video_reader_perf.json" JSON_HASH_LOW="video_reader_hash_low.json" JSON_HASH_HIGH="video_reader_hash_high.json" +LOG_REF="video_reader_ref.log" +JSON_REF="video_reader_ref.json" + +# Other libraries run for reference on the same samples (video_reader_bench.py --backends), on +# x86_64 only; one run each, not checked against minimums. Installed by install_reference_libs. +REF_SCENARIOS="lanes,single_stream,cache_key,cpu" +REF_ARGS="--repeat 1 --min_time 3 -j 1,16" + +# Installs PyNvVideoCodec and torchcodec (x86_64 only) and sets REF_BACKENDS. torchcodec must +# match the torch version installed through pip_packages, and is built for the same CUDA version. +install_reference_libs() { + local torch_ver cuda_tag tc_ver + torch_ver=$(python -c "import torch; print(torch.__version__.split('+')[0])") + cuda_tag=$(python -c "import torch; print('cu' + torch.version.cuda.replace('.', ''))") + case "${torch_ver}" in + 2.7.*) tc_ver=0.5 ;; + 2.11.*) tc_ver=0.11.1 ;; + *) echo "No torchcodec version known for torch ${torch_ver}; update test.sh"; return 1 ;; + esac + pip install PyNvVideoCodec==2.0.2 || return 1 + # --no-deps: keep the torch from pip_packages + pip install --no-deps "torchcodec==${tc_ver}+${cuda_tag}" \ + --index-url "https://download.pytorch.org/whl/${cuda_tag}" || return 1 + REF_BACKENDS="pynvvideocodec,torchcodec" + # torchcodec 0.5 has no set_cuda_backend, i.e. no torchcodec_beta + [ "${tc_ver}" != "0.5" ] && REF_BACKENDS="${REF_BACKENDS},torchcodec_beta" + echo "Reference backends: ${REF_BACKENDS}" +} # Scenarios of internal_tools/video_reader_bench.py run for performance; see its --help PERF_SCENARIOS="lanes,engines,single_stream,reader_depth,cache_key,startup,seq_longgop,cpu" @@ -33,7 +67,7 @@ THREADS_HIGH=16 BENCH_ARGS="--repeat 3 --min_time 3 -j ${THREADS_LOW},4,${THREADS_HIGH} --readers 1,2,4" function CLEAN_AND_EXIT { - rm -rf ${LOG_PERF} ${JSON_PERF} ${JSON_HASH_LOW} ${JSON_HASH_HIGH} + rm -rf ${LOG_PERF} ${JSON_PERF} ${JSON_HASH_LOW} ${JSON_HASH_HIGH} ${LOG_REF} ${JSON_REF} exit $1 } @@ -90,11 +124,26 @@ test_body() { --min lanes=${MIN_LANES} --min single_stream=${MIN_SINGLE_STREAM} \ --min cpu=${MIN_CPU} --min engines=${MIN_ENGINES} || CHECK_RESULT="FAIL" + # The same samples decoded by other libraries, for reference (x86_64 only). A failure here + # (installation or decoding) fails the test, so that the comparison does not silently + # disappear. + REF_RUN="OK" + if [ "$(uname -m)" != "x86_64" ]; then + REF_RUN="SKIPPED" + elif install_reference_libs; then + run_bench "${LOG_REF}" ${TASKSET} python video_reader_bench.py ${REF_ARGS} \ + -s ${REF_SCENARIOS} -b dali,${REF_BACKENDS} --json ${JSON_REF} || REF_RUN="FAIL" + else + REF_RUN="FAIL" + fi + echo "PERF_RUN=${PERF_RUN}" echo "HASH_RESULT=${HASH_RESULT}" echo "CHECK_RESULT=${CHECK_RESULT}" + echo "REF_RUN=${REF_RUN}" - if [[ "$PERF_RUN" == "OK" && "$HASH_RESULT" == "OK" && "$CHECK_RESULT" == "OK" ]]; then + if [[ "$PERF_RUN" == "OK" && "$HASH_RESULT" == "OK" && "$CHECK_RESULT" == "OK" \ + && "$REF_RUN" != "FAIL" ]]; then CLEAN_AND_EXIT 0 fi From 4cfc6968abc0ec663e543742871a74e4a132e1c4 Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Thu, 22 Oct 2026 10:08:29 +0200 Subject: [PATCH 06/25] Flag GPU benchmark rows where NVDEC stays idle A GPU backend can fall back to CPU decoding for a format it can't decode on the GPU (e.g. torchcodec's FFmpeg CUDA backend on HEVC 10-bit). Also leave build_s empty for the reference libraries, which open decoders lazily. Signed-off-by: Joaquin Anton Guirao --- internal_tools/video_reader_bench.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/internal_tools/video_reader_bench.py b/internal_tools/video_reader_bench.py index d8bf5d18a37..102adeaa221 100644 --- a/internal_tools/video_reader_bench.py +++ b/internal_tools/video_reader_bench.py @@ -654,9 +654,10 @@ def run_one(cfg, specs, args): runner = make_runner(cfg, specs, args) # initializes CUDA before the clock starts res["backend_version"] = runner.version - t0 = time.perf_counter() # build_s: the reader or the decoders, not the CUDA runtime + t0 = time.perf_counter() # build_s: the DALI reader (indexing), not the CUDA runtime runner.build() - res["build_s"] = time.perf_counter() - t0 + if cfg["backend"] == "dali": # other libraries open their decoders lazily, in run() + res["build_s"] = time.perf_counter() - t0 t0 = time.perf_counter() runner.run() res["first_batch_ms"] = (time.perf_counter() - t0) * 1e3 @@ -851,6 +852,10 @@ def format_row(r): flags.append("CONTAMINATED") if r.get("frames_per_s_spread_pct", 0) > 3: flags.append("spread>3%") + if r.get("device") == "gpu" and r.get("proc_nvdec_util") is not None: + if r["proc_nvdec_util"] < 10: + # e.g. a GPU backend that falls back to CPU decoding for a format it can't decode + flags.append("NVDEC<10%") if r.get("errors"): flags.append(f"{len(r['errors'])} failed run(s)") if r.get("error"): From 98c33f70effb2d338dbcf8ffe14646e9cac10d54 Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Thu, 22 Oct 2026 10:19:42 +0200 Subject: [PATCH 07/25] Tighten the video reader benchmark and its TL1 test - Reference libraries: min(threads, batch) decoder threads, cache_key at one thread, and no PyNvVideoCodec runs that seek mid-file (its SimpleDecoder raises IndexError there). Flag PyNvVideoCodec rows on clips whose lead-in frames are hidden by an MP4 edit list: it returns different frames. - --hash checks at least 40 samples, so that the reader's prefetch queue wraps; hash seq_longgop and random_access (with the HEVC trailer) too. - TL1: one hash run and compare, lanes at batch 16 only, --iters 8 (engines 20), a short reference run without DALI rows, no startup scenario, and random_access in the performance run. Check seq_longgop and random_access over threads and engines scaling. Re-run under nsys only when the performance checks fail, save all results on failure and print them to the log on success. - Remove dead code. Signed-off-by: Joaquin Anton Guirao --- internal_tools/video_reader_bench.py | 119 +++++++++++++--------- qa/TL1_video_reader_perf/check_results.py | 32 +++++- qa/TL1_video_reader_perf/test.sh | 73 +++++++------ 3 files changed, 139 insertions(+), 85 deletions(-) diff --git a/internal_tools/video_reader_bench.py b/internal_tools/video_reader_bench.py index 102adeaa221..cc7e8a9f9ff 100644 --- a/internal_tools/video_reader_bench.py +++ b/internal_tools/video_reader_bench.py @@ -18,11 +18,15 @@ lanes many distinct files, one keyframe-aligned sequence each, so every batch holds `batch` different files and no sample needs a seek: - decode parallelism across samples vs. num_threads + per-file cost (opening, indexing, decoder setup, decoding the lead-in + frames an edit list hides, see LEAD_IN_FILES) plus decoding, vs. + num_threads. A per-file speedup also shows here; check `cache_key` + and `engines` before attributing a gain to decode parallelism seq_longgop 15 H.264 clips read sequentially (long GOPs, consecutive sequences of the same file): no-regression check for sequential access - random_access the same clips shuffled: parallelism and keyframe-seek waste - together (do not read a lanes speedup from this one alone) + random_access the same clips, and the HEVC trailer (open GOPs), shuffled: + parallelism and keyframe-seek waste together (do not read a lanes + speedup from this one alone) single_stream one clip, batch 1: per-stream throughput, i.e. per-frame serialization. Includes a 512x512 clip, where per-frame overheads matter more than decode time @@ -54,11 +58,12 @@ * NVML metrics are per process where possible (decoder utilization, memory). A result is flagged as contaminated if other processes use the GPU or NVDEC is busy before the run starts - * files are read once before each scenario, so all runs see a warm page cache + * all input files are read once before the first run, so all runs see a warm page cache * DALI_extra has no 1080p content, so absolute rates are lower than for 1080p data; compare builds, not datasets ---hash records a checksum of every sample (frames, label, frame_num). Compare two +--hash records a checksum of every sample (frames, label, frame_num) of at least +HASH_MIN_SAMPLES samples, so that the reader's prefetch queue wraps around. Compare two --hash result files (e.g. two builds, or --ignore threads to check that the output does not depend on num_threads) with --compare. @@ -68,11 +73,14 @@ torchcodec_beta torchcodec VideoDecoder with its own NVDEC backend (set_cuda_backend) They decode the same samples as the DALI reader (the same sequences of the same files, in the same order, or shuffled with a fixed seed), each sample with frames [start, end) - of one file. A batch is decoded by a pool of `threads` threads, each with its own - decoder, kept while consecutive samples come from the same file (PyNvVideoCodec - reconfigures its decoder for a new file, torchcodec opens a new one). Batches are not - prefetched: unlike DALI, a batch is decoded when it is requested. They run only the - scenarios that mean the same for them (BACKEND_SCENARIOS), and need torch. + of one file. A batch is decoded by a pool of min(threads, batch) threads, each with its + own decoder, kept while consecutive samples come from the same file (PyNvVideoCodec + reconfigures its decoder for a new file, torchcodec opens a new one, scanning it, as + its default seek_mode="exact" does). Batches are not prefetched: unlike DALI, a batch + is decoded when it is requested. They run only the configurations that mean the same + for them (backend_runs), and need torch. GPU rows where NVDEC stays nearly idle are + flagged, as a GPU backend may fall back to CPU decoding, and so are PyNvVideoCodec rows on + clips with hidden lead-in frames (LEAD_IN_FILES), where it returns different frames. """ import argparse @@ -93,6 +101,11 @@ VIDEO_DIR = "db/video" +# --hash checks at least this many samples per configuration: twice the batches the +# pipeline and the reader can hold (depth 2 + reader_depth up to 8) at batch 1, so that +# every slot of the reader's prefetch queue is reused +HASH_MIN_SAMPLES = 40 + H264_CLIPS = ["sintel/video_files/*.mp4", "sintel/labelled_videos/*/*.mp4"] VPX_CLIPS = [ "vp9/vp9_0.mp4", @@ -132,6 +145,11 @@ # VP8/VP9 clips (H.264 is not enabled on the CPU backend) "vpx_clips": VPX_CLIPS, } +# File sets where most clips (cut without re-encoding) start before the frame they show first, +# with an MP4 edit list hiding the lead-in frames: DALI and torchcodec start at the first +# visible frame, so they decode the lead-in before frame 0, while PyNvVideoCodec returns the +# lead-in frames themselves (different frames, less decoding) +LEAD_IN_FILES = {"h264_clips", "h264_mixed"} def _cfg(files, layout="plain", device="gpu", batch=8, seq=32, threads=4, **kw): @@ -163,18 +181,17 @@ def _cfg(files, layout="plain", device="gpu", batch=8, seq=32, threads=4, **kw): return cfg -def scenarios(threads, readers): +def scenarios(threads, readers, quick=False): t_max = max(threads) t_pair = sorted({1, t_max}) + iters = {} if quick else dict(iters=20) return { - "lanes": [ - _cfg("h264_clips", "many:256", batch=b, seq=32, threads=t) - for b in (8, 16) - for t in threads - ], + "lanes": [_cfg("h264_clips", "many:256", batch=16, seq=32, threads=t) for t in threads], "seq_longgop": [_cfg("h264_clips", batch=16, seq=32, threads=t) for t in t_pair], "random_access": [ - _cfg("h264_clips", batch=16, seq=32, threads=t, shuffle=True) for t in t_pair + _cfg(f, batch=16, seq=32, threads=t, shuffle=True) + for f in ("h264_clips", "hevc_long") + for t in t_pair ], "single_stream": [ _cfg(f, batch=1, seq=64, threads=4) @@ -186,19 +203,21 @@ def scenarios(threads, readers): for d in (1, 2, 4, 8) ], # ms_per_sample(alternate) - ms_per_sample(same), per file pair: cost of opening the - # other file relative to rewinding the same one (see the module docstring) + # other file relative to rewinding the same one (see the module docstring). One + # thread: with more, other libraries keep each file of a pair on its own decoder "cache_key": [ - _cfg(f, f"{kind}:64", batch=8, seq=8, threads=t) + _cfg(f, f"{kind}:64", batch=8, seq=8, threads=1) for f in ("h264_cropped_pair", "h264_aligned_pair") for kind in ("alternate", "same") - for t in t_pair ], "switch": [_cfg("h264_mixed", batch=8, seq=8, threads=t, shuffle=True) for t in t_pair], "startup": [ _cfg("h264_clips", "many:256", batch=1, seq=8, threads=t, iters=0) for t in t_pair ], + # the noisiest scenario: more iterations than the --iters default "engines": [ - _cfg("h264_clips", "many:64", batch=4, seq=32, threads=4, readers=k) for k in readers + _cfg("h264_clips", "many:64", batch=4, seq=32, threads=4, readers=k, **iters) + for k in readers ], "cpu": [ _cfg("vpx_clips", "many:128", device="cpu", batch=b, seq=16, threads=t) @@ -226,9 +245,17 @@ def scenarios(threads, readers): } -def backend_runs(backend, scenario, device): +def backend_runs(backend, scenario, cfg): names, devices = BACKEND_SCENARIOS[backend] - return (names is None or scenario in names) and device in devices + if (names is not None and scenario not in names) or cfg["device"] not in devices: + return False + # PyNvVideoCodec 2.0 SimpleDecoder fails (IndexError) when a decoder that was just + # created or reconfigured for a file is asked for frames from the middle of it, so it + # runs only configurations whose decoders start every file at frame 0 + seeks = scenario in ("random_access", "switch") or ( + scenario == "seq_longgop" and min(cfg["threads"], cfg["batch"]) > 1 + ) + return not (backend == "pynvvideocodec" and seeks) # ---------------------------------------------------------------------------------------- @@ -284,13 +311,8 @@ def materialize(cfg, dali_extra, workdir): lines.append((dst, i)) elif kind == "alternate": lines = [(files[i % len(files)], i) for i in range(n)] - elif kind == "same": + else: # same lines = [(files[0], i) for i in range(n)] - else: - raise ValueError( - f"Unknown layout {cfg['layout']!r}; expected 'plain', 'many:N', 'alternate:N' " - f"or 'same:N'." - ) specs = [] per_reader = len(lines) // cfg["readers"] for r in range(cfg["readers"]): @@ -367,7 +389,7 @@ def _error(self, msg): self.errors.append(msg) def snapshot(self): - """Pre/post-run state: other processes, NVDEC utilization, clocks.""" + """Pre/post-run state: other processes, NVDEC utilization.""" if not self.nvml: return {} nvml, h = self.nvml, self.handle @@ -375,8 +397,6 @@ def snapshot(self): queries = dict( other_pids=lambda: other_gpu_pids(nvml, h), dev_nvdec_util=lambda: nvml.nvmlDeviceGetDecoderUtilization(h)[0], - clock_video_mhz=lambda: nvml.nvmlDeviceGetClockInfo(h, nvml.NVML_CLOCK_VIDEO), - clock_sm_mhz=lambda: nvml.nvmlDeviceGetClockInfo(h, nvml.NVML_CLOCK_SM), ) for key, query in queries.items(): try: # older NVML bindings may lack some queries for newer drivers @@ -466,8 +486,6 @@ def pipe(): else: kw["file_list_format"] = "frames" kw["file_list_include_end"] = False # explicit: the default changed over time - if "initial_fill" in cfg: - kw["initial_fill"] = cfg["initial_fill"] if frame_num: kw["enable_frame_num"] = True outs += fn.experimental.readers.video( @@ -578,7 +596,10 @@ def __init__(self, cfg, specs, args): self.local = threading.local() samples = sample_list(specs, cfg["seq"], cfg["shuffle"], self._num_frames) self.samples = itertools.cycle(samples) - self.pool = ThreadPoolExecutor(cfg["threads"]) if cfg["threads"] > 1 else None + # no more threads than samples per batch: an idle extra thread would take the next + # batch's samples of the same file and make its decoder seek + workers = min(cfg["threads"], cfg["batch"]) + self.pool = ThreadPoolExecutor(workers) if workers > 1 else None def _open(self, path): if self.backend == "pynvvideocodec": @@ -685,7 +706,6 @@ def run_one(cfg, specs, args): if stamps[-1] - t_ref >= args.min_time: break res.update(mon.stop()) - t_ref = stamps[skip - 1] if skip else t_start span = stamps[-1] - t_ref n = len(stamps) - skip times_ms = sorted((b - a) * 1e3 for a, b in zip([t_ref] + stamps[skip:], stamps[skip:])) @@ -697,7 +717,6 @@ def run_one(cfg, specs, args): ms_per_sample=span * 1e3 / (n * cfg["readers"] * cfg["batch"]), iter_p50_ms=statistics.median(times_ms), iter_p90_ms=percentile(times_ms, 90), - iter_p99_ms=percentile(times_ms, 99) if len(times_ms) >= 200 else None, iter_max_ms=times_ms[-1], ) res["post"] = mon.snapshot() @@ -712,11 +731,13 @@ def run_one(cfg, specs, args): def hash_one(cfg, specs, args): - """Returns one checksum per sample for the first --hash_batches batches.""" + """Returns one checksum per sample of the first batches (see HASH_MIN_SAMPLES).""" pipe = build_pipeline(cfg, specs, args.device_id, args.seed, frame_num=True) pipe.build() hashes = [] - for _ in range(args.hash_batches): + # Not dependent on threads or reader_depth, so that --compare --ignore can match runs + batches = max(args.hash_batches, -(-HASH_MIN_SAMPLES // (cfg["batch"] * cfg["readers"]))) + for _ in range(batches): outs = [_as_cpu(t) for t in pipe.run()] for r in range(cfg["readers"]): video, label, frame_num = outs[3 * r : 3 * r + 3] @@ -725,7 +746,7 @@ def hash_one(cfg, specs, args): h.update(label.at(i).tobytes()) h.update(frame_num.at(i).tobytes()) hashes.append(h.hexdigest()) - return dict(hashes=hashes) + return dict(hashes=hashes, hash_batches=batches) def _as_cpu(tensor_list): @@ -779,7 +800,6 @@ def aggregate(cfg, reps): "ms_per_sample", "iter_p50_ms", "iter_p90_ms", - "iter_p99_ms", "iter_max_ms", "build_s", "first_batch_ms", @@ -856,6 +876,8 @@ def format_row(r): if r["proc_nvdec_util"] < 10: # e.g. a GPU backend that falls back to CPU decoding for a format it can't decode flags.append("NVDEC<10%") + if r.get("backend") == "pynvvideocodec" and r.get("files") in LEAD_IN_FILES: + flags.append("lead-in frames") # not the same frames as DALI, see LEAD_IN_FILES if r.get("errors"): flags.append(f"{len(r['errors'])} failed run(s)") if r.get("error"): @@ -870,7 +892,6 @@ def provenance(args): python=sys.version.split()[0], cpu_count=os.cpu_count(), argv=sys.argv, - isolation="subprocess per run", env={k: v for k, v in os.environ.items() if k.startswith("DALI_") or k.startswith("CUDA_")}, ) try: @@ -918,7 +939,7 @@ def compare(a_path, b_path, ignore): """ with open(a_path) as fa, open(b_path) as fb: a, b = json.load(fa), json.load(fb) - skip = {"hashes", "scenario", "error", "errors", "backend_version", "repeats"} | set(ignore) + skip = {"hashes", "scenario", "error", "errors"} | set(ignore) def key(r): return json.dumps({k: v for k, v in r.items() if k not in skip}, sort_keys=True) @@ -1041,7 +1062,7 @@ def main(): threads = [int(t) for t in args.threads.split(",")] readers = [int(r) for r in args.readers.split(",")] - all_scenarios = scenarios(threads, readers) + all_scenarios = scenarios(threads, readers, args.quick) names = list(all_scenarios) if args.scenarios == "all" else args.scenarios.split(",") unknown = [n for n in names if n not in all_scenarios] if unknown: @@ -1076,8 +1097,8 @@ def main(): jobs, not_run = [], set() for name in names: for base, backend in [(c, b) for c in all_scenarios[name] for b in args.backends]: - if not backend_runs(backend, name, base["device"]): - not_run.add((backend, name)) + if not backend_runs(backend, name, base): + not_run.add((backend, name, base["threads"])) continue cfg = dict(base, scenario=name, backend=backend) try: @@ -1086,8 +1107,8 @@ def main(): jobs.append((cfg, None, f"{type(e).__name__}: {e}")) continue jobs.append((cfg, specs, None)) - for backend, name in sorted(not_run): - print(f"# not run: {name} with {backend} (DALI reader specific or unsupported)") + for backend, name, threads in sorted(not_run): + print(f"# not run: {name} threads={threads} with {backend} (see backend_runs)") for cfg, specs, err in jobs: if specs: paths = [] @@ -1122,7 +1143,7 @@ def main(): print_header() for i, (cfg, specs, err) in enumerate(jobs): if args.hash: - r = dict(cfg, seed=args.seed, hash_batches=args.hash_batches, **reps[i][0]) + r = dict(cfg, seed=args.seed, **reps[i][0]) # with the hash_batches used n = len(r.get("hashes", [])) print( f"{cfg['scenario']} {cfg['files']} threads={cfg['threads']}: " diff --git a/qa/TL1_video_reader_perf/check_results.py b/qa/TL1_video_reader_perf/check_results.py index 9d6005e22e0..8525d8f857f 100644 --- a/qa/TL1_video_reader_perf/check_results.py +++ b/qa/TL1_video_reader_perf/check_results.py @@ -17,10 +17,12 @@ Checks: * no configuration failed - * hardware-independent relative checks, with a tolerance: - - lanes, cpu: throughput at the highest num_threads is not lower than at the - lowest one (more threads must not make the reader slower) + * relative checks, with a tolerance: + - lanes, seq_longgop, random_access, cpu: throughput at any higher num_threads is + not lower than at the lowest one (more threads must not make the reader slower) - reader_depth: a deeper reader queue is not slower than depth 1 + * optional: engines with the most readers reach at least --engines_scaling times the + throughput of one reader (GPUs with several NVDEC engines; e.g. H100: 3.6x at 4) * optional absolute minimums (frames/s) for named metrics, passed with --min: lanes lanes, batch 16, highest num_threads single_stream single_stream, h264_long @@ -80,6 +82,11 @@ def main(): parser.add_argument( "--min", action="append", default=[], metavar="NAME=FPS", help="absolute minimum" ) + parser.add_argument( + "--engines_scaling", + type=float, + help="minimum speedup of engines with the most readers over one reader", + ) args = parser.parse_args() with open(args.results) as f: @@ -133,10 +140,25 @@ def relative(scenario, keys, param): f"tolerance {args.tolerance:g}%", ) - relative("lanes", ("files", "layout", "batch"), "threads") - relative("cpu", ("files", "layout", "batch"), "threads") + for scenario in ("lanes", "seq_longgop", "random_access", "cpu"): + relative(scenario, ("files", "layout", "batch"), "threads") relative("reader_depth", ("files", "layout"), "reader_depth") + if args.engines_scaling is not None: + rows = sorted( + (r for r in by(results, "engines") if r.get("frames_per_s")), key=lambda r: r["readers"] + ) + if len(rows) < 2 or rows[0]["readers"] != 1: + check("engines scaling", False, "needs engines results with 1 and more readers") + else: + one, most = rows[0], rows[-1] + scaling = most["frames_per_s"] / one["frames_per_s"] + check( + f"engines readers={most['readers']} vs 1", + scaling >= args.engines_scaling, + f"{scaling:.2f}x, minimum {args.engines_scaling:g}x", + ) + for name, value in metrics.items(): if name in minimums: check( diff --git a/qa/TL1_video_reader_perf/test.sh b/qa/TL1_video_reader_perf/test.sh index 4c6df077936..07761c296c5 100644 --- a/qa/TL1_video_reader_perf/test.sh +++ b/qa/TL1_video_reader_perf/test.sh @@ -27,15 +27,15 @@ do_once() { LOG_PERF="video_reader_perf.log" JSON_PERF="video_reader_perf.json" -JSON_HASH_LOW="video_reader_hash_low.json" -JSON_HASH_HIGH="video_reader_hash_high.json" +JSON_HASH="video_reader_hash.json" LOG_REF="video_reader_ref.log" JSON_REF="video_reader_ref.json" # Other libraries run for reference on the same samples (video_reader_bench.py --backends), on -# x86_64 only; one run each, not checked against minimums. Installed by install_reference_libs. +# x86_64 only; one short run each, not checked. The DALI numbers are in the performance run. +# Installed by install_reference_libs. REF_SCENARIOS="lanes,single_stream,cache_key,cpu" -REF_ARGS="--repeat 1 --min_time 3 -j 1,16" +REF_ARGS="--repeat 1 --iters 4 --warmup 1 --min_time 2 -j 1,16" # Installs PyNvVideoCodec and torchcodec (x86_64 only) and sets REF_BACKENDS. torchcodec must # match the torch version installed through pip_packages, and is built for the same CUDA version. @@ -59,18 +59,29 @@ install_reference_libs() { } # Scenarios of internal_tools/video_reader_bench.py run for performance; see its --help -PERF_SCENARIOS="lanes,engines,single_stream,reader_depth,cache_key,startup,seq_longgop,cpu" +PERF_SCENARIOS="lanes,engines,single_stream,reader_depth,cache_key,seq_longgop,random_access,cpu" # Scenarios whose output is checked to be independent of num_threads and reader_depth -HASH_SCENARIOS="lanes,reader_depth,cache_key,cpu" -THREADS_LOW=1 -THREADS_HIGH=16 -BENCH_ARGS="--repeat 3 --min_time 3 -j ${THREADS_LOW},4,${THREADS_HIGH} --readers 1,2,4" +HASH_SCENARIOS="lanes,reader_depth,cache_key,cpu,seq_longgop,random_access" +BENCH_ARGS="--repeat 3 --iters 8 --min_time 3 -j 1,4,16 --readers 1,2,4" +# engines: one H100 or GH200 reaches 3.6x with 4 readers (several NVDEC engines) +ENGINES_SCALING=2.5 function CLEAN_AND_EXIT { - rm -rf ${LOG_PERF} ${JSON_PERF} ${JSON_HASH_LOW} ${JSON_HASH_HIGH} ${LOG_REF} ${JSON_REF} + rm -rf ${LOG_PERF} ${JSON_PERF} ${JSON_HASH} ${LOG_REF} ${JSON_REF} exit $1 } +# Prints the results (without per-run data) to the log, so that they are kept on success too +print_results() { + for json in "$@"; do + [ -f "${json}" ] && python -c "import json, sys +d = json.load(open(sys.argv[1])) +for r in d['results']: + r.pop('runs', None) +print('RESULTS', sys.argv[1], json.dumps(d, separators=(',', ':')))" "${json}" + done +} + # Run the benchmark; if NSYS_REP is set, wrap it with nsys profiling (child processes included). run_bench() { local log_file="$1" @@ -88,7 +99,9 @@ test_body() { # Hopper TASKSET="taskset --cpu-list 0-127" # Absolute minimums in frames/s (see qa/TL1_video_reader_perf/check_results.py); - # empty: reported for reference only, not checked + # empty: reported for reference only, not checked. + # TODO: set them (e.g. 0.85x the REFERENCE values) once a few CI runs establish the + # baseline on these machines MIN_LANES= MIN_SINGLE_STREAM= MIN_CPU= @@ -107,20 +120,17 @@ test_body() { run_bench "${LOG_PERF}" ${TASKSET} python video_reader_bench.py ${BENCH_ARGS} \ -s ${PERF_SCENARIOS} --json ${JSON_PERF} || PERF_RUN="FAIL" - # The output must not depend on num_threads or on the reader's prefetch_queue_depth + # The output must not depend on num_threads or on the reader's prefetch_queue_depth: every + # configuration is compared with all that differ from it only in those HASH_RESULT="OK" - for threads in ${THREADS_LOW} ${THREADS_HIGH}; do - [ ${threads} == ${THREADS_LOW} ] && json=${JSON_HASH_LOW} || json=${JSON_HASH_HIGH} - ${TASKSET} python video_reader_bench.py -s ${HASH_SCENARIOS} -j ${threads} \ - --hash --hash_batches 4 --json ${json} || HASH_RESULT="FAIL" - done - python video_reader_bench.py --compare ${JSON_HASH_LOW} ${JSON_HASH_HIGH} --ignore threads \ - || HASH_RESULT="FAIL" - python video_reader_bench.py --compare ${JSON_HASH_HIGH} ${JSON_HASH_HIGH} \ - --ignore reader_depth || HASH_RESULT="FAIL" + ${TASKSET} python video_reader_bench.py -s ${HASH_SCENARIOS} -j 1,16 --hash --hash_batches 4 \ + --json ${JSON_HASH} || HASH_RESULT="FAIL" + python video_reader_bench.py --compare ${JSON_HASH} ${JSON_HASH} \ + --ignore threads,reader_depth || HASH_RESULT="FAIL" CHECK_RESULT="OK" python ${topdir}/qa/TL1_video_reader_perf/check_results.py ${JSON_PERF} --tolerance 10 \ + --engines_scaling ${ENGINES_SCALING} \ --min lanes=${MIN_LANES} --min single_stream=${MIN_SINGLE_STREAM} \ --min cpu=${MIN_CPU} --min engines=${MIN_ENGINES} || CHECK_RESULT="FAIL" @@ -132,7 +142,7 @@ test_body() { REF_RUN="SKIPPED" elif install_reference_libs; then run_bench "${LOG_REF}" ${TASKSET} python video_reader_bench.py ${REF_ARGS} \ - -s ${REF_SCENARIOS} -b dali,${REF_BACKENDS} --json ${JSON_REF} || REF_RUN="FAIL" + -s ${REF_SCENARIOS} -b ${REF_BACKENDS} --json ${JSON_REF} || REF_RUN="FAIL" else REF_RUN="FAIL" fi @@ -141,23 +151,24 @@ test_body() { echo "HASH_RESULT=${HASH_RESULT}" echo "CHECK_RESULT=${CHECK_RESULT}" echo "REF_RUN=${REF_RUN}" + print_results ${JSON_PERF} ${JSON_REF} if [[ "$PERF_RUN" == "OK" && "$HASH_RESULT" == "OK" && "$CHECK_RESULT" == "OK" \ && "$REF_RUN" != "FAIL" ]]; then CLEAN_AND_EXIT 0 fi - # On failure: re-run the performance scenarios once with nsys and save the profile - echo "Video reader checks failed; re-running the benchmark with nsys profiling..." + # On failure: save the results; if the performance run or its checks failed, also re-run the + # performance scenarios once with nsys and save the profile ARTIFACTS_DIR="${topdir}/core_artifacts" mkdir -p "${ARTIFACTS_DIR}" - - NSYS_REP="enabled" run_bench "${LOG_PERF}" ${TASKSET} python video_reader_bench.py \ - --repeat 1 --min_time 3 -j ${THREADS_LOW},${THREADS_HIGH} --readers 1,4 \ - -s ${PERF_SCENARIOS} || true - - cp -f *.nsys-rep ${JSON_PERF} "${ARTIFACTS_DIR}/" 2>/dev/null || true - echo "nsys profile and results saved to ${ARTIFACTS_DIR}" + if [[ "$PERF_RUN" != "OK" || "$CHECK_RESULT" != "OK" ]]; then + echo "Video reader performance checks failed; re-running with nsys profiling..." + NSYS_REP="enabled" run_bench "${LOG_PERF}" ${TASKSET} python video_reader_bench.py \ + --repeat 1 --min_time 3 -j 1,16 --readers 1,4 -s ${PERF_SCENARIOS} || true + fi + cp -f *.nsys-rep ${JSON_PERF} ${JSON_HASH} ${JSON_REF} "${ARTIFACTS_DIR}/" 2>/dev/null || true + echo "Results (and nsys profile, if any) saved to ${ARTIFACTS_DIR}" CLEAN_AND_EXIT 1 } pushd ../.. From 9d36532c32b41fd01162956d904ef9f5702e9ecb Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Thu, 22 Oct 2026 10:46:48 +0200 Subject: [PATCH 08/25] Trim the video reader benchmark; pin torch/torchcodec for the comparison - Remove the switch and startup scenarios, the pixel rate, cpu at batch 1, --hash_batches (HASH_MIN_SAMPLES only), the device NVDEC sampler and --require_idle, which no check or result needs. - torchcodec: select the CUDA backend explicitly (its default changed from FFmpeg to its own NVDEC decoder), rename torchcodec_beta to torchcodec_nvdec and report torchcodec's own cpu_fallback status. - TL1: install torch 2.11.0 with torchcodec 0.15.0 from the PyTorch cu129 or cu130 index for the comparison, instead of the torch pinned for the framework tests, so both CUDA versions run the same torchcodec. Signed-off-by: Joaquin Anton Guirao --- internal_tools/video_reader_bench.py | 162 +++++++++++---------------- qa/TL1_video_reader_perf/test.sh | 57 ++++------ 2 files changed, 91 insertions(+), 128 deletions(-) diff --git a/internal_tools/video_reader_bench.py b/internal_tools/video_reader_bench.py index cc7e8a9f9ff..4d0bae91882 100644 --- a/internal_tools/video_reader_bench.py +++ b/internal_tools/video_reader_bench.py @@ -38,9 +38,6 @@ the cost of opening another file (decoder creation or NVDEC decoder cache reuse) relative to a flush and seek within the same file. This is a workload difference, not an isolated switch cost - switch clips of mixed resolutions, shuffled, short sequences: per-sample - cost under constant file switching (random access) - startup many distinct files: pipeline build time and first batch latency engines K independent readers in one pipeline, each on its own files: how throughput scales with concurrent NVDEC sessions (works today, because every reader decodes on its own thread) @@ -52,7 +49,7 @@ * the first `depth` + `reader_depth` timed iterations are excluded, as up to that many batches (pipeline queue and the reader's own queue) may have been produced off the clock; measuring continues until both --iters and --min_time are reached after them - * throughput and pixel rates count the actual output shapes of the timed batches + * throughput counts the actual output frames of the timed batches * Pipeline.run() without cuda_stream returns host-synchronized outputs, so iteration times include GPU completion * NVML metrics are per process where possible (decoder utilization, memory). A @@ -70,7 +67,7 @@ Other libraries (--backends), for reference: pynvvideocodec PyNvVideoCodec SimpleDecoder (NVDEC), RGB interleaved output (as DALI) torchcodec torchcodec VideoDecoder, FFmpeg CUDA backend on GPU, FFmpeg on CPU - torchcodec_beta torchcodec VideoDecoder with its own NVDEC backend (set_cuda_backend) + torchcodec_nvdec torchcodec VideoDecoder with its own NVDEC backend (torchcodec >= 0.8) They decode the same samples as the DALI reader (the same sequences of the same files, in the same order, or shuffled with a fixed seed), each sample with frames [start, end) of one file. A batch is decoded by a pool of min(threads, batch) threads, each with its @@ -106,7 +103,6 @@ # every slot of the reader's prefetch queue is reused HASH_MIN_SAMPLES = 40 -H264_CLIPS = ["sintel/video_files/*.mp4", "sintel/labelled_videos/*/*.mp4"] VPX_CLIPS = [ "vp9/vp9_0.mp4", "vp9/vp9_2.mp4", @@ -119,7 +115,7 @@ # file sets: glob patterns relative to DALI_extra/db/video FILES = { # 15 H.264 1280x720 clips, 122-430 frames, B-frames, 1-4 keyframes (GOP up to 250) - "h264_clips": H264_CLIPS, + "h264_clips": ["sintel/video_files/*.mp4", "sintel/labelled_videos/*/*.mp4"], # one H.264 1280x720 clip, 1253 frames, GOP 250 "h264_long": ["sintel/sintel_trailer-720p.mp4"], # the same clip as HEVC 8-bit (keyframe every ~9 frames) and HEVC 10-bit (GOP 250) @@ -127,17 +123,6 @@ "hevc10_long": ["hevc/sintel_trailer-720p_10b.mp4"], # one H.264 512x512 clip, 250 frames: small frames, per-frame overheads dominate "h264_small": ["test_label.mp4"], - # H.264 clips with mixed resolutions (1280x720, 512x512, 800x600 coded as 800x608) - "h264_mixed": H264_CLIPS - + [ - "cfr_test.mp4", - "vfr_test.mp4", - "test_label.mp4", - "frame_num_timestamp/test_25fps.mp4", - "multiple_framerate/50/50fps.mp4", - "cfr/test_2.mp4", - "vfr/test_2.mp4", - ], # two H.264 800x600 clips, coded as 800x608 (display area cropped) "h264_cropped_pair": ["cfr/test_2.mp4", "vfr/test_2.mp4"], # two H.264 1280x720 clips, coded size == display size @@ -149,7 +134,7 @@ # with an MP4 edit list hiding the lead-in frames: DALI and torchcodec start at the first # visible frame, so they decode the lead-in before frame 0, while PyNvVideoCodec returns the # lead-in frames themselves (different frames, less decoding) -LEAD_IN_FILES = {"h264_clips", "h264_mixed"} +LEAD_IN_FILES = {"h264_clips"} def _cfg(files, layout="plain", device="gpu", batch=8, seq=32, threads=4, **kw): @@ -210,38 +195,25 @@ def scenarios(threads, readers, quick=False): for f in ("h264_cropped_pair", "h264_aligned_pair") for kind in ("alternate", "same") ], - "switch": [_cfg("h264_mixed", batch=8, seq=8, threads=t, shuffle=True) for t in t_pair], - "startup": [ - _cfg("h264_clips", "many:256", batch=1, seq=8, threads=t, iters=0) for t in t_pair - ], # the noisiest scenario: more iterations than the --iters default "engines": [ _cfg("h264_clips", "many:64", batch=4, seq=32, threads=4, readers=k, **iters) for k in readers ], "cpu": [ - _cfg("vpx_clips", "many:128", device="cpu", batch=b, seq=16, threads=t) - for b in (1, 8) - for t in threads + _cfg("vpx_clips", "many:128", device="cpu", batch=8, seq=16, threads=t) for t in threads ], } -# scenarios that other backends run as well (reader_depth, startup and engines measure -# properties of the DALI reader), and the devices they decode on -_REFERENCE_SCENARIOS = { - "lanes", - "seq_longgop", - "random_access", - "single_stream", - "cache_key", - "switch", -} +# scenarios that other backends run as well (reader_depth and engines measure properties of +# the DALI reader), and the devices they decode on +_REFERENCE_SCENARIOS = {"lanes", "seq_longgop", "random_access", "single_stream", "cache_key"} BACKEND_SCENARIOS = { "dali": (None, ("gpu", "cpu")), # None: all scenarios "pynvvideocodec": (_REFERENCE_SCENARIOS, ("gpu",)), "torchcodec": (_REFERENCE_SCENARIOS | {"cpu"}, ("gpu", "cpu")), - "torchcodec_beta": (_REFERENCE_SCENARIOS, ("gpu",)), + "torchcodec_nvdec": (_REFERENCE_SCENARIOS, ("gpu",)), } @@ -252,7 +224,7 @@ def backend_runs(backend, scenario, cfg): # PyNvVideoCodec 2.0 SimpleDecoder fails (IndexError) when a decoder that was just # created or reconfigured for a file is asked for frames from the middle of it, so it # runs only configurations whose decoders start every file at frame 0 - seeks = scenario in ("random_access", "switch") or ( + seeks = scenario == "random_access" or ( scenario == "seq_longgop" and min(cfg["threads"], cfg["batch"]) > 1 ) return not (backend == "pynvvideocodec" and seeks) @@ -374,7 +346,6 @@ def __init__(self, device_id, period=0.1): self.errors = [] # NVML problems; a metric may be missing, the run is still valid self.period = period self.mem = [] - self.dev_dec = [] self.others = set() self._stop = threading.Event() self._thread = threading.Thread(target=self._run, daemon=True) @@ -416,9 +387,6 @@ def own_memory(): samplers = { "own memory": own_memory, "other processes": lambda: self.others.update(other_gpu_pids(nvml, h)), - "decoder utilization": lambda: self.dev_dec.append( - nvml.nvmlDeviceGetDecoderUtilization(h)[0] - ), } failures = dict.fromkeys(samplers, 0) while not self._stop.is_set(): @@ -448,8 +416,6 @@ def stop(self): res = {} if self.mem: res["proc_mem_max_mb"] = max(self.mem) / 2**20 - if self.dev_dec: - res["dev_nvdec_util"] = statistics.mean(self.dev_dec) try: samples = self.nvml.nvmlDeviceGetProcessUtilization(self.handle, self._t_start_us) dec = [s.decUtil for s in samples if s.pid == os.getpid()] @@ -594,6 +560,13 @@ def __init__(self, cfg, specs, args): self.version = f"torchcodec {torchcodec.__version__}" self.version += f", torch {torch.__version__}" self.local = threading.local() + # torchcodec on the GPU: whether a decoder reported decoding on the CPU instead + # (None: not known, e.g. older torchcodec without VideoDecoder.cpu_fallback) + self.cpu_fallback = None + if self.gpu and self.backend != "pynvvideocodec": + from torchcodec.decoders import VideoDecoder + + self.cpu_fallback = False if hasattr(VideoDecoder, "cpu_fallback") else None samples = sample_list(specs, cfg["seq"], cfg["shuffle"], self._num_frames) self.samples = itertools.cycle(samples) # no more threads than samples per batch: an idle extra thread would take the next @@ -609,16 +582,18 @@ def _open(self, path): use_device_memory=True, output_color_type=self.nvc.OutputColorType.RGB, # interleaved, as DALI ) - from torchcodec.decoders import VideoDecoder + from torchcodec import decoders if not self.gpu: - return VideoDecoder(path, device="cpu") - if self.backend == "torchcodec": # the default CUDA backend, FFmpeg's NVDEC support - return VideoDecoder(path, device=f"cuda:{self.device_id}") - from torchcodec.decoders import set_cuda_backend # torchcodec >= 0.8 - - with set_cuda_backend("beta"): - return VideoDecoder(path, device=f"cuda:{self.device_id}") + return decoders.VideoDecoder(path, device="cpu") + # the CUDA backend is chosen explicitly, as the default changed: FFmpeg's NVDEC support + # up to torchcodec 0.11 (the only one before 0.8), torchcodec's own NVDEC decoder later + # ("nvdec", called "beta" before, which later versions still accept) + if not hasattr(decoders, "set_cuda_backend"): + return decoders.VideoDecoder(path, device=f"cuda:{self.device_id}") + backend = "ffmpeg" if self.backend == "torchcodec" else "beta" + with decoders.set_cuda_backend(backend): + return decoders.VideoDecoder(path, device=f"cuda:{self.device_id}") def _num_frames(self, path): return len(self._decoder(path)) @@ -645,6 +620,8 @@ def _decode(self, sample): frames = decoder.get_batch_frames_by_index(list(range(start, end))) return (len(frames),) + tuple(frames[0].shape[:2]) # frames of H, W, C video = decoder.get_frames_in_range(start, end).data + if self.gpu and getattr(decoder, "cpu_fallback", None): # torchcodec >= 0.11 + self.cpu_fallback = True return (video.shape[0], video.shape[2], video.shape[3]) # F, C, H, W def build(self): @@ -685,40 +662,39 @@ def run_one(cfg, specs, args): res["shape"] = list(runner.run()[0]) iters = cfg.get("iters", args.iters) - if iters > 0: - for _ in range(args.warmup): - runner.run() - # Up to runner.prefetched batches (DALI: `depth` in the pipeline queue + - # `reader_depth` in the reader's own queue) may have been produced before the clock - # started. They are excluded, and --iters/--min_time apply to the remaining window. - skip = runner.prefetched - stamps, frames, pixels = [], [], [] - mon.start() - t_start = time.perf_counter() - while True: - shapes = runner.run() # actual (frames, height, width) of every sample - stamps.append(time.perf_counter()) - frames.append(sum(s[0] for s in shapes)) - pixels.append(sum(s[0] * s[1] * s[2] for s in shapes)) - kept = len(stamps) - skip - if kept > 0 and kept >= iters: - t_ref = stamps[skip - 1] if skip else t_start - if stamps[-1] - t_ref >= args.min_time: - break - res.update(mon.stop()) - span = stamps[-1] - t_ref - n = len(stamps) - skip - times_ms = sorted((b - a) * 1e3 for a, b in zip([t_ref] + stamps[skip:], stamps[skip:])) - res.update( - iters=n, - iters_skipped=skip, - frames_per_s=sum(frames[skip:]) / span, - mpix_per_s=sum(pixels[skip:]) / span / 1e6, - ms_per_sample=span * 1e3 / (n * cfg["readers"] * cfg["batch"]), - iter_p50_ms=statistics.median(times_ms), - iter_p90_ms=percentile(times_ms, 90), - iter_max_ms=times_ms[-1], - ) + for _ in range(args.warmup): + runner.run() + # Up to runner.prefetched batches (DALI: `depth` in the pipeline queue + `reader_depth` + # in the reader's own queue) may have been produced before the clock started. They are + # excluded, and --iters/--min_time apply to the remaining window. + skip = runner.prefetched + stamps, frames = [], [] + mon.start() + t_start = time.perf_counter() + while True: + shapes = runner.run() # actual (frames, height, width) of every sample + stamps.append(time.perf_counter()) + frames.append(sum(s[0] for s in shapes)) + kept = len(stamps) - skip + if kept > 0 and kept >= iters: + t_ref = stamps[skip - 1] if skip else t_start + if stamps[-1] - t_ref >= args.min_time: + break + res.update(mon.stop()) + span = stamps[-1] - t_ref + n = len(stamps) - skip + times_ms = sorted((b - a) * 1e3 for a, b in zip([t_ref] + stamps[skip:], stamps[skip:])) + res.update( + iters=n, + iters_skipped=skip, + frames_per_s=sum(frames[skip:]) / span, + ms_per_sample=span * 1e3 / (n * cfg["readers"] * cfg["batch"]), + iter_p50_ms=statistics.median(times_ms), + iter_p90_ms=percentile(times_ms, 90), + iter_max_ms=times_ms[-1], + ) + if getattr(runner, "cpu_fallback", None) is not None: + res["cpu_fallback"] = runner.cpu_fallback res["post"] = mon.snapshot() if mon.errors: # after the last NVML query, so that every failure is recorded res["nvml_error"] = "; ".join(mon.errors) @@ -736,7 +712,7 @@ def hash_one(cfg, specs, args): pipe.build() hashes = [] # Not dependent on threads or reader_depth, so that --compare --ignore can match runs - batches = max(args.hash_batches, -(-HASH_MIN_SAMPLES // (cfg["batch"] * cfg["readers"]))) + batches = -(-HASH_MIN_SAMPLES // (cfg["batch"] * cfg["readers"])) # ceil for _ in range(batches): outs = [_as_cpu(t) for t in pipe.run()] for r in range(cfg["readers"]): @@ -770,7 +746,7 @@ def run_child(cfg, specs, args): cmd += [json.dumps(dict(cfg=cfg, specs=specs))] for flag in ("dali_extra", "device_id", "iters", "warmup", "min_time", "seed"): cmd += [f"--{flag}", str(getattr(args, flag))] - cmd += ["--hash_batches", str(args.hash_batches)] + (["--hash"] if args.hash else []) + cmd += ["--hash"] if args.hash else [] try: p = subprocess.run(cmd, capture_output=True, text=True, timeout=args.timeout) except subprocess.TimeoutExpired: @@ -793,10 +769,11 @@ def aggregate(cfg, reps): return res res["errors"] = [r["error"] for r in reps if "error" in r] res["contaminated"] = any(r.get("contaminated") for r in ok) + if any("cpu_fallback" in r for r in ok): + res["cpu_fallback"] = any(r.get("cpu_fallback") for r in ok) res["runs"] = ok for key in ( "frames_per_s", - "mpix_per_s", "ms_per_sample", "iter_p50_ms", "iter_p90_ms", @@ -804,7 +781,6 @@ def aggregate(cfg, reps): "build_s", "first_batch_ms", "proc_nvdec_util", - "dev_nvdec_util", "proc_mem_max_mb", ): vals = [r[key] for r in ok if r.get(key) is not None] @@ -876,6 +852,8 @@ def format_row(r): if r["proc_nvdec_util"] < 10: # e.g. a GPU backend that falls back to CPU decoding for a format it can't decode flags.append("NVDEC<10%") + if r.get("cpu_fallback"): + flags.append("CPU fallback") # reported by torchcodec if r.get("backend") == "pynvvideocodec" and r.get("files") in LEAD_IN_FILES: flags.append("lead-in frames") # not the same frames as DALI, see LEAD_IN_FILES if r.get("errors"): @@ -1018,9 +996,7 @@ def parse_args(): parser.add_argument("--dali_extra", default=os.environ.get("DALI_EXTRA_PATH", "")) parser.add_argument("--json", help="write results (with provenance) to this file") parser.add_argument("--quick", action="store_true", help="smoke test: tiny sweep, 1 run") - parser.add_argument("--require_idle", action="store_true", help="abort if the GPU is busy") parser.add_argument("--hash", action="store_true", help="record per-sample checksums") - parser.add_argument("--hash_batches", default=8, type=int) parser.add_argument("--compare", nargs=2, metavar=("A", "B"), help="compare two --hash files") parser.add_argument( "--ignore", @@ -1088,8 +1064,6 @@ def main(): f"The GPU is in use by other processes ({', '.join(details)}); " f"the results will be skewed." ) - if args.require_idle: - sys.exit("The GPU is in use by other processes and --require_idle is set; aborting.") workdir = tempfile.mkdtemp(prefix="video_reader_bench_") results, failed = [], 0 diff --git a/qa/TL1_video_reader_perf/test.sh b/qa/TL1_video_reader_perf/test.sh index 07761c296c5..c54d12d0ae7 100644 --- a/qa/TL1_video_reader_perf/test.sh +++ b/qa/TL1_video_reader_perf/test.sh @@ -1,7 +1,6 @@ #!/bin/bash -e -# used pip packages; torch (pinned per CUDA version in qa/setup_packages.py) is needed by the -# reference libraries, see install_reference_libs -pip_packages='numpy nvidia-ml-py==11.450.51 torch' +# used pip packages (the reference libraries are installed by install_reference_libs) +pip_packages='numpy nvidia-ml-py==11.450.51' target_dir=./internal_tools # One-time pre-step: FFmpeg shared libraries for torchcodec (x86_64 only, see @@ -37,24 +36,20 @@ JSON_REF="video_reader_ref.json" REF_SCENARIOS="lanes,single_stream,cache_key,cpu" REF_ARGS="--repeat 1 --iters 4 --warmup 1 --min_time 2 -j 1,16" -# Installs PyNvVideoCodec and torchcodec (x86_64 only) and sets REF_BACKENDS. torchcodec must -# match the torch version installed through pip_packages, and is built for the same CUDA version. +# Installs PyNvVideoCodec, and torch with the matching CUDA-enabled torchcodec from the PyTorch +# index for this CUDA version (x86_64 only), and sets REF_BACKENDS. torch is needed only here, +# so it is pinned here rather than in pip_packages, which would follow the DALI framework tests. install_reference_libs() { - local torch_ver cuda_tag tc_ver - torch_ver=$(python -c "import torch; print(torch.__version__.split('+')[0])") - cuda_tag=$(python -c "import torch; print('cu' + torch.version.cuda.replace('.', ''))") - case "${torch_ver}" in - 2.7.*) tc_ver=0.5 ;; - 2.11.*) tc_ver=0.11.1 ;; - *) echo "No torchcodec version known for torch ${torch_ver}; update test.sh"; return 1 ;; + local cuda_tag + case "${CUDA_VERSION}" in + 12*) cuda_tag=cu129 ;; + 13*) cuda_tag=cu130 ;; + *) echo "No PyTorch index known for CUDA_VERSION=${CUDA_VERSION}; update test.sh"; return 1 ;; esac pip install PyNvVideoCodec==2.0.2 || return 1 - # --no-deps: keep the torch from pip_packages - pip install --no-deps "torchcodec==${tc_ver}+${cuda_tag}" \ + pip install "torch==2.11.0+${cuda_tag}" "torchcodec==0.15.0+${cuda_tag}" \ --index-url "https://download.pytorch.org/whl/${cuda_tag}" || return 1 - REF_BACKENDS="pynvvideocodec,torchcodec" - # torchcodec 0.5 has no set_cuda_backend, i.e. no torchcodec_beta - [ "${tc_ver}" != "0.5" ] && REF_BACKENDS="${REF_BACKENDS},torchcodec_beta" + REF_BACKENDS="pynvvideocodec,torchcodec,torchcodec_nvdec" echo "Reference backends: ${REF_BACKENDS}" } @@ -96,24 +91,18 @@ run_bench() { test_body() { if [ "$(uname -p)" == "x86_64" ]; then - # Hopper - TASKSET="taskset --cpu-list 0-127" - # Absolute minimums in frames/s (see qa/TL1_video_reader_perf/check_results.py); - # empty: reported for reference only, not checked. - # TODO: set them (e.g. 0.85x the REFERENCE values) once a few CI runs establish the - # baseline on these machines - MIN_LANES= - MIN_SINGLE_STREAM= - MIN_CPU= - MIN_ENGINES= + TASKSET="taskset --cpu-list 0-127" # Hopper else - # GraceHopper - TASKSET="taskset --cpu-list 0-71" - MIN_LANES= - MIN_SINGLE_STREAM= - MIN_CPU= - MIN_ENGINES= + TASKSET="taskset --cpu-list 0-71" # GraceHopper fi + # Absolute minimums in frames/s (see qa/TL1_video_reader_perf/check_results.py); empty: + # reported for reference only, not checked. + # TODO: set them per architecture (e.g. 0.85x the REFERENCE values) once a few CI runs + # establish the baseline on these machines + MIN_LANES= + MIN_SINGLE_STREAM= + MIN_CPU= + MIN_ENGINES= unset NSYS_REP PERF_RUN="OK" @@ -123,7 +112,7 @@ test_body() { # The output must not depend on num_threads or on the reader's prefetch_queue_depth: every # configuration is compared with all that differ from it only in those HASH_RESULT="OK" - ${TASKSET} python video_reader_bench.py -s ${HASH_SCENARIOS} -j 1,16 --hash --hash_batches 4 \ + ${TASKSET} python video_reader_bench.py -s ${HASH_SCENARIOS} -j 1,16 --hash \ --json ${JSON_HASH} || HASH_RESULT="FAIL" python video_reader_bench.py --compare ${JSON_HASH} ${JSON_HASH} \ --ignore threads,reader_depth || HASH_RESULT="FAIL" From 118c7d58d1ac254c9f0086249041c13ebc412617 Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Thu, 22 Oct 2026 10:50:46 +0200 Subject: [PATCH 09/25] Widen the backend column of the video reader benchmark table Signed-off-by: Joaquin Anton Guirao --- internal_tools/video_reader_bench.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal_tools/video_reader_bench.py b/internal_tools/video_reader_bench.py index 4d0bae91882..411bf9a372b 100644 --- a/internal_tools/video_reader_bench.py +++ b/internal_tools/video_reader_bench.py @@ -801,7 +801,7 @@ def aggregate(cfg, reps): COLUMNS = [ ("scenario", "scenario", "{:<13}"), - ("backend", "backend", "{:<15}"), + ("backend", "backend", "{:<16}"), ("files", "files", "{:<17}"), ("layout", "layout", "{:<12}"), ("device", "dev", "{:<3}"), From ec9f5ceb6369c6cf9bd43f25aebf2214bb8bb55f Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Thu, 22 Oct 2026 10:52:50 +0200 Subject: [PATCH 10/25] Don't fail the TL1 video reader test on a missing results file print_results ended with a failing [ -f ] when the last file was missing (no reference run on GraceHopper, or a failed one), which exits test_body under set -e. Also give the empty except in provenance() a reason. Signed-off-by: Joaquin Anton Guirao --- internal_tools/video_reader_bench.py | 2 +- qa/TL1_video_reader_perf/test.sh | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/internal_tools/video_reader_bench.py b/internal_tools/video_reader_bench.py index 411bf9a372b..8afec02f346 100644 --- a/internal_tools/video_reader_bench.py +++ b/internal_tools/video_reader_bench.py @@ -886,7 +886,7 @@ def provenance(args): try: info.setdefault("packages", {})[pkg] = metadata.version(pkg) except metadata.PackageNotFoundError: - pass + continue # optional: needed only by the --backends that use it try: nvml, h = nvml_handle(args.device_id) except Exception as e: diff --git a/qa/TL1_video_reader_perf/test.sh b/qa/TL1_video_reader_perf/test.sh index c54d12d0ae7..e73db6c650d 100644 --- a/qa/TL1_video_reader_perf/test.sh +++ b/qa/TL1_video_reader_perf/test.sh @@ -66,14 +66,18 @@ function CLEAN_AND_EXIT { exit $1 } -# Prints the results (without per-run data) to the log, so that they are kept on success too +# Prints the results (without per-run data) to the log, so that they are kept on success too. +# A missing or unreadable file (e.g. no reference run on GraceHopper) is skipped, not an error. print_results() { for json in "$@"; do - [ -f "${json}" ] && python -c "import json, sys + if [ -f "${json}" ]; then + python -c "import json, sys d = json.load(open(sys.argv[1])) for r in d['results']: r.pop('runs', None) -print('RESULTS', sys.argv[1], json.dumps(d, separators=(',', ':')))" "${json}" +print('RESULTS', sys.argv[1], json.dumps(d, separators=(',', ':')))" "${json}" \ + || echo "Could not print ${json}" + fi done } From 1565e3a83788a9ff230b5b8e8cc1e3e067ffbf1a Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Thu, 22 Oct 2026 11:28:50 +0200 Subject: [PATCH 11/25] Install the shared libpython for torchcodec in the TL1 video reader test torchcodec 0.15 links against libpython3.X.so, which Ubuntu's python3 package does not include. Signed-off-by: Joaquin Anton Guirao --- qa/TL1_video_reader_perf/test.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/qa/TL1_video_reader_perf/test.sh b/qa/TL1_video_reader_perf/test.sh index e73db6c650d..45cdd1dd09f 100644 --- a/qa/TL1_video_reader_perf/test.sh +++ b/qa/TL1_video_reader_perf/test.sh @@ -8,7 +8,9 @@ target_dir=./internal_tools do_once() { apt update if [ "$(uname -m)" == "x86_64" ]; then - apt install -y --no-install-recommends ffmpeg + # torchcodec loads the FFmpeg shared libraries and links against the shared libpython + apt install -y --no-install-recommends ffmpeg \ + "libpython$(python -c 'import sys; print("%d.%d" % sys.version_info[:2])')" fi if command -v nsys &>/dev/null; then echo "nsys already installed: $(nsys --version)" From 33a0e53c960b6dbf22f45a24363318f53aad4e0b Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Thu, 22 Oct 2026 12:10:05 +0200 Subject: [PATCH 12/25] Drop the NVDEC utilization flag from the video reader benchmark Per-process NVDEC utilization is averaged over the GPU's NVDEC engines, so on GPUs with several engines a fully busy session reads well under 10% and the flag misfired. Only torchcodec can fall back to CPU decoding, and it reports that itself (CPU fallback flag). Signed-off-by: Joaquin Anton Guirao --- internal_tools/video_reader_bench.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/internal_tools/video_reader_bench.py b/internal_tools/video_reader_bench.py index 8afec02f346..d95ad69767c 100644 --- a/internal_tools/video_reader_bench.py +++ b/internal_tools/video_reader_bench.py @@ -54,7 +54,8 @@ times include GPU completion * NVML metrics are per process where possible (decoder utilization, memory). A result is flagged as contaminated if other processes use the GPU or NVDEC is busy - before the run starts + before the run starts. Decoder utilization (dec%) is averaged over the GPU's NVDEC + engines, so on GPUs with several engines it is low even for a fully busy session * all input files are read once before the first run, so all runs see a warm page cache * DALI_extra has no 1080p content, so absolute rates are lower than for 1080p data; compare builds, not datasets @@ -75,9 +76,9 @@ reconfigures its decoder for a new file, torchcodec opens a new one, scanning it, as its default seek_mode="exact" does). Batches are not prefetched: unlike DALI, a batch is decoded when it is requested. They run only the configurations that mean the same - for them (backend_runs), and need torch. GPU rows where NVDEC stays nearly idle are - flagged, as a GPU backend may fall back to CPU decoding, and so are PyNvVideoCodec rows on - clips with hidden lead-in frames (LEAD_IN_FILES), where it returns different frames. + for them (backend_runs), and need torch. Rows are flagged when torchcodec reports a CPU + fallback, and PyNvVideoCodec rows on clips with hidden lead-in frames (LEAD_IN_FILES), + where it returns different frames. """ import argparse @@ -848,12 +849,8 @@ def format_row(r): flags.append("CONTAMINATED") if r.get("frames_per_s_spread_pct", 0) > 3: flags.append("spread>3%") - if r.get("device") == "gpu" and r.get("proc_nvdec_util") is not None: - if r["proc_nvdec_util"] < 10: - # e.g. a GPU backend that falls back to CPU decoding for a format it can't decode - flags.append("NVDEC<10%") if r.get("cpu_fallback"): - flags.append("CPU fallback") # reported by torchcodec + flags.append("CPU fallback") # reported by torchcodec (>= 0.11) if r.get("backend") == "pynvvideocodec" and r.get("files") in LEAD_IN_FILES: flags.append("lead-in frames") # not the same frames as DALI, see LEAD_IN_FILES if r.get("errors"): From 8034caf30f73b82686550861fe73947c5301c9d9 Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Thu, 22 Oct 2026 12:19:02 +0200 Subject: [PATCH 13/25] Add DALI's legacy fn.readers.video to the video reader benchmark dali_legacy runs the legacy reader through the same pipeline as the DALI reader, for reference: GPU only, no cache_key (both pairs include a variable frame rate file, which the legacy reader rejects), and no seeking in the open-GOP HEVC trailer, where it fails. The TL1 comparison step runs it too. Signed-off-by: Joaquin Anton Guirao --- internal_tools/video_reader_bench.py | 29 ++++++++++++++++++++++------ qa/TL1_video_reader_perf/test.sh | 8 ++++---- 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/internal_tools/video_reader_bench.py b/internal_tools/video_reader_bench.py index d95ad69767c..ea3c86a347a 100644 --- a/internal_tools/video_reader_bench.py +++ b/internal_tools/video_reader_bench.py @@ -65,7 +65,9 @@ --hash result files (e.g. two builds, or --ignore threads to check that the output does not depend on num_threads) with --compare. -Other libraries (--backends), for reference: +Other readers and libraries (--backends), for reference: + dali_legacy DALI's legacy fn.readers.video (GPU, constant frame rate only: no cpu, + cache_key), through the same pipeline as the DALI reader pynvvideocodec PyNvVideoCodec SimpleDecoder (NVDEC), RGB interleaved output (as DALI) torchcodec torchcodec VideoDecoder, FFmpeg CUDA backend on GPU, FFmpeg on CPU torchcodec_nvdec torchcodec VideoDecoder with its own NVDEC backend (torchcodec >= 0.8) @@ -212,6 +214,9 @@ def scenarios(threads, readers, quick=False): _REFERENCE_SCENARIOS = {"lanes", "seq_longgop", "random_access", "single_stream", "cache_key"} BACKEND_SCENARIOS = { "dali": (None, ("gpu", "cpu")), # None: all scenarios + # the legacy fn.readers.video: GPU only, and constant frame rate only (both cache_key + # pairs include a variable frame rate file) + "dali_legacy": (_REFERENCE_SCENARIOS - {"cache_key"} | {"reader_depth", "engines"}, ("gpu",)), "pynvvideocodec": (_REFERENCE_SCENARIOS, ("gpu",)), "torchcodec": (_REFERENCE_SCENARIOS | {"cpu"}, ("gpu", "cpu")), "torchcodec_nvdec": (_REFERENCE_SCENARIOS, ("gpu",)), @@ -228,7 +233,11 @@ def backend_runs(backend, scenario, cfg): seeks = scenario == "random_access" or ( scenario == "seq_longgop" and min(cfg["threads"], cfg["batch"]) > 1 ) - return not (backend == "pynvvideocodec" and seeks) + if backend == "pynvvideocodec" and seeks: + return False + # the legacy reader fails seeking in the HEVC trailer (open GOPs): "The decoder returned a + # frame that is past the expected one"; reading it sequentially works + return not (backend == "dali_legacy" and seeks and cfg["files"] == "hevc_long") # ---------------------------------------------------------------------------------------- @@ -437,6 +446,9 @@ def build_pipeline(cfg, specs, device_id, seed, frame_num=False): import nvidia.dali.fn as fn from nvidia.dali.pipeline import pipeline_def + legacy = cfg["backend"] == "dali_legacy" + reader = fn.readers.video if legacy else fn.experimental.readers.video + @pipeline_def( batch_size=cfg["batch"], num_threads=cfg["threads"], @@ -450,12 +462,14 @@ def pipe(): kw = dict(spec) if "filenames" in kw: kw["labels"] = list(range(len(kw["filenames"]))) + elif legacy: + kw["file_list_frame_num"] = True # frame numbers, not timestamps; end exclusive else: kw["file_list_format"] = "frames" kw["file_list_include_end"] = False # explicit: the default changed over time if frame_num: kw["enable_frame_num"] = True - outs += fn.experimental.readers.video( + outs += reader( device=cfg["device"], sequence_length=cfg["seq"], random_shuffle=cfg["shuffle"], @@ -489,7 +503,9 @@ def __init__(self, cfg, specs, args): import nvidia.dali as dali self.cfg = cfg - self.version = dali.__version__ + self.version = dali.__version__ + ( + " (fn.readers.video)" if cfg["backend"] != "dali" else "" + ) # up to `depth` (pipeline queue) + `reader_depth` (the reader's own queue) batches # may be produced before the clock starts self.prefetched = cfg["depth"] + cfg["reader_depth"] @@ -637,7 +653,8 @@ def run(self): def make_runner(cfg, specs, args): - return (DaliRunner if cfg["backend"] == "dali" else ReferenceRunner)(cfg, specs, args) + dali = cfg["backend"] in ("dali", "dali_legacy") + return (DaliRunner if dali else ReferenceRunner)(cfg, specs, args) def percentile(sorted_vals, q): @@ -655,7 +672,7 @@ def run_one(cfg, specs, args): res["backend_version"] = runner.version t0 = time.perf_counter() # build_s: the DALI reader (indexing), not the CUDA runtime runner.build() - if cfg["backend"] == "dali": # other libraries open their decoders lazily, in run() + if isinstance(runner, DaliRunner): # other libraries open their decoders lazily, in run() res["build_s"] = time.perf_counter() - t0 t0 = time.perf_counter() runner.run() diff --git a/qa/TL1_video_reader_perf/test.sh b/qa/TL1_video_reader_perf/test.sh index 45cdd1dd09f..694c7168ce4 100644 --- a/qa/TL1_video_reader_perf/test.sh +++ b/qa/TL1_video_reader_perf/test.sh @@ -32,9 +32,9 @@ JSON_HASH="video_reader_hash.json" LOG_REF="video_reader_ref.log" JSON_REF="video_reader_ref.json" -# Other libraries run for reference on the same samples (video_reader_bench.py --backends), on -# x86_64 only; one short run each, not checked. The DALI numbers are in the performance run. -# Installed by install_reference_libs. +# DALI's legacy fn.readers.video and other libraries run for reference on the same samples +# (video_reader_bench.py --backends), on x86_64 only; one short run each, not checked. The DALI +# numbers are in the performance run. The libraries are installed by install_reference_libs. REF_SCENARIOS="lanes,single_stream,cache_key,cpu" REF_ARGS="--repeat 1 --iters 4 --warmup 1 --min_time 2 -j 1,16" @@ -137,7 +137,7 @@ test_body() { REF_RUN="SKIPPED" elif install_reference_libs; then run_bench "${LOG_REF}" ${TASKSET} python video_reader_bench.py ${REF_ARGS} \ - -s ${REF_SCENARIOS} -b ${REF_BACKENDS} --json ${JSON_REF} || REF_RUN="FAIL" + -s ${REF_SCENARIOS} -b dali_legacy,${REF_BACKENDS} --json ${JSON_REF} || REF_RUN="FAIL" else REF_RUN="FAIL" fi From 910e64fd4b6da74fe335c7a4f54fcbc57b55a3f5 Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Thu, 22 Oct 2026 14:05:13 +0200 Subject: [PATCH 14/25] Keep the reference libraries' batch outputs until the next batch The reference runner returned only the output shapes, so every sample's frames were freed right after decoding and the process memory of the libraries never included a batch, while DALI's includes its returned batch and its queues. A batch's outputs are now kept until the next batch is requested, as Pipeline.run() outputs are. PyNvVideoCodec's frames stay in its decoder's buffers, which its next call reuses, so its memory still does not include a batch; documented. Signed-off-by: Joaquin Anton Guirao --- internal_tools/video_reader_bench.py | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/internal_tools/video_reader_bench.py b/internal_tools/video_reader_bench.py index ea3c86a347a..8374c5d14e8 100644 --- a/internal_tools/video_reader_bench.py +++ b/internal_tools/video_reader_bench.py @@ -77,10 +77,14 @@ own decoder, kept while consecutive samples come from the same file (PyNvVideoCodec reconfigures its decoder for a new file, torchcodec opens a new one, scanning it, as its default seek_mode="exact" does). Batches are not prefetched: unlike DALI, a batch - is decoded when it is requested. They run only the configurations that mean the same - for them (backend_runs), and need torch. Rows are flagged when torchcodec reports a CPU - fallback, and PyNvVideoCodec rows on clips with hidden lead-in frames (LEAD_IN_FILES), - where it returns different frames. + is decoded when it is requested. A batch's outputs are kept until the next batch is + requested, as DALI's, so the process memory (mem MB) includes one batch for them, and + for DALI also the batches in its queues (`depth` + `reader_depth`). PyNvVideoCodec is an + exception: its frames stay in its decoder's buffers, which the decoder's next call + reuses, so its memory does not include a batch. They run only the configurations that + mean the same for them (backend_runs), and need torch. Rows are flagged when torchcodec + reports a CPU fallback, and PyNvVideoCodec rows on clips with hidden lead-in frames + (LEAD_IN_FILES), where it returns different frames. """ import argparse @@ -630,26 +634,33 @@ def _decoder(self, path): return local.decoder def _decode(self, sample): + """Returns the sample's (frames, height, width) and the library's output.""" path, start, end = sample decoder = self._decoder(path) # the libraries' own outputs, not converted or copied if self.backend == "pynvvideocodec": frames = decoder.get_batch_frames_by_index(list(range(start, end))) - return (len(frames),) + tuple(frames[0].shape[:2]) # frames of H, W, C + return (len(frames),) + tuple(frames[0].shape[:2]), frames # frames of H, W, C video = decoder.get_frames_in_range(start, end).data if self.gpu and getattr(decoder, "cpu_fallback", None): # torchcodec >= 0.11 self.cpu_fallback = True - return (video.shape[0], video.shape[2], video.shape[3]) # F, C, H, W + return (video.shape[0], video.shape[2], video.shape[3]), video # F, C, H, W def build(self): pass def run(self): + # The outputs of a batch are kept until the next run(), as Pipeline.run() outputs, so + # that the process memory includes a whole batch (DALI's also includes its queues) + self.outputs = None batch = [next(self.samples) for _ in range(self.cfg["batch"])] - shapes = list(self.pool.map(self._decode, batch) if self.pool else map(self._decode, batch)) + decoded = list( + self.pool.map(self._decode, batch) if self.pool else map(self._decode, batch) + ) if self.gpu: self.torch.cuda.synchronize() # as Pipeline.run(): outputs are ready - return shapes + self.outputs = [out for _, out in decoded] + return [shape for shape, _ in decoded] def make_runner(cfg, specs, args): From 0b13f353548a6f4bacb80925be6f736af93b1800 Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Thu, 22 Oct 2026 16:33:28 +0200 Subject: [PATCH 15/25] Add DALI's video decoder (fn.readers.file + fn.decoders.video) to the video reader benchmark The dali_decoder backend reads every sample's whole file with fn.readers.file and decodes its frames with fn.decoders.video (fn.experimental.decoders.video in older DALI), which decodes the samples of a batch in parallel on num_threads threads, with a new decoder per sample. It runs the same samples as the reader, in the reference scenarios plus engines and cpu, and the TL1 reference step. Like PyNvVideoCodec, it returns the lead-in frames that an MP4 edit list hides, so its rows on those clips are flagged. Signed-off-by: Joaquin Anton Guirao --- internal_tools/video_reader_bench.py | 76 ++++++++++++++++++++++++---- qa/TL1_video_reader_perf/test.sh | 5 +- 2 files changed, 70 insertions(+), 11 deletions(-) diff --git a/internal_tools/video_reader_bench.py b/internal_tools/video_reader_bench.py index 8374c5d14e8..cd407f56d29 100644 --- a/internal_tools/video_reader_bench.py +++ b/internal_tools/video_reader_bench.py @@ -68,6 +68,12 @@ Other readers and libraries (--backends), for reference: dali_legacy DALI's legacy fn.readers.video (GPU, constant frame rate only: no cpu, cache_key), through the same pipeline as the DALI reader + dali_decoder fn.readers.file + fn.decoders.video (fn.experimental.decoders.video in + older DALI): every sample is a whole file, read by the file reader, and + decoded with a new decoder per sample, the samples of a batch in parallel + on num_threads threads. Same samples (file, first and last frame) as the + DALI reader, shuffled by the file reader; `reader_depth` is the file + reader's prefetch_queue_depth pynvvideocodec PyNvVideoCodec SimpleDecoder (NVDEC), RGB interleaved output (as DALI) torchcodec torchcodec VideoDecoder, FFmpeg CUDA backend on GPU, FFmpeg on CPU torchcodec_nvdec torchcodec VideoDecoder with its own NVDEC backend (torchcodec >= 0.8) @@ -83,8 +89,8 @@ exception: its frames stay in its decoder's buffers, which the decoder's next call reuses, so its memory does not include a batch. They run only the configurations that mean the same for them (backend_runs), and need torch. Rows are flagged when torchcodec - reports a CPU fallback, and PyNvVideoCodec rows on clips with hidden lead-in frames - (LEAD_IN_FILES), where it returns different frames. + reports a CPU fallback, and PyNvVideoCodec and dali_decoder rows on clips with hidden + lead-in frames (LEAD_IN_FILES), where they return different frames. """ import argparse @@ -138,10 +144,12 @@ "vpx_clips": VPX_CLIPS, } # File sets where most clips (cut without re-encoding) start before the frame they show first, -# with an MP4 edit list hiding the lead-in frames: DALI and torchcodec start at the first -# visible frame, so they decode the lead-in before frame 0, while PyNvVideoCodec returns the -# lead-in frames themselves (different frames, less decoding) +# with an MP4 edit list hiding the lead-in frames: the DALI reader and torchcodec start at the +# first visible frame, so they decode the lead-in before frame 0, while PyNvVideoCodec and +# DALI's video decoder (dali_decoder) return the lead-in frames themselves (different frames, +# less decoding) LEAD_IN_FILES = {"h264_clips"} +LEAD_IN_BACKENDS = {"pynvvideocodec", "dali_decoder"} def _cfg(files, layout="plain", device="gpu", batch=8, seq=32, threads=4, **kw): @@ -221,6 +229,8 @@ def scenarios(threads, readers, quick=False): # the legacy fn.readers.video: GPU only, and constant frame rate only (both cache_key # pairs include a variable frame rate file) "dali_legacy": (_REFERENCE_SCENARIOS - {"cache_key"} | {"reader_depth", "engines"}, ("gpu",)), + # fn.readers.file + fn.decoders.video; reader_depth would prefetch file contents only + "dali_decoder": (_REFERENCE_SCENARIOS | {"engines", "cpu"}, ("gpu", "cpu")), "pynvvideocodec": (_REFERENCE_SCENARIOS, ("gpu",)), "torchcodec": (_REFERENCE_SCENARIOS | {"cpu"}, ("gpu", "cpu")), "torchcodec_nvdec": (_REFERENCE_SCENARIOS, ("gpu",)), @@ -446,12 +456,37 @@ def stop(self): # running +def num_frames_dali(path, device, device_id): + """The frames of a file as the DALI reader counts them: its sequences of one frame.""" + import nvidia.dali.fn as fn + from nvidia.dali.pipeline import pipeline_def + + @pipeline_def(batch_size=1, num_threads=1, device_id=device_id) + def count(): + return fn.experimental.readers.video( + filenames=[path], sequence_length=1, device=device, name="Count" + ) + + p = count() + p.build() + return p.reader_meta("Count")["epoch_size"] + + def build_pipeline(cfg, specs, device_id, seed, frame_num=False): import nvidia.dali.fn as fn from nvidia.dali.pipeline import pipeline_def legacy = cfg["backend"] == "dali_legacy" reader = fn.readers.video if legacy else fn.experimental.readers.video + if cfg["backend"] == "dali_decoder": + + def num_frames(path): + return num_frames_dali(path, cfg["device"], device_id) + + # the samples the DALI reader would produce, (path, start, end), one list per reader + samples = [sample_list([s], cfg["seq"], False, num_frames) for s in specs] + # moved out of experimental in DALI 2.0 + decoder = getattr(fn.decoders, "video", None) or fn.experimental.decoders.video @pipeline_def( batch_size=cfg["batch"], @@ -462,6 +497,24 @@ def build_pipeline(cfg, specs, device_id, seed, frame_num=False): ) def pipe(): outs = [] + if cfg["backend"] == "dali_decoder": + for r, reader_samples in enumerate(samples): + # whole files, with the sample's first frame as the label + encoded, start = fn.readers.file( + files=[path for path, _, _ in reader_samples], + labels=[start for _, start, _ in reader_samples], + random_shuffle=cfg["shuffle"], + prefetch_queue_depth=cfg["reader_depth"], + name=f"Reader{r}", + ) + video = decoder( + encoded, + device="mixed" if cfg["device"] == "gpu" else "cpu", + start_frame=start[0], + sequence_length=cfg["seq"], + ) + outs += [video, start] + return tuple(outs) for r, spec in enumerate(specs): kw = dict(spec) if "filenames" in kw: @@ -507,8 +560,13 @@ def __init__(self, cfg, specs, args): import nvidia.dali as dali self.cfg = cfg - self.version = dali.__version__ + ( - " (fn.readers.video)" if cfg["backend"] != "dali" else "" + self.version = ( + dali.__version__ + + { + "dali": "", + "dali_legacy": " (fn.readers.video)", + "dali_decoder": " (fn.readers.file + video decoder)", + }[cfg["backend"]] ) # up to `depth` (pipeline queue) + `reader_depth` (the reader's own queue) batches # may be produced before the clock starts @@ -664,7 +722,7 @@ def run(self): def make_runner(cfg, specs, args): - dali = cfg["backend"] in ("dali", "dali_legacy") + dali = cfg["backend"] in ("dali", "dali_legacy", "dali_decoder") return (DaliRunner if dali else ReferenceRunner)(cfg, specs, args) @@ -879,7 +937,7 @@ def format_row(r): flags.append("spread>3%") if r.get("cpu_fallback"): flags.append("CPU fallback") # reported by torchcodec (>= 0.11) - if r.get("backend") == "pynvvideocodec" and r.get("files") in LEAD_IN_FILES: + if r.get("backend") in LEAD_IN_BACKENDS and r.get("files") in LEAD_IN_FILES: flags.append("lead-in frames") # not the same frames as DALI, see LEAD_IN_FILES if r.get("errors"): flags.append(f"{len(r['errors'])} failed run(s)") diff --git a/qa/TL1_video_reader_perf/test.sh b/qa/TL1_video_reader_perf/test.sh index 694c7168ce4..06e4d9124b2 100644 --- a/qa/TL1_video_reader_perf/test.sh +++ b/qa/TL1_video_reader_perf/test.sh @@ -32,7 +32,8 @@ JSON_HASH="video_reader_hash.json" LOG_REF="video_reader_ref.log" JSON_REF="video_reader_ref.json" -# DALI's legacy fn.readers.video and other libraries run for reference on the same samples +# DALI's legacy fn.readers.video, its video decoder (fn.readers.file + fn.decoders.video) and +# other libraries run for reference on the same samples # (video_reader_bench.py --backends), on x86_64 only; one short run each, not checked. The DALI # numbers are in the performance run. The libraries are installed by install_reference_libs. REF_SCENARIOS="lanes,single_stream,cache_key,cpu" @@ -137,7 +138,7 @@ test_body() { REF_RUN="SKIPPED" elif install_reference_libs; then run_bench "${LOG_REF}" ${TASKSET} python video_reader_bench.py ${REF_ARGS} \ - -s ${REF_SCENARIOS} -b dali_legacy,${REF_BACKENDS} --json ${JSON_REF} || REF_RUN="FAIL" + -s ${REF_SCENARIOS} -b dali_legacy,dali_decoder,${REF_BACKENDS} --json ${JSON_REF} || REF_RUN="FAIL" else REF_RUN="FAIL" fi From a67ba9616c7e9d264bceed965fb89b234fd11035 Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Tue, 27 Oct 2026 03:09:08 +0100 Subject: [PATCH 16/25] Check video reader output over two epochs and with reader depth 2 The checksum run of the video reader benchmark covered at least 40 samples per configuration: three batches at batch 16. That is enough for the reader's queues to wrap around, but in 'lanes' no decoder is ever reused in three batches, seq_longgop and random_access reach the next epoch in none, and no checksummed configuration runs several decode lanes with more than one batch in the reader's queue. --hash_epochs E checksums at least E epochs and one more batch of every configuration, so that every file of 'lanes' is opened again and decoders are reused across batches and epochs. A new scenario, lanes_depth, run only with --hash, decodes batches of 16 on several lanes with the reader's prefetch_queue_depth 1 and 2: whole files in order and shuffled, and many distinct files. TL1_video_reader_perf checks two epochs and lanes_depth. Signed-off-by: Joaquin Anton Guirao --- internal_tools/video_reader_bench.py | 50 ++++++++++++++++++++++++---- qa/TL1_video_reader_perf/test.sh | 8 +++-- 2 files changed, 49 insertions(+), 9 deletions(-) diff --git a/internal_tools/video_reader_bench.py b/internal_tools/video_reader_bench.py index cd407f56d29..0101e66f231 100644 --- a/internal_tools/video_reader_bench.py +++ b/internal_tools/video_reader_bench.py @@ -61,9 +61,13 @@ compare builds, not datasets --hash records a checksum of every sample (frames, label, frame_num) of at least -HASH_MIN_SAMPLES samples, so that the reader's prefetch queue wraps around. Compare two ---hash result files (e.g. two builds, or --ignore threads to check that the output -does not depend on num_threads) with --compare. +HASH_MIN_SAMPLES samples, so that the reader's prefetch queue wraps around, and with +--hash_epochs E of E epochs and one more batch, so that decoders are reused across +batches and the reader wraps around its epochs. Compare two --hash result files (e.g. two +builds, or --ignore threads to check that the output does not depend on num_threads) with +--compare. --hash also runs the configurations only checksummed: + lanes_depth batch 16 on several decode lanes with the reader's prefetch_queue_depth + 1 and 2: whole files in order and shuffled, and many distinct files Other readers and libraries (--backends), for reference: dali_legacy DALI's legacy fn.readers.video (GPU, constant frame rate only: no cpu, @@ -182,6 +186,7 @@ def _cfg(files, layout="plain", device="gpu", batch=8, seq=32, threads=4, **kw): def scenarios(threads, readers, quick=False): + """The configurations of every scenario (HASH_ONLY_SCENARIOS only with --hash).""" t_max = max(threads) t_pair = sorted({1, t_max}) iters = {} if quick else dict(iters=20) @@ -218,9 +223,25 @@ def scenarios(threads, readers, quick=False): "cpu": [ _cfg("vpx_clips", "many:128", device="cpu", batch=8, seq=16, threads=t) for t in threads ], + # several decode lanes with up to two batches in the reader's queue (the other + # scenarios with several lanes use reader_depth 1) + "lanes_depth": [ + _cfg(f, layout, batch=16, seq=32, threads=t, reader_depth=d, shuffle=s) + for f, layout, s in ( + ("h264_clips", "plain", False), + ("h264_clips", "plain", True), + ("h264_clips", "many:256", False), + ) + for d in (1, 2) + for t in threads + ], } +# scenarios only checksummed (--hash), not measured +HASH_ONLY_SCENARIOS = {"lanes_depth"} + + # scenarios that other backends run as well (reader_depth and engines measure properties of # the DALI reader), and the devices they decode on _REFERENCE_SCENARIOS = {"lanes", "seq_longgop", "random_access", "single_stream", "cache_key"} @@ -794,12 +815,16 @@ def run_one(cfg, specs, args): def hash_one(cfg, specs, args): - """Returns one checksum per sample of the first batches (see HASH_MIN_SAMPLES).""" + """Returns one checksum per sample of the first batches (see HASH_MIN_SAMPLES and + --hash_epochs).""" pipe = build_pipeline(cfg, specs, args.device_id, args.seed, frame_num=True) pipe.build() hashes = [] # Not dependent on threads or reader_depth, so that --compare --ignore can match runs batches = -(-HASH_MIN_SAMPLES // (cfg["batch"] * cfg["readers"])) # ceil + if args.hash_epochs: + epoch = max(pipe.epoch_size(f"Reader{r}") for r in range(cfg["readers"])) + batches = max(batches, -(-args.hash_epochs * epoch // cfg["batch"]) + 1) for _ in range(batches): outs = [_as_cpu(t) for t in pipe.run()] for r in range(cfg["readers"]): @@ -831,7 +856,7 @@ def child_main(args): def run_child(cfg, specs, args): cmd = [sys.executable, os.path.abspath(__file__), "--_single"] cmd += [json.dumps(dict(cfg=cfg, specs=specs))] - for flag in ("dali_extra", "device_id", "iters", "warmup", "min_time", "seed"): + for flag in ("dali_extra", "device_id", "iters", "warmup", "min_time", "seed", "hash_epochs"): cmd += [f"--{flag}", str(getattr(args, flag))] cmd += ["--hash"] if args.hash else [] try: @@ -1080,6 +1105,13 @@ def parse_args(): parser.add_argument("--json", help="write results (with provenance) to this file") parser.add_argument("--quick", action="store_true", help="smoke test: tiny sweep, 1 run") parser.add_argument("--hash", action="store_true", help="record per-sample checksums") + parser.add_argument( + "--hash_epochs", + default=0, + type=int, + help="with --hash: checksum at least this many epochs of every configuration, and " + "one more batch", + ) parser.add_argument("--compare", nargs=2, metavar=("A", "B"), help="compare two --hash files") parser.add_argument( "--ignore", @@ -1122,13 +1154,19 @@ def main(): threads = [int(t) for t in args.threads.split(",")] readers = [int(r) for r in args.readers.split(",")] all_scenarios = scenarios(threads, readers, args.quick) - names = list(all_scenarios) if args.scenarios == "all" else args.scenarios.split(",") + if args.scenarios == "all": + names = [n for n in all_scenarios if args.hash or n not in HASH_ONLY_SCENARIOS] + else: + names = args.scenarios.split(",") unknown = [n for n in names if n not in all_scenarios] if unknown: parser.error( f"--scenarios got unknown scenario(s) {unknown}; expected 'all' or a subset of " f"{list(all_scenarios)}." ) + hash_only = [n for n in names if n in HASH_ONLY_SCENARIOS] + if hash_only and not args.hash: + parser.error(f"--scenarios {hash_only} run only with --hash.") if args.hash: args.repeat = 1 diff --git a/qa/TL1_video_reader_perf/test.sh b/qa/TL1_video_reader_perf/test.sh index 06e4d9124b2..c636ee36b61 100644 --- a/qa/TL1_video_reader_perf/test.sh +++ b/qa/TL1_video_reader_perf/test.sh @@ -58,8 +58,10 @@ install_reference_libs() { # Scenarios of internal_tools/video_reader_bench.py run for performance; see its --help PERF_SCENARIOS="lanes,engines,single_stream,reader_depth,cache_key,seq_longgop,random_access,cpu" -# Scenarios whose output is checked to be independent of num_threads and reader_depth -HASH_SCENARIOS="lanes,reader_depth,cache_key,cpu,seq_longgop,random_access" +# Scenarios whose output is checked to be independent of num_threads and reader_depth, over two +# epochs (decoders reused across batches, the epochs wrapping around) +HASH_SCENARIOS="lanes,lanes_depth,reader_depth,cache_key,cpu,seq_longgop,random_access" +HASH_EPOCHS=2 BENCH_ARGS="--repeat 3 --iters 8 --min_time 3 -j 1,4,16 --readers 1,2,4" # engines: one H100 or GH200 reaches 3.6x with 4 readers (several NVDEC engines) ENGINES_SCALING=2.5 @@ -120,7 +122,7 @@ test_body() { # configuration is compared with all that differ from it only in those HASH_RESULT="OK" ${TASKSET} python video_reader_bench.py -s ${HASH_SCENARIOS} -j 1,16 --hash \ - --json ${JSON_HASH} || HASH_RESULT="FAIL" + --hash_epochs ${HASH_EPOCHS} --json ${JSON_HASH} || HASH_RESULT="FAIL" python video_reader_bench.py --compare ${JSON_HASH} ${JSON_HASH} \ --ignore threads,reader_depth || HASH_RESULT="FAIL" From ee423dfe98a94c278c3a7848b521053d1a562f66 Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Tue, 27 Oct 2026 11:47:08 +0100 Subject: [PATCH 17/25] Pass the TL1 engines check when NVDEC is saturated With decode lanes, one reader already spreads its decoding over the NVDEC engines, so 4 readers can no longer reach 2.5x one reader (H100 PCIe: 2016 frames/s with 1 reader, 3824 with 4 at 98% NVDEC, 1.9x). The check now also passes when the run with the most readers keeps NVDEC at least 90% busy. Signed-off-by: Joaquin Anton Guirao --- qa/TL1_video_reader_perf/check_results.py | 24 +++++++++++++++++++---- qa/TL1_video_reader_perf/test.sh | 7 +++++-- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/qa/TL1_video_reader_perf/check_results.py b/qa/TL1_video_reader_perf/check_results.py index 8525d8f857f..31e6f665232 100644 --- a/qa/TL1_video_reader_perf/check_results.py +++ b/qa/TL1_video_reader_perf/check_results.py @@ -21,8 +21,11 @@ - lanes, seq_longgop, random_access, cpu: throughput at any higher num_threads is not lower than at the lowest one (more threads must not make the reader slower) - reader_depth: a deeper reader queue is not slower than depth 1 - * optional: engines with the most readers reach at least --engines_scaling times the - throughput of one reader (GPUs with several NVDEC engines; e.g. H100: 3.6x at 4) + * optional: engines with the most readers use the GPU's NVDEC engines: they reach at least + --engines_scaling times the throughput of one reader (GPUs with several NVDEC engines), or, + with --engines_busy, keep NVDEC at least that busy (percent). The latter is for when one + reader already spreads its decode lanes over the engines, so that more readers cannot scale + as much. * optional absolute minimums (frames/s) for named metrics, passed with --min: lanes lanes, batch 16, highest num_threads single_stream single_stream, h264_long @@ -87,6 +90,12 @@ def main(): type=float, help="minimum speedup of engines with the most readers over one reader", ) + parser.add_argument( + "--engines_busy", + type=float, + help="NVDEC utilization (percent) of engines with the most readers that passes the " + "engines check even below --engines_scaling (NVDEC is saturated)", + ) args = parser.parse_args() with open(args.results) as f: @@ -153,10 +162,17 @@ def relative(scenario, keys, param): else: one, most = rows[0], rows[-1] scaling = most["frames_per_s"] / one["frames_per_s"] + busy = most.get("proc_nvdec_util") + saturated = ( + args.engines_busy is not None and busy is not None and busy >= args.engines_busy + ) + detail = f"{scaling:.2f}x, minimum {args.engines_scaling:g}x" + if args.engines_busy is not None: + detail += f"; NVDEC {busy}% busy, saturated at {args.engines_busy:g}%" check( f"engines readers={most['readers']} vs 1", - scaling >= args.engines_scaling, - f"{scaling:.2f}x, minimum {args.engines_scaling:g}x", + scaling >= args.engines_scaling or saturated, + detail, ) for name, value in metrics.items(): diff --git a/qa/TL1_video_reader_perf/test.sh b/qa/TL1_video_reader_perf/test.sh index c636ee36b61..e0921afb6fd 100644 --- a/qa/TL1_video_reader_perf/test.sh +++ b/qa/TL1_video_reader_perf/test.sh @@ -63,8 +63,11 @@ PERF_SCENARIOS="lanes,engines,single_stream,reader_depth,cache_key,seq_longgop,r HASH_SCENARIOS="lanes,lanes_depth,reader_depth,cache_key,cpu,seq_longgop,random_access" HASH_EPOCHS=2 BENCH_ARGS="--repeat 3 --iters 8 --min_time 3 -j 1,4,16 --readers 1,2,4" -# engines: one H100 or GH200 reaches 3.6x with 4 readers (several NVDEC engines) +# engines: more readers use more NVDEC engines: 4 readers reach 2.5x one reader, or keep NVDEC +# saturated (one reader's decode lanes already use several engines, e.g. H100 PCIe: 2016 frames/s +# with 1 reader, 3824 at 98% NVDEC with 4, which is 1.9x) ENGINES_SCALING=2.5 +ENGINES_BUSY=90 function CLEAN_AND_EXIT { rm -rf ${LOG_PERF} ${JSON_PERF} ${JSON_HASH} ${LOG_REF} ${JSON_REF} @@ -128,7 +131,7 @@ test_body() { CHECK_RESULT="OK" python ${topdir}/qa/TL1_video_reader_perf/check_results.py ${JSON_PERF} --tolerance 10 \ - --engines_scaling ${ENGINES_SCALING} \ + --engines_scaling ${ENGINES_SCALING} --engines_busy ${ENGINES_BUSY} \ --min lanes=${MIN_LANES} --min single_stream=${MIN_SINGLE_STREAM} \ --min cpu=${MIN_CPU} --min engines=${MIN_ENGINES} || CHECK_RESULT="FAIL" From 9b6a8eec37a0e5dee13263b1e9634a4b5137c4de Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Wed, 28 Oct 2026 10:54:28 +0100 Subject: [PATCH 18/25] Replace the video reader benchmark with a smaller one Trim internal_tools/video_reader_bench.py from 1266 to 314 lines and the TL1 test.sh from 177 to 83, and drop check_results.py. The benchmark keeps the timing and checksum sweeps, a --check gate (more threads or a deeper reader queue may be at most 10% slower) and the torchcodec reference rows, which never fail the test. Signed-off-by: Joaquin Anton Guirao --- internal_tools/video_reader_bench.py | 1474 ++++----------------- qa/TL1_video_reader_perf/check_results.py | 193 --- qa/TL1_video_reader_perf/test.sh | 174 +-- 3 files changed, 301 insertions(+), 1540 deletions(-) delete mode 100644 qa/TL1_video_reader_perf/check_results.py diff --git a/internal_tools/video_reader_bench.py b/internal_tools/video_reader_bench.py index 0101e66f231..34200550000 100644 --- a/internal_tools/video_reader_bench.py +++ b/internal_tools/video_reader_bench.py @@ -12,1254 +12,302 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Benchmark for fn.experimental.readers.video, using DALI_extra data only. - -Scenarios (each isolates one aspect of the reader): - - lanes many distinct files, one keyframe-aligned sequence each, so every - batch holds `batch` different files and no sample needs a seek: - per-file cost (opening, indexing, decoder setup, decoding the lead-in - frames an edit list hides, see LEAD_IN_FILES) plus decoding, vs. - num_threads. A per-file speedup also shows here; check `cache_key` - and `engines` before attributing a gain to decode parallelism - seq_longgop 15 H.264 clips read sequentially (long GOPs, consecutive sequences - of the same file): no-regression check for sequential access - random_access the same clips, and the HEVC trailer (open GOPs), shuffled: - parallelism and keyframe-seek waste together (do not read a lanes - speedup from this one alone) - single_stream one clip, batch 1: per-stream throughput, i.e. per-frame - serialization. Includes a 512x512 clip, where per-frame overheads - matter more than decode time - reader_depth batch 1, the reader's own prefetch_queue_depth (batches in flight), - on many distinct files and on one sequential long-GOP file - cache_key every sample switches between two files with the same format, either - cropped (800x600 coded as 800x608) or not (1280x720), vs. a control - that rewinds the same file to frame 0 for every sample instead: - the cost of opening another file (decoder creation or NVDEC decoder - cache reuse) relative to a flush and seek within the same file. This - is a workload difference, not an isolated switch cost - engines K independent readers in one pipeline, each on its own files: - how throughput scales with concurrent NVDEC sessions (works today, - because every reader decodes on its own thread) - cpu many distinct VP8/VP9 files on the CPU backend vs. num_threads - -Methodology: - * every configuration runs in a fresh subprocess (no cache carry-over), --repeat - times, round-robin over configurations; the median and the spread are reported - * the first `depth` + `reader_depth` timed iterations are excluded, as up to that many - batches (pipeline queue and the reader's own queue) may have been produced off the - clock; measuring continues until both --iters and --min_time are reached after them - * throughput counts the actual output frames of the timed batches - * Pipeline.run() without cuda_stream returns host-synchronized outputs, so iteration - times include GPU completion - * NVML metrics are per process where possible (decoder utilization, memory). A - result is flagged as contaminated if other processes use the GPU or NVDEC is busy - before the run starts. Decoder utilization (dec%) is averaged over the GPU's NVDEC - engines, so on GPUs with several engines it is low even for a fully busy session - * all input files are read once before the first run, so all runs see a warm page cache - * DALI_extra has no 1080p content, so absolute rates are lower than for 1080p data; - compare builds, not datasets - ---hash records a checksum of every sample (frames, label, frame_num) of at least -HASH_MIN_SAMPLES samples, so that the reader's prefetch queue wraps around, and with ---hash_epochs E of E epochs and one more batch, so that decoders are reused across -batches and the reader wraps around its epochs. Compare two --hash result files (e.g. two -builds, or --ignore threads to check that the output does not depend on num_threads) with ---compare. --hash also runs the configurations only checksummed: - lanes_depth batch 16 on several decode lanes with the reader's prefetch_queue_depth - 1 and 2: whole files in order and shuffled, and many distinct files - -Other readers and libraries (--backends), for reference: - dali_legacy DALI's legacy fn.readers.video (GPU, constant frame rate only: no cpu, - cache_key), through the same pipeline as the DALI reader - dali_decoder fn.readers.file + fn.decoders.video (fn.experimental.decoders.video in - older DALI): every sample is a whole file, read by the file reader, and - decoded with a new decoder per sample, the samples of a batch in parallel - on num_threads threads. Same samples (file, first and last frame) as the - DALI reader, shuffled by the file reader; `reader_depth` is the file - reader's prefetch_queue_depth - pynvvideocodec PyNvVideoCodec SimpleDecoder (NVDEC), RGB interleaved output (as DALI) - torchcodec torchcodec VideoDecoder, FFmpeg CUDA backend on GPU, FFmpeg on CPU - torchcodec_nvdec torchcodec VideoDecoder with its own NVDEC backend (torchcodec >= 0.8) - They decode the same samples as the DALI reader (the same sequences of the same files, - in the same order, or shuffled with a fixed seed), each sample with frames [start, end) - of one file. A batch is decoded by a pool of min(threads, batch) threads, each with its - own decoder, kept while consecutive samples come from the same file (PyNvVideoCodec - reconfigures its decoder for a new file, torchcodec opens a new one, scanning it, as - its default seek_mode="exact" does). Batches are not prefetched: unlike DALI, a batch - is decoded when it is requested. A batch's outputs are kept until the next batch is - requested, as DALI's, so the process memory (mem MB) includes one batch for them, and - for DALI also the batches in its queues (`depth` + `reader_depth`). PyNvVideoCodec is an - exception: its frames stay in its decoder's buffers, which the decoder's next call - reuses, so its memory does not include a batch. They run only the configurations that - mean the same for them (backend_runs), and need torch. Rows are flagged when torchcodec - reports a CPU fallback, and PyNvVideoCodec and dali_decoder rows on clips with hidden - lead-in frames (LEAD_IN_FILES), where they return different frames. +"""Benchmark of fn.experimental.readers.video, using DALI_extra data only. + + video_reader_bench.py --json a.json throughput (frames/s) of every configuration + video_reader_bench.py --json a.json --check ... and fail if more threads (or a deeper reader + queue) are more than 10% slower than fewer + video_reader_bench.py --json a.json --hash checksums of the output instead of throughput + video_reader_bench.py --compare a.json b.json two builds: checksums equal, throughput ratio + +Configurations (a group is one set of samples, read with different num_threads / reader_depth): + + lanes 256 distinct files, one sequence each: every sample is a new file, so the cost + of opening a file and the parallelism of decoding dominate + cpu the same on the CPU backend (VP8/VP9 files) + seq, random the 15 H.264 clips as they are, read in order or shuffled + random_hevc the HEVC trailer, shuffled (keyframe seeks, open GOPs) + reader_depth the reader's own prefetch_queue_depth at batch 1 + single_* one long file, batch 1: a single stream, so what is measured is how fast one + decoder runs per frame, not how many run in parallel + +The GPU configurations also run through torchcodec on the GPU (groups *_torchcodec, FFmpeg's +NVDEC), on the same samples: for reference only, never checked, skipped if it is not installed. + +Every configuration runs in a fresh process, so the frame index and NVDEC decoder caches do not +carry over, and is repeated --repeat times: the median is reported. The first batches (those +the pipeline and the reader's queue may have produced before the clock starts) are not timed. + +With --hash, the output of every configuration of a group must be identical, whatever +num_threads and reader_depth: each configuration records a checksum of the first >= 40 samples +(so that the reader's prefetch queue wraps around). The configurations marked "checksums only" +add reader_depth 2 with batches of 16, which the timed ones do not run. """ import argparse +import contextlib import glob import hashlib +import itertools import json import os import random -import shutil -import socket import statistics import subprocess import sys import tempfile import threading import time -import warnings +from concurrent.futures import ThreadPoolExecutor VIDEO_DIR = "db/video" - -# --hash checks at least this many samples per configuration: twice the batches the -# pipeline and the reader can hold (depth 2 + reader_depth up to 8) at batch 1, so that -# every slot of the reader's prefetch queue is reused -HASH_MIN_SAMPLES = 40 - -VPX_CLIPS = [ - "vp9/vp9_0.mp4", - "vp9/vp9_2.mp4", - "vp8/vp8.webm", - "cfr/test_1_vp9.mp4", - "cfr/test_2_vp9.mp4", - "vfr/test_1_vp9.mp4", - "vfr/test_2_vp9.mp4", -] # file sets: glob patterns relative to DALI_extra/db/video -FILES = { - # 15 H.264 1280x720 clips, 122-430 frames, B-frames, 1-4 keyframes (GOP up to 250) - "h264_clips": ["sintel/video_files/*.mp4", "sintel/labelled_videos/*/*.mp4"], - # one H.264 1280x720 clip, 1253 frames, GOP 250 - "h264_long": ["sintel/sintel_trailer-720p.mp4"], - # the same clip as HEVC 8-bit (keyframe every ~9 frames) and HEVC 10-bit (GOP 250) - "hevc_long": ["hevc/sintel_trailer-720p.mp4"], - "hevc10_long": ["hevc/sintel_trailer-720p_10b.mp4"], - # one H.264 512x512 clip, 250 frames: small frames, per-frame overheads dominate - "h264_small": ["test_label.mp4"], - # two H.264 800x600 clips, coded as 800x608 (display area cropped) - "h264_cropped_pair": ["cfr/test_2.mp4", "vfr/test_2.mp4"], - # two H.264 1280x720 clips, coded size == display size - "h264_aligned_pair": ["cfr_test.mp4", "vfr_test.mp4"], - # VP8/VP9 clips (H.264 is not enabled on the CPU backend) - "vpx_clips": VPX_CLIPS, -} -# File sets where most clips (cut without re-encoding) start before the frame they show first, -# with an MP4 edit list hiding the lead-in frames: the DALI reader and torchcodec start at the -# first visible frame, so they decode the lead-in before frame 0, while PyNvVideoCodec and -# DALI's video decoder (dali_decoder) return the lead-in frames themselves (different frames, -# less decoding) -LEAD_IN_FILES = {"h264_clips"} -LEAD_IN_BACKENDS = {"pynvvideocodec", "dali_decoder"} - +CLIPS = ["sintel/video_files/*.mp4", "sintel/labelled_videos/*/*.mp4"] # 15 H.264 720p clips +H264_LONG = ["sintel/sintel_trailer-720p.mp4"] # 1253 frames, GOP 250 +HEVC_LONG = ["hevc/sintel_trailer-720p.mp4"] # the same clip as HEVC, a keyframe every ~9 frames +VPX = ["vp9/vp9_0.mp4", "vp9/vp9_2.mp4", "vp8/vp8.webm", "cfr/test_1_vp9.mp4", "vfr/test_1_vp9.mp4"] +HASH_MIN_SAMPLES = 40 -def _cfg(files, layout="plain", device="gpu", batch=8, seq=32, threads=4, **kw): - """A benchmark configuration. Only JSON-serializable values. - layout: "plain" - the files as they are (filenames=...) - "many:N" - N distinct symlinks cycling over the files, each with one - sequence starting at frame 0 (a keyframe) - "alternate:N" - N lines alternating between the files, each starting at - frame 0, read in order: every sample switches files - "same:N" - N lines, all of the first file, each starting at frame 0: the - control for "alternate" (no file switches, but every sample - after the first rewinds the same decoder to frame 0) - """ - cfg = dict( - backend="dali", - files=files, - layout=layout, - device=device, - batch=batch, - seq=seq, - threads=threads, - depth=2, # pipeline prefetch_queue_depth (DALI default) - reader_depth=1, # the reader's own prefetch_queue_depth (DALI default) - readers=1, - shuffle=False, - ) - cfg.update(kw) - return cfg +def dali_configs(threads): + """(group, configuration) of the DALI reader. - -def scenarios(threads, readers, quick=False): - """The configurations of every scenario (HASH_ONLY_SCENARIOS only with --hash).""" + `many`: that number of symlinks to the files, one sequence each. Without it, the files as + they are.""" + base = dict(batch=16, seq=32, rd=1, shuffle=False, device="gpu") t_max = max(threads) - t_pair = sorted({1, t_max}) - iters = {} if quick else dict(iters=20) - return { - "lanes": [_cfg("h264_clips", "many:256", batch=16, seq=32, threads=t) for t in threads], - "seq_longgop": [_cfg("h264_clips", batch=16, seq=32, threads=t) for t in t_pair], - "random_access": [ - _cfg(f, batch=16, seq=32, threads=t, shuffle=True) - for f in ("h264_clips", "hevc_long") - for t in t_pair - ], - "single_stream": [ - _cfg(f, batch=1, seq=64, threads=4) - for f in ("h264_long", "hevc_long", "hevc10_long", "h264_small") - ], - "reader_depth": [ - _cfg(f, layout, batch=1, seq=32, threads=8, reader_depth=d) - for f, layout in (("h264_clips", "many:256"), ("h264_long", "plain")) - for d in (1, 2, 4, 8) - ], - # ms_per_sample(alternate) - ms_per_sample(same), per file pair: cost of opening the - # other file relative to rewinding the same one (see the module docstring). One - # thread: with more, other libraries keep each file of a pair on its own decoder - "cache_key": [ - _cfg(f, f"{kind}:64", batch=8, seq=8, threads=1) - for f in ("h264_cropped_pair", "h264_aligned_pair") - for kind in ("alternate", "same") - ], - # the noisiest scenario: more iterations than the --iters default - "engines": [ - _cfg("h264_clips", "many:64", batch=4, seq=32, threads=4, readers=k, **iters) - for k in readers - ], - "cpu": [ - _cfg("vpx_clips", "many:128", device="cpu", batch=8, seq=16, threads=t) for t in threads - ], - # several decode lanes with up to two batches in the reader's queue (the other - # scenarios with several lanes use reader_depth 1) - "lanes_depth": [ - _cfg(f, layout, batch=16, seq=32, threads=t, reader_depth=d, shuffle=s) - for f, layout, s in ( - ("h264_clips", "plain", False), - ("h264_clips", "plain", True), - ("h264_clips", "many:256", False), - ) - for d in (1, 2) - for t in threads - ], - } - - -# scenarios only checksummed (--hash), not measured -HASH_ONLY_SCENARIOS = {"lanes_depth"} - - -# scenarios that other backends run as well (reader_depth and engines measure properties of -# the DALI reader), and the devices they decode on -_REFERENCE_SCENARIOS = {"lanes", "seq_longgop", "random_access", "single_stream", "cache_key"} -BACKEND_SCENARIOS = { - "dali": (None, ("gpu", "cpu")), # None: all scenarios - # the legacy fn.readers.video: GPU only, and constant frame rate only (both cache_key - # pairs include a variable frame rate file) - "dali_legacy": (_REFERENCE_SCENARIOS - {"cache_key"} | {"reader_depth", "engines"}, ("gpu",)), - # fn.readers.file + fn.decoders.video; reader_depth would prefetch file contents only - "dali_decoder": (_REFERENCE_SCENARIOS | {"engines", "cpu"}, ("gpu", "cpu")), - "pynvvideocodec": (_REFERENCE_SCENARIOS, ("gpu",)), - "torchcodec": (_REFERENCE_SCENARIOS | {"cpu"}, ("gpu", "cpu")), - "torchcodec_nvdec": (_REFERENCE_SCENARIOS, ("gpu",)), -} - - -def backend_runs(backend, scenario, cfg): - names, devices = BACKEND_SCENARIOS[backend] - if (names is not None and scenario not in names) or cfg["device"] not in devices: - return False - # PyNvVideoCodec 2.0 SimpleDecoder fails (IndexError) when a decoder that was just - # created or reconfigured for a file is asked for frames from the middle of it, so it - # runs only configurations whose decoders start every file at frame 0 - seeks = scenario == "random_access" or ( - scenario == "seq_longgop" and min(cfg["threads"], cfg["batch"]) > 1 - ) - if backend == "pynvvideocodec" and seeks: - return False - # the legacy reader fails seeking in the HEVC trailer (open GOPs): "The decoder returned a - # frame that is past the expected one"; reading it sequentially works - return not (backend == "dali_legacy" and seeks and cfg["files"] == "hevc_long") - - -# ---------------------------------------------------------------------------------------- -# data - - -def resolve_files(dali_extra, name): - # absolute paths: they are used as symlink targets in a temporary directory - root = os.path.abspath(os.path.join(dali_extra, VIDEO_DIR)) - files, missing = [], [] - for pattern in FILES[name]: - matches = sorted(glob.glob(os.path.join(root, pattern))) - if not matches: - missing.append(pattern) - files += matches - if missing: - raise FileNotFoundError( - f"File set '{name}': no files match {missing} under '{root}'. " - f"Check that --dali_extra (or DALI_EXTRA_PATH) points to a DALI_extra checkout." - ) - return files - - -def materialize(cfg, dali_extra, workdir): - """Creates the inputs of a configuration. Returns one input spec per reader. - - A spec is either {"filenames": [...]} or {"file_list": path}. The result is fully - determined by the configuration, so repeated runs (and --hash) see the same samples. - """ - files = resolve_files(dali_extra, cfg["files"]) - kind, _, n = cfg["layout"].partition(":") - if kind == "plain": - if cfg["readers"] != 1: - raise ValueError(f"layout='plain' requires readers=1; got readers={cfg['readers']}.") - return [{"filenames": files}] - if kind not in ("many", "alternate", "same") or not n.isdigit() or int(n) < 1: - raise ValueError( - f"Unknown layout {cfg['layout']!r}; expected 'plain', 'many:N', 'alternate:N' " - f"or 'same:N' with a positive integer N." - ) - n = int(n) - seq = cfg["seq"] - tag = "_".join(str(cfg[k]) for k in ("files", "layout", "seq", "readers")) - out_dir = os.path.join(workdir, tag.replace(":", "-")) - os.makedirs(out_dir, exist_ok=True) - lines = [] - if kind == "many": - for i in range(n * cfg["readers"]): - src = files[i % len(files)] - dst = os.path.join(out_dir, f"{i:05d}_{os.path.basename(src)}") - if not os.path.lexists(dst): - os.symlink(src, dst) - lines.append((dst, i)) - elif kind == "alternate": - lines = [(files[i % len(files)], i) for i in range(n)] - else: # same - lines = [(files[0], i) for i in range(n)] - specs = [] - per_reader = len(lines) // cfg["readers"] - for r in range(cfg["readers"]): - path = os.path.join(out_dir, f"file_list_{r}.txt") - with open(path, "w") as f: - for name, label in lines[r * per_reader : (r + 1) * per_reader]: - # frames [0, seq) (file_list_include_end=False): exactly one sequence, starting - # at frame 0, a keyframe, so no sample decodes frames it doesn't return - f.write(f"{name} {label} 0 {seq}\n") - specs.append({"file_list": path}) - return specs - - -def prewarm(paths): - """Reads files once so that every run sees a warm page cache.""" - for p in sorted(set(paths)): - with open(p, "rb") as f: - while f.read(1 << 24): - pass - - -# ---------------------------------------------------------------------------------------- -# GPU monitoring - - -def nvml_handle(device_id): - """NVML handle of the CUDA device `device_id`, honoring CUDA_VISIBLE_DEVICES. - - Assumes CUDA_DEVICE_ORDER=PCI_BUS_ID or a machine where CUDA and NVML enumerate - devices the same way. Returns (pynvml, handle) or raises. - """ - import pynvml - - pynvml.nvmlInit() - visible = os.environ.get("CUDA_VISIBLE_DEVICES") - if visible: - entry = visible.split(",")[device_id].strip() - if entry.isdigit(): - return pynvml, pynvml.nvmlDeviceGetHandleByIndex(int(entry)) - # older nvidia-ml-py versions take and return bytes, newer ones accept both - return pynvml, pynvml.nvmlDeviceGetHandleByUUID(entry.encode()) - return pynvml, pynvml.nvmlDeviceGetHandleByIndex(device_id) - - -def other_gpu_pids(nvml, handle): - procs = nvml.nvmlDeviceGetComputeRunningProcesses(handle) - procs += nvml.nvmlDeviceGetGraphicsRunningProcesses(handle) - return sorted({p.pid for p in procs if p.pid != os.getpid()}) - - -class GpuMonitor: - """Samples per-process GPU memory and collects per-process NVDEC utilization. - - While sampling, it also records any other process that uses the GPU, so that a - competing process that starts and exits during a run is detected. - """ - - def __init__(self, device_id, period=0.1): - self.errors = [] # NVML problems; a metric may be missing, the run is still valid - self.period = period - self.mem = [] - self.others = set() - self._stop = threading.Event() - self._thread = threading.Thread(target=self._run, daemon=True) - try: - self.nvml, self.handle = nvml_handle(device_id) - except Exception as e: - self.nvml = None - self._error(f"NVML unavailable: {e}") - - def _error(self, msg): - if msg not in self.errors: - self.errors.append(msg) - - def snapshot(self): - """Pre/post-run state: other processes, NVDEC utilization.""" - if not self.nvml: - return {} - nvml, h = self.nvml, self.handle - res = {} - queries = dict( - other_pids=lambda: other_gpu_pids(nvml, h), - dev_nvdec_util=lambda: nvml.nvmlDeviceGetDecoderUtilization(h)[0], - ) - for key, query in queries.items(): - try: # older NVML bindings may lack some queries for newer drivers - res[key] = query() - except Exception as e: - self._error(f"NVML {key} failed: {e}") - return res - - def _run(self): - nvml, h, pid = self.nvml, self.handle, os.getpid() - - def own_memory(): - for p in nvml.nvmlDeviceGetComputeRunningProcesses(h): - if p.pid == pid and p.usedGpuMemory is not None: - self.mem.append(p.usedGpuMemory) - - samplers = { - "own memory": own_memory, - "other processes": lambda: self.others.update(other_gpu_pids(nvml, h)), - } - failures = dict.fromkeys(samplers, 0) - while not self._stop.is_set(): - for name, sample in list(samplers.items()): - try: - sample() - failures[name] = 0 - except Exception as e: - # e.g. a process list that changed size during the query; only give up - # on a query that keeps failing (unsupported by these bindings/driver) - failures[name] += 1 - if failures[name] >= 3: - self._error(f"NVML sampling of {name} failed: {e}") - del samplers[name] - time.sleep(self.period) - - def start(self): - self._t_start_us = int(time.time() * 1e6) - if self.nvml: - self._thread.start() - - def stop(self): - if not self.nvml: - return {} - self._stop.set() - self._thread.join() - res = {} - if self.mem: - res["proc_mem_max_mb"] = max(self.mem) / 2**20 - try: - samples = self.nvml.nvmlDeviceGetProcessUtilization(self.handle, self._t_start_us) - dec = [s.decUtil for s in samples if s.pid == os.getpid()] - if dec: - res["proc_nvdec_util"] = statistics.mean(dec) - except Exception: - # per-process utilization is optional: not every driver/binding supports it, or - # there may be no samples yet; the result is still valid without it - res["proc_nvdec_util"] = None - return res - - -# ---------------------------------------------------------------------------------------- -# running - - -def num_frames_dali(path, device, device_id): - """The frames of a file as the DALI reader counts them: its sequences of one frame.""" - import nvidia.dali.fn as fn - from nvidia.dali.pipeline import pipeline_def - - @pipeline_def(batch_size=1, num_threads=1, device_id=device_id) - def count(): - return fn.experimental.readers.video( - filenames=[path], sequence_length=1, device=device, name="Count" + for t in threads: + yield "lanes", dict(base, files=CLIPS, many=256, t=t) + yield "cpu", dict(base, files=VPX, many=128, t=t, device="cpu", batch=8, seq=16) + for name, files in (("seq", CLIPS), ("random", CLIPS), ("random_hevc", HEVC_LONG)): + for t in sorted({1, t_max}): + yield name, dict(base, files=files, t=t, shuffle=name != "seq") + for rd in (1, 2, 4): + yield "reader_depth", dict(base, files=CLIPS, many=256, t=8, rd=rd, batch=1) + for name, files in (("single_h264", H264_LONG), ("single_hevc", HEVC_LONG)): + yield name, dict(base, files=files, t=4, batch=1, seq=64) + # checksums only: several decode lanes with two batches in the reader's queue + for name, extra in (("lanes", dict(many=256)), ("seq", {}), ("random", dict(shuffle=True))): + yield name, dict(base, files=CLIPS, t=t_max, rd=2, hash_only=True, **extra) + + +def configs(threads, torchcodec=True): + cfgs = list(dali_configs(threads)) + yield from cfgs + for group, c in cfgs if torchcodec else (): + if c["device"] == "gpu" and group != "reader_depth" and not c.get("hash_only"): + yield group + "_torchcodec", dict(c, lib="torchcodec") + + +def key(group, c): + return f"{group} t={c['t']} rd={c['rd']}" + + +def torchcodec_step(cfg, samples): + """Returns a function that decodes a batch of (path, start, end) samples with torchcodec on + min(t, batch) threads, each keeping its decoder while consecutive samples are of one file.""" + import torch + from torchcodec import decoders + + if cfg["shuffle"]: + random.Random(0).shuffle(samples) + batches, local, keep = itertools.cycle(samples), threading.local(), [] + pool = ThreadPoolExecutor(min(cfg["t"], cfg["batch"])) + # FFmpeg's NVDEC, as the default CUDA backend differs between torchcodec versions + backend = getattr(decoders, "set_cuda_backend", None) + + def decode(sample): + path, first, end = sample + if getattr(local, "path", None) != path: + torch.cuda.set_device(0) + with backend("ffmpeg") if backend else contextlib.nullcontext(): + local.decoder = decoders.VideoDecoder(path, device="cuda:0") + local.path = path + frames = local.decoder.get_frames_in_range(first, end).data + assert len(frames) == end - first, f"{path}[{first}:{end}]: {len(frames)} frames" + return frames + + def run_batch(): + keep[:] = pool.map(decode, [next(batches) for _ in range(cfg["batch"])]) # as DALI's + torch.cuda.synchronize() + + torch.zeros(1, device="cuda") # the CUDA context, before the clock starts + return run_batch + + +def child(cfg, root, hash_out): + """Runs one configuration in this process: {"fps", "build_s"}, or {"hash"}.""" + import numpy as np + + files = sorted(f for p in cfg["files"] for f in glob.glob(os.path.join(root, VIDEO_DIR, p))) + assert files, f"no files for {cfg['files']} under {root}/{VIDEO_DIR}" + samples = [] # (path, start, end) + if cfg.get("many"): # distinct symlinks: nothing keyed by the file name is shared + tmp = tempfile.mkdtemp() + lines = [] + for i in range(cfg["many"]): + link = os.path.join(tmp, f"{i}_{os.path.basename(files[i % len(files)])}") + os.symlink(files[i % len(files)], link) + lines.append(f"{link} {i} 0 {cfg['seq']}\n") # frames [0, seq) + samples.append((link, 0, cfg["seq"])) + with open(os.path.join(tmp, "list.txt"), "w") as f: + f.writelines(lines) + src = dict( + file_list=os.path.join(tmp, "list.txt"), + file_list_format="frames", + file_list_include_end=False, ) + else: + src = dict(filenames=files, labels=list(range(len(files)))) - p = count() - p.build() - return p.reader_meta("Count")["epoch_size"] - - -def build_pipeline(cfg, specs, device_id, seed, frame_num=False): - import nvidia.dali.fn as fn - from nvidia.dali.pipeline import pipeline_def - - legacy = cfg["backend"] == "dali_legacy" - reader = fn.readers.video if legacy else fn.experimental.readers.video - if cfg["backend"] == "dali_decoder": - - def num_frames(path): - return num_frames_dali(path, cfg["device"], device_id) - - # the samples the DALI reader would produce, (path, start, end), one list per reader - samples = [sample_list([s], cfg["seq"], False, num_frames) for s in specs] - # moved out of experimental in DALI 2.0 - decoder = getattr(fn.decoders, "video", None) or fn.experimental.decoders.video + t0 = time.perf_counter() + if cfg.get("lib") == "torchcodec": + if not cfg.get("many"): # whole files in full sequences, as the DALI reader splits them + from torchcodec.decoders import VideoDecoder - @pipeline_def( - batch_size=cfg["batch"], - num_threads=cfg["threads"], - device_id=device_id, - prefetch_queue_depth=cfg["depth"], - seed=seed, - ) - def pipe(): - outs = [] - if cfg["backend"] == "dali_decoder": - for r, reader_samples in enumerate(samples): - # whole files, with the sample's first frame as the label - encoded, start = fn.readers.file( - files=[path for path, _, _ in reader_samples], - labels=[start for _, start, _ in reader_samples], - random_shuffle=cfg["shuffle"], - prefetch_queue_depth=cfg["reader_depth"], - name=f"Reader{r}", - ) - video = decoder( - encoded, - device="mixed" if cfg["device"] == "gpu" else "cpu", - start_frame=start[0], + for f in files: + for first in range(0, len(VideoDecoder(f)) - cfg["seq"] + 1, cfg["seq"]): + samples.append((f, first, first + cfg["seq"])) + run_batch = torchcodec_step(cfg, samples) + else: + import nvidia.dali.fn as fn + from nvidia.dali.pipeline import pipeline_def + + @pipeline_def(batch_size=cfg["batch"], num_threads=cfg["t"], device_id=0, seed=42) + def pipe(): + return tuple( + fn.experimental.readers.video( + device=cfg["device"], sequence_length=cfg["seq"], + random_shuffle=cfg["shuffle"], + prefetch_queue_depth=cfg["rd"], + **src, ) - outs += [video, start] - return tuple(outs) - for r, spec in enumerate(specs): - kw = dict(spec) - if "filenames" in kw: - kw["labels"] = list(range(len(kw["filenames"]))) - elif legacy: - kw["file_list_frame_num"] = True # frame numbers, not timestamps; end exclusive - else: - kw["file_list_format"] = "frames" - kw["file_list_include_end"] = False # explicit: the default changed over time - if frame_num: - kw["enable_frame_num"] = True - outs += reader( - device=cfg["device"], - sequence_length=cfg["seq"], - random_shuffle=cfg["shuffle"], - prefetch_queue_depth=cfg["reader_depth"], - name=f"Reader{r}", - **kw, ) - return tuple(outs) - - return pipe() - - -def warm_up_cuda(device_id): - """Initializes CUDA and the executor so build_s measures the reader, not the runtime.""" - import nvidia.dali.fn as fn - from nvidia.dali.pipeline import pipeline_def - - @pipeline_def(batch_size=1, num_threads=1, device_id=device_id) - def tiny(): - return fn.random.uniform(shape=[1]).gpu() - p = tiny() - p.build() - p.run() - - -class DaliRunner: - """The DALI reader. run() returns the (frames, height, width) of every sample.""" - - def __init__(self, cfg, specs, args): - import nvidia.dali as dali - - self.cfg = cfg - self.version = ( - dali.__version__ - + { - "dali": "", - "dali_legacy": " (fn.readers.video)", - "dali_decoder": " (fn.readers.file + video decoder)", - }[cfg["backend"]] + p = pipe() + p.build() + run_batch = p.run + build_s = time.perf_counter() - t0 + + if hash_out: # from batch 0: the same samples whatever the queue depths + h = hashlib.sha1() + for _ in range(-(-HASH_MIN_SAMPLES // cfg["batch"]) + 2): + for o in run_batch(): + o = o.as_cpu() if hasattr(o, "as_cpu") else o + for i in range(len(o)): + h.update(np.ascontiguousarray(np.array(o[i])).tobytes()) + return dict(hash=h.hexdigest()) + # batches that may have been produced before the clock starts: pipeline + reader queues + for _ in range(2 + cfg["rd"] + 1): + run_batch() + n, t0 = 0, time.perf_counter() + while n < 20 or time.perf_counter() - t0 < 5: # >= 20 batches and >= 5 s + run_batch() + n += 1 + return dict(fps=n * cfg["batch"] * cfg["seq"] / (time.perf_counter() - t0), build_s=build_s) + + +def check_scaling(out, tolerance=0.10): + """More threads, or a deeper reader queue, must not be slower than fewer (flat is fine).""" + ok = True + for group in sorted({v["group"] for v in out.values() if "fps" in v}): + rows = sorted((v["t"], v["rd"], v["fps"]) for v in out.values() if v["group"] == group) + if group.endswith("_torchcodec") or len(rows) < 2: + continue + (t0, rd0, slow), (t1, rd1, fast) = rows[0], rows[-1] + bad = fast < (1 - tolerance) * slow + ok &= not bad + print( + f"{'FAIL' if bad else 'ok '} {group}: {slow:.0f} frames/s (t={t0}, rd={rd0})" + f" -> {fast:.0f} (t={t1}, rd={rd1})" ) - # up to `depth` (pipeline queue) + `reader_depth` (the reader's own queue) batches - # may be produced before the clock starts - self.prefetched = cfg["depth"] + cfg["reader_depth"] - warm_up_cuda(args.device_id) - self.pipe = build_pipeline(cfg, specs, args.device_id, args.seed) - - def build(self): - self.pipe.build() - - def run(self): - out = self.pipe.run() - # shapes are host-side metadata; video outputs are every other output (video, label) - return [tuple(s[:3]) for r in range(self.cfg["readers"]) for s in out[2 * r].shape()] - - -def sample_list(specs, seq, shuffle, num_frames): - """The samples the DALI reader produces for `specs`: (path, start, end) tuples. - - Whole files are split into full sequences of `seq` frames (the reader's default step - and stride); a file list gives one sample per line. DALI shuffles with its own seed, so - a shuffled list has the same samples in a different order. - """ - (spec,) = specs - samples = [] - if "filenames" in spec: - for path in spec["filenames"]: - n = num_frames(path) - samples += [(path, s, s + seq) for s in range(0, n - seq + 1, seq)] - else: - with open(spec["file_list"]) as f: - for line in f: - path, _, start, end = line.split() - samples.append((path, int(start), int(end))) - if shuffle: - random.Random(0).shuffle(samples) - return samples - - -class ReferenceRunner: - """PyNvVideoCodec or torchcodec, called directly through their own APIs. - - Each sample of a batch is decoded by one of `threads` threads; every thread keeps its - own decoder while consecutive samples come from the same file. Outputs are what the - libraries return; torch is used only to wait for the GPU. - """ - - prefetched = 0 # a batch is decoded when it is requested - - def __init__(self, cfg, specs, args): - import itertools - from concurrent.futures import ThreadPoolExecutor - - import torch - - self.cfg, self.torch = cfg, torch - self.backend, self.gpu = cfg["backend"], cfg["device"] == "gpu" - self.device_id = args.device_id - if self.gpu: - torch.cuda.set_device(self.device_id) - torch.zeros(1, device="cuda") # CUDA context, so build_s measures the decoders - torch.cuda.synchronize() - if self.backend == "pynvvideocodec": - import PyNvVideoCodec as nvc - - self.nvc, self.version = nvc, f"PyNvVideoCodec {nvc.__version__}" - else: - import torchcodec - - self.version = f"torchcodec {torchcodec.__version__}" - self.version += f", torch {torch.__version__}" - self.local = threading.local() - # torchcodec on the GPU: whether a decoder reported decoding on the CPU instead - # (None: not known, e.g. older torchcodec without VideoDecoder.cpu_fallback) - self.cpu_fallback = None - if self.gpu and self.backend != "pynvvideocodec": - from torchcodec.decoders import VideoDecoder - - self.cpu_fallback = False if hasattr(VideoDecoder, "cpu_fallback") else None - samples = sample_list(specs, cfg["seq"], cfg["shuffle"], self._num_frames) - self.samples = itertools.cycle(samples) - # no more threads than samples per batch: an idle extra thread would take the next - # batch's samples of the same file and make its decoder seek - workers = min(cfg["threads"], cfg["batch"]) - self.pool = ThreadPoolExecutor(workers) if workers > 1 else None - - def _open(self, path): - if self.backend == "pynvvideocodec": - return self.nvc.SimpleDecoder( - path, - gpu_id=self.device_id, - use_device_memory=True, - output_color_type=self.nvc.OutputColorType.RGB, # interleaved, as DALI + return ok + + +def run(a): + todo = [ + (group, c) + for group, c in configs(a.threads, not (a.hash or a.no_torchcodec)) + if a.only in group and (a.hash or not c.get("hash_only")) + ] + out = {} + for group, c in todo: + reference = c.get("lib") == "torchcodec" # never fails the run + reps = [] + for _ in range(1 if a.hash else a.repeat): + r = subprocess.run( + [sys.executable, __file__, "--child", json.dumps(c), "--dali_extra", a.dali_extra] + + ["--hash"] * a.hash, + capture_output=True, + text=True, ) - from torchcodec import decoders - - if not self.gpu: - return decoders.VideoDecoder(path, device="cpu") - # the CUDA backend is chosen explicitly, as the default changed: FFmpeg's NVDEC support - # up to torchcodec 0.11 (the only one before 0.8), torchcodec's own NVDEC decoder later - # ("nvdec", called "beta" before, which later versions still accept) - if not hasattr(decoders, "set_cuda_backend"): - return decoders.VideoDecoder(path, device=f"cuda:{self.device_id}") - backend = "ffmpeg" if self.backend == "torchcodec" else "beta" - with decoders.set_cuda_backend(backend): - return decoders.VideoDecoder(path, device=f"cuda:{self.device_id}") - - def _num_frames(self, path): - return len(self._decoder(path)) - - def _decoder(self, path): - """This thread's decoder, switched to `path`.""" - local = self.local - if getattr(local, "path", None) == path: - return local.decoder - if self.gpu: - self.torch.cuda.set_device(self.device_id) # this thread's current device - if self.backend == "pynvvideocodec" and getattr(local, "decoder", None) is not None: - local.decoder.reconfigure_decoder(path) # picks a cached NVDEC decoder if any - else: - local.decoder = self._open(path) - local.path = path - return local.decoder - - def _decode(self, sample): - """Returns the sample's (frames, height, width) and the library's output.""" - path, start, end = sample - decoder = self._decoder(path) - # the libraries' own outputs, not converted or copied - if self.backend == "pynvvideocodec": - frames = decoder.get_batch_frames_by_index(list(range(start, end))) - return (len(frames),) + tuple(frames[0].shape[:2]), frames # frames of H, W, C - video = decoder.get_frames_in_range(start, end).data - if self.gpu and getattr(decoder, "cpu_fallback", None): # torchcodec >= 0.11 - self.cpu_fallback = True - return (video.shape[0], video.shape[2], video.shape[3]), video # F, C, H, W - - def build(self): - pass - - def run(self): - # The outputs of a batch are kept until the next run(), as Pipeline.run() outputs, so - # that the process memory includes a whole batch (DALI's also includes its queues) - self.outputs = None - batch = [next(self.samples) for _ in range(self.cfg["batch"])] - decoded = list( - self.pool.map(self._decode, batch) if self.pool else map(self._decode, batch) - ) - if self.gpu: - self.torch.cuda.synchronize() # as Pipeline.run(): outputs are ready - self.outputs = [out for _, out in decoded] - return [shape for shape, _ in decoded] - - -def make_runner(cfg, specs, args): - dali = cfg["backend"] in ("dali", "dali_legacy", "dali_decoder") - return (DaliRunner if dali else ReferenceRunner)(cfg, specs, args) - - -def percentile(sorted_vals, q): - if len(sorted_vals) == 1: - return sorted_vals[0] - return statistics.quantiles(sorted_vals, n=100, method="inclusive")[q - 1] - - -def run_one(cfg, specs, args): - """Runs one configuration in this process and returns its metrics.""" - mon = GpuMonitor(args.device_id) - res = dict(pre=mon.snapshot()) - - runner = make_runner(cfg, specs, args) # initializes CUDA before the clock starts - res["backend_version"] = runner.version - t0 = time.perf_counter() # build_s: the DALI reader (indexing), not the CUDA runtime - runner.build() - if isinstance(runner, DaliRunner): # other libraries open their decoders lazily, in run() - res["build_s"] = time.perf_counter() - t0 - t0 = time.perf_counter() - runner.run() - res["first_batch_ms"] = (time.perf_counter() - t0) * 1e3 - res["shape"] = list(runner.run()[0]) - - iters = cfg.get("iters", args.iters) - for _ in range(args.warmup): - runner.run() - # Up to runner.prefetched batches (DALI: `depth` in the pipeline queue + `reader_depth` - # in the reader's own queue) may have been produced before the clock started. They are - # excluded, and --iters/--min_time apply to the remaining window. - skip = runner.prefetched - stamps, frames = [], [] - mon.start() - t_start = time.perf_counter() - while True: - shapes = runner.run() # actual (frames, height, width) of every sample - stamps.append(time.perf_counter()) - frames.append(sum(s[0] for s in shapes)) - kept = len(stamps) - skip - if kept > 0 and kept >= iters: - t_ref = stamps[skip - 1] if skip else t_start - if stamps[-1] - t_ref >= args.min_time: + if r.returncode: + reps = None + print(f"FAILED {key(group, c)}{' (reference)' if reference else ''}:") + print(r.stderr[-1500:]) break - res.update(mon.stop()) - span = stamps[-1] - t_ref - n = len(stamps) - skip - times_ms = sorted((b - a) * 1e3 for a, b in zip([t_ref] + stamps[skip:], stamps[skip:])) - res.update( - iters=n, - iters_skipped=skip, - frames_per_s=sum(frames[skip:]) / span, - ms_per_sample=span * 1e3 / (n * cfg["readers"] * cfg["batch"]), - iter_p50_ms=statistics.median(times_ms), - iter_p90_ms=percentile(times_ms, 90), - iter_max_ms=times_ms[-1], - ) - if getattr(runner, "cpu_fallback", None) is not None: - res["cpu_fallback"] = runner.cpu_fallback - res["post"] = mon.snapshot() - if mon.errors: # after the last NVML query, so that every failure is recorded - res["nvml_error"] = "; ".join(mon.errors) - pre, post = res["pre"], res["post"] - res["other_pids_seen"] = sorted( - mon.others | set(pre.get("other_pids", [])) | set(post.get("other_pids", [])) - ) - res["contaminated"] = bool(res["other_pids_seen"] or pre.get("dev_nvdec_util", 0) > 5) - return res - - -def hash_one(cfg, specs, args): - """Returns one checksum per sample of the first batches (see HASH_MIN_SAMPLES and - --hash_epochs).""" - pipe = build_pipeline(cfg, specs, args.device_id, args.seed, frame_num=True) - pipe.build() - hashes = [] - # Not dependent on threads or reader_depth, so that --compare --ignore can match runs - batches = -(-HASH_MIN_SAMPLES // (cfg["batch"] * cfg["readers"])) # ceil - if args.hash_epochs: - epoch = max(pipe.epoch_size(f"Reader{r}") for r in range(cfg["readers"])) - batches = max(batches, -(-args.hash_epochs * epoch // cfg["batch"]) + 1) - for _ in range(batches): - outs = [_as_cpu(t) for t in pipe.run()] - for r in range(cfg["readers"]): - video, label, frame_num = outs[3 * r : 3 * r + 3] - for i in range(len(video)): - h = hashlib.sha1(video.at(i).tobytes()) - h.update(label.at(i).tobytes()) - h.update(frame_num.at(i).tobytes()) - hashes.append(h.hexdigest()) - return dict(hashes=hashes, hash_batches=batches) - - -def _as_cpu(tensor_list): - return tensor_list.as_cpu() if hasattr(tensor_list, "as_cpu") else tensor_list - - -def child_main(args): - job = json.loads(args._single) - cfg, specs = job["cfg"], job["specs"] - try: - res = hash_one(cfg, specs, args) if args.hash else run_one(cfg, specs, args) - except Exception as e: - lines = [ln for ln in str(e).strip().splitlines() if ln.strip()] - detail = [ln for ln in lines if "rror" in ln or "ssert" in ln or "failed" in ln] - res = dict(error=f"{type(e).__name__}: " + " | ".join((detail or lines)[-3:])) - print("RESULT " + json.dumps(res)) - - -def run_child(cfg, specs, args): - cmd = [sys.executable, os.path.abspath(__file__), "--_single"] - cmd += [json.dumps(dict(cfg=cfg, specs=specs))] - for flag in ("dali_extra", "device_id", "iters", "warmup", "min_time", "seed", "hash_epochs"): - cmd += [f"--{flag}", str(getattr(args, flag))] - cmd += ["--hash"] if args.hash else [] - try: - p = subprocess.run(cmd, capture_output=True, text=True, timeout=args.timeout) - except subprocess.TimeoutExpired: - return dict(error=f"timeout after {args.timeout} s") - for line in reversed(p.stdout.splitlines()): - if line.startswith("RESULT "): - return json.loads(line[len("RESULT ") :]) - tail = [ln for ln in (p.stderr or p.stdout).splitlines() if ln.strip()][-15:] - detail = [ln for ln in tail if "rror" in ln or "ssert" in ln] or tail[-3:] - return dict(error=f"exit code {p.returncode}: " + " | ".join(detail[-3:])) - - -def aggregate(cfg, reps): - """Median and spread over repetitions.""" - ok = [r for r in reps if "error" not in r] - res = dict(cfg) - res["repeats"] = len(reps) - if not ok: - res["error"] = reps[-1]["error"] if reps else "no runs" - return res - res["errors"] = [r["error"] for r in reps if "error" in r] - res["contaminated"] = any(r.get("contaminated") for r in ok) - if any("cpu_fallback" in r for r in ok): - res["cpu_fallback"] = any(r.get("cpu_fallback") for r in ok) - res["runs"] = ok - for key in ( - "frames_per_s", - "ms_per_sample", - "iter_p50_ms", - "iter_p90_ms", - "iter_max_ms", - "build_s", - "first_batch_ms", - "proc_nvdec_util", - "proc_mem_max_mb", - ): - vals = [r[key] for r in ok if r.get(key) is not None] - if vals: - res[key] = statistics.median(vals) - if key in ("frames_per_s", "build_s", "first_batch_ms") and len(vals) > 1: - res[key + "_spread_pct"] = (max(vals) - min(vals)) / res[key] * 100 - res["shape"] = ok[0].get("shape") - for key in ("backend_version", "nvml_error"): - if ok[0].get(key): - res[key] = ok[0][key] - return res - - -# ---------------------------------------------------------------------------------------- -# reporting - - -COLUMNS = [ - ("scenario", "scenario", "{:<13}"), - ("backend", "backend", "{:<16}"), - ("files", "files", "{:<17}"), - ("layout", "layout", "{:<12}"), - ("device", "dev", "{:<3}"), - ("batch", "bs", "{:>2}"), - ("seq", "seq", "{:>3}"), - ("threads", "thr", "{:>3}"), - ("reader_depth", "rq", "{:>2}"), - ("readers", "rd", "{:>2}"), - ("shuffle", "sh", "{:<2}"), - ("frames_per_s", "frames/s", "{:>8.1f}"), - ("frames_per_s_spread_pct", "spr%", "{:>4.1f}"), - ("ms_per_sample", "ms/smp", "{:>7.2f}"), - ("iter_p50_ms", "p50 ms", "{:>8.2f}"), - ("iter_p90_ms", "p90 ms", "{:>8.2f}"), - ("iter_max_ms", "max ms", "{:>8.2f}"), - ("build_s", "build s", "{:>7.3f}"), - ("first_batch_ms", "1st ms", "{:>7.1f}"), - ("proc_nvdec_util", "dec%", "{:>4.0f}"), - ("proc_mem_max_mb", "mem MB", "{:>6.0f}"), -] - - -def _width(fmt): - return int(fmt[3:].split(".")[0].rstrip("}")) - - -def print_header(): - print(" ".join(head.rjust(_width(fmt)) for _, head, fmt in COLUMNS) + " flags") - - -def format_row(r): - cells = [] - for key, head, fmt in COLUMNS: - v = r.get(key) - if key == "shuffle": - v = "y" if v else "n" - width = _width(fmt) - try: - cells.append(fmt.format(v) if v is not None else "-".rjust(width)) - except (ValueError, TypeError): - cells.append(str(v).ljust(width)) - flags = [] - if r.get("contaminated"): - flags.append("CONTAMINATED") - if r.get("frames_per_s_spread_pct", 0) > 3: - flags.append("spread>3%") - if r.get("cpu_fallback"): - flags.append("CPU fallback") # reported by torchcodec (>= 0.11) - if r.get("backend") in LEAD_IN_BACKENDS and r.get("files") in LEAD_IN_FILES: - flags.append("lead-in frames") # not the same frames as DALI, see LEAD_IN_FILES - if r.get("errors"): - flags.append(f"{len(r['errors'])} failed run(s)") - if r.get("error"): - flags.append("ERROR: " + r["error"][:160]) - return " ".join(cells) + (" " + ", ".join(flags) if flags else "") - - -def provenance(args): - info = dict( - time=time.strftime("%Y-%m-%d %H:%M:%S"), - host=socket.gethostname(), - python=sys.version.split()[0], - cpu_count=os.cpu_count(), - argv=sys.argv, - env={k: v for k, v in os.environ.items() if k.startswith("DALI_") or k.startswith("CUDA_")}, - ) - try: - import nvidia.dali as dali - - info["dali_version"] = dali.__version__ - info["dali_git_sha"] = getattr(dali, "__git_sha__", None) - info["dali_path"] = os.path.dirname(dali.__file__) - except Exception as e: - info["dali_error"] = str(e) - from importlib import metadata - - for pkg in ("PyNvVideoCodec", "torchcodec", "torch"): - try: - info.setdefault("packages", {})[pkg] = metadata.version(pkg) - except metadata.PackageNotFoundError: - continue # optional: needed only by the --backends that use it - try: - nvml, h = nvml_handle(args.device_id) - except Exception as e: - info["nvml_error"] = str(e) - return info - queries = dict( - gpu=lambda: nvml.nvmlDeviceGetName(h), - gpu_uuid=lambda: nvml.nvmlDeviceGetUUID(h), - driver=lambda: nvml.nvmlSystemGetDriverVersion(), - other_gpu_pids=lambda: other_gpu_pids(nvml, h), - nvdec_util_at_start=lambda: nvml.nvmlDeviceGetDecoderUtilization(h)[0], - ) - for key, query in queries.items(): - try: # one unsupported query must not hide the others - value = query() - info[key] = value.decode() if isinstance(value, bytes) else value - except Exception as e: - info.setdefault("nvml_query_errors", {})[key] = str(e) - return info - - -def compare(a_path, b_path, ignore): - """Compares per-sample hashes of matching configurations in two --hash files. - - Fields listed in `ignore` (e.g. threads, reader_depth) are left out when matching, - so a configuration in A is compared against every configuration in B that differs - only in those fields. The output must not depend on them. - """ - with open(a_path) as fa, open(b_path) as fb: - a, b = json.load(fa), json.load(fb) - skip = {"hashes", "scenario", "error", "errors"} | set(ignore) - - def key(r): - return json.dumps({k: v for k, v in r.items() if k not in skip}, sort_keys=True) - - def label(r): - return ", ".join(f"{k}={r[k]}" for k in sorted(r) if k not in ("hashes", "errors", "error")) - - b_runs = {} - for r in b["results"]: - b_runs.setdefault(key(r), []).append(r) - a_keys = {key(r) for r in a["results"]} - bad = compared = 0 - # a configuration that failed or is missing on either side fails the comparison: a - # build that cannot decode a workload must not pass as "identical" - for r in b["results"]: - if key(r) not in a_keys: - bad += 1 - print(f"MISSING in {a_path}: {label(r)}") - failed_runs = set() # (file, index): each failed run is reported and counted once - for path, results in ((a_path, a["results"]), (b_path, b["results"])): - for i, r in enumerate(results): - if "hashes" not in r and (path, i) not in failed_runs: - failed_runs.add((path, i)) - bad += 1 - print(f"FAILED in {path}: {label(r)} ({r.get('error', 'no checksums')})") - for r in a["results"]: - matches = b_runs.get(key(r), []) - if not matches: - bad += 1 - print(f"MISSING in {b_path}: {label(r)}") - for other in matches: - if "hashes" not in r or "hashes" not in other: - continue # reported above - n = max(len(r["hashes"]), len(other["hashes"])) - diff = sum(x != y for x, y in zip(r["hashes"], other["hashes"])) - diff += abs(len(r["hashes"]) - len(other["hashes"])) - compared += 1 - bad += diff > 0 - vs = ", ".join(f"{k}={other[k]}" for k in ignore if other.get(k) != r.get(k)) - status = "OK " if diff == 0 else "DIFF" - print(f"{status} {diff:4d}/{n} samples {label(r)}" + (f" vs {vs}" if vs else "")) - if compared == 0 and bad == 0: - print("nothing to compare: no matching configurations") - return 2 - print("all identical" if bad == 0 else f"{bad} comparison(s) differ, failed or are missing") - return 1 if bad else 0 - - -# ---------------------------------------------------------------------------------------- - - -def parse_args(): - parser = argparse.ArgumentParser( - description="fn.experimental.readers.video benchmark on DALI_extra data", - formatter_class=argparse.RawDescriptionHelpFormatter, - epilog=__doc__, - ) - parser.add_argument("-s", "--scenarios", default="all", help="comma separated, or all") - parser.add_argument( - "-j", "--threads", default="1,2,4,8,16", help="num_threads sweep (B200: 1,2,4,7,8,14,16)" - ) - parser.add_argument("--readers", default="1,2,4,8", help="reader counts for 'engines'") - parser.add_argument( - "-b", - "--backends", - default="dali", - help=f"comma separated, any of {list(BACKEND_SCENARIOS)}; others for reference", - ) - parser.add_argument("-d", "--device_id", default=0, type=int) - parser.add_argument("--repeat", default=3, type=int, help="fresh-process runs per config") - parser.add_argument("--iters", default=20, type=int, help="minimum timed iterations") - parser.add_argument("--min_time", default=5.0, type=float, help="minimum timed seconds") - parser.add_argument("--warmup", default=3, type=int, help="untimed iterations") - parser.add_argument("--timeout", default=900, type=int, help="seconds per run") - parser.add_argument("--seed", default=42, type=int) - parser.add_argument("--dali_extra", default=os.environ.get("DALI_EXTRA_PATH", "")) - parser.add_argument("--json", help="write results (with provenance) to this file") - parser.add_argument("--quick", action="store_true", help="smoke test: tiny sweep, 1 run") - parser.add_argument("--hash", action="store_true", help="record per-sample checksums") - parser.add_argument( - "--hash_epochs", - default=0, - type=int, - help="with --hash: checksum at least this many epochs of every configuration, and " - "one more batch", - ) - parser.add_argument("--compare", nargs=2, metavar=("A", "B"), help="compare two --hash files") - parser.add_argument( - "--ignore", - default="", - help="with --compare: comma separated config fields to ignore when matching runs, " - "e.g. threads,reader_depth to check that output does not depend on them", - ) - parser.add_argument("--_single", help=argparse.SUPPRESS) - args = parser.parse_args() - if args.compare: - return parser, args - if not args.dali_extra: - parser.error("--dali_extra is empty; set DALI_EXTRA_PATH or pass --dali_extra.") - args.dali_extra = os.path.abspath(args.dali_extra) - if args.hash and not args.json and not args._single: - parser.error("--hash requires --json to store the checksums; got no --json.") - args.backends = args.backends.split(",") - unknown = [b for b in args.backends if b not in BACKEND_SCENARIOS] - if unknown: - parser.error( - f"--backends got unknown backend(s) {unknown}; expected a subset of " - f"{list(BACKEND_SCENARIOS)}." - ) - if args.hash and args.backends != ["dali"]: - parser.error(f"--hash supports only the dali backend; got --backends {args.backends}.") - if args.quick: - args.threads, args.readers, args.repeat = "1,4", "1,2", 1 - args.iters, args.min_time, args.warmup = 3, 0.0, 1 - return parser, args + reps.append(json.loads(r.stdout.splitlines()[-1])) + if reps: + out[key(group, c)] = dict(group=group, t=c["t"], rd=c["rd"]) + for k in reps[0]: + out[key(group, c)][k] = ( + reps[0][k] if k == "hash" else statistics.median(x[k] for x in reps) + ) + print(key(group, c), {k: v for k, v in out[key(group, c)].items() if k != "group"}) + failed = any( + key(g, c) not in out for g, c in todo if c.get("lib") != "torchcodec" + ) # a failed or missing configuration + for g in {v["group"] for v in out.values()} if a.hash else (): + # one group, one output, whatever num_threads / reader_depth + if len({v["hash"] for v in out.values() if v["group"] == g}) > 1: + print(f"MISMATCH within group {g}") + failed = True + if a.check and not a.hash: + failed |= not check_scaling(out) + with open(a.json, "w") as f: + json.dump(out, f, indent=1) + return int(failed) + + +def compare(path_a, path_b): + with open(path_a) as f: + a = json.load(f) + with open(path_b) as f: + b = json.load(f) + # a configuration missing on one side is a failure (the torchcodec rows are for reference) + bad = [k for k in a.keys() ^ b.keys() if "_torchcodec" not in k] + for k in sorted(a.keys() & b.keys()): + if "hash" in a[k] and a[k]["hash"] != b[k]["hash"]: + bad.append(k) + if "fps" in a[k]: + ratio = b[k]["fps"] / a[k]["fps"] + print(f"{k:34} {a[k]['fps']:9.1f} -> {b[k]['fps']:9.1f} frames/s x{ratio:.2f}") + print("DIFFERENT / MISSING:", bad if bad else "none") + return int(bool(bad)) def main(): - parser, args = parse_args() - if args.compare: - sys.exit(compare(*args.compare, [f for f in args.ignore.split(",") if f])) - if args._single: - child_main(args) - return - - threads = [int(t) for t in args.threads.split(",")] - readers = [int(r) for r in args.readers.split(",")] - all_scenarios = scenarios(threads, readers, args.quick) - if args.scenarios == "all": - names = [n for n in all_scenarios if args.hash or n not in HASH_ONLY_SCENARIOS] + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawTextHelpFormatter) + ap.add_argument("--json", default="video_reader_bench.json", help="where to write the results") + ap.add_argument("--hash", action="store_true", help="checksums instead of throughput") + ap.add_argument("--check", action="store_true", help="fail if threads / depth slow it down") + ap.add_argument("--compare", nargs=2, metavar=("A", "B"), help="two result files") + ap.add_argument( + "--threads", type=lambda s: [int(x) for x in s.split(",")], default=[1, 2, 4, 8] + ) + ap.add_argument("--repeat", type=int, default=3, help="fresh-process runs per configuration") + ap.add_argument("--only", default="", help="only groups containing this text") + ap.add_argument("--no_torchcodec", action="store_true", help="DALI only") + ap.add_argument("--dali_extra", default=os.environ.get("DALI_EXTRA_PATH", "")) + ap.add_argument("--child", help=argparse.SUPPRESS) + a = ap.parse_args() + if a.child: + print(json.dumps(child(json.loads(a.child), a.dali_extra, a.hash))) + elif a.compare: + sys.exit(compare(*a.compare)) else: - names = args.scenarios.split(",") - unknown = [n for n in names if n not in all_scenarios] - if unknown: - parser.error( - f"--scenarios got unknown scenario(s) {unknown}; expected 'all' or a subset of " - f"{list(all_scenarios)}." - ) - hash_only = [n for n in names if n in HASH_ONLY_SCENARIOS] - if hash_only and not args.hash: - parser.error(f"--scenarios {hash_only} run only with --hash.") - if args.hash: - args.repeat = 1 - - prov = provenance(args) - print("# " + json.dumps(prov)) - if prov.get("nvml_error"): - warnings.warn(f"NVML is unavailable ({prov['nvml_error']}); GPU checks are disabled.") - busy = prov.get("other_gpu_pids") or (prov.get("nvdec_util_at_start") or 0) > 5 - if busy: - details = [] - if prov.get("other_gpu_pids"): - details.append(f"pids {prov['other_gpu_pids']}") - if prov.get("nvdec_util_at_start") is not None: - details.append(f"NVDEC utilization {prov['nvdec_util_at_start']}%") - warnings.warn( - f"The GPU is in use by other processes ({', '.join(details)}); " - f"the results will be skewed." - ) - - workdir = tempfile.mkdtemp(prefix="video_reader_bench_") - results, failed = [], 0 - try: - jobs, not_run = [], set() - for name in names: - for base, backend in [(c, b) for c in all_scenarios[name] for b in args.backends]: - if not backend_runs(backend, name, base): - not_run.add((backend, name, base["threads"])) - continue - cfg = dict(base, scenario=name, backend=backend) - try: - specs = materialize(cfg, args.dali_extra, workdir) - except Exception as e: - jobs.append((cfg, None, f"{type(e).__name__}: {e}")) - continue - jobs.append((cfg, specs, None)) - for backend, name, threads in sorted(not_run): - print(f"# not run: {name} threads={threads} with {backend} (see backend_runs)") - for cfg, specs, err in jobs: - if specs: - paths = [] - for s in specs: - if "filenames" in s: - paths += s["filenames"] - else: - with open(s["file_list"]) as f: - paths += [ln.split()[0] for ln in f] - prewarm(os.path.realpath(p) for p in paths) - reps = {i: [] for i in range(len(jobs))} - rng = random.Random(args.seed) - for rep in range(args.repeat): - order = list(range(len(jobs))) - if rep > 0: - rng.shuffle(order) # round-robin, so drift does not line up with a config - for i in order: - cfg, specs, err = jobs[i] - res = dict(error=err) if err else run_child(cfg, specs, args) - reps[i].append(res) - note = res.get("error") or ( - f"{res['frames_per_s']:.1f} frames/s" if "frames_per_s" in res else "done" - ) - print( - f"[run {rep + 1}/{args.repeat}] {cfg['scenario']} {cfg['backend']} " - f"{cfg['files']} {cfg['layout']} bs={cfg['batch']} thr={cfg['threads']} " - f"rq={cfg['reader_depth']} rd={cfg['readers']}: {note}", - file=sys.stderr, - flush=True, - ) - if not args.hash: - print_header() - for i, (cfg, specs, err) in enumerate(jobs): - if args.hash: - r = dict(cfg, seed=args.seed, **reps[i][0]) # with the hash_batches used - n = len(r.get("hashes", [])) - print( - f"{cfg['scenario']} {cfg['files']} threads={cfg['threads']}: " - f"{n} sample hashes" + (f" ERROR: {r['error']}" if r.get("error") else "") - ) - else: - r = aggregate(cfg, reps[i]) - print(format_row(r)) - # a configuration fails if any of its repetitions failed - failed += bool(r.get("error") or r.get("errors")) - results.append(r) - finally: - shutil.rmtree(workdir, ignore_errors=True) - - if args.json: - with open(args.json, "w") as f: - json.dump(dict(provenance=prov, results=results), f, indent=1) - print(f"wrote {args.json}") - if failed: - print(f"{failed} configuration(s) failed") - sys.exit(1) + sys.exit(run(a)) if __name__ == "__main__": diff --git a/qa/TL1_video_reader_perf/check_results.py b/qa/TL1_video_reader_perf/check_results.py deleted file mode 100644 index 31e6f665232..00000000000 --- a/qa/TL1_video_reader_perf/check_results.py +++ /dev/null @@ -1,193 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Checks the results of internal_tools/video_reader_bench.py (--json output). - -Checks: - * no configuration failed - * relative checks, with a tolerance: - - lanes, seq_longgop, random_access, cpu: throughput at any higher num_threads is - not lower than at the lowest one (more threads must not make the reader slower) - - reader_depth: a deeper reader queue is not slower than depth 1 - * optional: engines with the most readers use the GPU's NVDEC engines: they reach at least - --engines_scaling times the throughput of one reader (GPUs with several NVDEC engines), or, - with --engines_busy, keep NVDEC at least that busy (percent). The latter is for when one - reader already spreads its decode lanes over the engines, so that more readers cannot scale - as much. - * optional absolute minimums (frames/s) for named metrics, passed with --min: - lanes lanes, batch 16, highest num_threads - single_stream single_stream, h264_long - cpu cpu, batch 8, highest num_threads - engines engines, highest number of readers - Metrics without a minimum are printed for reference only. - -Results flagged as contaminated (other processes used the GPU) only produce warnings. -Exits with 1 if any check failed. -""" - -import argparse -import json -import sys -import warnings - - -def by(results, scenario, **match): - """DALI results only; results of other libraries (--backends) are for reference.""" - return [ - r - for r in results - if r.get("scenario") == scenario - and r.get("backend", "dali") == "dali" - and all(r.get(k) == v for k, v in match.items()) - ] - - -def groups(results, scenario, keys): - out = {} - for r in by(results, scenario): - out.setdefault(tuple(r.get(k) for k in keys), []).append(r) - return out - - -def named_metrics(results): - """The metrics that can have absolute minimums.""" - - def at_max(rows, key): - rows = [r for r in rows if r.get("frames_per_s") is not None] - return max(rows, key=lambda r: r[key])["frames_per_s"] if rows else None - - return { - "lanes": at_max(by(results, "lanes", batch=16), "threads"), - "single_stream": next( - (r.get("frames_per_s") for r in by(results, "single_stream", files="h264_long")), None - ), - "cpu": at_max(by(results, "cpu", batch=8), "threads"), - "engines": at_max(by(results, "engines"), "readers"), - } - - -def main(): - parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) - parser.add_argument("results", help="--json output of video_reader_bench.py") - parser.add_argument("--tolerance", type=float, default=10.0, help="percent, relative checks") - parser.add_argument( - "--min", action="append", default=[], metavar="NAME=FPS", help="absolute minimum" - ) - parser.add_argument( - "--engines_scaling", - type=float, - help="minimum speedup of engines with the most readers over one reader", - ) - parser.add_argument( - "--engines_busy", - type=float, - help="NVDEC utilization (percent) of engines with the most readers that passes the " - "engines check even below --engines_scaling (NVDEC is saturated)", - ) - args = parser.parse_args() - - with open(args.results) as f: - results = json.load(f)["results"] - metrics = named_metrics(results) - minimums = {} - for item in args.min: - name, _, value = item.partition("=") - if name not in metrics: - parser.error(f"--min got unknown metric {name!r}; expected one of {list(metrics)}.") - if value: # an empty value means: no minimum, report only - try: - minimums[name] = float(value) - except ValueError: - parser.error(f"--min {name} expects a number of frames/s; got {value!r}.") - tol = args.tolerance / 100 - failed = [] - - def check(name, ok, detail): - print(f"CHECK {name}: {'OK' if ok else 'FAIL'} ({detail})") - if not ok: - failed.append(name) - - # a configuration fails if all its repetitions failed (`error`) or only some (`errors`) - errors = [r for r in results if r.get("error") or r.get("errors")] - check("no failed configurations", not errors, f"{len(errors)} of {len(results)} failed") - for r in errors: - for e in [r["error"]] if r.get("error") else r["errors"]: - print(f" {r.get('scenario')} {r.get('files')} {r.get('layout')}: {e}") - - contaminated = [r for r in results if r.get("contaminated")] - if contaminated: - warnings.warn( - f"{len(contaminated)} of {len(results)} results were measured while other " - f"processes used the GPU; their numbers may be skewed." - ) - - def relative(scenario, keys, param): - for key, rows in sorted(groups(results, scenario, keys).items(), key=str): - rows = sorted((r for r in rows if r.get("frames_per_s")), key=lambda r: r[param]) - if len(rows) < 2: - continue - base = rows[0] - for r in rows[1:]: - ok = r["frames_per_s"] >= (1 - tol) * base["frames_per_s"] - label = "/".join(str(k) for k in key) - check( - f"{scenario} {label} {param}={r[param]} vs {base[param]}", - ok, - f"{r['frames_per_s']:.1f} vs {base['frames_per_s']:.1f} frames/s, " - f"tolerance {args.tolerance:g}%", - ) - - for scenario in ("lanes", "seq_longgop", "random_access", "cpu"): - relative(scenario, ("files", "layout", "batch"), "threads") - relative("reader_depth", ("files", "layout"), "reader_depth") - - if args.engines_scaling is not None: - rows = sorted( - (r for r in by(results, "engines") if r.get("frames_per_s")), key=lambda r: r["readers"] - ) - if len(rows) < 2 or rows[0]["readers"] != 1: - check("engines scaling", False, "needs engines results with 1 and more readers") - else: - one, most = rows[0], rows[-1] - scaling = most["frames_per_s"] / one["frames_per_s"] - busy = most.get("proc_nvdec_util") - saturated = ( - args.engines_busy is not None and busy is not None and busy >= args.engines_busy - ) - detail = f"{scaling:.2f}x, minimum {args.engines_scaling:g}x" - if args.engines_busy is not None: - detail += f"; NVDEC {busy}% busy, saturated at {args.engines_busy:g}%" - check( - f"engines readers={most['readers']} vs 1", - scaling >= args.engines_scaling or saturated, - detail, - ) - - for name, value in metrics.items(): - if name in minimums: - check( - f"{name} minimum", - value is not None and value >= minimums[name], - f"{value} vs min {minimums[name]:g} frames/s", - ) - else: - print(f"REFERENCE {name}: {value} frames/s (no minimum set)") - - print("all checks passed" if not failed else f"{len(failed)} check(s) failed") - sys.exit(1 if failed else 0) - - -if __name__ == "__main__": - main() diff --git a/qa/TL1_video_reader_perf/test.sh b/qa/TL1_video_reader_perf/test.sh index e0921afb6fd..3e0a8e0e85f 100644 --- a/qa/TL1_video_reader_perf/test.sh +++ b/qa/TL1_video_reader_perf/test.sh @@ -1,104 +1,42 @@ #!/bin/bash -e -# used pip packages (the reference libraries are installed by install_reference_libs) -pip_packages='numpy nvidia-ml-py==11.450.51' +# used pip packages (torch and torchcodec are installed by install_torchcodec) +pip_packages='numpy' target_dir=./internal_tools -# One-time pre-step: FFmpeg shared libraries for torchcodec (x86_64 only, see -# install_reference_libs), and nsys (NVIDIA Nsight Systems) +# One-time pre-step: the FFmpeg shared libraries and the shared libpython that torchcodec needs +# (x86_64 only). The torchcodec rows are for reference, so a failure here is not an error. do_once() { - apt update if [ "$(uname -m)" == "x86_64" ]; then - # torchcodec loads the FFmpeg shared libraries and links against the shared libpython - apt install -y --no-install-recommends ffmpeg \ - "libpython$(python -c 'import sys; print("%d.%d" % sys.version_info[:2])')" + (apt update && apt install -y --no-install-recommends ffmpeg \ + "libpython$(python -c 'import sys; print("%d.%d" % sys.version_info[:2])')") \ + || echo "Could not install FFmpeg: no torchcodec rows" fi - if command -v nsys &>/dev/null; then - echo "nsys already installed: $(nsys --version)" - return - fi - DISTRO=$(source /etc/lsb-release && echo "$DISTRIB_RELEASE" | tr -d .) - ARCH=$(dpkg --print-architecture) - apt install -y --no-install-recommends gnupg curl - echo "deb [signed-by=/usr/share/keyrings/nvidia-devtools.gpg] https://developer.download.nvidia.com/devtools/repos/ubuntu${DISTRO}/${ARCH} /" \ - | tee /etc/apt/sources.list.d/nvidia-devtools.list - curl -fsSL "https://developer.download.nvidia.com/devtools/repos/ubuntu${DISTRO}/${ARCH}/nvidia.pub" \ - | gpg --dearmor -o /usr/share/keyrings/nvidia-devtools.gpg - apt update && apt install -y nsight-systems-cli } LOG_PERF="video_reader_perf.log" JSON_PERF="video_reader_perf.json" +LOG_HASH="video_reader_hash.log" JSON_HASH="video_reader_hash.json" -LOG_REF="video_reader_ref.log" -JSON_REF="video_reader_ref.json" -# DALI's legacy fn.readers.video, its video decoder (fn.readers.file + fn.decoders.video) and -# other libraries run for reference on the same samples -# (video_reader_bench.py --backends), on x86_64 only; one short run each, not checked. The DALI -# numbers are in the performance run. The libraries are installed by install_reference_libs. -REF_SCENARIOS="lanes,single_stream,cache_key,cpu" -REF_ARGS="--repeat 1 --iters 4 --warmup 1 --min_time 2 -j 1,16" +function CLEAN_AND_EXIT { + rm -rf ${LOG_PERF} ${JSON_PERF} ${LOG_HASH} ${JSON_HASH} + exit $1 +} -# Installs PyNvVideoCodec, and torch with the matching CUDA-enabled torchcodec from the PyTorch -# index for this CUDA version (x86_64 only), and sets REF_BACKENDS. torch is needed only here, -# so it is pinned here rather than in pip_packages, which would follow the DALI framework tests. -install_reference_libs() { +# Installs torch and the matching CUDA-enabled torchcodec from the PyTorch index for this CUDA +# version (x86_64 only). torch is needed only here, so it is pinned here rather than in +# pip_packages, which would follow the DALI framework tests. A failure is not an error: the +# benchmark then runs without the torchcodec rows (--no_torchcodec). +install_torchcodec() { local cuda_tag case "${CUDA_VERSION}" in 12*) cuda_tag=cu129 ;; 13*) cuda_tag=cu130 ;; - *) echo "No PyTorch index known for CUDA_VERSION=${CUDA_VERSION}; update test.sh"; return 1 ;; + *) echo "No PyTorch index known for CUDA_VERSION=${CUDA_VERSION}"; return 1 ;; esac - pip install PyNvVideoCodec==2.0.2 || return 1 + [ "$(uname -m)" == "x86_64" ] || return 1 pip install "torch==2.11.0+${cuda_tag}" "torchcodec==0.15.0+${cuda_tag}" \ - --index-url "https://download.pytorch.org/whl/${cuda_tag}" || return 1 - REF_BACKENDS="pynvvideocodec,torchcodec,torchcodec_nvdec" - echo "Reference backends: ${REF_BACKENDS}" -} - -# Scenarios of internal_tools/video_reader_bench.py run for performance; see its --help -PERF_SCENARIOS="lanes,engines,single_stream,reader_depth,cache_key,seq_longgop,random_access,cpu" -# Scenarios whose output is checked to be independent of num_threads and reader_depth, over two -# epochs (decoders reused across batches, the epochs wrapping around) -HASH_SCENARIOS="lanes,lanes_depth,reader_depth,cache_key,cpu,seq_longgop,random_access" -HASH_EPOCHS=2 -BENCH_ARGS="--repeat 3 --iters 8 --min_time 3 -j 1,4,16 --readers 1,2,4" -# engines: more readers use more NVDEC engines: 4 readers reach 2.5x one reader, or keep NVDEC -# saturated (one reader's decode lanes already use several engines, e.g. H100 PCIe: 2016 frames/s -# with 1 reader, 3824 at 98% NVDEC with 4, which is 1.9x) -ENGINES_SCALING=2.5 -ENGINES_BUSY=90 - -function CLEAN_AND_EXIT { - rm -rf ${LOG_PERF} ${JSON_PERF} ${JSON_HASH} ${LOG_REF} ${JSON_REF} - exit $1 -} - -# Prints the results (without per-run data) to the log, so that they are kept on success too. -# A missing or unreadable file (e.g. no reference run on GraceHopper) is skipped, not an error. -print_results() { - for json in "$@"; do - if [ -f "${json}" ]; then - python -c "import json, sys -d = json.load(open(sys.argv[1])) -for r in d['results']: - r.pop('runs', None) -print('RESULTS', sys.argv[1], json.dumps(d, separators=(',', ':')))" "${json}" \ - || echo "Could not print ${json}" - fi - done -} - -# Run the benchmark; if NSYS_REP is set, wrap it with nsys profiling (child processes included). -run_bench() { - local log_file="$1" - shift - if [ -n "${NSYS_REP}" ]; then - local profile_name="${log_file%.log}.nsys-rep" - nsys profile -o "${profile_name}" --trace=cuda,nvtx,osrt "$@" | tee "${log_file}" - else - "$@" | tee "${log_file}" - fi + --index-url "https://download.pytorch.org/whl/${cuda_tag}" && python -c "import torchcodec" } test_body() { @@ -107,69 +45,37 @@ test_body() { else TASKSET="taskset --cpu-list 0-71" # GraceHopper fi - # Absolute minimums in frames/s (see qa/TL1_video_reader_perf/check_results.py); empty: - # reported for reference only, not checked. - # TODO: set them per architecture (e.g. 0.85x the REFERENCE values) once a few CI runs - # establish the baseline on these machines - MIN_LANES= - MIN_SINGLE_STREAM= - MIN_CPU= - MIN_ENGINES= + TORCHCODEC_ARGS="" + install_torchcodec || { echo "No torchcodec rows"; TORCHCODEC_ARGS="--no_torchcodec"; } - unset NSYS_REP + # Throughput of every configuration. Failing: a configuration that fails, or more threads + # (or a deeper reader queue) that are more than 10% slower than fewer. The torchcodec rows + # are only reported. PERF_RUN="OK" - run_bench "${LOG_PERF}" ${TASKSET} python video_reader_bench.py ${BENCH_ARGS} \ - -s ${PERF_SCENARIOS} --json ${JSON_PERF} || PERF_RUN="FAIL" - - # The output must not depend on num_threads or on the reader's prefetch_queue_depth: every - # configuration is compared with all that differ from it only in those - HASH_RESULT="OK" - ${TASKSET} python video_reader_bench.py -s ${HASH_SCENARIOS} -j 1,16 --hash \ - --hash_epochs ${HASH_EPOCHS} --json ${JSON_HASH} || HASH_RESULT="FAIL" - python video_reader_bench.py --compare ${JSON_HASH} ${JSON_HASH} \ - --ignore threads,reader_depth || HASH_RESULT="FAIL" + ${TASKSET} python video_reader_bench.py --check ${TORCHCODEC_ARGS} --json ${JSON_PERF} \ + | tee ${LOG_PERF} || PERF_RUN="FAIL" - CHECK_RESULT="OK" - python ${topdir}/qa/TL1_video_reader_perf/check_results.py ${JSON_PERF} --tolerance 10 \ - --engines_scaling ${ENGINES_SCALING} --engines_busy ${ENGINES_BUSY} \ - --min lanes=${MIN_LANES} --min single_stream=${MIN_SINGLE_STREAM} \ - --min cpu=${MIN_CPU} --min engines=${MIN_ENGINES} || CHECK_RESULT="FAIL" - - # The same samples decoded by other libraries, for reference (x86_64 only). A failure here - # (installation or decoding) fails the test, so that the comparison does not silently - # disappear. - REF_RUN="OK" - if [ "$(uname -m)" != "x86_64" ]; then - REF_RUN="SKIPPED" - elif install_reference_libs; then - run_bench "${LOG_REF}" ${TASKSET} python video_reader_bench.py ${REF_ARGS} \ - -s ${REF_SCENARIOS} -b dali_legacy,dali_decoder,${REF_BACKENDS} --json ${JSON_REF} || REF_RUN="FAIL" - else - REF_RUN="FAIL" - fi + # The output must not depend on num_threads or on the reader's prefetch_queue_depth: the + # configurations of a group are compared with each other + HASH_RUN="OK" + ${TASKSET} python video_reader_bench.py --hash --json ${JSON_HASH} \ + | tee ${LOG_HASH} || HASH_RUN="FAIL" echo "PERF_RUN=${PERF_RUN}" - echo "HASH_RESULT=${HASH_RESULT}" - echo "CHECK_RESULT=${CHECK_RESULT}" - echo "REF_RUN=${REF_RUN}" - print_results ${JSON_PERF} ${JSON_REF} + echo "HASH_RUN=${HASH_RUN}" + for json in ${JSON_PERF} ${JSON_HASH}; do + [ -f ${json} ] && echo "RESULTS ${json} $(tr -d ' \n' < ${json})" + done - if [[ "$PERF_RUN" == "OK" && "$HASH_RESULT" == "OK" && "$CHECK_RESULT" == "OK" \ - && "$REF_RUN" != "FAIL" ]]; then + if [[ "$PERF_RUN" == "OK" && "$HASH_RUN" == "OK" ]]; then CLEAN_AND_EXIT 0 fi - # On failure: save the results; if the performance run or its checks failed, also re-run the - # performance scenarios once with nsys and save the profile + # On failure: save the logs and results ARTIFACTS_DIR="${topdir}/core_artifacts" mkdir -p "${ARTIFACTS_DIR}" - if [[ "$PERF_RUN" != "OK" || "$CHECK_RESULT" != "OK" ]]; then - echo "Video reader performance checks failed; re-running with nsys profiling..." - NSYS_REP="enabled" run_bench "${LOG_PERF}" ${TASKSET} python video_reader_bench.py \ - --repeat 1 --min_time 3 -j 1,16 --readers 1,4 -s ${PERF_SCENARIOS} || true - fi - cp -f *.nsys-rep ${JSON_PERF} ${JSON_HASH} ${JSON_REF} "${ARTIFACTS_DIR}/" 2>/dev/null || true - echo "Results (and nsys profile, if any) saved to ${ARTIFACTS_DIR}" + cp -f ${LOG_PERF} ${JSON_PERF} ${LOG_HASH} ${JSON_HASH} "${ARTIFACTS_DIR}/" 2>/dev/null || true + echo "Results saved to ${ARTIFACTS_DIR}" CLEAN_AND_EXIT 1 } pushd ../.. From 1d05835e29ef1997fa8f4ad926e37c7bcc8e10a1 Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Wed, 28 Oct 2026 10:57:15 +0100 Subject: [PATCH 19/25] Add Vera Rubin support to TL1 video reader perf test Select the CPU list by CPU architecture and GPU compute capability, as in the TL1 decoder perf test, and fail on an unsupported platform instead of using the Grace Hopper settings. Signed-off-by: Joaquin Anton Guirao --- qa/TL1_video_reader_perf/test.sh | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/qa/TL1_video_reader_perf/test.sh b/qa/TL1_video_reader_perf/test.sh index 3e0a8e0e85f..4e06dedd855 100644 --- a/qa/TL1_video_reader_perf/test.sh +++ b/qa/TL1_video_reader_perf/test.sh @@ -39,12 +39,33 @@ install_torchcodec() { --index-url "https://download.pytorch.org/whl/${cuda_tag}" && python -c "import torchcodec" } +# Per-platform benchmark settings, keyed by "/". +set_platform_params() { + local arch compute_cap + arch=$(uname -m) + compute_cap=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader -i 0) + case "${arch}/${compute_cap}" in + x86_64/*) # x86 + any GPU (tuned on Hopper) + CPU_LIST="0-127" + ;; + aarch64/10.7) # Vera Rubin + # 2 sockets x 88 cores x 2 threads; 0-87 + 176-263 are socket 0 and its hyperthread + # siblings (local to GPU 0) + CPU_LIST="0-87,176-263" + ;; + aarch64/9.0|aarch64/10.0|aarch64/10.3) # Grace Hopper (GH200), Grace Blackwell (GB200, GB300) + CPU_LIST="0-71" + ;; + *) + echo "Unsupported platform: got ${arch}/${compute_cap}, expected x86_64 with any GPU, or aarch64 with GPU compute capability 9.0, 10.0, 10.3 or 10.7." >&2 + CLEAN_AND_EXIT 1 + ;; + esac +} + test_body() { - if [ "$(uname -p)" == "x86_64" ]; then - TASKSET="taskset --cpu-list 0-127" # Hopper - else - TASKSET="taskset --cpu-list 0-71" # GraceHopper - fi + set_platform_params + TASKSET="taskset --cpu-list ${CPU_LIST}" TORCHCODEC_ARGS="" install_torchcodec || { echo "No torchcodec rows"; TORCHCODEC_ARGS="--no_torchcodec"; } From d48e3f1fd9760db11a5b1f9fa8a75f362af45151 Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Wed, 28 Oct 2026 11:04:00 +0100 Subject: [PATCH 20/25] Address review findings in the video reader benchmark Make --dali_extra absolute so symlink targets do not depend on the working directory, remove the per-configuration temporary directory, set the pipeline's prefetch queue depth explicitly, make --check compare every row with the fastest row that has fewer threads or a shallower queue, and make --compare report a hash/throughput file mismatch instead of raising KeyError. Signed-off-by: Joaquin Anton Guirao --- internal_tools/video_reader_bench.py | 42 ++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/internal_tools/video_reader_bench.py b/internal_tools/video_reader_bench.py index 34200550000..83022a0f4b0 100644 --- a/internal_tools/video_reader_bench.py +++ b/internal_tools/video_reader_bench.py @@ -67,6 +67,7 @@ HEVC_LONG = ["hevc/sintel_trailer-720p.mp4"] # the same clip as HEVC, a keyframe every ~9 frames VPX = ["vp9/vp9_0.mp4", "vp9/vp9_2.mp4", "vp8/vp8.webm", "cfr/test_1_vp9.mp4", "vfr/test_1_vp9.mp4"] HASH_MIN_SAMPLES = 40 +PIPELINE_DEPTH = 2 # the pipeline's own prefetch queue, set explicitly def dali_configs(threads): @@ -137,13 +138,17 @@ def run_batch(): def child(cfg, root, hash_out): """Runs one configuration in this process: {"fps", "build_s"}, or {"hash"}.""" + with tempfile.TemporaryDirectory() as tmp: + return run_config(cfg, root, hash_out, tmp) + + +def run_config(cfg, root, hash_out, tmp): import numpy as np files = sorted(f for p in cfg["files"] for f in glob.glob(os.path.join(root, VIDEO_DIR, p))) assert files, f"no files for {cfg['files']} under {root}/{VIDEO_DIR}" samples = [] # (path, start, end) if cfg.get("many"): # distinct symlinks: nothing keyed by the file name is shared - tmp = tempfile.mkdtemp() lines = [] for i in range(cfg["many"]): link = os.path.join(tmp, f"{i}_{os.path.basename(files[i % len(files)])}") @@ -173,7 +178,13 @@ def child(cfg, root, hash_out): import nvidia.dali.fn as fn from nvidia.dali.pipeline import pipeline_def - @pipeline_def(batch_size=cfg["batch"], num_threads=cfg["t"], device_id=0, seed=42) + @pipeline_def( + batch_size=cfg["batch"], + num_threads=cfg["t"], + device_id=0, + seed=42, + prefetch_queue_depth=PIPELINE_DEPTH, + ) def pipe(): return tuple( fn.experimental.readers.video( @@ -199,7 +210,7 @@ def pipe(): h.update(np.ascontiguousarray(np.array(o[i])).tobytes()) return dict(hash=h.hexdigest()) # batches that may have been produced before the clock starts: pipeline + reader queues - for _ in range(2 + cfg["rd"] + 1): + for _ in range(PIPELINE_DEPTH + cfg["rd"] + 1): run_batch() n, t0 = 0, time.perf_counter() while n < 20 or time.perf_counter() - t0 < 5: # >= 20 batches and >= 5 s @@ -215,13 +226,18 @@ def check_scaling(out, tolerance=0.10): rows = sorted((v["t"], v["rd"], v["fps"]) for v in out.values() if v["group"] == group) if group.endswith("_torchcodec") or len(rows) < 2: continue - (t0, rd0, slow), (t1, rd1, fast) = rows[0], rows[-1] - bad = fast < (1 - tolerance) * slow - ok &= not bad - print( - f"{'FAIL' if bad else 'ok '} {group}: {slow:.0f} frames/s (t={t0}, rd={rd0})" - f" -> {fast:.0f} (t={t1}, rd={rd1})" - ) + best = rows[0] # the fastest of the rows with fewer threads / a shallower queue + slower = [] + for row in rows[1:]: + if row[2] < (1 - tolerance) * best[2]: + slower.append((best, row)) + best = max(best, row, key=lambda r: r[2]) + ok &= not slower + for (t0, rd0, slow), (t1, rd1, fast) in slower or [(rows[0], rows[-1])]: + print( + f"{'FAIL' if slower else 'ok '} {group}: {slow:.0f} frames/s (t={t0}, rd={rd0})" + f" -> {fast:.0f} (t={t1}, rd={rd1})" + ) return ok @@ -278,9 +294,9 @@ def compare(path_a, path_b): # a configuration missing on one side is a failure (the torchcodec rows are for reference) bad = [k for k in a.keys() ^ b.keys() if "_torchcodec" not in k] for k in sorted(a.keys() & b.keys()): - if "hash" in a[k] and a[k]["hash"] != b[k]["hash"]: + if a[k].get("hash") != b[k].get("hash"): bad.append(k) - if "fps" in a[k]: + if "fps" in a[k] and "fps" in b[k]: ratio = b[k]["fps"] / a[k]["fps"] print(f"{k:34} {a[k]['fps']:9.1f} -> {b[k]['fps']:9.1f} frames/s x{ratio:.2f}") print("DIFFERENT / MISSING:", bad if bad else "none") @@ -302,6 +318,8 @@ def main(): ap.add_argument("--dali_extra", default=os.environ.get("DALI_EXTRA_PATH", "")) ap.add_argument("--child", help=argparse.SUPPRESS) a = ap.parse_args() + if a.dali_extra: # symlink targets and file lists must not depend on the working directory + a.dali_extra = os.path.abspath(a.dali_extra) if a.child: print(json.dumps(child(json.loads(a.child), a.dali_extra, a.hash))) elif a.compare: From 45bafa667d9359e651980c3921de23895ce8994a Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Wed, 28 Oct 2026 11:30:11 +0100 Subject: [PATCH 21/25] Address remaining review findings in the video reader benchmark Raise FileNotFoundError instead of asserting on missing data, hash more than an epoch so the reader's restart is covered, use a positive boolean for the torchcodec option and add the copyright header to the TL1 test script. Signed-off-by: Joaquin Anton Guirao --- internal_tools/video_reader_bench.py | 17 +++++++++++++---- qa/TL1_video_reader_perf/test.sh | 1 + 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/internal_tools/video_reader_bench.py b/internal_tools/video_reader_bench.py index 83022a0f4b0..a510e35e20c 100644 --- a/internal_tools/video_reader_bench.py +++ b/internal_tools/video_reader_bench.py @@ -146,7 +146,12 @@ def run_config(cfg, root, hash_out, tmp): import numpy as np files = sorted(f for p in cfg["files"] for f in glob.glob(os.path.join(root, VIDEO_DIR, p))) - assert files, f"no files for {cfg['files']} under {root}/{VIDEO_DIR}" + if not files: + raise FileNotFoundError( + f"--dali_extra={root!r}: expected files matching {cfg['files']} under " + f"{os.path.join(root, VIDEO_DIR)!r}, but found none. Point --dali_extra (or " + "DALI_EXTRA_PATH) at a DALI_extra checkout." + ) samples = [] # (path, start, end) if cfg.get("many"): # distinct symlinks: nothing keyed by the file name is shared lines = [] @@ -174,6 +179,7 @@ def run_config(cfg, root, hash_out, tmp): for first in range(0, len(VideoDecoder(f)) - cfg["seq"] + 1, cfg["seq"]): samples.append((f, first, first + cfg["seq"])) run_batch = torchcodec_step(cfg, samples) + epoch_size = len(samples) else: import nvidia.dali.fn as fn from nvidia.dali.pipeline import pipeline_def @@ -199,11 +205,14 @@ def pipe(): p = pipe() p.build() run_batch = p.run + epoch_size = sum(p.epoch_size().values()) build_s = time.perf_counter() - t0 if hash_out: # from batch 0: the same samples whatever the queue depths h = hashlib.sha1() - for _ in range(-(-HASH_MIN_SAMPLES // cfg["batch"]) + 2): + # more than an epoch, so that the reader's restart is covered too + n_samples = max(HASH_MIN_SAMPLES, epoch_size) + for _ in range(-(-n_samples // cfg["batch"]) + 2): for o in run_batch(): o = o.as_cpu() if hasattr(o, "as_cpu") else o for i in range(len(o)): @@ -244,7 +253,7 @@ def check_scaling(out, tolerance=0.10): def run(a): todo = [ (group, c) - for group, c in configs(a.threads, not (a.hash or a.no_torchcodec)) + for group, c in configs(a.threads, a.torchcodec and not a.hash) if a.only in group and (a.hash or not c.get("hash_only")) ] out = {} @@ -314,7 +323,7 @@ def main(): ) ap.add_argument("--repeat", type=int, default=3, help="fresh-process runs per configuration") ap.add_argument("--only", default="", help="only groups containing this text") - ap.add_argument("--no_torchcodec", action="store_true", help="DALI only") + ap.add_argument("--no_torchcodec", dest="torchcodec", action="store_false", help="DALI only") ap.add_argument("--dali_extra", default=os.environ.get("DALI_EXTRA_PATH", "")) ap.add_argument("--child", help=argparse.SUPPRESS) a = ap.parse_args() diff --git a/qa/TL1_video_reader_perf/test.sh b/qa/TL1_video_reader_perf/test.sh index 4e06dedd855..38664ac1e74 100644 --- a/qa/TL1_video_reader_perf/test.sh +++ b/qa/TL1_video_reader_perf/test.sh @@ -1,4 +1,5 @@ #!/bin/bash -e +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # used pip packages (torch and torchcodec are installed by install_torchcodec) pip_packages='numpy' target_dir=./internal_tools From 0535d1ebe484e1ba2c4ec9890738301dee368715 Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Fri, 30 Oct 2026 11:20:14 +0100 Subject: [PATCH 22/25] Rename the parsed arguments to args in the video reader benchmark Signed-off-by: Joaquin Anton Guirao --- internal_tools/video_reader_bench.py | 41 ++++++++++++++++------------ 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/internal_tools/video_reader_bench.py b/internal_tools/video_reader_bench.py index a510e35e20c..a73b446cdf9 100644 --- a/internal_tools/video_reader_bench.py +++ b/internal_tools/video_reader_bench.py @@ -250,20 +250,27 @@ def check_scaling(out, tolerance=0.10): return ok -def run(a): +def run(args): todo = [ (group, c) - for group, c in configs(a.threads, a.torchcodec and not a.hash) - if a.only in group and (a.hash or not c.get("hash_only")) + for group, c in configs(args.threads, args.torchcodec and not args.hash) + if args.only in group and (args.hash or not c.get("hash_only")) ] out = {} for group, c in todo: reference = c.get("lib") == "torchcodec" # never fails the run reps = [] - for _ in range(1 if a.hash else a.repeat): + for _ in range(1 if args.hash else args.repeat): r = subprocess.run( - [sys.executable, __file__, "--child", json.dumps(c), "--dali_extra", a.dali_extra] - + ["--hash"] * a.hash, + [ + sys.executable, + __file__, + "--child", + json.dumps(c), + "--dali_extra", + args.dali_extra, + ] + + ["--hash"] * args.hash, capture_output=True, text=True, ) @@ -283,14 +290,14 @@ def run(a): failed = any( key(g, c) not in out for g, c in todo if c.get("lib") != "torchcodec" ) # a failed or missing configuration - for g in {v["group"] for v in out.values()} if a.hash else (): + for g in {v["group"] for v in out.values()} if args.hash else (): # one group, one output, whatever num_threads / reader_depth if len({v["hash"] for v in out.values() if v["group"] == g}) > 1: print(f"MISMATCH within group {g}") failed = True - if a.check and not a.hash: + if args.check and not args.hash: failed |= not check_scaling(out) - with open(a.json, "w") as f: + with open(args.json, "w") as f: json.dump(out, f, indent=1) return int(failed) @@ -326,15 +333,15 @@ def main(): ap.add_argument("--no_torchcodec", dest="torchcodec", action="store_false", help="DALI only") ap.add_argument("--dali_extra", default=os.environ.get("DALI_EXTRA_PATH", "")) ap.add_argument("--child", help=argparse.SUPPRESS) - a = ap.parse_args() - if a.dali_extra: # symlink targets and file lists must not depend on the working directory - a.dali_extra = os.path.abspath(a.dali_extra) - if a.child: - print(json.dumps(child(json.loads(a.child), a.dali_extra, a.hash))) - elif a.compare: - sys.exit(compare(*a.compare)) + args = ap.parse_args() + if args.dali_extra: # symlink targets and file lists must not depend on the working directory + args.dali_extra = os.path.abspath(args.dali_extra) + if args.child: + print(json.dumps(child(json.loads(args.child), args.dali_extra, args.hash))) + elif args.compare: + sys.exit(compare(*args.compare)) else: - sys.exit(run(a)) + sys.exit(run(args)) if __name__ == "__main__": From 852a3c96e5433fd703ee2b90c1bd08a7f62e255a Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Fri, 30 Oct 2026 14:29:36 +0100 Subject: [PATCH 23/25] Address review: cfg naming, parent-side data check, OK/FAIL formatting Rename the config parameter c to cfg, check that the data files exist once in the parent process instead of in every child, and format the scaling check status with a width specifier. Signed-off-by: Joaquin Anton Guirao --- internal_tools/video_reader_bench.py | 55 ++++++++++++++++------------ 1 file changed, 31 insertions(+), 24 deletions(-) diff --git a/internal_tools/video_reader_bench.py b/internal_tools/video_reader_bench.py index a73b446cdf9..44e1e99599b 100644 --- a/internal_tools/video_reader_bench.py +++ b/internal_tools/video_reader_bench.py @@ -95,13 +95,24 @@ def dali_configs(threads): def configs(threads, torchcodec=True): cfgs = list(dali_configs(threads)) yield from cfgs - for group, c in cfgs if torchcodec else (): - if c["device"] == "gpu" and group != "reader_depth" and not c.get("hash_only"): - yield group + "_torchcodec", dict(c, lib="torchcodec") + for group, cfg in cfgs if torchcodec else (): + if cfg["device"] == "gpu" and group != "reader_depth" and not cfg.get("hash_only"): + yield group + "_torchcodec", dict(cfg, lib="torchcodec") -def key(group, c): - return f"{group} t={c['t']} rd={c['rd']}" +def key(group, cfg): + return f"{group} t={cfg['t']} rd={cfg['rd']}" + + +def find_files(cfg, root): + files = sorted(f for p in cfg["files"] for f in glob.glob(os.path.join(root, VIDEO_DIR, p))) + if not files: + raise FileNotFoundError( + f"--dali_extra={root!r}: expected files matching {cfg['files']} under " + f"{os.path.join(root, VIDEO_DIR)!r}, but found none. Point --dali_extra (or " + "DALI_EXTRA_PATH) at a DALI_extra checkout." + ) + return files def torchcodec_step(cfg, samples): @@ -145,13 +156,7 @@ def child(cfg, root, hash_out): def run_config(cfg, root, hash_out, tmp): import numpy as np - files = sorted(f for p in cfg["files"] for f in glob.glob(os.path.join(root, VIDEO_DIR, p))) - if not files: - raise FileNotFoundError( - f"--dali_extra={root!r}: expected files matching {cfg['files']} under " - f"{os.path.join(root, VIDEO_DIR)!r}, but found none. Point --dali_extra (or " - "DALI_EXTRA_PATH) at a DALI_extra checkout." - ) + files = find_files(cfg, root) samples = [] # (path, start, end) if cfg.get("many"): # distinct symlinks: nothing keyed by the file name is shared lines = [] @@ -244,7 +249,7 @@ def check_scaling(out, tolerance=0.10): ok &= not slower for (t0, rd0, slow), (t1, rd1, fast) in slower or [(rows[0], rows[-1])]: print( - f"{'FAIL' if slower else 'ok '} {group}: {slow:.0f} frames/s (t={t0}, rd={rd0})" + f"{'FAIL' if slower else 'OK':4} {group}: {slow:.0f} frames/s (t={t0}, rd={rd0})" f" -> {fast:.0f} (t={t1}, rd={rd1})" ) return ok @@ -252,13 +257,15 @@ def check_scaling(out, tolerance=0.10): def run(args): todo = [ - (group, c) - for group, c in configs(args.threads, args.torchcodec and not args.hash) - if args.only in group and (args.hash or not c.get("hash_only")) + (group, cfg) + for group, cfg in configs(args.threads, args.torchcodec and not args.hash) + if args.only in group and (args.hash or not cfg.get("hash_only")) ] + for _, cfg in todo: # once here, not as a failure of every child process + find_files(cfg, args.dali_extra) out = {} - for group, c in todo: - reference = c.get("lib") == "torchcodec" # never fails the run + for group, cfg in todo: + reference = cfg.get("lib") == "torchcodec" # never fails the run reps = [] for _ in range(1 if args.hash else args.repeat): r = subprocess.run( @@ -266,7 +273,7 @@ def run(args): sys.executable, __file__, "--child", - json.dumps(c), + json.dumps(cfg), "--dali_extra", args.dali_extra, ] @@ -276,19 +283,19 @@ def run(args): ) if r.returncode: reps = None - print(f"FAILED {key(group, c)}{' (reference)' if reference else ''}:") + print(f"FAILED {key(group, cfg)}{' (reference)' if reference else ''}:") print(r.stderr[-1500:]) break reps.append(json.loads(r.stdout.splitlines()[-1])) if reps: - out[key(group, c)] = dict(group=group, t=c["t"], rd=c["rd"]) + out[key(group, cfg)] = dict(group=group, t=cfg["t"], rd=cfg["rd"]) for k in reps[0]: - out[key(group, c)][k] = ( + out[key(group, cfg)][k] = ( reps[0][k] if k == "hash" else statistics.median(x[k] for x in reps) ) - print(key(group, c), {k: v for k, v in out[key(group, c)].items() if k != "group"}) + print(key(group, cfg), {k: v for k, v in out[key(group, cfg)].items() if k != "group"}) failed = any( - key(g, c) not in out for g, c in todo if c.get("lib") != "torchcodec" + key(g, cfg) not in out for g, cfg in todo if cfg.get("lib") != "torchcodec" ) # a failed or missing configuration for g in {v["group"] for v in out.values()} if args.hash else (): # one group, one output, whatever num_threads / reader_depth From dcd34d0501323c747ac1276d263a306134c39dcb Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Fri, 30 Oct 2026 15:09:07 +0100 Subject: [PATCH 24/25] Address review: name the hashed output, document the config keys Signed-off-by: Joaquin Anton Guirao --- internal_tools/video_reader_bench.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/internal_tools/video_reader_bench.py b/internal_tools/video_reader_bench.py index 44e1e99599b..9bb02ddc168 100644 --- a/internal_tools/video_reader_bench.py +++ b/internal_tools/video_reader_bench.py @@ -73,8 +73,10 @@ def dali_configs(threads): """(group, configuration) of the DALI reader. - `many`: that number of symlinks to the files, one sequence each. Without it, the files as - they are.""" + A configuration is a dict of: files (glob patterns), device, t (num_threads), rd (the reader's + prefetch_queue_depth), batch (batch_size), seq (sequence_length), shuffle (random_shuffle), + many (optional: that number of symlinks to the files, one sequence each; without it, the + files as they are), hash_only (optional: not timed).""" base = dict(batch=16, seq=32, rd=1, shuffle=False, device="gpu") t_max = max(threads) for t in threads: @@ -218,10 +220,10 @@ def pipe(): # more than an epoch, so that the reader's restart is covered too n_samples = max(HASH_MIN_SAMPLES, epoch_size) for _ in range(-(-n_samples // cfg["batch"]) + 2): - for o in run_batch(): - o = o.as_cpu() if hasattr(o, "as_cpu") else o - for i in range(len(o)): - h.update(np.ascontiguousarray(np.array(o[i])).tobytes()) + for output in run_batch(): + output = output.as_cpu() if hasattr(output, "as_cpu") else output + for i in range(len(output)): + h.update(np.ascontiguousarray(np.array(output[i])).tobytes()) return dict(hash=h.hexdigest()) # batches that may have been produced before the clock starts: pipeline + reader queues for _ in range(PIPELINE_DEPTH + cfg["rd"] + 1): From 996eb3d84c84ae701d41d79328407ebe9d18bf0c Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Fri, 30 Oct 2026 15:10:19 +0100 Subject: [PATCH 25/25] Comment what run_batch is where it is assigned Signed-off-by: Joaquin Anton Guirao --- internal_tools/video_reader_bench.py | 1 + 1 file changed, 1 insertion(+) diff --git a/internal_tools/video_reader_bench.py b/internal_tools/video_reader_bench.py index 9bb02ddc168..5b4e7ffba6a 100644 --- a/internal_tools/video_reader_bench.py +++ b/internal_tools/video_reader_bench.py @@ -178,6 +178,7 @@ def run_config(cfg, root, hash_out, tmp): src = dict(filenames=files, labels=list(range(len(files)))) t0 = time.perf_counter() + # run_batch() produces one batch: the DALI pipeline's run, or the same work in torchcodec if cfg.get("lib") == "torchcodec": if not cfg.get("many"): # whole files in full sequences, as the DALI reader splits them from torchcodec.decoders import VideoDecoder