From befe6fb91ecf5672b2639a20c2eda2cee1f03a02 Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Wed, 21 Oct 2026 12:49:55 +0200 Subject: [PATCH 01/46] Fix NVDEC decoder cache never matching cropped videos The NVDEC decoder cache looked up decoders by the coded size of the stream (CUVIDEOFORMAT::coded_height/width), but stored the display area size (ulTargetHeight/Width) of the decoder it created. The two differ whenever the stream is cropped, e.g. 1080p H.264 is coded as 1920x1088 and displayed as 1920x1080, so for such videos no cached decoder ever matched. As the cuvidReconfigureDecoder fallback is compiled out (ENABLE_NVDEC_RECONFIGURE is never defined), every video opened by FramesDecoderGpu got a freshly created decoder, and unused ones piled up in the cache. In an nsys profile of experimental.readers.video on 1080p H.264 clips, each file switch stalled the reader for ~70 ms before the first kernel, ~57 ms of which were ioctls inside libnvcuvid, consistent with creating a new decoder. Store both the coded size and the display area in DecInstance and require both to match, so a decoder is reused only for streams it is really suited for. Add NVDECCacheSize() and a test that decodes the same 800x600 (coded 800x608) H.264 file with two consecutive decoders and checks that the second one reuses the cached decoder. Signed-off-by: Joaquin Anton Guirao --- dali/operators/video/frames_decoder_gpu.cc | 31 +++++++++++++++++++-- dali/operators/video/frames_decoder_gpu.h | 15 ++++++++++ dali/operators/video/frames_decoder_test.cc | 21 ++++++++++++++ 3 files changed, 65 insertions(+), 2 deletions(-) diff --git a/dali/operators/video/frames_decoder_gpu.cc b/dali/operators/video/frames_decoder_gpu.cc index d209bd99b18..950d21c45e3 100644 --- a/dali/operators/video/frames_decoder_gpu.cc +++ b/dali/operators/video/frames_decoder_gpu.cc @@ -107,6 +107,11 @@ class NVDECCache { DALI_FAIL("Cannot return decoder that is not from the cache"); } + size_t Size() { + std::lock_guard lock(access_lock); + return dec_cache.size(); + } + private: NVDECCache() {} @@ -123,6 +128,13 @@ class NVDECCache { auto codec_type = video_format->codec; unsigned height = video_format->coded_height; unsigned width = video_format->coded_width; + // The decoder is created with the display area as the output size (see ulTargetHeight + // and ulTargetWidth below), which differs from the coded size when the stream is cropped, + // e.g. 1080p H.264 is coded as 1920x1088. A cached decoder can be reused only if it matches + // both. + const auto &display_area = video_format->display_area; + unsigned display_height = display_area.bottom - display_area.top; + unsigned display_width = display_area.right - display_area.left; auto num_decode_surfaces = video_format->min_num_decode_surfaces; auto chroma_format = video_format->chroma_format; auto bit_depth_luma_minus8 = video_format->bit_depth_luma_minus8; @@ -135,8 +147,11 @@ class NVDECCache { if (best_match == range.second && it->second.used == false) { best_match = it; } - if (it->second.used == false && it->second.height == height && - it->second.width == width && it->second.num_decode_surfaces == num_decode_surfaces && + if (it->second.used == false && it->second.coded_height == height && + it->second.coded_width == width && it->second.height == display_height && + it->second.width == display_width && it->second.display_left == display_area.left && + it->second.display_top == display_area.top && + it->second.num_decode_surfaces == num_decode_surfaces && it->second.chroma_format == chroma_format && it->second.bit_depth_luma_minus8 == bit_depth_luma_minus8) { it->second.used = true; @@ -157,8 +172,12 @@ class NVDECCache { reconfigParams.ulTargetWidth = reconfigParams.ulWidth = width; reconfigParams.ulTargetHeight = reconfigParams.ulHeight = height; reconfigParams.ulNumDecodeSurfaces = num_decode_surfaces; + best_match->second.coded_height = height; + best_match->second.coded_width = width; best_match->second.height = height; best_match->second.width = width; + best_match->second.display_left = 0; + best_match->second.display_top = 0; best_match->second.num_decode_surfaces = num_decode_surfaces; CUDA_CALL(cuvidReconfigureDecoder(best_match->second.decoder, &reconfigParams)); return NVDECLease(this, &best_match->second); @@ -228,8 +247,12 @@ class NVDECCache { DecInstance decoder_inst = {}; CUDA_CALL(cuvidCreateDecoder(&(decoder_inst.decoder), &decoder_info)); + decoder_inst.coded_height = decoder_info.ulHeight; + decoder_inst.coded_width = decoder_info.ulWidth; decoder_inst.height = decoder_info.ulTargetHeight; decoder_inst.width = decoder_info.ulTargetWidth; + decoder_inst.display_left = area.left; + decoder_inst.display_top = area.top; decoder_inst.max_height = decoder_info.ulMaxHeight; decoder_inst.max_width = decoder_info.ulMaxWidth; decoder_inst.num_decode_surfaces = num_decode_surfaces; @@ -270,6 +293,10 @@ void NVDECLease::Return() { } } +size_t NVDECCacheSize(int device_id) { + return NVDECCache::GetCache(device_id).Size(); +} + int process_video_sequence(void *user_data, CUVIDEOFORMAT *video_format) { FramesDecoderGpu *frames_decoder = static_cast(user_data); frames_decoder->InitGpuDecoder(video_format); diff --git a/dali/operators/video/frames_decoder_gpu.h b/dali/operators/video/frames_decoder_gpu.h index a6da7b2adc6..a637f428004 100644 --- a/dali/operators/video/frames_decoder_gpu.h +++ b/dali/operators/video/frames_decoder_gpu.h @@ -57,8 +57,16 @@ struct DecInstance { CUvideodecoder decoder = {}; cudaVideoCodec codec_type = {}; cudaVideoChromaFormat chroma_format = {}; + // Size of the coded frame (CUVIDEOFORMAT::coded_*), which the decoder is created for. It is + // aligned to the macroblock/CTB size, e.g. 1920x1088 for a 1920x1080 H.264 stream. + unsigned coded_height = 0; + unsigned coded_width = 0; + // Output size and origin of the display area (CUVIDEOFORMAT::display_area), which the decoder + // crops the coded frame to, e.g. 1920x1080 at (0, 0). unsigned height = 0; unsigned width = 0; + int display_left = 0; + int display_top = 0; unsigned num_decode_surfaces = 0; unsigned max_height = 0; unsigned max_width = 0; @@ -108,6 +116,13 @@ class NVDECLease { DecInstance *decoder = nullptr; }; +/** + * @brief Returns the number of decoders (leased and idle) in the NVDEC decoder cache of a device. + * + * @param device_id CUDA device ordinal or -1 for the current device. + */ +DLL_PUBLIC size_t NVDECCacheSize(int device_id = -1); + } // namespace frame_dec_gpu_impl struct NvDecodeState { diff --git a/dali/operators/video/frames_decoder_test.cc b/dali/operators/video/frames_decoder_test.cc index b858e94385f..12088040b48 100644 --- a/dali/operators/video/frames_decoder_test.cc +++ b/dali/operators/video/frames_decoder_test.cc @@ -226,6 +226,27 @@ TEST_F(FramesDecoderGpuTest, VariableFrameRate) { RunTest(decoder, vfr_videos_[1]); } +// H.264 is coded in whole macroblocks, so an 800x600 stream is coded as 800x608 and cropped to the +// display area. The NVDEC decoder cache has to recognize such a stream, so that a decoder is reused +// when the next file is opened and not created from scratch every time. +TEST_F(FramesDecoderGpuTest, DecoderCacheReusedForCroppedVideo) { + std::string path = testing::dali_extra_path() + "/db/video/cfr/test_2.mp4"; + auto decode_first_frame = [&]() { + FramesDecoderGpu decoder(path); + ASSERT_TRUE(decoder.IsValid()); + decoder.BuildIndex(); + ASSERT_EQ(decoder.Height(), 600); + ASSERT_EQ(decoder.Width(), 800); + decoder.ReadNextFrame(FrameData()); + }; + + decode_first_frame(); + size_t cache_size = frame_dec_gpu_impl::NVDECCacheSize(); + ASSERT_GT(cache_size, 0u); + decode_first_frame(); + EXPECT_EQ(frame_dec_gpu_impl::NVDECCacheSize(), cache_size); +} + TEST_F(FramesDecoderGpuTest, ConstantFrameRateHevc) { if (!FramesDecoderGpu::SupportsHevc()) { GTEST_SKIP(); From efb81e3f4e117137a9544580031adb21ecd97dd2 Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Wed, 21 Oct 2026 13:05:16 +0200 Subject: [PATCH 02/46] Count created NVDEC decoders in the cache reuse test Comparing the cache size before and after the second decode does not detect a cache miss when the cache is full: the miss inserts a new decoder and evicts an idle one, so the size stays the same. Replace NVDECCacheSize() with NVDECDecodersCreated(), a cumulative count of decoders created by the cache, and check that the second decoder does not increase it. Signed-off-by: Joaquin Anton Guirao --- dali/operators/video/frames_decoder_gpu.cc | 10 ++++++---- dali/operators/video/frames_decoder_gpu.h | 7 +++++-- dali/operators/video/frames_decoder_test.cc | 10 ++++++---- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/dali/operators/video/frames_decoder_gpu.cc b/dali/operators/video/frames_decoder_gpu.cc index 950d21c45e3..a7eb29ec4cc 100644 --- a/dali/operators/video/frames_decoder_gpu.cc +++ b/dali/operators/video/frames_decoder_gpu.cc @@ -107,9 +107,9 @@ class NVDECCache { DALI_FAIL("Cannot return decoder that is not from the cache"); } - size_t Size() { + size_t NumCreated() { std::lock_guard lock(access_lock); - return dec_cache.size(); + return num_created; } private: @@ -264,6 +264,7 @@ class NVDECCache { lock.lock(); auto inserted = dec_cache.insert({codec_type, decoder_inst}); + num_created++; if (dec_cache.size() > CACHE_SIZE_LIMIT) { for (auto it = dec_cache.begin(); it != dec_cache.end(); ++it) { if (it->second.used == false) { @@ -281,6 +282,7 @@ class NVDECCache { using codec_map = std::multimap; codec_map dec_cache; std::mutex access_lock; + size_t num_created = 0; // decoders created so far, including those already evicted static constexpr int CACHE_SIZE_LIMIT = 100; }; @@ -293,8 +295,8 @@ void NVDECLease::Return() { } } -size_t NVDECCacheSize(int device_id) { - return NVDECCache::GetCache(device_id).Size(); +size_t NVDECDecodersCreated(int device_id) { + return NVDECCache::GetCache(device_id).NumCreated(); } int process_video_sequence(void *user_data, CUVIDEOFORMAT *video_format) { diff --git a/dali/operators/video/frames_decoder_gpu.h b/dali/operators/video/frames_decoder_gpu.h index a637f428004..1e30780e8b5 100644 --- a/dali/operators/video/frames_decoder_gpu.h +++ b/dali/operators/video/frames_decoder_gpu.h @@ -117,11 +117,14 @@ class NVDECLease { }; /** - * @brief Returns the number of decoders (leased and idle) in the NVDEC decoder cache of a device. + * @brief Returns how many NVDEC decoders were created so far for a device, i.e. how many times + * the decoder cache could not provide a matching one. + * + * The count includes decoders that were evicted from the cache since. * * @param device_id CUDA device ordinal or -1 for the current device. */ -DLL_PUBLIC size_t NVDECCacheSize(int device_id = -1); +DLL_PUBLIC size_t NVDECDecodersCreated(int device_id = -1); } // namespace frame_dec_gpu_impl diff --git a/dali/operators/video/frames_decoder_test.cc b/dali/operators/video/frames_decoder_test.cc index 12088040b48..99492494def 100644 --- a/dali/operators/video/frames_decoder_test.cc +++ b/dali/operators/video/frames_decoder_test.cc @@ -228,7 +228,8 @@ TEST_F(FramesDecoderGpuTest, VariableFrameRate) { // H.264 is coded in whole macroblocks, so an 800x600 stream is coded as 800x608 and cropped to the // display area. The NVDEC decoder cache has to recognize such a stream, so that a decoder is reused -// when the next file is opened and not created from scratch every time. +// when the next file is opened and not created from scratch every time. The number of created +// decoders is checked, as the size of the cache would not change if it is full and evicts one. TEST_F(FramesDecoderGpuTest, DecoderCacheReusedForCroppedVideo) { std::string path = testing::dali_extra_path() + "/db/video/cfr/test_2.mp4"; auto decode_first_frame = [&]() { @@ -240,11 +241,12 @@ TEST_F(FramesDecoderGpuTest, DecoderCacheReusedForCroppedVideo) { decoder.ReadNextFrame(FrameData()); }; + // The first decoder may or may not be created, depending on what ran before. decode_first_frame(); - size_t cache_size = frame_dec_gpu_impl::NVDECCacheSize(); - ASSERT_GT(cache_size, 0u); + size_t num_created = frame_dec_gpu_impl::NVDECDecodersCreated(); + // The decoder returned to the cache by the first one has to be reused. decode_first_frame(); - EXPECT_EQ(frame_dec_gpu_impl::NVDECCacheSize(), cache_size); + EXPECT_EQ(frame_dec_gpu_impl::NVDECDecodersCreated(), num_created); } TEST_F(FramesDecoderGpuTest, ConstantFrameRateHevc) { From a33ae415efab31c58d618badc739aebb748ed829 Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Wed, 21 Oct 2026 15:12:06 +0200 Subject: [PATCH 03/46] Add a benchmark for fn.experimental.readers.video internal_tools/video_reader_bench.py measures the experimental video reader on DALI_extra data only, with one scenario per aspect of its performance: - lanes: many distinct files, one keyframe-aligned sequence each, so a batch holds `batch` different files: decode parallelism across samples vs. num_threads. - engines: K independent readers in one pipeline: scaling with the number of concurrent NVDEC sessions. - single_stream: per-stream throughput (H.264, HEVC 8/10-bit, 512x512). - reader_depth: the reader's own prefetch_queue_depth at batch 1. - cache_key: per-switch cost between two files of the same format, cropped (800x600 coded as 800x608) or not, against a no-switch control. - startup: build time and first batch latency with many files. - cpu: the CPU backend on many VP8/VP9 files vs. num_threads. - seq_longgop, random_access, switch: no-regression checks for sequential, shuffled and mixed-resolution access. Every configuration runs in a fresh process, --repeat times, round-robin; the median and spread are reported. NVDEC utilization and memory are sampled per process with NVML, and runs are flagged when other processes use the GPU. --hash records per-sample checksums and --compare checks that two builds, or two thread counts (--ignore threads), produce identical output. Signed-off-by: Joaquin Anton Guirao --- internal_tools/video_reader_bench.py | 836 +++++++++++++++++++++++++++ 1 file changed, 836 insertions(+) create mode 100644 internal_tools/video_reader_bench.py diff --git a/internal_tools/video_reader_bench.py b/internal_tools/video_reader_bench.py new file mode 100644 index 00000000000..196c0658bf8 --- /dev/null +++ b/internal_tools/video_reader_bench.py @@ -0,0 +1,836 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Benchmark for fn.experimental.readers.video, using DALI_extra data only. + +Scenarios (each isolates one aspect of the reader): + + lanes many distinct files, one keyframe-aligned sequence each, so every + batch holds `batch` different files and no sample needs a seek: + decode parallelism across samples vs. num_threads + seq_longgop 15 H.264 clips read sequentially (long GOPs, consecutive sequences + of the same file): no-regression check for sequential access + random_access the same clips shuffled: parallelism and keyframe-seek waste + together (do not read a lanes speedup from this one alone) + single_stream one clip, batch 1: per-stream throughput, i.e. per-frame + serialization. Includes a 512x512 clip, where per-frame overheads + matter more than decode time + reader_depth batch 1, the reader's own prefetch_queue_depth (batches in flight), + on many distinct files and on one sequential long-GOP file + cache_key every sample switches between two files with the same format, either + cropped (800x600 coded as 800x608) or not (1280x720): per-switch + cost and the NVDEC decoder cache + switch clips of mixed resolutions, shuffled, short sequences: per-sample + cost under constant file switching (random access) + startup many distinct files: pipeline build time and first batch latency + engines K independent readers in one pipeline, each on its own files: + how throughput scales with concurrent NVDEC sessions (works today, + because every reader decodes on its own thread) + cpu many distinct VP8/VP9 files on the CPU backend vs. num_threads + +Methodology: + * every configuration runs in a fresh subprocess (no cache carry-over), --repeat + times, round-robin over configurations; the median and the spread are reported + * timed iterations run until both --iters and --min_time are reached; throughput + excludes the first `depth` iterations, which may have been prefetched off the clock + * Pipeline.run() without cuda_stream returns host-synchronized outputs, so iteration + times include GPU completion + * NVML metrics are per process where possible (decoder utilization, memory). A + result is flagged as contaminated if other processes use the GPU or NVDEC is busy + before the run starts + * files are read once before each scenario, so all runs see a warm page cache + * DALI_extra has no 1080p content, so absolute rates are lower than for 1080p data; + compare builds, not datasets + +--hash records a checksum of every sample (frames, label, frame_num). Compare two +--hash result files (e.g. two builds, or --ignore threads to check that the output +does not depend on num_threads) with --compare. +""" + +import argparse +import glob +import hashlib +import json +import os +import random +import shutil +import socket +import statistics +import subprocess +import sys +import tempfile +import threading +import time + +VIDEO_DIR = "db/video" + +H264_CLIPS = ["sintel/video_files/*.mp4", "sintel/labelled_videos/*/*.mp4"] +VPX_CLIPS = [ + "vp9/vp9_0.mp4", + "vp9/vp9_2.mp4", + "vp8/vp8.webm", + "cfr/test_1_vp9.mp4", + "cfr/test_2_vp9.mp4", + "vfr/test_1_vp9.mp4", + "vfr/test_2_vp9.mp4", +] +# file sets: glob patterns relative to DALI_extra/db/video +FILES = { + # 15 H.264 1280x720 clips, 122-430 frames, B-frames, 1-4 keyframes (GOP up to 250) + "h264_clips": H264_CLIPS, + # one H.264 1280x720 clip, 1253 frames, GOP 250 + "h264_long": ["sintel/sintel_trailer-720p.mp4"], + # the same clip as HEVC 8-bit (keyframe every ~9 frames) and HEVC 10-bit (GOP 250) + "hevc_long": ["hevc/sintel_trailer-720p.mp4"], + "hevc10_long": ["hevc/sintel_trailer-720p_10b.mp4"], + # one H.264 512x512 clip, 250 frames: small frames, per-frame overheads dominate + "h264_small": ["test_label.mp4"], + # H.264 clips with mixed resolutions (1280x720, 512x512, 800x600 coded as 800x608) + "h264_mixed": H264_CLIPS + + [ + "cfr_test.mp4", + "vfr_test.mp4", + "test_label.mp4", + "frame_num_timestamp/test_25fps.mp4", + "multiple_framerate/50/50fps.mp4", + "cfr/test_2.mp4", + "vfr/test_2.mp4", + ], + # two H.264 800x600 clips, coded as 800x608 (display area cropped) + "h264_cropped_pair": ["cfr/test_2.mp4", "vfr/test_2.mp4"], + # two H.264 1280x720 clips, coded size == display size + "h264_aligned_pair": ["cfr_test.mp4", "vfr_test.mp4"], + # VP8/VP9 clips (H.264 is not enabled on the CPU backend) + "vpx_clips": VPX_CLIPS, +} + + +def _cfg(files, layout="plain", device="gpu", batch=8, seq=32, threads=4, **kw): + """A benchmark configuration. Only JSON-serializable values. + + layout: "plain" - the files as they are (filenames=...) + "many:N" - N distinct symlinks cycling over the files, each with one + sequence starting at frame 0 (a keyframe) + "alternate:N" - N lines alternating between the files, each starting at + frame 0, read in order: every sample switches files + "same:N" - N lines, all of the first file, each starting at frame 0: the + no-switch control for "alternate" (same seeks, no switches) + """ + cfg = dict( + files=files, + layout=layout, + device=device, + batch=batch, + seq=seq, + threads=threads, + depth=2, # pipeline prefetch_queue_depth (DALI default) + reader_depth=1, # the reader's own prefetch_queue_depth (DALI default) + readers=1, + shuffle=False, + ) + cfg.update(kw) + return cfg + + +def scenarios(threads, readers): + t_max = max(threads) + t_pair = sorted({1, t_max}) + return { + "lanes": [ + _cfg("h264_clips", "many:256", batch=b, seq=32, threads=t) + for b in (8, 16) + for t in threads + ], + "seq_longgop": [_cfg("h264_clips", batch=16, seq=32, threads=t) for t in t_pair], + "random_access": [ + _cfg("h264_clips", batch=16, seq=32, threads=t, shuffle=True) for t in t_pair + ], + "single_stream": [ + _cfg(f, batch=1, seq=64, threads=4) + for f in ("h264_long", "hevc_long", "hevc10_long", "h264_small") + ], + "reader_depth": [ + _cfg(f, layout, batch=1, seq=32, threads=8, reader_depth=d) + for f, layout in (("h264_clips", "many:256"), ("h264_long", "plain")) + for d in (1, 2, 4, 8) + ], + # switch cost = ms_per_sample(alternate) - ms_per_sample(same), per file pair + "cache_key": [ + _cfg(f, f"{kind}:64", batch=8, seq=8, threads=t) + for f in ("h264_cropped_pair", "h264_aligned_pair") + for kind in ("alternate", "same") + for t in t_pair + ], + "switch": [_cfg("h264_mixed", batch=8, seq=8, threads=t, shuffle=True) for t in t_pair], + "startup": [ + _cfg("h264_clips", "many:256", batch=1, seq=8, threads=t, iters=0) for t in t_pair + ], + "engines": [ + _cfg("h264_clips", "many:64", batch=4, seq=32, threads=4, readers=k) for k in readers + ], + "cpu": [ + _cfg("vpx_clips", "many:128", device="cpu", batch=b, seq=16, threads=t) + for b in (1, 8) + for t in threads + ], + } + + +# ---------------------------------------------------------------------------------------- +# data + + +def resolve_files(dali_extra, name): + root = os.path.join(dali_extra, VIDEO_DIR) + files, missing = [], [] + for pattern in FILES[name]: + matches = sorted(glob.glob(os.path.join(root, pattern))) + if not matches: + missing.append(pattern) + files += matches + if missing: + raise FileNotFoundError(f"file set {name}: not found under {root}: {missing}") + return files + + +def materialize(cfg, dali_extra, workdir): + """Creates the inputs of a configuration. Returns one input spec per reader. + + A spec is either {"filenames": [...]} or {"file_list": path}. The result is fully + determined by the configuration, so repeated runs (and --hash) see the same samples. + """ + files = resolve_files(dali_extra, cfg["files"]) + kind, _, n = cfg["layout"].partition(":") + if kind == "plain": + if cfg["readers"] != 1: + raise ValueError("layout 'plain' supports a single reader") + return [{"filenames": files}] + n = int(n) + seq = cfg["seq"] + tag = "_".join(str(cfg[k]) for k in ("files", "layout", "seq", "readers")) + out_dir = os.path.join(workdir, tag.replace(":", "-")) + os.makedirs(out_dir, exist_ok=True) + lines = [] + if kind == "many": + for i in range(n * cfg["readers"]): + src = files[i % len(files)] + dst = os.path.join(out_dir, f"{i:05d}_{os.path.basename(src)}") + if not os.path.lexists(dst): + os.symlink(src, dst) + lines.append((dst, i)) + elif kind == "alternate": + lines = [(files[i % len(files)], i) for i in range(n)] + elif kind == "same": + lines = [(files[0], i) for i in range(n)] + else: + raise ValueError(f"unknown layout {cfg['layout']}") + specs = [] + per_reader = len(lines) // cfg["readers"] + for r in range(cfg["readers"]): + path = os.path.join(out_dir, f"file_list_{r}.txt") + with open(path, "w") as f: + for name, label in lines[r * per_reader : (r + 1) * per_reader]: + # frames [0, seq) (file_list_include_end=False): exactly one sequence, starting + # at frame 0, a keyframe, so no sample decodes frames it doesn't return + f.write(f"{name} {label} 0 {seq}\n") + specs.append({"file_list": path}) + return specs + + +def prewarm(paths): + """Reads files once so that every run sees a warm page cache.""" + for p in sorted(set(paths)): + with open(p, "rb") as f: + while f.read(1 << 24): + pass + + +# ---------------------------------------------------------------------------------------- +# GPU monitoring + + +def nvml_handle(device_id): + """NVML handle of the CUDA device `device_id`, honoring CUDA_VISIBLE_DEVICES. + + Assumes CUDA_DEVICE_ORDER=PCI_BUS_ID or a machine where CUDA and NVML enumerate + devices the same way. Returns (pynvml, handle) or raises. + """ + import pynvml + + pynvml.nvmlInit() + visible = os.environ.get("CUDA_VISIBLE_DEVICES") + if visible: + entry = visible.split(",")[device_id].strip() + if entry.isdigit(): + return pynvml, pynvml.nvmlDeviceGetHandleByIndex(int(entry)) + return pynvml, pynvml.nvmlDeviceGetHandleByUUID(entry) + return pynvml, pynvml.nvmlDeviceGetHandleByIndex(device_id) + + +def other_gpu_pids(nvml, handle): + procs = nvml.nvmlDeviceGetComputeRunningProcesses(handle) + procs += nvml.nvmlDeviceGetGraphicsRunningProcesses(handle) + return sorted({p.pid for p in procs if p.pid != os.getpid()}) + + +class GpuMonitor: + """Samples per-process GPU memory and collects per-process NVDEC utilization.""" + + def __init__(self, device_id, period=0.1): + self.error = None + self.period = period + self.mem = [] + self.dev_dec = [] + self._stop = threading.Event() + self._thread = threading.Thread(target=self._run, daemon=True) + try: + self.nvml, self.handle = nvml_handle(device_id) + except Exception as e: + self.nvml = None + self.error = f"NVML unavailable: {e}" + + def snapshot(self): + """Pre/post-run state: other processes, NVDEC utilization, clocks.""" + if not self.nvml: + return {} + nvml, h = self.nvml, self.handle + return dict( + other_pids=other_gpu_pids(nvml, h), + dev_nvdec_util=nvml.nvmlDeviceGetDecoderUtilization(h)[0], + clock_video_mhz=nvml.nvmlDeviceGetClockInfo(h, nvml.NVML_CLOCK_VIDEO), + clock_sm_mhz=nvml.nvmlDeviceGetClockInfo(h, nvml.NVML_CLOCK_SM), + ) + + def _run(self): + nvml, h, pid = self.nvml, self.handle, os.getpid() + while not self._stop.is_set(): + try: + for p in nvml.nvmlDeviceGetComputeRunningProcesses(h): + if p.pid == pid and p.usedGpuMemory is not None: + self.mem.append(p.usedGpuMemory) + self.dev_dec.append(nvml.nvmlDeviceGetDecoderUtilization(h)[0]) + except Exception as e: + self.error = f"NVML sampling failed: {e}" + return + time.sleep(self.period) + + def start(self): + self._t_start_us = int(time.time() * 1e6) + if self.nvml: + self._thread.start() + + def stop(self): + if not self.nvml: + return {} + self._stop.set() + self._thread.join() + res = {} + if self.mem: + res["proc_mem_max_mb"] = max(self.mem) / 2**20 + if self.dev_dec: + res["dev_nvdec_util"] = statistics.mean(self.dev_dec) + try: # per-process decoder utilization since start, where the driver supports it + samples = self.nvml.nvmlDeviceGetProcessUtilization(self.handle, self._t_start_us) + dec = [s.decUtil for s in samples if s.pid == os.getpid()] + if dec: + res["proc_nvdec_util"] = statistics.mean(dec) + except Exception: + pass + return res + + +# ---------------------------------------------------------------------------------------- +# running + + +def build_pipeline(cfg, specs, device_id, seed, frame_num=False): + import nvidia.dali.fn as fn + from nvidia.dali.pipeline import pipeline_def + + @pipeline_def( + batch_size=cfg["batch"], + num_threads=cfg["threads"], + device_id=device_id, + prefetch_queue_depth=cfg["depth"], + seed=seed, + ) + def pipe(): + outs = [] + for r, spec in enumerate(specs): + kw = dict(spec) + if "filenames" in kw: + kw["labels"] = list(range(len(kw["filenames"]))) + else: + kw["file_list_format"] = "frames" + kw["file_list_include_end"] = False # explicit: the default changed over time + if "initial_fill" in cfg: + kw["initial_fill"] = cfg["initial_fill"] + if frame_num: + kw["enable_frame_num"] = True + outs += fn.experimental.readers.video( + device=cfg["device"], + sequence_length=cfg["seq"], + random_shuffle=cfg["shuffle"], + prefetch_queue_depth=cfg["reader_depth"], + name=f"Reader{r}", + **kw, + ) + return tuple(outs) + + return pipe() + + +def warm_up_cuda(device_id): + """Initializes CUDA and the executor so build_s measures the reader, not the runtime.""" + import nvidia.dali.fn as fn + from nvidia.dali.pipeline import pipeline_def + + @pipeline_def(batch_size=1, num_threads=1, device_id=device_id) + def tiny(): + return fn.random.uniform(shape=[1]).gpu() + + p = tiny() + p.build() + p.run() + del p + + +def percentile(sorted_vals, q): + if len(sorted_vals) == 1: + return sorted_vals[0] + return statistics.quantiles(sorted_vals, n=100, method="inclusive")[q - 1] + + +def run_one(cfg, specs, args): + """Runs one configuration in this process and returns its metrics.""" + import nvidia.dali as dali + + mon = GpuMonitor(args.device_id) + res = dict(dali_version=dali.__version__) + res["pre"] = mon.snapshot() + warm_up_cuda(args.device_id) + + t0 = time.perf_counter() + pipe = build_pipeline(cfg, specs, args.device_id, args.seed) + pipe.build() + res["build_s"] = time.perf_counter() - t0 + t0 = time.perf_counter() + pipe.run() + res["first_batch_ms"] = (time.perf_counter() - t0) * 1e3 + out = pipe.run() + res["shape"] = list(out[0].shape()[0]) + + iters = cfg.get("iters", args.iters) + if iters > 0: + for _ in range(args.warmup): + pipe.run() + frames_per_iter = cfg["readers"] * cfg["batch"] * cfg["seq"] + stamps = [] + mon.start() + t_start = time.perf_counter() + while len(stamps) < iters or stamps[-1] - t_start < args.min_time: + pipe.run() + stamps.append(time.perf_counter()) + res.update(mon.stop()) + times = sorted(b - a for a, b in zip([t_start] + stamps, stamps)) + times_ms = [t * 1e3 for t in times] + # the first `depth` batches may have been produced off the clock + skip = min(cfg["depth"], len(stamps) - 1) + n = len(stamps) - skip + span = stamps[-1] - (stamps[skip - 1] if skip else t_start) + h, w = res["shape"][1:3] + res.update( + iters=len(stamps), + frames_per_s=n * frames_per_iter / span, + mpix_per_s=n * frames_per_iter * h * w / span / 1e6, + ms_per_sample=span * 1e3 / (n * cfg["readers"] * cfg["batch"]), + iter_p50_ms=statistics.median(times_ms), + iter_p90_ms=percentile(times_ms, 90), + iter_p99_ms=percentile(times_ms, 99) if len(times_ms) >= 200 else None, + iter_max_ms=times_ms[-1], + ) + if mon.error: + res["nvml_error"] = mon.error + res["post"] = mon.snapshot() + pre, post = res["pre"], res["post"] + res["contaminated"] = bool( + pre.get("other_pids") or post.get("other_pids") or pre.get("dev_nvdec_util", 0) > 5 + ) + return res + + +def hash_one(cfg, specs, args): + """Returns one checksum per sample for the first --hash_batches batches.""" + pipe = build_pipeline(cfg, specs, args.device_id, args.seed, frame_num=True) + pipe.build() + hashes = [] + for _ in range(args.hash_batches): + outs = [_as_cpu(t) for t in pipe.run()] + for r in range(cfg["readers"]): + video, label, frame_num = outs[3 * r : 3 * r + 3] + for i in range(len(video)): + h = hashlib.sha1(video.at(i).tobytes()) + h.update(label.at(i).tobytes()) + h.update(frame_num.at(i).tobytes()) + hashes.append(h.hexdigest()) + return dict(hashes=hashes) + + +def _as_cpu(tensor_list): + return tensor_list.as_cpu() if hasattr(tensor_list, "as_cpu") else tensor_list + + +def child_main(args): + job = json.loads(args._single) + cfg, specs = job["cfg"], job["specs"] + try: + res = hash_one(cfg, specs, args) if args.hash else run_one(cfg, specs, args) + except Exception as e: + lines = [ln for ln in str(e).strip().splitlines() if ln.strip()] + detail = [ln for ln in lines if "rror" in ln or "ssert" in ln or "failed" in ln] + res = dict(error=f"{type(e).__name__}: " + " | ".join((detail or lines)[-3:])) + print("RESULT " + json.dumps(res)) + + +def run_child(cfg, specs, args): + cmd = [sys.executable, os.path.abspath(__file__), "--_single"] + cmd += [json.dumps(dict(cfg=cfg, specs=specs))] + for flag in ("dali_extra", "device_id", "iters", "warmup", "min_time", "seed"): + cmd += [f"--{flag}", str(getattr(args, flag))] + cmd += ["--hash_batches", str(args.hash_batches)] + (["--hash"] if args.hash else []) + try: + p = subprocess.run(cmd, capture_output=True, text=True, timeout=args.timeout) + except subprocess.TimeoutExpired: + return dict(error=f"timeout after {args.timeout} s") + for line in reversed(p.stdout.splitlines()): + if line.startswith("RESULT "): + return json.loads(line[len("RESULT ") :]) + tail = [ln for ln in (p.stderr or p.stdout).splitlines() if ln.strip()][-15:] + detail = [ln for ln in tail if "rror" in ln or "ssert" in ln] or tail[-3:] + return dict(error=f"exit code {p.returncode}: " + " | ".join(detail[-3:])) + + +def aggregate(cfg, reps): + """Median and spread over repetitions.""" + ok = [r for r in reps if "error" not in r] + res = dict(cfg) + res["repeats"] = len(reps) + if not ok: + res["error"] = reps[-1]["error"] if reps else "no runs" + return res + res["errors"] = [r["error"] for r in reps if "error" in r] + res["contaminated"] = any(r.get("contaminated") for r in ok) + res["runs"] = ok + for key in ( + "frames_per_s", + "mpix_per_s", + "ms_per_sample", + "iter_p50_ms", + "iter_p90_ms", + "iter_p99_ms", + "iter_max_ms", + "build_s", + "first_batch_ms", + "proc_nvdec_util", + "dev_nvdec_util", + "proc_mem_max_mb", + ): + vals = [r[key] for r in ok if r.get(key) is not None] + if vals: + res[key] = statistics.median(vals) + if key in ("frames_per_s", "build_s", "first_batch_ms") and len(vals) > 1: + res[key + "_spread_pct"] = (max(vals) - min(vals)) / res[key] * 100 + res["shape"] = ok[0].get("shape") + for key in ("dali_version", "nvml_error"): + if ok[0].get(key): + res[key] = ok[0][key] + return res + + +# ---------------------------------------------------------------------------------------- +# reporting + + +COLUMNS = [ + ("scenario", "scenario", "{:<13}"), + ("files", "files", "{:<17}"), + ("layout", "layout", "{:<12}"), + ("device", "dev", "{:<3}"), + ("batch", "bs", "{:>2}"), + ("seq", "seq", "{:>3}"), + ("threads", "thr", "{:>3}"), + ("reader_depth", "rq", "{:>2}"), + ("readers", "rd", "{:>2}"), + ("shuffle", "sh", "{:<2}"), + ("frames_per_s", "frames/s", "{:>8.1f}"), + ("frames_per_s_spread_pct", "spr%", "{:>4.1f}"), + ("ms_per_sample", "ms/smp", "{:>7.2f}"), + ("iter_p50_ms", "p50 ms", "{:>8.2f}"), + ("iter_p90_ms", "p90 ms", "{:>8.2f}"), + ("iter_max_ms", "max ms", "{:>8.2f}"), + ("build_s", "build s", "{:>7.3f}"), + ("first_batch_ms", "1st ms", "{:>7.1f}"), + ("proc_nvdec_util", "dec%", "{:>4.0f}"), + ("proc_mem_max_mb", "mem MB", "{:>6.0f}"), +] + + +def _width(fmt): + return int(fmt[3:].split(".")[0].rstrip("}")) + + +def print_header(): + print(" ".join(head.rjust(_width(fmt)) for _, head, fmt in COLUMNS) + " flags") + + +def format_row(r): + cells = [] + for key, head, fmt in COLUMNS: + v = r.get(key) + if key == "shuffle": + v = "y" if v else "n" + width = _width(fmt) + try: + cells.append(fmt.format(v) if v is not None else "-".rjust(width)) + except (ValueError, TypeError): + cells.append(str(v).ljust(width)) + flags = [] + if r.get("contaminated"): + flags.append("CONTAMINATED") + if r.get("frames_per_s_spread_pct", 0) > 3: + flags.append("spread>3%") + if r.get("errors"): + flags.append(f"{len(r['errors'])} failed run(s)") + if r.get("error"): + flags.append("ERROR: " + r["error"][:160]) + return " ".join(cells) + (" " + ", ".join(flags) if flags else "") + + +def provenance(args): + info = dict( + time=time.strftime("%Y-%m-%d %H:%M:%S"), + host=socket.gethostname(), + python=sys.version.split()[0], + cpu_count=os.cpu_count(), + argv=sys.argv, + isolation="subprocess per run", + env={k: v for k, v in os.environ.items() if k.startswith("DALI_") or k.startswith("CUDA_")}, + ) + try: + import nvidia.dali as dali + + info["dali_version"] = dali.__version__ + info["dali_git_sha"] = getattr(dali, "__git_sha__", None) + info["dali_path"] = os.path.dirname(dali.__file__) + except Exception as e: + info["dali_error"] = str(e) + try: + nvml, h = nvml_handle(args.device_id) + info["gpu"] = nvml.nvmlDeviceGetName(h) + info["gpu_uuid"] = nvml.nvmlDeviceGetUUID(h) + info["driver"] = nvml.nvmlSystemGetDriverVersion() + info["other_gpu_pids"] = other_gpu_pids(nvml, h) + info["nvdec_util_at_start"] = nvml.nvmlDeviceGetDecoderUtilization(h)[0] + except Exception as e: + info["nvml_error"] = str(e) + return info + + +def compare(a_path, b_path, ignore): + """Compares per-sample hashes of matching configurations in two --hash files. + + Fields listed in `ignore` (e.g. threads, reader_depth) are left out when matching, + so a configuration in A is compared against every configuration in B that differs + only in those fields. The output must not depend on them. + """ + a, b = (json.load(open(p)) for p in (a_path, b_path)) + skip = {"hashes", "scenario", "error", "errors", "dali_version", "repeats"} | set(ignore) + + def key(r): + return json.dumps({k: v for k, v in r.items() if k not in skip}, sort_keys=True) + + def label(r): + return ", ".join(f"{k}={r[k]}" for k in sorted(r) if k not in ("hashes", "errors")) + + b_runs = {} + for r in b["results"]: + b_runs.setdefault(key(r), []).append(r) + bad = compared = 0 + for r in a["results"]: + for other in b_runs.get(key(r), []): + if "hashes" not in r or "hashes" not in other: + print(f"skip (failed run): {label(r)}") + continue + n = max(len(r["hashes"]), len(other["hashes"])) + diff = sum(x != y for x, y in zip(r["hashes"], other["hashes"])) + diff += abs(len(r["hashes"]) - len(other["hashes"])) + compared += 1 + bad += diff > 0 + vs = ", ".join(f"{k}={other[k]}" for k in ignore if other.get(k) != r.get(k)) + status = "OK " if diff == 0 else "DIFF" + print(f"{status} {diff:4d}/{n} samples {label(r)}" + (f" vs {vs}" if vs else "")) + if compared == 0: + print("nothing to compare: no matching configurations") + return 2 + print("all identical" if bad == 0 else f"{bad} comparison(s) differ") + return 1 if bad else 0 + + +# ---------------------------------------------------------------------------------------- + + +def parse_args(): + parser = argparse.ArgumentParser( + description="fn.experimental.readers.video benchmark on DALI_extra data", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=__doc__, + ) + parser.add_argument("-s", "--scenarios", default="all", help="comma separated, or all") + parser.add_argument( + "-j", "--threads", default="1,2,4,8,16", help="num_threads sweep (B200: 1,2,4,7,8,14,16)" + ) + parser.add_argument("--readers", default="1,2,4,8", help="reader counts for 'engines'") + parser.add_argument("-d", "--device_id", default=0, type=int) + parser.add_argument("--repeat", default=3, type=int, help="fresh-process runs per config") + parser.add_argument("--iters", default=20, type=int, help="minimum timed iterations") + parser.add_argument("--min_time", default=5.0, type=float, help="minimum timed seconds") + parser.add_argument("--warmup", default=3, type=int, help="untimed iterations") + parser.add_argument("--timeout", default=900, type=int, help="seconds per run") + parser.add_argument("--seed", default=42, type=int) + parser.add_argument("--dali_extra", default=os.environ.get("DALI_EXTRA_PATH", "")) + parser.add_argument("--json", help="write results (with provenance) to this file") + parser.add_argument("--quick", action="store_true", help="smoke test: tiny sweep, 1 run") + parser.add_argument("--require_idle", action="store_true", help="abort if the GPU is busy") + parser.add_argument("--hash", action="store_true", help="record per-sample checksums") + parser.add_argument("--hash_batches", default=8, type=int) + parser.add_argument("--compare", nargs=2, metavar=("A", "B"), help="compare two --hash files") + parser.add_argument( + "--ignore", + default="", + help="with --compare: comma separated config fields to ignore when matching runs, " + "e.g. threads,reader_depth to check that output does not depend on them", + ) + parser.add_argument("--_single", help=argparse.SUPPRESS) + args = parser.parse_args() + if args.compare: + return parser, args + if not args.dali_extra: + parser.error("set DALI_EXTRA_PATH or pass --dali_extra") + if args.hash and not args.json and not args._single: + parser.error("--hash needs --json to store the checksums") + if args.quick: + args.threads, args.readers, args.repeat = "1,4", "1,2", 1 + args.iters, args.min_time, args.warmup = 3, 0.0, 1 + return parser, args + + +def main(): + parser, args = parse_args() + if args.compare: + sys.exit(compare(*args.compare, [f for f in args.ignore.split(",") if f])) + if args._single: + child_main(args) + return + + threads = [int(t) for t in args.threads.split(",")] + readers = [int(r) for r in args.readers.split(",")] + all_scenarios = scenarios(threads, readers) + names = list(all_scenarios) if args.scenarios == "all" else args.scenarios.split(",") + unknown = [n for n in names if n not in all_scenarios] + if unknown: + parser.error(f"unknown scenario(s) {unknown}; choose from {list(all_scenarios)}") + if args.hash: + args.repeat = 1 + + prov = provenance(args) + print("# " + json.dumps(prov)) + if prov.get("nvml_error"): + print(f"# WARNING: NVML unavailable ({prov['nvml_error']}); GPU checks are disabled") + busy = prov.get("other_gpu_pids") or prov.get("nvdec_util_at_start", 0) > 5 + if busy: + print("# WARNING: the GPU is in use by other processes; numbers will be skewed") + if args.require_idle: + sys.exit("GPU busy, aborting (--require_idle)") + + workdir = tempfile.mkdtemp(prefix="video_reader_bench_") + results, failed = [], 0 + try: + jobs = [] + for name in names: + for cfg in all_scenarios[name]: + cfg = dict(cfg, scenario=name) + try: + specs = materialize(cfg, args.dali_extra, workdir) + except Exception as e: + jobs.append((cfg, None, f"{type(e).__name__}: {e}")) + continue + jobs.append((cfg, specs, None)) + for cfg, specs, err in jobs: + if specs: + paths = [] + for s in specs: + if "filenames" in s: + paths += s["filenames"] + else: + paths += [ln.split()[0] for ln in open(s["file_list"])] + prewarm(os.path.realpath(p) for p in paths) + reps = {i: [] for i in range(len(jobs))} + rng = random.Random(args.seed) + for rep in range(args.repeat): + order = list(range(len(jobs))) + if rep > 0: + rng.shuffle(order) # round-robin, so drift does not line up with a config + for i in order: + cfg, specs, err = jobs[i] + res = dict(error=err) if err else run_child(cfg, specs, args) + reps[i].append(res) + note = res.get("error") or ( + f"{res['frames_per_s']:.1f} frames/s" if "frames_per_s" in res else "done" + ) + print( + f"[run {rep + 1}/{args.repeat}] {cfg['scenario']} {cfg['files']} " + f"{cfg['layout']} bs={cfg['batch']} thr={cfg['threads']} " + f"rq={cfg['reader_depth']} rd={cfg['readers']}: {note}", + file=sys.stderr, + flush=True, + ) + if not args.hash: + print_header() + for i, (cfg, specs, err) in enumerate(jobs): + if args.hash: + r = dict(cfg, seed=args.seed, hash_batches=args.hash_batches, **reps[i][0]) + n = len(r.get("hashes", [])) + print( + f"{cfg['scenario']} {cfg['files']} threads={cfg['threads']}: " + f"{n} sample hashes" + (f" ERROR: {r['error']}" if r.get("error") else "") + ) + else: + r = aggregate(cfg, reps[i]) + print(format_row(r)) + failed += bool(r.get("error")) + results.append(r) + finally: + shutil.rmtree(workdir, ignore_errors=True) + + if args.json: + with open(args.json, "w") as f: + json.dump(dict(provenance=prov, results=results), f, indent=1) + print(f"wrote {args.json}") + if failed: + print(f"{failed} configuration(s) failed") + sys.exit(1) + + +if __name__ == "__main__": + main() From e8bf6c8805074f8ee738bece309db7290ad8959d Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Wed, 21 Oct 2026 15:31:54 +0200 Subject: [PATCH 04/46] Add TL1_video_reader_perf qa test Runs internal_tools/video_reader_bench.py the way TL1_decoder_perf runs hw_decoder_bench.py: pinned to the same CPUs, re-run under nsys on failure with the profile and results saved to core_artifacts. The test fails if: - any benchmark configuration fails, - the output depends on num_threads (1 vs 16) or on the reader's prefetch_queue_depth (per-sample checksums, --hash/--compare), - throughput drops by more than 10% when num_threads (lanes, cpu) or the reader's prefetch_queue_depth grow, - an absolute minimum is set and not met. The minimums are empty for now, so the values are only printed for reference until they are calibrated on the CI machines. The checks live in qa/TL1_video_reader_perf/check_results.py, which reads the benchmark's JSON output. Also make the benchmark tolerate NVML queries that older bindings (nvidia-ml-py 11.450, as used in qa) do not support with newer drivers. Signed-off-by: Joaquin Anton Guirao --- internal_tools/video_reader_bench.py | 17 ++- qa/TL1_video_reader_perf/check_results.py | 140 ++++++++++++++++++++++ qa/TL1_video_reader_perf/test.sh | 116 ++++++++++++++++++ 3 files changed, 268 insertions(+), 5 deletions(-) create mode 100644 qa/TL1_video_reader_perf/check_results.py create mode 100644 qa/TL1_video_reader_perf/test.sh diff --git a/internal_tools/video_reader_bench.py b/internal_tools/video_reader_bench.py index 196c0658bf8..bf9613dc177 100644 --- a/internal_tools/video_reader_bench.py +++ b/internal_tools/video_reader_bench.py @@ -305,12 +305,19 @@ def snapshot(self): if not self.nvml: return {} nvml, h = self.nvml, self.handle - return dict( - other_pids=other_gpu_pids(nvml, h), - dev_nvdec_util=nvml.nvmlDeviceGetDecoderUtilization(h)[0], - clock_video_mhz=nvml.nvmlDeviceGetClockInfo(h, nvml.NVML_CLOCK_VIDEO), - clock_sm_mhz=nvml.nvmlDeviceGetClockInfo(h, nvml.NVML_CLOCK_SM), + res = {} + queries = dict( + other_pids=lambda: other_gpu_pids(nvml, h), + dev_nvdec_util=lambda: nvml.nvmlDeviceGetDecoderUtilization(h)[0], + clock_video_mhz=lambda: nvml.nvmlDeviceGetClockInfo(h, nvml.NVML_CLOCK_VIDEO), + clock_sm_mhz=lambda: nvml.nvmlDeviceGetClockInfo(h, nvml.NVML_CLOCK_SM), ) + for key, query in queries.items(): + try: # older NVML bindings may lack some queries for newer drivers + res[key] = query() + except Exception as e: + self.error = f"NVML {key} failed: {e}" + return res def _run(self): nvml, h, pid = self.nvml, self.handle, os.getpid() diff --git a/qa/TL1_video_reader_perf/check_results.py b/qa/TL1_video_reader_perf/check_results.py new file mode 100644 index 00000000000..4cbc9ea4fd3 --- /dev/null +++ b/qa/TL1_video_reader_perf/check_results.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Checks the results of internal_tools/video_reader_bench.py (--json output). + +Checks: + * no configuration failed + * hardware-independent relative checks, with a tolerance: + - lanes, cpu: throughput at the highest num_threads is not lower than at the + lowest one (more threads must not make the reader slower) + - reader_depth: a deeper reader queue is not slower than depth 1 + * optional absolute minimums (frames/s) for named metrics, passed with --min: + lanes lanes, batch 16, highest num_threads + single_stream single_stream, h264_long + cpu cpu, batch 8, highest num_threads + engines engines, highest number of readers + Metrics without a minimum are printed for reference only. + +Results flagged as contaminated (other processes used the GPU) only produce warnings. +Exits with 1 if any check failed. +""" + +import argparse +import json +import sys + + +def by(results, scenario, **match): + return [ + r + for r in results + if r.get("scenario") == scenario and all(r.get(k) == v for k, v in match.items()) + ] + + +def groups(results, scenario, keys): + out = {} + for r in by(results, scenario): + out.setdefault(tuple(r.get(k) for k in keys), []).append(r) + return out + + +def named_metrics(results): + """The metrics that can have absolute minimums.""" + + def at_max(rows, key): + rows = [r for r in rows if r.get("frames_per_s") is not None] + return max(rows, key=lambda r: r[key])["frames_per_s"] if rows else None + + return { + "lanes": at_max(by(results, "lanes", batch=16), "threads"), + "single_stream": next( + (r.get("frames_per_s") for r in by(results, "single_stream", files="h264_long")), None + ), + "cpu": at_max(by(results, "cpu", batch=8), "threads"), + "engines": at_max(by(results, "engines"), "readers"), + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("results", help="--json output of video_reader_bench.py") + parser.add_argument("--tolerance", type=float, default=10.0, help="percent, relative checks") + parser.add_argument( + "--min", action="append", default=[], metavar="NAME=FPS", help="absolute minimum" + ) + args = parser.parse_args() + + data = json.load(open(args.results)) + results = data["results"] + tol = args.tolerance / 100 + failed = [] + + def check(name, ok, detail): + print(f"CHECK {name}: {'OK' if ok else 'FAIL'} ({detail})") + if not ok: + failed.append(name) + + errors = [r for r in results if r.get("error")] + check("no failed configurations", not errors, f"{len(errors)} of {len(results)} failed") + for r in errors: + print(f" {r.get('scenario')} {r.get('files')} {r.get('layout')}: {r['error']}") + + contaminated = [r for r in results if r.get("contaminated")] + if contaminated: + print(f"WARNING: {len(contaminated)} result(s) contaminated by other GPU processes") + + def relative(scenario, keys, param): + for key, rows in sorted(groups(results, scenario, keys).items(), key=str): + rows = sorted((r for r in rows if r.get("frames_per_s")), key=lambda r: r[param]) + if len(rows) < 2: + continue + base = rows[0] + for r in rows[1:]: + ok = r["frames_per_s"] >= (1 - tol) * base["frames_per_s"] + label = "/".join(str(k) for k in key) + check( + f"{scenario} {label} {param}={r[param]} vs {base[param]}", + ok, + f"{r['frames_per_s']:.1f} vs {base['frames_per_s']:.1f} frames/s, " + f"tolerance {args.tolerance:g}%", + ) + + relative("lanes", ("files", "layout", "batch"), "threads") + relative("cpu", ("files", "layout", "batch"), "threads") + relative("reader_depth", ("files", "layout"), "reader_depth") + + minimums = {} + for item in args.min: + name, _, value = item.partition("=") + if value: + minimums[name] = float(value) + for name, value in named_metrics(results).items(): + if name in minimums: + check( + f"{name} minimum", + value is not None and value >= minimums[name], + f"{value} vs min {minimums[name]:g} frames/s", + ) + else: + print(f"REFERENCE {name}: {value} frames/s (no minimum set)") + + print("all checks passed" if not failed else f"{len(failed)} check(s) failed") + sys.exit(1 if failed else 0) + + +if __name__ == "__main__": + main() diff --git a/qa/TL1_video_reader_perf/test.sh b/qa/TL1_video_reader_perf/test.sh new file mode 100644 index 00000000000..183161a5af5 --- /dev/null +++ b/qa/TL1_video_reader_perf/test.sh @@ -0,0 +1,116 @@ +#!/bin/bash -e +# used pip packages +pip_packages='numpy nvidia-ml-py==11.450.51' +target_dir=./internal_tools + +# One-time pre-step: install nsys (NVIDIA Nsight Systems) if not present +do_once() { + if command -v nsys &>/dev/null; then + echo "nsys already installed: $(nsys --version)" + return + fi + DISTRO=$(source /etc/lsb-release && echo "$DISTRIB_RELEASE" | tr -d .) + ARCH=$(dpkg --print-architecture) + apt update && apt install -y --no-install-recommends gnupg curl + echo "deb [signed-by=/usr/share/keyrings/nvidia-devtools.gpg] https://developer.download.nvidia.com/devtools/repos/ubuntu${DISTRO}/${ARCH} /" \ + | tee /etc/apt/sources.list.d/nvidia-devtools.list + curl -fsSL "https://developer.download.nvidia.com/devtools/repos/ubuntu${DISTRO}/${ARCH}/nvidia.pub" \ + | gpg --dearmor -o /usr/share/keyrings/nvidia-devtools.gpg + apt update && apt install -y nsight-systems-cli +} + +LOG_PERF="video_reader_perf.log" +JSON_PERF="video_reader_perf.json" +JSON_HASH_LOW="video_reader_hash_low.json" +JSON_HASH_HIGH="video_reader_hash_high.json" + +# Scenarios of internal_tools/video_reader_bench.py run for performance; see its --help +PERF_SCENARIOS="lanes,engines,single_stream,reader_depth,cache_key,startup,seq_longgop,cpu" +# Scenarios whose output is checked to be independent of num_threads and reader_depth +HASH_SCENARIOS="lanes,reader_depth,cache_key,cpu" +THREADS_LOW=1 +THREADS_HIGH=16 +BENCH_ARGS="--repeat 3 --min_time 3 -j ${THREADS_LOW},4,${THREADS_HIGH} --readers 1,2,4" + +function CLEAN_AND_EXIT { + rm -rf ${LOG_PERF} ${JSON_PERF} ${JSON_HASH_LOW} ${JSON_HASH_HIGH} + exit $1 +} + +# Run the benchmark; if NSYS_REP is set, wrap it with nsys profiling (child processes included). +run_bench() { + local log_file="$1" + shift + if [ -n "${NSYS_REP}" ]; then + local profile_name="${log_file%.log}.nsys-rep" + nsys profile -o "${profile_name}" --trace=cuda,nvtx,osrt "$@" | tee "${log_file}" + else + "$@" | tee "${log_file}" + fi +} + +test_body() { + if [ "$(uname -p)" == "x86_64" ]; then + # Hopper + TASKSET="taskset --cpu-list 0-127" + # Absolute minimums in frames/s (see qa/TL1_video_reader_perf/check_results.py); + # empty: reported for reference only, not checked + MIN_LANES= + MIN_SINGLE_STREAM= + MIN_CPU= + MIN_ENGINES= + else + # GraceHopper + TASKSET="taskset --cpu-list 0-71" + MIN_LANES= + MIN_SINGLE_STREAM= + MIN_CPU= + MIN_ENGINES= + fi + + unset NSYS_REP + PERF_RUN="OK" + run_bench "${LOG_PERF}" ${TASKSET} python video_reader_bench.py ${BENCH_ARGS} \ + -s ${PERF_SCENARIOS} --json ${JSON_PERF} || PERF_RUN="FAIL" + + # The output must not depend on num_threads or on the reader's prefetch_queue_depth + HASH_RESULT="OK" + for threads in ${THREADS_LOW} ${THREADS_HIGH}; do + [ ${threads} == ${THREADS_LOW} ] && json=${JSON_HASH_LOW} || json=${JSON_HASH_HIGH} + ${TASKSET} python video_reader_bench.py -s ${HASH_SCENARIOS} -j ${threads} \ + --hash --hash_batches 4 --json ${json} || HASH_RESULT="FAIL" + done + python video_reader_bench.py --compare ${JSON_HASH_LOW} ${JSON_HASH_HIGH} --ignore threads \ + || HASH_RESULT="FAIL" + python video_reader_bench.py --compare ${JSON_HASH_HIGH} ${JSON_HASH_HIGH} \ + --ignore reader_depth || HASH_RESULT="FAIL" + + CHECK_RESULT="OK" + python ${topdir}/qa/TL1_video_reader_perf/check_results.py ${JSON_PERF} --tolerance 10 \ + --min lanes=${MIN_LANES} --min single_stream=${MIN_SINGLE_STREAM} \ + --min cpu=${MIN_CPU} --min engines=${MIN_ENGINES} || CHECK_RESULT="FAIL" + + echo "PERF_RUN=${PERF_RUN}" + echo "HASH_RESULT=${HASH_RESULT}" + echo "CHECK_RESULT=${CHECK_RESULT}" + + if [[ "$PERF_RUN" == "OK" && "$HASH_RESULT" == "OK" && "$CHECK_RESULT" == "OK" ]]; then + CLEAN_AND_EXIT 0 + fi + + # On failure: re-run the performance scenarios once with nsys and save the profile + echo "Video reader checks failed; re-running the benchmark with nsys profiling..." + ARTIFACTS_DIR="${topdir}/core_artifacts" + mkdir -p "${ARTIFACTS_DIR}" + + NSYS_REP="enabled" run_bench "${LOG_PERF}" ${TASKSET} python video_reader_bench.py \ + --repeat 1 --min_time 3 -j ${THREADS_LOW},${THREADS_HIGH} --readers 1,4 \ + -s ${PERF_SCENARIOS} || true + + cp -f *.nsys-rep ${JSON_PERF} "${ARTIFACTS_DIR}/" 2>/dev/null || true + echo "nsys profile and results saved to ${ARTIFACTS_DIR}" + CLEAN_AND_EXIT 1 +} +pushd ../.. +source ./qa/test_template.sh +popd From 2d49b4eb2c8d42b55aa4a909414b9a6fe0404a7a Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Wed, 21 Oct 2026 15:56:05 +0200 Subject: [PATCH 05/46] Address review comments on the video reader benchmark Results and checks: - --compare fails (exit 1) when a configuration failed or is missing in either file, reporting each failed run once with the file it came from. A build that cannot decode a workload no longer passes as "all identical". - A configuration counts as failed if any repetition failed: the benchmark exits with 1, and check_results.py rejects results with `errors`. Measurement: - Excluded iterations cover both the pipeline queue (`depth`) and the reader's own queue (`reader_depth`), and --iters/--min_time apply to the remaining window. - Frame and pixel rates count the actual output shapes of the timed batches, so mixed-resolution sets are reported correctly. - The GPU monitor records other GPU processes during the whole run, so a process that starts and exits mid-run marks the result as contaminated. A transient NVML failure no longer disables a sampler, and every NVML error is recorded. Robustness and style: - --dali_extra is made absolute, so symlinks in the temporary directory are valid with relative paths. - Validation errors name the parameter and show the expected and actual values; malformed layouts get a descriptive error. - Warnings use warnings.warn; files are always closed; no empty except. - The cache_key docs describe alternate - same as the cost of opening another file relative to rewinding the same one, not as an isolated switch cost. Signed-off-by: Joaquin Anton Guirao --- internal_tools/video_reader_bench.py | 224 ++++++++++++++++------ qa/TL1_video_reader_perf/check_results.py | 34 ++-- 2 files changed, 184 insertions(+), 74 deletions(-) diff --git a/internal_tools/video_reader_bench.py b/internal_tools/video_reader_bench.py index bf9613dc177..9426657bd84 100644 --- a/internal_tools/video_reader_bench.py +++ b/internal_tools/video_reader_bench.py @@ -29,8 +29,11 @@ reader_depth batch 1, the reader's own prefetch_queue_depth (batches in flight), on many distinct files and on one sequential long-GOP file cache_key every sample switches between two files with the same format, either - cropped (800x600 coded as 800x608) or not (1280x720): per-switch - cost and the NVDEC decoder cache + cropped (800x600 coded as 800x608) or not (1280x720), vs. a control + that rewinds the same file to frame 0 for every sample instead: + the cost of opening another file (decoder creation or NVDEC decoder + cache reuse) relative to a flush and seek within the same file. This + is a workload difference, not an isolated switch cost switch clips of mixed resolutions, shuffled, short sequences: per-sample cost under constant file switching (random access) startup many distinct files: pipeline build time and first batch latency @@ -42,8 +45,10 @@ Methodology: * every configuration runs in a fresh subprocess (no cache carry-over), --repeat times, round-robin over configurations; the median and the spread are reported - * timed iterations run until both --iters and --min_time are reached; throughput - excludes the first `depth` iterations, which may have been prefetched off the clock + * the first `depth` + `reader_depth` timed iterations are excluded, as up to that many + batches (pipeline queue and the reader's own queue) may have been produced off the + clock; measuring continues until both --iters and --min_time are reached after them + * throughput and pixel rates count the actual output shapes of the timed batches * Pipeline.run() without cuda_stream returns host-synchronized outputs, so iteration times include GPU completion * NVML metrics are per process where possible (decoder utilization, memory). A @@ -72,6 +77,7 @@ import tempfile import threading import time +import warnings VIDEO_DIR = "db/video" @@ -125,7 +131,8 @@ def _cfg(files, layout="plain", device="gpu", batch=8, seq=32, threads=4, **kw): "alternate:N" - N lines alternating between the files, each starting at frame 0, read in order: every sample switches files "same:N" - N lines, all of the first file, each starting at frame 0: the - no-switch control for "alternate" (same seeks, no switches) + control for "alternate" (no file switches, but every sample + after the first rewinds the same decoder to frame 0) """ cfg = dict( files=files, @@ -165,7 +172,8 @@ def scenarios(threads, readers): for f, layout in (("h264_clips", "many:256"), ("h264_long", "plain")) for d in (1, 2, 4, 8) ], - # switch cost = ms_per_sample(alternate) - ms_per_sample(same), per file pair + # ms_per_sample(alternate) - ms_per_sample(same), per file pair: cost of opening the + # other file relative to rewinding the same one (see the module docstring) "cache_key": [ _cfg(f, f"{kind}:64", batch=8, seq=8, threads=t) for f in ("h264_cropped_pair", "h264_aligned_pair") @@ -192,7 +200,8 @@ def scenarios(threads, readers): def resolve_files(dali_extra, name): - root = os.path.join(dali_extra, VIDEO_DIR) + # absolute paths: they are used as symlink targets in a temporary directory + root = os.path.abspath(os.path.join(dali_extra, VIDEO_DIR)) files, missing = [], [] for pattern in FILES[name]: matches = sorted(glob.glob(os.path.join(root, pattern))) @@ -200,7 +209,10 @@ def resolve_files(dali_extra, name): missing.append(pattern) files += matches if missing: - raise FileNotFoundError(f"file set {name}: not found under {root}: {missing}") + raise FileNotFoundError( + f"File set '{name}': no files match {missing} under '{root}'. " + f"Check that --dali_extra (or DALI_EXTRA_PATH) points to a DALI_extra checkout." + ) return files @@ -214,8 +226,13 @@ def materialize(cfg, dali_extra, workdir): kind, _, n = cfg["layout"].partition(":") if kind == "plain": if cfg["readers"] != 1: - raise ValueError("layout 'plain' supports a single reader") + raise ValueError(f"layout='plain' requires readers=1; got readers={cfg['readers']}.") return [{"filenames": files}] + if kind not in ("many", "alternate", "same") or not n.isdigit() or int(n) < 1: + raise ValueError( + f"Unknown layout {cfg['layout']!r}; expected 'plain', 'many:N', 'alternate:N' " + f"or 'same:N' with a positive integer N." + ) n = int(n) seq = cfg["seq"] tag = "_".join(str(cfg[k]) for k in ("files", "layout", "seq", "readers")) @@ -234,7 +251,10 @@ def materialize(cfg, dali_extra, workdir): elif kind == "same": lines = [(files[0], i) for i in range(n)] else: - raise ValueError(f"unknown layout {cfg['layout']}") + raise ValueError( + f"Unknown layout {cfg['layout']!r}; expected 'plain', 'many:N', 'alternate:N' " + f"or 'same:N'." + ) specs = [] per_reader = len(lines) // cfg["readers"] for r in range(cfg["readers"]): @@ -285,20 +305,29 @@ def other_gpu_pids(nvml, handle): class GpuMonitor: - """Samples per-process GPU memory and collects per-process NVDEC utilization.""" + """Samples per-process GPU memory and collects per-process NVDEC utilization. + + While sampling, it also records any other process that uses the GPU, so that a + competing process that starts and exits during a run is detected. + """ def __init__(self, device_id, period=0.1): - self.error = None + self.errors = [] # NVML problems; a metric may be missing, the run is still valid self.period = period self.mem = [] self.dev_dec = [] + self.others = set() self._stop = threading.Event() self._thread = threading.Thread(target=self._run, daemon=True) try: self.nvml, self.handle = nvml_handle(device_id) except Exception as e: self.nvml = None - self.error = f"NVML unavailable: {e}" + self._error(f"NVML unavailable: {e}") + + def _error(self, msg): + if msg not in self.errors: + self.errors.append(msg) def snapshot(self): """Pre/post-run state: other processes, NVDEC utilization, clocks.""" @@ -316,20 +345,37 @@ def snapshot(self): try: # older NVML bindings may lack some queries for newer drivers res[key] = query() except Exception as e: - self.error = f"NVML {key} failed: {e}" + self._error(f"NVML {key} failed: {e}") return res def _run(self): nvml, h, pid = self.nvml, self.handle, os.getpid() + + def own_memory(): + for p in nvml.nvmlDeviceGetComputeRunningProcesses(h): + if p.pid == pid and p.usedGpuMemory is not None: + self.mem.append(p.usedGpuMemory) + + samplers = { + "own memory": own_memory, + "other processes": lambda: self.others.update(other_gpu_pids(nvml, h)), + "decoder utilization": lambda: self.dev_dec.append( + nvml.nvmlDeviceGetDecoderUtilization(h)[0] + ), + } + failures = dict.fromkeys(samplers, 0) while not self._stop.is_set(): - try: - for p in nvml.nvmlDeviceGetComputeRunningProcesses(h): - if p.pid == pid and p.usedGpuMemory is not None: - self.mem.append(p.usedGpuMemory) - self.dev_dec.append(nvml.nvmlDeviceGetDecoderUtilization(h)[0]) - except Exception as e: - self.error = f"NVML sampling failed: {e}" - return + for name, sample in list(samplers.items()): + try: + sample() + failures[name] = 0 + except Exception as e: + # e.g. a process list that changed size during the query; only give up + # on a query that keeps failing (unsupported by these bindings/driver) + failures[name] += 1 + if failures[name] >= 3: + self._error(f"NVML sampling of {name} failed: {e}") + del samplers[name] time.sleep(self.period) def start(self): @@ -347,13 +393,15 @@ def stop(self): res["proc_mem_max_mb"] = max(self.mem) / 2**20 if self.dev_dec: res["dev_nvdec_util"] = statistics.mean(self.dev_dec) - try: # per-process decoder utilization since start, where the driver supports it + try: samples = self.nvml.nvmlDeviceGetProcessUtilization(self.handle, self._t_start_us) dec = [s.decUtil for s in samples if s.pid == os.getpid()] if dec: res["proc_nvdec_util"] = statistics.mean(dec) except Exception: - pass + # per-process utilization is optional: not every driver/binding supports it, or + # there may be no samples yet; the result is still valid without it + res["proc_nvdec_util"] = None return res @@ -410,7 +458,6 @@ def tiny(): p = tiny() p.build() p.run() - del p def percentile(sorted_vals, q): @@ -442,38 +489,49 @@ def run_one(cfg, specs, args): if iters > 0: for _ in range(args.warmup): pipe.run() - frames_per_iter = cfg["readers"] * cfg["batch"] * cfg["seq"] - stamps = [] + # Up to `depth` (pipeline queue) + `reader_depth` (the reader's own queue, filled + # by its prefetch thread) batches may have been produced before the clock started. + # They are excluded, and --iters/--min_time apply to the remaining window. + skip = cfg["depth"] + cfg["reader_depth"] + stamps, frames, pixels = [], [], [] mon.start() t_start = time.perf_counter() - while len(stamps) < iters or stamps[-1] - t_start < args.min_time: - pipe.run() + while True: + out = pipe.run() stamps.append(time.perf_counter()) + # actual output sizes (mixed resolutions); shapes are host-side metadata + shapes = [s for r in range(cfg["readers"]) for s in out[2 * r].shape()] + frames.append(sum(s[0] for s in shapes)) + pixels.append(sum(s[0] * s[1] * s[2] for s in shapes)) + kept = len(stamps) - skip + if kept > 0 and kept >= iters: + t_ref = stamps[skip - 1] if skip else t_start + if stamps[-1] - t_ref >= args.min_time: + break res.update(mon.stop()) - times = sorted(b - a for a, b in zip([t_start] + stamps, stamps)) - times_ms = [t * 1e3 for t in times] - # the first `depth` batches may have been produced off the clock - skip = min(cfg["depth"], len(stamps) - 1) + t_ref = stamps[skip - 1] if skip else t_start + span = stamps[-1] - t_ref n = len(stamps) - skip - span = stamps[-1] - (stamps[skip - 1] if skip else t_start) - h, w = res["shape"][1:3] + times_ms = sorted((b - a) * 1e3 for a, b in zip([t_ref] + stamps[skip:], stamps[skip:])) res.update( - iters=len(stamps), - frames_per_s=n * frames_per_iter / span, - mpix_per_s=n * frames_per_iter * h * w / span / 1e6, + iters=n, + iters_skipped=skip, + frames_per_s=sum(frames[skip:]) / span, + mpix_per_s=sum(pixels[skip:]) / span / 1e6, ms_per_sample=span * 1e3 / (n * cfg["readers"] * cfg["batch"]), iter_p50_ms=statistics.median(times_ms), iter_p90_ms=percentile(times_ms, 90), iter_p99_ms=percentile(times_ms, 99) if len(times_ms) >= 200 else None, iter_max_ms=times_ms[-1], ) - if mon.error: - res["nvml_error"] = mon.error res["post"] = mon.snapshot() + if mon.errors: # after the last NVML query, so that every failure is recorded + res["nvml_error"] = "; ".join(mon.errors) pre, post = res["pre"], res["post"] - res["contaminated"] = bool( - pre.get("other_pids") or post.get("other_pids") or pre.get("dev_nvdec_util", 0) > 5 + res["other_pids_seen"] = sorted( + mon.others | set(pre.get("other_pids", [])) | set(post.get("other_pids", [])) ) + res["contaminated"] = bool(res["other_pids_seen"] or pre.get("dev_nvdec_util", 0) > 5) return res @@ -644,13 +702,21 @@ def provenance(args): info["dali_error"] = str(e) try: nvml, h = nvml_handle(args.device_id) - info["gpu"] = nvml.nvmlDeviceGetName(h) - info["gpu_uuid"] = nvml.nvmlDeviceGetUUID(h) - info["driver"] = nvml.nvmlSystemGetDriverVersion() - info["other_gpu_pids"] = other_gpu_pids(nvml, h) - info["nvdec_util_at_start"] = nvml.nvmlDeviceGetDecoderUtilization(h)[0] except Exception as e: info["nvml_error"] = str(e) + return info + queries = dict( + gpu=lambda: nvml.nvmlDeviceGetName(h), + gpu_uuid=lambda: nvml.nvmlDeviceGetUUID(h), + driver=lambda: nvml.nvmlSystemGetDriverVersion(), + other_gpu_pids=lambda: other_gpu_pids(nvml, h), + nvdec_util_at_start=lambda: nvml.nvmlDeviceGetDecoderUtilization(h)[0], + ) + for key, query in queries.items(): + try: # one unsupported query must not hide the others + info[key] = query() + except Exception as e: + info.setdefault("nvml_query_errors", {})[key] = str(e) return info @@ -661,24 +727,42 @@ def compare(a_path, b_path, ignore): so a configuration in A is compared against every configuration in B that differs only in those fields. The output must not depend on them. """ - a, b = (json.load(open(p)) for p in (a_path, b_path)) + with open(a_path) as fa, open(b_path) as fb: + a, b = json.load(fa), json.load(fb) skip = {"hashes", "scenario", "error", "errors", "dali_version", "repeats"} | set(ignore) def key(r): return json.dumps({k: v for k, v in r.items() if k not in skip}, sort_keys=True) def label(r): - return ", ".join(f"{k}={r[k]}" for k in sorted(r) if k not in ("hashes", "errors")) + return ", ".join(f"{k}={r[k]}" for k in sorted(r) if k not in ("hashes", "errors", "error")) b_runs = {} for r in b["results"]: b_runs.setdefault(key(r), []).append(r) + a_keys = {key(r) for r in a["results"]} bad = compared = 0 + # a configuration that failed or is missing on either side fails the comparison: a + # build that cannot decode a workload must not pass as "identical" + for r in b["results"]: + if key(r) not in a_keys: + bad += 1 + print(f"MISSING in {a_path}: {label(r)}") + failed_runs = set() # (file, index): each failed run is reported and counted once + for path, results in ((a_path, a["results"]), (b_path, b["results"])): + for i, r in enumerate(results): + if "hashes" not in r and (path, i) not in failed_runs: + failed_runs.add((path, i)) + bad += 1 + print(f"FAILED in {path}: {label(r)} ({r.get('error', 'no checksums')})") for r in a["results"]: - for other in b_runs.get(key(r), []): + matches = b_runs.get(key(r), []) + if not matches: + bad += 1 + print(f"MISSING in {b_path}: {label(r)}") + for other in matches: if "hashes" not in r or "hashes" not in other: - print(f"skip (failed run): {label(r)}") - continue + continue # reported above n = max(len(r["hashes"]), len(other["hashes"])) diff = sum(x != y for x, y in zip(r["hashes"], other["hashes"])) diff += abs(len(r["hashes"]) - len(other["hashes"])) @@ -687,10 +771,10 @@ def label(r): vs = ", ".join(f"{k}={other[k]}" for k in ignore if other.get(k) != r.get(k)) status = "OK " if diff == 0 else "DIFF" print(f"{status} {diff:4d}/{n} samples {label(r)}" + (f" vs {vs}" if vs else "")) - if compared == 0: + if compared == 0 and bad == 0: print("nothing to compare: no matching configurations") return 2 - print("all identical" if bad == 0 else f"{bad} comparison(s) differ") + print("all identical" if bad == 0 else f"{bad} comparison(s) differ, failed or are missing") return 1 if bad else 0 @@ -733,9 +817,10 @@ def parse_args(): if args.compare: return parser, args if not args.dali_extra: - parser.error("set DALI_EXTRA_PATH or pass --dali_extra") + parser.error("--dali_extra is empty; set DALI_EXTRA_PATH or pass --dali_extra.") + args.dali_extra = os.path.abspath(args.dali_extra) if args.hash and not args.json and not args._single: - parser.error("--hash needs --json to store the checksums") + parser.error("--hash requires --json to store the checksums; got no --json.") if args.quick: args.threads, args.readers, args.repeat = "1,4", "1,2", 1 args.iters, args.min_time, args.warmup = 3, 0.0, 1 @@ -756,19 +841,30 @@ def main(): names = list(all_scenarios) if args.scenarios == "all" else args.scenarios.split(",") unknown = [n for n in names if n not in all_scenarios] if unknown: - parser.error(f"unknown scenario(s) {unknown}; choose from {list(all_scenarios)}") + parser.error( + f"--scenarios got unknown scenario(s) {unknown}; expected 'all' or a subset of " + f"{list(all_scenarios)}." + ) if args.hash: args.repeat = 1 prov = provenance(args) print("# " + json.dumps(prov)) if prov.get("nvml_error"): - print(f"# WARNING: NVML unavailable ({prov['nvml_error']}); GPU checks are disabled") - busy = prov.get("other_gpu_pids") or prov.get("nvdec_util_at_start", 0) > 5 + warnings.warn(f"NVML is unavailable ({prov['nvml_error']}); GPU checks are disabled.") + busy = prov.get("other_gpu_pids") or (prov.get("nvdec_util_at_start") or 0) > 5 if busy: - print("# WARNING: the GPU is in use by other processes; numbers will be skewed") + details = [] + if prov.get("other_gpu_pids"): + details.append(f"pids {prov['other_gpu_pids']}") + if prov.get("nvdec_util_at_start") is not None: + details.append(f"NVDEC utilization {prov['nvdec_util_at_start']}%") + warnings.warn( + f"The GPU is in use by other processes ({', '.join(details)}); " + f"the results will be skewed." + ) if args.require_idle: - sys.exit("GPU busy, aborting (--require_idle)") + sys.exit("The GPU is in use by other processes and --require_idle is set; aborting.") workdir = tempfile.mkdtemp(prefix="video_reader_bench_") results, failed = [], 0 @@ -790,7 +886,8 @@ def main(): if "filenames" in s: paths += s["filenames"] else: - paths += [ln.split()[0] for ln in open(s["file_list"])] + with open(s["file_list"]) as f: + paths += [ln.split()[0] for ln in f] prewarm(os.path.realpath(p) for p in paths) reps = {i: [] for i in range(len(jobs))} rng = random.Random(args.seed) @@ -825,7 +922,8 @@ def main(): else: r = aggregate(cfg, reps[i]) print(format_row(r)) - failed += bool(r.get("error")) + # a configuration fails if any of its repetitions failed + failed += bool(r.get("error") or r.get("errors")) results.append(r) finally: shutil.rmtree(workdir, ignore_errors=True) diff --git a/qa/TL1_video_reader_perf/check_results.py b/qa/TL1_video_reader_perf/check_results.py index 4cbc9ea4fd3..a7a7196136b 100644 --- a/qa/TL1_video_reader_perf/check_results.py +++ b/qa/TL1_video_reader_perf/check_results.py @@ -35,6 +35,7 @@ import argparse import json import sys +import warnings def by(results, scenario, **match): @@ -78,8 +79,19 @@ def main(): ) args = parser.parse_args() - data = json.load(open(args.results)) - results = data["results"] + with open(args.results) as f: + results = json.load(f)["results"] + metrics = named_metrics(results) + minimums = {} + for item in args.min: + name, _, value = item.partition("=") + if name not in metrics: + parser.error(f"--min got unknown metric {name!r}; expected one of {list(metrics)}.") + if value: # an empty value means: no minimum, report only + try: + minimums[name] = float(value) + except ValueError: + parser.error(f"--min {name} expects a number of frames/s; got {value!r}.") tol = args.tolerance / 100 failed = [] @@ -88,14 +100,19 @@ def check(name, ok, detail): if not ok: failed.append(name) - errors = [r for r in results if r.get("error")] + # a configuration fails if all its repetitions failed (`error`) or only some (`errors`) + errors = [r for r in results if r.get("error") or r.get("errors")] check("no failed configurations", not errors, f"{len(errors)} of {len(results)} failed") for r in errors: - print(f" {r.get('scenario')} {r.get('files')} {r.get('layout')}: {r['error']}") + for e in [r["error"]] if r.get("error") else r["errors"]: + print(f" {r.get('scenario')} {r.get('files')} {r.get('layout')}: {e}") contaminated = [r for r in results if r.get("contaminated")] if contaminated: - print(f"WARNING: {len(contaminated)} result(s) contaminated by other GPU processes") + warnings.warn( + f"{len(contaminated)} of {len(results)} results were measured while other " + f"processes used the GPU; their numbers may be skewed." + ) def relative(scenario, keys, param): for key, rows in sorted(groups(results, scenario, keys).items(), key=str): @@ -117,12 +134,7 @@ def relative(scenario, keys, param): relative("cpu", ("files", "layout", "batch"), "threads") relative("reader_depth", ("files", "layout"), "reader_depth") - minimums = {} - for item in args.min: - name, _, value = item.partition("=") - if value: - minimums[name] = float(value) - for name, value in named_metrics(results).items(): + for name, value in metrics.items(): if name in minimums: check( f"{name} minimum", From 5d61dc8042efedd112bd660b42556c7e729aa2c9 Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Wed, 21 Oct 2026 16:01:21 +0200 Subject: [PATCH 06/46] Explain the NVDEC decoder cache reuse condition Move the check whether a cached decoder can be reused into NVDECCache::IsReusable and document each criterion, in particular why both the coded size and the display area have to match. No functional change. Signed-off-by: Joaquin Anton Guirao --- dali/operators/video/frames_decoder_gpu.cc | 42 ++++++++++++++-------- 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/dali/operators/video/frames_decoder_gpu.cc b/dali/operators/video/frames_decoder_gpu.cc index a7eb29ec4cc..aef4d900675 100644 --- a/dali/operators/video/frames_decoder_gpu.cc +++ b/dali/operators/video/frames_decoder_gpu.cc @@ -122,19 +122,39 @@ class NVDECCache { } } + /** + * @brief Checks if a cached decoder can be reused for a stream with the given format. + * + * It can if it is not in use and it was created for exactly the same stream parameters + * (the codec is already matched by the cache key): + * - the coded size: the size of the decode surfaces (ulWidth/ulHeight of the decoder), + * - the display area, i.e. the size and the origin of the output that the decoder crops + * the coded frame to (ulTargetWidth/ulTargetHeight and display_area of the decoder). + * It differs from the coded size when the stream is cropped, e.g. 1080p H.264 is coded + * as 1920x1088 and displayed as 1920x1080, so both sizes have to be compared, + * - the number of decode surfaces, the chroma format and the bit depth. + */ + static bool IsReusable(const DecInstance &dec, const CUVIDEOFORMAT &format, + unsigned num_decode_surfaces) { + const auto &area = format.display_area; + return !dec.used && + dec.coded_width == format.coded_width && + dec.coded_height == format.coded_height && + dec.width == static_cast(area.right - area.left) && + dec.height == static_cast(area.bottom - area.top) && + dec.display_left == area.left && + dec.display_top == area.top && + dec.num_decode_surfaces == num_decode_surfaces && + dec.chroma_format == format.chroma_format && + dec.bit_depth_luma_minus8 == format.bit_depth_luma_minus8; + } + NVDECLease GetDecoder(CUVIDEOFORMAT *video_format, int device_id) { std::unique_lock lock(access_lock); auto codec_type = video_format->codec; unsigned height = video_format->coded_height; unsigned width = video_format->coded_width; - // The decoder is created with the display area as the output size (see ulTargetHeight - // and ulTargetWidth below), which differs from the coded size when the stream is cropped, - // e.g. 1080p H.264 is coded as 1920x1088. A cached decoder can be reused only if it matches - // both. - const auto &display_area = video_format->display_area; - unsigned display_height = display_area.bottom - display_area.top; - unsigned display_width = display_area.right - display_area.left; auto num_decode_surfaces = video_format->min_num_decode_surfaces; auto chroma_format = video_format->chroma_format; auto bit_depth_luma_minus8 = video_format->bit_depth_luma_minus8; @@ -147,13 +167,7 @@ class NVDECCache { if (best_match == range.second && it->second.used == false) { best_match = it; } - if (it->second.used == false && it->second.coded_height == height && - it->second.coded_width == width && it->second.height == display_height && - it->second.width == display_width && it->second.display_left == display_area.left && - it->second.display_top == display_area.top && - it->second.num_decode_surfaces == num_decode_surfaces && - it->second.chroma_format == chroma_format && - it->second.bit_depth_luma_minus8 == bit_depth_luma_minus8) { + if (IsReusable(it->second, *video_format, num_decode_surfaces)) { it->second.used = true; assert(it->second.device_id == device_id); return NVDECLease(this, &it->second); From 9f418178bea229f80d887d15ba622c9c1659ec4e Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Wed, 21 Oct 2026 17:02:29 +0200 Subject: [PATCH 07/46] Handle bytes from older nvidia-ml-py in the video reader benchmark nvidia-ml-py 11.450.51, which the TL1 tests install, returns bytes from nvmlDeviceGetName and friends and expects bytes in nvmlDeviceGetHandleByUUID. Signed-off-by: Joaquin Anton Guirao --- internal_tools/video_reader_bench.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/internal_tools/video_reader_bench.py b/internal_tools/video_reader_bench.py index 9426657bd84..d59ab50af31 100644 --- a/internal_tools/video_reader_bench.py +++ b/internal_tools/video_reader_bench.py @@ -294,7 +294,8 @@ def nvml_handle(device_id): entry = visible.split(",")[device_id].strip() if entry.isdigit(): return pynvml, pynvml.nvmlDeviceGetHandleByIndex(int(entry)) - return pynvml, pynvml.nvmlDeviceGetHandleByUUID(entry) + # older nvidia-ml-py versions take and return bytes, newer ones accept both + return pynvml, pynvml.nvmlDeviceGetHandleByUUID(entry.encode()) return pynvml, pynvml.nvmlDeviceGetHandleByIndex(device_id) @@ -714,7 +715,8 @@ def provenance(args): ) for key, query in queries.items(): try: # one unsupported query must not hide the others - info[key] = query() + value = query() + info[key] = value.decode() if isinstance(value, bytes) else value except Exception as e: info.setdefault("nvml_query_errors", {})[key] = str(e) return info From 0316837a77d60da44b6ac935dad422181932f3ac Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Thu, 22 Oct 2026 09:40:14 +0200 Subject: [PATCH 08/46] Compare the video reader with PyNvVideoCodec and torchcodec video_reader_bench.py gets --backends: besides the DALI reader, PyNvVideoCodec (SimpleDecoder) and torchcodec (FFmpeg CUDA backend, its own NVDEC backend, and CPU) decode the same samples through their own APIs, one decoder per thread of a thread pool. They run the scenarios that mean the same for them. The default stays dali only. TL1_video_reader_perf installs torch, FFmpeg, PyNvVideoCodec and a torchcodec matching the torch version on x86_64, and runs the comparison once there. The checks still apply to the DALI results only. Signed-off-by: Joaquin Anton Guirao --- internal_tools/video_reader_bench.py | 254 +++++++++++++++++++--- qa/TL1_video_reader_perf/check_results.py | 5 +- qa/TL1_video_reader_perf/test.sh | 61 +++++- 3 files changed, 287 insertions(+), 33 deletions(-) diff --git a/internal_tools/video_reader_bench.py b/internal_tools/video_reader_bench.py index d59ab50af31..d8bf5d18a37 100644 --- a/internal_tools/video_reader_bench.py +++ b/internal_tools/video_reader_bench.py @@ -61,6 +61,18 @@ --hash records a checksum of every sample (frames, label, frame_num). Compare two --hash result files (e.g. two builds, or --ignore threads to check that the output does not depend on num_threads) with --compare. + +Other libraries (--backends), for reference: + pynvvideocodec PyNvVideoCodec SimpleDecoder (NVDEC), RGB interleaved output (as DALI) + torchcodec torchcodec VideoDecoder, FFmpeg CUDA backend on GPU, FFmpeg on CPU + torchcodec_beta torchcodec VideoDecoder with its own NVDEC backend (set_cuda_backend) + They decode the same samples as the DALI reader (the same sequences of the same files, + in the same order, or shuffled with a fixed seed), each sample with frames [start, end) + of one file. A batch is decoded by a pool of `threads` threads, each with its own + decoder, kept while consecutive samples come from the same file (PyNvVideoCodec + reconfigures its decoder for a new file, torchcodec opens a new one). Batches are not + prefetched: unlike DALI, a batch is decoded when it is requested. They run only the + scenarios that mean the same for them (BACKEND_SCENARIOS), and need torch. """ import argparse @@ -135,6 +147,7 @@ def _cfg(files, layout="plain", device="gpu", batch=8, seq=32, threads=4, **kw): after the first rewinds the same decoder to frame 0) """ cfg = dict( + backend="dali", files=files, layout=layout, device=device, @@ -195,6 +208,29 @@ def scenarios(threads, readers): } +# scenarios that other backends run as well (reader_depth, startup and engines measure +# properties of the DALI reader), and the devices they decode on +_REFERENCE_SCENARIOS = { + "lanes", + "seq_longgop", + "random_access", + "single_stream", + "cache_key", + "switch", +} +BACKEND_SCENARIOS = { + "dali": (None, ("gpu", "cpu")), # None: all scenarios + "pynvvideocodec": (_REFERENCE_SCENARIOS, ("gpu",)), + "torchcodec": (_REFERENCE_SCENARIOS | {"cpu"}, ("gpu", "cpu")), + "torchcodec_beta": (_REFERENCE_SCENARIOS, ("gpu",)), +} + + +def backend_runs(backend, scenario, device): + names, devices = BACKEND_SCENARIOS[backend] + return (names is None or scenario in names) and device in devices + + # ---------------------------------------------------------------------------------------- # data @@ -461,6 +497,150 @@ def tiny(): p.run() +class DaliRunner: + """The DALI reader. run() returns the (frames, height, width) of every sample.""" + + def __init__(self, cfg, specs, args): + import nvidia.dali as dali + + self.cfg = cfg + self.version = dali.__version__ + # up to `depth` (pipeline queue) + `reader_depth` (the reader's own queue) batches + # may be produced before the clock starts + self.prefetched = cfg["depth"] + cfg["reader_depth"] + warm_up_cuda(args.device_id) + self.pipe = build_pipeline(cfg, specs, args.device_id, args.seed) + + def build(self): + self.pipe.build() + + def run(self): + out = self.pipe.run() + # shapes are host-side metadata; video outputs are every other output (video, label) + return [tuple(s[:3]) for r in range(self.cfg["readers"]) for s in out[2 * r].shape()] + + +def sample_list(specs, seq, shuffle, num_frames): + """The samples the DALI reader produces for `specs`: (path, start, end) tuples. + + Whole files are split into full sequences of `seq` frames (the reader's default step + and stride); a file list gives one sample per line. DALI shuffles with its own seed, so + a shuffled list has the same samples in a different order. + """ + (spec,) = specs + samples = [] + if "filenames" in spec: + for path in spec["filenames"]: + n = num_frames(path) + samples += [(path, s, s + seq) for s in range(0, n - seq + 1, seq)] + else: + with open(spec["file_list"]) as f: + for line in f: + path, _, start, end = line.split() + samples.append((path, int(start), int(end))) + if shuffle: + random.Random(0).shuffle(samples) + return samples + + +class ReferenceRunner: + """PyNvVideoCodec or torchcodec, called directly through their own APIs. + + Each sample of a batch is decoded by one of `threads` threads; every thread keeps its + own decoder while consecutive samples come from the same file. Outputs are what the + libraries return; torch is used only to wait for the GPU. + """ + + prefetched = 0 # a batch is decoded when it is requested + + def __init__(self, cfg, specs, args): + import itertools + from concurrent.futures import ThreadPoolExecutor + + import torch + + self.cfg, self.torch = cfg, torch + self.backend, self.gpu = cfg["backend"], cfg["device"] == "gpu" + self.device_id = args.device_id + if self.gpu: + torch.cuda.set_device(self.device_id) + torch.zeros(1, device="cuda") # CUDA context, so build_s measures the decoders + torch.cuda.synchronize() + if self.backend == "pynvvideocodec": + import PyNvVideoCodec as nvc + + self.nvc, self.version = nvc, f"PyNvVideoCodec {nvc.__version__}" + else: + import torchcodec + + self.version = f"torchcodec {torchcodec.__version__}" + self.version += f", torch {torch.__version__}" + self.local = threading.local() + samples = sample_list(specs, cfg["seq"], cfg["shuffle"], self._num_frames) + self.samples = itertools.cycle(samples) + self.pool = ThreadPoolExecutor(cfg["threads"]) if cfg["threads"] > 1 else None + + def _open(self, path): + if self.backend == "pynvvideocodec": + return self.nvc.SimpleDecoder( + path, + gpu_id=self.device_id, + use_device_memory=True, + output_color_type=self.nvc.OutputColorType.RGB, # interleaved, as DALI + ) + from torchcodec.decoders import VideoDecoder + + if not self.gpu: + return VideoDecoder(path, device="cpu") + if self.backend == "torchcodec": # the default CUDA backend, FFmpeg's NVDEC support + return VideoDecoder(path, device=f"cuda:{self.device_id}") + from torchcodec.decoders import set_cuda_backend # torchcodec >= 0.8 + + with set_cuda_backend("beta"): + return VideoDecoder(path, device=f"cuda:{self.device_id}") + + def _num_frames(self, path): + return len(self._decoder(path)) + + def _decoder(self, path): + """This thread's decoder, switched to `path`.""" + local = self.local + if getattr(local, "path", None) == path: + return local.decoder + if self.gpu: + self.torch.cuda.set_device(self.device_id) # this thread's current device + if self.backend == "pynvvideocodec" and getattr(local, "decoder", None) is not None: + local.decoder.reconfigure_decoder(path) # picks a cached NVDEC decoder if any + else: + local.decoder = self._open(path) + local.path = path + return local.decoder + + def _decode(self, sample): + path, start, end = sample + decoder = self._decoder(path) + # the libraries' own outputs, not converted or copied + if self.backend == "pynvvideocodec": + frames = decoder.get_batch_frames_by_index(list(range(start, end))) + return (len(frames),) + tuple(frames[0].shape[:2]) # frames of H, W, C + video = decoder.get_frames_in_range(start, end).data + return (video.shape[0], video.shape[2], video.shape[3]) # F, C, H, W + + def build(self): + pass + + def run(self): + batch = [next(self.samples) for _ in range(self.cfg["batch"])] + shapes = list(self.pool.map(self._decode, batch) if self.pool else map(self._decode, batch)) + if self.gpu: + self.torch.cuda.synchronize() # as Pipeline.run(): outputs are ready + return shapes + + +def make_runner(cfg, specs, args): + return (DaliRunner if cfg["backend"] == "dali" else ReferenceRunner)(cfg, specs, args) + + def percentile(sorted_vals, q): if len(sorted_vals) == 1: return sorted_vals[0] @@ -469,39 +649,33 @@ def percentile(sorted_vals, q): def run_one(cfg, specs, args): """Runs one configuration in this process and returns its metrics.""" - import nvidia.dali as dali - mon = GpuMonitor(args.device_id) - res = dict(dali_version=dali.__version__) - res["pre"] = mon.snapshot() - warm_up_cuda(args.device_id) + res = dict(pre=mon.snapshot()) - t0 = time.perf_counter() - pipe = build_pipeline(cfg, specs, args.device_id, args.seed) - pipe.build() + runner = make_runner(cfg, specs, args) # initializes CUDA before the clock starts + res["backend_version"] = runner.version + t0 = time.perf_counter() # build_s: the reader or the decoders, not the CUDA runtime + runner.build() res["build_s"] = time.perf_counter() - t0 t0 = time.perf_counter() - pipe.run() + runner.run() res["first_batch_ms"] = (time.perf_counter() - t0) * 1e3 - out = pipe.run() - res["shape"] = list(out[0].shape()[0]) + res["shape"] = list(runner.run()[0]) iters = cfg.get("iters", args.iters) if iters > 0: for _ in range(args.warmup): - pipe.run() - # Up to `depth` (pipeline queue) + `reader_depth` (the reader's own queue, filled - # by its prefetch thread) batches may have been produced before the clock started. - # They are excluded, and --iters/--min_time apply to the remaining window. - skip = cfg["depth"] + cfg["reader_depth"] + runner.run() + # Up to runner.prefetched batches (DALI: `depth` in the pipeline queue + + # `reader_depth` in the reader's own queue) may have been produced before the clock + # started. They are excluded, and --iters/--min_time apply to the remaining window. + skip = runner.prefetched stamps, frames, pixels = [], [], [] mon.start() t_start = time.perf_counter() while True: - out = pipe.run() + shapes = runner.run() # actual (frames, height, width) of every sample stamps.append(time.perf_counter()) - # actual output sizes (mixed resolutions); shapes are host-side metadata - shapes = [s for r in range(cfg["readers"]) for s in out[2 * r].shape()] frames.append(sum(s[0] for s in shapes)) pixels.append(sum(s[0] * s[1] * s[2] for s in shapes)) kept = len(stamps) - skip @@ -618,7 +792,7 @@ def aggregate(cfg, reps): if key in ("frames_per_s", "build_s", "first_batch_ms") and len(vals) > 1: res[key + "_spread_pct"] = (max(vals) - min(vals)) / res[key] * 100 res["shape"] = ok[0].get("shape") - for key in ("dali_version", "nvml_error"): + for key in ("backend_version", "nvml_error"): if ok[0].get(key): res[key] = ok[0][key] return res @@ -630,6 +804,7 @@ def aggregate(cfg, reps): COLUMNS = [ ("scenario", "scenario", "{:<13}"), + ("backend", "backend", "{:<15}"), ("files", "files", "{:<17}"), ("layout", "layout", "{:<12}"), ("device", "dev", "{:<3}"), @@ -701,6 +876,13 @@ def provenance(args): info["dali_path"] = os.path.dirname(dali.__file__) except Exception as e: info["dali_error"] = str(e) + from importlib import metadata + + for pkg in ("PyNvVideoCodec", "torchcodec", "torch"): + try: + info.setdefault("packages", {})[pkg] = metadata.version(pkg) + except metadata.PackageNotFoundError: + pass try: nvml, h = nvml_handle(args.device_id) except Exception as e: @@ -731,7 +913,7 @@ def compare(a_path, b_path, ignore): """ with open(a_path) as fa, open(b_path) as fb: a, b = json.load(fa), json.load(fb) - skip = {"hashes", "scenario", "error", "errors", "dali_version", "repeats"} | set(ignore) + skip = {"hashes", "scenario", "error", "errors", "backend_version", "repeats"} | set(ignore) def key(r): return json.dumps({k: v for k, v in r.items() if k not in skip}, sort_keys=True) @@ -794,6 +976,12 @@ def parse_args(): "-j", "--threads", default="1,2,4,8,16", help="num_threads sweep (B200: 1,2,4,7,8,14,16)" ) parser.add_argument("--readers", default="1,2,4,8", help="reader counts for 'engines'") + parser.add_argument( + "-b", + "--backends", + default="dali", + help=f"comma separated, any of {list(BACKEND_SCENARIOS)}; others for reference", + ) parser.add_argument("-d", "--device_id", default=0, type=int) parser.add_argument("--repeat", default=3, type=int, help="fresh-process runs per config") parser.add_argument("--iters", default=20, type=int, help="minimum timed iterations") @@ -823,6 +1011,15 @@ def parse_args(): args.dali_extra = os.path.abspath(args.dali_extra) if args.hash and not args.json and not args._single: parser.error("--hash requires --json to store the checksums; got no --json.") + args.backends = args.backends.split(",") + unknown = [b for b in args.backends if b not in BACKEND_SCENARIOS] + if unknown: + parser.error( + f"--backends got unknown backend(s) {unknown}; expected a subset of " + f"{list(BACKEND_SCENARIOS)}." + ) + if args.hash and args.backends != ["dali"]: + parser.error(f"--hash supports only the dali backend; got --backends {args.backends}.") if args.quick: args.threads, args.readers, args.repeat = "1,4", "1,2", 1 args.iters, args.min_time, args.warmup = 3, 0.0, 1 @@ -871,16 +1068,21 @@ def main(): workdir = tempfile.mkdtemp(prefix="video_reader_bench_") results, failed = [], 0 try: - jobs = [] + jobs, not_run = [], set() for name in names: - for cfg in all_scenarios[name]: - cfg = dict(cfg, scenario=name) + for base, backend in [(c, b) for c in all_scenarios[name] for b in args.backends]: + if not backend_runs(backend, name, base["device"]): + not_run.add((backend, name)) + continue + cfg = dict(base, scenario=name, backend=backend) try: specs = materialize(cfg, args.dali_extra, workdir) except Exception as e: jobs.append((cfg, None, f"{type(e).__name__}: {e}")) continue jobs.append((cfg, specs, None)) + for backend, name in sorted(not_run): + print(f"# not run: {name} with {backend} (DALI reader specific or unsupported)") for cfg, specs, err in jobs: if specs: paths = [] @@ -905,8 +1107,8 @@ def main(): f"{res['frames_per_s']:.1f} frames/s" if "frames_per_s" in res else "done" ) print( - f"[run {rep + 1}/{args.repeat}] {cfg['scenario']} {cfg['files']} " - f"{cfg['layout']} bs={cfg['batch']} thr={cfg['threads']} " + f"[run {rep + 1}/{args.repeat}] {cfg['scenario']} {cfg['backend']} " + f"{cfg['files']} {cfg['layout']} bs={cfg['batch']} thr={cfg['threads']} " f"rq={cfg['reader_depth']} rd={cfg['readers']}: {note}", file=sys.stderr, flush=True, diff --git a/qa/TL1_video_reader_perf/check_results.py b/qa/TL1_video_reader_perf/check_results.py index a7a7196136b..9d6005e22e0 100644 --- a/qa/TL1_video_reader_perf/check_results.py +++ b/qa/TL1_video_reader_perf/check_results.py @@ -39,10 +39,13 @@ def by(results, scenario, **match): + """DALI results only; results of other libraries (--backends) are for reference.""" return [ r for r in results - if r.get("scenario") == scenario and all(r.get(k) == v for k, v in match.items()) + if r.get("scenario") == scenario + and r.get("backend", "dali") == "dali" + and all(r.get(k) == v for k, v in match.items()) ] diff --git a/qa/TL1_video_reader_perf/test.sh b/qa/TL1_video_reader_perf/test.sh index 183161a5af5..4c6df077936 100644 --- a/qa/TL1_video_reader_perf/test.sh +++ b/qa/TL1_video_reader_perf/test.sh @@ -1,17 +1,23 @@ #!/bin/bash -e -# used pip packages -pip_packages='numpy nvidia-ml-py==11.450.51' +# used pip packages; torch (pinned per CUDA version in qa/setup_packages.py) is needed by the +# reference libraries, see install_reference_libs +pip_packages='numpy nvidia-ml-py==11.450.51 torch' target_dir=./internal_tools -# One-time pre-step: install nsys (NVIDIA Nsight Systems) if not present +# One-time pre-step: FFmpeg shared libraries for torchcodec (x86_64 only, see +# install_reference_libs), and nsys (NVIDIA Nsight Systems) do_once() { + apt update + if [ "$(uname -m)" == "x86_64" ]; then + apt install -y --no-install-recommends ffmpeg + fi if command -v nsys &>/dev/null; then echo "nsys already installed: $(nsys --version)" return fi DISTRO=$(source /etc/lsb-release && echo "$DISTRIB_RELEASE" | tr -d .) ARCH=$(dpkg --print-architecture) - apt update && apt install -y --no-install-recommends gnupg curl + apt install -y --no-install-recommends gnupg curl echo "deb [signed-by=/usr/share/keyrings/nvidia-devtools.gpg] https://developer.download.nvidia.com/devtools/repos/ubuntu${DISTRO}/${ARCH} /" \ | tee /etc/apt/sources.list.d/nvidia-devtools.list curl -fsSL "https://developer.download.nvidia.com/devtools/repos/ubuntu${DISTRO}/${ARCH}/nvidia.pub" \ @@ -23,6 +29,34 @@ LOG_PERF="video_reader_perf.log" JSON_PERF="video_reader_perf.json" JSON_HASH_LOW="video_reader_hash_low.json" JSON_HASH_HIGH="video_reader_hash_high.json" +LOG_REF="video_reader_ref.log" +JSON_REF="video_reader_ref.json" + +# Other libraries run for reference on the same samples (video_reader_bench.py --backends), on +# x86_64 only; one run each, not checked against minimums. Installed by install_reference_libs. +REF_SCENARIOS="lanes,single_stream,cache_key,cpu" +REF_ARGS="--repeat 1 --min_time 3 -j 1,16" + +# Installs PyNvVideoCodec and torchcodec (x86_64 only) and sets REF_BACKENDS. torchcodec must +# match the torch version installed through pip_packages, and is built for the same CUDA version. +install_reference_libs() { + local torch_ver cuda_tag tc_ver + torch_ver=$(python -c "import torch; print(torch.__version__.split('+')[0])") + cuda_tag=$(python -c "import torch; print('cu' + torch.version.cuda.replace('.', ''))") + case "${torch_ver}" in + 2.7.*) tc_ver=0.5 ;; + 2.11.*) tc_ver=0.11.1 ;; + *) echo "No torchcodec version known for torch ${torch_ver}; update test.sh"; return 1 ;; + esac + pip install PyNvVideoCodec==2.0.2 || return 1 + # --no-deps: keep the torch from pip_packages + pip install --no-deps "torchcodec==${tc_ver}+${cuda_tag}" \ + --index-url "https://download.pytorch.org/whl/${cuda_tag}" || return 1 + REF_BACKENDS="pynvvideocodec,torchcodec" + # torchcodec 0.5 has no set_cuda_backend, i.e. no torchcodec_beta + [ "${tc_ver}" != "0.5" ] && REF_BACKENDS="${REF_BACKENDS},torchcodec_beta" + echo "Reference backends: ${REF_BACKENDS}" +} # Scenarios of internal_tools/video_reader_bench.py run for performance; see its --help PERF_SCENARIOS="lanes,engines,single_stream,reader_depth,cache_key,startup,seq_longgop,cpu" @@ -33,7 +67,7 @@ THREADS_HIGH=16 BENCH_ARGS="--repeat 3 --min_time 3 -j ${THREADS_LOW},4,${THREADS_HIGH} --readers 1,2,4" function CLEAN_AND_EXIT { - rm -rf ${LOG_PERF} ${JSON_PERF} ${JSON_HASH_LOW} ${JSON_HASH_HIGH} + rm -rf ${LOG_PERF} ${JSON_PERF} ${JSON_HASH_LOW} ${JSON_HASH_HIGH} ${LOG_REF} ${JSON_REF} exit $1 } @@ -90,11 +124,26 @@ test_body() { --min lanes=${MIN_LANES} --min single_stream=${MIN_SINGLE_STREAM} \ --min cpu=${MIN_CPU} --min engines=${MIN_ENGINES} || CHECK_RESULT="FAIL" + # The same samples decoded by other libraries, for reference (x86_64 only). A failure here + # (installation or decoding) fails the test, so that the comparison does not silently + # disappear. + REF_RUN="OK" + if [ "$(uname -m)" != "x86_64" ]; then + REF_RUN="SKIPPED" + elif install_reference_libs; then + run_bench "${LOG_REF}" ${TASKSET} python video_reader_bench.py ${REF_ARGS} \ + -s ${REF_SCENARIOS} -b dali,${REF_BACKENDS} --json ${JSON_REF} || REF_RUN="FAIL" + else + REF_RUN="FAIL" + fi + echo "PERF_RUN=${PERF_RUN}" echo "HASH_RESULT=${HASH_RESULT}" echo "CHECK_RESULT=${CHECK_RESULT}" + echo "REF_RUN=${REF_RUN}" - if [[ "$PERF_RUN" == "OK" && "$HASH_RESULT" == "OK" && "$CHECK_RESULT" == "OK" ]]; then + if [[ "$PERF_RUN" == "OK" && "$HASH_RESULT" == "OK" && "$CHECK_RESULT" == "OK" \ + && "$REF_RUN" != "FAIL" ]]; then CLEAN_AND_EXIT 0 fi From 4cfc6968abc0ec663e543742871a74e4a132e1c4 Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Thu, 22 Oct 2026 10:08:29 +0200 Subject: [PATCH 09/46] Flag GPU benchmark rows where NVDEC stays idle A GPU backend can fall back to CPU decoding for a format it can't decode on the GPU (e.g. torchcodec's FFmpeg CUDA backend on HEVC 10-bit). Also leave build_s empty for the reference libraries, which open decoders lazily. Signed-off-by: Joaquin Anton Guirao --- internal_tools/video_reader_bench.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/internal_tools/video_reader_bench.py b/internal_tools/video_reader_bench.py index d8bf5d18a37..102adeaa221 100644 --- a/internal_tools/video_reader_bench.py +++ b/internal_tools/video_reader_bench.py @@ -654,9 +654,10 @@ def run_one(cfg, specs, args): runner = make_runner(cfg, specs, args) # initializes CUDA before the clock starts res["backend_version"] = runner.version - t0 = time.perf_counter() # build_s: the reader or the decoders, not the CUDA runtime + t0 = time.perf_counter() # build_s: the DALI reader (indexing), not the CUDA runtime runner.build() - res["build_s"] = time.perf_counter() - t0 + if cfg["backend"] == "dali": # other libraries open their decoders lazily, in run() + res["build_s"] = time.perf_counter() - t0 t0 = time.perf_counter() runner.run() res["first_batch_ms"] = (time.perf_counter() - t0) * 1e3 @@ -851,6 +852,10 @@ def format_row(r): flags.append("CONTAMINATED") if r.get("frames_per_s_spread_pct", 0) > 3: flags.append("spread>3%") + if r.get("device") == "gpu" and r.get("proc_nvdec_util") is not None: + if r["proc_nvdec_util"] < 10: + # e.g. a GPU backend that falls back to CPU decoding for a format it can't decode + flags.append("NVDEC<10%") if r.get("errors"): flags.append(f"{len(r['errors'])} failed run(s)") if r.get("error"): From 98c33f70effb2d338dbcf8ffe14646e9cac10d54 Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Thu, 22 Oct 2026 10:19:42 +0200 Subject: [PATCH 10/46] Tighten the video reader benchmark and its TL1 test - Reference libraries: min(threads, batch) decoder threads, cache_key at one thread, and no PyNvVideoCodec runs that seek mid-file (its SimpleDecoder raises IndexError there). Flag PyNvVideoCodec rows on clips whose lead-in frames are hidden by an MP4 edit list: it returns different frames. - --hash checks at least 40 samples, so that the reader's prefetch queue wraps; hash seq_longgop and random_access (with the HEVC trailer) too. - TL1: one hash run and compare, lanes at batch 16 only, --iters 8 (engines 20), a short reference run without DALI rows, no startup scenario, and random_access in the performance run. Check seq_longgop and random_access over threads and engines scaling. Re-run under nsys only when the performance checks fail, save all results on failure and print them to the log on success. - Remove dead code. Signed-off-by: Joaquin Anton Guirao --- internal_tools/video_reader_bench.py | 119 +++++++++++++--------- qa/TL1_video_reader_perf/check_results.py | 32 +++++- qa/TL1_video_reader_perf/test.sh | 73 +++++++------ 3 files changed, 139 insertions(+), 85 deletions(-) diff --git a/internal_tools/video_reader_bench.py b/internal_tools/video_reader_bench.py index 102adeaa221..cc7e8a9f9ff 100644 --- a/internal_tools/video_reader_bench.py +++ b/internal_tools/video_reader_bench.py @@ -18,11 +18,15 @@ lanes many distinct files, one keyframe-aligned sequence each, so every batch holds `batch` different files and no sample needs a seek: - decode parallelism across samples vs. num_threads + per-file cost (opening, indexing, decoder setup, decoding the lead-in + frames an edit list hides, see LEAD_IN_FILES) plus decoding, vs. + num_threads. A per-file speedup also shows here; check `cache_key` + and `engines` before attributing a gain to decode parallelism seq_longgop 15 H.264 clips read sequentially (long GOPs, consecutive sequences of the same file): no-regression check for sequential access - random_access the same clips shuffled: parallelism and keyframe-seek waste - together (do not read a lanes speedup from this one alone) + random_access the same clips, and the HEVC trailer (open GOPs), shuffled: + parallelism and keyframe-seek waste together (do not read a lanes + speedup from this one alone) single_stream one clip, batch 1: per-stream throughput, i.e. per-frame serialization. Includes a 512x512 clip, where per-frame overheads matter more than decode time @@ -54,11 +58,12 @@ * NVML metrics are per process where possible (decoder utilization, memory). A result is flagged as contaminated if other processes use the GPU or NVDEC is busy before the run starts - * files are read once before each scenario, so all runs see a warm page cache + * all input files are read once before the first run, so all runs see a warm page cache * DALI_extra has no 1080p content, so absolute rates are lower than for 1080p data; compare builds, not datasets ---hash records a checksum of every sample (frames, label, frame_num). Compare two +--hash records a checksum of every sample (frames, label, frame_num) of at least +HASH_MIN_SAMPLES samples, so that the reader's prefetch queue wraps around. Compare two --hash result files (e.g. two builds, or --ignore threads to check that the output does not depend on num_threads) with --compare. @@ -68,11 +73,14 @@ torchcodec_beta torchcodec VideoDecoder with its own NVDEC backend (set_cuda_backend) They decode the same samples as the DALI reader (the same sequences of the same files, in the same order, or shuffled with a fixed seed), each sample with frames [start, end) - of one file. A batch is decoded by a pool of `threads` threads, each with its own - decoder, kept while consecutive samples come from the same file (PyNvVideoCodec - reconfigures its decoder for a new file, torchcodec opens a new one). Batches are not - prefetched: unlike DALI, a batch is decoded when it is requested. They run only the - scenarios that mean the same for them (BACKEND_SCENARIOS), and need torch. + of one file. A batch is decoded by a pool of min(threads, batch) threads, each with its + own decoder, kept while consecutive samples come from the same file (PyNvVideoCodec + reconfigures its decoder for a new file, torchcodec opens a new one, scanning it, as + its default seek_mode="exact" does). Batches are not prefetched: unlike DALI, a batch + is decoded when it is requested. They run only the configurations that mean the same + for them (backend_runs), and need torch. GPU rows where NVDEC stays nearly idle are + flagged, as a GPU backend may fall back to CPU decoding, and so are PyNvVideoCodec rows on + clips with hidden lead-in frames (LEAD_IN_FILES), where it returns different frames. """ import argparse @@ -93,6 +101,11 @@ VIDEO_DIR = "db/video" +# --hash checks at least this many samples per configuration: twice the batches the +# pipeline and the reader can hold (depth 2 + reader_depth up to 8) at batch 1, so that +# every slot of the reader's prefetch queue is reused +HASH_MIN_SAMPLES = 40 + H264_CLIPS = ["sintel/video_files/*.mp4", "sintel/labelled_videos/*/*.mp4"] VPX_CLIPS = [ "vp9/vp9_0.mp4", @@ -132,6 +145,11 @@ # VP8/VP9 clips (H.264 is not enabled on the CPU backend) "vpx_clips": VPX_CLIPS, } +# File sets where most clips (cut without re-encoding) start before the frame they show first, +# with an MP4 edit list hiding the lead-in frames: DALI and torchcodec start at the first +# visible frame, so they decode the lead-in before frame 0, while PyNvVideoCodec returns the +# lead-in frames themselves (different frames, less decoding) +LEAD_IN_FILES = {"h264_clips", "h264_mixed"} def _cfg(files, layout="plain", device="gpu", batch=8, seq=32, threads=4, **kw): @@ -163,18 +181,17 @@ def _cfg(files, layout="plain", device="gpu", batch=8, seq=32, threads=4, **kw): return cfg -def scenarios(threads, readers): +def scenarios(threads, readers, quick=False): t_max = max(threads) t_pair = sorted({1, t_max}) + iters = {} if quick else dict(iters=20) return { - "lanes": [ - _cfg("h264_clips", "many:256", batch=b, seq=32, threads=t) - for b in (8, 16) - for t in threads - ], + "lanes": [_cfg("h264_clips", "many:256", batch=16, seq=32, threads=t) for t in threads], "seq_longgop": [_cfg("h264_clips", batch=16, seq=32, threads=t) for t in t_pair], "random_access": [ - _cfg("h264_clips", batch=16, seq=32, threads=t, shuffle=True) for t in t_pair + _cfg(f, batch=16, seq=32, threads=t, shuffle=True) + for f in ("h264_clips", "hevc_long") + for t in t_pair ], "single_stream": [ _cfg(f, batch=1, seq=64, threads=4) @@ -186,19 +203,21 @@ def scenarios(threads, readers): for d in (1, 2, 4, 8) ], # ms_per_sample(alternate) - ms_per_sample(same), per file pair: cost of opening the - # other file relative to rewinding the same one (see the module docstring) + # other file relative to rewinding the same one (see the module docstring). One + # thread: with more, other libraries keep each file of a pair on its own decoder "cache_key": [ - _cfg(f, f"{kind}:64", batch=8, seq=8, threads=t) + _cfg(f, f"{kind}:64", batch=8, seq=8, threads=1) for f in ("h264_cropped_pair", "h264_aligned_pair") for kind in ("alternate", "same") - for t in t_pair ], "switch": [_cfg("h264_mixed", batch=8, seq=8, threads=t, shuffle=True) for t in t_pair], "startup": [ _cfg("h264_clips", "many:256", batch=1, seq=8, threads=t, iters=0) for t in t_pair ], + # the noisiest scenario: more iterations than the --iters default "engines": [ - _cfg("h264_clips", "many:64", batch=4, seq=32, threads=4, readers=k) for k in readers + _cfg("h264_clips", "many:64", batch=4, seq=32, threads=4, readers=k, **iters) + for k in readers ], "cpu": [ _cfg("vpx_clips", "many:128", device="cpu", batch=b, seq=16, threads=t) @@ -226,9 +245,17 @@ def scenarios(threads, readers): } -def backend_runs(backend, scenario, device): +def backend_runs(backend, scenario, cfg): names, devices = BACKEND_SCENARIOS[backend] - return (names is None or scenario in names) and device in devices + if (names is not None and scenario not in names) or cfg["device"] not in devices: + return False + # PyNvVideoCodec 2.0 SimpleDecoder fails (IndexError) when a decoder that was just + # created or reconfigured for a file is asked for frames from the middle of it, so it + # runs only configurations whose decoders start every file at frame 0 + seeks = scenario in ("random_access", "switch") or ( + scenario == "seq_longgop" and min(cfg["threads"], cfg["batch"]) > 1 + ) + return not (backend == "pynvvideocodec" and seeks) # ---------------------------------------------------------------------------------------- @@ -284,13 +311,8 @@ def materialize(cfg, dali_extra, workdir): lines.append((dst, i)) elif kind == "alternate": lines = [(files[i % len(files)], i) for i in range(n)] - elif kind == "same": + else: # same lines = [(files[0], i) for i in range(n)] - else: - raise ValueError( - f"Unknown layout {cfg['layout']!r}; expected 'plain', 'many:N', 'alternate:N' " - f"or 'same:N'." - ) specs = [] per_reader = len(lines) // cfg["readers"] for r in range(cfg["readers"]): @@ -367,7 +389,7 @@ def _error(self, msg): self.errors.append(msg) def snapshot(self): - """Pre/post-run state: other processes, NVDEC utilization, clocks.""" + """Pre/post-run state: other processes, NVDEC utilization.""" if not self.nvml: return {} nvml, h = self.nvml, self.handle @@ -375,8 +397,6 @@ def snapshot(self): queries = dict( other_pids=lambda: other_gpu_pids(nvml, h), dev_nvdec_util=lambda: nvml.nvmlDeviceGetDecoderUtilization(h)[0], - clock_video_mhz=lambda: nvml.nvmlDeviceGetClockInfo(h, nvml.NVML_CLOCK_VIDEO), - clock_sm_mhz=lambda: nvml.nvmlDeviceGetClockInfo(h, nvml.NVML_CLOCK_SM), ) for key, query in queries.items(): try: # older NVML bindings may lack some queries for newer drivers @@ -466,8 +486,6 @@ def pipe(): else: kw["file_list_format"] = "frames" kw["file_list_include_end"] = False # explicit: the default changed over time - if "initial_fill" in cfg: - kw["initial_fill"] = cfg["initial_fill"] if frame_num: kw["enable_frame_num"] = True outs += fn.experimental.readers.video( @@ -578,7 +596,10 @@ def __init__(self, cfg, specs, args): self.local = threading.local() samples = sample_list(specs, cfg["seq"], cfg["shuffle"], self._num_frames) self.samples = itertools.cycle(samples) - self.pool = ThreadPoolExecutor(cfg["threads"]) if cfg["threads"] > 1 else None + # no more threads than samples per batch: an idle extra thread would take the next + # batch's samples of the same file and make its decoder seek + workers = min(cfg["threads"], cfg["batch"]) + self.pool = ThreadPoolExecutor(workers) if workers > 1 else None def _open(self, path): if self.backend == "pynvvideocodec": @@ -685,7 +706,6 @@ def run_one(cfg, specs, args): if stamps[-1] - t_ref >= args.min_time: break res.update(mon.stop()) - t_ref = stamps[skip - 1] if skip else t_start span = stamps[-1] - t_ref n = len(stamps) - skip times_ms = sorted((b - a) * 1e3 for a, b in zip([t_ref] + stamps[skip:], stamps[skip:])) @@ -697,7 +717,6 @@ def run_one(cfg, specs, args): ms_per_sample=span * 1e3 / (n * cfg["readers"] * cfg["batch"]), iter_p50_ms=statistics.median(times_ms), iter_p90_ms=percentile(times_ms, 90), - iter_p99_ms=percentile(times_ms, 99) if len(times_ms) >= 200 else None, iter_max_ms=times_ms[-1], ) res["post"] = mon.snapshot() @@ -712,11 +731,13 @@ def run_one(cfg, specs, args): def hash_one(cfg, specs, args): - """Returns one checksum per sample for the first --hash_batches batches.""" + """Returns one checksum per sample of the first batches (see HASH_MIN_SAMPLES).""" pipe = build_pipeline(cfg, specs, args.device_id, args.seed, frame_num=True) pipe.build() hashes = [] - for _ in range(args.hash_batches): + # Not dependent on threads or reader_depth, so that --compare --ignore can match runs + batches = max(args.hash_batches, -(-HASH_MIN_SAMPLES // (cfg["batch"] * cfg["readers"]))) + for _ in range(batches): outs = [_as_cpu(t) for t in pipe.run()] for r in range(cfg["readers"]): video, label, frame_num = outs[3 * r : 3 * r + 3] @@ -725,7 +746,7 @@ def hash_one(cfg, specs, args): h.update(label.at(i).tobytes()) h.update(frame_num.at(i).tobytes()) hashes.append(h.hexdigest()) - return dict(hashes=hashes) + return dict(hashes=hashes, hash_batches=batches) def _as_cpu(tensor_list): @@ -779,7 +800,6 @@ def aggregate(cfg, reps): "ms_per_sample", "iter_p50_ms", "iter_p90_ms", - "iter_p99_ms", "iter_max_ms", "build_s", "first_batch_ms", @@ -856,6 +876,8 @@ def format_row(r): if r["proc_nvdec_util"] < 10: # e.g. a GPU backend that falls back to CPU decoding for a format it can't decode flags.append("NVDEC<10%") + if r.get("backend") == "pynvvideocodec" and r.get("files") in LEAD_IN_FILES: + flags.append("lead-in frames") # not the same frames as DALI, see LEAD_IN_FILES if r.get("errors"): flags.append(f"{len(r['errors'])} failed run(s)") if r.get("error"): @@ -870,7 +892,6 @@ def provenance(args): python=sys.version.split()[0], cpu_count=os.cpu_count(), argv=sys.argv, - isolation="subprocess per run", env={k: v for k, v in os.environ.items() if k.startswith("DALI_") or k.startswith("CUDA_")}, ) try: @@ -918,7 +939,7 @@ def compare(a_path, b_path, ignore): """ with open(a_path) as fa, open(b_path) as fb: a, b = json.load(fa), json.load(fb) - skip = {"hashes", "scenario", "error", "errors", "backend_version", "repeats"} | set(ignore) + skip = {"hashes", "scenario", "error", "errors"} | set(ignore) def key(r): return json.dumps({k: v for k, v in r.items() if k not in skip}, sort_keys=True) @@ -1041,7 +1062,7 @@ def main(): threads = [int(t) for t in args.threads.split(",")] readers = [int(r) for r in args.readers.split(",")] - all_scenarios = scenarios(threads, readers) + all_scenarios = scenarios(threads, readers, args.quick) names = list(all_scenarios) if args.scenarios == "all" else args.scenarios.split(",") unknown = [n for n in names if n not in all_scenarios] if unknown: @@ -1076,8 +1097,8 @@ def main(): jobs, not_run = [], set() for name in names: for base, backend in [(c, b) for c in all_scenarios[name] for b in args.backends]: - if not backend_runs(backend, name, base["device"]): - not_run.add((backend, name)) + if not backend_runs(backend, name, base): + not_run.add((backend, name, base["threads"])) continue cfg = dict(base, scenario=name, backend=backend) try: @@ -1086,8 +1107,8 @@ def main(): jobs.append((cfg, None, f"{type(e).__name__}: {e}")) continue jobs.append((cfg, specs, None)) - for backend, name in sorted(not_run): - print(f"# not run: {name} with {backend} (DALI reader specific or unsupported)") + for backend, name, threads in sorted(not_run): + print(f"# not run: {name} threads={threads} with {backend} (see backend_runs)") for cfg, specs, err in jobs: if specs: paths = [] @@ -1122,7 +1143,7 @@ def main(): print_header() for i, (cfg, specs, err) in enumerate(jobs): if args.hash: - r = dict(cfg, seed=args.seed, hash_batches=args.hash_batches, **reps[i][0]) + r = dict(cfg, seed=args.seed, **reps[i][0]) # with the hash_batches used n = len(r.get("hashes", [])) print( f"{cfg['scenario']} {cfg['files']} threads={cfg['threads']}: " diff --git a/qa/TL1_video_reader_perf/check_results.py b/qa/TL1_video_reader_perf/check_results.py index 9d6005e22e0..8525d8f857f 100644 --- a/qa/TL1_video_reader_perf/check_results.py +++ b/qa/TL1_video_reader_perf/check_results.py @@ -17,10 +17,12 @@ Checks: * no configuration failed - * hardware-independent relative checks, with a tolerance: - - lanes, cpu: throughput at the highest num_threads is not lower than at the - lowest one (more threads must not make the reader slower) + * relative checks, with a tolerance: + - lanes, seq_longgop, random_access, cpu: throughput at any higher num_threads is + not lower than at the lowest one (more threads must not make the reader slower) - reader_depth: a deeper reader queue is not slower than depth 1 + * optional: engines with the most readers reach at least --engines_scaling times the + throughput of one reader (GPUs with several NVDEC engines; e.g. H100: 3.6x at 4) * optional absolute minimums (frames/s) for named metrics, passed with --min: lanes lanes, batch 16, highest num_threads single_stream single_stream, h264_long @@ -80,6 +82,11 @@ def main(): parser.add_argument( "--min", action="append", default=[], metavar="NAME=FPS", help="absolute minimum" ) + parser.add_argument( + "--engines_scaling", + type=float, + help="minimum speedup of engines with the most readers over one reader", + ) args = parser.parse_args() with open(args.results) as f: @@ -133,10 +140,25 @@ def relative(scenario, keys, param): f"tolerance {args.tolerance:g}%", ) - relative("lanes", ("files", "layout", "batch"), "threads") - relative("cpu", ("files", "layout", "batch"), "threads") + for scenario in ("lanes", "seq_longgop", "random_access", "cpu"): + relative(scenario, ("files", "layout", "batch"), "threads") relative("reader_depth", ("files", "layout"), "reader_depth") + if args.engines_scaling is not None: + rows = sorted( + (r for r in by(results, "engines") if r.get("frames_per_s")), key=lambda r: r["readers"] + ) + if len(rows) < 2 or rows[0]["readers"] != 1: + check("engines scaling", False, "needs engines results with 1 and more readers") + else: + one, most = rows[0], rows[-1] + scaling = most["frames_per_s"] / one["frames_per_s"] + check( + f"engines readers={most['readers']} vs 1", + scaling >= args.engines_scaling, + f"{scaling:.2f}x, minimum {args.engines_scaling:g}x", + ) + for name, value in metrics.items(): if name in minimums: check( diff --git a/qa/TL1_video_reader_perf/test.sh b/qa/TL1_video_reader_perf/test.sh index 4c6df077936..07761c296c5 100644 --- a/qa/TL1_video_reader_perf/test.sh +++ b/qa/TL1_video_reader_perf/test.sh @@ -27,15 +27,15 @@ do_once() { LOG_PERF="video_reader_perf.log" JSON_PERF="video_reader_perf.json" -JSON_HASH_LOW="video_reader_hash_low.json" -JSON_HASH_HIGH="video_reader_hash_high.json" +JSON_HASH="video_reader_hash.json" LOG_REF="video_reader_ref.log" JSON_REF="video_reader_ref.json" # Other libraries run for reference on the same samples (video_reader_bench.py --backends), on -# x86_64 only; one run each, not checked against minimums. Installed by install_reference_libs. +# x86_64 only; one short run each, not checked. The DALI numbers are in the performance run. +# Installed by install_reference_libs. REF_SCENARIOS="lanes,single_stream,cache_key,cpu" -REF_ARGS="--repeat 1 --min_time 3 -j 1,16" +REF_ARGS="--repeat 1 --iters 4 --warmup 1 --min_time 2 -j 1,16" # Installs PyNvVideoCodec and torchcodec (x86_64 only) and sets REF_BACKENDS. torchcodec must # match the torch version installed through pip_packages, and is built for the same CUDA version. @@ -59,18 +59,29 @@ install_reference_libs() { } # Scenarios of internal_tools/video_reader_bench.py run for performance; see its --help -PERF_SCENARIOS="lanes,engines,single_stream,reader_depth,cache_key,startup,seq_longgop,cpu" +PERF_SCENARIOS="lanes,engines,single_stream,reader_depth,cache_key,seq_longgop,random_access,cpu" # Scenarios whose output is checked to be independent of num_threads and reader_depth -HASH_SCENARIOS="lanes,reader_depth,cache_key,cpu" -THREADS_LOW=1 -THREADS_HIGH=16 -BENCH_ARGS="--repeat 3 --min_time 3 -j ${THREADS_LOW},4,${THREADS_HIGH} --readers 1,2,4" +HASH_SCENARIOS="lanes,reader_depth,cache_key,cpu,seq_longgop,random_access" +BENCH_ARGS="--repeat 3 --iters 8 --min_time 3 -j 1,4,16 --readers 1,2,4" +# engines: one H100 or GH200 reaches 3.6x with 4 readers (several NVDEC engines) +ENGINES_SCALING=2.5 function CLEAN_AND_EXIT { - rm -rf ${LOG_PERF} ${JSON_PERF} ${JSON_HASH_LOW} ${JSON_HASH_HIGH} ${LOG_REF} ${JSON_REF} + rm -rf ${LOG_PERF} ${JSON_PERF} ${JSON_HASH} ${LOG_REF} ${JSON_REF} exit $1 } +# Prints the results (without per-run data) to the log, so that they are kept on success too +print_results() { + for json in "$@"; do + [ -f "${json}" ] && python -c "import json, sys +d = json.load(open(sys.argv[1])) +for r in d['results']: + r.pop('runs', None) +print('RESULTS', sys.argv[1], json.dumps(d, separators=(',', ':')))" "${json}" + done +} + # Run the benchmark; if NSYS_REP is set, wrap it with nsys profiling (child processes included). run_bench() { local log_file="$1" @@ -88,7 +99,9 @@ test_body() { # Hopper TASKSET="taskset --cpu-list 0-127" # Absolute minimums in frames/s (see qa/TL1_video_reader_perf/check_results.py); - # empty: reported for reference only, not checked + # empty: reported for reference only, not checked. + # TODO: set them (e.g. 0.85x the REFERENCE values) once a few CI runs establish the + # baseline on these machines MIN_LANES= MIN_SINGLE_STREAM= MIN_CPU= @@ -107,20 +120,17 @@ test_body() { run_bench "${LOG_PERF}" ${TASKSET} python video_reader_bench.py ${BENCH_ARGS} \ -s ${PERF_SCENARIOS} --json ${JSON_PERF} || PERF_RUN="FAIL" - # The output must not depend on num_threads or on the reader's prefetch_queue_depth + # The output must not depend on num_threads or on the reader's prefetch_queue_depth: every + # configuration is compared with all that differ from it only in those HASH_RESULT="OK" - for threads in ${THREADS_LOW} ${THREADS_HIGH}; do - [ ${threads} == ${THREADS_LOW} ] && json=${JSON_HASH_LOW} || json=${JSON_HASH_HIGH} - ${TASKSET} python video_reader_bench.py -s ${HASH_SCENARIOS} -j ${threads} \ - --hash --hash_batches 4 --json ${json} || HASH_RESULT="FAIL" - done - python video_reader_bench.py --compare ${JSON_HASH_LOW} ${JSON_HASH_HIGH} --ignore threads \ - || HASH_RESULT="FAIL" - python video_reader_bench.py --compare ${JSON_HASH_HIGH} ${JSON_HASH_HIGH} \ - --ignore reader_depth || HASH_RESULT="FAIL" + ${TASKSET} python video_reader_bench.py -s ${HASH_SCENARIOS} -j 1,16 --hash --hash_batches 4 \ + --json ${JSON_HASH} || HASH_RESULT="FAIL" + python video_reader_bench.py --compare ${JSON_HASH} ${JSON_HASH} \ + --ignore threads,reader_depth || HASH_RESULT="FAIL" CHECK_RESULT="OK" python ${topdir}/qa/TL1_video_reader_perf/check_results.py ${JSON_PERF} --tolerance 10 \ + --engines_scaling ${ENGINES_SCALING} \ --min lanes=${MIN_LANES} --min single_stream=${MIN_SINGLE_STREAM} \ --min cpu=${MIN_CPU} --min engines=${MIN_ENGINES} || CHECK_RESULT="FAIL" @@ -132,7 +142,7 @@ test_body() { REF_RUN="SKIPPED" elif install_reference_libs; then run_bench "${LOG_REF}" ${TASKSET} python video_reader_bench.py ${REF_ARGS} \ - -s ${REF_SCENARIOS} -b dali,${REF_BACKENDS} --json ${JSON_REF} || REF_RUN="FAIL" + -s ${REF_SCENARIOS} -b ${REF_BACKENDS} --json ${JSON_REF} || REF_RUN="FAIL" else REF_RUN="FAIL" fi @@ -141,23 +151,24 @@ test_body() { echo "HASH_RESULT=${HASH_RESULT}" echo "CHECK_RESULT=${CHECK_RESULT}" echo "REF_RUN=${REF_RUN}" + print_results ${JSON_PERF} ${JSON_REF} if [[ "$PERF_RUN" == "OK" && "$HASH_RESULT" == "OK" && "$CHECK_RESULT" == "OK" \ && "$REF_RUN" != "FAIL" ]]; then CLEAN_AND_EXIT 0 fi - # On failure: re-run the performance scenarios once with nsys and save the profile - echo "Video reader checks failed; re-running the benchmark with nsys profiling..." + # On failure: save the results; if the performance run or its checks failed, also re-run the + # performance scenarios once with nsys and save the profile ARTIFACTS_DIR="${topdir}/core_artifacts" mkdir -p "${ARTIFACTS_DIR}" - - NSYS_REP="enabled" run_bench "${LOG_PERF}" ${TASKSET} python video_reader_bench.py \ - --repeat 1 --min_time 3 -j ${THREADS_LOW},${THREADS_HIGH} --readers 1,4 \ - -s ${PERF_SCENARIOS} || true - - cp -f *.nsys-rep ${JSON_PERF} "${ARTIFACTS_DIR}/" 2>/dev/null || true - echo "nsys profile and results saved to ${ARTIFACTS_DIR}" + if [[ "$PERF_RUN" != "OK" || "$CHECK_RESULT" != "OK" ]]; then + echo "Video reader performance checks failed; re-running with nsys profiling..." + NSYS_REP="enabled" run_bench "${LOG_PERF}" ${TASKSET} python video_reader_bench.py \ + --repeat 1 --min_time 3 -j 1,16 --readers 1,4 -s ${PERF_SCENARIOS} || true + fi + cp -f *.nsys-rep ${JSON_PERF} ${JSON_HASH} ${JSON_REF} "${ARTIFACTS_DIR}/" 2>/dev/null || true + echo "Results (and nsys profile, if any) saved to ${ARTIFACTS_DIR}" CLEAN_AND_EXIT 1 } pushd ../.. From 9d36532c32b41fd01162956d904ef9f5702e9ecb Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Thu, 22 Oct 2026 10:46:48 +0200 Subject: [PATCH 11/46] Trim the video reader benchmark; pin torch/torchcodec for the comparison - Remove the switch and startup scenarios, the pixel rate, cpu at batch 1, --hash_batches (HASH_MIN_SAMPLES only), the device NVDEC sampler and --require_idle, which no check or result needs. - torchcodec: select the CUDA backend explicitly (its default changed from FFmpeg to its own NVDEC decoder), rename torchcodec_beta to torchcodec_nvdec and report torchcodec's own cpu_fallback status. - TL1: install torch 2.11.0 with torchcodec 0.15.0 from the PyTorch cu129 or cu130 index for the comparison, instead of the torch pinned for the framework tests, so both CUDA versions run the same torchcodec. Signed-off-by: Joaquin Anton Guirao --- internal_tools/video_reader_bench.py | 162 +++++++++++---------------- qa/TL1_video_reader_perf/test.sh | 57 ++++------ 2 files changed, 91 insertions(+), 128 deletions(-) diff --git a/internal_tools/video_reader_bench.py b/internal_tools/video_reader_bench.py index cc7e8a9f9ff..4d0bae91882 100644 --- a/internal_tools/video_reader_bench.py +++ b/internal_tools/video_reader_bench.py @@ -38,9 +38,6 @@ the cost of opening another file (decoder creation or NVDEC decoder cache reuse) relative to a flush and seek within the same file. This is a workload difference, not an isolated switch cost - switch clips of mixed resolutions, shuffled, short sequences: per-sample - cost under constant file switching (random access) - startup many distinct files: pipeline build time and first batch latency engines K independent readers in one pipeline, each on its own files: how throughput scales with concurrent NVDEC sessions (works today, because every reader decodes on its own thread) @@ -52,7 +49,7 @@ * the first `depth` + `reader_depth` timed iterations are excluded, as up to that many batches (pipeline queue and the reader's own queue) may have been produced off the clock; measuring continues until both --iters and --min_time are reached after them - * throughput and pixel rates count the actual output shapes of the timed batches + * throughput counts the actual output frames of the timed batches * Pipeline.run() without cuda_stream returns host-synchronized outputs, so iteration times include GPU completion * NVML metrics are per process where possible (decoder utilization, memory). A @@ -70,7 +67,7 @@ Other libraries (--backends), for reference: pynvvideocodec PyNvVideoCodec SimpleDecoder (NVDEC), RGB interleaved output (as DALI) torchcodec torchcodec VideoDecoder, FFmpeg CUDA backend on GPU, FFmpeg on CPU - torchcodec_beta torchcodec VideoDecoder with its own NVDEC backend (set_cuda_backend) + torchcodec_nvdec torchcodec VideoDecoder with its own NVDEC backend (torchcodec >= 0.8) They decode the same samples as the DALI reader (the same sequences of the same files, in the same order, or shuffled with a fixed seed), each sample with frames [start, end) of one file. A batch is decoded by a pool of min(threads, batch) threads, each with its @@ -106,7 +103,6 @@ # every slot of the reader's prefetch queue is reused HASH_MIN_SAMPLES = 40 -H264_CLIPS = ["sintel/video_files/*.mp4", "sintel/labelled_videos/*/*.mp4"] VPX_CLIPS = [ "vp9/vp9_0.mp4", "vp9/vp9_2.mp4", @@ -119,7 +115,7 @@ # file sets: glob patterns relative to DALI_extra/db/video FILES = { # 15 H.264 1280x720 clips, 122-430 frames, B-frames, 1-4 keyframes (GOP up to 250) - "h264_clips": H264_CLIPS, + "h264_clips": ["sintel/video_files/*.mp4", "sintel/labelled_videos/*/*.mp4"], # one H.264 1280x720 clip, 1253 frames, GOP 250 "h264_long": ["sintel/sintel_trailer-720p.mp4"], # the same clip as HEVC 8-bit (keyframe every ~9 frames) and HEVC 10-bit (GOP 250) @@ -127,17 +123,6 @@ "hevc10_long": ["hevc/sintel_trailer-720p_10b.mp4"], # one H.264 512x512 clip, 250 frames: small frames, per-frame overheads dominate "h264_small": ["test_label.mp4"], - # H.264 clips with mixed resolutions (1280x720, 512x512, 800x600 coded as 800x608) - "h264_mixed": H264_CLIPS - + [ - "cfr_test.mp4", - "vfr_test.mp4", - "test_label.mp4", - "frame_num_timestamp/test_25fps.mp4", - "multiple_framerate/50/50fps.mp4", - "cfr/test_2.mp4", - "vfr/test_2.mp4", - ], # two H.264 800x600 clips, coded as 800x608 (display area cropped) "h264_cropped_pair": ["cfr/test_2.mp4", "vfr/test_2.mp4"], # two H.264 1280x720 clips, coded size == display size @@ -149,7 +134,7 @@ # with an MP4 edit list hiding the lead-in frames: DALI and torchcodec start at the first # visible frame, so they decode the lead-in before frame 0, while PyNvVideoCodec returns the # lead-in frames themselves (different frames, less decoding) -LEAD_IN_FILES = {"h264_clips", "h264_mixed"} +LEAD_IN_FILES = {"h264_clips"} def _cfg(files, layout="plain", device="gpu", batch=8, seq=32, threads=4, **kw): @@ -210,38 +195,25 @@ def scenarios(threads, readers, quick=False): for f in ("h264_cropped_pair", "h264_aligned_pair") for kind in ("alternate", "same") ], - "switch": [_cfg("h264_mixed", batch=8, seq=8, threads=t, shuffle=True) for t in t_pair], - "startup": [ - _cfg("h264_clips", "many:256", batch=1, seq=8, threads=t, iters=0) for t in t_pair - ], # the noisiest scenario: more iterations than the --iters default "engines": [ _cfg("h264_clips", "many:64", batch=4, seq=32, threads=4, readers=k, **iters) for k in readers ], "cpu": [ - _cfg("vpx_clips", "many:128", device="cpu", batch=b, seq=16, threads=t) - for b in (1, 8) - for t in threads + _cfg("vpx_clips", "many:128", device="cpu", batch=8, seq=16, threads=t) for t in threads ], } -# scenarios that other backends run as well (reader_depth, startup and engines measure -# properties of the DALI reader), and the devices they decode on -_REFERENCE_SCENARIOS = { - "lanes", - "seq_longgop", - "random_access", - "single_stream", - "cache_key", - "switch", -} +# scenarios that other backends run as well (reader_depth and engines measure properties of +# the DALI reader), and the devices they decode on +_REFERENCE_SCENARIOS = {"lanes", "seq_longgop", "random_access", "single_stream", "cache_key"} BACKEND_SCENARIOS = { "dali": (None, ("gpu", "cpu")), # None: all scenarios "pynvvideocodec": (_REFERENCE_SCENARIOS, ("gpu",)), "torchcodec": (_REFERENCE_SCENARIOS | {"cpu"}, ("gpu", "cpu")), - "torchcodec_beta": (_REFERENCE_SCENARIOS, ("gpu",)), + "torchcodec_nvdec": (_REFERENCE_SCENARIOS, ("gpu",)), } @@ -252,7 +224,7 @@ def backend_runs(backend, scenario, cfg): # PyNvVideoCodec 2.0 SimpleDecoder fails (IndexError) when a decoder that was just # created or reconfigured for a file is asked for frames from the middle of it, so it # runs only configurations whose decoders start every file at frame 0 - seeks = scenario in ("random_access", "switch") or ( + seeks = scenario == "random_access" or ( scenario == "seq_longgop" and min(cfg["threads"], cfg["batch"]) > 1 ) return not (backend == "pynvvideocodec" and seeks) @@ -374,7 +346,6 @@ def __init__(self, device_id, period=0.1): self.errors = [] # NVML problems; a metric may be missing, the run is still valid self.period = period self.mem = [] - self.dev_dec = [] self.others = set() self._stop = threading.Event() self._thread = threading.Thread(target=self._run, daemon=True) @@ -416,9 +387,6 @@ def own_memory(): samplers = { "own memory": own_memory, "other processes": lambda: self.others.update(other_gpu_pids(nvml, h)), - "decoder utilization": lambda: self.dev_dec.append( - nvml.nvmlDeviceGetDecoderUtilization(h)[0] - ), } failures = dict.fromkeys(samplers, 0) while not self._stop.is_set(): @@ -448,8 +416,6 @@ def stop(self): res = {} if self.mem: res["proc_mem_max_mb"] = max(self.mem) / 2**20 - if self.dev_dec: - res["dev_nvdec_util"] = statistics.mean(self.dev_dec) try: samples = self.nvml.nvmlDeviceGetProcessUtilization(self.handle, self._t_start_us) dec = [s.decUtil for s in samples if s.pid == os.getpid()] @@ -594,6 +560,13 @@ def __init__(self, cfg, specs, args): self.version = f"torchcodec {torchcodec.__version__}" self.version += f", torch {torch.__version__}" self.local = threading.local() + # torchcodec on the GPU: whether a decoder reported decoding on the CPU instead + # (None: not known, e.g. older torchcodec without VideoDecoder.cpu_fallback) + self.cpu_fallback = None + if self.gpu and self.backend != "pynvvideocodec": + from torchcodec.decoders import VideoDecoder + + self.cpu_fallback = False if hasattr(VideoDecoder, "cpu_fallback") else None samples = sample_list(specs, cfg["seq"], cfg["shuffle"], self._num_frames) self.samples = itertools.cycle(samples) # no more threads than samples per batch: an idle extra thread would take the next @@ -609,16 +582,18 @@ def _open(self, path): use_device_memory=True, output_color_type=self.nvc.OutputColorType.RGB, # interleaved, as DALI ) - from torchcodec.decoders import VideoDecoder + from torchcodec import decoders if not self.gpu: - return VideoDecoder(path, device="cpu") - if self.backend == "torchcodec": # the default CUDA backend, FFmpeg's NVDEC support - return VideoDecoder(path, device=f"cuda:{self.device_id}") - from torchcodec.decoders import set_cuda_backend # torchcodec >= 0.8 - - with set_cuda_backend("beta"): - return VideoDecoder(path, device=f"cuda:{self.device_id}") + return decoders.VideoDecoder(path, device="cpu") + # the CUDA backend is chosen explicitly, as the default changed: FFmpeg's NVDEC support + # up to torchcodec 0.11 (the only one before 0.8), torchcodec's own NVDEC decoder later + # ("nvdec", called "beta" before, which later versions still accept) + if not hasattr(decoders, "set_cuda_backend"): + return decoders.VideoDecoder(path, device=f"cuda:{self.device_id}") + backend = "ffmpeg" if self.backend == "torchcodec" else "beta" + with decoders.set_cuda_backend(backend): + return decoders.VideoDecoder(path, device=f"cuda:{self.device_id}") def _num_frames(self, path): return len(self._decoder(path)) @@ -645,6 +620,8 @@ def _decode(self, sample): frames = decoder.get_batch_frames_by_index(list(range(start, end))) return (len(frames),) + tuple(frames[0].shape[:2]) # frames of H, W, C video = decoder.get_frames_in_range(start, end).data + if self.gpu and getattr(decoder, "cpu_fallback", None): # torchcodec >= 0.11 + self.cpu_fallback = True return (video.shape[0], video.shape[2], video.shape[3]) # F, C, H, W def build(self): @@ -685,40 +662,39 @@ def run_one(cfg, specs, args): res["shape"] = list(runner.run()[0]) iters = cfg.get("iters", args.iters) - if iters > 0: - for _ in range(args.warmup): - runner.run() - # Up to runner.prefetched batches (DALI: `depth` in the pipeline queue + - # `reader_depth` in the reader's own queue) may have been produced before the clock - # started. They are excluded, and --iters/--min_time apply to the remaining window. - skip = runner.prefetched - stamps, frames, pixels = [], [], [] - mon.start() - t_start = time.perf_counter() - while True: - shapes = runner.run() # actual (frames, height, width) of every sample - stamps.append(time.perf_counter()) - frames.append(sum(s[0] for s in shapes)) - pixels.append(sum(s[0] * s[1] * s[2] for s in shapes)) - kept = len(stamps) - skip - if kept > 0 and kept >= iters: - t_ref = stamps[skip - 1] if skip else t_start - if stamps[-1] - t_ref >= args.min_time: - break - res.update(mon.stop()) - span = stamps[-1] - t_ref - n = len(stamps) - skip - times_ms = sorted((b - a) * 1e3 for a, b in zip([t_ref] + stamps[skip:], stamps[skip:])) - res.update( - iters=n, - iters_skipped=skip, - frames_per_s=sum(frames[skip:]) / span, - mpix_per_s=sum(pixels[skip:]) / span / 1e6, - ms_per_sample=span * 1e3 / (n * cfg["readers"] * cfg["batch"]), - iter_p50_ms=statistics.median(times_ms), - iter_p90_ms=percentile(times_ms, 90), - iter_max_ms=times_ms[-1], - ) + for _ in range(args.warmup): + runner.run() + # Up to runner.prefetched batches (DALI: `depth` in the pipeline queue + `reader_depth` + # in the reader's own queue) may have been produced before the clock started. They are + # excluded, and --iters/--min_time apply to the remaining window. + skip = runner.prefetched + stamps, frames = [], [] + mon.start() + t_start = time.perf_counter() + while True: + shapes = runner.run() # actual (frames, height, width) of every sample + stamps.append(time.perf_counter()) + frames.append(sum(s[0] for s in shapes)) + kept = len(stamps) - skip + if kept > 0 and kept >= iters: + t_ref = stamps[skip - 1] if skip else t_start + if stamps[-1] - t_ref >= args.min_time: + break + res.update(mon.stop()) + span = stamps[-1] - t_ref + n = len(stamps) - skip + times_ms = sorted((b - a) * 1e3 for a, b in zip([t_ref] + stamps[skip:], stamps[skip:])) + res.update( + iters=n, + iters_skipped=skip, + frames_per_s=sum(frames[skip:]) / span, + ms_per_sample=span * 1e3 / (n * cfg["readers"] * cfg["batch"]), + iter_p50_ms=statistics.median(times_ms), + iter_p90_ms=percentile(times_ms, 90), + iter_max_ms=times_ms[-1], + ) + if getattr(runner, "cpu_fallback", None) is not None: + res["cpu_fallback"] = runner.cpu_fallback res["post"] = mon.snapshot() if mon.errors: # after the last NVML query, so that every failure is recorded res["nvml_error"] = "; ".join(mon.errors) @@ -736,7 +712,7 @@ def hash_one(cfg, specs, args): pipe.build() hashes = [] # Not dependent on threads or reader_depth, so that --compare --ignore can match runs - batches = max(args.hash_batches, -(-HASH_MIN_SAMPLES // (cfg["batch"] * cfg["readers"]))) + batches = -(-HASH_MIN_SAMPLES // (cfg["batch"] * cfg["readers"])) # ceil for _ in range(batches): outs = [_as_cpu(t) for t in pipe.run()] for r in range(cfg["readers"]): @@ -770,7 +746,7 @@ def run_child(cfg, specs, args): cmd += [json.dumps(dict(cfg=cfg, specs=specs))] for flag in ("dali_extra", "device_id", "iters", "warmup", "min_time", "seed"): cmd += [f"--{flag}", str(getattr(args, flag))] - cmd += ["--hash_batches", str(args.hash_batches)] + (["--hash"] if args.hash else []) + cmd += ["--hash"] if args.hash else [] try: p = subprocess.run(cmd, capture_output=True, text=True, timeout=args.timeout) except subprocess.TimeoutExpired: @@ -793,10 +769,11 @@ def aggregate(cfg, reps): return res res["errors"] = [r["error"] for r in reps if "error" in r] res["contaminated"] = any(r.get("contaminated") for r in ok) + if any("cpu_fallback" in r for r in ok): + res["cpu_fallback"] = any(r.get("cpu_fallback") for r in ok) res["runs"] = ok for key in ( "frames_per_s", - "mpix_per_s", "ms_per_sample", "iter_p50_ms", "iter_p90_ms", @@ -804,7 +781,6 @@ def aggregate(cfg, reps): "build_s", "first_batch_ms", "proc_nvdec_util", - "dev_nvdec_util", "proc_mem_max_mb", ): vals = [r[key] for r in ok if r.get(key) is not None] @@ -876,6 +852,8 @@ def format_row(r): if r["proc_nvdec_util"] < 10: # e.g. a GPU backend that falls back to CPU decoding for a format it can't decode flags.append("NVDEC<10%") + if r.get("cpu_fallback"): + flags.append("CPU fallback") # reported by torchcodec if r.get("backend") == "pynvvideocodec" and r.get("files") in LEAD_IN_FILES: flags.append("lead-in frames") # not the same frames as DALI, see LEAD_IN_FILES if r.get("errors"): @@ -1018,9 +996,7 @@ def parse_args(): parser.add_argument("--dali_extra", default=os.environ.get("DALI_EXTRA_PATH", "")) parser.add_argument("--json", help="write results (with provenance) to this file") parser.add_argument("--quick", action="store_true", help="smoke test: tiny sweep, 1 run") - parser.add_argument("--require_idle", action="store_true", help="abort if the GPU is busy") parser.add_argument("--hash", action="store_true", help="record per-sample checksums") - parser.add_argument("--hash_batches", default=8, type=int) parser.add_argument("--compare", nargs=2, metavar=("A", "B"), help="compare two --hash files") parser.add_argument( "--ignore", @@ -1088,8 +1064,6 @@ def main(): f"The GPU is in use by other processes ({', '.join(details)}); " f"the results will be skewed." ) - if args.require_idle: - sys.exit("The GPU is in use by other processes and --require_idle is set; aborting.") workdir = tempfile.mkdtemp(prefix="video_reader_bench_") results, failed = [], 0 diff --git a/qa/TL1_video_reader_perf/test.sh b/qa/TL1_video_reader_perf/test.sh index 07761c296c5..c54d12d0ae7 100644 --- a/qa/TL1_video_reader_perf/test.sh +++ b/qa/TL1_video_reader_perf/test.sh @@ -1,7 +1,6 @@ #!/bin/bash -e -# used pip packages; torch (pinned per CUDA version in qa/setup_packages.py) is needed by the -# reference libraries, see install_reference_libs -pip_packages='numpy nvidia-ml-py==11.450.51 torch' +# used pip packages (the reference libraries are installed by install_reference_libs) +pip_packages='numpy nvidia-ml-py==11.450.51' target_dir=./internal_tools # One-time pre-step: FFmpeg shared libraries for torchcodec (x86_64 only, see @@ -37,24 +36,20 @@ JSON_REF="video_reader_ref.json" REF_SCENARIOS="lanes,single_stream,cache_key,cpu" REF_ARGS="--repeat 1 --iters 4 --warmup 1 --min_time 2 -j 1,16" -# Installs PyNvVideoCodec and torchcodec (x86_64 only) and sets REF_BACKENDS. torchcodec must -# match the torch version installed through pip_packages, and is built for the same CUDA version. +# Installs PyNvVideoCodec, and torch with the matching CUDA-enabled torchcodec from the PyTorch +# index for this CUDA version (x86_64 only), and sets REF_BACKENDS. torch is needed only here, +# so it is pinned here rather than in pip_packages, which would follow the DALI framework tests. install_reference_libs() { - local torch_ver cuda_tag tc_ver - torch_ver=$(python -c "import torch; print(torch.__version__.split('+')[0])") - cuda_tag=$(python -c "import torch; print('cu' + torch.version.cuda.replace('.', ''))") - case "${torch_ver}" in - 2.7.*) tc_ver=0.5 ;; - 2.11.*) tc_ver=0.11.1 ;; - *) echo "No torchcodec version known for torch ${torch_ver}; update test.sh"; return 1 ;; + local cuda_tag + case "${CUDA_VERSION}" in + 12*) cuda_tag=cu129 ;; + 13*) cuda_tag=cu130 ;; + *) echo "No PyTorch index known for CUDA_VERSION=${CUDA_VERSION}; update test.sh"; return 1 ;; esac pip install PyNvVideoCodec==2.0.2 || return 1 - # --no-deps: keep the torch from pip_packages - pip install --no-deps "torchcodec==${tc_ver}+${cuda_tag}" \ + pip install "torch==2.11.0+${cuda_tag}" "torchcodec==0.15.0+${cuda_tag}" \ --index-url "https://download.pytorch.org/whl/${cuda_tag}" || return 1 - REF_BACKENDS="pynvvideocodec,torchcodec" - # torchcodec 0.5 has no set_cuda_backend, i.e. no torchcodec_beta - [ "${tc_ver}" != "0.5" ] && REF_BACKENDS="${REF_BACKENDS},torchcodec_beta" + REF_BACKENDS="pynvvideocodec,torchcodec,torchcodec_nvdec" echo "Reference backends: ${REF_BACKENDS}" } @@ -96,24 +91,18 @@ run_bench() { test_body() { if [ "$(uname -p)" == "x86_64" ]; then - # Hopper - TASKSET="taskset --cpu-list 0-127" - # Absolute minimums in frames/s (see qa/TL1_video_reader_perf/check_results.py); - # empty: reported for reference only, not checked. - # TODO: set them (e.g. 0.85x the REFERENCE values) once a few CI runs establish the - # baseline on these machines - MIN_LANES= - MIN_SINGLE_STREAM= - MIN_CPU= - MIN_ENGINES= + TASKSET="taskset --cpu-list 0-127" # Hopper else - # GraceHopper - TASKSET="taskset --cpu-list 0-71" - MIN_LANES= - MIN_SINGLE_STREAM= - MIN_CPU= - MIN_ENGINES= + TASKSET="taskset --cpu-list 0-71" # GraceHopper fi + # Absolute minimums in frames/s (see qa/TL1_video_reader_perf/check_results.py); empty: + # reported for reference only, not checked. + # TODO: set them per architecture (e.g. 0.85x the REFERENCE values) once a few CI runs + # establish the baseline on these machines + MIN_LANES= + MIN_SINGLE_STREAM= + MIN_CPU= + MIN_ENGINES= unset NSYS_REP PERF_RUN="OK" @@ -123,7 +112,7 @@ test_body() { # The output must not depend on num_threads or on the reader's prefetch_queue_depth: every # configuration is compared with all that differ from it only in those HASH_RESULT="OK" - ${TASKSET} python video_reader_bench.py -s ${HASH_SCENARIOS} -j 1,16 --hash --hash_batches 4 \ + ${TASKSET} python video_reader_bench.py -s ${HASH_SCENARIOS} -j 1,16 --hash \ --json ${JSON_HASH} || HASH_RESULT="FAIL" python video_reader_bench.py --compare ${JSON_HASH} ${JSON_HASH} \ --ignore threads,reader_depth || HASH_RESULT="FAIL" From 118c7d58d1ac254c9f0086249041c13ebc412617 Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Thu, 22 Oct 2026 10:50:46 +0200 Subject: [PATCH 12/46] Widen the backend column of the video reader benchmark table Signed-off-by: Joaquin Anton Guirao --- internal_tools/video_reader_bench.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal_tools/video_reader_bench.py b/internal_tools/video_reader_bench.py index 4d0bae91882..411bf9a372b 100644 --- a/internal_tools/video_reader_bench.py +++ b/internal_tools/video_reader_bench.py @@ -801,7 +801,7 @@ def aggregate(cfg, reps): COLUMNS = [ ("scenario", "scenario", "{:<13}"), - ("backend", "backend", "{:<15}"), + ("backend", "backend", "{:<16}"), ("files", "files", "{:<17}"), ("layout", "layout", "{:<12}"), ("device", "dev", "{:<3}"), From ec9f5ceb6369c6cf9bd43f25aebf2214bb8bb55f Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Thu, 22 Oct 2026 10:52:50 +0200 Subject: [PATCH 13/46] Don't fail the TL1 video reader test on a missing results file print_results ended with a failing [ -f ] when the last file was missing (no reference run on GraceHopper, or a failed one), which exits test_body under set -e. Also give the empty except in provenance() a reason. Signed-off-by: Joaquin Anton Guirao --- internal_tools/video_reader_bench.py | 2 +- qa/TL1_video_reader_perf/test.sh | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/internal_tools/video_reader_bench.py b/internal_tools/video_reader_bench.py index 411bf9a372b..8afec02f346 100644 --- a/internal_tools/video_reader_bench.py +++ b/internal_tools/video_reader_bench.py @@ -886,7 +886,7 @@ def provenance(args): try: info.setdefault("packages", {})[pkg] = metadata.version(pkg) except metadata.PackageNotFoundError: - pass + continue # optional: needed only by the --backends that use it try: nvml, h = nvml_handle(args.device_id) except Exception as e: diff --git a/qa/TL1_video_reader_perf/test.sh b/qa/TL1_video_reader_perf/test.sh index c54d12d0ae7..e73db6c650d 100644 --- a/qa/TL1_video_reader_perf/test.sh +++ b/qa/TL1_video_reader_perf/test.sh @@ -66,14 +66,18 @@ function CLEAN_AND_EXIT { exit $1 } -# Prints the results (without per-run data) to the log, so that they are kept on success too +# Prints the results (without per-run data) to the log, so that they are kept on success too. +# A missing or unreadable file (e.g. no reference run on GraceHopper) is skipped, not an error. print_results() { for json in "$@"; do - [ -f "${json}" ] && python -c "import json, sys + if [ -f "${json}" ]; then + python -c "import json, sys d = json.load(open(sys.argv[1])) for r in d['results']: r.pop('runs', None) -print('RESULTS', sys.argv[1], json.dumps(d, separators=(',', ':')))" "${json}" +print('RESULTS', sys.argv[1], json.dumps(d, separators=(',', ':')))" "${json}" \ + || echo "Could not print ${json}" + fi done } From 1565e3a83788a9ff230b5b8e8cc1e3e067ffbf1a Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Thu, 22 Oct 2026 11:28:50 +0200 Subject: [PATCH 14/46] Install the shared libpython for torchcodec in the TL1 video reader test torchcodec 0.15 links against libpython3.X.so, which Ubuntu's python3 package does not include. Signed-off-by: Joaquin Anton Guirao --- qa/TL1_video_reader_perf/test.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/qa/TL1_video_reader_perf/test.sh b/qa/TL1_video_reader_perf/test.sh index e73db6c650d..45cdd1dd09f 100644 --- a/qa/TL1_video_reader_perf/test.sh +++ b/qa/TL1_video_reader_perf/test.sh @@ -8,7 +8,9 @@ target_dir=./internal_tools do_once() { apt update if [ "$(uname -m)" == "x86_64" ]; then - apt install -y --no-install-recommends ffmpeg + # torchcodec loads the FFmpeg shared libraries and links against the shared libpython + apt install -y --no-install-recommends ffmpeg \ + "libpython$(python -c 'import sys; print("%d.%d" % sys.version_info[:2])')" fi if command -v nsys &>/dev/null; then echo "nsys already installed: $(nsys --version)" From 33a0e53c960b6dbf22f45a24363318f53aad4e0b Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Thu, 22 Oct 2026 12:10:05 +0200 Subject: [PATCH 15/46] Drop the NVDEC utilization flag from the video reader benchmark Per-process NVDEC utilization is averaged over the GPU's NVDEC engines, so on GPUs with several engines a fully busy session reads well under 10% and the flag misfired. Only torchcodec can fall back to CPU decoding, and it reports that itself (CPU fallback flag). Signed-off-by: Joaquin Anton Guirao --- internal_tools/video_reader_bench.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/internal_tools/video_reader_bench.py b/internal_tools/video_reader_bench.py index 8afec02f346..d95ad69767c 100644 --- a/internal_tools/video_reader_bench.py +++ b/internal_tools/video_reader_bench.py @@ -54,7 +54,8 @@ times include GPU completion * NVML metrics are per process where possible (decoder utilization, memory). A result is flagged as contaminated if other processes use the GPU or NVDEC is busy - before the run starts + before the run starts. Decoder utilization (dec%) is averaged over the GPU's NVDEC + engines, so on GPUs with several engines it is low even for a fully busy session * all input files are read once before the first run, so all runs see a warm page cache * DALI_extra has no 1080p content, so absolute rates are lower than for 1080p data; compare builds, not datasets @@ -75,9 +76,9 @@ reconfigures its decoder for a new file, torchcodec opens a new one, scanning it, as its default seek_mode="exact" does). Batches are not prefetched: unlike DALI, a batch is decoded when it is requested. They run only the configurations that mean the same - for them (backend_runs), and need torch. GPU rows where NVDEC stays nearly idle are - flagged, as a GPU backend may fall back to CPU decoding, and so are PyNvVideoCodec rows on - clips with hidden lead-in frames (LEAD_IN_FILES), where it returns different frames. + for them (backend_runs), and need torch. Rows are flagged when torchcodec reports a CPU + fallback, and PyNvVideoCodec rows on clips with hidden lead-in frames (LEAD_IN_FILES), + where it returns different frames. """ import argparse @@ -848,12 +849,8 @@ def format_row(r): flags.append("CONTAMINATED") if r.get("frames_per_s_spread_pct", 0) > 3: flags.append("spread>3%") - if r.get("device") == "gpu" and r.get("proc_nvdec_util") is not None: - if r["proc_nvdec_util"] < 10: - # e.g. a GPU backend that falls back to CPU decoding for a format it can't decode - flags.append("NVDEC<10%") if r.get("cpu_fallback"): - flags.append("CPU fallback") # reported by torchcodec + flags.append("CPU fallback") # reported by torchcodec (>= 0.11) if r.get("backend") == "pynvvideocodec" and r.get("files") in LEAD_IN_FILES: flags.append("lead-in frames") # not the same frames as DALI, see LEAD_IN_FILES if r.get("errors"): From 8034caf30f73b82686550861fe73947c5301c9d9 Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Thu, 22 Oct 2026 12:19:02 +0200 Subject: [PATCH 16/46] Add DALI's legacy fn.readers.video to the video reader benchmark dali_legacy runs the legacy reader through the same pipeline as the DALI reader, for reference: GPU only, no cache_key (both pairs include a variable frame rate file, which the legacy reader rejects), and no seeking in the open-GOP HEVC trailer, where it fails. The TL1 comparison step runs it too. Signed-off-by: Joaquin Anton Guirao --- internal_tools/video_reader_bench.py | 29 ++++++++++++++++++++++------ qa/TL1_video_reader_perf/test.sh | 8 ++++---- 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/internal_tools/video_reader_bench.py b/internal_tools/video_reader_bench.py index d95ad69767c..ea3c86a347a 100644 --- a/internal_tools/video_reader_bench.py +++ b/internal_tools/video_reader_bench.py @@ -65,7 +65,9 @@ --hash result files (e.g. two builds, or --ignore threads to check that the output does not depend on num_threads) with --compare. -Other libraries (--backends), for reference: +Other readers and libraries (--backends), for reference: + dali_legacy DALI's legacy fn.readers.video (GPU, constant frame rate only: no cpu, + cache_key), through the same pipeline as the DALI reader pynvvideocodec PyNvVideoCodec SimpleDecoder (NVDEC), RGB interleaved output (as DALI) torchcodec torchcodec VideoDecoder, FFmpeg CUDA backend on GPU, FFmpeg on CPU torchcodec_nvdec torchcodec VideoDecoder with its own NVDEC backend (torchcodec >= 0.8) @@ -212,6 +214,9 @@ def scenarios(threads, readers, quick=False): _REFERENCE_SCENARIOS = {"lanes", "seq_longgop", "random_access", "single_stream", "cache_key"} BACKEND_SCENARIOS = { "dali": (None, ("gpu", "cpu")), # None: all scenarios + # the legacy fn.readers.video: GPU only, and constant frame rate only (both cache_key + # pairs include a variable frame rate file) + "dali_legacy": (_REFERENCE_SCENARIOS - {"cache_key"} | {"reader_depth", "engines"}, ("gpu",)), "pynvvideocodec": (_REFERENCE_SCENARIOS, ("gpu",)), "torchcodec": (_REFERENCE_SCENARIOS | {"cpu"}, ("gpu", "cpu")), "torchcodec_nvdec": (_REFERENCE_SCENARIOS, ("gpu",)), @@ -228,7 +233,11 @@ def backend_runs(backend, scenario, cfg): seeks = scenario == "random_access" or ( scenario == "seq_longgop" and min(cfg["threads"], cfg["batch"]) > 1 ) - return not (backend == "pynvvideocodec" and seeks) + if backend == "pynvvideocodec" and seeks: + return False + # the legacy reader fails seeking in the HEVC trailer (open GOPs): "The decoder returned a + # frame that is past the expected one"; reading it sequentially works + return not (backend == "dali_legacy" and seeks and cfg["files"] == "hevc_long") # ---------------------------------------------------------------------------------------- @@ -437,6 +446,9 @@ def build_pipeline(cfg, specs, device_id, seed, frame_num=False): import nvidia.dali.fn as fn from nvidia.dali.pipeline import pipeline_def + legacy = cfg["backend"] == "dali_legacy" + reader = fn.readers.video if legacy else fn.experimental.readers.video + @pipeline_def( batch_size=cfg["batch"], num_threads=cfg["threads"], @@ -450,12 +462,14 @@ def pipe(): kw = dict(spec) if "filenames" in kw: kw["labels"] = list(range(len(kw["filenames"]))) + elif legacy: + kw["file_list_frame_num"] = True # frame numbers, not timestamps; end exclusive else: kw["file_list_format"] = "frames" kw["file_list_include_end"] = False # explicit: the default changed over time if frame_num: kw["enable_frame_num"] = True - outs += fn.experimental.readers.video( + outs += reader( device=cfg["device"], sequence_length=cfg["seq"], random_shuffle=cfg["shuffle"], @@ -489,7 +503,9 @@ def __init__(self, cfg, specs, args): import nvidia.dali as dali self.cfg = cfg - self.version = dali.__version__ + self.version = dali.__version__ + ( + " (fn.readers.video)" if cfg["backend"] != "dali" else "" + ) # up to `depth` (pipeline queue) + `reader_depth` (the reader's own queue) batches # may be produced before the clock starts self.prefetched = cfg["depth"] + cfg["reader_depth"] @@ -637,7 +653,8 @@ def run(self): def make_runner(cfg, specs, args): - return (DaliRunner if cfg["backend"] == "dali" else ReferenceRunner)(cfg, specs, args) + dali = cfg["backend"] in ("dali", "dali_legacy") + return (DaliRunner if dali else ReferenceRunner)(cfg, specs, args) def percentile(sorted_vals, q): @@ -655,7 +672,7 @@ def run_one(cfg, specs, args): res["backend_version"] = runner.version t0 = time.perf_counter() # build_s: the DALI reader (indexing), not the CUDA runtime runner.build() - if cfg["backend"] == "dali": # other libraries open their decoders lazily, in run() + if isinstance(runner, DaliRunner): # other libraries open their decoders lazily, in run() res["build_s"] = time.perf_counter() - t0 t0 = time.perf_counter() runner.run() diff --git a/qa/TL1_video_reader_perf/test.sh b/qa/TL1_video_reader_perf/test.sh index 45cdd1dd09f..694c7168ce4 100644 --- a/qa/TL1_video_reader_perf/test.sh +++ b/qa/TL1_video_reader_perf/test.sh @@ -32,9 +32,9 @@ JSON_HASH="video_reader_hash.json" LOG_REF="video_reader_ref.log" JSON_REF="video_reader_ref.json" -# Other libraries run for reference on the same samples (video_reader_bench.py --backends), on -# x86_64 only; one short run each, not checked. The DALI numbers are in the performance run. -# Installed by install_reference_libs. +# DALI's legacy fn.readers.video and other libraries run for reference on the same samples +# (video_reader_bench.py --backends), on x86_64 only; one short run each, not checked. The DALI +# numbers are in the performance run. The libraries are installed by install_reference_libs. REF_SCENARIOS="lanes,single_stream,cache_key,cpu" REF_ARGS="--repeat 1 --iters 4 --warmup 1 --min_time 2 -j 1,16" @@ -137,7 +137,7 @@ test_body() { REF_RUN="SKIPPED" elif install_reference_libs; then run_bench "${LOG_REF}" ${TASKSET} python video_reader_bench.py ${REF_ARGS} \ - -s ${REF_SCENARIOS} -b ${REF_BACKENDS} --json ${JSON_REF} || REF_RUN="FAIL" + -s ${REF_SCENARIOS} -b dali_legacy,${REF_BACKENDS} --json ${JSON_REF} || REF_RUN="FAIL" else REF_RUN="FAIL" fi From 910e64fd4b6da74fe335c7a4f54fcbc57b55a3f5 Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Thu, 22 Oct 2026 14:05:13 +0200 Subject: [PATCH 17/46] Keep the reference libraries' batch outputs until the next batch The reference runner returned only the output shapes, so every sample's frames were freed right after decoding and the process memory of the libraries never included a batch, while DALI's includes its returned batch and its queues. A batch's outputs are now kept until the next batch is requested, as Pipeline.run() outputs are. PyNvVideoCodec's frames stay in its decoder's buffers, which its next call reuses, so its memory still does not include a batch; documented. Signed-off-by: Joaquin Anton Guirao --- internal_tools/video_reader_bench.py | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/internal_tools/video_reader_bench.py b/internal_tools/video_reader_bench.py index ea3c86a347a..8374c5d14e8 100644 --- a/internal_tools/video_reader_bench.py +++ b/internal_tools/video_reader_bench.py @@ -77,10 +77,14 @@ own decoder, kept while consecutive samples come from the same file (PyNvVideoCodec reconfigures its decoder for a new file, torchcodec opens a new one, scanning it, as its default seek_mode="exact" does). Batches are not prefetched: unlike DALI, a batch - is decoded when it is requested. They run only the configurations that mean the same - for them (backend_runs), and need torch. Rows are flagged when torchcodec reports a CPU - fallback, and PyNvVideoCodec rows on clips with hidden lead-in frames (LEAD_IN_FILES), - where it returns different frames. + is decoded when it is requested. A batch's outputs are kept until the next batch is + requested, as DALI's, so the process memory (mem MB) includes one batch for them, and + for DALI also the batches in its queues (`depth` + `reader_depth`). PyNvVideoCodec is an + exception: its frames stay in its decoder's buffers, which the decoder's next call + reuses, so its memory does not include a batch. They run only the configurations that + mean the same for them (backend_runs), and need torch. Rows are flagged when torchcodec + reports a CPU fallback, and PyNvVideoCodec rows on clips with hidden lead-in frames + (LEAD_IN_FILES), where it returns different frames. """ import argparse @@ -630,26 +634,33 @@ def _decoder(self, path): return local.decoder def _decode(self, sample): + """Returns the sample's (frames, height, width) and the library's output.""" path, start, end = sample decoder = self._decoder(path) # the libraries' own outputs, not converted or copied if self.backend == "pynvvideocodec": frames = decoder.get_batch_frames_by_index(list(range(start, end))) - return (len(frames),) + tuple(frames[0].shape[:2]) # frames of H, W, C + return (len(frames),) + tuple(frames[0].shape[:2]), frames # frames of H, W, C video = decoder.get_frames_in_range(start, end).data if self.gpu and getattr(decoder, "cpu_fallback", None): # torchcodec >= 0.11 self.cpu_fallback = True - return (video.shape[0], video.shape[2], video.shape[3]) # F, C, H, W + return (video.shape[0], video.shape[2], video.shape[3]), video # F, C, H, W def build(self): pass def run(self): + # The outputs of a batch are kept until the next run(), as Pipeline.run() outputs, so + # that the process memory includes a whole batch (DALI's also includes its queues) + self.outputs = None batch = [next(self.samples) for _ in range(self.cfg["batch"])] - shapes = list(self.pool.map(self._decode, batch) if self.pool else map(self._decode, batch)) + decoded = list( + self.pool.map(self._decode, batch) if self.pool else map(self._decode, batch) + ) if self.gpu: self.torch.cuda.synchronize() # as Pipeline.run(): outputs are ready - return shapes + self.outputs = [out for _, out in decoded] + return [shape for shape, _ in decoded] def make_runner(cfg, specs, args): From 0b13f353548a6f4bacb80925be6f736af93b1800 Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Thu, 22 Oct 2026 16:33:28 +0200 Subject: [PATCH 18/46] Add DALI's video decoder (fn.readers.file + fn.decoders.video) to the video reader benchmark The dali_decoder backend reads every sample's whole file with fn.readers.file and decodes its frames with fn.decoders.video (fn.experimental.decoders.video in older DALI), which decodes the samples of a batch in parallel on num_threads threads, with a new decoder per sample. It runs the same samples as the reader, in the reference scenarios plus engines and cpu, and the TL1 reference step. Like PyNvVideoCodec, it returns the lead-in frames that an MP4 edit list hides, so its rows on those clips are flagged. Signed-off-by: Joaquin Anton Guirao --- internal_tools/video_reader_bench.py | 76 ++++++++++++++++++++++++---- qa/TL1_video_reader_perf/test.sh | 5 +- 2 files changed, 70 insertions(+), 11 deletions(-) diff --git a/internal_tools/video_reader_bench.py b/internal_tools/video_reader_bench.py index 8374c5d14e8..cd407f56d29 100644 --- a/internal_tools/video_reader_bench.py +++ b/internal_tools/video_reader_bench.py @@ -68,6 +68,12 @@ Other readers and libraries (--backends), for reference: dali_legacy DALI's legacy fn.readers.video (GPU, constant frame rate only: no cpu, cache_key), through the same pipeline as the DALI reader + dali_decoder fn.readers.file + fn.decoders.video (fn.experimental.decoders.video in + older DALI): every sample is a whole file, read by the file reader, and + decoded with a new decoder per sample, the samples of a batch in parallel + on num_threads threads. Same samples (file, first and last frame) as the + DALI reader, shuffled by the file reader; `reader_depth` is the file + reader's prefetch_queue_depth pynvvideocodec PyNvVideoCodec SimpleDecoder (NVDEC), RGB interleaved output (as DALI) torchcodec torchcodec VideoDecoder, FFmpeg CUDA backend on GPU, FFmpeg on CPU torchcodec_nvdec torchcodec VideoDecoder with its own NVDEC backend (torchcodec >= 0.8) @@ -83,8 +89,8 @@ exception: its frames stay in its decoder's buffers, which the decoder's next call reuses, so its memory does not include a batch. They run only the configurations that mean the same for them (backend_runs), and need torch. Rows are flagged when torchcodec - reports a CPU fallback, and PyNvVideoCodec rows on clips with hidden lead-in frames - (LEAD_IN_FILES), where it returns different frames. + reports a CPU fallback, and PyNvVideoCodec and dali_decoder rows on clips with hidden + lead-in frames (LEAD_IN_FILES), where they return different frames. """ import argparse @@ -138,10 +144,12 @@ "vpx_clips": VPX_CLIPS, } # File sets where most clips (cut without re-encoding) start before the frame they show first, -# with an MP4 edit list hiding the lead-in frames: DALI and torchcodec start at the first -# visible frame, so they decode the lead-in before frame 0, while PyNvVideoCodec returns the -# lead-in frames themselves (different frames, less decoding) +# with an MP4 edit list hiding the lead-in frames: the DALI reader and torchcodec start at the +# first visible frame, so they decode the lead-in before frame 0, while PyNvVideoCodec and +# DALI's video decoder (dali_decoder) return the lead-in frames themselves (different frames, +# less decoding) LEAD_IN_FILES = {"h264_clips"} +LEAD_IN_BACKENDS = {"pynvvideocodec", "dali_decoder"} def _cfg(files, layout="plain", device="gpu", batch=8, seq=32, threads=4, **kw): @@ -221,6 +229,8 @@ def scenarios(threads, readers, quick=False): # the legacy fn.readers.video: GPU only, and constant frame rate only (both cache_key # pairs include a variable frame rate file) "dali_legacy": (_REFERENCE_SCENARIOS - {"cache_key"} | {"reader_depth", "engines"}, ("gpu",)), + # fn.readers.file + fn.decoders.video; reader_depth would prefetch file contents only + "dali_decoder": (_REFERENCE_SCENARIOS | {"engines", "cpu"}, ("gpu", "cpu")), "pynvvideocodec": (_REFERENCE_SCENARIOS, ("gpu",)), "torchcodec": (_REFERENCE_SCENARIOS | {"cpu"}, ("gpu", "cpu")), "torchcodec_nvdec": (_REFERENCE_SCENARIOS, ("gpu",)), @@ -446,12 +456,37 @@ def stop(self): # running +def num_frames_dali(path, device, device_id): + """The frames of a file as the DALI reader counts them: its sequences of one frame.""" + import nvidia.dali.fn as fn + from nvidia.dali.pipeline import pipeline_def + + @pipeline_def(batch_size=1, num_threads=1, device_id=device_id) + def count(): + return fn.experimental.readers.video( + filenames=[path], sequence_length=1, device=device, name="Count" + ) + + p = count() + p.build() + return p.reader_meta("Count")["epoch_size"] + + def build_pipeline(cfg, specs, device_id, seed, frame_num=False): import nvidia.dali.fn as fn from nvidia.dali.pipeline import pipeline_def legacy = cfg["backend"] == "dali_legacy" reader = fn.readers.video if legacy else fn.experimental.readers.video + if cfg["backend"] == "dali_decoder": + + def num_frames(path): + return num_frames_dali(path, cfg["device"], device_id) + + # the samples the DALI reader would produce, (path, start, end), one list per reader + samples = [sample_list([s], cfg["seq"], False, num_frames) for s in specs] + # moved out of experimental in DALI 2.0 + decoder = getattr(fn.decoders, "video", None) or fn.experimental.decoders.video @pipeline_def( batch_size=cfg["batch"], @@ -462,6 +497,24 @@ def build_pipeline(cfg, specs, device_id, seed, frame_num=False): ) def pipe(): outs = [] + if cfg["backend"] == "dali_decoder": + for r, reader_samples in enumerate(samples): + # whole files, with the sample's first frame as the label + encoded, start = fn.readers.file( + files=[path for path, _, _ in reader_samples], + labels=[start for _, start, _ in reader_samples], + random_shuffle=cfg["shuffle"], + prefetch_queue_depth=cfg["reader_depth"], + name=f"Reader{r}", + ) + video = decoder( + encoded, + device="mixed" if cfg["device"] == "gpu" else "cpu", + start_frame=start[0], + sequence_length=cfg["seq"], + ) + outs += [video, start] + return tuple(outs) for r, spec in enumerate(specs): kw = dict(spec) if "filenames" in kw: @@ -507,8 +560,13 @@ def __init__(self, cfg, specs, args): import nvidia.dali as dali self.cfg = cfg - self.version = dali.__version__ + ( - " (fn.readers.video)" if cfg["backend"] != "dali" else "" + self.version = ( + dali.__version__ + + { + "dali": "", + "dali_legacy": " (fn.readers.video)", + "dali_decoder": " (fn.readers.file + video decoder)", + }[cfg["backend"]] ) # up to `depth` (pipeline queue) + `reader_depth` (the reader's own queue) batches # may be produced before the clock starts @@ -664,7 +722,7 @@ def run(self): def make_runner(cfg, specs, args): - dali = cfg["backend"] in ("dali", "dali_legacy") + dali = cfg["backend"] in ("dali", "dali_legacy", "dali_decoder") return (DaliRunner if dali else ReferenceRunner)(cfg, specs, args) @@ -879,7 +937,7 @@ def format_row(r): flags.append("spread>3%") if r.get("cpu_fallback"): flags.append("CPU fallback") # reported by torchcodec (>= 0.11) - if r.get("backend") == "pynvvideocodec" and r.get("files") in LEAD_IN_FILES: + if r.get("backend") in LEAD_IN_BACKENDS and r.get("files") in LEAD_IN_FILES: flags.append("lead-in frames") # not the same frames as DALI, see LEAD_IN_FILES if r.get("errors"): flags.append(f"{len(r['errors'])} failed run(s)") diff --git a/qa/TL1_video_reader_perf/test.sh b/qa/TL1_video_reader_perf/test.sh index 694c7168ce4..06e4d9124b2 100644 --- a/qa/TL1_video_reader_perf/test.sh +++ b/qa/TL1_video_reader_perf/test.sh @@ -32,7 +32,8 @@ JSON_HASH="video_reader_hash.json" LOG_REF="video_reader_ref.log" JSON_REF="video_reader_ref.json" -# DALI's legacy fn.readers.video and other libraries run for reference on the same samples +# DALI's legacy fn.readers.video, its video decoder (fn.readers.file + fn.decoders.video) and +# other libraries run for reference on the same samples # (video_reader_bench.py --backends), on x86_64 only; one short run each, not checked. The DALI # numbers are in the performance run. The libraries are installed by install_reference_libs. REF_SCENARIOS="lanes,single_stream,cache_key,cpu" @@ -137,7 +138,7 @@ test_body() { REF_RUN="SKIPPED" elif install_reference_libs; then run_bench "${LOG_REF}" ${TASKSET} python video_reader_bench.py ${REF_ARGS} \ - -s ${REF_SCENARIOS} -b dali_legacy,${REF_BACKENDS} --json ${JSON_REF} || REF_RUN="FAIL" + -s ${REF_SCENARIOS} -b dali_legacy,dali_decoder,${REF_BACKENDS} --json ${JSON_REF} || REF_RUN="FAIL" else REF_RUN="FAIL" fi From 1c578592a7dcc51084f104cfe85a0be2d33cba46 Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Thu, 22 Oct 2026 17:28:07 +0200 Subject: [PATCH 19/46] Make the video reader's frame index cache thread-safe FrameIndexCache::find returned an iterator after releasing its lock, and callers copied the index through it without the lock, while insert() could overwrite the same entry (two readers indexing the same file) or rehash the map. The cache moves to its own header with GetOrBuild, which builds every index once per file, outside the lock, and returns an immutable shared_ptr, and Find, which never builds or waits. Signed-off-by: Joaquin Anton Guirao --- .../video/reader/frame_index_cache.h | 116 +++++++++++++++++ .../video/reader/frame_index_cache_test.cc | 120 ++++++++++++++++++ .../video/reader/video_reader_decoder_op.cc | 70 ++-------- 3 files changed, 247 insertions(+), 59 deletions(-) create mode 100644 dali/operators/video/reader/frame_index_cache.h create mode 100644 dali/operators/video/reader/frame_index_cache_test.cc diff --git a/dali/operators/video/reader/frame_index_cache.h b/dali/operators/video/reader/frame_index_cache.h new file mode 100644 index 00000000000..98a5ef018be --- /dev/null +++ b/dali/operators/video/reader/frame_index_cache.h @@ -0,0 +1,116 @@ +// Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef DALI_OPERATORS_VIDEO_READER_FRAME_INDEX_CACHE_H_ +#define DALI_OPERATORS_VIDEO_READER_FRAME_INDEX_CACHE_H_ + +#include +#include +#include +#include +#include +#include +#include + +#include "dali/operators/video/frames_decoder_base.h" + +namespace dali { + +/** + * @brief Process-wide cache of frame indices, keyed by file name. + * + * Shared by all video readers in the process and safe to use from several threads at once. + * Every index is built at most once (by the first caller that asks for it), outside the lock, + * so building the indices of different files can run in parallel. Callers get an immutable, + * reference-counted index that stays valid regardless of other insertions. + */ +class FrameIndexCache { + public: + using IndexPtr = std::shared_ptr; + + static FrameIndexCache &instance() { + static FrameIndexCache cache; + return cache; + } + + /** + * @brief Returns the index of `filename`, building it with `build()` if it is not cached. + * + * `build` returns a FrameIndex. If another thread is building the same index, this call waits + * for it. If building fails, the exception is rethrown to every caller waiting for that index + * and the entry is removed, so that a later call can try again. + */ + template + IndexPtr GetOrBuild(const std::string &filename, BuildFn &&build) { + std::promise promise; + std::shared_future index; + bool build_here = false; + { + std::lock_guard guard(mutex_); + auto it = cache_.find(filename); + if (it != cache_.end()) { + index = it->second; + } else { + index = promise.get_future().share(); + cache_.emplace(filename, index); + build_here = true; + } + } + if (build_here) { + try { + promise.set_value(std::make_shared(build())); + } catch (...) { + { + std::lock_guard guard(mutex_); + cache_.erase(filename); + } + promise.set_exception(std::current_exception()); + } + } + return index.get(); + } + + /** + * @brief Returns the index of `filename` if it is cached and ready, nullptr otherwise. + * + * Never builds and never waits. + */ + IndexPtr Find(const std::string &filename) const { + std::shared_future index; + { + std::lock_guard guard(mutex_); + auto it = cache_.find(filename); + if (it == cache_.end()) + return nullptr; + index = it->second; + } + if (index.wait_for(std::chrono::seconds(0)) != std::future_status::ready) + return nullptr; + try { + return index.get(); + } catch (...) { + return nullptr; + } + } + + private: + FrameIndexCache() = default; + + mutable std::mutex mutex_; + std::unordered_map> cache_; +}; + +} // namespace dali + +#endif // DALI_OPERATORS_VIDEO_READER_FRAME_INDEX_CACHE_H_ diff --git a/dali/operators/video/reader/frame_index_cache_test.cc b/dali/operators/video/reader/frame_index_cache_test.cc new file mode 100644 index 00000000000..0c53693d58c --- /dev/null +++ b/dali/operators/video/reader/frame_index_cache_test.cc @@ -0,0 +1,120 @@ +// Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include +#include +#include +#include + +#include "dali/operators/video/reader/frame_index_cache.h" + +namespace dali { + +namespace { + +// Unique keys per test: the cache is process-wide +std::string Key(const std::string &test, int i) { + return "frame_index_cache_test/" + test + "/" + std::to_string(i); +} + +FrameIndex MakeIndex(const std::string &filename, int num_frames) { + FrameIndex index; + index.filename = filename; + index.timebase = {1, 25}; + for (int i = 0; i < num_frames; i++) + index.index.push_back({i, 0, i == 0, i == num_frames - 1}); + return index; +} + +} // namespace + +TEST(FrameIndexCacheTest, BuildsEachIndexOnceUnderConcurrency) { + constexpr int kThreads = 16, kFiles = 64, kRounds = 4; + auto &cache = FrameIndexCache::instance(); + std::vector> builds(kFiles); + std::atomic errors{0}; + std::vector threads; + for (int t = 0; t < kThreads; t++) { + threads.emplace_back([&, t]() { + for (int r = 0; r < kRounds; r++) { + for (int f = 0; f < kFiles; f++) { + int file = (f + t * 7) % kFiles; // threads start on different files + auto index = cache.GetOrBuild(Key("concurrent", file), [&]() { + builds[file]++; + std::this_thread::sleep_for(std::chrono::microseconds(200)); + return MakeIndex(Key("concurrent", file), file + 1); + }); + if (!index || index->filename != Key("concurrent", file) || + static_cast(index->size()) != file + 1) + errors++; + } + } + }); + } + for (auto &t : threads) + t.join(); + EXPECT_EQ(errors, 0); + for (int f = 0; f < kFiles; f++) + EXPECT_EQ(builds[f], 1) << "file " << f; +} + +TEST(FrameIndexCacheTest, IndexStaysValidAfterOtherInsertions) { + auto &cache = FrameIndexCache::instance(); + auto first = cache.GetOrBuild(Key("valid", 0), [] { return MakeIndex(Key("valid", 0), 10); }); + for (int i = 1; i < 1000; i++) // forces rehashing of the underlying map + cache.GetOrBuild(Key("valid", i), [i] { return MakeIndex(Key("valid", i), 1); }); + ASSERT_EQ(first->size(), 10u); + EXPECT_EQ(first->filename, Key("valid", 0)); + EXPECT_EQ(cache.Find(Key("valid", 0)), first); +} + +TEST(FrameIndexCacheTest, FailedBuildIsReportedAndRetried) { + auto &cache = FrameIndexCache::instance(); + auto key = Key("failure", 0); + EXPECT_THROW( + cache.GetOrBuild(key, []() -> FrameIndex { throw std::runtime_error("cannot index"); }), + std::runtime_error); + EXPECT_EQ(cache.Find(key), nullptr); + auto index = cache.GetOrBuild(key, [&] { return MakeIndex(key, 3); }); + ASSERT_NE(index, nullptr); + EXPECT_EQ(index->size(), 3u); +} + +TEST(FrameIndexCacheTest, FindDoesNotBuildOrWait) { + auto &cache = FrameIndexCache::instance(); + auto key = Key("find", 0); + EXPECT_EQ(cache.Find(key), nullptr); + std::atomic building{false}, release{false}; + std::thread builder([&]() { + cache.GetOrBuild(key, [&] { + building = true; + while (!release) + std::this_thread::yield(); + return MakeIndex(key, 2); + }); + }); + while (!building) + std::this_thread::yield(); + EXPECT_EQ(cache.Find(key), nullptr); // still being built + release = true; + builder.join(); + auto index = cache.Find(key); + ASSERT_NE(index, nullptr); + EXPECT_EQ(index->size(), 2u); +} + +} // namespace dali diff --git a/dali/operators/video/reader/video_reader_decoder_op.cc b/dali/operators/video/reader/video_reader_decoder_op.cc index 30516e7f97c..5b917d5ca44 100644 --- a/dali/operators/video/reader/video_reader_decoder_op.cc +++ b/dali/operators/video/reader/video_reader_decoder_op.cc @@ -13,8 +13,6 @@ // limitations under the License. #include -#include -#include #include #include "dali/core/boundary.h" @@ -25,6 +23,7 @@ #include "dali/operators/video/frames_decoder_base.h" #include "dali/operators/video/frames_decoder_cpu.h" #include "dali/operators/video/frames_decoder_gpu.h" +#include "dali/operators/video/reader/frame_index_cache.h" #include "dali/operators/video/video_utils.h" #if NVML_ENABLED #include "dali/util/nvml.h" @@ -34,49 +33,6 @@ namespace dali { -class FrameIndexCache { - private: - std::unordered_map index_cache_; - mutable std::shared_mutex rw_mutex_; - - FrameIndexCache() = default; - - ~FrameIndexCache() { -#ifdef NDEBUG - size_t total_size = 0; - for (const auto& entry : index_cache_) { - total_size += entry.first.size(); // filename string size - total_size += entry.second.size() * sizeof(IndexEntry); // index entries size - } - LOG_LINE << "FrameIndexCache stats: " << std::endl; - LOG_LINE << " Number of cached files: " << index_cache_.size() << std::endl; - LOG_LINE << " Total memory used: " << total_size << " bytes" << std::endl; -#endif - } - - public: - static FrameIndexCache& instance() { - static FrameIndexCache cache; - return cache; - } - - std::unordered_map::iterator find(const std::string& filename) { - std::shared_lock read_lock(rw_mutex_); - return index_cache_.find(filename); - } - - std::unordered_map::iterator end() { - std::shared_lock read_lock(rw_mutex_); - return index_cache_.end(); - } - - void insert(const std::string& filename, const FrameIndex& index) { - std::unique_lock write_lock(rw_mutex_); - index_cache_[filename] = index; - } -}; - - struct VideoSampleDesc { VideoSampleDesc(const VideoFileMeta *video_file_meta = nullptr, int start = -1, int end = -1, int stride = -1) : video_file_meta_(video_file_meta), start_(start), end_(end), stride_(stride), frame_idxs_() {} @@ -280,15 +236,13 @@ class VideoLoaderDecoder : public Loader { LOG_LINE << "Invalid video file: " << entry.filename << std::endl; continue; } - auto it = FrameIndexCache::instance().find(entry.filename); - if (it == FrameIndexCache::instance().end()) { + auto index = FrameIndexCache::instance().GetOrBuild(entry.filename, [&]() { LOG_LINE << "Building index for " << entry.filename << std::endl; decoder->BuildIndex(); - FrameIndexCache::instance().insert(entry.filename, decoder->GetIndex()); - } else { - LOG_LINE << "Reusing index for " << entry.filename << std::endl; - decoder->SetIndex(it->second); - } + return decoder->GetIndex(); + }); + if (!decoder->HasIndex()) + decoder->SetIndex(*index); int64_t num_frames = decoder->NumFrames(); entry.start_frame = 0; entry.end_frame = num_frames; @@ -613,15 +567,13 @@ class VideoReaderDecoder } LOG_LINE << "Initialized decoder to " << decoder_->Filename() << " ptr: " << decoder_.get() << " num_frames: " << decoder_->NumFrames() << std::endl; - auto it = FrameIndexCache::instance().find(filename); - if (it == FrameIndexCache::instance().end()) { + auto index = FrameIndexCache::instance().GetOrBuild(filename, [&]() { LOG_LINE << "Building index for " << filename << std::endl; decoder_->BuildIndex(); - FrameIndexCache::instance().insert(filename, decoder_->GetIndex()); - } else { - LOG_LINE << "Reusing index for " << filename << std::endl; - decoder_->SetIndex(it->second); - } + return decoder_->GetIndex(); + }); + if (!decoder_->HasIndex()) + decoder_->SetIndex(*index); } else { LOG_LINE << "Reusing decoder for " << decoder_->Filename() << " ptr: " << decoder_.get() << " num_frames: " << decoder_->NumFrames() << std::endl; From 31f6173293e56c194ddcd6e2ef0ab2b063d1e566 Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Thu, 22 Oct 2026 17:28:44 +0200 Subject: [PATCH 20/46] Wait for pending copies before a GPU video decoder frees its frame buffers FramesDecoderGpu's frame_buffer_ is returned to the memory pool without stream ordering when the decoder is destroyed (the reader replaces its decoder on every file switch), while an asynchronous copy out of it may still be pending on the decoder's stream. Another stream could then be given that memory. The destructor now synchronizes the stream first. Signed-off-by: Joaquin Anton Guirao --- dali/operators/video/frames_decoder_gpu.cc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/dali/operators/video/frames_decoder_gpu.cc b/dali/operators/video/frames_decoder_gpu.cc index d209bd99b18..ea8899ddaeb 100644 --- a/dali/operators/video/frames_decoder_gpu.cc +++ b/dali/operators/video/frames_decoder_gpu.cc @@ -901,6 +901,12 @@ void FramesDecoderGpu::Flush() { } FramesDecoderGpu::~FramesDecoderGpu() { + // frame_buffer_ goes back to the memory pool right after this, without stream ordering: wait + // for the copies out of it (CopyFrame, copyD2D) that may still be pending on stream_. Nothing + // is thrown from the destructor: a failing sync means the CUDA context is already broken, and + // that error is reported by the call that caused it. + if (cudaStreamSynchronize(stream_) != cudaSuccess) + (void)cudaGetLastError(); filtered_packet_.reset(); bsfc_.reset(); } From 65579016774b4d59f2d3d71bba6cf1b15681ecf8 Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Thu, 22 Oct 2026 17:29:01 +0200 Subject: [PATCH 21/46] Make the video decoders' FFmpeg helpers safe to call from several threads av_error_string formatted into a function-static buffer, and every decoder constructor set FFmpeg's process-wide log level. Use a local buffer, and set the log level once. Signed-off-by: Joaquin Anton Guirao --- dali/operators/video/frames_decoder_base.cc | 15 +++++++++++++-- dali/operators/video/video_utils.cc | 4 ++-- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/dali/operators/video/frames_decoder_base.cc b/dali/operators/video/frames_decoder_base.cc index b7c27f3989a..01e3ba868f5 100644 --- a/dali/operators/video/frames_decoder_base.cc +++ b/dali/operators/video/frames_decoder_base.cc @@ -15,6 +15,7 @@ #include "dali/operators/video/frames_decoder_base.h" #include #include +#include #include "dali/core/error_handling.h" #include "dali/core/small_vector.h" #include "dali/operators/video/video_utils.h" @@ -212,8 +213,18 @@ bool FramesDecoderBase::CheckDimensions() { return true; } +namespace { + +// FFmpeg's log level is process-wide; set it once, not from every decoder (several threads) +void SetAvLogLevel() { + static std::once_flag once; + std::call_once(once, [] { av_log_set_level(AV_LOG_ERROR); }); +} + +} // namespace + FramesDecoderBase::FramesDecoderBase(const std::string &filename, DALIImageType image_type) { - av_log_set_level(AV_LOG_ERROR); + SetAvLogLevel(); filename_ = filename; DALI_ENFORCE(image_type == DALI_YCbCr || image_type == DALI_RGB, make_string("Invalid image type: ", image_type)); @@ -234,7 +245,7 @@ FramesDecoderBase::FramesDecoderBase(const std::string &filename, DALIImageType } FramesDecoderBase::FramesDecoderBase(const char *memory_file, size_t memory_file_size, std::string_view source_info, DALIImageType image_type) { - av_log_set_level(AV_LOG_ERROR); + SetAvLogLevel(); filename_ = source_info; DALI_ENFORCE(image_type == DALI_YCbCr || image_type == DALI_RGB, make_string("Invalid image type: ", image_type)); diff --git a/dali/operators/video/video_utils.cc b/dali/operators/video/video_utils.cc index d0c4b5392b0..c17ec31d89b 100644 --- a/dali/operators/video/video_utils.cc +++ b/dali/operators/video/video_utils.cc @@ -19,6 +19,7 @@ #include #include #include +#include namespace dali { @@ -148,8 +149,7 @@ std::vector GetVideoFiles(const std::string& file_root, } std::string av_error_string(int ret) { - static char msg[AV_ERROR_MAX_STRING_SIZE]; - memset(msg, 0, sizeof(msg)); + char msg[AV_ERROR_MAX_STRING_SIZE] = {}; // not static: decoders run on several threads return std::string(av_make_error_string(msg, AV_ERROR_MAX_STRING_SIZE, ret)); } From 4fae8ce5fea0dacb37e5bb5ddea45fa3ae6c6a87 Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Thu, 22 Oct 2026 17:56:26 +0200 Subject: [PATCH 22/46] Restructure the video reader's decoding into planned decode lanes VideoReaderDecoder::Prefetch decoded the batch in one loop with one decoder, one stream and shared scratch. It now plans the batch first, as the NemoAsr reader prepares its batch before decoding it: samples of one file that a decoder can decode in a row without seeking form a run, runs are assigned to decode lanes (longest first, preferring a lane whose decoder already holds the file), and every lane decodes its runs with its own decoder, stream, constant frame and scratch. With one lane, as in this change, the samples of a batch are grouped by file, so files alternating within a batch are reopened less often. A sample repeated in the batch is decoded once, and a sample the loader repeats in the next batch (pad_last_batch) is not decoded again while the previous batch may still be read. An error is reported for the first failing sample in the batch. FramesDecoderBase::HandleBoundary becomes static and the seek threshold a named constant, so that the planner can use them without a decoder. Signed-off-by: Joaquin Anton Guirao --- dali/operators/video/frames_decoder_base.cc | 5 +- dali/operators/video/frames_decoder_base.h | 12 +- .../video/reader/video_reader_decoder_op.cc | 551 ++++++++++++++---- 3 files changed, 454 insertions(+), 114 deletions(-) diff --git a/dali/operators/video/frames_decoder_base.cc b/dali/operators/video/frames_decoder_base.cc index 01e3ba868f5..339e5db0119 100644 --- a/dali/operators/video/frames_decoder_base.cc +++ b/dali/operators/video/frames_decoder_base.cc @@ -527,13 +527,12 @@ void FramesDecoderBase::SeekFrame(int frame_id) { assert(next_frame_idx_ >= 0); // If we are seeking to a frame that is before the current frame, - // or we are seeking to a frame that is more than MINIMUM_SEEK_LEAP frames away, + // or we are seeking to a frame that is more than kMinimumSeekLeap frames away, // or the current frame index is invalid (e.g. end of file), // we will to seek to the nearest keyframe first LOG_LINE << "SeekFrame: frame_id=" << frame_id << ", next_frame_idx=" << next_frame_idx_ << std::endl; - constexpr int MINIMUM_SEEK_LEAP = 10; - if (frame_id < next_frame_idx_ || frame_id > next_frame_idx_ + MINIMUM_SEEK_LEAP) { + if (frame_id < next_frame_idx_ || frame_id > next_frame_idx_ + kMinimumSeekLeap) { // If we have an index, we can seek to the nearest keyframe first if (HasIndex()) { LOG_LINE << "Using index to find nearest keyframe" << std::endl; diff --git a/dali/operators/video/frames_decoder_base.h b/dali/operators/video/frames_decoder_base.h index 466c43037bc..2b7b0d25121 100644 --- a/dali/operators/video/frames_decoder_base.h +++ b/dali/operators/video/frames_decoder_base.h @@ -262,6 +262,14 @@ class DLL_PUBLIC FramesDecoderBase { */ virtual void SeekFrame(int frame_id); + /** + * @brief Seeking forward by at most this many frames decodes through them instead of seeking. + * + * With an index, a decoder also decodes through to any later frame that has the same last + * keyframe as its next frame (see SeekFrame). + */ + static constexpr int kMinimumSeekLeap = 10; + /** * @brief Handles boundary conditions for a given frame index. * @@ -270,8 +278,8 @@ class DLL_PUBLIC FramesDecoderBase { * @param roi_start The start of the region of interest. * @param roi_end The end of the region of interest. */ - int HandleBoundary(boundary::BoundaryType boundary_type, int frame_id, - int roi_start, int roi_end); + static int HandleBoundary(boundary::BoundaryType boundary_type, int frame_id, + int roi_start, int roi_end); /** * @brief Decodes a collection of frames, not necessarily in ascending order, applying a boundary type diff --git a/dali/operators/video/reader/video_reader_decoder_op.cc b/dali/operators/video/reader/video_reader_decoder_op.cc index 5b917d5ca44..cc4fff83433 100644 --- a/dali/operators/video/reader/video_reader_decoder_op.cc +++ b/dali/operators/video/reader/video_reader_decoder_op.cc @@ -12,7 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include +#include +#include +#include +#include +#include #include +#include +#include +#include #include #include "dali/core/boundary.h" @@ -25,6 +34,7 @@ #include "dali/operators/video/frames_decoder_gpu.h" #include "dali/operators/video/reader/frame_index_cache.h" #include "dali/operators/video/video_utils.h" +#include "dali/pipeline/util/thread_pool.h" #if NVML_ENABLED #include "dali/util/nvml.h" #endif @@ -59,6 +69,9 @@ struct VideoSample : public VideoSampleDesc { Tensor data_; std::vector timestamps_; std::vector frame_idx_; + // Set once decoded. With pad_last_batch, the loader returns the last sample again, also in the + // next batch, which must not decode it again while the previous batch may still be read. + bool decoded_ = false; }; enum class FileListFormat { @@ -414,7 +427,7 @@ class VideoReaderDecoder DALI_ENFORCE(image_type_ == DALI_RGB || image_type_ == DALI_YCbCr, make_string("Invalid image_type: ", image_type_)); - constant_frame_.set_pinned(std::is_same_v); + InitLanes(spec); } ~VideoReaderDecoder() override { @@ -440,8 +453,7 @@ class VideoReaderDecoder return; } #endif - int device_id = spec.GetArgument("device_id"); - cuda_stream_ = CUDAStreamPool::instance().Get(device_id); + use_streams_ = true; } } @@ -551,122 +563,442 @@ class VideoReaderDecoder void Prefetch() override { Base::Prefetch(); - auto ¤t_batch = prefetched_batch_queue_[curr_batch_producer_]; - size_t i = 0; - for (auto &sample : current_batch) { - LOG_LINE << "Processing sample " << i++ << " with filename " << sample->video_file_meta_->filename - << " and previous decoder " << decoder_.get() << " filename " - << (decoder_ ? decoder_->Filename() : "none") << std::endl; - auto prev_filename = decoder_ ? decoder_->Filename() : ""; - const auto &filename = sample->video_file_meta_->filename; - if (prev_filename != filename) { - if constexpr (std::is_same_v) { - decoder_ = std::make_unique(filename, image_type_); - } else { - decoder_ = std::make_unique(filename, cuda_stream_, image_type_); - } - LOG_LINE << "Initialized decoder to " << decoder_->Filename() << " ptr: " << decoder_.get() - << " num_frames: " << decoder_->NumFrames() << std::endl; - auto index = FrameIndexCache::instance().GetOrBuild(filename, [&]() { - LOG_LINE << "Building index for " << filename << std::endl; - decoder_->BuildIndex(); - return decoder_->GetIndex(); - }); - if (!decoder_->HasIndex()) - decoder_->SetIndex(*index); - } else { - LOG_LINE << "Reusing decoder for " << decoder_->Filename() << " ptr: " << decoder_.get() - << " num_frames: " << decoder_->NumFrames() << std::endl; + PlanBatch(); + AssignRuns(); + DecodeBatch(); + LOG_LINE << "Prefetch done" << std::endl; + } + + private: + /** + * @brief A sample to decode, with the frames it decodes (for planning only). + */ + struct SampleTask { + VideoSample *sample; + int batch_idx; // first position of the sample in the batch + int first; // smallest frame index decoded; -1 if none (padding only) + int last; // largest frame index decoded; -1 if none + }; + + /** + * @brief Samples of one file that one decoder decodes in a row, without seeking. + */ + struct DecodeRun { + const std::string *filename; + std::vector tasks; // indices into tasks_, in decoding order + int first; + int last; + int batch_idx; // the smallest position of its samples in the batch + }; + + /** + * @brief A part of a batch's decoding that runs on one thread. + * + * A lane keeps its decoder while consecutive samples come from the same file, so that they + * are decoded without seeking, and decodes on its own stream, so that lanes do not wait for + * each other's work. Which samples a lane decodes is decided before decoding starts, from the + * samples alone, so a decoder's history (and with it, the output) never depends on thread + * timing. + */ + struct DecodeLane { + CUDAStreamLease stream; // GPU only; empty: the default stream + std::unique_ptr decoder; // after `stream`: destroyed (and synced) first + Tensor constant_frame; + std::vector frame_idxs; // scratch + // Planning: the runs of the current batch, in decoding order, the estimated work, and where + // the lane's decoder will be after them + std::vector runs; + int64_t load = 0; + std::string next_filename; + int next_frame = -1; + // Errors of the current batch: (position in the batch, exception) + std::vector> errors; + }; + + // Planning costs, in frames decoded: opening a decoder (and its first frame) costs about as + // much as decoding a few frames + static constexpr int64_t kOpenCost = 16; + + int NumLanes(const OpSpec &spec) { + return 1; // more lanes in the next step + } + + void InitLanes(const OpSpec &spec) { + int num_lanes = NumLanes(spec); + lanes_.resize(num_lanes); + int device_id = spec.GetArgument("device_id"); + for (auto &lane : lanes_) { + lane.constant_frame.set_pinned(std::is_same_v); + if (use_streams_) + lane.stream = CUDAStreamPool::instance().Get(device_id); + } + if (num_lanes > 1) + thread_pool_ = std::make_unique(num_lanes, device_id, false, + "VideoReaderDecoder"); + } + + /** + * @brief The range of frames a sample decodes: [first, last], or first = last = -1 if none. + */ + void FrameRange(const VideoSample &sample, int &first, int &last) const { + int roi_start = sample.video_file_meta_->start_frame; + int roi_end = sample.video_file_meta_->end_frame; + first = -1; + last = -1; + auto add = [&](int frame) { + int idx = FramesDecoderBase::HandleBoundary(boundary_type_, frame, roi_start, roi_end); + if (idx < 0) + return; // a constant frame + first = first < 0 ? idx : std::min(first, idx); + last = std::max(last, idx); + }; + if (!sample.frame_idxs_.empty()) { + for (int frame : sample.frame_idxs_) + add(frame); + } else { + for (int frame = sample.start_; frame < sample.end_; frame += sample.stride_) + add(frame); + } + } + + /** + * @brief Whether a decoder whose next frame is `next` decodes through to `frame` without + * seeking (see FramesDecoderBase::SeekFrame). + */ + static bool ReachesWithoutSeek(const FrameIndex *index, int next, int frame) { + if (next < 0 || frame < next) + return false; + if (frame - next <= FramesDecoderBase::kMinimumSeekLeap) + return true; + return index && frame < static_cast(index->size()) && + (*index)[frame].last_keyframe_id == (*index)[next].last_keyframe_id; + } + + /** + * @brief Groups the batch's samples into runs: samples of one file that one decoder decodes in + * a row without seeking. + * + * A sample repeated in the batch (pad_last_batch) is decoded once, and a sample decoded in an + * earlier batch (padding repeats the last sample, also in the next batch) is not decoded again. + */ + void PlanBatch() { + auto &batch = prefetched_batch_queue_[curr_batch_producer_]; + tasks_.clear(); + runs_.clear(); + for (int i = 0; i < static_cast(batch.size()); i++) { + auto *sample = batch[i].get(); + if (sample->decoded_) + continue; + bool repeated = false; + for (auto &task : tasks_) + repeated = repeated || task.sample == sample; + if (repeated) + continue; + SampleTask task{sample, i, -1, -1}; + try { + FrameRange(*sample, task.first, task.last); + } catch (...) { + // decoding the sample reports the error, in its lane } - DALI_ENFORCE(decoder_->IsValid(), - make_string("Invalid decoder for filename ", filename)); - - int64_t num_frames = sample->frame_idxs_.empty() - ? (sample->end_ - sample->start_ + sample->stride_ - 1) / sample->stride_ - : static_cast(sample->frame_idxs_.size()); - sample->data_.Resize( - {num_frames, decoder_->Height(), decoder_->Width(), decoder_->Channels()}, DALI_UINT8); - sample->data_.SetSourceInfo(decoder_->Filename()); - sample->data_.SetLayout("FHWC"); - - const uint8_t *constant_frame = - boundary_type_ == boundary::BoundaryType::CONSTANT ? - ConstantFrame(constant_frame_, decoder_->FrameShape(), make_cspan(fill_value_), - cuda_stream_, true) : - nullptr; - if (has_timestamps_) { - sample->timestamps_.resize(num_frames); - } else { - sample->timestamps_.clear(); + tasks_.push_back(task); + } + + std::vector order(tasks_.size()); + std::iota(order.begin(), order.end(), 0); + std::sort(order.begin(), order.end(), [&](int a, int b) { + const auto &ta = tasks_[a], &tb = tasks_[b]; + const auto &fa = ta.sample->video_file_meta_->filename; + const auto &fb = tb.sample->video_file_meta_->filename; + return std::tie(fa, ta.first, ta.batch_idx) < std::tie(fb, tb.first, tb.batch_idx); + }); + FrameIndexCache::IndexPtr index; + size_t file_runs = 0; // the first run of the current file + for (int t : order) { + auto &task = tasks_[t]; + const auto &filename = task.sample->video_file_meta_->filename; + if (runs_.empty() || *runs_.back().filename != filename) { + file_runs = runs_.size(); + index = FrameIndexCache::instance().Find(filename); } - if (!sample->frame_idxs_.empty()) { - LOG_LINE << "Decoding frames (uniform) num_frames=" << num_frames - << ", frame_idxs=[" << sample->frame_idxs_.front() << ".." - << sample->frame_idxs_.back() << "]" - << ", filename=" << sample->video_file_meta_->filename - << ", label=" << sample->video_file_meta_->label - << ", boundary_type=" << to_string(boundary_type_) << std::endl; + // join a run of the file that a decoder continues to this sample without seeking + auto run = std::find_if(runs_.begin() + file_runs, runs_.end(), [&](const DecodeRun &r) { + return task.first < 0 || ReachesWithoutSeek(index.get(), r.last + 1, task.first); + }); + if (run != runs_.end()) { + run->tasks.push_back(t); + run->last = std::max(run->last, task.last); + run->batch_idx = std::min(run->batch_idx, task.batch_idx); } else { - LOG_LINE << "Decoding frames start=" << sample->start_ << ", end=" << sample->end_ - << ", stride=" << sample->stride_ << ", num_frames=" << num_frames - << ", filename=" << sample->video_file_meta_->filename - << ", label=" << sample->video_file_meta_->label - << ", start=" << sample->video_file_meta_->start_frame - << ", end=" << sample->video_file_meta_->end_frame - << ", boundary_type=" << to_string(boundary_type_) << std::endl; + runs_.push_back({&filename, {t}, task.first, task.last, task.batch_idx}); + } + } + } + + /** + * @brief Assigns the runs to lanes: longest first, each to the lane that would finish it + * first, counting what the lane's decoder saves (no reopening, no seeking). + * + * A lane then decodes the runs of one file together, by first frame: first the file its + * decoder holds (starting with the run that continues where the decoder is), then the others in + * batch order. + */ + void AssignRuns() { + for (auto &lane : lanes_) { + lane.runs.clear(); + lane.load = 0; + lane.next_filename = lane.decoder ? lane.decoder->Filename() : std::string(); + lane.next_frame = lane.decoder ? lane.decoder->NextFrameIdx() : -1; + } + std::vector> start(lanes_.size()); // where the decoders are + for (size_t l = 0; l < lanes_.size(); l++) + start[l] = {lanes_[l].next_filename, lanes_[l].next_frame}; + struct RunCost { + int run; + const FrameIndex *index; + int64_t frames; // frames decoded, from the first to the last + int64_t preroll; // frames decoded before the first one, after a seek + }; + std::vector costs; + std::vector indices; // keeps the indices alive while planning + for (int r = 0; r < static_cast(runs_.size()); r++) { + const auto &run = runs_[r]; + indices.push_back(FrameIndexCache::instance().Find(*run.filename)); + const FrameIndex *index = indices.back().get(); + int64_t frames = run.first < 0 ? 0 : run.last - run.first + 1; + int64_t preroll = 0; + if (index && run.first >= 0 && run.first < static_cast(index->size())) + preroll = run.first - (*index)[run.first].last_keyframe_id; + costs.push_back({r, index, frames, preroll}); + } + std::stable_sort(costs.begin(), costs.end(), [](const RunCost &a, const RunCost &b) { + return a.frames + a.preroll > b.frames + b.preroll; + }); + for (const auto &cost : costs) { + const auto &run = runs_[cost.run]; + int best = 0; + int64_t best_end = std::numeric_limits::max(); + for (int l = 0; l < static_cast(lanes_.size()); l++) { + const auto &lane = lanes_[l]; + int64_t work = cost.frames; + if (lane.next_filename != *run.filename) + work += kOpenCost + cost.preroll; + else if (!ReachesWithoutSeek(cost.index, lane.next_frame, run.first)) + work += cost.preroll; + if (lane.load + work < best_end) { + best = l; + best_end = lane.load + work; + } + } + auto &lane = lanes_[best]; + lane.runs.push_back(cost.run); + lane.load = best_end; + lane.next_filename = *run.filename; + lane.next_frame = run.last < 0 ? lane.next_frame : run.last + 1; + } + for (size_t l = 0; l < lanes_.size(); l++) { + auto &runs = lanes_[l].runs; + auto continues = [&](int r) { + const auto &run = runs_[r]; + return *run.filename == start[l].first && + ReachesWithoutSeek(FrameIndexCache::instance().Find(*run.filename).get(), + start[l].second, run.first); + }; + // files in the order of their first sample in the batch, the decoder's file first + std::map file_order; + for (int r : runs) { + auto [it, inserted] = file_order.emplace(*runs_[r].filename, runs_[r].batch_idx); + if (!inserted) + it->second = std::min(it->second, runs_[r].batch_idx); + } + auto held = file_order.find(start[l].first); + if (held != file_order.end()) + held->second = -1; + std::sort(runs.begin(), runs.end(), [&](int a, int b) { + const auto &ra = runs_[a], &rb = runs_[b]; + return std::make_tuple(file_order[*ra.filename], *ra.filename, ra.first) < + std::make_tuple(file_order[*rb.filename], *rb.filename, rb.first); + }); + auto it = std::find_if(runs.begin(), runs.end(), continues); + if (it != runs.end()) + std::rotate(runs.begin(), it, it + 1); + } + } + + /** + * @brief Decodes the batch, every lane on its own thread, and waits for all of them. + * + * An error is reported for the first failing sample in the batch, whatever the lanes' timing. + */ + void DecodeBatch() { + std::vector active; + for (auto &lane : lanes_) { + if (!lane.runs.empty()) + active.push_back(&lane); + } + if (active.size() == 1 || !thread_pool_) { + for (auto *lane : active) + RunLane(*lane); + } else { + for (auto *lane : active) + thread_pool_->AddWork([this, lane](int) { RunLane(*lane); }, lane->load); + thread_pool_->RunAll(); + } + const std::pair *first_error = nullptr; + for (auto *lane : active) { + for (auto &error : lane->errors) { + if (!first_error || error.first < first_error->first) + first_error = &error; + } + } + if (first_error) { + auto error = first_error->second; + for (auto *lane : active) + lane->errors.clear(); + std::rethrow_exception(error); + } + } + + void RunLane(DecodeLane &lane) { + lane.errors.clear(); + for (int r : lane.runs) { + for (int t : runs_[r].tasks) { + auto &task = tasks_[t]; + try { + DecodeSample(*task.sample, lane); + task.sample->decoded_ = true; + } catch (...) { + lane.errors.emplace_back(task.batch_idx, std::current_exception()); + lane.decoder.reset(); // in an unknown state; its destructor waits for its stream + } } - int roi_start = sample->video_file_meta_->start_frame; - int roi_end = sample->video_file_meta_->end_frame; - if (frame_num_policy_ == FrameNumPolicy::Sequence) { - sample->frame_idx_.resize(num_frames); - if (!sample->frame_idxs_.empty()) { - // Uniform indices are always within [roi_start, roi_end-1], so HandleBoundary - // is a no-op here — but we call it for consistency with the stride path below. - for (int64_t i = 0; i < num_frames; ++i) { - sample->frame_idx_[i] = static_cast(decoder_->HandleBoundary( - boundary_type_, sample->frame_idxs_[i], roi_start, roi_end)); - } - } else { - for (int64_t i = 0; i < num_frames; ++i) { - sample->frame_idx_[i] = static_cast(decoder_->HandleBoundary( - boundary_type_, - static_cast(sample->start_ + i * sample->stride_), - roi_start, roi_end)); - } + } + if constexpr (std::is_same_v) { + // the samples are read on the workspace stream once the batch is consumed + cudaStream_t stream = lane.stream ? lane.stream.get() : 0; + if (lane.errors.empty()) { + try { + CUDA_CALL(cudaStreamSynchronize(stream)); + } catch (...) { + lane.errors.emplace_back(std::numeric_limits::max(), std::current_exception()); } + } else if (cudaStreamSynchronize(stream) != cudaSuccess) { + (void)cudaGetLastError(); // already failing: report the sample's error + } + } + } + + /** + * @brief Decodes one sample on a lane, reusing the lane's decoder if it has the same file. + */ + void DecodeSample(VideoSample &sample, DecodeLane &lane) { + const auto &filename = sample.video_file_meta_->filename; + LOG_LINE << "Processing sample with filename " << filename << " and previous decoder " + << lane.decoder.get() << " filename " + << (lane.decoder ? lane.decoder->Filename() : "none") << std::endl; + if (!lane.decoder || lane.decoder->Filename() != filename) { + lane.decoder.reset(); // first: its frame buffers are freed before the new ones + if constexpr (std::is_same_v) { + lane.decoder = std::make_unique(filename, image_type_); } else { - sample->frame_idx_.clear(); + lane.decoder = std::make_unique(filename, lane.stream, image_type_); } - if (!sample->frame_idxs_.empty()) { - // Uniform sampling: explicit frame indices already include ROI offset (start_frame) - decoder_->DecodeFrames(sample->data_.template mutable_data(), - make_cspan(sample->frame_idxs_), boundary_type_, constant_frame, - make_span(sample->timestamps_)); - } else if (roi_start != 0 || roi_end != decoder_->NumFrames()) { - frame_idxs_.clear(); - for (int frame_idx = sample->start_; frame_idx < sample->end_; - frame_idx += sample->stride_) { - frame_idxs_.push_back(decoder_->HandleBoundary( - boundary_type_, frame_idx, roi_start, roi_end)); + auto &decoder = *lane.decoder; + LOG_LINE << "Initialized decoder to " << decoder.Filename() << " ptr: " << &decoder + << " num_frames: " << decoder.NumFrames() << std::endl; + auto index = FrameIndexCache::instance().GetOrBuild(filename, [&]() { + LOG_LINE << "Building index for " << filename << std::endl; + decoder.BuildIndex(); + return decoder.GetIndex(); + }); + if (!decoder.HasIndex()) + decoder.SetIndex(*index); + } else { + LOG_LINE << "Reusing decoder for " << lane.decoder->Filename() << " ptr: " + << lane.decoder.get() << " num_frames: " << lane.decoder->NumFrames() + << std::endl; + } + auto &decoder = *lane.decoder; + DALI_ENFORCE(decoder.IsValid(), make_string("Invalid decoder for filename ", filename)); + + int64_t num_frames = sample.frame_idxs_.empty() + ? (sample.end_ - sample.start_ + sample.stride_ - 1) / sample.stride_ + : static_cast(sample.frame_idxs_.size()); + sample.data_.Resize( + {num_frames, decoder.Height(), decoder.Width(), decoder.Channels()}, DALI_UINT8); + sample.data_.SetSourceInfo(decoder.Filename()); + sample.data_.SetLayout("FHWC"); + + const uint8_t *constant_frame = + boundary_type_ == boundary::BoundaryType::CONSTANT ? + ConstantFrame(lane.constant_frame, decoder.FrameShape(), make_cspan(fill_value_), + lane.stream, true) : + nullptr; + if (has_timestamps_) { + sample.timestamps_.resize(num_frames); + } else { + sample.timestamps_.clear(); + } + if (!sample.frame_idxs_.empty()) { + LOG_LINE << "Decoding frames (uniform) num_frames=" << num_frames + << ", frame_idxs=[" << sample.frame_idxs_.front() << ".." + << sample.frame_idxs_.back() << "]" + << ", filename=" << sample.video_file_meta_->filename + << ", label=" << sample.video_file_meta_->label + << ", boundary_type=" << to_string(boundary_type_) << std::endl; + } else { + LOG_LINE << "Decoding frames start=" << sample.start_ << ", end=" << sample.end_ + << ", stride=" << sample.stride_ << ", num_frames=" << num_frames + << ", filename=" << sample.video_file_meta_->filename + << ", label=" << sample.video_file_meta_->label + << ", start=" << sample.video_file_meta_->start_frame + << ", end=" << sample.video_file_meta_->end_frame + << ", boundary_type=" << to_string(boundary_type_) << std::endl; + } + int roi_start = sample.video_file_meta_->start_frame; + int roi_end = sample.video_file_meta_->end_frame; + if (frame_num_policy_ == FrameNumPolicy::Sequence) { + sample.frame_idx_.resize(num_frames); + if (!sample.frame_idxs_.empty()) { + // Uniform indices are always within [roi_start, roi_end-1], so HandleBoundary + // is a no-op here — but we call it for consistency with the stride path below. + for (int64_t i = 0; i < num_frames; ++i) { + sample.frame_idx_[i] = static_cast(decoder.HandleBoundary( + boundary_type_, sample.frame_idxs_[i], roi_start, roi_end)); } - decoder_->DecodeFrames(sample->data_.template mutable_data(), - make_cspan(frame_idxs_), boundary_type_, constant_frame, - make_span(sample->timestamps_)); } else { - decoder_->DecodeFrames(sample->data_.template mutable_data(), sample->start_, - sample->end_, sample->stride_, boundary_type_, constant_frame, - make_span(sample->timestamps_)); + for (int64_t i = 0; i < num_frames; ++i) { + sample.frame_idx_[i] = static_cast(decoder.HandleBoundary( + boundary_type_, + static_cast(sample.start_ + i * sample.stride_), + roi_start, roi_end)); + } } - LOG_LINE << "Decoding frames done" << std::endl; + } else { + sample.frame_idx_.clear(); } - - if (cuda_stream_) { - CUDA_CALL(cudaStreamSynchronize(cuda_stream_.get())); + if (!sample.frame_idxs_.empty()) { + // Uniform sampling: explicit frame indices already include ROI offset (start_frame) + decoder.DecodeFrames(sample.data_.template mutable_data(), + make_cspan(sample.frame_idxs_), boundary_type_, constant_frame, + make_span(sample.timestamps_)); + } else if (roi_start != 0 || roi_end != decoder.NumFrames()) { + lane.frame_idxs.clear(); + for (int frame_idx = sample.start_; frame_idx < sample.end_; + frame_idx += sample.stride_) { + lane.frame_idxs.push_back(decoder.HandleBoundary( + boundary_type_, frame_idx, roi_start, roi_end)); + } + decoder.DecodeFrames(sample.data_.template mutable_data(), + make_cspan(lane.frame_idxs), boundary_type_, constant_frame, + make_span(sample.timestamps_)); + } else { + decoder.DecodeFrames(sample.data_.template mutable_data(), sample.start_, + sample.end_, sample.stride_, boundary_type_, constant_frame, + make_span(sample.timestamps_)); } - LOG_LINE << "Prefetch done" << std::endl; + LOG_LINE << "Decoding frames done" << std::endl; } - private: FrameNumPolicy frame_num_policy_; bool has_timestamps_; boundary::BoundaryType boundary_type_; @@ -674,10 +1006,11 @@ class VideoReaderDecoder std::vector fill_value_; bool has_labels_ = false; - Tensor constant_frame_; - CUDAStreamLease cuda_stream_; - std::unique_ptr decoder_; // keeping one decoder open. - std::vector frame_idxs_; + bool use_streams_ = false; // GPU: a stream per lane (false: the default stream, old drivers) + std::vector tasks_; + std::vector runs_; + std::vector lanes_; + std::unique_ptr thread_pool_; // with more than one lane }; DALI_SCHEMA(experimental__readers__Video) From f10ac01521dbc6902fb665f20102f7ba73269ad9 Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Thu, 22 Oct 2026 18:25:26 +0200 Subject: [PATCH 23/46] Decode the video reader's batches with up to num_threads decode lanes Every decode lane now runs on its own thread of an operator-owned pool, as the NemoAsr reader decodes its batch: up to num_threads lanes, no more than the batch size, and on the GPU at most two per NVDEC engine (cuvidGetDecoderCaps reports the engine count). With one engine, two lanes already keep it busy while a lane does its host work between pictures; more lanes add memory, one decoder each. The planning from the previous change makes the output independent of the number of lanes: which samples a lane decodes, and so its decoder's history, depends only on the samples. Tests check that the output is the same with 1 and 4 threads for consecutive, sparse, overlapping, shuffled, uniformly sampled and padded samples, and that a sample repeated by pad_last_batch into the next batch is the same sample. Signed-off-by: Joaquin Anton Guirao --- dali/operators/video/frames_decoder_gpu.cc | 14 ++++ dali/operators/video/frames_decoder_gpu.h | 5 ++ .../video/reader/video_reader_decoder_op.cc | 26 +++++- dali/test/python/decoder/test_video.py | 83 +++++++++++++++++++ 4 files changed, 126 insertions(+), 2 deletions(-) diff --git a/dali/operators/video/frames_decoder_gpu.cc b/dali/operators/video/frames_decoder_gpu.cc index ea8899ddaeb..c77f10bd801 100644 --- a/dali/operators/video/frames_decoder_gpu.cc +++ b/dali/operators/video/frames_decoder_gpu.cc @@ -22,6 +22,7 @@ #include #include "dali/core/error_handling.h" #include "dali/core/cuda_error.h" +#include "dali/core/device_guard.h" #include "dali/pipeline/data/backend.h" #include "dali/pipeline/data/tensor.h" #include "dali/operators/video/color_space.h" @@ -911,6 +912,19 @@ FramesDecoderGpu::~FramesDecoderGpu() { bsfc_.reset(); } +int FramesDecoderGpu::NumEngines(int device_id) { + if (!cuvidInitChecked()) + return 0; + DeviceGuard guard(device_id); + CUVIDDECODECAPS decoder_caps = {}; + decoder_caps.eCodecType = cudaVideoCodec_H264; + decoder_caps.eChromaFormat = cudaVideoChromaFormat_420; + decoder_caps.nBitDepthMinus8 = 0; + if (cuvidGetDecoderCaps(&decoder_caps) != CUDA_SUCCESS || !decoder_caps.bIsSupported) + return 0; + return decoder_caps.nNumNVDECs; +} + bool FramesDecoderGpu::SupportsHevc() { CUVIDDECODECAPS decoder_caps = {}; decoder_caps.eCodecType = cudaVideoCodec_HEVC; diff --git a/dali/operators/video/frames_decoder_gpu.h b/dali/operators/video/frames_decoder_gpu.h index a6da7b2adc6..2e00e0fec5e 100644 --- a/dali/operators/video/frames_decoder_gpu.h +++ b/dali/operators/video/frames_decoder_gpu.h @@ -169,6 +169,11 @@ class DLL_PUBLIC FramesDecoderGpu : public FramesDecoderBase { static bool SupportsHevc(); + /** + * @brief Number of NVDEC engines of the device; 0 if unknown. + */ + static int NumEngines(int device_id); + static bool SupportsCodec(AVCodecID codec_id, uint8_t bit_depth = 8); void InitGpuDecoder(CUVIDEOFORMAT *video_format); diff --git a/dali/operators/video/reader/video_reader_decoder_op.cc b/dali/operators/video/reader/video_reader_decoder_op.cc index cc4fff83433..2a616c24ce8 100644 --- a/dali/operators/video/reader/video_reader_decoder_op.cc +++ b/dali/operators/video/reader/video_reader_decoder_op.cc @@ -619,8 +619,23 @@ class VideoReaderDecoder // much as decoding a few frames static constexpr int64_t kOpenCost = 16; + // GPU: decode lanes per NVDEC engine. More sessions than engines keep an engine busy while a + // lane does its host work between pictures; many more only add memory. + static constexpr int kLanesPerEngine = 2; + + /** + * @brief Number of decode lanes: num_threads, but no more than the batch size and, on the GPU, + * kLanesPerEngine per NVDEC engine. + */ int NumLanes(const OpSpec &spec) { - return 1; // more lanes in the next step + int lanes = std::min(spec.GetArgument("num_threads"), + spec.GetArgument("max_batch_size")); + if constexpr (std::is_same_v) { + int engines = FramesDecoderGpu::NumEngines(spec.GetArgument("device_id")); + if (engines > 0) + lanes = std::min(lanes, kLanesPerEngine * engines); + } + return std::max(lanes, 1); } void InitLanes(const OpSpec &spec) { @@ -740,7 +755,7 @@ class VideoReaderDecoder * * A lane then decodes the runs of one file together, by first frame: first the file its * decoder holds (starting with the run that continues where the decoder is), then the others in - * batch order. + * batch order. A lane without runs releases its decoder. */ void AssignRuns() { for (auto &lane : lanes_) { @@ -821,6 +836,8 @@ class VideoReaderDecoder auto it = std::find_if(runs.begin(), runs.end(), continues); if (it != runs.end()) std::rotate(runs.begin(), it, it + 1); + if (runs.empty() && lanes_.size() > 1) + lanes_[l].decoder.reset(); // do not keep its frame buffers and NVDEC decoder idle } } @@ -1032,6 +1049,11 @@ The following codecs are supported by the GPU backend only: * H.264/AVC * H.265/HEVC +The samples of a batch are decoded in parallel, by up to ``num_threads`` decoders (on the GPU, at +most two per NVDEC engine of the device). Consecutive samples of one file that can be decoded +without seeking are decoded by the same decoder, one after another. Every decoder has its own +frame buffers, so the memory used grows with the number of decoders. + The outputs of the operator are: video, [labels], [frame_num], [timestamps]. * ``video``: A sequence of frames with shape ``(F, H, W, C)`` where ``F`` is the number of frames in the sequence diff --git a/dali/test/python/decoder/test_video.py b/dali/test/python/decoder/test_video.py index ce755d8bc74..04bff455fc9 100644 --- a/dali/test/python/decoder/test_video.py +++ b/dali/test/python/decoder/test_video.py @@ -21,6 +21,7 @@ import glob import os import random +import tempfile from itertools import cycle from test_utils import ( get_dali_extra_path, @@ -1319,3 +1320,85 @@ def video_pipe(filenames): seed=123456, ) pipe.run() + + +# The reader decodes the samples of a batch in parallel, by up to num_threads decoders (decode +# lanes). Its output must not depend on how many there are. +_lanes_reader_cases = [ + dict(sequence_length=5), # consecutive sequences of each file + dict(sequence_length=4, stride=3, step=20), # sparse samples of each file + dict(sequence_length=5, step=2), # overlapping samples + dict(sequence_length=6, random_shuffle=True, initial_fill=8), + dict(sequence_length=6, uniform_sample=True), + dict(sequence_length=7, pad_mode="constant", fill_value=[30, 60, 90], enable_timestamps=True), + dict(sequence_length=5, pad_last_batch=True, num_shards=3, shard_id=2), +] + + +def _lanes_reader_files(device): + # H.264 is not enabled on the CPU backend + return cfr_files + vfr_files + (codec_files["h264"] if device == "gpu" else []) + + +@cartesian_params(["cpu", "gpu"], range(len(_lanes_reader_cases))) +def test_reader_output_independent_of_num_threads(device, case): + skip_if_m60() + kwargs = _lanes_reader_cases[case] + batch_size = 4 + + @pipeline_def(batch_size=batch_size, device_id=0, seed=42) + def pipe(): + # videos, frame numbers and, with enable_timestamps, timestamps + return tuple( + fn.experimental.readers.video( + filenames=_lanes_reader_files(device), + device=device, + enable_frame_num="sequence", + **kwargs, + ) + ) + + compare_pipelines( + pipe(num_threads=1), pipe(num_threads=4), batch_size=batch_size, N_iterations=8 + ) + + +@params("cpu", "gpu") +def test_reader_pad_last_batch_repeats_last_sample(device): + """With pad_last_batch, a shard's last sample is repeated, here as the whole next batch: the + same sample, which must not be decoded again while the earlier batch may still be in use.""" + skip_if_m60() + batch_size = 4 + files = _lanes_reader_files(device) + # 26 samples in 3 shards: shard 0 has samples 0-7 and is padded to 9 samples, so its third + # batch is sample 7, four times + with tempfile.NamedTemporaryFile("w", suffix=".txt", delete=False) as file_list: + for i in range(26): + start = (i // len(files)) * 4 + file_list.write(f"{files[i % len(files)]} {i} {start} {start + 4}\n") + + @pipeline_def(batch_size=batch_size, num_threads=4, device_id=0, prefetch_queue_depth=3) + def pipe(): + videos, labels = fn.experimental.readers.video( + file_list=file_list.name, + file_list_format="frames", + file_list_include_end=False, + device=device, + sequence_length=4, + pad_last_batch=True, + num_shards=3, + shard_id=0, + ) + return videos, labels + + p = pipe() + samples = [] + for _ in range(3): + videos, labels = p.run() + videos = videos.as_cpu() if device == "gpu" else videos + labels = labels.as_cpu() if device == "gpu" else labels + samples += [(np.array(videos[i]), int(np.array(labels[i])[0])) for i in range(batch_size)] + os.remove(file_list.name) + assert [label for _, label in samples] == list(range(8)) + [7] * 4 + for video, _ in samples[8:]: + np.testing.assert_array_equal(video, samples[7][0]) From 7e2d8f74aec347e1cf22191a77592b67d170129d Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Fri, 23 Oct 2026 08:57:24 +0200 Subject: [PATCH 24/46] Address review: check decode-lane test output against PyAV, naming fixes Compare the reader's frames to PyAV-decoded reference frames in the num_threads and pad_last_batch tests, not only to its own other runs. Use make_string in the frame index cache test, rename the public decoded_ member to decoded and ReachesWithoutSeek to CanReachWithoutSeek. Signed-off-by: Joaquin Anton Guirao --- .../video/reader/frame_index_cache_test.cc | 3 +- .../video/reader/video_reader_decoder_op.cc | 14 ++--- dali/test/python/decoder/test_video.py | 53 ++++++++++++++++++- 3 files changed, 60 insertions(+), 10 deletions(-) diff --git a/dali/operators/video/reader/frame_index_cache_test.cc b/dali/operators/video/reader/frame_index_cache_test.cc index 0c53693d58c..51539949bb2 100644 --- a/dali/operators/video/reader/frame_index_cache_test.cc +++ b/dali/operators/video/reader/frame_index_cache_test.cc @@ -21,6 +21,7 @@ #include #include "dali/operators/video/reader/frame_index_cache.h" +#include "dali/core/format.h" namespace dali { @@ -28,7 +29,7 @@ namespace { // Unique keys per test: the cache is process-wide std::string Key(const std::string &test, int i) { - return "frame_index_cache_test/" + test + "/" + std::to_string(i); + return make_string("frame_index_cache_test/", test, "/", i); } FrameIndex MakeIndex(const std::string &filename, int num_frames) { diff --git a/dali/operators/video/reader/video_reader_decoder_op.cc b/dali/operators/video/reader/video_reader_decoder_op.cc index 2a616c24ce8..bc5d0559f34 100644 --- a/dali/operators/video/reader/video_reader_decoder_op.cc +++ b/dali/operators/video/reader/video_reader_decoder_op.cc @@ -71,7 +71,7 @@ struct VideoSample : public VideoSampleDesc { std::vector frame_idx_; // Set once decoded. With pad_last_batch, the loader returns the last sample again, also in the // next batch, which must not decode it again while the previous batch may still be read. - bool decoded_ = false; + bool decoded = false; }; enum class FileListFormat { @@ -680,7 +680,7 @@ class VideoReaderDecoder * @brief Whether a decoder whose next frame is `next` decodes through to `frame` without * seeking (see FramesDecoderBase::SeekFrame). */ - static bool ReachesWithoutSeek(const FrameIndex *index, int next, int frame) { + static bool CanReachWithoutSeek(const FrameIndex *index, int next, int frame) { if (next < 0 || frame < next) return false; if (frame - next <= FramesDecoderBase::kMinimumSeekLeap) @@ -702,7 +702,7 @@ class VideoReaderDecoder runs_.clear(); for (int i = 0; i < static_cast(batch.size()); i++) { auto *sample = batch[i].get(); - if (sample->decoded_) + if (sample->decoded) continue; bool repeated = false; for (auto &task : tasks_) @@ -737,7 +737,7 @@ class VideoReaderDecoder } // join a run of the file that a decoder continues to this sample without seeking auto run = std::find_if(runs_.begin() + file_runs, runs_.end(), [&](const DecodeRun &r) { - return task.first < 0 || ReachesWithoutSeek(index.get(), r.last + 1, task.first); + return task.first < 0 || CanReachWithoutSeek(index.get(), r.last + 1, task.first); }); if (run != runs_.end()) { run->tasks.push_back(t); @@ -797,7 +797,7 @@ class VideoReaderDecoder int64_t work = cost.frames; if (lane.next_filename != *run.filename) work += kOpenCost + cost.preroll; - else if (!ReachesWithoutSeek(cost.index, lane.next_frame, run.first)) + else if (!CanReachWithoutSeek(cost.index, lane.next_frame, run.first)) work += cost.preroll; if (lane.load + work < best_end) { best = l; @@ -815,7 +815,7 @@ class VideoReaderDecoder auto continues = [&](int r) { const auto &run = runs_[r]; return *run.filename == start[l].first && - ReachesWithoutSeek(FrameIndexCache::instance().Find(*run.filename).get(), + CanReachWithoutSeek(FrameIndexCache::instance().Find(*run.filename).get(), start[l].second, run.first); }; // files in the order of their first sample in the batch, the decoder's file first @@ -882,7 +882,7 @@ class VideoReaderDecoder auto &task = tasks_[t]; try { DecodeSample(*task.sample, lane); - task.sample->decoded_ = true; + task.sample->decoded = true; } catch (...) { lane.errors.emplace_back(task.batch_idx, std::current_exception()); lane.decoder.reset(); // in an unknown state; its destructor waits for its stream diff --git a/dali/test/python/decoder/test_video.py b/dali/test/python/decoder/test_video.py index 04bff455fc9..b282eb824ee 100644 --- a/dali/test/python/decoder/test_video.py +++ b/dali/test/python/decoder/test_video.py @@ -1322,6 +1322,37 @@ def video_pipe(filenames): pipe.run() +_reference_cache = {} + + +def _reference_frames(filename): + """All frames of the file, decoded with PyAV, in presentation order.""" + if filename not in _reference_cache: + with av.open(filename) as container: + _reference_cache[filename] = [ + f.to_ndarray(format="rgb24") for f in container.decode(video=0) + ] + return _reference_cache[filename] + + +def _assert_frames_match_file(frames, frame_nums, filename): + """Checks that frames[i] is the frame frame_nums[i] of the file (negative: padding, skipped). + + Decoders differ slightly in color conversion, so a frame must be close to the reference and + closer to it than to any other frame of the file.""" + reference = _reference_frames(filename) + for frame, num in zip(frames, frame_nums): + if num < 0: + continue + diffs = [np.abs(frame.astype(np.int32) - ref).mean() for ref in reference] + assert ( + diffs[num] < 10 + ), f"{filename}: frame {num} differs from the reference by {diffs[num]}" + assert ( + diffs[num] <= min(diffs) + 0.5 + ), f"{filename}: frame {num} is closer to frame {int(np.argmin(diffs))}" + + # The reader decodes the samples of a batch in parallel, by up to num_threads decoders (decode # lanes). Its output must not depend on how many there are. _lanes_reader_cases = [ @@ -1346,12 +1377,16 @@ def test_reader_output_independent_of_num_threads(device, case): kwargs = _lanes_reader_cases[case] batch_size = 4 + files = _lanes_reader_files(device) + @pipeline_def(batch_size=batch_size, device_id=0, seed=42) def pipe(): - # videos, frame numbers and, with enable_timestamps, timestamps + # videos, labels (the index of the file), frame numbers and, with enable_timestamps, + # timestamps return tuple( fn.experimental.readers.video( - filenames=_lanes_reader_files(device), + filenames=files, + labels=list(range(len(files))), device=device, enable_frame_num="sequence", **kwargs, @@ -1362,6 +1397,17 @@ def pipe(): pipe(num_threads=1), pipe(num_threads=4), batch_size=batch_size, N_iterations=8 ) + # The two runs could agree on wrong frames, so check the frames against PyAV as well + p = pipe(num_threads=4) + for _ in range(4): + videos, labels, frame_nums, *_ = (t.as_cpu() for t in p.run()) + for i in range(batch_size): + _assert_frames_match_file( + np.array(videos[i]), + np.array(frame_nums[i]), + files[int(np.array(labels[i])[0])], + ) + @params("cpu", "gpu") def test_reader_pad_last_batch_repeats_last_sample(device): @@ -1402,3 +1448,6 @@ def pipe(): assert [label for _, label in samples] == list(range(8)) + [7] * 4 for video, _ in samples[8:]: np.testing.assert_array_equal(video, samples[7][0]) + for video, label in samples[:8]: + start = (label // len(files)) * 4 + _assert_frames_match_file(video, range(start, start + 4), files[label % len(files)]) From 8c874adc7151bd1f5839d96f06070c721ffbdc98 Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Mon, 26 Oct 2026 12:15:57 +0100 Subject: [PATCH 25/46] Record decoding order in the video frame index The reader plans its decode lanes by an estimate of the work of each sample. To estimate it in decoded pictures, the frame index needs two things it does not have: where each frame's packet is in decoding order, counting the packets hidden by an edit list (negative timestamps), which are decoded but not indexed, and where a seek to a keyframe really starts decoding. IndexEntry gets decode_idx and seek_decode_idx, so it grows from 16 to 24 bytes: the cached index of a dataset takes 8 more bytes per frame. BuildIndex counts the stream's packets as it reads them. A seek to a frame that SeekFrame seeks to (the last keyframe of a frame) lands on the keyframe itself, unless the stream has hidden packets. Then the demuxer is asked where av_seek_frame lands, as AvSeekFrame does, since its rule is not exposed: mov searches its sync samples by decoding timestamp offset by the edit list, so in some Sintel clips a seek to the shown IDR lands on the hidden one, 118 packets earlier. Asking costs a seek and a packet read per keyframe, which only these streams pay; in all-intra streams it would read the whole file a second time. Nothing that affects the output reads the new fields; the index order, keyframes and last_keyframe_id are unchanged. An index without decoding order (e.g. hand-built) is taken to be in decoding order. Signed-off-by: Joaquin Anton Guirao --- dali/operators/video/frames_decoder_base.cc | 54 +++++++- dali/operators/video/frames_decoder_base.h | 26 +++- dali/operators/video/frames_decoder_test.cc | 126 +++++++++++++++++- .../video/reader/frame_index_cache_test.cc | 39 ++++++ 4 files changed, 242 insertions(+), 3 deletions(-) diff --git a/dali/operators/video/frames_decoder_base.cc b/dali/operators/video/frames_decoder_base.cc index 339e5db0119..4c02d8aca24 100644 --- a/dali/operators/video/frames_decoder_base.cc +++ b/dali/operators/video/frames_decoder_base.cc @@ -1,4 +1,4 @@ -// Copyright (c) 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// Copyright (c) 2021-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -16,6 +16,7 @@ #include #include #include +#include #include "dali/core/error_handling.h" #include "dali/core/small_vector.h" #include "dali/operators/video/video_utils.h" @@ -303,6 +304,16 @@ static inline uint32_t read_nal_unit_length(uint8_t *buf) { return length; } +/** + * @brief Identifies a packet of the stream when it is read again (e.g. after a seek): by its + * position in the file or, if unknown, by its timestamp. + */ +static inline int64_t packet_key(const AVPacket &packet) { + if (packet.pos >= 0) + return packet.pos; + return packet.dts != AV_NOPTS_VALUE ? packet.dts : packet.pts; +} + void FramesDecoderBase::BuildIndex() { if (HasIndex()) { return; @@ -317,6 +328,10 @@ void FramesDecoderBase::BuildIndex() { int last_keyframe = -1; int frame_count = 0; num_frames_ = 0; + int decode_idx = 0; // packets of the stream so far, in decoding order + // The decoding position of each sync packet, where a seek can land + std::unordered_map sync_packets; + bool has_hidden_packets = false; // with negative timestamps (e.g. hidden by an edit list) while (true) { // Read the next frame from the video @@ -336,6 +351,10 @@ void FramesDecoderBase::BuildIndex() { IndexEntry entry; entry.is_keyframe = false; // Default to false, set true only if confirmed + // Counted before the negative timestamps are skipped: hidden packets are decoded too + entry.decode_idx = decode_idx++; + if (packet->flags & AV_PKT_FLAG_KEY) + sync_packets.emplace(packet_key(*packet), entry.decode_idx); // Check if this packet contains a keyframe if (packet->flags & AV_PKT_FLAG_KEY) { @@ -393,6 +412,7 @@ void FramesDecoderBase::BuildIndex() { if (entry.pts < 0) { LOG_LINE << "Negative timestamp: " << entry.pts << ", skipping" << std::endl; + has_hidden_packets = true; continue; } @@ -444,6 +464,38 @@ void FramesDecoderBase::BuildIndex() { } } + // Where AvSeekFrame lands for the frames that SeekFrame seeks to (the last keyframe of a frame), + // for estimating the decoding work: on the keyframe itself, unless packets are hidden. Then the + // demuxer is asked, as its rule is not exposed: e.g. mov searches its sync samples by decoding + // timestamp, offset by the edit list, so a seek can land a whole GOP (or a hidden lead-in) + // before the keyframe. Asking reads a packet per target, which in an all-intra stream would + // read the whole file again. + bool ask_demuxer = has_hidden_packets; + int prev_target = -1; + for (size_t i = 0; i < index_.size(); i++) { + int target = index_[i].last_keyframe_id; + if (target == prev_target) + continue; // last_keyframe_id does not decrease + prev_target = target; + auto &entry = index_[target]; + entry.seek_decode_idx = entry.decode_idx; + if (!ask_demuxer) + continue; + if (av_seek_frame(ctx_, stream_id_, entry.pts, AVSEEK_FLAG_BACKWARD) < 0) { + ask_demuxer = false; // not seekable + continue; + } + while (av_read_frame(ctx_, packet_) == 0) { + auto packet = AVPacketScope(packet_, av_packet_unref); + if (packet->stream_index != stream_id_) + continue; + auto landing = sync_packets.find(packet_key(*packet)); + if (landing != sync_packets.end()) + entry.seek_decode_idx = landing->second; + break; + } + } + // Detect if video has variable frame rate (VFR) DetectVariableFrameRate(); Reset(); diff --git a/dali/operators/video/frames_decoder_base.h b/dali/operators/video/frames_decoder_base.h index 2b7b0d25121..ad0dfdf44e3 100644 --- a/dali/operators/video/frames_decoder_base.h +++ b/dali/operators/video/frames_decoder_base.h @@ -1,4 +1,4 @@ -// Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// Copyright (c) 2021-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -42,6 +42,12 @@ struct IndexEntry { int last_keyframe_id; bool is_keyframe; bool is_flush_frame; + // Position of the frame's packet in decoding order, counting all the packets of the stream, + // also those decoded but not shown (e.g. hidden by an edit list); -1 if unknown + int decode_idx = -1; + // For a frame that SeekFrame seeks to (the last keyframe of some frame): the decoding position + // of the packet where a seek to its timestamp lands (see AvSeekFrame); -1 otherwise or if unknown + int seek_decode_idx = -1; }; struct FrameIndex { @@ -63,6 +69,24 @@ struct FrameIndex { return index[idx]; } + /** + * @brief Position of frame i's packet in decoding order. + * + * Without decoding order (e.g. a hand-built index), the frames are assumed in decoding order. + */ + int DecodeIdx(int i) const { + return index[i].decode_idx >= 0 ? index[i].decode_idx : i; + } + + /** + * @brief Decoding position where a seek to frame i starts, for a frame that SeekFrame seeks to. + * + * If unknown, the seek is assumed to start at the frame itself. + */ + int SeekDecodeIdx(int i) const { + return index[i].seek_decode_idx >= 0 ? index[i].seek_decode_idx : DecodeIdx(i); + } + /** * @brief Returns the index of the frame that has the given timestamp * diff --git a/dali/operators/video/frames_decoder_test.cc b/dali/operators/video/frames_decoder_test.cc index 99492494def..a777e2c8323 100644 --- a/dali/operators/video/frames_decoder_test.cc +++ b/dali/operators/video/frames_decoder_test.cc @@ -1,4 +1,4 @@ -// Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// Copyright (c) 2021-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -13,8 +13,12 @@ // limitations under the License. #include +#include #include #include +#include +#include +#include #include "dali/core/cuda_error.h" #include "dali/core/dev_buffer.h" @@ -194,6 +198,126 @@ class FramesDecoderGpuTest : public FramesDecoderTestBase { DeviceBuffer frame_gpu_buffer_; }; +namespace { + +std::string SintelClip(int label, int clip) { + return make_string(testing::dali_extra_path(), "/db/video/sintel/labelled_videos/", label, + "/sintel_trailer-720p_", clip, ".mp4"); +} + +// The frames that SeekFrame seeks to: the last keyframes of the frames +std::set SeekTargets(const FrameIndex &index) { + std::set targets; + for (size_t i = 0; i < index.size(); i++) + targets.insert(index[i].last_keyframe_id); + return targets; +} + +// Checks the decoding order in the index of a file without hidden packets, where a seek to a +// keyframe lands on the keyframe itself +void CheckDecodeOrder(const FrameIndex &index) { + int n = index.size(); + std::vector decode_idx; + for (int i = 0; i < n; i++) + decode_idx.push_back(index[i].decode_idx); + std::sort(decode_idx.begin(), decode_idx.end()); + for (int i = 0; i < n; i++) + ASSERT_EQ(decode_idx[i], i) << "not a permutation of the decoding positions [0, " << n << ")"; + auto targets = SeekTargets(index); + for (int i = 0; i < n; i++) { + if (index[i].is_keyframe) { + EXPECT_TRUE(targets.count(i)) << "keyframe " << i; + } + if (targets.count(i)) { + EXPECT_EQ(index[i].seek_decode_idx, index[i].decode_idx) << "seek to frame " << i; + } else { + EXPECT_EQ(index[i].seek_decode_idx, -1) << "frame " << i << " is not a seek target"; + } + } +} + +} // namespace + +TEST_F(FramesDecoderTest_CpuOnlyTests, IndexDecodeOrder) { + for (auto &path : {cfr_videos_paths_[0], vfr_videos_paths_[0]}) { + FramesDecoderCpu decoder(path); + decoder.BuildIndex(); + auto &index = decoder.GetIndex(); + CheckDecodeOrder(index); + for (size_t i = 0; i < index.size(); i++) // VP9 has no reordering + EXPECT_EQ(index[i].decode_idx, static_cast(i)) << path; + } +} + +TEST_F(FramesDecoderGpuTest, IndexDecodeOrder) { + auto h264_path = testing::dali_extra_path() + "/db/video/cfr/test_1.mp4"; + for (auto &path : {cfr_videos_paths_[0], h264_path}) { + FramesDecoderGpu decoder(path); + decoder.BuildIndex(); + CheckDecodeOrder(decoder.GetIndex()); + } +} + +// The Sintel clips start with packets that an edit list hides (negative timestamps): they are not +// in the index, but they are decoded, before the first frame or after a seek that lands on them +TEST_F(FramesDecoderGpuTest, IndexDecodeOrderLeadIn) { + { + // 310 packets, of which 188 hidden, and only the IDR at frame 62 is shown + FramesDecoderGpu decoder(SintelClip(2, 8)); + decoder.BuildIndex(); + auto &index = decoder.GetIndex(); + ASSERT_EQ(index.size(), 122u); + std::set positions; + for (size_t i = 0; i < index.size(); i++) { + EXPECT_GE(index.DecodeIdx(i), 0); + EXPECT_LT(index.DecodeIdx(i), 310); + positions.insert(index.DecodeIdx(i)); + } + EXPECT_EQ(positions.size(), index.size()); // the other 188 are hidden + EXPECT_FALSE(index[0].is_keyframe); + EXPECT_EQ(index[0].last_keyframe_id, 0); + EXPECT_EQ(index.SeekDecodeIdx(0), 0); // the hidden IDR + int decoded = 0; + for (int i = 0; i < 32; i++) + decoded = std::max(decoded, index.DecodeIdx(i) + 1); + EXPECT_EQ(decoded, 221); // the pictures decoded for the first 32 frames + ASSERT_TRUE(index[62].is_keyframe); + EXPECT_EQ(index.DecodeIdx(62), 250); + EXPECT_EQ(index.SeekDecodeIdx(62), 250); + } + // Only the IDR is hidden: every frame is decoded from the start + for (auto &path : {SintelClip(0, 3), SintelClip(1, 7)}) { + FramesDecoderGpu decoder(path); + decoder.BuildIndex(); + auto &index = decoder.GetIndex(); + for (size_t i = 0; i < index.size(); i++) + EXPECT_EQ(index[i].last_keyframe_id, 0) << path << " frame " << i; + EXPECT_EQ(index.SeekDecodeIdx(0), 0) << path; + } + { + // The seek to the shown IDR (frame 60, packet 118) lands on the hidden one: mov searches the + // sync samples by decoding timestamp, offset by the edit list + FramesDecoderGpu decoder(SintelClip(0, 9)); + decoder.BuildIndex(); + auto &index = decoder.GetIndex(); + ASSERT_TRUE(index[60].is_keyframe); + EXPECT_EQ(index.DecodeIdx(60), 118); + EXPECT_EQ(index.SeekDecodeIdx(60), 0); + } + { + // Here the seeks to the shown IDRs land on them + FramesDecoderGpu decoder(SintelClip(2, 5)); + decoder.BuildIndex(); + auto &index = decoder.GetIndex(); + for (int k : {21, 58}) { + ASSERT_TRUE(index[k].is_keyframe) << k; + EXPECT_EQ(index.SeekDecodeIdx(k), index.DecodeIdx(k)) << k; + } + EXPECT_EQ(index.DecodeIdx(21), 37); + EXPECT_EQ(index.DecodeIdx(58), 74); + } +} + TEST_F(FramesDecoderTest_CpuOnlyTests, ConstantFrameRate) { FramesDecoderCpu decoder(cfr_videos_paths_[0]); decoder.BuildIndex(); diff --git a/dali/operators/video/reader/frame_index_cache_test.cc b/dali/operators/video/reader/frame_index_cache_test.cc index 51539949bb2..5f2507ab2f1 100644 --- a/dali/operators/video/reader/frame_index_cache_test.cc +++ b/dali/operators/video/reader/frame_index_cache_test.cc @@ -22,6 +22,8 @@ #include "dali/operators/video/reader/frame_index_cache.h" #include "dali/core/format.h" +#include "dali/operators/video/frames_decoder_cpu.h" +#include "dali/test/dali_test_config.h" namespace dali { @@ -95,6 +97,43 @@ TEST(FrameIndexCacheTest, FailedBuildIsReportedAndRetried) { EXPECT_EQ(index->size(), 3u); } +TEST(FrameIndexCacheTest, HandBuiltIndexIsInDecodingOrder) { + // Without decoding order, the frames are in decoding order and a seek lands on the frame + auto index = MakeIndex(Key("hand_built", 0), 5); + for (int i = 0; i < 5; i++) { + EXPECT_EQ(index[i].decode_idx, -1); + EXPECT_EQ(index.DecodeIdx(i), i); + EXPECT_EQ(index.SeekDecodeIdx(i), i); + } +} + +TEST(FrameIndexCacheTest, KeepsDecodingOrder) { + auto path = testing::dali_extra_path() + "/db/video/vfr/test_1_vp9.mp4"; + FramesDecoderCpu decoder(path); + ASSERT_TRUE(decoder.IsValid()); + auto &cache = FrameIndexCache::instance(); + auto key = Key("decoding_order", 0); + auto index = cache.GetOrBuild(key, [&] { + decoder.BuildIndex(); + return decoder.GetIndex(); + }); + FramesDecoderCpu other(path); + other.SetIndex(*cache.Find(key)); + auto &built = decoder.GetIndex(); + ASSERT_EQ(index->size(), built.size()); + ASSERT_EQ(other.GetIndex().size(), built.size()); + int seek_targets = 0; + for (size_t i = 0; i < built.size(); i++) { + EXPECT_GE(built[i].decode_idx, 0); + seek_targets += built[i].seek_decode_idx >= 0; + for (auto *copy : {&(*index)[i], &other.GetIndex()[i]}) { + EXPECT_EQ(copy->decode_idx, built[i].decode_idx) << i; + EXPECT_EQ(copy->seek_decode_idx, built[i].seek_decode_idx) << i; + } + } + EXPECT_GT(seek_targets, 1); +} + TEST(FrameIndexCacheTest, FindDoesNotBuildOrWait) { auto &cache = FrameIndexCache::instance(); auto key = Key("find", 0); From 68f8a080e432ea48357bec5798e7816b4ea5e0ac Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Mon, 26 Oct 2026 12:26:08 +0100 Subject: [PATCH 26/46] Count the pictures a video decoder decodes The planner of the reader's decode lanes estimates the work of a sample as the frames from the first to the last plus a preroll. What a decoder really does differs a lot: a frame costs every picture up to its own in decoding order (B-frames), clips with an edit list decode a hidden lead-in of up to ~190 pictures before their first frame, a seek can land a GOP before the keyframe, and a duplicate of the last frame (edge or reflect padding) reopens the file. FramesDecoderBase::AddDecodeWork estimates it: it mirrors SeekFrame (Reset, kMinimumSeekLeap, same keyframe, backward seek) on a DecodePosition and counts the pictures decoded, from the decoding order and seek landings recorded in the index, and the reopenings. PositionAt gives the position of a decoder from its next frame. Both decoders count the pictures they decode (DecodedPictures), so the tests check the estimate against real decoding on the Sintel clips, VP9, H.264 and HEVC files, in every path of SeekFrame. On the CPU it is exact. The GPU parser decodes a picture when the next packet arrives, and now and then only with the one after, so a GPU decoder can be one picture ahead of the estimate. By default the GPU tests run one Sintel clip of each kind (no lead-in, a single GOP, a seek landing on the hidden IDR, seeks landing on the shown IDRs, a long lead-in), which takes ~9 s; disabled tests sweep all 15 clips (~30 s). Signed-off-by: Joaquin Anton Guirao --- dali/operators/video/frames_decoder_base.cc | 35 ++++ dali/operators/video/frames_decoder_base.h | 45 +++++ dali/operators/video/frames_decoder_cpu.cc | 1 + dali/operators/video/frames_decoder_gpu.cc | 1 + dali/operators/video/frames_decoder_test.cc | 188 ++++++++++++++++++++ 5 files changed, 270 insertions(+) diff --git a/dali/operators/video/frames_decoder_base.cc b/dali/operators/video/frames_decoder_base.cc index 4c02d8aca24..9871c6b7cb9 100644 --- a/dali/operators/video/frames_decoder_base.cc +++ b/dali/operators/video/frames_decoder_base.cc @@ -557,6 +557,7 @@ void FramesDecoderBase::Reset() { SelectVideoStream(stream_id); } +// Keep FramesDecoderBase::AddDecodeWork in sync with how this seeks void FramesDecoderBase::SeekFrame(int frame_id) { LOG_LINE << "SeekFrame: Seeking to frame " << frame_id << " (current=" << next_frame_idx_ << ")" << std::endl; @@ -630,6 +631,40 @@ void FramesDecoderBase::SeekFrame(int frame_id) { assert(next_frame_idx_ == frame_id); } +FramesDecoderBase::DecodePosition FramesDecoderBase::PositionAt(const FrameIndex &index, + int next_frame) { + if (next_frame <= 0 || next_frame >= static_cast(index.size())) + return {next_frame, -1}; + // it decoded from the keyframe of the last frame read (or from before it) + int decoded = -1; + for (int i = index[next_frame - 1].last_keyframe_id; i < next_frame; i++) + decoded = std::max(decoded, index.DecodeIdx(i)); + return {next_frame, decoded}; +} + +void FramesDecoderBase::AddDecodeWork(const FrameIndex &index, DecodePosition &pos, int frame, + DecodeWork &work) { + int n = index.size(); + if (frame != pos.next_frame) { + if (pos.next_frame < 0 || pos.next_frame >= n) { // Reset(): reopens the file + pos = {0, -1}; + work.reopens++; + } + if (frame < pos.next_frame || frame > pos.next_frame + kMinimumSeekLeap) { + int keyframe = index[frame].last_keyframe_id; + if (index[pos.next_frame].last_keyframe_id != keyframe || frame < pos.next_frame) + pos = {keyframe, index.SeekDecodeIdx(keyframe) - 1}; // AvSeekFrame and Flush + } + } + // A frame is returned once the pictures up to its own (in decoding order) are decoded, so the + // decoder decodes up to the furthest picture of the frames it reads, skipped ones included + int start = pos.decoded; + for (int i = pos.next_frame; i <= frame; i++) + pos.decoded = std::max(pos.decoded, index.DecodeIdx(i)); + work.pictures += pos.decoded - start; + pos.next_frame = frame + 1 < n ? frame + 1 : -1; +} + int FramesDecoderBase::HandleBoundary(boundary::BoundaryType boundary_type, int frame_id, int roi_start, int roi_end) { DALI_ENFORCE(boundary_type == boundary::BoundaryType::CLAMP || boundary_type == boundary::BoundaryType::CONSTANT || diff --git a/dali/operators/video/frames_decoder_base.h b/dali/operators/video/frames_decoder_base.h index ad0dfdf44e3..5de2b06a0cf 100644 --- a/dali/operators/video/frames_decoder_base.h +++ b/dali/operators/video/frames_decoder_base.h @@ -294,6 +294,42 @@ class DLL_PUBLIC FramesDecoderBase { */ static constexpr int kMinimumSeekLeap = 10; + /** + * @brief Where a decoder is, for estimating its work: the frame it returns next (as + * NextFrameIdx, -1 after the last one) and the decoding position of the last picture it + * decoded (-1: none). + */ + struct DecodePosition { + int next_frame = 0; + int decoded = -1; + }; + + /** + * @brief Estimated work of a decoder: the pictures it decodes and the times it reopens the file. + */ + struct DecodeWork { + int64_t pictures = 0; + int reopens = 0; + }; + + /** + * @brief The position of a decoder that read the frames up to `next_frame - 1` in a row. + * + * It is taken to have decoded from the keyframe of the last frame read; a decoder that + * decoded through from an earlier (open GOP) keyframe may be a few pictures further. + */ + static DecodePosition PositionAt(const FrameIndex &index, int next_frame); + + /** + * @brief Adds the work of returning `frame` from `pos` and advances `pos`. + * + * Mirrors SeekFrame and ReadNextFrame (keep them in sync), counting the pictures decoded, also + * the hidden ones (see IndexEntry::decode_idx), and a Reset as a reopening. Not modelled: a + * failed av_seek_frame (after which every seek resets the decoder). + */ + static void AddDecodeWork(const FrameIndex &index, DecodePosition &pos, int frame, + DecodeWork &work); + /** * @brief Handles boundary conditions for a given frame index. * @@ -430,6 +466,13 @@ class DLL_PUBLIC FramesDecoderBase { dtype_ = dtype; } + /** + * @brief Number of pictures decoded so far (for tests and diagnostics). + */ + int64_t DecodedPictures() const { + return decoded_pictures_; + } + protected: void DecodeFramesImpl(uint8_t *data, SmallVector, 32> frame_ids, boundary::BoundaryType boundary_type, @@ -462,6 +505,8 @@ class DLL_PUBLIC FramesDecoderBase { // False when the file doesn't have any correct content or doesn't have a valid video stream bool is_valid_ = false; + int64_t decoded_pictures_ = 0; // see DecodedPictures + private: void DetectVariableFrameRate(); diff --git a/dali/operators/video/frames_decoder_cpu.cc b/dali/operators/video/frames_decoder_cpu.cc index 3a0e99ce470..9c3ee18009e 100644 --- a/dali/operators/video/frames_decoder_cpu.cc +++ b/dali/operators/video/frames_decoder_cpu.cc @@ -163,6 +163,7 @@ bool FramesDecoderCpu::ReadRegularFrame(uint8_t *data) { ret = avcodec_send_packet(codec_ctx_, packet.get()); DALI_ENFORCE(ret >= 0, make_string("Failed to send packet to decoder: ", av_error_string(ret))); + decoded_pictures_++; ret = avcodec_receive_frame(codec_ctx_, frame_); if (ret == AVERROR(EAGAIN)) { diff --git a/dali/operators/video/frames_decoder_gpu.cc b/dali/operators/video/frames_decoder_gpu.cc index 28aa0b1f90a..bf39ea1c25b 100644 --- a/dali/operators/video/frames_decoder_gpu.cc +++ b/dali/operators/video/frames_decoder_gpu.cc @@ -495,6 +495,7 @@ int FramesDecoderGpu::ProcessPictureDecode(CUVIDPICPARAMS *picture_params) { } CUDA_CALL(cuvidDecodePicture(nvdecode_state_->decoder, picture_params)); + decoded_pictures_++; CUVIDPARSERDISPINFO picture_display_info; memset(&picture_display_info, 0, sizeof(picture_display_info)); picture_display_info.picture_index = picture_params->CurrPicIdx; diff --git a/dali/operators/video/frames_decoder_test.cc b/dali/operators/video/frames_decoder_test.cc index a777e2c8323..393c9a569ba 100644 --- a/dali/operators/video/frames_decoder_test.cc +++ b/dali/operators/video/frames_decoder_test.cc @@ -236,8 +236,196 @@ void CheckDecodeOrder(const FrameIndex &index) { } } +std::string SintelVideoFile(int clip) { + return make_string(testing::dali_extra_path(), "/db/video/sintel/video_files/", + "sintel_trailer-720p_", clip, ".mp4"); +} + +// The frames of the Sintel clips: B-frames, some with hidden lead-ins, some with a single IDR +std::vector SintelClips() { + std::vector paths; + for (int clip = 0; clip < 10; clip++) + paths.push_back(SintelClip(clip % 3, clip)); + for (int clip = 0; clip < 5; clip++) + paths.push_back(SintelVideoFile(clip)); + return paths; +} + +// One Sintel clip of each kind: no lead-in, only the IDR hidden (a single GOP), a seek that lands +// on the hidden IDR, seeks that land on the shown IDRs, and a long lead-in (188 hidden pictures). +// SintelClips() has more of the same kinds; the DISABLED_*AllSintelClips tests sweep them all. +std::vector SintelClipKinds() { + return {SintelVideoFile(0), SintelClip(0, 3), SintelClip(0, 9), SintelClip(2, 5), + SintelClip(2, 8)}; +} + +constexpr int kMaxDecodeWorkFrames = 32; // in a decode work scenario + +std::vector FrameRange(int start, int end) { + std::vector frames; + for (int i = start; i < end; i++) + frames.push_back(i); + return frames; +} + +// The GPU parser is sent packets without an end-of-picture flag, so it decodes a picture once the +// next packet arrives, but now and then (e.g. some B-frames of the Sintel clips) only with the one +// after: a decoder may have decoded one picture more than AddDecodeWork estimates, which its next +// frames then do not cost. +constexpr int kGpuPictureSlack = 1; + +/** + * @brief Decodes `frame_ids` as the reader does (padded by clamping) and checks that the decoder + * decodes the pictures that AddDecodeWork estimates from `pos` (give or take `slack`), and ends + * where `pos` does. + */ +void CheckDecodeWork(FramesDecoderBase &decoder, FramesDecoderBase::DecodePosition &pos, + const std::vector &frame_ids, uint8_t *buffer, int slack, + const std::string &what) { + ASSERT_LE(frame_ids.size(), static_cast(kMaxDecodeWorkFrames)); + auto &index = decoder.GetIndex(); + int n = index.size(); + // the frames that DecodeFrames seeks to, in order + std::vector frames; + for (int frame : frame_ids) + frames.push_back(FramesDecoderBase::HandleBoundary(boundary::BoundaryType::CLAMP, frame, 0, n)); + std::sort(frames.begin(), frames.end()); + FramesDecoderBase::DecodeWork work; + for (int frame : frames) + FramesDecoderBase::AddDecodeWork(index, pos, frame, work); + + int64_t before = decoder.DecodedPictures(); + decoder.DecodeFrames(buffer, make_cspan(frame_ids), boundary::BoundaryType::CLAMP); + EXPECT_NEAR(decoder.DecodedPictures() - before, work.pictures, slack) << what; + EXPECT_EQ(decoder.NextFrameIdx(), pos.next_frame) << what; +} + +/** + * @brief Runs, on one decoder, samples that take every path of SeekFrame: decoding through, + * seeking forward and backward, to the same and to another keyframe, and resetting. + */ +void CheckDecodeWorkScenarios(FramesDecoderBase &decoder, uint8_t *buffer, int slack, + const std::string &path) { + auto &index = decoder.GetIndex(); + int n = index.size(); + ASSERT_GE(n, 40); + FramesDecoderBase::DecodePosition pos; // a new decoder + int a = std::min(kMaxDecodeWorkFrames, n / 5); + int next = a; + auto check = [&](const std::string &what, const std::vector &frames) { + CheckDecodeWork(decoder, pos, frames, buffer, slack, make_string(path, ": ", what)); + next = pos.next_frame; + }; + check("fresh", FrameRange(0, a)); + check("continue", FrameRange(next, next + 8)); + static_assert(FramesDecoderBase::kMinimumSeekLeap >= 5); + check("short leap", FrameRange(next + 5, next + 9)); + check("far jump", FrameRange(next + 20, std::min(next + 23, n))); + int keyframe = next < 0 ? n : next + 1; + while (keyframe < n && !index[keyframe].is_keyframe) + keyframe++; + if (keyframe < n - 4) + check("across a keyframe", FrameRange(keyframe + 1, keyframe + 4)); + else + check("near the end", FrameRange(3 * n / 4, 3 * n / 4 + 3)); + check("backward", FrameRange(5, 9)); + check("duplicate", {12, 12, 13}); + check("last frame, twice", {n - 1, n - 1}); + check("first frame after the end", {0, 1}); + std::vector uniform; + for (int i = 0; i < 8; i++) + uniform.push_back(std::lround(i * (n - 1) / 7.0)); + check("uniform", uniform); + check("edge-padded tail", FrameRange(n - 4, n + 3)); + check("from the start", FrameRange(0, 4)); +} + +/** + * @brief Reads the frames in a row and checks that, from PositionAt, AddDecodeWork estimates the + * pictures decoded for each frame: in total, the decoder is never behind the estimate and at most + * `slack` pictures ahead. Returns the number of frames that cost nothing (decoded earlier, for a + * frame shown before them). + */ +int CheckPositionAt(FramesDecoderBase &decoder, uint8_t *frame, int slack, + const std::string &path) { + auto &index = decoder.GetIndex(); + int free_frames = 0; + int64_t start = decoder.DecodedPictures(), estimate = 0; + for (int k = 0; k < static_cast(index.size()); k++) { + auto pos = FramesDecoderBase::PositionAt(index, k); + FramesDecoderBase::DecodeWork work; + FramesDecoderBase::AddDecodeWork(index, pos, k, work); + estimate += work.pictures; + EXPECT_EQ(decoder.NextFrameIdx(), k) << path; + decoder.ReadNextFrame(frame); + int64_t ahead = decoder.DecodedPictures() - start - estimate; + EXPECT_GE(ahead, 0) << path << ": frame " << k; + EXPECT_LE(ahead, slack) << path << ": frame " << k; + free_frames += work.pictures == 0; + } + return free_frames; +} + +void CheckGpuDecodeWork(const std::vector &paths) { + for (auto &path : paths) { + FramesDecoderGpu decoder(path); + decoder.BuildIndex(); + DeviceBuffer buffer; + buffer.resize(kMaxDecodeWorkFrames * decoder.FrameSize()); + CheckDecodeWorkScenarios(decoder, buffer.data(), kGpuPictureSlack, path); + } +} + +void CheckGpuPositionAt(const std::vector &paths) { + for (auto &path : paths) { + FramesDecoderGpu decoder(path); + decoder.BuildIndex(); + DeviceBuffer buffer; + buffer.resize(decoder.FrameSize()); + EXPECT_GT(CheckPositionAt(decoder, buffer.data(), kGpuPictureSlack, path), 0) + << path << ": no B-frames"; + } +} + } // namespace +TEST_F(FramesDecoderTest_CpuOnlyTests, DecodeWork) { + for (auto &path : {cfr_videos_paths_[0], vfr_videos_paths_[0], + testing::dali_extra_path() + "/db/video/cfr/test_2_vp9.mp4", + testing::dali_extra_path() + "/db/video/vfr/test_2_vp9.mp4"}) { + FramesDecoderCpu decoder(path); + decoder.BuildIndex(); + std::vector buffer(kMaxDecodeWorkFrames * decoder.FrameSize()); + CheckDecodeWorkScenarios(decoder, buffer.data(), 0, path); + decoder.Reset(); + CheckPositionAt(decoder, buffer.data(), 0, path); + } +} + +TEST_F(FramesDecoderGpuTest, DecodeWork) { + std::vector paths = SintelClipKinds(); + // VP9, H.264 without a lead-in and HEVC (the CPU test covers the other VP9 files) + paths.push_back(vfr_videos_paths_[0]); + paths.push_back(testing::dali_extra_path() + "/db/video/cfr/test_1.mp4"); + if (FramesDecoderGpu::SupportsHevc()) + paths.push_back(vfr_hevc_videos_paths_[0]); + CheckGpuDecodeWork(paths); +} + +TEST_F(FramesDecoderGpuTest, DecodeWorkFromPosition) { + CheckGpuPositionAt(SintelClipKinds()); +} + +// The same, on every Sintel clip: too slow to run by default (~30 s), so run it with +// --gtest_also_run_disabled_tests after changing SeekFrame or AddDecodeWork +TEST_F(FramesDecoderGpuTest, DISABLED_DecodeWorkAllSintelClips) { + CheckGpuDecodeWork(SintelClips()); +} + +TEST_F(FramesDecoderGpuTest, DISABLED_DecodeWorkFromPositionAllSintelClips) { + CheckGpuPositionAt(SintelClips()); +} + TEST_F(FramesDecoderTest_CpuOnlyTests, IndexDecodeOrder) { for (auto &path : {cfr_videos_paths_[0], vfr_videos_paths_[0]}) { FramesDecoderCpu decoder(path); From 2c2b20da8de4887e4818cf936efb0e642c26b5c5 Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Mon, 26 Oct 2026 12:35:38 +0100 Subject: [PATCH 27/46] Move decode lane planning out of the video reader The video reader assigns the runs of a batch to its decode lanes in AssignRuns, a private member of the operator. Nothing tested it: by design the output does not depend on the plan, so every test passes whatever the planner does. Move the planning, unchanged, into PlanLanes (decode_lane_planner.h), which takes the runs, where the lanes' decoders are and the open cost, and returns each lane's runs in decoding order and its load. CanReachWithoutSeek moves with it. AssignRuns now only fills the inputs from the batch and the decoders, copies the plans back and releases the decoders of empty lanes, as before. The new unit tests cover the longest-first, earliest-finish assignment, the continuation of a held file, the missing index and, against a copy of the previous code, the order in which a lane decodes its runs. Signed-off-by: Joaquin Anton Guirao --- .../video/reader/decode_lane_planner.cc | 105 +++++++++++ .../video/reader/decode_lane_planner.h | 78 ++++++++ .../video/reader/decode_lane_planner_test.cc | 172 ++++++++++++++++++ .../video/reader/video_reader_decoder_op.cc | 115 ++---------- 4 files changed, 373 insertions(+), 97 deletions(-) create mode 100644 dali/operators/video/reader/decode_lane_planner.cc create mode 100644 dali/operators/video/reader/decode_lane_planner.h create mode 100644 dali/operators/video/reader/decode_lane_planner_test.cc diff --git a/dali/operators/video/reader/decode_lane_planner.cc b/dali/operators/video/reader/decode_lane_planner.cc new file mode 100644 index 00000000000..aab5fe8ed80 --- /dev/null +++ b/dali/operators/video/reader/decode_lane_planner.cc @@ -0,0 +1,105 @@ +// Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "dali/operators/video/reader/decode_lane_planner.h" + +#include +#include +#include +#include +#include + +namespace dali { + +bool CanReachWithoutSeek(const FrameIndex *index, int next, int frame) { + if (next < 0 || frame < next) + return false; + if (frame - next <= FramesDecoderBase::kMinimumSeekLeap) + return true; + return index && frame < static_cast(index->size()) && + (*index)[frame].last_keyframe_id == (*index)[next].last_keyframe_id; +} + +void PlanLanes(span runs, span lanes, int64_t open_cost, + std::vector &plans) { + int num_lanes = lanes.size(); + plans.clear(); + plans.resize(num_lanes); + std::vector next(lanes.begin(), lanes.end()); // where the decoders will be + struct RunCost { + int run; + int64_t frames; // frames decoded, from the first to the last + int64_t preroll; // frames decoded before the first one, after a seek + }; + std::vector costs; + for (int r = 0; r < static_cast(runs.size()); r++) { + const auto &run = runs[r]; + int64_t frames = run.first < 0 ? 0 : run.last - run.first + 1; + int64_t preroll = 0; + if (run.index && run.first >= 0 && run.first < static_cast(run.index->size())) + preroll = run.first - (*run.index)[run.first].last_keyframe_id; + costs.push_back({r, frames, preroll}); + } + std::stable_sort(costs.begin(), costs.end(), [](const RunCost &a, const RunCost &b) { + return a.frames + a.preroll > b.frames + b.preroll; + }); + for (const auto &cost : costs) { + const auto &run = runs[cost.run]; + int best = 0; + int64_t best_end = std::numeric_limits::max(); + for (int l = 0; l < num_lanes; l++) { + int64_t work = cost.frames; + if (next[l].filename != run.filename) + work += open_cost + cost.preroll; + else if (!CanReachWithoutSeek(run.index, next[l].next_frame, run.first)) + work += cost.preroll; + if (plans[l].load + work < best_end) { + best = l; + best_end = plans[l].load + work; + } + } + plans[best].runs.push_back(cost.run); + plans[best].load = best_end; + next[best].filename = run.filename; + next[best].next_frame = run.last < 0 ? next[best].next_frame : run.last + 1; + } + for (int l = 0; l < num_lanes; l++) { + auto &lane_runs = plans[l].runs; + auto continues = [&](int r) { + const auto &run = runs[r]; + return run.filename == lanes[l].filename && + CanReachWithoutSeek(run.index, lanes[l].next_frame, run.first); + }; + // files in the order of their first sample in the batch, the decoder's file first + std::map file_order; + for (int r : lane_runs) { + auto [it, inserted] = file_order.emplace(runs[r].filename, runs[r].batch_idx); + if (!inserted) + it->second = std::min(it->second, runs[r].batch_idx); + } + auto held = file_order.find(lanes[l].filename); + if (held != file_order.end()) + held->second = -1; + std::sort(lane_runs.begin(), lane_runs.end(), [&](int a, int b) { + const auto &ra = runs[a], &rb = runs[b]; + return std::make_tuple(file_order[ra.filename], ra.filename, ra.first) < + std::make_tuple(file_order[rb.filename], rb.filename, rb.first); + }); + auto it = std::find_if(lane_runs.begin(), lane_runs.end(), continues); + if (it != lane_runs.end()) + std::rotate(lane_runs.begin(), it, it + 1); + } +} + +} // namespace dali diff --git a/dali/operators/video/reader/decode_lane_planner.h b/dali/operators/video/reader/decode_lane_planner.h new file mode 100644 index 00000000000..b5a74687123 --- /dev/null +++ b/dali/operators/video/reader/decode_lane_planner.h @@ -0,0 +1,78 @@ +// Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef DALI_OPERATORS_VIDEO_READER_DECODE_LANE_PLANNER_H_ +#define DALI_OPERATORS_VIDEO_READER_DECODE_LANE_PLANNER_H_ + +#include +#include +#include + +#include "dali/core/api_helper.h" +#include "dali/core/span.h" +#include "dali/operators/video/frames_decoder_base.h" + +namespace dali { + +/** + * @brief Samples of one file that one decoder decodes in a row, as the planner sees them. + */ +struct PlannedRun { + std::string_view filename; + const FrameIndex *index; // null: no index, the work is estimated from the frame range + int batch_idx; // the smallest position of its samples in the batch + int first, last; // the frames decoded, from first to last; -1 if none +}; + +/** + * @brief Where a lane's decoder is when the batch starts. + */ +struct LaneStart { + std::string_view filename; // empty: no decoder + int next_frame = -1; // as FramesDecoderBase::NextFrameIdx +}; + +/** + * @brief The runs a lane decodes, in the order it decodes them, and their estimated work. + */ +struct LanePlan { + std::vector runs; // indices into the planned runs + int64_t load = 0; +}; + +/** + * @brief Whether a decoder whose next frame is `next` decodes through to `frame` without + * seeking (see FramesDecoderBase::SeekFrame). + */ +DLL_PUBLIC bool CanReachWithoutSeek(const FrameIndex *index, int next, int frame); + +/** + * @brief Assigns the runs to lanes: longest first, each to the lane that would finish it first, + * counting what the lane's decoder saves (no reopening, no seeking). + * + * A lane then decodes the runs of one file together, by first frame: first the file its decoder + * holds (starting with the run that continues where the decoder is), then the others in batch + * order. + * + * @param runs the runs of the batch + * @param lanes where the lanes' decoders are + * @param open_cost the work of opening a decoder, in frames + * @param plans the plan of every lane (resized to the number of lanes) + */ +DLL_PUBLIC void PlanLanes(span runs, span lanes, + int64_t open_cost, std::vector &plans); + +} // namespace dali + +#endif // DALI_OPERATORS_VIDEO_READER_DECODE_LANE_PLANNER_H_ diff --git a/dali/operators/video/reader/decode_lane_planner_test.cc b/dali/operators/video/reader/decode_lane_planner_test.cc new file mode 100644 index 00000000000..7e80e338df2 --- /dev/null +++ b/dali/operators/video/reader/decode_lane_planner_test.cc @@ -0,0 +1,172 @@ +// Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "dali/operators/video/reader/decode_lane_planner.h" + +namespace dali { + +namespace { + +constexpr int64_t kOpenCost = 16; + +// An index of `num_frames` frames in decoding order, with a keyframe every `gop` frames +FrameIndex MakeIndex(int num_frames, int gop) { + FrameIndex index; + index.timebase = {1, 25}; + for (int i = 0; i < num_frames; i++) + index.index.push_back({i, i / gop * gop, i % gop == 0, i == num_frames - 1}); + return index; +} + +std::vector Plan(const std::vector &runs, + const std::vector &lanes) { + std::vector plans; + PlanLanes(make_cspan(runs), make_cspan(lanes), kOpenCost, plans); + EXPECT_EQ(plans.size(), lanes.size()); + return plans; +} + +/** + * @brief The order in which the reader decoded a lane's runs before the planner was moved out of + * it: the files by their first sample in the batch, the decoder's file first, the runs of a file + * by first frame, and the run that continues where the decoder is moved to the front. + */ +std::vector ReferenceOrder(const std::vector &runs_, const LaneStart &start, + std::vector runs) { + auto continues = [&](int r) { + const auto &run = runs_[r]; + return run.filename == start.filename && + CanReachWithoutSeek(run.index, start.next_frame, run.first); + }; + std::map file_order; + for (int r : runs) { + auto [it, inserted] = file_order.emplace(runs_[r].filename, runs_[r].batch_idx); + if (!inserted) + it->second = std::min(it->second, runs_[r].batch_idx); + } + auto held = file_order.find(start.filename); + if (held != file_order.end()) + held->second = -1; + std::sort(runs.begin(), runs.end(), [&](int a, int b) { + const auto &ra = runs_[a], &rb = runs_[b]; + return std::make_tuple(file_order[ra.filename], ra.filename, ra.first) < + std::make_tuple(file_order[rb.filename], rb.filename, rb.first); + }); + auto it = std::find_if(runs.begin(), runs.end(), continues); + if (it != runs.end()) + std::rotate(runs.begin(), it, it + 1); + return runs; +} + +} // namespace + +TEST(DecodeLanePlannerTest, CanReachWithoutSeek) { + auto index = MakeIndex(100, 30); + EXPECT_TRUE(CanReachWithoutSeek(&index, 5, 5)); + EXPECT_TRUE(CanReachWithoutSeek(&index, 5, 5 + FramesDecoderBase::kMinimumSeekLeap)); + EXPECT_TRUE(CanReachWithoutSeek(&index, 5, 29)); // the same keyframe + EXPECT_FALSE(CanReachWithoutSeek(&index, 5, 31)); // the next keyframe + EXPECT_FALSE(CanReachWithoutSeek(&index, 5, 4)); // backward + EXPECT_FALSE(CanReachWithoutSeek(&index, -1, 0)); // after the end + EXPECT_FALSE(CanReachWithoutSeek(nullptr, 5, 29)); +} + +TEST(DecodeLanePlannerTest, LongestFirstToEarliestFinish) { + auto index = MakeIndex(100, 100); + // three files, a run of each from frame 0: 30, 20 and 10 frames + std::vector runs = { + {"b", &index, 1, 0, 19}, + {"a", &index, 0, 0, 29}, + {"c", &index, 2, 0, 9}, + }; + auto plans = Plan(runs, {{}, {}}); + // a: lane 0 (a tie, to the lowest lane), b: lane 1, c: lane 1 (36 + 26 < 46 + 26) + EXPECT_EQ(plans[0].runs, std::vector({1})); + EXPECT_EQ(plans[0].load, 30 + kOpenCost); + EXPECT_EQ(plans[1].runs, std::vector({0, 2})); + EXPECT_EQ(plans[1].load, 20 + 10 + 2 * kOpenCost); +} + +TEST(DecodeLanePlannerTest, HeldFileContinues) { + auto index = MakeIndex(100, 50); + // frames 20-39 of the GOP from frame 0: 20 frames of preroll after a seek + std::vector runs = {{"a", &index, 0, 20, 39}}; + auto plans = Plan(runs, {{"b", 0}, {"a", 20}}); + EXPECT_TRUE(plans[0].runs.empty()); + EXPECT_EQ(plans[1].runs, std::vector({0})); + EXPECT_EQ(plans[1].load, 20); // no opening, no preroll + // a decoder that has to seek back pays the preroll, but not the opening + plans = Plan(runs, {{"b", 0}, {"a", 45}}); + EXPECT_EQ(plans[1].runs, std::vector({0})); + EXPECT_EQ(plans[1].load, 20 + 20); +} + +TEST(DecodeLanePlannerTest, NoIndex) { + // without an index, a run costs its frames (and the opening), with no preroll + std::vector runs = {{"a", nullptr, 0, 20, 39}}; + auto plans = Plan(runs, {{}}); + EXPECT_EQ(plans[0].load, 20 + kOpenCost); +} + +TEST(DecodeLanePlannerTest, LaneOrder) { + // Random batches: runs of a few files with distinct first frames (no ties in the order) + std::mt19937 rng(1234); + auto index = MakeIndex(300, 30); + std::vector files = {"a", "b", "c", "d"}; + for (int iter = 0; iter < 200; iter++) { + std::vector runs; + int num_runs = std::uniform_int_distribution<>(1, 12)(rng); + std::vector batch_idx(num_runs); + for (int i = 0; i < num_runs; i++) + batch_idx[i] = i; + std::shuffle(batch_idx.begin(), batch_idx.end(), rng); + std::map> firsts; + for (int r = 0; r < num_runs; r++) { + std::string_view file = files[rng() % files.size()]; + int first; + do { + first = rng() % 280; + } while (std::count(firsts[file].begin(), firsts[file].end(), first)); + firsts[file].push_back(first); + runs.push_back({file, &index, batch_idx[r], first, first + static_cast(rng() % 20)}); + } + std::vector lanes(std::uniform_int_distribution<>(1, 4)(rng)); + for (auto &lane : lanes) { + if (rng() % 3) + lane = {files[rng() % files.size()], static_cast(rng() % 300)}; + } + auto plans = Plan(runs, lanes); + std::vector planned; + for (size_t l = 0; l < lanes.size(); l++) { + EXPECT_EQ(plans[l].runs, ReferenceOrder(runs, lanes[l], plans[l].runs)) + << "iteration " << iter << ", lane " << l; + planned.insert(planned.end(), plans[l].runs.begin(), plans[l].runs.end()); + } + std::sort(planned.begin(), planned.end()); + ASSERT_EQ(planned.size(), runs.size()); + for (int r = 0; r < num_runs; r++) + EXPECT_EQ(planned[r], r); + } +} + +} // namespace dali diff --git a/dali/operators/video/reader/video_reader_decoder_op.cc b/dali/operators/video/reader/video_reader_decoder_op.cc index bc5d0559f34..5c668b8e9da 100644 --- a/dali/operators/video/reader/video_reader_decoder_op.cc +++ b/dali/operators/video/reader/video_reader_decoder_op.cc @@ -15,7 +15,6 @@ #include #include #include -#include #include #include #include @@ -32,6 +31,7 @@ #include "dali/operators/video/frames_decoder_base.h" #include "dali/operators/video/frames_decoder_cpu.h" #include "dali/operators/video/frames_decoder_gpu.h" +#include "dali/operators/video/reader/decode_lane_planner.h" #include "dali/operators/video/reader/frame_index_cache.h" #include "dali/operators/video/video_utils.h" #include "dali/pipeline/util/thread_pool.h" @@ -605,12 +605,10 @@ class VideoReaderDecoder std::unique_ptr decoder; // after `stream`: destroyed (and synced) first Tensor constant_frame; std::vector frame_idxs; // scratch - // Planning: the runs of the current batch, in decoding order, the estimated work, and where - // the lane's decoder will be after them + // Planning: the runs of the current batch, in the order the lane decodes them, and the + // estimated work std::vector runs; int64_t load = 0; - std::string next_filename; - int next_frame = -1; // Errors of the current batch: (position in the batch, exception) std::vector> errors; }; @@ -676,19 +674,6 @@ class VideoReaderDecoder } } - /** - * @brief Whether a decoder whose next frame is `next` decodes through to `frame` without - * seeking (see FramesDecoderBase::SeekFrame). - */ - static bool CanReachWithoutSeek(const FrameIndex *index, int next, int frame) { - if (next < 0 || frame < next) - return false; - if (frame - next <= FramesDecoderBase::kMinimumSeekLeap) - return true; - return index && frame < static_cast(index->size()) && - (*index)[frame].last_keyframe_id == (*index)[next].last_keyframe_id; - } - /** * @brief Groups the batch's samples into runs: samples of one file that one decoder decodes in * a row without seeking. @@ -750,93 +735,28 @@ class VideoReaderDecoder } /** - * @brief Assigns the runs to lanes: longest first, each to the lane that would finish it - * first, counting what the lane's decoder saves (no reopening, no seeking). - * - * A lane then decodes the runs of one file together, by first frame: first the file its - * decoder holds (starting with the run that continues where the decoder is), then the others in - * batch order. A lane without runs releases its decoder. + * @brief Assigns the runs to lanes (see PlanLanes). A lane without runs releases its decoder. */ void AssignRuns() { - for (auto &lane : lanes_) { - lane.runs.clear(); - lane.load = 0; - lane.next_filename = lane.decoder ? lane.decoder->Filename() : std::string(); - lane.next_frame = lane.decoder ? lane.decoder->NextFrameIdx() : -1; - } - std::vector> start(lanes_.size()); // where the decoders are - for (size_t l = 0; l < lanes_.size(); l++) - start[l] = {lanes_[l].next_filename, lanes_[l].next_frame}; - struct RunCost { - int run; - const FrameIndex *index; - int64_t frames; // frames decoded, from the first to the last - int64_t preroll; // frames decoded before the first one, after a seek - }; - std::vector costs; std::vector indices; // keeps the indices alive while planning - for (int r = 0; r < static_cast(runs_.size()); r++) { - const auto &run = runs_[r]; + std::vector runs; + for (const auto &run : runs_) { indices.push_back(FrameIndexCache::instance().Find(*run.filename)); - const FrameIndex *index = indices.back().get(); - int64_t frames = run.first < 0 ? 0 : run.last - run.first + 1; - int64_t preroll = 0; - if (index && run.first >= 0 && run.first < static_cast(index->size())) - preroll = run.first - (*index)[run.first].last_keyframe_id; - costs.push_back({r, index, frames, preroll}); + runs.push_back({*run.filename, indices.back().get(), run.batch_idx, run.first, run.last}); } - std::stable_sort(costs.begin(), costs.end(), [](const RunCost &a, const RunCost &b) { - return a.frames + a.preroll > b.frames + b.preroll; - }); - for (const auto &cost : costs) { - const auto &run = runs_[cost.run]; - int best = 0; - int64_t best_end = std::numeric_limits::max(); - for (int l = 0; l < static_cast(lanes_.size()); l++) { - const auto &lane = lanes_[l]; - int64_t work = cost.frames; - if (lane.next_filename != *run.filename) - work += kOpenCost + cost.preroll; - else if (!CanReachWithoutSeek(cost.index, lane.next_frame, run.first)) - work += cost.preroll; - if (lane.load + work < best_end) { - best = l; - best_end = lane.load + work; - } + std::vector filenames(lanes_.size()); // of the lanes' decoders + std::vector starts(lanes_.size()); + for (size_t l = 0; l < lanes_.size(); l++) { + if (lanes_[l].decoder) { + filenames[l] = lanes_[l].decoder->Filename(); + starts[l] = {filenames[l], lanes_[l].decoder->NextFrameIdx()}; } - auto &lane = lanes_[best]; - lane.runs.push_back(cost.run); - lane.load = best_end; - lane.next_filename = *run.filename; - lane.next_frame = run.last < 0 ? lane.next_frame : run.last + 1; } + PlanLanes(make_cspan(runs), make_cspan(starts), kOpenCost, plans_); for (size_t l = 0; l < lanes_.size(); l++) { - auto &runs = lanes_[l].runs; - auto continues = [&](int r) { - const auto &run = runs_[r]; - return *run.filename == start[l].first && - CanReachWithoutSeek(FrameIndexCache::instance().Find(*run.filename).get(), - start[l].second, run.first); - }; - // files in the order of their first sample in the batch, the decoder's file first - std::map file_order; - for (int r : runs) { - auto [it, inserted] = file_order.emplace(*runs_[r].filename, runs_[r].batch_idx); - if (!inserted) - it->second = std::min(it->second, runs_[r].batch_idx); - } - auto held = file_order.find(start[l].first); - if (held != file_order.end()) - held->second = -1; - std::sort(runs.begin(), runs.end(), [&](int a, int b) { - const auto &ra = runs_[a], &rb = runs_[b]; - return std::make_tuple(file_order[*ra.filename], *ra.filename, ra.first) < - std::make_tuple(file_order[*rb.filename], *rb.filename, rb.first); - }); - auto it = std::find_if(runs.begin(), runs.end(), continues); - if (it != runs.end()) - std::rotate(runs.begin(), it, it + 1); - if (runs.empty() && lanes_.size() > 1) + lanes_[l].runs = std::move(plans_[l].runs); + lanes_[l].load = plans_[l].load; + if (lanes_[l].runs.empty() && lanes_.size() > 1) lanes_[l].decoder.reset(); // do not keep its frame buffers and NVDEC decoder idle } } @@ -1027,6 +947,7 @@ class VideoReaderDecoder std::vector tasks_; std::vector runs_; std::vector lanes_; + std::vector plans_; // scratch std::unique_ptr thread_pool_; // with more than one lane }; From 7a876f629a649804ed6fa55f1f5203226fcce5c1 Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Mon, 26 Oct 2026 12:56:32 +0100 Subject: [PATCH 28/46] Plan decode lanes by the pictures they decode The planner charged a run its frame range plus, after a seek, the frames back to the keyframe, and simulated a lane's runs in the order it assigned them (longest first), not in the order the lane decodes them. Both are far off for real files: a frame costs every picture up to its own in decoding order, clips with an edit list decode up to ~190 hidden pictures before their first frame, a duplicate of the last frame (edge or reflect padding) reopens the file, and a lane holding file X that is given a run of Y and then a run of X decodes X first on its decoder, while it was charged for reopening X after Y. Each sample now carries the frames its decoder seeks to (SampleFrames), and the lanes' starting positions come from PositionAt (DecoderPosition). PlanLanes charges a run on a lane by re-simulating, with AddDecodeWork, the runs of that file on that lane in the order the lane will decode them (the file the decoder holds first, with the continuing run in front, the others from a newly opened decoder), so the work of a lane is the sum over its files. The assignment stays longest first to the lane that finishes first, and the final order of a lane is the same as before, with ties within a file broken by run number. Without an index, a run still costs its frame range. kOpenCost (16) is now in pictures, also charged per reset. A lane's plan also gives its pictures, the load without kOpenCost per open and reset. The reader logs them next to the pictures the lane decoded (LOG_LINE, with DALI_VERBOSE_LOGS), and a test checks that they match on CPU decoders driven as the reader drives them, from SampleFrames and DecoderPosition. Checked against the decoders in the TL1 lanes, random_access, seq_longgop and cpu scenarios, with 1 to 16 threads: per lane and batch, they match exactly on the CPU and within 2 on the GPU (the parser holds one picture back at some seeks). The exception is +3 to +4 pictures for an HEVC decoder held from the previous batch whose last frames precede a keyframe in display order but follow it in decoding order: PositionAt takes it to have decoded only from that keyframe. Only the plan changes; a decoder's output does not depend on its history, which the reader's tests check across thread counts, now also on clips with hidden lead-ins and with lanes that decode several runs. Signed-off-by: Joaquin Anton Guirao --- .../video/reader/decode_lane_planner.cc | 236 ++++++++++---- .../video/reader/decode_lane_planner.h | 46 ++- .../video/reader/decode_lane_planner_test.cc | 292 ++++++++++++++++-- .../video/reader/video_reader_decoder_op.cc | 60 ++-- dali/test/python/decoder/test_video.py | 48 ++- 5 files changed, 561 insertions(+), 121 deletions(-) diff --git a/dali/operators/video/reader/decode_lane_planner.cc b/dali/operators/video/reader/decode_lane_planner.cc index aab5fe8ed80..58d8e24cd2b 100644 --- a/dali/operators/video/reader/decode_lane_planner.cc +++ b/dali/operators/video/reader/decode_lane_planner.cc @@ -16,7 +16,6 @@ #include #include -#include #include #include @@ -31,74 +30,207 @@ bool CanReachWithoutSeek(const FrameIndex *index, int next, int frame) { (*index)[frame].last_keyframe_id == (*index)[next].last_keyframe_id; } +void SampleFrames(span frame_idxs, int start, int end, int stride, + boundary::BoundaryType boundary_type, int roi_start, int roi_end, + SmallVector &frames) { + frames.clear(); + auto add = [&](int frame) { + int idx = FramesDecoderBase::HandleBoundary(boundary_type, frame, roi_start, roi_end); + if (idx >= 0) // not a constant frame + frames.push_back(idx); + }; + if (!frame_idxs.empty()) { + for (int frame : frame_idxs) + add(frame); + } else { + for (int frame = start; frame < end; frame += stride) + add(frame); + } + std::sort(frames.begin(), frames.end()); +} + +FramesDecoderBase::DecodePosition DecoderPosition(FramesDecoderBase &decoder) { + int next = decoder.NextFrameIdx(); + return decoder.HasIndex() ? FramesDecoderBase::PositionAt(decoder.GetIndex(), next) + : FramesDecoderBase::DecodePosition{next, -1}; +} + +namespace { + +using DecodePosition = FramesDecoderBase::DecodePosition; +using DecodeWork = FramesDecoderBase::DecodeWork; + +// Whether the decoder of a lane continues to the run without seeking +bool Continues(const PlannedRun &run, const LaneStart &start) { + return run.filename == start.filename && + CanReachWithoutSeek(run.index, start.pos.next_frame, run.first); +} + +/** + * @brief Orders the runs of one file on a lane as the lane decodes them: by first frame (then run + * number), the run that continues from the lane's decoder first. + */ +void OrderFileRuns(span runs, const LaneStart &start, + SmallVector &order) { + std::sort(order.begin(), order.end(), [&](int a, int b) { + return std::make_pair(runs[a].first, a) < std::make_pair(runs[b].first, b); + }); + auto it = std::find_if(order.begin(), order.end(), + [&](int r) { return Continues(runs[r], start); }); + if (it != order.end()) + std::rotate(order.begin(), it, it + 1); +} + +// The work in pictures, an opening or a reset counted as `open_cost` +int64_t Cost(const DecodeWork &work, int64_t open_cost) { + return work.pictures + open_cost * work.reopens; +} + +/** + * @brief Without an index (or with frames outside of it): the frames of each run, from the first + * to the last, and the opening of a decoder. + */ +DecodeWork FrameRangeWork(span runs, bool held, span order) { + DecodeWork work; + work.reopens = held ? 0 : 1; + for (int r : order) { + if (runs[r].first >= 0) + work.pictures += runs[r].last - runs[r].first + 1; + } + return work; +} + +/** + * @brief The work of a lane for the runs of one file, decoded in `order`: from the lane's decoder + * if it holds the file, otherwise from a newly opened one. + */ +DecodeWork FileWork(span runs, const LaneStart &start, span order) { + const auto &file = runs[order[0]]; + bool held = start.filename == file.filename; + if (!file.index) + return FrameRangeWork(runs, held, order); + const auto &index = *file.index; + int num_frames = index.size(); + DecodePosition pos = held ? start.pos : DecodePosition{}; + DecodeWork work; + work.reopens = held ? 0 : 1; + for (int r : order) { + for (auto frames : runs[r].samples) { + for (int frame : frames) { + if (frame < 0 || frame >= num_frames) + return FrameRangeWork(runs, held, order); + FramesDecoderBase::AddDecodeWork(index, pos, frame, work); + } + } + } + return work; +} + +/** + * @brief The runs of one file on a lane, in the order the lane decodes them, and their work. + */ +struct FileGroup { + std::string_view filename; + SmallVector runs; + DecodeWork work; +}; + +FileGroup *FindGroup(SmallVector &groups, std::string_view filename) { + for (auto &group : groups) { + if (group.filename == filename) + return &group; + } + return nullptr; +} + +} // namespace + void PlanLanes(span runs, span lanes, int64_t open_cost, std::vector &plans) { int num_lanes = lanes.size(); + int num_runs = runs.size(); plans.clear(); plans.resize(num_lanes); - std::vector next(lanes.begin(), lanes.end()); // where the decoders will be - struct RunCost { - int run; - int64_t frames; // frames decoded, from the first to the last - int64_t preroll; // frames decoded before the first one, after a seek - }; - std::vector costs; - for (int r = 0; r < static_cast(runs.size()); r++) { - const auto &run = runs[r]; - int64_t frames = run.first < 0 ? 0 : run.last - run.first + 1; - int64_t preroll = 0; - if (run.index && run.first >= 0 && run.first < static_cast(run.index->size())) - preroll = run.first - (*run.index)[run.first].last_keyframe_id; - costs.push_back({r, frames, preroll}); + // The runs of different files on a lane do not share a decoder: the file the lane's decoder + // holds goes first, the others are opened anew, one after another. So the work of a lane is the + // sum of the work of its files, each simulated in the order the lane decodes its runs. + std::vector> groups(num_lanes); + + // the work of each run on a newly opened decoder; the longest first + std::vector fresh_work(num_runs); + std::vector fresh(num_runs); + std::vector order(num_runs); + for (int r = 0; r < num_runs; r++) { + fresh_work[r] = FileWork(runs, LaneStart{}, make_cspan(&r, 1)); + fresh[r] = Cost(fresh_work[r], open_cost); + order[r] = r; + LOG_LINE << "PlanLanes: run " << r << " of " << runs[r].filename << " [" << runs[r].first + << ", " << runs[r].last << "]: " << fresh[r] << std::endl; } - std::stable_sort(costs.begin(), costs.end(), [](const RunCost &a, const RunCost &b) { - return a.frames + a.preroll > b.frames + b.preroll; - }); - for (const auto &cost : costs) { - const auto &run = runs[cost.run]; + std::stable_sort(order.begin(), order.end(), [&](int a, int b) { return fresh[a] > fresh[b]; }); + + SmallVector candidate, best_runs; + for (int r : order) { + const auto &run = runs[r]; int best = 0; int64_t best_end = std::numeric_limits::max(); + DecodeWork best_work; for (int l = 0; l < num_lanes; l++) { - int64_t work = cost.frames; - if (next[l].filename != run.filename) - work += open_cost + cost.preroll; - else if (!CanReachWithoutSeek(run.index, next[l].next_frame, run.first)) - work += cost.preroll; - if (plans[l].load + work < best_end) { + FileGroup *group = FindGroup(groups[l], run.filename); + DecodeWork work; + int64_t delta; + candidate.clear(); + if (!group && lanes[l].filename != run.filename) { + candidate.push_back(r); + work = fresh_work[r]; + delta = fresh[r]; + } else { + if (group) + candidate = group->runs; + candidate.push_back(r); + OrderFileRuns(runs, lanes[l], candidate); + work = FileWork(runs, lanes[l], make_cspan(candidate)); + delta = Cost(work, open_cost) - (group ? Cost(group->work, open_cost) : 0); + } + if (plans[l].load + delta < best_end) { best = l; - best_end = plans[l].load + work; + best_end = plans[l].load + delta; + best_work = work; + best_runs = candidate; } } - plans[best].runs.push_back(cost.run); + FileGroup *group = FindGroup(groups[best], run.filename); + if (!group) { + groups[best].push_back({run.filename, {}, {}}); + group = &groups[best].back(); + } + group->runs = best_runs; + group->work = best_work; plans[best].load = best_end; - next[best].filename = run.filename; - next[best].next_frame = run.last < 0 ? next[best].next_frame : run.last + 1; } + for (int l = 0; l < num_lanes; l++) { - auto &lane_runs = plans[l].runs; - auto continues = [&](int r) { - const auto &run = runs[r]; - return run.filename == lanes[l].filename && - CanReachWithoutSeek(run.index, lanes[l].next_frame, run.first); + // the file the decoder holds first, then the others by their first sample in the batch + auto file_order = [&](const FileGroup &group) { + int batch_idx = std::numeric_limits::max(); + for (int r : group.runs) + batch_idx = std::min(batch_idx, runs[r].batch_idx); + return std::make_tuple(group.filename == lanes[l].filename ? -1 : batch_idx, + group.filename); }; - // files in the order of their first sample in the batch, the decoder's file first - std::map file_order; - for (int r : lane_runs) { - auto [it, inserted] = file_order.emplace(runs[r].filename, runs[r].batch_idx); - if (!inserted) - it->second = std::min(it->second, runs[r].batch_idx); - } - auto held = file_order.find(lanes[l].filename); - if (held != file_order.end()) - held->second = -1; - std::sort(lane_runs.begin(), lane_runs.end(), [&](int a, int b) { - const auto &ra = runs[a], &rb = runs[b]; - return std::make_tuple(file_order[ra.filename], ra.filename, ra.first) < - std::make_tuple(file_order[rb.filename], rb.filename, rb.first); + auto &lane_groups = groups[l]; + std::sort(lane_groups.begin(), lane_groups.end(), [&](const FileGroup &a, const FileGroup &b) { + return file_order(a) < file_order(b); }); - auto it = std::find_if(lane_runs.begin(), lane_runs.end(), continues); - if (it != lane_runs.end()) - std::rotate(lane_runs.begin(), it, it + 1); + for (auto &group : lane_groups) { + plans[l].runs.insert(plans[l].runs.end(), group.runs.begin(), group.runs.end()); + plans[l].pictures += group.work.pictures; + LOG_LINE << "PlanLanes: lane " << l << ": " << group.filename << ", " << group.runs.size() + << " runs: " << group.work.pictures << " pictures, " << group.work.reopens + << " openings" << std::endl; + } + LOG_LINE << "PlanLanes: lane " << l << ": load " << plans[l].load << ", " + << plans[l].pictures << " pictures" << std::endl; } } diff --git a/dali/operators/video/reader/decode_lane_planner.h b/dali/operators/video/reader/decode_lane_planner.h index b5a74687123..8860d575b1e 100644 --- a/dali/operators/video/reader/decode_lane_planner.h +++ b/dali/operators/video/reader/decode_lane_planner.h @@ -20,6 +20,8 @@ #include #include "dali/core/api_helper.h" +#include "dali/core/boundary.h" +#include "dali/core/small_vector.h" #include "dali/core/span.h" #include "dali/operators/video/frames_decoder_base.h" @@ -33,6 +35,9 @@ struct PlannedRun { const FrameIndex *index; // null: no index, the work is estimated from the frame range int batch_idx; // the smallest position of its samples in the batch int first, last; // the frames decoded, from first to last; -1 if none + // The frames that each sample decodes, in the order its decoder seeks to them: in the range of + // the file, sorted by frame index, duplicates kept, no constant padding + SmallVector, 4> samples; }; /** @@ -40,7 +45,9 @@ struct PlannedRun { */ struct LaneStart { std::string_view filename; // empty: no decoder - int next_frame = -1; // as FramesDecoderBase::NextFrameIdx + // The frame it returns next (as FramesDecoderBase::NextFrameIdx) and, with an index, the last + // picture it decoded (see FramesDecoderBase::PositionAt) + FramesDecoderBase::DecodePosition pos = {-1, -1}; }; /** @@ -49,8 +56,29 @@ struct LaneStart { struct LanePlan { std::vector runs; // indices into the planned runs int64_t load = 0; + int64_t pictures = 0; // the part of `load` that is pictures decoded (not openings or resets) }; +/** + * @brief The frames a sample decodes, in the order its decoder seeks to them (see + * FramesDecoderBase::DecodeFrames): in the range of the file, sorted, duplicates kept, no constant + * padding. + * + * @param frame_idxs the frames of the sample; if empty, from `start` to `end` (exclusive) by + * `stride` + * @param boundary_type the padding of the frames outside of [roi_start, roi_end) + * @param frames the frames (cleared first) + */ +DLL_PUBLIC void SampleFrames(span frame_idxs, int start, int end, int stride, + boundary::BoundaryType boundary_type, int roi_start, int roi_end, + SmallVector &frames); + +/** + * @brief Where a decoder is, for LaneStart: from its index (see FramesDecoderBase::PositionAt) or, + * without one, only its next frame. + */ +DLL_PUBLIC FramesDecoderBase::DecodePosition DecoderPosition(FramesDecoderBase &decoder); + /** * @brief Whether a decoder whose next frame is `next` decodes through to `frame` without * seeking (see FramesDecoderBase::SeekFrame). @@ -58,16 +86,20 @@ struct LanePlan { DLL_PUBLIC bool CanReachWithoutSeek(const FrameIndex *index, int next, int frame); /** - * @brief Assigns the runs to lanes: longest first, each to the lane that would finish it first, - * counting what the lane's decoder saves (no reopening, no seeking). + * @brief Assigns the runs to lanes: longest first, each to the lane that would finish it first. + * + * A lane decodes the runs of one file together, by first frame: first the file its decoder holds + * (starting with the run that continues where the decoder is), then the others in batch order, + * each from a newly opened decoder. * - * A lane then decodes the runs of one file together, by first frame: first the file its decoder - * holds (starting with the run that continues where the decoder is), then the others in batch - * order. + * The work is counted in pictures decoded (see FramesDecoderBase::AddDecodeWork), simulated in + * that order: B-frames, hidden lead-ins, seeks and resets, plus `open_cost` per opening or reset. + * Without an index, a run costs its frame range. With one, the `pictures` of a lane's plan are + * the pictures its decoders decode (see FramesDecoderBase::DecodedPictures). * * @param runs the runs of the batch * @param lanes where the lanes' decoders are - * @param open_cost the work of opening a decoder, in frames + * @param open_cost the work of opening a decoder, in pictures * @param plans the plan of every lane (resized to the number of lanes) */ DLL_PUBLIC void PlanLanes(span runs, span lanes, diff --git a/dali/operators/video/reader/decode_lane_planner_test.cc b/dali/operators/video/reader/decode_lane_planner_test.cc index 7e80e338df2..28da3487ca0 100644 --- a/dali/operators/video/reader/decode_lane_planner_test.cc +++ b/dali/operators/video/reader/decode_lane_planner_test.cc @@ -14,14 +14,21 @@ #include #include +#include #include +#include #include #include #include #include #include +#include "dali/core/boundary.h" +#include "dali/core/format.h" +#include "dali/core/small_vector.h" +#include "dali/operators/video/frames_decoder_cpu.h" #include "dali/operators/video/reader/decode_lane_planner.h" +#include "dali/test/dali_test_config.h" namespace dali { @@ -38,10 +45,46 @@ FrameIndex MakeIndex(int num_frames, int gop) { return index; } -std::vector Plan(const std::vector &runs, - const std::vector &lanes) { +// A decoder of `file` that read the frames up to `next_frame - 1` in a row +LaneStart Held(std::string_view file, const FrameIndex &index, int next_frame) { + return {file, FramesDecoderBase::PositionAt(index, next_frame)}; +} + +/** + * @brief The runs of a batch, and the frames of their samples. + */ +struct Batch { + std::vector runs; + std::deque> frames; // of the samples, referenced by the runs + + // Adds a run of samples, each the given frames + void Add(std::string_view file, const FrameIndex *index, int batch_idx, + const std::vector> &samples) { + PlannedRun run{file, index, batch_idx, -1, -1, {}}; + for (auto &sample : samples) { + frames.push_back(sample); + std::sort(frames.back().begin(), frames.back().end()); + run.samples.push_back(make_cspan(frames.back())); + for (int frame : sample) { + run.first = run.first < 0 ? frame : std::min(run.first, frame); + run.last = std::max(run.last, frame); + } + } + runs.push_back(run); + } + + // Adds a run of one sample, the frames from `first` to `last` + void Add(std::string_view file, const FrameIndex *index, int batch_idx, int first, int last) { + std::vector sample; + for (int frame = first; frame <= last; frame++) + sample.push_back(frame); + Add(file, index, batch_idx, {sample}); + } +}; + +std::vector Plan(const Batch &batch, const std::vector &lanes) { std::vector plans; - PlanLanes(make_cspan(runs), make_cspan(lanes), kOpenCost, plans); + PlanLanes(make_cspan(batch.runs), make_cspan(lanes), kOpenCost, plans); EXPECT_EQ(plans.size(), lanes.size()); return plans; } @@ -56,7 +99,7 @@ std::vector ReferenceOrder(const std::vector &runs_, const Lane auto continues = [&](int r) { const auto &run = runs_[r]; return run.filename == start.filename && - CanReachWithoutSeek(run.index, start.next_frame, run.first); + CanReachWithoutSeek(run.index, start.pos.next_frame, run.first); }; std::map file_order; for (int r : runs) { @@ -78,8 +121,120 @@ std::vector ReferenceOrder(const std::vector &runs_, const Lane return runs; } +std::vector Frames(const std::vector &frame_idxs, int start, int end, int stride, + boundary::BoundaryType boundary_type, int roi_start, int roi_end) { + SmallVector frames = {-100}; // cleared + SampleFrames(make_cspan(frame_idxs), start, end, stride, boundary_type, roi_start, roi_end, + frames); + return std::vector(frames.begin(), frames.end()); +} + +/** + * @brief A sample of the reader: the frames from `start` to `end` by `stride`, or `frame_idxs`, + * padded by clamping to the file. + */ +struct ReaderSample { + std::string path; + int start, end, stride; + std::vector frame_idxs = {}; +}; + +constexpr int kMaxSampleFrames = 16; + +/** + * @brief A lane of the reader, with the same decoder from batch to batch. + */ +struct ReaderLane { + std::unique_ptr decoder; + std::string filename; // of the decoder +}; + +/** + * @brief Plans a batch as the reader does, one run per sample, then decodes it, lane by lane, as + * the reader does, and checks that the lanes decode the pictures planned. + */ +void CheckPlannedPictures(const std::vector &batch, std::vector &lanes, + const std::map &indices, + std::vector &buffer, const std::string &what) { + std::deque> frames; + std::vector runs; + for (int i = 0; i < static_cast(batch.size()); i++) { + auto &sample = batch[i]; + auto &index = indices.at(sample.path); + frames.emplace_back(); + SampleFrames(make_cspan(sample.frame_idxs), sample.start, sample.end, sample.stride, + boundary::BoundaryType::CLAMP, 0, index.size(), frames.back()); + ASSERT_FALSE(frames.back().empty()); + runs.push_back({sample.path, &index, i, frames.back().front(), frames.back().back(), + {make_cspan(frames.back())}}); + } + std::vector starts(lanes.size()); + for (size_t l = 0; l < lanes.size(); l++) { + if (lanes[l].decoder) + starts[l] = {lanes[l].filename, DecoderPosition(*lanes[l].decoder)}; + } + std::vector plans; + PlanLanes(make_cspan(runs), make_cspan(starts), kOpenCost, plans); + + size_t planned_runs = 0; + for (size_t l = 0; l < lanes.size(); l++) { + auto &lane = lanes[l]; + int64_t decoded = 0; + for (int r : plans[l].runs) { + auto &sample = batch[r]; + if (!lane.decoder || lane.filename != sample.path) { + lane.decoder = std::make_unique(sample.path); + lane.decoder->SetIndex(indices.at(sample.path)); + lane.filename = sample.path; + } + auto &decoder = *lane.decoder; + int64_t before = decoder.DecodedPictures(); + if (sample.frame_idxs.empty()) { + ASSERT_LE((sample.end - sample.start + sample.stride - 1) / sample.stride, + kMaxSampleFrames); + decoder.DecodeFrames(buffer.data(), sample.start, sample.end, sample.stride, + boundary::BoundaryType::CLAMP); + } else { + ASSERT_LE(sample.frame_idxs.size(), static_cast(kMaxSampleFrames)); + decoder.DecodeFrames(buffer.data(), make_cspan(sample.frame_idxs), + boundary::BoundaryType::CLAMP); + } + decoded += decoder.DecodedPictures() - before; + } + EXPECT_EQ(decoded, plans[l].pictures) << what << ", lane " << l; + EXPECT_GE(plans[l].load - plans[l].pictures, 0) << what << ", lane " << l; + EXPECT_EQ((plans[l].load - plans[l].pictures) % kOpenCost, 0) << what << ", lane " << l; + planned_runs += plans[l].runs.size(); + } + EXPECT_EQ(planned_runs, runs.size()) << what; +} + } // namespace +TEST(DecodeLanePlannerTest, SampleFrames) { + using boundary::BoundaryType; + // the given frames (uniform sampling), sorted, duplicates kept + EXPECT_EQ(Frames({7, 3, 9, 3}, 0, 0, 1, BoundaryType::CLAMP, 0, 10), + std::vector({3, 3, 7, 9})); + // a range, with a stride + EXPECT_EQ(Frames({}, 2, 10, 3, BoundaryType::ISOLATED, 0, 10), std::vector({2, 5, 8})); + // padded past the end of the file + EXPECT_EQ(Frames({}, 7, 13, 1, BoundaryType::CLAMP, 0, 10), + std::vector({7, 8, 9, 9, 9, 9})); + EXPECT_EQ(Frames({}, 7, 13, 1, BoundaryType::REFLECT_101, 0, 10), + std::vector({6, 7, 7, 8, 8, 9})); + EXPECT_EQ(Frames({}, 7, 13, 1, BoundaryType::REFLECT_1001, 0, 10), + std::vector({7, 7, 8, 8, 9, 9})); + EXPECT_EQ(Frames({}, 7, 13, 1, BoundaryType::CONSTANT, 0, 10), std::vector({7, 8, 9})); + EXPECT_EQ(Frames({}, 10, 13, 1, BoundaryType::CONSTANT, 0, 10), std::vector()); + // a region of the file: clamped to it + EXPECT_EQ(Frames({}, 18, 22, 1, BoundaryType::CLAMP, 20, 30), + std::vector({20, 20, 20, 21})); + EXPECT_EQ(Frames({}, 28, 32, 1, BoundaryType::CLAMP, 20, 30), + std::vector({28, 29, 29, 29})); + EXPECT_THROW(Frames({}, 7, 13, 1, BoundaryType::ISOLATED, 0, 10), std::exception); +} + TEST(DecodeLanePlannerTest, CanReachWithoutSeek) { auto index = MakeIndex(100, 30); EXPECT_TRUE(CanReachWithoutSeek(&index, 5, 5)); @@ -94,12 +249,11 @@ TEST(DecodeLanePlannerTest, CanReachWithoutSeek) { TEST(DecodeLanePlannerTest, LongestFirstToEarliestFinish) { auto index = MakeIndex(100, 100); // three files, a run of each from frame 0: 30, 20 and 10 frames - std::vector runs = { - {"b", &index, 1, 0, 19}, - {"a", &index, 0, 0, 29}, - {"c", &index, 2, 0, 9}, - }; - auto plans = Plan(runs, {{}, {}}); + Batch batch; + batch.Add("b", &index, 1, 0, 19); + batch.Add("a", &index, 0, 0, 29); + batch.Add("c", &index, 2, 0, 9); + auto plans = Plan(batch, {{}, {}}); // a: lane 0 (a tie, to the lowest lane), b: lane 1, c: lane 1 (36 + 26 < 46 + 26) EXPECT_EQ(plans[0].runs, std::vector({1})); EXPECT_EQ(plans[0].load, 30 + kOpenCost); @@ -110,22 +264,76 @@ TEST(DecodeLanePlannerTest, LongestFirstToEarliestFinish) { TEST(DecodeLanePlannerTest, HeldFileContinues) { auto index = MakeIndex(100, 50); // frames 20-39 of the GOP from frame 0: 20 frames of preroll after a seek - std::vector runs = {{"a", &index, 0, 20, 39}}; - auto plans = Plan(runs, {{"b", 0}, {"a", 20}}); + Batch batch; + batch.Add("a", &index, 0, 20, 39); + auto plans = Plan(batch, {{"b", {0, -1}}, Held("a", index, 20)}); EXPECT_TRUE(plans[0].runs.empty()); EXPECT_EQ(plans[1].runs, std::vector({0})); EXPECT_EQ(plans[1].load, 20); // no opening, no preroll // a decoder that has to seek back pays the preroll, but not the opening - plans = Plan(runs, {{"b", 0}, {"a", 45}}); + plans = Plan(batch, {{"b", {0, -1}}, Held("a", index, 45)}); EXPECT_EQ(plans[1].runs, std::vector({0})); EXPECT_EQ(plans[1].load, 20 + 20); } TEST(DecodeLanePlannerTest, NoIndex) { // without an index, a run costs its frames (and the opening), with no preroll - std::vector runs = {{"a", nullptr, 0, 20, 39}}; - auto plans = Plan(runs, {{}}); + Batch batch; + batch.Add("a", nullptr, 0, 20, 39); + auto plans = Plan(batch, {{}}); EXPECT_EQ(plans[0].load, 20 + kOpenCost); + plans = Plan(batch, {{"a", {45, -1}}}); + EXPECT_EQ(plans[0].load, 20); +} + +TEST(DecodeLanePlannerTest, HeldFileGoesFirst) { + // A lane that holds X gets a run of Y, the longer one, and then the run of X that continues + // where its decoder is. It decodes X first, on its decoder, and then opens Y. + auto index = MakeIndex(100, 100); + Batch batch; + batch.Add("y", &index, 0, 0, 29); + batch.Add("x", &index, 1, 10, 19); + auto plans = Plan(batch, {Held("x", index, 10)}); + EXPECT_EQ(plans[0].runs, std::vector({1, 0})); + EXPECT_EQ(plans[0].load, 10 + kOpenCost + 30); // not reopening X after Y +} + +TEST(DecodeLanePlannerTest, RunsOfAFileInAscendingOrder) { + // Two runs of a file with a single GOP: the lane decodes them in ascending order, through from + // the start, not the later one first and then back from the start again + auto index = MakeIndex(200, 200); + Batch batch; + batch.Add("a", &index, 0, 100, 109); + batch.Add("a", &index, 1, 0, 9); + auto plans = Plan(batch, {{}}); + EXPECT_EQ(plans[0].runs, std::vector({1, 0})); + EXPECT_EQ(plans[0].load, kOpenCost + 110); +} + +TEST(DecodeLanePlannerTest, LeadIn) { + // 20 packets hidden by an edit list before the first frame, which is not a keyframe: every + // frame is decoded from the start of the stream + FrameIndex index; + index.timebase = {1, 25}; + for (int i = 0; i < 50; i++) + index.index.push_back({i, 0, false, i == 49, i + 20, i == 0 ? 0 : -1}); + Batch batch; + batch.Add("a", &index, 0, 30, 39); + auto plans = Plan(batch, {{}}); + EXPECT_EQ(plans[0].load, kOpenCost + 60); + // seeking back from frame 45 also starts from the hidden packets + plans = Plan(batch, {Held("a", index, 45)}); + EXPECT_EQ(plans[0].load, 60); +} + +TEST(DecodeLanePlannerTest, ResetAfterTheLastFrame) { + // Edge padding repeats the last frame; a decoder that returned it reopens the file to return + // it again, and decodes from the start + auto index = MakeIndex(50, 50); + Batch batch; + batch.Add("a", &index, 0, {{45, 46, 47, 48, 49, 49, 49}}); + auto plans = Plan(batch, {{}}); + EXPECT_EQ(plans[0].load, kOpenCost + 50 + 2 * (kOpenCost + 50)); } TEST(DecodeLanePlannerTest, LaneOrder) { @@ -134,7 +342,7 @@ TEST(DecodeLanePlannerTest, LaneOrder) { auto index = MakeIndex(300, 30); std::vector files = {"a", "b", "c", "d"}; for (int iter = 0; iter < 200; iter++) { - std::vector runs; + Batch batch; int num_runs = std::uniform_int_distribution<>(1, 12)(rng); std::vector batch_idx(num_runs); for (int i = 0; i < num_runs; i++) @@ -148,25 +356,67 @@ TEST(DecodeLanePlannerTest, LaneOrder) { first = rng() % 280; } while (std::count(firsts[file].begin(), firsts[file].end(), first)); firsts[file].push_back(first); - runs.push_back({file, &index, batch_idx[r], first, first + static_cast(rng() % 20)}); + batch.Add(file, &index, batch_idx[r], first, first + static_cast(rng() % 20)); } std::vector lanes(std::uniform_int_distribution<>(1, 4)(rng)); for (auto &lane : lanes) { if (rng() % 3) - lane = {files[rng() % files.size()], static_cast(rng() % 300)}; + lane = Held(files[rng() % files.size()], index, rng() % 300); } - auto plans = Plan(runs, lanes); + auto plans = Plan(batch, lanes); std::vector planned; for (size_t l = 0; l < lanes.size(); l++) { - EXPECT_EQ(plans[l].runs, ReferenceOrder(runs, lanes[l], plans[l].runs)) + EXPECT_EQ(plans[l].runs, ReferenceOrder(batch.runs, lanes[l], plans[l].runs)) << "iteration " << iter << ", lane " << l; planned.insert(planned.end(), plans[l].runs.begin(), plans[l].runs.end()); } std::sort(planned.begin(), planned.end()); - ASSERT_EQ(planned.size(), runs.size()); + ASSERT_EQ(planned.size(), batch.runs.size()); for (int r = 0; r < num_runs; r++) EXPECT_EQ(planned[r], r); } } +TEST(DecodeLanePlannerTest, PlannedPicturesAreDecoded) { + // VP9 (the codecs of the CPU decoder): 60 frames with a keyframe every 10, and 50 frames with + // a single keyframe + std::string gops = testing::dali_extra_path() + "/db/video/vfr/test_2_vp9.mp4"; + std::string single_gop = testing::dali_extra_path() + "/db/video/cfr/test_1_vp9.mp4"; + std::map indices; + size_t frame_size = 0; + for (auto &path : {gops, single_gop}) { + FramesDecoderCpu decoder(path); + decoder.BuildIndex(); + indices[path] = decoder.GetIndex(); + frame_size = std::max(frame_size, decoder.FrameSize()); + ASSERT_GE(decoder.NumFrames(), 50) << path; + } + std::vector buffer(kMaxSampleFrames * frame_size); + int n = indices[gops].size(); + + for (int num_lanes : {1, 2}) { + // the first lane holds a decoder of the first file, which decoded frames 20 to 29 + std::vector lanes(num_lanes); + lanes[0].decoder = std::make_unique(gops); + lanes[0].decoder->SetIndex(indices[gops]); + lanes[0].filename = gops; + lanes[0].decoder->DecodeFrames(buffer.data(), 20, 30, 1); + + // continuing on that decoder, the last frame repeated (edge padding: the decoder resets), a + // stride, uniform sampling, seeking forward and back, and continuing from the previous batch + std::string what = make_string(num_lanes, " lanes, batch "); + CheckPlannedPictures({{gops, n - 6, n + 2, 1}, + {gops, 30, 46, 2}, + {gops, 0, 0, 1, {0, n / 3, 2 * n / 3, n - 1}}, + {single_gop, 0, 16, 1}, + {single_gop, 25, 41, 1}}, + lanes, indices, buffer, what + "1"); + CheckPlannedPictures({{gops, 30, 46, 1}, + {gops, 5, 21, 1}, + {single_gop, 16, 32, 1}, + {single_gop, 34, 50, 1}}, + lanes, indices, buffer, what + "2"); + } +} + } // namespace dali diff --git a/dali/operators/video/reader/video_reader_decoder_op.cc b/dali/operators/video/reader/video_reader_decoder_op.cc index 5c668b8e9da..0810e6ef978 100644 --- a/dali/operators/video/reader/video_reader_decoder_op.cc +++ b/dali/operators/video/reader/video_reader_decoder_op.cc @@ -25,6 +25,7 @@ #include "dali/core/boundary.h" #include "dali/core/cuda_stream_pool.h" +#include "dali/core/small_vector.h" #include "dali/core/span.h" #include "dali/operators/reader/reader_op.h" @@ -578,6 +579,7 @@ class VideoReaderDecoder int batch_idx; // first position of the sample in the batch int first; // smallest frame index decoded; -1 if none (padding only) int last; // largest frame index decoded; -1 if none + SmallVector frames; // the frames the decoder seeks to, in order (see SampleFrames) }; /** @@ -609,12 +611,15 @@ class VideoReaderDecoder // estimated work std::vector runs; int64_t load = 0; + // Diagnostics: the pictures the plan expects the lane to decode, and those it decodes + int64_t planned_pictures = 0; + int64_t decoded_pictures = 0; // Errors of the current batch: (position in the batch, exception) std::vector> errors; }; - // Planning costs, in frames decoded: opening a decoder (and its first frame) costs about as - // much as decoding a few frames + // Planning costs, in pictures decoded: opening a decoder (or resetting it, which reopens the + // file) costs about as much as decoding a few pictures static constexpr int64_t kOpenCost = 16; // GPU: decode lanes per NVDEC engine. More sessions than engines keep an engine busy while a @@ -650,30 +655,6 @@ class VideoReaderDecoder "VideoReaderDecoder"); } - /** - * @brief The range of frames a sample decodes: [first, last], or first = last = -1 if none. - */ - void FrameRange(const VideoSample &sample, int &first, int &last) const { - int roi_start = sample.video_file_meta_->start_frame; - int roi_end = sample.video_file_meta_->end_frame; - first = -1; - last = -1; - auto add = [&](int frame) { - int idx = FramesDecoderBase::HandleBoundary(boundary_type_, frame, roi_start, roi_end); - if (idx < 0) - return; // a constant frame - first = first < 0 ? idx : std::min(first, idx); - last = std::max(last, idx); - }; - if (!sample.frame_idxs_.empty()) { - for (int frame : sample.frame_idxs_) - add(frame); - } else { - for (int frame = sample.start_; frame < sample.end_; frame += sample.stride_) - add(frame); - } - } - /** * @brief Groups the batch's samples into runs: samples of one file that one decoder decodes in * a row without seeking. @@ -694,13 +675,19 @@ class VideoReaderDecoder repeated = repeated || task.sample == sample; if (repeated) continue; - SampleTask task{sample, i, -1, -1}; + SampleTask task{sample, i, -1, -1, {}}; try { - FrameRange(*sample, task.first, task.last); + SampleFrames(sample->frame_idxs_, sample->start_, sample->end_, sample->stride_, + boundary_type_, sample->video_file_meta_->start_frame, + sample->video_file_meta_->end_frame, task.frames); } catch (...) { - // decoding the sample reports the error, in its lane + task.frames.clear(); // decoding the sample reports the error, in its lane + } + if (!task.frames.empty()) { + task.first = task.frames.front(); + task.last = task.frames.back(); } - tasks_.push_back(task); + tasks_.push_back(std::move(task)); } std::vector order(tasks_.size()); @@ -742,20 +729,23 @@ class VideoReaderDecoder std::vector runs; for (const auto &run : runs_) { indices.push_back(FrameIndexCache::instance().Find(*run.filename)); - runs.push_back({*run.filename, indices.back().get(), run.batch_idx, run.first, run.last}); + runs.push_back({*run.filename, indices.back().get(), run.batch_idx, run.first, run.last, {}}); + for (int t : run.tasks) + runs.back().samples.push_back(make_cspan(tasks_[t].frames)); } std::vector filenames(lanes_.size()); // of the lanes' decoders std::vector starts(lanes_.size()); for (size_t l = 0; l < lanes_.size(); l++) { if (lanes_[l].decoder) { filenames[l] = lanes_[l].decoder->Filename(); - starts[l] = {filenames[l], lanes_[l].decoder->NextFrameIdx()}; + starts[l] = {filenames[l], DecoderPosition(*lanes_[l].decoder)}; } } PlanLanes(make_cspan(runs), make_cspan(starts), kOpenCost, plans_); for (size_t l = 0; l < lanes_.size(); l++) { lanes_[l].runs = std::move(plans_[l].runs); lanes_[l].load = plans_[l].load; + lanes_[l].planned_pictures = plans_[l].pictures; if (lanes_[l].runs.empty() && lanes_.size() > 1) lanes_[l].decoder.reset(); // do not keep its frame buffers and NVDEC decoder idle } @@ -797,6 +787,7 @@ class VideoReaderDecoder void RunLane(DecodeLane &lane) { lane.errors.clear(); + lane.decoded_pictures = 0; for (int r : lane.runs) { for (int t : runs_[r].tasks) { auto &task = tasks_[t]; @@ -822,6 +813,9 @@ class VideoReaderDecoder (void)cudaGetLastError(); // already failing: report the sample's error } } + // Planned with the files' indices, the two match (on the GPU, within a picture or two) + LOG_LINE << "Lane decoded " << lane.decoded_pictures << " pictures, planned " + << lane.planned_pictures << " (load " << lane.load << ")" << std::endl; } /** @@ -856,6 +850,7 @@ class VideoReaderDecoder } auto &decoder = *lane.decoder; DALI_ENFORCE(decoder.IsValid(), make_string("Invalid decoder for filename ", filename)); + int64_t decoded_pictures = decoder.DecodedPictures(); int64_t num_frames = sample.frame_idxs_.empty() ? (sample.end_ - sample.start_ + sample.stride_ - 1) / sample.stride_ @@ -933,6 +928,7 @@ class VideoReaderDecoder sample.end_, sample.stride_, boundary_type_, constant_frame, make_span(sample.timestamps_)); } + lane.decoded_pictures += decoder.DecodedPictures() - decoded_pictures; LOG_LINE << "Decoding frames done" << std::endl; } diff --git a/dali/test/python/decoder/test_video.py b/dali/test/python/decoder/test_video.py index b282eb824ee..4f4fcceee10 100644 --- a/dali/test/python/decoder/test_video.py +++ b/dali/test/python/decoder/test_video.py @@ -1366,17 +1366,21 @@ def _assert_frames_match_file(frames, frame_nums, filename): ] +# H.264 clips that start with packets hidden by an edit list (62 and 16), which are decoded before +# the first frame, and after a seek that lands on them +lead_in_files = [ + f"{get_dali_extra_path()}/db/video/sintel/labelled_videos/0/sintel_trailer-720p_6.mp4", + f"{get_dali_extra_path()}/db/video/sintel/labelled_videos/2/sintel_trailer-720p_5.mp4", +] + + def _lanes_reader_files(device): # H.264 is not enabled on the CPU backend - return cfr_files + vfr_files + (codec_files["h264"] if device == "gpu" else []) + gpu_files = codec_files["h264"] + lead_in_files + return cfr_files + vfr_files + (gpu_files if device == "gpu" else []) -@cartesian_params(["cpu", "gpu"], range(len(_lanes_reader_cases))) -def test_reader_output_independent_of_num_threads(device, case): - skip_if_m60() - kwargs = _lanes_reader_cases[case] - batch_size = 4 - +def _check_reader_output_independent_of_num_threads(device, kwargs, batch_size, num_threads): files = _lanes_reader_files(device) @pipeline_def(batch_size=batch_size, device_id=0, seed=42) @@ -1394,11 +1398,11 @@ def pipe(): ) compare_pipelines( - pipe(num_threads=1), pipe(num_threads=4), batch_size=batch_size, N_iterations=8 + pipe(num_threads=1), pipe(num_threads=num_threads), batch_size=batch_size, N_iterations=8 ) # The two runs could agree on wrong frames, so check the frames against PyAV as well - p = pipe(num_threads=4) + p = pipe(num_threads=num_threads) for _ in range(4): videos, labels, frame_nums, *_ = (t.as_cpu() for t in p.run()) for i in range(batch_size): @@ -1409,6 +1413,32 @@ def pipe(): ) +@cartesian_params(["cpu", "gpu"], range(len(_lanes_reader_cases))) +def test_reader_output_independent_of_num_threads(device, case): + skip_if_m60() + _check_reader_output_independent_of_num_threads( + device, _lanes_reader_cases[case], batch_size=4, num_threads=4 + ) + + +# With fewer lanes than samples, a lane decodes several runs, of one file or more, in an order +# picked from their estimated work: tails padded with the last frame (decoding it again reopens +# the file) and uniformly sampled frames +_multi_run_lanes_cases = [ + dict(sequence_length=7, pad_mode="edge", random_shuffle=True, initial_fill=16), + dict(sequence_length=7, step=4, pad_mode="edge"), + dict(sequence_length=6, uniform_sample=True), +] + + +@cartesian_params(["cpu", "gpu"], range(len(_multi_run_lanes_cases))) +def test_reader_multi_run_lanes_output_independent_of_num_threads(device, case): + skip_if_m60() + _check_reader_output_independent_of_num_threads( + device, _multi_run_lanes_cases[case], batch_size=8, num_threads=2 + ) + + @params("cpu", "gpu") def test_reader_pad_last_batch_repeats_last_sample(device): """With pad_last_batch, a shard's last sample is repeated, here as the whole next batch: the From 7d1c30cf483903e9b76c8fef44366ffc83bc9588 Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Mon, 26 Oct 2026 19:22:32 +0100 Subject: [PATCH 29/46] Keep the NVDEC output surface mapped until its conversion is done FramesDecoderGpu waited for the color space conversion of every picture (cudaStreamSynchronize on its stream) before it unmapped the output surface, so the host was idle while each conversion ran. The surface now stays mapped after the conversion is launched, and an event is recorded behind it. The wait and the unmap move to ReleaseMappedFrame, which runs right before the next cuvidDecodePicture, in Flush, and in the destructor (after its stream sync), before the decoder goes back to the NVDEC cache. So NVDEC is still used only after the host has seen every earlier conversion finish, as before, and at most one surface is mapped at a time; only host work (reading, filtering and parsing the next packet) now overlaps the conversion. The output does not change. The destructor unmaps the surface on the decoder's device, as it may run on a thread where another device is current. A surface counts as unmapped only once cuvidUnmapVideoFrame succeeds: one that failed to unmap stays tracked, so the destructor tries again and the tests see it. A conversion that fails after the map (e.g. for an unsupported output type) no longer leaks the mapping: the surface is tracked before anything can throw, and released after a sync of the whole stream. NVDECFramesMapped counts the mapped surfaces, for tests. The move constructor is deleted: the parser calls back into the object, and a moved-from copy of a mapping would be unmapped twice. Signed-off-by: Joaquin Anton Guirao --- dali/operators/video/frames_decoder_gpu.cc | 73 +++++++++++++++++++--- dali/operators/video/frames_decoder_gpu.h | 28 ++++++++- 2 files changed, 92 insertions(+), 9 deletions(-) diff --git a/dali/operators/video/frames_decoder_gpu.cc b/dali/operators/video/frames_decoder_gpu.cc index bf39ea1c25b..deb1a0a5cc5 100644 --- a/dali/operators/video/frames_decoder_gpu.cc +++ b/dali/operators/video/frames_decoder_gpu.cc @@ -15,6 +15,7 @@ #include "dali/operators/video/frames_decoder_gpu.h" #include #include +#include #include #include #include @@ -314,6 +315,24 @@ size_t NVDECDecodersCreated(int device_id) { return NVDECCache::GetCache(device_id).NumCreated(); } +namespace { + +std::atomic frames_mapped{0}; // see NVDECFramesMapped + +void FrameMapped() { + frames_mapped.fetch_add(1, std::memory_order_relaxed); +} + +void FrameUnmapped() { + frames_mapped.fetch_sub(1, std::memory_order_relaxed); +} + +} // namespace + +int64_t NVDECFramesMapped() { + return frames_mapped.load(std::memory_order_relaxed); +} + int process_video_sequence(void *user_data, CUVIDEOFORMAT *video_format) { FramesDecoderGpu *frames_decoder = static_cast(user_data); frames_decoder->InitGpuDecoder(video_format); @@ -423,6 +442,8 @@ void FramesDecoderGpu::InitGpuDecoder(CUVIDEOFORMAT *video_format) { void FramesDecoderGpu::InitGpuParser() { nvdecode_state_ = std::make_unique(); + // on the current device, which is also the one that the decoder is taken from (NVDECCache) + conversion_done_ = CUDAEvent::CreateWithFlags(cudaEventDisableTiming); InitBitStreamFilter(); @@ -487,6 +508,22 @@ FramesDecoderGpu::FramesDecoderGpu(const char *memory_file, size_t memory_file_s } } +void FramesDecoderGpu::ReleaseMappedFrame() { + if (!mapped_frame_) + return; + assert(nvdecode_state_ && nvdecode_state_->decoder); + // Once unmapped, NVDEC may write the surface again: the conversion has to be done first + if (conversion_recorded_) + CUDA_CALL(cudaEventSynchronize(conversion_done_)); + else + CUDA_CALL(cudaStreamSynchronize(stream_)); + conversion_recorded_ = false; + // a surface that failed to unmap stays tracked: the destructor tries again + CUDA_CALL(cuvidUnmapVideoFrame(nvdecode_state_->decoder, mapped_frame_)); + mapped_frame_ = 0; + frame_dec_gpu_impl::FrameUnmapped(); +} + int FramesDecoderGpu::ProcessPictureDecode(CUVIDPICPARAMS *picture_params) { // Sending empty packet will call this callback. // If we want to flush the decoder, we do not need to do anything here @@ -494,6 +531,8 @@ int FramesDecoderGpu::ProcessPictureDecode(CUVIDPICPARAMS *picture_params) { return 0; } + // NVDEC is not used while the previous picture's surface may still be read + ReleaseMappedFrame(); CUDA_CALL(cuvidDecodePicture(nvdecode_state_->decoder, picture_params)); decoded_pictures_++; CUVIDPARSERDISPINFO picture_display_info; @@ -554,12 +593,16 @@ int FramesDecoderGpu::HandlePictureDisplay(CUVIDPARSERDISPINFO *picture_display_ CUdeviceptr frame = {}; unsigned int pitch = 0; + assert(!mapped_frame_); // released before this picture was decoded CUDA_CALL(cuvidMapVideoFrame( nvdecode_state_->decoder, picture_display_info->picture_index, &frame, &pitch, &videoProcessingParameters)); + mapped_frame_ = frame; // tracked before anything can throw + frame_dec_gpu_impl::FrameMapped(); + conversion_recorded_ = false; LOG_LINE << "VideoColorSpaceConversion with conversion_type_=" << static_cast(conversion_type_) << " and dimensions " << Height() << "x" << Width() @@ -575,11 +618,10 @@ int FramesDecoderGpu::HandlePictureDisplay(CUVIDPARSERDISPINFO *picture_display_ stream_); ), DALI_FAIL(make_string("Unsupported type: ", dtype_))); - // TODO(awolant): Alternative is to copy the data to a buffer - // and then process it on the stream. Check, if this is faster, when - // the benchmark is ready. - CUDA_CALL(cudaStreamSynchronize(stream_)); - CUDA_CALL(cuvidUnmapVideoFrame(nvdecode_state_->decoder, frame)); + // The host does not wait for the conversion here: the surface is unmapped by ReleaseMappedFrame, + // before NVDEC is used again + CUDA_CALL(cudaEventRecord(conversion_done_, stream_)); + conversion_recorded_ = true; return 1; } @@ -939,6 +981,7 @@ void FramesDecoderGpu::Reset() { void FramesDecoderGpu::Flush() { LOG_LINE << "Flush" << std::endl; + ReleaseMappedFrame(); // a flushed decoder holds no surface SendLastPacket(true); more_frames_to_decode_ = true; frame_index_if_no_pts_ = 0; @@ -947,11 +990,25 @@ void FramesDecoderGpu::Flush() { FramesDecoderGpu::~FramesDecoderGpu() { // frame_buffer_ goes back to the memory pool right after this, without stream ordering: wait - // for the copies out of it (CopyFrame, copyD2D) that may still be pending on stream_. Nothing - // is thrown from the destructor: a failing sync means the CUDA context is already broken, and - // that error is reported by the call that caused it. + // for the copies out of it (CopyFrame, copyD2D) that may still be pending on stream_, and for the + // conversion of a surface that is still mapped. Nothing is thrown from the destructor: a failing + // sync means the CUDA context is already broken, and that error is reported by the call that + // caused it. if (cudaStreamSynchronize(stream_) != cudaSuccess) (void)cudaGetLastError(); + // stream_ is idle: the surface can be unmapped before the decoder goes back to the cache. The + // destructor may run on a thread with another device current (e.g. when the operator is + // destroyed), so the decoder's device is set. A failed unmap is left counted in + // NVDECFramesMapped, so that the tests see it. + if (mapped_frame_ && nvdecode_state_ && nvdecode_state_->decoder) { + try { + DeviceGuard dg(nvdecode_state_->decoder.device_id()); + if (cuvidUnmapVideoFrame(nvdecode_state_->decoder, mapped_frame_) == CUDA_SUCCESS) + frame_dec_gpu_impl::FrameUnmapped(); + } catch (...) { + // the device could not be set: the context is broken, see above + } + } filtered_packet_.reset(); bsfc_.reset(); } diff --git a/dali/operators/video/frames_decoder_gpu.h b/dali/operators/video/frames_decoder_gpu.h index 8a263ef770f..9846227be27 100644 --- a/dali/operators/video/frames_decoder_gpu.h +++ b/dali/operators/video/frames_decoder_gpu.h @@ -29,6 +29,7 @@ extern "C" { #include "dali/operators/video/dynlink_nvcuvid/cuviddec.h" #include "dali/operators/video/dynlink_nvcuvid/dynlink_nvcuvid.h" +#include "dali/core/cuda_event.h" #include "dali/core/unique_handle.h" #include "dali/core/dev_buffer.h" @@ -107,6 +108,10 @@ class NVDECLease { operator CUvideodecoder() && = delete; + int device_id() const noexcept { + return decoder->device_id; + } + explicit operator bool() const noexcept { return decoder != nullptr; } @@ -126,6 +131,12 @@ class NVDECLease { */ DLL_PUBLIC size_t NVDECDecodersCreated(int device_id = -1); +/** + * @brief Returns how many NVDEC output surfaces are mapped by FramesDecoderGpu objects in this + * process. Meant for tests: it is 0 when no decoder is alive. + */ +DLL_PUBLIC int64_t NVDECFramesMapped(); + } // namespace frame_dec_gpu_impl struct NvDecodeState { @@ -181,7 +192,8 @@ class DLL_PUBLIC FramesDecoderGpu : public FramesDecoderBase { int HandlePictureDisplay(CUVIDPARSERDISPINFO *picture_display_info); - FramesDecoderGpu(FramesDecoderGpu&&) = default; + // not movable: the parser calls back into this object (and it may hold a mapped surface) + FramesDecoderGpu(FramesDecoderGpu&&) = delete; ~FramesDecoderGpu(); @@ -203,6 +215,20 @@ class DLL_PUBLIC FramesDecoderGpu : public FramesDecoderBase { private: std::unique_ptr nvdecode_state_; + + /** + * @brief Unmaps the output surface of the last converted picture, once its conversion is done. + * + * The surface stays mapped after the conversion is launched, so that the host does not wait for + * each picture; it is released before NVDEC is used again (the next picture, a flush), which is + * when NVDEC could overwrite it. + */ + void ReleaseMappedFrame(); + + CUdeviceptr mapped_frame_ = 0; // the mapped output surface, 0 if none + CUDAEvent conversion_done_; // recorded on stream_ after mapped_frame_ is converted + bool conversion_recorded_ = false; // false: the conversion failed, wait for the whole stream + void *current_frame_output_ = nullptr; bool current_copy_to_output_ = false; bool frame_returned_ = false; From a043d1c2f07638d6a68fcc079b537d7bff9b2a6b Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Mon, 26 Oct 2026 19:40:15 +0100 Subject: [PATCH 30/46] Report asynchronous GPU errors of a video sample at that sample The GPU frames decoder no longer waits for each picture, so an asynchronous error in a sample's conversions or copies would surface at a later sample of the same decoder, or at the end of a reader lane, where it is not attributed to any sample. The video reader lanes and decoders.video now wait for the decoder's stream after each sample, inside that sample's error handling. In the reader this is a wait for the last conversion and copies of the sample; the lane-end sync stays as a safety net. In decoders.video the decoder's destructor waited for that stream anyway, but swallowed its errors. inputs.video keeps its decoder across Run calls, so it waits for its stream at the end of each Run: the errors of the last pictures of a Run are reported by this operator, not by the next Run or by whatever touches the context next. Signed-off-by: Joaquin Anton Guirao --- dali/operators/video/decoder/video_decoder_base.h | 5 +++++ dali/operators/video/frames_decoder_gpu.h | 7 +++++++ dali/operators/video/input/video_input.h | 6 ++++++ dali/operators/video/reader/video_reader_decoder_op.cc | 5 +++++ 4 files changed, 23 insertions(+) diff --git a/dali/operators/video/decoder/video_decoder_base.h b/dali/operators/video/decoder/video_decoder_base.h index cea16902326..1dc2be75945 100644 --- a/dali/operators/video/decoder/video_decoder_base.h +++ b/dali/operators/video/decoder/video_decoder_base.h @@ -309,6 +309,11 @@ class DLL_PUBLIC VideoDecoderBase : public Operator { start_frame, end_frame, stride, boundary_type_, constant_frame); } + if constexpr (!std::is_same_v) { + // the decoder does not wait for its GPU work: its asynchronous errors are reported + // for this sample (the destructor would only swallow them) + CUDA_CALL(cudaStreamSynchronize(ctx.stream)); + } frames_decoders_[s].reset(); }, output[s].shape().num_elements()); diff --git a/dali/operators/video/frames_decoder_gpu.h b/dali/operators/video/frames_decoder_gpu.h index 9846227be27..7ef0d5ecc0c 100644 --- a/dali/operators/video/frames_decoder_gpu.h +++ b/dali/operators/video/frames_decoder_gpu.h @@ -210,6 +210,13 @@ class DLL_PUBLIC FramesDecoderGpu : public FramesDecoderBase { void CopyFrame(uint8_t *dst, const uint8_t *src) override; + /** + * @brief The stream the decoder converts and copies frames on. + */ + cudaStream_t stream() const noexcept { + return stream_; + } + protected: bool SelectVideoStream(int stream_id = -1) override; diff --git a/dali/operators/video/input/video_input.h b/dali/operators/video/input/video_input.h index 8891bb93e76..56d60a5d82a 100644 --- a/dali/operators/video/input/video_input.h +++ b/dali/operators/video/input/video_input.h @@ -337,6 +337,12 @@ void VideoInput::VideoInputRunImpl(Workspace &ws) { } } } + if constexpr (!is_cpu) { + // the decoder does not wait for its GPU work: its asynchronous errors are reported by this + // operator, in this Run. Its own stream is waited for (it was fixed when the decoder was + // created), so the frames are complete even if ws.stream() is not that stream. + CUDA_CALL(cudaStreamSynchronize(frames_decoder.stream())); + } // If true, this operator can be run again, after this Run. bool will_return_next = true; diff --git a/dali/operators/video/reader/video_reader_decoder_op.cc b/dali/operators/video/reader/video_reader_decoder_op.cc index 0810e6ef978..2e468f2d886 100644 --- a/dali/operators/video/reader/video_reader_decoder_op.cc +++ b/dali/operators/video/reader/video_reader_decoder_op.cc @@ -793,6 +793,11 @@ class VideoReaderDecoder auto &task = tasks_[t]; try { DecodeSample(*task.sample, lane); + if constexpr (std::is_same_v) { + // the decoder does not wait for its GPU work: its asynchronous errors (conversions, + // copies) are reported for this sample + CUDA_CALL(cudaStreamSynchronize(lane.stream ? lane.stream.get() : 0)); + } task.sample->decoded = true; } catch (...) { lane.errors.emplace_back(task.batch_idx, std::current_exception()); From 84dc6591b6cf31d3e6e05ec13faeec1a1af1935e Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Mon, 26 Oct 2026 19:40:15 +0100 Subject: [PATCH 31/46] Test the GPU frames decoder with conversions pending The decoder no longer waits for each picture's conversion. Hold up the decoder's stream before every frame, so that the conversions are still pending when the next picture is decoded, when the decoder seeks and flushes, and when it is destroyed, and check that the frames are the same as those decoded on the default stream without the delay, with and without an index. Also check that the output surface is unmapped before the NVDEC decoder goes back to the cache, where the next decoder takes it on another stream; that decoders can be destroyed in another thread than the one that decoded; and that a conversion failing after the map does not leave the surface mapped. Signed-off-by: Joaquin Anton Guirao --- dali/operators/video/frames_decoder_test.cc | 191 ++++++++++++++++++++ 1 file changed, 191 insertions(+) diff --git a/dali/operators/video/frames_decoder_test.cc b/dali/operators/video/frames_decoder_test.cc index 393c9a569ba..39eff74641a 100644 --- a/dali/operators/video/frames_decoder_test.cc +++ b/dali/operators/video/frames_decoder_test.cc @@ -14,13 +14,18 @@ #include #include +#include #include +#include +#include #include #include #include +#include #include #include "dali/core/cuda_error.h" +#include "dali/core/cuda_stream.h" #include "dali/core/dev_buffer.h" #include "dali/core/device_guard.h" #include "dali/core/dynlink_cuda.h" @@ -387,6 +392,96 @@ void CheckGpuPositionAt(const std::vector &paths) { } } +// Holds up the work enqueued on `stream` after this call for `ms` milliseconds +void DelayStream(cudaStream_t stream, int ms) { + auto sleep = [](void *ms) { + std::this_thread::sleep_for(std::chrono::milliseconds(reinterpret_cast(ms))); + }; + CUDA_CALL(cudaLaunchHostFunc(stream, sleep, reinterpret_cast(intptr_t{ms}))); +} + +constexpr int kConversionDelayMs = 5; + +// Frames to read in this order: in a row, seeking forward and back, the last frame, from the start +// after the end and across a short leap (the skipped frames are decoded, but not returned) +std::vector ReadOrder(int num_frames) { + std::vector frames = FrameRange(0, 10); + for (int frame : {25, 26, 12, 13, num_frames - 1, 0, 1, 30, 33}) + frames.push_back(std::min(frame, num_frames - 1)); + return frames; +} + +using GpuDecoderFactory = std::function(cudaStream_t)>; + +/** + * @brief Reads `frame_ids` in this order (seeking to a frame that is not the next one) into `out`, + * one frame after another, on the stream of the decoder. + * + * With `delay_ms`, the stream is held up before each frame is read, so that the conversions of the + * pictures are still pending when the decoder decodes the next picture (or is destroyed). Checks + * that at most `max_mapped` output surfaces are mapped in the process meanwhile. + */ +void ReadFrames(FramesDecoderGpu &decoder, cudaStream_t stream, const std::vector &frame_ids, + uint8_t *out, int delay_ms, int max_mapped = 1) { + for (int frame : frame_ids) { + if (decoder.NextFrameIdx() != frame) + decoder.SeekFrame(frame); + if (delay_ms > 0) + DelayStream(stream, delay_ms); + ASSERT_TRUE(decoder.ReadNextFrame(out)) << "frame " << frame; + EXPECT_LE(frame_dec_gpu_impl::NVDECFramesMapped(), max_mapped); + out += decoder.FrameSize(); + } +} + +/** + * @brief Decodes `frame_ids` (see ReadFrames) with a decoder on `stream` and returns them on the + * host. The decoder is destroyed first, while the conversion of its last picture may be pending. + */ +std::vector DecodeToHost(const GpuDecoderFactory &make_decoder, cudaStream_t stream, + const std::vector &frame_ids, int delay_ms) { + auto decoder = make_decoder(stream); + size_t frame_size = decoder->FrameSize(); + DeviceBuffer frames; + frames.resize(frame_ids.size() * frame_size); + ReadFrames(*decoder, stream, frame_ids, frames.data(), delay_ms); + decoder.reset(); + EXPECT_EQ(frame_dec_gpu_impl::NVDECFramesMapped(), 0) << "a surface was not unmapped"; + std::vector host(frames.size()); + MemCopy(host.data(), frames.data(), host.size(), stream); + CUDA_CALL(cudaStreamSynchronize(stream)); + return host; +} + +/** + * @brief Checks that the frames (see ReadOrder) decoded on a stream of their own, held up so that + * the conversions are pending, are the same as those decoded on the default stream without the + * delay, and match the ground truth. + */ +void CheckConversionPending(const GpuDecoderFactory &make_decoder, TestVideo &ground_truth, + double eps) { + auto frame_ids = ReadOrder(ground_truth.NumFrames()); + auto reference = DecodeToHost(make_decoder, 0, frame_ids, 0); + auto stream = CUDAStream::Create(true); + auto frames = DecodeToHost(make_decoder, stream, frame_ids, kConversionDelayMs); + ASSERT_EQ(frames.size(), reference.size()); + EXPECT_TRUE(frames == reference); + size_t frame_size = ground_truth.FrameSize(); + for (size_t i = 0; i < frame_ids.size(); i++) { + CompareFrameAvgError(frame_ids[i], frame_size, ground_truth.Width(), ground_truth.Height(), + frames.data() + i * frame_size, ground_truth.frames_[frame_ids[i]].data, + eps); + } +} + +GpuDecoderFactory FileDecoder(const std::string &path) { + return [path](cudaStream_t stream) { + auto decoder = std::make_unique(path, stream); + decoder->BuildIndex(); + return decoder; + }; +} + } // namespace TEST_F(FramesDecoderTest_CpuOnlyTests, DecodeWork) { @@ -506,6 +601,102 @@ TEST_F(FramesDecoderGpuTest, IndexDecodeOrderLeadIn) { } } +// The GPU decoder does not wait for the conversion of a picture before it decodes the next one: +// held up on a stream of its own, the conversions are pending when the decoder decodes the next +// picture, seeks, flushes and is destroyed. The frames are the same as when nothing is pending. +TEST_F(FramesDecoderGpuTest, ConversionPendingAtNextPicture) { + CheckConversionPending(FileDecoder(cfr_videos_paths_[0]), cfr_videos_[0], 1.5); +} + +// The same without an index: the frames go through the frame buffer of the decoder +TEST_F(FramesDecoderGpuTest, ConversionPendingAtNextPictureNoIndex) { + auto &ground_truth = vfr_videos_[1]; + auto memory_video = MemoryVideo(vfr_mpeg4_mkv_videos_paths_[1]); + auto make_decoder = [&](cudaStream_t stream) { + auto decoder = + std::make_unique(memory_video.data(), memory_video.size(), "", stream); + decoder->SetNumFrames(ground_truth.NumFrames()); + return decoder; + }; + CheckConversionPending(make_decoder, ground_truth, 3.0); +} + +// A decoder destroyed while the conversion of its last picture is pending unmaps the surface +// before NVDEC goes back to the cache, where the next decoder, on another stream, takes it +TEST_F(FramesDecoderGpuTest, DecoderReturnedToCacheIsClean) { + std::vector frame_ids = FrameRange(0, 8); + frame_ids.push_back(20); + std::vector> reference; + for (int clip = 0; clip < 5; clip++) // also puts a decoder for these clips in the cache + reference.push_back(DecodeToHost(FileDecoder(SintelClip(clip % 3, clip)), 0, frame_ids, 0)); + size_t num_created = frame_dec_gpu_impl::NVDECDecodersCreated(); + CUDAStream streams[2] = {CUDAStream::Create(true), CUDAStream::Create(true)}; + for (int clip = 0; clip < 5; clip++) { + auto frames = DecodeToHost(FileDecoder(SintelClip(clip % 3, clip)), streams[clip % 2], + frame_ids, kConversionDelayMs); + EXPECT_TRUE(frames == reference[clip]) << "clip " << clip; + } + EXPECT_EQ(frame_dec_gpu_impl::NVDECDecodersCreated(), num_created); +} + +// Decoders that run in threads of their own, on streams of their own, are destroyed in another +// thread (as the video reader does with the decoders of idle lanes), while the conversion of their +// last picture is likely pending +TEST_F(FramesDecoderGpuTest, DecodersDestroyedInAnotherThread) { + constexpr int kThreads = 4; + std::vector frame_ids = FrameRange(0, 6); + frame_ids.push_back(15); + std::vector> reference; + for (int clip = 0; clip < kThreads; clip++) + reference.push_back(DecodeToHost(FileDecoder(SintelClip(clip % 3, clip)), 0, frame_ids, 0)); + + std::vector streams; + std::vector> decoders(kThreads); + std::vector> frames(kThreads); + for (int t = 0; t < kThreads; t++) + streams.push_back(CUDAStream::Create(true)); + auto run = [&](std::function body) { + std::vector threads; + for (int t = 0; t < kThreads; t++) { + threads.emplace_back([&, t]() { + DeviceGuard guard(0); + body(t); + }); + } + for (auto &thread : threads) + thread.join(); + }; + run([&](int t) { + decoders[t] = FileDecoder(SintelClip(t % 3, t))(streams[t]); + frames[t].resize(frame_ids.size() * decoders[t]->FrameSize()); + ReadFrames(*decoders[t], streams[t], frame_ids, frames[t].data(), 20, kThreads); + }); + run([&](int t) { + decoders[(t + 1) % kThreads].reset(); + }); + EXPECT_EQ(frame_dec_gpu_impl::NVDECFramesMapped(), 0); + for (int t = 0; t < kThreads; t++) { + std::vector host(frames[t].size()); + MemCopy(host.data(), frames[t].data(), host.size(), streams[t]); + CUDA_CALL(cudaStreamSynchronize(streams[t])); + EXPECT_TRUE(host == reference[t]) << "clip " << t; + } +} + +// A conversion that fails once the surface is mapped (here: for an output type that it does not +// support) does not leave the surface mapped when the decoder goes back to the cache +TEST_F(FramesDecoderGpuTest, SurfaceUnmappedAfterFailedConversion) { + auto stream = CUDAStream::Create(true); + { + FramesDecoderGpu decoder(cfr_videos_paths_[0], stream); + decoder.BuildIndex(); + decoder.SetOutputType(DALI_INT16); + EXPECT_THROW(decoder.ReadNextFrame(FrameData()), std::exception); + EXPECT_EQ(frame_dec_gpu_impl::NVDECFramesMapped(), 1); + } + EXPECT_EQ(frame_dec_gpu_impl::NVDECFramesMapped(), 0); +} + TEST_F(FramesDecoderTest_CpuOnlyTests, ConstantFrameRate) { FramesDecoderCpu decoder(cfr_videos_paths_[0]); decoder.BuildIndex(); From 55f5c3c450cb033fd97584db71b9987d557cc55f Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Mon, 26 Oct 2026 22:21:36 +0100 Subject: [PATCH 32/46] Do not map the video frames that the GPU decoder never reads The GPU frames decoder mapped and color-converted every picture that it did not return as it came, also those that are never read. With an index, these are e.g. the pictures that an edit list hides at the start of a clip (up to 188 in the Sintel clips), which are decoded before the first frames, and again after each seek that lands on them. Each of them also took a frame buffer of its own, kept until the decoder is destroyed: up to ~190 buffers (~500 MB at 720p) per decoder. A buffered picture is now mapped and converted only if it may be read with its data later: if its timestamp is that of a frame at or after the next one in the index. Until the decoder is flushed, the buffered frames are only looked up for those frames (the next frame only moves forward between flushes), so the other pictures are never read. Such a picture still takes its slot in the frame buffer, with the same timestamp as before, so which frames are found, in which order, and what is decoded do not change; only the frame buffer is not needed. The slots now share the frame buffers: a slot that needs one takes the buffer of a free slot, or of one that keeps only a timestamp, and allocates one only if there is none. Without an index, every picture is mapped, as before. NVDEC decodes the same pictures, and the output does not change. Reading a frame of which only the timestamp was kept is an internal error, so a wrong decision gives an error, not a wrong frame. MappedPictures, NumFrameBuffers and SetBufferedFrameDataForTesting are meant for tests: the last one lets a test map every buffered picture, to compare the frames read with those of the decoder as it was, or none. Signed-off-by: Joaquin Anton Guirao --- dali/operators/video/frames_decoder_base.h | 4 ++ dali/operators/video/frames_decoder_gpu.cc | 56 +++++++++++++++++++++- dali/operators/video/frames_decoder_gpu.h | 53 +++++++++++++++++++- 3 files changed, 110 insertions(+), 3 deletions(-) diff --git a/dali/operators/video/frames_decoder_base.h b/dali/operators/video/frames_decoder_base.h index 5de2b06a0cf..4d31ce92522 100644 --- a/dali/operators/video/frames_decoder_base.h +++ b/dali/operators/video/frames_decoder_base.h @@ -23,6 +23,7 @@ extern "C" { #include } +#include #include #include #include @@ -446,6 +447,9 @@ class DLL_PUBLIC FramesDecoderBase { } void SetIndex(const FrameIndex& index) { + // sorted by timestamp, as BuildIndex makes it (the GPU decoder searches it) + assert(std::is_sorted(index.index.begin(), index.index.end(), + [](const IndexEntry &a, const IndexEntry &b) { return a.pts < b.pts; })); index_ = index; num_frames_ = index.size(); } diff --git a/dali/operators/video/frames_decoder_gpu.cc b/dali/operators/video/frames_decoder_gpu.cc index deb1a0a5cc5..bb25ea8cbc1 100644 --- a/dali/operators/video/frames_decoder_gpu.cc +++ b/dali/operators/video/frames_decoder_gpu.cc @@ -15,6 +15,7 @@ #include "dali/operators/video/frames_decoder_gpu.h" #include #include +#include #include #include #include @@ -587,6 +588,17 @@ int FramesDecoderGpu::HandlePictureDisplay(CUVIDPARSERDISPINFO *picture_display_ // Put currently decoded frame to the buffer for later auto &slot = FindEmptySlot(); slot.pts_ = current_pts_; + // A frame that is never read with its data (e.g. hidden by an edit list) keeps its place in + // the buffer, but it is neither mapped nor converted + slot.has_data_ = buffered_frame_data_ == BufferedFrameData::kIfRead + ? MayBeReadWithData(slot.pts_) + : buffered_frame_data_ == BufferedFrameData::kAll; + if (!slot.has_data_) { + LOG_LINE << "Not mapping frame with timestamp " << slot.pts_ << ": it is not read" + << std::endl; + return 1; + } + AttachFrameBuffer(slot); frame_output = slot.frame_.data(); } @@ -602,6 +614,7 @@ int FramesDecoderGpu::HandlePictureDisplay(CUVIDPARSERDISPINFO *picture_display_ &videoProcessingParameters)); mapped_frame_ = frame; // tracked before anything can throw frame_dec_gpu_impl::FrameMapped(); + mapped_pictures_++; conversion_recorded_ = false; LOG_LINE << "VideoColorSpaceConversion with conversion_type_=" @@ -648,6 +661,9 @@ bool FramesDecoderGpu::ReadNextFrameWithIndex(uint8_t *data) { for (auto &frame : frame_buffer_) { if (frame.pts_ != -1 && frame.pts_ == index_[next_frame_idx_].pts) { if (copy_to_output) { + DALI_ENFORCE(frame.has_data_, make_string("Internal error: frame ", next_frame_idx_, + " of \"", Filename(), "\" is read, but only its " + "timestamp was kept.")); LOG_LINE << "Copying from frame buffer to frame_output" << std::endl; copyD2D(data, frame.frame_.data(), FrameSize(), stream_); } @@ -824,6 +840,8 @@ bool FramesDecoderGpu::ReadNextFrameWithoutIndex(uint8_t *data) { if (dtype_ == DALI_FLOAT) { total_size *= 4; } + // without an index, every buffered frame keeps its data (see MayBeReadWithData) + assert(frame_buffer_[frame_to_return_index].has_data_); LOG_LINE << "Copying from frame buffer at index " << frame_to_return_index << " to frame_output at " << current_frame_output_ << " (frame_idx=" << next_frame_idx_ << ")" << std::endl; @@ -937,11 +955,45 @@ BufferedFrame& FramesDecoderGpu::FindEmptySlot() { } frame_buffer_ = std::move(new_frame_buffer); auto &new_frame = frame_buffer_.back(); - new_frame.frame_.resize(FrameSize()); - new_frame.pts_ = -1; + new_frame.pts_ = -1; // its frame buffer is attached if the frame keeps its data return new_frame; } +void FramesDecoderGpu::AttachFrameBuffer(BufferedFrame &slot) { + if (!slot.frame_.empty()) + return; + for (auto &other : frame_buffer_) { + // the buffer of an empty slot, or of a slot that keeps only a timestamp: a conversion into it + // runs on stream_, after any pending copy out of it + if (!other.frame_.empty() && (other.pts_ == -1 || !other.has_data_)) { + slot.frame_ = std::move(other.frame_); + return; + } + } + slot.frame_.resize(FrameSize()); +} + +bool FramesDecoderGpu::MayBeReadWithData(int64_t pts) const { + if (!HasIndex() || next_frame_idx_ < 0 || next_frame_idx_ >= static_cast(index_.size())) + return true; + // The index is sorted by timestamp and, until the decoder is flushed, a buffered frame is only + // looked up for the frames from the next one on: a frame that is not among them (e.g. hidden by + // an edit list, or shown before the next one) is never read + auto first = index_.index.begin() + next_frame_idx_; + auto entry = std::lower_bound(first, index_.index.end(), pts, + [](const IndexEntry &e, int64_t pts) { return e.pts < pts; }); + return entry != index_.index.end() && entry->pts == pts; +} + +int FramesDecoderGpu::NumFrameBuffers() const { + int num_buffers = 0; + for (auto &frame : frame_buffer_) { + if (!frame.frame_.empty()) + num_buffers++; + } + return num_buffers; +} + bool FramesDecoderGpu::HasEmptySlot() const { for (auto &frame : frame_buffer_) { if (frame.pts_ == -1) { diff --git a/dali/operators/video/frames_decoder_gpu.h b/dali/operators/video/frames_decoder_gpu.h index 7ef0d5ecc0c..33f38d058d2 100644 --- a/dali/operators/video/frames_decoder_gpu.h +++ b/dali/operators/video/frames_decoder_gpu.h @@ -149,8 +149,9 @@ struct NvDecodeState { }; struct BufferedFrame { - DeviceBuffer frame_; + DeviceBuffer frame_; // may be empty when the slot keeps no frame data int pts_; + bool has_data_ = false; // false: the frame is never read with its data, only pts_ is kept }; class DLL_PUBLIC FramesDecoderGpu : public FramesDecoderBase { @@ -217,9 +218,52 @@ class DLL_PUBLIC FramesDecoderGpu : public FramesDecoderBase { return stream_; } + /** + * @brief Number of frame buffers (device memory for one frame each) held by the decoder, for + * tests. + */ + int NumFrameBuffers() const; + + /** + * @brief Number of pictures mapped and converted so far (for tests and diagnostics). + * + * Pictures that are decoded, but never read (see MayBeReadWithData) or skipped as they come, + * are not. + */ + int64_t MappedPictures() const { + return mapped_pictures_; + } + protected: bool SelectVideoStream(int stream_id = -1) override; + /** + * @brief Which of the buffered frames (those not returned as they are decoded) are mapped and + * converted + */ + enum class BufferedFrameData { + kIfRead, // those that may be read with their data (see MayBeReadWithData) + kAll, // all of them, as before the decoder left out those that are not read + kNone, // none of them + }; + + /** + * @brief For tests: maps all the buffered frames (to compare the frames read with the default) + * or none of them (to check that reading a frame without its data fails). + */ + void SetBufferedFrameDataForTesting(BufferedFrameData data) { + buffered_frame_data_ = data; + } + + /** + * @brief Whether a buffered frame with this timestamp may be read with its data later. + * + * Decoding a frame that is not read (e.g. one hidden by an edit list) is still needed for the + * frames that refer to it, but mapping and converting it is not. Reading a frame without its + * data is an internal error. + */ + bool MayBeReadWithData(int64_t pts) const; + private: std::unique_ptr nvdecode_state_; @@ -235,6 +279,8 @@ class DLL_PUBLIC FramesDecoderGpu : public FramesDecoderBase { CUdeviceptr mapped_frame_ = 0; // the mapped output surface, 0 if none CUDAEvent conversion_done_; // recorded on stream_ after mapped_frame_ is converted bool conversion_recorded_ = false; // false: the conversion failed, wait for the whole stream + int64_t mapped_pictures_ = 0; // see MappedPictures + BufferedFrameData buffered_frame_data_ = BufferedFrameData::kIfRead; void *current_frame_output_ = nullptr; bool current_copy_to_output_ = false; @@ -264,6 +310,11 @@ class DLL_PUBLIC FramesDecoderGpu : public FramesDecoderBase { BufferedFrame& FindEmptySlot(); + /** + * @brief Gives the slot a frame buffer, taking a spare one from another slot if there is any. + */ + void AttachFrameBuffer(BufferedFrame &slot); + bool HasEmptySlot() const; bool IsBufferEmpty() const; From 6141f89aa3840c2a1fa5bf29aa712912616f5a04 Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Mon, 26 Oct 2026 23:06:11 +0100 Subject: [PATCH 33/46] Do not map the video frames that the GPU decoder only skips To read a frame, the decoder decodes the pictures from its keyframe on, and to read the next frames of a sample with gaps (a stride, a random or uniform sampling) it decodes through the frames in between. Those skipped frames that it decodes before it reaches them (e.g. P-frames decoded ahead of B-frames) are buffered, mapped and converted, although they are only skipped later. DecodeFrames now tells the decoder which frames it reads next, and SeekFrame, when called on its own, the frame it seeks to. A buffered frame up to the last of those frames that is not one of them is only skipped, so the GPU decoder keeps only its timestamp, as it does for the frames it never reads. Frames after the last one are still mapped: a later DecodeFrames on the same decoder (the next sample of a decode lane) may read them. A frame left without its data is below the next frame once DecodeFrames or SeekFrame returns, and reading a frame below the next one flushes the decoder first, so it is never read with its data afterwards either: not by a later sample, a duplicate or padded frame, nor after a checkpoint is restored. As before, which frames are buffered and decoded does not change, and neither does the output. Signed-off-by: Joaquin Anton Guirao --- dali/operators/video/frames_decoder_base.cc | 20 ++++++++++++++++++++ dali/operators/video/frames_decoder_base.h | 8 ++++++++ dali/operators/video/frames_decoder_gpu.cc | 19 ++++++++++++++----- dali/operators/video/frames_decoder_gpu.h | 6 +++--- 4 files changed, 45 insertions(+), 8 deletions(-) diff --git a/dali/operators/video/frames_decoder_base.cc b/dali/operators/video/frames_decoder_base.cc index 9871c6b7cb9..328d68bb928 100644 --- a/dali/operators/video/frames_decoder_base.cc +++ b/dali/operators/video/frames_decoder_base.cc @@ -17,6 +17,7 @@ #include #include #include +#include "dali/core/call_at_exit.h" #include "dali/core/error_handling.h" #include "dali/core/small_vector.h" #include "dali/operators/video/video_utils.h" @@ -621,6 +622,14 @@ void FramesDecoderBase::SeekFrame(int frame_id) { LOG_LINE << "After seeking: next_frame_idx_=" << next_frame_idx_ << ", frame_id=" << frame_id << std::endl; assert(next_frame_idx_ <= frame_id); + // Unless DecodeFrames knows more, the requested frame is the next one read with its data + bool own_frames_to_read = frames_to_read_.empty(); + if (own_frames_to_read) + frames_to_read_ = make_cspan(&frame_id, 1); + auto clear_frames_to_read = AtScopeExit([&] { + if (own_frames_to_read) + frames_to_read_ = {}; + }); // Skip all remaining frames until the requested frame LOG_LINE << "Skipping frames from " << next_frame_idx_ << " to " << frame_id << std::endl; for (int i = next_frame_idx_; i < frame_id; i++) { @@ -703,12 +712,23 @@ void FramesDecoderBase::DecodeFramesImpl(uint8_t *data, DALI_ENFORCE(constant_frame != nullptr || boundary_type != boundary::BoundaryType::CONSTANT, make_string("Constant frame must be provided if boundary type is CONSTANT")); + // The frames read with their data, in order (see frames_to_read_) + SmallVector to_read; + for (auto &[frame_id, i] : frame_ids) { + if (frame_id >= 0 && frame_id < NumFrames()) + to_read.push_back(frame_id); + } + auto clear_frames_to_read = AtScopeExit([&] { frames_to_read_ = {}; }); + int num_read = 0; + uint8_t *last_out_frame_start = nullptr; for (auto &[frame_id, i] : frame_ids) { uint8_t* out_frame_start = data + ptrdiff_t(i) * FrameSize(); assert(out_frame_start >= data); if (frame_id >= 0 && frame_id < NumFrames()) { LOG_LINE << "Decoding frame " << frame_id << " to position " << i << std::endl; + frames_to_read_ = make_cspan(to_read.data() + num_read, to_read.size() - num_read); + num_read++; SeekFrame(frame_id); ReadNextFrame(out_frame_start); last_out_frame_start = out_frame_start; diff --git a/dali/operators/video/frames_decoder_base.h b/dali/operators/video/frames_decoder_base.h index 4d31ce92522..ee3750d6102 100644 --- a/dali/operators/video/frames_decoder_base.h +++ b/dali/operators/video/frames_decoder_base.h @@ -511,6 +511,14 @@ class DLL_PUBLIC FramesDecoderBase { int64_t decoded_pictures_ = 0; // see DecodedPictures + /** + * @brief The frames read with their data next, in order, when known: DecodeFrames sets those it + * has yet to read and SeekFrame the one it seeks to. Empty: any frame may be read. + * + * A frame decoded on the way to them that is not one of them is only skipped. + */ + span frames_to_read_; + private: void DetectVariableFrameRate(); diff --git a/dali/operators/video/frames_decoder_gpu.cc b/dali/operators/video/frames_decoder_gpu.cc index bb25ea8cbc1..f393b47a021 100644 --- a/dali/operators/video/frames_decoder_gpu.cc +++ b/dali/operators/video/frames_decoder_gpu.cc @@ -588,14 +588,15 @@ int FramesDecoderGpu::HandlePictureDisplay(CUVIDPARSERDISPINFO *picture_display_ // Put currently decoded frame to the buffer for later auto &slot = FindEmptySlot(); slot.pts_ = current_pts_; - // A frame that is never read with its data (e.g. hidden by an edit list) keeps its place in - // the buffer, but it is neither mapped nor converted + // A frame that is never read with its data (e.g. hidden by an edit list, or only skipped on + // the way to the frames to read) keeps its place in the buffer, but it is neither mapped nor + // converted slot.has_data_ = buffered_frame_data_ == BufferedFrameData::kIfRead ? MayBeReadWithData(slot.pts_) : buffered_frame_data_ == BufferedFrameData::kAll; if (!slot.has_data_) { - LOG_LINE << "Not mapping frame with timestamp " << slot.pts_ << ": it is not read" - << std::endl; + LOG_LINE << "Not mapping frame with timestamp " << slot.pts_ + << ": it is not read with its data" << std::endl; return 1; } AttachFrameBuffer(slot); @@ -982,7 +983,15 @@ bool FramesDecoderGpu::MayBeReadWithData(int64_t pts) const { auto first = index_.index.begin() + next_frame_idx_; auto entry = std::lower_bound(first, index_.index.end(), pts, [](const IndexEntry &e, int64_t pts) { return e.pts < pts; }); - return entry != index_.index.end() && entry->pts == pts; + if (entry == index_.index.end() || entry->pts != pts) + return false; + // Up to the last of the frames to read next, a frame that is not one of them is only skipped; + // after it, a later DecodeFrames (e.g. the next sample of a decode lane) may read it + if (frames_to_read_.empty() || pts > index_[frames_to_read_.back()].pts) + return true; + auto read = std::lower_bound(frames_to_read_.begin(), frames_to_read_.end(), pts, + [&](int frame, int64_t pts) { return index_[frame].pts < pts; }); + return index_[*read].pts == pts; } int FramesDecoderGpu::NumFrameBuffers() const { diff --git a/dali/operators/video/frames_decoder_gpu.h b/dali/operators/video/frames_decoder_gpu.h index 33f38d058d2..9d00a3fc46b 100644 --- a/dali/operators/video/frames_decoder_gpu.h +++ b/dali/operators/video/frames_decoder_gpu.h @@ -258,9 +258,9 @@ class DLL_PUBLIC FramesDecoderGpu : public FramesDecoderBase { /** * @brief Whether a buffered frame with this timestamp may be read with its data later. * - * Decoding a frame that is not read (e.g. one hidden by an edit list) is still needed for the - * frames that refer to it, but mapping and converting it is not. Reading a frame without its - * data is an internal error. + * Decoding a frame that is not read (e.g. one hidden by an edit list, or one that is only + * skipped on the way to the frames to read) is still needed for the frames that refer to it, but + * mapping and converting it is not. Reading a frame without its data is an internal error. */ bool MayBeReadWithData(int64_t pts) const; From 24659da3387f5d331646168176e4a231a5a3d70d Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Mon, 26 Oct 2026 23:17:05 +0100 Subject: [PATCH 34/46] Test the GPU decoder with the frames it does not map Check that a GPU frames decoder returns the same frames as one that maps every picture (as the decoder did before), after decoding the same pictures, when it decodes samples with skipped frames, duplicates and padding, when it seeks, also with AvSeekFrame to a frame that is not a keyframe, and when it decodes one sample after another, also one that starts before the end of the previous one. The files: H.264 clips with long edit-list lead-ins, a VFR H.264 file and, if supported, HEVC ones. For a clip with a lead-in of 188 hidden pictures, check that the decoder maps no more than the frames it reads and the few decoded ahead of the last one, also when it skips frames, and that it keeps at most 10 frame buffers (before, one for each hidden picture). Reading a frame that was not mapped fails with an internal error. The reader tests also run on such a clip now, checked against PyAV around each frame (the whole file is slow to compare against). Signed-off-by: Joaquin Anton Guirao --- dali/operators/video/frames_decoder_test.cc | 208 ++++++++++++++++++++ dali/test/python/decoder/test_video.py | 43 +++- 2 files changed, 244 insertions(+), 7 deletions(-) diff --git a/dali/operators/video/frames_decoder_test.cc b/dali/operators/video/frames_decoder_test.cc index 39eff74641a..0853972145d 100644 --- a/dali/operators/video/frames_decoder_test.cc +++ b/dali/operators/video/frames_decoder_test.cc @@ -21,7 +21,9 @@ #include #include #include +#include #include +#include #include #include "dali/core/cuda_error.h" @@ -697,6 +699,212 @@ TEST_F(FramesDecoderGpuTest, SurfaceUnmappedAfterFailedConversion) { EXPECT_EQ(frame_dec_gpu_impl::NVDECFramesMapped(), 0); } +namespace { + +// A decoder that maps and converts every picture that it decodes, but does not return as it comes +// (as the decoder did before it left out the pictures that are not read) +class GpuDecoderMappingAll : public FramesDecoderGpu { + public: + template + explicit GpuDecoderMappingAll(Args &&...args) : FramesDecoderGpu(std::forward(args)...) { + SetBufferedFrameDataForTesting(BufferedFrameData::kAll); + } +}; + +// A decoder that maps none of the pictures that it does not return as they come +class GpuDecoderMappingNone : public FramesDecoderGpu { + public: + template + explicit GpuDecoderMappingNone(Args &&...args) : FramesDecoderGpu(std::forward(args)...) { + SetBufferedFrameDataForTesting(BufferedFrameData::kNone); + } +}; + +// The pictures that a decoder maps past the last frame it reads: those decoded before it, but shown +// after it (e.g. a P-frame decoded before the B-frames shown before it), which a next sample of a +// decode lane may read +constexpr int kMaxMappedAhead = 4; + +// Hashes of the `num_frames` frames in `frames` (on the device) +std::vector HashFrames(const uint8_t *frames, int num_frames, size_t frame_size, + cudaStream_t stream) { + std::vector host(frame_size); + std::vector hashes; + for (int i = 0; i < num_frames; i++) { + MemCopy(host.data(), frames + i * frame_size, frame_size, stream); + CUDA_CALL(cudaStreamSynchronize(stream)); + hashes.push_back(std::hash()(std::string_view(host.data(), frame_size))); + } + return hashes; +} + +std::vector Strided(int start, int end, int stride) { + std::vector frames; + for (int i = start; i < end; i += stride) + frames.push_back(i); + return frames; +} + +// Decodes `frame_ids` with DecodeFrames (padding with a constant frame) and hashes them +std::vector DecodeAndHash( + FramesDecoderGpu &decoder, const std::vector &frame_ids, + boundary::BoundaryType boundary_type = boundary::BoundaryType::ISOLATED) { + size_t frame_size = decoder.FrameSize(); + DeviceBuffer frames, constant; + frames.resize(frame_ids.size() * frame_size); + constant.resize(frame_size); + CUDA_CALL(cudaMemsetAsync(constant.data(), 77, frame_size, decoder.stream())); + decoder.DecodeFrames(frames.data(), make_cspan(frame_ids), boundary_type, constant.data()); + return HashFrames(frames.data(), frame_ids.size(), frame_size, decoder.stream()); +} + +// Reads `frame_ids` with SeekFrame and ReadNextFrame (see ReadFrames) and hashes them +std::vector ReadAndHash(FramesDecoderGpu &decoder, const std::vector &frame_ids) { + DeviceBuffer frames; + frames.resize(frame_ids.size() * decoder.FrameSize()); + ReadFrames(decoder, decoder.stream(), frame_ids, frames.data(), 0); + return HashFrames(frames.data(), frame_ids.size(), decoder.FrameSize(), decoder.stream()); +} + +using HashedFramesScenario = std::function(FramesDecoderGpu &)>; + +struct ScenarioResult { + std::vector frames; + int64_t decoded = 0, mapped = 0; +}; + +// Runs `scenario` with a new decoder (one at a time: ReadFrames expects one mapped surface at most) +template +ScenarioResult RunScenario(const std::string &path, const FrameIndex &index, + const HashedFramesScenario &scenario) { + Decoder decoder(path); + decoder.SetIndex(index); + ScenarioResult result; + result.frames = scenario(decoder); + result.decoded = decoder.DecodedPictures(); + result.mapped = decoder.MappedPictures(); + return result; +} + +/** + * @brief Runs `scenario` with a decoder and with one that maps every picture (GpuDecoderMappingAll) + * and checks that they return the same frames, after decoding the same pictures, of which the + * decoder maps no more. Returns the pictures mapped by the decoder. + */ +int64_t CheckSameFramesAsMappingAll(const std::string &path, const FrameIndex &index, + const std::string &what, const HashedFramesScenario &scenario) { + auto result = RunScenario(path, index, scenario); + auto expected = RunScenario(path, index, scenario); + EXPECT_EQ(result.frames, expected.frames) << path << ": " << what; + EXPECT_EQ(result.decoded, expected.decoded) << path << ": " << what; + EXPECT_LE(result.mapped, expected.mapped) << path << ": " << what; + return result.mapped; +} + +/** + * @brief Checks that a decoder that does not map the pictures it does not read returns the same + * frames as one that maps them all, when it decodes samples (with skipped frames, duplicates and + * padding), when it seeks, also with AvSeekFrame to a frame that is not a keyframe, and when one + * decoder decodes a sample after another (a decode lane), also one that starts before the end of + * the previous one. + */ +void CheckSameFramesAsMappingAll(const std::string &path) { + FramesDecoderGpu indexed(path); + indexed.BuildIndex(); + auto &index = indexed.GetIndex(); + int n = index.size(); + ASSERT_GE(n, 40) << path; + auto check = [&](const std::string &what, const HashedFramesScenario &scenario) { + CheckSameFramesAsMappingAll(path, index, what, scenario); + }; + check("stride 3", [&](FramesDecoderGpu &d) { return DecodeAndHash(d, Strided(0, 40, 3)); }); + check("a duplicate", [&](FramesDecoderGpu &d) { + return DecodeAndHash(d, {5, 5, 7, 30, 31, std::min(90, n - 1)}); + }); + for (auto pad : {boundary::BoundaryType::REFLECT_101, boundary::BoundaryType::CLAMP, + boundary::BoundaryType::CONSTANT}) { + check(make_string("padded with ", boundary::to_string(pad)), [&](FramesDecoderGpu &d) { + return DecodeAndHash(d, FrameRange(n - 4, n + 4), pad); + }); + } + static_assert(FramesDecoderBase::kMinimumSeekLeap >= 3); + constexpr int leap = FramesDecoderBase::kMinimumSeekLeap; + check("seeks", [&](FramesDecoderGpu &d) { + return ReadAndHash(d, {0, leap, 2 * leap + 1, 2 * leap + 3, n / 2, n / 2 + 3, 1}); + }); + check("a decode lane", [&](FramesDecoderGpu &d) { + auto frames = DecodeAndHash(d, Strided(0, 9, 2)); + for (auto &sample : {FrameRange(9, 17), FrameRange(16, 20)}) { + auto more = DecodeAndHash(d, sample); + frames.insert(frames.end(), more.begin(), more.end()); + } + return frames; + }); + int frame = n / 2; + while (frame < n - 3 && index[frame].is_keyframe) + frame++; + check("AvSeekFrame to a frame that is not a keyframe", [&](FramesDecoderGpu &d) { + EXPECT_TRUE(d.AvSeekFrame(index[frame].pts, frame)); + return ReadAndHash(d, FrameRange(frame, frame + 3)); + }); +} + +} // namespace + +// A decoder decodes the pictures that it does not return (hidden by an edit list, or skipped), but +// it does not map or convert them, nor keep a frame buffer for them +TEST_F(FramesDecoderGpuTest, PicturesNotReadAreNotMapped) { + // 188 pictures hidden by an edit list, decoded before the first frame + auto path = SintelVideoFile(4); + FramesDecoderGpu indexed(path); + indexed.BuildIndex(); + auto &index = indexed.GetIndex(); + auto check = [&](const std::string &what, const std::vector &frame_ids) { + auto mapped = CheckSameFramesAsMappingAll(path, index, what, [&](FramesDecoderGpu &d) { + auto frames = DecodeAndHash(d, frame_ids); + EXPECT_GT(d.DecodedPictures(), 188); + if (dynamic_cast(&d)) + EXPECT_GT(d.NumFrameBuffers(), 188); // one for each picture hidden by the edit list + else + EXPECT_LE(d.NumFrameBuffers(), 10); + return frames; + }); + EXPECT_LE(mapped, static_cast(frame_ids.size()) + kMaxMappedAhead) << what; + }; + check("in a row", FrameRange(0, 32)); + // the frames skipped are not mapped either: between the frames read, and before the first one + check("stride 3", Strided(0, 40, 3)); + check("from frame 30", {30, 31}); +} + +TEST_F(FramesDecoderGpuTest, SameFramesAsMappingAll) { + std::vector paths = {SintelVideoFile(4), SintelClip(2, 8), + testing::dali_extra_path() + "/db/video/vfr/test_1.mp4"}; + if (FramesDecoderGpu::SupportsHevc()) { + paths.push_back(testing::dali_extra_path() + "/db/video/hevc/sintel_trailer-720p.mp4"); + paths.push_back(testing::dali_extra_path() + "/db/video/vfr/test_2_hevc.mp4"); + } + for (auto &path : paths) + CheckSameFramesAsMappingAll(path); +} + +// Reading a frame that the decoder did not map is an internal error, not a frame of garbage +TEST_F(FramesDecoderGpuTest, ReadingFrameNotMappedFails) { + // B-frames: the frames shown after them are decoded before them, and buffered + GpuDecoderMappingNone decoder(SintelVideoFile(0)); + decoder.BuildIndex(); + DeviceBuffer frame; + frame.resize(decoder.FrameSize()); + try { + for (int i = 0; i < 60; i++) + decoder.ReadNextFrame(frame.data()); + FAIL() << "a frame that was not mapped was read"; + } catch (std::exception &e) { + EXPECT_NE(std::string(e.what()).find("only its timestamp was kept"), std::string::npos) + << e.what(); + } +} + TEST_F(FramesDecoderTest_CpuOnlyTests, ConstantFrameRate) { FramesDecoderCpu decoder(cfr_videos_paths_[0]); decoder.BuildIndex(); diff --git a/dali/test/python/decoder/test_video.py b/dali/test/python/decoder/test_video.py index 4f4fcceee10..69ff5ce16e6 100644 --- a/dali/test/python/decoder/test_video.py +++ b/dali/test/python/decoder/test_video.py @@ -1335,22 +1335,26 @@ def _reference_frames(filename): return _reference_cache[filename] -def _assert_frames_match_file(frames, frame_nums, filename): +def _assert_frames_match_file(frames, frame_nums, filename, window=None): """Checks that frames[i] is the frame frame_nums[i] of the file (negative: padding, skipped). Decoders differ slightly in color conversion, so a frame must be close to the reference and - closer to it than to any other frame of the file.""" + closer to it than to any other frame of the file (with `window`, to any other frame up to + `window` frames away, which is faster for long files).""" reference = _reference_frames(filename) for frame, num in zip(frames, frame_nums): if num < 0: continue - diffs = [np.abs(frame.astype(np.int32) - ref).mean() for ref in reference] + start = 0 if window is None else max(0, num - window) + end = len(reference) if window is None else min(len(reference), num + window + 1) + diffs = {i: np.abs(frame.astype(np.int32) - reference[i]).mean() for i in range(start, end)} + closest = min(diffs, key=diffs.get) assert ( diffs[num] < 10 ), f"{filename}: frame {num} differs from the reference by {diffs[num]}" assert ( - diffs[num] <= min(diffs) + 0.5 - ), f"{filename}: frame {num} is closer to frame {int(np.argmin(diffs))}" + diffs[num] <= diffs[closest] + 0.5 + ), f"{filename}: frame {num} is closer to frame {closest}" # The reader decodes the samples of a batch in parallel, by up to num_threads decoders (decode @@ -1380,8 +1384,10 @@ def _lanes_reader_files(device): return cfr_files + vfr_files + (gpu_files if device == "gpu" else []) -def _check_reader_output_independent_of_num_threads(device, kwargs, batch_size, num_threads): - files = _lanes_reader_files(device) +def _check_reader_output_independent_of_num_threads( + device, kwargs, batch_size, num_threads, files=None, window=None +): + files = files or _lanes_reader_files(device) @pipeline_def(batch_size=batch_size, device_id=0, seed=42) def pipe(): @@ -1410,6 +1416,7 @@ def pipe(): np.array(videos[i]), np.array(frame_nums[i]), files[int(np.array(labels[i])[0])], + window, ) @@ -1421,6 +1428,28 @@ def test_reader_output_independent_of_num_threads(device, case): ) +# An H.264 clip with a long lead-in hidden by an edit list (188 pictures). The GPU decoder decodes +# those pictures, but does not map or convert them, nor the pictures that it only skips +long_lead_in_files = [ + f"{get_dali_extra_path()}/db/video/sintel/video_files/sintel_trailer-720p_4.mp4", +] + + +# Consecutive sequences (from the start of a file, after its lead-in), sparse samples (frames +# skipped in between) and shuffled samples (seeks, also back to the lead-in) +@params(0, 1, 3) +def test_reader_lead_in_output_independent_of_num_threads(case): + skip_if_m60() + _check_reader_output_independent_of_num_threads( + "gpu", + _lanes_reader_cases[case], + batch_size=4, + num_threads=4, + files=long_lead_in_files, + window=16, + ) + + # With fewer lanes than samples, a lane decodes several runs, of one file or more, in an order # picked from their estimated work: tails padded with the last frame (decoding it again reopens # the file) and uniformly sampled frames From 9ca376ed162ce5986800bc0775b65aaee0aeeb78 Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Tue, 27 Oct 2026 03:19:01 +0100 Subject: [PATCH 35/46] Let readers read a batch and store its snapshot separately DataReader::Prefetch reads a batch into the producer's slot of the prefetch queue and stores the loader's state after it. A reader that reads its next batch ahead of the slot it is published in needs the two steps on their own: ReadBatch reads a batch into any batch vector, and SaveLoaderSnapshot takes the snapshot to store, taken when that batch was read. No behaviour change: Prefetch does the same through the new functions. Signed-off-by: Joaquin Anton Guirao --- dali/operators/reader/reader_op.h | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/dali/operators/reader/reader_op.h b/dali/operators/reader/reader_op.h index 33033d6a450..bb4fe36398d 100644 --- a/dali/operators/reader/reader_op.h +++ b/dali/operators/reader/reader_op.h @@ -57,6 +57,7 @@ template { public: using LoadTargetPtr = std::shared_ptr; + using BatchQueueElement = std::vector; inline explicit DataReader(const OpSpec& spec) : Operator(spec), @@ -93,12 +94,7 @@ class DataReader : public Operator { // We actually prepare the next batch DomainTimeRange tr("[DALI][DataReader] Prefetch #" + to_string(curr_batch_producer_), DomainTimeRange::kRed); - auto &curr_batch = prefetched_batch_queue_[curr_batch_producer_]; - curr_batch.clear(); - curr_batch.reserve(max_batch_size_); - for (int i = 0; i < max_batch_size_; ++i) { - curr_batch.push_back(loader_->ReadOne(i == 0, i == max_batch_size_ - 1)); - } + ReadBatch(prefetched_batch_queue_[curr_batch_producer_]); if (IsCheckpointingEnabled()) { SaveLoaderSnapshot(); } @@ -307,8 +303,27 @@ class DataReader : public Operator { } } + /** + * @brief Reads the next batch from the loader into `batch` (cleared first). + */ + void ReadBatch(BatchQueueElement &batch) { + batch.clear(); + batch.reserve(max_batch_size_); + for (int i = 0; i < max_batch_size_; ++i) { + batch.push_back(loader_->ReadOne(i == 0, i == max_batch_size_ - 1)); + } + } + + /** + * @brief Stores the loader's state after the batch that is being produced: the state the + * checkpoint of that batch restores. + */ + void SaveLoaderSnapshot(const LoaderStateSnapshot &snapshot) { + loader_snapshot_queue_[snapshot_producer_] = snapshot; + } + void SaveLoaderSnapshot() { - loader_snapshot_queue_[snapshot_producer_] = loader_->GetStateSnapshot(); + SaveLoaderSnapshot(loader_->GetStateSnapshot()); } void ParseIfNeeded(const Tensor& tensor, SampleWorkspace* ws) { @@ -432,7 +447,6 @@ class DataReader : public Operator { // prefetched batch int prefetch_queue_depth_; bool skip_cached_images_; - using BatchQueueElement = std::vector; std::vector prefetched_batch_queue_; int curr_batch_consumer_; int curr_batch_producer_; From f5d125f954869a30703bf58d50e9dbae242f47a9 Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Tue, 27 Oct 2026 03:33:44 +0100 Subject: [PATCH 36/46] Plan video decode lanes from where their batches leave them The video reader plans which lane decodes which samples of a batch from where the lanes' decoders are. It read that from the decoders themselves, so a batch could be planned only once the lanes were done with the batch before it. Where decoding a sample leaves its decoder is known before decoding: after the largest frame it decodes (DecodeFrames maps the frames into the file once more and decodes them in order), or where it was if it decodes none, or at frame 0 of a newly opened decoder. So the reader now keeps, for each lane, where its decoder will be once it is done with the batches planned so far, and plans the next batch from that. The plans are the same as before: the positions are normalized as the planner sees them (any frame past the end is the end). A decoder may be elsewhere only if it does not decode every frame it is asked for, which already leaves that sample's frames undefined. Decoding does not depend on it (it seeks from where the decoder is), only the plan can be worse. Each lane checks its decoder against the plan before decoding; DALI_VIDEO_READER_STRICT_PLANS, for tests, turns a mismatch into an error, and the tests of the lanes' output set it. Without it, a mismatch is only logged (in debug builds). A sample is marked when it is planned, not when it is decoded, which also covers samples repeated within a batch. DecoderPosition, now used only by the planner test, moves there. Signed-off-by: Joaquin Anton Guirao --- .../video/reader/decode_lane_planner.cc | 6 - .../video/reader/decode_lane_planner.h | 6 - .../video/reader/decode_lane_planner_test.cc | 5 + .../video/reader/video_reader_decoder_op.cc | 162 +++++++++++++++--- dali/test/python/decoder/test_video.py | 18 ++ 5 files changed, 162 insertions(+), 35 deletions(-) diff --git a/dali/operators/video/reader/decode_lane_planner.cc b/dali/operators/video/reader/decode_lane_planner.cc index 58d8e24cd2b..cb325fb53e7 100644 --- a/dali/operators/video/reader/decode_lane_planner.cc +++ b/dali/operators/video/reader/decode_lane_planner.cc @@ -49,12 +49,6 @@ void SampleFrames(span frame_idxs, int start, int end, int stride, std::sort(frames.begin(), frames.end()); } -FramesDecoderBase::DecodePosition DecoderPosition(FramesDecoderBase &decoder) { - int next = decoder.NextFrameIdx(); - return decoder.HasIndex() ? FramesDecoderBase::PositionAt(decoder.GetIndex(), next) - : FramesDecoderBase::DecodePosition{next, -1}; -} - namespace { using DecodePosition = FramesDecoderBase::DecodePosition; diff --git a/dali/operators/video/reader/decode_lane_planner.h b/dali/operators/video/reader/decode_lane_planner.h index 8860d575b1e..bf965323cec 100644 --- a/dali/operators/video/reader/decode_lane_planner.h +++ b/dali/operators/video/reader/decode_lane_planner.h @@ -73,12 +73,6 @@ DLL_PUBLIC void SampleFrames(span frame_idxs, int start, int end, int boundary::BoundaryType boundary_type, int roi_start, int roi_end, SmallVector &frames); -/** - * @brief Where a decoder is, for LaneStart: from its index (see FramesDecoderBase::PositionAt) or, - * without one, only its next frame. - */ -DLL_PUBLIC FramesDecoderBase::DecodePosition DecoderPosition(FramesDecoderBase &decoder); - /** * @brief Whether a decoder whose next frame is `next` decodes through to `frame` without * seeking (see FramesDecoderBase::SeekFrame). diff --git a/dali/operators/video/reader/decode_lane_planner_test.cc b/dali/operators/video/reader/decode_lane_planner_test.cc index 28da3487ca0..b4037c9d101 100644 --- a/dali/operators/video/reader/decode_lane_planner_test.cc +++ b/dali/operators/video/reader/decode_lane_planner_test.cc @@ -50,6 +50,11 @@ LaneStart Held(std::string_view file, const FrameIndex &index, int next_frame) { return {file, FramesDecoderBase::PositionAt(index, next_frame)}; } +// Where a decoder with an index is, for LaneStart +FramesDecoderBase::DecodePosition DecoderPosition(FramesDecoderBase &decoder) { + return FramesDecoderBase::PositionAt(decoder.GetIndex(), decoder.NextFrameIdx()); +} + /** * @brief The runs of a batch, and the frames of their samples. */ diff --git a/dali/operators/video/reader/video_reader_decoder_op.cc b/dali/operators/video/reader/video_reader_decoder_op.cc index 2e468f2d886..0c25841c9cd 100644 --- a/dali/operators/video/reader/video_reader_decoder_op.cc +++ b/dali/operators/video/reader/video_reader_decoder_op.cc @@ -13,6 +13,7 @@ // limitations under the License. #include +#include #include #include #include @@ -70,9 +71,10 @@ struct VideoSample : public VideoSampleDesc { Tensor data_; std::vector timestamps_; std::vector frame_idx_; - // Set once decoded. With pad_last_batch, the loader returns the last sample again, also in the - // next batch, which must not decode it again while the previous batch may still be read. - bool decoded = false; + // Set when the reader plans the sample's decoding. With pad_last_batch, the loader returns the + // last sample again, also in the next batch, which must not decode it again while the previous + // batch may still be read. + bool planned = false; }; enum class FileListFormat { @@ -408,6 +410,9 @@ class VideoReaderDecoder has_timestamps_(spec.GetArgument("enable_timestamps")), boundary_type_(GetBoundaryType(spec)), image_type_(spec.GetArgument("image_type")) { + // Test aid only: a lane's decoder that is not where the plan expects it fails the batch + const char *strict_plans = std::getenv("DALI_VIDEO_READER_STRICT_PLANS"); + strict_plans_ = strict_plans && *strict_plans && std::string_view(strict_plans) != "0"; loader_ = InitLoader(spec); this->SetInitialSnapshot(); @@ -579,9 +584,19 @@ class VideoReaderDecoder int batch_idx; // first position of the sample in the batch int first; // smallest frame index decoded; -1 if none (padding only) int last; // largest frame index decoded; -1 if none + int end; // where the sample leaves its decoder (see EndFrame) SmallVector frames; // the frames the decoder seeks to, in order (see SampleFrames) }; + /** + * @brief Where a lane's decoder is, as planned: its file (empty: no decoder) and the frame it + * returns next (see Normalize). + */ + struct LanePosition { + std::string filename; + int next_frame = -1; + }; + /** * @brief Samples of one file that one decoder decodes in a row, without seeking. */ @@ -611,6 +626,12 @@ class VideoReaderDecoder // estimated work std::vector runs; int64_t load = 0; + // Planning: where the decoder is when the lane starts the current batch, and where it is + // once the lane is done with it. The plans are made from these, not from the decoder. + LanePosition start, planned; + // Whether decoding a sample failed (and reset the decoder) since the lane last opened one: + // the decoder is then not where the plan expects it + bool failed = false; // Diagnostics: the pictures the plan expects the lane to decode, and those it decodes int64_t planned_pictures = 0; int64_t decoded_pictures = 0; @@ -626,6 +647,84 @@ class VideoReaderDecoder // lane does its host work between pictures; many more only add memory. static constexpr int kLanesPerEngine = 2; + // SampleTask::end of a sample that decodes no frame: its decoder stays where it was + static constexpr int kNoFrames = -2; + + /** + * @brief A decoder's next frame as the planner sees it: CanReachWithoutSeek and resetting + * decoders (see FramesDecoderBase::SeekFrame) treat any frame < 0 or >= num_frames as the end + * of the file, so these are all -1. + */ + static int Normalize(int next_frame, int num_frames) { + return next_frame < 0 || next_frame >= num_frames ? -1 : next_frame; + } + + /** + * @brief Where decoding a sample leaves its decoder: after the largest frame it decodes + * (Normalized), or kNoFrames. + * + * DecodeFrames maps the frames into [0, num_frames) once more, which differs from the sample's + * own range (see SampleFrames) when it ends past the end of the file. It decodes the frames in + * order, so that the largest one is read last. + */ + int EndFrame(const SampleTask &task, int num_frames) const { + int last = -1; + for (int frame : task.frames) { + last = std::max(last, FramesDecoderBase::HandleBoundary(boundary_type_, frame, 0, + num_frames)); + } + return last < 0 ? kNoFrames : Normalize(last + 1, num_frames); + } + + /** + * @brief Where a lane's decoder is once it decodes `runs`, in order, from `pos`. + */ + LanePosition PredictEnd(span runs, LanePosition pos) const { + for (int r : runs) { + for (int t : runs_[r].tasks) { + const auto &task = tasks_[t]; + const auto &filename = task.sample->video_file_meta_->filename; + if (pos.filename != filename) + pos = {filename, 0}; // a newly opened decoder + if (task.end != kNoFrames) + pos.next_frame = task.end; + } + } + return pos; + } + + /** + * @brief Whether a lane's decoder is at `pos`. + */ + static bool IsAt(DecodeLane &lane, const LanePosition &pos) { + if (!lane.decoder) + return pos.filename.empty(); + return lane.decoder->Filename() == pos.filename && + Normalize(lane.decoder->NextFrameIdx(), lane.decoder->NumFrames()) == pos.next_frame; + } + + /** + * @brief Checks that a lane's decoder is where the plan of its batch expects it. + * + * The plans are made from where decoding the samples leaves the decoders, which is where they + * are as long as they decode every frame they are asked for. If one is somewhere else, the lane + * still decodes the right frames (decoding seeks from where the decoder is), only the plan may + * be worse. With DALI_VIDEO_READER_STRICT_PLANS set (for tests), it is an error. + */ + void CheckPlan(DecodeLane &lane) { + if (lane.failed || IsAt(lane, lane.start)) + return; + LOG_LINE << "The decoder of a lane is at " + << (lane.decoder ? lane.decoder->Filename() : "no file") << ", frame " + << (lane.decoder ? lane.decoder->NextFrameIdx() : -1) << "; planned: " + << lane.start.filename << ", frame " << lane.start.next_frame << std::endl; + DALI_ENFORCE(!strict_plans_, make_string( + "Internal error: a video decoder is not where the plan expects it, at frame ", + lane.start.next_frame, " of \"", lane.start.filename, "\"; it is at frame ", + lane.decoder ? lane.decoder->NextFrameIdx() : -1, " of \"", + lane.decoder ? lane.decoder->Filename() : "", "\".")); + } + /** * @brief Number of decode lanes: num_threads, but no more than the batch size and, on the GPU, * kLanesPerEngine per NVDEC engine. @@ -659,7 +758,7 @@ class VideoReaderDecoder * @brief Groups the batch's samples into runs: samples of one file that one decoder decodes in * a row without seeking. * - * A sample repeated in the batch (pad_last_batch) is decoded once, and a sample decoded in an + * A sample repeated in the batch (pad_last_batch) is decoded once, and a sample planned in an * earlier batch (padding repeats the last sample, also in the next batch) is not decoded again. */ void PlanBatch() { @@ -668,20 +767,22 @@ class VideoReaderDecoder runs_.clear(); for (int i = 0; i < static_cast(batch.size()); i++) { auto *sample = batch[i].get(); - if (sample->decoded) + if (sample->planned) continue; - bool repeated = false; - for (auto &task : tasks_) - repeated = repeated || task.sample == sample; - if (repeated) - continue; - SampleTask task{sample, i, -1, -1, {}}; + sample->planned = true; + const auto &filename = sample->video_file_meta_->filename; + auto index = FrameIndexCache::instance().Find(filename); + DALI_ENFORCE(index, make_string("Internal error: no frame index for \"", filename, "\".")); + SampleTask task{sample, i, -1, -1, -1, {}}; try { SampleFrames(sample->frame_idxs_, sample->start_, sample->end_, sample->stride_, boundary_type_, sample->video_file_meta_->start_frame, sample->video_file_meta_->end_frame, task.frames); + task.end = EndFrame(task, index->size()); } catch (...) { - task.frames.clear(); // decoding the sample reports the error, in its lane + // decoding the sample reports the error, in its lane, and resets the decoder + task.frames.clear(); + task.end = -1; } if (!task.frames.empty()) { task.first = task.frames.front(); @@ -722,7 +823,8 @@ class VideoReaderDecoder } /** - * @brief Assigns the runs to lanes (see PlanLanes). A lane without runs releases its decoder. + * @brief Assigns the runs to lanes (see PlanLanes), from where the lanes' previous batches leave + * their decoders. A lane without runs releases its decoder. */ void AssignRuns() { std::vector indices; // keeps the indices alive while planning @@ -733,21 +835,27 @@ class VideoReaderDecoder for (int t : run.tasks) runs.back().samples.push_back(make_cspan(tasks_[t].frames)); } - std::vector filenames(lanes_.size()); // of the lanes' decoders std::vector starts(lanes_.size()); for (size_t l = 0; l < lanes_.size(); l++) { - if (lanes_[l].decoder) { - filenames[l] = lanes_[l].decoder->Filename(); - starts[l] = {filenames[l], DecoderPosition(*lanes_[l].decoder)}; + const auto &pos = lanes_[l].planned; + if (!pos.filename.empty()) { + indices.push_back(FrameIndexCache::instance().Find(pos.filename)); + starts[l] = {pos.filename, FramesDecoderBase::PositionAt(*indices.back(), pos.next_frame)}; } } PlanLanes(make_cspan(runs), make_cspan(starts), kOpenCost, plans_); for (size_t l = 0; l < lanes_.size(); l++) { - lanes_[l].runs = std::move(plans_[l].runs); - lanes_[l].load = plans_[l].load; - lanes_[l].planned_pictures = plans_[l].pictures; - if (lanes_[l].runs.empty() && lanes_.size() > 1) - lanes_[l].decoder.reset(); // do not keep its frame buffers and NVDEC decoder idle + auto &lane = lanes_[l]; + lane.runs = std::move(plans_[l].runs); + lane.load = plans_[l].load; + lane.planned_pictures = plans_[l].pictures; + lane.start = lane.planned; + if (lane.runs.empty() && lanes_.size() > 1) { + lane.decoder.reset(); // do not keep its frame buffers and NVDEC decoder idle + lane.planned = {}; + } else { + lane.planned = PredictEnd(make_cspan(lane.runs), std::move(lane.planned)); + } } } @@ -788,6 +896,12 @@ class VideoReaderDecoder void RunLane(DecodeLane &lane) { lane.errors.clear(); lane.decoded_pictures = 0; + try { + CheckPlan(lane); + } catch (...) { + lane.errors.emplace_back(tasks_[runs_[lane.runs[0]].tasks[0]].batch_idx, + std::current_exception()); + } for (int r : lane.runs) { for (int t : runs_[r].tasks) { auto &task = tasks_[t]; @@ -798,10 +912,10 @@ class VideoReaderDecoder // copies) are reported for this sample CUDA_CALL(cudaStreamSynchronize(lane.stream ? lane.stream.get() : 0)); } - task.sample->decoded = true; } catch (...) { lane.errors.emplace_back(task.batch_idx, std::current_exception()); lane.decoder.reset(); // in an unknown state; its destructor waits for its stream + lane.failed = true; } } } @@ -833,6 +947,7 @@ class VideoReaderDecoder << (lane.decoder ? lane.decoder->Filename() : "none") << std::endl; if (!lane.decoder || lane.decoder->Filename() != filename) { lane.decoder.reset(); // first: its frame buffers are freed before the new ones + lane.failed = false; if constexpr (std::is_same_v) { lane.decoder = std::make_unique(filename, image_type_); } else { @@ -943,6 +1058,7 @@ class VideoReaderDecoder DALIImageType image_type_; std::vector fill_value_; bool has_labels_ = false; + bool strict_plans_ = false; // see CheckPlan bool use_streams_ = false; // GPU: a stream per lane (false: the default stream, old drivers) std::vector tasks_; diff --git a/dali/test/python/decoder/test_video.py b/dali/test/python/decoder/test_video.py index 69ff5ce16e6..429f1d0ee55 100644 --- a/dali/test/python/decoder/test_video.py +++ b/dali/test/python/decoder/test_video.py @@ -17,6 +17,7 @@ import numpy as np import cv2 import av +import contextlib import nvidia.dali.types as types import glob import os @@ -1384,6 +1385,23 @@ def _lanes_reader_files(device): return cfr_files + vfr_files + (gpu_files if device == "gpu" else []) +@contextlib.contextmanager +def _strict_plans(): + """Readers created in it fail a batch if a decode lane's decoder is not where the batch's plan + expects it (otherwise, decoding only follows a worse plan).""" + name = "DALI_VIDEO_READER_STRICT_PLANS" + old = os.environ.get(name) + os.environ[name] = "1" + try: + yield + finally: + if old is None: + del os.environ[name] + else: + os.environ[name] = old + + +@_strict_plans() def _check_reader_output_independent_of_num_threads( device, kwargs, batch_size, num_threads, files=None, window=None ): From a3681f9fde7c11d6bd2facd3472dd8d8290ed92e Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Tue, 27 Oct 2026 03:55:49 +0100 Subject: [PATCH 37/46] Run video decode lanes on their own threads, a queue of batches each The video reader decoded a batch's lanes on a thread pool and waited for all of them (RunAll) before reading the next batch. A lane that finished early had nothing to do until the slowest lane finished too. Each lane now runs on its own thread, started with the first batch, and does its parts of the batches from its own queue, in the order the batches are read. A batch keeps its samples, its plan, the loader's snapshot and its errors until it is published, and is published once every lane is done with it. An idle lane releases its decoder on its thread, in its turn. With one lane, no thread is started: the batch is decoded on the prefetch thread, as before. The lanes' threads use the operator's device, as the thread pool's did. This commit still reads, decodes and publishes one batch at a time, so the output and the plans are as before. An error of a batch is still the first one by position in the batch; once a batch fails, or while the reader is destroyed, the lanes skip the samples they have not started and nothing is published any more. Tests: the output over two epochs with the reader's prefetch_queue_depth 1 and 2, pad_last_batch repeating the last sample in its own batch as well as in the next one, the first error of a batch reported with errors in several lanes and in the next batch, destroying the reader while it decodes or before it starts, and checkpoints with prefetch_queue_depth 1 and 2 and a shuffle buffer. Signed-off-by: Joaquin Anton Guirao --- .../video/reader/video_reader_decoder_op.cc | 383 +++++++++++++----- .../checkpointing/test_dali_checkpointing.py | 28 ++ dali/test/python/decoder/test_video.py | 130 +++++- 3 files changed, 420 insertions(+), 121 deletions(-) diff --git a/dali/operators/video/reader/video_reader_decoder_op.cc b/dali/operators/video/reader/video_reader_decoder_op.cc index 0c25841c9cd..2c057a52ca5 100644 --- a/dali/operators/video/reader/video_reader_decoder_op.cc +++ b/dali/operators/video/reader/video_reader_decoder_op.cc @@ -13,19 +13,27 @@ // limitations under the License. #include +#include +#include #include +#include #include #include #include +#include #include +#include #include #include +#include #include #include #include #include "dali/core/boundary.h" #include "dali/core/cuda_stream_pool.h" +#include "dali/core/device_guard.h" +#include "dali/core/nvtx.h" #include "dali/core/small_vector.h" #include "dali/core/span.h" @@ -36,7 +44,6 @@ #include "dali/operators/video/reader/decode_lane_planner.h" #include "dali/operators/video/reader/frame_index_cache.h" #include "dali/operators/video/video_utils.h" -#include "dali/pipeline/util/thread_pool.h" #if NVML_ENABLED #include "dali/util/nvml.h" #endif @@ -438,7 +445,10 @@ class VideoReaderDecoder ~VideoReaderDecoder() override { LOG_LINE << "VideoReaderDecoder destructor" << std::endl; - Base::StopPrefetchThread(); + cancel_ = true; // the lanes skip the samples they have not started + Base::StopPrefetchThread(); // a Prefetch in progress throws once its batch is done + StopLanes(); + pending_.clear(); // the samples go back to the loader, which is still alive LOG_LINE << "VideoReaderDecoder destructor done" << std::endl; } @@ -568,10 +578,24 @@ class VideoReaderDecoder } void Prefetch() override { - Base::Prefetch(); - PlanBatch(); - AssignRuns(); - DecodeBatch(); + DomainTimeRange tr("[DALI][VideoReaderDecoder] Prefetch #" + to_string(curr_batch_producer_), + DomainTimeRange::kRed); + auto &slot = prefetched_batch_queue_[curr_batch_producer_]; + slot.clear(); // first: a consumed batch's samples go back to the loader + while (static_cast(pending_.size()) <= decode_ahead_ && !read_failed_) + StartBatch(); + auto batch = std::move(pending_.front()); + pending_.pop_front(); + WaitForBatch(*batch); // after this, no lane uses the batch + if (cancel_) // the reader is being destroyed: nothing is published any more + throw std::runtime_error("The video reader is being destroyed."); + if (auto error = FirstError(*batch)) { + cancel_ = true; // the lanes skip the batches after it + std::rethrow_exception(error); + } + slot = std::move(batch->samples); + if (this->IsCheckpointingEnabled()) + this->SaveLoaderSnapshot(batch->snapshot); LOG_LINE << "Prefetch done" << std::endl; } @@ -602,41 +626,64 @@ class VideoReaderDecoder */ struct DecodeRun { const std::string *filename; - std::vector tasks; // indices into tasks_, in decoding order + std::vector tasks; // indices into the batch's tasks, in decoding order int first; int last; int batch_idx; // the smallest position of its samples in the batch }; /** - * @brief A part of a batch's decoding that runs on one thread. + * @brief An error of a sample, at its position in the batch (INT_MAX: of no sample), and the + * lane that decoded it. + */ + struct SampleError { + int batch_idx; + int lane; + std::exception_ptr error; + }; + + /** + * @brief A batch, from when it is read until it is published: its samples and their decoding. + */ + struct PendingBatch { + typename Base::BatchQueueElement samples; + LoaderStateSnapshot snapshot; // the loader's state after reading the batch (checkpointing) + std::vector tasks; + std::vector runs; + std::vector plans; // the runs of each lane, in decoding order + std::vector lane_start; // where the plan expects each lane's decoder + // Reading, planning or queuing the batch, or recording a lane's errors, failed (once the batch + // is queued, guarded by lanes_mutex_) + std::exception_ptr error; + // Guarded by lanes_mutex_: the lanes that have yet to do their part, and the errors + int lanes_left = 0; + std::vector errors; + }; + + /** + * @brief A decoder with its own thread (with more than one lane) and stream. * * A lane keeps its decoder while consecutive samples come from the same file, so that they * are decoded without seeking, and decodes on its own stream, so that lanes do not wait for * each other's work. Which samples a lane decodes is decided before decoding starts, from the * samples alone, so a decoder's history (and with it, the output) never depends on thread - * timing. + * timing. A lane does its parts of the batches in the order they are read. */ struct DecodeLane { CUDAStreamLease stream; // GPU only; empty: the default stream std::unique_ptr decoder; // after `stream`: destroyed (and synced) first Tensor constant_frame; std::vector frame_idxs; // scratch - // Planning: the runs of the current batch, in the order the lane decodes them, and the - // estimated work - std::vector runs; - int64_t load = 0; - // Planning: where the decoder is when the lane starts the current batch, and where it is - // once the lane is done with it. The plans are made from these, not from the decoder. - LanePosition start, planned; + // Planning: where the decoder is once the lane is done with the batches planned so far. The + // plans are made from it, not from the decoder. + LanePosition planned; // Whether decoding a sample failed (and reset the decoder) since the lane last opened one: // the decoder is then not where the plan expects it bool failed = false; - // Diagnostics: the pictures the plan expects the lane to decode, and those it decodes - int64_t planned_pictures = 0; + // Diagnostics: the pictures the lane decodes in the current batch int64_t decoded_pictures = 0; - // Errors of the current batch: (position in the batch, exception) - std::vector> errors; + std::deque queue; // guarded by lanes_mutex_: the batches to do, in order + std::thread thread; // with more than one lane }; // Planning costs, in pictures decoded: opening a decoder (or resetting it, which reopens the @@ -677,12 +724,13 @@ class VideoReaderDecoder } /** - * @brief Where a lane's decoder is once it decodes `runs`, in order, from `pos`. + * @brief Where a lane's decoder is once it decodes `runs` of `batch`, in order, from `pos`. */ - LanePosition PredictEnd(span runs, LanePosition pos) const { + static LanePosition PredictEnd(const PendingBatch &batch, span runs, + LanePosition pos) { for (int r : runs) { - for (int t : runs_[r].tasks) { - const auto &task = tasks_[t]; + for (int t : batch.runs[r].tasks) { + const auto &task = batch.tasks[t]; const auto &filename = task.sample->video_file_meta_->filename; if (pos.filename != filename) pos = {filename, 0}; // a newly opened decoder @@ -704,23 +752,23 @@ class VideoReaderDecoder } /** - * @brief Checks that a lane's decoder is where the plan of its batch expects it. + * @brief Checks that a lane's decoder is at `start`, where the plan of its batch expects it. * * The plans are made from where decoding the samples leaves the decoders, which is where they * are as long as they decode every frame they are asked for. If one is somewhere else, the lane * still decodes the right frames (decoding seeks from where the decoder is), only the plan may * be worse. With DALI_VIDEO_READER_STRICT_PLANS set (for tests), it is an error. */ - void CheckPlan(DecodeLane &lane) { - if (lane.failed || IsAt(lane, lane.start)) + void CheckPlan(DecodeLane &lane, const LanePosition &start) { + if (lane.failed || IsAt(lane, start)) return; LOG_LINE << "The decoder of a lane is at " << (lane.decoder ? lane.decoder->Filename() : "no file") << ", frame " << (lane.decoder ? lane.decoder->NextFrameIdx() : -1) << "; planned: " - << lane.start.filename << ", frame " << lane.start.next_frame << std::endl; + << start.filename << ", frame " << start.next_frame << std::endl; DALI_ENFORCE(!strict_plans_, make_string( "Internal error: a video decoder is not where the plan expects it, at frame ", - lane.start.next_frame, " of \"", lane.start.filename, "\"; it is at frame ", + start.next_frame, " of \"", start.filename, "\"; it is at frame ", lane.decoder ? lane.decoder->NextFrameIdx() : -1, " of \"", lane.decoder ? lane.decoder->Filename() : "", "\".")); } @@ -741,17 +789,60 @@ class VideoReaderDecoder } void InitLanes(const OpSpec &spec) { - int num_lanes = NumLanes(spec); - lanes_.resize(num_lanes); - int device_id = spec.GetArgument("device_id"); + lanes_.resize(NumLanes(spec)); + lane_device_id_ = spec.GetArgument("device_id"); for (auto &lane : lanes_) { lane.constant_frame.set_pinned(std::is_same_v); if (use_streams_) - lane.stream = CUDAStreamPool::instance().Get(device_id); + lane.stream = CUDAStreamPool::instance().Get(lane_device_id_); } - if (num_lanes > 1) - thread_pool_ = std::make_unique(num_lanes, device_id, false, - "VideoReaderDecoder"); + } + + /** + * @brief Starts the lanes' threads, with more than one lane, unless they run already. + */ + void StartLanes() { + if (lanes_.size() == 1 || lanes_started_) + return; + for (int l = 0; l < static_cast(lanes_.size()); l++) + lanes_[l].thread = std::thread(&VideoReaderDecoder::LaneMain, this, l); + lanes_started_ = true; + } + + void StopLanes() { + { + std::lock_guard lock(lanes_mutex_); + stop_lanes_ = true; + } + lanes_cv_.notify_all(); + for (auto &lane : lanes_) { + if (lane.thread.joinable()) + lane.thread.join(); + } + } + + /** + * @brief Reads the next batch, plans its decoding and queues it to the lanes. + * + * If reading or planning it fails, the batch fails when it is due, and no batch is read after + * it. + */ + void StartBatch() { + pending_.push_back(std::make_unique()); + auto &batch = *pending_.back(); + try { + this->ReadBatch(batch.samples); + if (this->IsCheckpointingEnabled()) + batch.snapshot = loader_->GetStateSnapshot(); + StartLanes(); + PlanBatch(batch); + AssignRuns(batch); + } catch (...) { + batch.error = std::current_exception(); + read_failed_ = true; + return; + } + QueueBatch(batch); } /** @@ -761,12 +852,11 @@ class VideoReaderDecoder * A sample repeated in the batch (pad_last_batch) is decoded once, and a sample planned in an * earlier batch (padding repeats the last sample, also in the next batch) is not decoded again. */ - void PlanBatch() { - auto &batch = prefetched_batch_queue_[curr_batch_producer_]; - tasks_.clear(); - runs_.clear(); - for (int i = 0; i < static_cast(batch.size()); i++) { - auto *sample = batch[i].get(); + void PlanBatch(PendingBatch &batch) { + auto &tasks = batch.tasks; + auto &runs = batch.runs; + for (int i = 0; i < static_cast(batch.samples.size()); i++) { + auto *sample = batch.samples[i].get(); if (sample->planned) continue; sample->planned = true; @@ -788,13 +878,13 @@ class VideoReaderDecoder task.first = task.frames.front(); task.last = task.frames.back(); } - tasks_.push_back(std::move(task)); + tasks.push_back(std::move(task)); } - std::vector order(tasks_.size()); + std::vector order(tasks.size()); std::iota(order.begin(), order.end(), 0); std::sort(order.begin(), order.end(), [&](int a, int b) { - const auto &ta = tasks_[a], &tb = tasks_[b]; + const auto &ta = tasks[a], &tb = tasks[b]; const auto &fa = ta.sample->video_file_meta_->filename; const auto &fb = tb.sample->video_file_meta_->filename; return std::tie(fa, ta.first, ta.batch_idx) < std::tie(fb, tb.first, tb.batch_idx); @@ -802,38 +892,38 @@ class VideoReaderDecoder FrameIndexCache::IndexPtr index; size_t file_runs = 0; // the first run of the current file for (int t : order) { - auto &task = tasks_[t]; + auto &task = tasks[t]; const auto &filename = task.sample->video_file_meta_->filename; - if (runs_.empty() || *runs_.back().filename != filename) { - file_runs = runs_.size(); + if (runs.empty() || *runs.back().filename != filename) { + file_runs = runs.size(); index = FrameIndexCache::instance().Find(filename); } // join a run of the file that a decoder continues to this sample without seeking - auto run = std::find_if(runs_.begin() + file_runs, runs_.end(), [&](const DecodeRun &r) { + auto run = std::find_if(runs.begin() + file_runs, runs.end(), [&](const DecodeRun &r) { return task.first < 0 || CanReachWithoutSeek(index.get(), r.last + 1, task.first); }); - if (run != runs_.end()) { + if (run != runs.end()) { run->tasks.push_back(t); run->last = std::max(run->last, task.last); run->batch_idx = std::min(run->batch_idx, task.batch_idx); } else { - runs_.push_back({&filename, {t}, task.first, task.last, task.batch_idx}); + runs.push_back({&filename, {t}, task.first, task.last, task.batch_idx}); } } } /** - * @brief Assigns the runs to lanes (see PlanLanes), from where the lanes' previous batches leave + * @brief Assigns the runs to lanes (see PlanLanes), from where the lanes' earlier batches leave * their decoders. A lane without runs releases its decoder. */ - void AssignRuns() { + void AssignRuns(PendingBatch &batch) { std::vector indices; // keeps the indices alive while planning std::vector runs; - for (const auto &run : runs_) { + for (const auto &run : batch.runs) { indices.push_back(FrameIndexCache::instance().Find(*run.filename)); runs.push_back({*run.filename, indices.back().get(), run.batch_idx, run.first, run.last, {}}); for (int t : run.tasks) - runs.back().samples.push_back(make_cspan(tasks_[t].frames)); + runs.back().samples.push_back(make_cspan(batch.tasks[t].frames)); } std::vector starts(lanes_.size()); for (size_t l = 0; l < lanes_.size(); l++) { @@ -843,68 +933,136 @@ class VideoReaderDecoder starts[l] = {pos.filename, FramesDecoderBase::PositionAt(*indices.back(), pos.next_frame)}; } } - PlanLanes(make_cspan(runs), make_cspan(starts), kOpenCost, plans_); + PlanLanes(make_cspan(runs), make_cspan(starts), kOpenCost, batch.plans); + batch.lane_start.resize(lanes_.size()); for (size_t l = 0; l < lanes_.size(); l++) { auto &lane = lanes_[l]; - lane.runs = std::move(plans_[l].runs); - lane.load = plans_[l].load; - lane.planned_pictures = plans_[l].pictures; - lane.start = lane.planned; - if (lane.runs.empty() && lanes_.size() > 1) { - lane.decoder.reset(); // do not keep its frame buffers and NVDEC decoder idle - lane.planned = {}; - } else { - lane.planned = PredictEnd(make_cspan(lane.runs), std::move(lane.planned)); + const auto &lane_runs = batch.plans[l].runs; + batch.lane_start[l] = lane.planned; + if (lane_runs.empty() && lanes_.size() > 1) + lane.planned = {}; // the lane releases its decoder (see DecodeLanePart) + else + lane.planned = PredictEnd(batch, make_cspan(lane_runs), std::move(lane.planned)); + } + } + + /** + * @brief Has the lanes decode the batch: on their threads, after the batches queued before it, + * or here, with one lane. + */ + void QueueBatch(PendingBatch &batch) { + if (lanes_.size() == 1) { + batch.lanes_left = 1; + RunLanePart(0, batch); + return; + } + { + std::lock_guard lock(lanes_mutex_); + for (auto &lane : lanes_) { + try { + lane.queue.push_back(&batch); + } catch (...) { + batch.error = std::current_exception(); + break; + } + batch.lanes_left++; // only the lanes that do their part } } + lanes_cv_.notify_all(); } /** - * @brief Decodes the batch, every lane on its own thread, and waits for all of them. - * - * An error is reported for the first failing sample in the batch, whatever the lanes' timing. + * @brief Waits until no lane uses the batch any more. */ - void DecodeBatch() { - std::vector active; - for (auto &lane : lanes_) { - if (!lane.runs.empty()) - active.push_back(&lane); + void WaitForBatch(PendingBatch &batch) { + std::unique_lock lock(lanes_mutex_); + batch_done_cv_.wait(lock, [&] { return batch.lanes_left == 0; }); + } + + /** + * @brief The error a batch fails with: of reading or planning it, or of its first failing sample + * (whatever the lanes' timing). + */ + static std::exception_ptr FirstError(const PendingBatch &batch) { + if (batch.error) + return batch.error; + const SampleError *first = nullptr; + for (const auto &error : batch.errors) { + if (!first || std::tie(error.batch_idx, error.lane) < std::tie(first->batch_idx, first->lane)) + first = &error; } - if (active.size() == 1 || !thread_pool_) { - for (auto *lane : active) - RunLane(*lane); - } else { - for (auto *lane : active) - thread_pool_->AddWork([this, lane](int) { RunLane(*lane); }, lane->load); - thread_pool_->RunAll(); + return first ? first->error : nullptr; + } + + void LaneMain(int l) { + SetThreadName(make_string("[DALI][VR lane ", l, "]").c_str()); + DeviceGuard g(lane_device_id_); + auto &lane = lanes_[l]; + std::unique_lock lock(lanes_mutex_); + for (;;) { + lanes_cv_.wait(lock, [&] { return stop_lanes_ || !lane.queue.empty(); }); + if (lane.queue.empty()) + return; // stopped + PendingBatch *batch = lane.queue.front(); + lane.queue.pop_front(); + lock.unlock(); + RunLanePart(l, *batch); + lock.lock(); } - const std::pair *first_error = nullptr; - for (auto *lane : active) { - for (auto &error : lane->errors) { - if (!first_error || error.first < first_error->first) - first_error = &error; - } + } + + /** + * @brief Does a lane's part of a batch, and records that it is done. Never throws: on a lane's + * thread, that would end the process, and the batch would never be done. + */ + void RunLanePart(int l, PendingBatch &batch) { + std::vector errors; + std::exception_ptr failure; // of the lane's part, not of a sample + try { + DecodeLanePart(l, batch, errors); + } catch (...) { + failure = std::current_exception(); } - if (first_error) { - auto error = first_error->second; - for (auto *lane : active) - lane->errors.clear(); - std::rethrow_exception(error); + std::lock_guard lock(lanes_mutex_); + try { + if (failure) + errors.push_back({std::numeric_limits::max(), l, failure}); + batch.errors.insert(batch.errors.end(), errors.begin(), errors.end()); + } catch (...) { + // out of memory: the batch fails all the same + if (!batch.error) + batch.error = failure ? failure : std::current_exception(); } + if (--batch.lanes_left == 0) + batch_done_cv_.notify_all(); } - void RunLane(DecodeLane &lane) { - lane.errors.clear(); + /** + * @brief Decodes a lane's runs of a batch. A sample that fails is recorded in `errors`. + */ + void DecodeLanePart(int l, PendingBatch &batch, std::vector &errors) { + auto &lane = lanes_[l]; + const auto &plan = batch.plans[l]; + if (plan.runs.empty()) { + if (lanes_.size() > 1) + lane.decoder.reset(); // do not keep its frame buffers and NVDEC decoder idle + return; + } + DomainTimeRange tr(make_string("[DALI][VideoReaderDecoder] lane ", l), + DomainTimeRange::kOrange); lane.decoded_pictures = 0; try { - CheckPlan(lane); + CheckPlan(lane, batch.lane_start[l]); } catch (...) { - lane.errors.emplace_back(tasks_[runs_[lane.runs[0]].tasks[0]].batch_idx, - std::current_exception()); + int first_task = batch.runs[plan.runs[0]].tasks[0]; + errors.push_back({batch.tasks[first_task].batch_idx, l, std::current_exception()}); } - for (int r : lane.runs) { - for (int t : runs_[r].tasks) { - auto &task = tasks_[t]; + bool cancelled = false; + for (size_t i = 0; i < plan.runs.size() && !cancelled; i++) { + for (int t : batch.runs[plan.runs[i]].tasks) { + if ((cancelled = cancel_)) + break; + auto &task = batch.tasks[t]; try { DecodeSample(*task.sample, lane); if constexpr (std::is_same_v) { @@ -913,28 +1071,28 @@ class VideoReaderDecoder CUDA_CALL(cudaStreamSynchronize(lane.stream ? lane.stream.get() : 0)); } } catch (...) { - lane.errors.emplace_back(task.batch_idx, std::current_exception()); lane.decoder.reset(); // in an unknown state; its destructor waits for its stream lane.failed = true; + errors.push_back({task.batch_idx, l, std::current_exception()}); } } } if constexpr (std::is_same_v) { // the samples are read on the workspace stream once the batch is consumed cudaStream_t stream = lane.stream ? lane.stream.get() : 0; - if (lane.errors.empty()) { + if (errors.empty()) { try { CUDA_CALL(cudaStreamSynchronize(stream)); } catch (...) { - lane.errors.emplace_back(std::numeric_limits::max(), std::current_exception()); + errors.push_back({std::numeric_limits::max(), l, std::current_exception()}); } } else if (cudaStreamSynchronize(stream) != cudaSuccess) { (void)cudaGetLastError(); // already failing: report the sample's error } } // Planned with the files' indices, the two match (on the GPU, within a picture or two) - LOG_LINE << "Lane decoded " << lane.decoded_pictures << " pictures, planned " - << lane.planned_pictures << " (load " << lane.load << ")" << std::endl; + LOG_LINE << "Lane " << l << " decoded " << lane.decoded_pictures << " pictures, planned " + << plan.pictures << " (load " << plan.load << ")" << std::endl; } /** @@ -1061,11 +1219,20 @@ class VideoReaderDecoder bool strict_plans_ = false; // see CheckPlan bool use_streams_ = false; // GPU: a stream per lane (false: the default stream, old drivers) - std::vector tasks_; - std::vector runs_; std::vector lanes_; - std::vector plans_; // scratch - std::unique_ptr thread_pool_; // with more than one lane + int lane_device_id_ = CPU_ONLY_DEVICE_ID; // the device of the lanes' threads + bool lanes_started_ = false; + std::mutex lanes_mutex_; + std::condition_variable lanes_cv_; // a lane's queue or stop_lanes_ changed + std::condition_variable batch_done_cv_; // a batch's lanes_left reached 0 + bool stop_lanes_ = false; // guarded by lanes_mutex_ + // The batches read, and queued to the lanes, that are not published yet, in reading order + std::deque> pending_; + int decode_ahead_ = 0; // the batches read and decoded ahead of the one being published + bool read_failed_ = false; // reading or planning a batch failed: no batch is read after it + // Set by the destructor, or by Prefetch when a batch fails: the lanes skip the samples they have + // not started, and Prefetch publishes nothing more + std::atomic cancel_{false}; }; DALI_SCHEMA(experimental__readers__Video) diff --git a/dali/test/python/checkpointing/test_dali_checkpointing.py b/dali/test/python/checkpointing/test_dali_checkpointing.py index 9ff20914ea9..f1cc695f734 100644 --- a/dali/test/python/checkpointing/test_dali_checkpointing.py +++ b/dali/test/python/checkpointing/test_dali_checkpointing.py @@ -819,6 +819,34 @@ def test_video_reader_resize_reader( ) +# With more than one decode lane, the reader decodes a batch ahead of the one it publishes: the +# checkpoints must still be those of the batches returned, also with batches in the reader's own +# queue and with a shuffle buffer (both hold samples of the loader) +@cartesian_params(("cpu", "gpu"), (1, 2), (False, True)) +def test_experimental_video_reader_prefetch_queue_depth(device, reader_depth, random_shuffle): + files = [ + os.path.join(get_dali_extra_path(), "db", "video", "vfr", f"test_{i}_vp9.mp4") + for i in (1, 2) + ] + + check_reader_checkpointing( + fn.experimental.readers.video, + 2, + 4, + 2, + device=device, + filenames=files, + labels=list(range(len(files))), + random_shuffle=random_shuffle, + initial_fill=8, + pad_last_batch=True, + prefetch_queue_depth=reader_depth, + sequence_length=3, + stride=1, + step=5, + ) + + @cartesian_params( ( "cpu", diff --git a/dali/test/python/decoder/test_video.py b/dali/test/python/decoder/test_video.py index 429f1d0ee55..10bff653d41 100644 --- a/dali/test/python/decoder/test_video.py +++ b/dali/test/python/decoder/test_video.py @@ -1403,8 +1403,9 @@ def _strict_plans(): @_strict_plans() def _check_reader_output_independent_of_num_threads( - device, kwargs, batch_size, num_threads, files=None, window=None + device, kwargs, batch_size, num_threads, files=None, window=None, epochs=None ): + """With `epochs`, compares that many epochs (and one more batch), otherwise 8 batches.""" files = files or _lanes_reader_files(device) @pipeline_def(batch_size=batch_size, device_id=0, seed=42) @@ -1417,12 +1418,18 @@ def pipe(): labels=list(range(len(files))), device=device, enable_frame_num="sequence", + name="Reader", **kwargs, ) ) + ref = pipe(num_threads=1) + iterations = 8 + if epochs: + ref.build() + iterations = -(-epochs * ref.epoch_size("Reader") // batch_size) + 1 compare_pipelines( - pipe(num_threads=1), pipe(num_threads=num_threads), batch_size=batch_size, N_iterations=8 + ref, pipe(num_threads=num_threads), batch_size=batch_size, N_iterations=iterations ) # The two runs could agree on wrong frames, so check the frames against PyAV as well @@ -1486,15 +1493,34 @@ def test_reader_multi_run_lanes_output_independent_of_num_threads(device, case): ) -@params("cpu", "gpu") -def test_reader_pad_last_batch_repeats_last_sample(device): - """With pad_last_batch, a shard's last sample is repeated, here as the whole next batch: the - same sample, which must not be decoded again while the earlier batch may still be in use.""" +# With more than one lane, the lanes decode the batches from their queues on their own threads, +# while the reader's queue (prefetch_queue_depth) holds earlier batches. Two epochs, so that +# decoders carry over from batch to batch and into the next epoch +@cartesian_params(["cpu", "gpu"], (0, 3, 6), (1, 2)) +def test_reader_output_independent_of_num_threads_reader_depth(device, case, reader_depth): + skip_if_m60() + _check_reader_output_independent_of_num_threads( + device, + dict(_lanes_reader_cases[case], prefetch_queue_depth=reader_depth), + batch_size=4, + num_threads=4, + files=cfr_files + vfr_files, + epochs=2, + ) + + +# 26 samples in 3 shards (pad_last_batch): shard 0 has samples 0-7, padded to 9 samples, so that +# its third batch is sample 7, four times (decoded in the batch before it); shard 2 has samples +# 17-25, so that its third batch is sample 25 and three repeats of it +@cartesian_params(("cpu", "gpu"), (0, 2), (1, 2)) +@_strict_plans() +def test_reader_pad_last_batch_repeats_last_sample(device, shard_id, reader_depth): + """With pad_last_batch, a shard's last sample is repeated, in its own batch or as the whole + next batch: the same sample, which must not be decoded again while an earlier batch may still + be in use.""" skip_if_m60() batch_size = 4 files = _lanes_reader_files(device) - # 26 samples in 3 shards: shard 0 has samples 0-7 and is padded to 9 samples, so its third - # batch is sample 7, four times with tempfile.NamedTemporaryFile("w", suffix=".txt", delete=False) as file_list: for i in range(26): start = (i // len(files)) * 4 @@ -1510,7 +1536,8 @@ def pipe(): sequence_length=4, pad_last_batch=True, num_shards=3, - shard_id=0, + shard_id=shard_id, + prefetch_queue_depth=reader_depth, ) return videos, labels @@ -1522,9 +1549,86 @@ def pipe(): labels = labels.as_cpu() if device == "gpu" else labels samples += [(np.array(videos[i]), int(np.array(labels[i])[0])) for i in range(batch_size)] os.remove(file_list.name) - assert [label for _, label in samples] == list(range(8)) + [7] * 4 - for video, _ in samples[8:]: - np.testing.assert_array_equal(video, samples[7][0]) - for video, label in samples[:8]: + first, end = 26 * shard_id // 3, 26 * (shard_id + 1) // 3 + num_read = end - first + assert [label for _, label in samples] == list(range(first, end)) + [end - 1] * (12 - num_read) + for video, _ in samples[num_read:]: + np.testing.assert_array_equal(video, samples[num_read - 1][0]) + for video, label in samples[:num_read]: start = (label // len(files)) * 4 _assert_frames_match_file(video, range(start, start + 4), files[label % len(files)]) + + +def _num_frames(filename, device): + """The frames of a file, as the reader counts them.""" + + @pipeline_def(batch_size=1, num_threads=1, device_id=0) + def pipe(): + return fn.experimental.readers.video( + filenames=[filename], sequence_length=1, device=device, name="Reader" + ) + + p = pipe() + p.build() + return p.reader_meta("Reader")["epoch_size"] + + +@cartesian_params(("cpu", "gpu"), (1, 4), (1, 2)) +@_strict_plans() +def test_reader_reports_first_error_of_batch(device, num_threads, reader_depth): + """Samples fail in two lanes of a batch and in the next batch: the error is the first one of + the batch, at that batch, whatever the lanes' timing.""" + skip_if_m60() + batch_size = 4 + files = cfr_files # with different numbers of frames + num_frames = [_num_frames(f, device) for f in files] + assert num_frames[0] != num_frames[1] + # (file, start) of each sample, with sequences that start past the end of the file + samples = [(0, 0), (1, 0), (0, 4), (1, 4)] # batch 0 + samples += [(0, 8), (0, num_frames[0] + 10), (1, 8), (1, num_frames[1] + 20)] # batch 1 + samples += [(0, num_frames[0] + 30), (1, 12), (0, 12), (1, 16)] # batch 2 + with tempfile.NamedTemporaryFile("w", suffix=".txt", delete=False) as file_list: + for i, (f, start) in enumerate(samples): + file_list.write(f"{files[f]} {i} {start} {start + 4}\n") + + @pipeline_def(batch_size=batch_size, num_threads=num_threads, device_id=0) + def pipe(): + videos, _ = fn.experimental.readers.video( + file_list=file_list.name, + file_list_format="frames", + file_list_include_end=False, + device=device, + sequence_length=4, + prefetch_queue_depth=reader_depth, + ) + return videos + + try: + p = pipe() + p.run() # batch 0 + expected = rf"frame index {num_frames[0] + 10} .*\[0, {num_frames[0]}\)" + with assert_raises(RuntimeError, regex=expected): + p.run() + del p # destroyed right after the error + finally: + os.remove(file_list.name) + + +@params("cpu", "gpu") +@_strict_plans() +def test_reader_destroyed_while_decoding(device): + """The reader is destroyed while its lanes decode, or before it starts them.""" + skip_if_m60() + + @pipeline_def(batch_size=4, num_threads=4, device_id=0) + def pipe(lazy_init): + return fn.experimental.readers.video( + filenames=cfr_files + vfr_files, device=device, sequence_length=5, lazy_init=lazy_init + ) + + for i in range(30): + p = pipe(lazy_init=i % 2 == 1) + p.build() + for _ in range(i % 3): # none: destroyed before the lanes are started + p.run() + del p From 17afbcf383010e21c5419d9f7b0734de008c621c Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Tue, 27 Oct 2026 04:19:12 +0100 Subject: [PATCH 38/46] Decode the next video batch while the slowest lane finishes With more than one decode lane, every batch ended with all lanes waiting for the slowest one: a lane that was done with its part had nothing to do until the batch was published and the next one read and planned. The reader now reads, plans and queues one batch ahead of the one it publishes. A lane that is done with its part of a batch goes on to its part of the next one, while the others finish theirs. Batches are still published in reading order, each once all lanes are done with it, with the loader's snapshot taken when it was read, and its first error by position in the batch; once a batch fails, the lanes skip the batches after it. The samples read and the plans are the same as before, so is the output. The lookahead is bounded by the reader's queue: the batch after the next one is read only when the queue has room. A consumed batch's samples go back to the loader before the next batch is read, so the loader's pool (2 * prefetch_queue_depth batches) suffices. With more than one lane, one more batch of decoded samples is held in memory; the documentation says so. With one lane nothing changes. The tests added with the lanes' threads now also cover the lookahead: the output with the next batch decoded ahead, an error in the batch after the failing one that the lanes may already be decoding, and destroying the reader while lanes decode ahead. Signed-off-by: Joaquin Anton Guirao --- .../video/reader/video_reader_decoder_op.cc | 11 ++++++++++- dali/test/python/decoder/test_video.py | 14 +++++++------- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/dali/operators/video/reader/video_reader_decoder_op.cc b/dali/operators/video/reader/video_reader_decoder_op.cc index 2c057a52ca5..7f321848b38 100644 --- a/dali/operators/video/reader/video_reader_decoder_op.cc +++ b/dali/operators/video/reader/video_reader_decoder_op.cc @@ -694,6 +694,12 @@ class VideoReaderDecoder // lane does its host work between pictures; many more only add memory. static constexpr int kLanesPerEngine = 2; + // With more than one lane: the batches read and decoded ahead of the one being published, so that + // a lane that is done with its part of a batch goes on to the next one, instead of waiting for + // the slowest lane. Each holds one more batch of samples (of the loader's 2 * + // prefetch_queue_depth batches). + static constexpr int kDecodeAhead = 1; + // SampleTask::end of a sample that decodes no frame: its decoder stays where it was static constexpr int kNoFrames = -2; @@ -790,6 +796,7 @@ class VideoReaderDecoder void InitLanes(const OpSpec &spec) { lanes_.resize(NumLanes(spec)); + decode_ahead_ = lanes_.size() > 1 ? kDecodeAhead : 0; lane_device_id_ = spec.GetArgument("device_id"); for (auto &lane : lanes_) { lane.constant_frame.set_pinned(std::is_same_v); @@ -1257,7 +1264,9 @@ The following codecs are supported by the GPU backend only: The samples of a batch are decoded in parallel, by up to ``num_threads`` decoders (on the GPU, at most two per NVDEC engine of the device). Consecutive samples of one file that can be decoded without seeking are decoded by the same decoder, one after another. Every decoder has its own -frame buffers, so the memory used grows with the number of decoders. +frame buffers, so the memory used grows with the number of decoders. With more than one decoder, +the decoders that are done with their part of a batch start on the next batch, so up to one more +batch of decoded samples is held in memory (host memory for the CPU backend). The outputs of the operator are: video, [labels], [frame_num], [timestamps]. diff --git a/dali/test/python/decoder/test_video.py b/dali/test/python/decoder/test_video.py index 10bff653d41..f0056d25ec0 100644 --- a/dali/test/python/decoder/test_video.py +++ b/dali/test/python/decoder/test_video.py @@ -1493,9 +1493,9 @@ def test_reader_multi_run_lanes_output_independent_of_num_threads(device, case): ) -# With more than one lane, the lanes decode the batches from their queues on their own threads, -# while the reader's queue (prefetch_queue_depth) holds earlier batches. Two epochs, so that -# decoders carry over from batch to batch and into the next epoch +# With more than one lane, the lanes decode the next batch while the slowest lane finishes the +# current one, and while the reader's queue (prefetch_queue_depth) holds earlier batches. Two +# epochs, so that decoders carry over from batch to batch and into the next epoch @cartesian_params(["cpu", "gpu"], (0, 3, 6), (1, 2)) def test_reader_output_independent_of_num_threads_reader_depth(device, case, reader_depth): skip_if_m60() @@ -1576,8 +1576,8 @@ def pipe(): @cartesian_params(("cpu", "gpu"), (1, 4), (1, 2)) @_strict_plans() def test_reader_reports_first_error_of_batch(device, num_threads, reader_depth): - """Samples fail in two lanes of a batch and in the next batch: the error is the first one of - the batch, at that batch, whatever the lanes' timing.""" + """Samples fail in two lanes of a batch and in the next batch, which lanes may be decoding + already: the error is the first one of the batch, at that batch, whatever the lanes' timing.""" skip_if_m60() batch_size = 4 files = cfr_files # with different numbers of frames @@ -1609,7 +1609,7 @@ def pipe(): expected = rf"frame index {num_frames[0] + 10} .*\[0, {num_frames[0]}\)" with assert_raises(RuntimeError, regex=expected): p.run() - del p # destroyed right after the error + del p # destroyed right after the error, while lanes may still be decoding batch 2 finally: os.remove(file_list.name) @@ -1617,7 +1617,7 @@ def pipe(): @params("cpu", "gpu") @_strict_plans() def test_reader_destroyed_while_decoding(device): - """The reader is destroyed while its lanes decode, or before it starts them.""" + """The reader is destroyed with its lanes decoding batches ahead, or before it starts them.""" skip_if_m60() @pipeline_def(batch_size=4, num_threads=4, device_id=0) From bff1e8d714171545b57b4d77f48a4bf5bf2edefb Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Tue, 27 Oct 2026 04:42:54 +0100 Subject: [PATCH 39/46] Give tied video decode runs to the lane with the least backlog The planner gives each run of a batch to the lane that would finish it first, and a tie to the lowest lane. Lanes without a decoder for the run's file often tie, e.g. at the start of every batch of distinct files, so lane 0 kept getting the longest runs. While the lanes had to wait for each other at the end of every batch, that did not matter. Now that a lane goes on to the next batch as soon as it is done with its part, the work a lane has more than another carries over. The reader keeps each lane's planned work beyond the least loaded lane's (its backlog), and the planner gives a tie to the lane with the least backlog. The backlog comes from the plans only, so the plans still depend only on the samples, not on timing; it starts at 0, also after a checkpoint is restored. This only picks among lanes the planner rates as finishing a run at the same time, but it changes which lane decodes which run, and so where the decoders are for later batches. Signed-off-by: Joaquin Anton Guirao --- .../video/reader/decode_lane_planner.cc | 5 +++-- .../video/reader/decode_lane_planner.h | 6 ++++++ .../video/reader/decode_lane_planner_test.cc | 21 +++++++++++++++++++ .../video/reader/video_reader_decoder_op.cc | 13 ++++++++++++ 4 files changed, 43 insertions(+), 2 deletions(-) diff --git a/dali/operators/video/reader/decode_lane_planner.cc b/dali/operators/video/reader/decode_lane_planner.cc index cb325fb53e7..dd787e63595 100644 --- a/dali/operators/video/reader/decode_lane_planner.cc +++ b/dali/operators/video/reader/decode_lane_planner.cc @@ -186,9 +186,10 @@ void PlanLanes(span runs, span lanes, int64_t work = FileWork(runs, lanes[l], make_cspan(candidate)); delta = Cost(work, open_cost) - (group ? Cost(group->work, open_cost) : 0); } - if (plans[l].load + delta < best_end) { + int64_t end = plans[l].load + delta; + if (end < best_end || (end == best_end && lanes[l].backlog < lanes[best].backlog)) { best = l; - best_end = plans[l].load + delta; + best_end = end; best_work = work; best_runs = candidate; } diff --git a/dali/operators/video/reader/decode_lane_planner.h b/dali/operators/video/reader/decode_lane_planner.h index bf965323cec..f409aa5977a 100644 --- a/dali/operators/video/reader/decode_lane_planner.h +++ b/dali/operators/video/reader/decode_lane_planner.h @@ -48,6 +48,9 @@ struct LaneStart { // The frame it returns next (as FramesDecoderBase::NextFrameIdx) and, with an index, the last // picture it decoded (see FramesDecoderBase::PositionAt) FramesDecoderBase::DecodePosition pos = {-1, -1}; + // The work of earlier batches that the lane may still be doing when the batch starts, beyond + // that of the lane with the least + int64_t backlog = 0; }; /** @@ -82,6 +85,9 @@ DLL_PUBLIC bool CanReachWithoutSeek(const FrameIndex *index, int next, int frame /** * @brief Assigns the runs to lanes: longest first, each to the lane that would finish it first. * + * Of the lanes that would finish a run at the same time, the one with the least backlog gets it + * (then the lowest one). + * * A lane decodes the runs of one file together, by first frame: first the file its decoder holds * (starting with the run that continues where the decoder is), then the others in batch order, * each from a newly opened decoder. diff --git a/dali/operators/video/reader/decode_lane_planner_test.cc b/dali/operators/video/reader/decode_lane_planner_test.cc index b4037c9d101..4d8cf2f1705 100644 --- a/dali/operators/video/reader/decode_lane_planner_test.cc +++ b/dali/operators/video/reader/decode_lane_planner_test.cc @@ -266,6 +266,27 @@ TEST(DecodeLanePlannerTest, LongestFirstToEarliestFinish) { EXPECT_EQ(plans[1].load, 20 + 10 + 2 * kOpenCost); } +TEST(DecodeLanePlannerTest, TiesToTheLeastBacklog) { + auto index = MakeIndex(100, 100); + Batch batch; + batch.Add("a", &index, 0, 0, 29); + // lanes without decoders, which would all finish the run at the same time: the lowest one + auto plans = Plan(batch, {{}, {}, {}}); + EXPECT_EQ(plans[0].runs, std::vector({0})); + // unless it has more work of earlier batches left than another + LaneStart behind, less_behind; + behind.backlog = 10; + less_behind.backlog = 5; + plans = Plan(batch, {behind, less_behind, behind}); + EXPECT_TRUE(plans[0].runs.empty()); + EXPECT_EQ(plans[1].runs, std::vector({0})); + // the backlog only breaks ties: the lane that continues the run gets it + auto held = Held("a", index, 0); + held.backlog = 1000; + plans = Plan(batch, {{}, held}); + EXPECT_EQ(plans[1].runs, std::vector({0})); +} + TEST(DecodeLanePlannerTest, HeldFileContinues) { auto index = MakeIndex(100, 50); // frames 20-39 of the GOP from frame 0: 20 frames of preroll after a seek diff --git a/dali/operators/video/reader/video_reader_decoder_op.cc b/dali/operators/video/reader/video_reader_decoder_op.cc index 7f321848b38..9b4f9c48793 100644 --- a/dali/operators/video/reader/video_reader_decoder_op.cc +++ b/dali/operators/video/reader/video_reader_decoder_op.cc @@ -677,6 +677,7 @@ class VideoReaderDecoder // Planning: where the decoder is once the lane is done with the batches planned so far. The // plans are made from it, not from the decoder. LanePosition planned; + int64_t backlog = 0; // planning: see LaneStart::backlog // Whether decoding a sample failed (and reset the decoder) since the lane last opened one: // the decoder is then not where the plan expects it bool failed = false; @@ -939,8 +940,20 @@ class VideoReaderDecoder indices.push_back(FrameIndexCache::instance().Find(pos.filename)); starts[l] = {pos.filename, FramesDecoderBase::PositionAt(*indices.back(), pos.next_frame)}; } + starts[l].backlog = lanes_[l].backlog; } PlanLanes(make_cspan(runs), make_cspan(starts), kOpenCost, batch.plans); + if (decode_ahead_ > 0) { + // A lane that is done with its part of a batch goes on to the next one: the work that one + // lane has more than another carries over to the next batch + int64_t least = std::numeric_limits::max(); + for (size_t l = 0; l < lanes_.size(); l++) { + lanes_[l].backlog += batch.plans[l].load; + least = std::min(least, lanes_[l].backlog); + } + for (auto &lane : lanes_) + lane.backlog -= least; + } batch.lane_start.resize(lanes_.size()); for (size_t l = 0; l < lanes_.size(); l++) { auto &lane = lanes_[l]; From b5fdb9946ff0a606a46696eb7f8c85d80683b4e4 Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Tue, 27 Oct 2026 08:55:13 +0100 Subject: [PATCH 40/46] Cache cuvidGetDecoderCaps instead of querying it for every file Signed-off-by: Joaquin Anton Guirao --- dali/operators/video/frames_decoder_gpu.cc | 45 +++++++++++++++++----- 1 file changed, 35 insertions(+), 10 deletions(-) diff --git a/dali/operators/video/frames_decoder_gpu.cc b/dali/operators/video/frames_decoder_gpu.cc index f393b47a021..a946be4bb86 100644 --- a/dali/operators/video/frames_decoder_gpu.cc +++ b/dali/operators/video/frames_decoder_gpu.cc @@ -22,6 +22,7 @@ #include #include #include +#include #include "dali/core/error_handling.h" #include "dali/core/cuda_error.h" #include "dali/core/device_guard.h" @@ -81,6 +82,37 @@ const char *codec_to_string(cudaVideoCodec in) { } } +/** + * @brief cuvidGetDecoderCaps of the current device, cached per process. + * + * The capabilities do not change, but the driver call does not scale: with many decoders opening + * files at once it takes tens of milliseconds (hundreds at worst), as it is serialized in the + * driver, and it is made for every file. + */ +CUVIDDECODECAPS GetDecoderCaps(cudaVideoCodec codec, cudaVideoChromaFormat chroma, + unsigned bit_depth_minus8) { + int device_id = -1; + CUDA_CALL(cudaGetDevice(&device_id)); + using Key = std::tuple; + static std::mutex mutex; + static std::map cache; + Key key{device_id, codec, chroma, bit_depth_minus8}; + { + std::lock_guard lock(mutex); + auto it = cache.find(key); + if (it != cache.end()) + return it->second; + } + CUVIDDECODECAPS caps = {}; + caps.eCodecType = codec; + caps.eChromaFormat = chroma; + caps.nBitDepthMinus8 = bit_depth_minus8; + CUDA_CALL(cuvidGetDecoderCaps(&caps)); + std::lock_guard lock(mutex); + cache.emplace(key, caps); + return caps; +} + class NVDECCache { public: static NVDECCache &GetCache(int device_id = -1) { @@ -201,11 +233,7 @@ class NVDECCache { } #endif lock.unlock(); - auto caps = CUVIDDECODECAPS{}; - caps.eCodecType = codec_type; - caps.eChromaFormat = chroma_format; - caps.nBitDepthMinus8 = bit_depth_luma_minus8; - CUDA_CALL(cuvidGetDecoderCaps(&caps)); + auto caps = GetDecoderCaps(codec_type, chroma_format, bit_depth_luma_minus8); DALI_ENFORCE(caps.bIsSupported, make_string("Codec configuration not supported on this GPU. ", "Codec: ", codec_to_string(codec_type), @@ -1124,11 +1152,8 @@ bool FramesDecoderGpu::SelectVideoStream(int stream_id) { return false; } - CUVIDDECODECAPS decoder_caps = {}; - decoder_caps.eCodecType = GetCodecType(codec_id); - decoder_caps.eChromaFormat = cudaVideoChromaFormat_420; - decoder_caps.nBitDepthMinus8 = 0; - CUDA_CALL(cuvidGetDecoderCaps(&decoder_caps)); + auto decoder_caps = frame_dec_gpu_impl::GetDecoderCaps(GetCodecType(codec_id), + cudaVideoChromaFormat_420, 0); if (!decoder_caps.bIsSupported) { DALI_WARN(make_string("Codec ", avcodec_get_name(codec_id), " is not supported by NVDEC on this platform.")); From d364d67e920846133c0be1dc6ddd92cd28663e70 Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Tue, 27 Oct 2026 11:11:16 +0100 Subject: [PATCH 41/46] Do not decode H.264 non-reference pictures that are not read A non-reference picture that is not read with its data (hidden by an edit list, or only skipped on the way to the frames to read) is no longer decoded: no other picture refers to it, and only its timestamp is kept. ParsedPictures (decoded + left out) is what AddDecodeWork estimates; the tests and the lanes' decode accounting compare it. Signed-off-by: Joaquin Anton Guirao --- dali/operators/video/frames_decoder_base.h | 10 ++++++ dali/operators/video/frames_decoder_gpu.cc | 27 ++++++++++++--- dali/operators/video/frames_decoder_gpu.h | 14 ++++++++ dali/operators/video/frames_decoder_test.cc | 33 +++++++++++-------- .../video/reader/video_reader_decoder_op.cc | 4 +-- 5 files changed, 69 insertions(+), 19 deletions(-) diff --git a/dali/operators/video/frames_decoder_base.h b/dali/operators/video/frames_decoder_base.h index ee3750d6102..e5366a9c0be 100644 --- a/dali/operators/video/frames_decoder_base.h +++ b/dali/operators/video/frames_decoder_base.h @@ -477,6 +477,15 @@ class DLL_PUBLIC FramesDecoderBase { return decoded_pictures_; } + /** + * @brief Number of pictures that the decoder went through so far: those decoded, and those it + * left out as no other picture needs them (see FramesDecoderGpu). AddDecodeWork estimates + * this number. + */ + int64_t ParsedPictures() const { + return decoded_pictures_ + skipped_pictures_; + } + protected: void DecodeFramesImpl(uint8_t *data, SmallVector, 32> frame_ids, boundary::BoundaryType boundary_type, @@ -510,6 +519,7 @@ class DLL_PUBLIC FramesDecoderBase { bool is_valid_ = false; int64_t decoded_pictures_ = 0; // see DecodedPictures + int64_t skipped_pictures_ = 0; // see ParsedPictures /** * @brief The frames read with their data next, in order, when known: DecodeFrames sets those it diff --git a/dali/operators/video/frames_decoder_gpu.cc b/dali/operators/video/frames_decoder_gpu.cc index a946be4bb86..4568995ec39 100644 --- a/dali/operators/video/frames_decoder_gpu.cc +++ b/dali/operators/video/frames_decoder_gpu.cc @@ -560,10 +560,20 @@ int FramesDecoderGpu::ProcessPictureDecode(CUVIDPICPARAMS *picture_params) { return 0; } - // NVDEC is not used while the previous picture's surface may still be read - ReleaseMappedFrame(); - CUDA_CALL(cuvidDecodePicture(nvdecode_state_->decoder, picture_params)); - decoded_pictures_++; + // A non-reference picture that is not read with its data (e.g. one hidden by an edit list, or one + // only skipped on the way to the frames to read) is not decoded at all: no other picture refers + // to it, and only its timestamp is kept + bool decode = picture_params->ref_pic_flag || picture_params->field_pic_flag || + codec_params_->codec_id != AV_CODEC_ID_H264 || + PictureReadWithData(piped_pts_.empty() ? current_pts_ : piped_pts_.front()); + if (decode) { + // NVDEC is not used while the previous picture's surface may still be read + ReleaseMappedFrame(); + CUDA_CALL(cuvidDecodePicture(nvdecode_state_->decoder, picture_params)); + decoded_pictures_++; + } else { + skipped_pictures_++; + } CUVIDPARSERDISPINFO picture_display_info; memset(&picture_display_info, 0, sizeof(picture_display_info)); picture_display_info.picture_index = picture_params->CurrPicIdx; @@ -1002,6 +1012,15 @@ void FramesDecoderGpu::AttachFrameBuffer(BufferedFrame &slot) { slot.frame_.resize(FrameSize()); } +bool FramesDecoderGpu::PictureReadWithData(int64_t pts) const { + // the same decision as HandlePictureDisplay's: returned now, or buffered with its data + if (HasIndex() && next_frame_idx_ >= 0 && pts == index_[next_frame_idx_].pts) + return current_copy_to_output_; + return buffered_frame_data_ == BufferedFrameData::kIfRead + ? MayBeReadWithData(pts) + : buffered_frame_data_ == BufferedFrameData::kAll; +} + bool FramesDecoderGpu::MayBeReadWithData(int64_t pts) const { if (!HasIndex() || next_frame_idx_ < 0 || next_frame_idx_ >= static_cast(index_.size())) return true; diff --git a/dali/operators/video/frames_decoder_gpu.h b/dali/operators/video/frames_decoder_gpu.h index 9d00a3fc46b..288b08f6cb1 100644 --- a/dali/operators/video/frames_decoder_gpu.h +++ b/dali/operators/video/frames_decoder_gpu.h @@ -234,6 +234,14 @@ class DLL_PUBLIC FramesDecoderGpu : public FramesDecoderBase { return mapped_pictures_; } + /** + * @brief Number of pictures not decoded at all so far (for tests and diagnostics): H.264 + * non-reference pictures that are not read with their data. + */ + int64_t SkippedPictures() const { + return skipped_pictures_; + } + protected: bool SelectVideoStream(int stream_id = -1) override; @@ -264,6 +272,12 @@ class DLL_PUBLIC FramesDecoderGpu : public FramesDecoderBase { */ bool MayBeReadWithData(int64_t pts) const; + /** + * @brief Whether the picture with this timestamp, decoded next, is mapped and converted: it is + * returned with its data now, or buffered with its data. + */ + bool PictureReadWithData(int64_t pts) const; + private: std::unique_ptr nvdecode_state_; diff --git a/dali/operators/video/frames_decoder_test.cc b/dali/operators/video/frames_decoder_test.cc index 0853972145d..972e62bb6dd 100644 --- a/dali/operators/video/frames_decoder_test.cc +++ b/dali/operators/video/frames_decoder_test.cc @@ -301,9 +301,9 @@ void CheckDecodeWork(FramesDecoderBase &decoder, FramesDecoderBase::DecodePositi for (int frame : frames) FramesDecoderBase::AddDecodeWork(index, pos, frame, work); - int64_t before = decoder.DecodedPictures(); + int64_t before = decoder.ParsedPictures(); decoder.DecodeFrames(buffer, make_cspan(frame_ids), boundary::BoundaryType::CLAMP); - EXPECT_NEAR(decoder.DecodedPictures() - before, work.pictures, slack) << what; + EXPECT_NEAR(decoder.ParsedPictures() - before, work.pictures, slack) << what; EXPECT_EQ(decoder.NextFrameIdx(), pos.next_frame) << what; } @@ -357,7 +357,7 @@ int CheckPositionAt(FramesDecoderBase &decoder, uint8_t *frame, int slack, const std::string &path) { auto &index = decoder.GetIndex(); int free_frames = 0; - int64_t start = decoder.DecodedPictures(), estimate = 0; + int64_t start = decoder.ParsedPictures(), estimate = 0; for (int k = 0; k < static_cast(index.size()); k++) { auto pos = FramesDecoderBase::PositionAt(index, k); FramesDecoderBase::DecodeWork work; @@ -365,7 +365,7 @@ int CheckPositionAt(FramesDecoderBase &decoder, uint8_t *frame, int slack, estimate += work.pictures; EXPECT_EQ(decoder.NextFrameIdx(), k) << path; decoder.ReadNextFrame(frame); - int64_t ahead = decoder.DecodedPictures() - start - estimate; + int64_t ahead = decoder.ParsedPictures() - start - estimate; EXPECT_GE(ahead, 0) << path << ": frame " << k; EXPECT_LE(ahead, slack) << path << ": frame " << k; free_frames += work.pictures == 0; @@ -770,7 +770,7 @@ using HashedFramesScenario = std::function(FramesDecoderGpu struct ScenarioResult { std::vector frames; - int64_t decoded = 0, mapped = 0; + int64_t decoded = 0, skipped = 0, mapped = 0; }; // Runs `scenario` with a new decoder (one at a time: ReadFrames expects one mapped surface at most) @@ -782,21 +782,24 @@ ScenarioResult RunScenario(const std::string &path, const FrameIndex &index, ScenarioResult result; result.frames = scenario(decoder); result.decoded = decoder.DecodedPictures(); + result.skipped = decoder.SkippedPictures(); result.mapped = decoder.MappedPictures(); return result; } /** * @brief Runs `scenario` with a decoder and with one that maps every picture (GpuDecoderMappingAll) - * and checks that they return the same frames, after decoding the same pictures, of which the - * decoder maps no more. Returns the pictures mapped by the decoder. + * and checks that they return the same frames, after going through the same pictures, of which the + * decoder decodes and maps no more. Returns the pictures mapped by the decoder. */ int64_t CheckSameFramesAsMappingAll(const std::string &path, const FrameIndex &index, const std::string &what, const HashedFramesScenario &scenario) { auto result = RunScenario(path, index, scenario); auto expected = RunScenario(path, index, scenario); EXPECT_EQ(result.frames, expected.frames) << path << ": " << what; - EXPECT_EQ(result.decoded, expected.decoded) << path << ": " << what; + EXPECT_EQ(result.decoded + result.skipped, expected.decoded + expected.skipped) + << path << ": " << what; + EXPECT_LE(result.decoded, expected.decoded) << path << ": " << what; EXPECT_LE(result.mapped, expected.mapped) << path << ": " << what; return result.mapped; } @@ -851,8 +854,9 @@ void CheckSameFramesAsMappingAll(const std::string &path) { } // namespace -// A decoder decodes the pictures that it does not return (hidden by an edit list, or skipped), but -// it does not map or convert them, nor keep a frame buffer for them +// A decoder decodes the pictures that it does not return (hidden by an edit list, or skipped) only +// when other pictures refer to them (H.264 non-reference pictures are left out), and it does not +// map or convert them, nor keep a frame buffer for them TEST_F(FramesDecoderGpuTest, PicturesNotReadAreNotMapped) { // 188 pictures hidden by an edit list, decoded before the first frame auto path = SintelVideoFile(4); @@ -862,11 +866,14 @@ TEST_F(FramesDecoderGpuTest, PicturesNotReadAreNotMapped) { auto check = [&](const std::string &what, const std::vector &frame_ids) { auto mapped = CheckSameFramesAsMappingAll(path, index, what, [&](FramesDecoderGpu &d) { auto frames = DecodeAndHash(d, frame_ids); - EXPECT_GT(d.DecodedPictures(), 188); - if (dynamic_cast(&d)) + EXPECT_GT(d.ParsedPictures(), 188); + if (dynamic_cast(&d)) { EXPECT_GT(d.NumFrameBuffers(), 188); // one for each picture hidden by the edit list - else + } else { EXPECT_LE(d.NumFrameBuffers(), 10); + // 86 of the 188 hidden pictures are H.264 non-reference pictures + EXPECT_GE(d.SkippedPictures(), 80); + } return frames; }); EXPECT_LE(mapped, static_cast(frame_ids.size()) + kMaxMappedAhead) << what; diff --git a/dali/operators/video/reader/video_reader_decoder_op.cc b/dali/operators/video/reader/video_reader_decoder_op.cc index 9b4f9c48793..6736d73c7d2 100644 --- a/dali/operators/video/reader/video_reader_decoder_op.cc +++ b/dali/operators/video/reader/video_reader_decoder_op.cc @@ -1148,7 +1148,7 @@ class VideoReaderDecoder } auto &decoder = *lane.decoder; DALI_ENFORCE(decoder.IsValid(), make_string("Invalid decoder for filename ", filename)); - int64_t decoded_pictures = decoder.DecodedPictures(); + int64_t decoded_pictures = decoder.ParsedPictures(); int64_t num_frames = sample.frame_idxs_.empty() ? (sample.end_ - sample.start_ + sample.stride_ - 1) / sample.stride_ @@ -1226,7 +1226,7 @@ class VideoReaderDecoder sample.end_, sample.stride_, boundary_type_, constant_frame, make_span(sample.timestamps_)); } - lane.decoded_pictures += decoder.DecodedPictures() - decoded_pictures; + lane.decoded_pictures += decoder.ParsedPictures() - decoded_pictures; LOG_LINE << "Decoding frames done" << std::endl; } From f826110c37ffc3c3ed6b340c07fcaf6f33be4c67 Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Tue, 27 Oct 2026 12:02:19 +0100 Subject: [PATCH 42/46] Address review: frame index cache lifetime, locking and lookups - The cache keeps an index only while a reader holds it (weak_ptr), so indices are freed with their readers instead of living for the whole process, and an index is built again when the file's size or modification time changed since it was built. - Each reader holds the indices of its files and plans and decodes with those (no cache lookups after setup), so an index cannot change under a reader. - shared_mutex (lookups in parallel, double-checked insertion) and string_view lookups. - instance() is defined in a .cc file, so that there is one cache per process. Signed-off-by: Joaquin Anton Guirao --- .../video/reader/frame_index_cache.cc | 78 +++++++++ .../video/reader/frame_index_cache.h | 152 ++++++++++++------ .../video/reader/frame_index_cache_test.cc | 61 ++++++- .../video/reader/video_reader_decoder_op.cc | 46 ++++-- 4 files changed, 270 insertions(+), 67 deletions(-) create mode 100644 dali/operators/video/reader/frame_index_cache.cc diff --git a/dali/operators/video/reader/frame_index_cache.cc b/dali/operators/video/reader/frame_index_cache.cc new file mode 100644 index 00000000000..cdae95aab56 --- /dev/null +++ b/dali/operators/video/reader/frame_index_cache.cc @@ -0,0 +1,78 @@ +// Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "dali/operators/video/reader/frame_index_cache.h" +#include +#include +#include + +namespace dali { + +FrameIndexCache &FrameIndexCache::instance() { + static FrameIndexCache cache; + return cache; +} + +FrameIndexCache::FileStamp FrameIndexCache::FileStamp::Of(std::string_view filename) { + struct stat st; + if (stat(std::string(filename).c_str(), &st) != 0) + return {}; + return {static_cast(st.st_size), + static_cast(st.st_mtim.tv_sec) * 1000000000 + st.st_mtim.tv_nsec}; +} + +FrameIndexCache::IndexPtr FrameIndexCache::Lookup(std::string_view filename, + const FileStamp &stamp, + std::shared_future &pending) const { + auto it = cache_.find(filename); + if (it == cache_.end() || it->second.stamp != stamp) + return nullptr; // not cached, or built from another version of the file + if (it->second.pending.valid()) { + pending = it->second.pending; + return nullptr; + } + return it->second.index.lock(); // nullptr if no longer used +} + +void FrameIndexCache::Finish(std::string_view filename, uint64_t id, const IndexPtr &index) { + std::unique_lock guard(mutex_); + auto it = cache_.find(filename); + if (it == cache_.end() || it->second.id != id) + return; // replaced (the file changed) while it was being built + if (index) { + it->second.index = index; + it->second.pending = {}; // the cache must not keep the index alive + } else { + cache_.erase(it); + } +} + +FrameIndexCache::IndexPtr FrameIndexCache::Find(std::string_view filename) const { + std::shared_lock guard(mutex_); + auto it = cache_.find(filename); + if (it == cache_.end() || it->second.pending.valid()) + return nullptr; + return it->second.index.lock(); +} + +void FrameIndexCache::SweepExpired() { + if (cache_.size() < sweep_size_) + return; + std::erase_if(cache_, [](const auto &item) { + return !item.second.pending.valid() && item.second.index.expired(); + }); + sweep_size_ = std::max(64, 2 * cache_.size()); +} + +} // namespace dali diff --git a/dali/operators/video/reader/frame_index_cache.h b/dali/operators/video/reader/frame_index_cache.h index 98a5ef018be..6f84becc7ae 100644 --- a/dali/operators/video/reader/frame_index_cache.h +++ b/dali/operators/video/reader/frame_index_cache.h @@ -15,14 +15,18 @@ #ifndef DALI_OPERATORS_VIDEO_READER_FRAME_INDEX_CACHE_H_ #define DALI_OPERATORS_VIDEO_READER_FRAME_INDEX_CACHE_H_ -#include +#include +#include #include #include #include +#include #include +#include #include #include +#include "dali/core/api_helper.h" #include "dali/operators/video/frames_decoder_base.h" namespace dali { @@ -31,18 +35,24 @@ namespace dali { * @brief Process-wide cache of frame indices, keyed by file name. * * Shared by all video readers in the process and safe to use from several threads at once. - * Every index is built at most once (by the first caller that asks for it), outside the lock, - * so building the indices of different files can run in parallel. Callers get an immutable, - * reference-counted index that stays valid regardless of other insertions. + * Every index is built at most once while it is in use (by the first caller that asks for it), + * outside the lock, so building the indices of different files can run in parallel. Callers get + * an immutable, reference-counted index that stays valid regardless of other insertions. + * + * The cache does not own the indices: it keeps an index only while a caller (e.g. a reader) + * holds it, so the index is freed with the last reader that uses it, and a later reader of the + * file builds it again. An index is also built again when the file's size or modification time + * changed since it was built. */ -class FrameIndexCache { +class DLL_PUBLIC FrameIndexCache { public: using IndexPtr = std::shared_ptr; - static FrameIndexCache &instance() { - static FrameIndexCache cache; - return cache; - } + /** + * @brief The process-wide instance. It is defined in a .cc file, so that there is one per + * process, not one per shared object that includes this header. + */ + static FrameIndexCache &instance(); /** * @brief Returns the index of `filename`, building it with `build()` if it is not cached. @@ -52,63 +62,101 @@ class FrameIndexCache { * and the entry is removed, so that a later call can try again. */ template - IndexPtr GetOrBuild(const std::string &filename, BuildFn &&build) { - std::promise promise; - std::shared_future index; - bool build_here = false; + IndexPtr GetOrBuild(std::string_view filename, BuildFn &&build) { + auto stamp = FileStamp::Of(filename); + std::shared_future pending; { - std::lock_guard guard(mutex_); - auto it = cache_.find(filename); - if (it != cache_.end()) { - index = it->second; - } else { - index = promise.get_future().share(); - cache_.emplace(filename, index); - build_here = true; - } + std::shared_lock guard(mutex_); + if (auto index = Lookup(filename, stamp, pending)) + return index; } - if (build_here) { - try { - promise.set_value(std::make_shared(build())); - } catch (...) { - { - std::lock_guard guard(mutex_); - cache_.erase(filename); - } - promise.set_exception(std::current_exception()); + std::promise promise; + uint64_t id = 0; + if (!pending.valid()) { + std::unique_lock guard(mutex_); + // looked up again: another thread may have added the entry since the shared lock was + // released + if (auto index = Lookup(filename, stamp, pending)) + return index; + if (!pending.valid()) { + SweepExpired(); + id = ++last_id_; + pending = promise.get_future().share(); + cache_.insert_or_assign(std::string(filename), Entry{id, pending, {}, stamp}); } } - return index.get(); + if (!id) + return pending.get(); // built by another thread + IndexPtr index; + try { + index = std::make_shared(build()); + } catch (...) { + Finish(filename, id, nullptr); + promise.set_exception(std::current_exception()); + throw; + } + Finish(filename, id, index); + promise.set_value(index); + return index; } /** * @brief Returns the index of `filename` if it is cached and ready, nullptr otherwise. * - * Never builds and never waits. + * Never builds and never waits; does not check the file. */ - IndexPtr Find(const std::string &filename) const { - std::shared_future index; - { - std::lock_guard guard(mutex_); - auto it = cache_.find(filename); - if (it == cache_.end()) - return nullptr; - index = it->second; - } - if (index.wait_for(std::chrono::seconds(0)) != std::future_status::ready) - return nullptr; - try { - return index.get(); - } catch (...) { - return nullptr; - } - } + IndexPtr Find(std::string_view filename) const; private: FrameIndexCache() = default; - mutable std::mutex mutex_; - std::unordered_map> cache_; + /** + * @brief A version of a file: its size and modification time (-1 if the file cannot be + * checked, e.g. the name is not a path). + */ + struct FileStamp { + int64_t size = -1, mtime_ns = -1; + static FileStamp Of(std::string_view filename); + bool operator==(const FileStamp &other) const = default; + }; + + struct Entry { + uint64_t id = 0; // tells this entry from a later one of the same file + std::shared_future pending; // valid while the index is being built + std::weak_ptr index; // set once built + FileStamp stamp; // of the file the index is built from + }; + + /** + * @brief With the lock held (shared or exclusive): the cached index of this version of the file, + * or nullptr; `pending` is set if that index is being built. + */ + IndexPtr Lookup(std::string_view filename, const FileStamp &stamp, + std::shared_future &pending) const; + + /** + * @brief Publishes the index built for entry `id` (or removes the entry if `index` is nullptr), + * unless the entry was replaced in the meantime. + */ + void Finish(std::string_view filename, uint64_t id, const IndexPtr &index); + + /** + * @brief With the exclusive lock held: removes the entries whose indices are no longer used, + * once the map has grown to twice its size after the last such sweep. + */ + void SweepExpired(); + + struct Hash { + using is_transparent = void; + size_t operator()(std::string_view s) const { + return std::hash{}(s); + } + }; + + mutable std::shared_mutex mutex_; + std::unordered_map> cache_; + uint64_t last_id_ = 0; + size_t sweep_size_ = 64; }; } // namespace dali diff --git a/dali/operators/video/reader/frame_index_cache_test.cc b/dali/operators/video/reader/frame_index_cache_test.cc index 5f2507ab2f1..d790608be3e 100644 --- a/dali/operators/video/reader/frame_index_cache_test.cc +++ b/dali/operators/video/reader/frame_index_cache_test.cc @@ -13,8 +13,11 @@ // limitations under the License. #include +#include #include #include +#include +#include #include #include #include @@ -51,6 +54,8 @@ TEST(FrameIndexCacheTest, BuildsEachIndexOnceUnderConcurrency) { std::vector> builds(kFiles); std::atomic errors{0}; std::vector threads; + // the cache keeps an index only while it is used: every thread keeps the ones it got + std::vector> held(kThreads); for (int t = 0; t < kThreads; t++) { threads.emplace_back([&, t]() { for (int r = 0; r < kRounds; r++) { @@ -64,6 +69,7 @@ TEST(FrameIndexCacheTest, BuildsEachIndexOnceUnderConcurrency) { if (!index || index->filename != Key("concurrent", file) || static_cast(index->size()) != file + 1) errors++; + held[t].push_back(index); } } }); @@ -139,8 +145,9 @@ TEST(FrameIndexCacheTest, FindDoesNotBuildOrWait) { auto key = Key("find", 0); EXPECT_EQ(cache.Find(key), nullptr); std::atomic building{false}, release{false}; + FrameIndexCache::IndexPtr built; std::thread builder([&]() { - cache.GetOrBuild(key, [&] { + built = cache.GetOrBuild(key, [&] { building = true; while (!release) std::this_thread::yield(); @@ -154,7 +161,59 @@ TEST(FrameIndexCacheTest, FindDoesNotBuildOrWait) { builder.join(); auto index = cache.Find(key); ASSERT_NE(index, nullptr); + EXPECT_EQ(index, built); EXPECT_EQ(index->size(), 2u); } +TEST(FrameIndexCacheTest, IndexIsReleasedWithItsLastUser) { + auto &cache = FrameIndexCache::instance(); + auto key = Key("released", 0); + int builds = 0; + auto build = [&] { + builds++; + return MakeIndex(key, 4); + }; + auto index = cache.GetOrBuild(key, build); + auto other = cache.GetOrBuild(key, build); + EXPECT_EQ(index, other); + EXPECT_EQ(builds, 1); + std::weak_ptr weak = index; + index.reset(); + other.reset(); + EXPECT_TRUE(weak.expired()); // the cache does not keep it + EXPECT_EQ(cache.Find(key), nullptr); + index = cache.GetOrBuild(key, build); + ASSERT_NE(index, nullptr); + EXPECT_EQ(builds, 2); +} + +TEST(FrameIndexCacheTest, IndexIsRebuiltWhenTheFileChanges) { + char path[] = "/tmp/frame_index_cache_test_XXXXXX"; + int fd = mkstemp(path); + ASSERT_GE(fd, 0); + close(fd); + auto &cache = FrameIndexCache::instance(); + int builds = 0; + auto build = [&] { + builds++; + return MakeIndex(path, builds); + }; + { + std::ofstream(path) << "first"; + } + auto first = cache.GetOrBuild(path, build); + EXPECT_EQ(cache.GetOrBuild(path, build), first); + EXPECT_EQ(builds, 1); + { + std::ofstream(path, std::ios::app) << ", second"; // another size + } + auto second = cache.GetOrBuild(path, build); + EXPECT_EQ(builds, 2); + EXPECT_NE(second, first); + EXPECT_EQ(second->size(), 2u); + EXPECT_EQ(first->size(), 1u); // still valid for its users + EXPECT_EQ(cache.GetOrBuild(path, build), second); + unlink(path); +} + } // namespace dali diff --git a/dali/operators/video/reader/video_reader_decoder_op.cc b/dali/operators/video/reader/video_reader_decoder_op.cc index 6736d73c7d2..def0fd0f07f 100644 --- a/dali/operators/video/reader/video_reader_decoder_op.cc +++ b/dali/operators/video/reader/video_reader_decoder_op.cc @@ -27,6 +27,7 @@ #include #include #include +#include #include #include @@ -236,6 +237,15 @@ class VideoLoaderDecoder : public Loader { MoveToNextShard(++current_index_); } + /** + * @brief The frame index of one of the reader's files, nullptr if the file was not indexed + * (it is not a valid video). + */ + const FrameIndex *FileIndex(const std::string &filename) const { + auto it = indices_.find(filename); + return it != indices_.end() ? it->second.get() : nullptr; + } + Index SizeImpl() override { return samples_.size(); } @@ -266,6 +276,7 @@ class VideoLoaderDecoder : public Loader { }); if (!decoder->HasIndex()) decoder->SetIndex(*index); + indices_[entry.filename] = index; // kept for the reader's lifetime int64_t num_frames = decoder->NumFrames(); entry.start_frame = 0; entry.end_frame = num_frames; @@ -391,6 +402,9 @@ class VideoLoaderDecoder : public Loader { FileListOptions file_list_opts_; std::vector video_files_info_; + // the frame indices of the valid files, built (or found in FrameIndexCache) by + // PrepareMetadataImpl; the cache keeps an index only while a reader holds it + std::unordered_map indices_; std::vector> all_frame_idxs_; // owns frame index data; samples_ hold spans into these std::vector samples_; CUDAStreamLease cuda_stream_; @@ -853,6 +867,13 @@ class VideoReaderDecoder QueueBatch(batch); } + /** + * @brief The frame index of one of the reader's files (see VideoLoaderDecoder::FileIndex). + */ + const FrameIndex *FileIndex(const std::string &filename) const { + return static_cast(*loader_).FileIndex(filename); + } + /** * @brief Groups the batch's samples into runs: samples of one file that one decoder decodes in * a row without seeking. @@ -869,7 +890,7 @@ class VideoReaderDecoder continue; sample->planned = true; const auto &filename = sample->video_file_meta_->filename; - auto index = FrameIndexCache::instance().Find(filename); + auto *index = FileIndex(filename); DALI_ENFORCE(index, make_string("Internal error: no frame index for \"", filename, "\".")); SampleTask task{sample, i, -1, -1, -1, {}}; try { @@ -897,18 +918,18 @@ class VideoReaderDecoder const auto &fb = tb.sample->video_file_meta_->filename; return std::tie(fa, ta.first, ta.batch_idx) < std::tie(fb, tb.first, tb.batch_idx); }); - FrameIndexCache::IndexPtr index; + const FrameIndex *index = nullptr; size_t file_runs = 0; // the first run of the current file for (int t : order) { auto &task = tasks[t]; const auto &filename = task.sample->video_file_meta_->filename; if (runs.empty() || *runs.back().filename != filename) { file_runs = runs.size(); - index = FrameIndexCache::instance().Find(filename); + index = FileIndex(filename); } // join a run of the file that a decoder continues to this sample without seeking auto run = std::find_if(runs.begin() + file_runs, runs.end(), [&](const DecodeRun &r) { - return task.first < 0 || CanReachWithoutSeek(index.get(), r.last + 1, task.first); + return task.first < 0 || CanReachWithoutSeek(index, r.last + 1, task.first); }); if (run != runs.end()) { run->tasks.push_back(t); @@ -925,11 +946,10 @@ class VideoReaderDecoder * their decoders. A lane without runs releases its decoder. */ void AssignRuns(PendingBatch &batch) { - std::vector indices; // keeps the indices alive while planning std::vector runs; for (const auto &run : batch.runs) { - indices.push_back(FrameIndexCache::instance().Find(*run.filename)); - runs.push_back({*run.filename, indices.back().get(), run.batch_idx, run.first, run.last, {}}); + runs.push_back({*run.filename, FileIndex(*run.filename), run.batch_idx, run.first, run.last, + {}}); for (int t : run.tasks) runs.back().samples.push_back(make_cspan(batch.tasks[t].frames)); } @@ -937,8 +957,8 @@ class VideoReaderDecoder for (size_t l = 0; l < lanes_.size(); l++) { const auto &pos = lanes_[l].planned; if (!pos.filename.empty()) { - indices.push_back(FrameIndexCache::instance().Find(pos.filename)); - starts[l] = {pos.filename, FramesDecoderBase::PositionAt(*indices.back(), pos.next_frame)}; + starts[l] = {pos.filename, FramesDecoderBase::PositionAt(*FileIndex(pos.filename), + pos.next_frame)}; } starts[l].backlog = lanes_[l].backlog; } @@ -1134,11 +1154,9 @@ class VideoReaderDecoder auto &decoder = *lane.decoder; LOG_LINE << "Initialized decoder to " << decoder.Filename() << " ptr: " << &decoder << " num_frames: " << decoder.NumFrames() << std::endl; - auto index = FrameIndexCache::instance().GetOrBuild(filename, [&]() { - LOG_LINE << "Building index for " << filename << std::endl; - decoder.BuildIndex(); - return decoder.GetIndex(); - }); + // the index the batch was planned with (the file is not indexed again, even if it changed) + auto *index = FileIndex(filename); + DALI_ENFORCE(index, make_string("Internal error: no frame index for \"", filename, "\".")); if (!decoder.HasIndex()) decoder.SetIndex(*index); } else { From 096344fd0524519c47b53d751923b5f56c3c8f52 Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Tue, 27 Oct 2026 12:02:19 +0100 Subject: [PATCH 43/46] Address review: check every frame against PyAV, test batch size 1 zip(..., strict=True) in _assert_frames_match_file, so that no decoded frame goes unchecked, and the lanes reader with one sample per batch. Signed-off-by: Joaquin Anton Guirao --- dali/test/python/decoder/test_video.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/dali/test/python/decoder/test_video.py b/dali/test/python/decoder/test_video.py index f0056d25ec0..2e698708ba0 100644 --- a/dali/test/python/decoder/test_video.py +++ b/dali/test/python/decoder/test_video.py @@ -1343,7 +1343,7 @@ def _assert_frames_match_file(frames, frame_nums, filename, window=None): closer to it than to any other frame of the file (with `window`, to any other frame up to `window` frames away, which is faster for long files).""" reference = _reference_frames(filename) - for frame, num in zip(frames, frame_nums): + for frame, num in zip(frames, frame_nums, strict=True): if num < 0: continue start = 0 if window is None else max(0, num - window) @@ -1453,6 +1453,16 @@ def test_reader_output_independent_of_num_threads(device, case): ) +# One sample per batch: one lane decodes, whatever the number of threads, and its decoder carries +# over from batch to batch +@cartesian_params(["cpu", "gpu"], (0, 1, 3)) +def test_reader_batch_size_1_output_independent_of_num_threads(device, case): + skip_if_m60() + _check_reader_output_independent_of_num_threads( + device, _lanes_reader_cases[case], batch_size=1, num_threads=4 + ) + + # An H.264 clip with a long lead-in hidden by an edit list (188 pictures). The GPU decoder decodes # those pictures, but does not map or convert them, nor the pictures that it only skips long_lead_in_files = [ From a67ba9616c7e9d264bceed965fb89b234fd11035 Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Tue, 27 Oct 2026 03:09:08 +0100 Subject: [PATCH 44/46] Check video reader output over two epochs and with reader depth 2 The checksum run of the video reader benchmark covered at least 40 samples per configuration: three batches at batch 16. That is enough for the reader's queues to wrap around, but in 'lanes' no decoder is ever reused in three batches, seq_longgop and random_access reach the next epoch in none, and no checksummed configuration runs several decode lanes with more than one batch in the reader's queue. --hash_epochs E checksums at least E epochs and one more batch of every configuration, so that every file of 'lanes' is opened again and decoders are reused across batches and epochs. A new scenario, lanes_depth, run only with --hash, decodes batches of 16 on several lanes with the reader's prefetch_queue_depth 1 and 2: whole files in order and shuffled, and many distinct files. TL1_video_reader_perf checks two epochs and lanes_depth. Signed-off-by: Joaquin Anton Guirao --- internal_tools/video_reader_bench.py | 50 ++++++++++++++++++++++++---- qa/TL1_video_reader_perf/test.sh | 8 +++-- 2 files changed, 49 insertions(+), 9 deletions(-) diff --git a/internal_tools/video_reader_bench.py b/internal_tools/video_reader_bench.py index cd407f56d29..0101e66f231 100644 --- a/internal_tools/video_reader_bench.py +++ b/internal_tools/video_reader_bench.py @@ -61,9 +61,13 @@ compare builds, not datasets --hash records a checksum of every sample (frames, label, frame_num) of at least -HASH_MIN_SAMPLES samples, so that the reader's prefetch queue wraps around. Compare two ---hash result files (e.g. two builds, or --ignore threads to check that the output -does not depend on num_threads) with --compare. +HASH_MIN_SAMPLES samples, so that the reader's prefetch queue wraps around, and with +--hash_epochs E of E epochs and one more batch, so that decoders are reused across +batches and the reader wraps around its epochs. Compare two --hash result files (e.g. two +builds, or --ignore threads to check that the output does not depend on num_threads) with +--compare. --hash also runs the configurations only checksummed: + lanes_depth batch 16 on several decode lanes with the reader's prefetch_queue_depth + 1 and 2: whole files in order and shuffled, and many distinct files Other readers and libraries (--backends), for reference: dali_legacy DALI's legacy fn.readers.video (GPU, constant frame rate only: no cpu, @@ -182,6 +186,7 @@ def _cfg(files, layout="plain", device="gpu", batch=8, seq=32, threads=4, **kw): def scenarios(threads, readers, quick=False): + """The configurations of every scenario (HASH_ONLY_SCENARIOS only with --hash).""" t_max = max(threads) t_pair = sorted({1, t_max}) iters = {} if quick else dict(iters=20) @@ -218,9 +223,25 @@ def scenarios(threads, readers, quick=False): "cpu": [ _cfg("vpx_clips", "many:128", device="cpu", batch=8, seq=16, threads=t) for t in threads ], + # several decode lanes with up to two batches in the reader's queue (the other + # scenarios with several lanes use reader_depth 1) + "lanes_depth": [ + _cfg(f, layout, batch=16, seq=32, threads=t, reader_depth=d, shuffle=s) + for f, layout, s in ( + ("h264_clips", "plain", False), + ("h264_clips", "plain", True), + ("h264_clips", "many:256", False), + ) + for d in (1, 2) + for t in threads + ], } +# scenarios only checksummed (--hash), not measured +HASH_ONLY_SCENARIOS = {"lanes_depth"} + + # scenarios that other backends run as well (reader_depth and engines measure properties of # the DALI reader), and the devices they decode on _REFERENCE_SCENARIOS = {"lanes", "seq_longgop", "random_access", "single_stream", "cache_key"} @@ -794,12 +815,16 @@ def run_one(cfg, specs, args): def hash_one(cfg, specs, args): - """Returns one checksum per sample of the first batches (see HASH_MIN_SAMPLES).""" + """Returns one checksum per sample of the first batches (see HASH_MIN_SAMPLES and + --hash_epochs).""" pipe = build_pipeline(cfg, specs, args.device_id, args.seed, frame_num=True) pipe.build() hashes = [] # Not dependent on threads or reader_depth, so that --compare --ignore can match runs batches = -(-HASH_MIN_SAMPLES // (cfg["batch"] * cfg["readers"])) # ceil + if args.hash_epochs: + epoch = max(pipe.epoch_size(f"Reader{r}") for r in range(cfg["readers"])) + batches = max(batches, -(-args.hash_epochs * epoch // cfg["batch"]) + 1) for _ in range(batches): outs = [_as_cpu(t) for t in pipe.run()] for r in range(cfg["readers"]): @@ -831,7 +856,7 @@ def child_main(args): def run_child(cfg, specs, args): cmd = [sys.executable, os.path.abspath(__file__), "--_single"] cmd += [json.dumps(dict(cfg=cfg, specs=specs))] - for flag in ("dali_extra", "device_id", "iters", "warmup", "min_time", "seed"): + for flag in ("dali_extra", "device_id", "iters", "warmup", "min_time", "seed", "hash_epochs"): cmd += [f"--{flag}", str(getattr(args, flag))] cmd += ["--hash"] if args.hash else [] try: @@ -1080,6 +1105,13 @@ def parse_args(): parser.add_argument("--json", help="write results (with provenance) to this file") parser.add_argument("--quick", action="store_true", help="smoke test: tiny sweep, 1 run") parser.add_argument("--hash", action="store_true", help="record per-sample checksums") + parser.add_argument( + "--hash_epochs", + default=0, + type=int, + help="with --hash: checksum at least this many epochs of every configuration, and " + "one more batch", + ) parser.add_argument("--compare", nargs=2, metavar=("A", "B"), help="compare two --hash files") parser.add_argument( "--ignore", @@ -1122,13 +1154,19 @@ def main(): threads = [int(t) for t in args.threads.split(",")] readers = [int(r) for r in args.readers.split(",")] all_scenarios = scenarios(threads, readers, args.quick) - names = list(all_scenarios) if args.scenarios == "all" else args.scenarios.split(",") + if args.scenarios == "all": + names = [n for n in all_scenarios if args.hash or n not in HASH_ONLY_SCENARIOS] + else: + names = args.scenarios.split(",") unknown = [n for n in names if n not in all_scenarios] if unknown: parser.error( f"--scenarios got unknown scenario(s) {unknown}; expected 'all' or a subset of " f"{list(all_scenarios)}." ) + hash_only = [n for n in names if n in HASH_ONLY_SCENARIOS] + if hash_only and not args.hash: + parser.error(f"--scenarios {hash_only} run only with --hash.") if args.hash: args.repeat = 1 diff --git a/qa/TL1_video_reader_perf/test.sh b/qa/TL1_video_reader_perf/test.sh index 06e4d9124b2..c636ee36b61 100644 --- a/qa/TL1_video_reader_perf/test.sh +++ b/qa/TL1_video_reader_perf/test.sh @@ -58,8 +58,10 @@ install_reference_libs() { # Scenarios of internal_tools/video_reader_bench.py run for performance; see its --help PERF_SCENARIOS="lanes,engines,single_stream,reader_depth,cache_key,seq_longgop,random_access,cpu" -# Scenarios whose output is checked to be independent of num_threads and reader_depth -HASH_SCENARIOS="lanes,reader_depth,cache_key,cpu,seq_longgop,random_access" +# Scenarios whose output is checked to be independent of num_threads and reader_depth, over two +# epochs (decoders reused across batches, the epochs wrapping around) +HASH_SCENARIOS="lanes,lanes_depth,reader_depth,cache_key,cpu,seq_longgop,random_access" +HASH_EPOCHS=2 BENCH_ARGS="--repeat 3 --iters 8 --min_time 3 -j 1,4,16 --readers 1,2,4" # engines: one H100 or GH200 reaches 3.6x with 4 readers (several NVDEC engines) ENGINES_SCALING=2.5 @@ -120,7 +122,7 @@ test_body() { # configuration is compared with all that differ from it only in those HASH_RESULT="OK" ${TASKSET} python video_reader_bench.py -s ${HASH_SCENARIOS} -j 1,16 --hash \ - --json ${JSON_HASH} || HASH_RESULT="FAIL" + --hash_epochs ${HASH_EPOCHS} --json ${JSON_HASH} || HASH_RESULT="FAIL" python video_reader_bench.py --compare ${JSON_HASH} ${JSON_HASH} \ --ignore threads,reader_depth || HASH_RESULT="FAIL" From ee423dfe98a94c278c3a7848b521053d1a562f66 Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Tue, 27 Oct 2026 11:47:08 +0100 Subject: [PATCH 45/46] Pass the TL1 engines check when NVDEC is saturated With decode lanes, one reader already spreads its decoding over the NVDEC engines, so 4 readers can no longer reach 2.5x one reader (H100 PCIe: 2016 frames/s with 1 reader, 3824 with 4 at 98% NVDEC, 1.9x). The check now also passes when the run with the most readers keeps NVDEC at least 90% busy. Signed-off-by: Joaquin Anton Guirao --- qa/TL1_video_reader_perf/check_results.py | 24 +++++++++++++++++++---- qa/TL1_video_reader_perf/test.sh | 7 +++++-- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/qa/TL1_video_reader_perf/check_results.py b/qa/TL1_video_reader_perf/check_results.py index 8525d8f857f..31e6f665232 100644 --- a/qa/TL1_video_reader_perf/check_results.py +++ b/qa/TL1_video_reader_perf/check_results.py @@ -21,8 +21,11 @@ - lanes, seq_longgop, random_access, cpu: throughput at any higher num_threads is not lower than at the lowest one (more threads must not make the reader slower) - reader_depth: a deeper reader queue is not slower than depth 1 - * optional: engines with the most readers reach at least --engines_scaling times the - throughput of one reader (GPUs with several NVDEC engines; e.g. H100: 3.6x at 4) + * optional: engines with the most readers use the GPU's NVDEC engines: they reach at least + --engines_scaling times the throughput of one reader (GPUs with several NVDEC engines), or, + with --engines_busy, keep NVDEC at least that busy (percent). The latter is for when one + reader already spreads its decode lanes over the engines, so that more readers cannot scale + as much. * optional absolute minimums (frames/s) for named metrics, passed with --min: lanes lanes, batch 16, highest num_threads single_stream single_stream, h264_long @@ -87,6 +90,12 @@ def main(): type=float, help="minimum speedup of engines with the most readers over one reader", ) + parser.add_argument( + "--engines_busy", + type=float, + help="NVDEC utilization (percent) of engines with the most readers that passes the " + "engines check even below --engines_scaling (NVDEC is saturated)", + ) args = parser.parse_args() with open(args.results) as f: @@ -153,10 +162,17 @@ def relative(scenario, keys, param): else: one, most = rows[0], rows[-1] scaling = most["frames_per_s"] / one["frames_per_s"] + busy = most.get("proc_nvdec_util") + saturated = ( + args.engines_busy is not None and busy is not None and busy >= args.engines_busy + ) + detail = f"{scaling:.2f}x, minimum {args.engines_scaling:g}x" + if args.engines_busy is not None: + detail += f"; NVDEC {busy}% busy, saturated at {args.engines_busy:g}%" check( f"engines readers={most['readers']} vs 1", - scaling >= args.engines_scaling, - f"{scaling:.2f}x, minimum {args.engines_scaling:g}x", + scaling >= args.engines_scaling or saturated, + detail, ) for name, value in metrics.items(): diff --git a/qa/TL1_video_reader_perf/test.sh b/qa/TL1_video_reader_perf/test.sh index c636ee36b61..e0921afb6fd 100644 --- a/qa/TL1_video_reader_perf/test.sh +++ b/qa/TL1_video_reader_perf/test.sh @@ -63,8 +63,11 @@ PERF_SCENARIOS="lanes,engines,single_stream,reader_depth,cache_key,seq_longgop,r HASH_SCENARIOS="lanes,lanes_depth,reader_depth,cache_key,cpu,seq_longgop,random_access" HASH_EPOCHS=2 BENCH_ARGS="--repeat 3 --iters 8 --min_time 3 -j 1,4,16 --readers 1,2,4" -# engines: one H100 or GH200 reaches 3.6x with 4 readers (several NVDEC engines) +# engines: more readers use more NVDEC engines: 4 readers reach 2.5x one reader, or keep NVDEC +# saturated (one reader's decode lanes already use several engines, e.g. H100 PCIe: 2016 frames/s +# with 1 reader, 3824 at 98% NVDEC with 4, which is 1.9x) ENGINES_SCALING=2.5 +ENGINES_BUSY=90 function CLEAN_AND_EXIT { rm -rf ${LOG_PERF} ${JSON_PERF} ${JSON_HASH} ${LOG_REF} ${JSON_REF} @@ -128,7 +131,7 @@ test_body() { CHECK_RESULT="OK" python ${topdir}/qa/TL1_video_reader_perf/check_results.py ${JSON_PERF} --tolerance 10 \ - --engines_scaling ${ENGINES_SCALING} \ + --engines_scaling ${ENGINES_SCALING} --engines_busy ${ENGINES_BUSY} \ --min lanes=${MIN_LANES} --min single_stream=${MIN_SINGLE_STREAM} \ --min cpu=${MIN_CPU} --min engines=${MIN_ENGINES} || CHECK_RESULT="FAIL" From 12bde61f30712a874239ff87f12dd368d9bd6d34 Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Tue, 27 Oct 2026 18:58:32 +0100 Subject: [PATCH 46/46] Replace the video reader benchmark with a smaller one A single script (video_reader_bench.py) that runs each configuration in a fresh process, reports the median frames/s and checks that the output does not depend on num_threads or the reader's prefetch_queue_depth. The GPU configurations also run through torchcodec, for reference only. - Configurations: lanes, cpu, seq, random, random_hevc, reader_depth and two single-stream ones; reader_depth 2 with batches of 16 is checked for output only. - --check fails when more threads (or a deeper reader queue) are more than 10% slower than fewer; --compare compares two result files. - The TL1 test installs torch and torchcodec on a best-effort basis: without them the torchcodec rows are skipped and the test still passes. - check_results.py is gone: its checks are part of the script. Signed-off-by: Joaquin Anton Guirao --- internal_tools/video_reader_bench.py | 1474 ++++----------------- qa/TL1_video_reader_perf/check_results.py | 193 --- qa/TL1_video_reader_perf/test.sh | 174 +-- 3 files changed, 301 insertions(+), 1540 deletions(-) delete mode 100644 qa/TL1_video_reader_perf/check_results.py diff --git a/internal_tools/video_reader_bench.py b/internal_tools/video_reader_bench.py index 0101e66f231..34200550000 100644 --- a/internal_tools/video_reader_bench.py +++ b/internal_tools/video_reader_bench.py @@ -12,1254 +12,302 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Benchmark for fn.experimental.readers.video, using DALI_extra data only. - -Scenarios (each isolates one aspect of the reader): - - lanes many distinct files, one keyframe-aligned sequence each, so every - batch holds `batch` different files and no sample needs a seek: - per-file cost (opening, indexing, decoder setup, decoding the lead-in - frames an edit list hides, see LEAD_IN_FILES) plus decoding, vs. - num_threads. A per-file speedup also shows here; check `cache_key` - and `engines` before attributing a gain to decode parallelism - seq_longgop 15 H.264 clips read sequentially (long GOPs, consecutive sequences - of the same file): no-regression check for sequential access - random_access the same clips, and the HEVC trailer (open GOPs), shuffled: - parallelism and keyframe-seek waste together (do not read a lanes - speedup from this one alone) - single_stream one clip, batch 1: per-stream throughput, i.e. per-frame - serialization. Includes a 512x512 clip, where per-frame overheads - matter more than decode time - reader_depth batch 1, the reader's own prefetch_queue_depth (batches in flight), - on many distinct files and on one sequential long-GOP file - cache_key every sample switches between two files with the same format, either - cropped (800x600 coded as 800x608) or not (1280x720), vs. a control - that rewinds the same file to frame 0 for every sample instead: - the cost of opening another file (decoder creation or NVDEC decoder - cache reuse) relative to a flush and seek within the same file. This - is a workload difference, not an isolated switch cost - engines K independent readers in one pipeline, each on its own files: - how throughput scales with concurrent NVDEC sessions (works today, - because every reader decodes on its own thread) - cpu many distinct VP8/VP9 files on the CPU backend vs. num_threads - -Methodology: - * every configuration runs in a fresh subprocess (no cache carry-over), --repeat - times, round-robin over configurations; the median and the spread are reported - * the first `depth` + `reader_depth` timed iterations are excluded, as up to that many - batches (pipeline queue and the reader's own queue) may have been produced off the - clock; measuring continues until both --iters and --min_time are reached after them - * throughput counts the actual output frames of the timed batches - * Pipeline.run() without cuda_stream returns host-synchronized outputs, so iteration - times include GPU completion - * NVML metrics are per process where possible (decoder utilization, memory). A - result is flagged as contaminated if other processes use the GPU or NVDEC is busy - before the run starts. Decoder utilization (dec%) is averaged over the GPU's NVDEC - engines, so on GPUs with several engines it is low even for a fully busy session - * all input files are read once before the first run, so all runs see a warm page cache - * DALI_extra has no 1080p content, so absolute rates are lower than for 1080p data; - compare builds, not datasets - ---hash records a checksum of every sample (frames, label, frame_num) of at least -HASH_MIN_SAMPLES samples, so that the reader's prefetch queue wraps around, and with ---hash_epochs E of E epochs and one more batch, so that decoders are reused across -batches and the reader wraps around its epochs. Compare two --hash result files (e.g. two -builds, or --ignore threads to check that the output does not depend on num_threads) with ---compare. --hash also runs the configurations only checksummed: - lanes_depth batch 16 on several decode lanes with the reader's prefetch_queue_depth - 1 and 2: whole files in order and shuffled, and many distinct files - -Other readers and libraries (--backends), for reference: - dali_legacy DALI's legacy fn.readers.video (GPU, constant frame rate only: no cpu, - cache_key), through the same pipeline as the DALI reader - dali_decoder fn.readers.file + fn.decoders.video (fn.experimental.decoders.video in - older DALI): every sample is a whole file, read by the file reader, and - decoded with a new decoder per sample, the samples of a batch in parallel - on num_threads threads. Same samples (file, first and last frame) as the - DALI reader, shuffled by the file reader; `reader_depth` is the file - reader's prefetch_queue_depth - pynvvideocodec PyNvVideoCodec SimpleDecoder (NVDEC), RGB interleaved output (as DALI) - torchcodec torchcodec VideoDecoder, FFmpeg CUDA backend on GPU, FFmpeg on CPU - torchcodec_nvdec torchcodec VideoDecoder with its own NVDEC backend (torchcodec >= 0.8) - They decode the same samples as the DALI reader (the same sequences of the same files, - in the same order, or shuffled with a fixed seed), each sample with frames [start, end) - of one file. A batch is decoded by a pool of min(threads, batch) threads, each with its - own decoder, kept while consecutive samples come from the same file (PyNvVideoCodec - reconfigures its decoder for a new file, torchcodec opens a new one, scanning it, as - its default seek_mode="exact" does). Batches are not prefetched: unlike DALI, a batch - is decoded when it is requested. A batch's outputs are kept until the next batch is - requested, as DALI's, so the process memory (mem MB) includes one batch for them, and - for DALI also the batches in its queues (`depth` + `reader_depth`). PyNvVideoCodec is an - exception: its frames stay in its decoder's buffers, which the decoder's next call - reuses, so its memory does not include a batch. They run only the configurations that - mean the same for them (backend_runs), and need torch. Rows are flagged when torchcodec - reports a CPU fallback, and PyNvVideoCodec and dali_decoder rows on clips with hidden - lead-in frames (LEAD_IN_FILES), where they return different frames. +"""Benchmark of fn.experimental.readers.video, using DALI_extra data only. + + video_reader_bench.py --json a.json throughput (frames/s) of every configuration + video_reader_bench.py --json a.json --check ... and fail if more threads (or a deeper reader + queue) are more than 10% slower than fewer + video_reader_bench.py --json a.json --hash checksums of the output instead of throughput + video_reader_bench.py --compare a.json b.json two builds: checksums equal, throughput ratio + +Configurations (a group is one set of samples, read with different num_threads / reader_depth): + + lanes 256 distinct files, one sequence each: every sample is a new file, so the cost + of opening a file and the parallelism of decoding dominate + cpu the same on the CPU backend (VP8/VP9 files) + seq, random the 15 H.264 clips as they are, read in order or shuffled + random_hevc the HEVC trailer, shuffled (keyframe seeks, open GOPs) + reader_depth the reader's own prefetch_queue_depth at batch 1 + single_* one long file, batch 1: a single stream, so what is measured is how fast one + decoder runs per frame, not how many run in parallel + +The GPU configurations also run through torchcodec on the GPU (groups *_torchcodec, FFmpeg's +NVDEC), on the same samples: for reference only, never checked, skipped if it is not installed. + +Every configuration runs in a fresh process, so the frame index and NVDEC decoder caches do not +carry over, and is repeated --repeat times: the median is reported. The first batches (those +the pipeline and the reader's queue may have produced before the clock starts) are not timed. + +With --hash, the output of every configuration of a group must be identical, whatever +num_threads and reader_depth: each configuration records a checksum of the first >= 40 samples +(so that the reader's prefetch queue wraps around). The configurations marked "checksums only" +add reader_depth 2 with batches of 16, which the timed ones do not run. """ import argparse +import contextlib import glob import hashlib +import itertools import json import os import random -import shutil -import socket import statistics import subprocess import sys import tempfile import threading import time -import warnings +from concurrent.futures import ThreadPoolExecutor VIDEO_DIR = "db/video" - -# --hash checks at least this many samples per configuration: twice the batches the -# pipeline and the reader can hold (depth 2 + reader_depth up to 8) at batch 1, so that -# every slot of the reader's prefetch queue is reused -HASH_MIN_SAMPLES = 40 - -VPX_CLIPS = [ - "vp9/vp9_0.mp4", - "vp9/vp9_2.mp4", - "vp8/vp8.webm", - "cfr/test_1_vp9.mp4", - "cfr/test_2_vp9.mp4", - "vfr/test_1_vp9.mp4", - "vfr/test_2_vp9.mp4", -] # file sets: glob patterns relative to DALI_extra/db/video -FILES = { - # 15 H.264 1280x720 clips, 122-430 frames, B-frames, 1-4 keyframes (GOP up to 250) - "h264_clips": ["sintel/video_files/*.mp4", "sintel/labelled_videos/*/*.mp4"], - # one H.264 1280x720 clip, 1253 frames, GOP 250 - "h264_long": ["sintel/sintel_trailer-720p.mp4"], - # the same clip as HEVC 8-bit (keyframe every ~9 frames) and HEVC 10-bit (GOP 250) - "hevc_long": ["hevc/sintel_trailer-720p.mp4"], - "hevc10_long": ["hevc/sintel_trailer-720p_10b.mp4"], - # one H.264 512x512 clip, 250 frames: small frames, per-frame overheads dominate - "h264_small": ["test_label.mp4"], - # two H.264 800x600 clips, coded as 800x608 (display area cropped) - "h264_cropped_pair": ["cfr/test_2.mp4", "vfr/test_2.mp4"], - # two H.264 1280x720 clips, coded size == display size - "h264_aligned_pair": ["cfr_test.mp4", "vfr_test.mp4"], - # VP8/VP9 clips (H.264 is not enabled on the CPU backend) - "vpx_clips": VPX_CLIPS, -} -# File sets where most clips (cut without re-encoding) start before the frame they show first, -# with an MP4 edit list hiding the lead-in frames: the DALI reader and torchcodec start at the -# first visible frame, so they decode the lead-in before frame 0, while PyNvVideoCodec and -# DALI's video decoder (dali_decoder) return the lead-in frames themselves (different frames, -# less decoding) -LEAD_IN_FILES = {"h264_clips"} -LEAD_IN_BACKENDS = {"pynvvideocodec", "dali_decoder"} - +CLIPS = ["sintel/video_files/*.mp4", "sintel/labelled_videos/*/*.mp4"] # 15 H.264 720p clips +H264_LONG = ["sintel/sintel_trailer-720p.mp4"] # 1253 frames, GOP 250 +HEVC_LONG = ["hevc/sintel_trailer-720p.mp4"] # the same clip as HEVC, a keyframe every ~9 frames +VPX = ["vp9/vp9_0.mp4", "vp9/vp9_2.mp4", "vp8/vp8.webm", "cfr/test_1_vp9.mp4", "vfr/test_1_vp9.mp4"] +HASH_MIN_SAMPLES = 40 -def _cfg(files, layout="plain", device="gpu", batch=8, seq=32, threads=4, **kw): - """A benchmark configuration. Only JSON-serializable values. - layout: "plain" - the files as they are (filenames=...) - "many:N" - N distinct symlinks cycling over the files, each with one - sequence starting at frame 0 (a keyframe) - "alternate:N" - N lines alternating between the files, each starting at - frame 0, read in order: every sample switches files - "same:N" - N lines, all of the first file, each starting at frame 0: the - control for "alternate" (no file switches, but every sample - after the first rewinds the same decoder to frame 0) - """ - cfg = dict( - backend="dali", - files=files, - layout=layout, - device=device, - batch=batch, - seq=seq, - threads=threads, - depth=2, # pipeline prefetch_queue_depth (DALI default) - reader_depth=1, # the reader's own prefetch_queue_depth (DALI default) - readers=1, - shuffle=False, - ) - cfg.update(kw) - return cfg +def dali_configs(threads): + """(group, configuration) of the DALI reader. - -def scenarios(threads, readers, quick=False): - """The configurations of every scenario (HASH_ONLY_SCENARIOS only with --hash).""" + `many`: that number of symlinks to the files, one sequence each. Without it, the files as + they are.""" + base = dict(batch=16, seq=32, rd=1, shuffle=False, device="gpu") t_max = max(threads) - t_pair = sorted({1, t_max}) - iters = {} if quick else dict(iters=20) - return { - "lanes": [_cfg("h264_clips", "many:256", batch=16, seq=32, threads=t) for t in threads], - "seq_longgop": [_cfg("h264_clips", batch=16, seq=32, threads=t) for t in t_pair], - "random_access": [ - _cfg(f, batch=16, seq=32, threads=t, shuffle=True) - for f in ("h264_clips", "hevc_long") - for t in t_pair - ], - "single_stream": [ - _cfg(f, batch=1, seq=64, threads=4) - for f in ("h264_long", "hevc_long", "hevc10_long", "h264_small") - ], - "reader_depth": [ - _cfg(f, layout, batch=1, seq=32, threads=8, reader_depth=d) - for f, layout in (("h264_clips", "many:256"), ("h264_long", "plain")) - for d in (1, 2, 4, 8) - ], - # ms_per_sample(alternate) - ms_per_sample(same), per file pair: cost of opening the - # other file relative to rewinding the same one (see the module docstring). One - # thread: with more, other libraries keep each file of a pair on its own decoder - "cache_key": [ - _cfg(f, f"{kind}:64", batch=8, seq=8, threads=1) - for f in ("h264_cropped_pair", "h264_aligned_pair") - for kind in ("alternate", "same") - ], - # the noisiest scenario: more iterations than the --iters default - "engines": [ - _cfg("h264_clips", "many:64", batch=4, seq=32, threads=4, readers=k, **iters) - for k in readers - ], - "cpu": [ - _cfg("vpx_clips", "many:128", device="cpu", batch=8, seq=16, threads=t) for t in threads - ], - # several decode lanes with up to two batches in the reader's queue (the other - # scenarios with several lanes use reader_depth 1) - "lanes_depth": [ - _cfg(f, layout, batch=16, seq=32, threads=t, reader_depth=d, shuffle=s) - for f, layout, s in ( - ("h264_clips", "plain", False), - ("h264_clips", "plain", True), - ("h264_clips", "many:256", False), - ) - for d in (1, 2) - for t in threads - ], - } - - -# scenarios only checksummed (--hash), not measured -HASH_ONLY_SCENARIOS = {"lanes_depth"} - - -# scenarios that other backends run as well (reader_depth and engines measure properties of -# the DALI reader), and the devices they decode on -_REFERENCE_SCENARIOS = {"lanes", "seq_longgop", "random_access", "single_stream", "cache_key"} -BACKEND_SCENARIOS = { - "dali": (None, ("gpu", "cpu")), # None: all scenarios - # the legacy fn.readers.video: GPU only, and constant frame rate only (both cache_key - # pairs include a variable frame rate file) - "dali_legacy": (_REFERENCE_SCENARIOS - {"cache_key"} | {"reader_depth", "engines"}, ("gpu",)), - # fn.readers.file + fn.decoders.video; reader_depth would prefetch file contents only - "dali_decoder": (_REFERENCE_SCENARIOS | {"engines", "cpu"}, ("gpu", "cpu")), - "pynvvideocodec": (_REFERENCE_SCENARIOS, ("gpu",)), - "torchcodec": (_REFERENCE_SCENARIOS | {"cpu"}, ("gpu", "cpu")), - "torchcodec_nvdec": (_REFERENCE_SCENARIOS, ("gpu",)), -} - - -def backend_runs(backend, scenario, cfg): - names, devices = BACKEND_SCENARIOS[backend] - if (names is not None and scenario not in names) or cfg["device"] not in devices: - return False - # PyNvVideoCodec 2.0 SimpleDecoder fails (IndexError) when a decoder that was just - # created or reconfigured for a file is asked for frames from the middle of it, so it - # runs only configurations whose decoders start every file at frame 0 - seeks = scenario == "random_access" or ( - scenario == "seq_longgop" and min(cfg["threads"], cfg["batch"]) > 1 - ) - if backend == "pynvvideocodec" and seeks: - return False - # the legacy reader fails seeking in the HEVC trailer (open GOPs): "The decoder returned a - # frame that is past the expected one"; reading it sequentially works - return not (backend == "dali_legacy" and seeks and cfg["files"] == "hevc_long") - - -# ---------------------------------------------------------------------------------------- -# data - - -def resolve_files(dali_extra, name): - # absolute paths: they are used as symlink targets in a temporary directory - root = os.path.abspath(os.path.join(dali_extra, VIDEO_DIR)) - files, missing = [], [] - for pattern in FILES[name]: - matches = sorted(glob.glob(os.path.join(root, pattern))) - if not matches: - missing.append(pattern) - files += matches - if missing: - raise FileNotFoundError( - f"File set '{name}': no files match {missing} under '{root}'. " - f"Check that --dali_extra (or DALI_EXTRA_PATH) points to a DALI_extra checkout." - ) - return files - - -def materialize(cfg, dali_extra, workdir): - """Creates the inputs of a configuration. Returns one input spec per reader. - - A spec is either {"filenames": [...]} or {"file_list": path}. The result is fully - determined by the configuration, so repeated runs (and --hash) see the same samples. - """ - files = resolve_files(dali_extra, cfg["files"]) - kind, _, n = cfg["layout"].partition(":") - if kind == "plain": - if cfg["readers"] != 1: - raise ValueError(f"layout='plain' requires readers=1; got readers={cfg['readers']}.") - return [{"filenames": files}] - if kind not in ("many", "alternate", "same") or not n.isdigit() or int(n) < 1: - raise ValueError( - f"Unknown layout {cfg['layout']!r}; expected 'plain', 'many:N', 'alternate:N' " - f"or 'same:N' with a positive integer N." - ) - n = int(n) - seq = cfg["seq"] - tag = "_".join(str(cfg[k]) for k in ("files", "layout", "seq", "readers")) - out_dir = os.path.join(workdir, tag.replace(":", "-")) - os.makedirs(out_dir, exist_ok=True) - lines = [] - if kind == "many": - for i in range(n * cfg["readers"]): - src = files[i % len(files)] - dst = os.path.join(out_dir, f"{i:05d}_{os.path.basename(src)}") - if not os.path.lexists(dst): - os.symlink(src, dst) - lines.append((dst, i)) - elif kind == "alternate": - lines = [(files[i % len(files)], i) for i in range(n)] - else: # same - lines = [(files[0], i) for i in range(n)] - specs = [] - per_reader = len(lines) // cfg["readers"] - for r in range(cfg["readers"]): - path = os.path.join(out_dir, f"file_list_{r}.txt") - with open(path, "w") as f: - for name, label in lines[r * per_reader : (r + 1) * per_reader]: - # frames [0, seq) (file_list_include_end=False): exactly one sequence, starting - # at frame 0, a keyframe, so no sample decodes frames it doesn't return - f.write(f"{name} {label} 0 {seq}\n") - specs.append({"file_list": path}) - return specs - - -def prewarm(paths): - """Reads files once so that every run sees a warm page cache.""" - for p in sorted(set(paths)): - with open(p, "rb") as f: - while f.read(1 << 24): - pass - - -# ---------------------------------------------------------------------------------------- -# GPU monitoring - - -def nvml_handle(device_id): - """NVML handle of the CUDA device `device_id`, honoring CUDA_VISIBLE_DEVICES. - - Assumes CUDA_DEVICE_ORDER=PCI_BUS_ID or a machine where CUDA and NVML enumerate - devices the same way. Returns (pynvml, handle) or raises. - """ - import pynvml - - pynvml.nvmlInit() - visible = os.environ.get("CUDA_VISIBLE_DEVICES") - if visible: - entry = visible.split(",")[device_id].strip() - if entry.isdigit(): - return pynvml, pynvml.nvmlDeviceGetHandleByIndex(int(entry)) - # older nvidia-ml-py versions take and return bytes, newer ones accept both - return pynvml, pynvml.nvmlDeviceGetHandleByUUID(entry.encode()) - return pynvml, pynvml.nvmlDeviceGetHandleByIndex(device_id) - - -def other_gpu_pids(nvml, handle): - procs = nvml.nvmlDeviceGetComputeRunningProcesses(handle) - procs += nvml.nvmlDeviceGetGraphicsRunningProcesses(handle) - return sorted({p.pid for p in procs if p.pid != os.getpid()}) - - -class GpuMonitor: - """Samples per-process GPU memory and collects per-process NVDEC utilization. - - While sampling, it also records any other process that uses the GPU, so that a - competing process that starts and exits during a run is detected. - """ - - def __init__(self, device_id, period=0.1): - self.errors = [] # NVML problems; a metric may be missing, the run is still valid - self.period = period - self.mem = [] - self.others = set() - self._stop = threading.Event() - self._thread = threading.Thread(target=self._run, daemon=True) - try: - self.nvml, self.handle = nvml_handle(device_id) - except Exception as e: - self.nvml = None - self._error(f"NVML unavailable: {e}") - - def _error(self, msg): - if msg not in self.errors: - self.errors.append(msg) - - def snapshot(self): - """Pre/post-run state: other processes, NVDEC utilization.""" - if not self.nvml: - return {} - nvml, h = self.nvml, self.handle - res = {} - queries = dict( - other_pids=lambda: other_gpu_pids(nvml, h), - dev_nvdec_util=lambda: nvml.nvmlDeviceGetDecoderUtilization(h)[0], - ) - for key, query in queries.items(): - try: # older NVML bindings may lack some queries for newer drivers - res[key] = query() - except Exception as e: - self._error(f"NVML {key} failed: {e}") - return res - - def _run(self): - nvml, h, pid = self.nvml, self.handle, os.getpid() - - def own_memory(): - for p in nvml.nvmlDeviceGetComputeRunningProcesses(h): - if p.pid == pid and p.usedGpuMemory is not None: - self.mem.append(p.usedGpuMemory) - - samplers = { - "own memory": own_memory, - "other processes": lambda: self.others.update(other_gpu_pids(nvml, h)), - } - failures = dict.fromkeys(samplers, 0) - while not self._stop.is_set(): - for name, sample in list(samplers.items()): - try: - sample() - failures[name] = 0 - except Exception as e: - # e.g. a process list that changed size during the query; only give up - # on a query that keeps failing (unsupported by these bindings/driver) - failures[name] += 1 - if failures[name] >= 3: - self._error(f"NVML sampling of {name} failed: {e}") - del samplers[name] - time.sleep(self.period) - - def start(self): - self._t_start_us = int(time.time() * 1e6) - if self.nvml: - self._thread.start() - - def stop(self): - if not self.nvml: - return {} - self._stop.set() - self._thread.join() - res = {} - if self.mem: - res["proc_mem_max_mb"] = max(self.mem) / 2**20 - try: - samples = self.nvml.nvmlDeviceGetProcessUtilization(self.handle, self._t_start_us) - dec = [s.decUtil for s in samples if s.pid == os.getpid()] - if dec: - res["proc_nvdec_util"] = statistics.mean(dec) - except Exception: - # per-process utilization is optional: not every driver/binding supports it, or - # there may be no samples yet; the result is still valid without it - res["proc_nvdec_util"] = None - return res - - -# ---------------------------------------------------------------------------------------- -# running - - -def num_frames_dali(path, device, device_id): - """The frames of a file as the DALI reader counts them: its sequences of one frame.""" - import nvidia.dali.fn as fn - from nvidia.dali.pipeline import pipeline_def - - @pipeline_def(batch_size=1, num_threads=1, device_id=device_id) - def count(): - return fn.experimental.readers.video( - filenames=[path], sequence_length=1, device=device, name="Count" + for t in threads: + yield "lanes", dict(base, files=CLIPS, many=256, t=t) + yield "cpu", dict(base, files=VPX, many=128, t=t, device="cpu", batch=8, seq=16) + for name, files in (("seq", CLIPS), ("random", CLIPS), ("random_hevc", HEVC_LONG)): + for t in sorted({1, t_max}): + yield name, dict(base, files=files, t=t, shuffle=name != "seq") + for rd in (1, 2, 4): + yield "reader_depth", dict(base, files=CLIPS, many=256, t=8, rd=rd, batch=1) + for name, files in (("single_h264", H264_LONG), ("single_hevc", HEVC_LONG)): + yield name, dict(base, files=files, t=4, batch=1, seq=64) + # checksums only: several decode lanes with two batches in the reader's queue + for name, extra in (("lanes", dict(many=256)), ("seq", {}), ("random", dict(shuffle=True))): + yield name, dict(base, files=CLIPS, t=t_max, rd=2, hash_only=True, **extra) + + +def configs(threads, torchcodec=True): + cfgs = list(dali_configs(threads)) + yield from cfgs + for group, c in cfgs if torchcodec else (): + if c["device"] == "gpu" and group != "reader_depth" and not c.get("hash_only"): + yield group + "_torchcodec", dict(c, lib="torchcodec") + + +def key(group, c): + return f"{group} t={c['t']} rd={c['rd']}" + + +def torchcodec_step(cfg, samples): + """Returns a function that decodes a batch of (path, start, end) samples with torchcodec on + min(t, batch) threads, each keeping its decoder while consecutive samples are of one file.""" + import torch + from torchcodec import decoders + + if cfg["shuffle"]: + random.Random(0).shuffle(samples) + batches, local, keep = itertools.cycle(samples), threading.local(), [] + pool = ThreadPoolExecutor(min(cfg["t"], cfg["batch"])) + # FFmpeg's NVDEC, as the default CUDA backend differs between torchcodec versions + backend = getattr(decoders, "set_cuda_backend", None) + + def decode(sample): + path, first, end = sample + if getattr(local, "path", None) != path: + torch.cuda.set_device(0) + with backend("ffmpeg") if backend else contextlib.nullcontext(): + local.decoder = decoders.VideoDecoder(path, device="cuda:0") + local.path = path + frames = local.decoder.get_frames_in_range(first, end).data + assert len(frames) == end - first, f"{path}[{first}:{end}]: {len(frames)} frames" + return frames + + def run_batch(): + keep[:] = pool.map(decode, [next(batches) for _ in range(cfg["batch"])]) # as DALI's + torch.cuda.synchronize() + + torch.zeros(1, device="cuda") # the CUDA context, before the clock starts + return run_batch + + +def child(cfg, root, hash_out): + """Runs one configuration in this process: {"fps", "build_s"}, or {"hash"}.""" + import numpy as np + + files = sorted(f for p in cfg["files"] for f in glob.glob(os.path.join(root, VIDEO_DIR, p))) + assert files, f"no files for {cfg['files']} under {root}/{VIDEO_DIR}" + samples = [] # (path, start, end) + if cfg.get("many"): # distinct symlinks: nothing keyed by the file name is shared + tmp = tempfile.mkdtemp() + lines = [] + for i in range(cfg["many"]): + link = os.path.join(tmp, f"{i}_{os.path.basename(files[i % len(files)])}") + os.symlink(files[i % len(files)], link) + lines.append(f"{link} {i} 0 {cfg['seq']}\n") # frames [0, seq) + samples.append((link, 0, cfg["seq"])) + with open(os.path.join(tmp, "list.txt"), "w") as f: + f.writelines(lines) + src = dict( + file_list=os.path.join(tmp, "list.txt"), + file_list_format="frames", + file_list_include_end=False, ) + else: + src = dict(filenames=files, labels=list(range(len(files)))) - p = count() - p.build() - return p.reader_meta("Count")["epoch_size"] - - -def build_pipeline(cfg, specs, device_id, seed, frame_num=False): - import nvidia.dali.fn as fn - from nvidia.dali.pipeline import pipeline_def - - legacy = cfg["backend"] == "dali_legacy" - reader = fn.readers.video if legacy else fn.experimental.readers.video - if cfg["backend"] == "dali_decoder": - - def num_frames(path): - return num_frames_dali(path, cfg["device"], device_id) - - # the samples the DALI reader would produce, (path, start, end), one list per reader - samples = [sample_list([s], cfg["seq"], False, num_frames) for s in specs] - # moved out of experimental in DALI 2.0 - decoder = getattr(fn.decoders, "video", None) or fn.experimental.decoders.video + t0 = time.perf_counter() + if cfg.get("lib") == "torchcodec": + if not cfg.get("many"): # whole files in full sequences, as the DALI reader splits them + from torchcodec.decoders import VideoDecoder - @pipeline_def( - batch_size=cfg["batch"], - num_threads=cfg["threads"], - device_id=device_id, - prefetch_queue_depth=cfg["depth"], - seed=seed, - ) - def pipe(): - outs = [] - if cfg["backend"] == "dali_decoder": - for r, reader_samples in enumerate(samples): - # whole files, with the sample's first frame as the label - encoded, start = fn.readers.file( - files=[path for path, _, _ in reader_samples], - labels=[start for _, start, _ in reader_samples], - random_shuffle=cfg["shuffle"], - prefetch_queue_depth=cfg["reader_depth"], - name=f"Reader{r}", - ) - video = decoder( - encoded, - device="mixed" if cfg["device"] == "gpu" else "cpu", - start_frame=start[0], + for f in files: + for first in range(0, len(VideoDecoder(f)) - cfg["seq"] + 1, cfg["seq"]): + samples.append((f, first, first + cfg["seq"])) + run_batch = torchcodec_step(cfg, samples) + else: + import nvidia.dali.fn as fn + from nvidia.dali.pipeline import pipeline_def + + @pipeline_def(batch_size=cfg["batch"], num_threads=cfg["t"], device_id=0, seed=42) + def pipe(): + return tuple( + fn.experimental.readers.video( + device=cfg["device"], sequence_length=cfg["seq"], + random_shuffle=cfg["shuffle"], + prefetch_queue_depth=cfg["rd"], + **src, ) - outs += [video, start] - return tuple(outs) - for r, spec in enumerate(specs): - kw = dict(spec) - if "filenames" in kw: - kw["labels"] = list(range(len(kw["filenames"]))) - elif legacy: - kw["file_list_frame_num"] = True # frame numbers, not timestamps; end exclusive - else: - kw["file_list_format"] = "frames" - kw["file_list_include_end"] = False # explicit: the default changed over time - if frame_num: - kw["enable_frame_num"] = True - outs += reader( - device=cfg["device"], - sequence_length=cfg["seq"], - random_shuffle=cfg["shuffle"], - prefetch_queue_depth=cfg["reader_depth"], - name=f"Reader{r}", - **kw, ) - return tuple(outs) - - return pipe() - - -def warm_up_cuda(device_id): - """Initializes CUDA and the executor so build_s measures the reader, not the runtime.""" - import nvidia.dali.fn as fn - from nvidia.dali.pipeline import pipeline_def - - @pipeline_def(batch_size=1, num_threads=1, device_id=device_id) - def tiny(): - return fn.random.uniform(shape=[1]).gpu() - p = tiny() - p.build() - p.run() - - -class DaliRunner: - """The DALI reader. run() returns the (frames, height, width) of every sample.""" - - def __init__(self, cfg, specs, args): - import nvidia.dali as dali - - self.cfg = cfg - self.version = ( - dali.__version__ - + { - "dali": "", - "dali_legacy": " (fn.readers.video)", - "dali_decoder": " (fn.readers.file + video decoder)", - }[cfg["backend"]] + p = pipe() + p.build() + run_batch = p.run + build_s = time.perf_counter() - t0 + + if hash_out: # from batch 0: the same samples whatever the queue depths + h = hashlib.sha1() + for _ in range(-(-HASH_MIN_SAMPLES // cfg["batch"]) + 2): + for o in run_batch(): + o = o.as_cpu() if hasattr(o, "as_cpu") else o + for i in range(len(o)): + h.update(np.ascontiguousarray(np.array(o[i])).tobytes()) + return dict(hash=h.hexdigest()) + # batches that may have been produced before the clock starts: pipeline + reader queues + for _ in range(2 + cfg["rd"] + 1): + run_batch() + n, t0 = 0, time.perf_counter() + while n < 20 or time.perf_counter() - t0 < 5: # >= 20 batches and >= 5 s + run_batch() + n += 1 + return dict(fps=n * cfg["batch"] * cfg["seq"] / (time.perf_counter() - t0), build_s=build_s) + + +def check_scaling(out, tolerance=0.10): + """More threads, or a deeper reader queue, must not be slower than fewer (flat is fine).""" + ok = True + for group in sorted({v["group"] for v in out.values() if "fps" in v}): + rows = sorted((v["t"], v["rd"], v["fps"]) for v in out.values() if v["group"] == group) + if group.endswith("_torchcodec") or len(rows) < 2: + continue + (t0, rd0, slow), (t1, rd1, fast) = rows[0], rows[-1] + bad = fast < (1 - tolerance) * slow + ok &= not bad + print( + f"{'FAIL' if bad else 'ok '} {group}: {slow:.0f} frames/s (t={t0}, rd={rd0})" + f" -> {fast:.0f} (t={t1}, rd={rd1})" ) - # up to `depth` (pipeline queue) + `reader_depth` (the reader's own queue) batches - # may be produced before the clock starts - self.prefetched = cfg["depth"] + cfg["reader_depth"] - warm_up_cuda(args.device_id) - self.pipe = build_pipeline(cfg, specs, args.device_id, args.seed) - - def build(self): - self.pipe.build() - - def run(self): - out = self.pipe.run() - # shapes are host-side metadata; video outputs are every other output (video, label) - return [tuple(s[:3]) for r in range(self.cfg["readers"]) for s in out[2 * r].shape()] - - -def sample_list(specs, seq, shuffle, num_frames): - """The samples the DALI reader produces for `specs`: (path, start, end) tuples. - - Whole files are split into full sequences of `seq` frames (the reader's default step - and stride); a file list gives one sample per line. DALI shuffles with its own seed, so - a shuffled list has the same samples in a different order. - """ - (spec,) = specs - samples = [] - if "filenames" in spec: - for path in spec["filenames"]: - n = num_frames(path) - samples += [(path, s, s + seq) for s in range(0, n - seq + 1, seq)] - else: - with open(spec["file_list"]) as f: - for line in f: - path, _, start, end = line.split() - samples.append((path, int(start), int(end))) - if shuffle: - random.Random(0).shuffle(samples) - return samples - - -class ReferenceRunner: - """PyNvVideoCodec or torchcodec, called directly through their own APIs. - - Each sample of a batch is decoded by one of `threads` threads; every thread keeps its - own decoder while consecutive samples come from the same file. Outputs are what the - libraries return; torch is used only to wait for the GPU. - """ - - prefetched = 0 # a batch is decoded when it is requested - - def __init__(self, cfg, specs, args): - import itertools - from concurrent.futures import ThreadPoolExecutor - - import torch - - self.cfg, self.torch = cfg, torch - self.backend, self.gpu = cfg["backend"], cfg["device"] == "gpu" - self.device_id = args.device_id - if self.gpu: - torch.cuda.set_device(self.device_id) - torch.zeros(1, device="cuda") # CUDA context, so build_s measures the decoders - torch.cuda.synchronize() - if self.backend == "pynvvideocodec": - import PyNvVideoCodec as nvc - - self.nvc, self.version = nvc, f"PyNvVideoCodec {nvc.__version__}" - else: - import torchcodec - - self.version = f"torchcodec {torchcodec.__version__}" - self.version += f", torch {torch.__version__}" - self.local = threading.local() - # torchcodec on the GPU: whether a decoder reported decoding on the CPU instead - # (None: not known, e.g. older torchcodec without VideoDecoder.cpu_fallback) - self.cpu_fallback = None - if self.gpu and self.backend != "pynvvideocodec": - from torchcodec.decoders import VideoDecoder - - self.cpu_fallback = False if hasattr(VideoDecoder, "cpu_fallback") else None - samples = sample_list(specs, cfg["seq"], cfg["shuffle"], self._num_frames) - self.samples = itertools.cycle(samples) - # no more threads than samples per batch: an idle extra thread would take the next - # batch's samples of the same file and make its decoder seek - workers = min(cfg["threads"], cfg["batch"]) - self.pool = ThreadPoolExecutor(workers) if workers > 1 else None - - def _open(self, path): - if self.backend == "pynvvideocodec": - return self.nvc.SimpleDecoder( - path, - gpu_id=self.device_id, - use_device_memory=True, - output_color_type=self.nvc.OutputColorType.RGB, # interleaved, as DALI + return ok + + +def run(a): + todo = [ + (group, c) + for group, c in configs(a.threads, not (a.hash or a.no_torchcodec)) + if a.only in group and (a.hash or not c.get("hash_only")) + ] + out = {} + for group, c in todo: + reference = c.get("lib") == "torchcodec" # never fails the run + reps = [] + for _ in range(1 if a.hash else a.repeat): + r = subprocess.run( + [sys.executable, __file__, "--child", json.dumps(c), "--dali_extra", a.dali_extra] + + ["--hash"] * a.hash, + capture_output=True, + text=True, ) - from torchcodec import decoders - - if not self.gpu: - return decoders.VideoDecoder(path, device="cpu") - # the CUDA backend is chosen explicitly, as the default changed: FFmpeg's NVDEC support - # up to torchcodec 0.11 (the only one before 0.8), torchcodec's own NVDEC decoder later - # ("nvdec", called "beta" before, which later versions still accept) - if not hasattr(decoders, "set_cuda_backend"): - return decoders.VideoDecoder(path, device=f"cuda:{self.device_id}") - backend = "ffmpeg" if self.backend == "torchcodec" else "beta" - with decoders.set_cuda_backend(backend): - return decoders.VideoDecoder(path, device=f"cuda:{self.device_id}") - - def _num_frames(self, path): - return len(self._decoder(path)) - - def _decoder(self, path): - """This thread's decoder, switched to `path`.""" - local = self.local - if getattr(local, "path", None) == path: - return local.decoder - if self.gpu: - self.torch.cuda.set_device(self.device_id) # this thread's current device - if self.backend == "pynvvideocodec" and getattr(local, "decoder", None) is not None: - local.decoder.reconfigure_decoder(path) # picks a cached NVDEC decoder if any - else: - local.decoder = self._open(path) - local.path = path - return local.decoder - - def _decode(self, sample): - """Returns the sample's (frames, height, width) and the library's output.""" - path, start, end = sample - decoder = self._decoder(path) - # the libraries' own outputs, not converted or copied - if self.backend == "pynvvideocodec": - frames = decoder.get_batch_frames_by_index(list(range(start, end))) - return (len(frames),) + tuple(frames[0].shape[:2]), frames # frames of H, W, C - video = decoder.get_frames_in_range(start, end).data - if self.gpu and getattr(decoder, "cpu_fallback", None): # torchcodec >= 0.11 - self.cpu_fallback = True - return (video.shape[0], video.shape[2], video.shape[3]), video # F, C, H, W - - def build(self): - pass - - def run(self): - # The outputs of a batch are kept until the next run(), as Pipeline.run() outputs, so - # that the process memory includes a whole batch (DALI's also includes its queues) - self.outputs = None - batch = [next(self.samples) for _ in range(self.cfg["batch"])] - decoded = list( - self.pool.map(self._decode, batch) if self.pool else map(self._decode, batch) - ) - if self.gpu: - self.torch.cuda.synchronize() # as Pipeline.run(): outputs are ready - self.outputs = [out for _, out in decoded] - return [shape for shape, _ in decoded] - - -def make_runner(cfg, specs, args): - dali = cfg["backend"] in ("dali", "dali_legacy", "dali_decoder") - return (DaliRunner if dali else ReferenceRunner)(cfg, specs, args) - - -def percentile(sorted_vals, q): - if len(sorted_vals) == 1: - return sorted_vals[0] - return statistics.quantiles(sorted_vals, n=100, method="inclusive")[q - 1] - - -def run_one(cfg, specs, args): - """Runs one configuration in this process and returns its metrics.""" - mon = GpuMonitor(args.device_id) - res = dict(pre=mon.snapshot()) - - runner = make_runner(cfg, specs, args) # initializes CUDA before the clock starts - res["backend_version"] = runner.version - t0 = time.perf_counter() # build_s: the DALI reader (indexing), not the CUDA runtime - runner.build() - if isinstance(runner, DaliRunner): # other libraries open their decoders lazily, in run() - res["build_s"] = time.perf_counter() - t0 - t0 = time.perf_counter() - runner.run() - res["first_batch_ms"] = (time.perf_counter() - t0) * 1e3 - res["shape"] = list(runner.run()[0]) - - iters = cfg.get("iters", args.iters) - for _ in range(args.warmup): - runner.run() - # Up to runner.prefetched batches (DALI: `depth` in the pipeline queue + `reader_depth` - # in the reader's own queue) may have been produced before the clock started. They are - # excluded, and --iters/--min_time apply to the remaining window. - skip = runner.prefetched - stamps, frames = [], [] - mon.start() - t_start = time.perf_counter() - while True: - shapes = runner.run() # actual (frames, height, width) of every sample - stamps.append(time.perf_counter()) - frames.append(sum(s[0] for s in shapes)) - kept = len(stamps) - skip - if kept > 0 and kept >= iters: - t_ref = stamps[skip - 1] if skip else t_start - if stamps[-1] - t_ref >= args.min_time: + if r.returncode: + reps = None + print(f"FAILED {key(group, c)}{' (reference)' if reference else ''}:") + print(r.stderr[-1500:]) break - res.update(mon.stop()) - span = stamps[-1] - t_ref - n = len(stamps) - skip - times_ms = sorted((b - a) * 1e3 for a, b in zip([t_ref] + stamps[skip:], stamps[skip:])) - res.update( - iters=n, - iters_skipped=skip, - frames_per_s=sum(frames[skip:]) / span, - ms_per_sample=span * 1e3 / (n * cfg["readers"] * cfg["batch"]), - iter_p50_ms=statistics.median(times_ms), - iter_p90_ms=percentile(times_ms, 90), - iter_max_ms=times_ms[-1], - ) - if getattr(runner, "cpu_fallback", None) is not None: - res["cpu_fallback"] = runner.cpu_fallback - res["post"] = mon.snapshot() - if mon.errors: # after the last NVML query, so that every failure is recorded - res["nvml_error"] = "; ".join(mon.errors) - pre, post = res["pre"], res["post"] - res["other_pids_seen"] = sorted( - mon.others | set(pre.get("other_pids", [])) | set(post.get("other_pids", [])) - ) - res["contaminated"] = bool(res["other_pids_seen"] or pre.get("dev_nvdec_util", 0) > 5) - return res - - -def hash_one(cfg, specs, args): - """Returns one checksum per sample of the first batches (see HASH_MIN_SAMPLES and - --hash_epochs).""" - pipe = build_pipeline(cfg, specs, args.device_id, args.seed, frame_num=True) - pipe.build() - hashes = [] - # Not dependent on threads or reader_depth, so that --compare --ignore can match runs - batches = -(-HASH_MIN_SAMPLES // (cfg["batch"] * cfg["readers"])) # ceil - if args.hash_epochs: - epoch = max(pipe.epoch_size(f"Reader{r}") for r in range(cfg["readers"])) - batches = max(batches, -(-args.hash_epochs * epoch // cfg["batch"]) + 1) - for _ in range(batches): - outs = [_as_cpu(t) for t in pipe.run()] - for r in range(cfg["readers"]): - video, label, frame_num = outs[3 * r : 3 * r + 3] - for i in range(len(video)): - h = hashlib.sha1(video.at(i).tobytes()) - h.update(label.at(i).tobytes()) - h.update(frame_num.at(i).tobytes()) - hashes.append(h.hexdigest()) - return dict(hashes=hashes, hash_batches=batches) - - -def _as_cpu(tensor_list): - return tensor_list.as_cpu() if hasattr(tensor_list, "as_cpu") else tensor_list - - -def child_main(args): - job = json.loads(args._single) - cfg, specs = job["cfg"], job["specs"] - try: - res = hash_one(cfg, specs, args) if args.hash else run_one(cfg, specs, args) - except Exception as e: - lines = [ln for ln in str(e).strip().splitlines() if ln.strip()] - detail = [ln for ln in lines if "rror" in ln or "ssert" in ln or "failed" in ln] - res = dict(error=f"{type(e).__name__}: " + " | ".join((detail or lines)[-3:])) - print("RESULT " + json.dumps(res)) - - -def run_child(cfg, specs, args): - cmd = [sys.executable, os.path.abspath(__file__), "--_single"] - cmd += [json.dumps(dict(cfg=cfg, specs=specs))] - for flag in ("dali_extra", "device_id", "iters", "warmup", "min_time", "seed", "hash_epochs"): - cmd += [f"--{flag}", str(getattr(args, flag))] - cmd += ["--hash"] if args.hash else [] - try: - p = subprocess.run(cmd, capture_output=True, text=True, timeout=args.timeout) - except subprocess.TimeoutExpired: - return dict(error=f"timeout after {args.timeout} s") - for line in reversed(p.stdout.splitlines()): - if line.startswith("RESULT "): - return json.loads(line[len("RESULT ") :]) - tail = [ln for ln in (p.stderr or p.stdout).splitlines() if ln.strip()][-15:] - detail = [ln for ln in tail if "rror" in ln or "ssert" in ln] or tail[-3:] - return dict(error=f"exit code {p.returncode}: " + " | ".join(detail[-3:])) - - -def aggregate(cfg, reps): - """Median and spread over repetitions.""" - ok = [r for r in reps if "error" not in r] - res = dict(cfg) - res["repeats"] = len(reps) - if not ok: - res["error"] = reps[-1]["error"] if reps else "no runs" - return res - res["errors"] = [r["error"] for r in reps if "error" in r] - res["contaminated"] = any(r.get("contaminated") for r in ok) - if any("cpu_fallback" in r for r in ok): - res["cpu_fallback"] = any(r.get("cpu_fallback") for r in ok) - res["runs"] = ok - for key in ( - "frames_per_s", - "ms_per_sample", - "iter_p50_ms", - "iter_p90_ms", - "iter_max_ms", - "build_s", - "first_batch_ms", - "proc_nvdec_util", - "proc_mem_max_mb", - ): - vals = [r[key] for r in ok if r.get(key) is not None] - if vals: - res[key] = statistics.median(vals) - if key in ("frames_per_s", "build_s", "first_batch_ms") and len(vals) > 1: - res[key + "_spread_pct"] = (max(vals) - min(vals)) / res[key] * 100 - res["shape"] = ok[0].get("shape") - for key in ("backend_version", "nvml_error"): - if ok[0].get(key): - res[key] = ok[0][key] - return res - - -# ---------------------------------------------------------------------------------------- -# reporting - - -COLUMNS = [ - ("scenario", "scenario", "{:<13}"), - ("backend", "backend", "{:<16}"), - ("files", "files", "{:<17}"), - ("layout", "layout", "{:<12}"), - ("device", "dev", "{:<3}"), - ("batch", "bs", "{:>2}"), - ("seq", "seq", "{:>3}"), - ("threads", "thr", "{:>3}"), - ("reader_depth", "rq", "{:>2}"), - ("readers", "rd", "{:>2}"), - ("shuffle", "sh", "{:<2}"), - ("frames_per_s", "frames/s", "{:>8.1f}"), - ("frames_per_s_spread_pct", "spr%", "{:>4.1f}"), - ("ms_per_sample", "ms/smp", "{:>7.2f}"), - ("iter_p50_ms", "p50 ms", "{:>8.2f}"), - ("iter_p90_ms", "p90 ms", "{:>8.2f}"), - ("iter_max_ms", "max ms", "{:>8.2f}"), - ("build_s", "build s", "{:>7.3f}"), - ("first_batch_ms", "1st ms", "{:>7.1f}"), - ("proc_nvdec_util", "dec%", "{:>4.0f}"), - ("proc_mem_max_mb", "mem MB", "{:>6.0f}"), -] - - -def _width(fmt): - return int(fmt[3:].split(".")[0].rstrip("}")) - - -def print_header(): - print(" ".join(head.rjust(_width(fmt)) for _, head, fmt in COLUMNS) + " flags") - - -def format_row(r): - cells = [] - for key, head, fmt in COLUMNS: - v = r.get(key) - if key == "shuffle": - v = "y" if v else "n" - width = _width(fmt) - try: - cells.append(fmt.format(v) if v is not None else "-".rjust(width)) - except (ValueError, TypeError): - cells.append(str(v).ljust(width)) - flags = [] - if r.get("contaminated"): - flags.append("CONTAMINATED") - if r.get("frames_per_s_spread_pct", 0) > 3: - flags.append("spread>3%") - if r.get("cpu_fallback"): - flags.append("CPU fallback") # reported by torchcodec (>= 0.11) - if r.get("backend") in LEAD_IN_BACKENDS and r.get("files") in LEAD_IN_FILES: - flags.append("lead-in frames") # not the same frames as DALI, see LEAD_IN_FILES - if r.get("errors"): - flags.append(f"{len(r['errors'])} failed run(s)") - if r.get("error"): - flags.append("ERROR: " + r["error"][:160]) - return " ".join(cells) + (" " + ", ".join(flags) if flags else "") - - -def provenance(args): - info = dict( - time=time.strftime("%Y-%m-%d %H:%M:%S"), - host=socket.gethostname(), - python=sys.version.split()[0], - cpu_count=os.cpu_count(), - argv=sys.argv, - env={k: v for k, v in os.environ.items() if k.startswith("DALI_") or k.startswith("CUDA_")}, - ) - try: - import nvidia.dali as dali - - info["dali_version"] = dali.__version__ - info["dali_git_sha"] = getattr(dali, "__git_sha__", None) - info["dali_path"] = os.path.dirname(dali.__file__) - except Exception as e: - info["dali_error"] = str(e) - from importlib import metadata - - for pkg in ("PyNvVideoCodec", "torchcodec", "torch"): - try: - info.setdefault("packages", {})[pkg] = metadata.version(pkg) - except metadata.PackageNotFoundError: - continue # optional: needed only by the --backends that use it - try: - nvml, h = nvml_handle(args.device_id) - except Exception as e: - info["nvml_error"] = str(e) - return info - queries = dict( - gpu=lambda: nvml.nvmlDeviceGetName(h), - gpu_uuid=lambda: nvml.nvmlDeviceGetUUID(h), - driver=lambda: nvml.nvmlSystemGetDriverVersion(), - other_gpu_pids=lambda: other_gpu_pids(nvml, h), - nvdec_util_at_start=lambda: nvml.nvmlDeviceGetDecoderUtilization(h)[0], - ) - for key, query in queries.items(): - try: # one unsupported query must not hide the others - value = query() - info[key] = value.decode() if isinstance(value, bytes) else value - except Exception as e: - info.setdefault("nvml_query_errors", {})[key] = str(e) - return info - - -def compare(a_path, b_path, ignore): - """Compares per-sample hashes of matching configurations in two --hash files. - - Fields listed in `ignore` (e.g. threads, reader_depth) are left out when matching, - so a configuration in A is compared against every configuration in B that differs - only in those fields. The output must not depend on them. - """ - with open(a_path) as fa, open(b_path) as fb: - a, b = json.load(fa), json.load(fb) - skip = {"hashes", "scenario", "error", "errors"} | set(ignore) - - def key(r): - return json.dumps({k: v for k, v in r.items() if k not in skip}, sort_keys=True) - - def label(r): - return ", ".join(f"{k}={r[k]}" for k in sorted(r) if k not in ("hashes", "errors", "error")) - - b_runs = {} - for r in b["results"]: - b_runs.setdefault(key(r), []).append(r) - a_keys = {key(r) for r in a["results"]} - bad = compared = 0 - # a configuration that failed or is missing on either side fails the comparison: a - # build that cannot decode a workload must not pass as "identical" - for r in b["results"]: - if key(r) not in a_keys: - bad += 1 - print(f"MISSING in {a_path}: {label(r)}") - failed_runs = set() # (file, index): each failed run is reported and counted once - for path, results in ((a_path, a["results"]), (b_path, b["results"])): - for i, r in enumerate(results): - if "hashes" not in r and (path, i) not in failed_runs: - failed_runs.add((path, i)) - bad += 1 - print(f"FAILED in {path}: {label(r)} ({r.get('error', 'no checksums')})") - for r in a["results"]: - matches = b_runs.get(key(r), []) - if not matches: - bad += 1 - print(f"MISSING in {b_path}: {label(r)}") - for other in matches: - if "hashes" not in r or "hashes" not in other: - continue # reported above - n = max(len(r["hashes"]), len(other["hashes"])) - diff = sum(x != y for x, y in zip(r["hashes"], other["hashes"])) - diff += abs(len(r["hashes"]) - len(other["hashes"])) - compared += 1 - bad += diff > 0 - vs = ", ".join(f"{k}={other[k]}" for k in ignore if other.get(k) != r.get(k)) - status = "OK " if diff == 0 else "DIFF" - print(f"{status} {diff:4d}/{n} samples {label(r)}" + (f" vs {vs}" if vs else "")) - if compared == 0 and bad == 0: - print("nothing to compare: no matching configurations") - return 2 - print("all identical" if bad == 0 else f"{bad} comparison(s) differ, failed or are missing") - return 1 if bad else 0 - - -# ---------------------------------------------------------------------------------------- - - -def parse_args(): - parser = argparse.ArgumentParser( - description="fn.experimental.readers.video benchmark on DALI_extra data", - formatter_class=argparse.RawDescriptionHelpFormatter, - epilog=__doc__, - ) - parser.add_argument("-s", "--scenarios", default="all", help="comma separated, or all") - parser.add_argument( - "-j", "--threads", default="1,2,4,8,16", help="num_threads sweep (B200: 1,2,4,7,8,14,16)" - ) - parser.add_argument("--readers", default="1,2,4,8", help="reader counts for 'engines'") - parser.add_argument( - "-b", - "--backends", - default="dali", - help=f"comma separated, any of {list(BACKEND_SCENARIOS)}; others for reference", - ) - parser.add_argument("-d", "--device_id", default=0, type=int) - parser.add_argument("--repeat", default=3, type=int, help="fresh-process runs per config") - parser.add_argument("--iters", default=20, type=int, help="minimum timed iterations") - parser.add_argument("--min_time", default=5.0, type=float, help="minimum timed seconds") - parser.add_argument("--warmup", default=3, type=int, help="untimed iterations") - parser.add_argument("--timeout", default=900, type=int, help="seconds per run") - parser.add_argument("--seed", default=42, type=int) - parser.add_argument("--dali_extra", default=os.environ.get("DALI_EXTRA_PATH", "")) - parser.add_argument("--json", help="write results (with provenance) to this file") - parser.add_argument("--quick", action="store_true", help="smoke test: tiny sweep, 1 run") - parser.add_argument("--hash", action="store_true", help="record per-sample checksums") - parser.add_argument( - "--hash_epochs", - default=0, - type=int, - help="with --hash: checksum at least this many epochs of every configuration, and " - "one more batch", - ) - parser.add_argument("--compare", nargs=2, metavar=("A", "B"), help="compare two --hash files") - parser.add_argument( - "--ignore", - default="", - help="with --compare: comma separated config fields to ignore when matching runs, " - "e.g. threads,reader_depth to check that output does not depend on them", - ) - parser.add_argument("--_single", help=argparse.SUPPRESS) - args = parser.parse_args() - if args.compare: - return parser, args - if not args.dali_extra: - parser.error("--dali_extra is empty; set DALI_EXTRA_PATH or pass --dali_extra.") - args.dali_extra = os.path.abspath(args.dali_extra) - if args.hash and not args.json and not args._single: - parser.error("--hash requires --json to store the checksums; got no --json.") - args.backends = args.backends.split(",") - unknown = [b for b in args.backends if b not in BACKEND_SCENARIOS] - if unknown: - parser.error( - f"--backends got unknown backend(s) {unknown}; expected a subset of " - f"{list(BACKEND_SCENARIOS)}." - ) - if args.hash and args.backends != ["dali"]: - parser.error(f"--hash supports only the dali backend; got --backends {args.backends}.") - if args.quick: - args.threads, args.readers, args.repeat = "1,4", "1,2", 1 - args.iters, args.min_time, args.warmup = 3, 0.0, 1 - return parser, args + reps.append(json.loads(r.stdout.splitlines()[-1])) + if reps: + out[key(group, c)] = dict(group=group, t=c["t"], rd=c["rd"]) + for k in reps[0]: + out[key(group, c)][k] = ( + reps[0][k] if k == "hash" else statistics.median(x[k] for x in reps) + ) + print(key(group, c), {k: v for k, v in out[key(group, c)].items() if k != "group"}) + failed = any( + key(g, c) not in out for g, c in todo if c.get("lib") != "torchcodec" + ) # a failed or missing configuration + for g in {v["group"] for v in out.values()} if a.hash else (): + # one group, one output, whatever num_threads / reader_depth + if len({v["hash"] for v in out.values() if v["group"] == g}) > 1: + print(f"MISMATCH within group {g}") + failed = True + if a.check and not a.hash: + failed |= not check_scaling(out) + with open(a.json, "w") as f: + json.dump(out, f, indent=1) + return int(failed) + + +def compare(path_a, path_b): + with open(path_a) as f: + a = json.load(f) + with open(path_b) as f: + b = json.load(f) + # a configuration missing on one side is a failure (the torchcodec rows are for reference) + bad = [k for k in a.keys() ^ b.keys() if "_torchcodec" not in k] + for k in sorted(a.keys() & b.keys()): + if "hash" in a[k] and a[k]["hash"] != b[k]["hash"]: + bad.append(k) + if "fps" in a[k]: + ratio = b[k]["fps"] / a[k]["fps"] + print(f"{k:34} {a[k]['fps']:9.1f} -> {b[k]['fps']:9.1f} frames/s x{ratio:.2f}") + print("DIFFERENT / MISSING:", bad if bad else "none") + return int(bool(bad)) def main(): - parser, args = parse_args() - if args.compare: - sys.exit(compare(*args.compare, [f for f in args.ignore.split(",") if f])) - if args._single: - child_main(args) - return - - threads = [int(t) for t in args.threads.split(",")] - readers = [int(r) for r in args.readers.split(",")] - all_scenarios = scenarios(threads, readers, args.quick) - if args.scenarios == "all": - names = [n for n in all_scenarios if args.hash or n not in HASH_ONLY_SCENARIOS] + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawTextHelpFormatter) + ap.add_argument("--json", default="video_reader_bench.json", help="where to write the results") + ap.add_argument("--hash", action="store_true", help="checksums instead of throughput") + ap.add_argument("--check", action="store_true", help="fail if threads / depth slow it down") + ap.add_argument("--compare", nargs=2, metavar=("A", "B"), help="two result files") + ap.add_argument( + "--threads", type=lambda s: [int(x) for x in s.split(",")], default=[1, 2, 4, 8] + ) + ap.add_argument("--repeat", type=int, default=3, help="fresh-process runs per configuration") + ap.add_argument("--only", default="", help="only groups containing this text") + ap.add_argument("--no_torchcodec", action="store_true", help="DALI only") + ap.add_argument("--dali_extra", default=os.environ.get("DALI_EXTRA_PATH", "")) + ap.add_argument("--child", help=argparse.SUPPRESS) + a = ap.parse_args() + if a.child: + print(json.dumps(child(json.loads(a.child), a.dali_extra, a.hash))) + elif a.compare: + sys.exit(compare(*a.compare)) else: - names = args.scenarios.split(",") - unknown = [n for n in names if n not in all_scenarios] - if unknown: - parser.error( - f"--scenarios got unknown scenario(s) {unknown}; expected 'all' or a subset of " - f"{list(all_scenarios)}." - ) - hash_only = [n for n in names if n in HASH_ONLY_SCENARIOS] - if hash_only and not args.hash: - parser.error(f"--scenarios {hash_only} run only with --hash.") - if args.hash: - args.repeat = 1 - - prov = provenance(args) - print("# " + json.dumps(prov)) - if prov.get("nvml_error"): - warnings.warn(f"NVML is unavailable ({prov['nvml_error']}); GPU checks are disabled.") - busy = prov.get("other_gpu_pids") or (prov.get("nvdec_util_at_start") or 0) > 5 - if busy: - details = [] - if prov.get("other_gpu_pids"): - details.append(f"pids {prov['other_gpu_pids']}") - if prov.get("nvdec_util_at_start") is not None: - details.append(f"NVDEC utilization {prov['nvdec_util_at_start']}%") - warnings.warn( - f"The GPU is in use by other processes ({', '.join(details)}); " - f"the results will be skewed." - ) - - workdir = tempfile.mkdtemp(prefix="video_reader_bench_") - results, failed = [], 0 - try: - jobs, not_run = [], set() - for name in names: - for base, backend in [(c, b) for c in all_scenarios[name] for b in args.backends]: - if not backend_runs(backend, name, base): - not_run.add((backend, name, base["threads"])) - continue - cfg = dict(base, scenario=name, backend=backend) - try: - specs = materialize(cfg, args.dali_extra, workdir) - except Exception as e: - jobs.append((cfg, None, f"{type(e).__name__}: {e}")) - continue - jobs.append((cfg, specs, None)) - for backend, name, threads in sorted(not_run): - print(f"# not run: {name} threads={threads} with {backend} (see backend_runs)") - for cfg, specs, err in jobs: - if specs: - paths = [] - for s in specs: - if "filenames" in s: - paths += s["filenames"] - else: - with open(s["file_list"]) as f: - paths += [ln.split()[0] for ln in f] - prewarm(os.path.realpath(p) for p in paths) - reps = {i: [] for i in range(len(jobs))} - rng = random.Random(args.seed) - for rep in range(args.repeat): - order = list(range(len(jobs))) - if rep > 0: - rng.shuffle(order) # round-robin, so drift does not line up with a config - for i in order: - cfg, specs, err = jobs[i] - res = dict(error=err) if err else run_child(cfg, specs, args) - reps[i].append(res) - note = res.get("error") or ( - f"{res['frames_per_s']:.1f} frames/s" if "frames_per_s" in res else "done" - ) - print( - f"[run {rep + 1}/{args.repeat}] {cfg['scenario']} {cfg['backend']} " - f"{cfg['files']} {cfg['layout']} bs={cfg['batch']} thr={cfg['threads']} " - f"rq={cfg['reader_depth']} rd={cfg['readers']}: {note}", - file=sys.stderr, - flush=True, - ) - if not args.hash: - print_header() - for i, (cfg, specs, err) in enumerate(jobs): - if args.hash: - r = dict(cfg, seed=args.seed, **reps[i][0]) # with the hash_batches used - n = len(r.get("hashes", [])) - print( - f"{cfg['scenario']} {cfg['files']} threads={cfg['threads']}: " - f"{n} sample hashes" + (f" ERROR: {r['error']}" if r.get("error") else "") - ) - else: - r = aggregate(cfg, reps[i]) - print(format_row(r)) - # a configuration fails if any of its repetitions failed - failed += bool(r.get("error") or r.get("errors")) - results.append(r) - finally: - shutil.rmtree(workdir, ignore_errors=True) - - if args.json: - with open(args.json, "w") as f: - json.dump(dict(provenance=prov, results=results), f, indent=1) - print(f"wrote {args.json}") - if failed: - print(f"{failed} configuration(s) failed") - sys.exit(1) + sys.exit(run(a)) if __name__ == "__main__": diff --git a/qa/TL1_video_reader_perf/check_results.py b/qa/TL1_video_reader_perf/check_results.py deleted file mode 100644 index 31e6f665232..00000000000 --- a/qa/TL1_video_reader_perf/check_results.py +++ /dev/null @@ -1,193 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Checks the results of internal_tools/video_reader_bench.py (--json output). - -Checks: - * no configuration failed - * relative checks, with a tolerance: - - lanes, seq_longgop, random_access, cpu: throughput at any higher num_threads is - not lower than at the lowest one (more threads must not make the reader slower) - - reader_depth: a deeper reader queue is not slower than depth 1 - * optional: engines with the most readers use the GPU's NVDEC engines: they reach at least - --engines_scaling times the throughput of one reader (GPUs with several NVDEC engines), or, - with --engines_busy, keep NVDEC at least that busy (percent). The latter is for when one - reader already spreads its decode lanes over the engines, so that more readers cannot scale - as much. - * optional absolute minimums (frames/s) for named metrics, passed with --min: - lanes lanes, batch 16, highest num_threads - single_stream single_stream, h264_long - cpu cpu, batch 8, highest num_threads - engines engines, highest number of readers - Metrics without a minimum are printed for reference only. - -Results flagged as contaminated (other processes used the GPU) only produce warnings. -Exits with 1 if any check failed. -""" - -import argparse -import json -import sys -import warnings - - -def by(results, scenario, **match): - """DALI results only; results of other libraries (--backends) are for reference.""" - return [ - r - for r in results - if r.get("scenario") == scenario - and r.get("backend", "dali") == "dali" - and all(r.get(k) == v for k, v in match.items()) - ] - - -def groups(results, scenario, keys): - out = {} - for r in by(results, scenario): - out.setdefault(tuple(r.get(k) for k in keys), []).append(r) - return out - - -def named_metrics(results): - """The metrics that can have absolute minimums.""" - - def at_max(rows, key): - rows = [r for r in rows if r.get("frames_per_s") is not None] - return max(rows, key=lambda r: r[key])["frames_per_s"] if rows else None - - return { - "lanes": at_max(by(results, "lanes", batch=16), "threads"), - "single_stream": next( - (r.get("frames_per_s") for r in by(results, "single_stream", files="h264_long")), None - ), - "cpu": at_max(by(results, "cpu", batch=8), "threads"), - "engines": at_max(by(results, "engines"), "readers"), - } - - -def main(): - parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) - parser.add_argument("results", help="--json output of video_reader_bench.py") - parser.add_argument("--tolerance", type=float, default=10.0, help="percent, relative checks") - parser.add_argument( - "--min", action="append", default=[], metavar="NAME=FPS", help="absolute minimum" - ) - parser.add_argument( - "--engines_scaling", - type=float, - help="minimum speedup of engines with the most readers over one reader", - ) - parser.add_argument( - "--engines_busy", - type=float, - help="NVDEC utilization (percent) of engines with the most readers that passes the " - "engines check even below --engines_scaling (NVDEC is saturated)", - ) - args = parser.parse_args() - - with open(args.results) as f: - results = json.load(f)["results"] - metrics = named_metrics(results) - minimums = {} - for item in args.min: - name, _, value = item.partition("=") - if name not in metrics: - parser.error(f"--min got unknown metric {name!r}; expected one of {list(metrics)}.") - if value: # an empty value means: no minimum, report only - try: - minimums[name] = float(value) - except ValueError: - parser.error(f"--min {name} expects a number of frames/s; got {value!r}.") - tol = args.tolerance / 100 - failed = [] - - def check(name, ok, detail): - print(f"CHECK {name}: {'OK' if ok else 'FAIL'} ({detail})") - if not ok: - failed.append(name) - - # a configuration fails if all its repetitions failed (`error`) or only some (`errors`) - errors = [r for r in results if r.get("error") or r.get("errors")] - check("no failed configurations", not errors, f"{len(errors)} of {len(results)} failed") - for r in errors: - for e in [r["error"]] if r.get("error") else r["errors"]: - print(f" {r.get('scenario')} {r.get('files')} {r.get('layout')}: {e}") - - contaminated = [r for r in results if r.get("contaminated")] - if contaminated: - warnings.warn( - f"{len(contaminated)} of {len(results)} results were measured while other " - f"processes used the GPU; their numbers may be skewed." - ) - - def relative(scenario, keys, param): - for key, rows in sorted(groups(results, scenario, keys).items(), key=str): - rows = sorted((r for r in rows if r.get("frames_per_s")), key=lambda r: r[param]) - if len(rows) < 2: - continue - base = rows[0] - for r in rows[1:]: - ok = r["frames_per_s"] >= (1 - tol) * base["frames_per_s"] - label = "/".join(str(k) for k in key) - check( - f"{scenario} {label} {param}={r[param]} vs {base[param]}", - ok, - f"{r['frames_per_s']:.1f} vs {base['frames_per_s']:.1f} frames/s, " - f"tolerance {args.tolerance:g}%", - ) - - for scenario in ("lanes", "seq_longgop", "random_access", "cpu"): - relative(scenario, ("files", "layout", "batch"), "threads") - relative("reader_depth", ("files", "layout"), "reader_depth") - - if args.engines_scaling is not None: - rows = sorted( - (r for r in by(results, "engines") if r.get("frames_per_s")), key=lambda r: r["readers"] - ) - if len(rows) < 2 or rows[0]["readers"] != 1: - check("engines scaling", False, "needs engines results with 1 and more readers") - else: - one, most = rows[0], rows[-1] - scaling = most["frames_per_s"] / one["frames_per_s"] - busy = most.get("proc_nvdec_util") - saturated = ( - args.engines_busy is not None and busy is not None and busy >= args.engines_busy - ) - detail = f"{scaling:.2f}x, minimum {args.engines_scaling:g}x" - if args.engines_busy is not None: - detail += f"; NVDEC {busy}% busy, saturated at {args.engines_busy:g}%" - check( - f"engines readers={most['readers']} vs 1", - scaling >= args.engines_scaling or saturated, - detail, - ) - - for name, value in metrics.items(): - if name in minimums: - check( - f"{name} minimum", - value is not None and value >= minimums[name], - f"{value} vs min {minimums[name]:g} frames/s", - ) - else: - print(f"REFERENCE {name}: {value} frames/s (no minimum set)") - - print("all checks passed" if not failed else f"{len(failed)} check(s) failed") - sys.exit(1 if failed else 0) - - -if __name__ == "__main__": - main() diff --git a/qa/TL1_video_reader_perf/test.sh b/qa/TL1_video_reader_perf/test.sh index e0921afb6fd..3e0a8e0e85f 100644 --- a/qa/TL1_video_reader_perf/test.sh +++ b/qa/TL1_video_reader_perf/test.sh @@ -1,104 +1,42 @@ #!/bin/bash -e -# used pip packages (the reference libraries are installed by install_reference_libs) -pip_packages='numpy nvidia-ml-py==11.450.51' +# used pip packages (torch and torchcodec are installed by install_torchcodec) +pip_packages='numpy' target_dir=./internal_tools -# One-time pre-step: FFmpeg shared libraries for torchcodec (x86_64 only, see -# install_reference_libs), and nsys (NVIDIA Nsight Systems) +# One-time pre-step: the FFmpeg shared libraries and the shared libpython that torchcodec needs +# (x86_64 only). The torchcodec rows are for reference, so a failure here is not an error. do_once() { - apt update if [ "$(uname -m)" == "x86_64" ]; then - # torchcodec loads the FFmpeg shared libraries and links against the shared libpython - apt install -y --no-install-recommends ffmpeg \ - "libpython$(python -c 'import sys; print("%d.%d" % sys.version_info[:2])')" + (apt update && apt install -y --no-install-recommends ffmpeg \ + "libpython$(python -c 'import sys; print("%d.%d" % sys.version_info[:2])')") \ + || echo "Could not install FFmpeg: no torchcodec rows" fi - if command -v nsys &>/dev/null; then - echo "nsys already installed: $(nsys --version)" - return - fi - DISTRO=$(source /etc/lsb-release && echo "$DISTRIB_RELEASE" | tr -d .) - ARCH=$(dpkg --print-architecture) - apt install -y --no-install-recommends gnupg curl - echo "deb [signed-by=/usr/share/keyrings/nvidia-devtools.gpg] https://developer.download.nvidia.com/devtools/repos/ubuntu${DISTRO}/${ARCH} /" \ - | tee /etc/apt/sources.list.d/nvidia-devtools.list - curl -fsSL "https://developer.download.nvidia.com/devtools/repos/ubuntu${DISTRO}/${ARCH}/nvidia.pub" \ - | gpg --dearmor -o /usr/share/keyrings/nvidia-devtools.gpg - apt update && apt install -y nsight-systems-cli } LOG_PERF="video_reader_perf.log" JSON_PERF="video_reader_perf.json" +LOG_HASH="video_reader_hash.log" JSON_HASH="video_reader_hash.json" -LOG_REF="video_reader_ref.log" -JSON_REF="video_reader_ref.json" -# DALI's legacy fn.readers.video, its video decoder (fn.readers.file + fn.decoders.video) and -# other libraries run for reference on the same samples -# (video_reader_bench.py --backends), on x86_64 only; one short run each, not checked. The DALI -# numbers are in the performance run. The libraries are installed by install_reference_libs. -REF_SCENARIOS="lanes,single_stream,cache_key,cpu" -REF_ARGS="--repeat 1 --iters 4 --warmup 1 --min_time 2 -j 1,16" +function CLEAN_AND_EXIT { + rm -rf ${LOG_PERF} ${JSON_PERF} ${LOG_HASH} ${JSON_HASH} + exit $1 +} -# Installs PyNvVideoCodec, and torch with the matching CUDA-enabled torchcodec from the PyTorch -# index for this CUDA version (x86_64 only), and sets REF_BACKENDS. torch is needed only here, -# so it is pinned here rather than in pip_packages, which would follow the DALI framework tests. -install_reference_libs() { +# Installs torch and the matching CUDA-enabled torchcodec from the PyTorch index for this CUDA +# version (x86_64 only). torch is needed only here, so it is pinned here rather than in +# pip_packages, which would follow the DALI framework tests. A failure is not an error: the +# benchmark then runs without the torchcodec rows (--no_torchcodec). +install_torchcodec() { local cuda_tag case "${CUDA_VERSION}" in 12*) cuda_tag=cu129 ;; 13*) cuda_tag=cu130 ;; - *) echo "No PyTorch index known for CUDA_VERSION=${CUDA_VERSION}; update test.sh"; return 1 ;; + *) echo "No PyTorch index known for CUDA_VERSION=${CUDA_VERSION}"; return 1 ;; esac - pip install PyNvVideoCodec==2.0.2 || return 1 + [ "$(uname -m)" == "x86_64" ] || return 1 pip install "torch==2.11.0+${cuda_tag}" "torchcodec==0.15.0+${cuda_tag}" \ - --index-url "https://download.pytorch.org/whl/${cuda_tag}" || return 1 - REF_BACKENDS="pynvvideocodec,torchcodec,torchcodec_nvdec" - echo "Reference backends: ${REF_BACKENDS}" -} - -# Scenarios of internal_tools/video_reader_bench.py run for performance; see its --help -PERF_SCENARIOS="lanes,engines,single_stream,reader_depth,cache_key,seq_longgop,random_access,cpu" -# Scenarios whose output is checked to be independent of num_threads and reader_depth, over two -# epochs (decoders reused across batches, the epochs wrapping around) -HASH_SCENARIOS="lanes,lanes_depth,reader_depth,cache_key,cpu,seq_longgop,random_access" -HASH_EPOCHS=2 -BENCH_ARGS="--repeat 3 --iters 8 --min_time 3 -j 1,4,16 --readers 1,2,4" -# engines: more readers use more NVDEC engines: 4 readers reach 2.5x one reader, or keep NVDEC -# saturated (one reader's decode lanes already use several engines, e.g. H100 PCIe: 2016 frames/s -# with 1 reader, 3824 at 98% NVDEC with 4, which is 1.9x) -ENGINES_SCALING=2.5 -ENGINES_BUSY=90 - -function CLEAN_AND_EXIT { - rm -rf ${LOG_PERF} ${JSON_PERF} ${JSON_HASH} ${LOG_REF} ${JSON_REF} - exit $1 -} - -# Prints the results (without per-run data) to the log, so that they are kept on success too. -# A missing or unreadable file (e.g. no reference run on GraceHopper) is skipped, not an error. -print_results() { - for json in "$@"; do - if [ -f "${json}" ]; then - python -c "import json, sys -d = json.load(open(sys.argv[1])) -for r in d['results']: - r.pop('runs', None) -print('RESULTS', sys.argv[1], json.dumps(d, separators=(',', ':')))" "${json}" \ - || echo "Could not print ${json}" - fi - done -} - -# Run the benchmark; if NSYS_REP is set, wrap it with nsys profiling (child processes included). -run_bench() { - local log_file="$1" - shift - if [ -n "${NSYS_REP}" ]; then - local profile_name="${log_file%.log}.nsys-rep" - nsys profile -o "${profile_name}" --trace=cuda,nvtx,osrt "$@" | tee "${log_file}" - else - "$@" | tee "${log_file}" - fi + --index-url "https://download.pytorch.org/whl/${cuda_tag}" && python -c "import torchcodec" } test_body() { @@ -107,69 +45,37 @@ test_body() { else TASKSET="taskset --cpu-list 0-71" # GraceHopper fi - # Absolute minimums in frames/s (see qa/TL1_video_reader_perf/check_results.py); empty: - # reported for reference only, not checked. - # TODO: set them per architecture (e.g. 0.85x the REFERENCE values) once a few CI runs - # establish the baseline on these machines - MIN_LANES= - MIN_SINGLE_STREAM= - MIN_CPU= - MIN_ENGINES= + TORCHCODEC_ARGS="" + install_torchcodec || { echo "No torchcodec rows"; TORCHCODEC_ARGS="--no_torchcodec"; } - unset NSYS_REP + # Throughput of every configuration. Failing: a configuration that fails, or more threads + # (or a deeper reader queue) that are more than 10% slower than fewer. The torchcodec rows + # are only reported. PERF_RUN="OK" - run_bench "${LOG_PERF}" ${TASKSET} python video_reader_bench.py ${BENCH_ARGS} \ - -s ${PERF_SCENARIOS} --json ${JSON_PERF} || PERF_RUN="FAIL" - - # The output must not depend on num_threads or on the reader's prefetch_queue_depth: every - # configuration is compared with all that differ from it only in those - HASH_RESULT="OK" - ${TASKSET} python video_reader_bench.py -s ${HASH_SCENARIOS} -j 1,16 --hash \ - --hash_epochs ${HASH_EPOCHS} --json ${JSON_HASH} || HASH_RESULT="FAIL" - python video_reader_bench.py --compare ${JSON_HASH} ${JSON_HASH} \ - --ignore threads,reader_depth || HASH_RESULT="FAIL" + ${TASKSET} python video_reader_bench.py --check ${TORCHCODEC_ARGS} --json ${JSON_PERF} \ + | tee ${LOG_PERF} || PERF_RUN="FAIL" - CHECK_RESULT="OK" - python ${topdir}/qa/TL1_video_reader_perf/check_results.py ${JSON_PERF} --tolerance 10 \ - --engines_scaling ${ENGINES_SCALING} --engines_busy ${ENGINES_BUSY} \ - --min lanes=${MIN_LANES} --min single_stream=${MIN_SINGLE_STREAM} \ - --min cpu=${MIN_CPU} --min engines=${MIN_ENGINES} || CHECK_RESULT="FAIL" - - # The same samples decoded by other libraries, for reference (x86_64 only). A failure here - # (installation or decoding) fails the test, so that the comparison does not silently - # disappear. - REF_RUN="OK" - if [ "$(uname -m)" != "x86_64" ]; then - REF_RUN="SKIPPED" - elif install_reference_libs; then - run_bench "${LOG_REF}" ${TASKSET} python video_reader_bench.py ${REF_ARGS} \ - -s ${REF_SCENARIOS} -b dali_legacy,dali_decoder,${REF_BACKENDS} --json ${JSON_REF} || REF_RUN="FAIL" - else - REF_RUN="FAIL" - fi + # The output must not depend on num_threads or on the reader's prefetch_queue_depth: the + # configurations of a group are compared with each other + HASH_RUN="OK" + ${TASKSET} python video_reader_bench.py --hash --json ${JSON_HASH} \ + | tee ${LOG_HASH} || HASH_RUN="FAIL" echo "PERF_RUN=${PERF_RUN}" - echo "HASH_RESULT=${HASH_RESULT}" - echo "CHECK_RESULT=${CHECK_RESULT}" - echo "REF_RUN=${REF_RUN}" - print_results ${JSON_PERF} ${JSON_REF} + echo "HASH_RUN=${HASH_RUN}" + for json in ${JSON_PERF} ${JSON_HASH}; do + [ -f ${json} ] && echo "RESULTS ${json} $(tr -d ' \n' < ${json})" + done - if [[ "$PERF_RUN" == "OK" && "$HASH_RESULT" == "OK" && "$CHECK_RESULT" == "OK" \ - && "$REF_RUN" != "FAIL" ]]; then + if [[ "$PERF_RUN" == "OK" && "$HASH_RUN" == "OK" ]]; then CLEAN_AND_EXIT 0 fi - # On failure: save the results; if the performance run or its checks failed, also re-run the - # performance scenarios once with nsys and save the profile + # On failure: save the logs and results ARTIFACTS_DIR="${topdir}/core_artifacts" mkdir -p "${ARTIFACTS_DIR}" - if [[ "$PERF_RUN" != "OK" || "$CHECK_RESULT" != "OK" ]]; then - echo "Video reader performance checks failed; re-running with nsys profiling..." - NSYS_REP="enabled" run_bench "${LOG_PERF}" ${TASKSET} python video_reader_bench.py \ - --repeat 1 --min_time 3 -j 1,16 --readers 1,4 -s ${PERF_SCENARIOS} || true - fi - cp -f *.nsys-rep ${JSON_PERF} ${JSON_HASH} ${JSON_REF} "${ARTIFACTS_DIR}/" 2>/dev/null || true - echo "Results (and nsys profile, if any) saved to ${ARTIFACTS_DIR}" + cp -f ${LOG_PERF} ${JSON_PERF} ${LOG_HASH} ${JSON_HASH} "${ARTIFACTS_DIR}/" 2>/dev/null || true + echo "Results saved to ${ARTIFACTS_DIR}" CLEAN_AND_EXIT 1 } pushd ../..