From 409793e44c53106f25fae30469a7e24de4797999 Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Wed, 28 Oct 2026 09:32:38 +0100 Subject: [PATCH 1/4] Convert video to RGB on the CPU with the matrix of the stream The CPU video decoder always used the BT.601 matrix for the source in swscale, while HD video is normally BT.709, which gives visibly wrong colors for saturated colors (e.g. pure red 255,0,0 came out as ~234,0,3). Use the BT.709 matrix when the stream is tagged as such; streams that don't specify it keep using BT.601. Signed-off-by: Joaquin Anton Guirao --- dali/operators/video/frames_decoder_cpu.cc | 10 ++++-- dali/operators/video/frames_decoder_cpu.h | 1 + dali/test/python/decoder/test_video.py | 36 ++++++++++++++++++++++ 3 files changed, 45 insertions(+), 2 deletions(-) diff --git a/dali/operators/video/frames_decoder_cpu.cc b/dali/operators/video/frames_decoder_cpu.cc index 3a0e99ce470..a18c22d00bb 100644 --- a/dali/operators/video/frames_decoder_cpu.cc +++ b/dali/operators/video/frames_decoder_cpu.cc @@ -107,10 +107,15 @@ void FramesDecoderCpu::CopyToOutput(uint8_t *data) { bool src_full_range = frame_->color_range == AVCOL_RANGE_JPEG || (frame_->color_range == AVCOL_RANGE_UNSPECIFIED && codec_params_->color_range == AVCOL_RANGE_JPEG); - if (sws_src_full_range_ != src_full_range) { + // The matrix that the stream says it uses (BT.709 for most HD video); otherwise, BT.601. + AVColorSpace src_colorspace = frame_->colorspace != AVCOL_SPC_UNSPECIFIED + ? frame_->colorspace + : codec_params_->color_space; + int sws_colorspace = src_colorspace == AVCOL_SPC_BT709 ? SWS_CS_ITU709 : SWS_CS_DEFAULT; + if (sws_src_full_range_ != src_full_range || sws_src_colorspace_ != sws_colorspace) { int ret = sws_setColorspaceDetails( sws_ctx_.get(), - sws_getCoefficients(SWS_CS_DEFAULT), + sws_getCoefficients(sws_colorspace), src_full_range, sws_getCoefficients(SWS_CS_DEFAULT), 1, @@ -121,6 +126,7 @@ void FramesDecoderCpu::CopyToOutput(uint8_t *data) { make_string("Could not set color space conversion details: ", av_error_string(ret))); sws_src_full_range_ = src_full_range; + sws_src_colorspace_ = sws_colorspace; } uint8_t *dest[4] = {sws_output_data, nullptr, nullptr, nullptr}; diff --git a/dali/operators/video/frames_decoder_cpu.h b/dali/operators/video/frames_decoder_cpu.h index 3babe99e767..7548959670f 100644 --- a/dali/operators/video/frames_decoder_cpu.h +++ b/dali/operators/video/frames_decoder_cpu.h @@ -65,6 +65,7 @@ class DLL_PUBLIC FramesDecoderCpu : public FramesDecoderBase { std::unique_ptr sws_ctx_{ nullptr, sws_freeContext}; std::optional sws_src_full_range_; + std::optional sws_src_colorspace_; std::vector tmp_buffer_; }; diff --git a/dali/test/python/decoder/test_video.py b/dali/test/python/decoder/test_video.py index ce755d8bc74..8bdd8c51cc4 100644 --- a/dali/test/python/decoder/test_video.py +++ b/dali/test/python/decoder/test_video.py @@ -306,6 +306,42 @@ def test_pipeline(): assert np.mean(absdiff) < 2 +def color_matrix_bars_error(frame, reference): + """Worst difference between `frame` and `reference` of the mean colors of the interiors of the + 8 bars of the test pattern in db/video/color_matrix (the bars are 240 pixels wide)""" + worst = 0 + for i in range(8): + interior = (slice(100, 440), slice(i * 240 + 40, (i + 1) * 240 - 40)) + color = frame[interior].reshape(-1, 3).mean(axis=0) + ref_color = reference[interior].reshape(-1, 3).mean(axis=0) + worst = max(worst, np.max(np.abs(color - ref_color))) + return worst + + +@params("bt709_limited_1080p_vp9") +def test_color_matrix_bt709_cpu(name): + # The streams are tagged as BT.709 (limited range). The references are what ffmpeg decodes, + # which uses the matrix from the stream; converting with BT.601 gives errors up to 22 in the + # interior of the saturated bars. + skip_if_m60() + data_dir = get_dali_extra_path() + "/db/video/color_matrix" + + @pipeline_def + def test_pipeline(): + return fn.experimental.readers.video( + device="cpu", + filenames=[f"{data_dir}/{name}.mp4"], + sequence_length=1, + ) + + out = test_pipeline(batch_size=1, num_threads=1, device_id=None).run()[0] + frame = out.as_array()[0, 0] + ref = cv2.cvtColor(cv2.imread(f"{data_dir}/{name}_ref.png"), cv2.COLOR_BGR2RGB) + assert frame.shape == ref.shape + assert color_matrix_bars_error(frame, ref) <= 4 + assert np.mean(np.abs(frame.astype(int) - ref.astype(int))) < 2 + + @attr("multi_gpu") @params("cpu", "mixed") def test_multi_gpu_video(device): From 599b729540e85a2cfce3b24227fdf9482ccf2770 Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Wed, 28 Oct 2026 11:01:40 +0100 Subject: [PATCH 2/4] Convert to the stream's matrix only for RGB output YCbCr output is a copy of the decoded planes, so the source matrix must stay the one of the destination. Also, run the CPU test on hosts without a GPU and build the paths with os.path.join. Signed-off-by: Joaquin Anton Guirao --- dali/operators/video/frames_decoder_cpu.cc | 6 +++++- dali/test/python/decoder/test_video.py | 7 +++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/dali/operators/video/frames_decoder_cpu.cc b/dali/operators/video/frames_decoder_cpu.cc index a18c22d00bb..21b148d8587 100644 --- a/dali/operators/video/frames_decoder_cpu.cc +++ b/dali/operators/video/frames_decoder_cpu.cc @@ -108,10 +108,14 @@ void FramesDecoderCpu::CopyToOutput(uint8_t *data) { (frame_->color_range == AVCOL_RANGE_UNSPECIFIED && codec_params_->color_range == AVCOL_RANGE_JPEG); // The matrix that the stream says it uses (BT.709 for most HD video); otherwise, BT.601. + // It matters only when converting to RGB: YCbCr output is a copy of the decoded planes, which + // must not be converted to another matrix (the destination one is BT.601). AVColorSpace src_colorspace = frame_->colorspace != AVCOL_SPC_UNSPECIFIED ? frame_->colorspace : codec_params_->color_space; - int sws_colorspace = src_colorspace == AVCOL_SPC_BT709 ? SWS_CS_ITU709 : SWS_CS_DEFAULT; + int sws_colorspace = (image_type_ == DALI_RGB && src_colorspace == AVCOL_SPC_BT709) + ? SWS_CS_ITU709 + : SWS_CS_DEFAULT; if (sws_src_full_range_ != src_full_range || sws_src_colorspace_ != sws_colorspace) { int ret = sws_setColorspaceDetails( sws_ctx_.get(), diff --git a/dali/test/python/decoder/test_video.py b/dali/test/python/decoder/test_video.py index 8bdd8c51cc4..e378f41092e 100644 --- a/dali/test/python/decoder/test_video.py +++ b/dali/test/python/decoder/test_video.py @@ -323,20 +323,19 @@ def test_color_matrix_bt709_cpu(name): # The streams are tagged as BT.709 (limited range). The references are what ffmpeg decodes, # which uses the matrix from the stream; converting with BT.601 gives errors up to 22 in the # interior of the saturated bars. - skip_if_m60() - data_dir = get_dali_extra_path() + "/db/video/color_matrix" + data_dir = os.path.join(get_dali_extra_path(), "db", "video", "color_matrix") @pipeline_def def test_pipeline(): return fn.experimental.readers.video( device="cpu", - filenames=[f"{data_dir}/{name}.mp4"], + filenames=[os.path.join(data_dir, f"{name}.mp4")], sequence_length=1, ) out = test_pipeline(batch_size=1, num_threads=1, device_id=None).run()[0] frame = out.as_array()[0, 0] - ref = cv2.cvtColor(cv2.imread(f"{data_dir}/{name}_ref.png"), cv2.COLOR_BGR2RGB) + ref = cv2.cvtColor(cv2.imread(os.path.join(data_dir, f"{name}_ref.png")), cv2.COLOR_BGR2RGB) assert frame.shape == ref.shape assert color_matrix_bars_error(frame, ref) <= 4 assert np.mean(np.abs(frame.astype(int) - ref.astype(int))) < 2 From 608f37aae2c3a36e30e7f5165a48eb251cfcdbfd Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Wed, 28 Oct 2026 15:27:49 +0100 Subject: [PATCH 3/4] Check the black and white bars of the color matrix test pattern The test pattern in DALI_extra has 10 bars now (black and white were added): use their width in the helper that compares the bars, and check that black and white are mapped to the ends of the range (the luma is not left at 16 and 235). Signed-off-by: Joaquin Anton Guirao --- dali/test/python/decoder/test_video.py | 38 ++++++++++++++++++++------ 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/dali/test/python/decoder/test_video.py b/dali/test/python/decoder/test_video.py index e378f41092e..612c67104a7 100644 --- a/dali/test/python/decoder/test_video.py +++ b/dali/test/python/decoder/test_video.py @@ -306,16 +306,37 @@ def test_pipeline(): assert np.mean(absdiff) < 2 +COLOR_MATRIX_NUM_BARS = 10 +COLOR_MATRIX_BAR_WIDTH = 192 # pixels + + +def color_matrix_bar(frame, i): + """Mean color of the interior of the i-th bar of the test pattern in db/video/color_matrix: 10 + flat bars (red, green, blue, yellow, cyan, magenta, orange, grey, black, white), 192 pixels + wide""" + interior = ( + slice(100, 440), + slice(i * COLOR_MATRIX_BAR_WIDTH + 32, (i + 1) * COLOR_MATRIX_BAR_WIDTH - 32), + ) + return frame[interior].reshape(-1, 3).mean(axis=0) + + def color_matrix_bars_error(frame, reference): """Worst difference between `frame` and `reference` of the mean colors of the interiors of the - 8 bars of the test pattern in db/video/color_matrix (the bars are 240 pixels wide)""" - worst = 0 - for i in range(8): - interior = (slice(100, 440), slice(i * 240 + 40, (i + 1) * 240 - 40)) - color = frame[interior].reshape(-1, 3).mean(axis=0) - ref_color = reference[interior].reshape(-1, 3).mean(axis=0) - worst = max(worst, np.max(np.abs(color - ref_color))) - return worst + bars of the test pattern in db/video/color_matrix""" + return max( + np.max(np.abs(color_matrix_bar(frame, i) - color_matrix_bar(reference, i))) + for i in range(COLOR_MATRIX_NUM_BARS) + ) + + +def check_color_matrix_black_and_white(frame): + """Black and white are the ends of the range (Y=16 and Y=235 in the limited range): the luma + has to be mapped to the whole range and clamped there (not 16 and 235). They do not depend on + the matrix, unlike the saturated bars.""" + black, white = color_matrix_bar(frame, 8), color_matrix_bar(frame, 9) + assert np.all(black <= 2), f"black bar is {black}" + assert np.all(white >= 253), f"white bar is {white}" @params("bt709_limited_1080p_vp9") @@ -338,6 +359,7 @@ def test_pipeline(): ref = cv2.cvtColor(cv2.imread(os.path.join(data_dir, f"{name}_ref.png")), cv2.COLOR_BGR2RGB) assert frame.shape == ref.shape assert color_matrix_bars_error(frame, ref) <= 4 + check_color_matrix_black_and_white(frame) assert np.mean(np.abs(frame.astype(int) - ref.astype(int))) < 2 From cf0d4351209ce90bb45965c76c8d9b9d57e4b471 Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Wed, 28 Oct 2026 15:55:17 +0100 Subject: [PATCH 4/4] Update DALI_extra version for the BT.709 video clips The color matrix tests read db/video/color_matrix, which DALI_extra#137 adds. Signed-off-by: Joaquin Anton Guirao --- DALI_EXTRA_VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DALI_EXTRA_VERSION b/DALI_EXTRA_VERSION index 8b0ede18311..0f243b7648c 100644 --- a/DALI_EXTRA_VERSION +++ b/DALI_EXTRA_VERSION @@ -1 +1 @@ -7fa5101b6ffaf536e2e9edb34bf5bea7ed0a74ad +eda69271b798e84648e3c51445ce9cf183c36c1e