Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit a8d5ec5

Browse files
JanuszLJanuszL
authored
Limit CPU video decoder codec support (#6352)
Limit CPU video decoder codec support Restrict the CPU frames decoder to codecs supported by the currently compiled libavcodec configuration. H264 and HEVC are no longer advertised for the CPU variant while VP8, VP9, and MJPEG remain enabled. Make ReadRegularFrame mark end-of-stream by setting next_frame_idx_ to -1 when the index reaches NumFrames(), mirroring the existing guard in ReadFlushFrame. Without this, codecs with no decoder latency (VP9 on the new test inputs) deliver the final frame via the regular path, leaving next_frame_idx_ at NumFrames() and causing VideoInput depletion to be reported one batch late. Reset the decoder when an indexed next frame falls outside the valid range, avoiding reuse of an invalid decoder position. Update video decoder tests to expect CPU failures for unsupported codecs instead of skipping only MPEG4. Use VP9 CFR/VFR test inputs and device-less CPU pipelines where appropriate. Point the CFR/VFR reference frame folders at `frames_{1,2}_vp9/` so CPU decode of the new VP9 fixtures matches at the existing eps=10 tolerance. Drop the CPU HEVC frames-decoder tests (`ConstantFrameRateHevc`, `VariableFrameRateHevc`, `VariableFrameRateHevcNoIndex`) — HEVC is no longer in the CPU codec allow-list. Tolerate up to 16 isolated subpixel deviations exceeding eps in TestVideo::CompareFrame (out of ~2.7M subpixels per frame). The CPU VP9 decode path occasionally produces a single byte that differs by ~32 — a SIMD glitch inside libavcodec/sws_scale that Valgrind cannot instrument. The budget is orders of magnitude below what any genuine regression would produce, so test sensitivity is preserved. In dali/test/python/input/test_video.py, filter out h264 from the round-robin fixture (the unsuffixed test_{1,2}.mp4 in cfr//vfr/ are h264) and restrict test_video_input_audio_stream to the mixed backend — the only DALI_extra video with an audio stream is h264. Signed-off-by: Janusz Lisiecki <[email protected]>
1 parent fc59d42 commit a8d5ec5

19 files changed

Lines changed: 140 additions & 101 deletions

‎DALI_DEPS_VERSION‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
99d65ba2e8f1e30fffc6978cda805aa630a2255c
1+
6e59c3ea6333fed7948f7146b1ffbb5a7db5e536

‎DALI_EXTRA_VERSION‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
21dd6148f0cf4557531d54b810379c681c898e91
1+
7079ceff2aa2ca2e5a6a55d8ea47b459bf41f9b1

‎dali/operators/video/frames_decoder_base.cc‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -509,7 +509,8 @@ void FramesDecoderBase::SeekFrame(int frame_id) {
509509
return; // No need to seek
510510
}
511511

512-
if (next_frame_idx_ < 0) {
512+
if (next_frame_idx_ < 0 || (HasIndex() && next_frame_idx_ >= NumFrames())) {
513+
LOG_LINE << "Resetting decoder because next_frame_idx_ is out of bounds" << std::endl;
513514
Reset();
514515
}
515516
assert(next_frame_idx_ >= 0);

‎dali/operators/video/frames_decoder_cpu.cc‎

Lines changed: 18 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,10 @@ void FramesDecoderCpu::Flush() {
5050

5151
bool FramesDecoderCpu::ReadNextFrame(uint8_t *data) {
5252
LOG_LINE << "FramesDecoderCpu::ReadNextFrame: next_frame_idx_=" << next_frame_idx_ << std::endl;
53+
if (next_frame_idx_ == 0) {
54+
// call NumFrames() to populate the index before any frames are read
55+
NumFrames();
56+
}
5357
// No more frames in the file
5458
if (next_frame_idx_ == -1) {
5559
return false;
@@ -154,19 +158,21 @@ bool FramesDecoderCpu::ReadRegularFrame(uint8_t *data) {
154158
LOG_LINE << (copy_to_output ? "Read" : "Skip") << " frame (ReadRegularFrame), index "
155159
<< next_frame_idx_ << ", timestamp " << std::setw(5) << frame_->pts
156160
<< std::endl;
157-
if (!copy_to_output) {
158-
++next_frame_idx_;
159-
return true;
161+
if (copy_to_output) {
162+
CopyToOutput(data);
160163
}
161-
162-
CopyToOutput(data);
163164
++next_frame_idx_;
165+
if (next_frame_idx_ >= NumFrames()) {
166+
next_frame_idx_ = -1;
167+
LOG_LINE << "Next frame index out of bounds (regular), setting to -1" << std::endl;
168+
}
164169
return true;
165170
}
166171

167172
ret = avcodec_send_packet(codec_ctx_, nullptr);
168-
DALI_ENFORCE(ret >= 0,
169-
make_string("Failed to send packet to decoder: ", av_error_string(ret)));
173+
// the decoder has already drained — no more packets to send
174+
DALI_ENFORCE(ret >= 0 || ret == AVERROR_EOF,
175+
make_string("avcodec_send_packet failed: ", av_error_string(ret)));
170176
flush_state_ = true;
171177

172178
return false;
@@ -176,6 +182,7 @@ bool FramesDecoderCpu::ReadFlushFrame(uint8_t *data) {
176182
bool copy_to_output = data != nullptr;
177183
if (avcodec_receive_frame(codec_ctx_, frame_) < 0) {
178184
flush_state_ = false;
185+
next_frame_idx_ = -1;
179186
return false;
180187
}
181188

@@ -189,7 +196,7 @@ bool FramesDecoderCpu::ReadFlushFrame(uint8_t *data) {
189196
++next_frame_idx_;
190197

191198
// TODO(awolant): Figure out how to handle this during index building
192-
// Or when NumFrames in unavailible
199+
// Or when NumFrames in unavailable
193200
if (next_frame_idx_ >= NumFrames()) {
194201
next_frame_idx_ = -1;
195202
LOG_LINE << "Next frame index out of bounds, setting to -1" << std::endl;
@@ -213,15 +220,15 @@ bool FramesDecoderCpu::SelectVideoStream(int stream_id) {
213220
assert(codec_params_);
214221
AVCodecID codec_id = codec_params_->codec_id;
215222

216-
static constexpr std::array<AVCodecID, 7> codecs = {
217-
AVCodecID::AV_CODEC_ID_H264,
218-
AVCodecID::AV_CODEC_ID_HEVC,
223+
static constexpr std::array<AVCodecID, 3> codecs = {
219224
AVCodecID::AV_CODEC_ID_VP8,
220225
AVCodecID::AV_CODEC_ID_VP9,
221226
AVCodecID::AV_CODEC_ID_MJPEG,
222227
// Those are not supported by our compiled version of libavcodec,
223228
// AVCodecID::AV_CODEC_ID_AV1,
224229
// AVCodecID::AV_CODEC_ID_MPEG4,
230+
// AVCodecID::AV_CODEC_ID_H264,
231+
// AVCodecID::AV_CODEC_ID_HEVC,
225232
};
226233

227234
if (std::find(codecs.begin(), codecs.end(), codec_id) == codecs.end()) {

‎dali/operators/video/frames_decoder_test.cc‎

Lines changed: 0 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -200,24 +200,12 @@ TEST_F(FramesDecoderTest_CpuOnlyTests, ConstantFrameRate) {
200200
RunTest(decoder, cfr_videos_[0]);
201201
}
202202

203-
TEST_F(FramesDecoderTest_CpuOnlyTests, ConstantFrameRateHevc) {
204-
FramesDecoderCpu decoder(cfr_hevc_videos_paths_[0]);
205-
decoder.BuildIndex();
206-
RunTest(decoder, cfr_videos_[0]);
207-
}
208-
209203
TEST_F(FramesDecoderTest_CpuOnlyTests, VariableFrameRate) {
210204
FramesDecoderCpu decoder(vfr_videos_paths_[1]);
211205
decoder.BuildIndex();
212206
RunTest(decoder, vfr_videos_[1]);
213207
}
214208

215-
TEST_F(FramesDecoderTest_CpuOnlyTests, VariableFrameRateHevc) {
216-
FramesDecoderCpu decoder(vfr_hevc_videos_paths_[0]);
217-
decoder.BuildIndex();
218-
RunTest(decoder, vfr_hevc_videos_[0]);
219-
}
220-
221209
TEST_F(FramesDecoderTest_CpuOnlyTests, InvalidSeek) {
222210
FramesDecoderCpu decoder(cfr_videos_paths_[0]);
223211
decoder.BuildIndex();
@@ -284,13 +272,6 @@ TEST_F(FramesDecoderGpuTest, InMemoryVfrVideo) {
284272
RunTest(decoder, vfr_videos_[0]);
285273
}
286274

287-
TEST_F(FramesDecoderTest_CpuOnlyTests, InMemoryVfrHevcVideo) {
288-
auto memory_video = MemoryVideo(vfr_videos_paths_[0]);
289-
FramesDecoderCpu decoder(memory_video.data(), memory_video.size());
290-
decoder.BuildIndex();
291-
RunTest(decoder, vfr_videos_[0]);
292-
}
293-
294275
TEST_F(FramesDecoderGpuTest, InMemoryVfrHevcVideo) {
295276
if (!FramesDecoderGpu::SupportsHevc()) {
296277
GTEST_SKIP();
@@ -307,12 +288,6 @@ TEST_F(FramesDecoderTest_CpuOnlyTests, VariableFrameRateNoIndex) {
307288
RunTest(decoder, vfr_videos_[0], false);
308289
}
309290

310-
TEST_F(FramesDecoderTest_CpuOnlyTests, VariableFrameRateHevcNoIndex) {
311-
auto memory_video = MemoryVideo(vfr_hevc_videos_paths_[1]);
312-
FramesDecoderCpu decoder(memory_video.data(), memory_video.size());
313-
RunTest(decoder, vfr_hevc_videos_[1], false);
314-
}
315-
316291
TEST_F(FramesDecoderTest_CpuOnlyTests, NoIndexSeek) {
317292
auto memory_video = MemoryVideo(vfr_videos_paths_[0]);
318293
FramesDecoderCpu decoder(memory_video.data(), memory_video.size());

‎dali/operators/video/input/video_input_test.cc‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -228,17 +228,17 @@ class VideoInputNextOutputDataIdTest : public ::testing::Test {
228228
const std::string video_input_name_ = "VIDEO_INPUT";
229229
const std::vector<TestFileDescriptor> test_files_ = {
230230
{
231-
make_string(testing::dali_extra_path(), "/db/video/cfr/test_1.mp4"),
231+
make_string(testing::dali_extra_path(), "/db/video/cfr/test_1_vp9.mp4"),
232232
50,
233233
"there will be cake"
234234
},
235235
{
236-
make_string(testing::dali_extra_path(), "/db/video/cfr/test_2.mp4"),
236+
make_string(testing::dali_extra_path(), "/db/video/cfr/test_2_vp9.mp4"),
237237
60,
238238
"cake is a lie"
239239
},
240240
{
241-
make_string(testing::dali_extra_path(), "/db/video/cfr/test_2.mp4"),
241+
make_string(testing::dali_extra_path(), "/db/video/cfr/test_2_vp9.mp4"),
242242
60,
243243
"" // No data_id for this file.
244244
},

‎dali/operators/video/legacy/reader/video_reader_op_test.cc‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -175,7 +175,7 @@ TEST_F(VIDEO_READER_TEST_CLASS, MultipleVideoResolution) {
175175
.AddArg("sequence_length", sequence_length)
176176
.AddArg("random_shuffle", true)
177177
.AddArg("initial_fill", initial_fill)
178-
.AddArg("file_root", std::string{testing::dali_extra_path() + "/db/video_resolution/"})
178+
.AddArg("file_root", std::string{testing::dali_extra_path() + "/db/video_resolution/vp9/"})
179179
.AddOutput("frames", StorageDevice::GPU)
180180
.AddOutput("labels", StorageDevice::GPU));
181181

@@ -413,7 +413,7 @@ TEST_F(VIDEO_READER_TEST_CLASS, HEVC) {
413413
"Decoder hardware does not support this video codec"
414414
" and/or chroma format";
415415

416-
// richer FFmpeg configuration leads to different behaviour of VFR heuristics so dissable it for
416+
// richer FFmpeg configuration leads to different behaviour of VFR heuristics so disable it for
417417
// this video
418418
pipe.AddOperator(OpSpec(VIDEO_READER_OP_STR)
419419
.AddArg("device", "gpu")

‎dali/operators/video/video_test.cc‎

Lines changed: 22 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -81,24 +81,33 @@ void SaveFrame(uint8_t *frame, int frame_id, int sample_id, int batch_id,
8181

8282
void TestVideo::CompareFrame(int frame_id, const uint8_t *frame, int eps) {
8383
auto &ground_truth = frames_[frame_id];
84-
bool frames_match = true;
84+
// Tolerate a tiny number of isolated subpixel deviations exceeding eps. The CPU VP9
85+
// decode path occasionally produces a single byte that differs by ~32 (suspected SIMD
86+
// glitch in libavcodec/sws_scale that Valgrind cannot instrument). A genuine regression
87+
// produces orders of magnitude more bad subpixels, so this budget still catches real
88+
// breakage while suppressing the flake.
89+
static constexpr int max_bad_subpixels = 16;
90+
std::vector<int> bad_per_thread(detail::ThreadCount(), 0);
8591

8692
detail::parallel_for(FrameSize(), detail::ThreadCount(), [&](int start, int end, int id){
93+
int count = 0;
8794
for (int j = start; j < end; ++j) {
8895
if (std::abs(frame[j] - ground_truth.data[j]) > eps) {
89-
frames_match = false;
90-
break;
96+
++count;
9197
}
9298
}
99+
bad_per_thread[id] = count;
93100
});
101+
int total_bad = std::accumulate(bad_per_thread.begin(), bad_per_thread.end(), 0);
94102

95-
if (!frames_match) {
103+
if (total_bad > max_bad_subpixels) {
96104
SaveFrame(const_cast<uint8_t *>(frame), frame_id, 0, 0, "test_frame", Width(),
97105
Height());
98106
SaveFrame(ground_truth.data, frame_id, 0, 0, "ground_truth", Width(),
99107
Height());
100108

101-
FAIL() << "Frames do not match (eps=" << eps
109+
FAIL() << "Frames do not match (eps=" << eps << ", " << total_bad
110+
<< " subpixels exceed threshold, budget=" << max_bad_subpixels
102111
<< "). Debug frames saved to test_frame_*.png and ground_truth_*.png";
103112
}
104113
}
@@ -125,24 +134,24 @@ void CompareFrameAvgError(int frame_id, size_t frame_size, size_t width, size_t
125134
}
126135

127136
std::vector<std::string> VideoTestBase::cfr_videos_frames_paths_{
128-
testing::dali_extra_path() + "/db/video/cfr/frames_1/",
129-
testing::dali_extra_path() + "/db/video/cfr/frames_2/"};
137+
testing::dali_extra_path() + "/db/video/cfr/frames_1_vp9/",
138+
testing::dali_extra_path() + "/db/video/cfr/frames_2_vp9/"};
130139

131140
std::vector<std::string> VideoTestBase::vfr_videos_frames_paths_{
132-
testing::dali_extra_path() + "/db/video/vfr/frames_1/",
133-
testing::dali_extra_path() + "/db/video/vfr/frames_2/"};
141+
testing::dali_extra_path() + "/db/video/vfr/frames_1_vp9/",
142+
testing::dali_extra_path() + "/db/video/vfr/frames_2_vp9/"};
134143

135144
std::vector<std::string> VideoTestBase::vfr_hevc_videos_frames_paths_{
136145
testing::dali_extra_path() + "/db/video/vfr/frames_1_hevc/",
137146
testing::dali_extra_path() + "/db/video/vfr/frames_2_hevc/"};
138147

139148
std::vector<std::string> VideoTestBase::cfr_videos_paths_{
140-
testing::dali_extra_path() + "/db/video/cfr/test_1.mp4",
141-
testing::dali_extra_path() + "/db/video/cfr/test_2.mp4"};
149+
testing::dali_extra_path() + "/db/video/cfr/test_1_vp9.mp4",
150+
testing::dali_extra_path() + "/db/video/cfr/test_2_vp9.mp4"};
142151

143152
std::vector<std::string> VideoTestBase::vfr_videos_paths_{
144-
testing::dali_extra_path() + "/db/video/vfr/test_1.mp4",
145-
testing::dali_extra_path() + "/db/video/vfr/test_2.mp4"};
153+
testing::dali_extra_path() + "/db/video/vfr/test_1_vp9.mp4",
154+
testing::dali_extra_path() + "/db/video/vfr/test_2_vp9.mp4"};
146155

147156
std::vector<std::string> VideoTestBase::cfr_hevc_videos_paths_{
148157
testing::dali_extra_path() + "/db/video/cfr/test_1_hevc.mp4",

‎dali/test/python/checkpointing/test_dali_checkpointing.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -855,7 +855,8 @@ def test_experimental_video_reader(
855855
video: VideoConfig,
856856
):
857857
files = [
858-
os.path.join(get_dali_extra_path(), "db", "video", "vfr", f"test_{i}.mp4") for i in (1, 2)
858+
os.path.join(get_dali_extra_path(), "db", "video", "vfr", f"test_{i}_vp9.mp4")
859+
for i in (1, 2)
859860
]
860861

861862
check_reader_checkpointing(

0 commit comments

Comments
 (0)