Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 63f47fd

Browse files
cudawarpedcudawarped
committed
videoio: fix ffmpeg encapsulation timestamps
1 parent 94b7a2d commit 63f47fd

File tree

3 files changed

+62
-29
lines changed

3 files changed

+62
-29
lines changed

‎modules/videoio/include/opencv2/videoio.hpp‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -230,8 +230,10 @@ enum VideoWriterProperties {
230230
VIDEOWRITER_PROP_HW_DEVICE = 7, //!< (**open-only**) Hardware device index (select GPU if multiple available). Device enumeration is acceleration type specific.
231231
VIDEOWRITER_PROP_HW_ACCELERATION_USE_OPENCL= 8, //!< (**open-only**) If non-zero, create new OpenCL context and bind it to current thread. The OpenCL context created with Video Acceleration context attached it (if not attached yet) for optimized GPU data copy between cv::UMat and HW accelerated encoder.
232232
VIDEOWRITER_PROP_RAW_VIDEO = 9, //!< (**open-only**) Set to non-zero to enable encapsulation of an encoded raw video stream. Each raw encoded video frame should be passed to VideoWriter::write() as single row or column of a \ref CV_8UC1 Mat. \note If the key frame interval is not 1 then it must be manually specified by the user. This can either be performed during initialization passing \ref VIDEOWRITER_PROP_KEY_INTERVAL as one of the extra encoder params to \ref VideoWriter::VideoWriter(const String &, int, double, const Size &, const std::vector< int > &params) or afterwards by setting the \ref VIDEOWRITER_PROP_KEY_FLAG with \ref VideoWriter::set() before writing each frame. FFMpeg backend only.
233-
VIDEOWRITER_PROP_KEY_INTERVAL = 10, //!< (**open-only**) Set the key frame interval using raw video encapsulation (\ref VIDEOWRITER_PROP_RAW_VIDEO != 0). Defaults to 1 when not set. FFMpeg backend only.
234-
VIDEOWRITER_PROP_KEY_FLAG = 11, //!< Set to non-zero to signal that the following frames are key frames or zero if not, when encapsulating raw video (\ref VIDEOWRITER_PROP_RAW_VIDEO != 0). FFMpeg backend only.
233+
VIDEOWRITER_PROP_KEY_INTERVAL = 10, //!< (**open-only**) Set the key frame interval using raw video encapsulation (\ref VIDEOWRITER_PROP_RAW_VIDEO != 0). Defaults to 1 when not set. FFmpeg back-end only.
234+
VIDEOWRITER_PROP_KEY_FLAG = 11, //!< Set to non-zero to signal that the following frames are key frames or zero if not, when encapsulating raw video (\ref VIDEOWRITER_PROP_RAW_VIDEO != 0). FFmpeg back-end only.
235+
VIDEOWRITER_PROP_PTS_INDEX = 12, //!< Specifies the frame presentation index for each frame when encapsulating raw video (\ref VIDEOWRITER_PROP_RAW_VIDEO != 0). This flag is necessary when encapsulating externally encoded video where the decoding order differs from the presentation order, such as in GOP patterns with bi-directional B-frames. FFmpeg back-end only.
236+
VIDEOWRITER_PROP_B_FRAME_PRESENTATION_DELAY = 13, //!< Maximum delay (in frames) between a B-frame's decoding and presentation. For example, in a GOP with presentation order IBP and decoding order IPB, this value would be 1, as the B-frame is decoded third but presented second.
235237
#ifndef CV_DOXYGEN
236238
CV__VIDEOWRITER_PROP_LATEST
237239
#endif

‎modules/videoio/src/cap_ffmpeg_impl.hpp‎

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2111,6 +2111,8 @@ struct CvVideoWriter_FFMPEG
21112111
bool encode_video;
21122112
int idr_period;
21132113
bool key_frame;
2114+
int pts_index;
2115+
int b_frame_dts_delay;
21142116
};
21152117

21162118
static const char * icvFFMPEGErrStr(int err)
@@ -2179,6 +2181,8 @@ void CvVideoWriter_FFMPEG::init()
21792181
encode_video = true;
21802182
idr_period = 0;
21812183
key_frame = false;
2184+
pts_index = -1;
2185+
b_frame_dts_delay = 0;
21822186
}
21832187

21842188
/**
@@ -2347,7 +2351,7 @@ static AVCodecContext * icv_configure_video_stream_FFMPEG(AVFormatContext *oc,
23472351
static const int OPENCV_NO_FRAMES_WRITTEN_CODE = 1000;
23482352

23492353
static int icv_av_encapsulate_video_FFMPEG(AVFormatContext* oc, AVStream* video_st, AVCodecContext* c,
2350-
uint8_t* data, int sz, const int frame_idx, const bool key_frame)
2354+
uint8_t* data, int sz, const int frame_idx, const int pts_index, const int b_frame_dts_delay, const bool key_frame)
23512355
{
23522356
#if LIBAVFORMAT_BUILD < CALC_FFMPEG_VERSION(57, 0, 0)
23532357
AVPacket pkt_;
@@ -2358,7 +2362,9 @@ static int icv_av_encapsulate_video_FFMPEG(AVFormatContext* oc, AVStream* video_
23582362
#endif
23592363
if(key_frame)
23602364
pkt->flags |= PKT_FLAG_KEY;
2361-
pkt->pts = frame_idx;
2365+
pkt->pts = pts_index == -1 ? frame_idx : pts_index;
2366+
pkt->dts = frame_idx - b_frame_dts_delay;
2367+
pkt->duration = 1;
23622368
pkt->size = sz;
23632369
pkt->data = data;
23642370
av_packet_rescale_ts(pkt, c->time_base, video_st->time_base);
@@ -2453,7 +2459,7 @@ bool CvVideoWriter_FFMPEG::writeFrame( const unsigned char* data, int step, int
24532459
if (!encode_video) {
24542460
CV_Assert(cn == 1 && ((width > 0 && height == 1) || (width == 1 && height > 0 && step == 1)));
24552461
const bool set_key_frame = key_frame ? key_frame : idr_period ? frame_idx % idr_period == 0 : 1;
2456-
bool ret = icv_av_encapsulate_video_FFMPEG(oc, video_st, context, (uint8_t*)data, width, frame_idx, set_key_frame);
2462+
bool ret = icv_av_encapsulate_video_FFMPEG(oc, video_st, context, (uint8_t*)data, width, frame_idx, pts_index, b_frame_dts_delay, set_key_frame);
24572463
frame_idx++;
24582464
return ret;
24592465
}
@@ -2655,6 +2661,12 @@ bool CvVideoWriter_FFMPEG::setProperty(int property_id, double value)
26552661
case VIDEOWRITER_PROP_KEY_FLAG:
26562662
key_frame = static_cast<bool>(value);
26572663
break;
2664+
case VIDEOWRITER_PROP_PTS_INDEX:
2665+
pts_index = static_cast<int>(value);
2666+
break;
2667+
case VIDEOWRITER_PROP_B_FRAME_PRESENTATION_DELAY:
2668+
b_frame_dts_delay = static_cast<int>(value);
2669+
break;
26582670
default:
26592671
return false;
26602672
}

‎modules/videoio/test/test_ffmpeg.cpp‎

Lines changed: 43 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -293,9 +293,13 @@ const videoio_container_get_params_t videoio_container_get_params[] =
293293

294294
INSTANTIATE_TEST_CASE_P(/**/, videoio_container_get, testing::ValuesIn(videoio_container_get_params));
295295

296-
typedef tuple<string, string, int, int> videoio_encapsulate_params_t;
296+
typedef tuple<string, string, int, int, int, bool, bool> videoio_encapsulate_params_t;
297297
typedef testing::TestWithParam< videoio_encapsulate_params_t > videoio_encapsulate;
298298

299+
#if defined(WIN32) // remove when FFmpeg wrapper includes PR25874
300+
#define WIN32_WAIT_FOR_FFMPEG_WRAPPER_UPDATE
301+
#endif
302+
299303
TEST_P(videoio_encapsulate, write)
300304
{
301305
const VideoCaptureAPIs api = CAP_FFMPEG;
@@ -307,6 +311,9 @@ TEST_P(videoio_encapsulate, write)
307311
const int idrPeriod = get<2>(GetParam());
308312
const int nFrames = get<3>(GetParam());
309313
const string fileNameOut = tempfile(cv::format("test_encapsulated_stream.%s", ext.c_str()).c_str());
314+
const int bFrameDtsDelay = get<4>(GetParam());
315+
const bool setPts = get<5>(GetParam());
316+
const bool tsWorking = get<6>(GetParam());
310317

311318
// Use VideoWriter to encapsulate encoded video read with VideoReader
312319
{
@@ -315,6 +322,7 @@ TEST_P(videoio_encapsulate, write)
315322
const int width = static_cast<int>(capRaw.get(CAP_PROP_FRAME_WIDTH));
316323
const int height = static_cast<int>(capRaw.get(CAP_PROP_FRAME_HEIGHT));
317324
const double fps = capRaw.get(CAP_PROP_FPS);
325+
const double msPerFrame = 1000.0 / fps;
318326
const int codecExtradataIndex = static_cast<int>(capRaw.get(CAP_PROP_CODEC_EXTRADATA_INDEX));
319327
Mat extraData;
320328
capRaw.retrieve(extraData, codecExtradataIndex);
@@ -323,6 +331,10 @@ TEST_P(videoio_encapsulate, write)
323331

324332
VideoWriter container(fileNameOut, api, fourcc, fps, { width, height }, { VideoWriterProperties::VIDEOWRITER_PROP_RAW_VIDEO, 1, VideoWriterProperties::VIDEOWRITER_PROP_KEY_INTERVAL, idrPeriod });
325333
ASSERT_TRUE(container.isOpened());
334+
#if !defined(WIN32_WAIT_FOR_FFMPEG_WRAPPER_UPDATE)
335+
if (bFrameDtsDelay > 0)
336+
ASSERT_TRUE(container.set(VIDEOWRITER_PROP_B_FRAME_PRESENTATION_DELAY, bFrameDtsDelay));
337+
#endif
326338
Mat rawFrame;
327339
for (int i = 0; i < nFrames; i++) {
328340
ASSERT_TRUE(capRaw.read(rawFrame));
@@ -336,6 +348,10 @@ TEST_P(videoio_encapsulate, write)
336348
memcpy(rawFrame.data, extraData.data, extraData.total());
337349
memcpy(rawFrame.data + extraData.total(), tmp.data, tmp.total());
338350
}
351+
#if !defined(WIN32_WAIT_FOR_FFMPEG_WRAPPER_UPDATE)
352+
if(setPts)
353+
ASSERT_TRUE(container.set(VIDEOWRITER_PROP_PTS_INDEX, round(capRaw.get(CAP_PROP_POS_MSEC) / msPerFrame)));
354+
#endif
339355
container.write(rawFrame);
340356
}
341357
container.release();
@@ -367,6 +383,10 @@ TEST_P(videoio_encapsulate, write)
367383
const bool keyFrameActual = capActualRaw.get(CAP_PROP_LRF_HAS_KEY_FRAME) == 1.;
368384
const bool keyFrameReference = idrPeriod ? i % idrPeriod == 0 : 1;
369385
ASSERT_EQ(keyFrameReference, keyFrameActual);
386+
#if !defined(WIN32_WAIT_FOR_FFMPEG_WRAPPER_UPDATE)
387+
if (tsWorking)
388+
ASSERT_EQ(round(capReference.get(CAP_PROP_POS_MSEC)), round(capActual.get(CAP_PROP_POS_MSEC)));
389+
#endif
370390
}
371391
}
372392

@@ -375,30 +395,29 @@ TEST_P(videoio_encapsulate, write)
375395

376396
const videoio_encapsulate_params_t videoio_encapsulate_params[] =
377397
{
378-
videoio_encapsulate_params_t("video/big_buck_bunny.h264", "avi", 125, 125),
379-
videoio_encapsulate_params_t("video/big_buck_bunny.h265", "mp4", 125, 125),
380-
videoio_encapsulate_params_t("video/big_buck_bunny.wmv", "wmv", 12, 13),
381-
videoio_encapsulate_params_t("video/big_buck_bunny.mp4", "mp4", 12, 13),
382-
videoio_encapsulate_params_t("video/big_buck_bunny.mjpg.avi", "mp4", 0, 4),
383-
videoio_encapsulate_params_t("video/big_buck_bunny.mov", "mp4", 12, 13),
384-
videoio_encapsulate_params_t("video/big_buck_bunny.avi", "mp4", 125, 125),
385-
videoio_encapsulate_params_t("video/big_buck_bunny.mpg", "mp4", 12, 13),
386-
videoio_encapsulate_params_t("video/VID00003-20100701-2204.wmv", "wmv", 12, 13),
387-
videoio_encapsulate_params_t("video/VID00003-20100701-2204.mpg", "mp4", 12,13),
388-
videoio_encapsulate_params_t("video/VID00003-20100701-2204.avi", "mp4", 12, 13),
389-
videoio_encapsulate_params_t("video/VID00003-20100701-2204.3GP", "mp4", 51, 52),
390-
videoio_encapsulate_params_t("video/sample_sorenson.avi", "mp4", 12, 13),
391-
videoio_encapsulate_params_t("video/sample_322x242_15frames.yuv420p.libxvid.mp4", "mp4", 3, 4),
392-
videoio_encapsulate_params_t("video/sample_322x242_15frames.yuv420p.mpeg2video.mp4", "mp4", 12, 13),
393-
videoio_encapsulate_params_t("video/sample_322x242_15frames.yuv420p.mjpeg.mp4", "mp4", 0, 5),
394-
videoio_encapsulate_params_t("video/sample_322x242_15frames.yuv420p.libx264.mp4", "avi", 15, 15),
395-
videoio_encapsulate_params_t("../cv/tracking/faceocc2/data/faceocc2.webm", "webm", 128, 129),
396-
videoio_encapsulate_params_t("../cv/video/1920x1080.avi", "mp4", 12, 13),
397-
videoio_encapsulate_params_t("../cv/video/768x576.avi", "avi", 15, 16)
398+
videoio_encapsulate_params_t("video/big_buck_bunny.h264", "avi", 125, 125, 0, false, false), // tsWorking = false: no timestamp information
399+
videoio_encapsulate_params_t("video/big_buck_bunny.h265", "mp4", 125, 125, 0, false, false), // tsWorking = false: no timestamp information
400+
videoio_encapsulate_params_t("video/big_buck_bunny.wmv", "wmv", 12, 13, 0, false, true),
401+
videoio_encapsulate_params_t("video/big_buck_bunny.mp4", "mp4", 12, 13, 0, false, true),
402+
videoio_encapsulate_params_t("video/big_buck_bunny.mjpg.avi", "mp4", 0, 4, 0, false, true),
403+
videoio_encapsulate_params_t("video/big_buck_bunny.mov", "mp4", 12, 13, 0, false, true),
404+
videoio_encapsulate_params_t("video/big_buck_bunny.avi", "mp4", 125, 125, 0, false, false), // tsWorking = false: Source value of CAP_PROP_POS_MSEC is wrong
405+
videoio_encapsulate_params_t("video/big_buck_bunny.mpg", "mp4", 12, 13, 0, false, false), // tsWorking = false: Source PTS not equal to CAP_PROP_POS_MSEC / (1000.0/ fps)
406+
videoio_encapsulate_params_t("video/VID00003-20100701-2204.wmv", "wmv", 12, 13, 0, false, true),
407+
videoio_encapsulate_params_t("video/VID00003-20100701-2204.mpg", "mp4", 12, 13, 0, false, false), // tsWorking = false: Source PTS not equal to CAP_PROP_POS_MSEC / (1000.0/ fps)
408+
videoio_encapsulate_params_t("video/VID00003-20100701-2204.avi", "mp4", 12, 13, 0, false, false), // tsWorking = false: Unable to correctly set PTS when writing
409+
videoio_encapsulate_params_t("video/VID00003-20100701-2204.3GP", "mp4", 51, 52, 0, false, false), // tsWorking = false: Source with variable fps, unable to calculate VIDEOWRITER_PROP_PTS_INDEX from average fps returned by VideoCapture
410+
videoio_encapsulate_params_t("video/sample_sorenson.avi", "mp4", 12, 13, 0,false, true),
411+
videoio_encapsulate_params_t("video/sample_322x242_15frames.yuv420p.libxvid.mp4", "mp4", 3, 4, 0, false, true),
412+
videoio_encapsulate_params_t("video/sample_322x242_15frames.yuv420p.mpeg2video.mp4", "mpg", 12, 13, 0, false, true),
413+
videoio_encapsulate_params_t("video/sample_322x242_15frames.yuv420p.mjpeg.mp4", "mp4", 0, 5, 0, false, true),
414+
videoio_encapsulate_params_t("video/sample_322x242_15frames.yuv420p.libx264.mp4", "mp4", 15, 15, 2, true, true),
415+
videoio_encapsulate_params_t("../cv/tracking/faceocc2/data/faceocc2.webm", "webm", 128, 129, 0, false, true),
416+
videoio_encapsulate_params_t("../cv/video/1920x1080.avi", "mp4", 12, 13, 0, false, true),
417+
videoio_encapsulate_params_t("../cv/video/768x576.avi", "avi", 15, 16, 0, false, true),
398418
// Not supported by with FFmpeg:
399-
//videoio_encapsulate_params_t("video/sample_322x242_15frames.yuv420p.libx265.mp4", "mp4", 15, 15),
400-
//videoio_encapsulate_params_t("video/sample_322x242_15frames.yuv420p.libvpx-vp9.mp4", "mp4", 15, 15),
401-
419+
//videoio_encapsulate_params_t("video/sample_322x242_15frames.yuv420p.libx265.mp4", "mp4", 15, 15, 2, true, true),
420+
//videoio_encapsulate_params_t("video/sample_322x242_15frames.yuv420p.libvpx-vp9.mp4", "mp4", 15, 15, 0, false, true),
402421
};
403422

404423
INSTANTIATE_TEST_CASE_P(/**/, videoio_encapsulate, testing::ValuesIn(videoio_encapsulate_params));

0 commit comments

Comments
 (0)