Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 7f3da26

Browse files
natario1natario1
committed
New video pipeline
1 parent bfc68fc commit 7f3da26

28 files changed

+838
-76
lines changed

‎lib/src/main/java/com/otaliastudios/transcoder/internal/DefaultEngine.kt‎

Lines changed: 25 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,16 @@
11
package com.otaliastudios.transcoder.internal
22

3+
import android.media.MediaFormat
34
import com.otaliastudios.transcoder.Transcoder
45
import com.otaliastudios.transcoder.common.TrackStatus
56
import com.otaliastudios.transcoder.common.TrackType
7+
import com.otaliastudios.transcoder.internal.pipeline.EmptyPipeline
8+
import com.otaliastudios.transcoder.internal.pipeline.PassThroughPipeline
9+
import com.otaliastudios.transcoder.internal.pipeline.Pipeline
10+
import com.otaliastudios.transcoder.internal.pipeline.RegularPipeline
611
import com.otaliastudios.transcoder.internal.utils.Logger
712
import com.otaliastudios.transcoder.internal.utils.TrackMap
13+
import com.otaliastudios.transcoder.internal.utils.forcingEos
814
import com.otaliastudios.transcoder.internal.utils.ignoringEos
915
import com.otaliastudios.transcoder.resample.AudioResampler
1016
import com.otaliastudios.transcoder.sink.DataSink
@@ -28,7 +34,7 @@ internal class DefaultEngine(
2834

2935
private val tracks = Tracks(strategies, dataSources)
3036

31-
private val segments = Segments(dataSources, tracks, ::createTranscoder)
37+
private val segments = Segments(dataSources, tracks, ::createPipeline)
3238

3339
private val timer = Timer(interpolator, dataSources, tracks, segments.currentIndex)
3440

@@ -48,22 +54,29 @@ internal class DefaultEngine(
4854
return this.cause?.isInterrupted() ?: false
4955
}
5056

51-
private fun createTranscoder(type: TrackType, index: Int, status: TrackStatus): TrackTranscoder {
57+
private fun createPipeline(
58+
type: TrackType,
59+
index: Int,
60+
status: TrackStatus,
61+
outputFormat: MediaFormat
62+
): Pipeline {
5263
val interpolator = timer.interpolator(type, index)
5364
val sources = dataSources[type]
54-
val source = sources[index]
65+
val source = sources[index].forcingEos {
66+
// Enforce EOS if we exceed duration of other tracks,
67+
// with a little tolerance.
68+
timer.readUs[type] > timer.durationUs + 100L
69+
}
5570
val sink = when {
5671
index == sources.lastIndex -> dataSink
5772
else -> dataSink.ignoringEos()
5873
}
5974
return when (status) {
60-
TrackStatus.ABSENT -> NoOpTrackTranscoder()
61-
TrackStatus.REMOVING -> NoOpTrackTranscoder()
62-
TrackStatus.PASS_THROUGH -> PassThroughTrackTranscoder(source, sink, type, interpolator)
63-
TrackStatus.COMPRESSING -> when (type) {
64-
TrackType.VIDEO -> VideoTrackTranscoder(source, sink, interpolator, videoRotation)
65-
TrackType.AUDIO -> AudioTrackTranscoder(source, sink, interpolator, audioStretcher, audioResampler)
66-
}
75+
TrackStatus.ABSENT -> EmptyPipeline()
76+
TrackStatus.REMOVING -> EmptyPipeline()
77+
TrackStatus.PASS_THROUGH -> PassThroughPipeline(type, source, sink, interpolator)
78+
TrackStatus.COMPRESSING -> RegularPipeline(type,
79+
source, sink, interpolator, outputFormat, videoRotation)
6780
}
6881
}
6982

@@ -106,21 +119,14 @@ internal class DefaultEngine(
106119
* Retrieve next segment from [Segments] and call [Segment.advance] for each track.
107120
* We don't have to worry about which tracks are available and how. The [Segments] class
108121
* will simply return null if there's nothing to be done.
109-
*
110-
* Note that we need to check if we need to force the input EOS for some track.
111-
* This can happen if the user adds e.g. 1 minute of audio with 20 seconds of video.
112-
* In this case the video track must be stopped once the audio stops.
113122
*/
114123
private fun execute(progress: (Double) -> Unit) {
115124
var loop = 0L
116125
while (true) {
117126
LOG.v("new step: $loop")
118-
val totalUs = timer.durationUs + 100L // tolerance
119-
val forceAudioEos = timer.readUs.audio > totalUs
120-
val forceVideoEos = timer.readUs.video > totalUs
121127
val advanced =
122-
(segments.next(TrackType.AUDIO)?.advance(forceAudioEos) ?: false) or
123-
(segments.next(TrackType.VIDEO)?.advance(forceVideoEos) ?: false)
128+
(segments.next(TrackType.AUDIO)?.advance() ?: false) or
129+
(segments.next(TrackType.VIDEO)?.advance() ?: false)
124130
val completed = !advanced && !segments.hasNext() // avoid calling hasNext if we advanced.
125131

126132
if (Thread.interrupted()) {

‎lib/src/main/java/com/otaliastudios/transcoder/internal/Segment.kt‎

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -2,27 +2,28 @@ package com.otaliastudios.transcoder.internal
22

33
import android.media.MediaFormat
44
import com.otaliastudios.transcoder.common.TrackType
5+
import com.otaliastudios.transcoder.internal.pipeline.Pipeline
6+
import com.otaliastudios.transcoder.internal.pipeline.State
57
import com.otaliastudios.transcoder.transcode.TrackTranscoder
68

79
internal class Segment(
810
val type: TrackType,
911
val index: Int,
10-
private val transcoder: TrackTranscoder,
11-
outputFormat: MediaFormat
12+
private val pipeline: Pipeline,
1213
) {
13-
init {
14-
transcoder.setUp(outputFormat)
15-
}
1614

17-
fun advance(forceInputEos: Boolean): Boolean {
18-
return transcoder.transcode(forceInputEos)
15+
private var state: State<Unit>? = null
16+
17+
fun advance(): Boolean {
18+
state = pipeline.execute()
19+
return state is State.Ok
1920
}
2021

2122
fun canAdvance(): Boolean {
22-
return !transcoder.isFinished
23+
return state == null || state !is State.Eos
2324
}
2425

2526
fun release() {
26-
transcoder.tearDown()
27+
pipeline.release()
2728
}
2829
}

‎lib/src/main/java/com/otaliastudios/transcoder/internal/Segments.kt‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,17 @@
11
package com.otaliastudios.transcoder.internal
22

3+
import android.media.MediaFormat
34
import com.otaliastudios.transcoder.common.TrackStatus
45
import com.otaliastudios.transcoder.common.TrackType
6+
import com.otaliastudios.transcoder.internal.pipeline.Pipeline
57
import com.otaliastudios.transcoder.internal.utils.TrackMap
68
import com.otaliastudios.transcoder.internal.utils.mutableTrackMapOf
79
import com.otaliastudios.transcoder.transcode.*
810

911
internal class Segments(
1012
private val sources: DataSources,
1113
private val tracks: Tracks,
12-
private val factory: (TrackType, Int, TrackStatus) -> TrackTranscoder
14+
private val factory: (TrackType, Int, TrackStatus, MediaFormat) -> Pipeline
1315
) {
1416

1517
private val current = mutableTrackMapOf<Segment>(null, null)
@@ -72,8 +74,12 @@ internal class Segments(
7274
return Segment(
7375
type = type,
7476
index = index,
75-
transcoder = factory(type, index, tracks.all[type]),
76-
outputFormat = tracks.outputFormats[type]
77+
pipeline = factory(
78+
type,
79+
index,
80+
tracks.all[type],
81+
tracks.outputFormats[type]
82+
),
7783
)
7884
}
7985

‎lib/src/main/java/com/otaliastudios/transcoder/internal/codec/Decoder.kt‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
package com.otaliastudios.transcoder.internal.codec
2+
3+
import android.media.MediaCodec.*
4+
import android.media.MediaFormat
5+
import android.view.Surface
6+
import com.otaliastudios.transcoder.internal.data.ReaderChannel
7+
import com.otaliastudios.transcoder.internal.data.ReaderData
8+
import com.otaliastudios.transcoder.internal.media.MediaCodecBuffers
9+
import com.otaliastudios.transcoder.internal.pipeline.BaseStep
10+
import com.otaliastudios.transcoder.internal.pipeline.Channel
11+
import com.otaliastudios.transcoder.internal.pipeline.State
12+
import java.nio.ByteBuffer
13+
14+
15+
internal data class DecoderData(
16+
val buffer: ByteBuffer,
17+
val timeUs: Long,
18+
val release: (render: Boolean) -> Unit
19+
)
20+
21+
internal interface DecoderChannel : Channel {
22+
fun handleSourceFormat(format: MediaFormat): Surface?
23+
fun handleTargetFormat(format: MediaFormat)
24+
}
25+
26+
internal class Decoder(
27+
private val format: MediaFormat, // source.getTrackFormat(track)
28+
) : BaseStep<ReaderData, ReaderChannel, DecoderData, DecoderChannel>(), ReaderChannel {
29+
30+
override val channel = this
31+
private val codec = createDecoderByType(format.getString(MediaFormat.KEY_MIME)!!)
32+
private val buffers by lazy { MediaCodecBuffers(codec) }
33+
private var info = BufferInfo()
34+
private var inputEosSent = false
35+
36+
override fun initialize(next: DecoderChannel) {
37+
super.initialize(next)
38+
val surface = next.handleSourceFormat(format)
39+
codec.configure(format, surface, null, 0)
40+
codec.start()
41+
}
42+
43+
override fun buffer(): Pair<ByteBuffer, Int>? {
44+
val id = codec.dequeueInputBuffer(0)
45+
return if (id >= 0) buffers.getInputBuffer(id) to id else null
46+
}
47+
48+
override fun step(state: State.Ok<ReaderData>): State<DecoderData> {
49+
// Input - feedDecoder
50+
if (state is State.Eos) {
51+
if (!inputEosSent) {
52+
val flag = BUFFER_FLAG_END_OF_STREAM
53+
codec.queueInputBuffer(state.value.id, 0, 0, 0, flag)
54+
inputEosSent = true
55+
}
56+
} else {
57+
val (chunk, id) = state.value
58+
val flag = if (chunk.isKeyFrame) BUFFER_FLAG_SYNC_FRAME else 0
59+
codec.queueInputBuffer(id, 0, chunk.bytes, chunk.timestampUs, flag)
60+
}
61+
62+
// Output - drainDecoder
63+
val result = codec.dequeueOutputBuffer(info, 0)
64+
return when (result) {
65+
INFO_TRY_AGAIN_LATER -> State.Wait
66+
INFO_OUTPUT_FORMAT_CHANGED -> {
67+
next.handleTargetFormat(codec.outputFormat)
68+
State.Retry
69+
}
70+
INFO_OUTPUT_BUFFERS_CHANGED -> {
71+
buffers.onOutputBuffersChanged()
72+
State.Retry
73+
}
74+
else -> {
75+
val isEos = info.flags and BUFFER_FLAG_END_OF_STREAM != 0
76+
val hasSize = info.size > 0
77+
if (isEos || hasSize) {
78+
val buffer = buffers.getOutputBuffer(result)
79+
val timeUs = info.presentationTimeUs
80+
val data = DecoderData(buffer, timeUs) {
81+
codec.releaseOutputBuffer(result, it)
82+
}
83+
if (isEos) State.Eos(data) else State.Ok(data)
84+
} else {
85+
State.Wait
86+
}
87+
}
88+
}
89+
}
90+
91+
override fun release() {
92+
codec.stop()
93+
codec.release()
94+
}
95+
}

‎lib/src/main/java/com/otaliastudios/transcoder/internal/codec/DecoderTimer.kt‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
package com.otaliastudios.transcoder.internal.codec
2+
3+
import com.otaliastudios.transcoder.common.TrackType
4+
import com.otaliastudios.transcoder.internal.pipeline.DataStep
5+
import com.otaliastudios.transcoder.internal.pipeline.State
6+
import com.otaliastudios.transcoder.source.DataSource
7+
import com.otaliastudios.transcoder.time.TimeInterpolator
8+
9+
internal class DecoderTimer<Channel : com.otaliastudios.transcoder.internal.pipeline.Channel>(
10+
private val track: TrackType,
11+
private val interpolator: TimeInterpolator,
12+
) : DataStep<DecoderData, DecoderData, Channel>() {
13+
14+
override fun step(state: State.Ok<DecoderData>): State<DecoderData> {
15+
if (state is State.Eos) return state
16+
val timeUs = interpolator.interpolate(track, state.value.timeUs)
17+
return State.Ok(state.value.copy(timeUs = timeUs))
18+
}
19+
}

‎lib/src/main/java/com/otaliastudios/transcoder/internal/codec/Encoder.kt‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
package com.otaliastudios.transcoder.internal.codec
2+
3+
import android.media.MediaCodec.*
4+
import android.media.MediaFormat
5+
import android.view.Surface
6+
import com.otaliastudios.transcoder.internal.data.WriterChannel
7+
import com.otaliastudios.transcoder.internal.data.WriterData
8+
import com.otaliastudios.transcoder.internal.media.MediaCodecBuffers
9+
import com.otaliastudios.transcoder.internal.pipeline.BaseStep
10+
import com.otaliastudios.transcoder.internal.pipeline.Channel
11+
import com.otaliastudios.transcoder.internal.pipeline.State
12+
13+
14+
internal interface EncoderChannel : Channel {
15+
val surface: Surface?
16+
}
17+
18+
internal class Encoder(
19+
private val format: MediaFormat, // desired output format
20+
extraRotation: Int?,
21+
) : BaseStep<Unit, EncoderChannel, WriterData, WriterChannel>(), EncoderChannel {
22+
23+
override val channel = this
24+
25+
init {
26+
// TODO should be done by previous step
27+
if (extraRotation != null) {
28+
// Flip the width and height as needed. This means rotating the VideoStrategy rotation
29+
// by the amount that was set in the TranscoderOptions.
30+
// It is possible that the format has its own KEY_ROTATION, but we don't care, that will
31+
// be respected at playback time.
32+
val width = format.getInteger(MediaFormat.KEY_WIDTH)
33+
val height = format.getInteger(MediaFormat.KEY_HEIGHT)
34+
val flip = extraRotation % 180 != 0
35+
format.setInteger(MediaFormat.KEY_WIDTH, if (flip) height else width)
36+
format.setInteger(MediaFormat.KEY_HEIGHT, if (flip) width else height)
37+
}
38+
}
39+
40+
private val codec = createEncoderByType(format.getString(MediaFormat.KEY_MIME)!!).also {
41+
it.configure(format, null, null, CONFIGURE_FLAG_ENCODE)
42+
}
43+
44+
override val surface = if (extraRotation != null) codec.createInputSurface() else null
45+
private val buffers by lazy { MediaCodecBuffers(codec) }
46+
private var info = BufferInfo()
47+
private var inputEosSent = false
48+
49+
init {
50+
// TODO check if we should maybe start after [surface] has been configured
51+
codec.start()
52+
}
53+
54+
override fun step(state: State.Ok<Unit>): State<WriterData> {
55+
// Input - feedEncoder
56+
if (surface == null) {
57+
// Audio handling
58+
TODO("Do buffer communication with previous audio component!")
59+
} else {
60+
// Video handling. Nothing to do unless EOS.
61+
if (state is State.Eos && !inputEosSent) {
62+
codec.signalEndOfInputStream()
63+
inputEosSent = true
64+
}
65+
}
66+
67+
// Output - drainEncoder
68+
val result = codec.dequeueOutputBuffer(info, 0)
69+
return when (result) {
70+
INFO_TRY_AGAIN_LATER -> State.Wait
71+
INFO_OUTPUT_FORMAT_CHANGED -> {
72+
next.handleFormat(codec.outputFormat)
73+
State.Retry
74+
}
75+
INFO_OUTPUT_BUFFERS_CHANGED -> {
76+
buffers.onOutputBuffersChanged()
77+
State.Retry
78+
}
79+
else -> {
80+
val isConfig = info.flags and BUFFER_FLAG_CODEC_CONFIG != 0
81+
if (isConfig) {
82+
codec.releaseOutputBuffer(result, false)
83+
State.Retry
84+
} else {
85+
val isEos = info.flags and BUFFER_FLAG_END_OF_STREAM != 0
86+
val flags = info.flags and BUFFER_FLAG_END_OF_STREAM.inv()
87+
val buffer = buffers.getOutputBuffer(result)
88+
val timeUs = info.presentationTimeUs
89+
val data = WriterData(buffer, timeUs, flags)
90+
if (isEos) State.Eos(data) else State.Ok(data)
91+
}
92+
}
93+
}
94+
}
95+
96+
override fun release() {
97+
codec.stop()
98+
codec.release()
99+
}
100+
}

0 commit comments

Comments
 (0)