From 77d493c638f38222aab67c8a5959a6f89562b9fa Mon Sep 17 00:00:00 2001 From: Mattia Iavarone Date: Sat, 3 Aug 2019 01:38:46 +0200 Subject: [PATCH 01/13] Simplify Strategies to return TrackStatus --- .../transcoder/engine/Engine.java | 196 ++++++++---------- .../strategy/DefaultAudioStrategy.java | 18 +- .../strategy/DefaultVideoStrategy.java | 30 +-- .../strategy/PassThroughTrackStrategy.java | 8 +- .../strategy/RemoveTrackStrategy.java | 8 +- .../transcoder/strategy/TrackStrategy.java | 18 +- .../strategy/TrackStrategyException.java | 63 ------ 7 files changed, 136 insertions(+), 205 deletions(-) delete mode 100644 lib/src/main/java/com/otaliastudios/transcoder/strategy/TrackStrategyException.java diff --git a/lib/src/main/java/com/otaliastudios/transcoder/engine/Engine.java b/lib/src/main/java/com/otaliastudios/transcoder/engine/Engine.java index 5f638a35..dbad64b9 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/engine/Engine.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/engine/Engine.java @@ -25,7 +25,6 @@ import com.otaliastudios.transcoder.sink.MediaMuxerDataSink; import com.otaliastudios.transcoder.source.DataSource; import com.otaliastudios.transcoder.strategy.TrackStrategy; -import com.otaliastudios.transcoder.strategy.TrackStrategyException; import com.otaliastudios.transcoder.transcode.AudioTrackTranscoder; import com.otaliastudios.transcoder.transcode.NoOpTrackTranscoder; import com.otaliastudios.transcoder.transcode.PassThroughTrackTranscoder; @@ -80,6 +79,13 @@ public double getProgress() { return mProgress; } + private void setProgress(double progress) { + mProgress = progress; + if (mProgressCallback != null) { + mProgressCallback.onProgress(progress); + } + } + /** * Performs transcoding. Blocks current thread. * @@ -88,22 +94,64 @@ public double getProgress() { * @throws InterruptedException when cancel to transcode */ public void transcode(@NonNull TranscoderOptions options) throws InterruptedException { + mDataSink = new MediaMuxerDataSink(options.getOutputPath()); + + // Pass metadata from DataSource to DataSink + mDataSink.setOrientation((mDataSource.getOrientation() + options.getRotation()) % 360); + double[] location = mDataSource.getLocation(); + if (location != null) mDataSink.setLocation(location[0], location[1]); + mDurationUs = mDataSource.getDurationUs(); + LOG.v("Duration (us): " + mDurationUs); + + // Set up transcoders. + int tracks = 0; + setUpTrackTranscoder(TrackType.VIDEO, options.getVideoTrackStrategy(), options); + setUpTrackTranscoder(TrackType.AUDIO, options.getAudioTrackStrategy(), options); + TrackStatus videoStatus = mStatuses.require(TrackType.VIDEO); + TrackStatus audioStatus = mStatuses.require(TrackType.AUDIO); + TrackTranscoder videoTranscoder = mTranscoders.require(TrackType.VIDEO); + TrackTranscoder audioTranscoder = mTranscoders.require(TrackType.AUDIO); + if (videoStatus.isTranscoding()) tracks++; + if (audioStatus.isTranscoding()) tracks++; + tracks = Math.max(1, tracks); + + // Pass to Validator. + //noinspection UnusedAssignment + boolean ignoreValidatorResult = false; + // If we have to apply some rotation, and the video should be transcoded, + // ignore any Validator trying to abort the operation. The operation must happen + // because we must apply the rotation. + ignoreValidatorResult = videoStatus.isTranscoding() && options.getRotation() != 0; + if (!options.getValidator().validate(videoStatus, audioStatus) && !ignoreValidatorResult) { + throw new ValidatorException("Validator returned false."); + } + + // Do the actual transcoding work. + long loopCount = 0; + if (mDurationUs <= 0) { + setProgress(PROGRESS_UNKNOWN); + } try { - // NOTE: use single extractor to keep from running out audio track fast. - mDataSink = new MediaMuxerDataSink(options.getOutputPath()); - mDataSink.setOrientation((mDataSource.getOrientation() + options.getRotation()) % 360); - double[] location = mDataSource.getLocation(); - if (location != null) mDataSink.setLocation(location[0], location[1]); - mDurationUs = mDataSource.getDurationUs(); - LOG.v("Duration (us): " + mDurationUs); - setUpTrackTranscoders(options); - runPipelines(); + while (!(videoTranscoder.isFinished() && audioTranscoder.isFinished())) { + if (Thread.interrupted()) { + throw new InterruptedException(); + } + boolean stepped = videoTranscoder.transcode() || audioTranscoder.transcode(); + if (mDurationUs > 0 && ++loopCount % PROGRESS_INTERVAL_STEPS == 0) { + double videoProgress = getTranscoderProgress(videoTranscoder, videoStatus); + double audioProgress = getTranscoderProgress(audioTranscoder, audioStatus); + LOG.i("progress - video:" + videoProgress + " audio:" + audioProgress); + setProgress((videoProgress + audioProgress) / tracks); + } + if (!stepped) { + Thread.sleep(SLEEP_TO_WAIT_TRACK_TRANSCODERS); + } + } mDataSink.stop(); } finally { try { - mTranscoders.require(TrackType.VIDEO).release(); - mTranscoders.require(TrackType.AUDIO).release(); - mTranscoders.clear(); + videoTranscoder.release(); + audioTranscoder.release(); } catch (RuntimeException e) { // Too fatal to make alive the app, because it may leak native resources. //noinspection ThrowFromFinallyBlock @@ -113,104 +161,45 @@ public void transcode(@NonNull TranscoderOptions options) throws InterruptedExce } } - private void setUpTrackTranscoder(@NonNull TranscoderOptions options, - @NonNull TrackType type) { - TrackStatus status; - TrackTranscoder transcoder; - MediaFormat inputFormat = mDataSource.getFormat(type); - MediaFormat outputFormat = null; - if (inputFormat == null) { - transcoder = new NoOpTrackTranscoder(); - status = TrackStatus.ABSENT; - } else { - TrackStrategy strategy; - switch (type) { - case VIDEO: strategy = options.getVideoTrackStrategy(); break; - case AUDIO: strategy = options.getAudioTrackStrategy(); break; - default: throw new RuntimeException("Unknown type: " + type); - } - try { - outputFormat = strategy.createOutputFormat(inputFormat); - if (outputFormat == null) { - transcoder = new NoOpTrackTranscoder(); - status = TrackStatus.REMOVING; - } else if (outputFormat == inputFormat) { - transcoder = new PassThroughTrackTranscoder(mDataSource, mDataSink, type, options.getTimeInterpolator()); - status = TrackStatus.PASS_THROUGH; - } else { - switch (type) { - case VIDEO: transcoder = new VideoTrackTranscoder(mDataSource, mDataSink, options.getTimeInterpolator()); break; - case AUDIO: transcoder = new AudioTrackTranscoder(mDataSource, mDataSink, options.getTimeInterpolator(), options.getAudioStretcher()); break; - default: throw new RuntimeException("Unknown type: " + type); - } - status = TrackStatus.COMPRESSING; + private void setUpTrackTranscoder(@NonNull TrackType type, + @NonNull TrackStrategy strategy, + @NonNull TranscoderOptions options) { + TrackStatus status = TrackStatus.ABSENT; + TrackTranscoder transcoder = new NoOpTrackTranscoder(); + final MediaFormat inputFormat = mDataSource.getFormat(type); + MediaFormat outputFormat = new MediaFormat(); + if (inputFormat != null) { + status = strategy.createOutputFormat(inputFormat, outputFormat); + switch (status) { + case ABSENT: throw new IllegalArgumentException("Strategies should not return ABSENT."); + case REMOVING: break; // We'll use NoOpTrackTranscoder. + case PASS_THROUGH: { + transcoder = new PassThroughTrackTranscoder(mDataSource, + mDataSink, type, options.getTimeInterpolator()); + break; } - } catch (TrackStrategyException strategyException) { - if (strategyException.getType() == TrackStrategyException.TYPE_ALREADY_COMPRESSED) { - // Should not abort, because the other track might need compression. Use a pass through. - transcoder = new PassThroughTrackTranscoder(mDataSource, mDataSink, type, options.getTimeInterpolator()); - status = TrackStatus.PASS_THROUGH; - } else { // Abort. - throw strategyException; + case COMPRESSING: { + transcoder = createTrackTranscoder(type, options); + break; } } } mDataSource.setTrackStatus(type, status); mDataSink.setTrackStatus(type, status); mStatuses.set(type, status); - // Just to respect nullability in setUp(). - if (outputFormat == null) outputFormat = new MediaFormat(); transcoder.setUp(outputFormat); mTranscoders.set(type, transcoder); } - private void setUpTrackTranscoders(@NonNull TranscoderOptions options) { - setUpTrackTranscoder(options, TrackType.VIDEO); - setUpTrackTranscoder(options, TrackType.AUDIO); - - TrackStatus videoStatus = mStatuses.require(TrackType.VIDEO); - TrackStatus audioStatus = mStatuses.require(TrackType.AUDIO); - //noinspection UnusedAssignment - boolean ignoreValidatorResult = false; - - // If we have to apply some rotation, and the video should be transcoded, - // ignore any Validator trying to abort the operation. The operation must happen - // because we must apply the rotation. - ignoreValidatorResult = videoStatus.isTranscoding() && options.getRotation() != 0; - - // Validate and go on. - if (!options.getValidator().validate(videoStatus, audioStatus) - && !ignoreValidatorResult) { - throw new ValidatorException("Validator returned false."); - } - } - - private void runPipelines() throws InterruptedException { - long loopCount = 0; - if (mDurationUs <= 0) { - double progress = PROGRESS_UNKNOWN; - mProgress = progress; - if (mProgressCallback != null) mProgressCallback.onProgress(progress); // unknown - } - TrackTranscoder videoTranscoder = mTranscoders.require(TrackType.VIDEO); - TrackTranscoder audioTranscoder = mTranscoders.require(TrackType.AUDIO); - while (!(videoTranscoder.isFinished() && audioTranscoder.isFinished())) { - if (Thread.interrupted()) { - throw new InterruptedException(); - } - boolean stepped = videoTranscoder.transcode() || audioTranscoder.transcode(); - loopCount++; - if (mDurationUs > 0 && loopCount % PROGRESS_INTERVAL_STEPS == 0) { - double videoProgress = getTranscoderProgress(videoTranscoder, mStatuses.require(TrackType.VIDEO)); - double audioProgress = getTranscoderProgress(audioTranscoder, mStatuses.require(TrackType.AUDIO)); - LOG.i("progress - video:" + videoProgress + " audio:" + audioProgress); - double progress = (videoProgress + audioProgress) / getTranscodersCount(); - mProgress = progress; - if (mProgressCallback != null) mProgressCallback.onProgress(progress); - } - if (!stepped) { - Thread.sleep(SLEEP_TO_WAIT_TRACK_TRANSCODERS); - } + @NonNull + private TrackTranscoder createTrackTranscoder(@NonNull TrackType type, + @NonNull TranscoderOptions options) { + switch (type) { + case VIDEO: return new VideoTrackTranscoder(mDataSource, mDataSink, + options.getTimeInterpolator()); + case AUDIO: return new AudioTrackTranscoder(mDataSource, mDataSink, + options.getTimeInterpolator(), options.getAudioStretcher()); + default: throw new RuntimeException("Unknown type: " + type); } } @@ -219,11 +208,4 @@ private double getTranscoderProgress(@NonNull TrackTranscoder transcoder, @NonNu if (transcoder.isFinished()) return 1.0; return Math.min(1.0, (double) transcoder.getLastPresentationTime() / mDurationUs); } - - private int getTranscodersCount() { - int count = 0; - if (mStatuses.require(TrackType.AUDIO).isTranscoding()) count++; - if (mStatuses.require(TrackType.VIDEO).isTranscoding()) count++; - return (count > 0) ? count : 1; - } } diff --git a/lib/src/main/java/com/otaliastudios/transcoder/strategy/DefaultAudioStrategy.java b/lib/src/main/java/com/otaliastudios/transcoder/strategy/DefaultAudioStrategy.java index 9dd6669f..95d346d5 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/strategy/DefaultAudioStrategy.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/strategy/DefaultAudioStrategy.java @@ -3,10 +3,10 @@ import android.media.MediaCodecInfo; import android.media.MediaFormat; +import com.otaliastudios.transcoder.engine.TrackStatus; import com.otaliastudios.transcoder.internal.MediaFormatConstants; import androidx.annotation.NonNull; -import androidx.annotation.Nullable; /** * An {@link TrackStrategy} for audio that converts it to AAC with the given number @@ -22,15 +22,17 @@ public DefaultAudioStrategy(int channels) { this.channels = channels; } - @Nullable + @NonNull @Override - public MediaFormat createOutputFormat(@NonNull MediaFormat inputFormat) throws TrackStrategyException { + public TrackStatus createOutputFormat(@NonNull MediaFormat inputFormat, @NonNull MediaFormat outputFormat) { int inputChannels = inputFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT); int outputChannels = (channels == AUDIO_CHANNELS_AS_IS) ? inputChannels : channels; - final MediaFormat format = MediaFormat.createAudioFormat(MediaFormatConstants.MIMETYPE_AUDIO_AAC, - inputFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE), outputChannels); - format.setInteger(MediaFormat.KEY_AAC_PROFILE, MediaCodecInfo.CodecProfileLevel.AACObjectLC); - format.setInteger(MediaFormat.KEY_BIT_RATE, inputFormat.getInteger(MediaFormat.KEY_BIT_RATE)); - return format; + outputFormat.setString(MediaFormat.KEY_MIME, MediaFormatConstants.MIMETYPE_AUDIO_AAC); + outputFormat.setInteger(MediaFormat.KEY_SAMPLE_RATE, inputFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE)); + outputFormat.setInteger(MediaFormat.KEY_CHANNEL_COUNT, outputChannels); + outputFormat.setInteger(MediaFormat.KEY_AAC_PROFILE, MediaCodecInfo.CodecProfileLevel.AACObjectLC); + outputFormat.setInteger(MediaFormat.KEY_BIT_RATE, inputFormat.getInteger(MediaFormat.KEY_BIT_RATE)); + return TrackStatus.COMPRESSING; } + } diff --git a/lib/src/main/java/com/otaliastudios/transcoder/strategy/DefaultVideoStrategy.java b/lib/src/main/java/com/otaliastudios/transcoder/strategy/DefaultVideoStrategy.java index 51da74f1..3bd5a04e 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/strategy/DefaultVideoStrategy.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/strategy/DefaultVideoStrategy.java @@ -4,6 +4,7 @@ import android.media.MediaFormat; import android.os.Build; +import com.otaliastudios.transcoder.engine.TrackStatus; import com.otaliastudios.transcoder.strategy.size.AspectRatioResizer; import com.otaliastudios.transcoder.strategy.size.AtMostResizer; import com.otaliastudios.transcoder.strategy.size.ExactResizer; @@ -16,7 +17,6 @@ import com.otaliastudios.transcoder.internal.MediaFormatConstants; import androidx.annotation.NonNull; -import androidx.annotation.Nullable; /** * An {@link TrackStrategy} for video that converts it AVC with the given size. @@ -203,9 +203,9 @@ public DefaultVideoStrategy(@NonNull Options options) { this.options = options; } - @Nullable + @NonNull @Override - public MediaFormat createOutputFormat(@NonNull MediaFormat inputFormat) throws TrackStrategyException { + public TrackStatus createOutputFormat(@NonNull MediaFormat inputFormat, @NonNull MediaFormat outputFormat) { boolean typeDone = inputFormat.getString(MediaFormat.KEY_MIME).equals(MIME_TYPE); // Compute output size. @@ -217,7 +217,7 @@ public MediaFormat createOutputFormat(@NonNull MediaFormat inputFormat) throws T try { outSize = options.resizer.getOutputSize(inSize); } catch (Exception e) { - throw TrackStrategyException.unavailable(e); + throw new RuntimeException("Resizer error:", e); } int outWidth, outHeight; if (outSize instanceof ExactSize) { @@ -251,27 +251,29 @@ public MediaFormat createOutputFormat(@NonNull MediaFormat inputFormat) throws T } boolean frameIntervalDone = inputIFrameInterval >= options.targetIFrameInterval; - // See if we should go on. + // See if we should go on or if we're already compressed. if (typeDone && sizeDone && frameRateDone && frameIntervalDone) { - throw TrackStrategyException.alreadyCompressed( - "Input minSize: " + inSize.getMinor() + ", desired minSize: " + outSize.getMinor() + + LOG.i("Input minSize: " + inSize.getMinor() + ", desired minSize: " + outSize.getMinor() + "\nInput frameRate: " + inputFrameRate + ", desired frameRate: " + outFrameRate + "\nInput iFrameInterval: " + inputIFrameInterval + ", desired iFrameInterval: " + options.targetIFrameInterval); + return TrackStatus.PASS_THROUGH; } // Create the actual format. - MediaFormat format = MediaFormat.createVideoFormat(MIME_TYPE, outWidth, outHeight); - format.setInteger(MediaFormat.KEY_FRAME_RATE, outFrameRate); + outputFormat.setString(MediaFormat.KEY_MIME, MIME_TYPE); + outputFormat.setInteger(MediaFormat.KEY_WIDTH, outWidth); + outputFormat.setInteger(MediaFormat.KEY_HEIGHT, outHeight); + outputFormat.setInteger(MediaFormat.KEY_FRAME_RATE, outFrameRate); if (Build.VERSION.SDK_INT >= 25) { - format.setFloat(MediaFormat.KEY_I_FRAME_INTERVAL, options.targetIFrameInterval); + outputFormat.setFloat(MediaFormat.KEY_I_FRAME_INTERVAL, options.targetIFrameInterval); } else { - format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, (int) Math.ceil(options.targetIFrameInterval)); + outputFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, (int) Math.ceil(options.targetIFrameInterval)); } - format.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface); + outputFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface); int outBitRate = (int) (options.targetBitRate == BITRATE_UNKNOWN ? estimateBitRate(outWidth, outHeight, outFrameRate) : options.targetBitRate); - format.setInteger(MediaFormat.KEY_BIT_RATE, outBitRate); - return format; + outputFormat.setInteger(MediaFormat.KEY_BIT_RATE, outBitRate); + return TrackStatus.COMPRESSING; } // Depends on the codec, but for AVC this is a reasonable default ? diff --git a/lib/src/main/java/com/otaliastudios/transcoder/strategy/PassThroughTrackStrategy.java b/lib/src/main/java/com/otaliastudios/transcoder/strategy/PassThroughTrackStrategy.java index bcdd4269..e54c0ce3 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/strategy/PassThroughTrackStrategy.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/strategy/PassThroughTrackStrategy.java @@ -5,6 +5,8 @@ import androidx.annotation.NonNull; import androidx.annotation.Nullable; +import com.otaliastudios.transcoder.engine.TrackStatus; + /** * An {@link TrackStrategy} that asks the encoder to keep this track as is. * Note that this is risky, as the track type might not be supported by @@ -13,9 +15,9 @@ @SuppressWarnings("unused") public class PassThroughTrackStrategy implements TrackStrategy { - @Nullable + @NonNull @Override - public MediaFormat createOutputFormat(@NonNull MediaFormat inputFormat) throws TrackStrategyException { - return inputFormat; + public TrackStatus createOutputFormat(@NonNull MediaFormat inputFormat, @NonNull MediaFormat outputFormat) { + return TrackStatus.PASS_THROUGH; } } diff --git a/lib/src/main/java/com/otaliastudios/transcoder/strategy/RemoveTrackStrategy.java b/lib/src/main/java/com/otaliastudios/transcoder/strategy/RemoveTrackStrategy.java index d3515197..bc4c33ee 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/strategy/RemoveTrackStrategy.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/strategy/RemoveTrackStrategy.java @@ -5,15 +5,17 @@ import androidx.annotation.NonNull; import androidx.annotation.Nullable; +import com.otaliastudios.transcoder.engine.TrackStatus; + /** * An {@link TrackStrategy} that removes this track from output. */ @SuppressWarnings("unused") public class RemoveTrackStrategy implements TrackStrategy { - @Nullable + @NonNull @Override - public MediaFormat createOutputFormat(@NonNull MediaFormat inputFormat) throws TrackStrategyException { - return null; + public TrackStatus createOutputFormat(@NonNull MediaFormat inputFormat, @NonNull MediaFormat outputFormat) { + return TrackStatus.REMOVING; } } diff --git a/lib/src/main/java/com/otaliastudios/transcoder/strategy/TrackStrategy.java b/lib/src/main/java/com/otaliastudios/transcoder/strategy/TrackStrategy.java index a26293fc..0067053f 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/strategy/TrackStrategy.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/strategy/TrackStrategy.java @@ -2,6 +2,7 @@ import android.media.MediaFormat; +import com.otaliastudios.transcoder.engine.TrackStatus; import com.otaliastudios.transcoder.strategy.size.Resizer; import androidx.annotation.NonNull; @@ -16,14 +17,17 @@ public interface TrackStrategy { /** * Create the output format for this track (either audio or video). - * Implementors can: - * - throw a {@link TrackStrategyException} if the whole transcoding should be aborted - * - return {@code inputFormat} for remuxing this track as-is - * - returning {@code null} for removing this track from output + * Implementors should fill the outputFormat object and return a non-null {@link TrackStatus}: + * - {@link TrackStatus#COMPRESSING}: we want to compress this track. Output format will be used + * - {@link TrackStatus#PASS_THROUGH}: we want to use the input format. Output format will be ignored + * - {@link TrackStatus#REMOVING}: we want to remove this track. Output format will be ignored + * + * Subclasses can also throw to abort the whole transcoding operation. * * @param inputFormat the input format - * @return the output format + * @param outputFormat the output format to be filled + * @return the track status */ - @Nullable - MediaFormat createOutputFormat(@NonNull MediaFormat inputFormat) throws TrackStrategyException; + @NonNull + TrackStatus createOutputFormat(@NonNull MediaFormat inputFormat, @NonNull MediaFormat outputFormat); } diff --git a/lib/src/main/java/com/otaliastudios/transcoder/strategy/TrackStrategyException.java b/lib/src/main/java/com/otaliastudios/transcoder/strategy/TrackStrategyException.java deleted file mode 100644 index 5a009724..00000000 --- a/lib/src/main/java/com/otaliastudios/transcoder/strategy/TrackStrategyException.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright (C) 2014 Yuya Tanaka - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.otaliastudios.transcoder.strategy; - -import androidx.annotation.NonNull; -import androidx.annotation.Nullable; - -/** - * Base class for exceptions thrown by {@link TrackStrategy} by any - * strategy implementors. - * - * These are later caught internally. - */ -public class TrackStrategyException extends RuntimeException { - - @SuppressWarnings("WeakerAccess") - public final static int TYPE_UNAVAILABLE = 0; - - public final static int TYPE_ALREADY_COMPRESSED = 1; - - private int type; - - @SuppressWarnings("WeakerAccess") - public TrackStrategyException(int type, @Nullable String detailMessage) { - super(detailMessage); - this.type = type; - } - - @SuppressWarnings("WeakerAccess") - public TrackStrategyException(int type, @Nullable Exception cause) { - super(cause); - this.type = type; - } - - public int getType() { - return type; - } - - @NonNull - @SuppressWarnings("WeakerAccess") - public static TrackStrategyException unavailable(@Nullable Exception cause) { - return new TrackStrategyException(TYPE_UNAVAILABLE, cause); - } - - @NonNull - @SuppressWarnings("WeakerAccess") - public static TrackStrategyException alreadyCompressed(@Nullable String detailMessage) { - return new TrackStrategyException(TYPE_ALREADY_COMPRESSED, detailMessage); - } -} From 79a8280e0633eea98a2e3ba8de4d98ab48d2b76d Mon Sep 17 00:00:00 2001 From: Mattia Iavarone Date: Sat, 3 Aug 2019 01:54:22 +0200 Subject: [PATCH 02/13] Comment sources --- .../transcoder/engine/Engine.java | 4 +- .../transcoder/sink/DataSink.java | 20 ++--- .../transcoder/sink/MediaMuxerDataSink.java | 24 ++---- .../transcoder/source/AndroidDataSource.java | 24 +++--- .../transcoder/source/DataSource.java | 73 +++++++++++++++---- .../source/FileDescriptorDataSource.java | 5 +- .../transcoder/source/FilePathDataSource.java | 17 ++--- .../transcoder/source/UriDataSource.java | 5 +- .../transcode/BaseTrackTranscoder.java | 10 +-- .../transcode/PassThroughTrackTranscoder.java | 13 ++-- 10 files changed, 108 insertions(+), 87 deletions(-) diff --git a/lib/src/main/java/com/otaliastudios/transcoder/engine/Engine.java b/lib/src/main/java/com/otaliastudios/transcoder/engine/Engine.java index dbad64b9..c8314d07 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/engine/Engine.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/engine/Engine.java @@ -166,7 +166,7 @@ private void setUpTrackTranscoder(@NonNull TrackType type, @NonNull TranscoderOptions options) { TrackStatus status = TrackStatus.ABSENT; TrackTranscoder transcoder = new NoOpTrackTranscoder(); - final MediaFormat inputFormat = mDataSource.getFormat(type); + final MediaFormat inputFormat = mDataSource.getTrackFormat(type); MediaFormat outputFormat = new MediaFormat(); if (inputFormat != null) { status = strategy.createOutputFormat(inputFormat, outputFormat); @@ -184,7 +184,7 @@ private void setUpTrackTranscoder(@NonNull TrackType type, } } } - mDataSource.setTrackStatus(type, status); + if (status.isTranscoding()) mDataSource.selectTrack(type); mDataSink.setTrackStatus(type, status); mStatuses.set(type, status); transcoder.setUp(outputFormat); diff --git a/lib/src/main/java/com/otaliastudios/transcoder/sink/DataSink.java b/lib/src/main/java/com/otaliastudios/transcoder/sink/DataSink.java index b92c69c4..e45ce4ce 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/sink/DataSink.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/sink/DataSink.java @@ -50,27 +50,21 @@ void setTrackStatus(@NonNull TrackType type, * This is not the output format chosen by the library user but rather the * output format determined by {@link MediaCodec}, which contains more information, * and should be inspected to know what kind of data we're collecting. - * - * @param transcoder the transcoder - * @param type the track type + * @param type the track type * @param format the track format */ - void setTrackOutputFormat(@NonNull TrackTranscoder transcoder, - @NonNull TrackType type, - @NonNull MediaFormat format); + void setTrackFormat(@NonNull TrackType type, + @NonNull MediaFormat format); /** * Called by {@link TrackTranscoder}s to write data into this sink. - * - * @param transcoder the transcoder - * @param type the track type + * @param type the track type * @param byteBuffer the data * @param bufferInfo the metadata */ - void write(@NonNull TrackTranscoder transcoder, - @NonNull TrackType type, - @NonNull ByteBuffer byteBuffer, - @NonNull MediaCodec.BufferInfo bufferInfo); + void writeTrack(@NonNull TrackType type, + @NonNull ByteBuffer byteBuffer, + @NonNull MediaCodec.BufferInfo bufferInfo); /** * Called when transcoders have stopped writing. diff --git a/lib/src/main/java/com/otaliastudios/transcoder/sink/MediaMuxerDataSink.java b/lib/src/main/java/com/otaliastudios/transcoder/sink/MediaMuxerDataSink.java index 22700576..7b2fa5d9 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/sink/MediaMuxerDataSink.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/sink/MediaMuxerDataSink.java @@ -11,8 +11,6 @@ import com.otaliastudios.transcoder.engine.TrackType; import com.otaliastudios.transcoder.internal.TrackTypeMap; import com.otaliastudios.transcoder.internal.Logger; -import com.otaliastudios.transcoder.transcode.PassThroughTrackTranscoder; -import com.otaliastudios.transcoder.transcode.TrackTranscoder; import java.io.IOException; import java.nio.ByteBuffer; @@ -34,16 +32,13 @@ public class MediaMuxerDataSink implements DataSink { * the muxer is still being started (waiting for output formats). */ private static class QueuedSample { - private final TrackTranscoder mTranscoder; private final TrackType mType; private final int mSize; private final long mTimeUs; private final int mFlags; - private QueuedSample(@NonNull TrackTranscoder transcoder, - @NonNull TrackType type, + private QueuedSample(@NonNull TrackType type, @NonNull MediaCodec.BufferInfo bufferInfo) { - mTranscoder = transcoder; mType = type; mSize = bufferInfo.size; mTimeUs = bufferInfo.presentationTimeUs; @@ -92,9 +87,8 @@ public void setTrackStatus(@NonNull TrackType type, @NonNull TrackStatus status) } @Override - public void setTrackOutputFormat(@NonNull TrackTranscoder transcoder, - @NonNull TrackType type, - @NonNull MediaFormat format) { + public void setTrackFormat(@NonNull TrackType type, + @NonNull MediaFormat format) { boolean shouldValidate = mStatus.require(type) == TrackStatus.COMPRESSING; if (shouldValidate) { mMuxerChecks.checkOutputFormat(type, format); @@ -132,11 +126,11 @@ private void startIfNeeded() { } @Override - public void write(@NonNull TrackTranscoder transcoder, @NonNull TrackType type, @NonNull ByteBuffer byteBuffer, @NonNull MediaCodec.BufferInfo bufferInfo) { + public void writeTrack(@NonNull TrackType type, @NonNull ByteBuffer byteBuffer, @NonNull MediaCodec.BufferInfo bufferInfo) { if (mMuxerStarted) { mMuxer.writeSampleData(mMuxerIndex.require(type), byteBuffer, bufferInfo); } else { - enqueue(transcoder, type, byteBuffer, bufferInfo); + enqueue(type, byteBuffer, bufferInfo); } } @@ -144,13 +138,11 @@ public void write(@NonNull TrackTranscoder transcoder, @NonNull TrackType type, * Enqueues the given byffer by writing it into our own buffer and * just storing its position and size. * - * @param transcoder transcoder * @param type track type * @param buffer input buffer * @param bufferInfo input buffer info */ - private void enqueue(@NonNull TrackTranscoder transcoder, - @NonNull TrackType type, + private void enqueue(@NonNull TrackType type, @NonNull ByteBuffer buffer, @NonNull MediaCodec.BufferInfo bufferInfo) { if (mQueueBuffer == null) { @@ -159,7 +151,7 @@ private void enqueue(@NonNull TrackTranscoder transcoder, buffer.limit(bufferInfo.offset + bufferInfo.size); buffer.position(bufferInfo.offset); mQueueBuffer.put(buffer); - mQueue.add(new QueuedSample(transcoder, type, bufferInfo)); + mQueue.add(new QueuedSample(type, bufferInfo)); } /** @@ -176,7 +168,7 @@ private void drainQueue() { int offset = 0; for (QueuedSample sample : mQueue) { bufferInfo.set(offset, sample.mSize, sample.mTimeUs, sample.mFlags); - write(sample.mTranscoder, sample.mType, mQueueBuffer, bufferInfo); + writeTrack(sample.mType, mQueueBuffer, bufferInfo); offset += sample.mSize; } mQueue.clear(); diff --git a/lib/src/main/java/com/otaliastudios/transcoder/source/AndroidDataSource.java b/lib/src/main/java/com/otaliastudios/transcoder/source/AndroidDataSource.java index 3828314e..bacbde49 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/source/AndroidDataSource.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/source/AndroidDataSource.java @@ -7,7 +7,6 @@ import androidx.annotation.NonNull; import androidx.annotation.Nullable; -import com.otaliastudios.transcoder.engine.TrackStatus; import com.otaliastudios.transcoder.engine.TrackType; import com.otaliastudios.transcoder.internal.TrackTypeMap; import com.otaliastudios.transcoder.internal.ISO6709LocationParser; @@ -30,13 +29,10 @@ public abstract class AndroidDataSource implements DataSource { private final TrackTypeMap mFormats = new TrackTypeMap<>(); private final TrackTypeMap mIndex = new TrackTypeMap<>(); - @SuppressWarnings("WeakerAccess") - protected AndroidDataSource() { } - private void ensureMetadata() { if (!mMetadataApplied) { mMetadataApplied = true; - apply(mMetadata); + applyRetriever(mMetadata); } } @@ -44,7 +40,7 @@ private void ensureExtractor() { if (!mExtractorApplied) { mExtractorApplied = true; try { - apply(mExtractor); + applyExtractor(mExtractor); } catch (IOException e) { LOG.e("Got IOException while trying to open MediaExtractor.", e); throw new RuntimeException(e); @@ -52,15 +48,13 @@ private void ensureExtractor() { } } - protected abstract void apply(@NonNull MediaExtractor extractor) throws IOException; + protected abstract void applyExtractor(@NonNull MediaExtractor extractor) throws IOException; - protected abstract void apply(@NonNull MediaMetadataRetriever retriever); + protected abstract void applyRetriever(@NonNull MediaMetadataRetriever retriever); @Override - public void setTrackStatus(@NonNull TrackType type, @NonNull TrackStatus status) { - if (status.isTranscoding()) { - mExtractor.selectTrack(mIndex.require(type)); - } + public void selectTrack(@NonNull TrackType type) { + mExtractor.selectTrack(mIndex.require(type)); } @Override @@ -70,13 +64,13 @@ public boolean isDrained() { } @Override - public boolean canRead(@NonNull TrackType type) { + public boolean canReadTrack(@NonNull TrackType type) { ensureExtractor(); return mExtractor.getSampleTrackIndex() == mIndex.require(type); } @Override - public void read(@NonNull Chunk chunk) { + public void readTrack(@NonNull Chunk chunk) { ensureExtractor(); chunk.bytes = mExtractor.readSampleData(chunk.buffer, 0); chunk.isKeyFrame = (mExtractor.getSampleFlags() & MediaExtractor.SAMPLE_FLAG_SYNC) != 0; @@ -125,7 +119,7 @@ public long getDurationUs() { @Nullable @Override - public MediaFormat getFormat(@NonNull TrackType type) { + public MediaFormat getTrackFormat(@NonNull TrackType type) { if (mFormats.has(type)) return mFormats.get(type); ensureExtractor(); int trackCount = mExtractor.getTrackCount(); diff --git a/lib/src/main/java/com/otaliastudios/transcoder/source/DataSource.java b/lib/src/main/java/com/otaliastudios/transcoder/source/DataSource.java index 158765ad..1741a2d2 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/source/DataSource.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/source/DataSource.java @@ -20,34 +20,81 @@ public interface DataSource { /** - * Called before starting to set the status for the given - * track. The source object can check if the track is transcoding - * using {@link TrackStatus#isTranscoding()}. + * Metadata information. Returns the video orientation, or 0. * - * @param type track type - * @param status status + * @return video metadata orientation */ - void setTrackStatus(@NonNull TrackType type, @NonNull TrackStatus status); - - @Nullable - MediaFormat getFormat(@NonNull TrackType type); - int getOrientation(); + /** + * Metadata information. Returns the video location, or null. + * + * @return video location or null + */ @Nullable double[] getLocation(); + /** + * Returns the video total duration in microseconds. + * + * @return duration in us + */ long getDurationUs(); - boolean isDrained(); + /** + * Called before starting to inspect the input format for this track. + * Can return null if this media does not include this track type. + * + * @param type track type + * @return format or null + */ + @Nullable + MediaFormat getTrackFormat(@NonNull TrackType type); + /** + * Called before starting, but after {@link #getTrackFormat(TrackType)}, + * to select the given track. + * + * @param type track type + */ + void selectTrack(@NonNull TrackType type); + + /** + * Returns true if we can read the given track at this point. + * If true if returned, source should expect a {@link #readTrack(Chunk)} call. + * + * @param type track type + * @return true if we can read this track now + */ @SuppressWarnings("BooleanMethodIsAlwaysInverted") - boolean canRead(@NonNull TrackType type); + boolean canReadTrack(@NonNull TrackType type); + + /** + * Called to read contents for the current track type. + * Contents should be put inside {@link DataSource.Chunk#buffer}, and the + * other chunk flags should be filled. + * + * @param chunk output chunk + */ + void readTrack(@NonNull DataSource.Chunk chunk); - void read(@NonNull DataSource.Chunk chunk); + /** + * When this source has been totally read, it can return true here to + * notify an end of input stream. + * + * @return true if drained + */ + boolean isDrained(); + /** + * Called to release resources. + */ void release(); + /** + * Represents a chunk of data. + * Can be used to read input from {@link #readTrack(Chunk)}. + */ class Chunk { public ByteBuffer buffer; public boolean isKeyFrame; diff --git a/lib/src/main/java/com/otaliastudios/transcoder/source/FileDescriptorDataSource.java b/lib/src/main/java/com/otaliastudios/transcoder/source/FileDescriptorDataSource.java index 565c196d..df5bf6a7 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/source/FileDescriptorDataSource.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/source/FileDescriptorDataSource.java @@ -17,17 +17,16 @@ public class FileDescriptorDataSource extends AndroidDataSource { private FileDescriptor descriptor; public FileDescriptorDataSource(@NonNull FileDescriptor descriptor) { - super(); this.descriptor = descriptor; } @Override - public void apply(@NonNull MediaExtractor extractor) throws IOException { + public void applyExtractor(@NonNull MediaExtractor extractor) throws IOException { extractor.setDataSource(descriptor); } @Override - public void apply(@NonNull MediaMetadataRetriever retriever) { + public void applyRetriever(@NonNull MediaMetadataRetriever retriever) { retriever.setDataSource(descriptor); } } diff --git a/lib/src/main/java/com/otaliastudios/transcoder/source/FilePathDataSource.java b/lib/src/main/java/com/otaliastudios/transcoder/source/FilePathDataSource.java index faefe3d0..2ae93e61 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/source/FilePathDataSource.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/source/FilePathDataSource.java @@ -10,21 +10,18 @@ import java.io.IOException; import androidx.annotation.NonNull; -import androidx.annotation.Nullable; /** * A {@link DataSource} backed by a file absolute path. */ public class FilePathDataSource extends AndroidDataSource { - private static final String TAG = "FilePathDataSource"; + private static final String TAG = FilePathDataSource.class.getSimpleName(); private static final Logger LOG = new Logger(TAG); - @NonNull - private FileDescriptorDataSource descriptor; - @Nullable private FileInputStream stream; + private final FileDescriptorDataSource descriptor; + private FileInputStream stream; public FilePathDataSource(@NonNull String path) { - super(); FileDescriptor fileDescriptor; try { stream = new FileInputStream(path); @@ -37,13 +34,13 @@ public FilePathDataSource(@NonNull String path) { } @Override - public void apply(@NonNull MediaExtractor extractor) throws IOException { - descriptor.apply(extractor); + public void applyExtractor(@NonNull MediaExtractor extractor) throws IOException { + descriptor.applyExtractor(extractor); } @Override - public void apply(@NonNull MediaMetadataRetriever retriever) { - descriptor.apply(retriever); + public void applyRetriever(@NonNull MediaMetadataRetriever retriever) { + descriptor.applyRetriever(retriever); } @Override diff --git a/lib/src/main/java/com/otaliastudios/transcoder/source/UriDataSource.java b/lib/src/main/java/com/otaliastudios/transcoder/source/UriDataSource.java index 7e5bded7..45f4937e 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/source/UriDataSource.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/source/UriDataSource.java @@ -19,18 +19,17 @@ public class UriDataSource extends AndroidDataSource { @NonNull private Uri uri; public UriDataSource(@NonNull Context context, @NonNull Uri uri) { - super(); this.context = context.getApplicationContext(); this.uri = uri; } @Override - public void apply(@NonNull MediaExtractor extractor) throws IOException { + public void applyExtractor(@NonNull MediaExtractor extractor) throws IOException { extractor.setDataSource(context, uri, null); } @Override - public void apply(@NonNull MediaMetadataRetriever retriever) { + public void applyRetriever(@NonNull MediaMetadataRetriever retriever) { retriever.setDataSource(context, uri); } } diff --git a/lib/src/main/java/com/otaliastudios/transcoder/transcode/BaseTrackTranscoder.java b/lib/src/main/java/com/otaliastudios/transcoder/transcode/BaseTrackTranscoder.java index 1cf5688f..29ef5fbb 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/transcode/BaseTrackTranscoder.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/transcode/BaseTrackTranscoder.java @@ -65,7 +65,7 @@ public final void setUp(@NonNull MediaFormat desiredOutputFormat) { onConfigureEncoder(desiredOutputFormat, mEncoder); onStartEncoder(desiredOutputFormat, mEncoder); - final MediaFormat inputFormat = mDataSource.getFormat(mTrackType); + final MediaFormat inputFormat = mDataSource.getTrackFormat(mTrackType); if (inputFormat == null) { throw new IllegalArgumentException("Input format is null!"); } @@ -198,7 +198,7 @@ protected void onEncoderOutputFormatChanged(@NonNull MediaCodec encoder, @NonNul throw new RuntimeException("Audio output format changed twice."); } mActualOutputFormat = format; - mDataSink.setTrackOutputFormat(this, mTrackType, mActualOutputFormat); + mDataSink.setTrackFormat(mTrackType, mActualOutputFormat); } @SuppressWarnings("SameParameterValue") @@ -215,7 +215,7 @@ private int feedDecoder(long timeoutUs) { return DRAIN_STATE_NONE; } - if (!mDataSource.canRead(mTrackType)) { + if (!mDataSource.canReadTrack(mTrackType)) { return DRAIN_STATE_NONE; } @@ -223,7 +223,7 @@ private int feedDecoder(long timeoutUs) { if (result < 0) return DRAIN_STATE_NONE; mDataChunk.buffer = mDecoderBuffers.getInputBuffer(result); - mDataSource.read(mDataChunk); + mDataSource.readTrack(mDataChunk); mDecoder.queueInputBuffer(result, 0, mDataChunk.bytes, @@ -294,7 +294,7 @@ private int drainEncoder(long timeoutUs) { mEncoder.releaseOutputBuffer(result, false); return DRAIN_STATE_SHOULD_RETRY_IMMEDIATELY; } - mDataSink.write(this, mTrackType, mEncoderBuffers.getOutputBuffer(result), mBufferInfo); + mDataSink.writeTrack(mTrackType, mEncoderBuffers.getOutputBuffer(result), mBufferInfo); mEncoder.releaseOutputBuffer(result, false); return DRAIN_STATE_CONSUMED; } diff --git a/lib/src/main/java/com/otaliastudios/transcoder/transcode/PassThroughTrackTranscoder.java b/lib/src/main/java/com/otaliastudios/transcoder/transcode/PassThroughTrackTranscoder.java index 1320393f..882075c1 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/transcode/PassThroughTrackTranscoder.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/transcode/PassThroughTrackTranscoder.java @@ -16,7 +16,6 @@ package com.otaliastudios.transcoder.transcode; import android.media.MediaCodec; -import android.media.MediaExtractor; import android.media.MediaFormat; import androidx.annotation.NonNull; @@ -49,7 +48,7 @@ public PassThroughTrackTranscoder(@NonNull DataSource dataSource, mDataSource = dataSource; mDataSink = dataSink; mTrackType = trackType; - mOutputFormat = dataSource.getFormat(trackType); + mOutputFormat = dataSource.getTrackFormat(trackType); if (mOutputFormat == null) { throw new IllegalArgumentException("Output format is null!"); } @@ -66,26 +65,26 @@ public void setUp(@NonNull MediaFormat desiredOutputFormat) { } public boolean transcode() { if (mIsEOS) return false; if (!mOutputFormatSet) { - mDataSink.setTrackOutputFormat(this, mTrackType, mOutputFormat); + mDataSink.setTrackFormat(mTrackType, mOutputFormat); mOutputFormatSet = true; } if (mDataSource.isDrained()) { mDataChunk.buffer.clear(); mBufferInfo.set(0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM); - mDataSink.write(this, mTrackType, mDataChunk.buffer, mBufferInfo); + mDataSink.writeTrack(mTrackType, mDataChunk.buffer, mBufferInfo); mIsEOS = true; return true; } - if (!mDataSource.canRead(mTrackType)) { + if (!mDataSource.canReadTrack(mTrackType)) { return false; } mDataChunk.buffer.clear(); - mDataSource.read(mDataChunk); + mDataSource.readTrack(mDataChunk); long timestampUs = mTimeInterpolator.interpolate(mTrackType, mDataChunk.timestampUs); int flags = mDataChunk.isKeyFrame ? MediaCodec.BUFFER_FLAG_SYNC_FRAME : 0; mBufferInfo.set(0, mDataChunk.bytes, timestampUs, flags); - mDataSink.write(this, mTrackType, mDataChunk.buffer, mBufferInfo); + mDataSink.writeTrack(mTrackType, mDataChunk.buffer, mBufferInfo); mLastPresentationTime = mDataChunk.timestampUs; return true; } From 573fe871d797ba381d6d9ad69517bd022a470e96 Mon Sep 17 00:00:00 2001 From: Mattia Iavarone Date: Sat, 3 Aug 2019 02:20:08 +0200 Subject: [PATCH 03/13] Refactor strategies to accept more than one input format --- .../transcoder/engine/Engine.java | 5 +- .../transcoder/internal/TrackTypeMap.java | 5 -- .../strategy/DefaultAudioStrategy.java | 39 +++++++++-- .../strategy/DefaultVideoStrategy.java | 69 +++++++++++++++---- .../strategy/PassThroughTrackStrategy.java | 4 +- .../strategy/RemoveTrackStrategy.java | 4 +- .../transcoder/strategy/TrackStrategy.java | 6 +- 7 files changed, 105 insertions(+), 27 deletions(-) diff --git a/lib/src/main/java/com/otaliastudios/transcoder/engine/Engine.java b/lib/src/main/java/com/otaliastudios/transcoder/engine/Engine.java index c8314d07..074cbed6 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/engine/Engine.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/engine/Engine.java @@ -35,6 +35,8 @@ import androidx.annotation.NonNull; import androidx.annotation.Nullable; +import java.util.Arrays; + /** * Internal engine, do not use this directly. */ @@ -169,7 +171,8 @@ private void setUpTrackTranscoder(@NonNull TrackType type, final MediaFormat inputFormat = mDataSource.getTrackFormat(type); MediaFormat outputFormat = new MediaFormat(); if (inputFormat != null) { - status = strategy.createOutputFormat(inputFormat, outputFormat); + //noinspection ArraysAsListWithZeroOrOneArgument + status = strategy.createOutputFormat(Arrays.asList(inputFormat), outputFormat); switch (status) { case ABSENT: throw new IllegalArgumentException("Strategies should not return ABSENT."); case REMOVING: break; // We'll use NoOpTrackTranscoder. diff --git a/lib/src/main/java/com/otaliastudios/transcoder/internal/TrackTypeMap.java b/lib/src/main/java/com/otaliastudios/transcoder/internal/TrackTypeMap.java index 23296702..e4ad96cc 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/internal/TrackTypeMap.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/internal/TrackTypeMap.java @@ -34,11 +34,6 @@ public T require(@NonNull TrackType type) { return map.get(type); } - @SuppressWarnings("WeakerAccess") - public void clear() { - map.clear(); - } - public boolean has(@NonNull TrackType type) { return map.containsKey(type); } diff --git a/lib/src/main/java/com/otaliastudios/transcoder/strategy/DefaultAudioStrategy.java b/lib/src/main/java/com/otaliastudios/transcoder/strategy/DefaultAudioStrategy.java index 95d346d5..4ffbc771 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/strategy/DefaultAudioStrategy.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/strategy/DefaultAudioStrategy.java @@ -8,6 +8,8 @@ import androidx.annotation.NonNull; +import java.util.List; + /** * An {@link TrackStrategy} for audio that converts it to AAC with the given number * of channels. @@ -24,15 +26,42 @@ public DefaultAudioStrategy(int channels) { @NonNull @Override - public TrackStatus createOutputFormat(@NonNull MediaFormat inputFormat, @NonNull MediaFormat outputFormat) { - int inputChannels = inputFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT); - int outputChannels = (channels == AUDIO_CHANNELS_AS_IS) ? inputChannels : channels; + public TrackStatus createOutputFormat(@NonNull List inputFormats, @NonNull MediaFormat outputFormat) { + + int outputChannels = (channels == AUDIO_CHANNELS_AS_IS) ? getInputChannelCount(inputFormats) : channels; outputFormat.setString(MediaFormat.KEY_MIME, MediaFormatConstants.MIMETYPE_AUDIO_AAC); - outputFormat.setInteger(MediaFormat.KEY_SAMPLE_RATE, inputFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE)); + outputFormat.setInteger(MediaFormat.KEY_SAMPLE_RATE, getInputSampleRate(inputFormats)); outputFormat.setInteger(MediaFormat.KEY_CHANNEL_COUNT, outputChannels); outputFormat.setInteger(MediaFormat.KEY_AAC_PROFILE, MediaCodecInfo.CodecProfileLevel.AACObjectLC); - outputFormat.setInteger(MediaFormat.KEY_BIT_RATE, inputFormat.getInteger(MediaFormat.KEY_BIT_RATE)); + outputFormat.setInteger(MediaFormat.KEY_BIT_RATE, getInputBitRate(inputFormats)); return TrackStatus.COMPRESSING; } + private int getInputChannelCount(@NonNull List formats) { + int count = 0; + for (MediaFormat format : formats) { + count = Math.max(count, format.getInteger(MediaFormat.KEY_CHANNEL_COUNT)); + } + return count; + } + + private int getInputSampleRate(@NonNull List formats) { + int rate = formats.get(0).getInteger(MediaFormat.KEY_SAMPLE_RATE); + for (MediaFormat format : formats) { + if (rate != format.getInteger(MediaFormat.KEY_SAMPLE_RATE)) { + throw new IllegalArgumentException("All input formats should have the same sample rate."); + } + } + return rate; + } + + private int getInputBitRate(@NonNull List formats) { + int rate = formats.get(0).getInteger(MediaFormat.KEY_BIT_RATE); + for (MediaFormat format : formats) { + if (rate != format.getInteger(MediaFormat.KEY_BIT_RATE)) { + throw new IllegalArgumentException("All input formats should have the same bit rate."); + } + } + return rate; + } } diff --git a/lib/src/main/java/com/otaliastudios/transcoder/strategy/DefaultVideoStrategy.java b/lib/src/main/java/com/otaliastudios/transcoder/strategy/DefaultVideoStrategy.java index 3bd5a04e..122c33dc 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/strategy/DefaultVideoStrategy.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/strategy/DefaultVideoStrategy.java @@ -18,6 +18,8 @@ import androidx.annotation.NonNull; +import java.util.List; + /** * An {@link TrackStrategy} for video that converts it AVC with the given size. * The input and output aspect ratio must match. @@ -205,12 +207,12 @@ public DefaultVideoStrategy(@NonNull Options options) { @NonNull @Override - public TrackStatus createOutputFormat(@NonNull MediaFormat inputFormat, @NonNull MediaFormat outputFormat) { - boolean typeDone = inputFormat.getString(MediaFormat.KEY_MIME).equals(MIME_TYPE); + public TrackStatus createOutputFormat(@NonNull List inputFormats, @NonNull MediaFormat outputFormat) { + boolean typeDone = checkMimeType(inputFormats); // Compute output size. - int inWidth = inputFormat.getInteger(MediaFormat.KEY_WIDTH); - int inHeight = inputFormat.getInteger(MediaFormat.KEY_HEIGHT); + int inWidth = getMaxWidth(inputFormats); + int inHeight = getMaxHeight(inputFormats); LOG.i("Input width&height: " + inWidth + "x" + inHeight); Size inSize = new ExactSize(inWidth, inHeight); Size outSize; @@ -234,21 +236,17 @@ public TrackStatus createOutputFormat(@NonNull MediaFormat inputFormat, @NonNull boolean sizeDone = inSize.getMinor() <= outSize.getMinor(); // Compute output frame rate. It can't be bigger than input frame rate. - int inputFrameRate, outFrameRate; - if (inputFormat.containsKey(MediaFormat.KEY_FRAME_RATE)) { - inputFrameRate = inputFormat.getInteger(MediaFormat.KEY_FRAME_RATE); + int outFrameRate; + int inputFrameRate = getMinFrameRate(inputFormats); + if (inputFrameRate > 0) { outFrameRate = Math.min(inputFrameRate, options.targetFrameRate); } else { - inputFrameRate = -1; outFrameRate = options.targetFrameRate; } boolean frameRateDone = inputFrameRate <= outFrameRate; // Compute i frame. - int inputIFrameInterval = -1; - if (inputFormat.containsKey(MediaFormat.KEY_I_FRAME_INTERVAL)) { - inputIFrameInterval = inputFormat.getInteger(MediaFormat.KEY_I_FRAME_INTERVAL); - } + int inputIFrameInterval = getAverageIFrameInterval(inputFormats); boolean frameIntervalDone = inputIFrameInterval >= options.targetIFrameInterval; // See if we should go on or if we're already compressed. @@ -276,6 +274,53 @@ public TrackStatus createOutputFormat(@NonNull MediaFormat inputFormat, @NonNull return TrackStatus.COMPRESSING; } + private boolean checkMimeType(@NonNull List formats) { + for (MediaFormat format : formats) { + if (!format.getString(MediaFormat.KEY_MIME).equals(MIME_TYPE)) { + return false; + } + } + return true; + } + + private int getMaxWidth(@NonNull List formats) { + int width = 0; + for (MediaFormat format : formats) { + width = Math.max(width, format.getInteger(MediaFormat.KEY_WIDTH)); + } + return width; + } + + private int getMaxHeight(@NonNull List formats) { + int height = 0; + for (MediaFormat format : formats) { + height = Math.max(height, format.getInteger(MediaFormat.KEY_HEIGHT)); + } + return height; + } + + private int getMinFrameRate(@NonNull List formats) { + int frameRate = Integer.MAX_VALUE; + for (MediaFormat format : formats) { + if (format.containsKey(MediaFormat.KEY_FRAME_RATE)) { + frameRate = Math.min(frameRate, format.getInteger(MediaFormat.KEY_FRAME_RATE)); + } + } + return (frameRate == Integer.MAX_VALUE) ? -1 : frameRate; + } + + private int getAverageIFrameInterval(@NonNull List formats) { + int count = 0; + int sum = 0; + for (MediaFormat format : formats) { + if (format.containsKey(MediaFormat.KEY_I_FRAME_INTERVAL)) { + count++; + sum += format.getInteger(MediaFormat.KEY_I_FRAME_INTERVAL); + } + } + return (count > 0) ? Math.round((float) sum / count) : -1; + } + // Depends on the codec, but for AVC this is a reasonable default ? // https://stackoverflow.com/a/5220554/4288782 private static long estimateBitRate(int width, int height, int frameRate) { diff --git a/lib/src/main/java/com/otaliastudios/transcoder/strategy/PassThroughTrackStrategy.java b/lib/src/main/java/com/otaliastudios/transcoder/strategy/PassThroughTrackStrategy.java index e54c0ce3..afab5615 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/strategy/PassThroughTrackStrategy.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/strategy/PassThroughTrackStrategy.java @@ -7,6 +7,8 @@ import com.otaliastudios.transcoder.engine.TrackStatus; +import java.util.List; + /** * An {@link TrackStrategy} that asks the encoder to keep this track as is. * Note that this is risky, as the track type might not be supported by @@ -17,7 +19,7 @@ public class PassThroughTrackStrategy implements TrackStrategy { @NonNull @Override - public TrackStatus createOutputFormat(@NonNull MediaFormat inputFormat, @NonNull MediaFormat outputFormat) { + public TrackStatus createOutputFormat(@NonNull List inputFormats, @NonNull MediaFormat outputFormat) { return TrackStatus.PASS_THROUGH; } } diff --git a/lib/src/main/java/com/otaliastudios/transcoder/strategy/RemoveTrackStrategy.java b/lib/src/main/java/com/otaliastudios/transcoder/strategy/RemoveTrackStrategy.java index bc4c33ee..38b5114f 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/strategy/RemoveTrackStrategy.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/strategy/RemoveTrackStrategy.java @@ -7,6 +7,8 @@ import com.otaliastudios.transcoder.engine.TrackStatus; +import java.util.List; + /** * An {@link TrackStrategy} that removes this track from output. */ @@ -15,7 +17,7 @@ public class RemoveTrackStrategy implements TrackStrategy { @NonNull @Override - public TrackStatus createOutputFormat(@NonNull MediaFormat inputFormat, @NonNull MediaFormat outputFormat) { + public TrackStatus createOutputFormat(@NonNull List inputFormats, @NonNull MediaFormat outputFormat) { return TrackStatus.REMOVING; } } diff --git a/lib/src/main/java/com/otaliastudios/transcoder/strategy/TrackStrategy.java b/lib/src/main/java/com/otaliastudios/transcoder/strategy/TrackStrategy.java index 0067053f..fa2df2a4 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/strategy/TrackStrategy.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/strategy/TrackStrategy.java @@ -8,6 +8,8 @@ import androidx.annotation.NonNull; import androidx.annotation.Nullable; +import java.util.List; + /** * Base class for video/audio format strategy. * Video strategies should use a {@link Resizer} instance to compute the output @@ -24,10 +26,10 @@ public interface TrackStrategy { * * Subclasses can also throw to abort the whole transcoding operation. * - * @param inputFormat the input format + * @param inputFormats the input formats * @param outputFormat the output format to be filled * @return the track status */ @NonNull - TrackStatus createOutputFormat(@NonNull MediaFormat inputFormat, @NonNull MediaFormat outputFormat); + TrackStatus createOutputFormat(@NonNull List inputFormats, @NonNull MediaFormat outputFormat); } From bbaa5afa10dad3e644700a63df084daac4ca7ef6 Mon Sep 17 00:00:00 2001 From: Mattia Iavarone Date: Sat, 3 Aug 2019 13:03:42 +0200 Subject: [PATCH 04/13] Change algorithm for merging input sizes --- .../strategy/DefaultVideoStrategy.java | 49 +++++++++++++------ 1 file changed, 35 insertions(+), 14 deletions(-) diff --git a/lib/src/main/java/com/otaliastudios/transcoder/strategy/DefaultVideoStrategy.java b/lib/src/main/java/com/otaliastudios/transcoder/strategy/DefaultVideoStrategy.java index 122c33dc..64119e35 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/strategy/DefaultVideoStrategy.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/strategy/DefaultVideoStrategy.java @@ -211,8 +211,8 @@ public TrackStatus createOutputFormat(@NonNull List inputFormats, @ boolean typeDone = checkMimeType(inputFormats); // Compute output size. - int inWidth = getMaxWidth(inputFormats); - int inHeight = getMaxHeight(inputFormats); + int inWidth = getBestInputSize(inputFormats)[0]; + int inHeight = getBestInputSize(inputFormats)[1]; LOG.i("Input width&height: " + inWidth + "x" + inHeight); Size inSize = new ExactSize(inWidth, inHeight); Size outSize; @@ -283,20 +283,41 @@ private boolean checkMimeType(@NonNull List formats) { return true; } - private int getMaxWidth(@NonNull List formats) { - int width = 0; - for (MediaFormat format : formats) { - width = Math.max(width, format.getInteger(MediaFormat.KEY_WIDTH)); + private int[] getBestInputSize(@NonNull List formats) { + int[] result = new int[2]; + int count = formats.size(); + // After thinking about it, I think the best size is the one that is closer to the + // average aspect ratio. Respect the rotation of the first video for now + float averageAspectRatio = 0; + float[] aspectRatio = new float[count]; + int firstRotation = -1; + for (int i = 0; i < count; i++) { + MediaFormat format = formats.get(i); + float width = format.getInteger(MediaFormat.KEY_WIDTH); + float height = format.getInteger(MediaFormat.KEY_HEIGHT); + int rotation = 0; + if (format.containsKey(MediaFormatConstants.KEY_ROTATION_DEGREES)) { + rotation = format.getInteger(MediaFormatConstants.KEY_ROTATION_DEGREES); + } + if (firstRotation == -1) firstRotation = 0; + boolean flip = (rotation - firstRotation + 360) % 180 != 0; + aspectRatio[i] = flip ? height / width : width / height; + averageAspectRatio += aspectRatio[i]; } - return width; - } - - private int getMaxHeight(@NonNull List formats) { - int height = 0; - for (MediaFormat format : formats) { - height = Math.max(height, format.getInteger(MediaFormat.KEY_HEIGHT)); + averageAspectRatio = averageAspectRatio / count; + float bestDelta = Float.MAX_VALUE; + int bestMatch = 0; + for (int i = 0; i < count; i++) { + float delta = Math.abs(aspectRatio[i] - averageAspectRatio); + if (delta < bestDelta) { + bestMatch = i; + bestDelta = delta; + } } - return height; + MediaFormat bestFormat = formats.get(bestMatch); + result[0] = bestFormat.getInteger(MediaFormat.KEY_WIDTH); + result[1] = bestFormat.getInteger(MediaFormat.KEY_HEIGHT); + return result; } private int getMinFrameRate(@NonNull List formats) { From 92f52c176b6656bde8d852405f9be7197eb540b3 Mon Sep 17 00:00:00 2001 From: Mattia Iavarone Date: Sat, 3 Aug 2019 13:09:30 +0200 Subject: [PATCH 05/13] Small changes --- .../strategy/DefaultVideoStrategy.java | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/lib/src/main/java/com/otaliastudios/transcoder/strategy/DefaultVideoStrategy.java b/lib/src/main/java/com/otaliastudios/transcoder/strategy/DefaultVideoStrategy.java index 64119e35..0e25c630 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/strategy/DefaultVideoStrategy.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/strategy/DefaultVideoStrategy.java @@ -207,14 +207,15 @@ public DefaultVideoStrategy(@NonNull Options options) { @NonNull @Override - public TrackStatus createOutputFormat(@NonNull List inputFormats, @NonNull MediaFormat outputFormat) { + public TrackStatus createOutputFormat(@NonNull List inputFormats, + @NonNull MediaFormat outputFormat) { boolean typeDone = checkMimeType(inputFormats); // Compute output size. - int inWidth = getBestInputSize(inputFormats)[0]; - int inHeight = getBestInputSize(inputFormats)[1]; + ExactSize inSize = getBestInputSize(inputFormats); + int inWidth = inSize.getWidth(); + int inHeight = inSize.getHeight(); LOG.i("Input width&height: " + inWidth + "x" + inHeight); - Size inSize = new ExactSize(inWidth, inHeight); Size outSize; try { outSize = options.resizer.getOutputSize(inSize); @@ -283,8 +284,7 @@ private boolean checkMimeType(@NonNull List formats) { return true; } - private int[] getBestInputSize(@NonNull List formats) { - int[] result = new int[2]; + private ExactSize getBestInputSize(@NonNull List formats) { int count = formats.size(); // After thinking about it, I think the best size is the one that is closer to the // average aspect ratio. Respect the rotation of the first video for now @@ -315,9 +315,8 @@ private int[] getBestInputSize(@NonNull List formats) { } } MediaFormat bestFormat = formats.get(bestMatch); - result[0] = bestFormat.getInteger(MediaFormat.KEY_WIDTH); - result[1] = bestFormat.getInteger(MediaFormat.KEY_HEIGHT); - return result; + return new ExactSize(bestFormat.getInteger(MediaFormat.KEY_WIDTH), + bestFormat.getInteger(MediaFormat.KEY_HEIGHT)); } private int getMinFrameRate(@NonNull List formats) { From 61acfd695e637161fc6fbaca31addb3b67613b71 Mon Sep 17 00:00:00 2001 From: Mattia Iavarone Date: Sat, 3 Aug 2019 21:26:14 +0200 Subject: [PATCH 06/13] Add multiple DataSource APIs, rename iFrameInterval to keyFrameInterval --- README.md | 2 +- .../otaliastudios/transcoder/Transcoder.java | 16 +--- .../transcoder/TranscoderOptions.java | 75 +++++++++++++++---- .../transcoder/engine/Engine.java | 61 +++++++++++---- .../transcoder/internal/TrackTypeMap.java | 36 +++++++++ .../strategy/DefaultVideoStrategies.java | 4 +- .../strategy/DefaultVideoStrategy.java | 30 ++++---- 7 files changed, 167 insertions(+), 57 deletions(-) diff --git a/README.md b/README.md index bd237caf..13cab15c 100644 --- a/README.md +++ b/README.md @@ -312,7 +312,7 @@ DefaultVideoStrategy strategy = new DefaultVideoStrategy.Builder() .bitRate(bitRate) .bitRate(DefaultVideoStrategy.BITRATE_UNKNOWN) // tries to estimate .frameRate(frameRate) // will be capped to the input frameRate - .iFrameInterval(interval) // interval between I-frames in seconds + .keyFrameInterval(interval) // interval between key-frames in seconds .build(); ``` diff --git a/lib/src/main/java/com/otaliastudios/transcoder/Transcoder.java b/lib/src/main/java/com/otaliastudios/transcoder/Transcoder.java index 55b7452d..d86921e2 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/Transcoder.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/Transcoder.java @@ -107,12 +107,12 @@ public static TranscoderOptions.Builder into(@NonNull String outPath) { @NonNull public Future transcode(@NonNull final TranscoderOptions options) { final TranscoderListener listenerWrapper = new ListenerWrapper(options.listenerHandler, - options.listener, options.getDataSource()); + options.listener); return mExecutor.submit(new Callable() { @Override public Void call() throws Exception { try { - Engine engine = new Engine(options.getDataSource(), new Engine.ProgressCallback() { + Engine engine = new Engine(new Engine.ProgressCallback() { @Override public void onProgress(final double progress) { listenerWrapper.onTranscodeProgress(progress); @@ -154,21 +154,16 @@ public void onProgress(final double progress) { } /** - * Wraps a TranscoderListener and a DataSource object, ensuring that the source - * is released when transcoding ends, fails or is canceled. - * - * It posts events on the given handler. + * Wraps a TranscoderListener and posts events on the given handler. */ private static class ListenerWrapper implements TranscoderListener { private Handler mHandler; private TranscoderListener mListener; - private DataSource mDataSource; - private ListenerWrapper(@NonNull Handler handler, @NonNull TranscoderListener listener, @NonNull DataSource source) { + private ListenerWrapper(@NonNull Handler handler, @NonNull TranscoderListener listener) { mHandler = handler; mListener = listener; - mDataSource = source; } @Override @@ -176,7 +171,6 @@ public void onTranscodeCanceled() { mHandler.post(new Runnable() { @Override public void run() { - mDataSource.release(); mListener.onTranscodeCanceled(); } }); @@ -187,7 +181,6 @@ public void onTranscodeCompleted(final int successCode) { mHandler.post(new Runnable() { @Override public void run() { - mDataSource.release(); mListener.onTranscodeCompleted(successCode); } }); @@ -198,7 +191,6 @@ public void onTranscodeFailed(@NonNull final Throwable exception) { mHandler.post(new Runnable() { @Override public void run() { - mDataSource.release(); mListener.onTranscodeFailed(exception); } }); diff --git a/lib/src/main/java/com/otaliastudios/transcoder/TranscoderOptions.java b/lib/src/main/java/com/otaliastudios/transcoder/TranscoderOptions.java index 2bccb0ce..04b92f89 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/TranscoderOptions.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/TranscoderOptions.java @@ -5,6 +5,7 @@ import android.os.Handler; import android.os.Looper; +import com.otaliastudios.transcoder.engine.TrackType; import com.otaliastudios.transcoder.source.DataSource; import com.otaliastudios.transcoder.source.FileDescriptorDataSource; import com.otaliastudios.transcoder.source.FilePathDataSource; @@ -21,6 +22,8 @@ import com.otaliastudios.transcoder.validator.Validator; import java.io.FileDescriptor; +import java.util.ArrayList; +import java.util.List; import java.util.concurrent.Future; import androidx.annotation.NonNull; @@ -34,7 +37,8 @@ public class TranscoderOptions { private TranscoderOptions() {} private String outPath; - private DataSource dataSource; + private List videoDataSources; + private List audioDataSources; private TrackStrategy audioTrackStrategy; private TrackStrategy videoTrackStrategy; private Validator validator; @@ -52,8 +56,14 @@ public String getOutputPath() { @NonNull @SuppressWarnings("WeakerAccess") - public DataSource getDataSource() { - return dataSource; + public List getAudioDataSources() { + return audioDataSources; + } + + @NonNull + @SuppressWarnings("WeakerAccess") + public List getVideoDataSources() { + return videoDataSources; } @NonNull @@ -87,7 +97,8 @@ public AudioStretcher getAudioStretcher() { public static class Builder { private String outPath; - private DataSource dataSource; + private final List audioDataSources = new ArrayList<>(); + private final List videoDataSources = new ArrayList<>(); private TranscoderListener listener; private Handler listenerHandler; private TrackStrategy audioTrackStrategy; @@ -102,31 +113,60 @@ public static class Builder { } @NonNull - @SuppressWarnings("unused") + @SuppressWarnings("WeakerAccess") public Builder setDataSource(@NonNull DataSource dataSource) { - this.dataSource = dataSource; + audioDataSources.clear(); + videoDataSources.clear(); + audioDataSources.add(dataSource); + videoDataSources.add(dataSource); + return this; + } + + @NonNull + @SuppressWarnings("WeakerAccess") + public Builder addDataSource(@NonNull TrackType type, @NonNull DataSource dataSource) { + if (type == TrackType.AUDIO) { + audioDataSources.add(dataSource); + } else if (type == TrackType.VIDEO) { + videoDataSources.add(dataSource); + } return this; } @NonNull @SuppressWarnings("unused") public Builder setDataSource(@NonNull FileDescriptor fileDescriptor) { - this.dataSource = new FileDescriptorDataSource(fileDescriptor); - return this; + return setDataSource(new FileDescriptorDataSource(fileDescriptor)); + } + + @NonNull + @SuppressWarnings("unused") + public Builder addDataSource(@NonNull TrackType type, @NonNull FileDescriptor fileDescriptor) { + return addDataSource(type, new FileDescriptorDataSource(fileDescriptor)); } @NonNull @SuppressWarnings("unused") public Builder setDataSource(@NonNull String inPath) { - this.dataSource = new FilePathDataSource(inPath); - return this; + return setDataSource(new FilePathDataSource(inPath)); + } + + @NonNull + @SuppressWarnings("unused") + public Builder addDataSource(@NonNull TrackType type, @NonNull String inPath) { + return addDataSource(type, new FilePathDataSource(inPath)); } @NonNull @SuppressWarnings("unused") public Builder setDataSource(@NonNull Context context, @NonNull Uri uri) { - this.dataSource = new UriDataSource(context, uri); - return this; + return setDataSource(new UriDataSource(context, uri)); + } + + @NonNull + @SuppressWarnings("unused") + public Builder addDataSource(@NonNull TrackType type, @NonNull Context context, @NonNull Uri uri) { + return addDataSource(type, new UriDataSource(context, uri)); } /** @@ -248,8 +288,8 @@ public TranscoderOptions build() { if (listener == null) { throw new IllegalStateException("listener can't be null"); } - if (dataSource == null) { - throw new IllegalStateException("data source can't be null"); + if (audioDataSources.isEmpty() && videoDataSources.isEmpty()) { + throw new IllegalStateException("we need at least one data source"); } if (outPath == null) { throw new IllegalStateException("out path can't be null"); @@ -257,6 +297,10 @@ public TranscoderOptions build() { if (rotation != 0 && rotation != 90 && rotation != 180 && rotation != 270) { throw new IllegalArgumentException("Accepted values for rotation are 0, 90, 180, 270"); } + if (rotation != 0 && videoDataSources.size() > 1) { + // TODO support this, by not using metadata and instead rotating the GL texture + throw new IllegalStateException("Rotation should be 0 if you have more than one video source."); + } if (listenerHandler == null) { Looper looper = Looper.myLooper(); if (looper == null) looper = Looper.getMainLooper(); @@ -279,7 +323,8 @@ public TranscoderOptions build() { } TranscoderOptions options = new TranscoderOptions(); options.listener = listener; - options.dataSource = dataSource; + options.audioDataSources = audioDataSources; + options.videoDataSources = videoDataSources; options.outPath = outPath; options.listenerHandler = listenerHandler; options.audioTrackStrategy = audioTrackStrategy; diff --git a/lib/src/main/java/com/otaliastudios/transcoder/engine/Engine.java b/lib/src/main/java/com/otaliastudios/transcoder/engine/Engine.java index 074cbed6..b0f7fda5 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/engine/Engine.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/engine/Engine.java @@ -36,6 +36,9 @@ import androidx.annotation.Nullable; import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; /** * Internal engine, do not use this directly. @@ -59,16 +62,15 @@ public interface ProgressCallback { void onProgress(double progress); } - private DataSource mDataSource; private DataSink mDataSink; + private TrackTypeMap> mDataSources = new TrackTypeMap<>(); private TrackTypeMap mTranscoders = new TrackTypeMap<>(); private TrackTypeMap mStatuses = new TrackTypeMap<>(); private volatile double mProgress; private ProgressCallback mProgressCallback; - private long mDurationUs; + private long mTotalDurationUs; - public Engine(@NonNull DataSource dataSource, @Nullable ProgressCallback progressCallback) { - mDataSource = dataSource; + public Engine(@Nullable ProgressCallback progressCallback) { mProgressCallback = progressCallback; } @@ -88,6 +90,21 @@ private void setProgress(double progress) { } } + private boolean hasVideoSources() { + return !mDataSources.requireVideo().isEmpty(); + } + + private boolean hasAudioSources() { + return !mDataSources.requireAudio().isEmpty(); + } + + private Set getUniqueSources() { + Set sources = new HashSet<>(); + sources.addAll(mDataSources.requireVideo()); + sources.addAll(mDataSources.requireAudio()); + return sources; + } + /** * Performs transcoding. Blocks current thread. * @@ -97,18 +114,34 @@ private void setProgress(double progress) { */ public void transcode(@NonNull TranscoderOptions options) throws InterruptedException { mDataSink = new MediaMuxerDataSink(options.getOutputPath()); + mDataSources.setVideo(options.getVideoDataSources()); + mDataSources.setAudio(options.getAudioDataSources()); // Pass metadata from DataSource to DataSink - mDataSink.setOrientation((mDataSource.getOrientation() + options.getRotation()) % 360); - double[] location = mDataSource.getLocation(); - if (location != null) mDataSink.setLocation(location[0], location[1]); - mDurationUs = mDataSource.getDurationUs(); - LOG.v("Duration (us): " + mDurationUs); + if (hasVideoSources()) { + DataSource firstVideoSource = mDataSources.requireVideo().get(0); + mDataSink.setOrientation((firstVideoSource.getOrientation() + options.getRotation()) % 360); + } + for (DataSource locationSource : getUniqueSources()) { + double[] location = locationSource.getLocation(); + if (location != null) { + mDataSink.setLocation(location[0], location[1]); + break; + } + } + + // Compute total duration: it is the minimum between the two. + long audioDurationUs = hasAudioSources() ? 0 : Long.MAX_VALUE; + long videoDurationUs = hasVideoSources() ? 0 : Long.MAX_VALUE; + for (DataSource source : options.getVideoDataSources()) videoDurationUs += source.getDurationUs(); + for (DataSource source : options.getAudioDataSources()) audioDurationUs += source.getDurationUs(); + mTotalDurationUs = Math.min(audioDurationUs, videoDurationUs); + LOG.v("Duration (us): " + mTotalDurationUs); // Set up transcoders. int tracks = 0; - setUpTrackTranscoder(TrackType.VIDEO, options.getVideoTrackStrategy(), options); - setUpTrackTranscoder(TrackType.AUDIO, options.getAudioTrackStrategy(), options); + setUpTrackTranscoders(TrackType.VIDEO, options.getVideoTrackStrategy(), options); + setUpTrackTranscoders(TrackType.AUDIO, options.getAudioTrackStrategy(), options); TrackStatus videoStatus = mStatuses.require(TrackType.VIDEO); TrackStatus audioStatus = mStatuses.require(TrackType.AUDIO); TrackTranscoder videoTranscoder = mTranscoders.require(TrackType.VIDEO); @@ -130,7 +163,7 @@ public void transcode(@NonNull TranscoderOptions options) throws InterruptedExce // Do the actual transcoding work. long loopCount = 0; - if (mDurationUs <= 0) { + if (mTotalDurationUs <= 0) { setProgress(PROGRESS_UNKNOWN); } try { @@ -139,7 +172,7 @@ public void transcode(@NonNull TranscoderOptions options) throws InterruptedExce throw new InterruptedException(); } boolean stepped = videoTranscoder.transcode() || audioTranscoder.transcode(); - if (mDurationUs > 0 && ++loopCount % PROGRESS_INTERVAL_STEPS == 0) { + if (mTotalDurationUs > 0 && ++loopCount % PROGRESS_INTERVAL_STEPS == 0) { double videoProgress = getTranscoderProgress(videoTranscoder, videoStatus); double audioProgress = getTranscoderProgress(audioTranscoder, audioStatus); LOG.i("progress - video:" + videoProgress + " audio:" + audioProgress); @@ -209,6 +242,6 @@ private TrackTranscoder createTrackTranscoder(@NonNull TrackType type, private double getTranscoderProgress(@NonNull TrackTranscoder transcoder, @NonNull TrackStatus status) { if (!status.isTranscoding()) return 0.0; if (transcoder.isFinished()) return 1.0; - return Math.min(1.0, (double) transcoder.getLastPresentationTime() / mDurationUs); + return Math.min(1.0, (double) transcoder.getLastPresentationTime() / mTotalDurationUs); } } diff --git a/lib/src/main/java/com/otaliastudios/transcoder/internal/TrackTypeMap.java b/lib/src/main/java/com/otaliastudios/transcoder/internal/TrackTypeMap.java index e4ad96cc..1f57bd83 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/internal/TrackTypeMap.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/internal/TrackTypeMap.java @@ -23,18 +23,54 @@ public void set(@NonNull TrackType type, @Nullable T value) { map.put(type, value); } + public void setAudio(@Nullable T value) { + set(TrackType.AUDIO, value); + } + + public void setVideo(@Nullable T value) { + set(TrackType.VIDEO, value); + } + @Nullable public T get(@NonNull TrackType type) { return map.get(type); } + @Nullable + public T getAudio() { + return get(TrackType.AUDIO); + } + + @Nullable + public T getVideo() { + return get(TrackType.VIDEO); + } + @NonNull public T require(@NonNull TrackType type) { //noinspection ConstantConditions return map.get(type); } + @NonNull + public T requireAudio() { + return require(TrackType.AUDIO); + } + + @NonNull + public T requireVideo() { + return require(TrackType.VIDEO); + } + public boolean has(@NonNull TrackType type) { return map.containsKey(type); } + + public boolean hasAudio() { + return has(TrackType.AUDIO); + } + + public boolean hasVideo() { + return has(TrackType.VIDEO); + } } diff --git a/lib/src/main/java/com/otaliastudios/transcoder/strategy/DefaultVideoStrategies.java b/lib/src/main/java/com/otaliastudios/transcoder/strategy/DefaultVideoStrategies.java index 159a7ab9..1d1ae3dc 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/strategy/DefaultVideoStrategies.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/strategy/DefaultVideoStrategies.java @@ -21,7 +21,7 @@ public static DefaultVideoStrategy for720x1280() { return DefaultVideoStrategy.exact(720, 1280) .bitRate(2L * 1000 * 1000) .frameRate(30) - .iFrameInterval(3F) + .keyFrameInterval(3F) .build(); } @@ -38,7 +38,7 @@ public static DefaultVideoStrategy for360x480() { return DefaultVideoStrategy.exact(360, 480) .bitRate(500L * 1000) .frameRate(30) - .iFrameInterval(3F) + .keyFrameInterval(3F) .build(); } } diff --git a/lib/src/main/java/com/otaliastudios/transcoder/strategy/DefaultVideoStrategy.java b/lib/src/main/java/com/otaliastudios/transcoder/strategy/DefaultVideoStrategy.java index 0e25c630..11020696 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/strategy/DefaultVideoStrategy.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/strategy/DefaultVideoStrategy.java @@ -34,7 +34,7 @@ public class DefaultVideoStrategy implements TrackStrategy { public final static long BITRATE_UNKNOWN = Long.MIN_VALUE; @SuppressWarnings("WeakerAccess") - public final static float DEFAULT_I_FRAME_INTERVAL = 3; + public final static float DEFAULT_KEY_FRAME_INTERVAL = 3; public final static int DEFAULT_FRAME_RATE = 30; @@ -47,7 +47,7 @@ private Options() {} private Resizer resizer; private long targetBitRate; private int targetFrameRate; - private float targetIFrameInterval; + private float targetKeyFrameInterval; } /** @@ -121,7 +121,7 @@ public static class Builder { private MultiResizer resizer = new MultiResizer(); private int targetFrameRate = DEFAULT_FRAME_RATE; private long targetBitRate = BITRATE_UNKNOWN; - private float targetIFrameInterval = DEFAULT_I_FRAME_INTERVAL; + private float targetKeyFrameInterval = DEFAULT_KEY_FRAME_INTERVAL; @SuppressWarnings("unused") public Builder() { } @@ -170,14 +170,14 @@ public Builder frameRate(int frameRate) { } /** - * The interval between I-frames in seconds. - * @param iFrameInterval desired i-frame interval + * The interval between key-frames in seconds. + * @param keyFrameInterval desired key-frame interval * @return this for chaining */ @NonNull @SuppressWarnings("WeakerAccess") - public Builder iFrameInterval(float iFrameInterval) { - targetIFrameInterval = iFrameInterval; + public Builder keyFrameInterval(float keyFrameInterval) { + targetKeyFrameInterval = keyFrameInterval; return this; } @@ -188,7 +188,7 @@ public Options options() { options.resizer = resizer; options.targetFrameRate = targetFrameRate; options.targetBitRate = targetBitRate; - options.targetIFrameInterval = targetIFrameInterval; + options.targetKeyFrameInterval = targetKeyFrameInterval; return options; } @@ -248,13 +248,17 @@ public TrackStatus createOutputFormat(@NonNull List inputFormats, // Compute i frame. int inputIFrameInterval = getAverageIFrameInterval(inputFormats); - boolean frameIntervalDone = inputIFrameInterval >= options.targetIFrameInterval; + boolean frameIntervalDone = inputIFrameInterval >= options.targetKeyFrameInterval; // See if we should go on or if we're already compressed. - if (typeDone && sizeDone && frameRateDone && frameIntervalDone) { + // If we have more than 1 input format, we can't go through this branch, + // or, for example, each part would be copied into output with its own size, + // breaking the muxer. + boolean canPassThrough = inputFormats.size() == 1; + if (canPassThrough && typeDone && sizeDone && frameRateDone && frameIntervalDone) { LOG.i("Input minSize: " + inSize.getMinor() + ", desired minSize: " + outSize.getMinor() + "\nInput frameRate: " + inputFrameRate + ", desired frameRate: " + outFrameRate + - "\nInput iFrameInterval: " + inputIFrameInterval + ", desired iFrameInterval: " + options.targetIFrameInterval); + "\nInput iFrameInterval: " + inputIFrameInterval + ", desired iFrameInterval: " + options.targetKeyFrameInterval); return TrackStatus.PASS_THROUGH; } @@ -264,9 +268,9 @@ public TrackStatus createOutputFormat(@NonNull List inputFormats, outputFormat.setInteger(MediaFormat.KEY_HEIGHT, outHeight); outputFormat.setInteger(MediaFormat.KEY_FRAME_RATE, outFrameRate); if (Build.VERSION.SDK_INT >= 25) { - outputFormat.setFloat(MediaFormat.KEY_I_FRAME_INTERVAL, options.targetIFrameInterval); + outputFormat.setFloat(MediaFormat.KEY_I_FRAME_INTERVAL, options.targetKeyFrameInterval); } else { - outputFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, (int) Math.ceil(options.targetIFrameInterval)); + outputFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, (int) Math.ceil(options.targetKeyFrameInterval)); } outputFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface); int outBitRate = (int) (options.targetBitRate == BITRATE_UNKNOWN ? From f0325d133e4fe95733ee9a87ed30b0f87deb5b33 Mon Sep 17 00:00:00 2001 From: Mattia Iavarone Date: Sat, 3 Aug 2019 22:09:39 +0200 Subject: [PATCH 07/13] Add getLastTimestampUs to sources --- .../transcoder/engine/Engine.java | 63 ++++++++++++++----- .../transcoder/source/AndroidDataSource.java | 7 +++ .../transcoder/source/DataSource.java | 8 +++ 3 files changed, 64 insertions(+), 14 deletions(-) diff --git a/lib/src/main/java/com/otaliastudios/transcoder/engine/Engine.java b/lib/src/main/java/com/otaliastudios/transcoder/engine/Engine.java index b0f7fda5..5e0c38b4 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/engine/Engine.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/engine/Engine.java @@ -35,6 +35,7 @@ import androidx.annotation.NonNull; import androidx.annotation.Nullable; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; import java.util.List; @@ -66,6 +67,7 @@ public interface ProgressCallback { private TrackTypeMap> mDataSources = new TrackTypeMap<>(); private TrackTypeMap mTranscoders = new TrackTypeMap<>(); private TrackTypeMap mStatuses = new TrackTypeMap<>(); + private TrackTypeMap mOutputFormats = new TrackTypeMap<>(); private volatile double mProgress; private ProgressCallback mProgressCallback; private long mTotalDurationUs; @@ -138,17 +140,11 @@ public void transcode(@NonNull TranscoderOptions options) throws InterruptedExce mTotalDurationUs = Math.min(audioDurationUs, videoDurationUs); LOG.v("Duration (us): " + mTotalDurationUs); - // Set up transcoders. - int tracks = 0; - setUpTrackTranscoders(TrackType.VIDEO, options.getVideoTrackStrategy(), options); - setUpTrackTranscoders(TrackType.AUDIO, options.getAudioTrackStrategy(), options); - TrackStatus videoStatus = mStatuses.require(TrackType.VIDEO); - TrackStatus audioStatus = mStatuses.require(TrackType.AUDIO); - TrackTranscoder videoTranscoder = mTranscoders.require(TrackType.VIDEO); - TrackTranscoder audioTranscoder = mTranscoders.require(TrackType.AUDIO); - if (videoStatus.isTranscoding()) tracks++; - if (audioStatus.isTranscoding()) tracks++; - tracks = Math.max(1, tracks); + // Compute the TrackStatus. + computeTrackStatus(TrackType.AUDIO, options.getAudioTrackStrategy(), options.getAudioDataSources()); + computeTrackStatus(TrackType.VIDEO, options.getVideoTrackStrategy(), options.getVideoDataSources()); + TrackStatus videoStatus = mStatuses.requireVideo(); + TrackStatus audioStatus = mStatuses.requireAudio(); // Pass to Validator. //noinspection UnusedAssignment @@ -161,6 +157,21 @@ public void transcode(@NonNull TranscoderOptions options) throws InterruptedExce throw new ValidatorException("Validator returned false."); } + // Set up transcoders. + // int tracks = 0; + // setUpTrackTranscoders(TrackType.VIDEO, options.getVideoTrackStrategy(), options); + // setUpTrackTranscoders(TrackType.AUDIO, options.getAudioTrackStrategy(), options); + // TrackTranscoder videoTranscoder = mTranscoders.require(TrackType.VIDEO); + // TrackTranscoder audioTranscoder = mTranscoders.require(TrackType.AUDIO); + // if (videoStatus.isTranscoding()) tracks++; + // if (audioStatus.isTranscoding()) tracks++; + // tracks = Math.max(1, tracks); + + // TODO dataSource.selectTrack + // TODO dataSource.release + // TODO trackTranscoder.setUp + // TODO trackTranscoder timeOffsetUs + // Do the actual transcoding work. long loopCount = 0; if (mTotalDurationUs <= 0) { @@ -196,6 +207,29 @@ public void transcode(@NonNull TranscoderOptions options) throws InterruptedExce } } + private void computeTrackStatus(@NonNull TrackType type, + @NonNull TrackStrategy strategy, + @NonNull List sources) { + TrackStatus status = TrackStatus.ABSENT; + MediaFormat outputFormat = new MediaFormat(); + if (!sources.isEmpty()) { + List inputFormats = new ArrayList<>(); + for (DataSource source : sources) { + MediaFormat inputFormat = source.getTrackFormat(type); + if (inputFormat != null) { + inputFormats.add(inputFormat); + } else if (sources.size() > 1) { + throw new IllegalArgumentException("More than one source selected for type " + type + + ", but getTrackFormat returned null."); + } + } + status = strategy.createOutputFormat(inputFormats, outputFormat); + } + mOutputFormats.set(type, outputFormat); + mDataSink.setTrackStatus(type, status); + mStatuses.set(type, status); + } + private void setUpTrackTranscoder(@NonNull TrackType type, @NonNull TrackStrategy strategy, @NonNull TranscoderOptions options) { @@ -228,12 +262,13 @@ private void setUpTrackTranscoder(@NonNull TrackType type, } @NonNull - private TrackTranscoder createTrackTranscoder(@NonNull TrackType type, + private TrackTranscoder createTrackTranscoder(@NonNull DataSource dataSource, + @NonNull TrackType type, @NonNull TranscoderOptions options) { switch (type) { - case VIDEO: return new VideoTrackTranscoder(mDataSource, mDataSink, + case VIDEO: return new VideoTrackTranscoder(dataSource, mDataSink, options.getTimeInterpolator()); - case AUDIO: return new AudioTrackTranscoder(mDataSource, mDataSink, + case AUDIO: return new AudioTrackTranscoder(dataSource, mDataSink, options.getTimeInterpolator(), options.getAudioStretcher()); default: throw new RuntimeException("Unknown type: " + type); } diff --git a/lib/src/main/java/com/otaliastudios/transcoder/source/AndroidDataSource.java b/lib/src/main/java/com/otaliastudios/transcoder/source/AndroidDataSource.java index bacbde49..51a66536 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/source/AndroidDataSource.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/source/AndroidDataSource.java @@ -28,6 +28,7 @@ public abstract class AndroidDataSource implements DataSource { private boolean mExtractorApplied; private final TrackTypeMap mFormats = new TrackTypeMap<>(); private final TrackTypeMap mIndex = new TrackTypeMap<>(); + private long mLastTimestampUs; private void ensureMetadata() { if (!mMetadataApplied) { @@ -75,9 +76,15 @@ public void readTrack(@NonNull Chunk chunk) { chunk.bytes = mExtractor.readSampleData(chunk.buffer, 0); chunk.isKeyFrame = (mExtractor.getSampleFlags() & MediaExtractor.SAMPLE_FLAG_SYNC) != 0; chunk.timestampUs = mExtractor.getSampleTime(); + mLastTimestampUs = chunk.timestampUs; mExtractor.advance(); } + @Override + public long getLastTimestampUs() { + return mLastTimestampUs; + } + @Nullable @Override public double[] getLocation() { diff --git a/lib/src/main/java/com/otaliastudios/transcoder/source/DataSource.java b/lib/src/main/java/com/otaliastudios/transcoder/source/DataSource.java index 1741a2d2..5dadf2a5 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/source/DataSource.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/source/DataSource.java @@ -78,6 +78,14 @@ public interface DataSource { */ void readTrack(@NonNull DataSource.Chunk chunk); + /** + * Returns the latest timestamp that was read using + * {@link #readTrack(Chunk)}. + * + * @return latest timestamp + */ + long getLastTimestampUs(); + /** * When this source has been totally read, it can return true here to * notify an end of input stream. From 5484573616934f8c0c5c28406646a7553176d1ba Mon Sep 17 00:00:00 2001 From: Mattia Iavarone Date: Sat, 3 Aug 2019 23:56:30 +0200 Subject: [PATCH 08/13] Complete multiple sources implementation --- .../transcoder/engine/Engine.java | 304 +++++++++++------- .../transcoder/source/AndroidDataSource.java | 9 + .../transcoder/source/DataSource.java | 8 + .../transcode/BaseTrackTranscoder.java | 8 - .../transcode/NoOpTrackTranscoder.java | 5 - .../transcode/PassThroughTrackTranscoder.java | 7 - .../transcoder/transcode/TrackTranscoder.java | 8 - 7 files changed, 204 insertions(+), 145 deletions(-) diff --git a/lib/src/main/java/com/otaliastudios/transcoder/engine/Engine.java b/lib/src/main/java/com/otaliastudios/transcoder/engine/Engine.java index 5e0c38b4..f1c76771 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/engine/Engine.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/engine/Engine.java @@ -25,6 +25,7 @@ import com.otaliastudios.transcoder.sink.MediaMuxerDataSink; import com.otaliastudios.transcoder.source.DataSource; import com.otaliastudios.transcoder.strategy.TrackStrategy; +import com.otaliastudios.transcoder.time.TimeInterpolator; import com.otaliastudios.transcoder.transcode.AudioTrackTranscoder; import com.otaliastudios.transcoder.transcode.NoOpTrackTranscoder; import com.otaliastudios.transcoder.transcode.PassThroughTrackTranscoder; @@ -49,7 +50,6 @@ public class Engine { private static final String TAG = Engine.class.getSimpleName(); private static final Logger LOG = new Logger(TAG); - private static final double PROGRESS_UNKNOWN = -1.0; private static final long SLEEP_TO_WAIT_TRACK_TRANSCODERS = 10; private static final long PROGRESS_INTERVAL_STEPS = 10; @@ -64,20 +64,26 @@ public interface ProgressCallback { } private DataSink mDataSink; - private TrackTypeMap> mDataSources = new TrackTypeMap<>(); - private TrackTypeMap mTranscoders = new TrackTypeMap<>(); - private TrackTypeMap mStatuses = new TrackTypeMap<>(); - private TrackTypeMap mOutputFormats = new TrackTypeMap<>(); + private final TrackTypeMap> mDataSources = new TrackTypeMap<>(); + private final TrackTypeMap> mTranscoders = new TrackTypeMap<>(); + private final TrackTypeMap> mInterpolators = new TrackTypeMap<>(); + private final TrackTypeMap mCurrentStep = new TrackTypeMap<>(); + private final TrackTypeMap mStatuses = new TrackTypeMap<>(); + private final TrackTypeMap mOutputFormats = new TrackTypeMap<>(); private volatile double mProgress; - private ProgressCallback mProgressCallback; - private long mTotalDurationUs; + private final ProgressCallback mProgressCallback; public Engine(@Nullable ProgressCallback progressCallback) { mProgressCallback = progressCallback; + mCurrentStep.setVideo(0); + mCurrentStep.setAudio(0); + mTranscoders.setVideo(new ArrayList()); + mTranscoders.setAudio(new ArrayList()); } /** - * NOTE: This method is thread safe. + * Returns the current progress. + * Note: This method is thread safe. * @return the current progress */ @SuppressWarnings("unused") @@ -107,6 +113,164 @@ private Set getUniqueSources() { return sources; } + private void computeTrackStatus(@NonNull TrackType type, + @NonNull TrackStrategy strategy, + @NonNull List sources) { + TrackStatus status = TrackStatus.ABSENT; + MediaFormat outputFormat = new MediaFormat(); + if (!sources.isEmpty()) { + List inputFormats = new ArrayList<>(); + for (DataSource source : sources) { + MediaFormat inputFormat = source.getTrackFormat(type); + if (inputFormat != null) { + inputFormats.add(inputFormat); + } else if (sources.size() > 1) { + throw new IllegalArgumentException("More than one source selected for type " + type + + ", but getTrackFormat returned null."); + } + } + status = strategy.createOutputFormat(inputFormats, outputFormat); + } + mOutputFormats.set(type, outputFormat); + mDataSink.setTrackStatus(type, status); + mStatuses.set(type, status); + } + + private boolean isCompleted(@NonNull TrackType type) { + int current = mCurrentStep.require(type); + return !mDataSources.require(type).isEmpty() + && current == mDataSources.require(type).size() - 1 + && current == mTranscoders.require(type).size() - 1 + && mTranscoders.require(type).get(current).isFinished(); + } + + private void openCurrentStep(@NonNull TrackType type, @NonNull TranscoderOptions options) { + int current = mCurrentStep.require(type); + TrackStatus status = mStatuses.require(type); + + // Notify the data source that we'll be transcoding this track. + DataSource dataSource = mDataSources.require(type).get(current); + if (status.isTranscoding()) { + dataSource.selectTrack(type); + } + + // Create a TimeInterpolator, wrapping the external one. + TimeInterpolator interpolator = createStepTimeInterpolator(type, current, + options.getTimeInterpolator()); + mInterpolators.require(type).add(interpolator); + + // Create a Transcoder for this track. + TrackTranscoder transcoder; + switch (status) { + case PASS_THROUGH: { + transcoder = new PassThroughTrackTranscoder(dataSource, + mDataSink, type, interpolator); + break; + } + case COMPRESSING: { + switch (type) { + case VIDEO: + transcoder = new VideoTrackTranscoder(dataSource, mDataSink, interpolator); + break; + case AUDIO: + transcoder = new AudioTrackTranscoder(dataSource, mDataSink, + interpolator, options.getAudioStretcher()); + break; + default: + throw new RuntimeException("Unknown type: " + type); + } + break; + } + case ABSENT: + case REMOVING: + default: { + transcoder = new NoOpTrackTranscoder(); + break; + } + } + transcoder.setUp(mOutputFormats.require(type)); + mTranscoders.require(type).add(transcoder); + } + + private void closeCurrentStep(@NonNull TrackType type) { + int current = mCurrentStep.require(type); + TrackTranscoder transcoder = mTranscoders.require(type).get(current); + DataSource dataSource = mDataSources.require(type).get(current); + transcoder.release(); + dataSource.release(); + mCurrentStep.set(type, current + 1); + } + + @NonNull + private TrackTranscoder getCurrentTrackTranscoder(@NonNull TrackType type, @NonNull TranscoderOptions options) { + int current = mCurrentStep.require(type); + int last = mTranscoders.require(type).size() - 1; + if (last == current) { + // We have already created a transcoder for this step. + // But this step might be completed and we might need to create a new one. + TrackTranscoder transcoder = mTranscoders.require(type).get(last); + if (transcoder.isFinished()) { + closeCurrentStep(type); + return getCurrentTrackTranscoder(type, options); + } else { + return mTranscoders.require(type).get(current); + } + } else if (last < current) { + // We need to create a new step. + openCurrentStep(type, options); + return mTranscoders.require(type).get(current); + } else { + throw new IllegalStateException("This should never happen. last:" + last + ", current:" + current); + } + } + + @NonNull + private TimeInterpolator createStepTimeInterpolator(@NonNull TrackType type, int step, + final @NonNull TimeInterpolator wrap) { + final long timebase; + if (step > 0) { + TimeInterpolator previous = mInterpolators.require(type).get(step - 1); + timebase = previous.interpolate(type, Long.MAX_VALUE); + } else { + timebase = 0; + } + return new TimeInterpolator() { + + private long mLastInterpolatedTime; + private long mFirstInputTime = Long.MAX_VALUE; + private long mTimeBase = timebase + 10; + + @Override + public long interpolate(@NonNull TrackType type, long time) { + if (time == Long.MAX_VALUE) return mLastInterpolatedTime; + if (mFirstInputTime == Long.MAX_VALUE) mFirstInputTime = time; + mLastInterpolatedTime = mTimeBase + (time - mFirstInputTime); + return wrap.interpolate(type, mLastInterpolatedTime); + } + }; + } + + private double getTrackProgress(@NonNull TrackType type) { + if (!mStatuses.require(type).isTranscoding()) return 0.0D; + int current = mCurrentStep.require(type); + long totalDurationUs = 0; + long completedDurationUs = 0; + for (int i = 0; i < mDataSources.require(type).size(); i++) { + DataSource source = mDataSources.require(type).get(i); + if (i < current) { + totalDurationUs += source.getLastTimestampUs() - source.getFirstTimestampUs(); + completedDurationUs += source.getLastTimestampUs() - source.getFirstTimestampUs(); + } else if (i == current) { + totalDurationUs += source.getDurationUs(); + completedDurationUs += source.getLastTimestampUs() - source.getFirstTimestampUs(); + } else { + totalDurationUs += source.getDurationUs(); + completedDurationUs += 0; + } + } + return (double) completedDurationUs / (double) totalDurationUs; + } + /** * Performs transcoding. Blocks current thread. * @@ -137,14 +301,18 @@ public void transcode(@NonNull TranscoderOptions options) throws InterruptedExce long videoDurationUs = hasVideoSources() ? 0 : Long.MAX_VALUE; for (DataSource source : options.getVideoDataSources()) videoDurationUs += source.getDurationUs(); for (DataSource source : options.getAudioDataSources()) audioDurationUs += source.getDurationUs(); - mTotalDurationUs = Math.min(audioDurationUs, videoDurationUs); - LOG.v("Duration (us): " + mTotalDurationUs); + long totalDurationUs = Math.min(audioDurationUs, videoDurationUs); + LOG.v("Duration (us): " + totalDurationUs); + // TODO if audio and video have different lengths, we should clip the longer one! // Compute the TrackStatus. + int activeTracks = 0; computeTrackStatus(TrackType.AUDIO, options.getAudioTrackStrategy(), options.getAudioDataSources()); computeTrackStatus(TrackType.VIDEO, options.getVideoTrackStrategy(), options.getVideoDataSources()); TrackStatus videoStatus = mStatuses.requireVideo(); TrackStatus audioStatus = mStatuses.requireAudio(); + if (videoStatus.isTranscoding()) activeTracks++; + if (audioStatus.isTranscoding()) activeTracks++; // Pass to Validator. //noinspection UnusedAssignment @@ -157,37 +325,18 @@ public void transcode(@NonNull TranscoderOptions options) throws InterruptedExce throw new ValidatorException("Validator returned false."); } - // Set up transcoders. - // int tracks = 0; - // setUpTrackTranscoders(TrackType.VIDEO, options.getVideoTrackStrategy(), options); - // setUpTrackTranscoders(TrackType.AUDIO, options.getAudioTrackStrategy(), options); - // TrackTranscoder videoTranscoder = mTranscoders.require(TrackType.VIDEO); - // TrackTranscoder audioTranscoder = mTranscoders.require(TrackType.AUDIO); - // if (videoStatus.isTranscoding()) tracks++; - // if (audioStatus.isTranscoding()) tracks++; - // tracks = Math.max(1, tracks); - - // TODO dataSource.selectTrack - // TODO dataSource.release - // TODO trackTranscoder.setUp - // TODO trackTranscoder timeOffsetUs - // Do the actual transcoding work. - long loopCount = 0; - if (mTotalDurationUs <= 0) { - setProgress(PROGRESS_UNKNOWN); - } try { - while (!(videoTranscoder.isFinished() && audioTranscoder.isFinished())) { + long loopCount = 0; + while (!(isCompleted(TrackType.AUDIO) && isCompleted(TrackType.VIDEO))) { if (Thread.interrupted()) { throw new InterruptedException(); } - boolean stepped = videoTranscoder.transcode() || audioTranscoder.transcode(); - if (mTotalDurationUs > 0 && ++loopCount % PROGRESS_INTERVAL_STEPS == 0) { - double videoProgress = getTranscoderProgress(videoTranscoder, videoStatus); - double audioProgress = getTranscoderProgress(audioTranscoder, audioStatus); - LOG.i("progress - video:" + videoProgress + " audio:" + audioProgress); - setProgress((videoProgress + audioProgress) / tracks); + boolean stepped = getCurrentTrackTranscoder(TrackType.VIDEO, options).transcode() + || getCurrentTrackTranscoder(TrackType.AUDIO, options).transcode(); + if (++loopCount % PROGRESS_INTERVAL_STEPS == 0) { + setProgress((getTrackProgress(TrackType.VIDEO) + + getTrackProgress(TrackType.AUDIO)) / activeTracks); } if (!stepped) { Thread.sleep(SLEEP_TO_WAIT_TRACK_TRANSCODERS); @@ -195,88 +344,9 @@ public void transcode(@NonNull TranscoderOptions options) throws InterruptedExce } mDataSink.stop(); } finally { - try { - videoTranscoder.release(); - audioTranscoder.release(); - } catch (RuntimeException e) { - // Too fatal to make alive the app, because it may leak native resources. - //noinspection ThrowFromFinallyBlock - throw new Error("Could not shutdown extractor, codecs and muxer pipeline.", e); - } + closeCurrentStep(TrackType.VIDEO); + closeCurrentStep(TrackType.AUDIO); mDataSink.release(); } } - - private void computeTrackStatus(@NonNull TrackType type, - @NonNull TrackStrategy strategy, - @NonNull List sources) { - TrackStatus status = TrackStatus.ABSENT; - MediaFormat outputFormat = new MediaFormat(); - if (!sources.isEmpty()) { - List inputFormats = new ArrayList<>(); - for (DataSource source : sources) { - MediaFormat inputFormat = source.getTrackFormat(type); - if (inputFormat != null) { - inputFormats.add(inputFormat); - } else if (sources.size() > 1) { - throw new IllegalArgumentException("More than one source selected for type " + type - + ", but getTrackFormat returned null."); - } - } - status = strategy.createOutputFormat(inputFormats, outputFormat); - } - mOutputFormats.set(type, outputFormat); - mDataSink.setTrackStatus(type, status); - mStatuses.set(type, status); - } - - private void setUpTrackTranscoder(@NonNull TrackType type, - @NonNull TrackStrategy strategy, - @NonNull TranscoderOptions options) { - TrackStatus status = TrackStatus.ABSENT; - TrackTranscoder transcoder = new NoOpTrackTranscoder(); - final MediaFormat inputFormat = mDataSource.getTrackFormat(type); - MediaFormat outputFormat = new MediaFormat(); - if (inputFormat != null) { - //noinspection ArraysAsListWithZeroOrOneArgument - status = strategy.createOutputFormat(Arrays.asList(inputFormat), outputFormat); - switch (status) { - case ABSENT: throw new IllegalArgumentException("Strategies should not return ABSENT."); - case REMOVING: break; // We'll use NoOpTrackTranscoder. - case PASS_THROUGH: { - transcoder = new PassThroughTrackTranscoder(mDataSource, - mDataSink, type, options.getTimeInterpolator()); - break; - } - case COMPRESSING: { - transcoder = createTrackTranscoder(type, options); - break; - } - } - } - if (status.isTranscoding()) mDataSource.selectTrack(type); - mDataSink.setTrackStatus(type, status); - mStatuses.set(type, status); - transcoder.setUp(outputFormat); - mTranscoders.set(type, transcoder); - } - - @NonNull - private TrackTranscoder createTrackTranscoder(@NonNull DataSource dataSource, - @NonNull TrackType type, - @NonNull TranscoderOptions options) { - switch (type) { - case VIDEO: return new VideoTrackTranscoder(dataSource, mDataSink, - options.getTimeInterpolator()); - case AUDIO: return new AudioTrackTranscoder(dataSource, mDataSink, - options.getTimeInterpolator(), options.getAudioStretcher()); - default: throw new RuntimeException("Unknown type: " + type); - } - } - - private double getTranscoderProgress(@NonNull TrackTranscoder transcoder, @NonNull TrackStatus status) { - if (!status.isTranscoding()) return 0.0; - if (transcoder.isFinished()) return 1.0; - return Math.min(1.0, (double) transcoder.getLastPresentationTime() / mTotalDurationUs); - } } diff --git a/lib/src/main/java/com/otaliastudios/transcoder/source/AndroidDataSource.java b/lib/src/main/java/com/otaliastudios/transcoder/source/AndroidDataSource.java index 51a66536..c8402c03 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/source/AndroidDataSource.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/source/AndroidDataSource.java @@ -29,6 +29,7 @@ public abstract class AndroidDataSource implements DataSource { private final TrackTypeMap mFormats = new TrackTypeMap<>(); private final TrackTypeMap mIndex = new TrackTypeMap<>(); private long mLastTimestampUs; + private long mFirstTimestampUs = Long.MIN_VALUE; private void ensureMetadata() { if (!mMetadataApplied) { @@ -77,9 +78,17 @@ public void readTrack(@NonNull Chunk chunk) { chunk.isKeyFrame = (mExtractor.getSampleFlags() & MediaExtractor.SAMPLE_FLAG_SYNC) != 0; chunk.timestampUs = mExtractor.getSampleTime(); mLastTimestampUs = chunk.timestampUs; + if (mFirstTimestampUs == Long.MIN_VALUE) { + mFirstTimestampUs = mLastTimestampUs; + } mExtractor.advance(); } + @Override + public long getFirstTimestampUs() { + return mFirstTimestampUs; + } + @Override public long getLastTimestampUs() { return mLastTimestampUs; diff --git a/lib/src/main/java/com/otaliastudios/transcoder/source/DataSource.java b/lib/src/main/java/com/otaliastudios/transcoder/source/DataSource.java index 5dadf2a5..075c9315 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/source/DataSource.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/source/DataSource.java @@ -86,6 +86,14 @@ public interface DataSource { */ long getLastTimestampUs(); + /** + * Returns the first timestamp that was read using + * {@link #readTrack(Chunk)}. + * + * @return first timestamp + */ + long getFirstTimestampUs(); + /** * When this source has been totally read, it can return true here to * notify an end of input stream. diff --git a/lib/src/main/java/com/otaliastudios/transcoder/transcode/BaseTrackTranscoder.java b/lib/src/main/java/com/otaliastudios/transcoder/transcode/BaseTrackTranscoder.java index 29ef5fbb..c0b1ea57 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/transcode/BaseTrackTranscoder.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/transcode/BaseTrackTranscoder.java @@ -30,8 +30,6 @@ public abstract class BaseTrackTranscoder implements TrackTranscoder { private final DataSink mDataSink; private final TrackType mTrackType; - private long mLastPresentationTimeUs; - private final MediaCodec.BufferInfo mBufferInfo = new MediaCodec.BufferInfo(); private MediaCodec mDecoder; private MediaCodec mEncoder; @@ -134,11 +132,6 @@ protected void onCodecsStarted(@NonNull MediaFormat inputFormat, @NonNull MediaF @NonNull MediaCodec decoder, @NonNull MediaCodec encoder) { } - @Override - public final long getLastPresentationTime() { - return mLastPresentationTimeUs; - } - @Override public final boolean isFinished() { return mIsEncoderEOS; @@ -255,7 +248,6 @@ private int drainDecoder(long timeoutUs) { boolean hasSize = mBufferInfo.size > 0; if (isEos) mIsDecoderEOS = true; if (isEos || hasSize) { - mLastPresentationTimeUs = mBufferInfo.presentationTimeUs; onDrainDecoder(mDecoder, result, mDecoderBuffers.getOutputBuffer(result), diff --git a/lib/src/main/java/com/otaliastudios/transcoder/transcode/NoOpTrackTranscoder.java b/lib/src/main/java/com/otaliastudios/transcoder/transcode/NoOpTrackTranscoder.java index 60eecd3b..32307d90 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/transcode/NoOpTrackTranscoder.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/transcode/NoOpTrackTranscoder.java @@ -33,11 +33,6 @@ public boolean transcode() { return false; } - @Override - public long getLastPresentationTime() { - return 0; - } - @Override public boolean isFinished() { return true; diff --git a/lib/src/main/java/com/otaliastudios/transcoder/transcode/PassThroughTrackTranscoder.java b/lib/src/main/java/com/otaliastudios/transcoder/transcode/PassThroughTrackTranscoder.java index 882075c1..1b2f077d 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/transcode/PassThroughTrackTranscoder.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/transcode/PassThroughTrackTranscoder.java @@ -36,7 +36,6 @@ public class PassThroughTrackTranscoder implements TrackTranscoder { private final TrackType mTrackType; private final MediaCodec.BufferInfo mBufferInfo = new MediaCodec.BufferInfo(); private boolean mIsEOS; - private long mLastPresentationTime; private final MediaFormat mOutputFormat; private boolean mOutputFormatSet = false; private TimeInterpolator mTimeInterpolator; @@ -85,15 +84,9 @@ public boolean transcode() { int flags = mDataChunk.isKeyFrame ? MediaCodec.BUFFER_FLAG_SYNC_FRAME : 0; mBufferInfo.set(0, mDataChunk.bytes, timestampUs, flags); mDataSink.writeTrack(mTrackType, mDataChunk.buffer, mBufferInfo); - mLastPresentationTime = mDataChunk.timestampUs; return true; } - @Override - public long getLastPresentationTime() { - return mLastPresentationTime; - } - @Override public boolean isFinished() { return mIsEOS; diff --git a/lib/src/main/java/com/otaliastudios/transcoder/transcode/TrackTranscoder.java b/lib/src/main/java/com/otaliastudios/transcoder/transcode/TrackTranscoder.java index 9d42b20f..c1e36ba7 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/transcode/TrackTranscoder.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/transcode/TrackTranscoder.java @@ -34,14 +34,6 @@ public interface TrackTranscoder { */ boolean transcode(); - /** - * Get presentation time of last sample taken from encoder. - * This presentation time should not be affected by {@link com.otaliastudios.transcoder.time.TimeInterpolator}s. - * - * @return Presentation time in microseconds. Return value is undefined if finished writing. - */ - long getLastPresentationTime(); - boolean isFinished(); void release(); From da5f11424aa7f861619ac277b5be26f397e26e68 Mon Sep 17 00:00:00 2001 From: Mattia Iavarone Date: Sun, 4 Aug 2019 00:23:32 +0200 Subject: [PATCH 09/13] Fix bugs --- .../transcoder/engine/Engine.java | 33 ++++++++++++------- .../transcoder/internal/TrackTypeMap.java | 8 +++++ 2 files changed, 29 insertions(+), 12 deletions(-) diff --git a/lib/src/main/java/com/otaliastudios/transcoder/engine/Engine.java b/lib/src/main/java/com/otaliastudios/transcoder/engine/Engine.java index f1c76771..4aa46d37 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/engine/Engine.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/engine/Engine.java @@ -65,9 +65,9 @@ public interface ProgressCallback { private DataSink mDataSink; private final TrackTypeMap> mDataSources = new TrackTypeMap<>(); - private final TrackTypeMap> mTranscoders = new TrackTypeMap<>(); - private final TrackTypeMap> mInterpolators = new TrackTypeMap<>(); - private final TrackTypeMap mCurrentStep = new TrackTypeMap<>(); + private final TrackTypeMap> mTranscoders = new TrackTypeMap<>(new ArrayList(), new ArrayList()); + private final TrackTypeMap> mInterpolators = new TrackTypeMap<>(new ArrayList(), new ArrayList()); + private final TrackTypeMap mCurrentStep = new TrackTypeMap<>(0, 0); private final TrackTypeMap mStatuses = new TrackTypeMap<>(); private final TrackTypeMap mOutputFormats = new TrackTypeMap<>(); private volatile double mProgress; @@ -75,10 +75,6 @@ public interface ProgressCallback { public Engine(@Nullable ProgressCallback progressCallback) { mProgressCallback = progressCallback; - mCurrentStep.setVideo(0); - mCurrentStep.setAudio(0); - mTranscoders.setVideo(new ArrayList()); - mTranscoders.setAudio(new ArrayList()); } /** @@ -205,6 +201,7 @@ private void closeCurrentStep(@NonNull TrackType type) { private TrackTranscoder getCurrentTrackTranscoder(@NonNull TrackType type, @NonNull TranscoderOptions options) { int current = mCurrentStep.require(type); int last = mTranscoders.require(type).size() - 1; + int max = mDataSources.require(type).size(); if (last == current) { // We have already created a transcoder for this step. // But this step might be completed and we might need to create a new one. @@ -268,6 +265,7 @@ private double getTrackProgress(@NonNull TrackType type) { completedDurationUs += 0; } } + if (totalDurationUs == 0) totalDurationUs = 1; return (double) completedDurationUs / (double) totalDurationUs; } @@ -328,12 +326,21 @@ public void transcode(@NonNull TranscoderOptions options) throws InterruptedExce // Do the actual transcoding work. try { long loopCount = 0; - while (!(isCompleted(TrackType.AUDIO) && isCompleted(TrackType.VIDEO))) { + boolean stepped = false; + boolean audioCompleted = false, videoCompleted = false; + while (!(audioCompleted && videoCompleted)) { if (Thread.interrupted()) { throw new InterruptedException(); } - boolean stepped = getCurrentTrackTranscoder(TrackType.VIDEO, options).transcode() - || getCurrentTrackTranscoder(TrackType.AUDIO, options).transcode(); + stepped = false; + audioCompleted = isCompleted(TrackType.AUDIO); + videoCompleted = isCompleted(TrackType.VIDEO); + if (!audioCompleted) { + stepped |= getCurrentTrackTranscoder(TrackType.AUDIO, options).transcode(); + } + if (!videoCompleted) { + stepped |= getCurrentTrackTranscoder(TrackType.VIDEO, options).transcode(); + } if (++loopCount % PROGRESS_INTERVAL_STEPS == 0) { setProgress((getTrackProgress(TrackType.VIDEO) + getTrackProgress(TrackType.AUDIO)) / activeTracks); @@ -344,8 +351,10 @@ public void transcode(@NonNull TranscoderOptions options) throws InterruptedExce } mDataSink.stop(); } finally { - closeCurrentStep(TrackType.VIDEO); - closeCurrentStep(TrackType.AUDIO); + try { + closeCurrentStep(TrackType.VIDEO); + closeCurrentStep(TrackType.AUDIO); + } catch (Exception ignore) {} mDataSink.release(); } } diff --git a/lib/src/main/java/com/otaliastudios/transcoder/internal/TrackTypeMap.java b/lib/src/main/java/com/otaliastudios/transcoder/internal/TrackTypeMap.java index 1f57bd83..6a8c514e 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/internal/TrackTypeMap.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/internal/TrackTypeMap.java @@ -16,6 +16,14 @@ */ public class TrackTypeMap { + public TrackTypeMap() { + } + + public TrackTypeMap(@NonNull T videoValue, @NonNull T audioValue) { + set(TrackType.AUDIO, audioValue); + set(TrackType.VIDEO, videoValue); + } + private Map map = new HashMap<>(); public void set(@NonNull TrackType type, @Nullable T value) { From c5843d477e5705c5d78edb7ad694b73fd1456404 Mon Sep 17 00:00:00 2001 From: Mattia Iavarone Date: Sun, 4 Aug 2019 00:47:06 +0200 Subject: [PATCH 10/13] Remove sampleRate exception --- README.md | 14 +++---- .../transcoder/demo/TranscoderActivity.java | 37 ++++++++++++++----- .../transcoder/TranscoderOptions.java | 18 ++++----- .../strategy/DefaultAudioStrategy.java | 23 +++++++----- .../transcode/internal/AudioEngine.java | 3 +- 5 files changed, 57 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index 13cab15c..c132e86c 100644 --- a/README.md +++ b/README.md @@ -22,10 +22,10 @@ Using Transcoder in the most basic form is pretty simple: ```java Transcoder.into(filePath) - .setDataSource(context, uri) // or... - .setDataSource(filePath) // or... - .setDataSource(fileDescriptor) // or... - .setDataSource(dataSource) + .addDataSource(context, uri) // or... + .addDataSource(filePath) // or... + .addDataSource(fileDescriptor) // or... + .addDataSource(dataSource) .setListener(new TranscoderListener() { public void onTranscodeProgress(double progress) {} public void onTranscodeCompleted(int successCode) {} @@ -80,17 +80,17 @@ which is convenient but it means that they can not be used twice. #### `UriDataSource` The Android friendly source can be created with `new UriDataSource(context, uri)` or simply -using `setDataSource(context, uri)` in the transcoding builder. +using `addDataSource(context, uri)` in the transcoding builder. #### `FileDescriptorDataSource` A data source backed by a file descriptor. Use `new FileDescriptorDataSource(descriptor)` or -simply `setDataSource(descriptor)` in the transcoding builder. +simply `addDataSource(descriptor)` in the transcoding builder. #### `FilePathDataSource` A data source backed by a file absolute path. Use `new FilePathDataSource(path)` or -simply `setDataSource(path)` in the transcoding builder. +simply `addDataSource(path)` in the transcoding builder. ## Listening for events diff --git a/demo/src/main/java/com/otaliastudios/transcoder/demo/TranscoderActivity.java b/demo/src/main/java/com/otaliastudios/transcoder/demo/TranscoderActivity.java index e44495c7..252e6ad8 100644 --- a/demo/src/main/java/com/otaliastudios/transcoder/demo/TranscoderActivity.java +++ b/demo/src/main/java/com/otaliastudios/transcoder/demo/TranscoderActivity.java @@ -1,5 +1,6 @@ package com.otaliastudios.transcoder.demo; +import android.content.ClipData; import android.content.Intent; import android.net.Uri; import android.os.Bundle; @@ -11,6 +12,7 @@ import com.otaliastudios.transcoder.Transcoder; import com.otaliastudios.transcoder.TranscoderListener; +import com.otaliastudios.transcoder.TranscoderOptions; import com.otaliastudios.transcoder.internal.Logger; import com.otaliastudios.transcoder.strategy.DefaultAudioStrategy; import com.otaliastudios.transcoder.strategy.DefaultVideoStrategy; @@ -51,7 +53,9 @@ public class TranscoderActivity extends AppCompatActivity implements private boolean mIsTranscoding; private Future mTranscodeFuture; - private Uri mTranscodeInputUri; + private Uri mTranscodeInputUri1; + private Uri mTranscodeInputUri2; + private Uri mTranscodeInputUri3; private File mTranscodeOutputFile; private long mTranscodeStartTime; private TrackStrategy mTranscodeVideoStrategy; @@ -66,7 +70,9 @@ protected void onCreate(Bundle savedInstanceState) { mButtonView = findViewById(R.id.button); mButtonView.setOnClickListener(v -> { if (!mIsTranscoding) { - startActivityForResult(new Intent(Intent.ACTION_GET_CONTENT).setType("video/*"), REQUEST_CODE_PICK); + startActivityForResult(new Intent(Intent.ACTION_GET_CONTENT) + .setType("video/*") + .putExtra(Intent.EXTRA_ALLOW_MULTIPLE, true), REQUEST_CODE_PICK); } else { mTranscodeFuture.cancel(true); } @@ -141,10 +147,19 @@ protected void onActivityResult(int requestCode, int resultCode, final Intent da super.onActivityResult(requestCode, resultCode, data); if (requestCode == REQUEST_CODE_PICK && resultCode == RESULT_OK - && data != null - && data.getData() != null) { - mTranscodeInputUri = data.getData(); - transcode(); + && data != null) { + if (data.getData() != null) { + mTranscodeInputUri1 = data.getData(); + mTranscodeInputUri2 = null; + mTranscodeInputUri3 = null; + transcode(); + } else if (data.getClipData() != null) { + ClipData clipData = data.getClipData(); + mTranscodeInputUri1 = clipData.getItemAt(0).getUri(); + mTranscodeInputUri2 = clipData.getItemCount() >= 2 ? clipData.getItemAt(1).getUri() : null; + mTranscodeInputUri3 = clipData.getItemCount() >= 3 ? clipData.getItemAt(2).getUri() : null; + transcode(); + } } } @@ -180,9 +195,11 @@ private void transcode() { // Launch the transcoding operation. mTranscodeStartTime = SystemClock.uptimeMillis(); setIsTranscoding(true); - mTranscodeFuture = Transcoder.into(mTranscodeOutputFile.getAbsolutePath()) - .setDataSource(this, mTranscodeInputUri) - .setListener(this) + TranscoderOptions.Builder builder = Transcoder.into(mTranscodeOutputFile.getAbsolutePath()); + if (mTranscodeInputUri1 != null) builder.addDataSource(this, mTranscodeInputUri1); + if (mTranscodeInputUri2 != null) builder.addDataSource(this, mTranscodeInputUri2); + if (mTranscodeInputUri3 != null) builder.addDataSource(this, mTranscodeInputUri3); + mTranscodeFuture = builder.setListener(this) .setAudioTrackStrategy(mTranscodeAudioStrategy) .setVideoTrackStrategy(mTranscodeVideoStrategy) .setRotation(rotation) @@ -216,7 +233,7 @@ public void onTranscodeCompleted(int successCode) { LOG.i("Transcoding was not needed."); onTranscodeFinished(true, "Transcoding not needed, source file not touched."); startActivity(new Intent(Intent.ACTION_VIEW) - .setDataAndType(mTranscodeInputUri, "video/mp4") + .setDataAndType(mTranscodeInputUri1, "video/mp4") .setFlags(Intent.FLAG_GRANT_READ_URI_PERMISSION)); } } diff --git a/lib/src/main/java/com/otaliastudios/transcoder/TranscoderOptions.java b/lib/src/main/java/com/otaliastudios/transcoder/TranscoderOptions.java index 04b92f89..8177970d 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/TranscoderOptions.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/TranscoderOptions.java @@ -114,9 +114,7 @@ public static class Builder { @NonNull @SuppressWarnings("WeakerAccess") - public Builder setDataSource(@NonNull DataSource dataSource) { - audioDataSources.clear(); - videoDataSources.clear(); + public Builder addDataSource(@NonNull DataSource dataSource) { audioDataSources.add(dataSource); videoDataSources.add(dataSource); return this; @@ -135,8 +133,8 @@ public Builder addDataSource(@NonNull TrackType type, @NonNull DataSource dataSo @NonNull @SuppressWarnings("unused") - public Builder setDataSource(@NonNull FileDescriptor fileDescriptor) { - return setDataSource(new FileDescriptorDataSource(fileDescriptor)); + public Builder addDataSource(@NonNull FileDescriptor fileDescriptor) { + return addDataSource(new FileDescriptorDataSource(fileDescriptor)); } @NonNull @@ -147,8 +145,8 @@ public Builder addDataSource(@NonNull TrackType type, @NonNull FileDescriptor fi @NonNull @SuppressWarnings("unused") - public Builder setDataSource(@NonNull String inPath) { - return setDataSource(new FilePathDataSource(inPath)); + public Builder addDataSource(@NonNull String inPath) { + return addDataSource(new FilePathDataSource(inPath)); } @NonNull @@ -158,9 +156,9 @@ public Builder addDataSource(@NonNull TrackType type, @NonNull String inPath) { } @NonNull - @SuppressWarnings("unused") - public Builder setDataSource(@NonNull Context context, @NonNull Uri uri) { - return setDataSource(new UriDataSource(context, uri)); + @SuppressWarnings({"unused", "UnusedReturnValue"}) + public Builder addDataSource(@NonNull Context context, @NonNull Uri uri) { + return addDataSource(new UriDataSource(context, uri)); } @NonNull diff --git a/lib/src/main/java/com/otaliastudios/transcoder/strategy/DefaultAudioStrategy.java b/lib/src/main/java/com/otaliastudios/transcoder/strategy/DefaultAudioStrategy.java index 4ffbc771..f73a1dbc 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/strategy/DefaultAudioStrategy.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/strategy/DefaultAudioStrategy.java @@ -4,6 +4,7 @@ import android.media.MediaFormat; import com.otaliastudios.transcoder.engine.TrackStatus; +import com.otaliastudios.transcoder.internal.Logger; import com.otaliastudios.transcoder.internal.MediaFormatConstants; import androidx.annotation.NonNull; @@ -16,7 +17,10 @@ */ public class DefaultAudioStrategy implements TrackStrategy { - public static final int AUDIO_CHANNELS_AS_IS = -1; + private final static String TAG = DefaultAudioStrategy.class.getSimpleName(); + private final static Logger LOG = new Logger(TAG); + + public final static int AUDIO_CHANNELS_AS_IS = -1; private int channels; @@ -27,13 +31,12 @@ public DefaultAudioStrategy(int channels) { @NonNull @Override public TrackStatus createOutputFormat(@NonNull List inputFormats, @NonNull MediaFormat outputFormat) { - int outputChannels = (channels == AUDIO_CHANNELS_AS_IS) ? getInputChannelCount(inputFormats) : channels; outputFormat.setString(MediaFormat.KEY_MIME, MediaFormatConstants.MIMETYPE_AUDIO_AAC); outputFormat.setInteger(MediaFormat.KEY_SAMPLE_RATE, getInputSampleRate(inputFormats)); outputFormat.setInteger(MediaFormat.KEY_CHANNEL_COUNT, outputChannels); outputFormat.setInteger(MediaFormat.KEY_AAC_PROFILE, MediaCodecInfo.CodecProfileLevel.AACObjectLC); - outputFormat.setInteger(MediaFormat.KEY_BIT_RATE, getInputBitRate(inputFormats)); + outputFormat.setInteger(MediaFormat.KEY_BIT_RATE, getAverageInputBitRate(inputFormats)); return TrackStatus.COMPRESSING; } @@ -49,19 +52,19 @@ private int getInputSampleRate(@NonNull List formats) { int rate = formats.get(0).getInteger(MediaFormat.KEY_SAMPLE_RATE); for (MediaFormat format : formats) { if (rate != format.getInteger(MediaFormat.KEY_SAMPLE_RATE)) { - throw new IllegalArgumentException("All input formats should have the same sample rate."); + LOG.e("Audio sampleRate should be equal for all DataSources audio tracks"); + // throw new IllegalArgumentException("All input formats should have the same sample rate."); } } return rate; } - private int getInputBitRate(@NonNull List formats) { - int rate = formats.get(0).getInteger(MediaFormat.KEY_BIT_RATE); + private int getAverageInputBitRate(@NonNull List formats) { + int count = formats.size(); + double bitRate = 0; for (MediaFormat format : formats) { - if (rate != format.getInteger(MediaFormat.KEY_BIT_RATE)) { - throw new IllegalArgumentException("All input formats should have the same bit rate."); - } + bitRate += format.getInteger(MediaFormat.KEY_BIT_RATE); } - return rate; + return (int) (bitRate / count); } } diff --git a/lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/AudioEngine.java b/lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/AudioEngine.java index da57cf23..bae715e5 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/AudioEngine.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/AudioEngine.java @@ -70,7 +70,8 @@ public AudioEngine(@NonNull MediaCodec decoder, int outputSampleRate = encoderOutputFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE); int inputSampleRate = decoderOutputFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE); if (inputSampleRate != outputSampleRate) { - throw new UnsupportedOperationException("Audio sample rate conversion not supported yet."); + LOG.e("Audio sample rate conversion is not supported. The result might be corrupted."); + // throw new UnsupportedOperationException("Audio sample rate conversion not supported yet."); } mSampleRate = inputSampleRate; From de97eb15ce65e25c439f47a4a381ed4200ebd6fb Mon Sep 17 00:00:00 2001 From: Mattia Iavarone Date: Sun, 4 Aug 2019 01:10:10 +0200 Subject: [PATCH 11/13] Rotate through OpenGL instead of metadata --- README.md | 2 +- .../transcoder/demo/TranscoderActivity.java | 2 +- .../transcoder/TranscoderOptions.java | 8 ++---- .../transcoder/engine/Engine.java | 10 +++---- .../transcode/VideoTrackTranscoder.java | 26 ++++++++++++++----- .../internal/VideoDecoderOutput.java | 17 +++++++++++- 6 files changed, 44 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index c132e86c..de81a97c 100644 --- a/README.md +++ b/README.md @@ -325,7 +325,7 @@ rotation to the input video frames. Accepted values are `0`, `90`, `180`, `270`: ```java Transcoder.into(filePath) - .setRotation(rotation) // 0, 90, 180, 270 + .setVideoRotation(rotation) // 0, 90, 180, 270 // ... ``` diff --git a/demo/src/main/java/com/otaliastudios/transcoder/demo/TranscoderActivity.java b/demo/src/main/java/com/otaliastudios/transcoder/demo/TranscoderActivity.java index 252e6ad8..aa0b1bb0 100644 --- a/demo/src/main/java/com/otaliastudios/transcoder/demo/TranscoderActivity.java +++ b/demo/src/main/java/com/otaliastudios/transcoder/demo/TranscoderActivity.java @@ -202,7 +202,7 @@ private void transcode() { mTranscodeFuture = builder.setListener(this) .setAudioTrackStrategy(mTranscodeAudioStrategy) .setVideoTrackStrategy(mTranscodeVideoStrategy) - .setRotation(rotation) + .setVideoRotation(rotation) .setSpeed(speed) .transcode(); } diff --git a/lib/src/main/java/com/otaliastudios/transcoder/TranscoderOptions.java b/lib/src/main/java/com/otaliastudios/transcoder/TranscoderOptions.java index 8177970d..fa695d13 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/TranscoderOptions.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/TranscoderOptions.java @@ -81,7 +81,7 @@ public Validator getValidator() { return validator; } - public int getRotation() { + public int getVideoRotation() { return rotation; } @@ -240,7 +240,7 @@ public Builder setValidator(@Nullable Validator validator) { */ @NonNull @SuppressWarnings("unused") - public Builder setRotation(int rotation) { + public Builder setVideoRotation(int rotation) { this.rotation = rotation; return this; } @@ -295,10 +295,6 @@ public TranscoderOptions build() { if (rotation != 0 && rotation != 90 && rotation != 180 && rotation != 270) { throw new IllegalArgumentException("Accepted values for rotation are 0, 90, 180, 270"); } - if (rotation != 0 && videoDataSources.size() > 1) { - // TODO support this, by not using metadata and instead rotating the GL texture - throw new IllegalStateException("Rotation should be 0 if you have more than one video source."); - } if (listenerHandler == null) { Looper looper = Looper.myLooper(); if (looper == null) looper = Looper.getMainLooper(); diff --git a/lib/src/main/java/com/otaliastudios/transcoder/engine/Engine.java b/lib/src/main/java/com/otaliastudios/transcoder/engine/Engine.java index 4aa46d37..74bad12c 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/engine/Engine.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/engine/Engine.java @@ -166,7 +166,8 @@ private void openCurrentStep(@NonNull TrackType type, @NonNull TranscoderOptions case COMPRESSING: { switch (type) { case VIDEO: - transcoder = new VideoTrackTranscoder(dataSource, mDataSink, interpolator); + transcoder = new VideoTrackTranscoder(dataSource, mDataSink, + interpolator, options.getVideoRotation()); break; case AUDIO: transcoder = new AudioTrackTranscoder(dataSource, mDataSink, @@ -282,10 +283,7 @@ public void transcode(@NonNull TranscoderOptions options) throws InterruptedExce mDataSources.setAudio(options.getAudioDataSources()); // Pass metadata from DataSource to DataSink - if (hasVideoSources()) { - DataSource firstVideoSource = mDataSources.requireVideo().get(0); - mDataSink.setOrientation((firstVideoSource.getOrientation() + options.getRotation()) % 360); - } + mDataSink.setOrientation(0); // Explicitly set 0 to output - we rotate the textures. for (DataSource locationSource : getUniqueSources()) { double[] location = locationSource.getLocation(); if (location != null) { @@ -318,7 +316,7 @@ public void transcode(@NonNull TranscoderOptions options) throws InterruptedExce // If we have to apply some rotation, and the video should be transcoded, // ignore any Validator trying to abort the operation. The operation must happen // because we must apply the rotation. - ignoreValidatorResult = videoStatus.isTranscoding() && options.getRotation() != 0; + ignoreValidatorResult = videoStatus.isTranscoding() && options.getVideoRotation() != 0; if (!options.getValidator().validate(videoStatus, audioStatus) && !ignoreValidatorResult) { throw new ValidatorException("Validator returned false."); } diff --git a/lib/src/main/java/com/otaliastudios/transcoder/transcode/VideoTrackTranscoder.java b/lib/src/main/java/com/otaliastudios/transcoder/transcode/VideoTrackTranscoder.java index 8192abff..950088ff 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/transcode/VideoTrackTranscoder.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/transcode/VideoTrackTranscoder.java @@ -45,14 +45,29 @@ public class VideoTrackTranscoder extends BaseTrackTranscoder { private VideoEncoderInput mEncoderInputSurface; private MediaCodec mEncoder; // Keep this since we want to signal EOS on it. private VideoFrameDropper mFrameDropper; - private TimeInterpolator mTimeInterpolator; + private final TimeInterpolator mTimeInterpolator; + private final int mRotation; + private final boolean mFlip; public VideoTrackTranscoder( @NonNull DataSource dataSource, @NonNull DataSink dataSink, - @NonNull TimeInterpolator timeInterpolator) { + @NonNull TimeInterpolator timeInterpolator, + int rotation) { super(dataSource, dataSink, TrackType.VIDEO); mTimeInterpolator = timeInterpolator; + mRotation = (rotation + dataSource.getOrientation()) % 360; + mFlip = mRotation % 180 != 0; + } + + @Override + protected void onConfigureEncoder(@NonNull MediaFormat format, @NonNull MediaCodec encoder) { + // Flip the width and height as needed. + int width = format.getInteger(MediaFormat.KEY_WIDTH); + int height = format.getInteger(MediaFormat.KEY_HEIGHT); + format.setInteger(MediaFormat.KEY_WIDTH, mFlip ? height : width); + format.setInteger(MediaFormat.KEY_HEIGHT, mFlip ? width : height); + super.onConfigureEncoder(format, encoder); } @Override @@ -69,6 +84,7 @@ protected void onConfigureDecoder(@NonNull MediaFormat format, @NonNull MediaCod format.setInteger(MediaFormatConstants.KEY_ROTATION_DEGREES, 0); } mDecoderOutputSurface = new VideoDecoderOutput(); + mDecoderOutputSurface.setRotation(mRotation); decoder.configure(format, mDecoderOutputSurface.getSurface(), null, 0); } @@ -84,8 +100,8 @@ protected void onCodecsStarted(@NonNull MediaFormat inputFormat, @NonNull MediaF float inputWidth = inputFormat.getInteger(MediaFormat.KEY_WIDTH); float inputHeight = inputFormat.getInteger(MediaFormat.KEY_HEIGHT); float inputRatio = inputWidth / inputHeight; - float outputWidth = outputFormat.getInteger(MediaFormat.KEY_WIDTH); - float outputHeight = outputFormat.getInteger(MediaFormat.KEY_HEIGHT); + float outputWidth = mFlip ? outputFormat.getInteger(MediaFormat.KEY_HEIGHT) : outputFormat.getInteger(MediaFormat.KEY_WIDTH); + float outputHeight = mFlip ? outputFormat.getInteger(MediaFormat.KEY_WIDTH) : outputFormat.getInteger(MediaFormat.KEY_HEIGHT); float outputRatio = outputWidth / outputHeight; float scaleX = 1, scaleY = 1; if (inputRatio > outputRatio) { // Input wider. We have a scaleX. @@ -93,8 +109,6 @@ protected void onCodecsStarted(@NonNull MediaFormat inputFormat, @NonNull MediaF } else if (inputRatio < outputRatio) { // Input taller. We have a scaleY. scaleY = outputRatio / inputRatio; } - // I don't think we should consider rotation and flip these - we operate on non-rotated - // surfaces and pass the input rotation metadata to the output muxer, see Engine.setupMetadata. mDecoderOutputSurface.setScale(scaleX, scaleY); } diff --git a/lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/VideoDecoderOutput.java b/lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/VideoDecoderOutput.java index da7952b1..e6a7070b 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/VideoDecoderOutput.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/VideoDecoderOutput.java @@ -42,6 +42,7 @@ public class VideoDecoderOutput { private float mScaleX = 1F; private float mScaleY = 1F; + private int mRotation = 0; private int mTextureId; private float[] mTextureTransform = new float[16]; @@ -91,6 +92,15 @@ public void setScale(float scaleX, float scaleY) { mScaleY = scaleY; } + /** + * Sets the desired frame rotation with respect + * to its natural orientation. + * @param rotation rotation + */ + public void setRotation(int rotation) { + mRotation = rotation; + } + /** * Returns a Surface to draw onto. * @return the output surface @@ -164,8 +174,13 @@ private void drawNewFrame() { float glTranslX = (1F - glScaleX) / 2F; float glTranslY = (1F - glScaleY) / 2F; Matrix.translateM(mTextureTransform, 0, glTranslX, glTranslY, 0); - // Scale and draw. + // Scale. Matrix.scaleM(mTextureTransform, 0, glScaleX, glScaleY, 1); + // Apply rotation. + Matrix.translateM(mTextureTransform, 0, 0.5F, 0.5F, 0); + Matrix.rotateM(mTextureTransform, 0, mRotation, 0, 0, 1); + Matrix.translateM(mTextureTransform, 0, -0.5F, -0.5F, 0); + // Draw. mScene.drawTexture(mDrawable, mProgram, mTextureId, mTextureTransform); } } From 095928825cb3d9e13ce43f904163b562d24b9be5 Mon Sep 17 00:00:00 2001 From: Mattia Iavarone Date: Sun, 4 Aug 2019 11:52:19 +0200 Subject: [PATCH 12/13] Use DataSource getReadUs --- .../otaliastudios/transcoder/engine/Engine.java | 9 ++++++--- .../transcoder/source/AndroidDataSource.java | 12 +++++------- .../transcoder/source/DataSource.java | 15 +++------------ .../transcoder/stretch/AudioStretcher.java | 14 -------------- 4 files changed, 14 insertions(+), 36 deletions(-) diff --git a/lib/src/main/java/com/otaliastudios/transcoder/engine/Engine.java b/lib/src/main/java/com/otaliastudios/transcoder/engine/Engine.java index 74bad12c..7d14d45f 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/engine/Engine.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/engine/Engine.java @@ -256,11 +256,11 @@ private double getTrackProgress(@NonNull TrackType type) { for (int i = 0; i < mDataSources.require(type).size(); i++) { DataSource source = mDataSources.require(type).get(i); if (i < current) { - totalDurationUs += source.getLastTimestampUs() - source.getFirstTimestampUs(); - completedDurationUs += source.getLastTimestampUs() - source.getFirstTimestampUs(); + totalDurationUs += source.getReadUs(); + completedDurationUs += source.getReadUs(); } else if (i == current) { totalDurationUs += source.getDurationUs(); - completedDurationUs += source.getLastTimestampUs() - source.getFirstTimestampUs(); + completedDurationUs += source.getReadUs(); } else { totalDurationUs += source.getDurationUs(); completedDurationUs += 0; @@ -299,7 +299,10 @@ public void transcode(@NonNull TranscoderOptions options) throws InterruptedExce for (DataSource source : options.getAudioDataSources()) audioDurationUs += source.getDurationUs(); long totalDurationUs = Math.min(audioDurationUs, videoDurationUs); LOG.v("Duration (us): " + totalDurationUs); + // TODO if audio and video have different lengths, we should clip the longer one! + // TODO audio resampling + // TODO ClipDataSource or something like that, to choose // Compute the TrackStatus. int activeTracks = 0; diff --git a/lib/src/main/java/com/otaliastudios/transcoder/source/AndroidDataSource.java b/lib/src/main/java/com/otaliastudios/transcoder/source/AndroidDataSource.java index c8402c03..a9c4c702 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/source/AndroidDataSource.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/source/AndroidDataSource.java @@ -85,13 +85,11 @@ public void readTrack(@NonNull Chunk chunk) { } @Override - public long getFirstTimestampUs() { - return mFirstTimestampUs; - } - - @Override - public long getLastTimestampUs() { - return mLastTimestampUs; + public long getReadUs() { + if (mFirstTimestampUs == Long.MIN_VALUE) { + return 0; + } + return mLastTimestampUs - mFirstTimestampUs; } @Nullable diff --git a/lib/src/main/java/com/otaliastudios/transcoder/source/DataSource.java b/lib/src/main/java/com/otaliastudios/transcoder/source/DataSource.java index 075c9315..ff1e2869 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/source/DataSource.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/source/DataSource.java @@ -79,20 +79,11 @@ public interface DataSource { void readTrack(@NonNull DataSource.Chunk chunk); /** - * Returns the latest timestamp that was read using - * {@link #readTrack(Chunk)}. + * Returns the total number of microseconds that have been read until now. * - * @return latest timestamp + * @return total read us */ - long getLastTimestampUs(); - - /** - * Returns the first timestamp that was read using - * {@link #readTrack(Chunk)}. - * - * @return first timestamp - */ - long getFirstTimestampUs(); + long getReadUs(); /** * When this source has been totally read, it can return true here to diff --git a/lib/src/main/java/com/otaliastudios/transcoder/stretch/AudioStretcher.java b/lib/src/main/java/com/otaliastudios/transcoder/stretch/AudioStretcher.java index e35c4f26..d2413cae 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/stretch/AudioStretcher.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/stretch/AudioStretcher.java @@ -33,18 +33,4 @@ public interface AudioStretcher { AudioStretcher CUT = new CutAudioStretcher(); AudioStretcher INSERT = new InsertAudioStretcher(); - - - AudioStretcher CUT_OR_INSERT = new AudioStretcher() { - @Override - public void stretch(@NonNull ShortBuffer input, @NonNull ShortBuffer output, int channels) { - if (input.remaining() < output.remaining()) { - INSERT.stretch(input, output, channels); - } else if (input.remaining() > output.remaining()) { - CUT.stretch(input, output, channels); - } else { - PASSTHROUGH.stretch(input, output, channels); - } - } - }; } From d66fa866a6ceb1d7d559d048bc5581f1b67fb81b Mon Sep 17 00:00:00 2001 From: Mattia Iavarone Date: Sun, 4 Aug 2019 12:38:25 +0200 Subject: [PATCH 13/13] Add documentation --- CHANGELOG.md | 10 ++++++++++ README.md | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 06324394..0ae0a5fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,12 @@ +### v0.7.0 (to be released) + +- New: video concatenation to stitch together multiple media ([#14][14]) +- New: select a specific track type (`VIDEO` or `AUDIO`) for sources ([#14][14]) +- Breaking change: `TranscoderOptions.setDataSource()` renamed to `addDataSource()` ([#14][14]) +- Breaking change: `TranscoderOptions.setRotation()` renamed to `setVideoRotation()` ([#14][14]) +- Breaking change: `DefaultVideoStrategy.iFrameInterval()` renamed to `keyFrameInterval()` ([#14][14]) +- Improvement: rotate videos through OpenGL instead of using metadata ([#14][14]) + ### v0.6.0 - New: ability to change video/audio speed and change each frame timestamp ([#10][10]) @@ -23,3 +32,4 @@ [8]: https://github.com/natario1/Transcoder/pull/8 [9]: https://github.com/natario1/Transcoder/pull/9 [10]: https://github.com/natario1/Transcoder/pull/10 +[14]: https://github.com/natario1/Transcoder/pull/14 diff --git a/README.md b/README.md index de81a97c..c4b2e3c4 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,7 @@ Take a look at the demo app for a real example or keep reading below for documen - Hardware accelerated - Multithreaded - Convenient, fluent API +- Concatenate multiple video and audio tracks [[docs]](#video-concatenation) - Choose output size, with automatic cropping [[docs]](#video-size) - Choose output rotation [[docs]](#video-rotation) - Choose output speed [[docs]](#video-speed) @@ -92,6 +93,44 @@ simply `addDataSource(descriptor)` in the transcoding builder. A data source backed by a file absolute path. Use `new FilePathDataSource(path)` or simply `addDataSource(path)` in the transcoding builder. +## Video Concatenation + +As you might have guessed, you can use `addDataSource(source)` multiple times. All the source +files will be stitched together: + +```java +Transcoder.into(filePath) + .addDataSource(source1) + .addDataSource(source2) + .addDataSource(source3) + // ... +``` + +In the above example, the three videos will be stitched together in the order they are added +to the builder. Once `source1` ends, we'll append `source2` and so on. The library will take care +of applying consistent parameters (frame rate, bit rate, sample rate) during the conversion. + +This is a powerful tool since it can be used per-track: + +```java +Transcoder.into(filePath) + .addDataSource(source1) // Audio & Video, 20 seconds + .addDataSource(TrackType.VIDEO, source2) // Video, 5 seconds + .addDataSource(TrackType.VIDEO, source3) // Video, 5 seconds + .addDataSource(TrackType.AUDIO, source4) // Audio, 10 sceonds + // ... +``` + +In the above example, the output file will be 30 seconds long: + +``` + _____________________________________________________________________________________ +Video |___________________source1_____________________:_____source2_____:______source3______| +Audio |___________________source1_____________________:______________source4________________| +``` + +And that's all you need. + ## Listening for events Transcoding will happen on a background thread, but we will send updates through the `TranscoderListener`