diff --git a/CHANGELOG.md b/CHANGELOG.md index 06324394..0ae0a5fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,12 @@ +### v0.7.0 (to be released) + +- New: video concatenation to stitch together multiple media ([#14][14]) +- New: select a specific track type (`VIDEO` or `AUDIO`) for sources ([#14][14]) +- Breaking change: `TranscoderOptions.setDataSource()` renamed to `addDataSource()` ([#14][14]) +- Breaking change: `TranscoderOptions.setRotation()` renamed to `setVideoRotation()` ([#14][14]) +- Breaking change: `DefaultVideoStrategy.iFrameInterval()` renamed to `keyFrameInterval()` ([#14][14]) +- Improvement: rotate videos through OpenGL instead of using metadata ([#14][14]) + ### v0.6.0 - New: ability to change video/audio speed and change each frame timestamp ([#10][10]) @@ -23,3 +32,4 @@ [8]: https://github.com/natario1/Transcoder/pull/8 [9]: https://github.com/natario1/Transcoder/pull/9 [10]: https://github.com/natario1/Transcoder/pull/10 +[14]: https://github.com/natario1/Transcoder/pull/14 diff --git a/README.md b/README.md index bd237caf..c4b2e3c4 100644 --- a/README.md +++ b/README.md @@ -22,10 +22,10 @@ Using Transcoder in the most basic form is pretty simple: ```java Transcoder.into(filePath) - .setDataSource(context, uri) // or... - .setDataSource(filePath) // or... - .setDataSource(fileDescriptor) // or... - .setDataSource(dataSource) + .addDataSource(context, uri) // or... + .addDataSource(filePath) // or... + .addDataSource(fileDescriptor) // or... + .addDataSource(dataSource) .setListener(new TranscoderListener() { public void onTranscodeProgress(double progress) {} public void onTranscodeCompleted(int successCode) {} @@ -42,6 +42,7 @@ Take a look at the demo app for a real example or keep reading below for documen - Hardware accelerated - Multithreaded - Convenient, fluent API +- Concatenate multiple video and audio tracks [[docs]](#video-concatenation) - Choose output size, with automatic cropping [[docs]](#video-size) - Choose output rotation [[docs]](#video-rotation) - Choose output speed [[docs]](#video-speed) @@ -80,17 +81,55 @@ which is convenient but it means that they can not be used twice. #### `UriDataSource` The Android friendly source can be created with `new UriDataSource(context, uri)` or simply -using `setDataSource(context, uri)` in the transcoding builder. +using `addDataSource(context, uri)` in the transcoding builder. #### `FileDescriptorDataSource` A data source backed by a file descriptor. Use `new FileDescriptorDataSource(descriptor)` or -simply `setDataSource(descriptor)` in the transcoding builder. +simply `addDataSource(descriptor)` in the transcoding builder. #### `FilePathDataSource` A data source backed by a file absolute path. Use `new FilePathDataSource(path)` or -simply `setDataSource(path)` in the transcoding builder. +simply `addDataSource(path)` in the transcoding builder. + +## Video Concatenation + +As you might have guessed, you can use `addDataSource(source)` multiple times. All the source +files will be stitched together: + +```java +Transcoder.into(filePath) + .addDataSource(source1) + .addDataSource(source2) + .addDataSource(source3) + // ... +``` + +In the above example, the three videos will be stitched together in the order they are added +to the builder. Once `source1` ends, we'll append `source2` and so on. The library will take care +of applying consistent parameters (frame rate, bit rate, sample rate) during the conversion. + +This is a powerful tool since it can be used per-track: + +```java +Transcoder.into(filePath) + .addDataSource(source1) // Audio & Video, 20 seconds + .addDataSource(TrackType.VIDEO, source2) // Video, 5 seconds + .addDataSource(TrackType.VIDEO, source3) // Video, 5 seconds + .addDataSource(TrackType.AUDIO, source4) // Audio, 10 sceonds + // ... +``` + +In the above example, the output file will be 30 seconds long: + +``` + _____________________________________________________________________________________ +Video |___________________source1_____________________:_____source2_____:______source3______| +Audio |___________________source1_____________________:______________source4________________| +``` + +And that's all you need. ## Listening for events @@ -312,7 +351,7 @@ DefaultVideoStrategy strategy = new DefaultVideoStrategy.Builder() .bitRate(bitRate) .bitRate(DefaultVideoStrategy.BITRATE_UNKNOWN) // tries to estimate .frameRate(frameRate) // will be capped to the input frameRate - .iFrameInterval(interval) // interval between I-frames in seconds + .keyFrameInterval(interval) // interval between key-frames in seconds .build(); ``` @@ -325,7 +364,7 @@ rotation to the input video frames. Accepted values are `0`, `90`, `180`, `270`: ```java Transcoder.into(filePath) - .setRotation(rotation) // 0, 90, 180, 270 + .setVideoRotation(rotation) // 0, 90, 180, 270 // ... ``` diff --git a/demo/src/main/java/com/otaliastudios/transcoder/demo/TranscoderActivity.java b/demo/src/main/java/com/otaliastudios/transcoder/demo/TranscoderActivity.java index e44495c7..aa0b1bb0 100644 --- a/demo/src/main/java/com/otaliastudios/transcoder/demo/TranscoderActivity.java +++ b/demo/src/main/java/com/otaliastudios/transcoder/demo/TranscoderActivity.java @@ -1,5 +1,6 @@ package com.otaliastudios.transcoder.demo; +import android.content.ClipData; import android.content.Intent; import android.net.Uri; import android.os.Bundle; @@ -11,6 +12,7 @@ import com.otaliastudios.transcoder.Transcoder; import com.otaliastudios.transcoder.TranscoderListener; +import com.otaliastudios.transcoder.TranscoderOptions; import com.otaliastudios.transcoder.internal.Logger; import com.otaliastudios.transcoder.strategy.DefaultAudioStrategy; import com.otaliastudios.transcoder.strategy.DefaultVideoStrategy; @@ -51,7 +53,9 @@ public class TranscoderActivity extends AppCompatActivity implements private boolean mIsTranscoding; private Future mTranscodeFuture; - private Uri mTranscodeInputUri; + private Uri mTranscodeInputUri1; + private Uri mTranscodeInputUri2; + private Uri mTranscodeInputUri3; private File mTranscodeOutputFile; private long mTranscodeStartTime; private TrackStrategy mTranscodeVideoStrategy; @@ -66,7 +70,9 @@ protected void onCreate(Bundle savedInstanceState) { mButtonView = findViewById(R.id.button); mButtonView.setOnClickListener(v -> { if (!mIsTranscoding) { - startActivityForResult(new Intent(Intent.ACTION_GET_CONTENT).setType("video/*"), REQUEST_CODE_PICK); + startActivityForResult(new Intent(Intent.ACTION_GET_CONTENT) + .setType("video/*") + .putExtra(Intent.EXTRA_ALLOW_MULTIPLE, true), REQUEST_CODE_PICK); } else { mTranscodeFuture.cancel(true); } @@ -141,10 +147,19 @@ protected void onActivityResult(int requestCode, int resultCode, final Intent da super.onActivityResult(requestCode, resultCode, data); if (requestCode == REQUEST_CODE_PICK && resultCode == RESULT_OK - && data != null - && data.getData() != null) { - mTranscodeInputUri = data.getData(); - transcode(); + && data != null) { + if (data.getData() != null) { + mTranscodeInputUri1 = data.getData(); + mTranscodeInputUri2 = null; + mTranscodeInputUri3 = null; + transcode(); + } else if (data.getClipData() != null) { + ClipData clipData = data.getClipData(); + mTranscodeInputUri1 = clipData.getItemAt(0).getUri(); + mTranscodeInputUri2 = clipData.getItemCount() >= 2 ? clipData.getItemAt(1).getUri() : null; + mTranscodeInputUri3 = clipData.getItemCount() >= 3 ? clipData.getItemAt(2).getUri() : null; + transcode(); + } } } @@ -180,12 +195,14 @@ private void transcode() { // Launch the transcoding operation. mTranscodeStartTime = SystemClock.uptimeMillis(); setIsTranscoding(true); - mTranscodeFuture = Transcoder.into(mTranscodeOutputFile.getAbsolutePath()) - .setDataSource(this, mTranscodeInputUri) - .setListener(this) + TranscoderOptions.Builder builder = Transcoder.into(mTranscodeOutputFile.getAbsolutePath()); + if (mTranscodeInputUri1 != null) builder.addDataSource(this, mTranscodeInputUri1); + if (mTranscodeInputUri2 != null) builder.addDataSource(this, mTranscodeInputUri2); + if (mTranscodeInputUri3 != null) builder.addDataSource(this, mTranscodeInputUri3); + mTranscodeFuture = builder.setListener(this) .setAudioTrackStrategy(mTranscodeAudioStrategy) .setVideoTrackStrategy(mTranscodeVideoStrategy) - .setRotation(rotation) + .setVideoRotation(rotation) .setSpeed(speed) .transcode(); } @@ -216,7 +233,7 @@ public void onTranscodeCompleted(int successCode) { LOG.i("Transcoding was not needed."); onTranscodeFinished(true, "Transcoding not needed, source file not touched."); startActivity(new Intent(Intent.ACTION_VIEW) - .setDataAndType(mTranscodeInputUri, "video/mp4") + .setDataAndType(mTranscodeInputUri1, "video/mp4") .setFlags(Intent.FLAG_GRANT_READ_URI_PERMISSION)); } } diff --git a/lib/src/main/java/com/otaliastudios/transcoder/Transcoder.java b/lib/src/main/java/com/otaliastudios/transcoder/Transcoder.java index 55b7452d..d86921e2 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/Transcoder.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/Transcoder.java @@ -107,12 +107,12 @@ public static TranscoderOptions.Builder into(@NonNull String outPath) { @NonNull public Future transcode(@NonNull final TranscoderOptions options) { final TranscoderListener listenerWrapper = new ListenerWrapper(options.listenerHandler, - options.listener, options.getDataSource()); + options.listener); return mExecutor.submit(new Callable() { @Override public Void call() throws Exception { try { - Engine engine = new Engine(options.getDataSource(), new Engine.ProgressCallback() { + Engine engine = new Engine(new Engine.ProgressCallback() { @Override public void onProgress(final double progress) { listenerWrapper.onTranscodeProgress(progress); @@ -154,21 +154,16 @@ public void onProgress(final double progress) { } /** - * Wraps a TranscoderListener and a DataSource object, ensuring that the source - * is released when transcoding ends, fails or is canceled. - * - * It posts events on the given handler. + * Wraps a TranscoderListener and posts events on the given handler. */ private static class ListenerWrapper implements TranscoderListener { private Handler mHandler; private TranscoderListener mListener; - private DataSource mDataSource; - private ListenerWrapper(@NonNull Handler handler, @NonNull TranscoderListener listener, @NonNull DataSource source) { + private ListenerWrapper(@NonNull Handler handler, @NonNull TranscoderListener listener) { mHandler = handler; mListener = listener; - mDataSource = source; } @Override @@ -176,7 +171,6 @@ public void onTranscodeCanceled() { mHandler.post(new Runnable() { @Override public void run() { - mDataSource.release(); mListener.onTranscodeCanceled(); } }); @@ -187,7 +181,6 @@ public void onTranscodeCompleted(final int successCode) { mHandler.post(new Runnable() { @Override public void run() { - mDataSource.release(); mListener.onTranscodeCompleted(successCode); } }); @@ -198,7 +191,6 @@ public void onTranscodeFailed(@NonNull final Throwable exception) { mHandler.post(new Runnable() { @Override public void run() { - mDataSource.release(); mListener.onTranscodeFailed(exception); } }); diff --git a/lib/src/main/java/com/otaliastudios/transcoder/TranscoderOptions.java b/lib/src/main/java/com/otaliastudios/transcoder/TranscoderOptions.java index 2bccb0ce..fa695d13 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/TranscoderOptions.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/TranscoderOptions.java @@ -5,6 +5,7 @@ import android.os.Handler; import android.os.Looper; +import com.otaliastudios.transcoder.engine.TrackType; import com.otaliastudios.transcoder.source.DataSource; import com.otaliastudios.transcoder.source.FileDescriptorDataSource; import com.otaliastudios.transcoder.source.FilePathDataSource; @@ -21,6 +22,8 @@ import com.otaliastudios.transcoder.validator.Validator; import java.io.FileDescriptor; +import java.util.ArrayList; +import java.util.List; import java.util.concurrent.Future; import androidx.annotation.NonNull; @@ -34,7 +37,8 @@ public class TranscoderOptions { private TranscoderOptions() {} private String outPath; - private DataSource dataSource; + private List videoDataSources; + private List audioDataSources; private TrackStrategy audioTrackStrategy; private TrackStrategy videoTrackStrategy; private Validator validator; @@ -52,8 +56,14 @@ public String getOutputPath() { @NonNull @SuppressWarnings("WeakerAccess") - public DataSource getDataSource() { - return dataSource; + public List getAudioDataSources() { + return audioDataSources; + } + + @NonNull + @SuppressWarnings("WeakerAccess") + public List getVideoDataSources() { + return videoDataSources; } @NonNull @@ -71,7 +81,7 @@ public Validator getValidator() { return validator; } - public int getRotation() { + public int getVideoRotation() { return rotation; } @@ -87,7 +97,8 @@ public AudioStretcher getAudioStretcher() { public static class Builder { private String outPath; - private DataSource dataSource; + private final List audioDataSources = new ArrayList<>(); + private final List videoDataSources = new ArrayList<>(); private TranscoderListener listener; private Handler listenerHandler; private TrackStrategy audioTrackStrategy; @@ -102,31 +113,58 @@ public static class Builder { } @NonNull - @SuppressWarnings("unused") - public Builder setDataSource(@NonNull DataSource dataSource) { - this.dataSource = dataSource; + @SuppressWarnings("WeakerAccess") + public Builder addDataSource(@NonNull DataSource dataSource) { + audioDataSources.add(dataSource); + videoDataSources.add(dataSource); return this; } @NonNull - @SuppressWarnings("unused") - public Builder setDataSource(@NonNull FileDescriptor fileDescriptor) { - this.dataSource = new FileDescriptorDataSource(fileDescriptor); + @SuppressWarnings("WeakerAccess") + public Builder addDataSource(@NonNull TrackType type, @NonNull DataSource dataSource) { + if (type == TrackType.AUDIO) { + audioDataSources.add(dataSource); + } else if (type == TrackType.VIDEO) { + videoDataSources.add(dataSource); + } return this; } @NonNull @SuppressWarnings("unused") - public Builder setDataSource(@NonNull String inPath) { - this.dataSource = new FilePathDataSource(inPath); - return this; + public Builder addDataSource(@NonNull FileDescriptor fileDescriptor) { + return addDataSource(new FileDescriptorDataSource(fileDescriptor)); } @NonNull @SuppressWarnings("unused") - public Builder setDataSource(@NonNull Context context, @NonNull Uri uri) { - this.dataSource = new UriDataSource(context, uri); - return this; + public Builder addDataSource(@NonNull TrackType type, @NonNull FileDescriptor fileDescriptor) { + return addDataSource(type, new FileDescriptorDataSource(fileDescriptor)); + } + + @NonNull + @SuppressWarnings("unused") + public Builder addDataSource(@NonNull String inPath) { + return addDataSource(new FilePathDataSource(inPath)); + } + + @NonNull + @SuppressWarnings("unused") + public Builder addDataSource(@NonNull TrackType type, @NonNull String inPath) { + return addDataSource(type, new FilePathDataSource(inPath)); + } + + @NonNull + @SuppressWarnings({"unused", "UnusedReturnValue"}) + public Builder addDataSource(@NonNull Context context, @NonNull Uri uri) { + return addDataSource(new UriDataSource(context, uri)); + } + + @NonNull + @SuppressWarnings("unused") + public Builder addDataSource(@NonNull TrackType type, @NonNull Context context, @NonNull Uri uri) { + return addDataSource(type, new UriDataSource(context, uri)); } /** @@ -202,7 +240,7 @@ public Builder setValidator(@Nullable Validator validator) { */ @NonNull @SuppressWarnings("unused") - public Builder setRotation(int rotation) { + public Builder setVideoRotation(int rotation) { this.rotation = rotation; return this; } @@ -248,8 +286,8 @@ public TranscoderOptions build() { if (listener == null) { throw new IllegalStateException("listener can't be null"); } - if (dataSource == null) { - throw new IllegalStateException("data source can't be null"); + if (audioDataSources.isEmpty() && videoDataSources.isEmpty()) { + throw new IllegalStateException("we need at least one data source"); } if (outPath == null) { throw new IllegalStateException("out path can't be null"); @@ -279,7 +317,8 @@ public TranscoderOptions build() { } TranscoderOptions options = new TranscoderOptions(); options.listener = listener; - options.dataSource = dataSource; + options.audioDataSources = audioDataSources; + options.videoDataSources = videoDataSources; options.outPath = outPath; options.listenerHandler = listenerHandler; options.audioTrackStrategy = audioTrackStrategy; diff --git a/lib/src/main/java/com/otaliastudios/transcoder/engine/Engine.java b/lib/src/main/java/com/otaliastudios/transcoder/engine/Engine.java index 5f638a35..7d14d45f 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/engine/Engine.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/engine/Engine.java @@ -25,7 +25,7 @@ import com.otaliastudios.transcoder.sink.MediaMuxerDataSink; import com.otaliastudios.transcoder.source.DataSource; import com.otaliastudios.transcoder.strategy.TrackStrategy; -import com.otaliastudios.transcoder.strategy.TrackStrategyException; +import com.otaliastudios.transcoder.time.TimeInterpolator; import com.otaliastudios.transcoder.transcode.AudioTrackTranscoder; import com.otaliastudios.transcoder.transcode.NoOpTrackTranscoder; import com.otaliastudios.transcoder.transcode.PassThroughTrackTranscoder; @@ -36,6 +36,12 @@ import androidx.annotation.NonNull; import androidx.annotation.Nullable; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + /** * Internal engine, do not use this directly. */ @@ -44,7 +50,6 @@ public class Engine { private static final String TAG = Engine.class.getSimpleName(); private static final Logger LOG = new Logger(TAG); - private static final double PROGRESS_UNKNOWN = -1.0; private static final long SLEEP_TO_WAIT_TRACK_TRANSCODERS = 10; private static final long PROGRESS_INTERVAL_STEPS = 10; @@ -58,21 +63,23 @@ public interface ProgressCallback { void onProgress(double progress); } - private DataSource mDataSource; private DataSink mDataSink; - private TrackTypeMap mTranscoders = new TrackTypeMap<>(); - private TrackTypeMap mStatuses = new TrackTypeMap<>(); + private final TrackTypeMap> mDataSources = new TrackTypeMap<>(); + private final TrackTypeMap> mTranscoders = new TrackTypeMap<>(new ArrayList(), new ArrayList()); + private final TrackTypeMap> mInterpolators = new TrackTypeMap<>(new ArrayList(), new ArrayList()); + private final TrackTypeMap mCurrentStep = new TrackTypeMap<>(0, 0); + private final TrackTypeMap mStatuses = new TrackTypeMap<>(); + private final TrackTypeMap mOutputFormats = new TrackTypeMap<>(); private volatile double mProgress; - private ProgressCallback mProgressCallback; - private long mDurationUs; + private final ProgressCallback mProgressCallback; - public Engine(@NonNull DataSource dataSource, @Nullable ProgressCallback progressCallback) { - mDataSource = dataSource; + public Engine(@Nullable ProgressCallback progressCallback) { mProgressCallback = progressCallback; } /** - * NOTE: This method is thread safe. + * Returns the current progress. + * Note: This method is thread safe. * @return the current progress */ @SuppressWarnings("unused") @@ -80,150 +87,276 @@ public double getProgress() { return mProgress; } - /** - * Performs transcoding. Blocks current thread. - * - * @param options Transcoding options. - * @throws InvalidOutputFormatException when output format is not supported. - * @throws InterruptedException when cancel to transcode - */ - public void transcode(@NonNull TranscoderOptions options) throws InterruptedException { - try { - // NOTE: use single extractor to keep from running out audio track fast. - mDataSink = new MediaMuxerDataSink(options.getOutputPath()); - mDataSink.setOrientation((mDataSource.getOrientation() + options.getRotation()) % 360); - double[] location = mDataSource.getLocation(); - if (location != null) mDataSink.setLocation(location[0], location[1]); - mDurationUs = mDataSource.getDurationUs(); - LOG.v("Duration (us): " + mDurationUs); - setUpTrackTranscoders(options); - runPipelines(); - mDataSink.stop(); - } finally { - try { - mTranscoders.require(TrackType.VIDEO).release(); - mTranscoders.require(TrackType.AUDIO).release(); - mTranscoders.clear(); - } catch (RuntimeException e) { - // Too fatal to make alive the app, because it may leak native resources. - //noinspection ThrowFromFinallyBlock - throw new Error("Could not shutdown extractor, codecs and muxer pipeline.", e); - } - mDataSink.release(); + private void setProgress(double progress) { + mProgress = progress; + if (mProgressCallback != null) { + mProgressCallback.onProgress(progress); } } - private void setUpTrackTranscoder(@NonNull TranscoderOptions options, - @NonNull TrackType type) { - TrackStatus status; - TrackTranscoder transcoder; - MediaFormat inputFormat = mDataSource.getFormat(type); - MediaFormat outputFormat = null; - if (inputFormat == null) { - transcoder = new NoOpTrackTranscoder(); - status = TrackStatus.ABSENT; - } else { - TrackStrategy strategy; - switch (type) { - case VIDEO: strategy = options.getVideoTrackStrategy(); break; - case AUDIO: strategy = options.getAudioTrackStrategy(); break; - default: throw new RuntimeException("Unknown type: " + type); - } - try { - outputFormat = strategy.createOutputFormat(inputFormat); - if (outputFormat == null) { - transcoder = new NoOpTrackTranscoder(); - status = TrackStatus.REMOVING; - } else if (outputFormat == inputFormat) { - transcoder = new PassThroughTrackTranscoder(mDataSource, mDataSink, type, options.getTimeInterpolator()); - status = TrackStatus.PASS_THROUGH; - } else { - switch (type) { - case VIDEO: transcoder = new VideoTrackTranscoder(mDataSource, mDataSink, options.getTimeInterpolator()); break; - case AUDIO: transcoder = new AudioTrackTranscoder(mDataSource, mDataSink, options.getTimeInterpolator(), options.getAudioStretcher()); break; - default: throw new RuntimeException("Unknown type: " + type); - } - status = TrackStatus.COMPRESSING; - } - } catch (TrackStrategyException strategyException) { - if (strategyException.getType() == TrackStrategyException.TYPE_ALREADY_COMPRESSED) { - // Should not abort, because the other track might need compression. Use a pass through. - transcoder = new PassThroughTrackTranscoder(mDataSource, mDataSink, type, options.getTimeInterpolator()); - status = TrackStatus.PASS_THROUGH; - } else { // Abort. - throw strategyException; + private boolean hasVideoSources() { + return !mDataSources.requireVideo().isEmpty(); + } + + private boolean hasAudioSources() { + return !mDataSources.requireAudio().isEmpty(); + } + + private Set getUniqueSources() { + Set sources = new HashSet<>(); + sources.addAll(mDataSources.requireVideo()); + sources.addAll(mDataSources.requireAudio()); + return sources; + } + + private void computeTrackStatus(@NonNull TrackType type, + @NonNull TrackStrategy strategy, + @NonNull List sources) { + TrackStatus status = TrackStatus.ABSENT; + MediaFormat outputFormat = new MediaFormat(); + if (!sources.isEmpty()) { + List inputFormats = new ArrayList<>(); + for (DataSource source : sources) { + MediaFormat inputFormat = source.getTrackFormat(type); + if (inputFormat != null) { + inputFormats.add(inputFormat); + } else if (sources.size() > 1) { + throw new IllegalArgumentException("More than one source selected for type " + type + + ", but getTrackFormat returned null."); } } + status = strategy.createOutputFormat(inputFormats, outputFormat); } - mDataSource.setTrackStatus(type, status); + mOutputFormats.set(type, outputFormat); mDataSink.setTrackStatus(type, status); mStatuses.set(type, status); - // Just to respect nullability in setUp(). - if (outputFormat == null) outputFormat = new MediaFormat(); - transcoder.setUp(outputFormat); - mTranscoders.set(type, transcoder); } - private void setUpTrackTranscoders(@NonNull TranscoderOptions options) { - setUpTrackTranscoder(options, TrackType.VIDEO); - setUpTrackTranscoder(options, TrackType.AUDIO); + private boolean isCompleted(@NonNull TrackType type) { + int current = mCurrentStep.require(type); + return !mDataSources.require(type).isEmpty() + && current == mDataSources.require(type).size() - 1 + && current == mTranscoders.require(type).size() - 1 + && mTranscoders.require(type).get(current).isFinished(); + } - TrackStatus videoStatus = mStatuses.require(TrackType.VIDEO); - TrackStatus audioStatus = mStatuses.require(TrackType.AUDIO); - //noinspection UnusedAssignment - boolean ignoreValidatorResult = false; + private void openCurrentStep(@NonNull TrackType type, @NonNull TranscoderOptions options) { + int current = mCurrentStep.require(type); + TrackStatus status = mStatuses.require(type); - // If we have to apply some rotation, and the video should be transcoded, - // ignore any Validator trying to abort the operation. The operation must happen - // because we must apply the rotation. - ignoreValidatorResult = videoStatus.isTranscoding() && options.getRotation() != 0; + // Notify the data source that we'll be transcoding this track. + DataSource dataSource = mDataSources.require(type).get(current); + if (status.isTranscoding()) { + dataSource.selectTrack(type); + } - // Validate and go on. - if (!options.getValidator().validate(videoStatus, audioStatus) - && !ignoreValidatorResult) { - throw new ValidatorException("Validator returned false."); + // Create a TimeInterpolator, wrapping the external one. + TimeInterpolator interpolator = createStepTimeInterpolator(type, current, + options.getTimeInterpolator()); + mInterpolators.require(type).add(interpolator); + + // Create a Transcoder for this track. + TrackTranscoder transcoder; + switch (status) { + case PASS_THROUGH: { + transcoder = new PassThroughTrackTranscoder(dataSource, + mDataSink, type, interpolator); + break; + } + case COMPRESSING: { + switch (type) { + case VIDEO: + transcoder = new VideoTrackTranscoder(dataSource, mDataSink, + interpolator, options.getVideoRotation()); + break; + case AUDIO: + transcoder = new AudioTrackTranscoder(dataSource, mDataSink, + interpolator, options.getAudioStretcher()); + break; + default: + throw new RuntimeException("Unknown type: " + type); + } + break; + } + case ABSENT: + case REMOVING: + default: { + transcoder = new NoOpTrackTranscoder(); + break; + } } + transcoder.setUp(mOutputFormats.require(type)); + mTranscoders.require(type).add(transcoder); } - private void runPipelines() throws InterruptedException { - long loopCount = 0; - if (mDurationUs <= 0) { - double progress = PROGRESS_UNKNOWN; - mProgress = progress; - if (mProgressCallback != null) mProgressCallback.onProgress(progress); // unknown - } - TrackTranscoder videoTranscoder = mTranscoders.require(TrackType.VIDEO); - TrackTranscoder audioTranscoder = mTranscoders.require(TrackType.AUDIO); - while (!(videoTranscoder.isFinished() && audioTranscoder.isFinished())) { - if (Thread.interrupted()) { - throw new InterruptedException(); + private void closeCurrentStep(@NonNull TrackType type) { + int current = mCurrentStep.require(type); + TrackTranscoder transcoder = mTranscoders.require(type).get(current); + DataSource dataSource = mDataSources.require(type).get(current); + transcoder.release(); + dataSource.release(); + mCurrentStep.set(type, current + 1); + } + + @NonNull + private TrackTranscoder getCurrentTrackTranscoder(@NonNull TrackType type, @NonNull TranscoderOptions options) { + int current = mCurrentStep.require(type); + int last = mTranscoders.require(type).size() - 1; + int max = mDataSources.require(type).size(); + if (last == current) { + // We have already created a transcoder for this step. + // But this step might be completed and we might need to create a new one. + TrackTranscoder transcoder = mTranscoders.require(type).get(last); + if (transcoder.isFinished()) { + closeCurrentStep(type); + return getCurrentTrackTranscoder(type, options); + } else { + return mTranscoders.require(type).get(current); } - boolean stepped = videoTranscoder.transcode() || audioTranscoder.transcode(); - loopCount++; - if (mDurationUs > 0 && loopCount % PROGRESS_INTERVAL_STEPS == 0) { - double videoProgress = getTranscoderProgress(videoTranscoder, mStatuses.require(TrackType.VIDEO)); - double audioProgress = getTranscoderProgress(audioTranscoder, mStatuses.require(TrackType.AUDIO)); - LOG.i("progress - video:" + videoProgress + " audio:" + audioProgress); - double progress = (videoProgress + audioProgress) / getTranscodersCount(); - mProgress = progress; - if (mProgressCallback != null) mProgressCallback.onProgress(progress); + } else if (last < current) { + // We need to create a new step. + openCurrentStep(type, options); + return mTranscoders.require(type).get(current); + } else { + throw new IllegalStateException("This should never happen. last:" + last + ", current:" + current); + } + } + + @NonNull + private TimeInterpolator createStepTimeInterpolator(@NonNull TrackType type, int step, + final @NonNull TimeInterpolator wrap) { + final long timebase; + if (step > 0) { + TimeInterpolator previous = mInterpolators.require(type).get(step - 1); + timebase = previous.interpolate(type, Long.MAX_VALUE); + } else { + timebase = 0; + } + return new TimeInterpolator() { + + private long mLastInterpolatedTime; + private long mFirstInputTime = Long.MAX_VALUE; + private long mTimeBase = timebase + 10; + + @Override + public long interpolate(@NonNull TrackType type, long time) { + if (time == Long.MAX_VALUE) return mLastInterpolatedTime; + if (mFirstInputTime == Long.MAX_VALUE) mFirstInputTime = time; + mLastInterpolatedTime = mTimeBase + (time - mFirstInputTime); + return wrap.interpolate(type, mLastInterpolatedTime); } - if (!stepped) { - Thread.sleep(SLEEP_TO_WAIT_TRACK_TRANSCODERS); + }; + } + + private double getTrackProgress(@NonNull TrackType type) { + if (!mStatuses.require(type).isTranscoding()) return 0.0D; + int current = mCurrentStep.require(type); + long totalDurationUs = 0; + long completedDurationUs = 0; + for (int i = 0; i < mDataSources.require(type).size(); i++) { + DataSource source = mDataSources.require(type).get(i); + if (i < current) { + totalDurationUs += source.getReadUs(); + completedDurationUs += source.getReadUs(); + } else if (i == current) { + totalDurationUs += source.getDurationUs(); + completedDurationUs += source.getReadUs(); + } else { + totalDurationUs += source.getDurationUs(); + completedDurationUs += 0; } } + if (totalDurationUs == 0) totalDurationUs = 1; + return (double) completedDurationUs / (double) totalDurationUs; } - private double getTranscoderProgress(@NonNull TrackTranscoder transcoder, @NonNull TrackStatus status) { - if (!status.isTranscoding()) return 0.0; - if (transcoder.isFinished()) return 1.0; - return Math.min(1.0, (double) transcoder.getLastPresentationTime() / mDurationUs); - } + /** + * Performs transcoding. Blocks current thread. + * + * @param options Transcoding options. + * @throws InvalidOutputFormatException when output format is not supported. + * @throws InterruptedException when cancel to transcode + */ + public void transcode(@NonNull TranscoderOptions options) throws InterruptedException { + mDataSink = new MediaMuxerDataSink(options.getOutputPath()); + mDataSources.setVideo(options.getVideoDataSources()); + mDataSources.setAudio(options.getAudioDataSources()); + + // Pass metadata from DataSource to DataSink + mDataSink.setOrientation(0); // Explicitly set 0 to output - we rotate the textures. + for (DataSource locationSource : getUniqueSources()) { + double[] location = locationSource.getLocation(); + if (location != null) { + mDataSink.setLocation(location[0], location[1]); + break; + } + } + + // Compute total duration: it is the minimum between the two. + long audioDurationUs = hasAudioSources() ? 0 : Long.MAX_VALUE; + long videoDurationUs = hasVideoSources() ? 0 : Long.MAX_VALUE; + for (DataSource source : options.getVideoDataSources()) videoDurationUs += source.getDurationUs(); + for (DataSource source : options.getAudioDataSources()) audioDurationUs += source.getDurationUs(); + long totalDurationUs = Math.min(audioDurationUs, videoDurationUs); + LOG.v("Duration (us): " + totalDurationUs); - private int getTranscodersCount() { - int count = 0; - if (mStatuses.require(TrackType.AUDIO).isTranscoding()) count++; - if (mStatuses.require(TrackType.VIDEO).isTranscoding()) count++; - return (count > 0) ? count : 1; + // TODO if audio and video have different lengths, we should clip the longer one! + // TODO audio resampling + // TODO ClipDataSource or something like that, to choose + + // Compute the TrackStatus. + int activeTracks = 0; + computeTrackStatus(TrackType.AUDIO, options.getAudioTrackStrategy(), options.getAudioDataSources()); + computeTrackStatus(TrackType.VIDEO, options.getVideoTrackStrategy(), options.getVideoDataSources()); + TrackStatus videoStatus = mStatuses.requireVideo(); + TrackStatus audioStatus = mStatuses.requireAudio(); + if (videoStatus.isTranscoding()) activeTracks++; + if (audioStatus.isTranscoding()) activeTracks++; + + // Pass to Validator. + //noinspection UnusedAssignment + boolean ignoreValidatorResult = false; + // If we have to apply some rotation, and the video should be transcoded, + // ignore any Validator trying to abort the operation. The operation must happen + // because we must apply the rotation. + ignoreValidatorResult = videoStatus.isTranscoding() && options.getVideoRotation() != 0; + if (!options.getValidator().validate(videoStatus, audioStatus) && !ignoreValidatorResult) { + throw new ValidatorException("Validator returned false."); + } + + // Do the actual transcoding work. + try { + long loopCount = 0; + boolean stepped = false; + boolean audioCompleted = false, videoCompleted = false; + while (!(audioCompleted && videoCompleted)) { + if (Thread.interrupted()) { + throw new InterruptedException(); + } + stepped = false; + audioCompleted = isCompleted(TrackType.AUDIO); + videoCompleted = isCompleted(TrackType.VIDEO); + if (!audioCompleted) { + stepped |= getCurrentTrackTranscoder(TrackType.AUDIO, options).transcode(); + } + if (!videoCompleted) { + stepped |= getCurrentTrackTranscoder(TrackType.VIDEO, options).transcode(); + } + if (++loopCount % PROGRESS_INTERVAL_STEPS == 0) { + setProgress((getTrackProgress(TrackType.VIDEO) + + getTrackProgress(TrackType.AUDIO)) / activeTracks); + } + if (!stepped) { + Thread.sleep(SLEEP_TO_WAIT_TRACK_TRANSCODERS); + } + } + mDataSink.stop(); + } finally { + try { + closeCurrentStep(TrackType.VIDEO); + closeCurrentStep(TrackType.AUDIO); + } catch (Exception ignore) {} + mDataSink.release(); + } } } diff --git a/lib/src/main/java/com/otaliastudios/transcoder/internal/TrackTypeMap.java b/lib/src/main/java/com/otaliastudios/transcoder/internal/TrackTypeMap.java index 23296702..6a8c514e 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/internal/TrackTypeMap.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/internal/TrackTypeMap.java @@ -16,6 +16,14 @@ */ public class TrackTypeMap { + public TrackTypeMap() { + } + + public TrackTypeMap(@NonNull T videoValue, @NonNull T audioValue) { + set(TrackType.AUDIO, audioValue); + set(TrackType.VIDEO, videoValue); + } + private Map map = new HashMap<>(); public void set(@NonNull TrackType type, @Nullable T value) { @@ -23,23 +31,54 @@ public void set(@NonNull TrackType type, @Nullable T value) { map.put(type, value); } + public void setAudio(@Nullable T value) { + set(TrackType.AUDIO, value); + } + + public void setVideo(@Nullable T value) { + set(TrackType.VIDEO, value); + } + @Nullable public T get(@NonNull TrackType type) { return map.get(type); } + @Nullable + public T getAudio() { + return get(TrackType.AUDIO); + } + + @Nullable + public T getVideo() { + return get(TrackType.VIDEO); + } + @NonNull public T require(@NonNull TrackType type) { //noinspection ConstantConditions return map.get(type); } - @SuppressWarnings("WeakerAccess") - public void clear() { - map.clear(); + @NonNull + public T requireAudio() { + return require(TrackType.AUDIO); + } + + @NonNull + public T requireVideo() { + return require(TrackType.VIDEO); } public boolean has(@NonNull TrackType type) { return map.containsKey(type); } + + public boolean hasAudio() { + return has(TrackType.AUDIO); + } + + public boolean hasVideo() { + return has(TrackType.VIDEO); + } } diff --git a/lib/src/main/java/com/otaliastudios/transcoder/sink/DataSink.java b/lib/src/main/java/com/otaliastudios/transcoder/sink/DataSink.java index b92c69c4..e45ce4ce 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/sink/DataSink.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/sink/DataSink.java @@ -50,27 +50,21 @@ void setTrackStatus(@NonNull TrackType type, * This is not the output format chosen by the library user but rather the * output format determined by {@link MediaCodec}, which contains more information, * and should be inspected to know what kind of data we're collecting. - * - * @param transcoder the transcoder - * @param type the track type + * @param type the track type * @param format the track format */ - void setTrackOutputFormat(@NonNull TrackTranscoder transcoder, - @NonNull TrackType type, - @NonNull MediaFormat format); + void setTrackFormat(@NonNull TrackType type, + @NonNull MediaFormat format); /** * Called by {@link TrackTranscoder}s to write data into this sink. - * - * @param transcoder the transcoder - * @param type the track type + * @param type the track type * @param byteBuffer the data * @param bufferInfo the metadata */ - void write(@NonNull TrackTranscoder transcoder, - @NonNull TrackType type, - @NonNull ByteBuffer byteBuffer, - @NonNull MediaCodec.BufferInfo bufferInfo); + void writeTrack(@NonNull TrackType type, + @NonNull ByteBuffer byteBuffer, + @NonNull MediaCodec.BufferInfo bufferInfo); /** * Called when transcoders have stopped writing. diff --git a/lib/src/main/java/com/otaliastudios/transcoder/sink/MediaMuxerDataSink.java b/lib/src/main/java/com/otaliastudios/transcoder/sink/MediaMuxerDataSink.java index 22700576..7b2fa5d9 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/sink/MediaMuxerDataSink.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/sink/MediaMuxerDataSink.java @@ -11,8 +11,6 @@ import com.otaliastudios.transcoder.engine.TrackType; import com.otaliastudios.transcoder.internal.TrackTypeMap; import com.otaliastudios.transcoder.internal.Logger; -import com.otaliastudios.transcoder.transcode.PassThroughTrackTranscoder; -import com.otaliastudios.transcoder.transcode.TrackTranscoder; import java.io.IOException; import java.nio.ByteBuffer; @@ -34,16 +32,13 @@ public class MediaMuxerDataSink implements DataSink { * the muxer is still being started (waiting for output formats). */ private static class QueuedSample { - private final TrackTranscoder mTranscoder; private final TrackType mType; private final int mSize; private final long mTimeUs; private final int mFlags; - private QueuedSample(@NonNull TrackTranscoder transcoder, - @NonNull TrackType type, + private QueuedSample(@NonNull TrackType type, @NonNull MediaCodec.BufferInfo bufferInfo) { - mTranscoder = transcoder; mType = type; mSize = bufferInfo.size; mTimeUs = bufferInfo.presentationTimeUs; @@ -92,9 +87,8 @@ public void setTrackStatus(@NonNull TrackType type, @NonNull TrackStatus status) } @Override - public void setTrackOutputFormat(@NonNull TrackTranscoder transcoder, - @NonNull TrackType type, - @NonNull MediaFormat format) { + public void setTrackFormat(@NonNull TrackType type, + @NonNull MediaFormat format) { boolean shouldValidate = mStatus.require(type) == TrackStatus.COMPRESSING; if (shouldValidate) { mMuxerChecks.checkOutputFormat(type, format); @@ -132,11 +126,11 @@ private void startIfNeeded() { } @Override - public void write(@NonNull TrackTranscoder transcoder, @NonNull TrackType type, @NonNull ByteBuffer byteBuffer, @NonNull MediaCodec.BufferInfo bufferInfo) { + public void writeTrack(@NonNull TrackType type, @NonNull ByteBuffer byteBuffer, @NonNull MediaCodec.BufferInfo bufferInfo) { if (mMuxerStarted) { mMuxer.writeSampleData(mMuxerIndex.require(type), byteBuffer, bufferInfo); } else { - enqueue(transcoder, type, byteBuffer, bufferInfo); + enqueue(type, byteBuffer, bufferInfo); } } @@ -144,13 +138,11 @@ public void write(@NonNull TrackTranscoder transcoder, @NonNull TrackType type, * Enqueues the given byffer by writing it into our own buffer and * just storing its position and size. * - * @param transcoder transcoder * @param type track type * @param buffer input buffer * @param bufferInfo input buffer info */ - private void enqueue(@NonNull TrackTranscoder transcoder, - @NonNull TrackType type, + private void enqueue(@NonNull TrackType type, @NonNull ByteBuffer buffer, @NonNull MediaCodec.BufferInfo bufferInfo) { if (mQueueBuffer == null) { @@ -159,7 +151,7 @@ private void enqueue(@NonNull TrackTranscoder transcoder, buffer.limit(bufferInfo.offset + bufferInfo.size); buffer.position(bufferInfo.offset); mQueueBuffer.put(buffer); - mQueue.add(new QueuedSample(transcoder, type, bufferInfo)); + mQueue.add(new QueuedSample(type, bufferInfo)); } /** @@ -176,7 +168,7 @@ private void drainQueue() { int offset = 0; for (QueuedSample sample : mQueue) { bufferInfo.set(offset, sample.mSize, sample.mTimeUs, sample.mFlags); - write(sample.mTranscoder, sample.mType, mQueueBuffer, bufferInfo); + writeTrack(sample.mType, mQueueBuffer, bufferInfo); offset += sample.mSize; } mQueue.clear(); diff --git a/lib/src/main/java/com/otaliastudios/transcoder/source/AndroidDataSource.java b/lib/src/main/java/com/otaliastudios/transcoder/source/AndroidDataSource.java index 3828314e..a9c4c702 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/source/AndroidDataSource.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/source/AndroidDataSource.java @@ -7,7 +7,6 @@ import androidx.annotation.NonNull; import androidx.annotation.Nullable; -import com.otaliastudios.transcoder.engine.TrackStatus; import com.otaliastudios.transcoder.engine.TrackType; import com.otaliastudios.transcoder.internal.TrackTypeMap; import com.otaliastudios.transcoder.internal.ISO6709LocationParser; @@ -29,14 +28,13 @@ public abstract class AndroidDataSource implements DataSource { private boolean mExtractorApplied; private final TrackTypeMap mFormats = new TrackTypeMap<>(); private final TrackTypeMap mIndex = new TrackTypeMap<>(); - - @SuppressWarnings("WeakerAccess") - protected AndroidDataSource() { } + private long mLastTimestampUs; + private long mFirstTimestampUs = Long.MIN_VALUE; private void ensureMetadata() { if (!mMetadataApplied) { mMetadataApplied = true; - apply(mMetadata); + applyRetriever(mMetadata); } } @@ -44,7 +42,7 @@ private void ensureExtractor() { if (!mExtractorApplied) { mExtractorApplied = true; try { - apply(mExtractor); + applyExtractor(mExtractor); } catch (IOException e) { LOG.e("Got IOException while trying to open MediaExtractor.", e); throw new RuntimeException(e); @@ -52,15 +50,13 @@ private void ensureExtractor() { } } - protected abstract void apply(@NonNull MediaExtractor extractor) throws IOException; + protected abstract void applyExtractor(@NonNull MediaExtractor extractor) throws IOException; - protected abstract void apply(@NonNull MediaMetadataRetriever retriever); + protected abstract void applyRetriever(@NonNull MediaMetadataRetriever retriever); @Override - public void setTrackStatus(@NonNull TrackType type, @NonNull TrackStatus status) { - if (status.isTranscoding()) { - mExtractor.selectTrack(mIndex.require(type)); - } + public void selectTrack(@NonNull TrackType type) { + mExtractor.selectTrack(mIndex.require(type)); } @Override @@ -70,20 +66,32 @@ public boolean isDrained() { } @Override - public boolean canRead(@NonNull TrackType type) { + public boolean canReadTrack(@NonNull TrackType type) { ensureExtractor(); return mExtractor.getSampleTrackIndex() == mIndex.require(type); } @Override - public void read(@NonNull Chunk chunk) { + public void readTrack(@NonNull Chunk chunk) { ensureExtractor(); chunk.bytes = mExtractor.readSampleData(chunk.buffer, 0); chunk.isKeyFrame = (mExtractor.getSampleFlags() & MediaExtractor.SAMPLE_FLAG_SYNC) != 0; chunk.timestampUs = mExtractor.getSampleTime(); + mLastTimestampUs = chunk.timestampUs; + if (mFirstTimestampUs == Long.MIN_VALUE) { + mFirstTimestampUs = mLastTimestampUs; + } mExtractor.advance(); } + @Override + public long getReadUs() { + if (mFirstTimestampUs == Long.MIN_VALUE) { + return 0; + } + return mLastTimestampUs - mFirstTimestampUs; + } + @Nullable @Override public double[] getLocation() { @@ -125,7 +133,7 @@ public long getDurationUs() { @Nullable @Override - public MediaFormat getFormat(@NonNull TrackType type) { + public MediaFormat getTrackFormat(@NonNull TrackType type) { if (mFormats.has(type)) return mFormats.get(type); ensureExtractor(); int trackCount = mExtractor.getTrackCount(); diff --git a/lib/src/main/java/com/otaliastudios/transcoder/source/DataSource.java b/lib/src/main/java/com/otaliastudios/transcoder/source/DataSource.java index 158765ad..ff1e2869 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/source/DataSource.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/source/DataSource.java @@ -20,34 +20,88 @@ public interface DataSource { /** - * Called before starting to set the status for the given - * track. The source object can check if the track is transcoding - * using {@link TrackStatus#isTranscoding()}. + * Metadata information. Returns the video orientation, or 0. * - * @param type track type - * @param status status + * @return video metadata orientation */ - void setTrackStatus(@NonNull TrackType type, @NonNull TrackStatus status); - - @Nullable - MediaFormat getFormat(@NonNull TrackType type); - int getOrientation(); + /** + * Metadata information. Returns the video location, or null. + * + * @return video location or null + */ @Nullable double[] getLocation(); + /** + * Returns the video total duration in microseconds. + * + * @return duration in us + */ long getDurationUs(); - boolean isDrained(); + /** + * Called before starting to inspect the input format for this track. + * Can return null if this media does not include this track type. + * + * @param type track type + * @return format or null + */ + @Nullable + MediaFormat getTrackFormat(@NonNull TrackType type); + + /** + * Called before starting, but after {@link #getTrackFormat(TrackType)}, + * to select the given track. + * + * @param type track type + */ + void selectTrack(@NonNull TrackType type); + /** + * Returns true if we can read the given track at this point. + * If true if returned, source should expect a {@link #readTrack(Chunk)} call. + * + * @param type track type + * @return true if we can read this track now + */ @SuppressWarnings("BooleanMethodIsAlwaysInverted") - boolean canRead(@NonNull TrackType type); + boolean canReadTrack(@NonNull TrackType type); + + /** + * Called to read contents for the current track type. + * Contents should be put inside {@link DataSource.Chunk#buffer}, and the + * other chunk flags should be filled. + * + * @param chunk output chunk + */ + void readTrack(@NonNull DataSource.Chunk chunk); + + /** + * Returns the total number of microseconds that have been read until now. + * + * @return total read us + */ + long getReadUs(); - void read(@NonNull DataSource.Chunk chunk); + /** + * When this source has been totally read, it can return true here to + * notify an end of input stream. + * + * @return true if drained + */ + boolean isDrained(); + /** + * Called to release resources. + */ void release(); + /** + * Represents a chunk of data. + * Can be used to read input from {@link #readTrack(Chunk)}. + */ class Chunk { public ByteBuffer buffer; public boolean isKeyFrame; diff --git a/lib/src/main/java/com/otaliastudios/transcoder/source/FileDescriptorDataSource.java b/lib/src/main/java/com/otaliastudios/transcoder/source/FileDescriptorDataSource.java index 565c196d..df5bf6a7 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/source/FileDescriptorDataSource.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/source/FileDescriptorDataSource.java @@ -17,17 +17,16 @@ public class FileDescriptorDataSource extends AndroidDataSource { private FileDescriptor descriptor; public FileDescriptorDataSource(@NonNull FileDescriptor descriptor) { - super(); this.descriptor = descriptor; } @Override - public void apply(@NonNull MediaExtractor extractor) throws IOException { + public void applyExtractor(@NonNull MediaExtractor extractor) throws IOException { extractor.setDataSource(descriptor); } @Override - public void apply(@NonNull MediaMetadataRetriever retriever) { + public void applyRetriever(@NonNull MediaMetadataRetriever retriever) { retriever.setDataSource(descriptor); } } diff --git a/lib/src/main/java/com/otaliastudios/transcoder/source/FilePathDataSource.java b/lib/src/main/java/com/otaliastudios/transcoder/source/FilePathDataSource.java index faefe3d0..2ae93e61 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/source/FilePathDataSource.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/source/FilePathDataSource.java @@ -10,21 +10,18 @@ import java.io.IOException; import androidx.annotation.NonNull; -import androidx.annotation.Nullable; /** * A {@link DataSource} backed by a file absolute path. */ public class FilePathDataSource extends AndroidDataSource { - private static final String TAG = "FilePathDataSource"; + private static final String TAG = FilePathDataSource.class.getSimpleName(); private static final Logger LOG = new Logger(TAG); - @NonNull - private FileDescriptorDataSource descriptor; - @Nullable private FileInputStream stream; + private final FileDescriptorDataSource descriptor; + private FileInputStream stream; public FilePathDataSource(@NonNull String path) { - super(); FileDescriptor fileDescriptor; try { stream = new FileInputStream(path); @@ -37,13 +34,13 @@ public FilePathDataSource(@NonNull String path) { } @Override - public void apply(@NonNull MediaExtractor extractor) throws IOException { - descriptor.apply(extractor); + public void applyExtractor(@NonNull MediaExtractor extractor) throws IOException { + descriptor.applyExtractor(extractor); } @Override - public void apply(@NonNull MediaMetadataRetriever retriever) { - descriptor.apply(retriever); + public void applyRetriever(@NonNull MediaMetadataRetriever retriever) { + descriptor.applyRetriever(retriever); } @Override diff --git a/lib/src/main/java/com/otaliastudios/transcoder/source/UriDataSource.java b/lib/src/main/java/com/otaliastudios/transcoder/source/UriDataSource.java index 7e5bded7..45f4937e 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/source/UriDataSource.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/source/UriDataSource.java @@ -19,18 +19,17 @@ public class UriDataSource extends AndroidDataSource { @NonNull private Uri uri; public UriDataSource(@NonNull Context context, @NonNull Uri uri) { - super(); this.context = context.getApplicationContext(); this.uri = uri; } @Override - public void apply(@NonNull MediaExtractor extractor) throws IOException { + public void applyExtractor(@NonNull MediaExtractor extractor) throws IOException { extractor.setDataSource(context, uri, null); } @Override - public void apply(@NonNull MediaMetadataRetriever retriever) { + public void applyRetriever(@NonNull MediaMetadataRetriever retriever) { retriever.setDataSource(context, uri); } } diff --git a/lib/src/main/java/com/otaliastudios/transcoder/strategy/DefaultAudioStrategy.java b/lib/src/main/java/com/otaliastudios/transcoder/strategy/DefaultAudioStrategy.java index 9dd6669f..f73a1dbc 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/strategy/DefaultAudioStrategy.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/strategy/DefaultAudioStrategy.java @@ -3,10 +3,13 @@ import android.media.MediaCodecInfo; import android.media.MediaFormat; +import com.otaliastudios.transcoder.engine.TrackStatus; +import com.otaliastudios.transcoder.internal.Logger; import com.otaliastudios.transcoder.internal.MediaFormatConstants; import androidx.annotation.NonNull; -import androidx.annotation.Nullable; + +import java.util.List; /** * An {@link TrackStrategy} for audio that converts it to AAC with the given number @@ -14,7 +17,10 @@ */ public class DefaultAudioStrategy implements TrackStrategy { - public static final int AUDIO_CHANNELS_AS_IS = -1; + private final static String TAG = DefaultAudioStrategy.class.getSimpleName(); + private final static Logger LOG = new Logger(TAG); + + public final static int AUDIO_CHANNELS_AS_IS = -1; private int channels; @@ -22,15 +28,43 @@ public DefaultAudioStrategy(int channels) { this.channels = channels; } - @Nullable + @NonNull @Override - public MediaFormat createOutputFormat(@NonNull MediaFormat inputFormat) throws TrackStrategyException { - int inputChannels = inputFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT); - int outputChannels = (channels == AUDIO_CHANNELS_AS_IS) ? inputChannels : channels; - final MediaFormat format = MediaFormat.createAudioFormat(MediaFormatConstants.MIMETYPE_AUDIO_AAC, - inputFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE), outputChannels); - format.setInteger(MediaFormat.KEY_AAC_PROFILE, MediaCodecInfo.CodecProfileLevel.AACObjectLC); - format.setInteger(MediaFormat.KEY_BIT_RATE, inputFormat.getInteger(MediaFormat.KEY_BIT_RATE)); - return format; + public TrackStatus createOutputFormat(@NonNull List inputFormats, @NonNull MediaFormat outputFormat) { + int outputChannels = (channels == AUDIO_CHANNELS_AS_IS) ? getInputChannelCount(inputFormats) : channels; + outputFormat.setString(MediaFormat.KEY_MIME, MediaFormatConstants.MIMETYPE_AUDIO_AAC); + outputFormat.setInteger(MediaFormat.KEY_SAMPLE_RATE, getInputSampleRate(inputFormats)); + outputFormat.setInteger(MediaFormat.KEY_CHANNEL_COUNT, outputChannels); + outputFormat.setInteger(MediaFormat.KEY_AAC_PROFILE, MediaCodecInfo.CodecProfileLevel.AACObjectLC); + outputFormat.setInteger(MediaFormat.KEY_BIT_RATE, getAverageInputBitRate(inputFormats)); + return TrackStatus.COMPRESSING; + } + + private int getInputChannelCount(@NonNull List formats) { + int count = 0; + for (MediaFormat format : formats) { + count = Math.max(count, format.getInteger(MediaFormat.KEY_CHANNEL_COUNT)); + } + return count; + } + + private int getInputSampleRate(@NonNull List formats) { + int rate = formats.get(0).getInteger(MediaFormat.KEY_SAMPLE_RATE); + for (MediaFormat format : formats) { + if (rate != format.getInteger(MediaFormat.KEY_SAMPLE_RATE)) { + LOG.e("Audio sampleRate should be equal for all DataSources audio tracks"); + // throw new IllegalArgumentException("All input formats should have the same sample rate."); + } + } + return rate; + } + + private int getAverageInputBitRate(@NonNull List formats) { + int count = formats.size(); + double bitRate = 0; + for (MediaFormat format : formats) { + bitRate += format.getInteger(MediaFormat.KEY_BIT_RATE); + } + return (int) (bitRate / count); } } diff --git a/lib/src/main/java/com/otaliastudios/transcoder/strategy/DefaultVideoStrategies.java b/lib/src/main/java/com/otaliastudios/transcoder/strategy/DefaultVideoStrategies.java index 159a7ab9..1d1ae3dc 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/strategy/DefaultVideoStrategies.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/strategy/DefaultVideoStrategies.java @@ -21,7 +21,7 @@ public static DefaultVideoStrategy for720x1280() { return DefaultVideoStrategy.exact(720, 1280) .bitRate(2L * 1000 * 1000) .frameRate(30) - .iFrameInterval(3F) + .keyFrameInterval(3F) .build(); } @@ -38,7 +38,7 @@ public static DefaultVideoStrategy for360x480() { return DefaultVideoStrategy.exact(360, 480) .bitRate(500L * 1000) .frameRate(30) - .iFrameInterval(3F) + .keyFrameInterval(3F) .build(); } } diff --git a/lib/src/main/java/com/otaliastudios/transcoder/strategy/DefaultVideoStrategy.java b/lib/src/main/java/com/otaliastudios/transcoder/strategy/DefaultVideoStrategy.java index 51da74f1..11020696 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/strategy/DefaultVideoStrategy.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/strategy/DefaultVideoStrategy.java @@ -4,6 +4,7 @@ import android.media.MediaFormat; import android.os.Build; +import com.otaliastudios.transcoder.engine.TrackStatus; import com.otaliastudios.transcoder.strategy.size.AspectRatioResizer; import com.otaliastudios.transcoder.strategy.size.AtMostResizer; import com.otaliastudios.transcoder.strategy.size.ExactResizer; @@ -16,7 +17,8 @@ import com.otaliastudios.transcoder.internal.MediaFormatConstants; import androidx.annotation.NonNull; -import androidx.annotation.Nullable; + +import java.util.List; /** * An {@link TrackStrategy} for video that converts it AVC with the given size. @@ -32,7 +34,7 @@ public class DefaultVideoStrategy implements TrackStrategy { public final static long BITRATE_UNKNOWN = Long.MIN_VALUE; @SuppressWarnings("WeakerAccess") - public final static float DEFAULT_I_FRAME_INTERVAL = 3; + public final static float DEFAULT_KEY_FRAME_INTERVAL = 3; public final static int DEFAULT_FRAME_RATE = 30; @@ -45,7 +47,7 @@ private Options() {} private Resizer resizer; private long targetBitRate; private int targetFrameRate; - private float targetIFrameInterval; + private float targetKeyFrameInterval; } /** @@ -119,7 +121,7 @@ public static class Builder { private MultiResizer resizer = new MultiResizer(); private int targetFrameRate = DEFAULT_FRAME_RATE; private long targetBitRate = BITRATE_UNKNOWN; - private float targetIFrameInterval = DEFAULT_I_FRAME_INTERVAL; + private float targetKeyFrameInterval = DEFAULT_KEY_FRAME_INTERVAL; @SuppressWarnings("unused") public Builder() { } @@ -168,14 +170,14 @@ public Builder frameRate(int frameRate) { } /** - * The interval between I-frames in seconds. - * @param iFrameInterval desired i-frame interval + * The interval between key-frames in seconds. + * @param keyFrameInterval desired key-frame interval * @return this for chaining */ @NonNull @SuppressWarnings("WeakerAccess") - public Builder iFrameInterval(float iFrameInterval) { - targetIFrameInterval = iFrameInterval; + public Builder keyFrameInterval(float keyFrameInterval) { + targetKeyFrameInterval = keyFrameInterval; return this; } @@ -186,7 +188,7 @@ public Options options() { options.resizer = resizer; options.targetFrameRate = targetFrameRate; options.targetBitRate = targetBitRate; - options.targetIFrameInterval = targetIFrameInterval; + options.targetKeyFrameInterval = targetKeyFrameInterval; return options; } @@ -203,21 +205,22 @@ public DefaultVideoStrategy(@NonNull Options options) { this.options = options; } - @Nullable + @NonNull @Override - public MediaFormat createOutputFormat(@NonNull MediaFormat inputFormat) throws TrackStrategyException { - boolean typeDone = inputFormat.getString(MediaFormat.KEY_MIME).equals(MIME_TYPE); + public TrackStatus createOutputFormat(@NonNull List inputFormats, + @NonNull MediaFormat outputFormat) { + boolean typeDone = checkMimeType(inputFormats); // Compute output size. - int inWidth = inputFormat.getInteger(MediaFormat.KEY_WIDTH); - int inHeight = inputFormat.getInteger(MediaFormat.KEY_HEIGHT); + ExactSize inSize = getBestInputSize(inputFormats); + int inWidth = inSize.getWidth(); + int inHeight = inSize.getHeight(); LOG.i("Input width&height: " + inWidth + "x" + inHeight); - Size inSize = new ExactSize(inWidth, inHeight); Size outSize; try { outSize = options.resizer.getOutputSize(inSize); } catch (Exception e) { - throw TrackStrategyException.unavailable(e); + throw new RuntimeException("Resizer error:", e); } int outWidth, outHeight; if (outSize instanceof ExactSize) { @@ -234,44 +237,112 @@ public MediaFormat createOutputFormat(@NonNull MediaFormat inputFormat) throws T boolean sizeDone = inSize.getMinor() <= outSize.getMinor(); // Compute output frame rate. It can't be bigger than input frame rate. - int inputFrameRate, outFrameRate; - if (inputFormat.containsKey(MediaFormat.KEY_FRAME_RATE)) { - inputFrameRate = inputFormat.getInteger(MediaFormat.KEY_FRAME_RATE); + int outFrameRate; + int inputFrameRate = getMinFrameRate(inputFormats); + if (inputFrameRate > 0) { outFrameRate = Math.min(inputFrameRate, options.targetFrameRate); } else { - inputFrameRate = -1; outFrameRate = options.targetFrameRate; } boolean frameRateDone = inputFrameRate <= outFrameRate; // Compute i frame. - int inputIFrameInterval = -1; - if (inputFormat.containsKey(MediaFormat.KEY_I_FRAME_INTERVAL)) { - inputIFrameInterval = inputFormat.getInteger(MediaFormat.KEY_I_FRAME_INTERVAL); - } - boolean frameIntervalDone = inputIFrameInterval >= options.targetIFrameInterval; - - // See if we should go on. - if (typeDone && sizeDone && frameRateDone && frameIntervalDone) { - throw TrackStrategyException.alreadyCompressed( - "Input minSize: " + inSize.getMinor() + ", desired minSize: " + outSize.getMinor() + + int inputIFrameInterval = getAverageIFrameInterval(inputFormats); + boolean frameIntervalDone = inputIFrameInterval >= options.targetKeyFrameInterval; + + // See if we should go on or if we're already compressed. + // If we have more than 1 input format, we can't go through this branch, + // or, for example, each part would be copied into output with its own size, + // breaking the muxer. + boolean canPassThrough = inputFormats.size() == 1; + if (canPassThrough && typeDone && sizeDone && frameRateDone && frameIntervalDone) { + LOG.i("Input minSize: " + inSize.getMinor() + ", desired minSize: " + outSize.getMinor() + "\nInput frameRate: " + inputFrameRate + ", desired frameRate: " + outFrameRate + - "\nInput iFrameInterval: " + inputIFrameInterval + ", desired iFrameInterval: " + options.targetIFrameInterval); + "\nInput iFrameInterval: " + inputIFrameInterval + ", desired iFrameInterval: " + options.targetKeyFrameInterval); + return TrackStatus.PASS_THROUGH; } // Create the actual format. - MediaFormat format = MediaFormat.createVideoFormat(MIME_TYPE, outWidth, outHeight); - format.setInteger(MediaFormat.KEY_FRAME_RATE, outFrameRate); + outputFormat.setString(MediaFormat.KEY_MIME, MIME_TYPE); + outputFormat.setInteger(MediaFormat.KEY_WIDTH, outWidth); + outputFormat.setInteger(MediaFormat.KEY_HEIGHT, outHeight); + outputFormat.setInteger(MediaFormat.KEY_FRAME_RATE, outFrameRate); if (Build.VERSION.SDK_INT >= 25) { - format.setFloat(MediaFormat.KEY_I_FRAME_INTERVAL, options.targetIFrameInterval); + outputFormat.setFloat(MediaFormat.KEY_I_FRAME_INTERVAL, options.targetKeyFrameInterval); } else { - format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, (int) Math.ceil(options.targetIFrameInterval)); + outputFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, (int) Math.ceil(options.targetKeyFrameInterval)); } - format.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface); + outputFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface); int outBitRate = (int) (options.targetBitRate == BITRATE_UNKNOWN ? estimateBitRate(outWidth, outHeight, outFrameRate) : options.targetBitRate); - format.setInteger(MediaFormat.KEY_BIT_RATE, outBitRate); - return format; + outputFormat.setInteger(MediaFormat.KEY_BIT_RATE, outBitRate); + return TrackStatus.COMPRESSING; + } + + private boolean checkMimeType(@NonNull List formats) { + for (MediaFormat format : formats) { + if (!format.getString(MediaFormat.KEY_MIME).equals(MIME_TYPE)) { + return false; + } + } + return true; + } + + private ExactSize getBestInputSize(@NonNull List formats) { + int count = formats.size(); + // After thinking about it, I think the best size is the one that is closer to the + // average aspect ratio. Respect the rotation of the first video for now + float averageAspectRatio = 0; + float[] aspectRatio = new float[count]; + int firstRotation = -1; + for (int i = 0; i < count; i++) { + MediaFormat format = formats.get(i); + float width = format.getInteger(MediaFormat.KEY_WIDTH); + float height = format.getInteger(MediaFormat.KEY_HEIGHT); + int rotation = 0; + if (format.containsKey(MediaFormatConstants.KEY_ROTATION_DEGREES)) { + rotation = format.getInteger(MediaFormatConstants.KEY_ROTATION_DEGREES); + } + if (firstRotation == -1) firstRotation = 0; + boolean flip = (rotation - firstRotation + 360) % 180 != 0; + aspectRatio[i] = flip ? height / width : width / height; + averageAspectRatio += aspectRatio[i]; + } + averageAspectRatio = averageAspectRatio / count; + float bestDelta = Float.MAX_VALUE; + int bestMatch = 0; + for (int i = 0; i < count; i++) { + float delta = Math.abs(aspectRatio[i] - averageAspectRatio); + if (delta < bestDelta) { + bestMatch = i; + bestDelta = delta; + } + } + MediaFormat bestFormat = formats.get(bestMatch); + return new ExactSize(bestFormat.getInteger(MediaFormat.KEY_WIDTH), + bestFormat.getInteger(MediaFormat.KEY_HEIGHT)); + } + + private int getMinFrameRate(@NonNull List formats) { + int frameRate = Integer.MAX_VALUE; + for (MediaFormat format : formats) { + if (format.containsKey(MediaFormat.KEY_FRAME_RATE)) { + frameRate = Math.min(frameRate, format.getInteger(MediaFormat.KEY_FRAME_RATE)); + } + } + return (frameRate == Integer.MAX_VALUE) ? -1 : frameRate; + } + + private int getAverageIFrameInterval(@NonNull List formats) { + int count = 0; + int sum = 0; + for (MediaFormat format : formats) { + if (format.containsKey(MediaFormat.KEY_I_FRAME_INTERVAL)) { + count++; + sum += format.getInteger(MediaFormat.KEY_I_FRAME_INTERVAL); + } + } + return (count > 0) ? Math.round((float) sum / count) : -1; } // Depends on the codec, but for AVC this is a reasonable default ? diff --git a/lib/src/main/java/com/otaliastudios/transcoder/strategy/PassThroughTrackStrategy.java b/lib/src/main/java/com/otaliastudios/transcoder/strategy/PassThroughTrackStrategy.java index bcdd4269..afab5615 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/strategy/PassThroughTrackStrategy.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/strategy/PassThroughTrackStrategy.java @@ -5,6 +5,10 @@ import androidx.annotation.NonNull; import androidx.annotation.Nullable; +import com.otaliastudios.transcoder.engine.TrackStatus; + +import java.util.List; + /** * An {@link TrackStrategy} that asks the encoder to keep this track as is. * Note that this is risky, as the track type might not be supported by @@ -13,9 +17,9 @@ @SuppressWarnings("unused") public class PassThroughTrackStrategy implements TrackStrategy { - @Nullable + @NonNull @Override - public MediaFormat createOutputFormat(@NonNull MediaFormat inputFormat) throws TrackStrategyException { - return inputFormat; + public TrackStatus createOutputFormat(@NonNull List inputFormats, @NonNull MediaFormat outputFormat) { + return TrackStatus.PASS_THROUGH; } } diff --git a/lib/src/main/java/com/otaliastudios/transcoder/strategy/RemoveTrackStrategy.java b/lib/src/main/java/com/otaliastudios/transcoder/strategy/RemoveTrackStrategy.java index d3515197..38b5114f 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/strategy/RemoveTrackStrategy.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/strategy/RemoveTrackStrategy.java @@ -5,15 +5,19 @@ import androidx.annotation.NonNull; import androidx.annotation.Nullable; +import com.otaliastudios.transcoder.engine.TrackStatus; + +import java.util.List; + /** * An {@link TrackStrategy} that removes this track from output. */ @SuppressWarnings("unused") public class RemoveTrackStrategy implements TrackStrategy { - @Nullable + @NonNull @Override - public MediaFormat createOutputFormat(@NonNull MediaFormat inputFormat) throws TrackStrategyException { - return null; + public TrackStatus createOutputFormat(@NonNull List inputFormats, @NonNull MediaFormat outputFormat) { + return TrackStatus.REMOVING; } } diff --git a/lib/src/main/java/com/otaliastudios/transcoder/strategy/TrackStrategy.java b/lib/src/main/java/com/otaliastudios/transcoder/strategy/TrackStrategy.java index a26293fc..fa2df2a4 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/strategy/TrackStrategy.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/strategy/TrackStrategy.java @@ -2,11 +2,14 @@ import android.media.MediaFormat; +import com.otaliastudios.transcoder.engine.TrackStatus; import com.otaliastudios.transcoder.strategy.size.Resizer; import androidx.annotation.NonNull; import androidx.annotation.Nullable; +import java.util.List; + /** * Base class for video/audio format strategy. * Video strategies should use a {@link Resizer} instance to compute the output @@ -16,14 +19,17 @@ public interface TrackStrategy { /** * Create the output format for this track (either audio or video). - * Implementors can: - * - throw a {@link TrackStrategyException} if the whole transcoding should be aborted - * - return {@code inputFormat} for remuxing this track as-is - * - returning {@code null} for removing this track from output + * Implementors should fill the outputFormat object and return a non-null {@link TrackStatus}: + * - {@link TrackStatus#COMPRESSING}: we want to compress this track. Output format will be used + * - {@link TrackStatus#PASS_THROUGH}: we want to use the input format. Output format will be ignored + * - {@link TrackStatus#REMOVING}: we want to remove this track. Output format will be ignored + * + * Subclasses can also throw to abort the whole transcoding operation. * - * @param inputFormat the input format - * @return the output format + * @param inputFormats the input formats + * @param outputFormat the output format to be filled + * @return the track status */ - @Nullable - MediaFormat createOutputFormat(@NonNull MediaFormat inputFormat) throws TrackStrategyException; + @NonNull + TrackStatus createOutputFormat(@NonNull List inputFormats, @NonNull MediaFormat outputFormat); } diff --git a/lib/src/main/java/com/otaliastudios/transcoder/strategy/TrackStrategyException.java b/lib/src/main/java/com/otaliastudios/transcoder/strategy/TrackStrategyException.java deleted file mode 100644 index 5a009724..00000000 --- a/lib/src/main/java/com/otaliastudios/transcoder/strategy/TrackStrategyException.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright (C) 2014 Yuya Tanaka - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.otaliastudios.transcoder.strategy; - -import androidx.annotation.NonNull; -import androidx.annotation.Nullable; - -/** - * Base class for exceptions thrown by {@link TrackStrategy} by any - * strategy implementors. - * - * These are later caught internally. - */ -public class TrackStrategyException extends RuntimeException { - - @SuppressWarnings("WeakerAccess") - public final static int TYPE_UNAVAILABLE = 0; - - public final static int TYPE_ALREADY_COMPRESSED = 1; - - private int type; - - @SuppressWarnings("WeakerAccess") - public TrackStrategyException(int type, @Nullable String detailMessage) { - super(detailMessage); - this.type = type; - } - - @SuppressWarnings("WeakerAccess") - public TrackStrategyException(int type, @Nullable Exception cause) { - super(cause); - this.type = type; - } - - public int getType() { - return type; - } - - @NonNull - @SuppressWarnings("WeakerAccess") - public static TrackStrategyException unavailable(@Nullable Exception cause) { - return new TrackStrategyException(TYPE_UNAVAILABLE, cause); - } - - @NonNull - @SuppressWarnings("WeakerAccess") - public static TrackStrategyException alreadyCompressed(@Nullable String detailMessage) { - return new TrackStrategyException(TYPE_ALREADY_COMPRESSED, detailMessage); - } -} diff --git a/lib/src/main/java/com/otaliastudios/transcoder/stretch/AudioStretcher.java b/lib/src/main/java/com/otaliastudios/transcoder/stretch/AudioStretcher.java index e35c4f26..d2413cae 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/stretch/AudioStretcher.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/stretch/AudioStretcher.java @@ -33,18 +33,4 @@ public interface AudioStretcher { AudioStretcher CUT = new CutAudioStretcher(); AudioStretcher INSERT = new InsertAudioStretcher(); - - - AudioStretcher CUT_OR_INSERT = new AudioStretcher() { - @Override - public void stretch(@NonNull ShortBuffer input, @NonNull ShortBuffer output, int channels) { - if (input.remaining() < output.remaining()) { - INSERT.stretch(input, output, channels); - } else if (input.remaining() > output.remaining()) { - CUT.stretch(input, output, channels); - } else { - PASSTHROUGH.stretch(input, output, channels); - } - } - }; } diff --git a/lib/src/main/java/com/otaliastudios/transcoder/transcode/BaseTrackTranscoder.java b/lib/src/main/java/com/otaliastudios/transcoder/transcode/BaseTrackTranscoder.java index 1cf5688f..c0b1ea57 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/transcode/BaseTrackTranscoder.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/transcode/BaseTrackTranscoder.java @@ -30,8 +30,6 @@ public abstract class BaseTrackTranscoder implements TrackTranscoder { private final DataSink mDataSink; private final TrackType mTrackType; - private long mLastPresentationTimeUs; - private final MediaCodec.BufferInfo mBufferInfo = new MediaCodec.BufferInfo(); private MediaCodec mDecoder; private MediaCodec mEncoder; @@ -65,7 +63,7 @@ public final void setUp(@NonNull MediaFormat desiredOutputFormat) { onConfigureEncoder(desiredOutputFormat, mEncoder); onStartEncoder(desiredOutputFormat, mEncoder); - final MediaFormat inputFormat = mDataSource.getFormat(mTrackType); + final MediaFormat inputFormat = mDataSource.getTrackFormat(mTrackType); if (inputFormat == null) { throw new IllegalArgumentException("Input format is null!"); } @@ -134,11 +132,6 @@ protected void onCodecsStarted(@NonNull MediaFormat inputFormat, @NonNull MediaF @NonNull MediaCodec decoder, @NonNull MediaCodec encoder) { } - @Override - public final long getLastPresentationTime() { - return mLastPresentationTimeUs; - } - @Override public final boolean isFinished() { return mIsEncoderEOS; @@ -198,7 +191,7 @@ protected void onEncoderOutputFormatChanged(@NonNull MediaCodec encoder, @NonNul throw new RuntimeException("Audio output format changed twice."); } mActualOutputFormat = format; - mDataSink.setTrackOutputFormat(this, mTrackType, mActualOutputFormat); + mDataSink.setTrackFormat(mTrackType, mActualOutputFormat); } @SuppressWarnings("SameParameterValue") @@ -215,7 +208,7 @@ private int feedDecoder(long timeoutUs) { return DRAIN_STATE_NONE; } - if (!mDataSource.canRead(mTrackType)) { + if (!mDataSource.canReadTrack(mTrackType)) { return DRAIN_STATE_NONE; } @@ -223,7 +216,7 @@ private int feedDecoder(long timeoutUs) { if (result < 0) return DRAIN_STATE_NONE; mDataChunk.buffer = mDecoderBuffers.getInputBuffer(result); - mDataSource.read(mDataChunk); + mDataSource.readTrack(mDataChunk); mDecoder.queueInputBuffer(result, 0, mDataChunk.bytes, @@ -255,7 +248,6 @@ private int drainDecoder(long timeoutUs) { boolean hasSize = mBufferInfo.size > 0; if (isEos) mIsDecoderEOS = true; if (isEos || hasSize) { - mLastPresentationTimeUs = mBufferInfo.presentationTimeUs; onDrainDecoder(mDecoder, result, mDecoderBuffers.getOutputBuffer(result), @@ -294,7 +286,7 @@ private int drainEncoder(long timeoutUs) { mEncoder.releaseOutputBuffer(result, false); return DRAIN_STATE_SHOULD_RETRY_IMMEDIATELY; } - mDataSink.write(this, mTrackType, mEncoderBuffers.getOutputBuffer(result), mBufferInfo); + mDataSink.writeTrack(mTrackType, mEncoderBuffers.getOutputBuffer(result), mBufferInfo); mEncoder.releaseOutputBuffer(result, false); return DRAIN_STATE_CONSUMED; } diff --git a/lib/src/main/java/com/otaliastudios/transcoder/transcode/NoOpTrackTranscoder.java b/lib/src/main/java/com/otaliastudios/transcoder/transcode/NoOpTrackTranscoder.java index 60eecd3b..32307d90 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/transcode/NoOpTrackTranscoder.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/transcode/NoOpTrackTranscoder.java @@ -33,11 +33,6 @@ public boolean transcode() { return false; } - @Override - public long getLastPresentationTime() { - return 0; - } - @Override public boolean isFinished() { return true; diff --git a/lib/src/main/java/com/otaliastudios/transcoder/transcode/PassThroughTrackTranscoder.java b/lib/src/main/java/com/otaliastudios/transcoder/transcode/PassThroughTrackTranscoder.java index 1320393f..1b2f077d 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/transcode/PassThroughTrackTranscoder.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/transcode/PassThroughTrackTranscoder.java @@ -16,7 +16,6 @@ package com.otaliastudios.transcoder.transcode; import android.media.MediaCodec; -import android.media.MediaExtractor; import android.media.MediaFormat; import androidx.annotation.NonNull; @@ -37,7 +36,6 @@ public class PassThroughTrackTranscoder implements TrackTranscoder { private final TrackType mTrackType; private final MediaCodec.BufferInfo mBufferInfo = new MediaCodec.BufferInfo(); private boolean mIsEOS; - private long mLastPresentationTime; private final MediaFormat mOutputFormat; private boolean mOutputFormatSet = false; private TimeInterpolator mTimeInterpolator; @@ -49,7 +47,7 @@ public PassThroughTrackTranscoder(@NonNull DataSource dataSource, mDataSource = dataSource; mDataSink = dataSink; mTrackType = trackType; - mOutputFormat = dataSource.getFormat(trackType); + mOutputFormat = dataSource.getTrackFormat(trackType); if (mOutputFormat == null) { throw new IllegalArgumentException("Output format is null!"); } @@ -66,35 +64,29 @@ public void setUp(@NonNull MediaFormat desiredOutputFormat) { } public boolean transcode() { if (mIsEOS) return false; if (!mOutputFormatSet) { - mDataSink.setTrackOutputFormat(this, mTrackType, mOutputFormat); + mDataSink.setTrackFormat(mTrackType, mOutputFormat); mOutputFormatSet = true; } if (mDataSource.isDrained()) { mDataChunk.buffer.clear(); mBufferInfo.set(0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM); - mDataSink.write(this, mTrackType, mDataChunk.buffer, mBufferInfo); + mDataSink.writeTrack(mTrackType, mDataChunk.buffer, mBufferInfo); mIsEOS = true; return true; } - if (!mDataSource.canRead(mTrackType)) { + if (!mDataSource.canReadTrack(mTrackType)) { return false; } mDataChunk.buffer.clear(); - mDataSource.read(mDataChunk); + mDataSource.readTrack(mDataChunk); long timestampUs = mTimeInterpolator.interpolate(mTrackType, mDataChunk.timestampUs); int flags = mDataChunk.isKeyFrame ? MediaCodec.BUFFER_FLAG_SYNC_FRAME : 0; mBufferInfo.set(0, mDataChunk.bytes, timestampUs, flags); - mDataSink.write(this, mTrackType, mDataChunk.buffer, mBufferInfo); - mLastPresentationTime = mDataChunk.timestampUs; + mDataSink.writeTrack(mTrackType, mDataChunk.buffer, mBufferInfo); return true; } - @Override - public long getLastPresentationTime() { - return mLastPresentationTime; - } - @Override public boolean isFinished() { return mIsEOS; diff --git a/lib/src/main/java/com/otaliastudios/transcoder/transcode/TrackTranscoder.java b/lib/src/main/java/com/otaliastudios/transcoder/transcode/TrackTranscoder.java index 9d42b20f..c1e36ba7 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/transcode/TrackTranscoder.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/transcode/TrackTranscoder.java @@ -34,14 +34,6 @@ public interface TrackTranscoder { */ boolean transcode(); - /** - * Get presentation time of last sample taken from encoder. - * This presentation time should not be affected by {@link com.otaliastudios.transcoder.time.TimeInterpolator}s. - * - * @return Presentation time in microseconds. Return value is undefined if finished writing. - */ - long getLastPresentationTime(); - boolean isFinished(); void release(); diff --git a/lib/src/main/java/com/otaliastudios/transcoder/transcode/VideoTrackTranscoder.java b/lib/src/main/java/com/otaliastudios/transcoder/transcode/VideoTrackTranscoder.java index 8192abff..950088ff 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/transcode/VideoTrackTranscoder.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/transcode/VideoTrackTranscoder.java @@ -45,14 +45,29 @@ public class VideoTrackTranscoder extends BaseTrackTranscoder { private VideoEncoderInput mEncoderInputSurface; private MediaCodec mEncoder; // Keep this since we want to signal EOS on it. private VideoFrameDropper mFrameDropper; - private TimeInterpolator mTimeInterpolator; + private final TimeInterpolator mTimeInterpolator; + private final int mRotation; + private final boolean mFlip; public VideoTrackTranscoder( @NonNull DataSource dataSource, @NonNull DataSink dataSink, - @NonNull TimeInterpolator timeInterpolator) { + @NonNull TimeInterpolator timeInterpolator, + int rotation) { super(dataSource, dataSink, TrackType.VIDEO); mTimeInterpolator = timeInterpolator; + mRotation = (rotation + dataSource.getOrientation()) % 360; + mFlip = mRotation % 180 != 0; + } + + @Override + protected void onConfigureEncoder(@NonNull MediaFormat format, @NonNull MediaCodec encoder) { + // Flip the width and height as needed. + int width = format.getInteger(MediaFormat.KEY_WIDTH); + int height = format.getInteger(MediaFormat.KEY_HEIGHT); + format.setInteger(MediaFormat.KEY_WIDTH, mFlip ? height : width); + format.setInteger(MediaFormat.KEY_HEIGHT, mFlip ? width : height); + super.onConfigureEncoder(format, encoder); } @Override @@ -69,6 +84,7 @@ protected void onConfigureDecoder(@NonNull MediaFormat format, @NonNull MediaCod format.setInteger(MediaFormatConstants.KEY_ROTATION_DEGREES, 0); } mDecoderOutputSurface = new VideoDecoderOutput(); + mDecoderOutputSurface.setRotation(mRotation); decoder.configure(format, mDecoderOutputSurface.getSurface(), null, 0); } @@ -84,8 +100,8 @@ protected void onCodecsStarted(@NonNull MediaFormat inputFormat, @NonNull MediaF float inputWidth = inputFormat.getInteger(MediaFormat.KEY_WIDTH); float inputHeight = inputFormat.getInteger(MediaFormat.KEY_HEIGHT); float inputRatio = inputWidth / inputHeight; - float outputWidth = outputFormat.getInteger(MediaFormat.KEY_WIDTH); - float outputHeight = outputFormat.getInteger(MediaFormat.KEY_HEIGHT); + float outputWidth = mFlip ? outputFormat.getInteger(MediaFormat.KEY_HEIGHT) : outputFormat.getInteger(MediaFormat.KEY_WIDTH); + float outputHeight = mFlip ? outputFormat.getInteger(MediaFormat.KEY_WIDTH) : outputFormat.getInteger(MediaFormat.KEY_HEIGHT); float outputRatio = outputWidth / outputHeight; float scaleX = 1, scaleY = 1; if (inputRatio > outputRatio) { // Input wider. We have a scaleX. @@ -93,8 +109,6 @@ protected void onCodecsStarted(@NonNull MediaFormat inputFormat, @NonNull MediaF } else if (inputRatio < outputRatio) { // Input taller. We have a scaleY. scaleY = outputRatio / inputRatio; } - // I don't think we should consider rotation and flip these - we operate on non-rotated - // surfaces and pass the input rotation metadata to the output muxer, see Engine.setupMetadata. mDecoderOutputSurface.setScale(scaleX, scaleY); } diff --git a/lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/AudioEngine.java b/lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/AudioEngine.java index da57cf23..bae715e5 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/AudioEngine.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/AudioEngine.java @@ -70,7 +70,8 @@ public AudioEngine(@NonNull MediaCodec decoder, int outputSampleRate = encoderOutputFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE); int inputSampleRate = decoderOutputFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE); if (inputSampleRate != outputSampleRate) { - throw new UnsupportedOperationException("Audio sample rate conversion not supported yet."); + LOG.e("Audio sample rate conversion is not supported. The result might be corrupted."); + // throw new UnsupportedOperationException("Audio sample rate conversion not supported yet."); } mSampleRate = inputSampleRate; diff --git a/lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/VideoDecoderOutput.java b/lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/VideoDecoderOutput.java index da7952b1..e6a7070b 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/VideoDecoderOutput.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/VideoDecoderOutput.java @@ -42,6 +42,7 @@ public class VideoDecoderOutput { private float mScaleX = 1F; private float mScaleY = 1F; + private int mRotation = 0; private int mTextureId; private float[] mTextureTransform = new float[16]; @@ -91,6 +92,15 @@ public void setScale(float scaleX, float scaleY) { mScaleY = scaleY; } + /** + * Sets the desired frame rotation with respect + * to its natural orientation. + * @param rotation rotation + */ + public void setRotation(int rotation) { + mRotation = rotation; + } + /** * Returns a Surface to draw onto. * @return the output surface @@ -164,8 +174,13 @@ private void drawNewFrame() { float glTranslX = (1F - glScaleX) / 2F; float glTranslY = (1F - glScaleY) / 2F; Matrix.translateM(mTextureTransform, 0, glTranslX, glTranslY, 0); - // Scale and draw. + // Scale. Matrix.scaleM(mTextureTransform, 0, glScaleX, glScaleY, 1); + // Apply rotation. + Matrix.translateM(mTextureTransform, 0, 0.5F, 0.5F, 0); + Matrix.rotateM(mTextureTransform, 0, mRotation, 0, 0, 1); + Matrix.translateM(mTextureTransform, 0, -0.5F, -0.5F, 0); + // Draw. mScene.drawTexture(mDrawable, mProgram, mTextureId, mTextureTransform); } }