diff --git a/mp4compose/src/main/java/com/daasuu/mp4compose/Rotation.kt b/mp4compose/src/main/java/com/daasuu/mp4compose/Rotation.kt index 62d63f3b..4071e8b2 100644 --- a/mp4compose/src/main/java/com/daasuu/mp4compose/Rotation.kt +++ b/mp4compose/src/main/java/com/daasuu/mp4compose/Rotation.kt @@ -11,6 +11,7 @@ enum class Rotation private constructor(val rotation: Int) { ROTATION_270(270); companion object { + @JvmStatic fun fromInt(rotate: Int): Rotation { for (rotation in Rotation.values()) { if (rotate == rotation.rotation) return rotation diff --git a/mp4compose/src/main/java/com/daasuu/mp4compose/VideoFormatMimeType.java b/mp4compose/src/main/java/com/daasuu/mp4compose/VideoFormatMimeType.java new file mode 100644 index 00000000..360f1de2 --- /dev/null +++ b/mp4compose/src/main/java/com/daasuu/mp4compose/VideoFormatMimeType.java @@ -0,0 +1,22 @@ +package com.daasuu.mp4compose; + +import android.media.MediaFormat; + +@SuppressWarnings("MemberName") +public enum VideoFormatMimeType { + HEVC(MediaFormat.MIMETYPE_VIDEO_HEVC), + AVC(MediaFormat.MIMETYPE_VIDEO_AVC), + MPEG4(MediaFormat.MIMETYPE_VIDEO_MPEG4), + H263(MediaFormat.MIMETYPE_VIDEO_H263), + AUTO(""); + + private final String videoFormatMimeType; + + VideoFormatMimeType(String videoFormatMimeType) { + this.videoFormatMimeType = videoFormatMimeType; + } + + public String getFormat() { + return videoFormatMimeType; + } +} diff --git a/mp4compose/src/main/java/com/daasuu/mp4compose/composer/AudioChannelWithSP.java b/mp4compose/src/main/java/com/daasuu/mp4compose/composer/AudioChannelWithSP.java new file mode 100644 index 00000000..78130ede --- /dev/null +++ b/mp4compose/src/main/java/com/daasuu/mp4compose/composer/AudioChannelWithSP.java @@ -0,0 +1,209 @@ +package com.daasuu.mp4compose.composer; + +import android.media.MediaCodec; +import android.media.MediaFormat; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.ShortBuffer; + +/** + * Created by TAPOS DATTA on 22,May,2020 + */ +@SuppressWarnings({"MemberName", "MethodName", "SimplifyBooleanReturn"}) +public class AudioChannelWithSP extends BaseAudioChannel { + private static final String TAG = "AUDIO_CHANNEL_WITH_SONIC"; + + private SonicAudioProcessor stream = null; // SonicAudioProcessor can deal with stereo Audio + private float timeScale = 1f; + boolean isEOF = false; + private int BUFFER_CAPACITY = 2048; // in ShortBuffer size + private long totalDataAdded = 0; + private int pendingDecoderOutputBuffIndx = -1; + private ByteBuffer tempInputBuffer = null; + private boolean isPendingFeeding = true; + private boolean isAffectInPitch; // if true the scale will impact in speed with pitch + + AudioChannelWithSP( + MediaCodec decoder, + MediaCodec encoder, + MediaFormat encodeFormat, + float timeScale, + boolean isPitchChanged + ) { + super(decoder, encoder, encodeFormat); + this.isAffectInPitch = isPitchChanged; + this.timeScale = timeScale; + } + + @Override + public void setActualDecodedFormat(MediaFormat decodedFormat) { + super.setActualDecodedFormat(decodedFormat); + + if (inputChannelCount > 2) { + throw new UnsupportedOperationException("Input channel count (" + inputChannelCount + ") not supported."); + } + stream = new SonicAudioProcessor(inputSampleRate, outputChannelCount); + isEOF = false; + totalDataAdded = 0; + isPendingFeeding = true; + tempInputBuffer = ByteBuffer.allocateDirect(BUFFER_CAPACITY * 16).order(ByteOrder.nativeOrder()); + + if (isAffectInPitch) { + stream.setRate(timeScale); + } else { + stream.setSpeed(timeScale); + } + } + + @Override + protected long sampleCountToDurationUs(long sampleCount, int sampleRate, int channelCount) { + // considered short buffer as data + return (long) ((MICROSECS_PER_SEC * (sampleCount * 1f) / (sampleRate * 1f * channelCount))); + } + + @Override + public void drainDecoderBufferAndQueue(int bufferIndex, long presentationTimeUs) { + if (actualDecodedFormat == null) { + throw new RuntimeException("Buffer received before format!"); + } + + final ByteBuffer data = + bufferIndex == BUFFER_INDEX_END_OF_STREAM + ? null : decoder.getOutputBuffer(bufferIndex); + + if (data != null) { + writeToSonicSteam(data.asShortBuffer()); + pendingDecoderOutputBuffIndx = bufferIndex; + isEOF = false; + decoder.releaseOutputBuffer(bufferIndex, false); + } else { + stream.flushStream(); + isEOF = true; + } + } + + @Override + public boolean feedEncoder(long timeoutUs) { + if (stream == null || !isPendingFeeding || (!isEOF && stream.samplesAvailable() == 0)) { + // no data available + + updatePendingDecoderStatus(); + + return false; + } else if (!isEOF + && timeScale < 1f + && stream.samplesAvailable() > 0 + && (stream.samplesAvailable() * outputChannelCount) < BUFFER_CAPACITY + ) { + // few data remaining in stream wait for next stream data + updatePendingDecoderStatus(); + + return false; + } + + final int encoderInBuffIndex = encoder.dequeueInputBuffer(timeoutUs); + + if (encoderInBuffIndex < 0) { + // Encoder is full - Bail out + return false; + } + + boolean status = false; + if (timeScale < 1f) { + status = slowTimeBufferProcess(encoderInBuffIndex); + } else { + status = FastOrNormalTimeBufferProcess(encoderInBuffIndex); + } + + return status; + } + + private void updatePendingDecoderStatus() { + if (pendingDecoderOutputBuffIndx != -1) { + pendingDecoderOutputBuffIndx = -1; + } + } + + private boolean FastOrNormalTimeBufferProcess(int encoderInBuffIndex) { + int samplesNum = stream.samplesAvailable(); + + boolean status = false; + + int rawDataLen = samplesNum * outputChannelCount; + + if (rawDataLen >= BUFFER_CAPACITY) { + return readStreamDataAndQueueToEncoder(BUFFER_CAPACITY, encoderInBuffIndex); + } else if (rawDataLen > 0 && rawDataLen < BUFFER_CAPACITY) { + return readStreamDataAndQueueToEncoder(rawDataLen, encoderInBuffIndex); + } else if (isEOF && samplesNum == 0) { + return finalizeEncoderQueue(encoderInBuffIndex); + } else { + return status; + } + } + + private boolean slowTimeBufferProcess(final int encoderInBuffIndex) { + int samplesNum = stream.samplesAvailable(); + + boolean status = false; + + int rawDataLen = samplesNum * outputChannelCount; + + if (rawDataLen >= BUFFER_CAPACITY) { + return readStreamDataAndQueueToEncoder(BUFFER_CAPACITY, encoderInBuffIndex); + } else if (isEOF && (rawDataLen > 0 && rawDataLen < BUFFER_CAPACITY)) { + return readStreamDataAndQueueToEncoder(rawDataLen, encoderInBuffIndex); + } else if (isEOF && rawDataLen == 0) { + return finalizeEncoderQueue(encoderInBuffIndex); + } else { + return status; + } + } + + private boolean finalizeEncoderQueue(final int encoderInBuffIndex) { + isPendingFeeding = false; + return queueInputBufferInEncoder(null, encoderInBuffIndex); + } + + private boolean readStreamDataAndQueueToEncoder(final int capacity, final int encoderInBuffIndex) { + short[] rawData = new short[capacity]; + stream.readShortFromStream(rawData, (capacity / outputChannelCount)); + return queueInputBufferInEncoder(rawData, encoderInBuffIndex); + } + + private boolean queueInputBufferInEncoder(final short[] rawData, final int encoderInBuffIndex) { + final ShortBuffer outBuffer = encoder.getInputBuffer(encoderInBuffIndex).asShortBuffer(); + + outBuffer.clear(); + if (rawData != null) { + outBuffer.put(rawData); + totalDataAdded += rawData.length; + + long presentationTimeUs = sampleCountToDurationUs(totalDataAdded, inputSampleRate, outputChannelCount); + + encoder.queueInputBuffer(encoderInBuffIndex, 0, rawData.length * BYTES_PER_SHORT, + presentationTimeUs, 0); + return false; + } else { + encoder.queueInputBuffer(encoderInBuffIndex, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM); + return false; + } + } + + private void writeToSonicSteam(final ShortBuffer data) { + short[] temBuff = new short[data.capacity()]; + data.get(temBuff); + data.rewind(); + stream.writeShortToStream(temBuff, temBuff.length / outputChannelCount); + } + + public boolean isAnyPendingBuffIndex() { + // allow to decoder to send data into stream (e.i. sonicprocessor) + if (pendingDecoderOutputBuffIndx != -1) { + return true; + } else { + return false; + } + } +} diff --git a/mp4compose/src/main/java/com/daasuu/mp4compose/composer/AudioComposer.kt b/mp4compose/src/main/java/com/daasuu/mp4compose/composer/AudioComposer.kt index c05d635d..6ffb75fe 100755 --- a/mp4compose/src/main/java/com/daasuu/mp4compose/composer/AudioComposer.kt +++ b/mp4compose/src/main/java/com/daasuu/mp4compose/composer/AudioComposer.kt @@ -12,12 +12,13 @@ import java.nio.ByteOrder internal class AudioComposer( private val mediaExtractor: MediaExtractor, private val trackIndex: Int, - private val muxRender: MuxRender + private val muxRender: MuxRender, + private val useFallBacks: Boolean = false ) : IAudioComposer { private val sampleType = MuxRender.SampleType.AUDIO private val bufferInfo = MediaCodec.BufferInfo() - private val bufferSize: Int - private val buffer: ByteBuffer + private var bufferSize: Int + private var buffer: ByteBuffer override var isFinished: Boolean = false private set private val actualOutputFormat: MediaFormat @@ -27,7 +28,15 @@ internal class AudioComposer( init { actualOutputFormat = this.mediaExtractor.getTrackFormat(this.trackIndex) this.muxRender.setOutputFormat(this.sampleType, actualOutputFormat) - bufferSize = actualOutputFormat.getInteger(MediaFormat.KEY_MAX_INPUT_SIZE) + + // TODO: maybe the original assignement could work as well? + // bufferSize = actualOutputFormat.containsKey(MediaFormat.KEY_MAX_INPUT_SIZE) ? actualOutputFormat.getInteger(MediaFormat.KEY_MAX_INPUT_SIZE) : (64 * 1024); + bufferSize = if (useFallBacks && !actualOutputFormat.containsKey(MediaFormat.KEY_MAX_INPUT_SIZE)) { + (64 * 1024) + } else { + actualOutputFormat.getInteger(MediaFormat.KEY_MAX_INPUT_SIZE) + } + buffer = ByteBuffer.allocateDirect(bufferSize).order(ByteOrder.nativeOrder()) } @@ -46,11 +55,18 @@ internal class AudioComposer( buffer.clear() val sampleSize = mediaExtractor.readSampleData(buffer, 0) - assert(sampleSize <= bufferSize) + if (useFallBacks && sampleSize > bufferSize) { + bufferSize = 2 * sampleSize + buffer = ByteBuffer.allocateDirect(bufferSize).order(ByteOrder.nativeOrder()) + } else { + assert(sampleSize <= bufferSize) + } val isKeyFrame = mediaExtractor.sampleFlags and MediaExtractor.SAMPLE_FLAG_SYNC != 0 val flags = if (isKeyFrame) MediaCodec.BUFFER_FLAG_KEY_FRAME else 0 bufferInfo.set(0, sampleSize, mediaExtractor.sampleTime, flags) muxRender.writeSampleData(sampleType, buffer, bufferInfo) + // TODO: should we use the original writtenPresentationTimeUs = mediaExtractor.getSampleTime(); + // at least for the Video Compression use case? writtenPresentationTimeUs = bufferInfo.presentationTimeUs mediaExtractor.advance() diff --git a/mp4compose/src/main/java/com/daasuu/mp4compose/composer/BaseAudioChannel.java b/mp4compose/src/main/java/com/daasuu/mp4compose/composer/BaseAudioChannel.java new file mode 100644 index 00000000..dd933e34 --- /dev/null +++ b/mp4compose/src/main/java/com/daasuu/mp4compose/composer/BaseAudioChannel.java @@ -0,0 +1,75 @@ +package com.daasuu.mp4compose.composer; + +import android.media.MediaCodec; +import android.media.MediaFormat; + +import java.nio.ShortBuffer; +import java.util.ArrayDeque; +import java.util.Queue; + +/** + * Created by TAPOS DATTA on 22,May,2020 + */ +@SuppressWarnings("MemberName") +abstract class BaseAudioChannel { + protected static class AudioBuffer { + int bufferIndex; + long presentationTimeUs; + ShortBuffer data; + } + protected static class BufferInfo { + long totaldata; + long presentationTimeUs; + } + + static final int BUFFER_INDEX_END_OF_STREAM = -1; + protected static final int BYTE_PER_SAMPLE = 16 / 8; + protected static final int BYTES_PER_SHORT = 2; + protected static final long MICROSECS_PER_SEC = 1000000; + + protected final Queue emptyBuffers = new ArrayDeque<>(); + protected final Queue filledBuffers = new ArrayDeque<>(); + + protected final MediaCodec decoder; + protected final MediaCodec encoder; + protected final MediaFormat encodeFormat; + + protected int inputSampleRate; + protected int inputChannelCount; + protected int outputChannelCount; + + protected final AudioBuffer overflowBuffer = new AudioBuffer(); + + protected MediaFormat actualDecodedFormat; + + BaseAudioChannel(final MediaCodec decoder, + final MediaCodec encoder, final MediaFormat encodeFormat) { + this.decoder = decoder; + this.encoder = encoder; + this.encodeFormat = encodeFormat; + } + + public void setActualDecodedFormat(final MediaFormat decodedFormat) { + actualDecodedFormat = decodedFormat; + + inputSampleRate = actualDecodedFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE); + if (inputSampleRate != encodeFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE)) { + throw new UnsupportedOperationException("Audio sample rate conversion not supported yet."); + } + + inputChannelCount = actualDecodedFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT); + outputChannelCount = encodeFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT); + + if (outputChannelCount != 1 && outputChannelCount != 2) { + throw new UnsupportedOperationException("Output channel count (" + outputChannelCount + ") not supported."); + } + + overflowBuffer.presentationTimeUs = 0; + } + + protected abstract long sampleCountToDurationUs(long sampleCount, int sampleRate, int channelCount); + + protected abstract void drainDecoderBufferAndQueue(int bufferIndex, long presentationTimeUs); + + protected abstract boolean feedEncoder(long timeoutUs); +} diff --git a/mp4compose/src/main/java/com/daasuu/mp4compose/composer/ComposerProvider.kt b/mp4compose/src/main/java/com/daasuu/mp4compose/composer/ComposerProvider.kt new file mode 100644 index 00000000..1dce27bd --- /dev/null +++ b/mp4compose/src/main/java/com/daasuu/mp4compose/composer/ComposerProvider.kt @@ -0,0 +1,68 @@ +package com.daasuu.mp4compose.composer + +import android.content.Context +import android.graphics.Bitmap +import android.media.MediaCodecInfo.CodecProfileLevel +import android.net.Uri +import android.util.Size +import com.daasuu.mp4compose.FillMode +import com.daasuu.mp4compose.VideoFormatMimeType +import com.daasuu.mp4compose.composer.ComposerUseCase.CompressVideo +import com.daasuu.mp4compose.composer.ComposerUseCase.SaveVideoAsFile +import com.daasuu.mp4compose.composer.ComposerUseCase.SaveVideoFromBgAsFile +import com.daasuu.mp4compose.filter.GlFilter + +sealed class ComposerUseCase { + data class SaveVideoAsFile( + val srcUri: Uri, + val destPath: String, + val context: Context, + val headers: Map? + ) : ComposerUseCase() + + data class SaveVideoFromBgAsFile(val bkgBmp: Bitmap, val destPath: String) : ComposerUseCase() + + data class CompressVideo @JvmOverloads constructor ( + val srcPath: String, + val destPath: String, + val videoFormatMimeType: VideoFormatMimeType, + val bitrate: Int, + val iFrameInterval: Int = 1, + val audioBitRate: Int = 128000, + val aacProfile: Int = CodecProfileLevel.AACObjectELD, + val forceAudioEncoding: Boolean = false + ) : ComposerUseCase() +} + +interface ComposerInterface { + fun size(size: Size): ComposerInterface + fun fillMode(fillMode: FillMode): ComposerInterface + fun filter(filter: GlFilter?): ComposerInterface + fun mute(mute: Boolean): ComposerInterface + fun listener(listener: Listener): ComposerInterface + fun start(): ComposerInterface +} + +object ComposerProvider { + fun getComposerForUseCase(useCase: ComposerUseCase): ComposerInterface { + return when (useCase) { + is SaveVideoAsFile -> { + Mp4Composer(useCase.srcUri, useCase.destPath) + .with(useCase.context) + .addedHeaders(useCase.headers) + } + is SaveVideoFromBgAsFile -> { + Mp4Composer(useCase.bkgBmp, useCase.destPath) + } + is CompressVideo -> { + Mp4ComposerBasic(useCase.srcPath, useCase.destPath) + .videoFormatMimeType(useCase.videoFormatMimeType) + .videoBitrate(useCase.bitrate) + .iFrameInterval(useCase.iFrameInterval) + .audioBitRate(useCase.audioBitRate) + .aacProfile(useCase.aacProfile) + .forceAudioEncoding(useCase.forceAudioEncoding) + } + } + } +} diff --git a/mp4compose/src/main/java/com/daasuu/mp4compose/composer/Listener.kt b/mp4compose/src/main/java/com/daasuu/mp4compose/composer/Listener.kt new file mode 100644 index 00000000..71d24544 --- /dev/null +++ b/mp4compose/src/main/java/com/daasuu/mp4compose/composer/Listener.kt @@ -0,0 +1,24 @@ +package com.daasuu.mp4compose.composer + +interface Listener { + /** + * Called to notify progress. + * + * @param progress Progress in [0.0, 1.0] range, or negative value if progress is unknown. + */ + fun onProgress(progress: Double) + + /** + * Called when transcode completed. + */ + fun onCompleted() + + /** + * Called when transcode canceled. + */ + fun onCanceled() + + fun onFailed(exception: Exception) + + fun onStart() +} diff --git a/mp4compose/src/main/java/com/daasuu/mp4compose/composer/Mp4Composer.kt b/mp4compose/src/main/java/com/daasuu/mp4compose/composer/Mp4Composer.kt index 312fd9a9..4dfcf70b 100644 --- a/mp4compose/src/main/java/com/daasuu/mp4compose/composer/Mp4Composer.kt +++ b/mp4compose/src/main/java/com/daasuu/mp4compose/composer/Mp4Composer.kt @@ -21,7 +21,7 @@ import java.util.concurrent.Executors * Created by sudamasayuki on 2017/11/15. */ -class Mp4Composer { +class Mp4Composer : ComposerInterface { private val srcUri: Uri? private val destPath: String private var filter: GlFilter? = null @@ -68,7 +68,7 @@ class Mp4Composer { return this } - fun filter(filter: GlFilter?): Mp4Composer { + override fun filter(filter: GlFilter?): Mp4Composer { this.filter = filter return this } @@ -78,7 +78,7 @@ class Mp4Composer { return this } - fun size(size: Size): Mp4Composer { + override fun size(size: Size): Mp4Composer { this.outputResolution = size return this } @@ -88,7 +88,7 @@ class Mp4Composer { return this } - fun mute(mute: Boolean): Mp4Composer { + override fun mute(mute: Boolean): Mp4Composer { this.mute = mute return this } @@ -108,7 +108,7 @@ class Mp4Composer { return this } - fun fillMode(fillMode: FillMode): Mp4Composer { + override fun fillMode(fillMode: FillMode): Mp4Composer { this.fillMode = fillMode return this } @@ -119,7 +119,7 @@ class Mp4Composer { return this } - fun listener(listener: Listener): Mp4Composer { + override fun listener(listener: Listener): Mp4Composer { this.listener = listener return this } @@ -136,7 +136,7 @@ class Mp4Composer { return executorService!! } - fun start(): Mp4Composer { + override fun start(): Mp4Composer { getExecutorService().execute(Runnable { val engine = Mp4ComposerEngine() @@ -265,27 +265,6 @@ class Mp4Composer { getExecutorService().shutdownNow() } - interface Listener { - /** - * Called to notify progress. - * - * @param progress Progress in [0.0, 1.0] range, or negative value if progress is unknown. - */ - fun onProgress(progress: Double) - - /** - * Called when transcode completed. - */ - fun onCompleted() - - /** - * Called when transcode canceled. - */ - fun onCanceled() - - fun onFailed(exception: Exception) - } - private fun initializeUriDataSource(engine: Mp4ComposerEngine) { engine.setDataSource(srcUri, addedRequestHeaders) } diff --git a/mp4compose/src/main/java/com/daasuu/mp4compose/composer/Mp4ComposerBasic.java b/mp4compose/src/main/java/com/daasuu/mp4compose/composer/Mp4ComposerBasic.java new file mode 100644 index 00000000..1e207210 --- /dev/null +++ b/mp4compose/src/main/java/com/daasuu/mp4compose/composer/Mp4ComposerBasic.java @@ -0,0 +1,453 @@ +package com.daasuu.mp4compose.composer; + +import android.media.MediaCodec; +import android.media.MediaCodecInfo; +import android.media.MediaMetadataRetriever; +import android.util.Size; + +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; + +import com.daasuu.mp4compose.FillMode; +import com.daasuu.mp4compose.FillModeCustomItem; +import com.daasuu.mp4compose.Rotation; +import com.daasuu.mp4compose.VideoFormatMimeType; +import com.daasuu.mp4compose.filter.GlFilter; +import com.daasuu.mp4compose.logger.AndroidLogger; +import com.daasuu.mp4compose.logger.Logger; +import com.daasuu.mp4compose.source.DataSource; +import com.daasuu.mp4compose.source.FilePathDataSource; + +import java.io.FileDescriptor; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +/** + * Created by sudamasayuki on 2017/11/15. + */ +@SuppressWarnings("MemberName") +public class Mp4ComposerBasic implements ComposerInterface { + private static final String TAG = Mp4Composer.class.getSimpleName(); + + private final DataSource srcDataSource; + private final String destPath; + private FileDescriptor destFileDescriptor; + private GlFilter filter; + private Size outputResolution; + private int bitrate = -1; + private int iFrameInterval = 1; + private int audioBitRate = 128000; + private int aacProfile = MediaCodecInfo.CodecProfileLevel.AACObjectELD; + private boolean forceAudioEncoding = false; + private boolean mute = false; + private Rotation rotation = Rotation.NORMAL; + private Listener listener; + private FillMode fillMode = FillMode.PRESERVE_ASPECT_FIT; + private FillModeCustomItem fillModeCustomItem; + // TODO: currently we do not use the timeScale feature. Also the timeScale ends up + // being converted into an int in the VideoComposer layer. + // See https://github.com/Automattic/stories-android/issues/685 for more context. + private float timeScale = 1f; // should be in range 0.125 (-8X) to 8.0 (8X) + private boolean isPitchChanged = false; + private boolean flipVertical = false; + private boolean flipHorizontal = false; + private long trimStartMs = 0; + private long trimEndMs = -1; + private VideoFormatMimeType videoFormatMimeType = VideoFormatMimeType.AUTO; + + private ExecutorService executorService; + private Mp4ComposerEngineBasic engine; + + private Logger logger; + + private DataSource.Listener errorDataSource = new DataSource.Listener() { + @Override + public void onError(Exception e) { + notifyListenerOfFailureAndShutdown(e); + } + }; + + public Mp4ComposerBasic(@NonNull final String srcPath, @NonNull final String destPath) { + this(srcPath, destPath, new AndroidLogger()); + } + + public Mp4ComposerBasic( + @NonNull final String srcPath, + @NonNull final String destPath, + @NonNull final Logger logger + ) { + this.logger = logger; + this.srcDataSource = new FilePathDataSource(srcPath, logger, errorDataSource); + this.destPath = destPath; + } + + public Mp4ComposerBasic filter(@NonNull GlFilter filter) { + this.filter = filter; + return this; + } + + public Mp4ComposerBasic size(int width, int height) { + this.outputResolution = new Size(width, height); + return this; + } + + @Override + public Mp4ComposerBasic size(Size size) { + this.outputResolution = size; + return this; + } + + public Mp4ComposerBasic videoBitrate(int bitrate) { + this.bitrate = bitrate; + return this; + } + + public Mp4ComposerBasic iFrameInterval(int iFrameInterval) { + this.iFrameInterval = iFrameInterval; + return this; + } + + public Mp4ComposerBasic audioBitRate(int audioBitRate) { + this.audioBitRate = audioBitRate; + return this; + } + + public Mp4ComposerBasic aacProfile(int aacProfile) { + this.aacProfile = aacProfile; + return this; + } + + public Mp4ComposerBasic forceAudioEncoding(boolean forceAudioEncoding) { + this.forceAudioEncoding = forceAudioEncoding; + return this; + } + + public Mp4ComposerBasic mute(boolean mute) { + this.mute = mute; + return this; + } + + public Mp4ComposerBasic flipVertical(boolean flipVertical) { + this.flipVertical = flipVertical; + return this; + } + + public Mp4ComposerBasic flipHorizontal(boolean flipHorizontal) { + this.flipHorizontal = flipHorizontal; + return this; + } + + public Mp4ComposerBasic rotation(@NonNull Rotation rotation) { + this.rotation = rotation; + return this; + } + + public Mp4ComposerBasic fillMode(@NonNull FillMode fillMode) { + this.fillMode = fillMode; + return this; + } + + public Mp4ComposerBasic customFillMode(@NonNull FillModeCustomItem fillModeCustomItem) { + this.fillModeCustomItem = fillModeCustomItem; + this.fillMode = FillMode.CUSTOM; + return this; + } + + public Mp4ComposerBasic listener(@NonNull Listener listener) { + this.listener = listener; + return this; + } + + public Mp4ComposerBasic timeScale(final float timeScale) { + this.timeScale = timeScale; + return this; + } + + public Mp4ComposerBasic changePitch(final boolean isPitchChanged) { + this.isPitchChanged = isPitchChanged; + return this; + } + + public Mp4ComposerBasic videoFormatMimeType(@NonNull VideoFormatMimeType videoFormatMimeType) { + this.videoFormatMimeType = videoFormatMimeType; + return this; + } + + /** + * Set the {@link Logger} that should be used. Defaults to {@link AndroidLogger} if none is set. + * + * @param logger The logger that should be used to log. + * @return The composer instance. + */ + public Mp4ComposerBasic logger(@NonNull final Logger logger) { + this.logger = logger; + return this; + } + + /** + * Trim the video to the provided times. By default the video will not be trimmed. + * + * @param trimStartMs The start time of the trim in milliseconds. + * @param trimEndMs The end time of the trim in milliseconds, -1 for no end. + * @return The composer instance. + */ + public Mp4ComposerBasic trim(final long trimStartMs, final long trimEndMs) { + this.trimStartMs = trimStartMs; + this.trimEndMs = trimEndMs; + return this; + } + + @Nullable + public Size getSrcVideoResolution() { + return getVideoResolution(srcDataSource); + } + + private ExecutorService getExecutorService() { + if (executorService == null) { + executorService = Executors.newSingleThreadExecutor(); + } + return executorService; + } + + + public Mp4ComposerBasic start() { + // if we're already composing, calling this should do nothing + if (engine != null) { + return this; + } + + getExecutorService().execute(new Runnable() { + @Override + public void run() { + if (logger == null) { + logger = new AndroidLogger(); + } + engine = new Mp4ComposerEngineBasic(logger); + + engine.setProgressCallback(new Mp4ComposerEngineBasic.ProgressCallback() { + @Override + public void onProgress(final double progress) { + if (listener != null) { + listener.onProgress(progress); + } + } + }); + + final Integer videoRotate = getVideoRotation(srcDataSource); + final Size srcVideoResolution = getVideoResolution(srcDataSource); + + if (srcVideoResolution == null || videoRotate == null) { + notifyListenerOfFailureAndShutdown( + new UnsupportedOperationException("File type unsupported, path: " + srcDataSource) + ); + return; + } + + if (filter == null) { + filter = new GlFilter(); + } + + if (fillMode == null) { + fillMode = FillMode.PRESERVE_ASPECT_FIT; + } + if (fillMode == FillMode.CUSTOM && fillModeCustomItem == null) { + notifyListenerOfFailureAndShutdown( + new IllegalAccessException("FillMode.CUSTOM must need fillModeCustomItem.") + ); + return; + } + + if (fillModeCustomItem != null) { + fillMode = FillMode.CUSTOM; + } + + if (outputResolution == null) { + if (fillMode == FillMode.CUSTOM) { + outputResolution = srcVideoResolution; + } else { + Rotation rotate = Rotation.fromInt(rotation.getRotation() + videoRotate); + if (rotate == Rotation.ROTATION_90 || rotate == Rotation.ROTATION_270) { + outputResolution = new Size(srcVideoResolution.getHeight(), srcVideoResolution.getWidth()); + } else { + outputResolution = srcVideoResolution; + } + } + } + + if (timeScale < 0.125f) { + timeScale = 0.125f; + } else if (timeScale > 8f) { + timeScale = 8f; + } + + logger.debug(TAG, "rotation = " + (rotation.getRotation() + videoRotate)); + logger.debug(TAG, "rotation = " + Rotation.fromInt(rotation.getRotation() + videoRotate)); + logger.debug( + TAG, + "inputResolution width = " + srcVideoResolution.getWidth() + " height = " + + srcVideoResolution.getHeight() + ); + logger.debug( + TAG, + "outputResolution width = " + outputResolution.getWidth() + " height = " + + outputResolution.getHeight() + ); + logger.debug(TAG, "fillMode = " + fillMode); + + try { + if (bitrate < 0) { + bitrate = calcBitRate(outputResolution.getWidth(), outputResolution.getHeight()); + } + + if (listener != null) { + listener.onStart(); + } + + engine.compose( + srcDataSource, + destPath, + destFileDescriptor, + outputResolution, + filter, + bitrate, + mute, + Rotation.fromInt(rotation.getRotation() + videoRotate), + srcVideoResolution, + fillMode, + fillModeCustomItem, + timeScale, + isPitchChanged, + flipVertical, + flipHorizontal, + trimStartMs, + trimEndMs, + videoFormatMimeType, + iFrameInterval, + audioBitRate, + aacProfile, + forceAudioEncoding + ); + } catch (Exception e) { + if (e instanceof MediaCodec.CodecException) { + logger.error( + TAG, + "This devicel cannot codec with that setting. Check width, height, " + + "bitrate and video format.", e + ); + notifyListenerOfFailureAndShutdown(e); + return; + } + + logger.error(TAG, "Unable to compose the engine", e); + notifyListenerOfFailureAndShutdown(e); + return; + } + + if (listener != null) { + if (engine.isCanceled()) { + listener.onCanceled(); + } else { + listener.onCompleted(); + } + } + executorService.shutdown(); + engine = null; + } + }); + + return this; + } + + private void notifyListenerOfFailureAndShutdown(final Exception failure) { + if (listener != null) { + listener.onFailed(failure); + } + if (executorService != null) { + executorService.shutdown(); + } + } + + public void cancel() { + if (engine != null) { + engine.cancel(); + } + } + + @Nullable + private Integer getVideoRotation(DataSource dataSource) { + MediaMetadataRetriever mediaMetadataRetriever = null; + try { + mediaMetadataRetriever = new MediaMetadataRetriever(); + mediaMetadataRetriever.setDataSource(dataSource.getFileDescriptor()); + final String orientation = mediaMetadataRetriever.extractMetadata( + MediaMetadataRetriever.METADATA_KEY_VIDEO_ROTATION + ); + if (orientation == null) { + return null; + } + return Integer.valueOf(orientation); + } catch (IllegalArgumentException e) { + logger.error("MediaMetadataRetriever", "getVideoRotation IllegalArgumentException", e); + return 0; + } catch (RuntimeException e) { + logger.error("MediaMetadataRetriever", "getVideoRotation RuntimeException", e); + return 0; + } catch (Exception e) { + logger.error("MediaMetadataRetriever", "getVideoRotation Exception", e); + return 0; + } finally { + try { + if (mediaMetadataRetriever != null) { + mediaMetadataRetriever.release(); + } + } catch (RuntimeException e) { + logger.error(TAG, "Failed to release mediaMetadataRetriever.", e); + } + } + } + + private int calcBitRate(int width, int height) { + final int bitrate = (int) (0.25 * 30 * width * height); + logger.debug(TAG, "bitrate=" + bitrate); + return bitrate; + } + + /** + * Extract the resolution of the video at the provided path, or null if the format is + * unsupported. + */ + @Nullable + private Size getVideoResolution(DataSource dataSource) { + MediaMetadataRetriever retriever = null; + try { + retriever = new MediaMetadataRetriever(); + retriever.setDataSource(dataSource.getFileDescriptor()); + final String rawWidth = retriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_VIDEO_WIDTH); + final String rawHeight = retriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_VIDEO_HEIGHT); + if (rawWidth == null || rawHeight == null) { + return null; + } + final int width = Integer.parseInt(rawWidth); + final int height = Integer.parseInt(rawHeight); + + return new Size(width, height); + } catch (IllegalArgumentException e) { + logger.error("MediaMetadataRetriever", "getVideoResolution IllegalArgumentException", e); + return null; + } catch (RuntimeException e) { + logger.error("MediaMetadataRetriever", "getVideoResolution RuntimeException", e); + return null; + } catch (Exception e) { + logger.error("MediaMetadataRetriever", "getVideoResolution Exception", e); + return null; + } finally { + try { + if (retriever != null) { + retriever.release(); + } + } catch (RuntimeException e) { + logger.error(TAG, "Failed to release mediaMetadataRetriever.", e); + } + } + } +} diff --git a/mp4compose/src/main/java/com/daasuu/mp4compose/composer/Mp4ComposerEngineBasic.java b/mp4compose/src/main/java/com/daasuu/mp4compose/composer/Mp4ComposerEngineBasic.java new file mode 100644 index 00000000..9a755a25 --- /dev/null +++ b/mp4compose/src/main/java/com/daasuu/mp4compose/composer/Mp4ComposerEngineBasic.java @@ -0,0 +1,424 @@ +package com.daasuu.mp4compose.composer; + +import android.media.MediaCodecInfo; +import android.media.MediaCodecList; +import android.media.MediaExtractor; +import android.media.MediaFormat; +import android.media.MediaMetadataRetriever; +import android.media.MediaMuxer; +import android.os.Build; +import android.util.Size; + +import androidx.annotation.NonNull; + +import com.daasuu.mp4compose.FillMode; +import com.daasuu.mp4compose.FillModeCustomItem; +import com.daasuu.mp4compose.Rotation; +import com.daasuu.mp4compose.VideoFormatMimeType; +import com.daasuu.mp4compose.filter.GlFilter; +import com.daasuu.mp4compose.logger.Logger; +import com.daasuu.mp4compose.source.DataSource; + +import java.io.FileDescriptor; +import java.io.IOException; + +// Refer: https://github.com/ypresto/android-transcoder/blob/master/lib/src/main/ +// java/net/ypresto/androidtranscoder/engine/MediaTranscoderEngine.java + +/** + * Internal engine, do not use this directly. + */ +@SuppressWarnings("MemberName") +class Mp4ComposerEngineBasic { + private static final String TAG = "Mp4ComposerEngine"; + private static final String AUDIO_PREFIX = "audio/"; + private static final String VIDEO_PREFIX = "video/"; + private static final double PROGRESS_UNKNOWN = -1.0; + private static final long SLEEP_TO_WAIT_TRACK_TRANSCODERS = 10; + private static final long PROGRESS_INTERVAL_STEPS = 10; + private VideoComposer videoComposer; + private IAudioComposer audioComposer; + private MediaExtractor mediaExtractor; + private MediaMuxer mediaMuxer; + private ProgressCallback progressCallback; + private long durationUs; + private MediaMetadataRetriever mediaMetadataRetriever; + private volatile boolean canceled; + private final Logger logger; + + Mp4ComposerEngineBasic(@NonNull final Logger logger) { + this.logger = logger; + } + + void setProgressCallback(ProgressCallback progressCallback) { + this.progressCallback = progressCallback; + } + + // TODO: currently we do not use the timeScale feature. Also the timeScale ends up + // being converted into an int in the VideoComposer layer. + // See https://github.com/Automattic/stories-android/issues/685 for more context. + void compose( + final DataSource srcDataSource, + final String destSrc, + final FileDescriptor destFileDescriptor, + final Size outputResolution, + final GlFilter filter, + final int bitrate, + final boolean mute, + final Rotation rotation, + final Size inputResolution, + final FillMode fillMode, + final FillModeCustomItem fillModeCustomItem, + final float timeScale, + final boolean isPitchChanged, + final boolean flipVertical, + final boolean flipHorizontal, + final long trimStartMs, + final long trimEndMs, + final VideoFormatMimeType videoFormatMimeType, + final int iFrameInterval, + final int audioBitRate, + final int aacProfile, + final boolean forceAudioEncoding + ) throws IOException { + try { + mediaExtractor = new MediaExtractor(); + mediaExtractor.setDataSource(srcDataSource.getFileDescriptor()); + if (Build.VERSION.SDK_INT >= 26 && destSrc == null) { + mediaMuxer = new MediaMuxer(destFileDescriptor, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4); + } else { + mediaMuxer = new MediaMuxer(destSrc, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4); + } + mediaMetadataRetriever = new MediaMetadataRetriever(); + mediaMetadataRetriever.setDataSource(srcDataSource.getFileDescriptor()); + + if (trimEndMs != -1) { + durationUs = (trimEndMs - trimStartMs) * 1000; + } else { + try { + durationUs = Long.parseLong( + mediaMetadataRetriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_DURATION) + ) * 1000; + } catch (NumberFormatException e) { + durationUs = -1; + } + } + + logger.debug(TAG, "Duration (us): " + durationUs); + + MuxRender muxRender = new MuxRender(mediaMuxer); + + // identify track indices + int videoTrackIndex = -1; + int audioTrackIndex = -1; + for (int i = 0; i < mediaExtractor.getTrackCount(); i++) { + MediaFormat mediaFormat = mediaExtractor.getTrackFormat(i); + String mimeType = mediaFormat.getString(MediaFormat.KEY_MIME); + if (mimeType == null) continue; + if (mimeType.startsWith(VIDEO_PREFIX)) { + videoTrackIndex = i; + } else if (mimeType.startsWith(AUDIO_PREFIX)) { + audioTrackIndex = i; + } + } + + final MediaFormat actualVideoOutputFormat = createVideoOutputFormatWithAvailableEncoders( + videoFormatMimeType, + bitrate, + outputResolution, + iFrameInterval + ); + if (Build.VERSION.SDK_INT == Build.VERSION_CODES.LOLLIPOP) { + // Only LOLLIPOP sets KEY_FRAME_RATE here. + actualVideoOutputFormat.setInteger(MediaFormat.KEY_FRAME_RATE, 30); + } + + // setup video composer + videoComposer = new VideoComposer( + mediaExtractor, + videoTrackIndex, + actualVideoOutputFormat, + muxRender, + (int) timeScale + ); + videoComposer.setUp( + filter, + rotation, + outputResolution, + inputResolution, + fillMode, + fillModeCustomItem, + flipVertical, + flipHorizontal + ); + mediaExtractor.selectTrack(videoTrackIndex); + + // setup audio if present and not muted + if (audioTrackIndex >= 0 + && mediaMetadataRetriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_HAS_AUDIO) != null + && !mute + ) { + // has Audio video + final MediaFormat inputMediaFormat = mediaExtractor.getTrackFormat(audioTrackIndex); + final MediaFormat outputMediaFormat = createAudioOutputFormat( + inputMediaFormat, + audioBitRate, + aacProfile, + forceAudioEncoding + ); + + if (timeScale >= 0.99 && timeScale <= 1.01 && outputMediaFormat.equals(inputMediaFormat)) { + audioComposer = new AudioComposer( + mediaExtractor, + audioTrackIndex, + muxRender, + true + ); + } else { + audioComposer = new RemixAudioComposerBasic( + mediaExtractor, + audioTrackIndex, + outputMediaFormat, + muxRender, + timeScale, + isPitchChanged, + trimStartMs, + trimEndMs + ); + } + + audioComposer.setup(); + mediaExtractor.selectTrack(audioTrackIndex); + mediaExtractor.seekTo(trimStartMs * 1000, MediaExtractor.SEEK_TO_PREVIOUS_SYNC); + runPipelines(); + } else { + mediaExtractor.seekTo(trimStartMs * 1000, MediaExtractor.SEEK_TO_PREVIOUS_SYNC); + // no audio video + runPipelinesNoAudio(); + } + + mediaMuxer.stop(); + } finally { + try { + if (videoComposer != null) { + videoComposer.release(); + videoComposer = null; + } + if (audioComposer != null) { + audioComposer.release(); + audioComposer = null; + } + if (mediaExtractor != null) { + mediaExtractor.release(); + mediaExtractor = null; + } + } catch (RuntimeException e) { + logger.error(TAG, "Could not shutdown mediaExtractor, codecs and mediaMuxer pipeline.", e); + } + try { + if (mediaMuxer != null) { + mediaMuxer.release(); + mediaMuxer = null; + } + } catch (RuntimeException e) { + logger.error(TAG, "Failed to release mediaMuxer.", e); + } + try { + if (mediaMetadataRetriever != null) { + mediaMetadataRetriever.release(); + mediaMetadataRetriever = null; + } + } catch (RuntimeException e) { + logger.error(TAG, "Failed to release mediaMetadataRetriever.", e); + } + } + } + + void cancel() { + canceled = true; + } + + boolean isCanceled() { + return canceled; + } + + @NonNull + private static MediaFormat createVideoOutputFormatWithAvailableEncoders(@NonNull final VideoFormatMimeType mimeType, + final int bitrate, + @NonNull final Size outputResolution, + final int iFrameInterval) { + final MediaCodecList mediaCodecList = new MediaCodecList(MediaCodecList.REGULAR_CODECS); + + if (mimeType != VideoFormatMimeType.AUTO) { + final MediaFormat mediaFormat = createVideoFormat( + mimeType.getFormat(), + bitrate, + outputResolution, + iFrameInterval + ); + if (mediaCodecList.findEncoderForFormat(mediaFormat) != null) { + return mediaFormat; + } + } + + final MediaFormat hevcMediaFormat = createVideoFormat( + VideoFormatMimeType.HEVC.getFormat(), + bitrate, + outputResolution, + iFrameInterval + ); + if (mediaCodecList.findEncoderForFormat(hevcMediaFormat) != null) { + return hevcMediaFormat; + } + + final MediaFormat avcMediaFormat = createVideoFormat( + VideoFormatMimeType.AVC.getFormat(), + bitrate, + outputResolution, + iFrameInterval + ); + if (mediaCodecList.findEncoderForFormat(avcMediaFormat) != null) { + return avcMediaFormat; + } + + final MediaFormat mp4vesMediaFormat = createVideoFormat( + VideoFormatMimeType.MPEG4.getFormat(), + bitrate, + outputResolution, + iFrameInterval + ); + if (mediaCodecList.findEncoderForFormat(mp4vesMediaFormat) != null) { + return mp4vesMediaFormat; + } + + return createVideoFormat(VideoFormatMimeType.H263.getFormat(), bitrate, outputResolution, iFrameInterval); + } + + @NonNull + private static MediaFormat createAudioOutputFormat( + @NonNull final MediaFormat inputFormat, + final int bitRate, + final int aacProfile, + final boolean forceEncoding + ) { + if (!forceEncoding && MediaFormat.MIMETYPE_AUDIO_AAC.equals(inputFormat.getString(MediaFormat.KEY_MIME))) { + return inputFormat; + } else { + final MediaFormat outputFormat = new MediaFormat(); + outputFormat.setString(MediaFormat.KEY_MIME, MediaFormat.MIMETYPE_AUDIO_AAC); + outputFormat.setInteger(MediaFormat.KEY_AAC_PROFILE, aacProfile); + outputFormat.setInteger(MediaFormat.KEY_SAMPLE_RATE, + inputFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE)); + outputFormat.setInteger(MediaFormat.KEY_BIT_RATE, bitRate); + outputFormat.setInteger(MediaFormat.KEY_CHANNEL_COUNT, + inputFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT)); + + return outputFormat; + } + } + + @NonNull + private static MediaFormat createVideoFormat(@NonNull final String mimeType, + final int bitrate, + @NonNull final Size outputResolution, + final int iFrameInterval) { + final MediaFormat outputFormat = + MediaFormat.createVideoFormat(mimeType, + outputResolution.getWidth(), + outputResolution.getHeight()); + + outputFormat.setInteger(MediaFormat.KEY_BIT_RATE, bitrate); + // On Build.VERSION_CODES.LOLLIPOP, format must not contain a MediaFormat#KEY_FRAME_RATE. + // https://developer.android.com/reference/android/media/MediaCodecInfo.CodecCapabilities.html + // #isFormatSupported(android.media.MediaFormat) + if (Build.VERSION.SDK_INT != Build.VERSION_CODES.LOLLIPOP) { + // Required but ignored by the encoder + outputFormat.setInteger(MediaFormat.KEY_FRAME_RATE, 30); + } + outputFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, iFrameInterval); + outputFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, + MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface); + + return outputFormat; + } + + + private void runPipelines() { + long loopCount = 0; + if (durationUs <= 0) { + if (progressCallback != null) { + progressCallback.onProgress(PROGRESS_UNKNOWN); + } // unknown + } + while (!canceled && !(videoComposer.isFinished() && audioComposer.isFinished())) { + boolean stepped = videoComposer.stepPipeline() + || audioComposer.stepPipeline(); + loopCount++; + if (durationUs > 0 && loopCount % PROGRESS_INTERVAL_STEPS == 0) { + long writtenPresentationVideoTimeUs = videoComposer.getWrittenPresentationTimeUs(); + double videoProgress = videoComposer.isFinished() ? 1.0 : Math.min( + 1.0, + (double) getWrittenPresentationTimeUs(writtenPresentationVideoTimeUs) / durationUs + ); + double audioProgress = audioComposer.isFinished() ? 1.0 : Math.min( + 1.0, + (double) getWrittenPresentationTimeUs(audioComposer.getWrittenPresentationTimeUs()) / durationUs + ); + double progress = (videoProgress + audioProgress) / 2.0; + if (progressCallback != null) { + progressCallback.onProgress(progress); + } + } + if (!stepped) { + try { + Thread.sleep(SLEEP_TO_WAIT_TRACK_TRANSCODERS); + } catch (InterruptedException e) { + // nothing to do + } + } + } + } + + private long getWrittenPresentationTimeUs(long time) { + return Math.max(0, time); + } + + private void runPipelinesNoAudio() { + long loopCount = 0; + if (durationUs <= 0) { + if (progressCallback != null) { + progressCallback.onProgress(PROGRESS_UNKNOWN); + } // unknown + } + while (!canceled && !videoComposer.isFinished()) { + boolean stepped = videoComposer.stepPipeline(); + loopCount++; + if (durationUs > 0 && loopCount % PROGRESS_INTERVAL_STEPS == 0) { + long writtenPresentationVideoTimeUs = videoComposer.getWrittenPresentationTimeUs(); + double videoProgress = videoComposer.isFinished() ? 1.0 : Math.min( + 1.0, + (double) getWrittenPresentationTimeUs(writtenPresentationVideoTimeUs) / durationUs + ); + if (progressCallback != null) { + progressCallback.onProgress(videoProgress); + } + } + if (!stepped) { + try { + Thread.sleep(SLEEP_TO_WAIT_TRACK_TRANSCODERS); + } catch (InterruptedException e) { + // nothing to do + } + } + } + } + + interface ProgressCallback { + /** + * Called to notify progress. Same thread which initiated transcode is used. + * + * @param progress Progress in [0.0, 1.0] range, or negative value if progress is unknown. + */ + void onProgress(double progress); + } +} diff --git a/mp4compose/src/main/java/com/daasuu/mp4compose/composer/RemixAudioComposer.kt b/mp4compose/src/main/java/com/daasuu/mp4compose/composer/RemixAudioComposer.kt index 3c57f770..d6eafaa0 100644 --- a/mp4compose/src/main/java/com/daasuu/mp4compose/composer/RemixAudioComposer.kt +++ b/mp4compose/src/main/java/com/daasuu/mp4compose/composer/RemixAudioComposer.kt @@ -17,7 +17,7 @@ internal class RemixAudioComposer( private val trackIndex: Int, private val outputFormat: MediaFormat, private val muxer: MuxRender, - private val timeScale: Int + private val timeScale: Int // TODO: this (and in other places) was float in the original lib, should we restore it? ) : IAudioComposer { override var writtenPresentationTimeUs: Long = 0 private set diff --git a/mp4compose/src/main/java/com/daasuu/mp4compose/composer/RemixAudioComposerBasic.java b/mp4compose/src/main/java/com/daasuu/mp4compose/composer/RemixAudioComposerBasic.java new file mode 100644 index 00000000..9565b3c5 --- /dev/null +++ b/mp4compose/src/main/java/com/daasuu/mp4compose/composer/RemixAudioComposerBasic.java @@ -0,0 +1,235 @@ +package com.daasuu.mp4compose.composer; + + +/** + * // Refer: https://github.com/ypresto/android-transcoder/blob/master/lib/src/main/ + * java/net/ypresto/androidtranscoder/engine/AudioTrackTranscoder.java + * + */ +import android.media.MediaCodec; +import android.media.MediaExtractor; +import android.media.MediaFormat; + +import com.daasuu.mp4compose.composer.MuxRender.SampleType; + +import java.io.IOException; +import java.util.concurrent.TimeUnit; + +@SuppressWarnings({"FallThrough", "MemberName"}) +class RemixAudioComposerBasic implements IAudioComposer { + private static final SampleType SAMPLE_TYPE = SampleType.AUDIO; + + private static final int DRAIN_STATE_NONE = 0; + private static final int DRAIN_STATE_SHOULD_RETRY_IMMEDIATELY = 1; + private static final int DRAIN_STATE_CONSUMED = 2; + + private final MediaExtractor extractor; + private final MuxRender muxer; + private long writtenPresentationTimeUs; + + private final int trackIndex; + + private final MediaFormat outputFormat; + + private final MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo(); + private MediaCodec decoder; + private MediaCodec encoder; + private MediaFormat actualOutputFormat; + + private boolean isExtractorEOS; + private boolean isDecoderEOS; + private boolean isEncoderEOS; + private boolean decoderStarted; + private boolean encoderStarted; + + private AudioChannelWithSP audioChannel; + private final float timeScale; + private final boolean isPitchChanged; + + private final long trimStartUs; + private final long trimEndUs; + int numTracks = 0; + + // Used for AAC priming offset. + private boolean addPrimingDelay; + private int frameCounter; + private long primingDelay; + + RemixAudioComposerBasic(MediaExtractor extractor, int trackIndex, + MediaFormat outputFormat, MuxRender muxer, float timeScale, boolean isPitchChanged, + long trimStartMs, long trimEndMs) { + this.extractor = extractor; + this.trackIndex = trackIndex; + this.outputFormat = outputFormat; + this.muxer = muxer; + this.timeScale = timeScale; + this.isPitchChanged = isPitchChanged; + this.trimStartUs = TimeUnit.MILLISECONDS.toMicros(trimStartMs); + this.trimEndUs = trimEndMs == -1 ? trimEndMs : TimeUnit.MILLISECONDS.toMicros(trimEndMs); + } + + @Override + public void setup() { + extractor.selectTrack(trackIndex); + try { + encoder = MediaCodec.createEncoderByType(outputFormat.getString(MediaFormat.KEY_MIME)); + } catch (IOException e) { + throw new IllegalStateException(e); + } + encoder.configure(outputFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); + encoder.start(); + encoderStarted = true; + + final MediaFormat inputFormat = extractor.getTrackFormat(trackIndex); + try { + decoder = MediaCodec.createDecoderByType(inputFormat.getString(MediaFormat.KEY_MIME)); + } catch (IOException e) { + throw new IllegalStateException(e); + } + decoder.configure(inputFormat, null, null, 0); + decoder.start(); + decoderStarted = true; + + audioChannel = new AudioChannelWithSP(decoder, encoder, outputFormat, timeScale, isPitchChanged); + } + + @Override + public boolean stepPipeline() { + boolean busy = false; + + int status; + + while (drainEncoder(0) != DRAIN_STATE_NONE) busy = true; + do { + if (audioChannel.isAnyPendingBuffIndex()) { + break; + } + status = drainDecoder(0); + if (status != DRAIN_STATE_NONE) busy = true; + // NOTE: not repeating to keep from deadlock when encoder is full. + } while (status == DRAIN_STATE_SHOULD_RETRY_IMMEDIATELY); + + while (audioChannel.feedEncoder(0)) busy = true; + while (drainExtractor(0) != DRAIN_STATE_NONE) busy = true; + + return busy; + } + + private int drainExtractor(long timeoutUs) { + if (isExtractorEOS) return DRAIN_STATE_NONE; + int trackIndex = extractor.getSampleTrackIndex(); + if (trackIndex >= 0 && trackIndex != this.trackIndex) { + return DRAIN_STATE_NONE; + } + + final int result = decoder.dequeueInputBuffer(timeoutUs); + if (result < 0) return DRAIN_STATE_NONE; + if (trackIndex < 0 || (writtenPresentationTimeUs >= trimEndUs && trimEndUs != -1)) { + isExtractorEOS = true; + decoder.queueInputBuffer(result, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM); + extractor.unselectTrack(this.trackIndex); + return DRAIN_STATE_NONE; + } + + final int sampleSize = extractor.readSampleData(decoder.getInputBuffer(result), 0); + final boolean isKeyFrame = (extractor.getSampleFlags() & MediaExtractor.SAMPLE_FLAG_SYNC) != 0; + decoder.queueInputBuffer( + result, + 0, + sampleSize, + extractor.getSampleTime(), + isKeyFrame ? MediaCodec.BUFFER_FLAG_KEY_FRAME : 0 + ); + extractor.advance(); + numTracks++; + return DRAIN_STATE_CONSUMED; + } + + private int drainDecoder(long timeoutUs) { + if (isDecoderEOS) return DRAIN_STATE_NONE; + + int result = decoder.dequeueOutputBuffer(bufferInfo, timeoutUs); + switch (result) { + case MediaCodec.INFO_TRY_AGAIN_LATER: + return DRAIN_STATE_NONE; + case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED: + audioChannel.setActualDecodedFormat(decoder.getOutputFormat()); + case MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED: + return DRAIN_STATE_SHOULD_RETRY_IMMEDIATELY; + } + + if ((bufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { + isDecoderEOS = true; + audioChannel.drainDecoderBufferAndQueue(BaseAudioChannel.BUFFER_INDEX_END_OF_STREAM, 0); + } else if (bufferInfo.size > 0) { + audioChannel.drainDecoderBufferAndQueue(result, bufferInfo.presentationTimeUs); + } + + return DRAIN_STATE_CONSUMED; + } + + private int drainEncoder(long timeoutUs) { + if (isEncoderEOS) return DRAIN_STATE_NONE; + int result = encoder.dequeueOutputBuffer(bufferInfo, timeoutUs); + switch (result) { + case MediaCodec.INFO_TRY_AGAIN_LATER: + return DRAIN_STATE_NONE; + case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED: + if (actualOutputFormat != null) { + throw new RuntimeException("Audio output format changed twice."); + } + actualOutputFormat = encoder.getOutputFormat(); + addPrimingDelay = MediaFormat.MIMETYPE_AUDIO_AAC.equals( + actualOutputFormat.getString(MediaFormat.KEY_MIME) + ); + muxer.setOutputFormat(SAMPLE_TYPE, actualOutputFormat); + return DRAIN_STATE_SHOULD_RETRY_IMMEDIATELY; + case MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED: + return DRAIN_STATE_SHOULD_RETRY_IMMEDIATELY; + } + + if (actualOutputFormat == null) { + throw new RuntimeException("Could not determine actual output format."); + } + + if ((bufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { + isEncoderEOS = true; + + bufferInfo.set(0, 0, 0, bufferInfo.flags); + } + if ((bufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) { + // SPS or PPS, which should be passed by MediaFormat. + encoder.releaseOutputBuffer(result, false); + return DRAIN_STATE_SHOULD_RETRY_IMMEDIATELY; + } + muxer.writeSampleData(SAMPLE_TYPE, encoder.getOutputBuffer(result), bufferInfo); + + writtenPresentationTimeUs = bufferInfo.presentationTimeUs; + encoder.releaseOutputBuffer(result, false); + return DRAIN_STATE_CONSUMED; + } + + @Override + public long getWrittenPresentationTimeUs() { + return (long) (writtenPresentationTimeUs * timeScale); + } + + @Override + public boolean isFinished() { + return isEncoderEOS; + } + + @Override + public void release() { + if (decoder != null) { + if (decoderStarted) decoder.stop(); + decoder.release(); + decoder = null; + } + if (encoder != null) { + if (encoderStarted) encoder.stop(); + encoder.release(); + encoder = null; + } + } +} diff --git a/mp4compose/src/main/java/com/daasuu/mp4compose/composer/SonicAudioProcessor.java b/mp4compose/src/main/java/com/daasuu/mp4compose/composer/SonicAudioProcessor.java new file mode 100644 index 00000000..f6dc46ab --- /dev/null +++ b/mp4compose/src/main/java/com/daasuu/mp4compose/composer/SonicAudioProcessor.java @@ -0,0 +1,975 @@ +package com.daasuu.mp4compose.composer; + +import android.util.Log; + +/** + * ********************************************************* + * Sonic library + * Copyright 2010, 2011 + * Bill Cox + * This file is part of the Sonic Library. + * + * This file is licensed under the Apache 2.0 license. + * ********************************************************* + * + * Sonic audio stream processor for time/pitch stretching + * + * ref on https://github.com/waywardgeek/sonic. + * + * + */ +@SuppressWarnings("MemberName") +class SonicAudioProcessor { + private static final int SONIC_MIN_PITCH = 65; + private static final int SONIC_MAX_PITCH = 400; + // This is used to down-sample some inputs to improve speed + private static final int SONIC_AMDF_FREQ = 4000; + + private short[] inputBuffer; + private short[] outputBuffer; + private short[] pitchBuffer; + private short[] downSampleBuffer; + private float speed; + private float volume; + private float pitch; + private float rate; + private int oldRatePosition; + private int newRatePosition; + private boolean useChordPitch; + private int quality; + private int numChannels; + private int inputBufferSize; + private int pitchBufferSize; + private int outputBufferSize; + private int numInputSamples; + private int numOutputSamples; + private int numPitchSamples; + private int minPeriod; + private int maxPeriod; + private int maxRequired; + private int remainingInputToCopy; + private int sampleRate; + private int prevPeriod; + private int prevMinDiff; + private int minDiff; + private int maxDiff; + + // Create a sonic stream. + SonicAudioProcessor( + int sampleRate, + int numChannels + ) { + allocateStreamBuffers(sampleRate, numChannels); + speed = 1.0f; + pitch = 1.0f; + volume = 1.0f; + rate = 1.0f; + oldRatePosition = 0; + newRatePosition = 0; + useChordPitch = false; + quality = 0; + } + + // Resize the array. + private short[] resize( + short[] oldArray, + int newLength + ) { + newLength *= numChannels; + short[] newArray = new short[newLength]; + int length = oldArray.length <= newLength ? oldArray.length : newLength; + + System.arraycopy(oldArray, 0, newArray, 0, length); + return newArray; + } + + // Move samples from one array to another. May move samples down within an array, but not up. + private void move( + short[] dest, + int destPos, + short[] source, + int sourcePos, + int numSamples + ) { + System.arraycopy(source, sourcePos * numChannels, dest, destPos * numChannels, numSamples * numChannels); + } + + // Scale the samples by the factor. + private void scaleSamples( + short[] samples, + int position, + int numSamples, + float volume + ) { + int fixedPointVolume = (int) (volume * 4096.0f); + int start = position * numChannels; + int stop = start + numSamples * numChannels; + + for (int xSample = start; xSample < stop; xSample++) { + int value = (samples[xSample] * fixedPointVolume) >> 12; + if (value > 32767) { + value = 32767; + } else if (value < -32767) { + value = -32767; + } + samples[xSample] = (short) value; + } + } + + // Get the speed of the stream. + public float getSpeed() { + return speed; + } + + // Set the speed of the stream. + public void setSpeed( + float speed + ) { + this.speed = speed; + } + + // Get the pitch of the stream. + public float getPitch() { + return pitch; + } + + // Set the pitch of the stream. + public void setPitch( + float pitch + ) { + this.pitch = pitch; + } + + // Get the rate of the stream. + public float getRate() { + return rate; + } + + // Set the playback rate of the stream. This scales pitch and speed at the same time. + public void setRate( + float rate + ) { + this.rate = rate; + this.oldRatePosition = 0; + this.newRatePosition = 0; + } + + // Get the vocal chord pitch setting. + public boolean getChordPitch() { + return useChordPitch; + } + + // Set the vocal chord mode for pitch computation. Default is off. + public void setChordPitch( + boolean useChordPitch + ) { + this.useChordPitch = useChordPitch; + } + + // Get the quality setting. + public int getQuality() { + return quality; + } + + // Set the "quality". Default 0 is virtually as good as 1, but very much faster. + public void setQuality( + int quality + ) { + this.quality = quality; + } + + // Get the scaling factor of the stream. + public float getVolume() { + return volume; + } + + // Set the scaling factor of the stream. + public void setVolume( + float volume + ) { + this.volume = volume; + } + + // Allocate stream buffers. + private void allocateStreamBuffers( + int sampleRate, + int numChannels + ) { + this.sampleRate = sampleRate; + this.numChannels = numChannels; + minPeriod = sampleRate / SONIC_MAX_PITCH; + maxPeriod = sampleRate / SONIC_MIN_PITCH; + maxRequired = 2 * maxPeriod; + inputBufferSize = maxRequired; + inputBuffer = new short[maxRequired * numChannels]; + outputBufferSize = maxRequired; + outputBuffer = new short[maxRequired * numChannels]; + pitchBufferSize = maxRequired; + pitchBuffer = new short[maxRequired * numChannels]; + downSampleBuffer = new short[maxRequired]; + oldRatePosition = 0; + newRatePosition = 0; + prevPeriod = 0; + } + + // Get the sample rate of the stream. + public int getSampleRate() { + return sampleRate; + } + + // Get the number of channels. + public int getNumChannels() { + return numChannels; + } + + // Enlarge the output buffer if needed. + private void enlargeOutputBufferIfNeeded( + int numSamples + ) { + if (numOutputSamples + numSamples > outputBufferSize) { + outputBufferSize += (outputBufferSize >> 1) + numSamples; + outputBuffer = resize(outputBuffer, outputBufferSize); + } + } + + // Enlarge the input buffer if needed. + private void enlargeInputBufferIfNeeded( + int numSamples + ) { + if (numInputSamples + numSamples > inputBufferSize) { + inputBufferSize += (inputBufferSize >> 1) + numSamples; + inputBuffer = resize(inputBuffer, inputBufferSize); + } + } + + // Add the input samples to the input buffer. + private void addFloatSamplesToInputBuffer( + float[] samples, + int numSamples + ) { + if (numSamples == 0) { + return; + } + enlargeInputBufferIfNeeded(numSamples); + int xBuffer = numInputSamples * numChannels; + for (int xSample = 0; xSample < numSamples * numChannels; xSample++) { + inputBuffer[xBuffer++] = (short) (samples[xSample] * 32767.0f); + } + numInputSamples += numSamples; + } + + // Add the input samples to the input buffer. + private void addShortSamplesToInputBuffer( + short[] samples, + int numSamples + ) { + if (numSamples == 0) { + return; + } + enlargeInputBufferIfNeeded(numSamples); + move(inputBuffer, numInputSamples, samples, 0, numSamples); + numInputSamples += numSamples; + } + + // Add the input samples to the input buffer. + private void addUnsignedByteSamplesToInputBuffer( + byte[] samples, + int numSamples + ) { + short sample; + + enlargeInputBufferIfNeeded(numSamples); + int xBuffer = numInputSamples * numChannels; + for (int xSample = 0; xSample < numSamples * numChannels; xSample++) { + sample = (short) ((samples[xSample] & 0xff) - 128); // Convert from unsigned to signed + inputBuffer[xBuffer++] = (short) (sample << 8); + } + numInputSamples += numSamples; + } + + // Add the input samples to the input buffer. They must be 16-bit little-endian encoded in a byte array. + private void addBytesToInputBuffer( + byte[] inBuffer, + int numBytes + ) { + int numSamples = numBytes / (2 * numChannels); + short sample; + + enlargeInputBufferIfNeeded(numSamples); + int xBuffer = numInputSamples * numChannels; + for (int xByte = 0; xByte + 1 < numBytes; xByte += 2) { + sample = (short) ((inBuffer[xByte] & 0xff) | (inBuffer[xByte + 1] << 8)); + inputBuffer[xBuffer++] = sample; + } + numInputSamples += numSamples; + } + + // Remove input samples that we have already processed. + private void removeInputSamples( + int position + ) { + int remainingSamples = numInputSamples - position; + + move(inputBuffer, 0, inputBuffer, position, remainingSamples); + numInputSamples = remainingSamples; + } + + // Just copy from the array to the output buffer + private void copyToOutput( + short[] samples, + int position, + int numSamples + ) { + enlargeOutputBufferIfNeeded(numSamples); + move(outputBuffer, numOutputSamples, samples, position, numSamples); + numOutputSamples += numSamples; + } + + // Just copy from the input buffer to the output buffer. Return num samples copied. + private int copyInputToOutput( + int position + ) { + int numSamples = Math.min(maxRequired, remainingInputToCopy); + + copyToOutput(inputBuffer, position, numSamples); + remainingInputToCopy -= numSamples; + return numSamples; + } + + // Read data out of the stream. Sometimes no data will be available, and zero + // is returned, which is not an error condition. + private int readFloatFromStream( + float[] samples, + int maxSamples + ) { + int numSamples = numOutputSamples; + int remainingSamples = 0; + + if (numSamples == 0) { + return 0; + } + if (numSamples > maxSamples) { + remainingSamples = numSamples - maxSamples; + numSamples = maxSamples; + } + for (int xSample = 0; xSample < numSamples * numChannels; xSample++) { + samples[xSample] = (outputBuffer[xSample]) / 32767.0f; + } + move(outputBuffer, 0, outputBuffer, numSamples, remainingSamples); + numOutputSamples = remainingSamples; + return numSamples; + } + + // Read short data out of the stream. Sometimes no data will be available, and zero + // is returned, which is not an error condition. + public int readShortFromStream( + short[] samples, + int maxSamples + ) { + int numSamples = numOutputSamples; + int remainingSamples = 0; + + if (numSamples == 0) { + return 0; + } + if (numSamples > maxSamples) { + remainingSamples = numSamples - maxSamples; + numSamples = maxSamples; + } + move(samples, 0, outputBuffer, 0, numSamples); + move(outputBuffer, 0, outputBuffer, numSamples, remainingSamples); + numOutputSamples = remainingSamples; + return numSamples; + } + + // Read unsigned byte data out of the stream. Sometimes no data will be available, and zero + // is returned, which is not an error condition. + private int readUnsignedByteFromStream( + byte[] samples, + int maxSamples + ) { + int numSamples = numOutputSamples; + int remainingSamples = 0; + + if (numSamples == 0) { + return 0; + } + if (numSamples > maxSamples) { + remainingSamples = numSamples - maxSamples; + numSamples = maxSamples; + } + for (int xSample = 0; xSample < numSamples * numChannels; xSample++) { + samples[xSample] = (byte) ((outputBuffer[xSample] >> 8) + 128); + } + move(outputBuffer, 0, outputBuffer, numSamples, remainingSamples); + numOutputSamples = remainingSamples; + return numSamples; + } + + // Read unsigned byte data out of the stream. Sometimes no data will be available, and zero + // is returned, which is not an error condition. + private int readBytesFromStream( + byte[] outBuffer, + int maxBytes + ) { + int maxSamples = maxBytes / (2 * numChannels); + int numSamples = numOutputSamples; + int remainingSamples = 0; + + if (numSamples == 0 || maxSamples == 0) { + return 0; + } + if (numSamples > maxSamples) { + remainingSamples = numSamples - maxSamples; + numSamples = maxSamples; + } + for (int xSample = 0; xSample < numSamples * numChannels; xSample++) { + short sample = outputBuffer[xSample]; + outBuffer[xSample << 1] = (byte) (sample & 0xff); + outBuffer[(xSample << 1) + 1] = (byte) (sample >> 8); + } + move(outputBuffer, 0, outputBuffer, numSamples, remainingSamples); + numOutputSamples = remainingSamples; + return 2 * numSamples * numChannels; + } + + // Force the sonic stream to generate output using whatever data it currently + // has. No extra delay will be added to the output, but flushing in the middle of + // words could introduce distortion. + public void flushStream() { + int remainingSamples = numInputSamples; + float s = speed / pitch; + float r = rate * pitch; + int expectedOutputSamples = numOutputSamples + (int) ((remainingSamples / s + numPitchSamples) / r + 0.5f); + +// // Add enough silence to flush both input and pitch buffers. + enlargeInputBufferIfNeeded(remainingSamples + 2 * maxRequired); + for (int xSample = 0; xSample < 2 * maxRequired * numChannels; xSample++) { + inputBuffer[remainingSamples * numChannels + xSample] = 0; + } + numInputSamples += 2 * maxRequired; + writeShortToStream(null, 0); + // Throw away any extra samples we generated due to the silence we added. + if (numOutputSamples > expectedOutputSamples) { + numOutputSamples = expectedOutputSamples; + } + // Empty input and pitch buffers. + numInputSamples = 0; + remainingInputToCopy = 0; + numPitchSamples = 0; + } + + // Return the number of samples in the output buffer + public int samplesAvailable() { + return numOutputSamples; + } + + // If skip is greater than one, average skip samples together and write them to + // the down-sample buffer. If numChannels is greater than one, mix the channels + // together as we down sample. + private void downSampleInput( + short[] samples, + int position, + int skip + ) { + int numSamples = maxRequired / skip; + int samplesPerValue = numChannels * skip; + int value; + + position *= numChannels; + for (int i = 0; i < numSamples; i++) { + value = 0; + for (int j = 0; j < samplesPerValue; j++) { + value += samples[position + i * samplesPerValue + j]; + } + value /= samplesPerValue; + downSampleBuffer[i] = (short) value; + } + } + + // Find the best frequency match in the range, and given a sample skip multiple. + // For now, just find the pitch of the first channel. + private int findPitchPeriodInRange( + short[] samples, + int position, + int minPeriod, + int maxPeriod + ) { + int bestPeriod = 0; + int worstPeriod = 255; + int minDiff = 1; + int maxDiff = 0; + + position *= numChannels; + for (int period = minPeriod; period <= maxPeriod; period++) { + int diff = 0; + for (int i = 0; i < period; i++) { + short sVal = samples[position + i]; + short pVal = samples[position + period + i]; + diff += (sVal >= pVal) ? sVal - pVal : pVal - sVal; + } + /* Note that the highest number of samples we add into diff will be less + than 256, since we skip samples. Thus, diff is a 24 bit number, and + we can safely multiply by numSamples without overflow */ + if (diff * bestPeriod < minDiff * period) { + minDiff = diff; + bestPeriod = period; + } + if (diff * worstPeriod > maxDiff * period) { + maxDiff = diff; + worstPeriod = period; + } + } + this.minDiff = minDiff / bestPeriod; + this.maxDiff = maxDiff / worstPeriod; + + return bestPeriod; + } + + // At abrupt ends of voiced words, we can have pitch periods that are better + // approximated by the previous pitch period estimate. Try to detect this case. + private boolean prevPeriodBetter( + int minDiff, + int maxDiff, + boolean preferNewPeriod + ) { + if (minDiff == 0 || prevPeriod == 0) { + return false; + } + if (preferNewPeriod) { + if (maxDiff > minDiff * 3) { + // Got a reasonable match this period + return false; + } + if (minDiff * 2 <= prevMinDiff * 3) { + // Mismatch is not that much greater this period + return false; + } + } else { + if (minDiff <= prevMinDiff) { + return false; + } + } + return true; + } + + // Find the pitch period. This is a critical step, and we may have to try + // multiple ways to get a good answer. This version uses AMDF. To improve + // speed, we down sample by an integer factor get in the 11KHz range, and then + // do it again with a narrower frequency range without down sampling + private int findPitchPeriod( + short[] samples, + int position, + boolean preferNewPeriod + ) { + int period, retPeriod; + int skip = 1; + + if (sampleRate > SONIC_AMDF_FREQ && quality == 0) { + skip = sampleRate / SONIC_AMDF_FREQ; + } + if (numChannels == 1 && skip == 1) { + period = findPitchPeriodInRange(samples, position, minPeriod, maxPeriod); + } else { + downSampleInput(samples, position, skip); + period = findPitchPeriodInRange(downSampleBuffer, 0, minPeriod / skip, + maxPeriod / skip); + if (skip != 1) { + period *= skip; + int minP = period - (skip << 2); + int maxP = period + (skip << 2); + if (minP < minPeriod) { + minP = minPeriod; + } + if (maxP > maxPeriod) { + maxP = maxPeriod; + } + if (numChannels == 1) { + period = findPitchPeriodInRange(samples, position, minP, maxP); + } else { + downSampleInput(samples, position, 1); + period = findPitchPeriodInRange(downSampleBuffer, 0, minP, maxP); + } + } + } + if (prevPeriodBetter(minDiff, maxDiff, preferNewPeriod)) { + retPeriod = prevPeriod; + } else { + retPeriod = period; + } + prevMinDiff = minDiff; + prevPeriod = period; + return retPeriod; + } + + // Overlap two sound segments, ramp the volume of one down, while ramping the + // other one from zero up, and add them, storing the result at the output. + private static void overlapAdd( + int frameCount, + int channelCount, + short[] out, + int outPosition, + short[] rampDown, + int rampDownPosition, + short[] rampUp, + int rampUpPosition) { + for (int i = 0; i < channelCount; i++) { + int o = outPosition * channelCount + i; + int u = rampUpPosition * channelCount + i; + int d = rampDownPosition * channelCount + i; + for (int t = 0; t < frameCount; t++) { + out[o] = (short) ((rampDown[d] * (frameCount - t) + rampUp[u] * t) / frameCount); + o += channelCount; + d += channelCount; + u += channelCount; + } + } + } + + // Overlap two sound segments, ramp the volume of one down, while ramping the + // other one from zero up, and add them, storing the result at the output. + private void overlapAddWithSeparation( + int numSamples, + int numChannels, + int separation, + short[] out, + int outPos, + short[] rampDown, + int rampDownPos, + short[] rampUp, + int rampUpPos + ) { + for (int i = 0; i < numChannels; i++) { + int o = outPos * numChannels + i; + int u = rampUpPos * numChannels + i; + int d = rampDownPos * numChannels + i; + + for (int t = 0; t < numSamples + separation; t++) { + if (t < separation) { + out[o] = (short) (rampDown[d] * (numSamples - t) / numSamples); + d += numChannels; + } else if (t < numSamples) { + out[o] = (short) ((rampDown[d] * (numSamples - t) + rampUp[u] * (t - separation)) / numSamples); + d += numChannels; + u += numChannels; + } else { + out[o] = (short) (rampUp[u] * (t - separation) / numSamples); + u += numChannels; + } + o += numChannels; + } + } + } + + // Just move the new samples in the output buffer to the pitch buffer + private void moveNewSamplesToPitchBuffer( + int originalNumOutputSamples + ) { + int numSamples = numOutputSamples - originalNumOutputSamples; + + if (numPitchSamples + numSamples > pitchBufferSize) { + pitchBufferSize += (pitchBufferSize >> 1) + numSamples; + pitchBuffer = resize(pitchBuffer, pitchBufferSize); + } + move(pitchBuffer, numPitchSamples, outputBuffer, originalNumOutputSamples, numSamples); + numOutputSamples = originalNumOutputSamples; + numPitchSamples += numSamples; + } + + // Remove processed samples from the pitch buffer. + private void removePitchSamples( + int numSamples + ) { + if (numSamples == 0) { + return; + } + move(pitchBuffer, 0, pitchBuffer, numSamples, numPitchSamples - numSamples); + numPitchSamples -= numSamples; + } + + // Change the pitch. The latency this introduces could be reduced by looking at + // past samples to determine pitch, rather than future. + private void adjustPitch( + int originalNumOutputSamples + ) { + int period, newPeriod, separation; + int position = 0; + + if (numOutputSamples == originalNumOutputSamples) { + return; + } + moveNewSamplesToPitchBuffer(originalNumOutputSamples); + while (numPitchSamples - position >= maxRequired) { + period = findPitchPeriod(pitchBuffer, position, false); + newPeriod = (int) (period / pitch); + enlargeOutputBufferIfNeeded(newPeriod); + if (pitch >= 1.0f) { + overlapAdd(newPeriod, numChannels, outputBuffer, numOutputSamples, pitchBuffer, + position, pitchBuffer, position + period - newPeriod); + } else { + separation = newPeriod - period; + Log.d("audio r", "adjustPitch: "); + overlapAddWithSeparation(period, numChannels, separation, outputBuffer, numOutputSamples, + pitchBuffer, position, pitchBuffer, position); + } + numOutputSamples += newPeriod; + position += period; + } + removePitchSamples(position); + } + + + // Return 1 if value >= 0, else -1. This represents the sign of value. + private int getSign(int value) { + return value >= 0 ? 1 : -1; + } + + // Interpolate the new output sample. + private short interpolate( + short[] in, + int inPos, // Index to first sample which already includes channel offset. + int oldSampleRate, + int newSampleRate + ) { + short left = in[inPos]; + short right = in[inPos + numChannels]; + int position = newRatePosition * oldSampleRate; + int leftPosition = oldRatePosition * newSampleRate; + int rightPosition = (oldRatePosition + 1) * newSampleRate; + int ratio = rightPosition - position; + int width = rightPosition - leftPosition; + return (short) ((ratio * left + (width - ratio) * right) / width); + } + + // Change the rate. + private void adjustRate( + float rate, + int originalNumOutputSamples + ) { + if (numOutputSamples == originalNumOutputSamples) { + return; + } + + int newSampleRate = (int) (sampleRate / rate); + int oldSampleRate = sampleRate; + int position; + + // Set these values to help with the integer math + while (newSampleRate > (1 << 14) || oldSampleRate > (1 << 14)) { + newSampleRate >>= 1; + oldSampleRate >>= 1; + } + + moveNewSamplesToPitchBuffer(originalNumOutputSamples); + // Leave at least one pitch sample in the buffer + for (position = 0; position < numPitchSamples - 1; position++) { + while ((oldRatePosition + 1) * newSampleRate > newRatePosition * oldSampleRate) { + enlargeOutputBufferIfNeeded(1); + for (int i = 0; i < numChannels; i++) { + outputBuffer[numOutputSamples * numChannels + i] = interpolate(pitchBuffer, + position * numChannels + i, oldSampleRate, newSampleRate); + } + newRatePosition++; + numOutputSamples++; + } + oldRatePosition++; + if (oldRatePosition == oldSampleRate) { + oldRatePosition = 0; + if (newRatePosition != newSampleRate) { + System.out.printf("Assertion failed: newRatePosition != newSampleRate\n"); + assert false; + } + newRatePosition = 0; + } + } + removePitchSamples(numPitchSamples - 1); + } + + + // Skip over a pitch period, and copy period/speed samples to the output + private int skipPitchPeriod( + short[] samples, + int position, + float speed, + int period + ) { + int newSamples; + + if (speed >= 2.0f) { + newSamples = (int) (period / (speed - 1.0f)); + } else { + newSamples = period; + remainingInputToCopy = (int) (period * (2.0f - speed) / (speed - 1.0f)); + } + enlargeOutputBufferIfNeeded(newSamples); + overlapAdd(newSamples, numChannels, outputBuffer, numOutputSamples, samples, position, + samples, position + period); + numOutputSamples += newSamples; + return newSamples; + } + + // Insert a pitch period, and determine how much input to copy directly. + private int insertPitchPeriod( + short[] samples, + int position, + float speed, + int period + ) { + int newSamples; + + if (speed < 0.5f) { + newSamples = (int) (period * speed / (1.0f - speed)); + } else { + newSamples = period; + remainingInputToCopy = (int) (period * (2.0f * speed - 1.0f) / (1.0f - speed)); + } + enlargeOutputBufferIfNeeded(period + newSamples); + move(outputBuffer, numOutputSamples, samples, position, period); + overlapAdd(newSamples, numChannels, outputBuffer, numOutputSamples + period, samples, + position + period, samples, position); + numOutputSamples += period + newSamples; + return newSamples; + } + + // Resample as many pitch periods as we have buffered on the input. Return 0 if + // we fail to resize an input or output buffer. Also scale the output by the volume. + private void changeSpeed(float speed) { + if (numInputSamples < maxRequired) { + return; + } + int numSamples = numInputSamples; + int position = 0; + do { + if (remainingInputToCopy > 0) { + position += copyInputToOutput(position); + } else { + int period = findPitchPeriod(inputBuffer, position, true); + if (speed > 1.0) { + position += period + skipPitchPeriod(inputBuffer, position, speed, period); + } else { + position += insertPitchPeriod(inputBuffer, position, speed, period); + } + } + } while (position + maxRequired <= numSamples); + + removeInputSamples(position); + } + + // Resample as many pitch periods as we have buffered on the input. Scale the output by the volume. + private void processStreamInput() { + int originalNumOutputSamples = numOutputSamples; + float s = speed / pitch; + float r = rate; + + if (!useChordPitch) { + r *= pitch; + } + if (s > 1.00001 || s < 0.99999) { + changeSpeed(s); + } else { + copyToOutput(inputBuffer, 0, numInputSamples); + numInputSamples = 0; + } + if (useChordPitch) { + if (pitch != 1.0f) { + adjustPitch(originalNumOutputSamples); + } + } else if (r != 1.0f) { + adjustRate(r, originalNumOutputSamples); + } + if (volume != 1.0f) { + // Adjust output volume. + scaleSamples(outputBuffer, originalNumOutputSamples, numOutputSamples - originalNumOutputSamples, + volume); + } + } + + // Write floating point data to the input buffer and process it. + public void writeFloatToStream( + float[] samples, + int numSamples + ) { + addFloatSamplesToInputBuffer(samples, numSamples); + processStreamInput(); + } + + // Write the data to the input stream, and process it. + public void writeShortToStream( + short[] samples, + int numSamples + ) { + addShortSamplesToInputBuffer(samples, numSamples); + processStreamInput(); + } + + // Simple wrapper around sonicWriteFloatToStream that does the unsigned byte to short + // conversion for you. + private void writeUnsignedByteToStream( + byte[] samples, + int numSamples + ) { + addUnsignedByteSamplesToInputBuffer(samples, numSamples); + processStreamInput(); + } + + // Simple wrapper around sonicWriteBytesToStream that does the byte to 16-bit LE conversion. + private void writeBytesToStream( + byte[] inBuffer, + int numBytes + ) { + addBytesToInputBuffer(inBuffer, numBytes); + processStreamInput(); + } + + // This is a non-stream oriented interface to just change the speed of a sound sample + private static int changeFloatSpeed( + float[] samples, + int numSamples, + float speed, + float pitch, + float rate, + float volume, + boolean useChordPitch, + int sampleRate, + int numChannels + ) { + SonicAudioProcessor stream = new SonicAudioProcessor(sampleRate, numChannels); + + stream.setSpeed(speed); + stream.setPitch(pitch); + stream.setRate(rate); + stream.setVolume(volume); + stream.setChordPitch(useChordPitch); + stream.writeFloatToStream(samples, numSamples); + stream.flushStream(); + numSamples = stream.samplesAvailable(); + stream.readFloatFromStream(samples, numSamples); + return numSamples; + } + + /* This is a non-stream oriented interface to just change the speed of a sound sample */ + private int sonicChangeShortSpeed( + short[] samples, + int numSamples, + float speed, + float pitch, + float rate, + float volume, + boolean useChordPitch, + int sampleRate, + int numChannels + ) { + SonicAudioProcessor stream = new SonicAudioProcessor(sampleRate, numChannels); + + stream.setSpeed(speed); + stream.setPitch(pitch); + stream.setRate(rate); + stream.setVolume(volume); + stream.setChordPitch(useChordPitch); + stream.writeShortToStream(samples, numSamples); + stream.flushStream(); + numSamples = stream.samplesAvailable(); + stream.readShortFromStream(samples, numSamples); + return numSamples; + } +} diff --git a/mp4compose/src/main/java/com/daasuu/mp4compose/composer/VideoComposer.kt b/mp4compose/src/main/java/com/daasuu/mp4compose/composer/VideoComposer.kt index e6dda635..5459b4cb 100755 --- a/mp4compose/src/main/java/com/daasuu/mp4compose/composer/VideoComposer.kt +++ b/mp4compose/src/main/java/com/daasuu/mp4compose/composer/VideoComposer.kt @@ -36,6 +36,9 @@ internal class VideoComposer { private var encoderStarted: Boolean = false var writtenPresentationTimeUs: Long = 0 private set + // TODO: currently we do not use the timeScale feature. Also the timeScale ends up + // being converted into an int in here being a float in upper layers. + // See https://github.com/Automattic/stories-android/issues/685 for more context. private val timeScale: Int private var useStaticBkg: Boolean = false private var addedFrameCount = 0 diff --git a/mp4compose/src/main/java/com/daasuu/mp4compose/logger/AndroidLogger.java b/mp4compose/src/main/java/com/daasuu/mp4compose/logger/AndroidLogger.java new file mode 100644 index 00000000..3934114d --- /dev/null +++ b/mp4compose/src/main/java/com/daasuu/mp4compose/logger/AndroidLogger.java @@ -0,0 +1,23 @@ +package com.daasuu.mp4compose.logger; + +import android.util.Log; + +/** + * The default implementation of the {@link Logger} for Android. + */ +public class AndroidLogger implements Logger { + @Override + public void debug(String tag, String message) { + Log.d(tag, message); + } + + @Override + public void error(String tag, String message, Throwable error) { + Log.e(tag, message, error); + } + + @Override + public void warning(String tag, String message) { + Log.w(tag, message); + } +} diff --git a/mp4compose/src/main/java/com/daasuu/mp4compose/logger/Logger.java b/mp4compose/src/main/java/com/daasuu/mp4compose/logger/Logger.java new file mode 100644 index 00000000..3d7f2579 --- /dev/null +++ b/mp4compose/src/main/java/com/daasuu/mp4compose/logger/Logger.java @@ -0,0 +1,32 @@ +package com.daasuu.mp4compose.logger; + +/** + * The logger interface used to log information to the console. + */ + +public interface Logger { + /** + * Logs a debug message. + * + * @param tag The tag of the message. + * @param message The message body. + */ + void debug(String tag, String message); + + /** + * Logs an error message. + * + * @param tag The tag of the message. + * @param message The message body. + * @param error The cause of the error. + */ + void error(String tag, String message, Throwable error); + + /** + * Logs a warning message. + * + * @param tag The tag of the message. + * @param message The message body. + */ + void warning(String tag, String message); +} diff --git a/mp4compose/src/main/java/com/daasuu/mp4compose/source/DataSource.java b/mp4compose/src/main/java/com/daasuu/mp4compose/source/DataSource.java new file mode 100644 index 00000000..a2bad791 --- /dev/null +++ b/mp4compose/src/main/java/com/daasuu/mp4compose/source/DataSource.java @@ -0,0 +1,14 @@ +package com.daasuu.mp4compose.source; + +import androidx.annotation.NonNull; + +import java.io.FileDescriptor; + +public interface DataSource { + @NonNull + FileDescriptor getFileDescriptor(); + + interface Listener { + void onError(Exception e); + } +} diff --git a/mp4compose/src/main/java/com/daasuu/mp4compose/source/FilePathDataSource.java b/mp4compose/src/main/java/com/daasuu/mp4compose/source/FilePathDataSource.java new file mode 100644 index 00000000..43c7c4e0 --- /dev/null +++ b/mp4compose/src/main/java/com/daasuu/mp4compose/source/FilePathDataSource.java @@ -0,0 +1,43 @@ +package com.daasuu.mp4compose.source; + +import androidx.annotation.NonNull; + +import com.daasuu.mp4compose.logger.Logger; + +import java.io.File; +import java.io.FileDescriptor; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; + +@SuppressWarnings("MemberName") +public class FilePathDataSource implements DataSource { + private static final String TAG = FilePathDataSource.class.getSimpleName(); + + private FileDescriptor fileDescriptor; + + public FilePathDataSource(@NonNull String filePath, @NonNull Logger logger, @NonNull Listener listener) { + final File srcFile = new File(filePath); + final FileInputStream fileInputStream; + try { + fileInputStream = new FileInputStream(srcFile); + } catch (FileNotFoundException e) { + logger.error(TAG, "Unable to find file", e); + listener.onError(e); + return; + } + + try { + fileDescriptor = fileInputStream.getFD(); + } catch (IOException e) { + logger.error(TAG, "Unable to read input file", e); + listener.onError(e); + } + } + + @NonNull + @Override + public FileDescriptor getFileDescriptor() { + return fileDescriptor; + } +} diff --git a/photoeditor/src/main/java/com/automattic/photoeditor/PhotoEditor.kt b/photoeditor/src/main/java/com/automattic/photoeditor/PhotoEditor.kt index 3a551a80..064aa9fe 100644 --- a/photoeditor/src/main/java/com/automattic/photoeditor/PhotoEditor.kt +++ b/photoeditor/src/main/java/com/automattic/photoeditor/PhotoEditor.kt @@ -47,7 +47,10 @@ import com.automattic.photoeditor.views.filter.CustomEffect import com.automattic.photoeditor.views.filter.PhotoFilter import com.bumptech.glide.Glide import com.daasuu.mp4compose.FillMode -import com.daasuu.mp4compose.composer.Mp4Composer +import com.daasuu.mp4compose.composer.ComposerProvider +import com.daasuu.mp4compose.composer.ComposerUseCase.SaveVideoAsFile +import com.daasuu.mp4compose.composer.ComposerUseCase.SaveVideoFromBgAsFile +import com.daasuu.mp4compose.composer.Listener import com.daasuu.mp4compose.filter.GlFilter import com.daasuu.mp4compose.filter.GlFilterGroup import com.daasuu.mp4compose.filter.GlGifWatermarkFilter @@ -801,9 +804,16 @@ class PhotoEditor private constructor(builder: Builder) : } } - Mp4Composer(videoInputPath, videoOutputPath) - .with(context) - .addedHeaders(authenticationHeadersInterface?.getAuthHeaders(videoInputPath.toString())) + val composer = ComposerProvider.getComposerForUseCase( + SaveVideoAsFile( + videoInputPath, + videoOutputPath, + context, + authenticationHeadersInterface?.getAuthHeaders(videoInputPath.toString()) + ) + ) + + composer // .size(width, height) // IMPORTANT: as we aim at a WYSIWYG UX, we need to produce a video of size equal to that of the phone // screen, given the user may be seeing a letterbox landscape video and placing emoji / text around @@ -814,7 +824,7 @@ class PhotoEditor private constructor(builder: Builder) : .fillMode(FillMode.PRESERVE_ASPECT_FIT) .filter(if (customAddedViews.isNotEmpty()) GlFilterGroup(filterCollection) else null) .mute(muteAudio) - .listener(object : Mp4Composer.Listener { + .listener(object : Listener { override fun onProgress(progress: Double) { Log.d(TAG, "onProgress = $progress") onSaveListener.onProgress(progress) @@ -834,6 +844,10 @@ class PhotoEditor private constructor(builder: Builder) : Log.e(TAG, "onFailed()", exception) onSaveListener.onFailure(exception) } + + override fun onStart() { + Log.d(TAG, "onStart()") + } }) .start() } @@ -960,11 +974,13 @@ class PhotoEditor private constructor(builder: Builder) : // take the static background image val bmp = createBitmapFromView(parentView.source) - Mp4Composer(bmp, videoOutputPath) - .size(bmp.width, bmp.height) // FIXME check whether these are the right values or not + val composer = ComposerProvider.getComposerForUseCase(SaveVideoFromBgAsFile(bmp, videoOutputPath)) + + composer + .size(Size(bmp.width, bmp.height)) // FIXME check whether these are the right values or not .fillMode(FillMode.PRESERVE_ASPECT_FIT) .filter(GlFilterGroup(filterCollection)) - .listener(object : Mp4Composer.Listener { + .listener(object : Listener { override fun onProgress(progress: Double) { Log.d(TAG, "onProgress = $progress") // TODO: show progress to user @@ -984,8 +1000,11 @@ class PhotoEditor private constructor(builder: Builder) : Log.e(TAG, "onFailed()", exception) onSaveListener.onFailure(exception) } - }) - .start() + + override fun onStart() { + Log.d(TAG, "onStart()") + } + }).start() } // TODO to be used in conjunction with mp4composer