From 1b030bf1f499f44f7b4f3c3ade87387be07f787d Mon Sep 17 00:00:00 2001 From: develric Date: Sun, 14 Mar 2021 23:45:14 +0100 Subject: [PATCH 01/12] Restoring a few files. --- .../mp4compose/VideoFormatMimeType.java | 21 + .../composer/AudioChannelWithSP.java | 225 ++++ .../mp4compose/composer/BaseAudioChannel.java | 76 ++ .../composer/SonicAudioProcessor.java | 990 ++++++++++++++++++ .../mp4compose/logger/AndroidLogger.java | 25 + .../com/daasuu/mp4compose/logger/Logger.java | 33 + .../daasuu/mp4compose/source/DataSource.java | 14 + .../mp4compose/source/FilePathDataSource.java | 44 + 8 files changed, 1428 insertions(+) create mode 100644 mp4compose/src/main/java/com/daasuu/mp4compose/VideoFormatMimeType.java create mode 100644 mp4compose/src/main/java/com/daasuu/mp4compose/composer/AudioChannelWithSP.java create mode 100644 mp4compose/src/main/java/com/daasuu/mp4compose/composer/BaseAudioChannel.java create mode 100644 mp4compose/src/main/java/com/daasuu/mp4compose/composer/SonicAudioProcessor.java create mode 100644 mp4compose/src/main/java/com/daasuu/mp4compose/logger/AndroidLogger.java create mode 100644 mp4compose/src/main/java/com/daasuu/mp4compose/logger/Logger.java create mode 100644 mp4compose/src/main/java/com/daasuu/mp4compose/source/DataSource.java create mode 100644 mp4compose/src/main/java/com/daasuu/mp4compose/source/FilePathDataSource.java diff --git a/mp4compose/src/main/java/com/daasuu/mp4compose/VideoFormatMimeType.java b/mp4compose/src/main/java/com/daasuu/mp4compose/VideoFormatMimeType.java new file mode 100644 index 000000000..52860af9d --- /dev/null +++ b/mp4compose/src/main/java/com/daasuu/mp4compose/VideoFormatMimeType.java @@ -0,0 +1,21 @@ +package com.daasuu.mp4compose; + +import android.media.MediaFormat; + +public enum VideoFormatMimeType { + HEVC(MediaFormat.MIMETYPE_VIDEO_HEVC), + AVC(MediaFormat.MIMETYPE_VIDEO_AVC), + MPEG4(MediaFormat.MIMETYPE_VIDEO_MPEG4), + H263(MediaFormat.MIMETYPE_VIDEO_H263), + AUTO(""); + + private final String videoFormatMimeType; + + VideoFormatMimeType(String videoFormatMimeType) { + this.videoFormatMimeType = videoFormatMimeType; + } + + public String getFormat() { + return videoFormatMimeType; + } +} diff --git a/mp4compose/src/main/java/com/daasuu/mp4compose/composer/AudioChannelWithSP.java b/mp4compose/src/main/java/com/daasuu/mp4compose/composer/AudioChannelWithSP.java new file mode 100644 index 000000000..0e99432ba --- /dev/null +++ b/mp4compose/src/main/java/com/daasuu/mp4compose/composer/AudioChannelWithSP.java @@ -0,0 +1,225 @@ +package com.daasuu.mp4compose.composer; + +import android.media.MediaCodec; +import android.media.MediaFormat; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.ShortBuffer; + +/** + * Created by TAPOS DATTA on 22,May,2020 + */ + +public class AudioChannelWithSP extends BaseAudioChannel{ + + private static final String TAG = "AUDIO_CHANNEL_WITH_SONIC"; + + private SonicAudioProcessor stream = null; // SonicAudioProcessor can deal with stereo Audio + private float timeScale = 1f; + boolean isEOF = false; + private int BUFFER_CAPACITY = 2048; // in ShortBuffer size + private long totalDataAdded = 0; + private int pendingDecoderOutputBuffIndx = -1; + private ByteBuffer tempInputBuffer = null; + private boolean isPendingFeeding = true; + private boolean isAffectInPitch; // if true the scale will impact in speed with pitch + + AudioChannelWithSP(MediaCodec decoder, MediaCodec encoder, MediaFormat encodeFormat,float timeScale, boolean isPitchChanged) { + super(decoder, encoder, encodeFormat); + this.isAffectInPitch = isPitchChanged; + this.timeScale = timeScale; + } + + @Override + public void setActualDecodedFormat(MediaFormat decodedFormat) { + super.setActualDecodedFormat(decodedFormat); + + if (inputChannelCount > 2) { + throw new UnsupportedOperationException("Input channel count (" + inputChannelCount + ") not supported."); + } + stream = new SonicAudioProcessor(inputSampleRate, outputChannelCount); + isEOF = false; + totalDataAdded = 0; + isPendingFeeding = true; + tempInputBuffer = ByteBuffer.allocateDirect(BUFFER_CAPACITY * 16).order(ByteOrder.nativeOrder()); + + if(isAffectInPitch){ + stream.setRate(timeScale); + }else { + stream.setSpeed(timeScale); + } + } + + @Override + protected long sampleCountToDurationUs(long sampleCount, int sampleRate, int channelCount) { + //considered short buffer as data + return (long) ((MICROSECS_PER_SEC * (sampleCount * 1f)/(sampleRate * 1f * channelCount))); + } + + @Override + public void drainDecoderBufferAndQueue(int bufferIndex, long presentationTimeUs) { + + if (actualDecodedFormat == null) { + throw new RuntimeException("Buffer received before format!"); + } + + final ByteBuffer data = + bufferIndex == BUFFER_INDEX_END_OF_STREAM ? + null : decoder.getOutputBuffer(bufferIndex); + + if (data != null) { + writeToSonicSteam(data.asShortBuffer()); + pendingDecoderOutputBuffIndx = bufferIndex; + isEOF = false; + decoder.releaseOutputBuffer(bufferIndex, false); + } else { + stream.flushStream(); + isEOF = true; + } + } + + @Override + public boolean feedEncoder(long timeoutUs) { + + if (stream == null || !isPendingFeeding || (!isEOF && stream.samplesAvailable() == 0)) { + //no data available + + updatePendingDecoderStatus(); + + return false; + } else if (!isEOF && timeScale < 1f && stream.samplesAvailable() > 0 && (stream.samplesAvailable() * outputChannelCount) < BUFFER_CAPACITY) { + //few data remaining in stream wait for next stream data + updatePendingDecoderStatus(); + + return false; + } + + final int encoderInBuffIndex = encoder.dequeueInputBuffer(timeoutUs); + + if (encoderInBuffIndex < 0) { + // Encoder is full - Bail out + return false; + } + + boolean status = false; + if (timeScale < 1f) { + status = slowTimeBufferProcess(encoderInBuffIndex); + } else { + status = FastOrNormalTimeBufferProcess(encoderInBuffIndex); + } + + return status; + } + + private void updatePendingDecoderStatus() { + + if (pendingDecoderOutputBuffIndx != -1) { + pendingDecoderOutputBuffIndx = -1; + } + } + + private boolean FastOrNormalTimeBufferProcess(int encoderInBuffIndex) { + + int samplesNum = stream.samplesAvailable(); + + boolean status = false; + + int rawDataLen = samplesNum * outputChannelCount; + + if(rawDataLen >= BUFFER_CAPACITY){ + + return readStreamDataAndQueueToEncoder(BUFFER_CAPACITY, encoderInBuffIndex); + } + + else if (rawDataLen > 0 && rawDataLen < BUFFER_CAPACITY) { + + return readStreamDataAndQueueToEncoder(rawDataLen, encoderInBuffIndex); + + } else if (isEOF && samplesNum == 0) { + + return finalizeEncoderQueue(encoderInBuffIndex); + + } else { + + return status; + } + } + + private boolean slowTimeBufferProcess(final int encoderInBuffIndex) { + + int samplesNum = stream.samplesAvailable(); + + boolean status = false; + + int rawDataLen = samplesNum * outputChannelCount; + + if (rawDataLen >= BUFFER_CAPACITY) { + + return readStreamDataAndQueueToEncoder(BUFFER_CAPACITY, encoderInBuffIndex); + + } else if (isEOF && (rawDataLen > 0 && rawDataLen < BUFFER_CAPACITY)) { + + return readStreamDataAndQueueToEncoder(rawDataLen, encoderInBuffIndex); + + } else if (isEOF && rawDataLen == 0) { + + return finalizeEncoderQueue(encoderInBuffIndex); + + } else { + + return status; + } + } + + private boolean finalizeEncoderQueue(final int encoderInBuffIndex) { + + isPendingFeeding = false; + return queueInputBufferInEncoder(null, encoderInBuffIndex); + } + + private boolean readStreamDataAndQueueToEncoder(final int capacity, final int encoderInBuffIndex) { + + short[] rawData = new short[capacity]; + stream.readShortFromStream(rawData, (capacity / outputChannelCount)); + return queueInputBufferInEncoder(rawData, encoderInBuffIndex); + } + + private boolean queueInputBufferInEncoder(final short[] rawData, final int encoderInBuffIndex) { + + final ShortBuffer outBuffer = encoder.getInputBuffer(encoderInBuffIndex).asShortBuffer(); + + outBuffer.clear(); + if (rawData != null) { + + outBuffer.put(rawData); + totalDataAdded += rawData.length; + + long presentationTimeUs = sampleCountToDurationUs(totalDataAdded, inputSampleRate, outputChannelCount); + + encoder.queueInputBuffer(encoderInBuffIndex, 0, rawData.length * BYTES_PER_SHORT, + presentationTimeUs, 0); + return false; + } else { + encoder.queueInputBuffer(encoderInBuffIndex, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM); + return false; + } + } + + private void writeToSonicSteam(final ShortBuffer data) { + + short[] temBuff = new short[data.capacity()]; + data.get(temBuff); + data.rewind(); + stream.writeShortToStream(temBuff, temBuff.length / outputChannelCount); + } + + public boolean isAnyPendingBuffIndex() { + // allow to decoder to send data into stream (e.i. sonicprocessor) + if (pendingDecoderOutputBuffIndx != -1) { + return true; + } else { + return false; + } + } +} diff --git a/mp4compose/src/main/java/com/daasuu/mp4compose/composer/BaseAudioChannel.java b/mp4compose/src/main/java/com/daasuu/mp4compose/composer/BaseAudioChannel.java new file mode 100644 index 000000000..ed7f7435a --- /dev/null +++ b/mp4compose/src/main/java/com/daasuu/mp4compose/composer/BaseAudioChannel.java @@ -0,0 +1,76 @@ +package com.daasuu.mp4compose.composer; + +import android.media.MediaCodec; +import android.media.MediaFormat; + +import java.nio.ShortBuffer; +import java.util.ArrayDeque; +import java.util.Queue; + +/** + * Created by TAPOS DATTA on 22,May,2020 + */ + +abstract class BaseAudioChannel { + + protected static class AudioBuffer { + int bufferIndex; + long presentationTimeUs; + ShortBuffer data; + } + protected static class BufferInfo{ + long totaldata; + long presentationTimeUs; + } + + static final int BUFFER_INDEX_END_OF_STREAM = -1; + protected static final int BYTE_PER_SAMPLE = 16 / 8 ; + protected static final int BYTES_PER_SHORT = 2; + protected static final long MICROSECS_PER_SEC = 1000000; + + protected final Queue emptyBuffers = new ArrayDeque<>(); + protected final Queue filledBuffers = new ArrayDeque<>(); + + protected final MediaCodec decoder; + protected final MediaCodec encoder; + protected final MediaFormat encodeFormat; + + protected int inputSampleRate; + protected int inputChannelCount; + protected int outputChannelCount; + + protected final AudioBuffer overflowBuffer = new AudioBuffer(); + + protected MediaFormat actualDecodedFormat; + + BaseAudioChannel(final MediaCodec decoder, + final MediaCodec encoder, final MediaFormat encodeFormat) { + this.decoder = decoder; + this.encoder = encoder; + this.encodeFormat = encodeFormat; + } + + public void setActualDecodedFormat(final MediaFormat decodedFormat) { + actualDecodedFormat = decodedFormat; + + inputSampleRate = actualDecodedFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE); + if (inputSampleRate != encodeFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE)) { + throw new UnsupportedOperationException("Audio sample rate conversion not supported yet."); + } + + inputChannelCount = actualDecodedFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT); + outputChannelCount = encodeFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT); + + if (outputChannelCount != 1 && outputChannelCount != 2) { + throw new UnsupportedOperationException("Output channel count (" + outputChannelCount + ") not supported."); + } + + overflowBuffer.presentationTimeUs = 0; + } + + protected abstract long sampleCountToDurationUs(final long sampleCount, final int sampleRate, final int channelCount); + + protected abstract void drainDecoderBufferAndQueue(final int bufferIndex, final long presentationTimeUs); + + protected abstract boolean feedEncoder(long timeoutUs); +} diff --git a/mp4compose/src/main/java/com/daasuu/mp4compose/composer/SonicAudioProcessor.java b/mp4compose/src/main/java/com/daasuu/mp4compose/composer/SonicAudioProcessor.java new file mode 100644 index 000000000..2b5f7c787 --- /dev/null +++ b/mp4compose/src/main/java/com/daasuu/mp4compose/composer/SonicAudioProcessor.java @@ -0,0 +1,990 @@ +package com.daasuu.mp4compose.composer; + +import android.util.Log; + +/** + * ********************************************************* + * Sonic library + * Copyright 2010, 2011 + * Bill Cox + * This file is part of the Sonic Library. + * + * This file is licensed under the Apache 2.0 license. + * ********************************************************* + * + * Sonic audio stream processor for time/pitch stretching + * + * ref on https://github.com/waywardgeek/sonic. + * + * + */ + +class SonicAudioProcessor { + + private static final int SONIC_MIN_PITCH = 65; + private static final int SONIC_MAX_PITCH = 400; + // This is used to down-sample some inputs to improve speed + private static final int SONIC_AMDF_FREQ = 4000; + + private short inputBuffer[]; + private short outputBuffer[]; + private short pitchBuffer[]; + private short downSampleBuffer[]; + private float speed; + private float volume; + private float pitch; + private float rate; + private int oldRatePosition; + private int newRatePosition; + private boolean useChordPitch; + private int quality; + private int numChannels; + private int inputBufferSize; + private int pitchBufferSize; + private int outputBufferSize; + private int numInputSamples; + private int numOutputSamples; + private int numPitchSamples; + private int minPeriod; + private int maxPeriod; + private int maxRequired; + private int remainingInputToCopy; + private int sampleRate; + private int prevPeriod; + private int prevMinDiff; + private int minDiff; + private int maxDiff; + + // Create a sonic stream. + SonicAudioProcessor( + int sampleRate, + int numChannels) + { + allocateStreamBuffers(sampleRate, numChannels); + speed = 1.0f; + pitch = 1.0f; + volume = 1.0f; + rate = 1.0f; + oldRatePosition = 0; + newRatePosition = 0; + useChordPitch = false; + quality = 0; + } + + // Resize the array. + private short[] resize( + short[] oldArray, + int newLength) + { + newLength *= numChannels; + short[] newArray = new short[newLength]; + int length = oldArray.length <= newLength? oldArray.length : newLength; + + System.arraycopy(oldArray, 0, newArray, 0, length); + return newArray; + } + + // Move samples from one array to another. May move samples down within an array, but not up. + private void move( + short dest[], + int destPos, + short source[], + int sourcePos, + int numSamples) + { + System.arraycopy(source, sourcePos*numChannels, dest, destPos*numChannels, numSamples*numChannels); + } + + // Scale the samples by the factor. + private void scaleSamples( + short samples[], + int position, + int numSamples, + float volume) + { + int fixedPointVolume = (int)(volume*4096.0f); + int start = position*numChannels; + int stop = start + numSamples*numChannels; + + for(int xSample = start; xSample < stop; xSample++) { + int value = (samples[xSample]*fixedPointVolume) >> 12; + if(value > 32767) { + value = 32767; + } else if(value < -32767) { + value = -32767; + } + samples[xSample] = (short)value; + } + } + + // Get the speed of the stream. + public float getSpeed() + { + return speed; + } + + // Set the speed of the stream. + public void setSpeed( + float speed) + { + this.speed = speed; + } + + // Get the pitch of the stream. + public float getPitch() + { + return pitch; + } + + // Set the pitch of the stream. + public void setPitch( + float pitch) + { + this.pitch = pitch; + } + + // Get the rate of the stream. + public float getRate() + { + return rate; + } + + // Set the playback rate of the stream. This scales pitch and speed at the same time. + public void setRate( + float rate) + { + this.rate = rate; + this.oldRatePosition = 0; + this.newRatePosition = 0; + } + + // Get the vocal chord pitch setting. + public boolean getChordPitch() + { + return useChordPitch; + } + + // Set the vocal chord mode for pitch computation. Default is off. + public void setChordPitch( + boolean useChordPitch) + { + this.useChordPitch = useChordPitch; + } + + // Get the quality setting. + public int getQuality() + { + return quality; + } + + // Set the "quality". Default 0 is virtually as good as 1, but very much faster. + public void setQuality( + int quality) + { + this.quality = quality; + } + + // Get the scaling factor of the stream. + public float getVolume() + { + return volume; + } + + // Set the scaling factor of the stream. + public void setVolume( + float volume) + { + this.volume = volume; + } + + // Allocate stream buffers. + private void allocateStreamBuffers( + int sampleRate, + int numChannels) + { + this.sampleRate = sampleRate; + this.numChannels = numChannels; + minPeriod = sampleRate/SONIC_MAX_PITCH; + maxPeriod = sampleRate/SONIC_MIN_PITCH; + maxRequired = 2*maxPeriod; + inputBufferSize = maxRequired; + inputBuffer = new short[maxRequired*numChannels]; + outputBufferSize = maxRequired; + outputBuffer = new short[maxRequired*numChannels]; + pitchBufferSize = maxRequired; + pitchBuffer = new short[maxRequired*numChannels]; + downSampleBuffer = new short[maxRequired]; + oldRatePosition = 0; + newRatePosition = 0; + prevPeriod = 0; + } + + // Get the sample rate of the stream. + public int getSampleRate() + { + return sampleRate; + } + + // Get the number of channels. + public int getNumChannels() + { + return numChannels; + } + + // Enlarge the output buffer if needed. + private void enlargeOutputBufferIfNeeded( + int numSamples) + { + if(numOutputSamples + numSamples > outputBufferSize) { + outputBufferSize += (outputBufferSize >> 1) + numSamples; + outputBuffer = resize(outputBuffer, outputBufferSize); + } + } + + // Enlarge the input buffer if needed. + private void enlargeInputBufferIfNeeded( + int numSamples) + { + if(numInputSamples + numSamples > inputBufferSize) { + inputBufferSize += (inputBufferSize >> 1) + numSamples; + inputBuffer = resize(inputBuffer, inputBufferSize); + } + } + + // Add the input samples to the input buffer. + private void addFloatSamplesToInputBuffer( + float samples[], + int numSamples) + { + if(numSamples == 0) { + return; + } + enlargeInputBufferIfNeeded(numSamples); + int xBuffer = numInputSamples*numChannels; + for(int xSample = 0; xSample < numSamples*numChannels; xSample++) { + inputBuffer[xBuffer++] = (short)(samples[xSample]*32767.0f); + } + numInputSamples += numSamples; + } + + // Add the input samples to the input buffer. + private void addShortSamplesToInputBuffer( + short samples[], + int numSamples) + { + if(numSamples == 0) { + return; + } + enlargeInputBufferIfNeeded(numSamples); + move(inputBuffer, numInputSamples, samples, 0, numSamples); + numInputSamples += numSamples; + } + + // Add the input samples to the input buffer. + private void addUnsignedByteSamplesToInputBuffer( + byte samples[], + int numSamples) + { + short sample; + + enlargeInputBufferIfNeeded(numSamples); + int xBuffer = numInputSamples*numChannels; + for(int xSample = 0; xSample < numSamples*numChannels; xSample++) { + sample = (short)((samples[xSample] & 0xff) - 128); // Convert from unsigned to signed + inputBuffer[xBuffer++] = (short) (sample << 8); + } + numInputSamples += numSamples; + } + + // Add the input samples to the input buffer. They must be 16-bit little-endian encoded in a byte array. + private void addBytesToInputBuffer( + byte inBuffer[], + int numBytes) + { + int numSamples = numBytes/(2*numChannels); + short sample; + + enlargeInputBufferIfNeeded(numSamples); + int xBuffer = numInputSamples*numChannels; + for(int xByte = 0; xByte + 1 < numBytes; xByte += 2) { + sample = (short)((inBuffer[xByte] & 0xff) | (inBuffer[xByte + 1] << 8)); + inputBuffer[xBuffer++] = sample; + } + numInputSamples += numSamples; + } + + // Remove input samples that we have already processed. + private void removeInputSamples( + int position) + { + int remainingSamples = numInputSamples - position; + + move(inputBuffer, 0, inputBuffer, position, remainingSamples); + numInputSamples = remainingSamples; + } + + // Just copy from the array to the output buffer + private void copyToOutput( + short samples[], + int position, + int numSamples) + { + enlargeOutputBufferIfNeeded(numSamples); + move(outputBuffer, numOutputSamples, samples, position, numSamples); + numOutputSamples += numSamples; + } + + // Just copy from the input buffer to the output buffer. Return num samples copied. + private int copyInputToOutput( + int position) + { + int numSamples = Math.min(maxRequired,remainingInputToCopy); + + copyToOutput(inputBuffer, position, numSamples); + remainingInputToCopy -= numSamples; + return numSamples; + } + + // Read data out of the stream. Sometimes no data will be available, and zero + // is returned, which is not an error condition. + private int readFloatFromStream( + float samples[], + int maxSamples) + { + int numSamples = numOutputSamples; + int remainingSamples = 0; + + if(numSamples == 0) { + return 0; + } + if(numSamples > maxSamples) { + remainingSamples = numSamples - maxSamples; + numSamples = maxSamples; + } + for(int xSample = 0; xSample < numSamples*numChannels; xSample++) { + samples[xSample] = (outputBuffer[xSample])/32767.0f; + } + move(outputBuffer, 0, outputBuffer, numSamples, remainingSamples); + numOutputSamples = remainingSamples; + return numSamples; + } + + // Read short data out of the stream. Sometimes no data will be available, and zero + // is returned, which is not an error condition. + public int readShortFromStream( + short samples[], + int maxSamples) + { + int numSamples = numOutputSamples; + int remainingSamples = 0; + + if(numSamples == 0) { + return 0; + } + if(numSamples > maxSamples) { + remainingSamples = numSamples - maxSamples; + numSamples = maxSamples; + } + move(samples, 0, outputBuffer, 0, numSamples); + move(outputBuffer, 0, outputBuffer, numSamples, remainingSamples); + numOutputSamples = remainingSamples; + return numSamples; + } + + // Read unsigned byte data out of the stream. Sometimes no data will be available, and zero + // is returned, which is not an error condition. + private int readUnsignedByteFromStream( + byte samples[], + int maxSamples) + { + int numSamples = numOutputSamples; + int remainingSamples = 0; + + if(numSamples == 0) { + return 0; + } + if(numSamples > maxSamples) { + remainingSamples = numSamples - maxSamples; + numSamples = maxSamples; + } + for(int xSample = 0; xSample < numSamples*numChannels; xSample++) { + samples[xSample] = (byte)((outputBuffer[xSample] >> 8) + 128); + } + move(outputBuffer, 0, outputBuffer, numSamples, remainingSamples); + numOutputSamples = remainingSamples; + return numSamples; + } + + // Read unsigned byte data out of the stream. Sometimes no data will be available, and zero + // is returned, which is not an error condition. + private int readBytesFromStream( + byte outBuffer[], + int maxBytes) + { + int maxSamples = maxBytes/(2*numChannels); + int numSamples = numOutputSamples; + int remainingSamples = 0; + + if(numSamples == 0 || maxSamples == 0) { + return 0; + } + if(numSamples > maxSamples) { + remainingSamples = numSamples - maxSamples; + numSamples = maxSamples; + } + for(int xSample = 0; xSample < numSamples*numChannels; xSample++) { + short sample = outputBuffer[xSample]; + outBuffer[xSample << 1] = (byte)(sample & 0xff); + outBuffer[(xSample << 1) + 1] = (byte)(sample >> 8); + } + move(outputBuffer, 0, outputBuffer, numSamples, remainingSamples); + numOutputSamples = remainingSamples; + return 2*numSamples*numChannels; + } + + // Force the sonic stream to generate output using whatever data it currently + // has. No extra delay will be added to the output, but flushing in the middle of + // words could introduce distortion. + public void flushStream() + { + int remainingSamples = numInputSamples; + float s = speed/pitch; + float r = rate * pitch; + int expectedOutputSamples = numOutputSamples + (int)((remainingSamples / s + numPitchSamples) / r + 0.5f); + +// // Add enough silence to flush both input and pitch buffers. + enlargeInputBufferIfNeeded(remainingSamples + 2 * maxRequired); + for(int xSample = 0; xSample < 2 * maxRequired * numChannels; xSample++) { + inputBuffer[remainingSamples * numChannels + xSample] = 0; + } + numInputSamples += 2 * maxRequired; + writeShortToStream(null, 0); + // Throw away any extra samples we generated due to the silence we added. + if(numOutputSamples > expectedOutputSamples) { + numOutputSamples = expectedOutputSamples; + } + // Empty input and pitch buffers. + numInputSamples = 0; + remainingInputToCopy = 0; + numPitchSamples = 0; + } + + // Return the number of samples in the output buffer + public int samplesAvailable() + { + return numOutputSamples; + } + + // If skip is greater than one, average skip samples together and write them to + // the down-sample buffer. If numChannels is greater than one, mix the channels + // together as we down sample. + private void downSampleInput( + short samples[], + int position, + int skip) + { + int numSamples = maxRequired/skip; + int samplesPerValue = numChannels*skip; + int value; + + position *= numChannels; + for(int i = 0; i < numSamples; i++) { + value = 0; + for(int j = 0; j < samplesPerValue; j++) { + value += samples[position + i * samplesPerValue + j]; + } + value /= samplesPerValue; + downSampleBuffer[i] = (short) value; + } + } + + // Find the best frequency match in the range, and given a sample skip multiple. + // For now, just find the pitch of the first channel. + private int findPitchPeriodInRange( + short samples[], + int position, + int minPeriod, + int maxPeriod) + { + int bestPeriod = 0; + int worstPeriod = 255; + int minDiff = 1; + int maxDiff = 0; + + position *= numChannels; + for(int period = minPeriod; period <= maxPeriod; period++) { + int diff = 0; + for(int i = 0; i < period; i++) { + short sVal = samples[position + i]; + short pVal = samples[position + period + i]; + diff += (sVal >= pVal) ? sVal - pVal : pVal - sVal; + } + /* Note that the highest number of samples we add into diff will be less + than 256, since we skip samples. Thus, diff is a 24 bit number, and + we can safely multiply by numSamples without overflow */ + if(diff * bestPeriod < minDiff * period) { + minDiff = diff; + bestPeriod = period; + } + if(diff * worstPeriod > maxDiff * period) { + maxDiff = diff; + worstPeriod = period; + } + } + this.minDiff = minDiff/bestPeriod; + this.maxDiff = maxDiff/worstPeriod; + + return bestPeriod; + } + + // At abrupt ends of voiced words, we can have pitch periods that are better + // approximated by the previous pitch period estimate. Try to detect this case. + private boolean prevPeriodBetter( + int minDiff, + int maxDiff, + boolean preferNewPeriod) + { + if(minDiff == 0 || prevPeriod == 0) { + return false; + } + if(preferNewPeriod) { + if(maxDiff > minDiff * 3) { + // Got a reasonable match this period + return false; + } + if(minDiff * 2 <= prevMinDiff * 3) { + // Mismatch is not that much greater this period + return false; + } + } else { + if(minDiff <= prevMinDiff) { + return false; + } + } + return true; + } + + // Find the pitch period. This is a critical step, and we may have to try + // multiple ways to get a good answer. This version uses AMDF. To improve + // speed, we down sample by an integer factor get in the 11KHz range, and then + // do it again with a narrower frequency range without down sampling + private int findPitchPeriod( + short samples[], + int position, + boolean preferNewPeriod) + { + int period, retPeriod; + int skip = 1; + + if(sampleRate > SONIC_AMDF_FREQ && quality == 0) { + skip = sampleRate/SONIC_AMDF_FREQ; + } + if(numChannels == 1 && skip == 1) { + period = findPitchPeriodInRange(samples, position, minPeriod, maxPeriod); + } else { + downSampleInput(samples, position, skip); + period = findPitchPeriodInRange(downSampleBuffer, 0, minPeriod/skip, + maxPeriod/skip); + if(skip != 1) { + period *= skip; + int minP = period - (skip << 2); + int maxP = period + (skip << 2); + if(minP < minPeriod) { + minP = minPeriod; + } + if(maxP > maxPeriod) { + maxP = maxPeriod; + } + if(numChannels == 1) { + period = findPitchPeriodInRange(samples, position, minP, maxP); + } else { + downSampleInput(samples, position, 1); + period = findPitchPeriodInRange(downSampleBuffer, 0, minP, maxP); + } + } + } + if(prevPeriodBetter(minDiff, maxDiff, preferNewPeriod)) { + retPeriod = prevPeriod; + } else { + retPeriod = period; + } + prevMinDiff = minDiff; + prevPeriod = period; + return retPeriod; + } + + // Overlap two sound segments, ramp the volume of one down, while ramping the + // other one from zero up, and add them, storing the result at the output. + private static void overlapAdd( + int frameCount, + int channelCount, + short[] out, + int outPosition, + short[] rampDown, + int rampDownPosition, + short[] rampUp, + int rampUpPosition) { + + for (int i = 0; i < channelCount; i++) { + int o = outPosition * channelCount + i; + int u = rampUpPosition * channelCount + i; + int d = rampDownPosition * channelCount + i; + for (int t = 0; t < frameCount; t++) { + out[o] = (short) ((rampDown[d] * (frameCount - t) + rampUp[u] * t) / frameCount); + o += channelCount; + d += channelCount; + u += channelCount; + } + } + } + + // Overlap two sound segments, ramp the volume of one down, while ramping the + // other one from zero up, and add them, storing the result at the output. + private void overlapAddWithSeparation( + int numSamples, + int numChannels, + int separation, + short out[], + int outPos, + short rampDown[], + int rampDownPos, + short rampUp[], + int rampUpPos) + { + for(int i = 0; i < numChannels; i++) { + + int o = outPos * numChannels + i; + int u = rampUpPos * numChannels + i; + int d = rampDownPos * numChannels + i; + + for(int t = 0; t < numSamples + separation; t++) { + + if(t < separation) { + out[o] = (short)(rampDown[d] * (numSamples - t) / numSamples); + d += numChannels; + } else if(t < numSamples) { + out[o] = (short)((rampDown[d] * (numSamples - t) + rampUp[u] * (t - separation)) / numSamples); + d += numChannels; + u += numChannels; + } else { + out[o] = (short)(rampUp[u] * (t - separation) / numSamples); + u += numChannels; + } + o += numChannels; + } + } + } + + // Just move the new samples in the output buffer to the pitch buffer + private void moveNewSamplesToPitchBuffer( + int originalNumOutputSamples) + { + int numSamples = numOutputSamples - originalNumOutputSamples; + + if(numPitchSamples + numSamples > pitchBufferSize) { + pitchBufferSize += (pitchBufferSize >> 1) + numSamples; + pitchBuffer = resize(pitchBuffer, pitchBufferSize); + } + move(pitchBuffer, numPitchSamples, outputBuffer, originalNumOutputSamples, numSamples); + numOutputSamples = originalNumOutputSamples; + numPitchSamples += numSamples; + } + + // Remove processed samples from the pitch buffer. + private void removePitchSamples( + int numSamples) + { + if(numSamples == 0) { + return; + } + move(pitchBuffer, 0, pitchBuffer, numSamples, numPitchSamples - numSamples); + numPitchSamples -= numSamples; + } + + // Change the pitch. The latency this introduces could be reduced by looking at + // past samples to determine pitch, rather than future. + private void adjustPitch( + int originalNumOutputSamples) + { + int period, newPeriod, separation; + int position = 0; + + if(numOutputSamples == originalNumOutputSamples) { + return; + } + moveNewSamplesToPitchBuffer(originalNumOutputSamples); + while(numPitchSamples - position >= maxRequired) { + period = findPitchPeriod(pitchBuffer, position, false); + newPeriod = (int)(period/pitch); + enlargeOutputBufferIfNeeded(newPeriod); + if(pitch >= 1.0f) { + overlapAdd(newPeriod, numChannels, outputBuffer, numOutputSamples, pitchBuffer, + position, pitchBuffer, position + period - newPeriod); + } else { + separation = newPeriod - period; + Log.d("audio r", "adjustPitch: "); + overlapAddWithSeparation(period, numChannels, separation, outputBuffer, numOutputSamples, + pitchBuffer, position, pitchBuffer, position); + } + numOutputSamples += newPeriod; + position += period; + } + removePitchSamples(position); + } + + + // Return 1 if value >= 0, else -1. This represents the sign of value. + private int getSign(int value) { + return value >= 0? 1 : -1; + } + + // Interpolate the new output sample. + private short interpolate( + short in[], + int inPos, // Index to first sample which already includes channel offset. + int oldSampleRate, + int newSampleRate) + { + short left = in[inPos]; + short right = in[inPos + numChannels]; + int position = newRatePosition * oldSampleRate; + int leftPosition = oldRatePosition * newSampleRate; + int rightPosition = (oldRatePosition + 1) * newSampleRate; + int ratio = rightPosition - position; + int width = rightPosition - leftPosition; + return (short) ((ratio * left + (width - ratio) * right) / width); + } + + // Change the rate. + private void adjustRate( + float rate, + int originalNumOutputSamples) + { + if(numOutputSamples == originalNumOutputSamples) { + return; + } + + int newSampleRate = (int)(sampleRate/rate); + int oldSampleRate = sampleRate; + int position; + + // Set these values to help with the integer math + while(newSampleRate > (1 << 14) || oldSampleRate > (1 << 14)) { + newSampleRate >>= 1; + oldSampleRate >>= 1; + } + + moveNewSamplesToPitchBuffer(originalNumOutputSamples); + // Leave at least one pitch sample in the buffer + for(position = 0; position < numPitchSamples - 1; position++) { + while((oldRatePosition + 1)*newSampleRate > newRatePosition*oldSampleRate) { + enlargeOutputBufferIfNeeded(1); + for(int i = 0; i < numChannels; i++) { + outputBuffer[numOutputSamples*numChannels + i] = interpolate(pitchBuffer, + position * numChannels + i, oldSampleRate, newSampleRate); + } + newRatePosition++; + numOutputSamples++; + } + oldRatePosition++; + if(oldRatePosition == oldSampleRate) { + oldRatePosition = 0; + if(newRatePosition != newSampleRate) { + System.out.printf("Assertion failed: newRatePosition != newSampleRate\n"); + assert false; + } + newRatePosition = 0; + } + } + removePitchSamples(numPitchSamples - 1); + } + + + // Skip over a pitch period, and copy period/speed samples to the output + private int skipPitchPeriod( + short samples[], + int position, + float speed, + int period) + { + int newSamples; + + if(speed >= 2.0f) { + newSamples = (int)(period/(speed - 1.0f)); + } else { + newSamples = period; + remainingInputToCopy = (int)(period*(2.0f - speed)/(speed - 1.0f)); + } + enlargeOutputBufferIfNeeded(newSamples); + overlapAdd(newSamples, numChannels, outputBuffer, numOutputSamples, samples, position, + samples, position + period); + numOutputSamples += newSamples; + return newSamples; + } + + // Insert a pitch period, and determine how much input to copy directly. + private int insertPitchPeriod( + short samples[], + int position, + float speed, + int period) + { + int newSamples; + + if(speed < 0.5f) { + newSamples = (int)(period * speed /(1.0f - speed)); + } else { + newSamples = period; + remainingInputToCopy = (int)(period * (2.0f * speed - 1.0f) / (1.0f - speed)); + } + enlargeOutputBufferIfNeeded(period + newSamples); + move(outputBuffer, numOutputSamples, samples, position, period); + overlapAdd(newSamples, numChannels, outputBuffer, numOutputSamples + period, samples, + position + period, samples, position); + numOutputSamples += period + newSamples; + return newSamples; + } + + // Resample as many pitch periods as we have buffered on the input. Return 0 if + // we fail to resize an input or output buffer. Also scale the output by the volume. + private void changeSpeed(float speed) { + if (numInputSamples < maxRequired) { + return; + } + int numSamples = numInputSamples; + int position = 0; + do { + if (remainingInputToCopy > 0) { + position += copyInputToOutput(position); + } else { + int period = findPitchPeriod(inputBuffer, position, true); + if (speed > 1.0) { + position += period + skipPitchPeriod(inputBuffer, position, speed, period); + } else { + position += insertPitchPeriod(inputBuffer, position, speed, period); + } + } + } while (position + maxRequired <= numSamples); + + removeInputSamples(position); + } + + // Resample as many pitch periods as we have buffered on the input. Scale the output by the volume. + private void processStreamInput() + { + int originalNumOutputSamples = numOutputSamples; + float s = speed/pitch; + float r = rate; + + if(!useChordPitch) { + r *= pitch; + } + if(s > 1.00001 || s < 0.99999) { + changeSpeed(s); + } else { + copyToOutput(inputBuffer, 0, numInputSamples); + numInputSamples = 0; + } + if(useChordPitch) { + if(pitch != 1.0f) { + adjustPitch(originalNumOutputSamples); + } + } else if(r != 1.0f) { + adjustRate(r, originalNumOutputSamples); + } + if(volume != 1.0f) { + // Adjust output volume. + scaleSamples(outputBuffer, originalNumOutputSamples, numOutputSamples - originalNumOutputSamples, + volume); + } + } + + // Write floating point data to the input buffer and process it. + public void writeFloatToStream( + float samples[], + int numSamples) + { + addFloatSamplesToInputBuffer(samples, numSamples); + processStreamInput(); + } + + // Write the data to the input stream, and process it. + public void writeShortToStream( + short samples[], + int numSamples) + { + addShortSamplesToInputBuffer(samples, numSamples); + processStreamInput(); + } + + // Simple wrapper around sonicWriteFloatToStream that does the unsigned byte to short + // conversion for you. + private void writeUnsignedByteToStream( + byte samples[], + int numSamples) + { + addUnsignedByteSamplesToInputBuffer(samples, numSamples); + processStreamInput(); + } + + // Simple wrapper around sonicWriteBytesToStream that does the byte to 16-bit LE conversion. + private void writeBytesToStream( + byte inBuffer[], + int numBytes) + { + addBytesToInputBuffer(inBuffer, numBytes); + processStreamInput(); + } + + // This is a non-stream oriented interface to just change the speed of a sound sample + private static int changeFloatSpeed( + float samples[], + int numSamples, + float speed, + float pitch, + float rate, + float volume, + boolean useChordPitch, + int sampleRate, + int numChannels) + { + SonicAudioProcessor stream = new SonicAudioProcessor(sampleRate, numChannels); + + stream.setSpeed(speed); + stream.setPitch(pitch); + stream.setRate(rate); + stream.setVolume(volume); + stream.setChordPitch(useChordPitch); + stream.writeFloatToStream(samples, numSamples); + stream.flushStream(); + numSamples = stream.samplesAvailable(); + stream.readFloatFromStream(samples, numSamples); + return numSamples; + } + + /* This is a non-stream oriented interface to just change the speed of a sound sample */ + private int sonicChangeShortSpeed( + short samples[], + int numSamples, + float speed, + float pitch, + float rate, + float volume, + boolean useChordPitch, + int sampleRate, + int numChannels) + { + SonicAudioProcessor stream = new SonicAudioProcessor(sampleRate, numChannels); + + stream.setSpeed(speed); + stream.setPitch(pitch); + stream.setRate(rate); + stream.setVolume(volume); + stream.setChordPitch(useChordPitch); + stream.writeShortToStream(samples, numSamples); + stream.flushStream(); + numSamples = stream.samplesAvailable(); + stream.readShortFromStream(samples, numSamples); + return numSamples; + } +} diff --git a/mp4compose/src/main/java/com/daasuu/mp4compose/logger/AndroidLogger.java b/mp4compose/src/main/java/com/daasuu/mp4compose/logger/AndroidLogger.java new file mode 100644 index 000000000..7ed1fc8b2 --- /dev/null +++ b/mp4compose/src/main/java/com/daasuu/mp4compose/logger/AndroidLogger.java @@ -0,0 +1,25 @@ +package com.daasuu.mp4compose.logger; + +import android.util.Log; + +/** + * The default implementation of the {@link Logger} for Android. + */ +public class AndroidLogger implements Logger{ + + @Override + public void debug(String tag, String message) { + Log.d(tag, message); + } + + @Override + public void error(String tag, String message, Throwable error) { + Log.e(tag, message, error); + } + + @Override + public void warning(String tag, String message) { + Log.w(tag, message); + } + +} diff --git a/mp4compose/src/main/java/com/daasuu/mp4compose/logger/Logger.java b/mp4compose/src/main/java/com/daasuu/mp4compose/logger/Logger.java new file mode 100644 index 000000000..000a804ef --- /dev/null +++ b/mp4compose/src/main/java/com/daasuu/mp4compose/logger/Logger.java @@ -0,0 +1,33 @@ +package com.daasuu.mp4compose.logger; + +/** + * The logger interface used to log information to the console. + */ +public interface Logger { + + /** + * Logs a debug message. + * + * @param tag The tag of the message. + * @param message The message body. + */ + void debug(String tag, String message); + + /** + * Logs an error message. + * + * @param tag The tag of the message. + * @param message The message body. + * @param error The cause of the error. + */ + void error(String tag, String message, Throwable error); + + /** + * Logs a warning message. + * + * @param tag The tag of the message. + * @param message The message body. + */ + void warning(String tag, String message); + +} diff --git a/mp4compose/src/main/java/com/daasuu/mp4compose/source/DataSource.java b/mp4compose/src/main/java/com/daasuu/mp4compose/source/DataSource.java new file mode 100644 index 000000000..a2bad791f --- /dev/null +++ b/mp4compose/src/main/java/com/daasuu/mp4compose/source/DataSource.java @@ -0,0 +1,14 @@ +package com.daasuu.mp4compose.source; + +import androidx.annotation.NonNull; + +import java.io.FileDescriptor; + +public interface DataSource { + @NonNull + FileDescriptor getFileDescriptor(); + + interface Listener { + void onError(Exception e); + } +} diff --git a/mp4compose/src/main/java/com/daasuu/mp4compose/source/FilePathDataSource.java b/mp4compose/src/main/java/com/daasuu/mp4compose/source/FilePathDataSource.java new file mode 100644 index 000000000..23440de16 --- /dev/null +++ b/mp4compose/src/main/java/com/daasuu/mp4compose/source/FilePathDataSource.java @@ -0,0 +1,44 @@ +package com.daasuu.mp4compose.source; + +import androidx.annotation.NonNull; + +import com.daasuu.mp4compose.logger.Logger; + +import java.io.File; +import java.io.FileDescriptor; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; + +public class FilePathDataSource implements DataSource { + + private final static String TAG = FilePathDataSource.class.getSimpleName(); + + private FileDescriptor fileDescriptor; + + public FilePathDataSource(@NonNull String filePath, @NonNull Logger logger, @NonNull Listener listener) { + + final File srcFile = new File(filePath); + final FileInputStream fileInputStream; + try { + fileInputStream = new FileInputStream(srcFile); + } catch (FileNotFoundException e) { + logger.error(TAG, "Unable to find file", e); + listener.onError(e); + return; + } + + try { + fileDescriptor = fileInputStream.getFD(); + } catch (IOException e) { + logger.error(TAG, "Unable to read input file", e); + listener.onError(e); + } + } + + @NonNull + @Override + public FileDescriptor getFileDescriptor() { + return fileDescriptor; + } +} From 6bb2dd6908627f1c3a2bf1c81a240c27ccf84d17 Mon Sep 17 00:00:00 2001 From: develric Date: Sun, 14 Mar 2021 23:45:45 +0100 Subject: [PATCH 02/12] Extracting Listener interface --- .../daasuu/mp4compose/composer/Listener.kt | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 mp4compose/src/main/java/com/daasuu/mp4compose/composer/Listener.kt diff --git a/mp4compose/src/main/java/com/daasuu/mp4compose/composer/Listener.kt b/mp4compose/src/main/java/com/daasuu/mp4compose/composer/Listener.kt new file mode 100644 index 000000000..71d245443 --- /dev/null +++ b/mp4compose/src/main/java/com/daasuu/mp4compose/composer/Listener.kt @@ -0,0 +1,24 @@ +package com.daasuu.mp4compose.composer + +interface Listener { + /** + * Called to notify progress. + * + * @param progress Progress in [0.0, 1.0] range, or negative value if progress is unknown. + */ + fun onProgress(progress: Double) + + /** + * Called when transcode completed. + */ + fun onCompleted() + + /** + * Called when transcode canceled. + */ + fun onCanceled() + + fun onFailed(exception: Exception) + + fun onStart() +} From c5669b56a3f6cfc91ca0b666ba32dc75ec25aa90 Mon Sep 17 00:00:00 2001 From: develric Date: Sun, 14 Mar 2021 23:47:45 +0100 Subject: [PATCH 03/12] Restoring original (with few modifications) versions of Mp4Composer and Mp4ComposerEngine --- .../mp4compose/composer/Mp4ComposerBasic.java | 449 ++++++++++++++++++ .../composer/Mp4ComposerEngineBasic.java | 404 ++++++++++++++++ 2 files changed, 853 insertions(+) create mode 100644 mp4compose/src/main/java/com/daasuu/mp4compose/composer/Mp4ComposerBasic.java create mode 100644 mp4compose/src/main/java/com/daasuu/mp4compose/composer/Mp4ComposerEngineBasic.java diff --git a/mp4compose/src/main/java/com/daasuu/mp4compose/composer/Mp4ComposerBasic.java b/mp4compose/src/main/java/com/daasuu/mp4compose/composer/Mp4ComposerBasic.java new file mode 100644 index 000000000..6e4d144bd --- /dev/null +++ b/mp4compose/src/main/java/com/daasuu/mp4compose/composer/Mp4ComposerBasic.java @@ -0,0 +1,449 @@ +package com.daasuu.mp4compose.composer; + +import android.media.MediaCodec; +import android.media.MediaCodecInfo; +import android.media.MediaMetadataRetriever; +import android.util.Size; +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; +import com.daasuu.mp4compose.FillMode; +import com.daasuu.mp4compose.FillModeCustomItem; +import com.daasuu.mp4compose.Rotation; +import com.daasuu.mp4compose.VideoFormatMimeType; +import com.daasuu.mp4compose.filter.GlFilter; +import com.daasuu.mp4compose.logger.AndroidLogger; +import com.daasuu.mp4compose.logger.Logger; +import com.daasuu.mp4compose.source.DataSource; +import com.daasuu.mp4compose.source.FilePathDataSource; + +import java.io.FileDescriptor; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +/** + * Created by sudamasayuki on 2017/11/15. + */ + +public class Mp4ComposerBasic implements ComposerInterface { + private final static String TAG = Mp4Composer.class.getSimpleName(); + + private final DataSource srcDataSource; + private final String destPath; + private FileDescriptor destFileDescriptor; + private GlFilter filter; + private Size outputResolution; + private int bitrate = -1; + private int iFrameInterval = 1; + private int audioBitRate = 128000; + private int aacProfile = MediaCodecInfo.CodecProfileLevel.AACObjectELD; + private boolean forceAudioEncoding = false; + private boolean mute = false; + private Rotation rotation = Rotation.NORMAL; + private Listener listener; + private FillMode fillMode = FillMode.PRESERVE_ASPECT_FIT; + private FillModeCustomItem fillModeCustomItem; + private float timeScale = 1f; // should be in range 0.125 (-8X) to 8.0 (8X) + private boolean isPitchChanged = false; + private boolean flipVertical = false; + private boolean flipHorizontal = false; + private long trimStartMs = 0; + private long trimEndMs = -1; + private VideoFormatMimeType videoFormatMimeType = VideoFormatMimeType.AUTO; + + private ExecutorService executorService; + private Mp4ComposerEngineBasic engine; + + private Logger logger; + + private DataSource.Listener errorDataSource = new DataSource.Listener() { + @Override + public void onError(Exception e) { + notifyListenerOfFailureAndShutdown(e); + } + }; + + public Mp4ComposerBasic(@NonNull final String srcPath, @NonNull final String destPath) { + this(srcPath, destPath, new AndroidLogger()); + } + + public Mp4ComposerBasic( + @NonNull final String srcPath, + @NonNull final String destPath, + @NonNull final Logger logger + ) { + this.logger = logger; + this.srcDataSource = new FilePathDataSource(srcPath, logger, errorDataSource); + this.destPath = destPath; + } + + public Mp4ComposerBasic filter(@NonNull GlFilter filter) { + this.filter = filter; + return this; + } + + public Mp4ComposerBasic size(int width, int height) { + this.outputResolution = new Size(width, height); + return this; + } + + @Override + public Mp4ComposerBasic size(Size size) { + this.outputResolution = size; + return this; + } + + public Mp4ComposerBasic videoBitrate(int bitrate) { + this.bitrate = bitrate; + return this; + } + + public Mp4ComposerBasic iFrameInterval(int iFrameInterval) { + this.iFrameInterval = iFrameInterval; + return this; + } + + public Mp4ComposerBasic audioBitRate(int audioBitRate) { + this.audioBitRate = audioBitRate; + return this; + } + + public Mp4ComposerBasic aacProfile(int aacProfile) { + this.aacProfile = aacProfile; + return this; + } + + public Mp4ComposerBasic forceAudioEncoding(boolean forceAudioEncoding) { + this.forceAudioEncoding = forceAudioEncoding; + return this; + } + + public Mp4ComposerBasic mute(boolean mute) { + this.mute = mute; + return this; + } + + public Mp4ComposerBasic flipVertical(boolean flipVertical) { + this.flipVertical = flipVertical; + return this; + } + + public Mp4ComposerBasic flipHorizontal(boolean flipHorizontal) { + this.flipHorizontal = flipHorizontal; + return this; + } + + public Mp4ComposerBasic rotation(@NonNull Rotation rotation) { + this.rotation = rotation; + return this; + } + + public Mp4ComposerBasic fillMode(@NonNull FillMode fillMode) { + this.fillMode = fillMode; + return this; + } + + public Mp4ComposerBasic customFillMode(@NonNull FillModeCustomItem fillModeCustomItem) { + this.fillModeCustomItem = fillModeCustomItem; + this.fillMode = FillMode.CUSTOM; + return this; + } + + public Mp4ComposerBasic listener(@NonNull Listener listener) { + this.listener = listener; + return this; + } + + public Mp4ComposerBasic timeScale(final float timeScale) { + this.timeScale = timeScale; + return this; + } + + public Mp4ComposerBasic changePitch(final boolean isPitchChanged){ + this.isPitchChanged = isPitchChanged; + return this; + } + + public Mp4ComposerBasic videoFormatMimeType(@NonNull VideoFormatMimeType videoFormatMimeType) { + this.videoFormatMimeType = videoFormatMimeType; + return this; + } + + /** + * Set the {@link Logger} that should be used. Defaults to {@link AndroidLogger} if none is set. + * + * @param logger The logger that should be used to log. + * @return The composer instance. + */ + public Mp4ComposerBasic logger(@NonNull final Logger logger) { + this.logger = logger; + return this; + } + + /** + * Trim the video to the provided times. By default the video will not be trimmed. + * + * @param trimStartMs The start time of the trim in milliseconds. + * @param trimEndMs The end time of the trim in milliseconds, -1 for no end. + * @return The composer instance. + */ + public Mp4ComposerBasic trim(final long trimStartMs, final long trimEndMs) { + this.trimStartMs = trimStartMs; + this.trimEndMs = trimEndMs; + return this; + } + + @Nullable + public Size getSrcVideoResolution() { + return getVideoResolution(srcDataSource); + } + + private ExecutorService getExecutorService() { + if (executorService == null) { + executorService = Executors.newSingleThreadExecutor(); + } + return executorService; + } + + + public Mp4ComposerBasic start() { + //if we're already composing, calling this should do nothing + if (engine != null) { + return this; + } + + getExecutorService().execute(new Runnable() { + @Override + public void run() { + if (logger == null) { + logger = new AndroidLogger(); + } + engine = new Mp4ComposerEngineBasic(logger); + + engine.setProgressCallback(new Mp4ComposerEngineBasic.ProgressCallback() { + @Override + public void onProgress(final double progress) { + if (listener != null) { + listener.onProgress(progress); + } + } + }); + + final Integer videoRotate = getVideoRotation(srcDataSource); + final Size srcVideoResolution = getVideoResolution(srcDataSource); + + if (srcVideoResolution == null || videoRotate == null) { + notifyListenerOfFailureAndShutdown( + new UnsupportedOperationException("File type unsupported, path: " + srcDataSource) + ); + return; + } + + if (filter == null) { + filter = new GlFilter(); + } + + if (fillMode == null) { + fillMode = FillMode.PRESERVE_ASPECT_FIT; + } + if (fillMode == FillMode.CUSTOM && fillModeCustomItem == null) { + notifyListenerOfFailureAndShutdown( + new IllegalAccessException("FillMode.CUSTOM must need fillModeCustomItem.") + ); + return; + } + + if (fillModeCustomItem != null) { + fillMode = FillMode.CUSTOM; + } + + if (outputResolution == null) { + if (fillMode == FillMode.CUSTOM) { + outputResolution = srcVideoResolution; + } else { + Rotation rotate = Rotation.fromInt(rotation.getRotation() + videoRotate); + if (rotate == Rotation.ROTATION_90 || rotate == Rotation.ROTATION_270) { + outputResolution = new Size(srcVideoResolution.getHeight(), srcVideoResolution.getWidth()); + } else { + outputResolution = srcVideoResolution; + } + } + } + + if (timeScale < 0.125f) { + timeScale = 0.125f; + }else if(timeScale > 8f){ + timeScale = 8f; + } + + logger.debug(TAG, "rotation = " + (rotation.getRotation() + videoRotate)); + logger.debug(TAG, "rotation = " + Rotation.fromInt(rotation.getRotation() + videoRotate)); + logger.debug( + TAG, + "inputResolution width = " + srcVideoResolution.getWidth() + " height = " + + srcVideoResolution.getHeight() + ); + logger.debug( + TAG, + "outputResolution width = " + outputResolution.getWidth() + " height = " + + outputResolution.getHeight() + ); + logger.debug(TAG, "fillMode = " + fillMode); + + try { + if (bitrate < 0) { + bitrate = calcBitRate(outputResolution.getWidth(), outputResolution.getHeight()); + } + + if (listener != null) { + listener.onStart(); + } + + engine.compose( + srcDataSource, + destPath, + destFileDescriptor, + outputResolution, + filter, + bitrate, + mute, + Rotation.fromInt(rotation.getRotation() + videoRotate), + srcVideoResolution, + fillMode, + fillModeCustomItem, + timeScale, + isPitchChanged, + flipVertical, + flipHorizontal, + trimStartMs, + trimEndMs, + videoFormatMimeType, + iFrameInterval, + audioBitRate, + aacProfile, + forceAudioEncoding + ); + + } catch (Exception e) { + if (e instanceof MediaCodec.CodecException) { + logger.error( + TAG, + "This devicel cannot codec with that setting. Check width, height, " + + "bitrate and video format.", e + ); + notifyListenerOfFailureAndShutdown(e); + return; + } + + logger.error(TAG, "Unable to compose the engine", e); + notifyListenerOfFailureAndShutdown(e); + return; + } + + if (listener != null) { + if (engine.isCanceled()) { + listener.onCanceled(); + } else { + listener.onCompleted(); + } + } + executorService.shutdown(); + engine = null; + } + }); + + return this; + } + + private void notifyListenerOfFailureAndShutdown(final Exception failure) { + if (listener != null) { + listener.onFailed(failure); + } + if (executorService != null) { + executorService.shutdown(); + } + } + + public void cancel() { + if (engine != null) { + engine.cancel(); + } + } + + @Nullable + private Integer getVideoRotation(DataSource dataSource) { + MediaMetadataRetriever mediaMetadataRetriever = null; + try { + mediaMetadataRetriever = new MediaMetadataRetriever(); + mediaMetadataRetriever.setDataSource(dataSource.getFileDescriptor()); + final String orientation = mediaMetadataRetriever.extractMetadata( + MediaMetadataRetriever.METADATA_KEY_VIDEO_ROTATION + ); + if (orientation == null) { + return null; + } + return Integer.valueOf(orientation); + } catch (IllegalArgumentException e) { + logger.error("MediaMetadataRetriever", "getVideoRotation IllegalArgumentException", e); + return 0; + } catch (RuntimeException e) { + logger.error("MediaMetadataRetriever", "getVideoRotation RuntimeException", e); + return 0; + } catch (Exception e) { + logger.error("MediaMetadataRetriever", "getVideoRotation Exception", e); + return 0; + } finally { + try { + if (mediaMetadataRetriever != null) { + mediaMetadataRetriever.release(); + } + } catch (RuntimeException e) { + logger.error(TAG, "Failed to release mediaMetadataRetriever.", e); + } + } + } + + private int calcBitRate(int width, int height) { + final int bitrate = (int) (0.25 * 30 * width * height); + logger.debug(TAG, "bitrate=" + bitrate); + return bitrate; + } + + /** + * Extract the resolution of the video at the provided path, or null if the format is + * unsupported. + */ + @Nullable + private Size getVideoResolution(DataSource dataSource) { + MediaMetadataRetriever retriever = null; + try { + retriever = new MediaMetadataRetriever(); + retriever.setDataSource(dataSource.getFileDescriptor()); + final String rawWidth = retriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_VIDEO_WIDTH); + final String rawHeight = retriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_VIDEO_HEIGHT); + if (rawWidth == null || rawHeight == null) { + return null; + } + final int width = Integer.parseInt(rawWidth); + final int height = Integer.parseInt(rawHeight); + + return new Size(width, height); + } catch (IllegalArgumentException e) { + logger.error("MediaMetadataRetriever", "getVideoResolution IllegalArgumentException", e); + return null; + } catch (RuntimeException e) { + logger.error("MediaMetadataRetriever", "getVideoResolution RuntimeException", e); + return null; + } catch (Exception e) { + logger.error("MediaMetadataRetriever", "getVideoResolution Exception", e); + return null; + } finally { + try { + if (retriever != null) { + retriever.release(); + } + } catch (RuntimeException e) { + logger.error(TAG, "Failed to release mediaMetadataRetriever.", e); + } + } + } +} diff --git a/mp4compose/src/main/java/com/daasuu/mp4compose/composer/Mp4ComposerEngineBasic.java b/mp4compose/src/main/java/com/daasuu/mp4compose/composer/Mp4ComposerEngineBasic.java new file mode 100644 index 000000000..91cb57e2a --- /dev/null +++ b/mp4compose/src/main/java/com/daasuu/mp4compose/composer/Mp4ComposerEngineBasic.java @@ -0,0 +1,404 @@ +package com.daasuu.mp4compose.composer; + +import android.media.*; +import android.os.Build; +import android.util.Size; +import androidx.annotation.NonNull; +import com.daasuu.mp4compose.FillMode; +import com.daasuu.mp4compose.FillModeCustomItem; +import com.daasuu.mp4compose.Rotation; +import com.daasuu.mp4compose.VideoFormatMimeType; +import com.daasuu.mp4compose.filter.GlFilter; +import com.daasuu.mp4compose.logger.Logger; +import com.daasuu.mp4compose.source.DataSource; + +import java.io.FileDescriptor; +import java.io.IOException; + +// Refer: https://github.com/ypresto/android-transcoder/blob/master/lib/src/main/java/net/ypresto/androidtranscoder/engine/MediaTranscoderEngine.java + +/** + * Internal engine, do not use this directly. + */ +class Mp4ComposerEngineBasic { + private static final String TAG = "Mp4ComposerEngine"; + private static final String AUDIO_PREFIX = "audio/"; + private static final String VIDEO_PREFIX = "video/"; + private static final double PROGRESS_UNKNOWN = -1.0; + private static final long SLEEP_TO_WAIT_TRACK_TRANSCODERS = 10; + private static final long PROGRESS_INTERVAL_STEPS = 10; + private VideoComposer videoComposer; + private IAudioComposer audioComposer; + private MediaExtractor mediaExtractor; + private MediaMuxer mediaMuxer; + private ProgressCallback progressCallback; + private long durationUs; + private MediaMetadataRetriever mediaMetadataRetriever; + private volatile boolean canceled; + private final Logger logger; + + Mp4ComposerEngineBasic(@NonNull final Logger logger) { + this.logger = logger; + } + + void setProgressCallback(ProgressCallback progressCallback) { + this.progressCallback = progressCallback; + } + + void compose( + final DataSource srcDataSource, + final String destSrc, + final FileDescriptor destFileDescriptor, + final Size outputResolution, + final GlFilter filter, + final int bitrate, + final boolean mute, + final Rotation rotation, + final Size inputResolution, + final FillMode fillMode, + final FillModeCustomItem fillModeCustomItem, + final float timeScale, + final boolean isPitchChanged, + final boolean flipVertical, + final boolean flipHorizontal, + final long trimStartMs, + final long trimEndMs, + final VideoFormatMimeType videoFormatMimeType, + final int iFrameInterval, + final int audioBitRate, + final int aacProfile, + final boolean forceAudioEncoding + ) throws IOException { + + try { + mediaExtractor = new MediaExtractor(); + mediaExtractor.setDataSource(srcDataSource.getFileDescriptor()); + if (Build.VERSION.SDK_INT >= 26 && destSrc == null) { + mediaMuxer = new MediaMuxer(destFileDescriptor, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4); + } else { + mediaMuxer = new MediaMuxer(destSrc, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4); + } + mediaMetadataRetriever = new MediaMetadataRetriever(); + mediaMetadataRetriever.setDataSource(srcDataSource.getFileDescriptor()); + + if (trimEndMs != -1) { + durationUs = (trimEndMs - trimStartMs) * 1000; + } else { + try { + durationUs = Long.parseLong( + mediaMetadataRetriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_DURATION) + ) * 1000; + } catch (NumberFormatException e) { + durationUs = -1; + } + } + + logger.debug(TAG, "Duration (us): " + durationUs); + + MuxRender muxRender = new MuxRender(mediaMuxer); + + // identify track indices + int videoTrackIndex = -1; + int audioTrackIndex = -1; + for (int i = 0; i < mediaExtractor.getTrackCount(); i++) { + MediaFormat mediaFormat = mediaExtractor.getTrackFormat(i); + String mimeType = mediaFormat.getString(MediaFormat.KEY_MIME); + if (mimeType == null) continue; + if (mimeType.startsWith(VIDEO_PREFIX)) { + videoTrackIndex = i; + } else if (mimeType.startsWith(AUDIO_PREFIX)) { + audioTrackIndex = i; + } + } + + final MediaFormat actualVideoOutputFormat = createVideoOutputFormatWithAvailableEncoders( + videoFormatMimeType, + bitrate, + outputResolution, + iFrameInterval + ); + if (Build.VERSION.SDK_INT == Build.VERSION_CODES.LOLLIPOP) { + // Only LOLLIPOP sets KEY_FRAME_RATE here. + actualVideoOutputFormat.setInteger(MediaFormat.KEY_FRAME_RATE, 30); + } + + // setup video composer + videoComposer = new VideoComposer( + mediaExtractor, + videoTrackIndex, + actualVideoOutputFormat, + muxRender, + (int)timeScale + ); + videoComposer.setUp( + filter, + rotation, + outputResolution, + inputResolution, + fillMode, + fillModeCustomItem, + flipVertical, + flipHorizontal + ); + mediaExtractor.selectTrack(videoTrackIndex); + + // setup audio if present and not muted + if (audioTrackIndex >= 0 && mediaMetadataRetriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_HAS_AUDIO) != null && !mute) { + // has Audio video + final MediaFormat inputMediaFormat = mediaExtractor.getTrackFormat(audioTrackIndex); + final MediaFormat outputMediaFormat = createAudioOutputFormat(inputMediaFormat, audioBitRate, aacProfile, forceAudioEncoding); + + if( timeScale >= 0.99 && timeScale <= 1.01 && outputMediaFormat.equals(inputMediaFormat)) { + audioComposer = new AudioComposer( + mediaExtractor, + audioTrackIndex, + muxRender, + true + ); + } else { + audioComposer = new RemixAudioComposerBasic( + mediaExtractor, + audioTrackIndex, + outputMediaFormat, + muxRender, + timeScale, + isPitchChanged, + trimStartMs, + trimEndMs + ); + } + + audioComposer.setup(); + mediaExtractor.selectTrack(audioTrackIndex); + mediaExtractor.seekTo(trimStartMs * 1000, MediaExtractor.SEEK_TO_PREVIOUS_SYNC); + runPipelines(); + } else { + mediaExtractor.seekTo(trimStartMs * 1000, MediaExtractor.SEEK_TO_PREVIOUS_SYNC); + // no audio video + runPipelinesNoAudio(); + } + + mediaMuxer.stop(); + } finally { + try { + if (videoComposer != null) { + videoComposer.release(); + videoComposer = null; + } + if (audioComposer != null) { + audioComposer.release(); + audioComposer = null; + } + if (mediaExtractor != null) { + mediaExtractor.release(); + mediaExtractor = null; + } + } catch (RuntimeException e) { + logger.error(TAG, "Could not shutdown mediaExtractor, codecs and mediaMuxer pipeline.", e); + } + try { + if (mediaMuxer != null) { + mediaMuxer.release(); + mediaMuxer = null; + } + } catch (RuntimeException e) { + logger.error(TAG, "Failed to release mediaMuxer.", e); + } + try { + if (mediaMetadataRetriever != null) { + mediaMetadataRetriever.release(); + mediaMetadataRetriever = null; + } + } catch (RuntimeException e) { + logger.error(TAG, "Failed to release mediaMetadataRetriever.", e); + } + } + } + + void cancel() { + canceled = true; + } + + boolean isCanceled() { + return canceled; + } + + @NonNull + private static MediaFormat createVideoOutputFormatWithAvailableEncoders(@NonNull final VideoFormatMimeType mimeType, + final int bitrate, + @NonNull final Size outputResolution, + final int iFrameInterval) { + final MediaCodecList mediaCodecList = new MediaCodecList(MediaCodecList.REGULAR_CODECS); + + if (mimeType != VideoFormatMimeType.AUTO) { + final MediaFormat mediaFormat = createVideoFormat( + mimeType.getFormat(), + bitrate, + outputResolution, + iFrameInterval + ); + if (mediaCodecList.findEncoderForFormat(mediaFormat) != null) { + return mediaFormat; + } + } + + final MediaFormat hevcMediaFormat = createVideoFormat( + VideoFormatMimeType.HEVC.getFormat(), + bitrate, + outputResolution, + iFrameInterval + ); + if (mediaCodecList.findEncoderForFormat(hevcMediaFormat) != null) { + return hevcMediaFormat; + } + + final MediaFormat avcMediaFormat = createVideoFormat( + VideoFormatMimeType.AVC.getFormat(), + bitrate, + outputResolution, + iFrameInterval + ); + if (mediaCodecList.findEncoderForFormat(avcMediaFormat) != null) { + return avcMediaFormat; + } + + final MediaFormat mp4vesMediaFormat = createVideoFormat( + VideoFormatMimeType.MPEG4.getFormat(), + bitrate, + outputResolution, + iFrameInterval + ); + if (mediaCodecList.findEncoderForFormat(mp4vesMediaFormat) != null) { + return mp4vesMediaFormat; + } + + return createVideoFormat(VideoFormatMimeType.H263.getFormat(), bitrate, outputResolution, iFrameInterval); + } + + @NonNull + private static MediaFormat createAudioOutputFormat( + @NonNull final MediaFormat inputFormat, + final int bitRate, + final int aacProfile, + final boolean forceEncoding + ) { + if (!forceEncoding && MediaFormat.MIMETYPE_AUDIO_AAC.equals(inputFormat.getString(MediaFormat.KEY_MIME))) { + return inputFormat; + } else { + final MediaFormat outputFormat = new MediaFormat(); + outputFormat.setString(MediaFormat.KEY_MIME, MediaFormat.MIMETYPE_AUDIO_AAC); + outputFormat.setInteger(MediaFormat.KEY_AAC_PROFILE, aacProfile); + outputFormat.setInteger(MediaFormat.KEY_SAMPLE_RATE, + inputFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE)); + outputFormat.setInteger(MediaFormat.KEY_BIT_RATE, bitRate); + outputFormat.setInteger(MediaFormat.KEY_CHANNEL_COUNT, + inputFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT)); + + return outputFormat; + } + } + + @NonNull + private static MediaFormat createVideoFormat(@NonNull final String mimeType, + final int bitrate, + @NonNull final Size outputResolution, + final int iFrameInterval) { + final MediaFormat outputFormat = + MediaFormat.createVideoFormat(mimeType, + outputResolution.getWidth(), + outputResolution.getHeight()); + + outputFormat.setInteger(MediaFormat.KEY_BIT_RATE, bitrate); + // On Build.VERSION_CODES.LOLLIPOP, format must not contain a MediaFormat#KEY_FRAME_RATE. + // https://developer.android.com/reference/android/media/MediaCodecInfo.CodecCapabilities.html#isFormatSupported(android.media.MediaFormat) + if (Build.VERSION.SDK_INT != Build.VERSION_CODES.LOLLIPOP) { + // Required but ignored by the encoder + outputFormat.setInteger(MediaFormat.KEY_FRAME_RATE, 30); + } + outputFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, iFrameInterval); + outputFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, + MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface); + + return outputFormat; + } + + + private void runPipelines() { + long loopCount = 0; + if (durationUs <= 0) { + if (progressCallback != null) { + progressCallback.onProgress(PROGRESS_UNKNOWN); + }// unknown + } + while (!canceled && !(videoComposer.isFinished() && audioComposer.isFinished())) { + boolean stepped = videoComposer.stepPipeline() + || audioComposer.stepPipeline(); + loopCount++; + if (durationUs > 0 && loopCount % PROGRESS_INTERVAL_STEPS == 0) { + long writtenPresentationVideoTimeUs = videoComposer.getWrittenPresentationTimeUs(); + double videoProgress = videoComposer.isFinished() ? 1.0 : Math.min( + 1.0, + (double) getWrittenPresentationTimeUs(writtenPresentationVideoTimeUs) / durationUs + ); + double audioProgress = audioComposer.isFinished() ? 1.0 : Math.min( + 1.0, + (double) getWrittenPresentationTimeUs(audioComposer.getWrittenPresentationTimeUs()) / durationUs + ); + double progress = (videoProgress + audioProgress) / 2.0; + if (progressCallback != null) { + progressCallback.onProgress(progress); + } + } + if (!stepped) { + try { + Thread.sleep(SLEEP_TO_WAIT_TRACK_TRANSCODERS); + } catch (InterruptedException e) { + // nothing to do + } + } + } + } + + private long getWrittenPresentationTimeUs(long time) { + return Math.max(0, time); + } + + private void runPipelinesNoAudio() { + long loopCount = 0; + if (durationUs <= 0) { + if (progressCallback != null) { + progressCallback.onProgress(PROGRESS_UNKNOWN); + } // unknown + } + while (!canceled && !videoComposer.isFinished()) { + boolean stepped = videoComposer.stepPipeline(); + loopCount++; + if (durationUs > 0 && loopCount % PROGRESS_INTERVAL_STEPS == 0) { + long writtenPresentationVideoTimeUs = videoComposer.getWrittenPresentationTimeUs(); + double videoProgress = videoComposer.isFinished() ? 1.0 : Math.min( + 1.0, + (double) getWrittenPresentationTimeUs(writtenPresentationVideoTimeUs) / durationUs + ); + if (progressCallback != null) { + progressCallback.onProgress(videoProgress); + } + } + if (!stepped) { + try { + Thread.sleep(SLEEP_TO_WAIT_TRACK_TRANSCODERS); + } catch (InterruptedException e) { + // nothing to do + } + } + } + } + + interface ProgressCallback { + /** + * Called to notify progress. Same thread which initiated transcode is used. + * + * @param progress Progress in [0.0, 1.0] range, or negative value if progress is unknown. + */ + void onProgress(double progress); + } +} From 8fbb9254a1dcb72caafb8f030d48a6d6b4836f37 Mon Sep 17 00:00:00 2001 From: develric Date: Sun, 14 Mar 2021 23:48:29 +0100 Subject: [PATCH 04/12] Restoring (with minor changes) the original RemixAudioComposer --- .../composer/RemixAudioComposerBasic.java | 226 ++++++++++++++++++ 1 file changed, 226 insertions(+) create mode 100644 mp4compose/src/main/java/com/daasuu/mp4compose/composer/RemixAudioComposerBasic.java diff --git a/mp4compose/src/main/java/com/daasuu/mp4compose/composer/RemixAudioComposerBasic.java b/mp4compose/src/main/java/com/daasuu/mp4compose/composer/RemixAudioComposerBasic.java new file mode 100644 index 000000000..d91d55c98 --- /dev/null +++ b/mp4compose/src/main/java/com/daasuu/mp4compose/composer/RemixAudioComposerBasic.java @@ -0,0 +1,226 @@ +package com.daasuu.mp4compose.composer; + +/** + * // Refer: https://github.com/ypresto/android-transcoder/blob/master/lib/src/main/java/net/ypresto/androidtranscoder/engine/AudioTrackTranscoder.java + * + */ + +import android.media.MediaCodec; +import android.media.MediaExtractor; +import android.media.MediaFormat; + +import com.daasuu.mp4compose.composer.MuxRender.SampleType; + +import java.io.IOException; +import java.util.concurrent.TimeUnit; + + +class RemixAudioComposerBasic implements IAudioComposer { + private static final SampleType SAMPLE_TYPE = SampleType.AUDIO; + + private static final int DRAIN_STATE_NONE = 0; + private static final int DRAIN_STATE_SHOULD_RETRY_IMMEDIATELY = 1; + private static final int DRAIN_STATE_CONSUMED = 2; + + private final MediaExtractor extractor; + private final MuxRender muxer; + private long writtenPresentationTimeUs; + + private final int trackIndex; + + private final MediaFormat outputFormat; + + private final MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo(); + private MediaCodec decoder; + private MediaCodec encoder; + private MediaFormat actualOutputFormat; + + private boolean isExtractorEOS; + private boolean isDecoderEOS; + private boolean isEncoderEOS; + private boolean decoderStarted; + private boolean encoderStarted; + + private AudioChannelWithSP audioChannel; + private final float timeScale; + private final boolean isPitchChanged; + + private final long trimStartUs; + private final long trimEndUs; + int numTracks = 0; + + // Used for AAC priming offset. + private boolean addPrimingDelay; + private int frameCounter; + private long primingDelay; + + RemixAudioComposerBasic(MediaExtractor extractor, int trackIndex, + MediaFormat outputFormat, MuxRender muxer, float timeScale, boolean isPitchChanged, + long trimStartMs, long trimEndMs) { + this.extractor = extractor; + this.trackIndex = trackIndex; + this.outputFormat = outputFormat; + this.muxer = muxer; + this.timeScale = timeScale; + this.isPitchChanged = isPitchChanged; + this.trimStartUs = TimeUnit.MILLISECONDS.toMicros(trimStartMs); + this.trimEndUs = trimEndMs == -1 ? trimEndMs : TimeUnit.MILLISECONDS.toMicros(trimEndMs); + } + + @Override + public void setup() { + extractor.selectTrack(trackIndex); + try { + encoder = MediaCodec.createEncoderByType(outputFormat.getString(MediaFormat.KEY_MIME)); + } catch (IOException e) { + throw new IllegalStateException(e); + } + encoder.configure(outputFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); + encoder.start(); + encoderStarted = true; + + final MediaFormat inputFormat = extractor.getTrackFormat(trackIndex); + try { + decoder = MediaCodec.createDecoderByType(inputFormat.getString(MediaFormat.KEY_MIME)); + } catch (IOException e) { + throw new IllegalStateException(e); + } + decoder.configure(inputFormat, null, null, 0); + decoder.start(); + decoderStarted = true; + + audioChannel = new AudioChannelWithSP(decoder, encoder, outputFormat,timeScale,isPitchChanged); + } + + @Override + public boolean stepPipeline() { + boolean busy = false; + + int status; + + while (drainEncoder(0) != DRAIN_STATE_NONE) busy = true; + do { + if(audioChannel.isAnyPendingBuffIndex()){ + break; + } + status = drainDecoder(0); + if (status != DRAIN_STATE_NONE) busy = true; + // NOTE: not repeating to keep from deadlock when encoder is full. + } while (status == DRAIN_STATE_SHOULD_RETRY_IMMEDIATELY); + + while (audioChannel.feedEncoder(0)) busy = true; + while (drainExtractor(0) != DRAIN_STATE_NONE) busy = true; + + return busy; + } + + private int drainExtractor(long timeoutUs) { + if (isExtractorEOS) return DRAIN_STATE_NONE; + int trackIndex = extractor.getSampleTrackIndex(); + if (trackIndex >= 0 && trackIndex != this.trackIndex) { + return DRAIN_STATE_NONE; + } + + final int result = decoder.dequeueInputBuffer(timeoutUs); + if (result < 0) return DRAIN_STATE_NONE; + if (trackIndex < 0 || (writtenPresentationTimeUs >= trimEndUs && trimEndUs != -1)) { + isExtractorEOS = true; + decoder.queueInputBuffer(result, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM); + extractor.unselectTrack(this.trackIndex); + return DRAIN_STATE_NONE; + } + + final int sampleSize = extractor.readSampleData(decoder.getInputBuffer(result), 0); + final boolean isKeyFrame = (extractor.getSampleFlags() & MediaExtractor.SAMPLE_FLAG_SYNC) != 0; + decoder.queueInputBuffer(result, 0, sampleSize, extractor.getSampleTime(), isKeyFrame ? MediaCodec.BUFFER_FLAG_KEY_FRAME : 0); + extractor.advance(); + numTracks ++ ; + return DRAIN_STATE_CONSUMED; + } + + private int drainDecoder(long timeoutUs) { + if (isDecoderEOS) return DRAIN_STATE_NONE; + + int result = decoder.dequeueOutputBuffer(bufferInfo, timeoutUs); + switch (result) { + case MediaCodec.INFO_TRY_AGAIN_LATER: + return DRAIN_STATE_NONE; + case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED: + audioChannel.setActualDecodedFormat(decoder.getOutputFormat()); + case MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED: + return DRAIN_STATE_SHOULD_RETRY_IMMEDIATELY; + } + + if ((bufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { + isDecoderEOS = true; + audioChannel.drainDecoderBufferAndQueue(BaseAudioChannel.BUFFER_INDEX_END_OF_STREAM, 0); + } else if (bufferInfo.size > 0) { + audioChannel.drainDecoderBufferAndQueue(result,bufferInfo.presentationTimeUs); + } + + return DRAIN_STATE_CONSUMED; + } + + private int drainEncoder(long timeoutUs) { + if (isEncoderEOS) return DRAIN_STATE_NONE; + int result = encoder.dequeueOutputBuffer(bufferInfo, timeoutUs); + switch (result) { + case MediaCodec.INFO_TRY_AGAIN_LATER: + return DRAIN_STATE_NONE; + case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED: + if (actualOutputFormat != null) { + throw new RuntimeException("Audio output format changed twice."); + } + actualOutputFormat = encoder.getOutputFormat(); + addPrimingDelay = MediaFormat.MIMETYPE_AUDIO_AAC.equals(actualOutputFormat.getString(MediaFormat.KEY_MIME)); + muxer.setOutputFormat(SAMPLE_TYPE, actualOutputFormat); + return DRAIN_STATE_SHOULD_RETRY_IMMEDIATELY; + case MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED: + return DRAIN_STATE_SHOULD_RETRY_IMMEDIATELY; + } + + if (actualOutputFormat == null) { + throw new RuntimeException("Could not determine actual output format."); + } + + if ((bufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { + isEncoderEOS = true; + + bufferInfo.set(0, 0, 0, bufferInfo.flags); + } + if ((bufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) { + // SPS or PPS, which should be passed by MediaFormat. + encoder.releaseOutputBuffer(result, false); + return DRAIN_STATE_SHOULD_RETRY_IMMEDIATELY; + } + muxer.writeSampleData(SAMPLE_TYPE, encoder.getOutputBuffer(result), bufferInfo); + + writtenPresentationTimeUs = bufferInfo.presentationTimeUs; + encoder.releaseOutputBuffer(result, false); + return DRAIN_STATE_CONSUMED; + } + + @Override + public long getWrittenPresentationTimeUs() { + return (long)(writtenPresentationTimeUs * timeScale); + } + + @Override + public boolean isFinished() { + return isEncoderEOS; + } + + @Override + public void release() { + if (decoder != null) { + if (decoderStarted) decoder.stop(); + decoder.release(); + decoder = null; + } + if (encoder != null) { + if (encoderStarted) encoder.stop(); + encoder.release(); + encoder = null; + } + } +} From 09b47474a9e187d9a74f5ecb78bcecc0cf12cf87 Mon Sep 17 00:00:00 2001 From: develric Date: Sun, 14 Mar 2021 23:49:07 +0100 Subject: [PATCH 05/12] Adding a ComposerProvider to get the correct composer based on the use case requested. --- .../mp4compose/composer/ComposerProvider.kt | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 mp4compose/src/main/java/com/daasuu/mp4compose/composer/ComposerProvider.kt diff --git a/mp4compose/src/main/java/com/daasuu/mp4compose/composer/ComposerProvider.kt b/mp4compose/src/main/java/com/daasuu/mp4compose/composer/ComposerProvider.kt new file mode 100644 index 000000000..a9dbb2ff7 --- /dev/null +++ b/mp4compose/src/main/java/com/daasuu/mp4compose/composer/ComposerProvider.kt @@ -0,0 +1,69 @@ +package com.daasuu.mp4compose.composer + +import android.content.Context +import android.graphics.Bitmap +import android.media.MediaCodecInfo.CodecProfileLevel +import android.net.Uri +import android.util.Size +import com.daasuu.mp4compose.FillMode +import com.daasuu.mp4compose.VideoFormatMimeType +import com.daasuu.mp4compose.composer.ComposerUseCase.CompressVideo +import com.daasuu.mp4compose.composer.ComposerUseCase.SaveVideoAsFile +import com.daasuu.mp4compose.composer.ComposerUseCase.SaveVideoFromBgAsFile +import com.daasuu.mp4compose.filter.GlFilter +import com.daasuu.mp4compose.source.DataSource + +sealed class ComposerUseCase { + data class SaveVideoAsFile( + val srcUri: Uri, + val destPath: String, + val context: Context, + val headers: Map? + ) : ComposerUseCase() + + data class SaveVideoFromBgAsFile(val bkgBmp: Bitmap, val destPath: String) : ComposerUseCase() + + data class CompressVideo @JvmOverloads constructor ( + val srcPath: String, + val destPath: String, + val videoFormatMimeType: VideoFormatMimeType, + val bitrate: Int, + val iFrameInterval: Int = 1, + val audioBitRate: Int = 128000, + val aacProfile: Int = CodecProfileLevel.AACObjectELD, + val forceAudioEncoding: Boolean = false + ) : ComposerUseCase() +} + +interface ComposerInterface { + fun size(size: Size): ComposerInterface + fun fillMode(fillMode: FillMode): ComposerInterface + fun filter(filter: GlFilter?): ComposerInterface + fun mute(mute: Boolean): ComposerInterface + fun listener(listener: Listener): ComposerInterface + fun start(): ComposerInterface +} + +object ComposerProvider { + fun getComposerForUseCase(useCase: ComposerUseCase): ComposerInterface { + return when (useCase) { + is SaveVideoAsFile -> { + Mp4Composer(useCase.srcUri, useCase.destPath) + .with(useCase.context) + .addedHeaders(useCase.headers) + } + is SaveVideoFromBgAsFile -> { + Mp4Composer(useCase.bkgBmp, useCase.destPath) + } + is CompressVideo -> { + Mp4ComposerBasic(useCase.srcPath, useCase.destPath) + .videoFormatMimeType(useCase.videoFormatMimeType) + .videoBitrate(useCase.bitrate) + .iFrameInterval(useCase.iFrameInterval) + .audioBitRate(useCase.audioBitRate) + .aacProfile(useCase.aacProfile) + .forceAudioEncoding(useCase.forceAudioEncoding) + } + } + } +} From 0210110c545d01ff3db34818079424ee0ee37f87 Mon Sep 17 00:00:00 2001 From: develric Date: Sun, 14 Mar 2021 23:49:45 +0100 Subject: [PATCH 06/12] Making the fromInt method statically visible from Java. --- mp4compose/src/main/java/com/daasuu/mp4compose/Rotation.kt | 1 + 1 file changed, 1 insertion(+) diff --git a/mp4compose/src/main/java/com/daasuu/mp4compose/Rotation.kt b/mp4compose/src/main/java/com/daasuu/mp4compose/Rotation.kt index 62d63f3b3..4071e8b2e 100644 --- a/mp4compose/src/main/java/com/daasuu/mp4compose/Rotation.kt +++ b/mp4compose/src/main/java/com/daasuu/mp4compose/Rotation.kt @@ -11,6 +11,7 @@ enum class Rotation private constructor(val rotation: Int) { ROTATION_270(270); companion object { + @JvmStatic fun fromInt(rotate: Int): Rotation { for (rotation in Rotation.values()) { if (rotate == rotation.rotation) return rotation From 520c2009689384ba3fd5f06c021fcfc0e692a59d Mon Sep 17 00:00:00 2001 From: develric Date: Sun, 14 Mar 2021 23:53:00 +0100 Subject: [PATCH 07/12] Placing a comment about timeScale being int of float. --- .../java/com/daasuu/mp4compose/composer/RemixAudioComposer.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mp4compose/src/main/java/com/daasuu/mp4compose/composer/RemixAudioComposer.kt b/mp4compose/src/main/java/com/daasuu/mp4compose/composer/RemixAudioComposer.kt index 3c57f7704..d6eafaa01 100644 --- a/mp4compose/src/main/java/com/daasuu/mp4compose/composer/RemixAudioComposer.kt +++ b/mp4compose/src/main/java/com/daasuu/mp4compose/composer/RemixAudioComposer.kt @@ -17,7 +17,7 @@ internal class RemixAudioComposer( private val trackIndex: Int, private val outputFormat: MediaFormat, private val muxer: MuxRender, - private val timeScale: Int + private val timeScale: Int // TODO: this (and in other places) was float in the original lib, should we restore it? ) : IAudioComposer { override var writtenPresentationTimeUs: Long = 0 private set From 85c20a4a7fe8ddf5d4c05e065474330b5b093ee4 Mon Sep 17 00:00:00 2001 From: develric Date: Sun, 14 Mar 2021 23:55:07 +0100 Subject: [PATCH 08/12] Using fallbacks in AudioComposer at least in video compression scenario. --- .../mp4compose/composer/AudioComposer.kt | 26 +++++++++++++++---- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/mp4compose/src/main/java/com/daasuu/mp4compose/composer/AudioComposer.kt b/mp4compose/src/main/java/com/daasuu/mp4compose/composer/AudioComposer.kt index c05d635d9..b6172e099 100755 --- a/mp4compose/src/main/java/com/daasuu/mp4compose/composer/AudioComposer.kt +++ b/mp4compose/src/main/java/com/daasuu/mp4compose/composer/AudioComposer.kt @@ -12,12 +12,13 @@ import java.nio.ByteOrder internal class AudioComposer( private val mediaExtractor: MediaExtractor, private val trackIndex: Int, - private val muxRender: MuxRender + private val muxRender: MuxRender, + private val useFallBacks: Boolean = false ) : IAudioComposer { private val sampleType = MuxRender.SampleType.AUDIO private val bufferInfo = MediaCodec.BufferInfo() - private val bufferSize: Int - private val buffer: ByteBuffer + private var bufferSize: Int + private var buffer: ByteBuffer override var isFinished: Boolean = false private set private val actualOutputFormat: MediaFormat @@ -27,7 +28,15 @@ internal class AudioComposer( init { actualOutputFormat = this.mediaExtractor.getTrackFormat(this.trackIndex) this.muxRender.setOutputFormat(this.sampleType, actualOutputFormat) - bufferSize = actualOutputFormat.getInteger(MediaFormat.KEY_MAX_INPUT_SIZE) + + // TODO: maybe the original assignement could work as well? + // bufferSize = actualOutputFormat.containsKey(MediaFormat.KEY_MAX_INPUT_SIZE) ? actualOutputFormat.getInteger(MediaFormat.KEY_MAX_INPUT_SIZE) : (64 * 1024); + bufferSize = if (useFallBacks && !actualOutputFormat.containsKey(MediaFormat.KEY_MAX_INPUT_SIZE)) { + (64 * 1024) + } else { + actualOutputFormat.getInteger(MediaFormat.KEY_MAX_INPUT_SIZE) + } + buffer = ByteBuffer.allocateDirect(bufferSize).order(ByteOrder.nativeOrder()) } @@ -46,11 +55,18 @@ internal class AudioComposer( buffer.clear() val sampleSize = mediaExtractor.readSampleData(buffer, 0) - assert(sampleSize <= bufferSize) + if (useFallBacks && sampleSize > bufferSize) { + bufferSize = 2 * sampleSize; + buffer = ByteBuffer.allocateDirect(bufferSize).order(ByteOrder.nativeOrder()); + } else { + assert(sampleSize <= bufferSize) + } val isKeyFrame = mediaExtractor.sampleFlags and MediaExtractor.SAMPLE_FLAG_SYNC != 0 val flags = if (isKeyFrame) MediaCodec.BUFFER_FLAG_KEY_FRAME else 0 bufferInfo.set(0, sampleSize, mediaExtractor.sampleTime, flags) muxRender.writeSampleData(sampleType, buffer, bufferInfo) + // TODO: should we use the original writtenPresentationTimeUs = mediaExtractor.getSampleTime(); + // at least for the Video Compression use case? writtenPresentationTimeUs = bufferInfo.presentationTimeUs mediaExtractor.advance() From 53b7884d0694c8febca936fec2da80e3ef0b27b7 Mon Sep 17 00:00:00 2001 From: develric Date: Sun, 14 Mar 2021 23:55:33 +0100 Subject: [PATCH 09/12] Making Mp4Composer implement the ComposerInterface interface. --- .../daasuu/mp4compose/composer/Mp4Composer.kt | 35 ++++--------------- 1 file changed, 7 insertions(+), 28 deletions(-) diff --git a/mp4compose/src/main/java/com/daasuu/mp4compose/composer/Mp4Composer.kt b/mp4compose/src/main/java/com/daasuu/mp4compose/composer/Mp4Composer.kt index 312fd9a97..c60b51392 100644 --- a/mp4compose/src/main/java/com/daasuu/mp4compose/composer/Mp4Composer.kt +++ b/mp4compose/src/main/java/com/daasuu/mp4compose/composer/Mp4Composer.kt @@ -21,7 +21,7 @@ import java.util.concurrent.Executors * Created by sudamasayuki on 2017/11/15. */ -class Mp4Composer { +class Mp4Composer: ComposerInterface { private val srcUri: Uri? private val destPath: String private var filter: GlFilter? = null @@ -68,7 +68,7 @@ class Mp4Composer { return this } - fun filter(filter: GlFilter?): Mp4Composer { + override fun filter(filter: GlFilter?): Mp4Composer { this.filter = filter return this } @@ -78,7 +78,7 @@ class Mp4Composer { return this } - fun size(size: Size): Mp4Composer { + override fun size(size: Size): Mp4Composer { this.outputResolution = size return this } @@ -88,7 +88,7 @@ class Mp4Composer { return this } - fun mute(mute: Boolean): Mp4Composer { + override fun mute(mute: Boolean): Mp4Composer { this.mute = mute return this } @@ -108,7 +108,7 @@ class Mp4Composer { return this } - fun fillMode(fillMode: FillMode): Mp4Composer { + override fun fillMode(fillMode: FillMode): Mp4Composer { this.fillMode = fillMode return this } @@ -119,7 +119,7 @@ class Mp4Composer { return this } - fun listener(listener: Listener): Mp4Composer { + override fun listener(listener: Listener): Mp4Composer { this.listener = listener return this } @@ -136,7 +136,7 @@ class Mp4Composer { return executorService!! } - fun start(): Mp4Composer { + override fun start(): Mp4Composer { getExecutorService().execute(Runnable { val engine = Mp4ComposerEngine() @@ -265,27 +265,6 @@ class Mp4Composer { getExecutorService().shutdownNow() } - interface Listener { - /** - * Called to notify progress. - * - * @param progress Progress in [0.0, 1.0] range, or negative value if progress is unknown. - */ - fun onProgress(progress: Double) - - /** - * Called when transcode completed. - */ - fun onCompleted() - - /** - * Called when transcode canceled. - */ - fun onCanceled() - - fun onFailed(exception: Exception) - } - private fun initializeUriDataSource(engine: Mp4ComposerEngine) { engine.setDataSource(srcUri, addedRequestHeaders) } From 99c7f27f7f3d75b52e14d2434a3745d4f76b514a Mon Sep 17 00:00:00 2001 From: develric Date: Sun, 14 Mar 2021 23:55:59 +0100 Subject: [PATCH 10/12] Using ComposerProvider in photo editor. --- .../com/automattic/photoeditor/PhotoEditor.kt | 39 ++++++++++++++----- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/photoeditor/src/main/java/com/automattic/photoeditor/PhotoEditor.kt b/photoeditor/src/main/java/com/automattic/photoeditor/PhotoEditor.kt index 23f493bd1..278e427ff 100644 --- a/photoeditor/src/main/java/com/automattic/photoeditor/PhotoEditor.kt +++ b/photoeditor/src/main/java/com/automattic/photoeditor/PhotoEditor.kt @@ -47,7 +47,10 @@ import com.automattic.photoeditor.views.filter.CustomEffect import com.automattic.photoeditor.views.filter.PhotoFilter import com.bumptech.glide.Glide import com.daasuu.mp4compose.FillMode -import com.daasuu.mp4compose.composer.Mp4Composer +import com.daasuu.mp4compose.composer.ComposerProvider +import com.daasuu.mp4compose.composer.ComposerUseCase.SaveVideoAsFile +import com.daasuu.mp4compose.composer.ComposerUseCase.SaveVideoFromBgAsFile +import com.daasuu.mp4compose.composer.Listener import com.daasuu.mp4compose.filter.GlFilter import com.daasuu.mp4compose.filter.GlFilterGroup import com.daasuu.mp4compose.filter.GlGifWatermarkFilter @@ -788,9 +791,16 @@ class PhotoEditor private constructor(builder: Builder) : } } - Mp4Composer(videoInputPath, videoOutputPath) - .with(context) - .addedHeaders(authenticationHeadersInterface?.getAuthHeaders(videoInputPath.toString())) + val composer = ComposerProvider.getComposerForUseCase( + SaveVideoAsFile( + videoInputPath, + videoOutputPath, + context, + authenticationHeadersInterface?.getAuthHeaders(videoInputPath.toString()) + ) + ) + + composer // .size(width, height) // IMPORTANT: as we aim at a WYSIWYG UX, we need to produce a video of size equal to that of the phone // screen, given the user may be seeing a letterbox landscape video and placing emoji / text around @@ -801,7 +811,7 @@ class PhotoEditor private constructor(builder: Builder) : .fillMode(FillMode.PRESERVE_ASPECT_FIT) .filter(if (customAddedViews.isNotEmpty()) GlFilterGroup(filterCollection) else null) .mute(muteAudio) - .listener(object : Mp4Composer.Listener { + .listener(object : Listener { override fun onProgress(progress: Double) { Log.d(TAG, "onProgress = $progress") onSaveListener.onProgress(progress) @@ -821,6 +831,10 @@ class PhotoEditor private constructor(builder: Builder) : Log.e(TAG, "onFailed()", exception) onSaveListener.onFailure(exception) } + + override fun onStart() { + Log.d(TAG, "onStart()") + } }) .start() } @@ -945,11 +959,13 @@ class PhotoEditor private constructor(builder: Builder) : // take the static background image val bmp = createBitmapFromView(parentView.source) - Mp4Composer(bmp, videoOutputPath) - .size(bmp.width, bmp.height) // FIXME check whether these are the right values or not + val composer = ComposerProvider.getComposerForUseCase(SaveVideoFromBgAsFile(bmp, videoOutputPath)) + + composer + .size(Size(bmp.width, bmp.height)) // FIXME check whether these are the right values or not .fillMode(FillMode.PRESERVE_ASPECT_FIT) .filter(GlFilterGroup(filterCollection)) - .listener(object : Mp4Composer.Listener { + .listener(object : Listener { override fun onProgress(progress: Double) { Log.d(TAG, "onProgress = $progress") // TODO: show progress to user @@ -969,8 +985,11 @@ class PhotoEditor private constructor(builder: Builder) : Log.e(TAG, "onFailed()", exception) onSaveListener.onFailure(exception) } - }) - .start() + + override fun onStart() { + Log.d(TAG, "onStart()") + } + }).start() } // TODO to be used in conjunction with mp4composer From 2b9653fe82b31e0393682a473ed6ca5441e86083 Mon Sep 17 00:00:00 2001 From: develric Date: Mon, 15 Mar 2021 12:41:55 +0100 Subject: [PATCH 11/12] Fixing linter. --- .../mp4compose/VideoFormatMimeType.java | 1 + .../composer/AudioChannelWithSP.java | 64 +-- .../mp4compose/composer/AudioComposer.kt | 4 +- .../mp4compose/composer/BaseAudioChannel.java | 11 +- .../mp4compose/composer/ComposerProvider.kt | 1 - .../daasuu/mp4compose/composer/Mp4Composer.kt | 2 +- .../mp4compose/composer/Mp4ComposerBasic.java | 13 +- .../composer/Mp4ComposerEngineBasic.java | 35 +- .../composer/RemixAudioComposerBasic.java | 29 +- .../composer/SonicAudioProcessor.java | 499 +++++++++--------- .../mp4compose/logger/AndroidLogger.java | 4 +- .../com/daasuu/mp4compose/logger/Logger.java | 3 +- .../mp4compose/source/FilePathDataSource.java | 5 +- 13 files changed, 331 insertions(+), 340 deletions(-) diff --git a/mp4compose/src/main/java/com/daasuu/mp4compose/VideoFormatMimeType.java b/mp4compose/src/main/java/com/daasuu/mp4compose/VideoFormatMimeType.java index 52860af9d..360f1de21 100644 --- a/mp4compose/src/main/java/com/daasuu/mp4compose/VideoFormatMimeType.java +++ b/mp4compose/src/main/java/com/daasuu/mp4compose/VideoFormatMimeType.java @@ -2,6 +2,7 @@ import android.media.MediaFormat; +@SuppressWarnings("MemberName") public enum VideoFormatMimeType { HEVC(MediaFormat.MIMETYPE_VIDEO_HEVC), AVC(MediaFormat.MIMETYPE_VIDEO_AVC), diff --git a/mp4compose/src/main/java/com/daasuu/mp4compose/composer/AudioChannelWithSP.java b/mp4compose/src/main/java/com/daasuu/mp4compose/composer/AudioChannelWithSP.java index 0e99432ba..78130edec 100644 --- a/mp4compose/src/main/java/com/daasuu/mp4compose/composer/AudioChannelWithSP.java +++ b/mp4compose/src/main/java/com/daasuu/mp4compose/composer/AudioChannelWithSP.java @@ -10,9 +10,8 @@ /** * Created by TAPOS DATTA on 22,May,2020 */ - -public class AudioChannelWithSP extends BaseAudioChannel{ - +@SuppressWarnings({"MemberName", "MethodName", "SimplifyBooleanReturn"}) +public class AudioChannelWithSP extends BaseAudioChannel { private static final String TAG = "AUDIO_CHANNEL_WITH_SONIC"; private SonicAudioProcessor stream = null; // SonicAudioProcessor can deal with stereo Audio @@ -25,7 +24,13 @@ public class AudioChannelWithSP extends BaseAudioChannel{ private boolean isPendingFeeding = true; private boolean isAffectInPitch; // if true the scale will impact in speed with pitch - AudioChannelWithSP(MediaCodec decoder, MediaCodec encoder, MediaFormat encodeFormat,float timeScale, boolean isPitchChanged) { + AudioChannelWithSP( + MediaCodec decoder, + MediaCodec encoder, + MediaFormat encodeFormat, + float timeScale, + boolean isPitchChanged + ) { super(decoder, encoder, encodeFormat); this.isAffectInPitch = isPitchChanged; this.timeScale = timeScale; @@ -44,29 +49,28 @@ public void setActualDecodedFormat(MediaFormat decodedFormat) { isPendingFeeding = true; tempInputBuffer = ByteBuffer.allocateDirect(BUFFER_CAPACITY * 16).order(ByteOrder.nativeOrder()); - if(isAffectInPitch){ + if (isAffectInPitch) { stream.setRate(timeScale); - }else { + } else { stream.setSpeed(timeScale); } } @Override protected long sampleCountToDurationUs(long sampleCount, int sampleRate, int channelCount) { - //considered short buffer as data - return (long) ((MICROSECS_PER_SEC * (sampleCount * 1f)/(sampleRate * 1f * channelCount))); + // considered short buffer as data + return (long) ((MICROSECS_PER_SEC * (sampleCount * 1f) / (sampleRate * 1f * channelCount))); } @Override public void drainDecoderBufferAndQueue(int bufferIndex, long presentationTimeUs) { - if (actualDecodedFormat == null) { throw new RuntimeException("Buffer received before format!"); } final ByteBuffer data = - bufferIndex == BUFFER_INDEX_END_OF_STREAM ? - null : decoder.getOutputBuffer(bufferIndex); + bufferIndex == BUFFER_INDEX_END_OF_STREAM + ? null : decoder.getOutputBuffer(bufferIndex); if (data != null) { writeToSonicSteam(data.asShortBuffer()); @@ -81,15 +85,18 @@ public void drainDecoderBufferAndQueue(int bufferIndex, long presentationTimeUs) @Override public boolean feedEncoder(long timeoutUs) { - if (stream == null || !isPendingFeeding || (!isEOF && stream.samplesAvailable() == 0)) { - //no data available + // no data available updatePendingDecoderStatus(); return false; - } else if (!isEOF && timeScale < 1f && stream.samplesAvailable() > 0 && (stream.samplesAvailable() * outputChannelCount) < BUFFER_CAPACITY) { - //few data remaining in stream wait for next stream data + } else if (!isEOF + && timeScale < 1f + && stream.samplesAvailable() > 0 + && (stream.samplesAvailable() * outputChannelCount) < BUFFER_CAPACITY + ) { + // few data remaining in stream wait for next stream data updatePendingDecoderStatus(); return false; @@ -113,41 +120,30 @@ public boolean feedEncoder(long timeoutUs) { } private void updatePendingDecoderStatus() { - if (pendingDecoderOutputBuffIndx != -1) { pendingDecoderOutputBuffIndx = -1; } } private boolean FastOrNormalTimeBufferProcess(int encoderInBuffIndex) { - int samplesNum = stream.samplesAvailable(); boolean status = false; int rawDataLen = samplesNum * outputChannelCount; - if(rawDataLen >= BUFFER_CAPACITY){ - + if (rawDataLen >= BUFFER_CAPACITY) { return readStreamDataAndQueueToEncoder(BUFFER_CAPACITY, encoderInBuffIndex); - } - - else if (rawDataLen > 0 && rawDataLen < BUFFER_CAPACITY) { - + } else if (rawDataLen > 0 && rawDataLen < BUFFER_CAPACITY) { return readStreamDataAndQueueToEncoder(rawDataLen, encoderInBuffIndex); - } else if (isEOF && samplesNum == 0) { - return finalizeEncoderQueue(encoderInBuffIndex); - } else { - return status; } } private boolean slowTimeBufferProcess(final int encoderInBuffIndex) { - int samplesNum = stream.samplesAvailable(); boolean status = false; @@ -155,43 +151,32 @@ private boolean slowTimeBufferProcess(final int encoderInBuffIndex) { int rawDataLen = samplesNum * outputChannelCount; if (rawDataLen >= BUFFER_CAPACITY) { - return readStreamDataAndQueueToEncoder(BUFFER_CAPACITY, encoderInBuffIndex); - } else if (isEOF && (rawDataLen > 0 && rawDataLen < BUFFER_CAPACITY)) { - return readStreamDataAndQueueToEncoder(rawDataLen, encoderInBuffIndex); - } else if (isEOF && rawDataLen == 0) { - return finalizeEncoderQueue(encoderInBuffIndex); - } else { - return status; } } private boolean finalizeEncoderQueue(final int encoderInBuffIndex) { - isPendingFeeding = false; return queueInputBufferInEncoder(null, encoderInBuffIndex); } private boolean readStreamDataAndQueueToEncoder(final int capacity, final int encoderInBuffIndex) { - short[] rawData = new short[capacity]; stream.readShortFromStream(rawData, (capacity / outputChannelCount)); return queueInputBufferInEncoder(rawData, encoderInBuffIndex); } private boolean queueInputBufferInEncoder(final short[] rawData, final int encoderInBuffIndex) { - final ShortBuffer outBuffer = encoder.getInputBuffer(encoderInBuffIndex).asShortBuffer(); outBuffer.clear(); if (rawData != null) { - outBuffer.put(rawData); totalDataAdded += rawData.length; @@ -207,7 +192,6 @@ private boolean queueInputBufferInEncoder(final short[] rawData, final int encod } private void writeToSonicSteam(final ShortBuffer data) { - short[] temBuff = new short[data.capacity()]; data.get(temBuff); data.rewind(); diff --git a/mp4compose/src/main/java/com/daasuu/mp4compose/composer/AudioComposer.kt b/mp4compose/src/main/java/com/daasuu/mp4compose/composer/AudioComposer.kt index b6172e099..6ffb75feb 100755 --- a/mp4compose/src/main/java/com/daasuu/mp4compose/composer/AudioComposer.kt +++ b/mp4compose/src/main/java/com/daasuu/mp4compose/composer/AudioComposer.kt @@ -56,8 +56,8 @@ internal class AudioComposer( buffer.clear() val sampleSize = mediaExtractor.readSampleData(buffer, 0) if (useFallBacks && sampleSize > bufferSize) { - bufferSize = 2 * sampleSize; - buffer = ByteBuffer.allocateDirect(bufferSize).order(ByteOrder.nativeOrder()); + bufferSize = 2 * sampleSize + buffer = ByteBuffer.allocateDirect(bufferSize).order(ByteOrder.nativeOrder()) } else { assert(sampleSize <= bufferSize) } diff --git a/mp4compose/src/main/java/com/daasuu/mp4compose/composer/BaseAudioChannel.java b/mp4compose/src/main/java/com/daasuu/mp4compose/composer/BaseAudioChannel.java index ed7f7435a..dd933e341 100644 --- a/mp4compose/src/main/java/com/daasuu/mp4compose/composer/BaseAudioChannel.java +++ b/mp4compose/src/main/java/com/daasuu/mp4compose/composer/BaseAudioChannel.java @@ -10,21 +10,20 @@ /** * Created by TAPOS DATTA on 22,May,2020 */ - +@SuppressWarnings("MemberName") abstract class BaseAudioChannel { - protected static class AudioBuffer { int bufferIndex; long presentationTimeUs; ShortBuffer data; } - protected static class BufferInfo{ + protected static class BufferInfo { long totaldata; long presentationTimeUs; } static final int BUFFER_INDEX_END_OF_STREAM = -1; - protected static final int BYTE_PER_SAMPLE = 16 / 8 ; + protected static final int BYTE_PER_SAMPLE = 16 / 8; protected static final int BYTES_PER_SHORT = 2; protected static final long MICROSECS_PER_SEC = 1000000; @@ -68,9 +67,9 @@ public void setActualDecodedFormat(final MediaFormat decodedFormat) { overflowBuffer.presentationTimeUs = 0; } - protected abstract long sampleCountToDurationUs(final long sampleCount, final int sampleRate, final int channelCount); + protected abstract long sampleCountToDurationUs(long sampleCount, int sampleRate, int channelCount); - protected abstract void drainDecoderBufferAndQueue(final int bufferIndex, final long presentationTimeUs); + protected abstract void drainDecoderBufferAndQueue(int bufferIndex, long presentationTimeUs); protected abstract boolean feedEncoder(long timeoutUs); } diff --git a/mp4compose/src/main/java/com/daasuu/mp4compose/composer/ComposerProvider.kt b/mp4compose/src/main/java/com/daasuu/mp4compose/composer/ComposerProvider.kt index a9dbb2ff7..1dce27bd4 100644 --- a/mp4compose/src/main/java/com/daasuu/mp4compose/composer/ComposerProvider.kt +++ b/mp4compose/src/main/java/com/daasuu/mp4compose/composer/ComposerProvider.kt @@ -11,7 +11,6 @@ import com.daasuu.mp4compose.composer.ComposerUseCase.CompressVideo import com.daasuu.mp4compose.composer.ComposerUseCase.SaveVideoAsFile import com.daasuu.mp4compose.composer.ComposerUseCase.SaveVideoFromBgAsFile import com.daasuu.mp4compose.filter.GlFilter -import com.daasuu.mp4compose.source.DataSource sealed class ComposerUseCase { data class SaveVideoAsFile( diff --git a/mp4compose/src/main/java/com/daasuu/mp4compose/composer/Mp4Composer.kt b/mp4compose/src/main/java/com/daasuu/mp4compose/composer/Mp4Composer.kt index c60b51392..4dfcf70bc 100644 --- a/mp4compose/src/main/java/com/daasuu/mp4compose/composer/Mp4Composer.kt +++ b/mp4compose/src/main/java/com/daasuu/mp4compose/composer/Mp4Composer.kt @@ -21,7 +21,7 @@ import java.util.concurrent.Executors * Created by sudamasayuki on 2017/11/15. */ -class Mp4Composer: ComposerInterface { +class Mp4Composer : ComposerInterface { private val srcUri: Uri? private val destPath: String private var filter: GlFilter? = null diff --git a/mp4compose/src/main/java/com/daasuu/mp4compose/composer/Mp4ComposerBasic.java b/mp4compose/src/main/java/com/daasuu/mp4compose/composer/Mp4ComposerBasic.java index 6e4d144bd..f138cf1f0 100644 --- a/mp4compose/src/main/java/com/daasuu/mp4compose/composer/Mp4ComposerBasic.java +++ b/mp4compose/src/main/java/com/daasuu/mp4compose/composer/Mp4ComposerBasic.java @@ -4,8 +4,10 @@ import android.media.MediaCodecInfo; import android.media.MediaMetadataRetriever; import android.util.Size; + import androidx.annotation.NonNull; import androidx.annotation.Nullable; + import com.daasuu.mp4compose.FillMode; import com.daasuu.mp4compose.FillModeCustomItem; import com.daasuu.mp4compose.Rotation; @@ -23,9 +25,9 @@ /** * Created by sudamasayuki on 2017/11/15. */ - +@SuppressWarnings("MemberName") public class Mp4ComposerBasic implements ComposerInterface { - private final static String TAG = Mp4Composer.class.getSimpleName(); + private static final String TAG = Mp4Composer.class.getSimpleName(); private final DataSource srcDataSource; private final String destPath; @@ -158,7 +160,7 @@ public Mp4ComposerBasic timeScale(final float timeScale) { return this; } - public Mp4ComposerBasic changePitch(final boolean isPitchChanged){ + public Mp4ComposerBasic changePitch(final boolean isPitchChanged) { this.isPitchChanged = isPitchChanged; return this; } @@ -206,7 +208,7 @@ private ExecutorService getExecutorService() { public Mp4ComposerBasic start() { - //if we're already composing, calling this should do nothing + // if we're already composing, calling this should do nothing if (engine != null) { return this; } @@ -271,7 +273,7 @@ public void onProgress(final double progress) { if (timeScale < 0.125f) { timeScale = 0.125f; - }else if(timeScale > 8f){ + } else if (timeScale > 8f) { timeScale = 8f; } @@ -322,7 +324,6 @@ public void onProgress(final double progress) { aacProfile, forceAudioEncoding ); - } catch (Exception e) { if (e instanceof MediaCodec.CodecException) { logger.error( diff --git a/mp4compose/src/main/java/com/daasuu/mp4compose/composer/Mp4ComposerEngineBasic.java b/mp4compose/src/main/java/com/daasuu/mp4compose/composer/Mp4ComposerEngineBasic.java index 91cb57e2a..16a7dbfb3 100644 --- a/mp4compose/src/main/java/com/daasuu/mp4compose/composer/Mp4ComposerEngineBasic.java +++ b/mp4compose/src/main/java/com/daasuu/mp4compose/composer/Mp4ComposerEngineBasic.java @@ -1,9 +1,16 @@ package com.daasuu.mp4compose.composer; -import android.media.*; +import android.media.MediaCodecInfo; +import android.media.MediaCodecList; +import android.media.MediaExtractor; +import android.media.MediaFormat; +import android.media.MediaMetadataRetriever; +import android.media.MediaMuxer; import android.os.Build; import android.util.Size; + import androidx.annotation.NonNull; + import com.daasuu.mp4compose.FillMode; import com.daasuu.mp4compose.FillModeCustomItem; import com.daasuu.mp4compose.Rotation; @@ -15,11 +22,13 @@ import java.io.FileDescriptor; import java.io.IOException; -// Refer: https://github.com/ypresto/android-transcoder/blob/master/lib/src/main/java/net/ypresto/androidtranscoder/engine/MediaTranscoderEngine.java +// Refer: https://github.com/ypresto/android-transcoder/blob/master/lib/src/main/ +// java/net/ypresto/androidtranscoder/engine/MediaTranscoderEngine.java /** * Internal engine, do not use this directly. */ +@SuppressWarnings("MemberName") class Mp4ComposerEngineBasic { private static final String TAG = "Mp4ComposerEngine"; private static final String AUDIO_PREFIX = "audio/"; @@ -69,7 +78,6 @@ void compose( final int aacProfile, final boolean forceAudioEncoding ) throws IOException { - try { mediaExtractor = new MediaExtractor(); mediaExtractor.setDataSource(srcDataSource.getFileDescriptor()); @@ -128,7 +136,7 @@ void compose( videoTrackIndex, actualVideoOutputFormat, muxRender, - (int)timeScale + (int) timeScale ); videoComposer.setUp( filter, @@ -143,12 +151,20 @@ void compose( mediaExtractor.selectTrack(videoTrackIndex); // setup audio if present and not muted - if (audioTrackIndex >= 0 && mediaMetadataRetriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_HAS_AUDIO) != null && !mute) { + if (audioTrackIndex >= 0 + && mediaMetadataRetriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_HAS_AUDIO) != null + && !mute + ) { // has Audio video final MediaFormat inputMediaFormat = mediaExtractor.getTrackFormat(audioTrackIndex); - final MediaFormat outputMediaFormat = createAudioOutputFormat(inputMediaFormat, audioBitRate, aacProfile, forceAudioEncoding); + final MediaFormat outputMediaFormat = createAudioOutputFormat( + inputMediaFormat, + audioBitRate, + aacProfile, + forceAudioEncoding + ); - if( timeScale >= 0.99 && timeScale <= 1.01 && outputMediaFormat.equals(inputMediaFormat)) { + if (timeScale >= 0.99 && timeScale <= 1.01 && outputMediaFormat.equals(inputMediaFormat)) { audioComposer = new AudioComposer( mediaExtractor, audioTrackIndex, @@ -310,7 +326,8 @@ private static MediaFormat createVideoFormat(@NonNull final String mimeType, outputFormat.setInteger(MediaFormat.KEY_BIT_RATE, bitrate); // On Build.VERSION_CODES.LOLLIPOP, format must not contain a MediaFormat#KEY_FRAME_RATE. - // https://developer.android.com/reference/android/media/MediaCodecInfo.CodecCapabilities.html#isFormatSupported(android.media.MediaFormat) + // https://developer.android.com/reference/android/media/MediaCodecInfo.CodecCapabilities.html + // #isFormatSupported(android.media.MediaFormat) if (Build.VERSION.SDK_INT != Build.VERSION_CODES.LOLLIPOP) { // Required but ignored by the encoder outputFormat.setInteger(MediaFormat.KEY_FRAME_RATE, 30); @@ -328,7 +345,7 @@ private void runPipelines() { if (durationUs <= 0) { if (progressCallback != null) { progressCallback.onProgress(PROGRESS_UNKNOWN); - }// unknown + } // unknown } while (!canceled && !(videoComposer.isFinished() && audioComposer.isFinished())) { boolean stepped = videoComposer.stepPipeline() diff --git a/mp4compose/src/main/java/com/daasuu/mp4compose/composer/RemixAudioComposerBasic.java b/mp4compose/src/main/java/com/daasuu/mp4compose/composer/RemixAudioComposerBasic.java index d91d55c98..9565b3c5e 100644 --- a/mp4compose/src/main/java/com/daasuu/mp4compose/composer/RemixAudioComposerBasic.java +++ b/mp4compose/src/main/java/com/daasuu/mp4compose/composer/RemixAudioComposerBasic.java @@ -1,10 +1,11 @@ package com.daasuu.mp4compose.composer; + /** - * // Refer: https://github.com/ypresto/android-transcoder/blob/master/lib/src/main/java/net/ypresto/androidtranscoder/engine/AudioTrackTranscoder.java + * // Refer: https://github.com/ypresto/android-transcoder/blob/master/lib/src/main/ + * java/net/ypresto/androidtranscoder/engine/AudioTrackTranscoder.java * */ - import android.media.MediaCodec; import android.media.MediaExtractor; import android.media.MediaFormat; @@ -14,7 +15,7 @@ import java.io.IOException; import java.util.concurrent.TimeUnit; - +@SuppressWarnings({"FallThrough", "MemberName"}) class RemixAudioComposerBasic implements IAudioComposer { private static final SampleType SAMPLE_TYPE = SampleType.AUDIO; @@ -89,7 +90,7 @@ public void setup() { decoder.start(); decoderStarted = true; - audioChannel = new AudioChannelWithSP(decoder, encoder, outputFormat,timeScale,isPitchChanged); + audioChannel = new AudioChannelWithSP(decoder, encoder, outputFormat, timeScale, isPitchChanged); } @Override @@ -100,7 +101,7 @@ public boolean stepPipeline() { while (drainEncoder(0) != DRAIN_STATE_NONE) busy = true; do { - if(audioChannel.isAnyPendingBuffIndex()){ + if (audioChannel.isAnyPendingBuffIndex()) { break; } status = drainDecoder(0); @@ -132,9 +133,15 @@ private int drainExtractor(long timeoutUs) { final int sampleSize = extractor.readSampleData(decoder.getInputBuffer(result), 0); final boolean isKeyFrame = (extractor.getSampleFlags() & MediaExtractor.SAMPLE_FLAG_SYNC) != 0; - decoder.queueInputBuffer(result, 0, sampleSize, extractor.getSampleTime(), isKeyFrame ? MediaCodec.BUFFER_FLAG_KEY_FRAME : 0); + decoder.queueInputBuffer( + result, + 0, + sampleSize, + extractor.getSampleTime(), + isKeyFrame ? MediaCodec.BUFFER_FLAG_KEY_FRAME : 0 + ); extractor.advance(); - numTracks ++ ; + numTracks++; return DRAIN_STATE_CONSUMED; } @@ -155,7 +162,7 @@ private int drainDecoder(long timeoutUs) { isDecoderEOS = true; audioChannel.drainDecoderBufferAndQueue(BaseAudioChannel.BUFFER_INDEX_END_OF_STREAM, 0); } else if (bufferInfo.size > 0) { - audioChannel.drainDecoderBufferAndQueue(result,bufferInfo.presentationTimeUs); + audioChannel.drainDecoderBufferAndQueue(result, bufferInfo.presentationTimeUs); } return DRAIN_STATE_CONSUMED; @@ -172,7 +179,9 @@ private int drainEncoder(long timeoutUs) { throw new RuntimeException("Audio output format changed twice."); } actualOutputFormat = encoder.getOutputFormat(); - addPrimingDelay = MediaFormat.MIMETYPE_AUDIO_AAC.equals(actualOutputFormat.getString(MediaFormat.KEY_MIME)); + addPrimingDelay = MediaFormat.MIMETYPE_AUDIO_AAC.equals( + actualOutputFormat.getString(MediaFormat.KEY_MIME) + ); muxer.setOutputFormat(SAMPLE_TYPE, actualOutputFormat); return DRAIN_STATE_SHOULD_RETRY_IMMEDIATELY; case MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED: @@ -202,7 +211,7 @@ private int drainEncoder(long timeoutUs) { @Override public long getWrittenPresentationTimeUs() { - return (long)(writtenPresentationTimeUs * timeScale); + return (long) (writtenPresentationTimeUs * timeScale); } @Override diff --git a/mp4compose/src/main/java/com/daasuu/mp4compose/composer/SonicAudioProcessor.java b/mp4compose/src/main/java/com/daasuu/mp4compose/composer/SonicAudioProcessor.java index 2b5f7c787..f6dc46ab8 100644 --- a/mp4compose/src/main/java/com/daasuu/mp4compose/composer/SonicAudioProcessor.java +++ b/mp4compose/src/main/java/com/daasuu/mp4compose/composer/SonicAudioProcessor.java @@ -18,18 +18,17 @@ * * */ - +@SuppressWarnings("MemberName") class SonicAudioProcessor { - private static final int SONIC_MIN_PITCH = 65; private static final int SONIC_MAX_PITCH = 400; // This is used to down-sample some inputs to improve speed private static final int SONIC_AMDF_FREQ = 4000; - private short inputBuffer[]; - private short outputBuffer[]; - private short pitchBuffer[]; - private short downSampleBuffer[]; + private short[] inputBuffer; + private short[] outputBuffer; + private short[] pitchBuffer; + private short[] downSampleBuffer; private float speed; private float volume; private float pitch; @@ -58,8 +57,8 @@ class SonicAudioProcessor { // Create a sonic stream. SonicAudioProcessor( int sampleRate, - int numChannels) - { + int numChannels + ) { allocateStreamBuffers(sampleRate, numChannels); speed = 1.0f; pitch = 1.0f; @@ -74,11 +73,11 @@ class SonicAudioProcessor { // Resize the array. private short[] resize( short[] oldArray, - int newLength) - { + int newLength + ) { newLength *= numChannels; - short[] newArray = new short[newLength]; - int length = oldArray.length <= newLength? oldArray.length : newLength; + short[] newArray = new short[newLength]; + int length = oldArray.length <= newLength ? oldArray.length : newLength; System.arraycopy(oldArray, 0, newArray, 0, length); return newArray; @@ -86,133 +85,127 @@ private short[] resize( // Move samples from one array to another. May move samples down within an array, but not up. private void move( - short dest[], + short[] dest, int destPos, - short source[], + short[] source, int sourcePos, - int numSamples) - { - System.arraycopy(source, sourcePos*numChannels, dest, destPos*numChannels, numSamples*numChannels); + int numSamples + ) { + System.arraycopy(source, sourcePos * numChannels, dest, destPos * numChannels, numSamples * numChannels); } // Scale the samples by the factor. private void scaleSamples( - short samples[], + short[] samples, int position, int numSamples, - float volume) - { - int fixedPointVolume = (int)(volume*4096.0f); - int start = position*numChannels; - int stop = start + numSamples*numChannels; - - for(int xSample = start; xSample < stop; xSample++) { - int value = (samples[xSample]*fixedPointVolume) >> 12; - if(value > 32767) { + float volume + ) { + int fixedPointVolume = (int) (volume * 4096.0f); + int start = position * numChannels; + int stop = start + numSamples * numChannels; + + for (int xSample = start; xSample < stop; xSample++) { + int value = (samples[xSample] * fixedPointVolume) >> 12; + if (value > 32767) { value = 32767; - } else if(value < -32767) { + } else if (value < -32767) { value = -32767; } - samples[xSample] = (short)value; + samples[xSample] = (short) value; } } // Get the speed of the stream. - public float getSpeed() - { + public float getSpeed() { return speed; } // Set the speed of the stream. public void setSpeed( - float speed) - { + float speed + ) { this.speed = speed; } // Get the pitch of the stream. - public float getPitch() - { + public float getPitch() { return pitch; } // Set the pitch of the stream. public void setPitch( - float pitch) - { + float pitch + ) { this.pitch = pitch; } // Get the rate of the stream. - public float getRate() - { + public float getRate() { return rate; } // Set the playback rate of the stream. This scales pitch and speed at the same time. public void setRate( - float rate) - { + float rate + ) { this.rate = rate; this.oldRatePosition = 0; this.newRatePosition = 0; } // Get the vocal chord pitch setting. - public boolean getChordPitch() - { + public boolean getChordPitch() { return useChordPitch; } // Set the vocal chord mode for pitch computation. Default is off. public void setChordPitch( - boolean useChordPitch) - { + boolean useChordPitch + ) { this.useChordPitch = useChordPitch; } // Get the quality setting. - public int getQuality() - { + public int getQuality() { return quality; } // Set the "quality". Default 0 is virtually as good as 1, but very much faster. public void setQuality( - int quality) - { + int quality + ) { this.quality = quality; } // Get the scaling factor of the stream. - public float getVolume() - { + public float getVolume() { return volume; } // Set the scaling factor of the stream. public void setVolume( - float volume) - { + float volume + ) { this.volume = volume; } // Allocate stream buffers. private void allocateStreamBuffers( int sampleRate, - int numChannels) - { + int numChannels + ) { this.sampleRate = sampleRate; this.numChannels = numChannels; - minPeriod = sampleRate/SONIC_MAX_PITCH; - maxPeriod = sampleRate/SONIC_MIN_PITCH; - maxRequired = 2*maxPeriod; + minPeriod = sampleRate / SONIC_MAX_PITCH; + maxPeriod = sampleRate / SONIC_MIN_PITCH; + maxRequired = 2 * maxPeriod; inputBufferSize = maxRequired; - inputBuffer = new short[maxRequired*numChannels]; + inputBuffer = new short[maxRequired * numChannels]; outputBufferSize = maxRequired; - outputBuffer = new short[maxRequired*numChannels]; + outputBuffer = new short[maxRequired * numChannels]; pitchBufferSize = maxRequired; - pitchBuffer = new short[maxRequired*numChannels]; + pitchBuffer = new short[maxRequired * numChannels]; downSampleBuffer = new short[maxRequired]; oldRatePosition = 0; newRatePosition = 0; @@ -220,22 +213,20 @@ private void allocateStreamBuffers( } // Get the sample rate of the stream. - public int getSampleRate() - { + public int getSampleRate() { return sampleRate; } // Get the number of channels. - public int getNumChannels() - { + public int getNumChannels() { return numChannels; } // Enlarge the output buffer if needed. private void enlargeOutputBufferIfNeeded( - int numSamples) - { - if(numOutputSamples + numSamples > outputBufferSize) { + int numSamples + ) { + if (numOutputSamples + numSamples > outputBufferSize) { outputBufferSize += (outputBufferSize >> 1) + numSamples; outputBuffer = resize(outputBuffer, outputBufferSize); } @@ -243,9 +234,9 @@ private void enlargeOutputBufferIfNeeded( // Enlarge the input buffer if needed. private void enlargeInputBufferIfNeeded( - int numSamples) - { - if(numInputSamples + numSamples > inputBufferSize) { + int numSamples + ) { + if (numInputSamples + numSamples > inputBufferSize) { inputBufferSize += (inputBufferSize >> 1) + numSamples; inputBuffer = resize(inputBuffer, inputBufferSize); } @@ -253,26 +244,26 @@ private void enlargeInputBufferIfNeeded( // Add the input samples to the input buffer. private void addFloatSamplesToInputBuffer( - float samples[], - int numSamples) - { - if(numSamples == 0) { + float[] samples, + int numSamples + ) { + if (numSamples == 0) { return; } enlargeInputBufferIfNeeded(numSamples); - int xBuffer = numInputSamples*numChannels; - for(int xSample = 0; xSample < numSamples*numChannels; xSample++) { - inputBuffer[xBuffer++] = (short)(samples[xSample]*32767.0f); + int xBuffer = numInputSamples * numChannels; + for (int xSample = 0; xSample < numSamples * numChannels; xSample++) { + inputBuffer[xBuffer++] = (short) (samples[xSample] * 32767.0f); } numInputSamples += numSamples; } // Add the input samples to the input buffer. private void addShortSamplesToInputBuffer( - short samples[], - int numSamples) - { - if(numSamples == 0) { + short[] samples, + int numSamples + ) { + if (numSamples == 0) { return; } enlargeInputBufferIfNeeded(numSamples); @@ -282,15 +273,15 @@ private void addShortSamplesToInputBuffer( // Add the input samples to the input buffer. private void addUnsignedByteSamplesToInputBuffer( - byte samples[], - int numSamples) - { + byte[] samples, + int numSamples + ) { short sample; enlargeInputBufferIfNeeded(numSamples); - int xBuffer = numInputSamples*numChannels; - for(int xSample = 0; xSample < numSamples*numChannels; xSample++) { - sample = (short)((samples[xSample] & 0xff) - 128); // Convert from unsigned to signed + int xBuffer = numInputSamples * numChannels; + for (int xSample = 0; xSample < numSamples * numChannels; xSample++) { + sample = (short) ((samples[xSample] & 0xff) - 128); // Convert from unsigned to signed inputBuffer[xBuffer++] = (short) (sample << 8); } numInputSamples += numSamples; @@ -298,16 +289,16 @@ private void addUnsignedByteSamplesToInputBuffer( // Add the input samples to the input buffer. They must be 16-bit little-endian encoded in a byte array. private void addBytesToInputBuffer( - byte inBuffer[], - int numBytes) - { - int numSamples = numBytes/(2*numChannels); + byte[] inBuffer, + int numBytes + ) { + int numSamples = numBytes / (2 * numChannels); short sample; enlargeInputBufferIfNeeded(numSamples); - int xBuffer = numInputSamples*numChannels; - for(int xByte = 0; xByte + 1 < numBytes; xByte += 2) { - sample = (short)((inBuffer[xByte] & 0xff) | (inBuffer[xByte + 1] << 8)); + int xBuffer = numInputSamples * numChannels; + for (int xByte = 0; xByte + 1 < numBytes; xByte += 2) { + sample = (short) ((inBuffer[xByte] & 0xff) | (inBuffer[xByte + 1] << 8)); inputBuffer[xBuffer++] = sample; } numInputSamples += numSamples; @@ -315,8 +306,8 @@ private void addBytesToInputBuffer( // Remove input samples that we have already processed. private void removeInputSamples( - int position) - { + int position + ) { int remainingSamples = numInputSamples - position; move(inputBuffer, 0, inputBuffer, position, remainingSamples); @@ -325,10 +316,10 @@ private void removeInputSamples( // Just copy from the array to the output buffer private void copyToOutput( - short samples[], + short[] samples, int position, - int numSamples) - { + int numSamples + ) { enlargeOutputBufferIfNeeded(numSamples); move(outputBuffer, numOutputSamples, samples, position, numSamples); numOutputSamples += numSamples; @@ -336,9 +327,9 @@ private void copyToOutput( // Just copy from the input buffer to the output buffer. Return num samples copied. private int copyInputToOutput( - int position) - { - int numSamples = Math.min(maxRequired,remainingInputToCopy); + int position + ) { + int numSamples = Math.min(maxRequired, remainingInputToCopy); copyToOutput(inputBuffer, position, numSamples); remainingInputToCopy -= numSamples; @@ -348,21 +339,21 @@ private int copyInputToOutput( // Read data out of the stream. Sometimes no data will be available, and zero // is returned, which is not an error condition. private int readFloatFromStream( - float samples[], - int maxSamples) - { + float[] samples, + int maxSamples + ) { int numSamples = numOutputSamples; int remainingSamples = 0; - if(numSamples == 0) { + if (numSamples == 0) { return 0; } - if(numSamples > maxSamples) { + if (numSamples > maxSamples) { remainingSamples = numSamples - maxSamples; numSamples = maxSamples; } - for(int xSample = 0; xSample < numSamples*numChannels; xSample++) { - samples[xSample] = (outputBuffer[xSample])/32767.0f; + for (int xSample = 0; xSample < numSamples * numChannels; xSample++) { + samples[xSample] = (outputBuffer[xSample]) / 32767.0f; } move(outputBuffer, 0, outputBuffer, numSamples, remainingSamples); numOutputSamples = remainingSamples; @@ -372,16 +363,16 @@ private int readFloatFromStream( // Read short data out of the stream. Sometimes no data will be available, and zero // is returned, which is not an error condition. public int readShortFromStream( - short samples[], - int maxSamples) - { + short[] samples, + int maxSamples + ) { int numSamples = numOutputSamples; int remainingSamples = 0; - if(numSamples == 0) { + if (numSamples == 0) { return 0; } - if(numSamples > maxSamples) { + if (numSamples > maxSamples) { remainingSamples = numSamples - maxSamples; numSamples = maxSamples; } @@ -394,21 +385,21 @@ public int readShortFromStream( // Read unsigned byte data out of the stream. Sometimes no data will be available, and zero // is returned, which is not an error condition. private int readUnsignedByteFromStream( - byte samples[], - int maxSamples) - { + byte[] samples, + int maxSamples + ) { int numSamples = numOutputSamples; int remainingSamples = 0; - if(numSamples == 0) { + if (numSamples == 0) { return 0; } - if(numSamples > maxSamples) { + if (numSamples > maxSamples) { remainingSamples = numSamples - maxSamples; numSamples = maxSamples; } - for(int xSample = 0; xSample < numSamples*numChannels; xSample++) { - samples[xSample] = (byte)((outputBuffer[xSample] >> 8) + 128); + for (int xSample = 0; xSample < numSamples * numChannels; xSample++) { + samples[xSample] = (byte) ((outputBuffer[xSample] >> 8) + 128); } move(outputBuffer, 0, outputBuffer, numSamples, remainingSamples); numOutputSamples = remainingSamples; @@ -418,49 +409,48 @@ private int readUnsignedByteFromStream( // Read unsigned byte data out of the stream. Sometimes no data will be available, and zero // is returned, which is not an error condition. private int readBytesFromStream( - byte outBuffer[], - int maxBytes) - { - int maxSamples = maxBytes/(2*numChannels); + byte[] outBuffer, + int maxBytes + ) { + int maxSamples = maxBytes / (2 * numChannels); int numSamples = numOutputSamples; int remainingSamples = 0; - if(numSamples == 0 || maxSamples == 0) { + if (numSamples == 0 || maxSamples == 0) { return 0; } - if(numSamples > maxSamples) { + if (numSamples > maxSamples) { remainingSamples = numSamples - maxSamples; numSamples = maxSamples; } - for(int xSample = 0; xSample < numSamples*numChannels; xSample++) { + for (int xSample = 0; xSample < numSamples * numChannels; xSample++) { short sample = outputBuffer[xSample]; - outBuffer[xSample << 1] = (byte)(sample & 0xff); - outBuffer[(xSample << 1) + 1] = (byte)(sample >> 8); + outBuffer[xSample << 1] = (byte) (sample & 0xff); + outBuffer[(xSample << 1) + 1] = (byte) (sample >> 8); } move(outputBuffer, 0, outputBuffer, numSamples, remainingSamples); numOutputSamples = remainingSamples; - return 2*numSamples*numChannels; + return 2 * numSamples * numChannels; } // Force the sonic stream to generate output using whatever data it currently // has. No extra delay will be added to the output, but flushing in the middle of // words could introduce distortion. - public void flushStream() - { + public void flushStream() { int remainingSamples = numInputSamples; - float s = speed/pitch; + float s = speed / pitch; float r = rate * pitch; - int expectedOutputSamples = numOutputSamples + (int)((remainingSamples / s + numPitchSamples) / r + 0.5f); + int expectedOutputSamples = numOutputSamples + (int) ((remainingSamples / s + numPitchSamples) / r + 0.5f); // // Add enough silence to flush both input and pitch buffers. enlargeInputBufferIfNeeded(remainingSamples + 2 * maxRequired); - for(int xSample = 0; xSample < 2 * maxRequired * numChannels; xSample++) { + for (int xSample = 0; xSample < 2 * maxRequired * numChannels; xSample++) { inputBuffer[remainingSamples * numChannels + xSample] = 0; } numInputSamples += 2 * maxRequired; writeShortToStream(null, 0); // Throw away any extra samples we generated due to the silence we added. - if(numOutputSamples > expectedOutputSamples) { + if (numOutputSamples > expectedOutputSamples) { numOutputSamples = expectedOutputSamples; } // Empty input and pitch buffers. @@ -470,8 +460,7 @@ public void flushStream() } // Return the number of samples in the output buffer - public int samplesAvailable() - { + public int samplesAvailable() { return numOutputSamples; } @@ -479,18 +468,18 @@ public int samplesAvailable() // the down-sample buffer. If numChannels is greater than one, mix the channels // together as we down sample. private void downSampleInput( - short samples[], + short[] samples, int position, - int skip) - { - int numSamples = maxRequired/skip; - int samplesPerValue = numChannels*skip; + int skip + ) { + int numSamples = maxRequired / skip; + int samplesPerValue = numChannels * skip; int value; position *= numChannels; - for(int i = 0; i < numSamples; i++) { + for (int i = 0; i < numSamples; i++) { value = 0; - for(int j = 0; j < samplesPerValue; j++) { + for (int j = 0; j < samplesPerValue; j++) { value += samples[position + i * samplesPerValue + j]; } value /= samplesPerValue; @@ -501,20 +490,20 @@ private void downSampleInput( // Find the best frequency match in the range, and given a sample skip multiple. // For now, just find the pitch of the first channel. private int findPitchPeriodInRange( - short samples[], + short[] samples, int position, int minPeriod, - int maxPeriod) - { + int maxPeriod + ) { int bestPeriod = 0; int worstPeriod = 255; int minDiff = 1; int maxDiff = 0; position *= numChannels; - for(int period = minPeriod; period <= maxPeriod; period++) { + for (int period = minPeriod; period <= maxPeriod; period++) { int diff = 0; - for(int i = 0; i < period; i++) { + for (int i = 0; i < period; i++) { short sVal = samples[position + i]; short pVal = samples[position + period + i]; diff += (sVal >= pVal) ? sVal - pVal : pVal - sVal; @@ -522,17 +511,17 @@ private int findPitchPeriodInRange( /* Note that the highest number of samples we add into diff will be less than 256, since we skip samples. Thus, diff is a 24 bit number, and we can safely multiply by numSamples without overflow */ - if(diff * bestPeriod < minDiff * period) { + if (diff * bestPeriod < minDiff * period) { minDiff = diff; bestPeriod = period; } - if(diff * worstPeriod > maxDiff * period) { + if (diff * worstPeriod > maxDiff * period) { maxDiff = diff; worstPeriod = period; } } - this.minDiff = minDiff/bestPeriod; - this.maxDiff = maxDiff/worstPeriod; + this.minDiff = minDiff / bestPeriod; + this.maxDiff = maxDiff / worstPeriod; return bestPeriod; } @@ -542,22 +531,22 @@ private int findPitchPeriodInRange( private boolean prevPeriodBetter( int minDiff, int maxDiff, - boolean preferNewPeriod) - { - if(minDiff == 0 || prevPeriod == 0) { + boolean preferNewPeriod + ) { + if (minDiff == 0 || prevPeriod == 0) { return false; } - if(preferNewPeriod) { - if(maxDiff > minDiff * 3) { + if (preferNewPeriod) { + if (maxDiff > minDiff * 3) { // Got a reasonable match this period return false; } - if(minDiff * 2 <= prevMinDiff * 3) { + if (minDiff * 2 <= prevMinDiff * 3) { // Mismatch is not that much greater this period return false; } } else { - if(minDiff <= prevMinDiff) { + if (minDiff <= prevMinDiff) { return false; } } @@ -569,33 +558,33 @@ private boolean prevPeriodBetter( // speed, we down sample by an integer factor get in the 11KHz range, and then // do it again with a narrower frequency range without down sampling private int findPitchPeriod( - short samples[], + short[] samples, int position, - boolean preferNewPeriod) - { + boolean preferNewPeriod + ) { int period, retPeriod; int skip = 1; - if(sampleRate > SONIC_AMDF_FREQ && quality == 0) { - skip = sampleRate/SONIC_AMDF_FREQ; + if (sampleRate > SONIC_AMDF_FREQ && quality == 0) { + skip = sampleRate / SONIC_AMDF_FREQ; } - if(numChannels == 1 && skip == 1) { + if (numChannels == 1 && skip == 1) { period = findPitchPeriodInRange(samples, position, minPeriod, maxPeriod); } else { downSampleInput(samples, position, skip); - period = findPitchPeriodInRange(downSampleBuffer, 0, minPeriod/skip, - maxPeriod/skip); - if(skip != 1) { + period = findPitchPeriodInRange(downSampleBuffer, 0, minPeriod / skip, + maxPeriod / skip); + if (skip != 1) { period *= skip; int minP = period - (skip << 2); int maxP = period + (skip << 2); - if(minP < minPeriod) { + if (minP < minPeriod) { minP = minPeriod; } - if(maxP > maxPeriod) { + if (maxP > maxPeriod) { maxP = maxPeriod; } - if(numChannels == 1) { + if (numChannels == 1) { period = findPitchPeriodInRange(samples, position, minP, maxP); } else { downSampleInput(samples, position, 1); @@ -603,7 +592,7 @@ private int findPitchPeriod( } } } - if(prevPeriodBetter(minDiff, maxDiff, preferNewPeriod)) { + if (prevPeriodBetter(minDiff, maxDiff, preferNewPeriod)) { retPeriod = prevPeriod; } else { retPeriod = period; @@ -624,7 +613,6 @@ private static void overlapAdd( int rampDownPosition, short[] rampUp, int rampUpPosition) { - for (int i = 0; i < channelCount; i++) { int o = outPosition * channelCount + i; int u = rampUpPosition * channelCount + i; @@ -644,30 +632,28 @@ private void overlapAddWithSeparation( int numSamples, int numChannels, int separation, - short out[], + short[] out, int outPos, - short rampDown[], + short[] rampDown, int rampDownPos, - short rampUp[], - int rampUpPos) - { - for(int i = 0; i < numChannels; i++) { - + short[] rampUp, + int rampUpPos + ) { + for (int i = 0; i < numChannels; i++) { int o = outPos * numChannels + i; int u = rampUpPos * numChannels + i; int d = rampDownPos * numChannels + i; - for(int t = 0; t < numSamples + separation; t++) { - - if(t < separation) { - out[o] = (short)(rampDown[d] * (numSamples - t) / numSamples); + for (int t = 0; t < numSamples + separation; t++) { + if (t < separation) { + out[o] = (short) (rampDown[d] * (numSamples - t) / numSamples); d += numChannels; - } else if(t < numSamples) { - out[o] = (short)((rampDown[d] * (numSamples - t) + rampUp[u] * (t - separation)) / numSamples); + } else if (t < numSamples) { + out[o] = (short) ((rampDown[d] * (numSamples - t) + rampUp[u] * (t - separation)) / numSamples); d += numChannels; u += numChannels; } else { - out[o] = (short)(rampUp[u] * (t - separation) / numSamples); + out[o] = (short) (rampUp[u] * (t - separation) / numSamples); u += numChannels; } o += numChannels; @@ -677,11 +663,11 @@ private void overlapAddWithSeparation( // Just move the new samples in the output buffer to the pitch buffer private void moveNewSamplesToPitchBuffer( - int originalNumOutputSamples) - { + int originalNumOutputSamples + ) { int numSamples = numOutputSamples - originalNumOutputSamples; - if(numPitchSamples + numSamples > pitchBufferSize) { + if (numPitchSamples + numSamples > pitchBufferSize) { pitchBufferSize += (pitchBufferSize >> 1) + numSamples; pitchBuffer = resize(pitchBuffer, pitchBufferSize); } @@ -692,9 +678,9 @@ private void moveNewSamplesToPitchBuffer( // Remove processed samples from the pitch buffer. private void removePitchSamples( - int numSamples) - { - if(numSamples == 0) { + int numSamples + ) { + if (numSamples == 0) { return; } move(pitchBuffer, 0, pitchBuffer, numSamples, numPitchSamples - numSamples); @@ -704,20 +690,20 @@ private void removePitchSamples( // Change the pitch. The latency this introduces could be reduced by looking at // past samples to determine pitch, rather than future. private void adjustPitch( - int originalNumOutputSamples) - { + int originalNumOutputSamples + ) { int period, newPeriod, separation; int position = 0; - if(numOutputSamples == originalNumOutputSamples) { + if (numOutputSamples == originalNumOutputSamples) { return; } moveNewSamplesToPitchBuffer(originalNumOutputSamples); - while(numPitchSamples - position >= maxRequired) { + while (numPitchSamples - position >= maxRequired) { period = findPitchPeriod(pitchBuffer, position, false); - newPeriod = (int)(period/pitch); + newPeriod = (int) (period / pitch); enlargeOutputBufferIfNeeded(newPeriod); - if(pitch >= 1.0f) { + if (pitch >= 1.0f) { overlapAdd(newPeriod, numChannels, outputBuffer, numOutputSamples, pitchBuffer, position, pitchBuffer, position + period - newPeriod); } else { @@ -735,16 +721,16 @@ private void adjustPitch( // Return 1 if value >= 0, else -1. This represents the sign of value. private int getSign(int value) { - return value >= 0? 1 : -1; + return value >= 0 ? 1 : -1; } // Interpolate the new output sample. private short interpolate( - short in[], + short[] in, int inPos, // Index to first sample which already includes channel offset. int oldSampleRate, - int newSampleRate) - { + int newSampleRate + ) { short left = in[inPos]; short right = in[inPos + numChannels]; int position = newRatePosition * oldSampleRate; @@ -758,38 +744,38 @@ private short interpolate( // Change the rate. private void adjustRate( float rate, - int originalNumOutputSamples) - { - if(numOutputSamples == originalNumOutputSamples) { + int originalNumOutputSamples + ) { + if (numOutputSamples == originalNumOutputSamples) { return; } - int newSampleRate = (int)(sampleRate/rate); + int newSampleRate = (int) (sampleRate / rate); int oldSampleRate = sampleRate; int position; // Set these values to help with the integer math - while(newSampleRate > (1 << 14) || oldSampleRate > (1 << 14)) { + while (newSampleRate > (1 << 14) || oldSampleRate > (1 << 14)) { newSampleRate >>= 1; oldSampleRate >>= 1; } moveNewSamplesToPitchBuffer(originalNumOutputSamples); // Leave at least one pitch sample in the buffer - for(position = 0; position < numPitchSamples - 1; position++) { - while((oldRatePosition + 1)*newSampleRate > newRatePosition*oldSampleRate) { + for (position = 0; position < numPitchSamples - 1; position++) { + while ((oldRatePosition + 1) * newSampleRate > newRatePosition * oldSampleRate) { enlargeOutputBufferIfNeeded(1); - for(int i = 0; i < numChannels; i++) { - outputBuffer[numOutputSamples*numChannels + i] = interpolate(pitchBuffer, + for (int i = 0; i < numChannels; i++) { + outputBuffer[numOutputSamples * numChannels + i] = interpolate(pitchBuffer, position * numChannels + i, oldSampleRate, newSampleRate); } newRatePosition++; numOutputSamples++; } oldRatePosition++; - if(oldRatePosition == oldSampleRate) { + if (oldRatePosition == oldSampleRate) { oldRatePosition = 0; - if(newRatePosition != newSampleRate) { + if (newRatePosition != newSampleRate) { System.out.printf("Assertion failed: newRatePosition != newSampleRate\n"); assert false; } @@ -802,18 +788,18 @@ private void adjustRate( // Skip over a pitch period, and copy period/speed samples to the output private int skipPitchPeriod( - short samples[], + short[] samples, int position, float speed, - int period) - { + int period + ) { int newSamples; - if(speed >= 2.0f) { - newSamples = (int)(period/(speed - 1.0f)); + if (speed >= 2.0f) { + newSamples = (int) (period / (speed - 1.0f)); } else { newSamples = period; - remainingInputToCopy = (int)(period*(2.0f - speed)/(speed - 1.0f)); + remainingInputToCopy = (int) (period * (2.0f - speed) / (speed - 1.0f)); } enlargeOutputBufferIfNeeded(newSamples); overlapAdd(newSamples, numChannels, outputBuffer, numOutputSamples, samples, position, @@ -824,18 +810,18 @@ private int skipPitchPeriod( // Insert a pitch period, and determine how much input to copy directly. private int insertPitchPeriod( - short samples[], + short[] samples, int position, float speed, - int period) - { + int period + ) { int newSamples; - if(speed < 0.5f) { - newSamples = (int)(period * speed /(1.0f - speed)); + if (speed < 0.5f) { + newSamples = (int) (period * speed / (1.0f - speed)); } else { newSamples = period; - remainingInputToCopy = (int)(period * (2.0f * speed - 1.0f) / (1.0f - speed)); + remainingInputToCopy = (int) (period * (2.0f * speed - 1.0f) / (1.0f - speed)); } enlargeOutputBufferIfNeeded(period + newSamples); move(outputBuffer, numOutputSamples, samples, position, period); @@ -870,29 +856,28 @@ private void changeSpeed(float speed) { } // Resample as many pitch periods as we have buffered on the input. Scale the output by the volume. - private void processStreamInput() - { + private void processStreamInput() { int originalNumOutputSamples = numOutputSamples; - float s = speed/pitch; + float s = speed / pitch; float r = rate; - if(!useChordPitch) { + if (!useChordPitch) { r *= pitch; } - if(s > 1.00001 || s < 0.99999) { + if (s > 1.00001 || s < 0.99999) { changeSpeed(s); } else { copyToOutput(inputBuffer, 0, numInputSamples); numInputSamples = 0; } - if(useChordPitch) { - if(pitch != 1.0f) { + if (useChordPitch) { + if (pitch != 1.0f) { adjustPitch(originalNumOutputSamples); } - } else if(r != 1.0f) { + } else if (r != 1.0f) { adjustRate(r, originalNumOutputSamples); } - if(volume != 1.0f) { + if (volume != 1.0f) { // Adjust output volume. scaleSamples(outputBuffer, originalNumOutputSamples, numOutputSamples - originalNumOutputSamples, volume); @@ -901,18 +886,18 @@ private void processStreamInput() // Write floating point data to the input buffer and process it. public void writeFloatToStream( - float samples[], - int numSamples) - { + float[] samples, + int numSamples + ) { addFloatSamplesToInputBuffer(samples, numSamples); processStreamInput(); } // Write the data to the input stream, and process it. public void writeShortToStream( - short samples[], - int numSamples) - { + short[] samples, + int numSamples + ) { addShortSamplesToInputBuffer(samples, numSamples); processStreamInput(); } @@ -920,25 +905,25 @@ public void writeShortToStream( // Simple wrapper around sonicWriteFloatToStream that does the unsigned byte to short // conversion for you. private void writeUnsignedByteToStream( - byte samples[], - int numSamples) - { + byte[] samples, + int numSamples + ) { addUnsignedByteSamplesToInputBuffer(samples, numSamples); processStreamInput(); } // Simple wrapper around sonicWriteBytesToStream that does the byte to 16-bit LE conversion. private void writeBytesToStream( - byte inBuffer[], - int numBytes) - { + byte[] inBuffer, + int numBytes + ) { addBytesToInputBuffer(inBuffer, numBytes); processStreamInput(); } // This is a non-stream oriented interface to just change the speed of a sound sample private static int changeFloatSpeed( - float samples[], + float[] samples, int numSamples, float speed, float pitch, @@ -946,8 +931,8 @@ private static int changeFloatSpeed( float volume, boolean useChordPitch, int sampleRate, - int numChannels) - { + int numChannels + ) { SonicAudioProcessor stream = new SonicAudioProcessor(sampleRate, numChannels); stream.setSpeed(speed); @@ -964,7 +949,7 @@ private static int changeFloatSpeed( /* This is a non-stream oriented interface to just change the speed of a sound sample */ private int sonicChangeShortSpeed( - short samples[], + short[] samples, int numSamples, float speed, float pitch, @@ -972,8 +957,8 @@ private int sonicChangeShortSpeed( float volume, boolean useChordPitch, int sampleRate, - int numChannels) - { + int numChannels + ) { SonicAudioProcessor stream = new SonicAudioProcessor(sampleRate, numChannels); stream.setSpeed(speed); diff --git a/mp4compose/src/main/java/com/daasuu/mp4compose/logger/AndroidLogger.java b/mp4compose/src/main/java/com/daasuu/mp4compose/logger/AndroidLogger.java index 7ed1fc8b2..3934114de 100644 --- a/mp4compose/src/main/java/com/daasuu/mp4compose/logger/AndroidLogger.java +++ b/mp4compose/src/main/java/com/daasuu/mp4compose/logger/AndroidLogger.java @@ -5,8 +5,7 @@ /** * The default implementation of the {@link Logger} for Android. */ -public class AndroidLogger implements Logger{ - +public class AndroidLogger implements Logger { @Override public void debug(String tag, String message) { Log.d(tag, message); @@ -21,5 +20,4 @@ public void error(String tag, String message, Throwable error) { public void warning(String tag, String message) { Log.w(tag, message); } - } diff --git a/mp4compose/src/main/java/com/daasuu/mp4compose/logger/Logger.java b/mp4compose/src/main/java/com/daasuu/mp4compose/logger/Logger.java index 000a804ef..3d7f25790 100644 --- a/mp4compose/src/main/java/com/daasuu/mp4compose/logger/Logger.java +++ b/mp4compose/src/main/java/com/daasuu/mp4compose/logger/Logger.java @@ -3,8 +3,8 @@ /** * The logger interface used to log information to the console. */ -public interface Logger { +public interface Logger { /** * Logs a debug message. * @@ -29,5 +29,4 @@ public interface Logger { * @param message The message body. */ void warning(String tag, String message); - } diff --git a/mp4compose/src/main/java/com/daasuu/mp4compose/source/FilePathDataSource.java b/mp4compose/src/main/java/com/daasuu/mp4compose/source/FilePathDataSource.java index 23440de16..43c7c4e04 100644 --- a/mp4compose/src/main/java/com/daasuu/mp4compose/source/FilePathDataSource.java +++ b/mp4compose/src/main/java/com/daasuu/mp4compose/source/FilePathDataSource.java @@ -10,14 +10,13 @@ import java.io.FileNotFoundException; import java.io.IOException; +@SuppressWarnings("MemberName") public class FilePathDataSource implements DataSource { - - private final static String TAG = FilePathDataSource.class.getSimpleName(); + private static final String TAG = FilePathDataSource.class.getSimpleName(); private FileDescriptor fileDescriptor; public FilePathDataSource(@NonNull String filePath, @NonNull Logger logger, @NonNull Listener listener) { - final File srcFile = new File(filePath); final FileInputStream fileInputStream; try { From d8e90a185079cfdcb0a18c02899be5681f42c5e6 Mon Sep 17 00:00:00 2001 From: develric Date: Thu, 20 May 2021 19:50:06 +0200 Subject: [PATCH 12/12] Adding comment to reference a GH issue relevant to time scaling functionality. --- .../java/com/daasuu/mp4compose/composer/Mp4ComposerBasic.java | 3 +++ .../com/daasuu/mp4compose/composer/Mp4ComposerEngineBasic.java | 3 +++ .../main/java/com/daasuu/mp4compose/composer/VideoComposer.kt | 3 +++ 3 files changed, 9 insertions(+) diff --git a/mp4compose/src/main/java/com/daasuu/mp4compose/composer/Mp4ComposerBasic.java b/mp4compose/src/main/java/com/daasuu/mp4compose/composer/Mp4ComposerBasic.java index f138cf1f0..1e2072108 100644 --- a/mp4compose/src/main/java/com/daasuu/mp4compose/composer/Mp4ComposerBasic.java +++ b/mp4compose/src/main/java/com/daasuu/mp4compose/composer/Mp4ComposerBasic.java @@ -44,6 +44,9 @@ public class Mp4ComposerBasic implements ComposerInterface { private Listener listener; private FillMode fillMode = FillMode.PRESERVE_ASPECT_FIT; private FillModeCustomItem fillModeCustomItem; + // TODO: currently we do not use the timeScale feature. Also the timeScale ends up + // being converted into an int in the VideoComposer layer. + // See https://github.com/Automattic/stories-android/issues/685 for more context. private float timeScale = 1f; // should be in range 0.125 (-8X) to 8.0 (8X) private boolean isPitchChanged = false; private boolean flipVertical = false; diff --git a/mp4compose/src/main/java/com/daasuu/mp4compose/composer/Mp4ComposerEngineBasic.java b/mp4compose/src/main/java/com/daasuu/mp4compose/composer/Mp4ComposerEngineBasic.java index 16a7dbfb3..9a755a25b 100644 --- a/mp4compose/src/main/java/com/daasuu/mp4compose/composer/Mp4ComposerEngineBasic.java +++ b/mp4compose/src/main/java/com/daasuu/mp4compose/composer/Mp4ComposerEngineBasic.java @@ -54,6 +54,9 @@ void setProgressCallback(ProgressCallback progressCallback) { this.progressCallback = progressCallback; } + // TODO: currently we do not use the timeScale feature. Also the timeScale ends up + // being converted into an int in the VideoComposer layer. + // See https://github.com/Automattic/stories-android/issues/685 for more context. void compose( final DataSource srcDataSource, final String destSrc, diff --git a/mp4compose/src/main/java/com/daasuu/mp4compose/composer/VideoComposer.kt b/mp4compose/src/main/java/com/daasuu/mp4compose/composer/VideoComposer.kt index e6dda6358..5459b4cb1 100755 --- a/mp4compose/src/main/java/com/daasuu/mp4compose/composer/VideoComposer.kt +++ b/mp4compose/src/main/java/com/daasuu/mp4compose/composer/VideoComposer.kt @@ -36,6 +36,9 @@ internal class VideoComposer { private var encoderStarted: Boolean = false var writtenPresentationTimeUs: Long = 0 private set + // TODO: currently we do not use the timeScale feature. Also the timeScale ends up + // being converted into an int in here being a float in upper layers. + // See https://github.com/Automattic/stories-android/issues/685 for more context. private val timeScale: Int private var useStaticBkg: Boolean = false private var addedFrameCount = 0