Merge pull request #652 from Automattic/try/using-lib-in-video-compre…

…ssion-scenario

Using lib in WPAndroid video compression scenario

mp4compose/src/main/java/com/daasuu/mp4compose/Rotation.kt

@@ -11,6 +11,7 @@ enum class Rotation private constructor(val rotation: Int) {
     ROTATION_270(270);
 
     companion object {
+        @JvmStatic
         fun fromInt(rotate: Int): Rotation {
             for (rotation in Rotation.values()) {
                 if (rotate == rotation.rotation) return rotation

mp4compose/src/main/java/com/daasuu/mp4compose/VideoFormatMimeType.java

@@ -0,0 +1,22 @@
+package com.daasuu.mp4compose;
+
+import android.media.MediaFormat;
+
+@SuppressWarnings("MemberName")
+public enum VideoFormatMimeType {
+    HEVC(MediaFormat.MIMETYPE_VIDEO_HEVC),
+    AVC(MediaFormat.MIMETYPE_VIDEO_AVC),
+    MPEG4(MediaFormat.MIMETYPE_VIDEO_MPEG4),
+    H263(MediaFormat.MIMETYPE_VIDEO_H263),
+    AUTO("");
+
+    private final String videoFormatMimeType;
+
+    VideoFormatMimeType(String videoFormatMimeType) {
+        this.videoFormatMimeType = videoFormatMimeType;
+    }
+
+    public String getFormat() {
+        return videoFormatMimeType;
+    }
+}

mp4compose/src/main/java/com/daasuu/mp4compose/composer/AudioChannelWithSP.java

@@ -0,0 +1,209 @@
+package com.daasuu.mp4compose.composer;
+
+import android.media.MediaCodec;
+import android.media.MediaFormat;
+
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.ShortBuffer;
+
+/**
+ * Created by TAPOS DATTA on 22,May,2020
+ */
+@SuppressWarnings({"MemberName", "MethodName", "SimplifyBooleanReturn"})
+public class AudioChannelWithSP extends BaseAudioChannel {
+    private static final String TAG = "AUDIO_CHANNEL_WITH_SONIC";
+
+    private SonicAudioProcessor stream = null;  // SonicAudioProcessor can deal with stereo Audio
+    private float timeScale = 1f;
+    boolean isEOF = false;
+    private int BUFFER_CAPACITY = 2048; // in ShortBuffer size
+    private long totalDataAdded = 0;
+    private int pendingDecoderOutputBuffIndx = -1;
+    private ByteBuffer tempInputBuffer = null;
+    private boolean isPendingFeeding = true;
+    private boolean isAffectInPitch; // if true the scale will impact in speed with pitch
+
+    AudioChannelWithSP(
+            MediaCodec decoder,
+            MediaCodec encoder,
+            MediaFormat encodeFormat,
+            float timeScale,
+            boolean isPitchChanged
+    ) {
+        super(decoder, encoder, encodeFormat);
+        this.isAffectInPitch = isPitchChanged;
+        this.timeScale = timeScale;
+    }
+
+    @Override
+    public void setActualDecodedFormat(MediaFormat decodedFormat) {
+        super.setActualDecodedFormat(decodedFormat);
+
+        if (inputChannelCount > 2) {
+            throw new UnsupportedOperationException("Input channel count (" + inputChannelCount + ") not supported.");
+        }
+        stream = new SonicAudioProcessor(inputSampleRate, outputChannelCount);
+        isEOF = false;
+        totalDataAdded = 0;
+        isPendingFeeding = true;
+        tempInputBuffer = ByteBuffer.allocateDirect(BUFFER_CAPACITY * 16).order(ByteOrder.nativeOrder());
+
+        if (isAffectInPitch) {
+            stream.setRate(timeScale);
+        } else {
+            stream.setSpeed(timeScale);
+        }
+    }
+
+    @Override
+    protected long sampleCountToDurationUs(long sampleCount, int sampleRate, int channelCount) {
+        // considered short buffer as data
+        return (long) ((MICROSECS_PER_SEC * (sampleCount * 1f) / (sampleRate * 1f * channelCount)));
+    }
+
+    @Override
+    public void drainDecoderBufferAndQueue(int bufferIndex, long presentationTimeUs) {
+        if (actualDecodedFormat == null) {
+            throw new RuntimeException("Buffer received before format!");
+        }
+
+        final ByteBuffer data =
+                bufferIndex == BUFFER_INDEX_END_OF_STREAM
+                        ? null : decoder.getOutputBuffer(bufferIndex);
+
+        if (data != null) {
+            writeToSonicSteam(data.asShortBuffer());
+            pendingDecoderOutputBuffIndx = bufferIndex;
+            isEOF = false;
+            decoder.releaseOutputBuffer(bufferIndex, false);
+        } else {
+            stream.flushStream();
+            isEOF = true;
+        }
+    }
+
+    @Override
+    public boolean feedEncoder(long timeoutUs) {
+        if (stream == null || !isPendingFeeding || (!isEOF && stream.samplesAvailable() == 0)) {
+            // no data available
+
+            updatePendingDecoderStatus();
+
+            return false;
+        } else if (!isEOF
+                   && timeScale < 1f
+                   && stream.samplesAvailable() > 0
+                   && (stream.samplesAvailable() * outputChannelCount) < BUFFER_CAPACITY
+        ) {
+            // few data remaining in stream wait for next stream data
+            updatePendingDecoderStatus();
+
+            return false;
+        }
+
+        final int encoderInBuffIndex = encoder.dequeueInputBuffer(timeoutUs);
+
+        if (encoderInBuffIndex < 0) {
+            // Encoder is full - Bail out
+            return false;
+        }
+
+        boolean status = false;
+        if (timeScale < 1f) {
+            status = slowTimeBufferProcess(encoderInBuffIndex);
+        } else {
+            status = FastOrNormalTimeBufferProcess(encoderInBuffIndex);
+        }
+
+        return status;
+    }
+
+    private void updatePendingDecoderStatus() {
+        if (pendingDecoderOutputBuffIndx != -1) {
+            pendingDecoderOutputBuffIndx = -1;
+        }
+    }
+
+    private boolean FastOrNormalTimeBufferProcess(int encoderInBuffIndex) {
+        int samplesNum = stream.samplesAvailable();
+
+        boolean status = false;
+
+        int rawDataLen = samplesNum * outputChannelCount;
+
+        if (rawDataLen >= BUFFER_CAPACITY) {
+            return readStreamDataAndQueueToEncoder(BUFFER_CAPACITY, encoderInBuffIndex);
+        } else if (rawDataLen > 0 && rawDataLen < BUFFER_CAPACITY) {
+            return readStreamDataAndQueueToEncoder(rawDataLen, encoderInBuffIndex);
+        } else if (isEOF && samplesNum == 0) {
+            return finalizeEncoderQueue(encoderInBuffIndex);
+        } else {
+            return status;
+        }
+    }
+
+    private boolean slowTimeBufferProcess(final int encoderInBuffIndex) {
+        int samplesNum = stream.samplesAvailable();
+
+        boolean status = false;
+
+        int rawDataLen = samplesNum * outputChannelCount;
+
+        if (rawDataLen >= BUFFER_CAPACITY) {
+            return readStreamDataAndQueueToEncoder(BUFFER_CAPACITY, encoderInBuffIndex);
+        } else if (isEOF && (rawDataLen > 0 && rawDataLen < BUFFER_CAPACITY)) {
+            return readStreamDataAndQueueToEncoder(rawDataLen, encoderInBuffIndex);
+        } else if (isEOF && rawDataLen == 0) {
+            return finalizeEncoderQueue(encoderInBuffIndex);
+        } else {
+            return status;
+        }
+    }
+
+    private boolean finalizeEncoderQueue(final int encoderInBuffIndex) {
+        isPendingFeeding = false;
+        return queueInputBufferInEncoder(null, encoderInBuffIndex);
+    }
+
+    private boolean readStreamDataAndQueueToEncoder(final int capacity, final int encoderInBuffIndex) {
+        short[] rawData = new short[capacity];
+        stream.readShortFromStream(rawData, (capacity / outputChannelCount));
+        return queueInputBufferInEncoder(rawData, encoderInBuffIndex);
+    }
+
+    private boolean queueInputBufferInEncoder(final short[] rawData, final int encoderInBuffIndex) {
+        final ShortBuffer outBuffer = encoder.getInputBuffer(encoderInBuffIndex).asShortBuffer();
+
+        outBuffer.clear();
+        if (rawData != null) {
+            outBuffer.put(rawData);
+            totalDataAdded += rawData.length;
+
+            long presentationTimeUs = sampleCountToDurationUs(totalDataAdded, inputSampleRate, outputChannelCount);
+
+            encoder.queueInputBuffer(encoderInBuffIndex, 0, rawData.length * BYTES_PER_SHORT,
+                    presentationTimeUs, 0);
+            return false;
+        } else {
+            encoder.queueInputBuffer(encoderInBuffIndex, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM);
+            return false;
+        }
+    }
+
+    private void writeToSonicSteam(final ShortBuffer data) {
+        short[] temBuff = new short[data.capacity()];
+        data.get(temBuff);
+        data.rewind();
+        stream.writeShortToStream(temBuff, temBuff.length / outputChannelCount);
+    }
+
+    public boolean isAnyPendingBuffIndex() {
+        // allow to decoder to send data into stream (e.i. sonicprocessor)
+        if (pendingDecoderOutputBuffIndx != -1) {
+            return true;
+        } else {
+            return false;
+        }
+    }
+}

mp4compose/src/main/java/com/daasuu/mp4compose/composer/AudioComposer.kt

@@ -12,12 +12,13 @@ import java.nio.ByteOrder
 internal class AudioComposer(
     private val mediaExtractor: MediaExtractor,
     private val trackIndex: Int,
-    private val muxRender: MuxRender
+    private val muxRender: MuxRender,
+    private val useFallBacks: Boolean = false
 ) : IAudioComposer {
     private val sampleType = MuxRender.SampleType.AUDIO
     private val bufferInfo = MediaCodec.BufferInfo()
-    private val bufferSize: Int
-    private val buffer: ByteBuffer
+    private var bufferSize: Int
+    private var buffer: ByteBuffer
     override var isFinished: Boolean = false
         private set
     private val actualOutputFormat: MediaFormat
@@ -27,7 +28,15 @@ internal class AudioComposer(
     init {
         actualOutputFormat = this.mediaExtractor.getTrackFormat(this.trackIndex)
         this.muxRender.setOutputFormat(this.sampleType, actualOutputFormat)
-        bufferSize = actualOutputFormat.getInteger(MediaFormat.KEY_MAX_INPUT_SIZE)
+
+        // TODO: maybe the original assignement could work as well?
+        // bufferSize = actualOutputFormat.containsKey(MediaFormat.KEY_MAX_INPUT_SIZE) ? actualOutputFormat.getInteger(MediaFormat.KEY_MAX_INPUT_SIZE) : (64 * 1024);
+        bufferSize = if (useFallBacks && !actualOutputFormat.containsKey(MediaFormat.KEY_MAX_INPUT_SIZE)) {
+            (64 * 1024)
+        } else {
+            actualOutputFormat.getInteger(MediaFormat.KEY_MAX_INPUT_SIZE)
+        }
+
         buffer = ByteBuffer.allocateDirect(bufferSize).order(ByteOrder.nativeOrder())
     }
 
@@ -46,11 +55,18 @@ internal class AudioComposer(
 
         buffer.clear()
         val sampleSize = mediaExtractor.readSampleData(buffer, 0)
-        assert(sampleSize <= bufferSize)
+        if (useFallBacks && sampleSize > bufferSize) {
+            bufferSize = 2 * sampleSize
+            buffer = ByteBuffer.allocateDirect(bufferSize).order(ByteOrder.nativeOrder())
+        } else {
+            assert(sampleSize <= bufferSize)
+        }
         val isKeyFrame = mediaExtractor.sampleFlags and MediaExtractor.SAMPLE_FLAG_SYNC != 0
         val flags = if (isKeyFrame) MediaCodec.BUFFER_FLAG_KEY_FRAME else 0
         bufferInfo.set(0, sampleSize, mediaExtractor.sampleTime, flags)
         muxRender.writeSampleData(sampleType, buffer, bufferInfo)
+        // TODO: should we use the original writtenPresentationTimeUs = mediaExtractor.getSampleTime();
+        // at least for the Video Compression use case?
         writtenPresentationTimeUs = bufferInfo.presentationTimeUs
 
         mediaExtractor.advance()

mp4compose/src/main/java/com/daasuu/mp4compose/composer/BaseAudioChannel.java

@@ -0,0 +1,75 @@
+package com.daasuu.mp4compose.composer;
+
+import android.media.MediaCodec;
+import android.media.MediaFormat;
+
+import java.nio.ShortBuffer;
+import java.util.ArrayDeque;
+import java.util.Queue;
+
+/**
+ * Created by TAPOS DATTA on 22,May,2020
+ */
+@SuppressWarnings("MemberName")
+abstract class BaseAudioChannel {
+    protected static class AudioBuffer {
+        int bufferIndex;
+        long presentationTimeUs;
+        ShortBuffer data;
+    }
+    protected static class BufferInfo {
+        long totaldata;
+        long presentationTimeUs;
+    }
+
+    static final int BUFFER_INDEX_END_OF_STREAM = -1;
+    protected static final int BYTE_PER_SAMPLE = 16 / 8;
+    protected static final int BYTES_PER_SHORT = 2;
+    protected static final long MICROSECS_PER_SEC = 1000000;
+
+    protected final Queue<AudioBuffer> emptyBuffers = new ArrayDeque<>();
+    protected final Queue<AudioBuffer> filledBuffers = new ArrayDeque<>();
+
+    protected final MediaCodec decoder;
+    protected final MediaCodec encoder;
+    protected final MediaFormat encodeFormat;
+
+    protected int inputSampleRate;
+    protected int inputChannelCount;
+    protected int outputChannelCount;
+
+    protected final AudioBuffer overflowBuffer = new AudioBuffer();
+
+    protected MediaFormat actualDecodedFormat;
+
+    BaseAudioChannel(final MediaCodec decoder,
+                     final MediaCodec encoder, final MediaFormat encodeFormat) {
+        this.decoder = decoder;
+        this.encoder = encoder;
+        this.encodeFormat = encodeFormat;
+    }
+
+    public void setActualDecodedFormat(final MediaFormat decodedFormat) {
+        actualDecodedFormat = decodedFormat;
+
+        inputSampleRate = actualDecodedFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE);
+        if (inputSampleRate != encodeFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE)) {
+            throw new UnsupportedOperationException("Audio sample rate conversion not supported yet.");
+        }
+
+        inputChannelCount = actualDecodedFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT);
+        outputChannelCount = encodeFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT);
+
+        if (outputChannelCount != 1 && outputChannelCount != 2) {
+            throw new UnsupportedOperationException("Output channel count (" + outputChannelCount + ") not supported.");
+        }
+
+        overflowBuffer.presentationTimeUs = 0;
+    }
+
+    protected abstract long sampleCountToDurationUs(long sampleCount, int sampleRate, int channelCount);
+
+    protected abstract void drainDecoderBufferAndQueue(int bufferIndex, long presentationTimeUs);
+
+    protected abstract boolean feedEncoder(long timeoutUs);
+}