diff --git a/android/src/main/java/com/twiliorn/library/PatchedVideoView.java b/android/src/main/java/com/twiliorn/library/PatchedVideoView.java index 8f7a959a..35363ae0 100644 --- a/android/src/main/java/com/twiliorn/library/PatchedVideoView.java +++ b/android/src/main/java/com/twiliorn/library/PatchedVideoView.java @@ -12,18 +12,46 @@ import android.util.AttributeSet; import com.twilio.video.I420Frame; -import com.twilio.video.VideoView; +import com.twilio.video.VideoTextureView; +import android.view.View; + +import java.util.concurrent.atomic.AtomicBoolean; +import android.graphics.Bitmap; +import android.util.Base64; +import java.io.ByteArrayOutputStream; +import java.nio.ByteBuffer; +import android.graphics.BitmapFactory; +import android.graphics.Matrix; +import android.graphics.Rect; +import android.graphics.YuvImage; +import androidx.annotation.NonNull; + +import com.facebook.react.uimanager.events.RCTEventEmitter; +import com.facebook.react.bridge.WritableMap; +import com.facebook.react.bridge.WritableNativeMap; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import android.net.Uri; + +import static android.graphics.ImageFormat.NV21; /* * VideoView that notifies Listener of the first frame rendered and the first frame after a reset * request. */ -public class PatchedVideoView extends VideoView { +public class PatchedVideoView extends VideoTextureView { private boolean notifyFrameRendered = false; private Listener listener; private final Handler mainThreadHandler = new Handler(Looper.getMainLooper()); + private final AtomicBoolean snapshotRequsted = new AtomicBoolean(false); + private RCTEventEmitter eventEmitter; + private int viewId; + private File outputFile; + private VideoTextureView videoTextureView; + public PatchedVideoView(Context context) { super(context); } @@ -43,9 +71,134 @@ public void run() { } }); } + + if (snapshotRequsted.compareAndSet(true, false)) { + mainThreadHandler.post(new Runnable() { + @Override + public void run() { + final Bitmap bitmap = frame.yuvPlanes == null ? + captureBitmapFromTexture(frame) : + captureBitmapFromYuvFrame(frame); + + WritableMap event = new WritableNativeMap(); + try (FileOutputStream output = new FileOutputStream(outputFile)) { + bitmap.compress(Bitmap.CompressFormat.PNG, 100, output); + String uri = Uri.fromFile(outputFile).toString(); + event.putString("uri", uri); + } catch (final Throwable ex) { + event.putString("error", "Error saving snapshot."); + } + pushEvent("onSnapshot", event); + } + }); + } + super.renderFrame(frame); } + public Bitmap captureBitmapFromTexture(I420Frame frame) { + Bitmap bitmap = videoTextureView.getBitmap(); + return bitmap; + } + + public Bitmap captureBitmapFromYuvFrame(I420Frame frame) { + YuvImage yuvImage = i420ToYuvImage(frame.yuvPlanes, + frame.yuvStrides, + frame.width, + frame.height); + ByteArrayOutputStream stream = new ByteArrayOutputStream(); + Rect rect = new Rect(0, 0, yuvImage.getWidth(), yuvImage.getHeight()); + + // Compress YuvImage to jpeg + yuvImage.compressToJpeg(rect, 100, stream); + + // Convert jpeg to Bitmap + byte[] imageBytes = stream.toByteArray(); + Bitmap bitmap = BitmapFactory.decodeByteArray(imageBytes, 0, imageBytes.length); + Matrix matrix = new Matrix(); + + // Apply any needed rotation + matrix.postRotate(frame.rotationDegree); + bitmap = Bitmap.createBitmap(bitmap, 0, 0, bitmap.getWidth(), bitmap.getHeight(), matrix, + true); + + return bitmap; + } + + private YuvImage i420ToYuvImage(ByteBuffer[] yuvPlanes, + int[] yuvStrides, + int width, + int height) { + if (yuvStrides[0] != width) { + return fastI420ToYuvImage(yuvPlanes, yuvStrides, width, height); + } + if (yuvStrides[1] != width / 2) { + return fastI420ToYuvImage(yuvPlanes, yuvStrides, width, height); + } + if (yuvStrides[2] != width / 2) { + return fastI420ToYuvImage(yuvPlanes, yuvStrides, width, height); + } + + byte[] bytes = new byte[yuvStrides[0] * height + + yuvStrides[1] * height / 2 + + yuvStrides[2] * height / 2]; + ByteBuffer tmp = ByteBuffer.wrap(bytes, 0, width * height); + copyPlane(yuvPlanes[0], tmp); + + byte[] tmpBytes = new byte[width / 2 * height / 2]; + tmp = ByteBuffer.wrap(tmpBytes, 0, width / 2 * height / 2); + + copyPlane(yuvPlanes[2], tmp); + for (int row = 0 ; row < height / 2 ; row++) { + for (int col = 0 ; col < width / 2 ; col++) { + bytes[width * height + row * width + col * 2] + = tmpBytes[row * width / 2 + col]; + } + } + copyPlane(yuvPlanes[1], tmp); + for (int row = 0 ; row < height / 2 ; row++) { + for (int col = 0 ; col < width / 2 ; col++) { + bytes[width * height + row * width + col * 2 + 1] = + tmpBytes[row * width / 2 + col]; + } + } + return new YuvImage(bytes, NV21, width, height, null); + } + + private YuvImage fastI420ToYuvImage(ByteBuffer[] yuvPlanes, + int[] yuvStrides, + int width, + int height) { + byte[] bytes = new byte[width * height * 3 / 2]; + int i = 0; + for (int row = 0 ; row < height ; row++) { + for (int col = 0 ; col < width ; col++) { + bytes[i++] = yuvPlanes[0].get(col + row * yuvStrides[0]); + } + } + for (int row = 0 ; row < height / 2 ; row++) { + for (int col = 0 ; col < width / 2; col++) { + bytes[i++] = yuvPlanes[2].get(col + row * yuvStrides[2]); + bytes[i++] = yuvPlanes[1].get(col + row * yuvStrides[1]); + } + } + return new YuvImage(bytes, NV21, width, height, null); + } + + private void copyPlane(ByteBuffer src, ByteBuffer dst) { + src.position(0).limit(src.capacity()); + dst.put(src); + dst.position(0).limit(dst.capacity()); + } + + public void takeSnapshot(RCTEventEmitter eventEmitter, int viewId, File outputFile, VideoTextureView video) { + this.videoTextureView = video; + this.eventEmitter = eventEmitter; + this.viewId = viewId; + this.outputFile = outputFile; + snapshotRequsted.set(true); + } + /* * Set your listener */ @@ -63,4 +216,8 @@ public void resetListener() { public interface Listener { void onFirstFrame(); } + + void pushEvent(String name, WritableMap data) { + eventEmitter.receiveEvent(viewId, name, data); + } } diff --git a/android/src/main/java/com/twiliorn/library/TwilioRemotePreview.java b/android/src/main/java/com/twiliorn/library/TwilioRemotePreview.java index 0ade8d6c..c171c5c6 100644 --- a/android/src/main/java/com/twiliorn/library/TwilioRemotePreview.java +++ b/android/src/main/java/com/twiliorn/library/TwilioRemotePreview.java @@ -7,22 +7,65 @@ package com.twiliorn.library; +import android.content.Context; import android.util.Log; - +import com.facebook.react.uimanager.events.RCTEventEmitter; import com.facebook.react.uimanager.ThemedReactContext; - +import java.io.File; +import java.io.IOException; +import androidx.annotation.NonNull; public class TwilioRemotePreview extends RNVideoViewGroup { private static final String TAG = "TwilioRemotePreview"; - + private PatchedVideoView video; + private final RCTEventEmitter eventEmitter; + private final ThemedReactContext mContext; public TwilioRemotePreview(ThemedReactContext context, String trackSid) { super(context); Log.i("CustomTwilioVideoView", "Remote Prview Construct"); Log.i("CustomTwilioVideoView", trackSid); - + this.eventEmitter = context.getJSModule(RCTEventEmitter.class); + this.video = this.getSurfaceViewRenderer(); + this.mContext = context; CustomTwilioVideoView.registerPrimaryVideoView(this.getSurfaceViewRenderer(), trackSid); } + + public void takeSnapshot() { + try { + File outputFile = createTempFile(mContext); + video.takeSnapshot(eventEmitter, TwilioRemotePreview.this.getId(), outputFile, video); + } catch (final Throwable ex) { + Log.e(TAG, "Failed to take snapshot", ex); + } + } + + /** + * Create a temporary file in the cache directory on either internal or external storage, + * whichever is available and has more free space. + */ + @NonNull + private File createTempFile(@NonNull final Context context) throws IOException { + final File externalCacheDir = context.getExternalCacheDir(); + final File internalCacheDir = context.getCacheDir(); + final File cacheDir; + + if (externalCacheDir == null && internalCacheDir == null) { + throw new IOException("No cache directory available"); + } + + if (externalCacheDir == null) { + cacheDir = internalCacheDir; + } else if (internalCacheDir == null) { + cacheDir = externalCacheDir; + } else { + cacheDir = externalCacheDir.getFreeSpace() > internalCacheDir.getFreeSpace() ? + externalCacheDir : internalCacheDir; + } + + final String suffix = "." + "png"; + return File.createTempFile("snapshot_video", suffix, cacheDir); + } } diff --git a/android/src/main/java/com/twiliorn/library/TwilioRemotePreviewManager.java b/android/src/main/java/com/twiliorn/library/TwilioRemotePreviewManager.java index 71d47c2e..47fe7c3c 100644 --- a/android/src/main/java/com/twiliorn/library/TwilioRemotePreviewManager.java +++ b/android/src/main/java/com/twiliorn/library/TwilioRemotePreviewManager.java @@ -1,3 +1,4 @@ + /** * Component for Twilio Video participant views. *

@@ -15,12 +16,15 @@ import com.facebook.react.uimanager.annotations.ReactProp; import org.webrtc.RendererCommon; - +import com.facebook.react.bridge.ReadableArray; +import java.util.Map; +import com.facebook.react.common.MapBuilder; public class TwilioRemotePreviewManager extends SimpleViewManager { public static final String REACT_CLASS = "RNTwilioRemotePreview"; public String myTrackSid = ""; + private static final int TAKE_SNAPSHOT = 10001; @Override public String getName() { @@ -46,6 +50,32 @@ public void setTrackId(TwilioRemotePreview view, @Nullable String trackSid) { CustomTwilioVideoView.registerPrimaryVideoView(view.getSurfaceViewRenderer(), trackSid); } + @Override + public void receiveCommand(TwilioRemotePreview view, int commandId, @Nullable ReadableArray args) { + switch (commandId) { + case TAKE_SNAPSHOT: + view.takeSnapshot(); + break; + } + } + + @Override + @Nullable + public Map getCommandsMap() { + return MapBuilder.builder() + .put("takeSnapshot", TAKE_SNAPSHOT) + .build(); + } + + @Override + @Nullable + public Map getExportedCustomDirectEventTypeConstants() { + Map> map = MapBuilder.of( + "onSnapshot", MapBuilder.of("registrationName", "onSnapshot") + ); + + return map; + } @Override protected TwilioRemotePreview createViewInstance(ThemedReactContext reactContext) { diff --git a/android/src/main/java/com/twiliorn/library/TwilioVideoPreview.java b/android/src/main/java/com/twiliorn/library/TwilioVideoPreview.java index 6acddc74..90c3a716 100644 --- a/android/src/main/java/com/twiliorn/library/TwilioVideoPreview.java +++ b/android/src/main/java/com/twiliorn/library/TwilioVideoPreview.java @@ -16,6 +16,6 @@ public class TwilioVideoPreview extends RNVideoViewGroup { public TwilioVideoPreview(ThemedReactContext themedReactContext) { super(themedReactContext); CustomTwilioVideoView.registerThumbnailVideoView(this.getSurfaceViewRenderer()); - this.getSurfaceViewRenderer().applyZOrder(true); + // this.getSurfaceViewRenderer().applyZOrder(true); } } diff --git a/docs/README.md b/docs/README.md index b3a79740..7498163f 100644 --- a/docs/README.md +++ b/docs/README.md @@ -237,4 +237,22 @@ trackIdentifier: { } ``` +#### onSnapshot (Android only) + +```js +onSnapshot: Function +``` + +Callback when snapshot has been saved. + +@param {{error, uri}} Uri of temp image file, or error if failed to take snapshot + +#### takeSnapshot (Android only, method, not props) + +```js +takeSnapshot: Function +``` + +Take snapshot. +

diff --git a/src/TwilioVideoParticipantView.android.js b/src/TwilioVideoParticipantView.android.js index a70d8dc1..f4ca9402 100644 --- a/src/TwilioVideoParticipantView.android.js +++ b/src/TwilioVideoParticipantView.android.js @@ -5,10 +5,14 @@ * Jonathan Chang */ -import { requireNativeComponent } from 'react-native' +import { requireNativeComponent, Platform, UIManager, findNodeHandle } from 'react-native' import PropTypes from 'prop-types' import React from 'react' +const nativeEvents = { + takeSnapshot: 10001, +} + class TwilioRemotePreview extends React.Component { static propTypes = { trackIdentifier: PropTypes.shape({ @@ -29,8 +33,27 @@ class TwilioRemotePreview extends React.Component { testID: PropTypes.string } + takeSnapshot() { + this.runCommand(nativeEvents.takeSnapshot) + } + + runCommand(event, args) { + switch (Platform.OS) { + case 'android': + UIManager.dispatchViewManagerCommand( + findNodeHandle(this.refs.remoteParticipantView), + event, + args + ) + break + default: + break + } + } + buildNativeEventWrappers () { return [ + 'onSnapshot', 'onFrameDimensionsChanged' ].reduce((wrappedEvents, eventName) => { if (this.props[eventName]) { @@ -47,6 +70,7 @@ class TwilioRemotePreview extends React.Component { const { trackIdentifier } = this.props return (