Dev (#26)

* Fixed a memory leak * Fixed the last error regarding the leak fix * fix. * ring buffer as struct * to prevent null reference * Multithread refactor (#13) * disconnect support * Fix name * To handle better connection and disconnection from room * change to avoid using of Coroutines, using async methods instead * async OnAudioRead * Cleaning logs * Cancel tokens * Cancel token * remove Task.Run, since usually SendRequest is called from other task * getting rid of use YieldInstruction * adding audio filter in main thread * passing CancellationToken instead of CancellationTokenSource * added check for cancellation at beginning * Cleaning handles * fix await * remove cruft * Cancellation token cleanup * Factory async method * basic handling of remaining events * more async * wip * Compile pass * formatting * fix pending check * revert async changes * prep audio filter for threads * loggin * logs to debug publishing track * minor cleanup * added some methods to handle localtrack and SetSubscribed, Clenaing code * Cleaning up * fixes for yield instructions * macro for verbose logs * Audio data and pointer fix Still says samplerate and num of channels are incorrect * remove possibility of null * todo ref buffering * minor cleanup * cruft * todo object pooling * remove redundant Task usages on separated threads --------- Co-authored-by: cdga777 <[email protected]> Co-authored-by: nickkhalow <[email protected]> * feat: optimisations (#16) * remove redundant Task usages on separated threads * FfiRequestsPool * IFFIClient interface * disposable FFIClient * FFIClient initialize simplify * PoolableFFIClient * NativeMethods pass pointer * response pooling * tests for reject manual creation of protocol objects * FfiResponseWrap, remove manual new Request calls * remove IPoolableFFIClient * FFIBridge introduction, remove closures, RAII with FfiResponseWrap and FfiRequestWrap * Clear -> generic ClearMessage * Make the Ffi Handle type consistent. * room support spans * cleanup. * feat: optimisation - memory pooling, FfiHandles (#18) * fix clear message * optimise FfiNewRequest - no FfiHandle * LiveKitInitialize with IntPtr * const format * memory pool * inner message pooling * ArrayMemoryPool with thread safety * remove obsolete * thread safety * RingBuffer recycling * dangerous access for memory buffer (#26) * feat: Audio/Video Support (#29) * ensure channels and rate match * subscription change * fixing early subscription * track handle fix temp logs to confirm * track handle fix * streaming fixes * remix fix * resample log check * try off frame * handler fix * log cleanup * saving track for local Participant * remix potential fix * cleanup checks * testing without thread * revert * destroy cleanup and possible fix to ringbuffer * log cleanup * test * revert * disable thread for test * test * fix * adding threading back in for writing * video prep * handle fix * fix cast * removing streams from video for now * buffer fix * hack to get it to send a web texture * more consistent naming * format fix * bad merge fix * first pass on screen share and memory fix * debug and formatting * remove logs * updated protobuff and fixed compiler errors * video working * dynamic buffertype and stride * split video source and partial wip on cameravideoSource * camera support * merge cont. * wip merging * adding back stomped change * latest libs for mac * Fixes for Audio/Video Streaming * remove cruft * fix when an AudioTrack is created * mac fix * null check which can occur if you are closing app * format checks * utils * fix for metal * Subscription management * cleanup * log cleanup * cleanup * log --------- Co-authored-by: cdga777 <[email protected]> * update local metadata. * fix message truncation bug. * Update livekit_ffi.dll.meta * Update libwebrtc.jar.meta * update. * some minor improvements. * update. --------- Co-authored-by: Ashkan Saeidi Mazdeh <[email protected]> Co-authored-by: nickkhalow <[email protected]> Co-authored-by: cdga777 <[email protected]> Co-authored-by: Mark Grossnickle <[email protected]> Co-authored-by: Nick Khalow <[email protected]>
livekit · Apr 8, 2024 · b0f5508 · b0f5508
1 parent e8f771b
commit b0f5508
Show file tree

Hide file tree

Showing 81 changed files with 2,008 additions and 599 deletions.
diff --git a/README.md.meta b/README.md.meta
diff --git a/Runtime/Plugins/ffi-android-arm64/liblivekit_ffi.so.meta b/Runtime/Plugins/ffi-android-arm64/liblivekit_ffi.so.meta
diff --git a/Runtime/Plugins/ffi-linux-arm64/liblivekit_ffi.so.meta b/Runtime/Plugins/ffi-linux-arm64/liblivekit_ffi.so.meta
diff --git a/Runtime/Plugins/ffi-windows-x86_64/livekit_ffi.dll.meta b/Runtime/Plugins/ffi-windows-x86_64/livekit_ffi.dll.meta
diff --git a/Runtime/Plugins/libwebrtc.jar.meta b/Runtime/Plugins/libwebrtc.jar.meta
diff --git a/Runtime/Scripts/AudioFrame.cs b/Runtime/Scripts/AudioFrame.cs
@@ -1,5 +1,6 @@
 using System;
 using LiveKit.Proto;
+using LiveKit.Internal;
 using Unity.Collections;
 using Unity.Collections.LowLevel.Unsafe;
 
@@ -9,7 +10,7 @@ public class AudioFrame : IDisposable
     {
         private AudioFrameBufferInfo _info;
 
-        private FfiOwnedHandle _handle;
+        private FfiHandle _handle;
 
         private bool _disposed = false;
 
@@ -27,14 +28,14 @@ public class AudioFrame : IDisposable
 
         public int Length => (int) (SamplesPerChannel * NumChannels * sizeof(short));
 
-        internal AudioFrame(FfiOwnedHandle handle, AudioFrameBufferInfo info)
+        internal AudioFrame(OwnedAudioFrameBuffer info)
         {
-            _handle = handle;
-            _info = info;
+            _handle = FfiHandle.FromOwnedHandle(info.Handle);
+            _info = info.Info;
             _sampleRate = _info.SampleRate;
             _numChannels = _info.NumChannels;
             _samplesPerChannel = _info.SamplesPerChannel;
-            _dataPtr = (IntPtr)info.DataPtr;
+            _dataPtr = (IntPtr)_info.DataPtr;
         }
 
         internal AudioFrame(int sampleRate, int numChannels, int samplesPerChannel) {
@@ -66,4 +67,4 @@ protected virtual void Dispose(bool disposing)
             }
         }
     }
-}
+}
diff --git a/Runtime/Scripts/AudioSource.cs b/Runtime/Scripts/AudioSource.cs
@@ -2,6 +2,8 @@
 using UnityEngine;
 using LiveKit.Proto;
 using LiveKit.Internal;
+using System.Threading;
+using LiveKit.Internal.FFIClients.Requests;
 
 namespace LiveKit
 {
@@ -10,86 +12,143 @@ public class RtcAudioSource
         private AudioSource _audioSource;
         private AudioFilter _audioFilter;
 
-        internal readonly FfiOwnedHandle Handle;
+        internal readonly FfiHandle Handle;
         protected AudioSourceInfo _info;
 
         // Used on the AudioThread
         private AudioFrame _frame;
+        private Thread _readAudioThread;
+        private object _lock = new object();
+        private float[] _data;
+        private int _channels;
+        private int _sampleRate;
+        private volatile bool _pending = false;
 
         public RtcAudioSource(AudioSource source)
         {
-            var newAudioSource = new NewAudioSourceRequest();
+            using var request = FFIBridge.Instance.NewRequest<NewAudioSourceRequest>();
+            var newAudioSource = request.request;
             newAudioSource.Type = AudioSourceType.AudioSourceNative;
             newAudioSource.NumChannels = 2;
             newAudioSource.SampleRate = 48000;
-
-            var request = new FfiRequest();
-            request.NewAudioSource = newAudioSource;
-
-            var resp = FfiClient.SendRequest(request);
-            var respSource = resp.NewAudioSource.Source;
-            _info = respSource.Info;
-
-            Handle = respSource.Handle;
+            using var response = request.Send();
+            FfiResponse res = response;
+            _info = res.NewAudioSource.Source.Info;
+            //TODO pooling handles
+            Handle = FfiHandle.FromOwnedHandle(res.NewAudioSource.Source.Handle);
             UpdateSource(source);
         }
 
-        private void UpdateSource(AudioSource source)
+        public void Start()
         {
-            _audioSource = source;
-            _audioFilter = source.gameObject.AddComponent<AudioFilter>();
-            //_audioFilter.hideFlags = HideFlags.HideInInspector;
+            Stop();
+            _readAudioThread = new Thread(Update);
+            _readAudioThread.Start();
+
             _audioFilter.AudioRead += OnAudioRead;
-            source.Play();
+            _audioSource.Play();
         }
 
-
-        private void OnAudioRead(float[] data, int channels, int sampleRate)
+        public void Stop()
         {
-            var samplesPerChannel = data.Length / channels;
-            if (_frame == null || _frame.NumChannels != channels 
-                || _frame.SampleRate != sampleRate 
-                || _frame.SamplesPerChannel != samplesPerChannel)
+            if (_readAudioThread != null)
             {
-                _frame = new AudioFrame(sampleRate, channels, samplesPerChannel);
-            }
-
-            static short FloatToS16(float v) {
-                v *= 32768f;
-                v = Math.Min(v, 32767f);
-                v = Math.Max(v, -32768f);
-                return (short)(v + Math.Sign(v) * 0.5f);
+                _readAudioThread.Abort();
+                _readAudioThread = null;
             }
+            if(_audioFilter) _audioFilter.AudioRead -= OnAudioRead;
+            if(_audioSource) _audioSource.Stop();
+        }
 
-            unsafe {
-                var frameData = new Span<short>(_frame.Data.ToPointer(), _frame.Length / sizeof(short));
-                for (int i = 0; i < data.Length; i++)
+        private void Update()
+        {
+            while (true)
+            {
+                Thread.Sleep(Constants.TASK_DELAY);
+                if (_pending)
                 {
-                    frameData[i] = FloatToS16(data[i]);
+                    ReadAudio();
                 }
             }
+        }
 
-            // Don't play the audio locally
-            Array.Clear(data, 0, data.Length);
-
-            var audioFrameBufferInfo = new AudioFrameBufferInfo();
-
-            audioFrameBufferInfo.DataPtr = (ulong)_frame.Data;
-            audioFrameBufferInfo.NumChannels = _frame.NumChannels;
-            audioFrameBufferInfo.SampleRate = _frame.SampleRate;
-            audioFrameBufferInfo.SamplesPerChannel = _frame.SamplesPerChannel;
-
-            var pushFrame = new CaptureAudioFrameRequest();
-            pushFrame.SourceHandle = Handle.Id;
-            pushFrame.Buffer = audioFrameBufferInfo;
-
-            var request = new FfiRequest();
-            request.CaptureAudioFrame = pushFrame;
+        private void ReadAudio()
+        {
+            _pending = false;
+            lock (_lock)
+            {
+                var samplesPerChannel = _data.Length / _channels;
+                if (_frame == null
+                    || _frame.NumChannels != _channels
+                    || _frame.SampleRate != _sampleRate
+                    || _frame.SamplesPerChannel != samplesPerChannel)
+                {
+                    _frame = new AudioFrame(_sampleRate, _channels, samplesPerChannel);
+                }
+                try
+                {
 
-            FfiClient.SendRequest(request);
+                    static short FloatToS16(float v)
+                    {
+                        v *= 32768f;
+                        v = Math.Min(v, 32767f);
+                        v = Math.Max(v, -32768f);
+                        return (short)(v + Math.Sign(v) * 0.5f);
+                    }
+
+                    unsafe
+                    {
+                        var frameData = new Span<short>(_frame.Data.ToPointer(), _frame.Length / sizeof(short));
+                        for (int i = 0; i < _data.Length; i++)
+                        {
+                            frameData[i] = FloatToS16(_data[i]); 
+                        }
+
+                        // Don't play the audio locally
+                        Array.Clear(_data, 0, _data.Length);
+
+                        using var request = FFIBridge.Instance.NewRequest<CaptureAudioFrameRequest>();
+                        using var audioFrameBufferInfo = request.TempResource<AudioFrameBufferInfo>();
+
+                        var pushFrame = request.request;
+                        pushFrame.SourceHandle = (ulong)Handle.DangerousGetHandle();
+
+                        pushFrame.Buffer = audioFrameBufferInfo;
+                        pushFrame.Buffer.DataPtr = (ulong)_frame.Data;
+                        pushFrame.Buffer.NumChannels = _frame.NumChannels;
+                        pushFrame.Buffer.SampleRate = _frame.SampleRate;
+                        pushFrame.Buffer.SamplesPerChannel = _frame.SamplesPerChannel;
+
+                        using var response = request.Send();
+
+                        pushFrame.Buffer.DataPtr = 0;
+                        pushFrame.Buffer.NumChannels = 0;
+                        pushFrame.Buffer.SampleRate = 0;
+                        pushFrame.Buffer.SamplesPerChannel = 0;
+                    }
+                }
+                catch (Exception e)
+                {
+                    Utils.Error("Audio Framedata error: " + e.Message);
+                }
+            }
+        }
 
+        private void UpdateSource(AudioSource source)
+        {
+            _audioSource = source;
+            _audioFilter = source.gameObject.AddComponent<AudioFilter>();
+        }
 
-            //Debug.Log($"Pushed audio frame with {data.Length} samples");
+        private void OnAudioRead(float[] data, int channels, int sampleRate)
+        {
+            lock (_lock)
+            {
+                _data = data;
+                _channels = channels;
+                _sampleRate = sampleRate;
+                _pending = true;
+            }
         }
     }
 }