From 43375a9aa9f196a5bcf78c3c8a5252b1b1ac6ba2 Mon Sep 17 00:00:00 2001 From: Elias Holzer Date: Tue, 5 Nov 2024 15:58:14 +0100 Subject: [PATCH 1/3] Updates project file to net8.0, replacing video device enumeration code from SharpDX.MediaFoundation and DirectShowLib with Microsoft.Windows.CsWin32 The former package is no longer compatible with .net8 (needs awkward hacks in build system as well as current vvvv runtime code), the latter causes build warnings. --- .gitignore | 25 +-- VL.OpenCV.vl | 2 +- .../VideoInput/VideoInputDeviceDefinition.cs | 11 +- src/NativeMethods.json | 6 + src/NativeMethods.txt | 20 +++ src/VL.OpenCV.csproj | 40 +---- src/VideoInInfo.cs | 158 ++++++++++++++---- src/YOLO3Helper.cs | 1 - 8 files changed, 177 insertions(+), 86 deletions(-) create mode 100644 src/NativeMethods.json create mode 100644 src/NativeMethods.txt diff --git a/.gitignore b/.gitignore index 0ff0ccf..f9eed1e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,14 +1,15 @@ - -# .v4p backup files -*~.xml - -# Dynamic plugins .dll -bin/ -obj/ -.vs/ -packages/ -lib/net461/ -appveyor/nuget.exe -/lib/net472/ + +# .v4p backup files +*~.xml + +# Dynamic plugins .dll +bin/ +obj/ +.vs/ +packages/ +lib/net461/ +appveyor/nuget.exe +/lib/net472/ +/lib/net8.0-windows/ /.vl/* diff --git a/VL.OpenCV.vl b/VL.OpenCV.vl index fcd070d..0046659 100644 --- a/VL.OpenCV.vl +++ b/VL.OpenCV.vl @@ -49097,7 +49097,7 @@ - + diff --git a/src/Dynamic Enums/VideoInput/VideoInputDeviceDefinition.cs b/src/Dynamic Enums/VideoInput/VideoInputDeviceDefinition.cs index d416637..bb2ce79 100644 --- a/src/Dynamic Enums/VideoInput/VideoInputDeviceDefinition.cs +++ b/src/Dynamic Enums/VideoInput/VideoInputDeviceDefinition.cs @@ -1,8 +1,8 @@ -using SharpDX.MediaFoundation; -using System; +using System; using System.Collections.Generic; using VL.Lib; using VL.Lib.Collections; +using static Windows.Win32.PInvoke; namespace VL.OpenCV { @@ -12,21 +12,22 @@ namespace VL.OpenCV public class VideoInputDeviceDefinition : DynamicEnumDefinitionBase { //return the current enum entries - protected override IReadOnlyDictionary GetEntries() + protected override unsafe IReadOnlyDictionary GetEntries() { // Get the collection of video devices - Activate[] capDevices = VideoInInfo.EnumerateVideoDevices(); + var capDevices = VideoInInfo.EnumerateVideoDevices(); Dictionary devices = new Dictionary(capDevices.Length); for (int i = 0; i < capDevices.Length; i++) { var j = 1; - var friendlyName = capDevices[i].Get(CaptureDeviceAttributeKeys.FriendlyName); + var friendlyName = VideoInInfo.GetString(capDevices[i], in MF_DEVSOURCE_ATTRIBUTE_FRIENDLY_NAME); var finalName = friendlyName; while (devices.ContainsKey(finalName)) { finalName = friendlyName + " #" + j++; } devices[finalName] = i; + capDevices[i]->Release(); } return devices; } diff --git a/src/NativeMethods.json b/src/NativeMethods.json new file mode 100644 index 0000000..b757aa8 --- /dev/null +++ b/src/NativeMethods.json @@ -0,0 +1,6 @@ +{ + "$schema": "https://aka.ms/CsWin32.schema.json", + "allowMarshaling": false, + "comInterop": { "preserveSigMethods": [ "IAMCameraControl", "IAMVideoProcAmp", "IMFMediaEngine", "ICreateDevEnum" ] }, + "public": true +} \ No newline at end of file diff --git a/src/NativeMethods.txt b/src/NativeMethods.txt new file mode 100644 index 0000000..7188a68 --- /dev/null +++ b/src/NativeMethods.txt @@ -0,0 +1,20 @@ +CoCreateInstance +IMFActivate +IMFMediaSource +MFCreateAttributes +MFEnumDeviceSources +MFMediaType_Video +MF_MT_SUBTYPE +MF_MT_FRAME_RATE +MF_MT_FRAME_SIZE +MF_DEVSOURCE_ATTRIBUTE_FRIENDLY_NAME +MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE +MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE_VIDCAP_GUID + +ICreateDevEnum +IPropertyBag +CLSID_SystemDeviceEnum +CLSID_VideoInputDeviceCategory +VariantInit +VariantClear +S_OK \ No newline at end of file diff --git a/src/VL.OpenCV.csproj b/src/VL.OpenCV.csproj index 5d68a75..2332422 100644 --- a/src/VL.OpenCV.csproj +++ b/src/VL.OpenCV.csproj @@ -2,56 +2,32 @@ - {BF6BC14C-CC1F-4F5F-B491-452662B4D4A1} - VL.OpenCV - VL.OpenCV - net472 + net8.0-windows ..\lib\ - 2.5.0 - - + 3.0.0 true 1701;1702;MSB3270;MSB3026 + true - - true - - - - - - ..\lib\dx11\FeralTic.dll - - - - - - - - - - - - - ..\lib\dx11\VVVV.DX11.Core.dll - - + + all + - - + + \ No newline at end of file diff --git a/src/VideoInInfo.cs b/src/VideoInInfo.cs index 88307e0..0650b4b 100644 --- a/src/VideoInInfo.cs +++ b/src/VideoInInfo.cs @@ -1,15 +1,20 @@ -using DirectShowLib; -using SharpDX.MediaFoundation; -using SharpDX.Multimedia; +using SharpDX.Multimedia; using System; using System.Collections.Generic; using System.Linq; +using Windows.Win32; +using Windows.Win32.Foundation; +using Windows.Win32.Media.DirectShow; +using Windows.Win32.Media.MediaFoundation; +using Windows.Win32.System.Com; +using Windows.Win32.System.Com.StructuredStorage; +using Windows.Win32.System.Variant; +using static Windows.Win32.PInvoke; + namespace VL.OpenCV { - - - public class VideoInInfo + public unsafe class VideoInInfo { public class Format { @@ -37,63 +42,99 @@ public static string GetSupportedFormats(int deviceIndex) } } - static IEnumerable EnumerateSupportedFormats(int deviceIndex) + static unsafe IEnumerable EnumerateSupportedFormats(int deviceIndex) { + var result = new List(); var devices = EnumerateVideoDevices(); if (deviceIndex < devices.Length) { var device = devices[deviceIndex]; - var mediaSource = device.ActivateObject(); - mediaSource.CreatePresentationDescriptor(out PresentationDescriptor descriptor); - var streamDescriptor = descriptor.GetStreamDescriptorByIndex(0, out SharpDX.Mathematics.Interop.RawBool _); - var handler = streamDescriptor.MediaTypeHandler; - for (int i = 0; i < handler.MediaTypeCount; i++) + var mediaSource = (IMFMediaSource*)device->ActivateObject(in IMFMediaSource.IID_Guid); + + IMFPresentationDescriptor* descriptor = null; + IMFStreamDescriptor* streamDescriptor = null; + IMFMediaTypeHandler* handler = null; + + try { - var mediaType = handler.GetMediaTypeByIndex(i); - if (mediaType.MajorType == MediaTypeGuids.Video) + mediaSource->CreatePresentationDescriptor(&descriptor); + descriptor->GetStreamDescriptorByIndex(0, out _, &streamDescriptor); + streamDescriptor->GetMediaTypeHandler(&handler); + handler->GetMediaTypeCount(out var mediaTypeCount); + + for (uint i = 0; i < mediaTypeCount; i++) { - var frameRate = ParseFrameRate(mediaType.Get(MediaTypeAttributeKeys.FrameRate)); - ParseSize(mediaType.Get(MediaTypeAttributeKeys.FrameSize), out int width, out int height); + IMFMediaType* mediaType; + handler->GetMediaTypeByIndex(i, &mediaType); + try + { + mediaType->GetMajorType(out var majorType); + if (majorType == MFMediaType_Video) + { + mediaType->GetUINT64(MF_MT_FRAME_RATE, out var fr); + var frameRate = ParseFrameRate(fr); + mediaType->GetUINT64(MF_MT_FRAME_SIZE, out var fs); + ParseSize(fs, out int width, out int height); - var format = GetVideoFormat(mediaType); + var format = GetVideoFormat(mediaType); - yield return new Format() { w = width, h = height, fr = frameRate, format = format }; + result.Add(new Format() { w = width, h = height, fr = frameRate, format = format }); + } + } + finally + { + mediaType->Release(); + } } } + finally + { + handler->Release(); + streamDescriptor->Release(); + descriptor->Release(); + device->Release(); + } } else { throw new IndexOutOfRangeException(); } + return result; } - public static Activate[] EnumerateVideoDevices() + internal static unsafe IMFActivate*[] EnumerateVideoDevices() { - var attributes = new MediaAttributes(); - MediaFactory.CreateAttributes(attributes, 1); - attributes.Set(CaptureDeviceAttributeKeys.SourceType, CaptureDeviceAttributeKeys.SourceTypeVideoCapture.Guid); - var mediaFoundationActivates = MediaFactory.EnumDeviceSources(attributes); - Activate[] result = new Activate[mediaFoundationActivates.Length]; + IMFAttributes* attributes; + MFCreateAttributes(&attributes, 1).ThrowOnFailure(); + + attributes->SetGUID(MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE, MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE_VIDCAP_GUID); + MFEnumDeviceSources(attributes, out var devicePtr, out var count).ThrowOnFailure(); + + var result = new IMFActivate*[count]; + for (int i = 0; i < result.Length; i++) + result[i] = devicePtr[i]; + Dictionary order = new Dictionary(); - DsDevice[] capDevicesDS; - capDevicesDS = DsDevice.GetDevicesOfCat(FilterCategory.VideoInputDevice); + var capDevicesDS = EnumerateCaptureDevicesDS(); //tries to match the order of the found devices in DirectShow and MediaFoundation - for (int i = 0; i < mediaFoundationActivates.Length; i++) + for (int i = 0; i < result.Length; i++) { - var friendlyName = mediaFoundationActivates[i].Get(CaptureDeviceAttributeKeys.FriendlyName); + var friendlyName = GetString(result[i], in MF_DEVSOURCE_ATTRIBUTE_FRIENDLY_NAME); var suffix = ""; //used to handle multiple devices listed with the same name var counter = 1; - for (int j = 0; j < capDevicesDS.Length; j++) + for (int j = 0; j < capDevicesDS.Count; j++) { - var friendlyNameDS = capDevicesDS[j].Name + suffix; + var friendlyNameDS = capDevicesDS[j] + suffix; if (friendlyName + suffix == friendlyNameDS) { if (!order.ContainsKey(friendlyName + suffix)) { order.Add(friendlyName + suffix, new int[] { i, j }); - result[j] = mediaFoundationActivates[i]; + var tmp = result[j]; + result[j] = result[i]; + result[i] = tmp; suffix = ""; break; } @@ -108,27 +149,74 @@ public static Activate[] EnumerateVideoDevices() return result; } + private static List EnumerateCaptureDevicesDS() + { + var capDevicesDS = new List(); + CoCreateInstance(in CLSID_SystemDeviceEnum, null, CLSCTX.CLSCTX_INPROC_SERVER, out var devEnum).ThrowOnFailure(); + + IEnumMoniker* pEnum = null; + var hr = devEnum->CreateClassEnumerator(in CLSID_VideoInputDeviceCategory, &pEnum, 0); + if (pEnum != null) + { + IMoniker* moniker = null; + while (pEnum->Next(1, &moniker, null) == HRESULT.S_OK) + { + try + { + var id = typeof(IPropertyBag).GUID; + moniker->BindToStorage(null, null, in id, out var bag); + var propertyBag = (IPropertyBag*)bag; + VARIANT val = default; + VariantInit(&val); + propertyBag->Read("FriendlyName", ref val, null); + capDevicesDS.Add(val.Anonymous.Anonymous.Anonymous.bstrVal.AsSpan().ToString()); + VariantClear(&val); + propertyBag->Release(); + } + finally + { + moniker->Release(); + } + } + pEnum->Release(); + } + devEnum->Release(); + return capDevicesDS; + } - private static void ParseSize(long value, out int width, out int height) + private static void ParseSize(ulong value, out int width, out int height) { width = (int)(value >> 32); height = (int)(value & 0x00000000FFFFFFFF); } - private static float ParseFrameRate(long value) + private static float ParseFrameRate(ulong value) { var numerator = (int)(value >> 32); var denominator = (int)(value & 0x00000000FFFFFFFF); return numerator * 100 / denominator / 100f; } - private static string GetVideoFormat(SharpDX.MediaFoundation.MediaType mediaType) + private static string GetVideoFormat(IMFMediaType* mediaType) { // https://docs.microsoft.com/en-us/windows/desktop/medfound/video-subtype-guids - var subTypeId = mediaType.Get(MediaTypeAttributeKeys.Subtype); + mediaType->GetGUID(MF_MT_SUBTYPE, out var subTypeId); var fourccEncoded = BitConverter.ToInt32(subTypeId.ToByteArray(), 0); var fourcc = new FourCC(fourccEncoded); return fourcc.ToString(); } + + internal static unsafe string GetString(IMFActivate* activate, in Guid guid) + { + activate->GetStringLength(in guid, out var length); + + Span buffer = stackalloc char[(int)length + 1]; + fixed (char* c = buffer) + { + activate->GetString(in guid, new PWSTR(c), (uint)buffer.Length, &length); + } + + return buffer.Slice(0, (int)length).ToString(); + } } } diff --git a/src/YOLO3Helper.cs b/src/YOLO3Helper.cs index 72804d4..d2b1010 100644 --- a/src/YOLO3Helper.cs +++ b/src/YOLO3Helper.cs @@ -6,7 +6,6 @@ using FeralTic.DX11.Geometry; using OpenCvSharp; using OpenCvSharp.Dnn; -using SharpDX.Direct3D9; using Stride.Core.Mathematics; using VL.Lib.Collections; From 3e928d482b3d2c8098bb8bac5b9421fd1a66e3cf Mon Sep 17 00:00:00 2001 From: Elias Holzer Date: Tue, 5 Nov 2024 17:11:46 +0100 Subject: [PATCH 2/3] Adds `VideoSourceToCvImage` to efficiently connect the node set to video sources from VL.Video and others --- VL.OpenCV.vl | 92 +++++++++++++++++++++++++++++++++++-- src/VL.OpenCV.csproj | 3 +- src/VideoSourceToCvImage.cs | 58 +++++++++++++++++++++++ 3 files changed, 147 insertions(+), 6 deletions(-) create mode 100644 src/VideoSourceToCvImage.cs diff --git a/VL.OpenCV.vl b/VL.OpenCV.vl index 0046659..b5d5daf 100644 --- a/VL.OpenCV.vl +++ b/VL.OpenCV.vl @@ -1,4 +1,4 @@ - + @@ -21339,6 +21339,91 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/VideoSourceToCvImage.cs b/src/VideoSourceToCvImage.cs new file mode 100644 index 0000000..b03d4e7 --- /dev/null +++ b/src/VideoSourceToCvImage.cs @@ -0,0 +1,58 @@ +#nullable enable +using Microsoft.Extensions.DependencyInjection; +using OpenCvSharp; +using System; +using VL.Core; +using VL.Core.Utils; +using VL.Lib.Animation; +using VL.Lib.Basics.Resources; +using VL.Lib.Basics.Video; +using VL.Lib.Video; + +namespace VL.OpenCV +{ + public class VideoSourceToCvImage : VideoSourceToImage + { + public VideoSourceToCvImage(NodeContext nodeContext) + { + var frameClock = nodeContext.AppHost.Services.GetRequiredService(); + Context = new VideoPlaybackContext(frameClock); + } + + protected override VideoPlaybackContext Context { get; } + + protected override void OnPush(IResourceProvider videoFrameProvider, bool mipmapped) + { + var handle = videoFrameProvider.GetHandle(); + var imageHandle = ToCvImage(handle).GetHandle(); + if (!resultQueue.TryAddSafe(imageHandle, millisecondsTimeout: 10)) + imageHandle.Dispose(); + } + + protected override void OnPull(IResourceProvider? videoFrameProvider, bool mipmapped) + { + if (videoFrameProvider is null) + return; + + var handle = ToCvImage(videoFrameProvider.GetHandle()).GetHandle(); + if (handle != null && !resultQueue.TryAddSafe(handle)) + handle.Dispose(); + } + + private static unsafe IResourceProvider ToCvImage(IResourceHandle handle) + { + var videoFrame = handle.Resource; + if (!videoFrame.TryGetMemory(out var memory)) + throw new NotImplementedException(); + + var memoryHandle = memory.Pin(); + var mat = new Mat(videoFrame.Height, videoFrame.Width, videoFrame.PixelFormat.ToMatType(string.Empty), data: (nint)memoryHandle.Pointer, step: videoFrame.RowLengthInBytes); + return ResourceProvider.Return(new CvImage(mat), (mat, memoryHandle, handle), x => + { + x.mat.Dispose(); + x.memoryHandle.Dispose(); + x.handle.Dispose(); + }); + } + } +} From 275cf5d5d88dde70ba6c78f0021d2cb5cae8a5db Mon Sep 17 00:00:00 2001 From: Elias Holzer Date: Tue, 5 Nov 2024 17:19:39 +0100 Subject: [PATCH 3/3] No need for public COM interfaces --- src/NativeMethods.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/NativeMethods.json b/src/NativeMethods.json index b757aa8..d6c069d 100644 --- a/src/NativeMethods.json +++ b/src/NativeMethods.json @@ -1,6 +1,5 @@ { "$schema": "https://aka.ms/CsWin32.schema.json", "allowMarshaling": false, - "comInterop": { "preserveSigMethods": [ "IAMCameraControl", "IAMVideoProcAmp", "IMFMediaEngine", "ICreateDevEnum" ] }, - "public": true + "comInterop": { "preserveSigMethods": [ "ICreateDevEnum" ] } } \ No newline at end of file