Support all raw formats

Cargo.toml

@@ -32,8 +32,8 @@ actix-web = "4.1.0"
 actix-web-validator = "5.0.1"
 actix-service = "2.0.2"
 actix-extensible-rate-limit = "0.2.1"
-serde = { version = "1.0.140", features = ["derive"] }
-serde_json = "1.0.82"
+serde = { version = "1.0.183", features = ["derive"] }
+serde_json = "1.0.104"
 validator = { version = "0.16", features = ["derive"] }
 
 ## FINAL

src/stream/pipeline/fake_pipeline.rs

@@ -1,7 +1,7 @@
 use crate::{
     stream::types::CaptureConfiguration,
     video::{
-        types::{VideoEncodeType, VideoSourceType},
+        types::{VideoEncodeType, VideoSourceType, DEFAULT_RAW_FORMAT, KNOWN_RTP_RAW_FORMATS},
         video_source_gst::VideoSourceGstType,
     },
     video_stream::types::VideoAndStreamInformation,
@@ -70,14 +70,15 @@ impl FakePipeline {
                 format!(concat!(
                         "videotestsrc pattern={pattern} is-live=true do-timestamp=true",
                         " ! timeoverlay",
-                        " ! video/x-raw,format=I420",
+                        " ! video/x-raw,format={format}",
                         " ! x264enc tune=zerolatency speed-preset=ultrafast bitrate=5000",
                         " ! h264parse",
                         " ! capsfilter name={filter_name} caps=video/x-h264,profile={profile},stream-format=avc,alignment=au,width={width},height={height},framerate={interval_denominator}/{interval_numerator}",
                         " ! rtph264pay aggregate-mode=zero-latency config-interval=10 pt=96",
                         " ! tee name={sink_tee_name} allow-not-linked=true"
                     ),
                     pattern = pattern,
+                    format = DEFAULT_RAW_FORMAT,
                     profile = "constrained-baseline",
                     width = configuration.width,
                     height = configuration.height,
@@ -87,20 +88,22 @@ impl FakePipeline {
                     sink_tee_name = sink_tee_name,
                 )
             }
-            VideoEncodeType::Yuyv => {
+            VideoEncodeType::Raw(fourcc) => {
+                let mut fourcc = fourcc.clone();
+                if !KNOWN_RTP_RAW_FORMATS.contains(&fourcc.as_str()) {
+                    fourcc = DEFAULT_RAW_FORMAT.to_string();
+                }
+
                 format!(
                     concat!(
-                        // Because application-rtp templates doesn't accept "YUY2", we
-                        // need to transcode it. We are arbitrarily chosing the closest
-                        // format available ("UYVY").
                         "videotestsrc pattern={pattern} is-live=true do-timestamp=true",
                         " ! timeoverlay",
-                        " ! video/x-raw,format=I420",
-                        " ! capsfilter name={filter_name} caps=video/x-raw,format=I420,width={width},height={height},framerate={interval_denominator}/{interval_numerator}",
+                        " ! capsfilter name={filter_name} caps=video/x-raw,format={format},width={width},height={height},framerate={interval_denominator}/{interval_numerator}",
                         " ! rtpvrawpay pt=96",
                         " ! tee name={sink_tee_name} allow-not-linked=true",
                     ),
                     pattern = pattern,
+                    format = fourcc,
                     width = configuration.width,
                     height = configuration.height,
                     interval_denominator = configuration.frame_interval.denominator,
@@ -114,13 +117,14 @@ impl FakePipeline {
                     concat!(
                         "videotestsrc pattern={pattern} is-live=true do-timestamp=true",
                         " ! timeoverlay",
-                        " ! video/x-raw,format=I420",
+                        " ! video/x-raw,format={format}",
                         " ! jpegenc quality=85 idct-method=1",
                         " ! capsfilter name={filter_name} caps=image/jpeg,width={width},height={height},framerate={interval_denominator}/{interval_numerator}",
                         " ! rtpjpegpay pt=96",
                         " ! tee name={sink_tee_name} allow-not-linked=true",
                     ),
                     pattern = pattern,
+                    format = DEFAULT_RAW_FORMAT,
                     width = configuration.width,
                     height = configuration.height,
                     interval_denominator = configuration.frame_interval.denominator,

src/stream/pipeline/v4l_pipeline.rs

@@ -1,6 +1,6 @@
 use crate::{
     stream::types::CaptureConfiguration,
-    video::types::{VideoEncodeType, VideoSourceType},
+    video::types::{VideoEncodeType, VideoSourceType, DEFAULT_RAW_FORMAT, KNOWN_RTP_RAW_FORMATS},
     video_stream::types::VideoAndStreamInformation,
 };
 
@@ -69,16 +69,22 @@ impl V4lPipeline {
                     sink_tee_name = sink_tee_name,
                 )
             }
-            VideoEncodeType::Yuyv => {
+            VideoEncodeType::Raw(fourcc) => {
+                let mut fourcc = fourcc.clone();
+                if !KNOWN_RTP_RAW_FORMATS.contains(&fourcc.as_str()) {
+                    fourcc = DEFAULT_RAW_FORMAT.to_string();
+                }
+
                 format!(
                     concat!(
                         "v4l2src device={device} do-timestamp=false",
                         " ! videoconvert",
-                        " ! capsfilter name={filter_name} caps=video/x-raw,format=I420,width={width},height={height},framerate={interval_denominator}/{interval_numerator}",
+                        " ! capsfilter name={filter_name} caps=video/x-raw,format={format},width={width},height={height},framerate={interval_denominator}/{interval_numerator}",
                         " ! rtpvrawpay pt=96",
                         " ! tee name={sink_tee_name} allow-not-linked=true"
                     ),
                     device = device,
+                    format = fourcc,
                     width = width,
                     height = height,
                     interval_denominator = interval_denominator,

src/stream/sink/image_sink.rs

@@ -356,7 +356,7 @@ impl ImageSink {
                 _transcoding_elements.push(parser);
                 _transcoding_elements.push(decoder);
             }
-            VideoEncodeType::Yuyv => {
+            VideoEncodeType::Raw(_) => {
                 let depayloader = gst::ElementFactory::make("rtpvrawdepay").build()?;
                 _transcoding_elements.push(depayloader);
             }

src/video/types.rs

@@ -5,21 +5,28 @@ use super::video_source_redirect::VideoSourceRedirect;
 use paperclip::actix::Apiv2Schema;
 use serde::{Deserialize, Serialize};
 
+pub const KNOWN_RTP_RAW_FORMATS: [&str; 9] = [
+    "RGB", "RGBA", "BGR", "BGRA", "AYUV", "UYVY", "I420", "Y41B", "UYVP",
+];
+pub const DEFAULT_RAW_FORMAT: &str = "I420";
+
 #[derive(Apiv2Schema, Clone, Debug, PartialEq, Serialize, Deserialize)]
 pub enum VideoSourceType {
     Gst(VideoSourceGst),
     Local(VideoSourceLocal),
     Redirect(VideoSourceRedirect),
 }
 
-#[derive(Apiv2Schema, Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize)]
+#[derive(Apiv2Schema, Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize)]
 #[serde(rename_all = "UPPERCASE")]
 pub enum VideoEncodeType {
-    Unknown(String),
     H265,
     H264,
     Mjpg,
-    Yuyv,
+    #[serde(untagged)]
+    Raw(String),
+    #[serde(untagged)]
+    Unknown(String),
 }
 
 #[derive(Apiv2Schema, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize)]
@@ -102,15 +109,64 @@ impl VideoSourceType {
 }
 
 impl VideoEncodeType {
-    //TODO: use trait fromstr, check others places
     pub fn from_str(fourcc: &str) -> VideoEncodeType {
         let fourcc = fourcc.to_uppercase();
-        match fourcc.as_str() {
-            "H264" => VideoEncodeType::H264,
-            "MJPG" => VideoEncodeType::Mjpg,
-            "YUYV" => VideoEncodeType::Yuyv,
-            _ => VideoEncodeType::Unknown(fourcc),
+
+        let fourcc = match fourcc.as_str() {
+            "H264" => return VideoEncodeType::H264,
+            "MJPG" => return VideoEncodeType::Mjpg,
+            // TODO: We can implement a [GstDeviceMonitor](https://gstreamer.freedesktop.org/documentation/gstreamer/gstdevicemonitor.html?gi-language=c) and then this manual mapping between v4l's and gst's fourcc will not be neccessary anymore.
+            // A list of possible v4l fourcc from the Linux docs:
+            // - [YUV](https://www.kernel.org/doc/html/v4.8/media/uapi/v4l/yuv-formats.html)
+            // - [RGB](https://www.kernel.org/doc/html/v4.8/media/uapi/v4l/pixfmt-rgb.html)
+            // - [compressed](https://www.kernel.org/doc/html/v4.8/media/uapi/v4l/pixfmt-013.html)
+            "YUYV" => "YUY2".to_string(),
+            _ => fourcc,
+        };
+
+        // Use Gstreamer to recognize raw formats. [GStreamer raw formats](https://gstreamer.freedesktop.org/documentation/additional/design/mediatype-video-raw.html?gi-language=c#formats)
+        use std::ffi::CString;
+        if let Ok(c_char_array) = CString::new(fourcc.clone()).map(|c_str| c_str.into_raw()) {
+            use gst_video::ffi::*;
+
+            let gst_video_format = unsafe { gst_video_format_from_string(c_char_array) };
+
+            if !matches!(
+                gst_video_format,
+                GST_VIDEO_FORMAT_UNKNOWN | GST_VIDEO_FORMAT_ENCODED
+            ) {
+                return VideoEncodeType::Raw(fourcc);
+            }
+        }
+
+        VideoEncodeType::Unknown(fourcc)
+    }
+}
+impl<'de> Deserialize<'de> for VideoEncodeType {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        struct VideoEncodeTypeVisitor;
+
+        use std::fmt;
+
+        impl<'de> serde::de::Visitor<'de> for VideoEncodeTypeVisitor {
+            type Value = VideoEncodeType;
+
+            fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
+                formatter.write_str("valid video encode type")
+            }
+
+            fn visit_str<E>(self, value: &str) -> Result<Self::Value, E>
+            where
+                E: serde::de::Error,
+            {
+                Ok(Self::Value::from_str(value))
+            }
         }
+
+        deserializer.deserialize_str(VideoEncodeTypeVisitor)
     }
 }
 
@@ -141,3 +197,27 @@ pub static STANDARD_SIZES: &[(u32, u32); 16] = &[
     (320, 240),
     (256, 144),
 ];
+
+mod tests {
+
+    #[test]
+    fn test_video_encode_type_serde() {
+        use super::VideoEncodeType;
+
+        for (encode, encode_str) in [
+            (VideoEncodeType::Mjpg, "\"MJPG\""),
+            (VideoEncodeType::H264, "\"H264\""),
+            (VideoEncodeType::Raw("I420".to_string()), "\"I420\""),
+            (VideoEncodeType::Unknown("POTATO".to_string()), "\"POTATO\""),
+        ] {
+            debug_assert_eq!(encode_str, serde_json::to_string(&encode).unwrap());
+
+            debug_assert_eq!(encode, serde_json::from_str(encode_str).unwrap());
+        }
+
+        // Ensure UYUV is parsed as YUY2:
+        let encode = VideoEncodeType::Raw("YUY2".to_string());
+        let legacy_encode_str = "\"YUYV\"";
+        debug_assert_eq!(encode, serde_json::from_str(legacy_encode_str).unwrap());
+    }
+}

src/video/video_source_gst.rs

@@ -66,7 +66,7 @@ impl VideoSource for VideoSourceGst {
                         sizes: sizes.clone(),
                     },
                     Format {
-                        encode: VideoEncodeType::Yuyv,
+                        encode: VideoEncodeType::Raw(DEFAULT_RAW_FORMAT.to_string()),
                         sizes: sizes.clone(),
                     },
                     Format {