Video encoder frame

CMakeLists.txt

@@ -229,6 +229,7 @@ add_library(${TARGET_CORE_NAME}
     src/pipeline/node/UVC.cpp
     src/pipeline/datatype/Buffer.cpp
     src/pipeline/datatype/ImgFrame.cpp
+    src/pipeline/datatype/EncodedFrame.cpp
     src/pipeline/datatype/ImageManipConfig.cpp
     src/pipeline/datatype/CameraControl.cpp
     src/pipeline/datatype/NNData.cpp
@@ -247,6 +248,7 @@ add_library(${TARGET_CORE_NAME}
     src/pipeline/datatype/TrackedFeatures.cpp
     src/pipeline/datatype/FeatureTrackerConfig.cpp
     src/pipeline/datatype/ToFConfig.cpp
+    src/utility/H26xParsers.cpp
     src/utility/Initialization.cpp
     src/utility/Resources.cpp
     src/utility/Path.cpp

cmake/Depthai/DepthaiDeviceSideConfig.cmake

@@ -2,7 +2,7 @@
 set(DEPTHAI_DEVICE_SIDE_MATURITY "snapshot")
 
 # "full commit hash of device side binary"
-set(DEPTHAI_DEVICE_SIDE_COMMIT "d088dd017f964a3b51ecb1165b5e625e35d78af4")
+set(DEPTHAI_DEVICE_SIDE_COMMIT "1cf15832ab1f408d8e4dab72e901ce050bf8850b")
 
 # "version if applicable"
 set(DEPTHAI_DEVICE_SIDE_VERSION "")

include/depthai/pipeline/datatype/EncodedFrame.hpp

@@ -0,0 +1,144 @@
+#pragma once
+
+#include <chrono>
+
+#include "depthai/pipeline/datatype/Buffer.hpp"
+
+// shared
+#include "depthai-shared/datatype/RawEncodedFrame.hpp"
+
+// optional
+#ifdef DEPTHAI_HAVE_OPENCV_SUPPORT
+    #include <opencv2/opencv.hpp>
+#endif
+
+namespace dai {
+
+class EncodedFrame : public Buffer {
+    std::shared_ptr<RawBuffer> serialize() const override;
+    RawEncodedFrame& frame;
+
+   public:
+    // Raw* mirror
+    using Profile = RawEncodedFrame::Profile;
+    using FrameType = RawEncodedFrame::FrameType;
+
+    /**
+     * Construct EncodedFrame message.
+     * Timestamp is set to now
+     */
+    EncodedFrame();
+    explicit EncodedFrame(std::shared_ptr<RawEncodedFrame> ptr);
+    virtual ~EncodedFrame() = default;
+
+    // getters
+    /**
+     * Retrieves instance number
+     */
+    unsigned int getInstanceNum() const;
+    /**
+     * Retrieves exposure time
+     */
+    std::chrono::microseconds getExposureTime() const;
+
+    /**
+     * Retrieves sensitivity, as an ISO value
+     */
+    int getSensitivity() const;
+
+    /**
+     * Retrieves white-balance color temperature of the light source, in kelvins
+     */
+    int getColorTemperature() const;
+
+    /**
+     * Retrieves lens position, range 0..255. Returns -1 if not available
+     */
+    int getLensPosition() const;
+    /**
+     * Retrieves the encoding quality
+     */
+    unsigned int getQuality() const;
+
+    /**
+     * Retrieves the encoding bitrate
+     */
+    unsigned int getBitrate() const;
+
+    /**
+     * Returns true if encoding is lossless (JPEG only)
+     */
+    bool getLossless() const;
+
+    /**
+     * Retrieves frame type (H26x only)
+     */
+    FrameType getFrameType() const;
+
+    /**
+     * Retrieves the encoding profile (JPEG, AVC or HEVC)
+     */
+    Profile getProfile() const;
+
+    // setters
+    /**
+     * Retrieves image timestamp related to dai::Clock::now()
+     */
+    EncodedFrame& setTimestamp(std::chrono::time_point<std::chrono::steady_clock, std::chrono::steady_clock::duration> tp);
+
+    /**
+     * Sets image timestamp related to dai::Clock::now()
+     */
+    EncodedFrame& setTimestampDevice(std::chrono::time_point<std::chrono::steady_clock, std::chrono::steady_clock::duration> tp);
+
+    /**
+     * Specifies sequence number
+     *
+     * @param seq Sequence number
+     */
+    EncodedFrame& setSequenceNum(int64_t seq);
+
+    /**
+     * Instance number relates to the origin of the frame (which camera)
+     *
+     * @param instance Instance number
+     */
+    EncodedFrame& setInstanceNum(unsigned int instance);
+
+    /**
+     * Specifies the encoding quality
+     *
+     * @param quality Encoding quality
+     */
+    EncodedFrame& setQuality(unsigned int quality);
+
+    /**
+     * Specifies the encoding quality
+     *
+     * @param quality Encoding quality
+     */
+    EncodedFrame& setBitrate(unsigned int bitrate);
+
+    /**
+     * Specifies if encoding is lossless (JPEG only)
+     *
+     * @param lossless True if lossless
+     */
+    EncodedFrame& setLossless(bool lossless);
+
+    /**
+     * Specifies the frame type (H26x only)
+     *
+     * @param type Type of h26x frame (I, P, B)
+     */
+    EncodedFrame& setFrameType(FrameType type);
+
+    /**
+     * Specifies the encoding profile
+     *
+     * @param profile Encoding profile
+     */
+    EncodedFrame& setProfile(Profile profile);
+};
+
+}  // namespace dai

include/depthai/pipeline/node/VideoEncoder.hpp

@@ -25,10 +25,15 @@ class VideoEncoder : public NodeCRTP<Node, VideoEncoder, VideoEncoderProperties>
     Input input{*this, "in", Input::Type::SReceiver, true, 4, true, {{DatatypeEnum::ImgFrame, true}}};
 
     /**
-     * Outputs ImgFrame message that carries BITSTREAM encoded (MJPEG, H264 or H265) frame data.
+     * Outputs ImgFrame message that carries BITSTREAM encoded (MJPEG, H264 or H265) frame data. Mutually exclusive with out.
      */
     Output bitstream{*this, "bitstream", Output::Type::MSender, {{DatatypeEnum::ImgFrame, false}}};
 
+    /**
+     * Outputs EncodedFrame message that carries encoded (MJPEG, H264 or H265) frame data. Mutually exclusive with bitstream.
+     */
+    Output out{*this, "out", Output::Type::MSender, {{DatatypeEnum::EncodedFrame, false}}};
+
     // Sets default options for a specified size and profile
     /**
      * Sets a default preset based on specified frame rate and profile

shared/depthai-shared.cmake

@@ -63,4 +63,4 @@ foreach(source_file ${DEPTHAI_SHARED_SOURCES})
     if(NOT EXISTS ${source_file})
         message(FATAL_ERROR "depthai-shared submodule files missing. Make sure to download prepackaged release instead of \"Source code\" on GitHub. Example: depthai-core-vX.Y.Z.tar.gz")
     endif()
-endforeach()
+endforeach()

src/pipeline/datatype/EncodedFrame.cpp

@@ -0,0 +1,121 @@
+#include "depthai/pipeline/datatype/EncodedFrame.hpp"
+
+#include "utility/H26xParsers.hpp"
+
+namespace dai {
+
+std::shared_ptr<RawBuffer> EncodedFrame::serialize() const {
+    return raw;
+}
+
+EncodedFrame::EncodedFrame() : Buffer(std::make_shared<RawEncodedFrame>()), frame(*dynamic_cast<RawEncodedFrame*>(raw.get())) {
+    // set timestamp to now
+    setTimestamp(std::chrono::steady_clock::now());
+}
+EncodedFrame::EncodedFrame(std::shared_ptr<RawEncodedFrame> ptr) : Buffer(std::move(ptr)), frame(*dynamic_cast<RawEncodedFrame*>(raw.get())) {}
+
+// getters
+unsigned int EncodedFrame::getInstanceNum() const {
+    return frame.instanceNum;
+}
+std::chrono::microseconds EncodedFrame::getExposureTime() const {
+    return std::chrono::microseconds(frame.cam.exposureTimeUs);
+}
+int EncodedFrame::getSensitivity() const {
+    return frame.cam.sensitivityIso;
+}
+int EncodedFrame::getColorTemperature() const {
+    return frame.cam.wbColorTemp;
+}
+int EncodedFrame::getLensPosition() const {
+    return frame.cam.lensPosition;
+}
+unsigned int EncodedFrame::getQuality() const {
+    return frame.quality;
+}
+unsigned int EncodedFrame::getBitrate() const {
+    return frame.bitrate;
+}
+bool EncodedFrame::getLossless() const {
+    return frame.lossless;
+}
+EncodedFrame::FrameType EncodedFrame::getFrameType() const {
+    if(frame.type == FrameType::Unknown) {
+        utility::SliceType frameType;
+        switch(frame.profile) {
+            case RawEncodedFrame::Profile::JPEG:
+                frameType = utility::SliceType::I;
+                break;
+            case RawEncodedFrame::Profile::AVC:
+                frameType = utility::getTypesH264(frame.data, true)[0];
+                break;
+            case RawEncodedFrame::Profile::HEVC:
+                frameType = utility::getTypesH265(frame.data, true)[0];
+                break;
+        }
+        switch(frameType) {
+            case utility::SliceType::P:
+                frame.type = FrameType::P;
+                break;
+            case utility::SliceType::B:
+                frame.type = FrameType::B;
+                break;
+            case utility::SliceType::I:
+                frame.type = FrameType::I;
+                break;
+            case utility::SliceType::SP:
+                frame.type = FrameType::P;
+                break;
+            case utility::SliceType::SI:
+                frame.type = FrameType::I;
+                break;
+            case utility::SliceType::Unknown:
+                frame.type = FrameType::Unknown;
+                break;
+        }
+    }
+    return frame.type;
+}
+EncodedFrame::Profile EncodedFrame::getProfile() const {
+    return frame.profile;
+}
+
+// setters
+EncodedFrame& EncodedFrame::setTimestamp(std::chrono::time_point<std::chrono::steady_clock, std::chrono::steady_clock::duration> tp) {
+    // Set timestamp from timepoint
+    return static_cast<EncodedFrame&>(Buffer::setTimestamp(tp));
+}
+EncodedFrame& EncodedFrame::setTimestampDevice(std::chrono::time_point<std::chrono::steady_clock, std::chrono::steady_clock::duration> tp) {
+    // Set timestamp from timepoint
+    return static_cast<EncodedFrame&>(Buffer::setTimestampDevice(tp));
+}
+EncodedFrame& EncodedFrame::setSequenceNum(int64_t sequenceNum) {
+    return static_cast<EncodedFrame&>(Buffer::setSequenceNum(sequenceNum));
+}
+EncodedFrame& EncodedFrame::setInstanceNum(unsigned int instanceNum) {
+    frame.instanceNum = instanceNum;
+    return *this;
+}
+EncodedFrame& EncodedFrame::setQuality(unsigned int quality) {
+    frame.quality = quality;
+    return *this;
+}
+EncodedFrame& EncodedFrame::setBitrate(unsigned int bitrate) {
+    frame.bitrate = bitrate;
+    return *this;
+}
+
+EncodedFrame& EncodedFrame::setLossless(bool lossless) {
+    frame.lossless = lossless;
+    return *this;
+}
+EncodedFrame& EncodedFrame::setFrameType(FrameType frameType) {
+    frame.type = frameType;
+    return *this;
+}
+EncodedFrame& EncodedFrame::setProfile(Profile profile) {
+    frame.profile = profile;
+    return *this;
+}
+
+}  // namespace dai

src/pipeline/datatype/StreamMessageParser.cpp

@@ -15,6 +15,7 @@
 #include "depthai/pipeline/datatype/Buffer.hpp"
 #include "depthai/pipeline/datatype/CameraControl.hpp"
 #include "depthai/pipeline/datatype/EdgeDetectorConfig.hpp"
+#include "depthai/pipeline/datatype/EncodedFrame.hpp"
 #include "depthai/pipeline/datatype/FeatureTrackerConfig.hpp"
 #include "depthai/pipeline/datatype/IMUData.hpp"
 #include "depthai/pipeline/datatype/ImageManipConfig.hpp"
@@ -37,6 +38,7 @@
 #include "depthai-shared/datatype/RawBuffer.hpp"
 #include "depthai-shared/datatype/RawCameraControl.hpp"
 #include "depthai-shared/datatype/RawEdgeDetectorConfig.hpp"
+#include "depthai-shared/datatype/RawEncodedFrame.hpp"
 #include "depthai-shared/datatype/RawFeatureTrackerConfig.hpp"
 #include "depthai-shared/datatype/RawIMUData.hpp"
 #include "depthai-shared/datatype/RawImageManipConfig.hpp"
@@ -122,6 +124,10 @@ std::shared_ptr<RawBuffer> StreamMessageParser::parseMessage(streamPacketDesc_t*
             return parseDatatype<RawImgFrame>(metadataStart, serializedObjectSize, data);
             break;
 
+        case DatatypeEnum::EncodedFrame:
+            return parseDatatype<RawEncodedFrame>(metadataStart, serializedObjectSize, data);
+            break;
+
         case DatatypeEnum::NNData:
             return parseDatatype<RawNNData>(metadataStart, serializedObjectSize, data);
             break;
@@ -213,6 +219,10 @@ std::shared_ptr<ADatatype> StreamMessageParser::parseMessageToADatatype(streamPa
             return std::make_shared<ImgFrame>(parseDatatype<RawImgFrame>(metadataStart, serializedObjectSize, data));
             break;
 
+        case DatatypeEnum::EncodedFrame:
+            return std::make_shared<EncodedFrame>(parseDatatype<RawEncodedFrame>(metadataStart, serializedObjectSize, data));
+            break;
+
         case DatatypeEnum::NNData:
             return std::make_shared<NNData>(parseDatatype<RawNNData>(metadataStart, serializedObjectSize, data));
             break;

src/pipeline/node/VideoEncoder.cpp

@@ -15,7 +15,7 @@ VideoEncoder::VideoEncoder(const std::shared_ptr<PipelineImpl>& par, int64_t nod
 VideoEncoder::VideoEncoder(const std::shared_ptr<PipelineImpl>& par, int64_t nodeId, std::unique_ptr<Properties> props)
     : NodeCRTP<Node, VideoEncoder, VideoEncoderProperties>(par, nodeId, std::move(props)) {
     setInputRefs({&input});
-    setOutputRefs({&bitstream});
+    setOutputRefs({&bitstream, &out});
 }
 // node properties
 void VideoEncoder::setNumFramesPool(int frames) {