From 440d7b16687c92946292b066de550de845bc2ebb Mon Sep 17 00:00:00 2001 From: asahtik Date: Thu, 24 Aug 2023 13:51:23 +0200 Subject: [PATCH 01/16] Added host side EncodedFrame with parsing and output handling --- .../pipeline/datatype/EncodedFrame.hpp | 157 ++++++++++++++++++ .../depthai/pipeline/node/VideoEncoder.hpp | 5 + src/pipeline/datatype/EncodedFrame.cpp | 103 ++++++++++++ src/pipeline/datatype/StreamMessageParser.cpp | 10 ++ src/pipeline/node/VideoEncoder.cpp | 2 +- 5 files changed, 276 insertions(+), 1 deletion(-) create mode 100644 include/depthai/pipeline/datatype/EncodedFrame.hpp create mode 100644 src/pipeline/datatype/EncodedFrame.cpp diff --git a/include/depthai/pipeline/datatype/EncodedFrame.hpp b/include/depthai/pipeline/datatype/EncodedFrame.hpp new file mode 100644 index 000000000..49f3b2440 --- /dev/null +++ b/include/depthai/pipeline/datatype/EncodedFrame.hpp @@ -0,0 +1,157 @@ +#pragma once + +#include + +#include "depthai/pipeline/datatype/Buffer.hpp" + +// shared +#include "depthai-shared/datatype/RawEncodedFrame.hpp" + +// optional +#ifdef DEPTHAI_HAVE_OPENCV_SUPPORT + #include +#endif + +namespace dai { + +class EncodedFrame : public Buffer { + std::shared_ptr serialize() const override; + RawEncodedFrame& frame; + + public: + // Raw* mirror + using Profile = RawEncodedFrame::Profile; + using FrameType = RawEncodedFrame::FrameType; + + /** + * Construct EncodedFrame message. + * Timestamp is set to now + */ + EncodedFrame(); + explicit EncodedFrame(std::shared_ptr ptr); + virtual ~EncodedFrame() = default; + + // getters + /** + * Retrieves image timestamp related to dai::Clock::now() + */ + std::chrono::time_point getTimestamp() const; + + /** + * Retrieves image timestamp directly captured from device's monotonic clock, + * not synchronized to host time. Used when monotonicity is required. + */ + std::chrono::time_point getTimestampDevice() const; + + /** + * Retrieves image sequence number + */ + int64_t getSequenceNum() const; + + /** + * Retrieves image width in pixels + */ + unsigned int getWidth() const; + + /** + * Retrieves image height in pixels + */ + unsigned int getHeight() const; + + /** + * Retrieves the encoding quality + */ + unsigned int getQuality() const; + + /** + * Returns true if encoding is lossless (JPEG only) + */ + bool getLossless() const; + + /** + * Retrieves frame type (H26x only) + */ + FrameType getFrameType() const; + + /** + * Retrieves the encoding profile (JPEG, AVC or HEVC) + */ + Profile getProfile() const; + + // setters + /** + * Retrieves image timestamp related to dai::Clock::now() + */ + EncodedFrame& setTimestamp(std::chrono::time_point tp); + + /** + * Sets image timestamp related to dai::Clock::now() + */ + EncodedFrame& setTimestampDevice(std::chrono::time_point tp); + + /** + * Specifies sequence number + * + * @param seq Sequence number + */ + EncodedFrame& setSequenceNum(int64_t seq); + + /** + * Specifies frame width + * + * @param width frame width + */ + EncodedFrame& setWidth(unsigned int width); + + /** + * Specifies frame height + * + * @param height frame height + */ + EncodedFrame& setHeight(unsigned int height); + + /** + * Specifies frame size + * + * @param height frame height + * @param width frame width + */ + EncodedFrame& setSize(unsigned int width, unsigned int height); + + /** + * Specifies frame size + * + * @param size frame size + */ + EncodedFrame& setSize(std::tuple size); + + /** + * Specifies the encoding quality + * + * @param quality Encoding quality + */ + EncodedFrame& setQuality(unsigned int quality); + + /** + * Specifies if encoding is lossless (JPEG only) + * + * @param lossless True if lossless + */ + EncodedFrame& setLossless(bool lossless); + + /** + * Specifies the frame type (H26x only) + * + * @param type Type of h26x frame (I, P, B) + */ + EncodedFrame& setFrameType(FrameType type); + + /** + * Specifies the encoding profile + * + * @param profile Encoding profile + */ + EncodedFrame& setProfile(Profile profile); +}; + +} \ No newline at end of file diff --git a/include/depthai/pipeline/node/VideoEncoder.hpp b/include/depthai/pipeline/node/VideoEncoder.hpp index caa21ca82..7c0e55394 100644 --- a/include/depthai/pipeline/node/VideoEncoder.hpp +++ b/include/depthai/pipeline/node/VideoEncoder.hpp @@ -29,6 +29,11 @@ class VideoEncoder : public NodeCRTP */ Output bitstream{*this, "bitstream", Output::Type::MSender, {{DatatypeEnum::ImgFrame, false}}}; + /** + * Outputs EncodedFrame message that carries encoded (MJPEG, H264 or H265) frame data. + */ + Output out{*this, "out", Output::Type::MSender, {{DatatypeEnum::EncodedFrame, false}}}; + // Sets default options for a specified size and profile /** * Sets a default preset based on specified frame rate and profile diff --git a/src/pipeline/datatype/EncodedFrame.cpp b/src/pipeline/datatype/EncodedFrame.cpp new file mode 100644 index 000000000..260a4c791 --- /dev/null +++ b/src/pipeline/datatype/EncodedFrame.cpp @@ -0,0 +1,103 @@ +#include "depthai/pipeline/datatype/EncodedFrame.hpp" + +namespace dai { + +std::shared_ptr EncodedFrame::serialize() const { + return raw; +} + +EncodedFrame::EncodedFrame() : Buffer(std::make_shared()), frame(*dynamic_cast(raw.get())) { + // set timestamp to now + setTimestamp(std::chrono::steady_clock::now()); +} +EncodedFrame::EncodedFrame(std::shared_ptr ptr) : Buffer(std::move(ptr)), frame(*dynamic_cast(raw.get())) {} + +// Getters + +// getters +std::chrono::time_point EncodedFrame::getTimestamp() const { + using namespace std::chrono; + return time_point{seconds(frame.ts.sec) + nanoseconds(frame.ts.nsec)}; +} +std::chrono::time_point EncodedFrame::getTimestampDevice() const { + using namespace std::chrono; + return time_point{seconds(frame.tsDevice.sec) + nanoseconds(frame.tsDevice.nsec)}; +} +int64_t EncodedFrame::getSequenceNum() const { + return frame.sequenceNum; +} +unsigned int EncodedFrame::getWidth() const { + return frame.fb.width; +} +unsigned int EncodedFrame::getHeight() const { + return frame.fb.height; +} +unsigned int EncodedFrame::getQuality() const { + return frame.quality; +} +bool EncodedFrame::getLossless() const { + return frame.lossless; +} +EncodedFrame::FrameType EncodedFrame::getFrameType() const { + return frame.frameType; +} +EncodedFrame::Profile EncodedFrame::getProfile() const { + return frame.profile; +} + +// setters +EncodedFrame& EncodedFrame::setTimestamp(std::chrono::time_point tp) { + // Set timestamp from timepoint + using namespace std::chrono; + auto ts = tp.time_since_epoch(); + frame.ts.sec = duration_cast(ts).count(); + frame.ts.nsec = duration_cast(ts).count() % 1000000000; + return *this; +} +EncodedFrame& EncodedFrame::setTimestampDevice(std::chrono::time_point tp) { + // Set timestamp from timepoint + using namespace std::chrono; + auto ts = tp.time_since_epoch(); + frame.tsDevice.sec = duration_cast(ts).count(); + frame.tsDevice.nsec = duration_cast(ts).count() % 1000000000; + return *this; +} +EncodedFrame& EncodedFrame::setSequenceNum(int64_t sequenceNum) { + frame.sequenceNum = sequenceNum; + return *this; +} +EncodedFrame& EncodedFrame::setWidth(unsigned int width) { + frame.width = width; + return *this; +} +EncodedFrame& EncodedFrame::setHeight(unsigned int height) { + frame.height = height; + return *this; +} +EncodedFrame& EncodedFrame::setSize(unsigned int width, unsigned int height) { + setWidth(width); + setHeight(height); + return *this; +} +EncodedFrame& EncodedFrame::setSize(std::tuple size) { + setSize(std::get<0>(size), std::get<1>(size)); + return *this; +} +EncodedFrame& EncodedFrame::setQuality(unsigned int quality) { + frame.quality = quality; + return *this; +} +EncodedFrame& EncodedFrame::setLossless(bool lossless) { + frame.lossless = lossless; + return *this; +} +EncodedFrame& EncodedFrame::setFrameType(FrameType frameType) { + frame.frameType = frameType; + return *this; +} +EncodedFrame& EncodedFrame::setProfile(Profile profile) { + frame.profile = profile; + return *this; +} + +} \ No newline at end of file diff --git a/src/pipeline/datatype/StreamMessageParser.cpp b/src/pipeline/datatype/StreamMessageParser.cpp index a000b205d..e6b415874 100644 --- a/src/pipeline/datatype/StreamMessageParser.cpp +++ b/src/pipeline/datatype/StreamMessageParser.cpp @@ -20,6 +20,7 @@ #include "depthai/pipeline/datatype/ImageManipConfig.hpp" #include "depthai/pipeline/datatype/ImgDetections.hpp" #include "depthai/pipeline/datatype/ImgFrame.hpp" +#include "depthai/pipeline/datatype/EncodedFrame.hpp" #include "depthai/pipeline/datatype/NNData.hpp" #include "depthai/pipeline/datatype/SpatialImgDetections.hpp" #include "depthai/pipeline/datatype/SpatialLocationCalculatorConfig.hpp" @@ -42,6 +43,7 @@ #include "depthai-shared/datatype/RawImageManipConfig.hpp" #include "depthai-shared/datatype/RawImgDetections.hpp" #include "depthai-shared/datatype/RawImgFrame.hpp" +#include "depthai-shared/datatype/RawEncodedFrame.hpp" #include "depthai-shared/datatype/RawNNData.hpp" #include "depthai-shared/datatype/RawSpatialImgDetections.hpp" #include "depthai-shared/datatype/RawSpatialLocationCalculatorConfig.hpp" @@ -126,6 +128,10 @@ std::shared_ptr StreamMessageParser::parseMessage(streamPacketDesc_t* return parseDatatype(metadataStart, serializedObjectSize, data); break; + case DatatypeEnum::EncodedFrame: + return parseDatatype(metadataStart, serializedObjectSize, data); + break; + case DatatypeEnum::NNData: return parseDatatype(metadataStart, serializedObjectSize, data); break; @@ -220,6 +226,10 @@ std::shared_ptr StreamMessageParser::parseMessageToADatatype(streamPa return std::make_shared(parseDatatype(metadataStart, serializedObjectSize, data)); break; + case DatatypeEnum::EncodedFrame: + return std::make_shared(parseDatatype(metadataStart, serializedObjectSize, data)); + break; + case DatatypeEnum::NNData: return std::make_shared(parseDatatype(metadataStart, serializedObjectSize, data)); break; diff --git a/src/pipeline/node/VideoEncoder.cpp b/src/pipeline/node/VideoEncoder.cpp index da8efc018..0ff9afc7f 100644 --- a/src/pipeline/node/VideoEncoder.cpp +++ b/src/pipeline/node/VideoEncoder.cpp @@ -15,7 +15,7 @@ VideoEncoder::VideoEncoder(const std::shared_ptr& par, int64_t nod VideoEncoder::VideoEncoder(const std::shared_ptr& par, int64_t nodeId, std::unique_ptr props) : NodeCRTP(par, nodeId, std::move(props)) { setInputRefs({&input}); - setOutputRefs({&bitstream}); + setOutputRefs({&bitstream, &out}); } // node properties void VideoEncoder::setNumFramesPool(int frames) { From a63cf88b521c707f9a8b4850cc85843dcef1d792 Mon Sep 17 00:00:00 2001 From: asahtik Date: Fri, 25 Aug 2023 11:36:42 +0200 Subject: [PATCH 02/16] Bugfixes --- CMakeLists.txt | 1 + src/pipeline/datatype/EncodedFrame.cpp | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c32131618..80a46f7a6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -229,6 +229,7 @@ add_library(${TARGET_CORE_NAME} src/pipeline/node/UVC.cpp src/pipeline/datatype/Buffer.cpp src/pipeline/datatype/ImgFrame.cpp + src/pipeline/datatype/EncodedFrame.cpp src/pipeline/datatype/ImageManipConfig.cpp src/pipeline/datatype/CameraControl.cpp src/pipeline/datatype/NNData.cpp diff --git a/src/pipeline/datatype/EncodedFrame.cpp b/src/pipeline/datatype/EncodedFrame.cpp index 260a4c791..88ee37434 100644 --- a/src/pipeline/datatype/EncodedFrame.cpp +++ b/src/pipeline/datatype/EncodedFrame.cpp @@ -27,10 +27,10 @@ int64_t EncodedFrame::getSequenceNum() const { return frame.sequenceNum; } unsigned int EncodedFrame::getWidth() const { - return frame.fb.width; + return frame.width; } unsigned int EncodedFrame::getHeight() const { - return frame.fb.height; + return frame.height; } unsigned int EncodedFrame::getQuality() const { return frame.quality; @@ -39,7 +39,7 @@ bool EncodedFrame::getLossless() const { return frame.lossless; } EncodedFrame::FrameType EncodedFrame::getFrameType() const { - return frame.frameType; + return frame.type; } EncodedFrame::Profile EncodedFrame::getProfile() const { return frame.profile; @@ -92,7 +92,7 @@ EncodedFrame& EncodedFrame::setLossless(bool lossless) { return *this; } EncodedFrame& EncodedFrame::setFrameType(FrameType frameType) { - frame.frameType = frameType; + frame.type = frameType; return *this; } EncodedFrame& EncodedFrame::setProfile(Profile profile) { From 8fd2365243198147f5e4d062ff81b17538ff059e Mon Sep 17 00:00:00 2001 From: asahtik Date: Fri, 8 Sep 2023 13:09:29 +0200 Subject: [PATCH 03/16] Moved timestamps to buffer --- .../pipeline/datatype/EncodedFrame.hpp | 16 ------------ shared/depthai-shared | 2 +- src/pipeline/datatype/EncodedFrame.cpp | 26 +++---------------- 3 files changed, 4 insertions(+), 40 deletions(-) diff --git a/include/depthai/pipeline/datatype/EncodedFrame.hpp b/include/depthai/pipeline/datatype/EncodedFrame.hpp index 49f3b2440..322321c98 100644 --- a/include/depthai/pipeline/datatype/EncodedFrame.hpp +++ b/include/depthai/pipeline/datatype/EncodedFrame.hpp @@ -32,22 +32,6 @@ class EncodedFrame : public Buffer { virtual ~EncodedFrame() = default; // getters - /** - * Retrieves image timestamp related to dai::Clock::now() - */ - std::chrono::time_point getTimestamp() const; - - /** - * Retrieves image timestamp directly captured from device's monotonic clock, - * not synchronized to host time. Used when monotonicity is required. - */ - std::chrono::time_point getTimestampDevice() const; - - /** - * Retrieves image sequence number - */ - int64_t getSequenceNum() const; - /** * Retrieves image width in pixels */ diff --git a/shared/depthai-shared b/shared/depthai-shared index cb1bb4540..3170d119c 160000 --- a/shared/depthai-shared +++ b/shared/depthai-shared @@ -1 +1 @@ -Subproject commit cb1bb4540a6ffb4894b99155ba925e6eacdf0bc9 +Subproject commit 3170d119c2f6d453998fa3c291b47a9f36281eba diff --git a/src/pipeline/datatype/EncodedFrame.cpp b/src/pipeline/datatype/EncodedFrame.cpp index 88ee37434..839e525ef 100644 --- a/src/pipeline/datatype/EncodedFrame.cpp +++ b/src/pipeline/datatype/EncodedFrame.cpp @@ -15,17 +15,6 @@ EncodedFrame::EncodedFrame(std::shared_ptr ptr) : Buffer(std::m // Getters // getters -std::chrono::time_point EncodedFrame::getTimestamp() const { - using namespace std::chrono; - return time_point{seconds(frame.ts.sec) + nanoseconds(frame.ts.nsec)}; -} -std::chrono::time_point EncodedFrame::getTimestampDevice() const { - using namespace std::chrono; - return time_point{seconds(frame.tsDevice.sec) + nanoseconds(frame.tsDevice.nsec)}; -} -int64_t EncodedFrame::getSequenceNum() const { - return frame.sequenceNum; -} unsigned int EncodedFrame::getWidth() const { return frame.width; } @@ -48,23 +37,14 @@ EncodedFrame::Profile EncodedFrame::getProfile() const { // setters EncodedFrame& EncodedFrame::setTimestamp(std::chrono::time_point tp) { // Set timestamp from timepoint - using namespace std::chrono; - auto ts = tp.time_since_epoch(); - frame.ts.sec = duration_cast(ts).count(); - frame.ts.nsec = duration_cast(ts).count() % 1000000000; - return *this; + return static_cast(Buffer::setTimestamp(tp)); } EncodedFrame& EncodedFrame::setTimestampDevice(std::chrono::time_point tp) { // Set timestamp from timepoint - using namespace std::chrono; - auto ts = tp.time_since_epoch(); - frame.tsDevice.sec = duration_cast(ts).count(); - frame.tsDevice.nsec = duration_cast(ts).count() % 1000000000; - return *this; + return static_cast(Buffer::setTimestampDevice(tp)); } EncodedFrame& EncodedFrame::setSequenceNum(int64_t sequenceNum) { - frame.sequenceNum = sequenceNum; - return *this; + return static_cast(Buffer::setSequenceNum(sequenceNum)); } EncodedFrame& EncodedFrame::setWidth(unsigned int width) { frame.width = width; From 60c0b299c010edf27ae0ee88bc07d4e35517b02d Mon Sep 17 00:00:00 2001 From: asahtik Date: Tue, 19 Sep 2023 15:13:27 +0200 Subject: [PATCH 04/16] EncFrm fixes --- .../pipeline/datatype/EncodedFrame.hpp | 41 +------------------ shared/depthai-shared | 2 +- src/pipeline/datatype/EncodedFrame.cpp | 25 +---------- 3 files changed, 3 insertions(+), 65 deletions(-) diff --git a/include/depthai/pipeline/datatype/EncodedFrame.hpp b/include/depthai/pipeline/datatype/EncodedFrame.hpp index 322321c98..f1e441c64 100644 --- a/include/depthai/pipeline/datatype/EncodedFrame.hpp +++ b/include/depthai/pipeline/datatype/EncodedFrame.hpp @@ -32,16 +32,6 @@ class EncodedFrame : public Buffer { virtual ~EncodedFrame() = default; // getters - /** - * Retrieves image width in pixels - */ - unsigned int getWidth() const; - - /** - * Retrieves image height in pixels - */ - unsigned int getHeight() const; - /** * Retrieves the encoding quality */ @@ -80,35 +70,6 @@ class EncodedFrame : public Buffer { */ EncodedFrame& setSequenceNum(int64_t seq); - /** - * Specifies frame width - * - * @param width frame width - */ - EncodedFrame& setWidth(unsigned int width); - - /** - * Specifies frame height - * - * @param height frame height - */ - EncodedFrame& setHeight(unsigned int height); - - /** - * Specifies frame size - * - * @param height frame height - * @param width frame width - */ - EncodedFrame& setSize(unsigned int width, unsigned int height); - - /** - * Specifies frame size - * - * @param size frame size - */ - EncodedFrame& setSize(std::tuple size); - /** * Specifies the encoding quality * @@ -138,4 +99,4 @@ class EncodedFrame : public Buffer { EncodedFrame& setProfile(Profile profile); }; -} \ No newline at end of file +} diff --git a/shared/depthai-shared b/shared/depthai-shared index 06119fe9a..b889cd42b 160000 --- a/shared/depthai-shared +++ b/shared/depthai-shared @@ -1 +1 @@ -Subproject commit 06119fe9a7853b05f3dc640ba50b9b6ebaa90985 +Subproject commit b889cd42b1b06536ec42ca95cdf066ac1d155693 diff --git a/src/pipeline/datatype/EncodedFrame.cpp b/src/pipeline/datatype/EncodedFrame.cpp index 839e525ef..fc8ec6043 100644 --- a/src/pipeline/datatype/EncodedFrame.cpp +++ b/src/pipeline/datatype/EncodedFrame.cpp @@ -15,12 +15,6 @@ EncodedFrame::EncodedFrame(std::shared_ptr ptr) : Buffer(std::m // Getters // getters -unsigned int EncodedFrame::getWidth() const { - return frame.width; -} -unsigned int EncodedFrame::getHeight() const { - return frame.height; -} unsigned int EncodedFrame::getQuality() const { return frame.quality; } @@ -46,23 +40,6 @@ EncodedFrame& EncodedFrame::setTimestampDevice(std::chrono::time_point(Buffer::setSequenceNum(sequenceNum)); } -EncodedFrame& EncodedFrame::setWidth(unsigned int width) { - frame.width = width; - return *this; -} -EncodedFrame& EncodedFrame::setHeight(unsigned int height) { - frame.height = height; - return *this; -} -EncodedFrame& EncodedFrame::setSize(unsigned int width, unsigned int height) { - setWidth(width); - setHeight(height); - return *this; -} -EncodedFrame& EncodedFrame::setSize(std::tuple size) { - setSize(std::get<0>(size), std::get<1>(size)); - return *this; -} EncodedFrame& EncodedFrame::setQuality(unsigned int quality) { frame.quality = quality; return *this; @@ -80,4 +57,4 @@ EncodedFrame& EncodedFrame::setProfile(Profile profile) { return *this; } -} \ No newline at end of file +} From fd35a887a940adde624af7114f2887f78e7010a1 Mon Sep 17 00:00:00 2001 From: asahtik Date: Wed, 20 Sep 2023 12:35:57 +0200 Subject: [PATCH 05/16] Added tests, bugfixes --- .../pipeline/datatype/EncodedFrame.hpp | 14 ++ shared/depthai-shared | 2 +- src/pipeline/datatype/EncodedFrame.cpp | 8 ++ tests/CMakeLists.txt | 2 + tests/src/encoded_frame_test.cpp | 124 ++++++++++++++++++ 5 files changed, 149 insertions(+), 1 deletion(-) create mode 100644 tests/src/encoded_frame_test.cpp diff --git a/include/depthai/pipeline/datatype/EncodedFrame.hpp b/include/depthai/pipeline/datatype/EncodedFrame.hpp index f1e441c64..d43c467da 100644 --- a/include/depthai/pipeline/datatype/EncodedFrame.hpp +++ b/include/depthai/pipeline/datatype/EncodedFrame.hpp @@ -36,6 +36,12 @@ class EncodedFrame : public Buffer { * Retrieves the encoding quality */ unsigned int getQuality() const; + + /** + * Retrieves the encoding bitrate + */ + unsigned int getBitrate() const; + /** * Returns true if encoding is lossless (JPEG only) @@ -77,6 +83,14 @@ class EncodedFrame : public Buffer { */ EncodedFrame& setQuality(unsigned int quality); + /** + * Specifies the encoding quality + * + * @param quality Encoding quality + */ + EncodedFrame& setBitrate(unsigned int bitrate); + + /** * Specifies if encoding is lossless (JPEG only) * diff --git a/shared/depthai-shared b/shared/depthai-shared index b889cd42b..d48fa64b9 160000 --- a/shared/depthai-shared +++ b/shared/depthai-shared @@ -1 +1 @@ -Subproject commit b889cd42b1b06536ec42ca95cdf066ac1d155693 +Subproject commit d48fa64b9bae98a524c426b3b095e1c603189af0 diff --git a/src/pipeline/datatype/EncodedFrame.cpp b/src/pipeline/datatype/EncodedFrame.cpp index fc8ec6043..54aa55724 100644 --- a/src/pipeline/datatype/EncodedFrame.cpp +++ b/src/pipeline/datatype/EncodedFrame.cpp @@ -18,6 +18,9 @@ EncodedFrame::EncodedFrame(std::shared_ptr ptr) : Buffer(std::m unsigned int EncodedFrame::getQuality() const { return frame.quality; } +unsigned int EncodedFrame::getBitrate() const { + return frame.bitrate; +} bool EncodedFrame::getLossless() const { return frame.lossless; } @@ -44,6 +47,11 @@ EncodedFrame& EncodedFrame::setQuality(unsigned int quality) { frame.quality = quality; return *this; } +EncodedFrame& EncodedFrame::setBitrate(unsigned int bitrate) { + frame.bitrate = bitrate; + return *this; +} + EncodedFrame& EncodedFrame::setLossless(bool lossless) { frame.lossless = lossless; return *this; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index fc8bc81da..4920da7e4 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -222,6 +222,8 @@ dai_add_test(device_usbspeed_test src/device_usbspeed_test.cpp CONFORMING) dai_add_test(device_usbspeed_test_17 src/device_usbspeed_test.cpp CONFORMING CXX_STANDARD 17) dai_add_test(device_usbspeed_test_20 src/device_usbspeed_test.cpp CONFORMING CXX_STANDARD 20) +dai_add_test(encoded_frame_test src/encoded_frame_test.cpp CXX_STANDARD 17) + # Unlimited io connections test dai_add_test(unlimited_io_connection_test src/unlimited_io_connection_test.cpp) diff --git a/tests/src/encoded_frame_test.cpp b/tests/src/encoded_frame_test.cpp new file mode 100644 index 000000000..10e38ec32 --- /dev/null +++ b/tests/src/encoded_frame_test.cpp @@ -0,0 +1,124 @@ +#include +#include +#include +#include "depthai-shared/datatype/RawEncodedFrame.hpp" +#include "depthai-shared/properties/VideoEncoderProperties.hpp" +#include "depthai/device/Device.hpp" +#include "depthai/pipeline/Pipeline.hpp" +#include "depthai/pipeline/datatype/EncodedFrame.hpp" +#include "depthai/pipeline/datatype/ImgFrame.hpp" +#include "depthai/pipeline/node/ColorCamera.hpp" +#include "depthai/pipeline/node/VideoEncoder.hpp" +#include "depthai/pipeline/node/XLinkOut.hpp" + +dai::Pipeline getPipeline(dai::VideoEncoderProperties::Profile profile, unsigned int quality, bool lossless, unsigned int bitrate) { + dai::Pipeline pipeline; + auto camNode = pipeline.create(); + auto encNode = pipeline.create(); + auto xlinkOut = pipeline.create(); + camNode->video.link(encNode->input); + encNode->out.link(xlinkOut->input); + + camNode->setVideoSize(1280, 720); + encNode->setProfile(profile); + encNode->setBitrate(bitrate); + encNode->setQuality(quality); + encNode->setLossless(lossless); + encNode->setKeyframeFrequency(30); + xlinkOut->setStreamName("out"); + + return pipeline; +} + +TEST_CASE("OLD_OUTPUT") { + dai::Pipeline pipeline; + auto camNode = pipeline.create(); + auto encNode = pipeline.create(); + auto xlinkOut = pipeline.create(); + camNode->video.link(encNode->input); + encNode->bitstream.link(xlinkOut->input); + + camNode->setVideoSize(1280, 720); + encNode->setProfile(dai::VideoEncoderProperties::Profile::H264_MAIN); + xlinkOut->setStreamName("out"); + + dai::Device device(pipeline); + + auto outQ = device.getOutputQueue("out"); + for (int i = 0; i < 100; ++i) { + REQUIRE_NOTHROW(outQ->get()); + } +} + + +TEST_CASE("JPEG_ENCODING_LOSSLESS") { + dai::Device device(getPipeline(dai::VideoEncoderProperties::Profile::MJPEG, 30, true, 0)); + + auto outQ = device.getOutputQueue("out"); + for (int i = 0; i < 100; ++i) { + auto encfrm = outQ->get(); + REQUIRE(encfrm->getProfile() == dai::EncodedFrame::Profile::JPEG); + REQUIRE(encfrm->getLossless() == true); + REQUIRE(encfrm->getQuality() == 30); + } +} + +TEST_CASE("JPEG_ENCODING_LOSSY") { + dai::Device device(getPipeline(dai::VideoEncoderProperties::Profile::MJPEG, 30, false, 0)); + + auto outQ = device.getOutputQueue("out"); + for (int i = 0; i < 100; ++i) { + auto encfrm = outQ->get(); + REQUIRE(encfrm->getProfile() == dai::EncodedFrame::Profile::JPEG); + REQUIRE(encfrm->getLossless() == false); + REQUIRE(encfrm->getQuality() == 30); + } +} + +TEST_CASE("AVC_ENCODING") { + dai::Device device(getPipeline(dai::VideoEncoderProperties::Profile::H264_HIGH, 30, false, 8500000)); + + auto outQ = device.getOutputQueue("out"); + for (int i = 0; i < 100; ++i) { + auto encfrm = outQ->get(); + REQUIRE(encfrm->getProfile() == dai::EncodedFrame::Profile::AVC); + REQUIRE(encfrm->getLossless() == false); + if (i % 30 == 0) + REQUIRE(encfrm->getFrameType() == dai::EncodedFrame::FrameType::I); + REQUIRE(encfrm->getQuality() == 30); + REQUIRE(encfrm->getBitrate() == 8500000); + } +} + +TEST_CASE("HEVC_ENCODING") { + dai::Device device(getPipeline(dai::VideoEncoderProperties::Profile::H265_MAIN, 30, false, 8500000)); + + auto outQ = device.getOutputQueue("out"); + for (int i = 0; i < 100; ++i) { + auto encfrm = outQ->get(); + REQUIRE(encfrm->getProfile() == dai::EncodedFrame::Profile::HEVC); + REQUIRE(encfrm->getLossless() == false); + if (i % 30 == 0) + REQUIRE(encfrm->getFrameType() == dai::EncodedFrame::FrameType::I); + REQUIRE(encfrm->getQuality() == 30); + REQUIRE(encfrm->getBitrate() == 8500000); + } +} + +TEST_CASE("LINK_TO_BOTH") { + dai::Pipeline pipeline; + auto camNode = pipeline.create(); + auto encNode = pipeline.create(); + auto xlinkOut1 = pipeline.create(); + auto xlinkOut2 = pipeline.create(); + camNode->video.link(encNode->input); + encNode->bitstream.link(xlinkOut1->input); + encNode->out.link(xlinkOut2->input); + + camNode->setVideoSize(1280, 720); + encNode->setProfile(dai::VideoEncoderProperties::Profile::H264_MAIN); + xlinkOut1->setStreamName("out1"); + xlinkOut2->setStreamName("out2"); + + REQUIRE_THROWS(dai::Device(pipeline)); +} From acb82a5f9ab2717f4e262e72d2907c3a66627db9 Mon Sep 17 00:00:00 2001 From: asahtik Date: Thu, 21 Sep 2023 08:17:56 +0200 Subject: [PATCH 06/16] clangformat --- .../pipeline/datatype/EncodedFrame.hpp | 12 ++++---- shared/depthai-shared | 2 +- src/openvino/BlobReader.cpp | 2 +- src/pipeline/datatype/EncodedFrame.cpp | 2 +- src/pipeline/datatype/StreamMessageParser.cpp | 4 +-- src/utility/Path.cpp | 4 +-- tests/src/encoded_frame_test.cpp | 28 +++++++++---------- 7 files changed, 25 insertions(+), 29 deletions(-) diff --git a/include/depthai/pipeline/datatype/EncodedFrame.hpp b/include/depthai/pipeline/datatype/EncodedFrame.hpp index d43c467da..a3e52fb46 100644 --- a/include/depthai/pipeline/datatype/EncodedFrame.hpp +++ b/include/depthai/pipeline/datatype/EncodedFrame.hpp @@ -18,10 +18,10 @@ class EncodedFrame : public Buffer { std::shared_ptr serialize() const override; RawEncodedFrame& frame; - public: + public: // Raw* mirror using Profile = RawEncodedFrame::Profile; - using FrameType = RawEncodedFrame::FrameType; + using FrameType = RawEncodedFrame::FrameType; /** * Construct EncodedFrame message. @@ -36,16 +36,15 @@ class EncodedFrame : public Buffer { * Retrieves the encoding quality */ unsigned int getQuality() const; - + /** * Retrieves the encoding bitrate */ unsigned int getBitrate() const; - /** * Returns true if encoding is lossless (JPEG only) - */ + */ bool getLossless() const; /** @@ -90,7 +89,6 @@ class EncodedFrame : public Buffer { */ EncodedFrame& setBitrate(unsigned int bitrate); - /** * Specifies if encoding is lossless (JPEG only) * @@ -113,4 +111,4 @@ class EncodedFrame : public Buffer { EncodedFrame& setProfile(Profile profile); }; -} +} // namespace dai diff --git a/shared/depthai-shared b/shared/depthai-shared index d48fa64b9..458029432 160000 --- a/shared/depthai-shared +++ b/shared/depthai-shared @@ -1 +1 @@ -Subproject commit d48fa64b9bae98a524c426b3b095e1c603189af0 +Subproject commit 4580294329177acc8d324a0e4926fd54341376c1 diff --git a/src/openvino/BlobReader.cpp b/src/openvino/BlobReader.cpp index de6010002..148ce8c7e 100644 --- a/src/openvino/BlobReader.cpp +++ b/src/openvino/BlobReader.cpp @@ -18,7 +18,7 @@ #include "BlobFormat.hpp" // TODO(themarpe) -//#include +// #include namespace dai { diff --git a/src/pipeline/datatype/EncodedFrame.cpp b/src/pipeline/datatype/EncodedFrame.cpp index 54aa55724..e46535bd8 100644 --- a/src/pipeline/datatype/EncodedFrame.cpp +++ b/src/pipeline/datatype/EncodedFrame.cpp @@ -65,4 +65,4 @@ EncodedFrame& EncodedFrame::setProfile(Profile profile) { return *this; } -} +} // namespace dai diff --git a/src/pipeline/datatype/StreamMessageParser.cpp b/src/pipeline/datatype/StreamMessageParser.cpp index 7e955bb77..e40b3b9a1 100644 --- a/src/pipeline/datatype/StreamMessageParser.cpp +++ b/src/pipeline/datatype/StreamMessageParser.cpp @@ -15,12 +15,12 @@ #include "depthai/pipeline/datatype/Buffer.hpp" #include "depthai/pipeline/datatype/CameraControl.hpp" #include "depthai/pipeline/datatype/EdgeDetectorConfig.hpp" +#include "depthai/pipeline/datatype/EncodedFrame.hpp" #include "depthai/pipeline/datatype/FeatureTrackerConfig.hpp" #include "depthai/pipeline/datatype/IMUData.hpp" #include "depthai/pipeline/datatype/ImageManipConfig.hpp" #include "depthai/pipeline/datatype/ImgDetections.hpp" #include "depthai/pipeline/datatype/ImgFrame.hpp" -#include "depthai/pipeline/datatype/EncodedFrame.hpp" #include "depthai/pipeline/datatype/NNData.hpp" #include "depthai/pipeline/datatype/SpatialImgDetections.hpp" #include "depthai/pipeline/datatype/SpatialLocationCalculatorConfig.hpp" @@ -38,12 +38,12 @@ #include "depthai-shared/datatype/RawBuffer.hpp" #include "depthai-shared/datatype/RawCameraControl.hpp" #include "depthai-shared/datatype/RawEdgeDetectorConfig.hpp" +#include "depthai-shared/datatype/RawEncodedFrame.hpp" #include "depthai-shared/datatype/RawFeatureTrackerConfig.hpp" #include "depthai-shared/datatype/RawIMUData.hpp" #include "depthai-shared/datatype/RawImageManipConfig.hpp" #include "depthai-shared/datatype/RawImgDetections.hpp" #include "depthai-shared/datatype/RawImgFrame.hpp" -#include "depthai-shared/datatype/RawEncodedFrame.hpp" #include "depthai-shared/datatype/RawNNData.hpp" #include "depthai-shared/datatype/RawSpatialImgDetections.hpp" #include "depthai-shared/datatype/RawSpatialLocationCalculatorConfig.hpp" diff --git a/src/utility/Path.cpp b/src/utility/Path.cpp index f7b625530..fe0b970e8 100644 --- a/src/utility/Path.cpp +++ b/src/utility/Path.cpp @@ -10,13 +10,13 @@ namespace dai { std::wstring Path::convert_utf8_to_wide(const std::string& utf8string) { - //#pragma warning(suppress : 4996) + // #pragma warning(suppress : 4996) std::wstring_convert> converter; return converter.from_bytes(utf8string); } std::string Path::u8string() const { - //#pragma warning(suppress : 4996) + // #pragma warning(suppress : 4996) std::wstring_convert> converter; return converter.to_bytes(_nativePath); } diff --git a/tests/src/encoded_frame_test.cpp b/tests/src/encoded_frame_test.cpp index 10e38ec32..8c68bd79f 100644 --- a/tests/src/encoded_frame_test.cpp +++ b/tests/src/encoded_frame_test.cpp @@ -1,6 +1,7 @@ #include #include #include + #include "depthai-shared/datatype/RawEncodedFrame.hpp" #include "depthai-shared/properties/VideoEncoderProperties.hpp" #include "depthai/device/Device.hpp" @@ -43,19 +44,18 @@ TEST_CASE("OLD_OUTPUT") { xlinkOut->setStreamName("out"); dai::Device device(pipeline); - + auto outQ = device.getOutputQueue("out"); - for (int i = 0; i < 100; ++i) { + for(int i = 0; i < 100; ++i) { REQUIRE_NOTHROW(outQ->get()); } } - TEST_CASE("JPEG_ENCODING_LOSSLESS") { dai::Device device(getPipeline(dai::VideoEncoderProperties::Profile::MJPEG, 30, true, 0)); - + auto outQ = device.getOutputQueue("out"); - for (int i = 0; i < 100; ++i) { + for(int i = 0; i < 100; ++i) { auto encfrm = outQ->get(); REQUIRE(encfrm->getProfile() == dai::EncodedFrame::Profile::JPEG); REQUIRE(encfrm->getLossless() == true); @@ -65,9 +65,9 @@ TEST_CASE("JPEG_ENCODING_LOSSLESS") { TEST_CASE("JPEG_ENCODING_LOSSY") { dai::Device device(getPipeline(dai::VideoEncoderProperties::Profile::MJPEG, 30, false, 0)); - + auto outQ = device.getOutputQueue("out"); - for (int i = 0; i < 100; ++i) { + for(int i = 0; i < 100; ++i) { auto encfrm = outQ->get(); REQUIRE(encfrm->getProfile() == dai::EncodedFrame::Profile::JPEG); REQUIRE(encfrm->getLossless() == false); @@ -77,14 +77,13 @@ TEST_CASE("JPEG_ENCODING_LOSSY") { TEST_CASE("AVC_ENCODING") { dai::Device device(getPipeline(dai::VideoEncoderProperties::Profile::H264_HIGH, 30, false, 8500000)); - + auto outQ = device.getOutputQueue("out"); - for (int i = 0; i < 100; ++i) { + for(int i = 0; i < 100; ++i) { auto encfrm = outQ->get(); REQUIRE(encfrm->getProfile() == dai::EncodedFrame::Profile::AVC); REQUIRE(encfrm->getLossless() == false); - if (i % 30 == 0) - REQUIRE(encfrm->getFrameType() == dai::EncodedFrame::FrameType::I); + if(i % 30 == 0) REQUIRE(encfrm->getFrameType() == dai::EncodedFrame::FrameType::I); REQUIRE(encfrm->getQuality() == 30); REQUIRE(encfrm->getBitrate() == 8500000); } @@ -92,14 +91,13 @@ TEST_CASE("AVC_ENCODING") { TEST_CASE("HEVC_ENCODING") { dai::Device device(getPipeline(dai::VideoEncoderProperties::Profile::H265_MAIN, 30, false, 8500000)); - + auto outQ = device.getOutputQueue("out"); - for (int i = 0; i < 100; ++i) { + for(int i = 0; i < 100; ++i) { auto encfrm = outQ->get(); REQUIRE(encfrm->getProfile() == dai::EncodedFrame::Profile::HEVC); REQUIRE(encfrm->getLossless() == false); - if (i % 30 == 0) - REQUIRE(encfrm->getFrameType() == dai::EncodedFrame::FrameType::I); + if(i % 30 == 0) REQUIRE(encfrm->getFrameType() == dai::EncodedFrame::FrameType::I); REQUIRE(encfrm->getQuality() == 30); REQUIRE(encfrm->getBitrate() == 8500000); } From f4e89f561170d48573d1834b3484377aecd1cf41 Mon Sep 17 00:00:00 2001 From: asahtik Date: Thu, 21 Sep 2023 08:19:38 +0200 Subject: [PATCH 07/16] Bump device --- cmake/Depthai/DepthaiDeviceSideConfig.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/Depthai/DepthaiDeviceSideConfig.cmake b/cmake/Depthai/DepthaiDeviceSideConfig.cmake index b6efc0b2d..6b3b8e3ee 100644 --- a/cmake/Depthai/DepthaiDeviceSideConfig.cmake +++ b/cmake/Depthai/DepthaiDeviceSideConfig.cmake @@ -2,7 +2,7 @@ set(DEPTHAI_DEVICE_SIDE_MATURITY "snapshot") # "full commit hash of device side binary" -set(DEPTHAI_DEVICE_SIDE_COMMIT "dc4563dc83f22b31325b5cb06b20188eb10601b0") +set(DEPTHAI_DEVICE_SIDE_COMMIT "57bdf2cd2a064855592ad238fa87b05cc0f66d2f") # "version if applicable" set(DEPTHAI_DEVICE_SIDE_VERSION "") From 4a9ee8d0b72fcc422b08dc00e876b49cfb851d62 Mon Sep 17 00:00:00 2001 From: asahtik Date: Thu, 21 Sep 2023 08:29:47 +0200 Subject: [PATCH 08/16] Bump device again --- cmake/Depthai/DepthaiDeviceSideConfig.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/Depthai/DepthaiDeviceSideConfig.cmake b/cmake/Depthai/DepthaiDeviceSideConfig.cmake index 6b3b8e3ee..d8f01daa5 100644 --- a/cmake/Depthai/DepthaiDeviceSideConfig.cmake +++ b/cmake/Depthai/DepthaiDeviceSideConfig.cmake @@ -2,7 +2,7 @@ set(DEPTHAI_DEVICE_SIDE_MATURITY "snapshot") # "full commit hash of device side binary" -set(DEPTHAI_DEVICE_SIDE_COMMIT "57bdf2cd2a064855592ad238fa87b05cc0f66d2f") +set(DEPTHAI_DEVICE_SIDE_COMMIT "bef7c80f56dc2401254908c44902565c6d37f603") # "version if applicable" set(DEPTHAI_DEVICE_SIDE_VERSION "") From 4338f139be96e238a09f3c7fedbb7fe25e589812 Mon Sep 17 00:00:00 2001 From: asahtik Date: Tue, 10 Oct 2023 16:37:21 +0200 Subject: [PATCH 09/16] Added fields to EncodedFrame --- .../pipeline/datatype/EncodedFrame.hpp | 30 +++++++++++++++++++ shared/depthai-shared | 2 +- src/pipeline/datatype/EncodedFrame.cpp | 21 +++++++++++-- 3 files changed, 50 insertions(+), 3 deletions(-) diff --git a/include/depthai/pipeline/datatype/EncodedFrame.hpp b/include/depthai/pipeline/datatype/EncodedFrame.hpp index a3e52fb46..99db79ba0 100644 --- a/include/depthai/pipeline/datatype/EncodedFrame.hpp +++ b/include/depthai/pipeline/datatype/EncodedFrame.hpp @@ -32,6 +32,29 @@ class EncodedFrame : public Buffer { virtual ~EncodedFrame() = default; // getters + /** + * Retrieves instance number + */ + unsigned int getInstanceNum() const; + /** + * Retrieves exposure time + */ + std::chrono::microseconds getExposureTime() const; + + /** + * Retrieves sensitivity, as an ISO value + */ + int getSensitivity() const; + + /** + * Retrieves white-balance color temperature of the light source, in kelvins + */ + int getColorTemperature() const; + + /** + * Retrieves lens position, range 0..255. Returns -1 if not available + */ + int getLensPosition() const; /** * Retrieves the encoding quality */ @@ -75,6 +98,13 @@ class EncodedFrame : public Buffer { */ EncodedFrame& setSequenceNum(int64_t seq); + /** + * Instance number relates to the origin of the frame (which camera) + * + * @param instance Instance number + */ + EncodedFrame& setInstanceNum(unsigned int instance); + /** * Specifies the encoding quality * diff --git a/shared/depthai-shared b/shared/depthai-shared index ab1b4cd36..ddbeed9e9 160000 --- a/shared/depthai-shared +++ b/shared/depthai-shared @@ -1 +1 @@ -Subproject commit ab1b4cd362182036082eba790a74371db1d500fe +Subproject commit ddbeed9e9f4e25c741e7c46b2f50554e4af9cf57 diff --git a/src/pipeline/datatype/EncodedFrame.cpp b/src/pipeline/datatype/EncodedFrame.cpp index e46535bd8..276a85514 100644 --- a/src/pipeline/datatype/EncodedFrame.cpp +++ b/src/pipeline/datatype/EncodedFrame.cpp @@ -12,9 +12,22 @@ EncodedFrame::EncodedFrame() : Buffer(std::make_shared()), fram } EncodedFrame::EncodedFrame(std::shared_ptr ptr) : Buffer(std::move(ptr)), frame(*dynamic_cast(raw.get())) {} -// Getters - // getters +unsigned int EncodedFrame::getInstanceNum() const { + return frame.instanceNum; +} +std::chrono::microseconds EncodedFrame::getExposureTime() const { + return std::chrono::microseconds(frame.cam.exposureTimeUs); +} +int EncodedFrame::getSensitivity() const { + return frame.cam.sensitivityIso; +} +int EncodedFrame::getColorTemperature() const { + return frame.cam.wbColorTemp; +} +int EncodedFrame::getLensPosition() const { + return frame.cam.lensPosition; +} unsigned int EncodedFrame::getQuality() const { return frame.quality; } @@ -43,6 +56,10 @@ EncodedFrame& EncodedFrame::setTimestampDevice(std::chrono::time_point(Buffer::setSequenceNum(sequenceNum)); } +EncodedFrame& EncodedFrame::setInstanceNum(unsigned int instanceNum) { + frame.instanceNum = instanceNum; + return *this; +} EncodedFrame& EncodedFrame::setQuality(unsigned int quality) { frame.quality = quality; return *this; From 797638a0803bc846791c1b8446b1198ed086179b Mon Sep 17 00:00:00 2001 From: asahtik Date: Wed, 11 Oct 2023 11:02:52 +0200 Subject: [PATCH 10/16] Bump shared --- shared/depthai-shared | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/depthai-shared b/shared/depthai-shared index ddbeed9e9..b43db62b6 160000 --- a/shared/depthai-shared +++ b/shared/depthai-shared @@ -1 +1 @@ -Subproject commit ddbeed9e9f4e25c741e7c46b2f50554e4af9cf57 +Subproject commit b43db62b62599c3e6bc4e8011b7d1f788722adae From d519eedd73492a0ad7a828e8a18c4247d16b5716 Mon Sep 17 00:00:00 2001 From: asahtik Date: Sat, 28 Oct 2023 13:37:50 +0200 Subject: [PATCH 11/16] Moved frame type extraction into a getter --- CMakeLists.txt | 1 + shared/depthai-shared.cmake | 2 +- src/pipeline/datatype/EncodedFrame.cpp | 35 +++ src/utility/H26xParsers.cpp | 290 +++++++++++++++++++++++++ src/utility/H26xParsers.hpp | 14 ++ 5 files changed, 341 insertions(+), 1 deletion(-) create mode 100644 src/utility/H26xParsers.cpp create mode 100644 src/utility/H26xParsers.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index d055c44e6..e76b2725b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -248,6 +248,7 @@ add_library(${TARGET_CORE_NAME} src/pipeline/datatype/TrackedFeatures.cpp src/pipeline/datatype/FeatureTrackerConfig.cpp src/pipeline/datatype/ToFConfig.cpp + src/utility/H26xParsers.cpp src/utility/Initialization.cpp src/utility/Resources.cpp src/utility/Path.cpp diff --git a/shared/depthai-shared.cmake b/shared/depthai-shared.cmake index 72f46ce99..414e1ebf6 100644 --- a/shared/depthai-shared.cmake +++ b/shared/depthai-shared.cmake @@ -63,4 +63,4 @@ foreach(source_file ${DEPTHAI_SHARED_SOURCES}) if(NOT EXISTS ${source_file}) message(FATAL_ERROR "depthai-shared submodule files missing. Make sure to download prepackaged release instead of \"Source code\" on GitHub. Example: depthai-core-vX.Y.Z.tar.gz") endif() -endforeach() \ No newline at end of file +endforeach() diff --git a/src/pipeline/datatype/EncodedFrame.cpp b/src/pipeline/datatype/EncodedFrame.cpp index 276a85514..29bfd6bd5 100644 --- a/src/pipeline/datatype/EncodedFrame.cpp +++ b/src/pipeline/datatype/EncodedFrame.cpp @@ -1,4 +1,5 @@ #include "depthai/pipeline/datatype/EncodedFrame.hpp" +#include "utility/H26xParsers.hpp" namespace dai { @@ -38,6 +39,40 @@ bool EncodedFrame::getLossless() const { return frame.lossless; } EncodedFrame::FrameType EncodedFrame::getFrameType() const { + if (frame.type == FrameType::Unknown) { + SliceType frameType; + switch (frame.profile) { + case RawEncodedFrame::Profile::JPEG: + frameType = SliceType::I; + break; + case RawEncodedFrame::Profile::AVC: + frameType = getTypesH264(frame.data, true)[0]; + break; + case RawEncodedFrame::Profile::HEVC: + frameType = getTypesH265(frame.data, true)[0]; + break; + } + switch (frameType) { + case SliceType::P: + frame.type = FrameType::P; + break; + case SliceType::B: + frame.type = FrameType::B; + break; + case SliceType::I: + frame.type = FrameType::I; + break; + case SliceType::SP: + frame.type = FrameType::P; + break; + case SliceType::SI: + frame.type = FrameType::I; + break; + case SliceType::Unknown: + frame.type = FrameType::Unknown; + break; + } + } return frame.type; } EncodedFrame::Profile EncodedFrame::getProfile() const { diff --git a/src/utility/H26xParsers.cpp b/src/utility/H26xParsers.cpp new file mode 100644 index 000000000..2bbba63e8 --- /dev/null +++ b/src/utility/H26xParsers.cpp @@ -0,0 +1,290 @@ +#include "H26xParsers.hpp" + +#include +#include + +namespace dai { + +template +struct H26xParser { + protected: + virtual void parseNal(const std::vector& bs, unsigned int start, std::vector& out) = 0; + std::vector parseBytestream(const std::vector& bs, bool breakOnFirst); + + public: + static std::vector getTypes(const std::vector& bs, bool breakOnFirst); + virtual ~H26xParser() = default; +}; + +struct H264Parser : H26xParser { + void parseNal(const std::vector& bs, unsigned int start, std::vector& out); +}; + +struct H265Parser : H26xParser { + unsigned int nalUnitType = 0; // In NAL header + unsigned int dependentSliceSegmentsEnabledFlag = 0; // In picture parameter set + unsigned int numExtraSliceHeaderBits = 0; // In picture parameter set + // Getting the length of slice_segment address + unsigned int picWidthInLumaSamples = 0; // In sequence parameter set + unsigned int picHeightInLumaSamples = 0; // In sequence parameter set + unsigned int chromaFormatIdc = 0; // In sequence parameter set + unsigned int log2DiffMaxMinLumaCodingBlockSize = 0; // In sequence parameter set + unsigned int log2MinLumaCodingBlockSizeMinus3 = 0; // In sequence parameter set + + void parseNal(const std::vector& bs, unsigned int start, std::vector& out); +}; + +typedef unsigned int uint; +typedef unsigned long ulong; +typedef const std::vector buf; + +SliceType getSliceType(uint num, Profile p) { + switch(p) { + case Profile::H264: + switch(num) { + case 0: + case 5: + return SliceType::P; + case 1: + case 6: + return SliceType::B; + case 2: + case 7: + return SliceType::I; + case 3: + case 8: + return SliceType::SP; + case 4: + case 9: + return SliceType::SI; + default: + return SliceType::Unknown; + } + case Profile::H265: + switch(num) { + case 0: + return SliceType::B; + case 1: + return SliceType::P; + case 2: + return SliceType::I; + default: + return SliceType::Unknown; + } + default: + return SliceType::Unknown; + } +} + +bool scodeEq(buf& bs, uint pos, buf code) { + if(bs.size() - pos > code.size()) { + for(uint i = 0; i < code.size(); ++i) { + if(bs[pos + i] != code[i]) return false; + } + return true; + } else + return false; +} + +uint findStart(buf& bs, uint pos) { + buf codeLong = {0, 0, 0, 1}; + buf codeShort = {0, 0, 1}; + uint size = bs.size(); + for(uint i = pos; i < size; ++i) { + if(bs[i] == 0) { + if(scodeEq(bs, i, codeLong)) { + return i + 4; + } else if(scodeEq(bs, i, codeShort)) { + return i + 3; + } + } + } + return size; +} + +uint findEnd(buf& bs, uint pos) { + buf end1 = {0, 0, 0}; + buf end2 = {0, 0, 1}; + uint size = bs.size(); + for(uint i = pos; i < size; ++i) { + if(bs[i] == 0) { + if(scodeEq(bs, i, end1) || scodeEq(bs, i, end2)) { + return i; + } + } + } + return size; +} + +uint readUint(buf& bs, ulong start, ulong end) { + uint ret = 0; + for(ulong i = start; i < end; ++i) { + uint bit = (bs[(uint)(i / 8)] & (1 << (7 - i % 8))) > 0; + ret += bit * (1 << (end - i - 1)); + } + return ret; +} + +std::tuple readGE(buf& bs, ulong pos) { + uint count = 0; + ulong size = bs.size() * 8; + while(pos < size) { + std::uint8_t bit = bs[(uint)(pos / 8)] & (char)(1 << (7 - pos % 8)); + if(bit == 0) { + ++count; + ++pos; + } else + break; + } + ++count; + ulong start = pos; + ulong end = pos + count; + if(end > size) exit(30); + return std::make_tuple(readUint(bs, start, end) - 1, end); +} + +template +std::vector H26xParser::getTypes(buf& buffer, bool breakOnFirst) { + T p; + return p.parseBytestream(buffer, breakOnFirst); +} + +template +std::vector H26xParser::parseBytestream(buf& bs, bool breakOnFirst) { + uint pos = 0; + uint size = bs.size(); + std::vector ret; + while(pos < size) { + uint start = findStart(bs, pos); + uint end = findEnd(bs, start); + if(start >= end) break; + parseNal(bs, start, ret); + pos = end; + if(breakOnFirst && ret.size() > 0) break; + } + return ret; +} + +void H264Parser::parseNal(buf& bs, uint start, std::vector& out) { + uint pos = start; + uint nalUnitType = bs[pos++] & 31; + uint nalUnitHeaderBytes = 1; + if(nalUnitType == 14 || nalUnitType == 20 || nalUnitType == 21) { + uint avc3dExtensionFlag = 0; + if(nalUnitType == 21) avc3dExtensionFlag = readUint(bs, pos * 8, pos * 8 + 1); + if(avc3dExtensionFlag) + nalUnitHeaderBytes += 2; + else + nalUnitHeaderBytes += 3; + } + pos = start + nalUnitHeaderBytes; + if(nalUnitType == 1 || nalUnitType == 5) { + const auto bpos1 = std::get<1>(readGE(bs, pos * 8)); + uint tyNum; + ulong bpos2; + std::tie(tyNum, bpos2) = readGE(bs, bpos1); + pos = (uint)(bpos2 / 8) + (bpos2 % 8 > 0); + out.push_back(getSliceType(tyNum, Profile::H264)); + } +} + +void H265Parser::parseNal(buf& bs, uint start, std::vector& out) { + nalUnitType = (bs[start] & 126) >> 1; + uint pos = start + 2; + if(nalUnitType == 33) { + // Sequence parameter set + uint spsMaxSubLayersMinus1 = (bs[pos] & 14) >> 1; + ++pos; + const auto bpos1 = std::get<1>(readGE(bs, pos * 8)); // We don't need this value + uint chr; + ulong bpos2; + std::tie(chr, bpos2) = readGE(bs, bpos1); + chromaFormatIdc = chr; + if(chromaFormatIdc) ++bpos2; // We don't need this value + uint pw; + ulong bpos3; + std::tie(pw, bpos3) = readGE(bs, bpos2); + uint ph; + ulong bpos4; + std::tie(ph, bpos4) = readGE(bs, bpos3); + picWidthInLumaSamples = pw; + picHeightInLumaSamples = ph; + uint conformanceWindowFlag = readUint(bs, bpos4, bpos4 + 1); + ulong bpos8 = ++bpos4; + if(conformanceWindowFlag) { + // We don't care about any of these values + auto bpos5 = std::get<1>(readGE(bs, bpos4)); + auto bpos6 = std::get<1>(readGE(bs, bpos5)); + auto bpos7 = std::get<1>(readGE(bs, bpos6)); + auto tbpos8 = std::get<1>(readGE(bs, bpos7)); + bpos8 = tbpos8; + } + auto bpos9 = std::get<1>(readGE(bs, bpos8)); + auto bpos10 = std::get<1>(readGE(bs, bpos9)); + auto bpos11 = std::get<1>(readGE(bs, bpos10)); + uint spsSubLayerOrderingInfoPresentFlag = readUint(bs, bpos11, bpos11 + 1); + ulong bpos12 = ++bpos11; + for(uint i = spsSubLayerOrderingInfoPresentFlag ? 0 : spsMaxSubLayersMinus1; i <= spsMaxSubLayersMinus1; ++i) { + auto tbpos1 = std::get<1>(readGE(bs, bpos12)); + auto tbpos2 = std::get<1>(readGE(bs, tbpos1)); + auto tbpos3 = std::get<1>(readGE(bs, tbpos2)); + bpos12 = tbpos3; + } + uint lm, ldm; + ulong bpos13, bpos14; + std::tie(lm, bpos13) = readGE(bs, bpos12); + std::tie(ldm, bpos14) = readGE(bs, bpos13); + log2MinLumaCodingBlockSizeMinus3 = lm; + log2DiffMaxMinLumaCodingBlockSize = ldm; + pos = (uint)(bpos14 / 8) + (bpos14 % 8 > 0); // Update byte position with newest bit position + } else if(nalUnitType == 34) { + // Picture parameter set + auto bpos1 = std::get<1>(readGE(bs, pos * 8)); + auto bpos2 = std::get<1>(readGE(bs, bpos1)); + dependentSliceSegmentsEnabledFlag = readUint(bs, bpos2, bpos2 + 1); + bpos2 += 2; + numExtraSliceHeaderBits = readUint(bs, bpos2, bpos2 + 3); + bpos2 += 3; + pos = (uint)(bpos2 / 8) + (bpos2 % 8 > 0); + } else if((0 <= nalUnitType && nalUnitType <= 9) || (16 <= nalUnitType && nalUnitType <= 21)) { + // Coded slice segment + ulong bpos1 = pos * 8; + uint firstSliceSegmentInPicFlag = readUint(bs, bpos1, bpos1 + 1); + ++bpos1; + if(16 <= nalUnitType && nalUnitType <= 23) ++bpos1; + auto bpos2 = std::get<1>(readGE(bs, bpos1)); + uint dependentSliceSegmentFlag = 0; + if(!firstSliceSegmentInPicFlag) { + if(dependentSliceSegmentsEnabledFlag) { + dependentSliceSegmentFlag = readUint(bs, bpos2, bpos2 + 1); + ++bpos2; + } + uint ctbLog2SizeY = log2MinLumaCodingBlockSizeMinus3 + 3 + log2DiffMaxMinLumaCodingBlockSize; + uint ctbSizeY = 1 << ctbLog2SizeY; + uint picWidthInCtbsY = ceil(picWidthInLumaSamples / ctbSizeY); + uint picHeightInCtbsY = ceil(picHeightInLumaSamples / ctbSizeY); + uint picSizeInCtbsY = picWidthInCtbsY * picHeightInCtbsY; + uint vlen = ceil(log2(picSizeInCtbsY)); + bpos2 += vlen; + } + ulong bpos3 = bpos2; + if(!dependentSliceSegmentFlag) { + bpos2 += numExtraSliceHeaderBits; + uint tyNum; + ulong tbpos3; + std::tie(tyNum, tbpos3) = readGE(bs, bpos2); + bpos3 = tbpos3; + out.push_back(getSliceType(tyNum, Profile::H265)); + } + pos = (uint)(bpos3 / 8) + (bpos3 % 8 > 0); + } +} + +std::vector getTypesH264(buf& bs, bool breakOnFirst) { + return H264Parser::getTypes(bs, breakOnFirst); +} +std::vector getTypesH265(buf& bs, bool breakOnFirst) { + return H265Parser::getTypes(bs, breakOnFirst); +} + +} // namespace dai diff --git a/src/utility/H26xParsers.hpp b/src/utility/H26xParsers.hpp new file mode 100644 index 000000000..d2b90c1d9 --- /dev/null +++ b/src/utility/H26xParsers.hpp @@ -0,0 +1,14 @@ +#pragma once + +#include +#include + +namespace dai { + +enum class Profile { H264, H265 }; +enum class SliceType { P, B, I, SP, SI, Unknown }; + +std::vector getTypesH264(const std::vector& bs, bool breakOnFirst = false); +std::vector getTypesH265(const std::vector& bs, bool breakOnFirst = false); + +} // namespace dai From 79c1d67d5f58bb720ee783847fe4ec8430aa3663 Mon Sep 17 00:00:00 2001 From: asahtik Date: Fri, 3 Nov 2023 08:22:26 +0100 Subject: [PATCH 12/16] Bump shared --- shared/depthai-shared | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/depthai-shared b/shared/depthai-shared index 94bfa96c4..1148fba38 160000 --- a/shared/depthai-shared +++ b/shared/depthai-shared @@ -1 +1 @@ -Subproject commit 94bfa96c4e2f7673d1d934cfcd6cfee7fc6363d3 +Subproject commit 1148fba385d29d4ef652ff7edf03a007c78f1734 From af9de8c41bf1de44b319d13be691d34dca46405c Mon Sep 17 00:00:00 2001 From: asahtik Date: Fri, 3 Nov 2023 09:14:57 +0100 Subject: [PATCH 13/16] Bump device --- cmake/Depthai/DepthaiDeviceSideConfig.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/Depthai/DepthaiDeviceSideConfig.cmake b/cmake/Depthai/DepthaiDeviceSideConfig.cmake index 554164f21..142b4a997 100644 --- a/cmake/Depthai/DepthaiDeviceSideConfig.cmake +++ b/cmake/Depthai/DepthaiDeviceSideConfig.cmake @@ -2,7 +2,7 @@ set(DEPTHAI_DEVICE_SIDE_MATURITY "snapshot") # "full commit hash of device side binary" -set(DEPTHAI_DEVICE_SIDE_COMMIT "c9b6a2e0a4bdbf635914b80baa06efc2f76fa2bc") +set(DEPTHAI_DEVICE_SIDE_COMMIT "eb8642760be61cf175ca623e8a1f95b427109def") # "version if applicable" set(DEPTHAI_DEVICE_SIDE_VERSION "") From 06672b706362b9217e139a7ad730360d4eda351f Mon Sep 17 00:00:00 2001 From: asahtik Date: Fri, 3 Nov 2023 10:16:01 +0100 Subject: [PATCH 14/16] Bump device --- cmake/Depthai/DepthaiDeviceSideConfig.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/Depthai/DepthaiDeviceSideConfig.cmake b/cmake/Depthai/DepthaiDeviceSideConfig.cmake index 142b4a997..99876d274 100644 --- a/cmake/Depthai/DepthaiDeviceSideConfig.cmake +++ b/cmake/Depthai/DepthaiDeviceSideConfig.cmake @@ -2,7 +2,7 @@ set(DEPTHAI_DEVICE_SIDE_MATURITY "snapshot") # "full commit hash of device side binary" -set(DEPTHAI_DEVICE_SIDE_COMMIT "eb8642760be61cf175ca623e8a1f95b427109def") +set(DEPTHAI_DEVICE_SIDE_COMMIT "1cf15832ab1f408d8e4dab72e901ce050bf8850b") # "version if applicable" set(DEPTHAI_DEVICE_SIDE_VERSION "") From b910590ffddf758548d9269bd5393a3a55e8c769 Mon Sep 17 00:00:00 2001 From: asahtik Date: Fri, 3 Nov 2023 10:19:26 +0100 Subject: [PATCH 15/16] clangformat --- src/pipeline/datatype/EncodedFrame.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/pipeline/datatype/EncodedFrame.cpp b/src/pipeline/datatype/EncodedFrame.cpp index 29bfd6bd5..ab4cd27e7 100644 --- a/src/pipeline/datatype/EncodedFrame.cpp +++ b/src/pipeline/datatype/EncodedFrame.cpp @@ -1,4 +1,5 @@ #include "depthai/pipeline/datatype/EncodedFrame.hpp" + #include "utility/H26xParsers.hpp" namespace dai { @@ -39,9 +40,9 @@ bool EncodedFrame::getLossless() const { return frame.lossless; } EncodedFrame::FrameType EncodedFrame::getFrameType() const { - if (frame.type == FrameType::Unknown) { + if(frame.type == FrameType::Unknown) { SliceType frameType; - switch (frame.profile) { + switch(frame.profile) { case RawEncodedFrame::Profile::JPEG: frameType = SliceType::I; break; @@ -52,7 +53,7 @@ EncodedFrame::FrameType EncodedFrame::getFrameType() const { frameType = getTypesH265(frame.data, true)[0]; break; } - switch (frameType) { + switch(frameType) { case SliceType::P: frame.type = FrameType::P; break; From 3d85f9fcf00a74cb0584ff3a49cea061560c1110 Mon Sep 17 00:00:00 2001 From: asahtik Date: Tue, 7 Nov 2023 10:19:41 +0100 Subject: [PATCH 16/16] Applied suggestions --- .../depthai/pipeline/node/VideoEncoder.hpp | 4 ++-- src/pipeline/datatype/EncodedFrame.cpp | 20 +++++++++---------- src/utility/H26xParsers.cpp | 2 ++ src/utility/H26xParsers.hpp | 2 ++ 4 files changed, 16 insertions(+), 12 deletions(-) diff --git a/include/depthai/pipeline/node/VideoEncoder.hpp b/include/depthai/pipeline/node/VideoEncoder.hpp index 7c0e55394..59c186e52 100644 --- a/include/depthai/pipeline/node/VideoEncoder.hpp +++ b/include/depthai/pipeline/node/VideoEncoder.hpp @@ -25,12 +25,12 @@ class VideoEncoder : public NodeCRTP Input input{*this, "in", Input::Type::SReceiver, true, 4, true, {{DatatypeEnum::ImgFrame, true}}}; /** - * Outputs ImgFrame message that carries BITSTREAM encoded (MJPEG, H264 or H265) frame data. + * Outputs ImgFrame message that carries BITSTREAM encoded (MJPEG, H264 or H265) frame data. Mutually exclusive with out. */ Output bitstream{*this, "bitstream", Output::Type::MSender, {{DatatypeEnum::ImgFrame, false}}}; /** - * Outputs EncodedFrame message that carries encoded (MJPEG, H264 or H265) frame data. + * Outputs EncodedFrame message that carries encoded (MJPEG, H264 or H265) frame data. Mutually exclusive with bitstream. */ Output out{*this, "out", Output::Type::MSender, {{DatatypeEnum::EncodedFrame, false}}}; diff --git a/src/pipeline/datatype/EncodedFrame.cpp b/src/pipeline/datatype/EncodedFrame.cpp index ab4cd27e7..0473d7e63 100644 --- a/src/pipeline/datatype/EncodedFrame.cpp +++ b/src/pipeline/datatype/EncodedFrame.cpp @@ -41,35 +41,35 @@ bool EncodedFrame::getLossless() const { } EncodedFrame::FrameType EncodedFrame::getFrameType() const { if(frame.type == FrameType::Unknown) { - SliceType frameType; + utility::SliceType frameType; switch(frame.profile) { case RawEncodedFrame::Profile::JPEG: - frameType = SliceType::I; + frameType = utility::SliceType::I; break; case RawEncodedFrame::Profile::AVC: - frameType = getTypesH264(frame.data, true)[0]; + frameType = utility::getTypesH264(frame.data, true)[0]; break; case RawEncodedFrame::Profile::HEVC: - frameType = getTypesH265(frame.data, true)[0]; + frameType = utility::getTypesH265(frame.data, true)[0]; break; } switch(frameType) { - case SliceType::P: + case utility::SliceType::P: frame.type = FrameType::P; break; - case SliceType::B: + case utility::SliceType::B: frame.type = FrameType::B; break; - case SliceType::I: + case utility::SliceType::I: frame.type = FrameType::I; break; - case SliceType::SP: + case utility::SliceType::SP: frame.type = FrameType::P; break; - case SliceType::SI: + case utility::SliceType::SI: frame.type = FrameType::I; break; - case SliceType::Unknown: + case utility::SliceType::Unknown: frame.type = FrameType::Unknown; break; } diff --git a/src/utility/H26xParsers.cpp b/src/utility/H26xParsers.cpp index 2bbba63e8..62922d204 100644 --- a/src/utility/H26xParsers.cpp +++ b/src/utility/H26xParsers.cpp @@ -4,6 +4,7 @@ #include namespace dai { +namespace utility { template struct H26xParser { @@ -287,4 +288,5 @@ std::vector getTypesH265(buf& bs, bool breakOnFirst) { return H265Parser::getTypes(bs, breakOnFirst); } +} // namespace utility } // namespace dai diff --git a/src/utility/H26xParsers.hpp b/src/utility/H26xParsers.hpp index d2b90c1d9..a8b1e3b6c 100644 --- a/src/utility/H26xParsers.hpp +++ b/src/utility/H26xParsers.hpp @@ -4,6 +4,7 @@ #include namespace dai { +namespace utility { enum class Profile { H264, H265 }; enum class SliceType { P, B, I, SP, SI, Unknown }; @@ -11,4 +12,5 @@ enum class SliceType { P, B, I, SP, SI, Unknown }; std::vector getTypesH264(const std::vector& bs, bool breakOnFirst = false); std::vector getTypesH265(const std::vector& bs, bool breakOnFirst = false); +} // namespace utility } // namespace dai