forked from microsoft/onnxruntime
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Create Utils for Adding Range and Marker (microsoft#4013)
In this PR, we 1. create some APIs for creating NVTX objects 2. apply those APIs in pipeline-related operators and sequential executor. As a result, we can explicitly see how a pipeline schedule is run by GPUs in Nvidia's visual profiler. Note that these APIs are Linux only due to Nvidia's limited support.
- Loading branch information
Showing
14 changed files
with
461 additions
and
9 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,52 @@ | ||
| // Copyright (c) Microsoft Corporation. All rights reserved. | ||
| // Licensed under the MIT License. | ||
|
|
||
| #pragma once | ||
|
|
||
| #include <thread> | ||
| #include <mutex> | ||
| #include <string> | ||
| #include <unordered_map> | ||
|
|
||
| #ifdef ENABLE_NVTX_PROFILE | ||
|
|
||
| namespace onnxruntime { | ||
| namespace profile { | ||
|
|
||
| // Singleton class of managing global NVTX profiling information. | ||
| class Context { | ||
| public: | ||
| static Context& GetInstance() { | ||
| static Context instance_; | ||
| return instance_; | ||
| } | ||
|
|
||
| // Return tag for the specified thread. | ||
| // If the thread's tag doesn't exist, this function returns an empty string. | ||
| std::string GetThreadTagOrDefault(const std::thread::id& thread_id) { | ||
| const std::lock_guard<std::mutex> lock(mtx_); | ||
| return thread_tag_[thread_id]; | ||
| } | ||
|
|
||
| // Set tag for the specified thread. | ||
| void SetThreadTag( | ||
| const std::thread::id& thread_id, const std::string& tag) { | ||
| const std::lock_guard<std::mutex> lock(mtx_); | ||
| thread_tag_[thread_id] = tag; | ||
| } | ||
|
|
||
| private: | ||
| Context() = default; | ||
| ~Context() = default; | ||
| Context(const Context&) = delete; | ||
| Context& operator=(const Context&) = delete; | ||
|
|
||
| // map from thread's id to its human-readable tag. | ||
| std::unordered_map<std::thread::id, std::string> thread_tag_; | ||
| std::mutex mtx_; | ||
| }; | ||
|
|
||
| } // namespace profile | ||
| } // namespace onnxruntime | ||
|
|
||
| #endif |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,65 @@ | ||
| // Copyright (c) Microsoft Corporation. All rights reserved. | ||
| // Licensed under the MIT License. | ||
|
|
||
| #ifdef ENABLE_NVTX_PROFILE | ||
| #include "core/profile/profile.h" | ||
| #include "core/common/common.h" | ||
| #include <nvToolsExt.h> | ||
| #include <nvToolsExtCuda.h> | ||
|
|
||
| namespace onnxruntime { | ||
| namespace profile { | ||
|
|
||
| void NvtxRangeCreator::BeginImpl() { | ||
| // enable only for debug builds because this function is for profiling only. | ||
| nvtxEventAttributes_t eventAttrib; | ||
| eventAttrib.version = NVTX_VERSION; | ||
| eventAttrib.size = NVTX_EVENT_ATTRIB_STRUCT_SIZE; | ||
| eventAttrib.colorType = NVTX_COLOR_ARGB; | ||
| eventAttrib.color = static_cast<uint32_t>(color_); | ||
| eventAttrib.messageType = NVTX_MESSAGE_TYPE_ASCII; | ||
| eventAttrib.message.ascii = message_.c_str(); | ||
|
|
||
| range_id_ = nvtxRangeStartEx(&eventAttrib); | ||
| } | ||
|
|
||
| void NvtxRangeCreator::EndImpl() { | ||
| // enable only for debug builds because this function is for profiling only. | ||
| nvtxRangeEnd(range_id_); | ||
| } | ||
|
|
||
| void NvtxNestedRangeCreator::BeginImpl() { | ||
| // enable only for debug builds because this function is for profiling only. | ||
| nvtxEventAttributes_t eventAttrib; | ||
| eventAttrib.version = NVTX_VERSION; | ||
| eventAttrib.size = NVTX_EVENT_ATTRIB_STRUCT_SIZE; | ||
| eventAttrib.colorType = NVTX_COLOR_ARGB; | ||
| eventAttrib.color = static_cast<uint32_t>(color_); | ||
| eventAttrib.messageType = NVTX_MESSAGE_TYPE_ASCII; | ||
| eventAttrib.message.ascii = message_.c_str(); | ||
|
|
||
| nvtxRangePushEx(&eventAttrib); | ||
| } | ||
|
|
||
| void NvtxNestedRangeCreator::EndImpl() { | ||
| // enable only for debug builds because this function is for profiling only. | ||
| nvtxRangePop(); | ||
| } | ||
|
|
||
| void NvtxMarkerCreator::Mark() { | ||
| // enable only for debug builds because this function is for profiling only. | ||
| nvtxEventAttributes_t eventAttrib; | ||
| eventAttrib.version = NVTX_VERSION; | ||
| eventAttrib.size = NVTX_EVENT_ATTRIB_STRUCT_SIZE; | ||
| eventAttrib.colorType = NVTX_COLOR_ARGB; | ||
| eventAttrib.color = static_cast<uint32_t>(color_); | ||
| eventAttrib.messageType = NVTX_MESSAGE_TYPE_ASCII; | ||
| eventAttrib.message.ascii = message_.c_str(); | ||
|
|
||
| nvtxMarkEx(&eventAttrib); | ||
| } | ||
|
|
||
| } // namespace contrib | ||
| } // namespace onnxruntime | ||
|
|
||
| #endif |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,140 @@ | ||
| // Copyright (c) Microsoft Corporation. All rights reserved. | ||
| // Licensed under the MIT License. | ||
|
|
||
| // These enclosed classes are wrappers over | ||
| // generating Nvidia's visual profile APIs. | ||
| // They can be used to plot the time intervals of forward and backward passes. | ||
| // They can also be used to plot the time span of a specific operator. | ||
| // When writing this file, Nvidia only supports this tool on Linux. | ||
| #ifdef ENABLE_NVTX_PROFILE | ||
|
|
||
| #pragma once | ||
|
|
||
| #include <cinttypes> | ||
| #include <cstdlib> | ||
| #include <iostream> | ||
| #include <stdexcept> | ||
| #include <string> | ||
|
|
||
| #include "core/common/common.h" | ||
|
|
||
| namespace onnxruntime { | ||
| namespace profile { | ||
|
|
||
| // Color in ARGB space. | ||
| // A: first 8 bit. | ||
| // R: later 8 bit. | ||
| // G: later 8 bit. | ||
| // B: last 8 bits | ||
| // All colo channels has range [0, 255]. | ||
| enum class Color : uint32_t { | ||
| Black = 0x00000000, | ||
| Red = 0x00ff0000, | ||
| DarkGreen = 0x00009900, | ||
| Green = 0x0000ff00, | ||
| LightGreen = 0x00ccffcc, | ||
| Blue = 0x000000ff, | ||
| Amber = 0x00ffbf00, | ||
| LightAmber = 0x00fff2cc, | ||
| White = 0x00ffffff, | ||
| Cyan = 0x0000ffff, | ||
| Magenta = 0x00ff00ff | ||
| }; | ||
|
|
||
| class RangeCreatorBase { | ||
| public: | ||
| RangeCreatorBase(const std::string message, const Color color) | ||
| : message_(message), color_(color), | ||
| is_begin_called_(false), is_end_called_(false) {}; | ||
|
|
||
| // Check if Begin and End are both called. | ||
| // It's pointless if not all of them are called. | ||
| ~RangeCreatorBase() { | ||
| if (!is_begin_called_) { | ||
| std::cerr << "Begin must be called once." << std::endl; | ||
| } | ||
| if (!is_end_called_) { | ||
| std::cerr << "End must be called once." << std::endl; | ||
| } | ||
| } | ||
|
|
||
| // Mark the beginning of a range. | ||
| void Begin() { | ||
| ORT_ENFORCE(!is_begin_called_, "Begin cannot be called more than once."); | ||
| ORT_ENFORCE(!is_end_called_, "Begin cannot be called after calling End."); | ||
| BeginImpl(); | ||
| is_begin_called_ = true; | ||
| } | ||
|
|
||
| // Mark the end of a range. | ||
| void End() { | ||
| ORT_ENFORCE(is_begin_called_, "End must be called after calling Begin."); | ||
| ORT_ENFORCE(!is_end_called_, "End cannot be called more than once."); | ||
| EndImpl(); | ||
| is_end_called_ = true; | ||
| } | ||
|
|
||
| bool IsBeginCalled() const { | ||
| return is_begin_called_; | ||
| } | ||
|
|
||
| bool IsEndCalled() const { | ||
| return is_end_called_; | ||
| } | ||
|
|
||
| virtual void BeginImpl() = 0; | ||
|
|
||
| virtual void EndImpl() = 0; | ||
|
|
||
| protected: | ||
| // Text on this event. | ||
| const std::string message_; | ||
|
|
||
| // Color of event in ARGB space. | ||
| const Color color_; | ||
|
|
||
| bool is_begin_called_; | ||
| bool is_end_called_; | ||
| }; | ||
|
|
||
| class NvtxRangeCreator final : public RangeCreatorBase { | ||
| public: | ||
| NvtxRangeCreator(const std::string message, const Color color) | ||
| : RangeCreatorBase(message, color) {}; | ||
|
|
||
| void BeginImpl() override; | ||
| void EndImpl() override; | ||
|
|
||
| private: | ||
| // It records the event ID created by BeginImpl. | ||
| // EndImpl needs this value to end the right event. | ||
| uint64_t range_id_; | ||
| }; | ||
|
|
||
| class NvtxNestedRangeCreator final : public RangeCreatorBase { | ||
| public: | ||
| NvtxNestedRangeCreator(const std::string message, const Color color) | ||
| : RangeCreatorBase(message, color) {}; | ||
|
|
||
| void BeginImpl() override; | ||
| void EndImpl() override; | ||
| }; | ||
|
|
||
| class NvtxMarkerCreator final { | ||
| public: | ||
| NvtxMarkerCreator(const std::string message, const Color color) | ||
| : message_(message), color_(color) {}; | ||
| void Mark(); | ||
|
|
||
| private: | ||
| // Text on this marker. | ||
| const std::string message_; | ||
|
|
||
| // See nvtxRangeCreator.color_. | ||
| const Color color_; | ||
| }; | ||
|
|
||
| } // namespace profile | ||
| } // namespace onnxruntime | ||
|
|
||
| #endif |
Oops, something went wrong.