From e86ceba79dc5999c7b3655fd006847a360b351a0 Mon Sep 17 00:00:00 2001 From: WenTao Ou Date: Thu, 4 Apr 2024 02:45:07 +0800 Subject: [PATCH] [EXPORTER] OTLP file exporter (#2540) --- CHANGELOG.md | 2 + CMakeLists.txt | 8 +- ci/do_ci.sh | 9 + cmake/opentelemetry-cpp-config.cmake.in | 8 + exporters/CMakeLists.txt | 4 +- exporters/otlp/BUILD | 218 +++ exporters/otlp/CMakeLists.txt | 166 ++ .../exporters/otlp/otlp_file_client.h | 87 + .../exporters/otlp/otlp_file_client_options.h | 108 ++ .../exporters/otlp/otlp_file_exporter.h | 78 + .../otlp/otlp_file_exporter_factory.h | 37 + .../otlp/otlp_file_exporter_options.h | 30 + .../otlp/otlp_file_log_record_exporter.h | 78 + .../otlp_file_log_record_exporter_factory.h | 37 + .../otlp_file_log_record_exporter_options.h | 30 + .../otlp/otlp_file_metric_exporter.h | 73 + .../otlp/otlp_file_metric_exporter_factory.h | 37 + .../otlp/otlp_file_metric_exporter_options.h | 33 + exporters/otlp/src/otlp_file_client.cc | 1616 +++++++++++++++++ exporters/otlp/src/otlp_file_exporter.cc | 94 + .../otlp/src/otlp_file_exporter_factory.cc | 30 + .../otlp/src/otlp_file_exporter_options.cc | 34 + .../otlp/src/otlp_file_log_record_exporter.cc | 94 + .../otlp_file_log_record_exporter_factory.cc | 31 + .../otlp_file_log_record_exporter_options.cc | 34 + .../otlp/src/otlp_file_metric_exporter.cc | 99 + .../src/otlp_file_metric_exporter_factory.cc | 31 + .../src/otlp_file_metric_exporter_options.cc | 36 + exporters/otlp/src/otlp_http_client.cc | 4 + exporters/otlp/test/otlp_file_client_test.cc | 533 ++++++ .../test/otlp_file_exporter_factory_test.cc | 42 + .../otlp/test/otlp_file_exporter_test.cc | 145 ++ ...p_file_log_record_exporter_factory_test.cc | 42 + .../otlp_file_log_record_exporter_test.cc | 177 ++ .../otlp_file_metric_exporter_factory_test.cc | 42 + .../test/otlp_file_metric_exporter_test.cc | 390 ++++ 36 files changed, 4515 insertions(+), 2 deletions(-) create mode 100644 exporters/otlp/include/opentelemetry/exporters/otlp/otlp_file_client.h create mode 100644 exporters/otlp/include/opentelemetry/exporters/otlp/otlp_file_client_options.h create mode 100644 exporters/otlp/include/opentelemetry/exporters/otlp/otlp_file_exporter.h create mode 100644 exporters/otlp/include/opentelemetry/exporters/otlp/otlp_file_exporter_factory.h create mode 100644 exporters/otlp/include/opentelemetry/exporters/otlp/otlp_file_exporter_options.h create mode 100644 exporters/otlp/include/opentelemetry/exporters/otlp/otlp_file_log_record_exporter.h create mode 100644 exporters/otlp/include/opentelemetry/exporters/otlp/otlp_file_log_record_exporter_factory.h create mode 100644 exporters/otlp/include/opentelemetry/exporters/otlp/otlp_file_log_record_exporter_options.h create mode 100644 exporters/otlp/include/opentelemetry/exporters/otlp/otlp_file_metric_exporter.h create mode 100644 exporters/otlp/include/opentelemetry/exporters/otlp/otlp_file_metric_exporter_factory.h create mode 100644 exporters/otlp/include/opentelemetry/exporters/otlp/otlp_file_metric_exporter_options.h create mode 100644 exporters/otlp/src/otlp_file_client.cc create mode 100644 exporters/otlp/src/otlp_file_exporter.cc create mode 100644 exporters/otlp/src/otlp_file_exporter_factory.cc create mode 100644 exporters/otlp/src/otlp_file_exporter_options.cc create mode 100644 exporters/otlp/src/otlp_file_log_record_exporter.cc create mode 100644 exporters/otlp/src/otlp_file_log_record_exporter_factory.cc create mode 100644 exporters/otlp/src/otlp_file_log_record_exporter_options.cc create mode 100644 exporters/otlp/src/otlp_file_metric_exporter.cc create mode 100644 exporters/otlp/src/otlp_file_metric_exporter_factory.cc create mode 100644 exporters/otlp/src/otlp_file_metric_exporter_options.cc create mode 100644 exporters/otlp/test/otlp_file_client_test.cc create mode 100644 exporters/otlp/test/otlp_file_exporter_factory_test.cc create mode 100644 exporters/otlp/test/otlp_file_exporter_test.cc create mode 100644 exporters/otlp/test/otlp_file_log_record_exporter_factory_test.cc create mode 100644 exporters/otlp/test/otlp_file_log_record_exporter_test.cc create mode 100644 exporters/otlp/test/otlp_file_metric_exporter_factory_test.cc create mode 100644 exporters/otlp/test/otlp_file_metric_exporter_test.cc diff --git a/CHANGELOG.md b/CHANGELOG.md index ccb629074e..e0ba982b89 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,8 @@ Increment the: ## [Unreleased] +* [EXPORTER] Add OTLP File exporters + [#2540](https://github.com/open-telemetry/opentelemetry-cpp/pull/2540) * [EXPORTER] Gzip compression support for OTLP/HTTP and OTLP/gRPC exporter [#2530](https://github.com/open-telemetry/opentelemetry-cpp/pull/2530) * [EXPORTER] Support URL-encoded values for `OTEL_EXPORTER_OTLP_HEADERS` diff --git a/CMakeLists.txt b/CMakeLists.txt index 8a6a28510a..cc085b1af4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -204,6 +204,9 @@ option(WITH_OTLP_GRPC "Whether to include the OTLP gRPC exporter in the SDK" option(WITH_OTLP_HTTP "Whether to include the OTLP http exporter in the SDK" OFF) +option(WITH_OTLP_FILE "Whether to include the OTLP file exporter in the SDK" + OFF) + option( WITH_OTLP_HTTP_COMPRESSION "Whether to include gzip compression for the OTLP http exporter in the SDK" @@ -370,7 +373,9 @@ if(WITH_ABSEIL) find_package(absl CONFIG REQUIRED) endif() -if(WITH_OTLP_GRPC OR WITH_OTLP_HTTP) +if(WITH_OTLP_GRPC + OR WITH_OTLP_HTTP + OR WITH_OTLP_FILE) find_package(Protobuf) if(Protobuf_VERSION AND Protobuf_VERSION VERSION_GREATER_EQUAL "3.22.0") if(NOT WITH_ABSEIL) @@ -473,6 +478,7 @@ endif() if(WITH_ELASTICSEARCH OR WITH_ZIPKIN OR WITH_OTLP_HTTP + OR WITH_OTLP_FILE OR BUILD_W3CTRACECONTEXT_TEST OR WITH_ETW) set(USE_NLOHMANN_JSON ON) diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 7897bab440..87bf781da2 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -114,6 +114,7 @@ elif [[ "$1" == "cmake.maintainer.sync.test" ]]; then rm -rf * cmake "${CMAKE_OPTIONS[@]}" \ -DWITH_OTLP_HTTP=ON \ + -DWITH_OTLP_FILE=ON \ -DWITH_PROMETHEUS=ON \ -DWITH_EXAMPLES=ON \ -DWITH_EXAMPLES_HTTP=ON \ @@ -135,6 +136,7 @@ elif [[ "$1" == "cmake.maintainer.async.test" ]]; then rm -rf * cmake "${CMAKE_OPTIONS[@]}" \ -DWITH_OTLP_HTTP=ON \ + -DWITH_OTLP_FILE=ON \ -DWITH_PROMETHEUS=ON \ -DWITH_EXAMPLES=ON \ -DWITH_EXAMPLES_HTTP=ON \ @@ -157,6 +159,7 @@ elif [[ "$1" == "cmake.maintainer.cpp11.async.test" ]]; then cmake "${CMAKE_OPTIONS[@]}" \ -DCMAKE_CXX_STANDARD=11 \ -DWITH_OTLP_HTTP=ON \ + -DWITH_OTLP_FILE=ON \ -DWITH_PROMETHEUS=ON \ -DWITH_EXAMPLES=ON \ -DWITH_EXAMPLES_HTTP=ON \ @@ -177,6 +180,7 @@ elif [[ "$1" == "cmake.maintainer.abiv2.test" ]]; then rm -rf * cmake "${CMAKE_OPTIONS[@]}" \ -DWITH_OTLP_HTTP=ON \ + -DWITH_OTLP_FILE=ON \ -DWITH_PROMETHEUS=ON \ -DWITH_EXAMPLES=ON \ -DWITH_EXAMPLES_HTTP=ON \ @@ -327,6 +331,7 @@ elif [[ "$1" == "cmake.legacy.exporter.otprotocol.test" ]]; then -DCMAKE_CXX_STANDARD=11 \ -DWITH_OTLP_GRPC=ON \ -DWITH_OTLP_HTTP=ON \ + -DWITH_OTLP_FILE=ON \ -DWITH_ASYNC_EXPORT_PREVIEW=ON \ "${SRC_DIR}" grpc_cpp_plugin=`which grpc_cpp_plugin` @@ -344,6 +349,7 @@ elif [[ "$1" == "cmake.exporter.otprotocol.test" ]]; then cmake "${CMAKE_OPTIONS[@]}" \ -DWITH_OTLP_GRPC=ON \ -DWITH_OTLP_HTTP=ON \ + -DWITH_OTLP_FILE=ON \ -DWITH_OTLP_GRPC_SSL_MTLS_PREVIEW=ON \ "${SRC_DIR}" grpc_cpp_plugin=`which grpc_cpp_plugin` @@ -358,6 +364,7 @@ elif [[ "$1" == "cmake.exporter.otprotocol.shared_libs.with_static_grpc.test" ]] cmake "${CMAKE_OPTIONS[@]}" \ -DWITH_OTLP_GRPC=ON \ -DWITH_OTLP_HTTP=ON \ + -DWITH_OTLP_FILE=ON \ -DBUILD_SHARED_LIBS=ON \ "${SRC_DIR}" grpc_cpp_plugin=`which grpc_cpp_plugin` @@ -372,6 +379,7 @@ elif [[ "$1" == "cmake.exporter.otprotocol.with_async_export.test" ]]; then cmake "${CMAKE_OPTIONS[@]}" \ -DWITH_OTLP_GRPC=ON \ -DWITH_OTLP_HTTP=ON \ + -DWITH_OTLP_FILE=ON \ -DWITH_ASYNC_EXPORT_PREVIEW=ON \ "${SRC_DIR}" grpc_cpp_plugin=`which grpc_cpp_plugin` @@ -386,6 +394,7 @@ elif [[ "$1" == "cmake.do_not_install.test" ]]; then cmake "${CMAKE_OPTIONS[@]}" \ -DWITH_OTLP_GRPC=ON \ -DWITH_OTLP_HTTP=ON \ + -DWITH_OTLP_FILE=ON \ -DWITH_ASYNC_EXPORT_PREVIEW=ON \ -DOPENTELEMETRY_INSTALL=OFF \ "${SRC_DIR}" diff --git a/cmake/opentelemetry-cpp-config.cmake.in b/cmake/opentelemetry-cpp-config.cmake.in index 86098f7d7c..36215c8af7 100644 --- a/cmake/opentelemetry-cpp-config.cmake.in +++ b/cmake/opentelemetry-cpp-config.cmake.in @@ -41,6 +41,10 @@ # opentelemetry-cpp::otlp_http_exporter - Imported target of opentelemetry-cpp::otlp_http_exporter # opentelemetry-cpp::otlp_http_log_record_exporter - Imported target of opentelemetry-cpp::otlp_http_log_record_exporter # opentelemetry-cpp::otlp_http_metric_exporter - Imported target of opentelemetry-cpp::otlp_http_metric_exporter +# opentelemetry-cpp::otlp_file_client - Imported target of opentelemetry-cpp::otlp_file_client +# opentelemetry-cpp::otlp_file_exporter - Imported target of opentelemetry-cpp::otlp_file_exporter +# opentelemetry-cpp::otlp_file_log_record_exporter - Imported target of opentelemetry-cpp::otlp_file_log_record_exporter +# opentelemetry-cpp::otlp_file_metric_exporter - Imported target of opentelemetry-cpp::otlp_file_metric_exporter # opentelemetry-cpp::ostream_log_record_exporter - Imported target of opentelemetry-cpp::ostream_log_record_exporter # opentelemetry-cpp::ostream_metrics_exporter - Imported target of opentelemetry-cpp::ostream_metrics_exporter # opentelemetry-cpp::ostream_span_exporter - Imported target of opentelemetry-cpp::ostream_span_exporter @@ -93,6 +97,10 @@ set(_OPENTELEMETRY_CPP_LIBRARIES_TEST_TARGETS otlp_http_exporter otlp_http_log_record_exporter otlp_http_metric_exporter + otlp_file_client + otlp_file_exporter + otlp_file_log_record_exporter + otlp_file_metric_exporter ostream_log_record_exporter ostream_metrics_exporter ostream_span_exporter diff --git a/exporters/CMakeLists.txt b/exporters/CMakeLists.txt index c40ff48654..a59e1f9046 100644 --- a/exporters/CMakeLists.txt +++ b/exporters/CMakeLists.txt @@ -1,7 +1,9 @@ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 -if(WITH_OTLP_GRPC OR WITH_OTLP_HTTP) +if(WITH_OTLP_GRPC + OR WITH_OTLP_HTTP + OR WITH_OTLP_FILE) add_subdirectory(otlp) endif() diff --git a/exporters/otlp/BUILD b/exporters/otlp/BUILD index a9f1c887e3..a2dd8f8c65 100644 --- a/exporters/otlp/BUILD +++ b/exporters/otlp/BUILD @@ -169,6 +169,61 @@ cc_library( ], ) +cc_library( + name = "otlp_file_client", + srcs = [ + "src/otlp_file_client.cc", + ], + hdrs = [ + "include/opentelemetry/exporters/otlp/otlp_environment.h", + "include/opentelemetry/exporters/otlp/otlp_file_client.h", + "include/opentelemetry/exporters/otlp/otlp_file_client_options.h", + "include/opentelemetry/exporters/otlp/protobuf_include_prefix.h", + "include/opentelemetry/exporters/otlp/protobuf_include_suffix.h", + ], + strip_include_prefix = "include", + tags = [ + "otlp", + "otlp_file", + ], + deps = [ + "//api", + "//sdk:headers", + "//sdk/src/common:base64", + "@com_github_opentelemetry_proto//:common_proto_cc", + "@com_google_absl//absl/strings", + "@github_nlohmann_json//:json", + ], +) + +cc_library( + name = "otlp_file_exporter", + srcs = [ + "src/otlp_file_exporter.cc", + "src/otlp_file_exporter_factory.cc", + "src/otlp_file_exporter_options.cc", + ], + hdrs = [ + "include/opentelemetry/exporters/otlp/otlp_environment.h", + "include/opentelemetry/exporters/otlp/otlp_file_exporter.h", + "include/opentelemetry/exporters/otlp/otlp_file_exporter_factory.h", + "include/opentelemetry/exporters/otlp/otlp_file_exporter_options.h", + "include/opentelemetry/exporters/otlp/protobuf_include_prefix.h", + "include/opentelemetry/exporters/otlp/protobuf_include_suffix.h", + ], + strip_include_prefix = "include", + tags = [ + "otlp", + "otlp_file", + ], + deps = [ + ":otlp_file_client", + ":otlp_recordable", + "//sdk/src/trace", + "@com_github_opentelemetry_proto//:trace_service_proto_cc", + ], +) + cc_library( name = "otlp_grpc_metric_exporter", srcs = [ @@ -229,6 +284,34 @@ cc_library( ], ) +cc_library( + name = "otlp_file_metric_exporter", + srcs = [ + "src/otlp_file_metric_exporter.cc", + "src/otlp_file_metric_exporter_factory.cc", + "src/otlp_file_metric_exporter_options.cc", + ], + hdrs = [ + "include/opentelemetry/exporters/otlp/otlp_environment.h", + "include/opentelemetry/exporters/otlp/otlp_file_metric_exporter.h", + "include/opentelemetry/exporters/otlp/otlp_file_metric_exporter_factory.h", + "include/opentelemetry/exporters/otlp/otlp_file_metric_exporter_options.h", + "include/opentelemetry/exporters/otlp/protobuf_include_prefix.h", + "include/opentelemetry/exporters/otlp/protobuf_include_suffix.h", + ], + strip_include_prefix = "include", + tags = [ + "otlp", + "otlp_file_metric", + ], + deps = [ + ":otlp_file_client", + ":otlp_recordable", + "//sdk/src/metrics", + "@com_github_opentelemetry_proto//:metrics_service_proto_cc", + ], +) + cc_library( name = "otlp_http_log_record_exporter", srcs = [ @@ -257,6 +340,34 @@ cc_library( ], ) +cc_library( + name = "otlp_file_log_record_exporter", + srcs = [ + "src/otlp_file_log_record_exporter.cc", + "src/otlp_file_log_record_exporter_factory.cc", + "src/otlp_file_log_record_exporter_options.cc", + ], + hdrs = [ + "include/opentelemetry/exporters/otlp/otlp_environment.h", + "include/opentelemetry/exporters/otlp/otlp_file_log_record_exporter.h", + "include/opentelemetry/exporters/otlp/otlp_file_log_record_exporter_factory.h", + "include/opentelemetry/exporters/otlp/otlp_file_log_record_exporter_options.h", + "include/opentelemetry/exporters/otlp/protobuf_include_prefix.h", + "include/opentelemetry/exporters/otlp/protobuf_include_suffix.h", + ], + strip_include_prefix = "include", + tags = [ + "otlp", + "otlp_file_log", + ], + deps = [ + ":otlp_file_client", + ":otlp_recordable", + "//sdk/src/logs", + "@com_github_opentelemetry_proto//:logs_service_proto_cc", + ], +) + cc_library( name = "otlp_grpc_log_record_exporter", srcs = [ @@ -318,6 +429,23 @@ cc_test( ], ) +cc_test( + name = "otlp_file_client_test", + srcs = ["test/otlp_file_client_test.cc"], + tags = [ + "otlp", + "otlp_file", + "test", + ], + deps = [ + ":otlp_file_client", + ":otlp_file_exporter", + ":otlp_recordable", + "//api", + "@com_google_googletest//:gtest_main", + ], +) + cc_test( name = "otlp_grpc_exporter_test", srcs = ["test/otlp_grpc_exporter_test.cc"], @@ -380,6 +508,36 @@ cc_test( ], ) +cc_test( + name = "otlp_file_exporter_test", + srcs = ["test/otlp_file_exporter_test.cc"], + tags = [ + "otlp", + "otlp_file", + "test", + ], + deps = [ + ":otlp_file_exporter", + "//api", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "otlp_file_exporter_factory_test", + srcs = ["test/otlp_file_exporter_factory_test.cc"], + tags = [ + "otlp", + "otlp_file", + "test", + ], + deps = [ + ":otlp_file_exporter", + "//api", + "@com_google_googletest//:gtest_main", + ], +) + cc_test( name = "otlp_http_log_record_exporter_test", srcs = ["test/otlp_http_log_record_exporter_test.cc"], @@ -412,6 +570,36 @@ cc_test( ], ) +cc_test( + name = "otlp_file_log_record_exporter_test", + srcs = ["test/otlp_file_log_record_exporter_test.cc"], + tags = [ + "otlp", + "otlp_file_log", + "test", + ], + deps = [ + ":otlp_file_log_record_exporter", + "//api", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "otlp_file_log_record_exporter_factory_test", + srcs = ["test/otlp_file_log_record_exporter_factory_test.cc"], + tags = [ + "otlp", + "otlp_file_log", + "test", + ], + deps = [ + ":otlp_file_log_record_exporter", + "//api", + "@com_google_googletest//:gtest_main", + ], +) + cc_test( name = "otlp_grpc_log_record_exporter_test", srcs = ["test/otlp_grpc_log_record_exporter_test.cc"], @@ -508,6 +696,36 @@ cc_test( ], ) +cc_test( + name = "otlp_file_metric_exporter_test", + srcs = ["test/otlp_file_metric_exporter_test.cc"], + tags = [ + "otlp", + "otlp_file_metric", + "test", + ], + deps = [ + ":otlp_file_metric_exporter", + "//api", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "otlp_file_metric_exporter_factory_test", + srcs = ["test/otlp_file_metric_exporter_factory_test.cc"], + tags = [ + "otlp", + "otlp_file_metric", + "test", + ], + deps = [ + ":otlp_file_metric_exporter", + "//api", + "@com_google_googletest//:gtest_main", + ], +) + otel_cc_benchmark( name = "otlp_grpc_exporter_benchmark", srcs = ["test/otlp_grpc_exporter_benchmark.cc"], diff --git a/exporters/otlp/CMakeLists.txt b/exporters/otlp/CMakeLists.txt index 243a7a2bd3..228c5cc67f 100644 --- a/exporters/otlp/CMakeLists.txt +++ b/exporters/otlp/CMakeLists.txt @@ -186,6 +186,83 @@ if(WITH_OTLP_HTTP) opentelemetry_exporter_otlp_http_metric) endif() +if(WITH_OTLP_FILE) + add_library(opentelemetry_exporter_otlp_file_client src/otlp_file_client.cc) + set_target_properties(opentelemetry_exporter_otlp_file_client + PROPERTIES EXPORT_NAME otlp_file_client) + set_target_version(opentelemetry_exporter_otlp_file_client) + + target_link_libraries( + opentelemetry_exporter_otlp_file_client + PUBLIC opentelemetry_sdk opentelemetry_common + PRIVATE opentelemetry_proto nlohmann_json::nlohmann_json) + if(TARGET absl::strings) + target_link_libraries(opentelemetry_exporter_otlp_file_client + PUBLIC absl::strings) + endif() + if(nlohmann_json_clone) + add_dependencies(opentelemetry_exporter_otlp_file_client + nlohmann_json::nlohmann_json) + endif() + target_include_directories( + opentelemetry_exporter_otlp_file_client + PUBLIC "$" + "$") + + list(APPEND OPENTELEMETRY_OTLP_TARGETS + opentelemetry_exporter_otlp_file_client) + + add_library( + opentelemetry_exporter_otlp_file + src/otlp_file_exporter.cc src/otlp_file_exporter_factory.cc + src/otlp_file_exporter_options.cc) + + set_target_properties(opentelemetry_exporter_otlp_file + PROPERTIES EXPORT_NAME otlp_file_exporter) + set_target_version(opentelemetry_exporter_otlp_file) + + target_link_libraries( + opentelemetry_exporter_otlp_file + PUBLIC opentelemetry_otlp_recordable + opentelemetry_exporter_otlp_file_client) + + list(APPEND OPENTELEMETRY_OTLP_TARGETS opentelemetry_exporter_otlp_file) + + add_library( + opentelemetry_exporter_otlp_file_log + src/otlp_file_log_record_exporter.cc + src/otlp_file_log_record_exporter_factory.cc + src/otlp_file_log_record_exporter_options.cc) + + set_target_properties(opentelemetry_exporter_otlp_file_log + PROPERTIES EXPORT_NAME otlp_file_log_record_exporter) + set_target_version(opentelemetry_exporter_otlp_file_log) + + target_link_libraries( + opentelemetry_exporter_otlp_file_log + PUBLIC opentelemetry_otlp_recordable + opentelemetry_exporter_otlp_file_client) + + list(APPEND OPENTELEMETRY_OTLP_TARGETS opentelemetry_exporter_otlp_file_log) + + add_library( + opentelemetry_exporter_otlp_file_metric + src/otlp_file_metric_exporter.cc src/otlp_file_metric_exporter_factory.cc + src/otlp_file_metric_exporter_options.cc) + + set_target_properties(opentelemetry_exporter_otlp_file_metric + PROPERTIES EXPORT_NAME otlp_file_metric_exporter) + set_target_version(opentelemetry_exporter_otlp_file_metric) + + target_link_libraries( + opentelemetry_exporter_otlp_file_metric + PUBLIC opentelemetry_otlp_recordable + opentelemetry_exporter_otlp_file_client) + + list(APPEND OPENTELEMETRY_OTLP_TARGETS + opentelemetry_exporter_otlp_file_metric) +endif() + target_link_libraries( opentelemetry_otlp_recordable PUBLIC opentelemetry_trace opentelemetry_resources opentelemetry_proto) @@ -408,4 +485,93 @@ if(BUILD_TESTING) TEST_PREFIX exporter.otlp. TEST_LIST otlp_http_metric_exporter_factory_test) endif() + + if(WITH_OTLP_FILE) + add_executable(otlp_file_client_test test/otlp_file_client_test.cc) + target_link_libraries( + otlp_file_client_test ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} + ${GMOCK_LIB} opentelemetry_exporter_otlp_file + opentelemetry_otlp_recordable) + gtest_add_tests( + TARGET otlp_file_client_test + TEST_PREFIX exporter.otlp. + TEST_LIST otlp_file_client_test) + + add_executable(otlp_file_exporter_test test/otlp_file_exporter_test.cc) + target_link_libraries( + otlp_file_exporter_test + ${GTEST_BOTH_LIBRARIES} + ${CMAKE_THREAD_LIBS_INIT} + ${GMOCK_LIB} + opentelemetry_exporter_otlp_file + nlohmann_json::nlohmann_json + protobuf::libprotobuf) + gtest_add_tests( + TARGET otlp_file_exporter_test + TEST_PREFIX exporter.otlp. + TEST_LIST otlp_file_exporter_test) + + add_executable(otlp_file_exporter_factory_test + test/otlp_file_exporter_factory_test.cc) + target_link_libraries( + otlp_file_exporter_factory_test ${GTEST_BOTH_LIBRARIES} + ${CMAKE_THREAD_LIBS_INIT} ${GMOCK_LIB} opentelemetry_exporter_otlp_file) + gtest_add_tests( + TARGET otlp_file_exporter_factory_test + TEST_PREFIX exporter.otlp. + TEST_LIST otlp_file_exporter_factory_test) + + add_executable(otlp_file_log_record_exporter_test + test/otlp_file_log_record_exporter_test.cc) + target_link_libraries( + otlp_file_log_record_exporter_test + ${GTEST_BOTH_LIBRARIES} + ${CMAKE_THREAD_LIBS_INIT} + ${GMOCK_LIB} + opentelemetry_exporter_otlp_file_log + opentelemetry_logs + nlohmann_json::nlohmann_json) + gtest_add_tests( + TARGET otlp_file_log_record_exporter_test + TEST_PREFIX exporter.otlp. + TEST_LIST otlp_file_log_record_exporter_test) + + add_executable(otlp_file_log_record_exporter_factory_test + test/otlp_file_log_record_exporter_factory_test.cc) + target_link_libraries( + otlp_file_log_record_exporter_factory_test ${GTEST_BOTH_LIBRARIES} + ${CMAKE_THREAD_LIBS_INIT} ${GMOCK_LIB} + opentelemetry_exporter_otlp_file_log opentelemetry_logs) + gtest_add_tests( + TARGET otlp_file_log_record_exporter_factory_test + TEST_PREFIX exporter.otlp. + TEST_LIST otlp_file_log_record_exporter_factory_test) + + add_executable(otlp_file_metric_exporter_test + test/otlp_file_metric_exporter_test.cc) + target_link_libraries( + otlp_file_metric_exporter_test + ${GTEST_BOTH_LIBRARIES} + ${CMAKE_THREAD_LIBS_INIT} + ${GMOCK_LIB} + opentelemetry_exporter_otlp_file_metric + opentelemetry_metrics + nlohmann_json::nlohmann_json + protobuf::libprotobuf) + gtest_add_tests( + TARGET otlp_file_metric_exporter_test + TEST_PREFIX exporter.otlp. + TEST_LIST otlp_file_metric_exporter_test) + + add_executable(otlp_file_metric_exporter_factory_test + test/otlp_file_metric_exporter_factory_test.cc) + target_link_libraries( + otlp_file_metric_exporter_factory_test ${GTEST_BOTH_LIBRARIES} + ${CMAKE_THREAD_LIBS_INIT} ${GMOCK_LIB} + opentelemetry_exporter_otlp_file_metric opentelemetry_metrics) + gtest_add_tests( + TARGET otlp_file_metric_exporter_factory_test + TEST_PREFIX exporter.otlp. + TEST_LIST otlp_file_metric_exporter_factory_test) + endif() endif() # BUILD_TESTING diff --git a/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_file_client.h b/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_file_client.h new file mode 100644 index 0000000000..bedbe77e52 --- /dev/null +++ b/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_file_client.h @@ -0,0 +1,87 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include "opentelemetry/nostd/shared_ptr.h" +#include "opentelemetry/sdk/common/exporter_utils.h" + +#include "opentelemetry/exporters/otlp/otlp_file_client_options.h" + +#include + +// forward declare google::protobuf::Message +namespace google +{ +namespace protobuf +{ +class Message; +} +} // namespace google + +OPENTELEMETRY_BEGIN_NAMESPACE +namespace exporter +{ +namespace otlp +{ + +/** + * The OTLP File client exports data in OpenTelemetry Protocol (OTLP) format. + */ +class OtlpFileClient +{ +public: + /** + * Create an OtlpFileClient using the given options. + */ + explicit OtlpFileClient(OtlpFileClientOptions &&options); + + ~OtlpFileClient(); + + /** + * Sync export + * @param message message to export, it should be ExportTraceServiceRequest, + * ExportMetricsServiceRequest or ExportLogsServiceRequest + * @param record_count record count of the message + * @return return the status of this operation + */ + sdk::common::ExportResult Export(const google::protobuf::Message &message, + std::size_t record_count) noexcept; + + /** + * Force flush the file client. + */ + bool ForceFlush(std::chrono::microseconds timeout = (std::chrono::microseconds::max)()) noexcept; + + /** + * Shut down the file client. + * @param timeout an optional timeout, the default timeout of 0 means that no + * timeout is applied. + * @return return the status of this operation + */ + bool Shutdown(std::chrono::microseconds timeout = std::chrono::microseconds(0)) noexcept; + + /** + * Get options of current OTLP file client. + * @return options of current OTLP file client. + */ + inline const OtlpFileClientOptions &GetOptions() const noexcept { return options_; } + + /** + * Get if this OTLP file client is shutdown. + * @return return true after Shutdown is called. + */ + bool IsShutdown() const noexcept; + +private: + // Stores if this file client had its Shutdown() method called + bool is_shutdown_; + + // The configuration options associated with this file client. + const OtlpFileClientOptions options_; + + opentelemetry::nostd::shared_ptr backend_; +}; +} // namespace otlp +} // namespace exporter +OPENTELEMETRY_END_NAMESPACE diff --git a/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_file_client_options.h b/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_file_client_options.h new file mode 100644 index 0000000000..c355d515c8 --- /dev/null +++ b/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_file_client_options.h @@ -0,0 +1,108 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include "opentelemetry/common/macros.h" +#include "opentelemetry/nostd/shared_ptr.h" +#include "opentelemetry/nostd/string_view.h" +#include "opentelemetry/nostd/variant.h" + +#include +#include +#include +#include +#include + +// forward declare google::protobuf::Message +namespace google +{ +namespace protobuf +{ +class Message; +} +} // namespace google + +OPENTELEMETRY_BEGIN_NAMESPACE +namespace exporter +{ +namespace otlp +{ + +/** + * Struct to hold OTLP File client options for file system backend. + * @note Available placeholder for file_pattern and alias_pattern: + * %Y: writes year as a 4 digit decimal number + * %y: writes last 2 digits of year as a decimal number (range [00,99]) + * %m: writes month as a decimal number (range [01,12]) + * %j: writes day of the year as a decimal number (range [001,366]) + * %d: writes day of the month as a decimal number (range [01,31]) + * %w: writes weekday as a decimal number, where Sunday is 0 (range [0-6]) + * %H: writes hour as a decimal number, 24 hour clock (range [00-23]) + * %I: writes hour as a decimal number, 12 hour clock (range [01,12]) + * %M: writes minute as a decimal number (range [00,59]) + * %S: writes second as a decimal number (range [00,60]) + * %F: equivalent to "%Y-%m-%d" (the ISO 8601 date format) + * %T: equivalent to "%H:%M:%S" (the ISO 8601 time format) + * %R: equivalent to "%H:%M" + * %N: rotate index, start from 0 + * %n: rotate index, start from 1 + */ +struct OtlpFileClientFileSystemOptions +{ + // Pattern to create output file + std::string file_pattern; + + // Pattern to create alias file path for the latest file rotation. + std::string alias_pattern; + + // Flush interval + std::chrono::microseconds flush_interval = std::chrono::microseconds(30000000); + + // Flush record count + std::size_t flush_count = 256; + + // Maximum file size + std::size_t file_size = 1024 * 1024 * 20; + + // Maximum file count + std::size_t rotate_size = 3; + + inline OtlpFileClientFileSystemOptions() noexcept {} +}; + +/** + * Class to append data of OTLP format. + */ +class OtlpFileAppender +{ +public: + virtual ~OtlpFileAppender() = default; + + virtual void Export(opentelemetry::nostd::string_view data, std::size_t record_count) = 0; + + virtual bool ForceFlush(std::chrono::microseconds timeout) noexcept = 0; + + virtual bool Shutdown(std::chrono::microseconds timeout) noexcept = 0; +}; + +using OtlpFileClientBackendOptions = + nostd::variant, + opentelemetry::nostd::shared_ptr>; + +/** + * Struct to hold OTLP FILE client options. + */ +struct OtlpFileClientOptions +{ + // Whether to print the status of the FILE client in the console + bool console_debug = false; + + OtlpFileClientBackendOptions backend_options; + + inline OtlpFileClientOptions() noexcept {} +}; +} // namespace otlp +} // namespace exporter +OPENTELEMETRY_END_NAMESPACE diff --git a/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_file_exporter.h b/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_file_exporter.h new file mode 100644 index 0000000000..b5b87f9f88 --- /dev/null +++ b/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_file_exporter.h @@ -0,0 +1,78 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +// We need include exporter.h first, which will include Windows.h with NOMINMAX on Windows +#include "opentelemetry/sdk/trace/exporter.h" + +#include "opentelemetry/exporters/otlp/otlp_file_client.h" + +#include "opentelemetry/exporters/otlp/otlp_file_exporter_options.h" + +#include +#include + +OPENTELEMETRY_BEGIN_NAMESPACE +namespace exporter +{ +namespace otlp +{ + +/** + * The OTLP exporter exports span data in OpenTelemetry Protocol (OTLP) format. + */ +class OPENTELEMETRY_EXPORT OtlpFileExporter final : public opentelemetry::sdk::trace::SpanExporter +{ +public: + /** + * Create an OtlpFileExporter using all default options. + */ + OtlpFileExporter(); + + /** + * Create an OtlpFileExporter using the given options. + */ + explicit OtlpFileExporter(const OtlpFileExporterOptions &options); + + /** + * Create a span recordable. + * @return a newly initialized Recordable object + */ + std::unique_ptr MakeRecordable() noexcept override; + + /** + * Export + * @param spans a span of unique pointers to span recordables + */ + opentelemetry::sdk::common::ExportResult Export( + const nostd::span> &spans) noexcept + override; + + /** + * Force flush the exporter. + * @param timeout an option timeout, default to max. + * @return return true when all data are exported, and false when timeout + */ + bool ForceFlush( + std::chrono::microseconds timeout = (std::chrono::microseconds::max)()) noexcept override; + + /** + * Shut down the exporter. + * @param timeout an optional timeout, the default timeout of 0 means that no + * timeout is applied. + * @return return the status of this operation + */ + bool Shutdown( + std::chrono::microseconds timeout = (std::chrono::microseconds::max)()) noexcept override; + +private: + // The configuration options associated with this exporter. + const OtlpFileExporterOptions options_; + + // Object that stores the file context. + std::unique_ptr file_client_; +}; +} // namespace otlp +} // namespace exporter +OPENTELEMETRY_END_NAMESPACE diff --git a/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_file_exporter_factory.h b/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_file_exporter_factory.h new file mode 100644 index 0000000000..dc68609729 --- /dev/null +++ b/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_file_exporter_factory.h @@ -0,0 +1,37 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include + +#include "opentelemetry/exporters/otlp/otlp_file_exporter_options.h" +#include "opentelemetry/sdk/trace/exporter.h" + +OPENTELEMETRY_BEGIN_NAMESPACE +namespace exporter +{ +namespace otlp +{ + +/** + * Factory class for OtlpFileExporter. + */ +class OPENTELEMETRY_EXPORT OtlpFileExporterFactory +{ +public: + /** + * Create an OtlpFileExporter using all default options. + */ + static std::unique_ptr Create(); + + /** + * Create an OtlpFileExporter using the given options. + */ + static std::unique_ptr Create( + const OtlpFileExporterOptions &options); +}; + +} // namespace otlp +} // namespace exporter +OPENTELEMETRY_END_NAMESPACE diff --git a/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_file_exporter_options.h b/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_file_exporter_options.h new file mode 100644 index 0000000000..862ce1daf6 --- /dev/null +++ b/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_file_exporter_options.h @@ -0,0 +1,30 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include "opentelemetry/version.h" + +#include "opentelemetry/exporters/otlp/otlp_file_client_options.h" + +OPENTELEMETRY_BEGIN_NAMESPACE +namespace exporter +{ +namespace otlp +{ + +/** + * Struct to hold OTLP File traces exporter options. + * + * See + * https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/protocol/file-exporter.md + */ +struct OPENTELEMETRY_EXPORT OtlpFileExporterOptions : public OtlpFileClientOptions +{ + OtlpFileExporterOptions(); + ~OtlpFileExporterOptions(); +}; + +} // namespace otlp +} // namespace exporter +OPENTELEMETRY_END_NAMESPACE diff --git a/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_file_log_record_exporter.h b/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_file_log_record_exporter.h new file mode 100644 index 0000000000..79d816ae53 --- /dev/null +++ b/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_file_log_record_exporter.h @@ -0,0 +1,78 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include "opentelemetry/sdk/logs/exporter.h" + +#include "opentelemetry/exporters/otlp/otlp_file_client.h" + +#include "opentelemetry/exporters/otlp/otlp_file_log_record_exporter_options.h" + +#include +#include +#include +#include + +OPENTELEMETRY_BEGIN_NAMESPACE +namespace exporter +{ +namespace otlp +{ + +/** + * The OTLP exporter exports log data in OpenTelemetry Protocol (OTLP) format. + */ +class OtlpFileLogRecordExporter final : public opentelemetry::sdk::logs::LogRecordExporter +{ +public: + /** + * Create an OtlpFileLogRecordExporter with default exporter options. + */ + OtlpFileLogRecordExporter(); + + /** + * Create an OtlpFileLogRecordExporter with user specified options. + * @param options An object containing the user's configuration options. + */ + OtlpFileLogRecordExporter(const OtlpFileLogRecordExporterOptions &options); + + /** + * Creates a recordable that stores the data in a JSON object + */ + std::unique_ptr MakeRecordable() noexcept override; + + /** + * Exports a vector of log records to the Elasticsearch instance. Guaranteed to return after a + * timeout specified from the options passed from the constructor. + * @param records A list of log records to send to Elasticsearch. + */ + opentelemetry::sdk::common::ExportResult Export( + const nostd::span> &records) noexcept + override; + + /** + * Force flush the exporter. + * @param timeout an option timeout, default to max. + * @return return true when all data are exported, and false when timeout + */ + bool ForceFlush( + std::chrono::microseconds timeout = (std::chrono::microseconds::max)()) noexcept override; + + /** + * Shutdown this exporter. + * @param timeout The maximum time to wait for the shutdown method to return + */ + bool Shutdown( + std::chrono::microseconds timeout = (std::chrono::microseconds::max)()) noexcept override; + +private: + // Configuration options for the exporter + const OtlpFileLogRecordExporterOptions options_; + + // Object that stores the file context. + std::unique_ptr file_client_; +}; +} // namespace otlp +} // namespace exporter +OPENTELEMETRY_END_NAMESPACE diff --git a/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_file_log_record_exporter_factory.h b/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_file_log_record_exporter_factory.h new file mode 100644 index 0000000000..0e6c0143b9 --- /dev/null +++ b/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_file_log_record_exporter_factory.h @@ -0,0 +1,37 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include + +#include "opentelemetry/exporters/otlp/otlp_file_log_record_exporter_options.h" +#include "opentelemetry/sdk/logs/exporter.h" + +OPENTELEMETRY_BEGIN_NAMESPACE +namespace exporter +{ +namespace otlp +{ + +/** + * Factory class for OtlpFileExporter. + */ +class OPENTELEMETRY_EXPORT OtlpFileLogRecordExporterFactory +{ +public: + /** + * Create an OtlpFileExporter using all default options. + */ + static std::unique_ptr Create(); + + /** + * Create an OtlpFileExporter using the given options. + */ + static std::unique_ptr Create( + const OtlpFileLogRecordExporterOptions &options); +}; + +} // namespace otlp +} // namespace exporter +OPENTELEMETRY_END_NAMESPACE diff --git a/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_file_log_record_exporter_options.h b/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_file_log_record_exporter_options.h new file mode 100644 index 0000000000..7045ffdc16 --- /dev/null +++ b/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_file_log_record_exporter_options.h @@ -0,0 +1,30 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include "opentelemetry/version.h" + +#include "opentelemetry/exporters/otlp/otlp_file_client_options.h" + +OPENTELEMETRY_BEGIN_NAMESPACE +namespace exporter +{ +namespace otlp +{ + +/** + * Struct to hold OTLP File log record exporter options. + * + * See + * https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/protocol/file-exporter.md + */ +struct OPENTELEMETRY_EXPORT OtlpFileLogRecordExporterOptions : public OtlpFileClientOptions +{ + OtlpFileLogRecordExporterOptions(); + ~OtlpFileLogRecordExporterOptions(); +}; + +} // namespace otlp +} // namespace exporter +OPENTELEMETRY_END_NAMESPACE diff --git a/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_file_metric_exporter.h b/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_file_metric_exporter.h new file mode 100644 index 0000000000..165c7de99a --- /dev/null +++ b/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_file_metric_exporter.h @@ -0,0 +1,73 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include "opentelemetry/sdk/metrics/push_metric_exporter.h" + +#include "opentelemetry/exporters/otlp/otlp_file_client.h" + +#include "opentelemetry/exporters/otlp/otlp_file_metric_exporter_options.h" + +#include +#include +#include +#include + +OPENTELEMETRY_BEGIN_NAMESPACE +namespace exporter +{ +namespace otlp +{ +/** + * The OTLP exporter exports metrics data in OpenTelemetry Protocol (OTLP) format to file. + */ +class OtlpFileMetricExporter final : public opentelemetry::sdk::metrics::PushMetricExporter +{ +public: + /** + * Create an OtlpFileMetricExporter with default exporter options. + */ + OtlpFileMetricExporter(); + + /** + * Create an OtlpFileMetricExporter with user specified options. + * @param options An object containing the user's configuration options. + */ + OtlpFileMetricExporter(const OtlpFileMetricExporterOptions &options); + + /** + * Get the AggregationTemporality for exporter + * + * @return AggregationTemporality + */ + sdk::metrics::AggregationTemporality GetAggregationTemporality( + sdk::metrics::InstrumentType instrument_type) const noexcept override; + + opentelemetry::sdk::common::ExportResult Export( + const opentelemetry::sdk::metrics::ResourceMetrics &data) noexcept override; + + /** + * Force flush the exporter. + */ + bool ForceFlush( + std::chrono::microseconds timeout = (std::chrono::microseconds::max)()) noexcept override; + + bool Shutdown( + std::chrono::microseconds timeout = (std::chrono::microseconds::max)()) noexcept override; + +private: + friend class OtlpFileMetricExporterTestPeer; + + // Configuration options for the exporter + const OtlpFileMetricExporterOptions options_; + + // Aggregation Temporality Selector + const sdk::metrics::AggregationTemporalitySelector aggregation_temporality_selector_; + + // Object that stores the file context. + std::unique_ptr file_client_; +}; +} // namespace otlp +} // namespace exporter +OPENTELEMETRY_END_NAMESPACE diff --git a/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_file_metric_exporter_factory.h b/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_file_metric_exporter_factory.h new file mode 100644 index 0000000000..483e4aa5aa --- /dev/null +++ b/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_file_metric_exporter_factory.h @@ -0,0 +1,37 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include + +#include "opentelemetry/exporters/otlp/otlp_file_metric_exporter_options.h" +#include "opentelemetry/sdk/metrics/push_metric_exporter.h" + +OPENTELEMETRY_BEGIN_NAMESPACE +namespace exporter +{ +namespace otlp +{ + +/** + * Factory class for OtlpFileExporter. + */ +class OPENTELEMETRY_EXPORT OtlpFileMetricExporterFactory +{ +public: + /** + * Create an OtlpFileExporter using all default options. + */ + static std::unique_ptr Create(); + + /** + * Create an OtlpFileExporter using the given options. + */ + static std::unique_ptr Create( + const OtlpFileMetricExporterOptions &options); +}; + +} // namespace otlp +} // namespace exporter +OPENTELEMETRY_END_NAMESPACE diff --git a/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_file_metric_exporter_options.h b/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_file_metric_exporter_options.h new file mode 100644 index 0000000000..922c4bf84c --- /dev/null +++ b/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_file_metric_exporter_options.h @@ -0,0 +1,33 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include "opentelemetry/version.h" + +#include "opentelemetry/exporters/otlp/otlp_file_client_options.h" +#include "opentelemetry/exporters/otlp/otlp_preferred_temporality.h" + +OPENTELEMETRY_BEGIN_NAMESPACE +namespace exporter +{ +namespace otlp +{ + +/** + * Struct to hold OTLP File metrics exporter options. + * + * See + * https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/protocol/file-exporter.md + */ +struct OPENTELEMETRY_EXPORT OtlpFileMetricExporterOptions : public OtlpFileClientOptions +{ + OtlpFileMetricExporterOptions(); + ~OtlpFileMetricExporterOptions(); + + PreferredAggregationTemporality aggregation_temporality; +}; + +} // namespace otlp +} // namespace exporter +OPENTELEMETRY_END_NAMESPACE diff --git a/exporters/otlp/src/otlp_file_client.cc b/exporters/otlp/src/otlp_file_client.cc new file mode 100644 index 0000000000..2549c369d1 --- /dev/null +++ b/exporters/otlp/src/otlp_file_client.cc @@ -0,0 +1,1616 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +#include "opentelemetry/exporters/otlp/otlp_file_client.h" + +#if defined(HAVE_GSL) +# include +#else +# include +#endif + +// clang-format off +#include "opentelemetry/exporters/otlp/protobuf_include_prefix.h" +// clang-format on + +#include "google/protobuf/message.h" +#include "google/protobuf/reflection.h" +#include "google/protobuf/stubs/common.h" +#include "nlohmann/json.hpp" + +// clang-format off +#include "opentelemetry/exporters/otlp/protobuf_include_suffix.h" +// clang-format on + +#include "opentelemetry/nostd/string_view.h" +#include "opentelemetry/nostd/variant.h" +#include "opentelemetry/sdk/common/base64.h" +#include "opentelemetry/sdk/common/global_log_handler.h" +#include "opentelemetry/sdk_config.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if !defined(__CYGWIN__) && defined(_WIN32) +# ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +# endif +# ifndef NOMINMAX +# define NOMINMAX +# endif + +# include +# include +# include + +# ifdef UNICODE +# include +# define VC_TEXT(x) A2W(x) +# else +# define VC_TEXT(x) x +# endif + +# define FS_ACCESS(x) _access(x, 0) +# define SAFE_STRTOK_S(...) strtok_s(__VA_ARGS__) +# define FS_MKDIR(path, mode) _mkdir(path) + +#else + +# include +# include +# include +# include +# include +# include + +# define FS_ACCESS(x) access(x, F_OK) +# define SAFE_STRTOK_S(...) strtok_r(__VA_ARGS__) +# define FS_MKDIR(path, mode) ::mkdir(path, mode) + +# if defined(__ANDROID__) +# define FS_DISABLE_LINK 1 +# elif defined(__APPLE__) +# if __dest_os != __mac_os_x +# define FS_DISABLE_LINK 1 +# endif +# endif + +#endif + +#ifdef GetMessage +# undef GetMessage +#endif + +#ifdef _MSC_VER +# define strcasecmp _stricmp +#endif + +#if (defined(_MSC_VER) && _MSC_VER >= 1600) || \ + (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) || defined(__STDC_LIB_EXT1__) +# ifdef _MSC_VER +# define OTLP_FILE_SNPRINTF(buffer, bufsz, ...) \ + sprintf_s(buffer, static_cast(bufsz), __VA_ARGS__) +# else +# define OTLP_FILE_SNPRINTF(buffer, bufsz, fmt, args...) \ + snprintf_s(buffer, static_cast(bufsz), fmt, ##args) +# endif +#else +# define OTLP_FILE_SNPRINTF(buffer, bufsz, fmt, args...) \ + snprintf(buffer, static_cast(bufsz), fmt, ##args) +#endif + +OPENTELEMETRY_BEGIN_NAMESPACE +namespace exporter +{ +namespace otlp +{ + +namespace +{ +static std::tm GetLocalTime() +{ + std::time_t now = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()); +#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) || defined(__STDC_LIB_EXT1__) + std::tm ret; + localtime_s(&now, &ret); +#elif defined(_MSC_VER) && _MSC_VER >= 1300 + std::tm ret; + localtime_s(&ret, &now); +#elif defined(_XOPEN_SOURCE) || defined(_BSD_SOURCE) || defined(_SVID_SOURCE) || \ + defined(_POSIX_SOURCE) + std::tm ret; + localtime_r(&now, &ret); +#else + std::tm ret = *localtime(&now); +#endif + return ret; +} + +static std::size_t FormatPath(char *buff, + size_t bufz, + opentelemetry::nostd::string_view fmt, + std::size_t rotate_index) +{ + if (nullptr == buff || 0 == bufz) + { + return 0; + } + + if (fmt.empty()) + { + buff[0] = '\0'; + return 0; + } + + bool need_parse = false; + bool running = true; + std::size_t ret = 0; + std::tm tm_obj_cache; + std::tm *tm_obj_ptr = nullptr; + +#define LOG_FMT_FN_TM_MEM(VAR, EXPRESS) \ + \ + int VAR; \ + \ + if (nullptr == tm_obj_ptr) \ + { \ + tm_obj_cache = GetLocalTime(); \ + tm_obj_ptr = &tm_obj_cache; \ + VAR = tm_obj_ptr->EXPRESS; \ + } \ + else \ + { \ + VAR = tm_obj_ptr->EXPRESS; \ + } + + for (size_t i = 0; i < fmt.size() && ret < bufz && running; ++i) + { + if (!need_parse) + { + if ('%' == fmt[i]) + { + need_parse = true; + } + else + { + buff[ret++] = fmt[i]; + } + continue; + } + + need_parse = false; + switch (fmt[i]) + { + // =================== datetime =================== + case 'Y': { + if (bufz - ret < 4) + { + running = false; + } + else + { + LOG_FMT_FN_TM_MEM(year, tm_year + 1900); + buff[ret++] = static_cast(year / 1000 + '0'); + buff[ret++] = static_cast((year / 100) % 10 + '0'); + buff[ret++] = static_cast((year / 10) % 10 + '0'); + buff[ret++] = static_cast(year % 10 + '0'); + } + break; + } + case 'y': { + if (bufz - ret < 2) + { + running = false; + } + else + { + LOG_FMT_FN_TM_MEM(year, tm_year + 1900); + buff[ret++] = static_cast((year / 10) % 10 + '0'); + buff[ret++] = static_cast(year % 10 + '0'); + } + break; + } + case 'm': { + if (bufz - ret < 2) + { + running = false; + } + else + { + LOG_FMT_FN_TM_MEM(mon, tm_mon + 1); + buff[ret++] = static_cast(mon / 10 + '0'); + buff[ret++] = static_cast(mon % 10 + '0'); + } + break; + } + case 'j': { + if (bufz - ret < 3) + { + running = false; + } + else + { + LOG_FMT_FN_TM_MEM(yday, tm_yday); + buff[ret++] = static_cast(yday / 100 + '0'); + buff[ret++] = static_cast((yday / 10) % 10 + '0'); + buff[ret++] = static_cast(yday % 10 + '0'); + } + break; + } + case 'd': { + if (bufz - ret < 2) + { + running = false; + } + else + { + LOG_FMT_FN_TM_MEM(mday, tm_mday); + buff[ret++] = static_cast(mday / 10 + '0'); + buff[ret++] = static_cast(mday % 10 + '0'); + } + break; + } + case 'w': { + LOG_FMT_FN_TM_MEM(wday, tm_wday); + buff[ret++] = static_cast(wday + '0'); + break; + } + case 'H': { + if (bufz - ret < 2) + { + running = false; + } + else + { + LOG_FMT_FN_TM_MEM(hour, tm_hour); + buff[ret++] = static_cast(hour / 10 + '0'); + buff[ret++] = static_cast(hour % 10 + '0'); + } + break; + } + case 'I': { + if (bufz - ret < 2) + { + running = false; + } + else + { + LOG_FMT_FN_TM_MEM(hour, tm_hour % 12 + 1); + buff[ret++] = static_cast(hour / 10 + '0'); + buff[ret++] = static_cast(hour % 10 + '0'); + } + break; + } + case 'M': { + if (bufz - ret < 2) + { + running = false; + } + else + { + LOG_FMT_FN_TM_MEM(minite, tm_min); + buff[ret++] = static_cast(minite / 10 + '0'); + buff[ret++] = static_cast(minite % 10 + '0'); + } + break; + } + case 'S': { + if (bufz - ret < 2) + { + running = false; + } + else + { + LOG_FMT_FN_TM_MEM(sec, tm_sec); + buff[ret++] = static_cast(sec / 10 + '0'); + buff[ret++] = static_cast(sec % 10 + '0'); + } + break; + } + case 'F': { + if (bufz - ret < 10) + { + running = false; + } + else + { + LOG_FMT_FN_TM_MEM(year, tm_year + 1900); + LOG_FMT_FN_TM_MEM(mon, tm_mon + 1); + LOG_FMT_FN_TM_MEM(mday, tm_mday); + buff[ret++] = static_cast(year / 1000 + '0'); + buff[ret++] = static_cast((year / 100) % 10 + '0'); + buff[ret++] = static_cast((year / 10) % 10 + '0'); + buff[ret++] = static_cast(year % 10 + '0'); + buff[ret++] = '-'; + buff[ret++] = static_cast(mon / 10 + '0'); + buff[ret++] = static_cast(mon % 10 + '0'); + buff[ret++] = '-'; + buff[ret++] = static_cast(mday / 10 + '0'); + buff[ret++] = static_cast(mday % 10 + '0'); + } + break; + } + case 'T': { + if (bufz - ret < 8) + { + running = false; + } + else + { + LOG_FMT_FN_TM_MEM(hour, tm_hour); + LOG_FMT_FN_TM_MEM(minite, tm_min); + LOG_FMT_FN_TM_MEM(sec, tm_sec); + buff[ret++] = static_cast(hour / 10 + '0'); + buff[ret++] = static_cast(hour % 10 + '0'); + buff[ret++] = ':'; + buff[ret++] = static_cast(minite / 10 + '0'); + buff[ret++] = static_cast(minite % 10 + '0'); + buff[ret++] = ':'; + buff[ret++] = static_cast(sec / 10 + '0'); + buff[ret++] = static_cast(sec % 10 + '0'); + } + break; + } + case 'R': { + if (bufz - ret < 5) + { + running = false; + } + else + { + LOG_FMT_FN_TM_MEM(hour, tm_hour); + LOG_FMT_FN_TM_MEM(minite, tm_min); + buff[ret++] = static_cast(hour / 10 + '0'); + buff[ret++] = static_cast(hour % 10 + '0'); + buff[ret++] = ':'; + buff[ret++] = static_cast(minite / 10 + '0'); + buff[ret++] = static_cast(minite % 10 + '0'); + } + break; + } + + // =================== rotate index =================== + case 'n': + case 'N': { + std::size_t value = fmt[i] == 'n' ? rotate_index + 1 : rotate_index; +#if (defined(_MSC_VER) && _MSC_VER >= 1600) || \ + (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) || defined(__STDC_LIB_EXT1__) +# ifdef _MSC_VER + auto res = + sprintf_s(&buff[ret], bufz - ret, "%llu", static_cast(value)); +# else + auto res = + snprintf_s(&buff[ret], bufz - ret, "%llu", static_cast(value)); +# endif +#else + auto res = snprintf(&buff[ret], bufz - ret, "%llu", static_cast(value)); +#endif + if (res < 0) + { + running = false; + } + else + { + ret += static_cast(res); + } + break; + } + + // =================== unknown =================== + default: { + buff[ret++] = fmt[i]; + break; + } + } + } + +#undef LOG_FMT_FN_TM_MEM + + if (ret < bufz) + { + buff[ret] = '\0'; + } + else + { + buff[bufz - 1] = '\0'; + } + return ret; +} + +class OPENTELEMETRY_LOCAL_SYMBOL FileSystemUtil +{ +public: + // When LongPathsEnabled on Windows, it allow 32767 characters in a absolute path.But it still + // only allow 260 characters in a relative path. See + // https://docs.microsoft.com/en-us/windows/win32/fileio/maximum-file-path-limitation + + static constexpr const std::size_t kMaxPathSize = +#if defined(MAX_PATH) + MAX_PATH; +#elif defined(_MAX_PATH) + _MAX_PATH; +#elif defined(PATH_MAX) + PATH_MAX; +#else + 260; +#endif + + static constexpr const char kDirectorySeparator = +#if !defined(__CYGWIN__) && defined(_WIN32) + '\\'; +#else + '/'; +#endif + + static std::size_t GetFileSize(const char *file_path) + { + std::fstream file; + file.open(file_path, std::ios::binary | std::ios::in); + if (!file.is_open()) + { + return 0; + } + + file.seekg(std::ios::end); + auto size = file.tellg(); + file.close(); + + if (size > 0) + { + return static_cast(size); + } + else + { + return 0; + } + } + + static std::string DirName(opentelemetry::nostd::string_view file_path, int depth = 1) + { + if (file_path.empty()) + { + return ""; + } + + std::size_t sz = file_path.size() - 1; + + while (sz > 0 && ('/' == file_path[sz] || '\\' == file_path[sz])) + { + --sz; + } + + while (sz > 0 && depth > 0) + { + if ('/' == file_path[sz] || '\\' == file_path[sz]) + { + // DirName(a//\b) -> a + while (sz > 0 && ('/' == file_path[sz] || '\\' == file_path[sz])) + { + --sz; + } + + --depth; + if (depth <= 0) + { + ++sz; + break; + } + } + else + { + --sz; + } + } + + return static_cast(file_path.substr(0, sz)); + } + + static bool IsExist(const char *file_path) { return 0 == FS_ACCESS(file_path); } + + static std::vector SplitPath(opentelemetry::nostd::string_view path, + bool normalize = false) + { + std::vector out; + + std::string path_buffer = static_cast(path); + + char *saveptr = nullptr; + char *token = SAFE_STRTOK_S(&path_buffer[0], "\\/", &saveptr); + while (nullptr != token) + { + if (0 != strlen(token)) + { + if (normalize) + { + // Normalize path + if (0 == strcmp("..", token)) + { + if (!out.empty() && out.back() != "..") + { + out.pop_back(); + } + else + { + out.push_back(token); + } + } + else if (0 != strcmp(".", token)) + { + out.push_back(token); + } + } + else + { + out.push_back(token); + } + } + token = SAFE_STRTOK_S(nullptr, "\\/", &saveptr); + } + + return out; + } + + static bool MkDir(const char *dir_path, bool recursion, OPENTELEMETRY_MAYBE_UNUSED int mode) + { +#if !(!defined(__CYGWIN__) && defined(_WIN32)) + if (0 == mode) + { + mode = S_IRWXU | S_IRWXG | S_IRWXO; + } +#endif + if (!recursion) + { + return 0 == FS_MKDIR(dir_path, static_cast(mode)); + } + + std::vector path_segs = SplitPath(dir_path, true); + + if (path_segs.empty()) + { + return false; + } + + std::string current_path; + if (nullptr != dir_path && ('/' == *dir_path || '\\' == *dir_path)) + { + current_path.reserve(strlen(dir_path) + 4); + current_path = *dir_path; + + // NFS Supporting + char next_char = *(dir_path + 1); + if ('/' == next_char || '\\' == next_char) + { + current_path += next_char; + } + } + + for (size_t i = 0; i < path_segs.size(); ++i) + { + current_path += path_segs[i]; + + if (false == IsExist(current_path.c_str())) + { + if (0 != FS_MKDIR(current_path.c_str(), static_cast(mode))) + { + return false; + } + } + + current_path += kDirectorySeparator; + } + + return true; + } + +#if !defined(UTIL_FS_DISABLE_LINK) + enum class LinkOption : int32_t + { + kDefault = 0x00, // hard link for default + kSymbolicLink = 0x01, // or soft link + kDirectoryLink = 0x02, // it's used only for windows + kForceRewrite = 0x04, // delete the old file if it exists + }; + + /** + * @brief Create link + * @param oldpath source path + * @param newpath target path + * @param options options + * @return 0 for success, or error code + */ + static int Link(const char *oldpath, + const char *newpath, + int32_t options = static_cast(LinkOption::kDefault)) + { + if ((options & static_cast(LinkOption::kForceRewrite)) && IsExist(newpath)) + { + remove(newpath); + } + +# if !defined(__CYGWIN__) && defined(_WIN32) +# if defined(UNICODE) + USES_CONVERSION; +# endif + + if (options & static_cast(LinkOption::kSymbolicLink)) + { + DWORD dwFlags = 0; + if (options & static_cast(LinkOption::kDirectoryLink)) + { + dwFlags |= SYMBOLIC_LINK_FLAG_DIRECTORY; +# if defined(SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE) + dwFlags |= SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE; +# endif + } + + if (CreateSymbolicLink(VC_TEXT(newpath), VC_TEXT(oldpath), dwFlags)) + { + return 0; + } + + return static_cast(GetLastError()); + } + else + { + if (CreateHardLink(VC_TEXT(newpath), VC_TEXT(oldpath), nullptr)) + { + return 0; + } + + return static_cast(GetLastError()); + } + +# else + int opts = 0; + if (options & static_cast(LinkOption::kSymbolicLink)) + { + opts = AT_SYMLINK_FOLLOW; + } + + int res = ::linkat(AT_FDCWD, oldpath, AT_FDCWD, newpath, opts); + if (0 == res) + { + return 0; + } + + return errno; + +# endif + } +#endif +}; + +static inline char HexEncode(unsigned char byte) +{ +#if defined(HAVE_GSL) + Expects(byte <= 16); +#else + assert(byte <= 16); +#endif + if (byte >= 10) + { + return byte - 10 + 'a'; + } + else + { + return byte + '0'; + } +} + +static std::string HexEncode(const std::string &bytes) +{ + std::string ret; + ret.reserve(bytes.size() * 2); + for (std::string::size_type i = 0; i < bytes.size(); ++i) + { + unsigned char byte = static_cast(bytes[i]); + ret.push_back(HexEncode(byte >> 4)); + ret.push_back(HexEncode(byte & 0x0f)); + } + return ret; +} + +static std::string BytesMapping(const std::string &bytes, + const google::protobuf::FieldDescriptor *field_descriptor) +{ + if (field_descriptor->lowercase_name() == "trace_id" || + field_descriptor->lowercase_name() == "span_id" || + field_descriptor->lowercase_name() == "parent_span_id") + { + return HexEncode(bytes); + } + else + { + return opentelemetry::sdk::common::Base64Escape(bytes); + } +} + +static void ConvertGenericFieldToJson(nlohmann::json &value, + const google::protobuf::Message &message, + const google::protobuf::FieldDescriptor *field_descriptor); + +static void ConvertListFieldToJson(nlohmann::json &value, + const google::protobuf::Message &message, + const google::protobuf::FieldDescriptor *field_descriptor); + +static void ConvertGenericMessageToJson(nlohmann::json &value, + const google::protobuf::Message &message) +{ + std::vector fields_with_data; + message.GetReflection()->ListFields(message, &fields_with_data); + for (std::size_t i = 0; i < fields_with_data.size(); ++i) + { + const google::protobuf::FieldDescriptor *field_descriptor = fields_with_data[i]; + nlohmann::json &child_value = value[field_descriptor->camelcase_name()]; + if (field_descriptor->is_repeated()) + { + ConvertListFieldToJson(child_value, message, field_descriptor); + } + else + { + ConvertGenericFieldToJson(child_value, message, field_descriptor); + } + } +} + +void ConvertGenericFieldToJson(nlohmann::json &value, + const google::protobuf::Message &message, + const google::protobuf::FieldDescriptor *field_descriptor) +{ + switch (field_descriptor->cpp_type()) + { + case google::protobuf::FieldDescriptor::CPPTYPE_INT32: { + value = message.GetReflection()->GetInt32(message, field_descriptor); + break; + } + case google::protobuf::FieldDescriptor::CPPTYPE_INT64: { + // According to Protobuf specs 64-bit integer numbers in JSON-encoded payloads are encoded as + // decimal strings, and either numbers or strings are accepted when decoding. + value = std::to_string(message.GetReflection()->GetInt64(message, field_descriptor)); + break; + } + case google::protobuf::FieldDescriptor::CPPTYPE_UINT32: { + value = message.GetReflection()->GetUInt32(message, field_descriptor); + break; + } + case google::protobuf::FieldDescriptor::CPPTYPE_UINT64: { + // According to Protobuf specs 64-bit integer numbers in JSON-encoded payloads are encoded as + // decimal strings, and either numbers or strings are accepted when decoding. + value = std::to_string(message.GetReflection()->GetUInt64(message, field_descriptor)); + break; + } + case google::protobuf::FieldDescriptor::CPPTYPE_STRING: { + std::string empty; + if (field_descriptor->type() == google::protobuf::FieldDescriptor::TYPE_BYTES) + { + value = BytesMapping( + message.GetReflection()->GetStringReference(message, field_descriptor, &empty), + field_descriptor); + } + else + { + value = message.GetReflection()->GetStringReference(message, field_descriptor, &empty); + } + break; + } + case google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE: { + ConvertGenericMessageToJson( + value, message.GetReflection()->GetMessage(message, field_descriptor, nullptr)); + break; + } + case google::protobuf::FieldDescriptor::CPPTYPE_DOUBLE: { + value = message.GetReflection()->GetDouble(message, field_descriptor); + break; + } + case google::protobuf::FieldDescriptor::CPPTYPE_FLOAT: { + value = message.GetReflection()->GetFloat(message, field_descriptor); + break; + } + case google::protobuf::FieldDescriptor::CPPTYPE_BOOL: { + value = message.GetReflection()->GetBool(message, field_descriptor); + break; + } + case google::protobuf::FieldDescriptor::CPPTYPE_ENUM: { + value = message.GetReflection()->GetEnumValue(message, field_descriptor); + break; + } + default: { + break; + } + } +} + +void ConvertListFieldToJson(nlohmann::json &value, + const google::protobuf::Message &message, + const google::protobuf::FieldDescriptor *field_descriptor) +{ + auto field_size = message.GetReflection()->FieldSize(message, field_descriptor); + + switch (field_descriptor->cpp_type()) + { + case google::protobuf::FieldDescriptor::CPPTYPE_INT32: { + for (int i = 0; i < field_size; ++i) + { + value.push_back(message.GetReflection()->GetRepeatedInt32(message, field_descriptor, i)); + } + + break; + } + case google::protobuf::FieldDescriptor::CPPTYPE_INT64: { + for (int i = 0; i < field_size; ++i) + { + // According to Protobuf specs 64-bit integer numbers in JSON-encoded payloads are encoded + // as decimal strings, and either numbers or strings are accepted when decoding. + value.push_back(std::to_string( + message.GetReflection()->GetRepeatedInt64(message, field_descriptor, i))); + } + + break; + } + case google::protobuf::FieldDescriptor::CPPTYPE_UINT32: { + for (int i = 0; i < field_size; ++i) + { + value.push_back(message.GetReflection()->GetRepeatedUInt32(message, field_descriptor, i)); + } + + break; + } + case google::protobuf::FieldDescriptor::CPPTYPE_UINT64: { + for (int i = 0; i < field_size; ++i) + { + // According to Protobuf specs 64-bit integer numbers in JSON-encoded payloads are encoded + // as decimal strings, and either numbers or strings are accepted when decoding. + value.push_back(std::to_string( + message.GetReflection()->GetRepeatedUInt64(message, field_descriptor, i))); + } + + break; + } + case google::protobuf::FieldDescriptor::CPPTYPE_STRING: { + std::string empty; + if (field_descriptor->type() == google::protobuf::FieldDescriptor::TYPE_BYTES) + { + for (int i = 0; i < field_size; ++i) + { + value.push_back(BytesMapping(message.GetReflection()->GetRepeatedStringReference( + message, field_descriptor, i, &empty), + field_descriptor)); + } + } + else + { + for (int i = 0; i < field_size; ++i) + { + value.push_back(message.GetReflection()->GetRepeatedStringReference( + message, field_descriptor, i, &empty)); + } + } + break; + } + case google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE: { + for (int i = 0; i < field_size; ++i) + { + nlohmann::json sub_value; + ConvertGenericMessageToJson( + sub_value, message.GetReflection()->GetRepeatedMessage(message, field_descriptor, i)); + value.push_back(std::move(sub_value)); + } + + break; + } + case google::protobuf::FieldDescriptor::CPPTYPE_DOUBLE: { + for (int i = 0; i < field_size; ++i) + { + value.push_back(message.GetReflection()->GetRepeatedDouble(message, field_descriptor, i)); + } + + break; + } + case google::protobuf::FieldDescriptor::CPPTYPE_FLOAT: { + for (int i = 0; i < field_size; ++i) + { + value.push_back(message.GetReflection()->GetRepeatedFloat(message, field_descriptor, i)); + } + + break; + } + case google::protobuf::FieldDescriptor::CPPTYPE_BOOL: { + for (int i = 0; i < field_size; ++i) + { + value.push_back(message.GetReflection()->GetRepeatedBool(message, field_descriptor, i)); + } + + break; + } + case google::protobuf::FieldDescriptor::CPPTYPE_ENUM: { + for (int i = 0; i < field_size; ++i) + { + value.push_back( + message.GetReflection()->GetRepeatedEnumValue(message, field_descriptor, i)); + } + break; + } + default: { + break; + } + } +} + +} // namespace + +class OPENTELEMETRY_LOCAL_SYMBOL OtlpFileSystemBackend : public OtlpFileAppender +{ +public: + explicit OtlpFileSystemBackend(const OtlpFileClientFileSystemOptions &options) + : options_(options), is_initialized_{false}, check_file_path_interval_{0} + { + file_ = std::make_shared(); + file_->is_shutdown.store(false); + file_->rotate_index = 0; + file_->written_size = 0; + file_->left_flush_record_count = 0; + file_->last_checkpoint = 0; + file_->record_count.store(0); + file_->flushed_record_count.store(0); + } + + ~OtlpFileSystemBackend() override + { + if (file_) + { + file_->background_thread_waker_cv.notify_all(); + std::unique_ptr background_flush_thread; + { + std::lock_guard lock_guard{file_->background_thread_lock}; + file_->background_flush_thread.swap(background_flush_thread); + } + if (background_flush_thread && background_flush_thread->joinable()) + { + background_flush_thread->join(); + } + } + } + + void Export(nostd::string_view data, std::size_t record_count) override + { + if (!is_initialized_.load(std::memory_order_acquire)) + { + Initialize(); + } + + if (file_->written_size > 0 && file_->written_size + data.size() > options_.file_size) + { + RotateLog(); + } + CheckUpdate(); + + std::shared_ptr out = OpenLogFile(true); + if (!out) + { + return; + } + + out->write(data.data(), data.size()); + out->write("\n", 1); + + { + std::lock_guard lock_guard{file_->file_lock}; + + file_->record_count += record_count; + + // Pipe file size always returns 0, we ignore the size limit of it. + auto written_size = out->tellp(); + if (written_size >= 0) + { + file_->written_size = static_cast(written_size); + } + + if (options_.flush_count > 0) + { + if (file_->left_flush_record_count <= record_count) + { + file_->left_flush_record_count = options_.flush_count; + + out->flush(); + + file_->flushed_record_count.store(file_->record_count.load(std::memory_order_acquire), + std::memory_order_release); + } + else + { + file_->left_flush_record_count -= record_count; + } + } + } + + // Maybe need spawn a background thread to flush ostream + SpawnBackgroundWorkThread(); + } + + bool ForceFlush(std::chrono::microseconds timeout) noexcept override + { + std::chrono::microseconds wait_interval = timeout / 256; + if (wait_interval <= std::chrono::microseconds{0}) + { + wait_interval = timeout; + } + // If set timeout to a large value, we limit the check interval to 256ms. + // So we will not wait too long to shutdown the client when missing the finish notification. + if (wait_interval > std::chrono::microseconds{256000}) + { + wait_interval = std::chrono::microseconds{256000}; + } + + std::size_t current_wait_for_flush_count = file_->record_count.load(std::memory_order_acquire); + + while (timeout >= std::chrono::microseconds::zero()) + { + // No more data to flush + { + if (file_->flushed_record_count.load(std::memory_order_acquire) >= + current_wait_for_flush_count) + { + break; + } + } + + std::chrono::system_clock::time_point begin_time = std::chrono::system_clock::now(); + // Notify background thread to flush immediately + { + std::lock_guard lock_guard{file_->background_thread_lock}; + if (!file_->background_flush_thread) + { + break; + } + file_->background_thread_waker_cv.notify_all(); + } + + // Wait result + { + std::unique_lock lk(file_->background_thread_waiter_lock); + file_->background_thread_waiter_cv.wait_for(lk, wait_interval); + } + + std::chrono::system_clock::time_point end_time = std::chrono::system_clock::now(); + if (end_time - begin_time > std::chrono::microseconds{1}) + { + timeout -= std::chrono::duration_cast(end_time - begin_time); + } + else + { + timeout -= std::chrono::microseconds{1}; + } + } + + return timeout >= std::chrono::microseconds::zero(); + } + + bool Shutdown(std::chrono::microseconds timeout) noexcept override + { + file_->is_shutdown.store(true, std::memory_order_release); + + bool result = ForceFlush(timeout); + return result; + } + +private: + void Initialize() + { + if (is_initialized_.load(std::memory_order_acquire)) + { + return; + } + + // Double check + std::string file_pattern; + { + std::lock_guard lock_guard{file_->file_lock}; + if (is_initialized_.load(std::memory_order_acquire)) + { + return; + } + is_initialized_.store(true, std::memory_order_release); + + file_->rotate_index = 0; + ResetLogFile(); + + char file_path[FileSystemUtil::kMaxPathSize]; + for (std::size_t i = 0; options_.file_size > 0 && i < options_.rotate_size; ++i) + { + FormatPath(file_path, sizeof(file_path), options_.file_pattern, i); + std::size_t existed_file_size = FileSystemUtil::GetFileSize(file_path); + + // File size is also zero when it's not existed. + if (existed_file_size < options_.file_size) + { + file_->rotate_index = i; + break; + } + } + + file_pattern = options_.file_pattern; + } + + // Reset the interval to check + static std::time_t check_interval[128] = {0}; + // Some timezone contains half a hour, we use 1800s for the max check interval. + if (check_interval[static_cast('S')] == 0) + { + check_interval[static_cast('R')] = 60; + check_interval[static_cast('T')] = 1; + check_interval[static_cast('F')] = 1800; + check_interval[static_cast('S')] = 1; + check_interval[static_cast('M')] = 60; + check_interval[static_cast('I')] = 1800; + check_interval[static_cast('H')] = 1800; + check_interval[static_cast('w')] = 1800; + check_interval[static_cast('d')] = 1800; + check_interval[static_cast('j')] = 1800; + check_interval[static_cast('m')] = 1800; + check_interval[static_cast('y')] = 1800; + check_interval[static_cast('Y')] = 1800; + } + + { + check_file_path_interval_ = 0; + for (std::size_t i = 0; i + 1 < file_pattern.size(); ++i) + { + if (file_pattern[i] == '%') + { + int checked = static_cast(file_pattern[i + 1]); + if (checked > 0 && checked < 128 && check_interval[checked] > 0) + { + if (0 == check_file_path_interval_ || + check_interval[checked] < check_file_path_interval_) + { + check_file_path_interval_ = check_interval[checked]; + } + } + } + } + } + + OpenLogFile(false); + } + + std::shared_ptr OpenLogFile(bool destroy_content) + { + std::lock_guard lock_guard{file_->file_lock}; + + if (file_->current_file && file_->current_file->good()) + { + return file_->current_file; + } + + ResetLogFile(); + + char file_path[FileSystemUtil::kMaxPathSize + 1]; + std::size_t file_path_size = FormatPath(file_path, FileSystemUtil::kMaxPathSize, + options_.file_pattern, file_->rotate_index); + if (file_path_size <= 0) + { + OTEL_INTERNAL_LOG_ERROR("[OTLP FILE Client] Generate file path from pattern " + << options_.file_pattern << " failed"); + return std::shared_ptr(); + } + file_path[file_path_size] = 0; + + std::shared_ptr of = std::make_shared(); + + std::string directory_name = FileSystemUtil::DirName(file_path); + if (!directory_name.empty()) + { + int error_code = 0; + if (!FileSystemUtil::IsExist(directory_name.c_str())) + { + FileSystemUtil::MkDir(directory_name.c_str(), true, 0); + error_code = errno; + } + + if (!FileSystemUtil::IsExist(directory_name.c_str())) + { +#if !defined(__CYGWIN__) && defined(_WIN32) + char error_message[256] = {0}; + strerror_s(error_message, sizeof(error_message) - 1, error_code); +#else + char error_message[256] = {0}; + strerror_r(error_code, error_message, sizeof(error_message) - 1); +#endif + OTEL_INTERNAL_LOG_ERROR("[OTLP FILE Client] Create directory \"" + << directory_name << "\" failed.errno: " << error_code + << ", message: " << error_message); + } + } + + if (destroy_content && FileSystemUtil::IsExist(file_path)) + { + std::fstream trunc_file; + trunc_file.open(file_path, std::ios::binary | std::ios::out | std::ios::trunc); + if (!trunc_file.is_open()) + { + OTEL_INTERNAL_LOG_ERROR("[OTLP FILE Client] Open " + << static_cast(file_path) + << " failed with pattern: " << options_.file_pattern); + return std::shared_ptr(); + } + trunc_file.close(); + } + + of->open(file_path, std::ios::binary | std::ios::out | std::ios::app); + if (!of->is_open()) + { + std::string hint; + if (!directory_name.empty()) + { + hint = std::string(".The directory \"") + directory_name + + "\" may not exist or may not be writable."; + } + OTEL_INTERNAL_LOG_ERROR("[OTLP FILE Client] Open " + << static_cast(file_path) + << " failed with pattern: " << options_.file_pattern << hint); + return std::shared_ptr(); + } + + of->seekp(0, std::ios_base::end); + file_->written_size = static_cast(of->tellp()); + + file_->current_file = of; + file_->last_checkpoint = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()); + file_->file_path.assign(file_path, file_path_size); + + // Create hardlink for alias +#if !defined(FS_DISABLE_LINK) + if (!options_.alias_pattern.empty()) + { + char alias_file_path[FileSystemUtil::kMaxPathSize + 1]; + std::size_t file_path_len = FormatPath(alias_file_path, sizeof(alias_file_path) - 1, + options_.alias_pattern, file_->rotate_index); + if (file_path_len <= 0) + { + OTEL_INTERNAL_LOG_ERROR("[OTLP FILE Client] Generate alias file path from " + << options_.alias_pattern << " failed"); + return file_->current_file; + } + + if (file_path_len < sizeof(alias_file_path)) + { + alias_file_path[file_path_len] = 0; + } + + if (0 == strcasecmp(file_path, alias_file_path)) + { + return file_->current_file; + } + + int res = + FileSystemUtil::Link(file_->file_path.c_str(), alias_file_path, + static_cast(FileSystemUtil::LinkOption::kForceRewrite)); + if (res != 0) + { +# if !defined(__CYGWIN__) && defined(_WIN32) + // We can use FormatMessage to get error message.But it may be unicode and may not be + // printed correctly. See + // https://learn.microsoft.com/en-us/windows/win32/debug/retrieving-the-last-error-code for + // more details + OTEL_INTERNAL_LOG_ERROR("[OTLP FILE Client] Link " << file_->file_path << " to " + << alias_file_path + << " failed, errno: " << res); +# else + OTEL_INTERNAL_LOG_ERROR("[OTLP FILE Client] Link " + << file_->file_path << " to " << alias_file_path + << " failed, errno: " << res << ", message: " << strerror(res)); +# endif + return file_->current_file; + } + } +#endif + + return file_->current_file; + } + + void RotateLog() + { + std::lock_guard lock_guard{file_->file_lock}; + if (options_.rotate_size > 0) + { + file_->rotate_index = (file_->rotate_index + 1) % options_.rotate_size; + } + else + { + file_->rotate_index = 0; + } + ResetLogFile(); + } + + void CheckUpdate() + { + if (check_file_path_interval_ <= 0) + { + return; + } + + std::time_t current_checkpoint = + std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()); + if (current_checkpoint / check_file_path_interval_ == + file_->last_checkpoint / check_file_path_interval_) + { + return; + } + // Refresh checkpoint + file_->last_checkpoint = current_checkpoint; + + char file_path[FileSystemUtil::kMaxPathSize + 1]; + size_t file_path_len = + FormatPath(file_path, sizeof(file_path) - 1, options_.file_pattern, file_->rotate_index); + if (file_path_len <= 0) + { + return; + } + + std::string new_file_path; + std::string old_file_path; + new_file_path.assign(file_path, file_path_len); + + { + // Lock for a short time + std::lock_guard lock_guard{file_->file_lock}; + old_file_path = file_->file_path; + + if (new_file_path == old_file_path) + { + // Refresh checking time + return; + } + } + + std::string new_dir = FileSystemUtil::DirName(new_file_path); + std::string old_dir = FileSystemUtil::DirName(old_file_path); + + // Reset rotate index when directory changes + if (new_dir != old_dir) + { + file_->rotate_index = 0; + } + + ResetLogFile(); + } + + void ResetLogFile() + { + // ResetLogFile is called in lock, do not lock again + + file_->current_file.reset(); + file_->last_checkpoint = 0; + file_->written_size = 0; + } + + void SpawnBackgroundWorkThread() + { + if (options_.flush_interval <= std::chrono::microseconds{0}) + { + return; + } + + if (!file_) + { + return; + } + + std::lock_guard lock_guard_caller{file_->background_thread_lock}; + if (file_->background_flush_thread) + { + return; + } + + std::shared_ptr concurrency_file = file_; + std::chrono::microseconds flush_interval = options_.flush_interval; + file_->background_flush_thread.reset(new std::thread([concurrency_file, flush_interval]() { + std::chrono::system_clock::time_point last_free_job_timepoint = + std::chrono::system_clock::now(); + std::size_t last_record_count = 0; + + while (true) + { + std::chrono::system_clock::time_point now = std::chrono::system_clock::now(); + // Exit flush thread if there is not data to flush more than one minute. + if (now - last_free_job_timepoint > std::chrono::minutes{1}) + { + break; + } + + if (concurrency_file->is_shutdown.load(std::memory_order_acquire)) + { + break; + } + + { + std::unique_lock lk(concurrency_file->background_thread_waker_lock); + concurrency_file->background_thread_waker_cv.wait_for(lk, flush_interval); + } + + { + std::size_t current_record_count = + concurrency_file->record_count.load(std::memory_order_acquire); + std::lock_guard lock_guard{concurrency_file->file_lock}; + if (current_record_count != last_record_count) + { + last_record_count = current_record_count; + last_free_job_timepoint = std::chrono::system_clock::now(); + } + + if (concurrency_file->current_file) + { + concurrency_file->current_file->flush(); + } + + concurrency_file->flushed_record_count.store(current_record_count, + std::memory_order_release); + } + + concurrency_file->background_thread_waiter_cv.notify_all(); + } + + // Detach running thread because it will exit soon + std::unique_ptr background_flush_thread; + { + std::lock_guard lock_guard_inner{concurrency_file->background_thread_lock}; + background_flush_thread.swap(concurrency_file->background_flush_thread); + } + if (background_flush_thread && background_flush_thread->joinable()) + { + background_flush_thread->detach(); + } + })); + } + +private: + OtlpFileClientFileSystemOptions options_; + + struct FileStats + { + std::atomic is_shutdown; + std::size_t rotate_index; + std::size_t written_size; + std::size_t left_flush_record_count; + std::shared_ptr current_file; + std::mutex file_lock; + std::time_t last_checkpoint; + std::string file_path; + std::atomic record_count; + std::atomic flushed_record_count; + + std::unique_ptr background_flush_thread; + std::mutex background_thread_lock; + std::mutex background_thread_waker_lock; + std::condition_variable background_thread_waker_cv; + std::mutex background_thread_waiter_lock; + std::condition_variable background_thread_waiter_cv; + }; + std::shared_ptr file_; + + std::atomic is_initialized_; + std::time_t check_file_path_interval_; +}; + +class OPENTELEMETRY_LOCAL_SYMBOL OtlpFileOstreamBackend : public OtlpFileAppender +{ +public: + explicit OtlpFileOstreamBackend(const std::reference_wrapper &os) : os_(os) {} + + ~OtlpFileOstreamBackend() override {} + + void Export(nostd::string_view data, std::size_t /*record_count*/) override + { + os_.get().write(data.data(), data.size()); + os_.get().write("\n", 1); + } + + bool ForceFlush(std::chrono::microseconds /*timeout*/) noexcept override + { + os_.get().flush(); + + return true; + } + + bool Shutdown(std::chrono::microseconds timeout) noexcept override { return ForceFlush(timeout); } + +private: + std::reference_wrapper os_; +}; + +OtlpFileClient::OtlpFileClient(OtlpFileClientOptions &&options) + : is_shutdown_(false), options_(options) +{ + if (nostd::holds_alternative(options_.backend_options)) + { + backend_ = opentelemetry::nostd::shared_ptr(new OtlpFileSystemBackend( + nostd::get(options_.backend_options))); + } + else if (nostd::holds_alternative>(options_.backend_options)) + { + backend_ = opentelemetry::nostd::shared_ptr(new OtlpFileOstreamBackend( + nostd::get>(options_.backend_options))); + } + else if (nostd::holds_alternative>( + options_.backend_options)) + { + backend_ = + nostd::get>(options_.backend_options); + } +} + +OtlpFileClient::~OtlpFileClient() +{ + if (!IsShutdown()) + { + Shutdown(); + } +} + +// ----------------------------- File Client methods ------------------------------ +opentelemetry::sdk::common::ExportResult OtlpFileClient::Export( + const google::protobuf::Message &message, + std::size_t record_count) noexcept +{ + if (is_shutdown_) + { + return ::opentelemetry::sdk::common::ExportResult::kFailure; + } + + nlohmann::json json_request; + // Convert from proto into json object + ConvertGenericMessageToJson(json_request, message); + + std::string post_body_json = + json_request.dump(-1, ' ', false, nlohmann::detail::error_handler_t::replace); + if (options_.console_debug) + { + OTEL_INTERNAL_LOG_DEBUG("[OTLP FILE Client] Write body(Json)" << post_body_json); + } + + if (backend_) + { + backend_->Export(post_body_json, record_count); + return ::opentelemetry::sdk::common::ExportResult::kSuccess; + } + + return ::opentelemetry::sdk::common::ExportResult::kFailure; +} + +bool OtlpFileClient::ForceFlush(std::chrono::microseconds timeout) noexcept +{ + if (backend_) + { + return backend_->ForceFlush(timeout); + } + + return true; +} + +bool OtlpFileClient::Shutdown(std::chrono::microseconds timeout) noexcept +{ + is_shutdown_ = true; + + if (backend_) + { + return backend_->Shutdown(timeout); + } + + return true; +} + +bool OtlpFileClient::IsShutdown() const noexcept +{ + return is_shutdown_; +} + +} // namespace otlp +} // namespace exporter +OPENTELEMETRY_END_NAMESPACE diff --git a/exporters/otlp/src/otlp_file_exporter.cc b/exporters/otlp/src/otlp_file_exporter.cc new file mode 100644 index 0000000000..816151719a --- /dev/null +++ b/exporters/otlp/src/otlp_file_exporter.cc @@ -0,0 +1,94 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +#include "opentelemetry/exporters/otlp/otlp_file_exporter.h" +#include "opentelemetry/exporters/otlp/otlp_file_client.h" +#include "opentelemetry/exporters/otlp/otlp_recordable.h" +#include "opentelemetry/exporters/otlp/otlp_recordable_utils.h" + +#include "opentelemetry/exporters/otlp/protobuf_include_prefix.h" + +#include "google/protobuf/arena.h" +#include "opentelemetry/proto/collector/trace/v1/trace_service.pb.h" + +#include "opentelemetry/exporters/otlp/protobuf_include_suffix.h" + +#include "opentelemetry/sdk/common/global_log_handler.h" + +OPENTELEMETRY_BEGIN_NAMESPACE +namespace exporter +{ +namespace otlp +{ + +OtlpFileExporter::OtlpFileExporter() : OtlpFileExporter(OtlpFileExporterOptions()) {} + +OtlpFileExporter::OtlpFileExporter(const OtlpFileExporterOptions &options) + : options_(options), file_client_(new OtlpFileClient(OtlpFileClientOptions(options))) +{} + +// ----------------------------- Exporter methods ------------------------------ + +std::unique_ptr OtlpFileExporter::MakeRecordable() noexcept +{ + return std::unique_ptr( + new exporter::otlp::OtlpRecordable()); +} + +opentelemetry::sdk::common::ExportResult OtlpFileExporter::Export( + const nostd::span> &spans) noexcept +{ + if (file_client_->IsShutdown()) + { + std::size_t span_count = spans.size(); + OTEL_INTERNAL_LOG_ERROR("[OTLP TRACE FILE Exporter] ERROR: Export " + << span_count << " trace span(s) failed, exporter is shutdown"); + return opentelemetry::sdk::common::ExportResult::kFailure; + } + + if (spans.empty()) + { + return opentelemetry::sdk::common::ExportResult::kSuccess; + } + + google::protobuf::ArenaOptions arena_options; + // It's easy to allocate datas larger than 1024 when we populate basic resource and attributes + arena_options.initial_block_size = 1024; + // When in batch mode, it's easy to export a large number of spans at once, we can alloc a lager + // block to reduce memory fragments. + arena_options.max_block_size = 65536; + google::protobuf::Arena arena{arena_options}; + + proto::collector::trace::v1::ExportTraceServiceRequest *service_request = + google::protobuf::Arena::Create( + &arena); + OtlpRecordableUtils::PopulateRequest(spans, service_request); + std::size_t span_count = spans.size(); + opentelemetry::sdk::common::ExportResult result = + file_client_->Export(*service_request, span_count); + if (result != opentelemetry::sdk::common::ExportResult::kSuccess) + { + OTEL_INTERNAL_LOG_ERROR("[OTLP TRACE FILE Exporter] ERROR: Export " + << span_count << " trace span(s) error: " << static_cast(result)); + } + else + { + OTEL_INTERNAL_LOG_DEBUG("[OTLP TRACE FILE Exporter] Export " << span_count + << " trace span(s) success"); + } + return result; +} + +bool OtlpFileExporter::ForceFlush(std::chrono::microseconds timeout) noexcept +{ + return file_client_->ForceFlush(timeout); +} + +bool OtlpFileExporter::Shutdown(std::chrono::microseconds timeout) noexcept +{ + return file_client_->Shutdown(timeout); +} + +} // namespace otlp +} // namespace exporter +OPENTELEMETRY_END_NAMESPACE diff --git a/exporters/otlp/src/otlp_file_exporter_factory.cc b/exporters/otlp/src/otlp_file_exporter_factory.cc new file mode 100644 index 0000000000..245b3a9152 --- /dev/null +++ b/exporters/otlp/src/otlp_file_exporter_factory.cc @@ -0,0 +1,30 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +#include "opentelemetry/exporters/otlp/otlp_file_exporter_factory.h" + +#include "opentelemetry/exporters/otlp/otlp_file_exporter.h" +#include "opentelemetry/exporters/otlp/otlp_file_exporter_options.h" + +OPENTELEMETRY_BEGIN_NAMESPACE +namespace exporter +{ +namespace otlp +{ + +std::unique_ptr OtlpFileExporterFactory::Create() +{ + OtlpFileExporterOptions options; + return Create(options); +} + +std::unique_ptr OtlpFileExporterFactory::Create( + const OtlpFileExporterOptions &options) +{ + std::unique_ptr exporter(new OtlpFileExporter(options)); + return exporter; +} + +} // namespace otlp +} // namespace exporter +OPENTELEMETRY_END_NAMESPACE diff --git a/exporters/otlp/src/otlp_file_exporter_options.cc b/exporters/otlp/src/otlp_file_exporter_options.cc new file mode 100644 index 0000000000..b5e9fa725f --- /dev/null +++ b/exporters/otlp/src/otlp_file_exporter_options.cc @@ -0,0 +1,34 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +#include "opentelemetry/exporters/otlp/otlp_file_exporter_options.h" + +#include +#include + +OPENTELEMETRY_BEGIN_NAMESPACE +namespace exporter +{ +namespace otlp +{ + +OtlpFileExporterOptions::OtlpFileExporterOptions() +{ + console_debug = false; + + OtlpFileClientFileSystemOptions fs_options; + fs_options.file_pattern = "trace-%N.jsonl"; + fs_options.alias_pattern = "trace-latest.jsonl"; + fs_options.flush_interval = std::chrono::seconds(30); + fs_options.flush_count = 256; + fs_options.file_size = 1024 * 1024 * 20; + fs_options.rotate_size = 10; + + backend_options = fs_options; +} + +OtlpFileExporterOptions::~OtlpFileExporterOptions() {} + +} // namespace otlp +} // namespace exporter +OPENTELEMETRY_END_NAMESPACE diff --git a/exporters/otlp/src/otlp_file_log_record_exporter.cc b/exporters/otlp/src/otlp_file_log_record_exporter.cc new file mode 100644 index 0000000000..49cbfd3ab2 --- /dev/null +++ b/exporters/otlp/src/otlp_file_log_record_exporter.cc @@ -0,0 +1,94 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +#include "opentelemetry/exporters/otlp/otlp_file_log_record_exporter.h" +#include "opentelemetry/exporters/otlp/otlp_log_recordable.h" +#include "opentelemetry/exporters/otlp/otlp_recordable_utils.h" + +#include "opentelemetry/exporters/otlp/protobuf_include_prefix.h" + +#include "google/protobuf/arena.h" +#include "opentelemetry/proto/collector/logs/v1/logs_service.pb.h" + +#include "opentelemetry/exporters/otlp/protobuf_include_suffix.h" + +#include "opentelemetry/sdk/common/global_log_handler.h" + +OPENTELEMETRY_BEGIN_NAMESPACE +namespace exporter +{ +namespace otlp +{ + +OtlpFileLogRecordExporter::OtlpFileLogRecordExporter() + : OtlpFileLogRecordExporter(OtlpFileLogRecordExporterOptions()) +{} + +OtlpFileLogRecordExporter::OtlpFileLogRecordExporter( + const OtlpFileLogRecordExporterOptions &options) + : options_(options), file_client_(new OtlpFileClient(OtlpFileClientOptions(options))) +{} +// ----------------------------- Exporter methods ------------------------------ + +std::unique_ptr +OtlpFileLogRecordExporter::MakeRecordable() noexcept +{ + return std::unique_ptr(new OtlpLogRecordable()); +} + +opentelemetry::sdk::common::ExportResult OtlpFileLogRecordExporter::Export( + const nostd::span> &logs) noexcept +{ + if (file_client_->IsShutdown()) + { + std::size_t log_count = logs.size(); + OTEL_INTERNAL_LOG_ERROR("[OTLP LOG FILE Exporter] ERROR: Export " + << log_count << " log(s) failed, exporter is shutdown"); + return opentelemetry::sdk::common::ExportResult::kFailure; + } + + if (logs.empty()) + { + return opentelemetry::sdk::common::ExportResult::kSuccess; + } + + google::protobuf::ArenaOptions arena_options; + // It's easy to allocate datas larger than 1024 when we populate basic resource and attributes + arena_options.initial_block_size = 1024; + // When in batch mode, it's easy to export a large number of spans at once, we can alloc a lager + // block to reduce memory fragments. + arena_options.max_block_size = 65536; + google::protobuf::Arena arena{arena_options}; + + proto::collector::logs::v1::ExportLogsServiceRequest *service_request = + google::protobuf::Arena::Create(&arena); + OtlpRecordableUtils::PopulateRequest(logs, service_request); + std::size_t log_count = logs.size(); + + opentelemetry::sdk::common::ExportResult result = + file_client_->Export(*service_request, log_count); + if (result != opentelemetry::sdk::common::ExportResult::kSuccess) + { + OTEL_INTERNAL_LOG_ERROR("[OTLP LOG FILE Exporter] ERROR: Export " + << log_count << " log(s) error: " << static_cast(result)); + } + else + { + OTEL_INTERNAL_LOG_DEBUG("[OTLP LOG FILE Exporter] Export " << log_count << " log(s) success"); + } + return result; +} + +bool OtlpFileLogRecordExporter::ForceFlush(std::chrono::microseconds timeout) noexcept +{ + return file_client_->ForceFlush(timeout); +} + +bool OtlpFileLogRecordExporter::Shutdown(std::chrono::microseconds timeout) noexcept +{ + return file_client_->Shutdown(timeout); +} + +} // namespace otlp +} // namespace exporter +OPENTELEMETRY_END_NAMESPACE diff --git a/exporters/otlp/src/otlp_file_log_record_exporter_factory.cc b/exporters/otlp/src/otlp_file_log_record_exporter_factory.cc new file mode 100644 index 0000000000..145e52262a --- /dev/null +++ b/exporters/otlp/src/otlp_file_log_record_exporter_factory.cc @@ -0,0 +1,31 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +#include "opentelemetry/exporters/otlp/otlp_file_log_record_exporter_factory.h" +#include "opentelemetry/exporters/otlp/otlp_file_log_record_exporter.h" +#include "opentelemetry/exporters/otlp/otlp_file_log_record_exporter_options.h" + +OPENTELEMETRY_BEGIN_NAMESPACE +namespace exporter +{ +namespace otlp +{ + +std::unique_ptr +OtlpFileLogRecordExporterFactory::Create() +{ + OtlpFileLogRecordExporterOptions options; + return Create(options); +} + +std::unique_ptr +OtlpFileLogRecordExporterFactory::Create(const OtlpFileLogRecordExporterOptions &options) +{ + std::unique_ptr exporter( + new OtlpFileLogRecordExporter(options)); + return exporter; +} + +} // namespace otlp +} // namespace exporter +OPENTELEMETRY_END_NAMESPACE diff --git a/exporters/otlp/src/otlp_file_log_record_exporter_options.cc b/exporters/otlp/src/otlp_file_log_record_exporter_options.cc new file mode 100644 index 0000000000..be794b2872 --- /dev/null +++ b/exporters/otlp/src/otlp_file_log_record_exporter_options.cc @@ -0,0 +1,34 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +#include "opentelemetry/exporters/otlp/otlp_file_log_record_exporter_options.h" + +#include +#include + +OPENTELEMETRY_BEGIN_NAMESPACE +namespace exporter +{ +namespace otlp +{ + +OtlpFileLogRecordExporterOptions::OtlpFileLogRecordExporterOptions() +{ + console_debug = false; + + OtlpFileClientFileSystemOptions fs_options; + fs_options.file_pattern = "logs-%N.jsonl"; + fs_options.alias_pattern = "logs-latest.jsonl"; + fs_options.flush_interval = std::chrono::seconds(30); + fs_options.flush_count = 256; + fs_options.file_size = 1024 * 1024 * 20; + fs_options.rotate_size = 10; + + backend_options = fs_options; +} + +OtlpFileLogRecordExporterOptions::~OtlpFileLogRecordExporterOptions() {} + +} // namespace otlp +} // namespace exporter +OPENTELEMETRY_END_NAMESPACE diff --git a/exporters/otlp/src/otlp_file_metric_exporter.cc b/exporters/otlp/src/otlp_file_metric_exporter.cc new file mode 100644 index 0000000000..2c414c31b4 --- /dev/null +++ b/exporters/otlp/src/otlp_file_metric_exporter.cc @@ -0,0 +1,99 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +#include "opentelemetry/exporters/otlp/otlp_file_metric_exporter.h" +#include "opentelemetry/exporters/otlp/otlp_metric_utils.h" + +#include "opentelemetry/exporters/otlp/protobuf_include_prefix.h" + +#include "google/protobuf/arena.h" +#include "opentelemetry/proto/collector/metrics/v1/metrics_service.pb.h" + +#include "opentelemetry/exporters/otlp/protobuf_include_suffix.h" + +#include "opentelemetry/sdk/common/global_log_handler.h" + +OPENTELEMETRY_BEGIN_NAMESPACE +namespace exporter +{ +namespace otlp +{ + +OtlpFileMetricExporter::OtlpFileMetricExporter() + : OtlpFileMetricExporter(OtlpFileMetricExporterOptions()) +{} + +OtlpFileMetricExporter::OtlpFileMetricExporter(const OtlpFileMetricExporterOptions &options) + : options_(options), + aggregation_temporality_selector_{ + OtlpMetricUtils::ChooseTemporalitySelector(options_.aggregation_temporality)}, + file_client_(new OtlpFileClient(OtlpFileClientOptions(options))) +{} + +// ----------------------------- Exporter methods ------------------------------ + +sdk::metrics::AggregationTemporality OtlpFileMetricExporter::GetAggregationTemporality( + sdk::metrics::InstrumentType instrument_type) const noexcept +{ + + return aggregation_temporality_selector_(instrument_type); +} + +opentelemetry::sdk::common::ExportResult OtlpFileMetricExporter::Export( + const opentelemetry::sdk::metrics::ResourceMetrics &data) noexcept +{ + if (file_client_->IsShutdown()) + { + std::size_t metric_count = data.scope_metric_data_.size(); + OTEL_INTERNAL_LOG_ERROR("[OTLP METRIC FILE Exporter] ERROR: Export " + << metric_count << " metric(s) failed, exporter is shutdown"); + return opentelemetry::sdk::common::ExportResult::kFailure; + } + + if (data.scope_metric_data_.empty()) + { + return opentelemetry::sdk::common::ExportResult::kSuccess; + } + + google::protobuf::ArenaOptions arena_options; + // It's easy to allocate datas larger than 1024 when we populate basic resource and attributes + arena_options.initial_block_size = 1024; + // When in batch mode, it's easy to export a large number of spans at once, we can alloc a lager + // block to reduce memory fragments. + arena_options.max_block_size = 65536; + google::protobuf::Arena arena{arena_options}; + + proto::collector::metrics::v1::ExportMetricsServiceRequest *service_request = + google::protobuf::Arena::Create( + &arena); + OtlpMetricUtils::PopulateRequest(data, service_request); + std::size_t metric_count = data.scope_metric_data_.size(); + + opentelemetry::sdk::common::ExportResult result = + file_client_->Export(*service_request, metric_count); + if (result != opentelemetry::sdk::common::ExportResult::kSuccess) + { + OTEL_INTERNAL_LOG_ERROR("[OTLP METRIC FILE Exporter] ERROR: Export " + << metric_count << " metric(s) error: " << static_cast(result)); + } + else + { + OTEL_INTERNAL_LOG_DEBUG("[OTLP METRIC FILE Exporter] Export " << metric_count + << " metric(s) success"); + } + return result; +} + +bool OtlpFileMetricExporter::ForceFlush(std::chrono::microseconds timeout) noexcept +{ + return file_client_->ForceFlush(timeout); +} + +bool OtlpFileMetricExporter::Shutdown(std::chrono::microseconds timeout) noexcept +{ + return file_client_->Shutdown(timeout); +} + +} // namespace otlp +} // namespace exporter +OPENTELEMETRY_END_NAMESPACE diff --git a/exporters/otlp/src/otlp_file_metric_exporter_factory.cc b/exporters/otlp/src/otlp_file_metric_exporter_factory.cc new file mode 100644 index 0000000000..d474bbb914 --- /dev/null +++ b/exporters/otlp/src/otlp_file_metric_exporter_factory.cc @@ -0,0 +1,31 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +#include "opentelemetry/exporters/otlp/otlp_file_metric_exporter_factory.h" +#include "opentelemetry/exporters/otlp/otlp_file_metric_exporter.h" +#include "opentelemetry/exporters/otlp/otlp_file_metric_exporter_options.h" + +OPENTELEMETRY_BEGIN_NAMESPACE +namespace exporter +{ +namespace otlp +{ + +std::unique_ptr +OtlpFileMetricExporterFactory::Create() +{ + OtlpFileMetricExporterOptions options; + return Create(options); +} + +std::unique_ptr +OtlpFileMetricExporterFactory::Create(const OtlpFileMetricExporterOptions &options) +{ + std::unique_ptr exporter( + new OtlpFileMetricExporter(options)); + return exporter; +} + +} // namespace otlp +} // namespace exporter +OPENTELEMETRY_END_NAMESPACE diff --git a/exporters/otlp/src/otlp_file_metric_exporter_options.cc b/exporters/otlp/src/otlp_file_metric_exporter_options.cc new file mode 100644 index 0000000000..3ad8eaa457 --- /dev/null +++ b/exporters/otlp/src/otlp_file_metric_exporter_options.cc @@ -0,0 +1,36 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +#include "opentelemetry/exporters/otlp/otlp_file_metric_exporter_options.h" + +#include +#include + +OPENTELEMETRY_BEGIN_NAMESPACE +namespace exporter +{ +namespace otlp +{ + +OtlpFileMetricExporterOptions::OtlpFileMetricExporterOptions() +{ + console_debug = false; + + OtlpFileClientFileSystemOptions fs_options; + fs_options.file_pattern = "metrics-%N.jsonl"; + fs_options.alias_pattern = "metrics-latest.jsonl"; + fs_options.flush_interval = std::chrono::seconds(30); + fs_options.flush_count = 256; + fs_options.file_size = 1024 * 1024 * 20; + fs_options.rotate_size = 10; + + backend_options = fs_options; + + aggregation_temporality = PreferredAggregationTemporality::kCumulative; +} + +OtlpFileMetricExporterOptions::~OtlpFileMetricExporterOptions() {} + +} // namespace otlp +} // namespace exporter +OPENTELEMETRY_END_NAMESPACE diff --git a/exporters/otlp/src/otlp_http_client.cc b/exporters/otlp/src/otlp_http_client.cc index 4b4a92c2b0..6a1b89164e 100644 --- a/exporters/otlp/src/otlp_http_client.cc +++ b/exporters/otlp/src/otlp_http_client.cc @@ -12,14 +12,18 @@ #include "opentelemetry/ext/http/client/http_client_factory.h" #include "opentelemetry/ext/http/common/url_parser.h" +// clang-format off #include "opentelemetry/exporters/otlp/protobuf_include_prefix.h" +// clang-format on #include "google/protobuf/message.h" #include "google/protobuf/reflection.h" #include "google/protobuf/stubs/common.h" #include "nlohmann/json.hpp" +// clang-format off #include "opentelemetry/exporters/otlp/protobuf_include_suffix.h" +// clang-format on #include "opentelemetry/common/timestamp.h" #include "opentelemetry/nostd/string_view.h" diff --git a/exporters/otlp/test/otlp_file_client_test.cc b/exporters/otlp/test/otlp_file_client_test.cc new file mode 100644 index 0000000000..f38a682de0 --- /dev/null +++ b/exporters/otlp/test/otlp_file_client_test.cc @@ -0,0 +1,533 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +#include "opentelemetry/common/key_value_iterable_view.h" + +#include "opentelemetry/exporters/otlp/otlp_file_client.h" +#include "opentelemetry/exporters/otlp/otlp_file_exporter_options.h" +#include "opentelemetry/exporters/otlp/otlp_recordable.h" +#include "opentelemetry/exporters/otlp/otlp_recordable_utils.h" +#include "opentelemetry/nostd/unique_ptr.h" +#include "opentelemetry/nostd/variant.h" +#include "opentelemetry/sdk/resource/resource.h" + +#include "opentelemetry/exporters/otlp/protobuf_include_prefix.h" + +#include "opentelemetry/proto/collector/trace/v1/trace_service.pb.h" + +#include "opentelemetry/exporters/otlp/protobuf_include_suffix.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +#include "nlohmann/json.hpp" + +#include +#include +#include +#include +#include +#include +#include + +using namespace testing; + +OPENTELEMETRY_BEGIN_NAMESPACE +namespace exporter +{ +namespace otlp +{ + +namespace resource = opentelemetry::sdk::resource; + +class ProtobufGlobalSymbolGuard +{ +public: + ProtobufGlobalSymbolGuard() {} + ~ProtobufGlobalSymbolGuard() { google::protobuf::ShutdownProtobufLibrary(); } +}; + +static std::tm GetLocalTime(std::chrono::system_clock::time_point tp) +{ + std::time_t now = std::chrono::system_clock::to_time_t(tp); +#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) || defined(__STDC_LIB_EXT1__) + std::tm ret; + localtime_s(&now, &ret); +#elif defined(_MSC_VER) && _MSC_VER >= 1300 + std::tm ret; + localtime_s(&ret, &now); +#elif defined(_XOPEN_SOURCE) || defined(_BSD_SOURCE) || defined(_SVID_SOURCE) || \ + defined(_POSIX_SOURCE) + std::tm ret; + localtime_r(&now, &ret); +#else + std::tm ret = *localtime(&now); +#endif + return ret; +} + +template +static nostd::span MakeSpan(T (&array)[N]) +{ + return nostd::span(array); +} + +static resource::Resource MakeResource() +{ + static ProtobufGlobalSymbolGuard global_symbol_guard; + + resource::ResourceAttributes resource_attributes = {{"service.name", "unit_test_service"}, + {"tenant.id", "test_user"}}; + resource_attributes["bool_value"] = true; + resource_attributes["int32_value"] = static_cast(1); + resource_attributes["uint32_value"] = static_cast(2); + resource_attributes["int64_value"] = static_cast(0x1100000000LL); + resource_attributes["uint64_value"] = static_cast(0x1200000000ULL); + resource_attributes["double_value"] = static_cast(3.1); + resource_attributes["vec_bool_value"] = std::vector{true, false, true}; + resource_attributes["vec_int32_value"] = std::vector{1, 2}; + resource_attributes["vec_uint32_value"] = std::vector{3, 4}; + resource_attributes["vec_int64_value"] = std::vector{5, 6}; + resource_attributes["vec_uint64_value"] = std::vector{7, 8}; + resource_attributes["vec_double_value"] = std::vector{3.2, 3.3}; + resource_attributes["vec_string_value"] = std::vector{"vector", "string"}; + + return resource::Resource::Create(resource_attributes); +} + +static opentelemetry::nostd::unique_ptr< + opentelemetry::sdk::instrumentationscope::InstrumentationScope> +MakeInstrumentationScope() +{ + return opentelemetry::sdk::instrumentationscope::InstrumentationScope::Create( + "otlp_file_client_test", "1.11.0", "https://opentelemetry.io/schemas/1.11.0"); +} + +static std::unique_ptr MakeRecordable( + const resource::Resource &resource, + const opentelemetry::sdk::instrumentationscope::InstrumentationScope &instrumentation_scope) +{ + OtlpRecordable *recordable = new OtlpRecordable(); + recordable->SetResource(resource); + recordable->SetInstrumentationScope(instrumentation_scope); + + recordable->SetName("otlp_file_client_test_span"); + recordable->SetSpanKind(opentelemetry::trace::SpanKind::kInternal); + recordable->SetAttribute("test-attribute-key", "test-attribute-value"); + recordable->SetDuration(std::chrono::nanoseconds(1234567890)); + recordable->SetStartTime( + opentelemetry::common::SystemTimestamp(std::chrono::system_clock::now())); + recordable->SetStatus(opentelemetry::trace::StatusCode::kOk, "success"); + + { + constexpr uint8_t trace_id_buf[] = {1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8}; + constexpr uint8_t span_id_buf[] = {1, 2, 3, 4, 5, 6, 7, 8}; + constexpr uint8_t parent_span_id_buf[] = {8, 7, 6, 5, 4, 3, 2, 1}; + opentelemetry::trace::TraceId trace_id{trace_id_buf}; + opentelemetry::trace::SpanId span_id{span_id_buf}; + opentelemetry::trace::SpanId parent_span_id{parent_span_id_buf}; + const auto trace_state = opentelemetry::trace::TraceState::GetDefault()->Set("key1", "value"); + const opentelemetry::trace::SpanContext span_context{ + trace_id, span_id, + opentelemetry::trace::TraceFlags{opentelemetry::trace::TraceFlags::kIsSampled}, true, + trace_state}; + + recordable->SetIdentity(span_context, parent_span_id); + } + + return std::unique_ptr(recordable); +} + +TEST(OtlpFileClientTest, Shutdown) +{ + opentelemetry::proto::collector::trace::v1::ExportTraceServiceRequest request; + auto client = std::unique_ptr( + new opentelemetry::exporter::otlp::OtlpFileClient( + opentelemetry::exporter::otlp::OtlpFileClientOptions())); + ASSERT_FALSE(client->IsShutdown()); + ASSERT_TRUE(client->Shutdown()); + ASSERT_TRUE(client->IsShutdown()); + + auto result = client->Export(request, 1); + EXPECT_EQ(result, opentelemetry::sdk::common::ExportResult::kFailure); +} + +TEST(OtlpFileClientTest, ExportToOstreamTest) +{ + auto resource = MakeResource(); + auto instrumentation_scope = MakeInstrumentationScope(); + + std::unique_ptr recordable[] = { + MakeRecordable(resource, *instrumentation_scope)}; + + opentelemetry::proto::collector::trace::v1::ExportTraceServiceRequest request; + OtlpRecordableUtils::PopulateRequest(MakeSpan(recordable), &request); + + std::stringstream output_stream; + + opentelemetry::exporter::otlp::OtlpFileClientOptions opts; + opts.backend_options = std::ref(output_stream); + + auto client = std::unique_ptr( + new opentelemetry::exporter::otlp::OtlpFileClient(std::move(opts))); + client->Export(request, 1); + + { + auto check_json = nlohmann::json::parse(output_stream.str(), nullptr, false); + auto resource_span = *check_json["resourceSpans"].begin(); + auto scope_span = *resource_span["scopeSpans"].begin(); + auto span = *scope_span["spans"].begin(); + auto received_trace_id = span["traceId"].get(); + EXPECT_EQ(received_trace_id, "01020304050607080102030405060708"); + + auto received_name = span["name"].get(); + EXPECT_EQ(received_name, request.resource_spans(0).scope_spans(0).spans(0).name()); + + auto receivec_attributes = span["attributes"]; + int attribute_found = 0; + for (auto iter = receivec_attributes.begin(); iter != receivec_attributes.end(); ++iter) + { + auto key = (*iter)["key"].get(); + if (key == "test-attribute-key") + { + auto value = (*iter)["value"]["stringValue"]; + ++attribute_found; + EXPECT_EQ(value.get(), "test-attribute-value"); + } + } + EXPECT_EQ(attribute_found, 1); + + auto receivec_resource_attributes = resource_span["resource"]["attributes"]; + for (auto iter = receivec_resource_attributes.begin(); + iter != receivec_resource_attributes.end(); ++iter) + { + auto key = (*iter)["key"].get(); + if (key == "service.name") + { + auto value = (*iter)["value"]["stringValue"]; + ++attribute_found; + EXPECT_EQ(value.get(), "unit_test_service"); + } + else if (key == "tenant.id") + { + auto value = (*iter)["value"]["stringValue"]; + ++attribute_found; + EXPECT_EQ(value.get(), "test_user"); + } + else if (key == "int32_value") + { + auto value = (*iter)["value"]["intValue"]; + ++attribute_found; + if (value.is_number()) + { + EXPECT_EQ(value.get(), 1); + } + else + { + EXPECT_EQ(value.get(), "1"); + } + } + } + EXPECT_EQ(attribute_found, 4); + } +} + +TEST(OtlpFileClientTest, ExportToFileSystemRotateIndexTest) +{ + auto resource = MakeResource(); + auto instrumentation_scope = MakeInstrumentationScope(); + + std::unique_ptr recordable[] = { + MakeRecordable(resource, *instrumentation_scope)}; + + opentelemetry::proto::collector::trace::v1::ExportTraceServiceRequest request; + OtlpRecordableUtils::PopulateRequest(MakeSpan(recordable), &request); + + std::stringstream output_stream; + + // Clear old files + { + std::fstream clear_file1("otlp_file_client_test_dir/trace-1.jsonl", + std::ios::out | std::ios::trunc); + std::fstream clear_file2("otlp_file_client_test_dir/trace-2.jsonl", + std::ios::out | std::ios::trunc); + std::fstream clear_file3("otlp_file_client_test_dir/trace-3.jsonl", + std::ios::out | std::ios::trunc); + std::fstream clear_file4("otlp_file_client_test_dir/trace-latest.jsonl", + std::ios::out | std::ios::trunc); + } + + opentelemetry::exporter::otlp::OtlpFileClientFileSystemOptions backend_opts; + backend_opts.file_pattern = "otlp_file_client_test_dir/trace-%n.jsonl"; + backend_opts.alias_pattern = "otlp_file_client_test_dir/trace-latest.jsonl"; + // Smaller than the size of one record, so it will rotate after each record. + backend_opts.file_size = 1500; + backend_opts.rotate_size = 3; + + opentelemetry::exporter::otlp::OtlpFileClientOptions opts; + opts.backend_options = backend_opts; + + auto client = std::unique_ptr( + new opentelemetry::exporter::otlp::OtlpFileClient(std::move(opts))); + + // Write 5 records with rotatation index 1,2,3,1,2 + for (int i = 0; i < 4; ++i) + { + client->Export(request, 1); + } + request.mutable_resource_spans(0)->set_schema_url("https://opentelemetry.io/schemas/1.12.0"); + client->Export(request, 1); + client->ForceFlush(); + + std::unique_ptr input_file[5] = { + std::unique_ptr( + new std::ifstream("otlp_file_client_test_dir/trace-1.jsonl", std::ios::in)), + std::unique_ptr( + new std::ifstream("otlp_file_client_test_dir/trace-2.jsonl", std::ios::in)), + std::unique_ptr( + new std::ifstream("otlp_file_client_test_dir/trace-3.jsonl", std::ios::in)), + std::unique_ptr( + new std::ifstream("otlp_file_client_test_dir/trace-4.jsonl", std::ios::in)), + std::unique_ptr( + new std::ifstream("otlp_file_client_test_dir/trace-latest.jsonl", std::ios::in))}; + + EXPECT_TRUE(input_file[0]->is_open()); + EXPECT_TRUE(input_file[1]->is_open()); + EXPECT_TRUE(input_file[2]->is_open()); + EXPECT_FALSE(input_file[3]->is_open()); + EXPECT_TRUE(input_file[4]->is_open()); + + std::string jsonl[4]; + std::getline(*input_file[0], jsonl[0]); + std::getline(*input_file[1], jsonl[1]); + std::getline(*input_file[2], jsonl[2]); + std::getline(*input_file[4], jsonl[3]); + + EXPECT_EQ(jsonl[0], jsonl[2]); + EXPECT_EQ(jsonl[1], jsonl[3]); + + { + auto check_json = nlohmann::json::parse(jsonl[0], nullptr, false); + auto resource_span = *check_json["resourceSpans"].begin(); + auto scope_span = *resource_span["scopeSpans"].begin(); + auto span = *scope_span["spans"].begin(); + auto received_trace_id = span["traceId"].get(); + EXPECT_EQ(received_trace_id, "01020304050607080102030405060708"); + + auto received_name = span["name"].get(); + EXPECT_EQ(received_name, request.resource_spans(0).scope_spans(0).spans(0).name()); + + auto receivec_attributes = span["attributes"]; + int attribute_found = 0; + for (auto iter = receivec_attributes.begin(); iter != receivec_attributes.end(); ++iter) + { + auto key = (*iter)["key"].get(); + if (key == "test-attribute-key") + { + auto value = (*iter)["value"]["stringValue"]; + ++attribute_found; + EXPECT_EQ(value.get(), "test-attribute-value"); + } + } + EXPECT_EQ(attribute_found, 1); + + auto receivec_resource_attributes = resource_span["resource"]["attributes"]; + for (auto iter = receivec_resource_attributes.begin(); + iter != receivec_resource_attributes.end(); ++iter) + { + auto key = (*iter)["key"].get(); + if (key == "service.name") + { + auto value = (*iter)["value"]["stringValue"]; + ++attribute_found; + EXPECT_EQ(value.get(), "unit_test_service"); + } + else if (key == "tenant.id") + { + auto value = (*iter)["value"]["stringValue"]; + ++attribute_found; + EXPECT_EQ(value.get(), "test_user"); + } + else if (key == "int32_value") + { + auto value = (*iter)["value"]["intValue"]; + ++attribute_found; + if (value.is_number()) + { + EXPECT_EQ(value.get(), 1); + } + else + { + EXPECT_EQ(value.get(), "1"); + } + } + } + EXPECT_EQ(attribute_found, 4); + } +} + +TEST(OtlpFileClientTest, ExportToFileSystemRotateByTimeTest) +{ + auto resource = MakeResource(); + auto instrumentation_scope = MakeInstrumentationScope(); + + std::unique_ptr recordable[] = { + MakeRecordable(resource, *instrumentation_scope)}; + + opentelemetry::proto::collector::trace::v1::ExportTraceServiceRequest request; + OtlpRecordableUtils::PopulateRequest(MakeSpan(recordable), &request); + + std::stringstream output_stream; + + opentelemetry::exporter::otlp::OtlpFileClientFileSystemOptions backend_opts; + backend_opts.file_pattern = "otlp_file_client_test_dir/trace-%Y-%m-%d-%H-%M-%S.jsonl"; + backend_opts.alias_pattern = ""; + // Smaller than the size of one record, so it will rotate after each record. + backend_opts.file_size = 1500; + + opentelemetry::exporter::otlp::OtlpFileClientOptions opts; + opts.backend_options = backend_opts; + + auto client = std::unique_ptr( + new opentelemetry::exporter::otlp::OtlpFileClient(std::move(opts))); + + auto start_time = std::chrono::system_clock::now(); + client->Export(request, 1); + std::this_thread::sleep_for(std::chrono::seconds{1}); + client->Export(request, 1); + client->ForceFlush(); + + std::unique_ptr input_file[2]; + std::size_t found_file_index = 0; + // Try to load the file in 5s, it should finished. + for (int i = 0; i < 5; ++i) + { + char file_path_buf[256] = {0}; + std::tm local_tm = GetLocalTime(start_time); + std::strftime(file_path_buf, sizeof(file_path_buf) - 1, + "otlp_file_client_test_dir/trace-%Y-%m-%d-%H-%M-%S.jsonl", &local_tm); + start_time += std::chrono::seconds{1}; + + input_file[found_file_index] = + std::unique_ptr(new std::ifstream(file_path_buf, std::ios::in)); + if (input_file[found_file_index]->is_open()) + { + ++found_file_index; + } + if (found_file_index >= 2) + { + break; + } + } + + ASSERT_EQ(found_file_index, 2); + + std::string jsonl[2]; + std::getline(*input_file[0], jsonl[0]); + std::getline(*input_file[1], jsonl[1]); + + EXPECT_EQ(jsonl[0], jsonl[1]); + + { + auto check_json = nlohmann::json::parse(jsonl[0], nullptr, false); + auto resource_span = *check_json["resourceSpans"].begin(); + auto scope_span = *resource_span["scopeSpans"].begin(); + auto span = *scope_span["spans"].begin(); + auto received_trace_id = span["traceId"].get(); + EXPECT_EQ(received_trace_id, "01020304050607080102030405060708"); + + auto received_name = span["name"].get(); + EXPECT_EQ(received_name, request.resource_spans(0).scope_spans(0).spans(0).name()); + + auto receivec_attributes = span["attributes"]; + int attribute_found = 0; + for (auto iter = receivec_attributes.begin(); iter != receivec_attributes.end(); ++iter) + { + auto key = (*iter)["key"].get(); + if (key == "test-attribute-key") + { + auto value = (*iter)["value"]["stringValue"]; + ++attribute_found; + EXPECT_EQ(value.get(), "test-attribute-value"); + } + } + EXPECT_EQ(attribute_found, 1); + + auto receivec_resource_attributes = resource_span["resource"]["attributes"]; + for (auto iter = receivec_resource_attributes.begin(); + iter != receivec_resource_attributes.end(); ++iter) + { + auto key = (*iter)["key"].get(); + if (key == "service.name") + { + auto value = (*iter)["value"]["stringValue"]; + ++attribute_found; + EXPECT_EQ(value.get(), "unit_test_service"); + } + else if (key == "tenant.id") + { + auto value = (*iter)["value"]["stringValue"]; + ++attribute_found; + EXPECT_EQ(value.get(), "test_user"); + } + else if (key == "int32_value") + { + auto value = (*iter)["value"]["intValue"]; + ++attribute_found; + if (value.is_number()) + { + EXPECT_EQ(value.get(), 1); + } + else + { + EXPECT_EQ(value.get(), "1"); + } + } + } + EXPECT_EQ(attribute_found, 4); + } +} + +// Test client configuration options +TEST(OtlpFileClientTest, ConfigTest) +{ + { + opentelemetry::exporter::otlp::OtlpFileClientOptions opts; + opts.console_debug = true; + opts.backend_options = std::ref(std::cout); + + auto client = std::unique_ptr( + new opentelemetry::exporter::otlp::OtlpFileClient(std::move(opts))); + + ASSERT_TRUE(client->GetOptions().console_debug); + ASSERT_TRUE(opentelemetry::nostd::holds_alternative>( + client->GetOptions().backend_options)); + } + + { + opentelemetry::exporter::otlp::OtlpFileClientFileSystemOptions backend_opts; + backend_opts.file_pattern = "test_file_pattern.jsonl"; + + opentelemetry::exporter::otlp::OtlpFileClientOptions opts; + opts.console_debug = false; + opts.backend_options = backend_opts; + + auto client = std::unique_ptr( + new opentelemetry::exporter::otlp::OtlpFileClient(std::move(opts))); + + ASSERT_FALSE(client->GetOptions().console_debug); + ASSERT_TRUE(opentelemetry::nostd::holds_alternative< + opentelemetry::exporter::otlp::OtlpFileClientFileSystemOptions>( + client->GetOptions().backend_options)); + + EXPECT_EQ( + opentelemetry::nostd::get( + client->GetOptions().backend_options) + .file_pattern, + "test_file_pattern.jsonl"); + } +} + +} // namespace otlp +} // namespace exporter +OPENTELEMETRY_END_NAMESPACE diff --git a/exporters/otlp/test/otlp_file_exporter_factory_test.cc b/exporters/otlp/test/otlp_file_exporter_factory_test.cc new file mode 100644 index 0000000000..2d671c6e8f --- /dev/null +++ b/exporters/otlp/test/otlp_file_exporter_factory_test.cc @@ -0,0 +1,42 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +#include + +#include "opentelemetry/exporters/otlp/otlp_file_exporter_factory.h" +#include "opentelemetry/exporters/otlp/otlp_file_exporter_options.h" + +/* + Make sure OtlpFileExporterFactory does not require, + even indirectly, nlohmann/json headers. +*/ +#ifdef NLOHMANN_JSON_VERSION_MAJOR +# error "nlohmann/json should not be included" +#endif /* NLOHMANN_JSON_VERSION_MAJOR */ + +/* + Make sure OtlpFileExporterFactory does not require, + even indirectly, protobuf headers. +*/ +#ifdef GOOGLE_PROTOBUF_VERSION +# error "protobuf should not be included" +#endif + +OPENTELEMETRY_BEGIN_NAMESPACE +namespace exporter +{ +namespace otlp +{ + +TEST(OtlpFileExporterFactoryTest, BuildTest) +{ + OtlpFileExporterOptions opts; + std::unique_ptr exporter = + OtlpFileExporterFactory::Create(opts); + + EXPECT_TRUE(exporter != nullptr); +} + +} // namespace otlp +} // namespace exporter +OPENTELEMETRY_END_NAMESPACE diff --git a/exporters/otlp/test/otlp_file_exporter_test.cc b/exporters/otlp/test/otlp_file_exporter_test.cc new file mode 100644 index 0000000000..497bb0b203 --- /dev/null +++ b/exporters/otlp/test/otlp_file_exporter_test.cc @@ -0,0 +1,145 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +#include "opentelemetry/exporters/otlp/otlp_file_exporter_factory.h" +#include "opentelemetry/exporters/otlp/otlp_file_exporter_options.h" + +#include "opentelemetry/exporters/otlp/otlp_file_exporter.h" + +#include "opentelemetry/exporters/otlp/protobuf_include_prefix.h" + +#include "google/protobuf/message_lite.h" +#include "opentelemetry/proto/collector/trace/v1/trace_service.pb.h" + +#include "opentelemetry/exporters/otlp/protobuf_include_suffix.h" + +#include "opentelemetry/sdk/trace/batch_span_processor.h" +#include "opentelemetry/sdk/trace/batch_span_processor_options.h" +#include "opentelemetry/sdk/trace/tracer_provider.h" + +#include "opentelemetry/trace/provider.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +#include "nlohmann/json.hpp" + +#include +#include + +using namespace testing; + +OPENTELEMETRY_BEGIN_NAMESPACE +namespace exporter +{ +namespace otlp +{ + +namespace trace_api = opentelemetry::trace; +namespace resource = opentelemetry::sdk::resource; + +class ProtobufGlobalSymbolGuard +{ +public: + ProtobufGlobalSymbolGuard() {} + ~ProtobufGlobalSymbolGuard() { google::protobuf::ShutdownProtobufLibrary(); } +}; + +template +static nostd::span MakeSpan(T (&array)[N]) +{ + return nostd::span(array); +} + +class OtlpFileExporterTestPeer : public ::testing::Test +{ +public: + void ExportJsonIntegrationTest() + { + static ProtobufGlobalSymbolGuard global_symbol_guard; + + std::stringstream output; + OtlpFileExporterOptions opts; + opts.backend_options = std::ref(output); + + auto exporter = OtlpFileExporterFactory::Create(opts); + + resource::ResourceAttributes resource_attributes = {{"service.name", "unit_test_service"}, + {"tenant.id", "test_user"}}; + resource_attributes["bool_value"] = true; + resource_attributes["int32_value"] = static_cast(1); + resource_attributes["uint32_value"] = static_cast(2); + resource_attributes["int64_value"] = static_cast(0x1100000000LL); + resource_attributes["uint64_value"] = static_cast(0x1200000000ULL); + resource_attributes["double_value"] = static_cast(3.1); + resource_attributes["vec_bool_value"] = std::vector{true, false, true}; + resource_attributes["vec_int32_value"] = std::vector{1, 2}; + resource_attributes["vec_uint32_value"] = std::vector{3, 4}; + resource_attributes["vec_int64_value"] = std::vector{5, 6}; + resource_attributes["vec_uint64_value"] = std::vector{7, 8}; + resource_attributes["vec_double_value"] = std::vector{3.2, 3.3}; + resource_attributes["vec_string_value"] = std::vector{"vector", "string"}; + auto resource = resource::Resource::Create(resource_attributes); + + auto processor_opts = sdk::trace::BatchSpanProcessorOptions(); + processor_opts.max_export_batch_size = 5; + processor_opts.max_queue_size = 5; + processor_opts.schedule_delay_millis = std::chrono::milliseconds(256); + + auto processor = std::unique_ptr( + new sdk::trace::BatchSpanProcessor(std::move(exporter), processor_opts)); + auto provider = nostd::shared_ptr( + new sdk::trace::TracerProvider(std::move(processor), resource)); + + std::string report_trace_id; + + char trace_id_hex[2 * trace_api::TraceId::kSize] = {0}; + auto tracer = provider->GetTracer("test"); + auto parent_span = tracer->StartSpan("Test parent span"); + + trace_api::StartSpanOptions child_span_opts = {}; + child_span_opts.parent = parent_span->GetContext(); + + auto child_span = tracer->StartSpan("Test child span", child_span_opts); + + nostd::get(child_span_opts.parent) + .trace_id() + .ToLowerBase16(MakeSpan(trace_id_hex)); + report_trace_id.assign(trace_id_hex, sizeof(trace_id_hex)); + + child_span->End(); + parent_span->End(); + + static_cast(provider.get())->ForceFlush(); + + { + auto check_json = nlohmann::json::parse(output.str(), nullptr, false); + auto resource_span = *check_json["resourceSpans"].begin(); + auto scope_span = *resource_span["scopeSpans"].begin(); + auto span = *scope_span["spans"].begin(); + auto received_trace_id = span["traceId"].get(); + EXPECT_EQ(received_trace_id, report_trace_id); + } + } +}; + +TEST(OtlpFileExporterTest, Shutdown) +{ + auto exporter = std::unique_ptr(new OtlpFileExporter()); + ASSERT_TRUE(exporter->Shutdown()); + + nostd::span> spans = {}; + + auto result = exporter->Export(spans); + EXPECT_EQ(result, opentelemetry::sdk::common::ExportResult::kFailure); +} + +// Create spans, let processor call Export() +TEST_F(OtlpFileExporterTestPeer, ExportJsonIntegrationTestSync) +{ + ExportJsonIntegrationTest(); +} + +} // namespace otlp +} // namespace exporter +OPENTELEMETRY_END_NAMESPACE diff --git a/exporters/otlp/test/otlp_file_log_record_exporter_factory_test.cc b/exporters/otlp/test/otlp_file_log_record_exporter_factory_test.cc new file mode 100644 index 0000000000..ec4ced5f33 --- /dev/null +++ b/exporters/otlp/test/otlp_file_log_record_exporter_factory_test.cc @@ -0,0 +1,42 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +#include + +#include "opentelemetry/exporters/otlp/otlp_file_log_record_exporter_factory.h" +#include "opentelemetry/exporters/otlp/otlp_file_log_record_exporter_options.h" + +/* + Make sure OtlpFileExporterFactory does not require, + even indirectly, nlohmann/json headers. +*/ +#ifdef NLOHMANN_JSON_VERSION_MAJOR +# error "nlohmann/json should not be included" +#endif /* NLOHMANN_JSON_VERSION_MAJOR */ + +/* + Make sure OtlpFileExporterFactory does not require, + even indirectly, protobuf headers. +*/ +#ifdef GOOGLE_PROTOBUF_VERSION +# error "protobuf should not be included" +#endif + +OPENTELEMETRY_BEGIN_NAMESPACE +namespace exporter +{ +namespace otlp +{ + +TEST(OtlpFileLogRecordExporterFactoryTest, BuildTest) +{ + OtlpFileLogRecordExporterOptions opts; + std::unique_ptr exporter = + OtlpFileLogRecordExporterFactory::Create(opts); + + EXPECT_TRUE(exporter != nullptr); +} + +} // namespace otlp +} // namespace exporter +OPENTELEMETRY_END_NAMESPACE diff --git a/exporters/otlp/test/otlp_file_log_record_exporter_test.cc b/exporters/otlp/test/otlp_file_log_record_exporter_test.cc new file mode 100644 index 0000000000..9f04bcece4 --- /dev/null +++ b/exporters/otlp/test/otlp_file_log_record_exporter_test.cc @@ -0,0 +1,177 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +#include +#include + +#include "opentelemetry/exporters/otlp/otlp_file_log_record_exporter.h" +#include "opentelemetry/exporters/otlp/otlp_file_log_record_exporter_factory.h" + +#include "opentelemetry/exporters/otlp/protobuf_include_prefix.h" + +#include "opentelemetry/proto/collector/logs/v1/logs_service.pb.h" + +#include "opentelemetry/exporters/otlp/protobuf_include_suffix.h" + +#include "opentelemetry/common/key_value_iterable_view.h" + +#include "opentelemetry/logs/provider.h" +#include "opentelemetry/sdk/logs/batch_log_record_processor.h" +#include "opentelemetry/sdk/logs/exporter.h" +#include "opentelemetry/sdk/logs/logger_provider.h" +#include "opentelemetry/sdk/resource/resource.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +#include "nlohmann/json.hpp" + +#include +#include + +using namespace testing; + +OPENTELEMETRY_BEGIN_NAMESPACE +namespace exporter +{ +namespace otlp +{ + +class ProtobufGlobalSymbolGuard +{ +public: + ProtobufGlobalSymbolGuard() {} + ~ProtobufGlobalSymbolGuard() { google::protobuf::ShutdownProtobufLibrary(); } +}; + +template +static nostd::span MakeSpan(T (&array)[N]) +{ + return nostd::span(array); +} + +class OtlpFileLogRecordExporterTestPeer : public ::testing::Test +{ +public: + void ExportJsonIntegrationTest() + { + static ProtobufGlobalSymbolGuard global_symbol_guard; + + std::stringstream output; + OtlpFileLogRecordExporterOptions opts; + opts.backend_options = std::ref(output); + + auto exporter = OtlpFileLogRecordExporterFactory::Create(opts); + + bool attribute_storage_bool_value[] = {true, false, true}; + int32_t attribute_storage_int32_value[] = {1, 2}; + uint32_t attribute_storage_uint32_value[] = {3, 4}; + int64_t attribute_storage_int64_value[] = {5, 6}; + uint64_t attribute_storage_uint64_value[] = {7, 8}; + double attribute_storage_double_value[] = {3.2, 3.3}; + opentelemetry::nostd::string_view attribute_storage_string_value[] = {"vector", "string"}; + + auto provider = nostd::shared_ptr(new sdk::logs::LoggerProvider()); + + provider->AddProcessor( + std::unique_ptr(new sdk::logs::BatchLogRecordProcessor( + std::move(exporter), 5, std::chrono::milliseconds(256), 5))); + + std::string report_trace_id; + std::string report_span_id; + uint8_t trace_id_bin[opentelemetry::trace::TraceId::kSize] = { + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'}; + char trace_id_hex[2 * opentelemetry::trace::TraceId::kSize] = {0}; + opentelemetry::trace::TraceId trace_id{trace_id_bin}; + uint8_t span_id_bin[opentelemetry::trace::SpanId::kSize] = { + // Fix clang-format 10 has different behavior for this line + '7', '6', '5', '4', '3', '2', '1', '0'}; + char span_id_hex[2 * opentelemetry::trace::SpanId::kSize] = {0}; + opentelemetry::trace::SpanId span_id{span_id_bin}; + + const std::string schema_url{"https://opentelemetry.io/schemas/1.2.0"}; + auto logger = provider->GetLogger("test", "opentelelemtry_library", "", schema_url, + {{"scope_key1", "scope_value"}, {"scope_key2", 2}}); + + trace_id.ToLowerBase16(MakeSpan(trace_id_hex)); + report_trace_id.assign(trace_id_hex, sizeof(trace_id_hex)); + + span_id.ToLowerBase16(MakeSpan(span_id_hex)); + report_span_id.assign(span_id_hex, sizeof(span_id_hex)); + logger->EmitLogRecord( + opentelemetry::logs::Severity::kInfo, "Log message", + opentelemetry::common::MakeAttributes( + {{"service.name", "unit_test_service"}, + {"tenant.id", "test_user"}, + {"bool_value", true}, + {"int32_value", static_cast(1)}, + {"uint32_value", static_cast(2)}, + {"int64_value", static_cast(0x1100000000LL)}, + {"uint64_value", static_cast(0x1200000000ULL)}, + {"double_value", static_cast(3.1)}, + {"vec_bool_value", attribute_storage_bool_value}, + {"vec_int32_value", attribute_storage_int32_value}, + {"vec_uint32_value", attribute_storage_uint32_value}, + {"vec_int64_value", attribute_storage_int64_value}, + {"vec_uint64_value", attribute_storage_uint64_value}, + {"vec_double_value", attribute_storage_double_value}, + {"vec_string_value", attribute_storage_string_value}}), + trace_id, span_id, + opentelemetry::trace::TraceFlags{opentelemetry::trace::TraceFlags::kIsSampled}, + std::chrono::system_clock::now()); + + provider->ForceFlush(); + + { + auto check_json = nlohmann::json::parse(output.str(), nullptr, false); + auto resource_logs = *check_json["resourceLogs"].begin(); + auto scope_logs = *resource_logs["scopeLogs"].begin(); + auto scope = scope_logs["scope"]; + auto log = *scope_logs["logRecords"].begin(); + auto received_trace_id = log["traceId"].get(); + auto received_span_id = log["spanId"].get(); + EXPECT_EQ(received_trace_id, report_trace_id); + EXPECT_EQ(received_span_id, report_span_id); + EXPECT_EQ("Log message", log["body"]["stringValue"].get()); + EXPECT_LE(15, log["attributes"].size()); + + bool check_scope_attribute = false; + auto scope_attributes = scope["attributes"]; + for (auto &attribute : scope_attributes) + { + if (!attribute.is_object()) + { + continue; + } + if ("scope_key1" == attribute["key"]) + { + check_scope_attribute = true; + EXPECT_EQ("scope_value", attribute["value"]["stringValue"].get()); + } + } + ASSERT_TRUE(check_scope_attribute); + } + } +}; + +TEST(OtlpFileLogRecordExporterTest, Shutdown) +{ + auto exporter = + std::unique_ptr(new OtlpFileLogRecordExporter()); + ASSERT_TRUE(exporter->Shutdown()); + + nostd::span> logs = {}; + + auto result = exporter->Export(logs); + EXPECT_EQ(result, opentelemetry::sdk::common::ExportResult::kFailure); +} + +// Create log records, let processor call Export() +TEST_F(OtlpFileLogRecordExporterTestPeer, ExportJsonIntegrationTestSync) +{ + ExportJsonIntegrationTest(); +} + +} // namespace otlp +} // namespace exporter +OPENTELEMETRY_END_NAMESPACE diff --git a/exporters/otlp/test/otlp_file_metric_exporter_factory_test.cc b/exporters/otlp/test/otlp_file_metric_exporter_factory_test.cc new file mode 100644 index 0000000000..d14dc3056a --- /dev/null +++ b/exporters/otlp/test/otlp_file_metric_exporter_factory_test.cc @@ -0,0 +1,42 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +#include + +#include "opentelemetry/exporters/otlp/otlp_file_metric_exporter_factory.h" +#include "opentelemetry/exporters/otlp/otlp_file_metric_exporter_options.h" + +/* + Make sure OtlpFileExporterFactory does not require, + even indirectly, nlohmann/json headers. +*/ +#ifdef NLOHMANN_JSON_VERSION_MAJOR +# error "nlohmann/json should not be included" +#endif /* NLOHMANN_JSON_VERSION_MAJOR */ + +/* + Make sure OtlpFileExporterFactory does not require, + even indirectly, protobuf headers. +*/ +#ifdef GOOGLE_PROTOBUF_VERSION +# error "protobuf should not be included" +#endif + +OPENTELEMETRY_BEGIN_NAMESPACE +namespace exporter +{ +namespace otlp +{ + +TEST(OtlpFileMetricExporterFactory, BuildTest) +{ + OtlpFileMetricExporterOptions opts; + std::unique_ptr exporter = + OtlpFileMetricExporterFactory::Create(opts); + + EXPECT_TRUE(exporter != nullptr); +} + +} // namespace otlp +} // namespace exporter +OPENTELEMETRY_END_NAMESPACE diff --git a/exporters/otlp/test/otlp_file_metric_exporter_test.cc b/exporters/otlp/test/otlp_file_metric_exporter_test.cc new file mode 100644 index 0000000000..f08c8854ba --- /dev/null +++ b/exporters/otlp/test/otlp_file_metric_exporter_test.cc @@ -0,0 +1,390 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +#include +#include +#include + +#include "opentelemetry/exporters/otlp/otlp_file_metric_exporter.h" +#include "opentelemetry/exporters/otlp/otlp_file_metric_exporter_factory.h" + +#include "opentelemetry/exporters/otlp/protobuf_include_prefix.h" + +#include "google/protobuf/message_lite.h" +#include "opentelemetry/proto/collector/metrics/v1/metrics_service.pb.h" + +#include "opentelemetry/exporters/otlp/otlp_metric_utils.h" +#include "opentelemetry/exporters/otlp/protobuf_include_suffix.h" + +#include "opentelemetry/common/key_value_iterable_view.h" +#include "opentelemetry/sdk/instrumentationscope/instrumentation_scope.h" +#include "opentelemetry/sdk/metrics/aggregation/default_aggregation.h" +#include "opentelemetry/sdk/metrics/aggregation/histogram_aggregation.h" +#include "opentelemetry/sdk/metrics/data/metric_data.h" +#include "opentelemetry/sdk/metrics/export/metric_producer.h" +#include "opentelemetry/sdk/metrics/instruments.h" +#include "opentelemetry/sdk/resource/resource.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +#include "nlohmann/json.hpp" + +#include +#include + +using namespace testing; + +OPENTELEMETRY_BEGIN_NAMESPACE +namespace exporter +{ +namespace otlp +{ + +class ProtobufGlobalSymbolGuard +{ +public: + ProtobufGlobalSymbolGuard() {} + ~ProtobufGlobalSymbolGuard() { google::protobuf::ShutdownProtobufLibrary(); } +}; + +template +static IntegerType JsonToInteger(nlohmann::json value) +{ + if (value.is_string()) + { + return static_cast(strtol(value.get().c_str(), nullptr, 10)); + } + + return value.get(); +} + +class OtlpFileMetricExporterTestPeer : public ::testing::Test +{ +public: + // Get the options associated with the given exporter. + const OtlpFileMetricExporterOptions &GetOptions(std::unique_ptr &exporter) + { + return exporter->options_; + } + + void ExportJsonIntegrationTestExportSumPointData() + { + static ProtobufGlobalSymbolGuard global_symbol_guard; + + std::stringstream output; + OtlpFileMetricExporterOptions opts; + opts.backend_options = std::ref(output); + + auto exporter = OtlpFileMetricExporterFactory::Create(opts); + + opentelemetry::sdk::metrics::SumPointData sum_point_data{}; + sum_point_data.value_ = 10.0; + opentelemetry::sdk::metrics::SumPointData sum_point_data2{}; + sum_point_data2.value_ = 20.0; + opentelemetry::sdk::metrics::ResourceMetrics data; + auto resource = opentelemetry::sdk::resource::Resource::Create( + opentelemetry::sdk::resource::ResourceAttributes{}); + data.resource_ = &resource; + auto scope = opentelemetry::sdk::instrumentationscope::InstrumentationScope::Create( + "library_name", "1.5.0"); + opentelemetry::sdk::metrics::MetricData metric_data{ + opentelemetry::sdk::metrics::InstrumentDescriptor{ + "metrics_library_name", "metrics_description", "metrics_unit", + opentelemetry::sdk::metrics::InstrumentType::kCounter, + opentelemetry::sdk::metrics::InstrumentValueType::kDouble}, + opentelemetry::sdk::metrics::AggregationTemporality::kDelta, + opentelemetry::common::SystemTimestamp{}, opentelemetry::common::SystemTimestamp{}, + std::vector{ + {opentelemetry::sdk::metrics::PointAttributes{{"a1", "b1"}}, sum_point_data}, + {opentelemetry::sdk::metrics::PointAttributes{{"a2", "b2"}}, sum_point_data2}}}; + data.scope_metric_data_ = std::vector{ + {scope.get(), std::vector{metric_data}}}; + + auto result = exporter->Export(data); + EXPECT_EQ(result, opentelemetry::sdk::common::ExportResult::kSuccess); + + exporter->ForceFlush(); + + { + auto check_json = nlohmann::json::parse(output.str(), nullptr, false); + + auto resource_metrics = *check_json["resourceMetrics"].begin(); + auto scope_metrics = *resource_metrics["scopeMetrics"].begin(); + auto scope = scope_metrics["scope"]; + EXPECT_EQ("library_name", scope["name"].get()); + EXPECT_EQ("1.5.0", scope["version"].get()); + + auto metric = *scope_metrics["metrics"].begin(); + EXPECT_EQ("metrics_library_name", metric["name"].get()); + EXPECT_EQ("metrics_description", metric["description"].get()); + EXPECT_EQ("metrics_unit", metric["unit"].get()); + + auto data_points = metric["sum"]["dataPoints"]; + EXPECT_EQ(10.0, data_points[0]["asDouble"].get()); + EXPECT_EQ(20.0, data_points[1]["asDouble"].get()); + } + } + + void ExportJsonIntegrationTestExportLastValuePointData() + { + std::stringstream output; + OtlpFileMetricExporterOptions opts; + opts.backend_options = std::ref(output); + + auto exporter = OtlpFileMetricExporterFactory::Create(opts); + + opentelemetry::sdk::metrics::LastValuePointData last_value_point_data{}; + last_value_point_data.value_ = 10.0; + last_value_point_data.is_lastvalue_valid_ = true; + last_value_point_data.sample_ts_ = opentelemetry::common::SystemTimestamp{}; + opentelemetry::sdk::metrics::LastValuePointData last_value_point_data2{}; + last_value_point_data2.value_ = static_cast(20); + last_value_point_data2.is_lastvalue_valid_ = true; + last_value_point_data2.sample_ts_ = opentelemetry::common::SystemTimestamp{}; + opentelemetry::sdk::metrics::MetricData metric_data{ + opentelemetry::sdk::metrics::InstrumentDescriptor{ + "metrics_library_name", "metrics_description", "metrics_unit", + opentelemetry::sdk::metrics::InstrumentType::kObservableGauge, + opentelemetry::sdk::metrics::InstrumentValueType::kDouble}, + opentelemetry::sdk::metrics::AggregationTemporality::kDelta, + opentelemetry::common::SystemTimestamp{}, opentelemetry::common::SystemTimestamp{}, + std::vector{ + {opentelemetry::sdk::metrics::PointAttributes{{"a1", "b1"}}, last_value_point_data}, + {opentelemetry::sdk::metrics::PointAttributes{{"a2", "b2"}}, last_value_point_data2}}}; + + opentelemetry::sdk::metrics::ResourceMetrics data; + auto resource = opentelemetry::sdk::resource::Resource::Create( + opentelemetry::sdk::resource::ResourceAttributes{}); + data.resource_ = &resource; + auto scope = opentelemetry::sdk::instrumentationscope::InstrumentationScope::Create( + "library_name", "1.5.0"); + data.scope_metric_data_ = std::vector{ + {scope.get(), std::vector{metric_data}}}; + + auto result = exporter->Export(data); + EXPECT_EQ(result, opentelemetry::sdk::common::ExportResult::kSuccess); + + exporter->ForceFlush(); + + { + auto check_json = nlohmann::json::parse(output.str(), nullptr, false); + + auto resource_metrics = *check_json["resourceMetrics"].begin(); + auto scope_metrics = *resource_metrics["scopeMetrics"].begin(); + auto scope = scope_metrics["scope"]; + EXPECT_EQ("library_name", scope["name"].get()); + EXPECT_EQ("1.5.0", scope["version"].get()); + + auto metric = *scope_metrics["metrics"].begin(); + EXPECT_EQ("metrics_library_name", metric["name"].get()); + EXPECT_EQ("metrics_description", metric["description"].get()); + EXPECT_EQ("metrics_unit", metric["unit"].get()); + + auto data_points = metric["gauge"]["dataPoints"]; + EXPECT_EQ(10.0, data_points[0]["asDouble"].get()); + EXPECT_EQ(20l, JsonToInteger(data_points[1]["asInt"])); + } + } + + void ExportJsonIntegrationTestExportHistogramPointData() + { + std::stringstream output; + OtlpFileMetricExporterOptions opts; + opts.backend_options = std::ref(output); + + auto exporter = OtlpFileMetricExporterFactory::Create(opts); + + opentelemetry::sdk::metrics::HistogramPointData histogram_point_data{}; + histogram_point_data.boundaries_ = {10.1, 20.2, 30.2}; + histogram_point_data.count_ = 3; + histogram_point_data.counts_ = {200, 300, 400, 500}; + histogram_point_data.sum_ = 900.5; + histogram_point_data.min_ = 1.8; + histogram_point_data.max_ = 19.0; + opentelemetry::sdk::metrics::HistogramPointData histogram_point_data2{}; + histogram_point_data2.boundaries_ = {10.0, 20.0, 30.0}; + histogram_point_data2.count_ = 3; + histogram_point_data2.counts_ = {200, 300, 400, 500}; + histogram_point_data2.sum_ = static_cast(900); + + opentelemetry::sdk::metrics::MetricData metric_data{ + opentelemetry::sdk::metrics::InstrumentDescriptor{ + "metrics_library_name", "metrics_description", "metrics_unit", + opentelemetry::sdk::metrics::InstrumentType::kHistogram, + opentelemetry::sdk::metrics::InstrumentValueType::kDouble}, + opentelemetry::sdk::metrics::AggregationTemporality::kDelta, + opentelemetry::common::SystemTimestamp{}, opentelemetry::common::SystemTimestamp{}, + std::vector{ + {opentelemetry::sdk::metrics::PointAttributes{{"a1", "b1"}}, histogram_point_data}, + {opentelemetry::sdk::metrics::PointAttributes{{"a2", "b2"}}, histogram_point_data2}}}; + + opentelemetry::sdk::metrics::ResourceMetrics data; + auto resource = opentelemetry::sdk::resource::Resource::Create( + opentelemetry::sdk::resource::ResourceAttributes{}); + data.resource_ = &resource; + auto scope = opentelemetry::sdk::instrumentationscope::InstrumentationScope::Create( + "library_name", "1.5.0"); + data.scope_metric_data_ = std::vector{ + {scope.get(), std::vector{metric_data}}}; + + auto result = exporter->Export(data); + EXPECT_EQ(result, opentelemetry::sdk::common::ExportResult::kSuccess); + + exporter->ForceFlush(); + + { + auto check_json = nlohmann::json::parse(output.str(), nullptr, false); + + auto resource_metrics = *check_json["resourceMetrics"].begin(); + auto scope_metrics = *resource_metrics["scopeMetrics"].begin(); + auto scope = scope_metrics["scope"]; + EXPECT_EQ("library_name", scope["name"].get()); + EXPECT_EQ("1.5.0", scope["version"].get()); + + auto metric = *scope_metrics["metrics"].begin(); + EXPECT_EQ("metrics_library_name", metric["name"].get()); + EXPECT_EQ("metrics_description", metric["description"].get()); + EXPECT_EQ("metrics_unit", metric["unit"].get()); + + auto data_points = metric["histogram"]["dataPoints"]; + EXPECT_EQ(3, JsonToInteger(data_points[0]["count"])); + EXPECT_EQ(900.5, data_points[0]["sum"].get()); + EXPECT_EQ(1.8, data_points[0]["min"].get()); + EXPECT_EQ(19, data_points[0]["max"].get()); + EXPECT_EQ(4, data_points[0]["bucketCounts"].size()); + if (4 == data_points[0]["bucketCounts"].size()) + { + EXPECT_EQ(200, JsonToInteger(data_points[0]["bucketCounts"][0])); + EXPECT_EQ(300, JsonToInteger(data_points[0]["bucketCounts"][1])); + EXPECT_EQ(400, JsonToInteger(data_points[0]["bucketCounts"][2])); + EXPECT_EQ(500, JsonToInteger(data_points[0]["bucketCounts"][3])); + } + EXPECT_EQ(3, data_points[0]["explicitBounds"].size()); + if (3 == data_points[0]["explicitBounds"].size()) + { + EXPECT_EQ(10.1, data_points[0]["explicitBounds"][0].get()); + EXPECT_EQ(20.2, data_points[0]["explicitBounds"][1].get()); + EXPECT_EQ(30.2, data_points[0]["explicitBounds"][2].get()); + } + + EXPECT_EQ(3, JsonToInteger(data_points[1]["count"])); + EXPECT_EQ(900.0, data_points[1]["sum"].get()); + EXPECT_EQ(4, data_points[1]["bucketCounts"].size()); + if (4 == data_points[1]["bucketCounts"].size()) + { + EXPECT_EQ(200, JsonToInteger(data_points[1]["bucketCounts"][0])); + EXPECT_EQ(300, JsonToInteger(data_points[1]["bucketCounts"][1])); + EXPECT_EQ(400, JsonToInteger(data_points[1]["bucketCounts"][2])); + EXPECT_EQ(500, JsonToInteger(data_points[1]["bucketCounts"][3])); + } + EXPECT_EQ(3, data_points[1]["explicitBounds"].size()); + if (3 == data_points[1]["explicitBounds"].size()) + { + EXPECT_EQ(10.0, data_points[1]["explicitBounds"][0].get()); + EXPECT_EQ(20.0, data_points[1]["explicitBounds"][1].get()); + EXPECT_EQ(30.0, data_points[1]["explicitBounds"][2].get()); + } + } + } +}; + +TEST(OtlpFileMetricExporterTest, Shutdown) +{ + auto exporter = std::unique_ptr( + new OtlpFileMetricExporter()); + ASSERT_TRUE(exporter->Shutdown()); + auto result = exporter->Export(opentelemetry::sdk::metrics::ResourceMetrics{}); + EXPECT_EQ(result, opentelemetry::sdk::common::ExportResult::kFailure); +} + +TEST_F(OtlpFileMetricExporterTestPeer, ExportJsonIntegrationTestSumPointDataSync) +{ + ExportJsonIntegrationTestExportSumPointData(); +} + +TEST_F(OtlpFileMetricExporterTestPeer, ExportJsonIntegrationTestLastValuePointDataSync) +{ + ExportJsonIntegrationTestExportLastValuePointData(); +} + +TEST_F(OtlpFileMetricExporterTestPeer, ExportJsonIntegrationTestHistogramPointDataSync) +{ + ExportJsonIntegrationTestExportHistogramPointData(); +} + +// Test Preferred aggregtion temporality selection +TEST_F(OtlpFileMetricExporterTestPeer, PreferredAggergationTemporality) +{ + // Cummulative aggregation selector : use cummulative aggregation for all instruments. + std::unique_ptr exporter(new OtlpFileMetricExporter()); + EXPECT_EQ(GetOptions(exporter).aggregation_temporality, + PreferredAggregationTemporality::kCumulative); + auto cumm_selector = + OtlpMetricUtils::ChooseTemporalitySelector(GetOptions(exporter).aggregation_temporality); + EXPECT_EQ(cumm_selector(opentelemetry::sdk::metrics::InstrumentType::kCounter), + opentelemetry::sdk::metrics::AggregationTemporality::kCumulative); + EXPECT_EQ(cumm_selector(opentelemetry::sdk::metrics::InstrumentType::kHistogram), + opentelemetry::sdk::metrics::AggregationTemporality::kCumulative); + EXPECT_EQ(cumm_selector(opentelemetry::sdk::metrics::InstrumentType::kUpDownCounter), + opentelemetry::sdk::metrics::AggregationTemporality::kCumulative); + EXPECT_EQ(cumm_selector(opentelemetry::sdk::metrics::InstrumentType::kObservableCounter), + opentelemetry::sdk::metrics::AggregationTemporality::kCumulative); + EXPECT_EQ(cumm_selector(opentelemetry::sdk::metrics::InstrumentType::kObservableGauge), + opentelemetry::sdk::metrics::AggregationTemporality::kCumulative); + EXPECT_EQ(cumm_selector(opentelemetry::sdk::metrics::InstrumentType::kObservableUpDownCounter), + opentelemetry::sdk::metrics::AggregationTemporality::kCumulative); + + // LowMemory aggregation selector use: + // - cummulative aggregtion for Counter and Histogram + // - delta aggregation for up-down counter, observable counter, observable gauge, observable + // up-down counter + OtlpFileMetricExporterOptions opts2; + opts2.aggregation_temporality = PreferredAggregationTemporality::kLowMemory; + std::unique_ptr exporter2(new OtlpFileMetricExporter(opts2)); + EXPECT_EQ(GetOptions(exporter2).aggregation_temporality, + PreferredAggregationTemporality::kLowMemory); + auto lowmemory_selector = + OtlpMetricUtils::ChooseTemporalitySelector(GetOptions(exporter2).aggregation_temporality); + EXPECT_EQ(lowmemory_selector(opentelemetry::sdk::metrics::InstrumentType::kCounter), + opentelemetry::sdk::metrics::AggregationTemporality::kDelta); + EXPECT_EQ(lowmemory_selector(opentelemetry::sdk::metrics::InstrumentType::kHistogram), + opentelemetry::sdk::metrics::AggregationTemporality::kDelta); + + EXPECT_EQ(lowmemory_selector(opentelemetry::sdk::metrics::InstrumentType::kUpDownCounter), + opentelemetry::sdk::metrics::AggregationTemporality::kCumulative); + EXPECT_EQ(lowmemory_selector(opentelemetry::sdk::metrics::InstrumentType::kObservableCounter), + opentelemetry::sdk::metrics::AggregationTemporality::kCumulative); + EXPECT_EQ(lowmemory_selector(opentelemetry::sdk::metrics::InstrumentType::kObservableGauge), + opentelemetry::sdk::metrics::AggregationTemporality::kCumulative); + EXPECT_EQ( + lowmemory_selector(opentelemetry::sdk::metrics::InstrumentType::kObservableUpDownCounter), + opentelemetry::sdk::metrics::AggregationTemporality::kCumulative); + + // Delta aggregation selector use: + // - delta aggregtion for Counter, Histogram, Observable Counter, Observable Gauge + // - cummulative aggregation for up-down counter, observable up-down counter + OtlpFileMetricExporterOptions opts3; + opts3.aggregation_temporality = PreferredAggregationTemporality::kDelta; + std::unique_ptr exporter3(new OtlpFileMetricExporter(opts3)); + EXPECT_EQ(GetOptions(exporter3).aggregation_temporality, PreferredAggregationTemporality::kDelta); + auto delta_selector = + OtlpMetricUtils::ChooseTemporalitySelector(GetOptions(exporter3).aggregation_temporality); + EXPECT_EQ(delta_selector(opentelemetry::sdk::metrics::InstrumentType::kCounter), + opentelemetry::sdk::metrics::AggregationTemporality::kDelta); + EXPECT_EQ(delta_selector(opentelemetry::sdk::metrics::InstrumentType::kHistogram), + opentelemetry::sdk::metrics::AggregationTemporality::kDelta); + EXPECT_EQ(delta_selector(opentelemetry::sdk::metrics::InstrumentType::kObservableCounter), + opentelemetry::sdk::metrics::AggregationTemporality::kDelta); + EXPECT_EQ(delta_selector(opentelemetry::sdk::metrics::InstrumentType::kObservableGauge), + opentelemetry::sdk::metrics::AggregationTemporality::kDelta); + + EXPECT_EQ(delta_selector(opentelemetry::sdk::metrics::InstrumentType::kUpDownCounter), + opentelemetry::sdk::metrics::AggregationTemporality::kCumulative); + EXPECT_EQ(delta_selector(opentelemetry::sdk::metrics::InstrumentType::kObservableUpDownCounter), + opentelemetry::sdk::metrics::AggregationTemporality::kCumulative); +} + +} // namespace otlp +} // namespace exporter +OPENTELEMETRY_END_NAMESPACE