From 5e62859ffce873fd8f996d1c72d7a87f8f61f07b Mon Sep 17 00:00:00 2001 From: Maxime Leblanc Date: Sun, 27 Oct 2024 17:50:41 +0100 Subject: [PATCH] [EXPORTER] Elastic Search exporter follow ECS guidelines (#3107) * [EXPORTERS]: elastic search log message within `message` key instead of `body` According to ECS logging reference https://www.elastic.co/guide/en/ecs/8.11/ecs-base.html#field-message Refs #3091 * [EXPORTERS]: elastic search set severity within `log.level` key instead of `severity` According to ECS logging reference https://www.elastic.co/guide/en/ecs/8.11/ecs-log.html#field-log-level Refs #3091 * [EXPORTERS]: elastic search set timestamp within `@timestamp` instead of `timestamp` Also changes the format to be a Date string. According to ECS logging reference https://www.elastic.co/guide/en/ecs/8.11/ecs-base.html#field-timestamp Refs #3091 * [EXPORTERS]: elastic search set instrumentation scope within `log.logger` instead of `name` According to ECS logging reference https://www.elastic.co/guide/en/ecs/8.11/ecs-log.html#field-log-logger Refs #3091 * EXPORTERS]: elastic search recorable constructor sets `ecs.version` field to 8.11.0 According to ECS guidelines this field is mandatory https://www.elastic.co/guide/en/ecs/8.11/ecs-guidelines.html Refs #3091 * [EXPORTERS]: elastic search put attributes in json root instead of under `attributes` This allows user to set other fields that are part of the [ECS log documentation](https://www.elastic.co/guide/en/ecs/8.11/ecs-log.html). For instance, it allows to have an attribute with key `log.file`, that will, thanks to `nlohmann::json`, appear as : ``` { "log": { "file": "xxx" } } ``` Closes #3091 --- .../elasticsearch/es_log_recordable.h | 2 + .../elasticsearch/src/es_log_recordable.cc | 50 ++++++++++++++++--- 2 files changed, 46 insertions(+), 6 deletions(-) diff --git a/exporters/elasticsearch/include/opentelemetry/exporters/elasticsearch/es_log_recordable.h b/exporters/elasticsearch/include/opentelemetry/exporters/elasticsearch/es_log_recordable.h index af4ccb16ef..52d24cada6 100644 --- a/exporters/elasticsearch/include/opentelemetry/exporters/elasticsearch/es_log_recordable.h +++ b/exporters/elasticsearch/include/opentelemetry/exporters/elasticsearch/es_log_recordable.h @@ -42,6 +42,8 @@ class ElasticSearchRecordable final : public sdk::logs::Recordable void WriteValue(const opentelemetry::common::AttributeValue &value, const std::string &name); public: + ElasticSearchRecordable() noexcept; + /** * Returns a JSON object contain the log information */ diff --git a/exporters/elasticsearch/src/es_log_recordable.cc b/exporters/elasticsearch/src/es_log_recordable.cc index 663691273b..56d7cea71b 100644 --- a/exporters/elasticsearch/src/es_log_recordable.cc +++ b/exporters/elasticsearch/src/es_log_recordable.cc @@ -197,6 +197,11 @@ void ElasticSearchRecordable::WriteValue(const opentelemetry::common::AttributeV } } +ElasticSearchRecordable::ElasticSearchRecordable() noexcept : sdk::logs::Recordable() +{ + json_["ecs"]["version"] = "8.11.0"; +} + nlohmann::json ElasticSearchRecordable::GetJSON() noexcept { return json_; @@ -205,7 +210,38 @@ nlohmann::json ElasticSearchRecordable::GetJSON() noexcept void ElasticSearchRecordable::SetTimestamp( opentelemetry::common::SystemTimestamp timestamp) noexcept { - json_["timestamp"] = timestamp.time_since_epoch().count(); + const std::chrono::system_clock::time_point timePoint{timestamp}; + + // If built with with at least cpp 20 then use std::format + // Otherwise use the old style to format the timestamp in UTC +#if __cplusplus >= 202002L + const std::string dateStr = std::format("{:%FT%T%Ez}", timePoint); +#else + const static int dateToSecondsSize = 19; + const static int millisecondsSize = 8; + const static int timeZoneSize = 1; + const static int dateSize = dateToSecondsSize + millisecondsSize + timeZoneSize; + + std::time_t time = std::chrono::system_clock::to_time_t(timePoint); + std::tm tm = *std::gmtime(&time); + + char bufferDate[dateSize]; // example: 2024-10-18T07:26:00.123456Z + std::strftime(bufferDate, sizeof(bufferDate), "%Y-%m-%dT%H:%M:%S", &tm); + auto microseconds = + std::chrono::duration_cast(timePoint.time_since_epoch()) % + std::chrono::seconds(1); + + char bufferMilliseconds[millisecondsSize]; + std::snprintf(bufferMilliseconds, sizeof(bufferMilliseconds), ".%06ld", + static_cast(microseconds.count())); + + std::strcat(bufferDate, bufferMilliseconds); + std::strcat(bufferDate, "Z"); + + const std::string dateStr(bufferDate); +#endif + + json_["@timestamp"] = dateStr; } void ElasticSearchRecordable::SetObservedTimestamp( @@ -216,23 +252,25 @@ void ElasticSearchRecordable::SetObservedTimestamp( void ElasticSearchRecordable::SetSeverity(opentelemetry::logs::Severity severity) noexcept { + auto &severityField = json_["log"]["level"]; + // Convert the severity enum to a string std::uint32_t severity_index = static_cast(severity); if (severity_index >= std::extent::value) { std::stringstream sout; sout << "Invalid severity(" << severity_index << ")"; - json_["severity"] = sout.str(); + severityField = sout.str(); } else { - json_["severity"] = opentelemetry::logs::SeverityNumToText[severity_index]; + severityField = opentelemetry::logs::SeverityNumToText[severity_index]; } } void ElasticSearchRecordable::SetBody(const opentelemetry::common::AttributeValue &message) noexcept { - WriteValue(message, "body"); + WriteValue(message, "message"); } void ElasticSearchRecordable::SetTraceId(const opentelemetry::trace::TraceId &trace_id) noexcept @@ -275,7 +313,7 @@ void ElasticSearchRecordable::SetAttribute( nostd::string_view key, const opentelemetry::common::AttributeValue &value) noexcept { - WriteKeyValue(key, value, "attributes"); + WriteValue(value, key.data()); } void ElasticSearchRecordable::SetResource( @@ -291,7 +329,7 @@ void ElasticSearchRecordable::SetInstrumentationScope( const opentelemetry::sdk::instrumentationscope::InstrumentationScope &instrumentation_scope) noexcept { - json_["name"] = instrumentation_scope.GetName(); + json_["log"]["logger"] = instrumentation_scope.GetName(); } } // namespace logs