From 8854eedf80c5b3894af23a993e1cb8b1c61f3b67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Magnus=20B=C3=A4ck?= Date: Sun, 17 Apr 2016 20:38:47 +0200 Subject: [PATCH 1/2] Add 'metadata' option to include @metadata field in output Just like the rubydebug codec, being able to include the @metadata field in the output of the json_lines codec can be useful, for example for debugging or test tools that prefer machine-readable output. The new option is immediately marked as deprecated to discourage its use and to make it easier to replace it with something better once we figure out what that would be. Since the Event class's constructor mutates the input hash I had to refactor some existing tests to send in a clone of the hash since it'll otherwise end up without a @metadata field. --- docs/index.asciidoc | 9 +++++++++ lib/logstash/codecs/json_lines.rb | 26 +++++++++++++++++++++++--- spec/codecs/json_lines_spec.rb | 30 ++++++++++++++++++++++++++---- 3 files changed, 58 insertions(+), 7 deletions(-) diff --git a/docs/index.asciidoc b/docs/index.asciidoc index d3fc8a2..60fffac 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -36,6 +36,7 @@ Therefore this codec cannot work with line oriented inputs. |Setting |Input type|Required | <> |<>, one of `["ASCII-8BIT", "UTF-8", "US-ASCII", "Big5", "Big5-HKSCS", "Big5-UAO", "CP949", "Emacs-Mule", "EUC-JP", "EUC-KR", "EUC-TW", "GB2312", "GB18030", "GBK", "ISO-8859-1", "ISO-8859-2", "ISO-8859-3", "ISO-8859-4", "ISO-8859-5", "ISO-8859-6", "ISO-8859-7", "ISO-8859-8", "ISO-8859-9", "ISO-8859-10", "ISO-8859-11", "ISO-8859-13", "ISO-8859-14", "ISO-8859-15", "ISO-8859-16", "KOI8-R", "KOI8-U", "Shift_JIS", "UTF-16BE", "UTF-16LE", "UTF-32BE", "UTF-32LE", "Windows-31J", "Windows-1250", "Windows-1251", "Windows-1252", "IBM437", "IBM737", "IBM775", "CP850", "IBM852", "CP852", "IBM855", "CP855", "IBM857", "IBM860", "IBM861", "IBM862", "IBM863", "IBM864", "IBM865", "IBM866", "IBM869", "Windows-1258", "GB1988", "macCentEuro", "macCroatian", "macCyrillic", "macGreek", "macIceland", "macRoman", "macRomania", "macThai", "macTurkish", "macUkraine", "CP950", "CP951", "IBM037", "stateless-ISO-2022-JP", "eucJP-ms", "CP51932", "EUC-JIS-2004", "GB12345", "ISO-2022-JP", "ISO-2022-JP-2", "CP50220", "CP50221", "Windows-1256", "Windows-1253", "Windows-1255", "Windows-1254", "TIS-620", "Windows-874", "Windows-1257", "MacJapanese", "UTF-7", "UTF8-MAC", "UTF-16", "UTF-32", "UTF8-DoCoMo", "SJIS-DoCoMo", "UTF8-KDDI", "SJIS-KDDI", "ISO-2022-JP-KDDI", "stateless-ISO-2022-JP-KDDI", "UTF8-SoftBank", "SJIS-SoftBank", "BINARY", "CP437", "CP737", "CP775", "IBM850", "CP857", "CP860", "CP861", "CP862", "CP863", "CP864", "CP865", "CP866", "CP869", "CP1258", "Big5-HKSCS:2008", "ebcdic-cp-us", "eucJP", "euc-jp-ms", "EUC-JISX0213", "eucKR", "eucTW", "EUC-CN", "eucCN", "CP936", "ISO2022-JP", "ISO2022-JP2", "ISO8859-1", "ISO8859-2", "ISO8859-3", "ISO8859-4", "ISO8859-5", "ISO8859-6", "CP1256", "ISO8859-7", "CP1253", "ISO8859-8", "CP1255", "ISO8859-9", "CP1254", "ISO8859-10", "ISO8859-11", "CP874", "ISO8859-13", "CP1257", "ISO8859-14", "ISO8859-15", "ISO8859-16", "CP878", "MacJapan", "ASCII", "ANSI_X3.4-1968", "646", "CP65000", "CP65001", "UTF-8-MAC", "UTF-8-HFS", "UCS-2BE", "UCS-4BE", "UCS-4LE", "CP932", "csWindows31J", "SJIS", "PCK", "CP1250", "CP1251", "CP1252", "external", "locale"]`|No | <> |<>|No +| <> |<>|No |=======================================================================   @@ -64,4 +65,12 @@ For nxlog users, you'll want to set this to `CP1252` Change the delimiter that separates lines +[id="plugins-{type}s-{plugin}-metadata"] +===== `metadata` + * Value type is <> + * Default value is `false` + * This option is deprecated and may be removed in a future major release of the plugin. + +If true, the event's metadata (the `@metadata` field and any +subfields) will be included when used as an output codec. diff --git a/lib/logstash/codecs/json_lines.rb b/lib/logstash/codecs/json_lines.rb index 4c801ee..8373a84 100644 --- a/lib/logstash/codecs/json_lines.rb +++ b/lib/logstash/codecs/json_lines.rb @@ -28,12 +28,22 @@ class LogStash::Codecs::JSONLines < LogStash::Codecs::Base # Change the delimiter that separates lines config :delimiter, :validate => :string, :default => "\n" + # If true, the event's metadata (the `@metadata` field) will be + # included when used as an output codec. + config :metadata, :validate => :boolean, :default => false, + :deprecated => "Option may be removed in a future release." + public def register @buffer = FileWatch::BufferedTokenizer.new(@delimiter) @converter = LogStash::Util::Charset.new(@charset) @converter.logger = @logger + if @metadata + @encoder = method(:encode_with_metadata) + else + @encoder = method(:encode_default) + end end def decode(data, &block) @@ -43,9 +53,7 @@ def decode(data, &block) end def encode(event) - # Tack on a @delimiter for now because previously most of logstash's JSON - # outputs emitted one per line, and whitespace is OK in json. - @on_event.call(event, "#{event.to_json}#{@delimiter}") + @encoder.call(event) end def flush(&block) @@ -57,6 +65,18 @@ def flush(&block) private + def encode_default(event) + # Tack on a @delimiter for now because previously most of logstash's JSON + # outputs emitted one per line, and whitespace is OK in json. + @on_event.call(event, "#{event.to_json}#{@delimiter}") + end + + def encode_with_metadata(event) + # Tack on a @delimiter for now because previously most of logstash's JSON + # outputs emitted one per line, and whitespace is OK in json. + @on_event.call(event, "#{LogStash::Json.dump(event.to_hash_with_metadata)}#{@delimiter}") + end + # from_json_parse uses the Event#from_json method to deserialize and directly produce events def from_json_parse(json, &block) LogStash::Event.from_json(json).each { |event| yield event } diff --git a/spec/codecs/json_lines_spec.rb b/spec/codecs/json_lines_spec.rb index c5b6b52..951f11e 100644 --- a/spec/codecs/json_lines_spec.rb +++ b/spec/codecs/json_lines_spec.rb @@ -120,13 +120,13 @@ end context "#encode" do - let(:data) { { LogStash::Event::TIMESTAMP => "2015-12-07T11:37:00.000Z", "foo" => "bar", "baz" => {"bah" => ["a","b","c"]}} } - let(:event) { LogStash::Event.new(data) } + let(:data) { { LogStash::Event::TIMESTAMP => "2015-12-07T11:37:00.000Z", "foo" => "bar", "baz" => {"bah" => ["a","b","c"]}, "@metadata" => {"metafield" => "metavalue"} } } + let(:event) { LogStash::Event.new(data.clone) } it "should return json data" do got_event = false subject.on_event do |e, d| - insist { d } == "#{LogStash::Event.new(data).to_json}\n" + insist { d } == "#{LogStash::Event.new(data.clone).to_json}\n" insist { LogStash::Json.load(d)["foo"] } == data["foo"] insist { LogStash::Json.load(d)["baz"] } == data["baz"] insist { LogStash::Json.load(d)["bah"] } == data["bah"] @@ -136,13 +136,35 @@ insist { got_event } end + it "should not include metadata in the output" do + got_event = false + subject.on_event do |e, d| + insist { LogStash::Json.load(d).has_key? "@metadata" } == false + got_event = true + end + subject.encode(event) + insist { got_event } + end + context "when using custom delimiter" do let(:delimiter) { "|" } let(:codec_options) { { "delimiter" => delimiter } } it "should decode multiple lines separated by the delimiter" do subject.on_event do |e, d| - insist { d } == "#{LogStash::Event.new(data).to_json}#{delimiter}" + insist { d } == "#{LogStash::Event.new(data.clone).to_json}#{delimiter}" + end + subject.encode(event) + end + end + + context "when enabling metadata output" do + let(:codec_options) { { "metadata" => true } } + + it "should include the @metadata field" do + subject.on_event do |e, d| + insist { d } == "#{LogStash::Json.dump(LogStash::Event.new(data.clone).to_hash_with_metadata)}\n" + insist { LogStash::Json.load(d)["@metadata"] } == data["@metadata"] end subject.encode(event) end From 5faa9695f527ed4edb12087bf778ac7ee64d4734 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Magnus=20B=C3=A4ck?= Date: Tue, 19 Apr 2016 21:33:57 +0200 Subject: [PATCH 2/2] Bump version to 3.1.0 and add release notes --- CHANGELOG.md | 3 +++ logstash-codec-json_lines.gemspec | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 237ca89..29bfdb9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 3.1.0 + - feature: Add 'metadata' option that enables inclusion of the @metadata field in the output. + ## 3.0.6 - Support flush method, see https://github.com/logstash-plugins/logstash-codec-json_lines/pull/35 diff --git a/logstash-codec-json_lines.gemspec b/logstash-codec-json_lines.gemspec index 8bc529f..2494995 100644 --- a/logstash-codec-json_lines.gemspec +++ b/logstash-codec-json_lines.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-codec-json_lines' - s.version = '3.0.6' + s.version = '3.1.0' s.licenses = ['Apache License (2.0)'] s.summary = "Reads and writes newline-delimited JSON" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"