Skip to content

Commit

Permalink
Allow comments in hashes and before EOF (#16058)
Browse files Browse the repository at this point in the history 
In the grammar definitions for hashes, `whitespace` was replaced with `cs` to allow either whitespace _or_ comments. 
Additionally, the grammar definition for comments was previously required to end with a newline, now it can end with a newline _or_ EOF, using the "not anything" treetop rule `!.`.

Co-authored-by: Jonas Lundholm Bertelsen <[email protected]>
  • Loading branch information
@jonaslb
jonaslb and jonaslb authored May 8, 2024
1 parent 3068934 commit 0d6ba8d
Show file tree
Hide file tree
Showing 5 changed files with 233 additions and 52 deletions.
111 changes: 87 additions & 24 deletions logstash-core/lib/logstash/compiler/lscl/lscl_grammar.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,91 @@ def _nt_config
r0
end

module NewlineOrEoi0
end

def _nt_newline_or_eoi
start_index = index
if node_cache[:newline_or_eoi].has_key?(index)
cached = node_cache[:newline_or_eoi][index]
if cached
node_cache[:newline_or_eoi][index] = cached = SyntaxNode.new(input, index...(index + 1)) if cached == true
@index = cached.interval.end
end
return cached
end

i0 = index
i1, s1 = index, []
if (match_len = has_terminal?("\r", false, index))
r3 = true
@index += match_len
else
terminal_parse_failure('"\\r"')
r3 = nil
end
if r3
r2 = r3
else
r2 = instantiate_node(SyntaxNode,input, index...index)
end
s1 << r2
if r2
if (match_len = has_terminal?("\n", false, index))
r4 = true
@index += match_len
else
terminal_parse_failure('"\\n"')
r4 = nil
end
s1 << r4
end
if s1.last
r1 = instantiate_node(SyntaxNode,input, i1...index, s1)
r1.extend(NewlineOrEoi0)
else
@index = i1
r1 = nil
end
if r1
r1 = SyntaxNode.new(input, (index-1)...index) if r1 == true
r0 = r1
else
i5 = index
if index < input_length
r6 = true
@index += 1
else
terminal_parse_failure("any character")
r6 = nil
end
if r6
@index = i5
r5 = nil
terminal_parse_failure("any character", true)
else
@terminal_failures.pop
@index = i5
r5 = instantiate_node(SyntaxNode,input, index...index)
end
if r5
r5 = SyntaxNode.new(input, (index-1)...index) if r5 == true
r0 = r5
else
@index = i0
r0 = nil
end
end

node_cache[:newline_or_eoi][start_index] = r0

r0
end

module Comment0
def newline_or_eoi
elements[3]
end
end

def _nt_comment
Expand Down Expand Up @@ -132,29 +216,8 @@ def _nt_comment
r5 = instantiate_node(SyntaxNode,input, i5...index, s5)
s1 << r5
if r5
if (match_len = has_terminal?("\r", false, index))
r8 = true
@index += match_len
else
terminal_parse_failure('"\\r"')
r8 = nil
end
if r8
r7 = r8
else
r7 = instantiate_node(SyntaxNode,input, index...index)
end
r7 = _nt_newline_or_eoi
s1 << r7
if r7
if (match_len = has_terminal?("\n", false, index))
r9 = true
@index += match_len
else
terminal_parse_failure('"\\n"')
r9 = nil
end
s1 << r9
end
end
end
end
Expand Down Expand Up @@ -1674,7 +1737,7 @@ def _nt_hash
end

module Hashentries0
def whitespace
def cs
elements[0]
end

Expand Down Expand Up @@ -1708,7 +1771,7 @@ def _nt_hashentries
s2, i2 = [], index
loop do
i3, s3 = index, []
r4 = _nt_whitespace
r4 = _nt_cs
s3 << r4
if r4
r5 = _nt_hashentry
Expand Down
9 changes: 7 additions & 2 deletions logstash-core/lib/logstash/compiler/lscl/lscl_grammar.treetop
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,13 @@ grammar LogStashCompilerLSCLGrammar
(cs plugin_section)* cs <LogStash::Compiler::LSCL::AST::Config>
end

rule newline_or_eoi
# `!.` is a negative lookahead for 'anything', i.e. it matches at the end of input.
("\r"? "\n") / !.
end

rule comment
(whitespace? "#" [^\r\n]* "\r"? "\n")+ <LogStash::Compiler::LSCL::AST::Comment>
(whitespace? "#" [^\r\n]* newline_or_eoi)+ <LogStash::Compiler::LSCL::AST::Comment>
end

rule cs
Expand Down Expand Up @@ -114,7 +119,7 @@ grammar LogStashCompilerLSCLGrammar
end

rule hashentries
hashentry (whitespace hashentry)*
hashentry (cs hashentry)*
<LogStash::Compiler::LSCL::AST::HashEntries>
end

Expand Down
111 changes: 87 additions & 24 deletions logstash-core/lib/logstash/config/grammar.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,91 @@ def _nt_config
r0
end

module NewlineOrEoi0
end

def _nt_newline_or_eoi
start_index = index
if node_cache[:newline_or_eoi].has_key?(index)
cached = node_cache[:newline_or_eoi][index]
if cached
node_cache[:newline_or_eoi][index] = cached = SyntaxNode.new(input, index...(index + 1)) if cached == true
@index = cached.interval.end
end
return cached
end

i0 = index
i1, s1 = index, []
if (match_len = has_terminal?("\r", false, index))
r3 = true
@index += match_len
else
terminal_parse_failure('"\\r"')
r3 = nil
end
if r3
r2 = r3
else
r2 = instantiate_node(SyntaxNode,input, index...index)
end
s1 << r2
if r2
if (match_len = has_terminal?("\n", false, index))
r4 = true
@index += match_len
else
terminal_parse_failure('"\\n"')
r4 = nil
end
s1 << r4
end
if s1.last
r1 = instantiate_node(SyntaxNode,input, i1...index, s1)
r1.extend(NewlineOrEoi0)
else
@index = i1
r1 = nil
end
if r1
r1 = SyntaxNode.new(input, (index-1)...index) if r1 == true
r0 = r1
else
i5 = index
if index < input_length
r6 = true
@index += 1
else
terminal_parse_failure("any character")
r6 = nil
end
if r6
@index = i5
r5 = nil
terminal_parse_failure("any character", true)
else
@terminal_failures.pop
@index = i5
r5 = instantiate_node(SyntaxNode,input, index...index)
end
if r5
r5 = SyntaxNode.new(input, (index-1)...index) if r5 == true
r0 = r5
else
@index = i0
r0 = nil
end
end

node_cache[:newline_or_eoi][start_index] = r0

r0
end

module Comment0
def newline_or_eoi
elements[3]
end
end

def _nt_comment
Expand Down Expand Up @@ -156,29 +240,8 @@ def _nt_comment
r5 = instantiate_node(SyntaxNode,input, i5...index, s5)
s1 << r5
if r5
if (match_len = has_terminal?("\r", false, index))
r8 = true
@index += match_len
else
terminal_parse_failure('"\\r"')
r8 = nil
end
if r8
r7 = r8
else
r7 = instantiate_node(SyntaxNode,input, index...index)
end
r7 = _nt_newline_or_eoi
s1 << r7
if r7
if (match_len = has_terminal?("\n", false, index))
r9 = true
@index += match_len
else
terminal_parse_failure('"\\n"')
r9 = nil
end
s1 << r9
end
end
end
end
Expand Down Expand Up @@ -1698,7 +1761,7 @@ def _nt_hash
end

module Hashentries0
def whitespace
def cs
elements[0]
end

Expand Down Expand Up @@ -1732,7 +1795,7 @@ def _nt_hashentries
s2, i2 = [], index
loop do
i3, s3 = index, []
r4 = _nt_whitespace
r4 = _nt_cs
s3 << r4
if r4
r5 = _nt_hashentry
Expand Down
9 changes: 7 additions & 2 deletions logstash-core/lib/logstash/config/grammar.treetop
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,13 @@ grammar LogStashConfig
cs plugin_section cs (cs plugin_section)* cs <LogStash::Config::AST::Config>
end

rule newline_or_eoi
# `!.` is a negative lookahead for 'anything', i.e. it matches at the end of input.
("\r"? "\n") / !.
end

rule comment
(whitespace? "#" [^\r\n]* "\r"? "\n")+ <LogStash::Config::AST::Comment>
(whitespace? "#" [^\r\n]* newline_or_eoi)+ <LogStash::Config::AST::Comment>
end

rule cs
Expand Down Expand Up @@ -114,7 +119,7 @@ grammar LogStashConfig
end

rule hashentries
hashentry (whitespace hashentry)*
hashentry (cs hashentry)*
<LogStash::Config::AST::HashEntries>
end

Expand Down
45 changes: 45 additions & 0 deletions logstash-core/spec/logstash/config/config_ast_spec.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -211,6 +211,51 @@
end
end
end

context "commentary in odd places" do
let(:config) { %q(
# early
input # bla
{ # bla
plugin # bla
# bla
{ # bla
# bla
arg # bla
=> # bla
bareword # bla
# bla
hasharg => {
#preentry
"firstkey" => "firstvalue"
# bla
"middle" # bla
# bla
=> # bla
# bla
"midvalue" # bla
# bla
"lastkey" => "lastvalue"
# bla
}
} # bla
# bla
} # bla
# comment ending in EOF:
).strip }
subject { LogStashConfigParser.new }
def line_to_source(*args); end
def plugin(*args); end
it "hasn't got a trailing newline in the test config" do
# just to be clear that the last comment is supposed to end with EOF not \n
expect(config).not_to(end_with "\n")
end
it "should permit the comments" do
result = subject.parse(config)
expect(result).not_to(be_nil)
expect { eval(result.compile) }.not_to(raise_error)
end
end
end

context "when using two plugin sections of the same type" do
Expand Down

0 comments on commit 0d6ba8d

Please sign in to comment.