Finish 2.1.2

README.md

@@ -101,6 +101,8 @@ On a parsing failure, and exception is raised with information that may be usefu
 The [EBNF][] variant used here is based on [W3C](https://w3.org/) [EBNF][] (see {file:etc/ebnf.ebnf EBNF grammar}) as defined in the
 [XML 1.0 recommendation](https://www.w3.org/TR/REC-xml/), with minor extensions:
 
+Note that the grammar includes an optional `[identifer]` in front of rule names, which can be in conflict with the `RANGE` terminal. It is typically not a problem, but if it comes up, try parsing with the `native` parser,  add comments or sequences to disambiguate. EBNF does not have beginning of line checks as all whitespace is treated the same, so the common practice of identifying each rule inherently leads to such ambiguity.
+
 The character set for EBNF is UTF-8.
 
 The general form of a rule is:
@@ -259,7 +261,8 @@ This repository uses [Git Flow](https://github.com/nvie/gitflow) to mange develo
   list in the the `README`. Alphabetical order applies.
 * Do note that in order for us to merge any non-trivial changes (as a rule
   of thumb, additions larger than about 15 lines of code), we need an
-  explicit [public domain dedication][PDD] on record from you.
+  explicit [public domain dedication][PDD] on record from you,
+  which you will be asked to agree to on the first commit to a repo within the organization.
 
 ## License
 This is free and unencumbered public domain software. For more information,

Rakefile

@@ -60,27 +60,27 @@ desc "Build meta files for ABNF, EBNF and ISO EBNF"
 task :meta => %w{lib/ebnf/ebnf/meta.rb lib/ebnf/isoebnf/meta.rb lib/ebnf/abnf/meta.rb lib/ebnf/abnf/core.rb}
 
 file "lib/ebnf/abnf/meta.rb" => "etc/abnf.ebnf" do
-  %x(bin/ebnf --peg -f rb --mod-name ABNFMeta -o lib/ebnf/abnf/meta.rb etc/abnf.ebnf)
+  %x(bin/ebnf --input-format native --peg -f rb --mod-name ABNFMeta -o lib/ebnf/abnf/meta.rb etc/abnf.ebnf)
 end
 
 file "lib/ebnf/abnf/core.rb" => "etc/abnf-core.ebnf" do
-  %x(bin/ebnf -f rb --mod-name ABNFCore -o lib/ebnf/abnf/core.rb etc/abnf-core.ebnf)
+  %x(bin/ebnf --input-format native -f rb --mod-name ABNFCore -o lib/ebnf/abnf/core.rb etc/abnf-core.ebnf)
 end
 
 file "lib/ebnf/ebnf/meta.rb" => "etc/ebnf.peg.rb" do
   %x(cp etc/ebnf.peg.rb lib/ebnf/ebnf/meta.rb)
 end
 
 file "lib/ebnf/isoebnf/meta.rb" => "etc/iso-ebnf.ebnf" do
-  %x(bin/ebnf --peg -f rb --mod-name ISOEBNFMeta -o lib/ebnf/isoebnf/meta.rb etc/iso-ebnf.ebnf)
+  %x(bin/ebnf --input-format native --peg -f rb --mod-name ISOEBNFMeta -o lib/ebnf/isoebnf/meta.rb etc/iso-ebnf.ebnf)
 end
 
 
 # Build SXP output with leading space to allow for Markdown formatting.
 rule ".sxp" => %w{.ebnf} do |t|
   puts "build #{t.name}"
   File.open(t.name, "w") do |f|
-    IO.popen(%(bin/ebnf #{t.source})).each_line do |line|
+    IO.popen(%(bin/ebnf --input-format native #{t.source})).each_line do |line|
       f.puts '    ' + line
     end
   end
@@ -89,32 +89,32 @@ end
 rule ".peg.sxp" => %w{.ebnf} do |t|
   puts "build #{t.name}"
   File.open(t.name, "w") do |f|
-    IO.popen(%(bin/ebnf --peg #{t.source})).each_line do |line|
+    IO.popen(%(bin/ebnf --input-format native --peg #{t.source})).each_line do |line|
       f.puts '    ' + line
     end
   end
 end
 
 rule ".html" => %w{.ebnf} do |t|
   puts "build #{t.name}"
-  %x(bin/ebnf --format html -o #{t.name} #{t.source})
+  %x(bin/ebnf --input-format native --format html -o #{t.name} #{t.source})
 end
 
 file "etc/ebnf.ll1.sxp" => "etc/ebnf.ebnf" do |t|
   puts "build #{t.name}"
   File.open(t.name, "w") do |f|
-    IO.popen(%(bin/ebnf --ll1 ebnf #{t.source})).each_line do |line|
+    IO.popen(%(bin/ebnf --input-format native --ll1 ebnf #{t.source})).each_line do |line|
       f.puts '    ' + line
     end
   end
 end
 
 file "etc/ebnf.peg.rb" => "etc/ebnf.ebnf" do |t|
   puts "build #{t.name}"
-  %x(bin/ebnf --peg --mod-name EBNFMeta -f rb -o etc/ebnf.peg.rb etc/ebnf.ebnf)
+  %x(bin/ebnf --input-format native --peg --mod-name EBNFMeta -f rb -o etc/ebnf.peg.rb etc/ebnf.ebnf)
 end
 
 file "etc/ebnf.ll1.rb" => "etc/ebnf.ebnf" do |t|
   puts "build #{t.name}"
-  %x(bin/ebnf --ll1 ebnf -f rb -o etc/ebnf.ll1.rb etc/ebnf.ebnf)
+  %x(bin/ebnf --input-format native --ll1 ebnf -f rb -o etc/ebnf.ll1.rb etc/ebnf.ebnf)
 end

VERSION

@@ -1 +1 @@
-2.1.1
+2.1.2

lib/ebnf/peg/rule.rb

@@ -24,6 +24,7 @@ module Rule
     # * `opt`: returns the value matched, or `nil` if unmatched.
     # * `plus`: returns an array of the values matched for the specified production, or `:unmatched`, if none are matched. For Terminals, these are concatenated into a single string.
     # * `range`: returns a string composed of the values matched, or `:unmatched`, if less than `min` are matched.
+    # * `rept`: returns an array of the values matched for the speficied production, or `:unmatched`, if none are matched. For Terminals, these are concatenated into a single string.
     # * `seq`: returns an array composed of single-entry hashes for each matched production indexed by the production name, or `:unmatched` if any production fails to match. For Terminals, returns a string created by concatenating these values. Via option in a `production` or definition, the result can be a single hash with values for each matched production; note that this is not always possible due to the possibility of repeated productions within the sequence.
     # * `star`: returns an array of the values matched for the specified production. For Terminals, these are concatenated into a single string.
     #
@@ -142,6 +143,14 @@ def parse(input)
           parser.update_furthest_failure(input.pos, input.lineno, expr[1])
           :unmatched
         end
+      when :rept
+        # Result is an array of all expressions while they match,
+        # an empty array of none match
+        rept = rept(input, expr[1], expr[2], expr[3])
+
+        # # Update furthest failure for strings and terminals
+        parser.update_furthest_failure(input.pos, input.lineno, expr[3]) if terminal?
+        rept.is_a?(Array) && terminal? ? rept.join("") : rept
       when :seq
         # Evaluate each expression into an array of hashes where each hash contains a key from the associated production and the value is the parsed value of that production. Returns :unmatched if the input does not match the production. Value ordering is ensured by native Hash ordering.
         seq = expr[1..-1].each_with_object([]) do |prod, accumulator|

lib/ebnf/writer.rb

@@ -133,7 +133,10 @@ def initialize(rules, out: $stdout, html: false, format: :ebnf, **options)
               OpenStruct.new(id: ("@#{rule.kind}"),
                              sym: nil,
                              assign: nil,
-                             formatted: ("<strong># Productions for terminals</strong>" if rule.kind == :terminals))
+                             formatted: (
+                               rule.kind == :terminals ?
+                                 "<strong># Productions for terminals</strong>" :
+                                 self.send(format_meth, rule.expr)))
             else
               formatted_expr = self.send(format_meth, rule.expr)
               # Measure text without markup
@@ -685,9 +688,11 @@ def format_isoebnf_range(string)
           <% for rule in @rules %>
           <tr<%= %{ id="grammar-production-#{rule.sym}"} unless %w(=/ |).include?(rule.assign)%>>
             <% if rule.id %>
-            <td><%= rule.id %></td>
+            <td<%= " colspan=2" unless rule.sym %>><%= rule.id %></td>
             <% end %>
+            <% if rule.sym %>
             <td><code><%== rule.sym %></code></td>
+            <% end %>
             <td><%= rule.assign %></td>
             <td><%= rule.formatted %></td>
           </tr>

spec/base_spec.rb

@@ -83,7 +83,7 @@
   end
 
   describe "#validate!" do
-    let(:simple) {EBNF.parse("a ::= b")}
+    let(:simple) {EBNF.parse("a ::= b", format: :native)}
     it "notes invalid grammar" do
       expect do
         expect {simple.validate!}.to raise_error SyntaxError, "In rule a: No rule found for b"
@@ -96,7 +96,7 @@
   end
 
   describe "#valid?" do
-    let(:simple) {EBNF.parse("a ::= b")}
+    let(:simple) {EBNF.parse("a ::= b", format: :native)}
     it "notes invalid grammar" do
       expect do
         expect(simple.valid?).to be_falsey
@@ -152,7 +152,7 @@
 
   def parse(value, **options)
     @debug = []
-    options = {debug: @debug}.merge(options)
+    options = {debug: @debug, format: :native}.merge(options)
     EBNF::Base.new(value, **options)
   end
 end

spec/bnf_spec.rb

@@ -69,7 +69,7 @@
 
   def parse(value, **options)
     @debug = []
-    options = {debug: @debug}.merge(options)
+    options = {debug: @debug, format: :native}.merge(options)
     EBNF::Base.new(value, **options)
   end
 end

spec/peg/rule_spec.rb

@@ -79,6 +79,21 @@
           input: " A A A ",
           expect: %w(A A A)
         },
+        "(rept 1 2 A) with ' A A A '" => {
+          rule: [:rept, 1, 2, "A"],
+          input: " A A A ",
+          expect: %w(A A)
+        },
+        "(rept 1 4 A) with ' A A A '" => {
+          rule: [:rept, 1, 4, "A"],
+          input: " A A A ",
+          expect: %w(A A A)
+        },
+        "(rept 4 10 A) with ' A A A '" => {
+          rule: [:rept, 4, 10, "A"],
+          input: " A A A ",
+          expect: :unmatched
+        },
         "(seq 'A' 'B')" => {
           rule: [:seq, "A", "B"],
           input: "A B",
@@ -213,6 +228,21 @@
             input: " A A A ",
             expect: %w(A A A)
           },
+          "(rept 1 2 A) with ' A A A '" => {
+            rule: [:rept, 1, 2, "A"],
+            input: " A A A ",
+            expect: %w(A A)
+          },
+          "(rept 1 4 A) with ' A A A '" => {
+            rule: [:rept, 1, 4, "A"],
+            input: " A A A ",
+            expect: %w(A A A)
+          },
+          "(rept 4 10 A) with ' A A A '" => {
+            rule: [:rept, 4, 10, "A"],
+            input: " A A A ",
+            expect: :unmatched
+          },
           "(seq 'A' 'B')" => {
             rule: [:seq, "A", "B"],
             input: "A B",

spec/peg_spec.rb

@@ -54,7 +54,7 @@
 
   def parse(value, **options)
     @debug = []
-    options = {debug: @debug}.merge(options)
+    options = {debug: @debug, format: :native}.merge(options)
     EBNF::Base.new(value, **options)
   end
 end

spec/rule_spec.rb

@@ -6,7 +6,7 @@
 
 describe EBNF::Rule do
   let(:debug) {[]}
-  let(:ebnf) {EBNF.parse(File.open(File.expand_path("../../etc/ebnf.ebnf", __FILE__)))}
+  let(:ebnf) {EBNF.parse(File.open(File.expand_path("../../etc/ebnf.ebnf", __FILE__)), format: :native)}
   subject {EBNF::Rule.new(:rule, "0", [:seq, :foo])}
 
   describe ".from_sxp" do

spec/spec_helper.rb

@@ -38,4 +38,4 @@
 
 require 'ebnf'
 
-PARSED_EBNF_GRAMMAR = EBNF.parse(File.open(File.expand_path("../../etc/ebnf.ebnf", __FILE__))).freeze
+PARSED_EBNF_GRAMMAR = EBNF.parse(File.open(File.expand_path("../../etc/ebnf.ebnf", __FILE__)), format: :native).freeze

spec/writer_spec.rb

@@ -47,7 +47,7 @@
       ],
     }.each do |title, (grammar, plain)|
       context title do
-        subject {EBNF::Base.new(grammar).ast}
+        subject {EBNF::Base.new(grammar, format: :native).ast}
 
         it "generates plain" do
           expect(EBNF::Writer.string(*subject)).to eq plain
@@ -64,7 +64,7 @@
       ],
     }.each do |title, (grammar, plain)|
       context title do
-        subject {EBNF::Base.new(grammar).ast}
+        subject {EBNF::Base.new(grammar, format: :native).ast}
 
         it "generates plain" do
           expect {EBNF::Writer.print(*subject)}.to write(plain).to(:output)
@@ -89,7 +89,7 @@
       ],
     }.each do |title, (grammar, xpaths)|
       context title do
-        subject {EBNF::Writer.html(*EBNF::Base.new(grammar).ast)}
+        subject {EBNF::Writer.html(*EBNF::Base.new(grammar, format: :native).ast)}
         xpaths.each do |path, value|
           specify {is_expected.to have_xpath(path, value)}
         end
@@ -106,7 +106,7 @@
         ],
       }.each do |title, (grammar, plain)|
         context title do
-          subject {EBNF::Base.new(grammar).ast}
+          subject {EBNF::Base.new(grammar, format: :native).ast}
 
           it "generates plain" do
             expect {EBNF::Writer.new(subject)}.to write(plain).to(:output)