feat!: PP, modal scanner, #include

src/Std/Scan.java

@@ -11,13 +11,15 @@ public class Scan implements IScan {
 
     public int lno;              // current line number
     public Token tok;            // this is persistent across all calls to cur()
+    public Token lineMode;       // token to toggle line mode
 
     // create a scanner object on a buffered reader
     public Scan(BufferedReader rdr) {
         this.rdr = rdr;
         this.lno = 0;
-        s = null;
-        tok = null;
+        this.lineMode = null;
+        this.s = null;
+        this.tok = null;
         // force the enum Match class to compile its patterns
         String msg = Token.Match.init();
         if (msg != null) {
@@ -36,6 +38,7 @@ public void reset() {
         // force the scanner to process the next line
         s = null;
         tok = null;
+        lineMode = null;
     }
 
     // fill the string buffer from the reader if it's exhausted or null)
@@ -47,7 +50,7 @@ public void fillString() {
                 if (s == null)
                     return; // end of file
                 lno++;
-                s += "\n";
+                s += "\n";  // make sure the string has a newline
                 start = 0;
                 end = s.length();
             } catch (IOException e) {
@@ -73,20 +76,41 @@ public Token cur() {
                 return tok;
             }
             // s cannot be null here
+            // are we in line mode?
+            if (lineMode != null) {
+                Pattern cpat = lineMode.match.cPattern;
+                Matcher m = cpat.matcher(s);
+                m.region(0,end);
+                start = end; // consume the line before next match
+                if (m.lookingAt()) {
+                    // found the lineMode token, exit line mode
+                    // and return the matched lineMode token
+                    // System.out.println("leaving line mode...");
+                    tok = new Token(lineMode.match, m.group(), lno, s);
+                    lineMode = null;
+                    return tok;
+                } else {
+                    // return the entire line as a token
+                    tok = new Token(Token.Match.$LINE, s, lno, s);
+                    return tok;
+                }
+            }
             int matchEnd = start; // current end of match
             for (Token.Match match : Token.Match.values()) {
                 Pattern cpat = match.cPattern;
                 if (cpat == null)
                     break; // nothing matches, so can't find a token
-                if (match.skip && matchFound != null)
+                if (match.tokType == Token.TokType.SKIP && matchFound != null)
                     continue; // ignore skips if we have a pending token
+                if (start != 0 && match.pattern.charAt(0) == '^')
+                    continue; // '^' must match at start of line
                 Matcher m = cpat.matcher(s);
                 m.region(start, end);
                 if (m.lookingAt()) {
                     int e = m.end();
                     if (e == start)
                         continue; // empty match, so try next pattern
-                    if (match.skip) {
+                    if (match.tokType == Token.TokType.SKIP) {
                         // there's a non-empty skip match,
                         // so we skip over the matched part
                         // and get more stuff to read
@@ -114,6 +138,12 @@ public Token cur() {
             start = matchEnd; // start of next token match
             // matchString is the matching string
             tok = new Token(matchFound, matchString, lno, s); // persistent
+            // System.out.println(String.format("match=%s\n", toggle));
+            if (matchFound.tokType == Token.TokType.LINE_TOGGLE) {
+                // System.out.println("going to line mode...");
+                start = end; // swallow the rest of the line
+                lineMode = tok;
+            }
             return tok;
         }
     }

src/Std/Token.pattern

@@ -4,29 +4,50 @@ import java.util.regex.*;
 // Token class with match patterns (used with the built-in Scan class)
 public class Token {
 
-    // this is set to an error message string
+    // patternFail is set to an error message string
     // if there are pattern compile errors
     public static String patternFail = null; //
     public static final Match $eof = Match.$EOF;
 
+    public enum TokType {
+        TOKEN,
+        SKIP,
+        LINE_TOGGLE,
+        SPECIAL;
+    }
+
     public enum Match {
 %%Match%%
         $ERROR (null),
-        $EOF (null);
+        $EOF (null),
+        $LINE (null);
 
         public String pattern;
-        public boolean skip;
-        public Pattern cPattern; // compiled pattern
+        public TokType tokType;
+        public Pattern cPattern = null; // compiled pattern
 
-        // a token pattern (skip == false)
+        // a SPECIAL token type or a TOKEN/LINE_TOGGLE
         Match(String pattern) {
-            this(pattern, false);
-        }
+            this(pattern, null);
+        } 
 
+        // legacy ??
         Match(String pattern, boolean skip) {
-            this.pattern = pattern;
-            this.skip = skip;
+            this(pattern, TokType.SKIP);
+        }
+
+        Match(String pattern, TokType tokType) {
             if (pattern != null) {
+                if (tokType == TokType.SKIP) {
+                    this.tokType = TokType.SKIP;
+                } else if (pattern.length() >= 2 &&
+                           pattern.substring(0,2).equals("^^")) {
+                    pattern = pattern.substring(1);
+                    this.tokType = TokType.LINE_TOGGLE;
+                } else {
+                    this.tokType = TokType.TOKEN;
+                }
+                this.pattern = pattern;
                 try {
                     this.cPattern = Pattern.compile(pattern, Pattern.DOTALL);
                 } catch (PatternSyntaxException e) {
@@ -36,12 +57,14 @@ public class Token {
                     patternFail += (" " +this);
                     this.cPattern = null;
                 }
+            } else {
+                this.tokType = TokType.SPECIAL; // SPECIAL
             }
         }
 
         // Use this to force loading Match class to compile patterns.
         public static String init() {
-            return patternFail;
+            return patternFail; // returns null if no errors
         }
     }
 
@@ -88,22 +111,27 @@ public class Token {
 
     public static void main(String [] args) {
         String msg = Match.init();
-        if (msg != null)
+        if (msg != null) {
             System.out.println(msg);
+            System.exit(1);
+        }
         for (Match match: Match.values()) {
-            if (match.pattern == null)
-                continue;
-            String what;
-            if (match.skip)
-                what = "skip";
-            else
-                what = "token";
+            if (match.tokType == TokType.SPECIAL) {
+                System.out.println(
+                    String.format("special "+match.toString())
+                );
+                continue; // not a real token
+            }
+            String what = "??";
+            switch(match.tokType) {
+                case SKIP -> what = "skip";
+                case TOKEN -> what = "token";
+                case LINE_TOGGLE -> what = "token (line toggle)";
+            }
             System.out.println(
-                String.format("%s %s '%s'", what, match, match.pattern)
+                String.format("%s %s '%s'",what,match.toString(),match.pattern)
             );
         }
-        if (msg != null)
-            System.exit(1);
     }
 
 //Token//

src/Std/Token.template

@@ -5,7 +5,7 @@ public class Token {
 
     public enum Match {
 %%Match%%
-        }
+    }
 
     public Match match;      // token match
     public String str;       // this token's lexeme (never empty!)

src/plcc

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-LIB="${LIBPLCC:-/usr/local/pub/plcc/PLCC}"
+LIB="${LIBPLCC:-/home/fossumtv/PL/src}"
 PYTHON3=python3
 PLCC="$LIB/plcc.py"