diff --git a/doc/options.md b/doc/options.md index 3ec64fca5..5c14840da 100644 --- a/doc/options.md +++ b/doc/options.md @@ -20,7 +20,7 @@ Set the superclass of the generated parser or lexer. For combined grammars, it s $ cat Hi.g4 grammar Hi; a : 'hi' ; -$ antlr4 -DsuperClass=XX Hi.g4 +$ antlr5 -DsuperClass=XX Hi.g4 $ grep 'public class' HiParser.java public class HiParser extends XX { $ grep 'public class' HiLexer.java @@ -32,10 +32,90 @@ public class HiLexer extends Lexer { Generate code in the indicated language, if ANTLR is able to do so. Otherwise, you will see an error message like this: ``` -$ antlr4 -Dlanguage=C MyGrammar.g4 +$ antlr5 -Dlanguage=C MyGrammar.g4 error(31): ANTLR cannot generate C code as of version 4.0 ``` +### `actionTemplates` + +This option uses the provided [StringTemplate](https://www.stringtemplate.org/) group file (`*.stg`) to render templates inside the action blocks of an ANTLR grammar. + +This enables you to provide target-specific action logic by providing different `.stg` files for each target language. + +The syntax of group files is [described](https://github.com/antlr/stringtemplate4/blob/master/doc/groups.md) in the StringTemplate documentation. + +For example, if you provide the following group file when generating Java code: + +`ActionTemplates.stg`: +```string-template +normalize(s) ::= <, Form.NFKC)>> +setText(s) ::= <);>> +getText() ::= <> +normalizerImports ::= << +import java.text.Normalizer; +import java.text.Normalizer.Form; +>> +``` + +You can use the templates like so in your ANTLR grammar: + +```antlrv4 +ID: + (ID_START ID_CONTINUE* | '_' ID_CONTINUE+) { + + }; +``` + +The ANTLR tool must be invoked by providing the target language and StringTemplate group file: + +```bash +$ antlr5 -Dlanguage=Java -DactionTemplates=ActionTemplates.stg MyGrammar.g4 +``` + +The templates will be expanded into the following before the grammar is used to generate the target code: + +```antlrv4 +ID: + (ID_START ID_CONTINUE* | '_' ID_CONTINUE+) { + setText(Normalizer.normalize(getText(), Form.NFKC)); + }; +``` + +Templates can also be used in named actions, such as the `@header` or `@members` block, for example: + +```antlrv4 +@lexer::header { + +} +``` + +To use the same grammar to generate a different target language, you can provide a different StringTemplate group file. + +For example, to generate JavaScript code equivalent to the previous example the following group file could be used instead: + +`ActionTemplates.stg`: +```string-template +normalize(s) ::= <<.normalize("NFKC")>> +setText(s) ::= <;>> +getText() ::= <> +normalizerImports ::= "" +``` + +Now you can invoke the ANTLR tool with the new target language and your alternate StringTemplate group file: + +```bash +$ antlr5 -Dlanguage=JavaScript -DactionTemplates=ActionTemplates.stg MyGrammar.g4 +``` + +These templates will expand into the following before the grammar is used to generate the target code: + +```antlrv4 +ID: + (ID_START ID_CONTINUE* | '_' ID_CONTINUE+) { + this.text = this.text.normalize("NFKC"); + }; +``` + ### `tokenVocab` ANTLR assigns token type numbers to the tokens as it encounters them in a file. To use different token type values, such as with a separate lexer, use this option to have ANTLR pull in the tokens file. ANTLR generates a tokens file from each grammar. @@ -49,10 +129,10 @@ parser grammar R; options {tokenVocab=SomeLexer;} tokens {A,B,C} // normally, these would be token types 1, 2, 3 a : ID ; -$ antlr4 SomeLexer.g4 +$ antlr5 SomeLexer.g4 $ cat SomeLexer.tokens ID=1 -$ antlr4 R.g4 +$ antlr5 R.g4 $ cat R.tokens A=2 B=3 @@ -69,7 +149,7 @@ $ cat T2.g4 grammar T2; options {TokenLabelType=MyToken;} a : x=ID ; -$ antlr4 T2.g4 +$ antlr5 T2.g4 $ grep MyToken T2Parser.java public MyToken x; ``` diff --git a/runtime-testsuite/test/org/antlr/v5/test/runtime/Generator.java b/runtime-testsuite/test/org/antlr/v5/test/runtime/Generator.java index a7de7b96c..7486620ff 100644 --- a/runtime-testsuite/test/org/antlr/v5/test/runtime/Generator.java +++ b/runtime-testsuite/test/org/antlr/v5/test/runtime/Generator.java @@ -79,6 +79,15 @@ public static GeneratedState generate(RunOptions runOptions, String language, St options.add("-DsuperClass=" + superClass); } + if (runOptions.libDir != null && !runOptions.libDir.isEmpty()) { + options.add("-lib"); + options.add(runOptions.libDir); + } + + if (runOptions.actionTemplates != null && !runOptions.actionTemplates.isEmpty()) { + options.add("-DactionTemplates=" + runOptions.actionTemplates); + } + if (extraOptions != null) { options.addAll(Arrays.asList(extraOptions)); } diff --git a/runtime-testsuite/test/org/antlr/v5/test/runtime/RunOptions.java b/runtime-testsuite/test/org/antlr/v5/test/runtime/RunOptions.java index 7ed5e3b80..f6734de95 100644 --- a/runtime-testsuite/test/org/antlr/v5/test/runtime/RunOptions.java +++ b/runtime-testsuite/test/org/antlr/v5/test/runtime/RunOptions.java @@ -21,6 +21,8 @@ public class RunOptions { public final String superClass; public final PredictionMode predictionMode; public final boolean buildParseTree; + public final String libDir; + public final String actionTemplates; public final String[] extraGenerationOptions; public static RunOptions createGenerationOptions(String[] grammars, String[] slaveGrammars, boolean useListener, boolean useVisitor, @@ -39,6 +41,8 @@ public static RunOptions createGenerationOptions(String[] grammars, String[] sla false, Stage.Generate, superClass, + null, + null, PredictionMode.LL, false, extraGenerationOptions @@ -60,6 +64,8 @@ public static RunOptions createCompilationOptions(String[] grammars, String[] sl false, Stage.Compile, superClass, + null, + null, PredictionMode.LL, false, extraGenerationOptions @@ -70,7 +76,8 @@ public RunOptions(String[] grammars, String[] slaveGrammars, boolean useListener, boolean useVisitor, String startRuleName, String input, boolean profile, boolean showDiagnosticErrors, boolean traceATN, boolean showDFA, Stage endStage, - String superClass, PredictionMode predictionMode, boolean buildParseTree, + String superClass, String libDir, String actionTemplates, + PredictionMode predictionMode, boolean buildParseTree, String[] extraGenerationOptions) { this.grammars = grammars; this.slaveGrammars = slaveGrammars; @@ -84,6 +91,8 @@ public RunOptions(String[] grammars, String[] slaveGrammars, this.showDFA = showDFA; this.endStage = endStage; this.superClass = superClass; + this.libDir = libDir; + this.actionTemplates = actionTemplates; this.predictionMode = predictionMode; this.buildParseTree = buildParseTree; this.extraGenerationOptions = extraGenerationOptions; diff --git a/runtime-testsuite/test/org/antlr/v5/test/runtime/RuntimeTests.java b/runtime-testsuite/test/org/antlr/v5/test/runtime/RuntimeTests.java index 9bb1401f6..9033dbfe0 100644 --- a/runtime-testsuite/test/org/antlr/v5/test/runtime/RuntimeTests.java +++ b/runtime-testsuite/test/org/antlr/v5/test/runtime/RuntimeTests.java @@ -133,6 +133,8 @@ private static void test(RuntimeTestDescriptor descriptor, RuntimeRunner runner) descriptor.showDFA, Stage.Execute, null, + null, + null, descriptor.predictionMode, descriptor.buildParseTree, null diff --git a/tool-testsuite/test/org/antlr/v5/test/tool/TestActionTemplates.java b/tool-testsuite/test/org/antlr/v5/test/tool/TestActionTemplates.java new file mode 100644 index 000000000..6720a88c3 --- /dev/null +++ b/tool-testsuite/test/org/antlr/v5/test/tool/TestActionTemplates.java @@ -0,0 +1,348 @@ +package org.antlr.v5.test.tool; + +import org.antlr.v5.test.runtime.RunOptions; +import org.antlr.v5.test.runtime.java.JavaRunner; +import org.antlr.v5.test.runtime.states.ExecutedState; +import org.antlr.v5.test.runtime.states.GeneratedState; +import org.antlr.v5.test.runtime.states.State; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Path; + +import static org.antlr.v5.test.runtime.FileUtils.writeFile; +import static org.antlr.v5.test.tool.ToolTestUtils.createExecOptionsForJavaToolTests; +import static org.junit.jupiter.api.Assertions.*; + +public class TestActionTemplates { + @Test void testIncorrectActionTemplateGroupExtension(@TempDir Path tempDir) { + String actionTemplates = tempDir.resolve("Java.st").toString(); + + writeFile(actionTemplates, "", "UTF-8"); + + String grammar = + "lexer grammar L;" + + "WS : (' '|'\\n') -> skip ;"; + + State state = execLexer(grammar, "34 34", tempDir, "Java.st"); + + assertInstanceOf(GeneratedState.class, state, state.getErrorMessage()); + + GeneratedState generated = (GeneratedState) state; + + assertTrue(generated.containsErrors()); + + assertEquals( + "State: Generate; \n" + + "error(208): error reading action templates file Java.st: " + + "Group file names must end in .stg: " + actionTemplates + "\n", + generated.getErrorMessage()); + } + + @Test void testActionTemplateFileMissing(@TempDir Path tempDir) { + String grammar = + "lexer grammar L;" + + "WS : (' '|'\\n') -> skip ;"; + + State state = execLexer(grammar, "34 34", tempDir, "Java.stg"); + + assertInstanceOf(GeneratedState.class, state, state.getErrorMessage()); + + GeneratedState generated = (GeneratedState) state; + + assertTrue(generated.containsErrors()); + + assertEquals( + "State: Generate; \n" + + "error(206): cannot find action templates file Java.stg given for L\n", + generated.getErrorMessage()); + } + + @Test void testUnlexableActionTemplate(@TempDir Path tempDir) { + writeActionTemplatesFile(tempDir, "writeln(s) ::= <\");>>"); + + String grammar = + "lexer grammar L;\n" + + "I : '0'..'9'+ {<¢>} ;\n" + + "WS : (' '|'\\n') -> skip ;"; + + State state = execLexer(grammar, "34 34", tempDir, "Java.stg"); + + assertInstanceOf(GeneratedState.class, state, state.getErrorMessage()); + + GeneratedState generated = (GeneratedState) state; + + assertTrue(generated.containsErrors()); + + assertEquals( + "State: Generate; \n" + + "error(211): L.g4:2:14: error compiling action template: 2:16: invalid character '¢'\n" + + "error(211): L.g4:2:14: error compiling action template: 2:14: this doesn't look like a template: \" <¢> \"\n", + generated.getErrorMessage()); + } + + @Test void testUnlexableMultilineActionTemplate(@TempDir Path tempDir) { + writeActionTemplatesFile(tempDir, "writeln(s) ::= <\");>>"); + + String grammar = + "lexer grammar L;\n" + + "I : '0'..'9'+ {\n" + + " <¢>\n" + + "};\n" + + "WS : (' '|'\\n') -> skip ;"; + + State state = execLexer(grammar, "34 34", tempDir, "Java.stg"); + + assertInstanceOf(GeneratedState.class, state, state.getErrorMessage()); + + GeneratedState generated = (GeneratedState) state; + + assertTrue(generated.containsErrors()); + + assertEquals( + "State: Generate; \n" + + "error(211): L.g4:2:14: error compiling action template: 3:3: invalid character '¢'\n" + + "error(211): L.g4:2:14: error compiling action template: 3:0: mismatched input ' ' expecting EOF\n", + generated.getErrorMessage()); + } + + @Test void testInvalidActionTemplateGroup(@TempDir Path tempDir) { + String actionTemplates = tempDir.resolve("Java.stg").toString(); + + writeActionTemplatesFile(tempDir, "writeln(s) := <\");>>"); + + String grammar = + "lexer grammar L;\n" + + "I : '0'..'9'+ {} ;\n" + + "WS : (' '|'\\n') -> skip ;"; + + State state = execLexer(grammar, "34 34", tempDir, "Java.stg"); + + GeneratedState generated = (GeneratedState) state; + + assertTrue(generated.containsErrors()); + + assertEquals( + "State: Generate; \n" + + "error(209): error compiling action templates file "+ actionTemplates +": Java.stg 1:11: mismatched input ':' expecting '::='\n" + + "error(212): L.g4:2:14: error rendering action template: 2:16: no such template: /writeln\n", + generated.getErrorMessage()); + } + + @Test void testInvalidActionTemplate(@TempDir Path tempDir) { + writeActionTemplatesFile(tempDir, "writeln(s) ::= <\");>>"); + + String grammar = + "lexer grammar L;\n" + + "I : '0'..'9'+ { skip ;"; + + State state = execLexer(grammar, "34 34", tempDir, "Java.stg"); + + GeneratedState generated = (GeneratedState) state; + + assertTrue(generated.containsErrors()); + + assertEquals( + "State: Generate; \n" + + "error(211): L.g4:2:14: error compiling action template: 2:29: premature EOF\n", + generated.getErrorMessage()); + } + + @Test + void testInvalidMultilineActionTemplate(@TempDir Path tempDir) { + writeActionTemplatesFile(tempDir, "writeln(s) ::= <\");>>"); + + String grammar = + "lexer grammar L;\n" + + "I : '0'..'9'+ {\n" + + " skip ;"; + + State state = execLexer(grammar, "34 34", tempDir, "Java.stg"); + + GeneratedState generated = (GeneratedState) state; + + assertTrue(generated.containsErrors()); + + assertEquals( + "State: Generate; \n" + + "error(211): L.g4:2:14: error compiling action template: 4:1: premature EOF\n", + generated.getErrorMessage()); + } + + @Test + void testValidActionTemplate(@TempDir Path tempDir) { + writeActionTemplatesFile(tempDir, "writeln(s) ::= <\");>>"); + + String grammar = + "lexer grammar L;\n" + + "I : '0'..'9'+ {} ;\n" + + "WS : (' '|'\\n') -> skip ;"; + + State state = execLexer(grammar, "34 34", tempDir, "Java.stg"); + + // Should have identical output to TestLexerActions.testActionExecutedInDFA + String expecting = + "I\n" + + "I\n" + + "[@0,0:1='34',<1>,1:0]\n" + + "[@1,3:4='34',<1>,1:3]\n" + + "[@2,5:4='',<-1>,1:5]\n"; + + assertInstanceOf(ExecutedState.class, state, state.getErrorMessage()); + + assertEquals(expecting, ((ExecutedState) state).output); + } + + @Test + void testValidMultilineActionTemplate(@TempDir Path tempDir) { + writeActionTemplatesFile(tempDir, "writeln(s) ::= <\");>>"); + + String actionTemplates = tempDir.resolve("Java.stg").toString(); + + String grammar = + "lexer grammar L;\n" + + "I : '0'..'9'+ {\n" + + " \n" + + "};\n" + + "WS : (' '|'\\n') -> skip ;"; + + State state = execLexer(grammar, "34 34", tempDir, actionTemplates); + + // Should have identical output to TestLexerActions.testActionExecutedInDFA + String expecting = + "I\n" + + "I\n" + + "[@0,0:1='34',<1>,1:0]\n" + + "[@1,3:4='34',<1>,1:3]\n" + + "[@2,5:4='',<-1>,1:5]\n"; + + assertInstanceOf(ExecutedState.class, state, state.getErrorMessage()); + + assertEquals(expecting, ((ExecutedState) state).output); + } + + @Test + void testActionTemplateHeader(@TempDir Path tempDir) { + String actionTemplates = + "normalizerImports() ::= <<\n" + + "import java.text.Normalizer;\n" + + "import java.text.Normalizer.Form;\n" + + ">>\n" + + "normalize(s) ::= <, Form.NFKC)>>\n" + + "getText() ::= <>\n" + + "setText(s) ::= <);>>"; + + writeActionTemplatesFile(tempDir, actionTemplates); + + String grammar = + "lexer grammar L;\n" + + "@lexer::header {\n" + + "\n" + + "}\n" + + "ID : (ID_START ID_CONTINUE* | '_' ID_CONTINUE+) { } ;\n" + + "ID_START : [\\p{XID_Start}] ;\n" + + "ID_CONTINUE: [\\p{XID_Continue}] ;\n" + + "WS : (' '|'\\n') -> skip ;"; + + State state = execLexer(grammar, "This _is \ufb01ne", tempDir, "Java.stg"); + + String expecting = + "[@0,0:3='This',<1>,1:0]\n" + + "[@1,5:7='_is',<1>,1:5]\n" + + "[@2,9:11='fine',<1>,1:9]\n" + + "[@3,12:11='',<-1>,1:12]\n"; + + assertInstanceOf(ExecutedState.class, state, state.getErrorMessage()); + + assertEquals(expecting, ((ExecutedState) state).output); + } + + @Test + void testActionTemplateSemanticPredicate(@TempDir Path tempDir) { + String actionTemplates = "pred() ::= <>"; + + writeActionTemplatesFile(tempDir, actionTemplates); + + String actionTemplatesFile = tempDir.resolve("Java.stg").toString(); + + String grammar = + "grammar P;\n" + + "file : atom EOF ;\n" + + "atom : scientific | { }? variable ;\n" + + "variable: VARIABLE ;\n" + + "scientific: SCIENTIFIC_NUMBER ;\n" + + "VARIABLE : VALID_ID_START VALID_ID_CHAR* ;\n" + + "SCIENTIFIC_NUMBER : NUMBER (E SIGN? UNSIGNED_INTEGER)? ;\n" + + "fragment VALID_ID_START : ('a' .. 'z') | ('A' .. 'Z') | '_' ;\n" + + "fragment VALID_ID_CHAR : VALID_ID_START | ('0' .. '9') ;\n" + + "fragment NUMBER : ('0' .. '9') + ('.' ('0' .. '9') +)? ;\n" + + "fragment UNSIGNED_INTEGER : ('0' .. '9')+ ;\n" + + "fragment E : 'E' | 'e' ;\n" + + "fragment SIGN : ('+' | '-') ;\n" + + "WS : (' '|'\\n') -> skip ;"; + + State state = execParser(grammar, "Bla", tempDir, "file", actionTemplatesFile); + + assertInstanceOf(ExecutedState.class, state, state.getErrorMessage()); + + ExecutedState executedState = (ExecutedState) state; + + // We can never match the input unless pred() expands to true + assertEquals("", executedState.output); + assertEquals("", executedState.errors); + } + + @Test + void testValidActionTemplateInLibDir(@TempDir Path tempDir, @TempDir Path libDir) { + writeActionTemplatesFile(libDir, "writeln(s) ::= <\");>>"); + + String actionTemplates = "Java.stg"; + + String grammar = + "lexer grammar L;\n" + + "I : '0'..'9'+ {} ;\n" + + "WS : (' '|'\\n') -> skip ;"; + + State state = execLexer(grammar, "34 34", tempDir, libDir, actionTemplates); + + // Should have identical output to TestLexerActions.testActionExecutedInDFA + String expecting = + "I\n" + + "I\n" + + "[@0,0:1='34',<1>,1:0]\n" + + "[@1,3:4='34',<1>,1:3]\n" + + "[@2,5:4='',<-1>,1:5]\n"; + + assertInstanceOf(ExecutedState.class, state, state.getErrorMessage()); + + assertEquals(expecting, ((ExecutedState) state).output); + } + + void writeActionTemplatesFile(Path tempDir, String template) { + writeFile(tempDir.resolve("Java.stg").toString(), template, "UTF-8"); + } + + State execParser(String grammarStr, String input, Path tempDir, String startRule, String actionTemplates) { + RunOptions runOptions = createExecOptionsForJavaToolTests(grammarStr, false, true, startRule, input, null, actionTemplates, false, false); + try (JavaRunner runner = new JavaRunner(tempDir, false)) { + return runner.run(runOptions); + } + } + + State execLexer(String grammarStr, String input, Path tempDir, String actionTemplates) { + RunOptions runOptions = createExecOptionsForJavaToolTests(grammarStr, false, true, null, input, null, actionTemplates, false, false); + try (JavaRunner runner = new JavaRunner(tempDir, false)) { + return runner.run(runOptions); + } + } + + State execLexer(String grammarStr, String input, Path tempDir, Path libDir, String actionTemplates) { + RunOptions runOptions = createExecOptionsForJavaToolTests(grammarStr, false, true, null, input, libDir.toString(), actionTemplates, false, false); + try (JavaRunner runner = new JavaRunner(tempDir, false)) { + return runner.run(runOptions); + } + } +} diff --git a/tool-testsuite/test/org/antlr/v5/test/tool/TestParseTreeMatcher.java b/tool-testsuite/test/org/antlr/v5/test/tool/TestParseTreeMatcher.java index a0bd73bd4..453f7d947 100644 --- a/tool-testsuite/test/org/antlr/v5/test/tool/TestParseTreeMatcher.java +++ b/tool-testsuite/test/org/antlr/v5/test/tool/TestParseTreeMatcher.java @@ -386,7 +386,7 @@ private static ParseTreeMatch checkPatternMatch(String grammar, String startRule throws Exception { RunOptions runOptions = createExecOptionsForJavaToolTests(grammar, - false, false, startRule, input, false, false); + false, false, startRule, input, null, null, false, false); try (JavaRunner runner = new JavaRunner()) { State state = runner.run(runOptions); if(!(state instanceof JavaExecutedState)) diff --git a/tool-testsuite/test/org/antlr/v5/test/tool/TestXPath.java b/tool-testsuite/test/org/antlr/v5/test/tool/TestXPath.java index 46c76d416..c5306249e 100644 --- a/tool-testsuite/test/org/antlr/v5/test/tool/TestXPath.java +++ b/tool-testsuite/test/org/antlr/v5/test/tool/TestXPath.java @@ -197,7 +197,7 @@ private Pair> compileAndExtract(String grammar, String input, String xpath, String startRuleName ) throws Exception { RunOptions runOptions = createExecOptionsForJavaToolTests(grammar, - false, false, startRuleName, input, false, false); + false, false, startRuleName, input, null, null, false, false); try (JavaRunner runner = new JavaRunner()) { JavaExecutedState executedState = (JavaExecutedState)runner.run(runOptions); JavaCompiledState compiledState = (JavaCompiledState)executedState.previousState; diff --git a/tool-testsuite/test/org/antlr/v5/test/tool/ToolTestUtils.java b/tool-testsuite/test/org/antlr/v5/test/tool/ToolTestUtils.java index 96a5b9254..7be1753f1 100644 --- a/tool-testsuite/test/org/antlr/v5/test/tool/ToolTestUtils.java +++ b/tool-testsuite/test/org/antlr/v5/test/tool/ToolTestUtils.java @@ -64,6 +64,7 @@ private static ExecutedState execRecognizer(String grammarStr, String startRuleN Path workingDir, boolean saveTestDir, boolean profile) { RunOptions runOptions = createExecOptionsForJavaToolTests(grammarStr, false, true, startRuleName, input, + null, null, profile, showDiagnosticErrors); try (JavaRunner runner = new JavaRunner(workingDir, saveTestDir)) { State result = runner.run(runOptions); @@ -77,11 +78,11 @@ private static ExecutedState execRecognizer(String grammarStr, String startRuleN public static RunOptions createExecOptionsForJavaToolTests( String grammarStr, boolean useListener, boolean useVisitor, String startRuleName, - String input, boolean profile, boolean showDiagnosticErrors + String input, String libDir, String actionTemplates, boolean profile, boolean showDiagnosticErrors ) { return new RunOptions(new String[] {grammarStr}, null, useListener, useVisitor, startRuleName, input, profile, showDiagnosticErrors, false, false, Stage.Execute, - null, PredictionMode.LL, true, null); + null, libDir, actionTemplates, PredictionMode.LL, true, null); } public static void testErrors(String[] pairs, boolean printTree) { diff --git a/tool/src/org/antlr/v5/tool/ErrorType.java b/tool/src/org/antlr/v5/tool/ErrorType.java index e9f73bb7c..6030a10e3 100644 --- a/tool/src/org/antlr/v5/tool/ErrorType.java +++ b/tool/src/org/antlr/v5/tool/ErrorType.java @@ -1235,6 +1235,48 @@ public enum ErrorType { */ @Deprecated V3_SYNPRED(205, "(...)=> syntactic predicates are not supported in ANTLR 4", ErrorSeverity.ERROR), + /** + * Compiler Error 206. + * + *

cannot find action templates file filename given for grammar

+ */ + CANNOT_FIND_ACTION_TEMPLATES_FILE_GIVEN_ON_CMDLINE(206, "cannot find action templates file given for ", ErrorSeverity.ERROR), + /** + * Compiler Error 207. + * + *

cannot find action templates file filename

+ */ + CANNOT_FIND_ACTION_TEMPLATES_FILE_REFD_IN_GRAMMAR(207, "cannot find action templates file ", ErrorSeverity.ERROR), + /** + * Compiler Error 208. + * + *

error reading action templates file filename: reason

+ */ + ERROR_READING_ACTION_TEMPLATES_FILE(208, "error reading action templates file : ", ErrorSeverity.ERROR), + /** + * Compiler Error 209. + * + *

error compiling action templates file filename: reason

+ */ + ERROR_COMPILING_ACTION_TEMPLATES_FILE(209, "error compiling action templates file : ", ErrorSeverity.ERROR), + /** + * Compiler Error 210. + * + *

error rendering action templates file filename: reason

+ */ + ERROR_RENDERING_ACTION_TEMPLATES_FILE(210, "error rendering action templates file : ", ErrorSeverity.ERROR), + /** + * Compiler Error 211. + * + *

error compiling action template: reason

+ */ + ERROR_COMPILING_ACTION_TEMPLATE(211, "error compiling action template: ", ErrorSeverity.ERROR), + /** + * Compiler Error 212. + * + *

error rendering action template: reason

+ */ + ERROR_RENDERING_ACTION_TEMPLATE(212, "error rendering action template: ", ErrorSeverity.ERROR), // Dependency sorting errors diff --git a/tool/src/org/antlr/v5/tool/Grammar.java b/tool/src/org/antlr/v5/tool/Grammar.java index d64dc80af..c3020cff1 100644 --- a/tool/src/org/antlr/v5/tool/Grammar.java +++ b/tool/src/org/antlr/v5/tool/Grammar.java @@ -78,6 +78,7 @@ public class Grammar implements AttributeResolver { parserOptions.add("TokenLabelType"); parserOptions.add("tokenVocab"); parserOptions.add("language"); + parserOptions.add("actionTemplates"); parserOptions.add("accessLevel"); parserOptions.add("exportMacro"); parserOptions.add(caseInsensitiveOptionName); @@ -1172,6 +1173,10 @@ public String getLanguage() { return getOptionString("language"); } + public String getActionTemplates() { + return getOptionString("actionTemplates"); + } + public String getOptionString(String key) { return ast.getOptionString(key); } /** Given ^(TOKEN_REF ^(OPTIONS ^(ELEMENT_OPTIONS (= assoc right)))) diff --git a/tool/src/org/antlr/v5/tool/GrammarTransformPipeline.java b/tool/src/org/antlr/v5/tool/GrammarTransformPipeline.java index b4ab3dde8..6873edabf 100644 --- a/tool/src/org/antlr/v5/tool/GrammarTransformPipeline.java +++ b/tool/src/org/antlr/v5/tool/GrammarTransformPipeline.java @@ -7,6 +7,7 @@ package org.antlr.v5.tool; import org.antlr.runtime.CommonToken; +import org.antlr.runtime.RecognitionException; import org.antlr.runtime.tree.CommonTree; import org.antlr.runtime.tree.CommonTreeNodeStream; import org.antlr.runtime.tree.Tree; @@ -27,7 +28,16 @@ import org.antlr.v5.tool.ast.GrammarRootAST; import org.antlr.v5.tool.ast.RuleAST; import org.antlr.v5.tool.ast.TerminalAST; - +import org.stringtemplate.v4.ST; +import org.stringtemplate.v4.STErrorListener; +import org.stringtemplate.v4.STGroupFile; +import org.stringtemplate.v4.STGroupString; +import org.stringtemplate.v4.compiler.STException; +import org.stringtemplate.v4.misc.*; + +import java.io.File; +import java.io.PrintWriter; +import java.io.StringWriter; import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; @@ -54,6 +64,10 @@ public void process() { reduceBlocksToSets(root); expandParameterizedLoops(root); + if (g.getActionTemplates() != null) { + expandActionTemplates(root); + } + tool.log("grammar", "after: "+root.toStringTree()); } @@ -509,4 +523,332 @@ public GrammarRootAST extractImplicitLexer(Grammar combinedGrammar) { return lexerAST; } + /** + * Create an error listener for .stg action template group files provided via grammar options or via the command line. + */ + private STErrorListener createActionTemplateErrorListener(GrammarAST ast, STGroupFile actionTemplateGroupFile) { + return new STErrorListener() { + private final ErrorManager errorManager = ast.g.tool.errMgr; + + @Override + public void compileTimeError(STMessage stMessage) { + errorManager.toolError( + ErrorType.ERROR_COMPILING_ACTION_TEMPLATES_FILE, + actionTemplateGroupFile.fileName, + stMessage.toString()); + } + + @Override + public void runTimeError(STMessage stMessage) { + errorManager.toolError( + ErrorType.ERROR_RENDERING_ACTION_TEMPLATES_FILE, + actionTemplateGroupFile.fileName, + stMessage.toString()); + } + + @Override + public void IOError(STMessage stMessage) { + reportInternalError(stMessage); + } + + @Override + public void internalError(STMessage stMessage) { + reportInternalError(stMessage); + } + + private void reportInternalError(STMessage stMessage) { + errorManager.toolError(ErrorType.INTERNAL_ERROR, stMessage.cause, stMessage.toString()); + } + }; + } + + /** + * Create an error listener for action templates embedded inside a grammar's actions and semantic predicates. + */ + private STErrorListener createGrammarErrorListener(GrammarAST ast) { + return new STErrorListener() { + private final ErrorManager errorManager = ast.g.tool.errMgr; + + /** + * Get the STCompiletimeMesage error message content, translating the source location + * according to the action token's position in the grammar. + */ + private String getSTCompiletimeErrorMessage(STCompiletimeMessage stMsg) { + StringWriter sw = new StringWriter(); + PrintWriter pw = new PrintWriter(sw); + + if (stMsg.token != null) { + int linePos = ast.getLine() + stMsg.token.getLine() - 1; + int charPos = stMsg.token.getCharPositionInLine(); + if (stMsg.token.getLine() == 1) { + charPos += ast.getCharPositionInLine(); + } + pw.print(linePos + ":" + charPos + ": "); + } + + String msg = String.format(stMsg.error.message, stMsg.arg, stMsg.arg2); + + pw.print(msg); + + return sw.toString(); + } + + /** + * Get the STLexerMessage error message content, translating the source location + * according to the action token's position in the grammar. + */ + private String getSTLexerErrorMessage(STLexerMessage stMsg) { + StringWriter sw = new StringWriter(); + PrintWriter pw = new PrintWriter(sw); + + // From STLexerMessage.toString + RecognitionException re = (RecognitionException)stMsg.cause; + int linePos = ast.getLine() + re.line - 1; + int charPos = re.charPositionInLine; + if (re.line == 1) { + charPos += ast.getCharPositionInLine(); + } + pw.print(linePos + ":" + charPos + ": "); + + String msg = String.format(stMsg.error.message, stMsg.msg); + + pw.print(msg); + + return sw.toString(); + } + + @Override + public void compileTimeError(STMessage stMessage) { + if (stMessage instanceof STCompiletimeMessage) { + STCompiletimeMessage compileMessage = (STCompiletimeMessage) stMessage; + errorManager.grammarError(ErrorType.ERROR_COMPILING_ACTION_TEMPLATE, ast.g.fileName, ast.getToken(), getSTCompiletimeErrorMessage(compileMessage)); + } + else if (stMessage instanceof STLexerMessage) { + STLexerMessage lexerMessage = (STLexerMessage) stMessage; + errorManager.grammarError(ErrorType.ERROR_COMPILING_ACTION_TEMPLATE, ast.g.fileName, ast.getToken(), getSTLexerErrorMessage(lexerMessage)); + } + else { + errorManager.grammarError(ErrorType.ERROR_COMPILING_ACTION_TEMPLATE, ast.g.fileName, ast.getToken(), stMessage.toString()); + } + } + + /** + * Get the STRuntimeMessage error location Coordinate. + */ + private Coordinate getSTRuntimeMessageSourceLocation(STRuntimeMessage msg) { + // From STRuntimeMessage.getSourceLocation + if (msg.ip >= 0 && msg.self != null && msg.self.impl != null) { + Interval I = msg.self.impl.sourceMap[msg.ip]; + if (I == null) { + return null; + } else { + int i = I.a; + return Misc.getLineCharPosition(msg.self.impl.template, i); + } + } else { + return null; + } + } + + /** + * Get the STRuntimeMessage error message content, translating the source location + * according to the action token's position in the grammar. + */ + private String getSTRuntimeErrorMessage(STRuntimeMessage stMsg) { + StringWriter sw = new StringWriter(); + PrintWriter pw = new PrintWriter(sw); + Coordinate coord = getSTRuntimeMessageSourceLocation(stMsg); + + if (coord != null) { + int linePos = ast.getLine() + coord.line - 1; + int charPos = coord.charPosition; + if (coord.line == 1) { + charPos += ast.getCharPositionInLine(); + } + pw.print(linePos + ":" + charPos + ": "); + } + + // From STMessage.toString + String msg = String.format(stMsg.error.message, stMsg.arg, stMsg.arg2, stMsg.arg3); + + pw.print(msg); + + if (stMsg.cause != null) { + pw.print("\nCaused by: "); + stMsg.cause.printStackTrace(pw); + } + + return sw.toString(); + } + + @Override + public void runTimeError(STMessage stMessage) { + if (stMessage instanceof STRuntimeMessage) { + STRuntimeMessage runtimeMessage = (STRuntimeMessage) stMessage; + errorManager.grammarError( + ErrorType.ERROR_RENDERING_ACTION_TEMPLATE, + ast.g.fileName, + ast.getToken(), + getSTRuntimeErrorMessage(runtimeMessage)); + } else { + errorManager.grammarError( + ErrorType.ERROR_RENDERING_ACTION_TEMPLATE, + ast.g.fileName, + ast.getToken(), + stMessage.toString()); + } + } + + @Override + public void IOError(STMessage stMessage) { + reportError(stMessage); + } + + @Override + public void internalError(STMessage stMessage) { + reportError(stMessage); + } + + private void reportError(STMessage stMessage) { + errorManager.toolError(ErrorType.INTERNAL_ERROR, stMessage.cause, stMessage.toString()); + } + }; + } + + public File getActionTemplatesGroupFile(GrammarRootAST root, String actionTemplates) { + // Try for an absolute path + File importedFile = new File(actionTemplates); + + if (!importedFile.exists()) { + // Next try the input directory + importedFile = new File(root.g.tool.inputDirectory, actionTemplates); + if (!importedFile.exists()) { + // Next try the parent directory of the grammar file + File grammarFile = new File(root.g.fileName); + String parentDir = grammarFile.getParent(); + importedFile = new File(parentDir, actionTemplates); + if (!importedFile.exists()) { + // Next try the lib directory + importedFile = new File(root.g.tool.libDirectory, actionTemplates); + if (!importedFile.exists()) { + return null; + } + } + } + } + + return importedFile; + } + + public void cannotFindActionTemplatesFileError(GrammarRootAST root) { + Grammar grammar = root.g; + ErrorManager errorManager = grammar.tool.errMgr; + String actionTemplatesFile = grammar.getActionTemplates(); + + // Check whether this action template file came from options in the grammar file + GrammarAST optionAST = root.getOptionAST("actionTemplates"); + + if (optionAST != null && actionTemplatesFile.equals(optionAST.getToken().getText())) { + errorManager.grammarError( + ErrorType.CANNOT_FIND_ACTION_TEMPLATES_FILE_REFD_IN_GRAMMAR, + grammar.fileName, + optionAST.getToken(), actionTemplatesFile); + } else { + errorManager.toolError( + ErrorType.CANNOT_FIND_ACTION_TEMPLATES_FILE_GIVEN_ON_CMDLINE, + actionTemplatesFile, + grammar.name); + } + } + + public STGroupFile loadActionTemplatesGroupFile(GrammarRootAST root) { + Grammar grammar = root.g; + ErrorManager errorManager = grammar.tool.errMgr; + String actionTemplatesFile = grammar.getActionTemplates(); + File actionTemplatesGroupFile = getActionTemplatesGroupFile(root, actionTemplatesFile); + + if (actionTemplatesGroupFile == null) { + cannotFindActionTemplatesFileError(root); + return null; + } + + try { + STGroupFile actionTemplates = new STGroupFile(actionTemplatesGroupFile.getAbsolutePath()); + STErrorListener errorListener = createActionTemplateErrorListener(root, actionTemplates); + + // Force load the action templates group file + actionTemplates.setListener(errorListener); + actionTemplates.load(); + + return actionTemplates; + + } catch (IllegalArgumentException e) { + if (e.getMessage() != null && e.getMessage().startsWith("No such group file")) { + cannotFindActionTemplatesFileError(root); + } else { + errorManager.toolError( + ErrorType.ERROR_READING_ACTION_TEMPLATES_FILE, e, + actionTemplatesFile, + e.getMessage()); + } + } catch (STException e) { + errorManager.toolError( + ErrorType.ERROR_READING_ACTION_TEMPLATES_FILE, e, + actionTemplatesFile, + e.getMessage()); + } + + return null; + } + + public void expandActionTemplates(GrammarRootAST root) { + Grammar grammar = root.g; + ErrorManager errorManager = grammar.tool.errMgr; + + STGroupFile actionTemplates = loadActionTemplatesGroupFile(root); + + if (actionTemplates != null) { + TreeVisitor visitor = new TreeVisitor(new GrammarASTAdaptor()); + visitor.visit(root, new TreeVisitorAction() { + @Override + public Object pre(Object t) { + GrammarAST grammarAST = (GrammarAST) t; + int tokenType = grammarAST.getType(); + if (tokenType == ANTLRParser.ACTION || tokenType == ANTLRParser.SEMPRED) { + return expandActionTemplate((GrammarAST) t, errorManager, actionTemplates); + } + return t; + } + @Override + public Object post(Object t) { + return t; + } + }); + } + } + + public GrammarAST expandActionTemplate(GrammarAST ast, ErrorManager errMgr, STGroupFile actionTemplateGroupFile) { + // Trim the curly braces and trailing question mark + // from an action or semantic predicate + String actionText = ast.getText() + .substring(1, + ast.getType() == ANTLRParser.SEMPRED + ? ast.getText().length() - 2 + : ast.getText().length() - 1); + + STGroupString actionTemplateGroup = new STGroupString( + ast.g.fileName, "action() ::= << " + actionText + " >>"); + + actionTemplateGroup.importTemplates(actionTemplateGroupFile); + + actionTemplateGroup.setListener(createGrammarErrorListener(ast)); + + ST actionTemplate = actionTemplateGroup.getInstanceOf("action"); + + if (actionTemplate != null) { + ast.setText(actionTemplate.render()); + } + + return ast; + } }