[codegen] remder REs for comments into the tokenizer/buffer and parser

neogeny · Oct 13, 2023 · c777dfe · c777dfe
1 parent 0a34786
commit c777dfe
Showing 1 changed file with 9 additions and 3 deletions.
diff --git a/tatsu/codegen/python.py b/tatsu/codegen/python.py
@@ -471,6 +471,8 @@ def render_fields(self, fields):
         left_recursion = self.node.config.left_recursion
         parseinfo = self.node.config.parseinfo
         namechars = repr(self.node.config.namechars or '')
+        comments_re = repr(self.node.config.comments_re)
+        eol_comments_re = repr(self.node.config.eol_comments_re)
 
         rules = '\n'.join([
             self.get_renderer(rule).render() for rule in self.node.rules
@@ -494,6 +496,8 @@ def render_fields(self, fields):
                       parseinfo=parseinfo,
                       keywords=keywords,
                       namechars=namechars,
+                      comments_re=comments_re,
+                      eol_comments_re=eol_comments_re,
                       )
 
     abstract_rule_template = '''
@@ -513,8 +517,6 @@ def {name}(self, ast):  # noqa
                 # Any changes you make to it will be overwritten the next time
                 # the file is generated.
 
-                from __future__ import annotations
-
                 import sys
 
                 from tatsu.buffering import Buffer
@@ -528,7 +530,7 @@ def {name}(self, ast):  # noqa
                 KEYWORDS = {{{keywords}}}  # type: ignore
 
 
-                class {name}Buffer(Buffer):
+                class {name}Tokenizer(Buffer):
                     def __init__(self, text, /, config: ParserConfig | None = None, **settings):
                         config = ParserConfig.new(
                             config,
@@ -537,6 +539,8 @@ def __init__(self, text, /, config: ParserConfig | None = None, **settings):
                             ignorecase={ignorecase},
                             namechars={namechars},
                             parseinfo={parseinfo},
+                            comments_re={comments_re},
+                            eol_comments_re={eol_comments_re},
                         )
                         config = config.replace(**settings)
                         super().__init__(text, config=config)
@@ -551,6 +555,8 @@ def __init__(self, /, config: ParserConfig | None = None, **settings):
                             ignorecase={ignorecase},
                             namechars={namechars},
                             parseinfo={parseinfo},
+                            comments_re={comments_re},
+                            eol_comments_re={eol_comments_re},
                             keywords=KEYWORDS,
                             start={start!r},
                         )