Merge remote-tracking branch 'refs/remotes/origin/master'

docs/directives.rst

@@ -109,8 +109,15 @@ Enabling ``@@parseinfo`` will allow precise reporting over the input source-code
 ``@@whitespace :: <regexp>``
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Provides a regular expression for the whitespace to be ignored by the parser. It defaults to ``/(?s)\s+/``:
+Provides a regular expression for the whitespace to be ignored by the parser. If no definition is
+provided, then ``r'(?m)\s+'`` will be used as default:
 
 .. code::
 
     @@whitespace :: /[\t ]+/
+
+To disable any parsing of whitespace, use ``None`` for the definition:
+
+.. code::
+
+    @@whitespace :: None

docs/syntax.rst

@@ -699,6 +699,20 @@ overwrite the setting in the grammar::
 
   @@whitespace :: /[\t ]+/
 
+If no ``whitespace`` or ``@@whitespace`` is specified, |TatSu| will use
+``r'(?m)\s+'`` as a default. Use ``None`` to have *no whitespace definition*.
+
+
+.. code:: python
+
+   parser = MyParser(text, whitespace=None)
+
+or:
+
+.. code::
+
+    @@whitespace :: None
+
 Case Sensitivity
 ~~~~~~~~~~~~~~~~
 

tatsu/_version.py

@@ -1 +1 @@
-__version__ = '5.10.4b1'
+__version__ = '5.10.4'

tatsu/bootstrap.py

@@ -9,17 +9,17 @@
 # Any changes you make to it will be overwritten the next time
 # the file is generated.
 
-# ruff: noqa: I001, SIM117
+# ruff: noqa: I001, F401, SIM117
 
 import sys
 from pathlib import Path
 
 from tatsu.buffering import Buffer
 from tatsu.parsing import Parser
 from tatsu.parsing import tatsumasu
-from tatsu.parsing import leftrec, nomemo, isname  # noqa: F401
+from tatsu.parsing import leftrec, nomemo, isname
 from tatsu.infos import ParserConfig
-from tatsu.util import re, generic_main  # noqa: F401
+from tatsu.util import re, generic_main
 
 
 KEYWORDS: set[str] = set()

tatsu/buffering.py

@@ -13,7 +13,14 @@
 from typing import Any
 
 from .exceptions import ParseError
-from .infos import CommentInfo, LineIndexInfo, LineInfo, ParserConfig, PosLine
+from .infos import (
+    CommentInfo,
+    LineIndexInfo,
+    LineInfo,
+    ParserConfig,
+    PosLine,
+    UndefinedStr,
+)
 from .tokenizing import Tokenizer
 from .util import (
     RETYPE,
@@ -76,12 +83,17 @@ def whitespace(self):
 
     @staticmethod
     def build_whitespace_re(whitespace):
-        if whitespace is None:
+        if type(whitespace) is UndefinedStr:
             return WHITESPACE_RE
+        if whitespace in {None, ''}:
+            return None
         elif isinstance(whitespace, RETYPE):
             return whitespace
         elif whitespace:
             if not isinstance(whitespace, str):
+                # FIXME:
+                #   this feature is undocumented
+                #   only regular expressions should be supported
                 # a list or a set?
                 whitespace = f"[{''.join(c for c in whitespace)}]+"
             return re.compile(f'(?m){whitespace}')

tatsu/codegen/python.py

@@ -509,17 +509,17 @@ def {name}(self, ast):
                 # Any changes you make to it will be overwritten the next time
                 # the file is generated.
 
-                # ruff: noqa: I001, SIM117
+                # ruff: noqa: I001, F401, SIM117
 
                 import sys
                 from pathlib import Path
 
                 from tatsu.buffering import Buffer
                 from tatsu.parsing import Parser
                 from tatsu.parsing import tatsumasu
-                from tatsu.parsing import leftrec, nomemo, isname  # noqa: F401
+                from tatsu.parsing import leftrec, nomemo, isname
                 from tatsu.infos import ParserConfig
-                from tatsu.util import re, generic_main  # noqa: F401
+                from tatsu.util import re, generic_main
 
 
                 KEYWORDS: set[str] = set({keywords})

tatsu/infos.py

@@ -11,6 +11,13 @@
 from .util.unicode_characters import C_DERIVE
 
 
+class UndefinedStr(str):
+    pass
+
+
+_undefined_str = UndefinedStr()
+
+
 @dataclasses.dataclass
 class ParserConfig:
     owner: Any = None
@@ -48,7 +55,7 @@ class ParserConfig:
     ignorecase: bool | None = False
     namechars: str = ''
     nameguard: bool | None = None  # implied by namechars
-    whitespace: str | None = None
+    whitespace: str | None = _undefined_str
 
     parseinfo: bool = False
 

tatsu/parser_semantics.py

@@ -136,6 +136,7 @@ def grammar(self, ast, *args):
         keywords = list(flatten(ast.keywords)) or []
 
         if directives.get('whitespace') in {'None', 'False'}:
+            # NOTE: use '' because None will _not_ override defaults in configuration
             directives['whitespace'] = ''
 
         name = (

test/grammar/directive_test.py

@@ -44,6 +44,13 @@ def test_whitespace_none_directive(self):
             else:
                 self.fail('parsed through non-whitespace')
 
+    def test_default_whitespace(self):
+        grammar = r"""
+            start = {'x'}+ $;
+        """
+
+        tatsu.parse(grammar, "x x x")
+
     def test_eol_comments_re_directive(self):
         grammar = """
             @@eol_comments :: /#.*?$/