From 9f0ad496eed1699eebde94022139a548c246516b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juancarlo=20A=C3=B1ez?= Date: Mon, 18 Mar 2024 18:27:34 -0400 Subject: [PATCH] Keywords must be str (#336) * [test] upgrade keyword_test to pytest * [test] add unit test to check that keywords are str * [grammar] make keywords always have type str fixes #335 * [dist] bump up version for release --- grammar/tatsu.ebnf | 2 +- tatsu/_version.py | 2 +- tatsu/bootstrap.py | 11 +- test/grammar/keyword_test.py | 265 +++++++++++++++++++---------------- 4 files changed, 153 insertions(+), 127 deletions(-) diff --git a/grammar/tatsu.ebnf b/grammar/tatsu.ebnf index a70e42fc..32dac162 100644 --- a/grammar/tatsu.ebnf +++ b/grammar/tatsu.ebnf @@ -65,7 +65,7 @@ keywords keyword = - '@@keyword' ~ '::' ~ {@+:literal !(':'|'=')} + '@@keyword' ~ '::' ~ {@+:(word|string) !(':'|'=')} ; diff --git a/tatsu/_version.py b/tatsu/_version.py index cfb36a71..d1e994c6 100644 --- a/tatsu/_version.py +++ b/tatsu/_version.py @@ -1 +1 @@ -__version__ = '5.11.4b1' +__version__ = '5.12.0' diff --git a/tatsu/bootstrap.py b/tatsu/bootstrap.py index 2f74ba91..b34dfae4 100644 --- a/tatsu/bootstrap.py +++ b/tatsu/bootstrap.py @@ -238,7 +238,16 @@ def _keyword_(self): self._cut() def block0(): - self._literal_() + with self._group(): + with self._choice(): + with self._option(): + self._word_() + with self._option(): + self._string_() + self._error( + 'expecting one of: ' + ' ' + ) self.add_last_node_to_name('@') with self._ifnot(): with self._group(): diff --git a/test/grammar/keyword_test.py b/test/grammar/keyword_test.py index 4380354f..a5111955 100644 --- a/test/grammar/keyword_test.py +++ b/test/grammar/keyword_test.py @@ -1,152 +1,169 @@ -import unittest from ast import parse +import pytest + from tatsu.exceptions import FailedParse from tatsu.ngcodegen import codegen from tatsu.tool import compile -class KeywordTests(unittest.TestCase): - def test_keywords_in_rule_names(self): - grammar = """ - start - = - whitespace - ; - +def test_keywords_in_rule_names(): + grammar = """ + start + = whitespace - = - {'x'}+ - ; - """ - m = compile(grammar, 'Keywords') - m.parse('x') - - def test_python_keywords_in_rule_names(self): - # This is a regression test for - # https://bitbucket.org/neogeny/tatsu/issues/59 - # (semantic actions not called for rules with the same name as a python - # keyword). - grammar = """ - not = 'x' ; - """ - m = compile(grammar, 'Keywords') - - class Semantics: - def __init__(self): - self.called = False - - def not_(self, ast): - self.called = True - - semantics = Semantics() - m.parse('x', semantics=semantics) - assert semantics.called - - def test_define_keywords(self): - grammar = """ - @@keyword :: B C - @@keyword :: 'A' - - start = ('a' 'b').{'x'}+ ; - """ - model = compile(grammar, 'test') - c = codegen(model) - parse(c) - - grammar2 = str(model) - model2 = compile(grammar2, 'test') - c2 = codegen(model2) - parse(c2) - - self.assertEqual(grammar2, str(model2)) - - def test_check_keywords(self): - grammar = r""" - @@keyword :: A - - start = {id}+ $ ; - - @name - id = /\w+/ ; - """ - model = compile(grammar, 'test') - c = codegen(model) - print(c) - parse(c) - - ast = model.parse('hello world') - self.assertEqual(['hello', 'world'], ast) + ; - try: - ast = model.parse('hello A world') - self.assertEqual(['hello', 'A', 'world'], ast) - self.fail('accepted keyword as name') - except FailedParse as e: - self.assertTrue('"A" is a reserved word' in str(e)) + whitespace + = + {'x'}+ + ; + """ + m = compile(grammar, 'Keywords') + m.parse('x') + + +def test_python_keywords_in_rule_names(): + # This is a regression test for + # https://bitbucket.org/neogeny/tatsu/issues/59 + # (semantic actions not called for rules with the same name as a python + # keyword). + grammar = """ + not = 'x' ; + """ + m = compile(grammar, 'Keywords') + + class Semantics: + def __init__(self): + self.called = False + + def not_(self, ast): + self.called = True + + semantics = Semantics() + m.parse('x', semantics=semantics) + assert semantics.called + + +def test_define_keywords(): + grammar = """ + @@keyword :: B C + @@keyword :: 'A' + + start = ('a' 'b').{'x'}+ ; + """ + model = compile(grammar, 'test') + c = codegen(model) + parse(c) + + grammar2 = str(model) + model2 = compile(grammar2, 'test') + c2 = codegen(model2) + parse(c2) + + assert grammar2 == str(model2) - def test_check_unicode_name(self): - grammar = r""" - @@keyword :: A - start = {id}+ $ ; +def test_check_keywords(): + grammar = r""" + @@keyword :: A - @name - id = /\w+/ ; - """ - model = compile(grammar, 'test') - model.parse('hello Øresund') + start = {id}+ $ ; - def test_sparse_keywords(self): - grammar = r""" - @@keyword :: A + @name + id = /\w+/ ; + """ + model = compile(grammar, 'test') + c = codegen(model) + print(c) + parse(c) - @@ignorecase :: False + ast = model.parse('hello world') + assert ast == ['hello', 'world'] - start = {id}+ $ ; + try: + ast = model.parse('hello A world') + assert ast == ['hello', 'A', 'world'] + pytest.fail('accepted keyword as name') + except FailedParse as e: + assert '"A" is a reserved word' in str(e) + + +def test_check_unicode_name(): + grammar = r""" + @@keyword :: A + + start = {id}+ $ ; + + @name + id = /\w+/ ; + """ + model = compile(grammar, 'test') + model.parse('hello Øresund') + + +def test_sparse_keywords(): + grammar = r""" + @@keyword :: A + + @@ignorecase :: False + + start = {id}+ $ ; + + @@keyword :: B + + @name + id = /\w+/ ; + """ + model = compile(grammar, 'test', trace=False, colorize=True) + c = codegen(model) + parse(c) + + ast = model.parse('hello world') + assert ast == ['hello', 'world'] + + for k in ('A', 'B'): + try: + ast = model.parse('hello %s world' % k) + assert ['hello', k, 'world'] == ast + pytest.fail('accepted keyword "%s" as name' % k) + except FailedParse as e: + assert '"%s" is a reserved word' % k in str(e) - @@keyword :: B - @name - id = /\w+/ ; - """ - model = compile(grammar, 'test', trace=False, colorize=True) - c = codegen(model) - parse(c) +def test_ignorecase_keywords(): + grammar = r""" + @@ignorecase :: True + @@keyword :: if - ast = model.parse('hello world') - self.assertEqual(['hello', 'world'], ast) + start = rule ; - for k in ('A', 'B'): - try: - ast = model.parse('hello %s world' % k) - self.assertEqual(['hello', k, 'world'], ast) - self.fail('accepted keyword "%s" as name' % k) - except FailedParse as e: - self.assertTrue('"%s" is a reserved word' % k in str(e)) + @name + rule = @:word if_exp $ ; - def test_ignorecase_keywords(self): - grammar = r""" - @@ignorecase :: True - @@keyword :: if + if_exp = 'if' digit ; - start = rule ; + word = /\w+/ ; + digit = /\d/ ; + """ - @name - rule = @:word if_exp $ ; + model = compile(grammar, 'test') - if_exp = 'if' digit ; + model.parse('nonIF if 1', trace=False) - word = /\w+/ ; - digit = /\d/ ; - """ + with pytest.raises(FailedParse): + model.parse('i rf if 1', trace=False) - model = compile(grammar, 'test') + with pytest.raises(FailedParse): + model.parse('IF if 1', trace=False) - model.parse('nonIF if 1', trace=False) - with self.assertRaises(FailedParse): - model.parse('i rf if 1', trace=False) +def test_keywords_are_str(): + grammar = r""" + @@keyword :: True False - with self.assertRaises(FailedParse): - model.parse('IF if 1', trace=False) + start = $ ; + """ + model = compile(grammar, 'test') + assert model.keywords == ['True', 'False'] + assert all(isinstance(k, str) for k in model.keywords)