Keywords must be str (#336)

* [test] upgrade keyword_test to pytest

* [test] add unit test to check that keywords are str

* [grammar] make keywords always have type str

fixes #335

* [dist] bump up version for release

Author: apalala

grammar/tatsu.ebnf

@@ -65,7 +65,7 @@ keywords
 
 keyword
     =
-    '@@keyword' ~ '::' ~ {@+:literal !(':'|'=')}
+    '@@keyword' ~ '::' ~ {@+:(word|string) !(':'|'=')}
     ;
 
 

tatsu/_version.py

@@ -1 +1 @@
-__version__ = '5.11.4b1'
+__version__ = '5.12.0'

tatsu/bootstrap.py

@@ -238,7 +238,16 @@ def _keyword_(self):
         self._cut()
 
         def block0():
-            self._literal_()
+            with self._group():
+                with self._choice():
+                    with self._option():
+                        self._word_()
+                    with self._option():
+                        self._string_()
+                    self._error(
+                        'expecting one of: '
+                        '<string> <word>'
+                    )
             self.add_last_node_to_name('@')
             with self._ifnot():
                 with self._group():

test/grammar/keyword_test.py

@@ -1,152 +1,169 @@
-import unittest
 from ast import parse
 
+import pytest
+
 from tatsu.exceptions import FailedParse
 from tatsu.ngcodegen import codegen
 from tatsu.tool import compile
 
 
-class KeywordTests(unittest.TestCase):
-    def test_keywords_in_rule_names(self):
-        grammar = """
-            start
-                =
-                whitespace
-                ;
-
+def test_keywords_in_rule_names():
+    grammar = """
+        start
+            =
             whitespace
-                =
-                    {'x'}+
-                ;
-        """
-        m = compile(grammar, 'Keywords')
-        m.parse('x')
-
-    def test_python_keywords_in_rule_names(self):
-        # This is a regression test for
-        # https://bitbucket.org/neogeny/tatsu/issues/59
-        # (semantic actions not called for rules with the same name as a python
-        # keyword).
-        grammar = """
-            not = 'x' ;
-        """
-        m = compile(grammar, 'Keywords')
-
-        class Semantics:
-            def __init__(self):
-                self.called = False
-
-            def not_(self, ast):
-                self.called = True
-
-        semantics = Semantics()
-        m.parse('x', semantics=semantics)
-        assert semantics.called
-
-    def test_define_keywords(self):
-        grammar = """
-            @@keyword :: B C
-            @@keyword :: 'A'
-
-            start = ('a' 'b').{'x'}+ ;
-        """
-        model = compile(grammar, 'test')
-        c = codegen(model)
-        parse(c)
-
-        grammar2 = str(model)
-        model2 = compile(grammar2, 'test')
-        c2 = codegen(model2)
-        parse(c2)
-
-        self.assertEqual(grammar2, str(model2))
-
-    def test_check_keywords(self):
-        grammar = r"""
-            @@keyword :: A
-
-            start = {id}+ $ ;
-
-            @name
-            id = /\w+/ ;
-        """
-        model = compile(grammar, 'test')
-        c = codegen(model)
-        print(c)
-        parse(c)
-
-        ast = model.parse('hello world')
-        self.assertEqual(['hello', 'world'], ast)
+            ;
 
-        try:
-            ast = model.parse('hello A world')
-            self.assertEqual(['hello', 'A', 'world'], ast)
-            self.fail('accepted keyword as name')
-        except FailedParse as e:
-            self.assertTrue('"A" is a reserved word' in str(e))
+        whitespace
+            =
+                {'x'}+
+            ;
+    """
+    m = compile(grammar, 'Keywords')
+    m.parse('x')
+
+
+def test_python_keywords_in_rule_names():
+    # This is a regression test for
+    # https://bitbucket.org/neogeny/tatsu/issues/59
+    # (semantic actions not called for rules with the same name as a python
+    # keyword).
+    grammar = """
+        not = 'x' ;
+    """
+    m = compile(grammar, 'Keywords')
+
+    class Semantics:
+        def __init__(self):
+            self.called = False
+
+        def not_(self, ast):
+            self.called = True
+
+    semantics = Semantics()
+    m.parse('x', semantics=semantics)
+    assert semantics.called
+
+
+def test_define_keywords():
+    grammar = """
+        @@keyword :: B C
+        @@keyword :: 'A'
+
+        start = ('a' 'b').{'x'}+ ;
+    """
+    model = compile(grammar, 'test')
+    c = codegen(model)
+    parse(c)
+
+    grammar2 = str(model)
+    model2 = compile(grammar2, 'test')
+    c2 = codegen(model2)
+    parse(c2)
+
+    assert grammar2 == str(model2)
 
-    def test_check_unicode_name(self):
-        grammar = r"""
-            @@keyword :: A
 
-            start = {id}+ $ ;
+def test_check_keywords():
+    grammar = r"""
+        @@keyword :: A
 
-            @name
-            id = /\w+/ ;
-        """
-        model = compile(grammar, 'test')
-        model.parse('hello Øresund')
+        start = {id}+ $ ;
 
-    def test_sparse_keywords(self):
-        grammar = r"""
-            @@keyword :: A
+        @name
+        id = /\w+/ ;
+    """
+    model = compile(grammar, 'test')
+    c = codegen(model)
+    print(c)
+    parse(c)
 
-            @@ignorecase :: False
+    ast = model.parse('hello world')
+    assert ast == ['hello', 'world']
 
-            start = {id}+ $ ;
+    try:
+        ast = model.parse('hello A world')
+        assert ast == ['hello', 'A', 'world']
+        pytest.fail('accepted keyword as name')
+    except FailedParse as e:
+        assert '"A" is a reserved word' in str(e)
+
+
+def test_check_unicode_name():
+    grammar = r"""
+        @@keyword :: A
+
+        start = {id}+ $ ;
+
+        @name
+        id = /\w+/ ;
+    """
+    model = compile(grammar, 'test')
+    model.parse('hello Øresund')
+
+
+def test_sparse_keywords():
+    grammar = r"""
+        @@keyword :: A
+
+        @@ignorecase :: False
+
+        start = {id}+ $ ;
+
+        @@keyword :: B
+
+        @name
+        id = /\w+/ ;
+    """
+    model = compile(grammar, 'test', trace=False, colorize=True)
+    c = codegen(model)
+    parse(c)
+
+    ast = model.parse('hello world')
+    assert ast == ['hello', 'world']
+
+    for k in ('A', 'B'):
+        try:
+            ast = model.parse('hello %s world' % k)
+            assert ['hello', k, 'world'] == ast
+            pytest.fail('accepted keyword "%s" as name' % k)
+        except FailedParse as e:
+            assert '"%s" is a reserved word' % k in str(e)
 
-            @@keyword :: B
 
-            @name
-            id = /\w+/ ;
-        """
-        model = compile(grammar, 'test', trace=False, colorize=True)
-        c = codegen(model)
-        parse(c)
+def test_ignorecase_keywords():
+    grammar = r"""
+        @@ignorecase :: True
+        @@keyword :: if
 
-        ast = model.parse('hello world')
-        self.assertEqual(['hello', 'world'], ast)
+        start = rule ;
 
-        for k in ('A', 'B'):
-            try:
-                ast = model.parse('hello %s world' % k)
-                self.assertEqual(['hello', k, 'world'], ast)
-                self.fail('accepted keyword "%s" as name' % k)
-            except FailedParse as e:
-                self.assertTrue('"%s" is a reserved word' % k in str(e))
+        @name
+        rule = @:word if_exp $ ;
 
-    def test_ignorecase_keywords(self):
-        grammar = r"""
-            @@ignorecase :: True
-            @@keyword :: if
+        if_exp = 'if' digit ;
 
-            start = rule ;
+        word = /\w+/ ;
+        digit = /\d/ ;
+    """
 
-            @name
-            rule = @:word if_exp $ ;
+    model = compile(grammar, 'test')
 
-            if_exp = 'if' digit ;
+    model.parse('nonIF if 1', trace=False)
 
-            word = /\w+/ ;
-            digit = /\d/ ;
-        """
+    with pytest.raises(FailedParse):
+        model.parse('i rf if 1', trace=False)
 
-        model = compile(grammar, 'test')
+    with pytest.raises(FailedParse):
+        model.parse('IF if 1', trace=False)
 
-        model.parse('nonIF if 1', trace=False)
 
-        with self.assertRaises(FailedParse):
-            model.parse('i rf if 1', trace=False)
+def test_keywords_are_str():
+    grammar = r"""
+        @@keyword :: True False
 
-        with self.assertRaises(FailedParse):
-            model.parse('IF if 1', trace=False)
+        start = $ ;
+    """
+    model = compile(grammar, 'test')
+    assert model.keywords == ['True', 'False']
+    assert all(isinstance(k, str) for k in model.keywords)