[parserconfig] add memoization parameter/option and directive (#377)

There is no left recursion without memoization.

Author: apalala

grammar/tatsu.ebnf

@@ -42,7 +42,7 @@ directive
             ~
             '::' ~ value:(regex|string|'None'|'False'|`None`)
         |
-            name:('nameguard' | 'ignorecase' | 'left_recursion' | 'parseinfo')
+            name:('nameguard' | 'ignorecase' | 'left_recursion' | 'parseinfo' | 'memoization')
             ~
             ('::' ~ value:boolean|value:`True`)
         |

tatsu/bootstrap.py

@@ -36,7 +36,7 @@ def __init__(self, text, /, config: ParserConfig | None = None, **settings):
             namechars='',
             parseinfo=True,
             comments='(?sm)[(][*](?:.|\\n)*?[*][)]',
-            eol_comments='(?sm)*#[^\\n]*$',
+            eol_comments='(?m)#[^\\n]*$',
             keywords=KEYWORDS,
             start='start',
         )
@@ -50,7 +50,7 @@ def __init__(self, /, config: ParserConfig | None = None, **settings):
         config = ParserConfig.new(
             config,
             owner=self,
-            whitespace='\\s+',
+            whitespace='(?m)\\s+',
             nameguard=None,
             ignorecase=False,
             namechars='',
@@ -172,10 +172,12 @@ def _directive_(self):
                                 self._token('left_recursion')
                             with self._option():
                                 self._token('parseinfo')
+                            with self._option():
+                                self._token('memoization')
                             self._error(
                                 'expecting one of: '
                                 "'ignorecase' 'left_recursion'"
-                                "'nameguard' 'parseinfo'"
+                                "'memoization' 'nameguard' 'parseinfo'"
                             )
                     self.name_last_node('name')
                     self._cut()
@@ -219,8 +221,8 @@ def _directive_(self):
                     'expecting one of: '
                     "'comments' 'eol_comments' 'grammar'"
                     "'ignorecase' 'left_recursion'"
-                    "'namechars' 'nameguard' 'parseinfo'"
-                    "'whitespace'"
+                    "'memoization' 'namechars' 'nameguard'"
+                    "'parseinfo' 'whitespace'"
                 )
         self._cut()
         self._define(['name', 'value'], [])

tatsu/contexts.py

@@ -420,7 +420,10 @@ def _cut(self):
         self._cut_stack[-1] = True
 
     def _memoization(self):
-        return self.memoize_lookaheads or self._lookahead == 0
+        return self.config.memoization and (
+                    self.memoize_lookaheads or
+                    self._lookahead == 0
+                )
 
     def _rulestack(self):
         rulestack = [r.name for r in reversed(self._rule_stack)]

tatsu/grammars.py

@@ -1069,7 +1069,7 @@ def nodecount(self):
 
     def _to_str(self, lean=False):
         regex_directives = {'comments', 'eol_comments', 'whitespace'}
-        ustr_directives = {'comments', 'grammar'}
+        str_directives = {'comments', 'grammar'}
         string_directives = {'namechars'}
 
         directives = ''
@@ -1081,7 +1081,7 @@ def _to_str(self, lean=False):
                     repr(value)
                     if directive in string_directives
                     else str(value)
-                    if directive in ustr_directives
+                    if directive in str_directives
                     else value
                 ),
             )

tatsu/infos.py

@@ -40,7 +40,10 @@ class ParserConfig:
     semantics: type | None = None
 
     comment_recovery: bool = False
+
+    memoization: bool = True
     memoize_lookaheads: bool = True
+    memo_cache_size: int = MEMO_CACHE_SIZE
 
     colorize: bool = False
     trace: bool = False
@@ -62,7 +65,6 @@ class ParserConfig:
     whitespace: str | None = _undefined_str
 
     parseinfo: bool = False
-    memo_cache_size: int = MEMO_CACHE_SIZE
 
     def __post_init__(self):  # pylint: disable=W0235
         if self.ignorecase:
@@ -83,6 +85,9 @@ def __post_init__(self):  # pylint: disable=W0235
         if self.whitespace:
             cached_re_compile(self.whitespace)
 
+        if not self.memoization:
+            self.left_recursion = False
+
     @classmethod
     def new(
         cls,