Add newline, indent, dedent external tokens for raw_text rules (#3)

Attempt to capture inline source code nodes written within script and style tags. Recognizes where source code begins but not where it ends.

Author: deequez

binding.gyp

@@ -9,7 +9,7 @@
       "sources": [
         "bindings/node/binding.cc",
         "src/parser.c",
-        # If your language uses an external scanner, add it here.
+        "src/scanner.cc"
       ],
       "cflags_c": [
         "-std=c99",

example-file.pug

@@ -39,4 +39,4 @@ html
     include includes/footer.pug
 
     script(src='https://code.jquery.com/jquery-3.4.1.slim.min.js')
-    script(src='https://getbootstrap.com/docs/4.4/dist/js/bootstrap.bundle)
+    script(src='https://getbootstrap.com/docs/4.4/dist/js/bootstrap.bundle')

grammar.js

@@ -1,6 +1,8 @@
 module.exports = grammar({
   name: "pug",
 
+  externals: ($) => [$._newline, $._indent, $._dedent],
+
   rules: {
     source_file: ($) =>
       repeat(
@@ -20,12 +22,16 @@ module.exports = grammar({
 
     link_tag: ($) => seq("link", optional($._attributes)),
 
-    script_tag: ($) => seq("script", optional($._attributes)),
+    script_tag: ($) =>
+      seq("script", optional($._attributes), optional(seq(".", $._newline))),
 
-    style_tag: ($) => seq("style", optional($._attributes)),
+    style_tag: ($) =>
+      seq("style", optional($._attributes), optional(seq(".", $._newline))),
 
     path: ($) => seq(repeat1(choice(/\w/, "/")), ".", repeat1(/\w/)),
 
+    raw_text: ($) => choice(seq($._indent, /\w/, $._dedent)),
+
     _attributes: ($) =>
       seq("(", repeat(seq($.attribute, optional(choice(",", " ")))), ")"),
 

src/grammar.json

@@ -106,6 +106,27 @@
               "type": "BLANK"
             }
           ]
+        },
+        {
+          "type": "CHOICE",
+          "members": [
+            {
+              "type": "SEQ",
+              "members": [
+                {
+                  "type": "STRING",
+                  "value": "."
+                },
+                {
+                  "type": "SYMBOL",
+                  "name": "_newline"
+                }
+              ]
+            },
+            {
+              "type": "BLANK"
+            }
+          ]
         }
       ]
     },
@@ -127,6 +148,27 @@
               "type": "BLANK"
             }
           ]
+        },
+        {
+          "type": "CHOICE",
+          "members": [
+            {
+              "type": "SEQ",
+              "members": [
+                {
+                  "type": "STRING",
+                  "value": "."
+                },
+                {
+                  "type": "SYMBOL",
+                  "name": "_newline"
+                }
+              ]
+            },
+            {
+              "type": "BLANK"
+            }
+          ]
         }
       ]
     },
@@ -162,6 +204,28 @@
         }
       ]
     },
+    "raw_text": {
+      "type": "CHOICE",
+      "members": [
+        {
+          "type": "SEQ",
+          "members": [
+            {
+              "type": "SYMBOL",
+              "name": "_indent"
+            },
+            {
+              "type": "PATTERN",
+              "value": "\\w"
+            },
+            {
+              "type": "SYMBOL",
+              "name": "_dedent"
+            }
+          ]
+        }
+      ]
+    },
     "_attributes": {
       "type": "SEQ",
       "members": [
@@ -340,7 +404,20 @@
   ],
   "conflicts": [],
   "precedences": [],
-  "externals": [],
+  "externals": [
+    {
+      "type": "SYMBOL",
+      "name": "_newline"
+    },
+    {
+      "type": "SYMBOL",
+      "name": "_indent"
+    },
+    {
+      "type": "SYMBOL",
+      "name": "_dedent"
+    }
+  ],
   "inline": [],
   "supertypes": []
 }