parser: added support for comments, set up tree-sitter boilerplate

Author: kiahjh

Cargo.lock

@@ -5,3 +5,7 @@ members = [
   "fen_parser",
   "fen_lsp"
 ]
+exclude = [
+  "tree-sitter-fen",
+  "documentation"
+]

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "fen_cli"
-version = "0.2.1"
+version = "0.3.0"
 description = "The command line interface for Fen"
 edition = "2021"
 repository = "https://github.com/kiahjh/fen"
@@ -29,4 +29,4 @@ colored = "2.2.0"
 serde = "1.0.217"
 serde_json = "1.0.134"
 toml = "0.8.19"
-fen_parser = { path = "../fen_parser", version = "0.2.0" }
+fen_parser = { path = "../fen_parser", version = "0.3.0" }

fen_cli/Cargo.toml

@@ -1,5 +1,8 @@
 # Changelog
 
+## [0.3.0]
+- fen: support for comments with `//`
+
 ## [0.2.0]
 - all: appended `_fen_/` to all paths
 - all: groundwork laid for annotation support

changelog.md fen_cli/changelog.mdchangelog.md renamed to fen_cli/changelog.md

@@ -1,6 +1,6 @@
 [package]
 name = "fen_parser"
-version = "0.2.0"
+version = "0.3.0"
 description = "A parser for Fen"
 edition = "2021"
 repository = "https://github.com/kiahjh/fen"

fen_parser/Cargo.toml

@@ -45,6 +45,35 @@ impl Lexer {
         self.peek_token = None;
     }
 
+    fn skip_whitespace(&mut self) {
+        while self.peek_char().map_or(false, u8::is_ascii_whitespace) {
+            self.pos += 1;
+        }
+    }
+
+    fn skip_extras(&mut self) -> Result<(), Error> {
+        self.skip_whitespace();
+
+        // skip over comments
+        if self.peek_char().map_or(false, |c| c == &b'/') {
+            self.next_char();
+            if self.peek_char().map_or(false, |c| c == &b'/') {
+                while self.peek_char().map_or(false, |c| c != &b'\n') {
+                    self.pos += 1;
+                }
+                self.skip_whitespace();
+                // recurse to skip any other comments:
+                if self.peek_char().map_or(false, |c| c == &b'/') {
+                    return self.skip_extras();
+                }
+            } else {
+                return Err(Error::new("Expected '/'", self.pos - 1));
+            }
+        }
+
+        Ok(())
+    }
+
     pub(crate) fn peek_tok(&mut self) -> Result<Option<&Token>, Error> {
         if self.peek_token.is_none() {
             self.peek_token = self.next_tok()?;
@@ -57,10 +86,7 @@ impl Lexer {
             return Ok(Some(peeked));
         }
 
-        // skip over whitespace
-        while self.peek_char().map_or(false, u8::is_ascii_whitespace) {
-            self.pos += 1;
-        }
+        self.skip_extras()?;
 
         let mut return_val: Result<Option<Token>, Error> = Ok(None);
 
@@ -281,32 +307,57 @@ mod tests {
         assert_eq!(lexer.next_tok(), Ok(None));
     }
 
+    #[test]
+    fn comments() {
+        expect_tokens(
+            r#"
+            // this is a comment
+            foo
+            // another comment
+            "bar"
+            "#,
+            &[
+                TokenKind::Identifier("foo".to_string()),
+                TokenKind::StringLiteral("bar".to_string()),
+            ],
+        );
+    }
+
     #[test]
     fn all_syntax() {
         expect_tokens(
             r#"
+            // comment
             name: "GetTodos"
             description: "Get all todos for a user"
-            authed: true
+            authed: true // here's another comment
 
             ---
 
+            // this is the input (nice no?)
             @input { user_id: String }
 
             @output [Todo]
 
             ---
 
+            @someAnnotation
             Todo {
               name: String
               due: Date?
+              // refers to custom enum
               priority: Priority?
               subtasks: [Todo]
             }
 
+            // we add this because this is actually
+            // a postgres enum
+            // these comments can be many lines long btw
+            // just double checking that it works
+            @sqlxType
             Priority (
               low
-              medium
+              medium // for the bourgeois case
               high
               other(String)
             )
@@ -335,6 +386,8 @@ mod tests {
                 TokenKind::Identifier("Todo".to_string()),
                 TokenKind::RightBracket,
                 TokenKind::Rule,
+                TokenKind::At,
+                TokenKind::Identifier("someAnnotation".to_string()),
                 TokenKind::Identifier("Todo".to_string()),
                 TokenKind::LeftBrace,
                 TokenKind::Identifier("name".to_string()),
@@ -354,6 +407,8 @@ mod tests {
                 TokenKind::Identifier("Todo".to_string()),
                 TokenKind::RightBracket,
                 TokenKind::RightBrace,
+                TokenKind::At,
+                TokenKind::Identifier("sqlxType".to_string()),
                 TokenKind::Identifier("Priority".to_string()),
                 TokenKind::LeftParen,
                 TokenKind::Identifier("low".to_string()),

fen_parser/src/lexer.rs

@@ -0,0 +1,46 @@
+root = true
+
+[*]
+charset = utf-8
+
+[*.{json,toml,yml,gyp}]
+indent_style = space
+indent_size = 2
+
+[*.js]
+indent_style = space
+indent_size = 2
+
+[*.scm]
+indent_style = space
+indent_size = 2
+
+[*.{c,cc,h}]
+indent_style = space
+indent_size = 4
+
+[*.rs]
+indent_style = space
+indent_size = 4
+
+[*.{py,pyi}]
+indent_style = space
+indent_size = 4
+
+[*.swift]
+indent_style = space
+indent_size = 4
+
+[*.go]
+indent_style = tab
+indent_size = 8
+
+[Makefile]
+indent_style = tab
+indent_size = 8
+
+[parser.c]
+indent_size = 2
+
+[{alloc,array,parser}.h]
+indent_size = 2

tree-sitter-fen/.editorconfig

@@ -0,0 +1,41 @@
+* text=auto eol=lf
+
+# Generated source files
+src/*.json linguist-generated
+src/parser.c linguist-generated
+src/tree_sitter/* linguist-generated
+
+# C bindings
+bindings/c/** linguist-generated
+CMakeLists.txt linguist-generated
+Makefile linguist-generated
+
+# Rust bindings
+bindings/rust/* linguist-generated
+Cargo.toml linguist-generated
+Cargo.lock linguist-generated
+
+# Node.js bindings
+bindings/node/* linguist-generated
+binding.gyp linguist-generated
+package.json linguist-generated
+package-lock.json linguist-generated
+
+# Python bindings
+bindings/python/** linguist-generated
+setup.py linguist-generated
+pyproject.toml linguist-generated
+
+# Go bindings
+bindings/go/* linguist-generated
+go.mod linguist-generated
+go.sum linguist-generated
+
+# Swift bindings
+bindings/swift/** linguist-generated
+Package.swift linguist-generated
+Package.resolved linguist-generated
+
+# Zig bindings
+build.zig linguist-generated
+build.zig.zon linguist-generated

tree-sitter-fen/.gitattributes

@@ -0,0 +1,47 @@
+# Rust artifacts
+target/
+
+# Node artifacts
+build/
+prebuilds/
+node_modules/
+
+# Swift artifacts
+.build/
+
+# Go artifacts
+_obj/
+
+# Python artifacts
+.venv/
+dist/
+*.egg-info
+*.whl
+
+# C artifacts
+*.a
+*.so
+*.so.*
+*.dylib
+*.dll
+*.pc
+*.exp
+*.lib
+
+# Zig artifacts
+.zig-cache/
+zig-cache/
+zig-out/
+
+# Example dirs
+/examples/*/
+
+# Grammar volatiles
+*.wasm
+*.obj
+*.o
+
+# Archives
+*.tar.gz
+*.tgz
+*.zip