Add comments

For now it only supports non-nested comments. It seems like I *will*
need to write an external lexer to account for nesting properly.

grammar.js

@@ -25,12 +25,32 @@ module.exports = grammar({
     [$._lvalue, $.array_expression],
   ],
 
+  extras: ($) => [
+    /( |\n|\r|\t)+/,
+    $.comment,
+  ],
+
   rules: {
     source_file: ($) => choice(
       $._expr,
       optional($._declaration_chunks),
     ),
 
+    comment: ($) => token(
+      seq(
+        "/*",
+        repeat(
+          choice(
+            // Match anything but the end-delimiter
+            /(\*[^/]|[^*])+/,
+            // Comments can be nested
+            // $.comment,
+          ),
+        ),
+        "*/",
+      ),
+    ),
+
     // Expressions {{{
 
     _expr: ($) => choice(

src/grammar.json

@@ -23,6 +23,34 @@
         }
       ]
     },
+    "comment": {
+      "type": "TOKEN",
+      "content": {
+        "type": "SEQ",
+        "members": [
+          {
+            "type": "STRING",
+            "value": "/*"
+          },
+          {
+            "type": "REPEAT",
+            "content": {
+              "type": "CHOICE",
+              "members": [
+                {
+                  "type": "PATTERN",
+                  "value": "(\\*[^/]|[^*])+"
+                }
+              ]
+            }
+          },
+          {
+            "type": "STRING",
+            "value": "*/"
+          }
+        ]
+      }
+    },
     "_expr": {
       "type": "CHOICE",
       "members": [
@@ -1426,7 +1454,11 @@
   "extras": [
     {
       "type": "PATTERN",
-      "value": "\\s"
+      "value": "( |\\n|\\r|\\t)+"
+    },
+    {
+      "type": "SYMBOL",
+      "name": "comment"
     }
   ],
   "conflicts": [

src/node-types.json

@@ -2283,6 +2283,10 @@
     "type": "break_expression",
     "named": true
   },
+  {
+    "type": "comment",
+    "named": true
+  },
   {
     "type": "do",
     "named": false

test/corpus/comments.txt

@@ -0,0 +1,86 @@
+================================================================================
+Empty comment
+================================================================================
+
+/**/
+
+--------------------------------------------------------------------------------
+
+(source_file
+  (comment))
+
+================================================================================
+Whitespace comment
+================================================================================
+
+/*   */
+
+--------------------------------------------------------------------------------
+
+(source_file
+  (comment))
+
+================================================================================
+Comment
+================================================================================
+
+/* This is a comment */
+
+--------------------------------------------------------------------------------
+
+(source_file
+  (comment))
+
+================================================================================
+Comment with slashes
+================================================================================
+
+/* /// */
+
+--------------------------------------------------------------------------------
+
+(source_file
+  (comment))
+
+================================================================================
+Nested comment
+================================================================================
+
+/* This is /* a /* nested */ comment */*/
+
+--------------------------------------------------------------------------------
+
+(source_file
+  (comment)
+  (identifier)
+  (ERROR
+    (operator)
+    (operator)
+    (operator)
+    (operator)))
+
+================================================================================
+Unterminated comment
+================================================================================
+
+/*
+
+--------------------------------------------------------------------------------
+
+(source_file
+  (ERROR
+    (operator)
+    (operator)))
+
+================================================================================
+Comment end-delimiter only
+================================================================================
+
+*/
+
+--------------------------------------------------------------------------------
+
+(source_file
+  (ERROR
+    (operator)
+    (operator)))