Update grammar and tests

corpus/tests.txt

@@ -4,15 +4,13 @@ Comments
 
 # x = 1;
 # unassigned_variable
-x # xyz
+#xyz
 
 ---
 
-(source_file
+(source
 	(comment)
 	(comment)
-	(expression
-		(identifier))
 	(comment))
 
 ==================
@@ -28,19 +26,13 @@ x.x
 
 ---
 
-(source_file
+(source
-	(expression
+	(identifier)
-		(identifier))
+	(identifier)
-	(expression
+	(identifier)
-		(identifier))
+	(identifier)
-	(expression
+	(identifier)
-		(identifier))
+	(identifier))
-	(expression
-		(identifier))
-	(expression
-		(identifier))
-	(expression
-		(identifier)))
 
 ==================
 Operators
@@ -50,56 +42,58 @@ x = y + y;
 
 ---
 
-(source_file
+(source
-	(expression
+    (identifier)
-        (identifier)
+    (operator)
-        (operator)
+    (identifier)
-    (expression
+    (operator)
-        (identifier)
+    (identifier)
-        (operator)
+	(operator))
-    (expression
-        (identifier))))
-	(chain))
 
 ==================
 String
 ==================
 
 "string"
+'string'
 
 ---
 
-(source_file
+(source
-	(expression
+	(value
-		(value
+		(string))
-			(string))))
+	(value
+		(string)))
 
 ==================
 Integer
 ==================
 
 1
+123
 
 ---
 
-(source_file
+(source
-	(expression
+	(value
-		(value
+		(integer))
-			(integer))))
+	(value
+		(integer)))
 
 ==================
 Float
 ==================
 
 1.0
+123.123
 
 ---
 
-(source_file
+(source
-	(expression
+	(value
-		(value
+		(float))
-			(float))))
+	(value
-
+		(float)))
 
 ==================
 List
@@ -109,16 +103,13 @@ List
 
 ---
 
-(source_file
+(source
-	(expression
+	(value
-    	(value
+		(list
-      		(list
+			(value
-        		(expression
+    			(integer))
-          			(value
+			(value
-            			(integer)))
+       			(integer)))))
-        		(expression
-          			(value
-            	(integer)))))))
 
 ==================
 Empty
@@ -128,10 +119,9 @@ Empty
 
 ---
 
-(source_file
+(source
-	(expression
+	(value
-		(value
+		(empty)))
-			(empty))))
 
 ==================
 Tool
@@ -141,13 +131,11 @@ random_boolean();
 
 ---
 
-(source_file
+(source
-	(expression
+	(tool)
-		(tool))
+	(value
-	(expression
+		(empty))
-		(value
+	(operator))
-			(empty)))
-		(chain))
 
 ==================
 Boolean
@@ -157,10 +145,8 @@ true false
 
 ---
 
-(source_file
+(source
-	(expression
+	(value
-		(value
+		(boolean))
-			(boolean)))
+	(value
-	(expression
+		(boolean)))
-		(value
-			(boolean))))

grammar.js

@@ -2,19 +2,17 @@ module.exports = grammar({
   name: 'dust',
 
   rules: {
-    source_file: $ => repeat(choice(
+    source: $ => repeat(choice(
-      $.comment, 
+      $.comment,
-      $.expression,
+      $.identifier,
-      $.yield,
+      $.value,
-      $.chain
+      $.tool,
+      $.operator,
     )),
 
-    expression: $ => choice(
+    comment: $ => seq('#', /.*/),
-      prec(2, $.value),
+
-      prec(1, $.identifier),
+    identifier: $ => /[a-zA-Z|_|.]+(_[a-zA-Z]+)*/,
-      $.tool,
-      seq($.identifier, $.operator, $.expression),
-    ),
 
     value: $ => choice(
       $.integer,
@@ -23,15 +21,16 @@ module.exports = grammar({
       $.list,
       $.empty,
       $.boolean,
+      $.function,
+    ), 
+
+    tool: $ => choice(
+      "random",
+      "random_boolean",
+      "random_integer",
+      "random_string",
+      "random_float",
     ),
-  
-    comment: $ => /(#)(.+?)([\n\r])/,
-
-    yield: $ => "->",
-
-    chain: $ => ";",
-    
-    identifier: $ => /[a-zA-Z|_|.]+(_[a-zA-Z]+)*/,
 
     operator: $ => choice(
       '=',
@@ -39,24 +38,18 @@ module.exports = grammar({
       '+',
       '/',
       '|',
-      '&'
+      '&',
-    ),
+      ';',
-
+      "->",
-    tool: $ => choice(
-      "random",
-      "random_boolean",
-      "random_integer",
-      "random_string",
-      "random_float"      
     ),
 
     float: $ => /\d+\.\d*/,
     
     integer: $ => /\d+/,
 
-    string: $ => /"(.*?)\"/,
+    string: $ => /("|')(.*?)("|')/,
 
-    function: $ => /'(.*?)\'/,
+    function: $ => /{(.*?)}/,
 
     empty: $ => "()",
 
@@ -67,7 +60,7 @@ module.exports = grammar({
 
     list: $ => seq(
       '(',
-      repeat1(seq($.expression, optional(','))),
+      repeat1(seq($.value, optional(','))),
       ')'
     ),
   }

src/grammar.json

@@ -1,7 +1,7 @@
 {
   "name": "dust",
   "rules": {
-    "source_file": {
+    "source": {
       "type": "REPEAT",
       "content": {
         "type": "CHOICE",
@@ -12,61 +12,40 @@
           },
           {
             "type": "SYMBOL",
-            "name": "expression"
+            "name": "identifier"
           },
           {
             "type": "SYMBOL",
-            "name": "yield"
+            "name": "value"
           },
           {
             "type": "SYMBOL",
-            "name": "chain"
+            "name": "tool"
+          },
+          {
+            "type": "SYMBOL",
+            "name": "operator"
           }
         ]
       }
     },
-    "expression": {
+    "comment": {
-      "type": "CHOICE",
+      "type": "SEQ",
       "members": [
         {
-          "type": "PREC",
+          "type": "STRING",
-          "value": 2,
+          "value": "#"
-          "content": {
-            "type": "SYMBOL",
-            "name": "value"
-          }
         },
         {
-          "type": "PREC",
+          "type": "PATTERN",
-          "value": 1,
+          "value": ".*"
-          "content": {
-            "type": "SYMBOL",
-            "name": "identifier"
-          }
-        },
-        {
-          "type": "SYMBOL",
-          "name": "tool"
-        },
-        {
-          "type": "SEQ",
-          "members": [
-            {
-              "type": "SYMBOL",
-              "name": "identifier"
-            },
-            {
-              "type": "SYMBOL",
-              "name": "operator"
-            },
-            {
-              "type": "SYMBOL",
-              "name": "expression"
-            }
-          ]
         }
       ]
     },
+    "identifier": {
+      "type": "PATTERN",
+      "value": "[a-zA-Z|_|.]+(_[a-zA-Z]+)*"
+    },
     "value": {
       "type": "CHOICE",
       "members": [
@@ -93,51 +72,10 @@
         {
           "type": "SYMBOL",
           "name": "boolean"
-        }
-      ]
-    },
-    "comment": {
-      "type": "PATTERN",
-      "value": "(#)(.+?)([\\n\\r])"
-    },
-    "yield": {
-      "type": "STRING",
-      "value": "->"
-    },
-    "chain": {
-      "type": "STRING",
-      "value": ";"
-    },
-    "identifier": {
-      "type": "PATTERN",
-      "value": "[a-zA-Z|_|.]+(_[a-zA-Z]+)*"
-    },
-    "operator": {
-      "type": "CHOICE",
-      "members": [
-        {
-          "type": "STRING",
-          "value": "="
         },
         {
-          "type": "STRING",
+          "type": "SYMBOL",
-          "value": "-"
+          "name": "function"
-        },
-        {
-          "type": "STRING",
-          "value": "+"
-        },
-        {
-          "type": "STRING",
-          "value": "/"
-        },
-        {
-          "type": "STRING",
-          "value": "|"
-        },
-        {
-          "type": "STRING",
-          "value": "&"
         }
       ]
     },
@@ -166,6 +104,43 @@
         }
       ]
     },
+    "operator": {
+      "type": "CHOICE",
+      "members": [
+        {
+          "type": "STRING",
+          "value": "="
+        },
+        {
+          "type": "STRING",
+          "value": "-"
+        },
+        {
+          "type": "STRING",
+          "value": "+"
+        },
+        {
+          "type": "STRING",
+          "value": "/"
+        },
+        {
+          "type": "STRING",
+          "value": "|"
+        },
+        {
+          "type": "STRING",
+          "value": "&"
+        },
+        {
+          "type": "STRING",
+          "value": ";"
+        },
+        {
+          "type": "STRING",
+          "value": "->"
+        }
+      ]
+    },
     "float": {
       "type": "PATTERN",
       "value": "\\d+\\.\\d*"
@@ -176,11 +151,11 @@
     },
     "string": {
       "type": "PATTERN",
-      "value": "\"(.*?)\\\""
+      "value": "(\"|')(.*?)(\"|')"
     },
     "function": {
       "type": "PATTERN",
-      "value": "'(.*?)\\'"
+      "value": "{(.*?)}"
     },
     "empty": {
       "type": "STRING",
@@ -213,7 +188,7 @@
             "members": [
               {
                 "type": "SYMBOL",
-                "name": "expression"
+                "name": "value"
               },
               {
                 "type": "CHOICE",

src/node-types.json

@@ -5,7 +5,12 @@
     "fields": {}
   },
   {
-    "type": "expression",
+    "type": "comment",
+    "named": true,
+    "fields": {}
+  },
+  {
+    "type": "list",
     "named": true,
     "fields": {},
     "children": {
@@ -13,7 +18,27 @@
       "required": true,
       "types": [
         {
-          "type": "expression",
+          "type": "value",
+          "named": true
+        }
+      ]
+    }
+  },
+  {
+    "type": "operator",
+    "named": true,
+    "fields": {}
+  },
+  {
+    "type": "source",
+    "named": true,
+    "fields": {},
+    "children": {
+      "multiple": true,
+      "required": false,
+      "types": [
+        {
+          "type": "comment",
           "named": true
         },
         {
@@ -35,53 +60,6 @@
       ]
     }
   },
-  {
-    "type": "list",
-    "named": true,
-    "fields": {},
-    "children": {
-      "multiple": true,
-      "required": true,
-      "types": [
-        {
-          "type": "expression",
-          "named": true
-        }
-      ]
-    }
-  },
-  {
-    "type": "operator",
-    "named": true,
-    "fields": {}
-  },
-  {
-    "type": "source_file",
-    "named": true,
-    "fields": {},
-    "children": {
-      "multiple": true,
-      "required": false,
-      "types": [
-        {
-          "type": "chain",
-          "named": true
-        },
-        {
-          "type": "comment",
-          "named": true
-        },
-        {
-          "type": "expression",
-          "named": true
-        },
-        {
-          "type": "yield",
-          "named": true
-        }
-      ]
-    }
-  },
   {
     "type": "tool",
     "named": true,
@@ -107,6 +85,10 @@
           "type": "float",
           "named": true
         },
+        {
+          "type": "function",
+          "named": true
+        },
         {
           "type": "integer",
           "named": true
@@ -122,6 +104,10 @@
       ]
     }
   },
+  {
+    "type": "#",
+    "named": false
+  },
   {
     "type": "&",
     "named": false
@@ -146,22 +132,22 @@
     "type": "-",
     "named": false
   },
+  {
+    "type": "->",
+    "named": false
+  },
   {
     "type": "/",
     "named": false
   },
+  {
+    "type": ";",
+    "named": false
+  },
   {
     "type": "=",
     "named": false
   },
-  {
-    "type": "chain",
-    "named": true
-  },
-  {
-    "type": "comment",
-    "named": true
-  },
   {
     "type": "empty",
     "named": true
@@ -174,6 +160,10 @@
     "type": "float",
     "named": true
   },
+  {
+    "type": "function",
+    "named": true
+  },
   {
     "type": "identifier",
     "named": true
@@ -210,10 +200,6 @@
     "type": "true",
     "named": false
   },
-  {
-    "type": "yield",
-    "named": true
-  },
   {
     "type": "|",
     "named": false