Implement filter loop

This reverts commit 6ed376335b.

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "bindings/dust"]
+	path = bindings/dust
+	url = ssh://git@git.jeffa.io:22022/jeff/dust.git

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "tree-sitter-dust"
-description = "Dust grammar for the tree-sitter parsing library"
+description = "dust grammar for the tree-sitter parsing library"
 version = "0.0.1"
 keywords = ["incremental", "parsing", "dust"]
 categories = ["parsing", "text-editors"]
@@ -8,7 +8,7 @@ repository = "https://github.com/tree-sitter/tree-sitter-dust"
 edition = "2018"
 license = "MIT"
 
-build = "runtime/build.rs"
+build = "bindings/rust/build.rs"
 include = [
   "bindings/rust/*",
   "grammar.js",
@@ -16,17 +16,11 @@ include = [
   "src/*",
 ]
 
-[[bin]]
-name = "main"
-path = "runtime/main.rs"
-
 [lib]
-path = "runtime/lib.rs"
+path = "bindings/rust/lib.rs"
 
 [dependencies]
-clap = "4.4.4"
-dust-lang = "0.1.2"
-rustyline = "12.0.0"
+rayon = "1.8.0"
 tree-sitter = "~0.20.10"
 
 [build-dependencies]

binding.gyp

@@ -1,19 +0,0 @@
-{
-  "targets": [
-    {
-      "target_name": "tree_sitter_Dust_binding",
-      "include_dirs": [
-        "<!(node -e \"require('nan')\")",
-        "src"
-      ],
-      "sources": [
-        "bindings/node/binding.cc",
-        "src/parser.c",
-        # If your language uses an external scanner, add it here.
-      ],
-      "cflags_c": [
-        "-std=c99",
-      ]
-    }
-  ]
-}

corpus/async.txt

@@ -0,0 +1,169 @@
+==================
+Simple Async Statements
+==================
+
+async { (output 'Whaddup') }
+
+---
+
+(root
+  (item
+    (statement
+      (async
+        (statement
+          (expression
+            (function_call
+              (tool)
+              (expression
+                (value
+                  (string))))))))))
+
+==================
+Complex Async Statements
+==================
+
+async {
+  if 1 % 2 == 0 {
+    (output 'true')
+  } else {
+    (output 'false')
+  }
+
+  (output 'foobar')
+}
+
+---
+
+(root
+  (item
+    (statement
+      (async
+        (statement
+          (if_else
+            (if
+              (expression
+                (logic
+                  (expression
+                    (math
+                      (expression
+                        (value
+                          (integer)))
+                      (math_operator)
+                      (expression
+                        (value
+                          (integer)))))
+                  (logic_operator)
+                  (expression
+                    (value
+                      (integer)))))
+              (statement
+                (expression
+                  (function_call
+                    (tool)
+                    (expression
+                      (value
+                        (string)))))))
+            (else
+              (statement
+                (expression
+                  (function_call
+                    (tool)
+                    (expression
+                      (value
+                        (string)))))))))
+        (statement
+          (expression
+            (function_call
+              (tool)
+              (expression
+                (value
+                  (string))))))))))
+                    
+==================
+Simple Async Await Statements
+==================
+
+x = async {
+  1
+}
+
+
+---
+
+(root
+  (item
+    (statement
+      (assignment
+        (identifier)
+        (assignment_operator)
+        (statement
+          (async
+            (statement
+              (expression
+                (value
+                  (integer))))))))))
+                    
+==================
+Complex Async Await Statements
+==================
+
+x = async {
+  i = 0
+  while i < 5 {
+    (output i)
+    i += 1
+  }
+  (read "examples/assets/faithful.csv")
+}
+
+
+---
+
+(root
+  (item
+    (statement
+      (assignment
+        (identifier)
+        (assignment_operator)
+        (statement
+          (async
+            (statement
+              (assignment
+                (identifier)
+                (assignment_operator)
+                (statement
+                  (expression
+                    (value
+                      (integer))))))
+            (statement
+              (while
+                (expression
+                  (logic
+                    (expression
+                      (identifier))
+                    (logic_operator)
+                    (expression
+                      (value
+                        (integer)))))
+                (item
+                  (statement
+                    (expression
+                      (function_call
+                        (tool)
+                        (expression
+                          (identifier)))))
+                  (statement
+                    (assignment
+                      (identifier)
+                      (assignment_operator)
+                      (statement
+                        (expression
+                          (value
+                            (integer)))))))))
+            (statement
+              (expression
+                (function_call
+                  (tool)
+                  (expression
+                    (value
+                      (string))))))))))))

corpus/comments.txt

@@ -0,0 +1,34 @@
+==================
+Full Line Comments
+==================
+
+not_a_comment
+# comment
+
+---
+
+(root
+  (item
+    (statement
+      (expression
+        (identifier))))
+  (item
+    (statement
+      (comment))))
+
+==================
+Partial Line Comments
+==================
+
+not_a_comment # comment
+
+---
+
+(root
+  (item
+    (statement
+      (expression
+        (identifier))))
+  (item
+    (statement
+      (comment))))

corpus/control_flow.txt

@@ -0,0 +1,178 @@
+==================
+If
+==================
+
+if true { "True" }
+
+---
+
+(root
+  (item
+    (statement
+      (if_else
+        (if
+          (expression
+            (value
+              (boolean)))
+          (statement
+            (expression
+              (value
+                (string)))))))))
+
+==================
+If Assignment
+==================
+
+x = if true { 1 }
+
+---
+
+(root
+  (item
+    (statement
+      (assignment
+        (identifier)
+        (assignment_operator)
+        (statement
+          (if_else
+            (if
+              (expression
+                (value
+                  (boolean)))
+              (statement
+                (expression
+                  (value
+                    (integer)))))))))))
+
+==================
+If Else
+==================
+
+if false { "True" } else { "False" }
+
+---
+
+(root
+  (item
+    (statement
+      (if_else
+        (if
+          (expression
+            (value
+              (boolean)))
+          (statement
+            (expression
+              (value
+                (string)))))
+        (else
+          (statement
+            (expression
+              (value
+                (string)))))))))
+
+==================
+If Else If
+==================
+
+if 1 == 1 {
+  "math is fun"
+} else if 4 == 9 {
+  "math is broken"
+}
+
+---
+
+(root
+  (item
+    (statement
+      (if_else
+        (if
+          (expression
+            (logic
+              (expression
+                (value
+                  (integer)))
+              (logic_operator)
+              (expression
+                (value
+                  (integer)))))
+          (statement
+            (expression
+              (value
+                (string)))))
+        (else_if
+          (expression
+            (logic
+              (expression
+                (value
+                  (integer)))
+              (logic_operator)
+              (expression
+                (value
+                  (integer)))))
+          (statement
+            (expression
+              (value
+                (string)))))))))
+
+==================
+If Else Else If Else
+==================
+
+if false {
+  "no"
+} else if false {
+  "no"
+} else if 1 + 1 == 9 {
+  "not the answer"
+} else {
+  "42"
+}
+
+---
+
+(root
+  (item
+    (statement
+      (if_else
+        (if
+          (expression
+            (value
+              (boolean)))
+          (statement
+            (expression
+              (value
+                (string)))))
+        (else_if
+          (expression
+            (value
+              (boolean)))
+          (statement
+            (expression
+              (value
+                (string)))))
+        (else_if
+          (expression
+            (logic
+              (expression
+                (math
+                  (expression
+                    (value
+                      (integer)))
+                  (math_operator)
+                  (expression
+                    (value
+                      (integer)))))
+              (logic_operator)
+              (expression
+                (value
+                  (integer)))))
+          (statement
+            (expression
+              (value
+                (string)))))
+        (else
+          (statement
+            (expression
+              (value
+                (string)))))))))

corpus/filter.txt

@@ -0,0 +1,79 @@
+==================
+Filter Loop
+==================
+
+filter i in [1, 2, 3] {
+  i <= 1 
+}
+
+---
+
+(root
+  (item
+    (statement
+      (filter
+        (identifier)
+        (expression
+          (value
+            (list
+              (expression
+                (value
+                  (integer)))
+              (expression
+                (value
+                  (integer)))
+              (expression
+                (value
+                  (integer))))))
+        (item
+          (statement
+            (expression
+              (logic
+                (expression
+                  (identifier))
+                (logic_operator)
+                (expression
+                  (value
+                    (integer)))))))))))
+
+==================
+Filter Loop Assignment
+==================
+
+list = filter i in ["one", "two", "three"] {
+  i == "one"
+}
+
+---
+
+(root
+  (item
+    (statement
+      (assignment
+        (identifier)
+        (assignment_operator)
+        (statement
+          (filter
+            (identifier)
+            (expression
+              (value
+                (list
+                  (expression
+                    (value
+                      (string)))
+                  (expression
+                    (value
+                      (string)))
+                  (expression
+                    (value
+                      (string))))))
+            (item
+              (statement
+                (expression
+                  (logic
+                    (expression
+                      (identifier))
+                    (logic_operator)
+                    (expression
+                      (value
+                        (string)))))))))))))

corpus/for.txt

@@ -0,0 +1,92 @@
+==================
+Simple For Loop
+==================
+
+for i in [1, 2, 3] {
+  (output i)
+}
+
+---
+
+(root
+  (item
+    (statement
+      (for
+        (identifier)
+        (expression
+          (value
+            (list
+              (expression
+                (value
+                  (integer)))
+              (expression
+                (value
+                  (integer)))
+              (expression
+                (value
+                  (integer))))))
+        (item
+          (statement
+            (expression
+              (function_call
+                (tool)
+                (expression
+                  (identifier))))))))))
+
+==================
+Complex For Loop
+==================
+
+for list in list_of_lists {
+  for item in list {
+    (output item)
+  }
+
+  if (length list) > 1 {
+    (output "List is long...")
+  }
+}
+
+---
+
+(root
+  (item
+    (statement
+      (for
+        (identifier)
+        (expression
+          (identifier))
+        (item
+          (statement
+            (for
+              (identifier)
+              (expression
+                (identifier))
+              (item
+                (statement
+                  (expression
+                    (function_call
+                      (tool)
+                      (expression
+                        (identifier))))))))
+          (statement
+            (if_else
+              (if
+                (expression
+                  (logic
+                    (expression
+                      (function_call
+                        (tool)
+                        (expression
+                          (identifier))))
+                    (logic_operator)
+                    (expression
+                      (value
+                        (integer)))))
+                (statement
+                  (expression
+                    (function_call
+                      (tool)
+                      (expression
+                        (value
+                          (string))))))))))))))

corpus/functions.txt

@@ -0,0 +1,109 @@
+==================
+Simple Function
+==================
+
+function { "Hiya" }
+
+---
+
+(root
+  (item
+    (statement
+      (expression
+        (value
+          (function
+            (item
+              (statement
+                (expression
+                  (value
+                    (string)))))))))))
+
+==================
+Function Call
+==================
+
+(foobar "Hiya")
+
+---
+
+(root
+  (item
+    (statement
+      (expression
+        (function_call
+          (identifier)
+          (expression
+            (value
+              (string))))))))
+
+==================
+Complex Function
+==================
+
+function <message number> {
+  (output message)
+  (output number)
+}
+
+---
+
+(root
+  (item
+    (statement
+      (expression
+        (value
+          (function
+            (identifier)
+            (identifier)
+            (item
+              (statement
+                (expression
+                  (function_call
+                    (tool)
+                    (expression
+                      (identifier)))))
+              (statement
+                (expression
+                  (function_call
+                    (tool)
+                    (expression
+                      (identifier))))))))))))
+
+==================
+Complex Function Call
+==================
+
+(foobar
+    "hi"
+    42
+    {
+      x = 1
+      y = 2
+    }
+)
+
+---
+
+(root
+  (item
+    (statement
+      (expression
+        (function_call
+          (identifier)
+          (expression
+            (value
+              (string)))
+          (expression
+            (value
+              (integer)))
+          (expression
+            (value
+              (map
+                (identifier)
+                (expression
+                  (value
+                    (integer)))
+                (identifier)
+                (expression
+                  (value
+                    (integer)))))))))))

corpus/lists.txt

@@ -0,0 +1,72 @@
+==================
+List Declaration
+==================
+
+['answer', 42]
+
+---
+
+(root
+  (item
+    (statement
+      (expression
+        (value
+          (list
+            (expression
+              (value
+                (string)))
+            (expression
+              (value
+                (integer)))))))))
+
+==================
+List Assignment
+==================
+
+foobar = ['foobar']
+
+---
+
+(root
+  (item
+    (statement
+      (assignment
+        (identifier)
+        (assignment_operator)
+        (statement
+          (expression
+            (value
+              (list
+                (expression
+                  (value
+                    (string)))))))))))
+
+==================
+List Nesting
+==================
+
+['answers', [42, [666]]]
+
+---
+
+(root
+  (item
+    (statement
+      (expression
+        (value
+          (list
+            (expression
+              (value
+                (string)))
+            (expression
+              (value
+                (list
+                  (expression
+                    (value
+                      (integer)))
+                  (expression
+                    (value
+                      (list
+                        (expression
+                          (value
+                            (integer)))))))))))))))

corpus/maps.txt

@@ -0,0 +1,59 @@
+==================
+Simple Map
+==================
+
+{
+    answer = 42
+}
+
+---
+
+(root
+  (item
+    (statement
+      (expression
+        (value
+          (map
+            (identifier)
+            (expression
+              (value
+                (integer)))))))))
+
+==================
+Map Assignment
+==================
+
+x = {
+    answer = 42
+}
+
+---
+
+(root
+  (item
+    (statement
+      (assignment
+        (identifier)
+        (assignment_operator)
+        (statement
+          (expression
+            (value
+              (map
+                (identifier)
+                (expression
+                  (value
+                    (integer)))))))))))
+
+==================
+Map Access
+==================
+
+x.answer
+
+---
+
+(root
+    (item
+      (statement
+        (expression
+          (identifier)))))

corpus/operators.txt

@@ -0,0 +1,98 @@
+==================
+Equality
+==================
+
+3 == 1 + 1 + 1
+
+---
+
+(root
+  (item
+    (statement
+      (expression
+        (logic
+          (expression
+            (value
+              (integer)))
+          (logic_operator)
+          (expression
+            (math
+              (expression
+                (math
+                  (expression
+                    (value
+                      (integer)))
+                  (math_operator)
+                  (expression
+                    (value
+                      (integer)))))
+              (math_operator)
+              (expression
+                (value
+                  (integer))))))))))
+
+==================
+&&
+==================
+
+4 + 2 == 42 && true 
+
+---
+
+(root
+  (item
+    (statement
+      (expression
+        (logic
+          (expression
+            (math
+              (expression
+                (value
+                  (integer)))
+              (math_operator)
+              (expression
+                (value
+                  (integer)))))
+          (logic_operator)
+          (expression
+            (logic
+              (expression
+                (value
+                  (integer)))
+              (logic_operator)
+              (expression
+                (value
+                  (boolean))))))))))
+
+==================
+\||
+==================
+
+4 + 2 == 42 || true 
+
+---
+
+(root
+  (item
+    (statement
+      (expression
+        (logic
+          (expression
+            (math
+              (expression
+                (value
+                  (integer)))
+              (math_operator)
+              (expression
+                (value
+                  (integer)))))
+          (logic_operator)
+          (expression
+            (logic
+              (expression
+                (value
+                  (integer)))
+              (logic_operator)
+              (expression
+                (value
+                  (boolean))))))))))

corpus/simple_values.txt

@@ -0,0 +1,91 @@
+==================
+Booleans
+==================
+
+true
+false
+
+---
+
+(root
+  (item
+    (statement
+      (expression
+        (value
+          (boolean)))))
+  (item
+    (statement
+      (expression
+        (value
+          (boolean))))))
+
+==================
+Integers
+==================
+
+1 2 3
+456 7
+
+---
+
+(root
+  (item
+    (statement
+      (expression
+        (value
+          (integer)))))
+  (item
+    (statement
+      (expression
+        (value
+          (integer)))))
+  (item
+    (statement
+      (expression
+        (value
+          (integer)))))
+  (item
+    (statement
+      (expression
+        (value
+          (integer)))))
+  (item
+    (statement
+      (expression
+        (value
+          (integer))))))
+
+==================
+Strings
+==================
+
+"one" 'two' "three" `four` 'five'
+
+---
+
+(root
+  (item
+    (statement
+      (expression
+        (value
+          (string)))))
+  (item
+    (statement
+      (expression
+        (value
+          (string)))))
+  (item
+    (statement
+      (expression
+        (value
+          (string)))))
+  (item
+    (statement
+      (expression
+        (value
+          (string)))))
+  (item
+    (statement
+      (expression
+        (value
+          (string))))))

corpus/statements.txt

@@ -0,0 +1,116 @@
+==================
+Simple Statements
+==================
+
+1
+"one"
+x
+
+---
+
+(root
+  (item
+    (statement
+      (expression
+        (value
+          (integer)))))
+  (item
+    (statement
+      (expression
+        (value
+          (string)))))
+  (item
+    (statement
+      (expression
+        (identifier)))))
+
+==================
+Simple Assignment
+==================
+
+x = 1
+y = "one"
+
+---
+
+(root
+  (item
+    (statement
+      (assignment
+        (identifier)
+        (assignment_operator)
+        (statement
+          (expression
+            (value
+              (integer)))))))
+  (item
+    (statement
+      (assignment
+        (identifier)
+        (assignment_operator)
+        (statement
+          (expression
+            (value
+              (string))))))))
+
+==================
+Complex Assignment
+==================
+
+x = 1 + 1
+
+---
+
+(root
+  (item
+    (statement
+      (assignment
+        (identifier)
+        (assignment_operator)
+        (statement
+          (expression
+            (math
+              (expression
+                (value
+                  (integer)))
+              (math_operator)
+              (expression
+                (value
+                  (integer))))))))))
+
+==================
+Expression Precedence
+==================
+
+x = (3 == (1 + (2 + 2)))
+
+---
+
+(root
+  (item
+    (statement
+      (assignment
+        (identifier)
+        (assignment_operator)
+        (statement
+          (expression
+            (logic
+              (expression
+                (value
+                  (integer)))
+              (logic_operator)
+              (expression
+                (math
+                  (expression
+                    (value
+                      (integer)))
+                  (math_operator)
+                  (expression
+                    (math
+                      (expression
+                        (value
+                          (integer)))
+                      (math_operator)
+                      (expression
+                        (value
+                          (integer))))))))))))))

corpus/tables.txt

@@ -0,0 +1,104 @@
+==================
+Table Declaration
+==================
+
+table <text, number> {
+    ['answer', 42]
+}
+
+---
+
+(root
+  (item
+    (statement
+      (expression
+        (value
+          (table
+            (identifier)
+            (identifier)
+            (expression
+              (value
+                (list
+                  (expression
+                    (value
+                      (string)))
+                  (expression
+                    (value
+                      (integer))))))))))))
+
+==================
+Table Assignment
+==================
+
+foobar = table <text, number> {
+    ['answer', 42]
+}
+
+---
+
+(root
+  (item
+    (statement
+      (assignment
+        (identifier)
+        (assignment_operator)
+        (statement
+          (expression
+            (value
+              (table
+                (identifier)
+                (identifier)
+                (expression
+                  (value
+                    (list
+                      (expression
+                        (value
+                          (string)))
+                      (expression
+                        (value
+                          (integer))))))))))))))
+
+==================
+Table Access
+==================
+
+select number from foobar where text == 'answer'
+
+---
+
+(root
+  (item
+    (statement
+      (select
+        (identifier)
+        (identifier)
+        (expression
+          (logic
+            (expression
+              (identifier))
+            (logic_operator)
+            (expression
+              (value
+                (string)))))))))
+
+
+==================
+Table Insert
+==================
+
+insert ['bob was here', 0] into foobar
+
+---
+
+(root
+  (item
+    (statement
+      (insert
+        (list
+          (expression
+            (value
+              (string)))
+          (expression
+            (value
+              (integer))))
+        (identifier)))))

corpus/tests.txt

@@ -1,182 +0,0 @@
-==================
-Comments
-==================
-
-# x = 1;
-# unassigned_variable
-x # xyz
-
----
-
-(source_file
-	(comment)
-	(comment)
-	(expression
-		(identifier))
-	(comment))
-
-==================
-Identifiers
-==================
-
-variable_name
-_unused_variable
-__strange_format__
-a
-blahblah
-x.x
-
----
-
-(source_file
-	(expression
-		(identifier))
-	(expression
-		(identifier))
-	(expression
-		(identifier))
-	(expression
-		(identifier))
-	(expression
-		(identifier))
-	(expression
-		(identifier)))
-
-==================
-Operators
-==================
-
-x = y + y;
-
----
-
-(source_file
-	(expression
-        (identifier)
-        (operator)
-    (expression
-        (identifier)
-        (operator)
-    (expression
-        (identifier))))
-	(chain))
-
-==================
-String
-==================
-
-"string"
-
----
-
-(source_file
-	(expression
-		(value
-			(string))))
-
-==================
-Integer
-==================
-
-1
-
----
-
-(source_file
-	(expression
-		(value
-			(integer))))
-
-==================
-Float
-==================
-
-1.0
-
----
-
-(source_file
-	(expression
-		(value
-			(float))))
-
-
-==================
-List
-==================
-
-(1, 2)
-
----
-
-(source_file
-	(expression
-    	(value
-      		(list
-        		(expression
-          			(value
-            			(integer)))
-        		(expression
-          			(value
-            	(integer)))))))
-
-==================
-Empty
-==================
-
-()
-
----
-
-(source_file
-	(expression
-		(value
-			(empty))))
-
-==================
-Tool
-==================
-
-random_boolean();
-
----
-
-(source_file
-	(expression
-		(tool))
-	(expression
-		(value
-			(empty)))
-		(chain))
-
-==================
-Boolean
-==================
-
-true false
-
----
-
-(source_file
-	(expression
-		(value
-			(boolean)))
-	(expression
-		(value
-			(boolean))))
-
-==================
-Function
-==================
-
-{ output "hi" }
-
----
-
-(source_file
-	(expression
-		(value
-			(boolean)))
-	(expression
-		(value
-			(boolean))))

corpus/transform.txt

@@ -0,0 +1,73 @@
+==================
+Transform Loop
+==================
+
+transform i in [1, 2, 3] {
+  (output i)
+}
+
+---
+
+(root
+  (item
+    (statement
+      (transform
+        (identifier)
+        (expression
+          (value
+            (list
+              (expression
+                (value
+                  (integer)))
+              (expression
+                (value
+                  (integer)))
+              (expression
+                (value
+                  (integer))))))
+        (item
+          (statement
+            (expression
+              (function_call
+                (tool)
+                (expression
+                  (identifier))))))))))
+
+==================
+Transform Loop Assignment
+==================
+
+list = transform i in ["one", "two", "three"] {
+  (output i)
+}
+
+---
+
+(root
+  (item
+    (statement
+      (assignment
+        (identifier)
+        (assignment_operator)
+        (statement
+          (transform
+            (identifier)
+            (expression
+              (value
+                (list
+                  (expression
+                    (value
+                      (string)))
+                  (expression
+                    (value
+                      (string)))
+                  (expression
+                    (value
+                      (string))))))
+            (item
+              (statement
+                (expression
+                  (function_call
+                    (tool)
+                    (expression
+                      (identifier))))))))))))

corpus/while.txt

@@ -0,0 +1,52 @@
+==================
+While Loop
+==================
+
+while true {
+  (output "This is a bad idea...")
+}
+
+---
+
+(root
+  (item
+    (statement
+      (while
+        (expression
+          (value
+            (boolean)))
+        (item
+          (statement
+            (expression
+              (function_call
+                (tool)
+                (expression
+                  (value
+                    (string)))))))))))
+
+==================
+While Loop Assignment
+==================
+
+answer = while false {
+  42
+}
+
+---
+
+(root
+  (item
+    (statement
+      (assignment
+        (identifier)
+        (assignment_operator)
+        (statement
+          (while
+            (expression
+              (value
+                (boolean)))
+            (item
+              (statement
+                (expression
+                  (value
+                    (integer)))))))))))

example.dust

@@ -1,6 +0,0 @@
-# Comment
-(1, 2, 3);
-
-random_integer();
-true;
-false;

grammar.js

@@ -1,78 +1,269 @@
 module.exports = grammar({
   name: 'dust',
 
+  word: $ => $.identifier,
+
   rules: {
-    source_file: $ => repeat(choice(
-      $.comment, 
+    root: $ => repeat1($.item),
+
+    item: $ => prec.left(repeat1($.statement)),
+
+    statement: $ => choice(
+      $.comment,
+      $.assignment,
       $.expression,
-      $.yield,
-      $.chain
-    )),
+      $.if_else,
+      $.insert,
+      $.select,
+      $.while,
+      $.async,
+      $.for,
+      $.transform,
+      $.filter,
+    ),
+  
+    comment: $ => seq(/[#]+.*/),
 
     expression: $ => choice(
-      prec(2, $.value),
-      prec(1, $.identifier),
-      $.tool,
-      seq($.identifier, $.operator, $.expression),
+      $._expression_kind,
+      seq('(', $._expression_kind, ')'),
     ),
 
+    _expression_kind: $ => prec.right(choice(
+      $.value,
+      $.identifier,
+      $.function_call,
+      $.math,
+      $.logic,
+    )),
+
+    identifier: $ => /[a-z|_|.]+[0-9]?/,
+
     value: $ => choice(
       $.integer,
       $.float,
       $.string,
-      $.list,
-      $.empty,
       $.boolean,
-    ),
-  
-    comment: $ => /(#)(.+?)([\n\r])/,
-
-    yield: $ => "->",
-
-    chain: $ => ";",
-    
-    identifier: $ => /[a-zA-Z|_|.]+(_[a-zA-Z]+)*/,
-
-    operator: $ => choice(
-      '=',
-      '-',
-      '+',
-      '/',
-      '|',
-      '&'
+      $.list,
+      $.function,
+      $.table,
+      $.map,
     ),
 
-    tool: $ => choice(
-      "random",
-      "random_boolean",
-      "random_integer",
-      "random_string",
-      "random_float"      
-    ),
+    integer: $ => /[-]?[0-9]+/,
 
-    float: $ => /\d+\.\d*/,
-    
-    integer: $ => /\d+/,
+    float: $ => /[-]?[0-9]+[.]{1}[0-9]*/,
 
-    string: $ => seq(
-      '"',
-      repeat(),
-      '"'
-    ),
-
-    function: $ => /\{(.*?)\}/,
-
-    empty: $ => "()",
+    string: $ => /("[^"]*?")|('[^']*?')|(`[^`]*?`)/,
 
     boolean: $ => choice(
-      "true",
-      "false"
+      'true',
+      'false',
     ),
 
     list: $ => seq(
+      '[',
+      repeat(seq($.expression, optional(','))),
+      ']'
+    ),
+
+    function: $ => seq(
+      'function',
+      optional(seq('<', repeat(seq($.identifier, optional(','))), '>')),
+      '{',
+      $.item,
+      '}',
+    ),
+
+    table: $ => seq(
+      'table',
+      seq('<', repeat1(seq($.identifier, optional(','))), '>'),
+      '{',
+      repeat($.expression),
+      '}',
+    ),
+
+    map: $ => seq(
+      '{',
+      repeat(seq($.identifier, "=", $.expression)),
+      '}',
+    ),
+
+    math: $ => prec.left(seq(
+      $.expression,
+      $.math_operator,      
+      $.expression,
+    )),
+
+    math_operator: $ => choice(
+      '+',
+      '-',
+      '*',
+      '/',
+      '%',
+    ),
+
+    logic: $ => prec.right(seq(
+      $.expression,
+      $.logic_operator,
+      $.expression,
+    )),
+
+    logic_operator: $ => choice(
+      '==',
+      '!=',
+      '&&',
+      '||',
+      '>',
+      '<',
+      ">=",
+      "<=",
+    ),
+
+    assignment: $ => prec.right(seq(
+      $.identifier,
+      $.assignment_operator,
+      $.statement,
+    )),
+
+    assignment_operator: $ => choice(
+      "=",
+      "+=",
+      "-=",
+    ),
+
+    if_else: $ => prec.left(seq(
+      $.if,
+      repeat(seq($.else_if)),
+      optional(seq($.else)),
+    )),
+
+    if: $ => seq(
+      'if',
+      $.expression,
+      '{',
+      $.statement,
+      '}',
+    ),
+
+    else_if: $ => seq(
+      'else if',
+      $.expression,
+      '{',
+      $.statement,
+      '}',
+    ),
+
+    else: $ => seq(
+      'else',
+      '{',
+      $.statement,
+      '}',
+    ),
+
+    function_call: $ => prec.right(seq(
       '(',
-      repeat1(seq($.expression, optional(','))),
-      ')'
+      choice($.identifier, $.tool),
+      repeat(seq($.expression, optional(','))),
+      ')',
+    )),
+
+    while: $ => seq(
+      'while',
+      $.expression,
+      '{',
+      $.item,
+      '}',      
+    ),
+
+    for: $ => seq(
+      'for',
+      $.identifier,
+      'in',
+      $.expression,
+      '{',
+      $.item,
+      '}',
+    ),
+
+    transform: $ => seq(
+      'transform',
+      $.identifier,
+      'in',
+      $.expression,
+      '{',
+      $.item,
+      '}',
+    ),
+
+    filter: $ => seq(
+      'filter',
+      $.identifier,
+      'in',
+      $.expression,
+      '{',
+      $.item,
+      '}',
+    ),
+
+    tool: $ => choice(
+      'assert',
+      'assert_equal',
+      'output',
+
+      'read',
+      'write',
+
+      'raw',
+      'sh',
+      'bash',
+      'fish',
+      'zsh',
+
+      'random',
+      'random_boolean',
+      'random_float',
+      'random_string',
+      'random_integer',
+
+      'length',
+      'sort',
+      'transform',
+      'filter',
+
+      'to_csv',
+      'from_csv',
+      'to_json',
+      'from_json',
+
+      'help',
+    ),
+
+    select: $ => prec.right(seq(
+      'select',
+      $.identifier,
+      'from',
+      $.identifier,
+      optional(
+        seq('where', $.expression)
+      ),
+    )),
+
+    insert: $ => prec.right(seq(
+      'insert',
+      repeat1($.list),
+      'into',
+      $.identifier,
+      optional(
+        seq('where', $.logic)
+      ),
+    )),
+
+    async: $ => seq(
+      'async', 
+      '{', 
+      repeat($.statement), 
+      '}'
     ),
   }
-});
+});

package.json

@@ -2,12 +2,8 @@
   "name": "tree-sitter-dust",
   "version": "1.0.0",
   "description": "Tree Sitter grammar for the Dust programming language",
-  "main": "bindings/node",
-  "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1"
-  },
   "author": "Jeff Anderson",
-  "license": "ISC",
+  "license": "MIT",
   "tree-sitter": [
     {
       "scope": "source.dust",
@@ -23,5 +19,6 @@
   },
   "devDependencies": {
     "tree-sitter-cli": "^0.20.8"
-  }
+  },
+  "main": "bindings/node"
 }

runtime/build.rs

@@ -1,40 +0,0 @@
-fn main() {
-    let src_dir = std::path::Path::new("src");
-
-    let mut c_config = cc::Build::new();
-    c_config.include(&src_dir);
-    c_config
-        .flag_if_supported("-Wno-unused-parameter")
-        .flag_if_supported("-Wno-unused-but-set-variable")
-        .flag_if_supported("-Wno-trigraphs");
-    let parser_path = src_dir.join("parser.c");
-    c_config.file(&parser_path);
-
-    // If your language uses an external scanner written in C,
-    // then include this block of code:
-
-    /*
-    let scanner_path = src_dir.join("scanner.c");
-    c_config.file(&scanner_path);
-    println!("cargo:rerun-if-changed={}", scanner_path.to_str().unwrap());
-    */
-
-    c_config.compile("parser");
-    println!("cargo:rerun-if-changed={}", parser_path.to_str().unwrap());
-
-    // If your language uses an external scanner written in C++,
-    // then include this block of code:
-
-    /*
-    let mut cpp_config = cc::Build::new();
-    cpp_config.cpp(true);
-    cpp_config.include(&src_dir);
-    cpp_config
-        .flag_if_supported("-Wno-unused-parameter")
-        .flag_if_supported("-Wno-unused-but-set-variable");
-    let scanner_path = src_dir.join("scanner.cc");
-    cpp_config.file(&scanner_path);
-    cpp_config.compile("scanner");
-    println!("cargo:rerun-if-changed={}", scanner_path.to_str().unwrap());
-    */
-}

runtime/lib.rs

@@ -1,68 +0,0 @@
-//! This crate provides Dust language support for the [tree-sitter][] parsing library.
-//!
-//! Typically, you will use the [language][language func] function to add this language to a
-//! tree-sitter [Parser][], and then use the parser to parse some code:
-//!
-//! ```
-//! let code = "";
-//! let mut parser = tree_sitter::Parser::new();
-//! parser.set_language(tree_sitter_Dust::language()).expect("Error loading Dust grammar");
-//! let tree = parser.parse(code, None).unwrap();
-//! ```
-//!
-//! [Language]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Language.html
-//! [language func]: fn.language.html
-//! [Parser]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Parser.html
-//! [tree-sitter]: https://tree-sitter.github.io/
-
-use tree_sitter::{Language, Node};
-use dust_lib::Value;
-
-extern "C" {
-    fn tree_sitter_dust() -> Language;
-}
-
-/// Get the tree-sitter [Language][] for this grammar.
-///
-/// [Language]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Language.html
-pub fn language() -> Language {
-    unsafe { tree_sitter_dust() }
-}
-
-/// The content of the [`node-types.json`][] file for this grammar.
-///
-/// [`node-types.json`]: https://tree-sitter.github.io/tree-sitter/using-parsers#static-node-types
-pub const NODE_TYPES: &'static str = include_str!("../src/node-types.json");
-
-pub fn evaluate(root: Node) -> Value {
-    let cursor = root.walk();
-    
-
-    if let Some(name) = cursor.field_name() {
-        match name {
-            "source" => Value::Empty,
-            _ => Value::Empty,
-        }
-    }
-
-    
-}
-
-
-// Uncomment these to include any queries that this grammar contains
-
-// pub const HIGHLIGHTS_QUERY: &'static str = include_str!("../../queries/highlights.scm");
-// pub const INJECTIONS_QUERY: &'static str = include_str!("../../queries/injections.scm");
-// pub const LOCALS_QUERY: &'static str = include_str!("../../queries/locals.scm");
-// pub const TAGS_QUERY: &'static str = include_str!("../../queries/tags.scm");
-
-#[cfg(test)]
-mod tests {
-    #[test]
-    fn test_can_load_grammar() {
-        let mut parser = tree_sitter::Parser::new();
-        parser
-            .set_language(super::language())
-            .expect("Error loading Dust language");
-    }
-}

runtime/main.rs

@@ -1,38 +0,0 @@
-//! Command line interface for the dust programming language.
-use clap::Parser;
-use std::fs::read_to_string;
-use tree_sitter::{Parser as TSParser, Language};
-
-/// Command-line arguments to be parsed.
-#[derive(Parser, Debug)]
-#[command(author, version, about, long_about = None)]
-struct Args {
-    /// Whale source code to evaluate.
-    #[arg(short, long)]
-    command: Option<String>,
-
-    /// Location of the file to run.
-    path: Option<String>,
-}
-
-fn main() {
-    let args = Args::parse();
-
-    if args.path.is_none() && args.command.is_none() {
-        todo!();
-    }
-
-    if let Some(path) = args.path {
-        let file_contents = read_to_string(path).unwrap();
-
-        let mut parser = TSParser::new();
-        parser.set_language(tree_sitter_dust::language()).unwrap();
-        let tree = parser.parse(&file_contents, None).unwrap();
-        let root = tree.root_node();
-
-        tree_sitter_dust::evaluate(root);
-
-        println!("{tree:?}");
-    }
-}
-

src/node-types.json

@@ -1,13 +1,126 @@
 [
+  {
+    "type": "assignment",
+    "named": true,
+    "fields": {},
+    "children": {
+      "multiple": true,
+      "required": true,
+      "types": [
+        {
+          "type": "assignment_operator",
+          "named": true
+        },
+        {
+          "type": "identifier",
+          "named": true
+        },
+        {
+          "type": "statement",
+          "named": true
+        }
+      ]
+    }
+  },
+  {
+    "type": "assignment_operator",
+    "named": true,
+    "fields": {}
+  },
+  {
+    "type": "async",
+    "named": true,
+    "fields": {},
+    "children": {
+      "multiple": true,
+      "required": false,
+      "types": [
+        {
+          "type": "statement",
+          "named": true
+        }
+      ]
+    }
+  },
   {
     "type": "boolean",
     "named": true,
     "fields": {}
   },
+  {
+    "type": "comment",
+    "named": true,
+    "fields": {}
+  },
+  {
+    "type": "else",
+    "named": true,
+    "fields": {},
+    "children": {
+      "multiple": false,
+      "required": true,
+      "types": [
+        {
+          "type": "statement",
+          "named": true
+        }
+      ]
+    }
+  },
+  {
+    "type": "else_if",
+    "named": true,
+    "fields": {},
+    "children": {
+      "multiple": true,
+      "required": true,
+      "types": [
+        {
+          "type": "expression",
+          "named": true
+        },
+        {
+          "type": "statement",
+          "named": true
+        }
+      ]
+    }
+  },
   {
     "type": "expression",
     "named": true,
     "fields": {},
+    "children": {
+      "multiple": false,
+      "required": true,
+      "types": [
+        {
+          "type": "function_call",
+          "named": true
+        },
+        {
+          "type": "identifier",
+          "named": true
+        },
+        {
+          "type": "logic",
+          "named": true
+        },
+        {
+          "type": "math",
+          "named": true
+        },
+        {
+          "type": "value",
+          "named": true
+        }
+      ]
+    }
+  },
+  {
+    "type": "filter",
+    "named": true,
+    "fields": {},
     "children": {
       "multiple": true,
       "required": true,
@@ -21,15 +134,152 @@
           "named": true
         },
         {
-          "type": "operator",
+          "type": "item",
+          "named": true
+        }
+      ]
+    }
+  },
+  {
+    "type": "for",
+    "named": true,
+    "fields": {},
+    "children": {
+      "multiple": true,
+      "required": true,
+      "types": [
+        {
+          "type": "expression",
+          "named": true
+        },
+        {
+          "type": "identifier",
+          "named": true
+        },
+        {
+          "type": "item",
+          "named": true
+        }
+      ]
+    }
+  },
+  {
+    "type": "function",
+    "named": true,
+    "fields": {},
+    "children": {
+      "multiple": true,
+      "required": true,
+      "types": [
+        {
+          "type": "identifier",
+          "named": true
+        },
+        {
+          "type": "item",
+          "named": true
+        }
+      ]
+    }
+  },
+  {
+    "type": "function_call",
+    "named": true,
+    "fields": {},
+    "children": {
+      "multiple": true,
+      "required": true,
+      "types": [
+        {
+          "type": "expression",
+          "named": true
+        },
+        {
+          "type": "identifier",
           "named": true
         },
         {
           "type": "tool",
           "named": true
+        }
+      ]
+    }
+  },
+  {
+    "type": "if",
+    "named": true,
+    "fields": {},
+    "children": {
+      "multiple": true,
+      "required": true,
+      "types": [
+        {
+          "type": "expression",
+          "named": true
         },
         {
-          "type": "value",
+          "type": "statement",
+          "named": true
+        }
+      ]
+    }
+  },
+  {
+    "type": "if_else",
+    "named": true,
+    "fields": {},
+    "children": {
+      "multiple": true,
+      "required": true,
+      "types": [
+        {
+          "type": "else",
+          "named": true
+        },
+        {
+          "type": "else_if",
+          "named": true
+        },
+        {
+          "type": "if",
+          "named": true
+        }
+      ]
+    }
+  },
+  {
+    "type": "insert",
+    "named": true,
+    "fields": {},
+    "children": {
+      "multiple": true,
+      "required": true,
+      "types": [
+        {
+          "type": "identifier",
+          "named": true
+        },
+        {
+          "type": "list",
+          "named": true
+        },
+        {
+          "type": "logic",
+          "named": true
+        }
+      ]
+    }
+  },
+  {
+    "type": "item",
+    "named": true,
+    "fields": {},
+    "children": {
+      "multiple": true,
+      "required": true,
+      "types": [
+        {
+          "type": "statement",
           "named": true
         }
       ]
@@ -41,7 +291,7 @@
     "fields": {},
     "children": {
       "multiple": true,
-      "required": true,
+      "required": false,
       "types": [
         {
           "type": "expression",
@@ -51,12 +301,31 @@
     }
   },
   {
-    "type": "operator",
+    "type": "logic",
+    "named": true,
+    "fields": {},
+    "children": {
+      "multiple": true,
+      "required": true,
+      "types": [
+        {
+          "type": "expression",
+          "named": true
+        },
+        {
+          "type": "logic_operator",
+          "named": true
+        }
+      ]
+    }
+  },
+  {
+    "type": "logic_operator",
     "named": true,
     "fields": {}
   },
   {
-    "type": "source_file",
+    "type": "map",
     "named": true,
     "fields": {},
     "children": {
@@ -64,7 +333,88 @@
       "required": false,
       "types": [
         {
-          "type": "chain",
+          "type": "expression",
+          "named": true
+        },
+        {
+          "type": "identifier",
+          "named": true
+        }
+      ]
+    }
+  },
+  {
+    "type": "math",
+    "named": true,
+    "fields": {},
+    "children": {
+      "multiple": true,
+      "required": true,
+      "types": [
+        {
+          "type": "expression",
+          "named": true
+        },
+        {
+          "type": "math_operator",
+          "named": true
+        }
+      ]
+    }
+  },
+  {
+    "type": "math_operator",
+    "named": true,
+    "fields": {}
+  },
+  {
+    "type": "root",
+    "named": true,
+    "fields": {},
+    "children": {
+      "multiple": true,
+      "required": true,
+      "types": [
+        {
+          "type": "item",
+          "named": true
+        }
+      ]
+    }
+  },
+  {
+    "type": "select",
+    "named": true,
+    "fields": {},
+    "children": {
+      "multiple": true,
+      "required": true,
+      "types": [
+        {
+          "type": "expression",
+          "named": true
+        },
+        {
+          "type": "identifier",
+          "named": true
+        }
+      ]
+    }
+  },
+  {
+    "type": "statement",
+    "named": true,
+    "fields": {},
+    "children": {
+      "multiple": false,
+      "required": true,
+      "types": [
+        {
+          "type": "assignment",
+          "named": true
+        },
+        {
+          "type": "async",
           "named": true
         },
         {
@@ -76,7 +426,50 @@
           "named": true
         },
         {
-          "type": "yield",
+          "type": "filter",
+          "named": true
+        },
+        {
+          "type": "for",
+          "named": true
+        },
+        {
+          "type": "if_else",
+          "named": true
+        },
+        {
+          "type": "insert",
+          "named": true
+        },
+        {
+          "type": "select",
+          "named": true
+        },
+        {
+          "type": "transform",
+          "named": true
+        },
+        {
+          "type": "while",
+          "named": true
+        }
+      ]
+    }
+  },
+  {
+    "type": "table",
+    "named": true,
+    "fields": {},
+    "children": {
+      "multiple": true,
+      "required": true,
+      "types": [
+        {
+          "type": "expression",
+          "named": true
+        },
+        {
+          "type": "identifier",
           "named": true
         }
       ]
@@ -87,6 +480,29 @@
     "named": true,
     "fields": {}
   },
+  {
+    "type": "transform",
+    "named": true,
+    "fields": {},
+    "children": {
+      "multiple": true,
+      "required": true,
+      "types": [
+        {
+          "type": "expression",
+          "named": true
+        },
+        {
+          "type": "identifier",
+          "named": true
+        },
+        {
+          "type": "item",
+          "named": true
+        }
+      ]
+    }
+  },
   {
     "type": "value",
     "named": true,
@@ -100,11 +516,11 @@
           "named": true
         },
         {
-          "type": "empty",
+          "type": "float",
           "named": true
         },
         {
-          "type": "float",
+          "type": "function",
           "named": true
         },
         {
@@ -115,15 +531,50 @@
           "type": "list",
           "named": true
         },
+        {
+          "type": "map",
+          "named": true
+        },
         {
           "type": "string",
           "named": true
+        },
+        {
+          "type": "table",
+          "named": true
         }
       ]
     }
   },
   {
-    "type": "&",
+    "type": "while",
+    "named": true,
+    "fields": {},
+    "children": {
+      "multiple": true,
+      "required": true,
+      "types": [
+        {
+          "type": "expression",
+          "named": true
+        },
+        {
+          "type": "item",
+          "named": true
+        }
+      ]
+    }
+  },
+  {
+    "type": "!=",
+    "named": false
+  },
+  {
+    "type": "%",
+    "named": false
+  },
+  {
+    "type": "&&",
     "named": false
   },
   {
@@ -134,10 +585,18 @@
     "type": ")",
     "named": false
   },
+  {
+    "type": "*",
+    "named": false
+  },
   {
     "type": "+",
     "named": false
   },
+  {
+    "type": "+=",
+    "named": false
+  },
   {
     "type": ",",
     "named": false
@@ -146,42 +605,142 @@
     "type": "-",
     "named": false
   },
+  {
+    "type": "-=",
+    "named": false
+  },
   {
     "type": "/",
     "named": false
   },
+  {
+    "type": "<",
+    "named": false
+  },
+  {
+    "type": "<=",
+    "named": false
+  },
   {
     "type": "=",
     "named": false
   },
   {
-    "type": "chain",
-    "named": true
+    "type": "==",
+    "named": false
   },
   {
-    "type": "comment",
-    "named": true
+    "type": ">",
+    "named": false
   },
   {
-    "type": "empty",
-    "named": true
+    "type": ">=",
+    "named": false
+  },
+  {
+    "type": "[",
+    "named": false
+  },
+  {
+    "type": "]",
+    "named": false
+  },
+  {
+    "type": "assert",
+    "named": false
+  },
+  {
+    "type": "assert_equal",
+    "named": false
+  },
+  {
+    "type": "async",
+    "named": false
+  },
+  {
+    "type": "bash",
+    "named": false
+  },
+  {
+    "type": "else",
+    "named": false
+  },
+  {
+    "type": "else if",
+    "named": false
   },
   {
     "type": "false",
     "named": false
   },
+  {
+    "type": "filter",
+    "named": false
+  },
+  {
+    "type": "fish",
+    "named": false
+  },
   {
     "type": "float",
     "named": true
   },
+  {
+    "type": "for",
+    "named": false
+  },
+  {
+    "type": "from",
+    "named": false
+  },
+  {
+    "type": "from_csv",
+    "named": false
+  },
+  {
+    "type": "from_json",
+    "named": false
+  },
+  {
+    "type": "function",
+    "named": false
+  },
+  {
+    "type": "help",
+    "named": false
+  },
   {
     "type": "identifier",
     "named": true
   },
+  {
+    "type": "if",
+    "named": false
+  },
+  {
+    "type": "in",
+    "named": false
+  },
+  {
+    "type": "insert",
+    "named": false
+  },
   {
     "type": "integer",
     "named": true
   },
+  {
+    "type": "into",
+    "named": false
+  },
+  {
+    "type": "length",
+    "named": false
+  },
+  {
+    "type": "output",
+    "named": false
+  },
   {
     "type": "random",
     "named": false
@@ -202,20 +761,76 @@
     "type": "random_string",
     "named": false
   },
+  {
+    "type": "raw",
+    "named": false
+  },
+  {
+    "type": "read",
+    "named": false
+  },
+  {
+    "type": "select",
+    "named": false
+  },
+  {
+    "type": "sh",
+    "named": false
+  },
+  {
+    "type": "sort",
+    "named": false
+  },
   {
     "type": "string",
     "named": true
   },
+  {
+    "type": "table",
+    "named": false
+  },
+  {
+    "type": "to_csv",
+    "named": false
+  },
+  {
+    "type": "to_json",
+    "named": false
+  },
+  {
+    "type": "transform",
+    "named": false
+  },
   {
     "type": "true",
     "named": false
   },
   {
-    "type": "yield",
-    "named": true
+    "type": "where",
+    "named": false
   },
   {
-    "type": "|",
+    "type": "while",
+    "named": false
+  },
+  {
+    "type": "write",
+    "named": false
+  },
+  {
+    "type": "zsh",
+    "named": false
+  },
+  {
+    "type": "{",
+    "named": false
+  },
+  {
+    "type": "||",
+    "named": false
+  },
+  {
+    "type": "}",
     "named": false
   }
 ]