Add tests; Improve parsing

binding.gyp

@@ -1,19 +0,0 @@
-{
-  "targets": [
-    {
-      "target_name": "tree_sitter_dust_binding",
-      "include_dirs": [
-        "<!(node -e \"require('nan')\")",
-        "src"
-      ],
-      "sources": [
-        "bindings/node/binding.cc",
-        "src/parser.c",
-        # If your language uses an external scanner, add it here.
-      ],
-      "cflags_c": [
-        "-std=c99",
-      ]
-    }
-  ]
-}

bindings/node/binding.cc

@@ -1,28 +0,0 @@
-#include "tree_sitter/parser.h"
-#include <node.h>
-#include "nan.h"
-
-using namespace v8;
-
-extern "C" TSLanguage * tree_sitter_dust();
-
-namespace {
-
-NAN_METHOD(New) {}
-
-void Init(Local<Object> exports, Local<Object> module) {
-  Local<FunctionTemplate> tpl = Nan::New<FunctionTemplate>(New);
-  tpl->SetClassName(Nan::New("Language").ToLocalChecked());
-  tpl->InstanceTemplate()->SetInternalFieldCount(1);
-
-  Local<Function> constructor = Nan::GetFunction(tpl).ToLocalChecked();
-  Local<Object> instance = constructor->NewInstance(Nan::GetCurrentContext()).ToLocalChecked();
-  Nan::SetInternalFieldPointer(instance, 0, tree_sitter_dust());
-
-  Nan::Set(instance, Nan::New("name").ToLocalChecked(), Nan::New("dust").ToLocalChecked());
-  Nan::Set(module, Nan::New("exports").ToLocalChecked(), instance);
-}
-
-NODE_MODULE(tree_sitter_dust_binding, Init)
-
-}  // namespace

bindings/node/index.js

@@ -1,19 +0,0 @@
-try {
-  module.exports = require("../../build/Release/tree_sitter_dust_binding");
-} catch (error1) {
-  if (error1.code !== 'MODULE_NOT_FOUND') {
-    throw error1;
-  }
-  try {
-    module.exports = require("../../build/Debug/tree_sitter_dust_binding");
-  } catch (error2) {
-    if (error2.code !== 'MODULE_NOT_FOUND') {
-      throw error2;
-    }
-    throw error1
-  }
-}
-
-try {
-  module.exports.nodeTypeInfo = require("../../src/node-types.json");
-} catch (_) {}

bindings/rust/build.rs

@@ -1,40 +0,0 @@
-fn main() {
-    let src_dir = std::path::Path::new("src");
-
-    let mut c_config = cc::Build::new();
-    c_config.include(&src_dir);
-    c_config
-        .flag_if_supported("-Wno-unused-parameter")
-        .flag_if_supported("-Wno-unused-but-set-variable")
-        .flag_if_supported("-Wno-trigraphs");
-    let parser_path = src_dir.join("parser.c");
-    c_config.file(&parser_path);
-
-    // If your language uses an external scanner written in C,
-    // then include this block of code:
-
-    /*
-    let scanner_path = src_dir.join("scanner.c");
-    c_config.file(&scanner_path);
-    println!("cargo:rerun-if-changed={}", scanner_path.to_str().unwrap());
-    */
-
-    c_config.compile("parser");
-    println!("cargo:rerun-if-changed={}", parser_path.to_str().unwrap());
-
-    // If your language uses an external scanner written in C++,
-    // then include this block of code:
-
-    /*
-    let mut cpp_config = cc::Build::new();
-    cpp_config.cpp(true);
-    cpp_config.include(&src_dir);
-    cpp_config
-        .flag_if_supported("-Wno-unused-parameter")
-        .flag_if_supported("-Wno-unused-but-set-variable");
-    let scanner_path = src_dir.join("scanner.cc");
-    cpp_config.file(&scanner_path);
-    cpp_config.compile("scanner");
-    println!("cargo:rerun-if-changed={}", scanner_path.to_str().unwrap());
-    */
-}

bindings/rust/lib.rs

@@ -1,52 +0,0 @@
-//! This crate provides dust language support for the [tree-sitter][] parsing library.
-//!
-//! Typically, you will use the [language][language func] function to add this language to a
-//! tree-sitter [Parser][], and then use the parser to parse some code:
-//!
-//! ```
-//! let code = "";
-//! let mut parser = tree_sitter::Parser::new();
-//! parser.set_language(tree_sitter_dust::language()).expect("Error loading dust grammar");
-//! let tree = parser.parse(code, None).unwrap();
-//! ```
-//!
-//! [Language]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Language.html
-//! [language func]: fn.language.html
-//! [Parser]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Parser.html
-//! [tree-sitter]: https://tree-sitter.github.io/
-
-use tree_sitter::Language;
-
-extern "C" {
-    fn tree_sitter_dust() -> Language;
-}
-
-/// Get the tree-sitter [Language][] for this grammar.
-///
-/// [Language]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Language.html
-pub fn language() -> Language {
-    unsafe { tree_sitter_dust() }
-}
-
-/// The content of the [`node-types.json`][] file for this grammar.
-///
-/// [`node-types.json`]: https://tree-sitter.github.io/tree-sitter/using-parsers#static-node-types
-pub const NODE_TYPES: &'static str = include_str!("../../src/node-types.json");
-
-// Uncomment these to include any queries that this grammar contains
-
-// pub const HIGHLIGHTS_QUERY: &'static str = include_str!("../../queries/highlights.scm");
-// pub const INJECTIONS_QUERY: &'static str = include_str!("../../queries/injections.scm");
-// pub const LOCALS_QUERY: &'static str = include_str!("../../queries/locals.scm");
-// pub const TAGS_QUERY: &'static str = include_str!("../../queries/tags.scm");
-
-#[cfg(test)]
-mod tests {
-    #[test]
-    fn test_can_load_grammar() {
-        let mut parser = tree_sitter::Parser::new();
-        parser
-            .set_language(super::language())
-            .expect("Error loading dust language");
-    }
-}

corpus/functions.txt

@@ -57,15 +57,15 @@ function <message number> {
             (statement
               (expression            
                 (tool_call
-                  (tool)
+                  (output
                     (expression
-                    (identifier)))))
+                      (identifier))))))
             (statement
               (expression
                 (tool_call
-                  (tool)
+                  (output
                     (expression
-                    (identifier)))))))))))
+                      (identifier))))))))))))
 
 ==================
 Complex Function Call

corpus/maps.txt

@@ -54,4 +54,3 @@ x.answer
       (statement
         (expression
           (identifier)))))
-

corpus/operators.txt

@@ -1,8 +1,8 @@
 ==================
-Simple Equality
+Equality
 ==================
 
-1 == 1
+3 == 1 + 1 + 1
 
 ---
 
@@ -15,23 +15,6 @@ Simple Equality
             (value
               (integer)))
           (logic_operator)
-          (expression
-            (value
-              (integer))))))))
-
-==================
-Complex Equality
-==================
-
-1 + 1 + 1 != 3
-
----
-
-(root
-  (item
-    (statement
-      (expression
-        (logic
           (expression
             (math
               (expression
@@ -44,10 +27,72 @@ Complex Equality
                     (value
                       (integer)))))
               (math_operator)
+              (expression
+                (value
+                  (integer))))))))))
+
+==================
+&&
+==================
+
+4 + 2 == 42 && true 
+
+---
+
+(root
+  (item
+    (statement
+      (expression
+        (logic
+          (expression
+            (math
+              (expression
+                (value
+                  (integer)))
+              (math_operator)
               (expression
                 (value
                   (integer)))))
           (logic_operator)
+          (expression
+            (logic
               (expression
                 (value
-              (integer))))))))
+                  (integer)))
+              (logic_operator)
+              (expression
+                (value
+                  (boolean))))))))))
+
+==================
+\||
+==================
+
+4 + 2 == 42 || true 
+
+---
+
+(root
+  (item
+    (statement
+      (expression
+        (logic
+          (expression
+            (math
+              (expression
+                (value
+                  (integer)))
+              (math_operator)
+              (expression
+                (value
+                  (integer)))))
+          (logic_operator)
+          (expression
+            (logic
+              (expression
+                (value
+                  (integer)))
+              (logic_operator)
+              (expression
+                (value
+                  (boolean))))))))))

grammar.js

@@ -31,14 +31,14 @@ module.exports = grammar({
       repeat(prec.left(seq('->', $.expression)))
     ),
 
-    expression: $ => choice(
+    expression: $ => prec.right(choice(
       $.value,
       $.identifier,
       $.function_call,
       $.tool_call,
       $.math,
       $.logic,
-    ),
+    )),
 
     identifier: $ => /[a-z|_|.]+[0-9]?/,
 
@@ -106,7 +106,7 @@ module.exports = grammar({
       '%',
     ),
 
-    logic: $ => prec.left(seq(
+    logic: $ => prec.right(seq(
       $.expression,
       $.logic_operator,
       $.expression,
@@ -117,8 +117,6 @@ module.exports = grammar({
       '!=',
       '&&',
       '||',
-      'and',
-      'or',
     ),
 
     assignment: $ => prec.right(seq(
@@ -197,10 +195,12 @@ module.exports = grammar({
     )),
 
     _tool: $ => choice(
-      'input',
+      $.input,
       $.output,
     ),
 
+    input: $ => seq('input', $.expression),
     output: $ => seq('output', $.expression),
+    read: $ => seq('read', $.expression),
   }
 });

src/grammar.json

@@ -110,6 +110,9 @@
       ]
     },
     "expression": {
+      "type": "PREC_RIGHT",
+      "value": 0,
+      "content": {
         "type": "CHOICE",
         "members": [
           {
@@ -137,6 +140,7 @@
             "name": "logic"
           }
         ]
+      }
     },
     "identifier": {
       "type": "PATTERN",
@@ -451,7 +455,7 @@
       ]
     },
     "logic": {
-      "type": "PREC_LEFT",
+      "type": "PREC_RIGHT",
       "value": 0,
       "content": {
         "type": "SEQ",
@@ -489,14 +493,6 @@
         {
           "type": "STRING",
           "value": "||"
-        },
-        {
-          "type": "STRING",
-          "value": "and"
-        },
-        {
-          "type": "STRING",
-          "value": "or"
         }
       ]
     },
@@ -853,6 +849,19 @@
     },
     "_tool": {
       "type": "CHOICE",
+      "members": [
+        {
+          "type": "SYMBOL",
+          "name": "input"
+        },
+        {
+          "type": "SYMBOL",
+          "name": "output"
+        }
+      ]
+    },
+    "input": {
+      "type": "SEQ",
       "members": [
         {
           "type": "STRING",
@@ -860,7 +869,7 @@
         },
         {
           "type": "SYMBOL",
-          "name": "output"
+          "name": "expression"
         }
       ]
     },
@@ -876,6 +885,19 @@
           "name": "expression"
         }
       ]
+    },
+    "read": {
+      "type": "SEQ",
+      "members": [
+        {
+          "type": "STRING",
+          "value": "read"
+        },
+        {
+          "type": "SYMBOL",
+          "name": "expression"
+        }
+      ]
     }
   },
   "extras": [

src/node-types.json

@@ -120,6 +120,21 @@
       ]
     }
   },
+  {
+    "type": "input",
+    "named": true,
+    "fields": {},
+    "children": {
+      "multiple": false,
+      "required": true,
+      "types": [
+        {
+          "type": "expression",
+          "named": true
+        }
+      ]
+    }
+  },
   {
     "type": "insert",
     "named": true,
@@ -399,8 +414,12 @@
     "fields": {},
     "children": {
       "multiple": false,
-      "required": false,
+      "required": true,
       "types": [
+        {
+          "type": "input",
+          "named": true
+        },
         {
           "type": "output",
           "named": true
@@ -550,10 +569,6 @@
     "type": "]",
     "named": false
   },
-  {
-    "type": "and",
-    "named": false
-  },
   {
     "type": "break",
     "named": false
@@ -615,11 +630,11 @@
     "named": false
   },
   {
-    "type": "or",
+    "type": "output",
     "named": false
   },
   {
-    "type": "output",
+    "type": "read",
     "named": false
   },
   {