Begin new grammar

bindings/node/binding.cc

@@ -0,0 +1,28 @@
+#include "tree_sitter/parser.h"
+#include <node.h>
+#include "nan.h"
+
+using namespace v8;
+
+extern "C" TSLanguage * tree_sitter_dust();
+
+namespace {
+
+NAN_METHOD(New) {}
+
+void Init(Local<Object> exports, Local<Object> module) {
+  Local<FunctionTemplate> tpl = Nan::New<FunctionTemplate>(New);
+  tpl->SetClassName(Nan::New("Language").ToLocalChecked());
+  tpl->InstanceTemplate()->SetInternalFieldCount(1);
+
+  Local<Function> constructor = Nan::GetFunction(tpl).ToLocalChecked();
+  Local<Object> instance = constructor->NewInstance(Nan::GetCurrentContext()).ToLocalChecked();
+  Nan::SetInternalFieldPointer(instance, 0, tree_sitter_dust());
+
+  Nan::Set(instance, Nan::New("name").ToLocalChecked(), Nan::New("dust").ToLocalChecked());
+  Nan::Set(module, Nan::New("exports").ToLocalChecked(), instance);
+}
+
+NODE_MODULE(tree_sitter_dust_binding, Init)
+
+}  // namespace

bindings/node/index.js

@@ -0,0 +1,19 @@
+try {
+  module.exports = require("../../build/Release/tree_sitter_dust_binding");
+} catch (error1) {
+  if (error1.code !== 'MODULE_NOT_FOUND') {
+    throw error1;
+  }
+  try {
+    module.exports = require("../../build/Debug/tree_sitter_dust_binding");
+  } catch (error2) {
+    if (error2.code !== 'MODULE_NOT_FOUND') {
+      throw error2;
+    }
+    throw error1
+  }
+}
+
+try {
+  module.exports.nodeTypeInfo = require("../../src/node-types.json");
+} catch (_) {}

bindings/rust/build.rs

@@ -0,0 +1,40 @@
+fn main() {
+    let src_dir = std::path::Path::new("src");
+
+    let mut c_config = cc::Build::new();
+    c_config.include(&src_dir);
+    c_config
+        .flag_if_supported("-Wno-unused-parameter")
+        .flag_if_supported("-Wno-unused-but-set-variable")
+        .flag_if_supported("-Wno-trigraphs");
+    let parser_path = src_dir.join("parser.c");
+    c_config.file(&parser_path);
+
+    // If your language uses an external scanner written in C,
+    // then include this block of code:
+
+    /*
+    let scanner_path = src_dir.join("scanner.c");
+    c_config.file(&scanner_path);
+    println!("cargo:rerun-if-changed={}", scanner_path.to_str().unwrap());
+    */
+
+    c_config.compile("parser");
+    println!("cargo:rerun-if-changed={}", parser_path.to_str().unwrap());
+
+    // If your language uses an external scanner written in C++,
+    // then include this block of code:
+
+    /*
+    let mut cpp_config = cc::Build::new();
+    cpp_config.cpp(true);
+    cpp_config.include(&src_dir);
+    cpp_config
+        .flag_if_supported("-Wno-unused-parameter")
+        .flag_if_supported("-Wno-unused-but-set-variable");
+    let scanner_path = src_dir.join("scanner.cc");
+    cpp_config.file(&scanner_path);
+    cpp_config.compile("scanner");
+    println!("cargo:rerun-if-changed={}", scanner_path.to_str().unwrap());
+    */
+}

bindings/rust/lib.rs

@@ -0,0 +1,52 @@
+//! This crate provides dust language support for the [tree-sitter][] parsing library.
+//!
+//! Typically, you will use the [language][language func] function to add this language to a
+//! tree-sitter [Parser][], and then use the parser to parse some code:
+//!
+//! ```
+//! let code = "";
+//! let mut parser = tree_sitter::Parser::new();
+//! parser.set_language(tree_sitter_dust::language()).expect("Error loading dust grammar");
+//! let tree = parser.parse(code, None).unwrap();
+//! ```
+//!
+//! [Language]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Language.html
+//! [language func]: fn.language.html
+//! [Parser]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Parser.html
+//! [tree-sitter]: https://tree-sitter.github.io/
+
+use tree_sitter::Language;
+
+extern "C" {
+    fn tree_sitter_dust() -> Language;
+}
+
+/// Get the tree-sitter [Language][] for this grammar.
+///
+/// [Language]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Language.html
+pub fn language() -> Language {
+    unsafe { tree_sitter_dust() }
+}
+
+/// The content of the [`node-types.json`][] file for this grammar.
+///
+/// [`node-types.json`]: https://tree-sitter.github.io/tree-sitter/using-parsers#static-node-types
+pub const NODE_TYPES: &'static str = include_str!("../../src/node-types.json");
+
+// Uncomment these to include any queries that this grammar contains
+
+// pub const HIGHLIGHTS_QUERY: &'static str = include_str!("../../queries/highlights.scm");
+// pub const INJECTIONS_QUERY: &'static str = include_str!("../../queries/injections.scm");
+// pub const LOCALS_QUERY: &'static str = include_str!("../../queries/locals.scm");
+// pub const TAGS_QUERY: &'static str = include_str!("../../queries/tags.scm");
+
+#[cfg(test)]
+mod tests {
+    #[test]
+    fn test_can_load_grammar() {
+        let mut parser = tree_sitter::Parser::new();
+        parser
+            .set_language(super::language())
+            .expect("Error loading dust language");
+    }
+}

corpus/statements.txt

@@ -0,0 +1,50 @@
+==================
+Simple Statements
+==================
+
+1;
+"one";
+x;
+
+---
+
+(source
+	(statement
+		(value
+			(integer))
+		(close))
+	(statement
+		(value
+			(string))
+		(close))
+	(statement
+		(identifier)
+		(close)))
+
+==================
+Simple Assignment
+==================
+
+x = 1;
+y = "one";
+
+---
+
+(source
+	(statement
+		(identifier)
+		(operator
+			(assignment)) 
+		(statement
+			(value
+				(integer))
+			(close)))
+	(statement
+		(identifier)
+		(operator
+			(assignment)) 
+		(statement
+			(value
+				(string))
+			(close))))
+

corpus/tests.txt

@@ -1,152 +0,0 @@
-==================
-Comments
-==================
-
-# x = 1;
-# unassigned_variable
-#xyz
-
----
-
-(source
-	(comment)
-	(comment)
-	(comment))
-
-==================
-Identifiers
-==================
-
-variable_name
-_unused_variable
-__strange_format__
-a
-blahblah
-x.x
-
----
-
-(source
-	(identifier)
-	(identifier)
-	(identifier)
-	(identifier)
-	(identifier)
-	(identifier))
-
-==================
-Operators
-==================
-
-x = y + y;
-
----
-
-(source
-    (identifier)
-    (operator)
-    (identifier)
-    (operator)
-    (identifier)
-	(operator))
-
-==================
-String
-==================
-
-"string"
-'string'
-
----
-
-(source
-	(value
-		(string))
-	(value
-		(string)))
-
-==================
-Integer
-==================
-
-1
-123
-
----
-
-(source
-	(value
-		(integer))
-	(value
-		(integer)))
-
-==================
-Float
-==================
-
-1.0
-123.123
-
----
-
-(source
-	(value
-		(float))
-	(value
-		(float)))
-
-==================
-List
-==================
-
-(1, 2)
-
----
-
-(source
-	(value
-		(list
-			(value
-    			(integer))
-			(value
-       			(integer)))))
-
-==================
-Empty
-==================
-
-()
-
----
-
-(source
-	(value
-		(empty)))
-
-==================
-Tool
-==================
-
-random_boolean();
-
----
-
-(source
-	(tool)
-	(value
-		(empty))
-	(operator))
-
-==================
-Boolean
-==================
-
-true false
-
----
-
-(source
-	(value
-		(boolean))
-	(value
-		(boolean)))

grammar.js

@@ -3,14 +3,16 @@ module.exports = grammar({
 
   rules: {
     source: $ => repeat(choice(
-      $.comment,
+      $.statement,
-      $.identifier,
-      $.value,
-      $.tool,
-      $.operator,
     )),
 
-    comment: $ => seq('#', /.*/),
+    statement: $ => choice(
+      seq($.value, $.close),
+      seq($.identifier, $.close),
+      seq($.identifier, $.operator, $.statement),
+    ),
+
+    close: $ => ";",
 
     identifier: $ => /[a-zA-Z|_|.]+(_[a-zA-Z]+)*/,
 
@@ -24,38 +26,19 @@ module.exports = grammar({
       $.function,
     ), 
 
-    tool: $ => choice(
-      "random",
-      "random_boolean",
-      "random_integer",
-      "random_string",
-      "random_float",
-    ),
-
-    operator: $ => choice(
-      '=',
-      '-',
-      '+',
-      '/',
-      '|',
-      '&',
-      ';',
-      "->",
-    ),
-
     float: $ => /\d+\.\d*/,
     
     integer: $ => /\d+/,
 
-    string: $ => /("|')(.*?)("|')/,
+    string: $ => /("|'|`)(.*?)("|'|`)/,
 
     function: $ => /{(.*?)}/,
 
-    empty: $ => "()",
+    empty: $ => '()',
 
     boolean: $ => choice(
-      "true",
+      'true',
-      "false"
+      'false',
     ),
 
     list: $ => seq(
@@ -63,5 +46,17 @@ module.exports = grammar({
       repeat1(seq($.value, optional(','))),
       ')'
     ),
+
+    operator: $ => choice(
+      '+',
+      '-',
+      $.assignment,
+    ),
+
+    assignment: $ => choice(
+      '=',
+      '+=',
+      '-=',
+    ),
   }
 });

src/grammar.json

@@ -8,40 +8,63 @@
         "members": [
           {
             "type": "SYMBOL",
-            "name": "comment"
+            "name": "statement"
-          },
-          {
-            "type": "SYMBOL",
-            "name": "identifier"
-          },
-          {
-            "type": "SYMBOL",
-            "name": "value"
-          },
-          {
-            "type": "SYMBOL",
-            "name": "tool"
-          },
-          {
-            "type": "SYMBOL",
-            "name": "operator"
           }
         ]
       }
     },
-    "comment": {
+    "statement": {
-      "type": "SEQ",
+      "type": "CHOICE",
       "members": [
         {
-          "type": "STRING",
+          "type": "SEQ",
-          "value": "#"
+          "members": [
+            {
+              "type": "SYMBOL",
+              "name": "value"
+            },
+            {
+              "type": "SYMBOL",
+              "name": "close"
+            }
+          ]
         },
         {
-          "type": "PATTERN",
+          "type": "SEQ",
-          "value": ".*"
+          "members": [
+            {
+              "type": "SYMBOL",
+              "name": "identifier"
+            },
+            {
+              "type": "SYMBOL",
+              "name": "close"
+            }
+          ]
+        },
+        {
+          "type": "SEQ",
+          "members": [
+            {
+              "type": "SYMBOL",
+              "name": "identifier"
+            },
+            {
+              "type": "SYMBOL",
+              "name": "operator"
+            },
+            {
+              "type": "SYMBOL",
+              "name": "statement"
+            }
+          ]
         }
       ]
     },
+    "close": {
+      "type": "STRING",
+      "value": ";"
+    },
     "identifier": {
       "type": "PATTERN",
       "value": "[a-zA-Z|_|.]+(_[a-zA-Z]+)*"
@@ -79,68 +102,6 @@
         }
       ]
     },
-    "tool": {
-      "type": "CHOICE",
-      "members": [
-        {
-          "type": "STRING",
-          "value": "random"
-        },
-        {
-          "type": "STRING",
-          "value": "random_boolean"
-        },
-        {
-          "type": "STRING",
-          "value": "random_integer"
-        },
-        {
-          "type": "STRING",
-          "value": "random_string"
-        },
-        {
-          "type": "STRING",
-          "value": "random_float"
-        }
-      ]
-    },
-    "operator": {
-      "type": "CHOICE",
-      "members": [
-        {
-          "type": "STRING",
-          "value": "="
-        },
-        {
-          "type": "STRING",
-          "value": "-"
-        },
-        {
-          "type": "STRING",
-          "value": "+"
-        },
-        {
-          "type": "STRING",
-          "value": "/"
-        },
-        {
-          "type": "STRING",
-          "value": "|"
-        },
-        {
-          "type": "STRING",
-          "value": "&"
-        },
-        {
-          "type": "STRING",
-          "value": ";"
-        },
-        {
-          "type": "STRING",
-          "value": "->"
-        }
-      ]
-    },
     "float": {
       "type": "PATTERN",
       "value": "\\d+\\.\\d*"
@@ -151,7 +112,7 @@
     },
     "string": {
       "type": "PATTERN",
-      "value": "(\"|')(.*?)(\"|')"
+      "value": "(\"|'|`)(.*?)(\"|'|`)"
     },
     "function": {
       "type": "PATTERN",
@@ -210,6 +171,40 @@
           "value": ")"
         }
       ]
+    },
+    "operator": {
+      "type": "CHOICE",
+      "members": [
+        {
+          "type": "STRING",
+          "value": "+"
+        },
+        {
+          "type": "STRING",
+          "value": "-"
+        },
+        {
+          "type": "SYMBOL",
+          "name": "assignment"
+        }
+      ]
+    },
+    "assignment": {
+      "type": "CHOICE",
+      "members": [
+        {
+          "type": "STRING",
+          "value": "="
+        },
+        {
+          "type": "STRING",
+          "value": "+="
+        },
+        {
+          "type": "STRING",
+          "value": "-="
+        }
+      ]
     }
   },
   "extras": [

src/node-types.json

@@ -1,11 +1,11 @@
 [
   {
-    "type": "boolean",
+    "type": "assignment",
     "named": true,
     "fields": {}
   },
   {
-    "type": "comment",
+    "type": "boolean",
     "named": true,
     "fields": {}
   },
@@ -27,7 +27,17 @@
   {
     "type": "operator",
     "named": true,
-    "fields": {}
+    "fields": {},
+    "children": {
+      "multiple": false,
+      "required": false,
+      "types": [
+        {
+          "type": "assignment",
+          "named": true
+        }
+      ]
+    }
   },
   {
     "type": "source",
@@ -38,7 +48,22 @@
       "required": false,
       "types": [
         {
-          "type": "comment",
+          "type": "statement",
+          "named": true
+        }
+      ]
+    }
+  },
+  {
+    "type": "statement",
+    "named": true,
+    "fields": {},
+    "children": {
+      "multiple": true,
+      "required": true,
+      "types": [
+        {
+          "type": "close",
           "named": true
         },
         {
@@ -50,7 +75,7 @@
           "named": true
         },
         {
-          "type": "tool",
+          "type": "statement",
           "named": true
         },
         {
@@ -60,11 +85,6 @@
       ]
     }
   },
-  {
-    "type": "tool",
-    "named": true,
-    "fields": {}
-  },
   {
     "type": "value",
     "named": true,
@@ -104,14 +124,6 @@
       ]
     }
   },
-  {
-    "type": "#",
-    "named": false
-  },
-  {
-    "type": "&",
-    "named": false
-  },
   {
     "type": "(",
     "named": false
@@ -124,6 +136,10 @@
     "type": "+",
     "named": false
   },
+  {
+    "type": "+=",
+    "named": false
+  },
   {
     "type": ",",
     "named": false
@@ -133,21 +149,17 @@
     "named": false
   },
   {
-    "type": "->",
+    "type": "-=",
-    "named": false
-  },
-  {
-    "type": "/",
-    "named": false
-  },
-  {
-    "type": ";",
     "named": false
   },
   {
     "type": "=",
     "named": false
   },
+  {
+    "type": "close",
+    "named": true
+  },
   {
     "type": "empty",
     "named": true
@@ -172,26 +184,6 @@
     "type": "integer",
     "named": true
   },
-  {
-    "type": "random",
-    "named": false
-  },
-  {
-    "type": "random_boolean",
-    "named": false
-  },
-  {
-    "type": "random_float",
-    "named": false
-  },
-  {
-    "type": "random_integer",
-    "named": false
-  },
-  {
-    "type": "random_string",
-    "named": false
-  },
   {
     "type": "string",
     "named": true
@@ -199,9 +191,5 @@
   {
     "type": "true",
     "named": false
-  },
-  {
-    "type": "|",
-    "named": false
   }
 ]