Add tests; Improve parsing

This commit is contained in:
Jeff 2023-10-06 16:19:17 -04:00
parent 03c5664851
commit a3dbb19ecc
12 changed files with 4609 additions and 4765 deletions

View File

@ -1,19 +0,0 @@
{
"targets": [
{
"target_name": "tree_sitter_dust_binding",
"include_dirs": [
"<!(node -e \"require('nan')\")",
"src"
],
"sources": [
"bindings/node/binding.cc",
"src/parser.c",
# If your language uses an external scanner, add it here.
],
"cflags_c": [
"-std=c99",
]
}
]
}

View File

@ -1,28 +0,0 @@
#include "tree_sitter/parser.h"
#include <node.h>
#include "nan.h"
using namespace v8;
extern "C" TSLanguage * tree_sitter_dust();
namespace {
NAN_METHOD(New) {}
void Init(Local<Object> exports, Local<Object> module) {
Local<FunctionTemplate> tpl = Nan::New<FunctionTemplate>(New);
tpl->SetClassName(Nan::New("Language").ToLocalChecked());
tpl->InstanceTemplate()->SetInternalFieldCount(1);
Local<Function> constructor = Nan::GetFunction(tpl).ToLocalChecked();
Local<Object> instance = constructor->NewInstance(Nan::GetCurrentContext()).ToLocalChecked();
Nan::SetInternalFieldPointer(instance, 0, tree_sitter_dust());
Nan::Set(instance, Nan::New("name").ToLocalChecked(), Nan::New("dust").ToLocalChecked());
Nan::Set(module, Nan::New("exports").ToLocalChecked(), instance);
}
NODE_MODULE(tree_sitter_dust_binding, Init)
} // namespace

View File

@ -1,19 +0,0 @@
try {
module.exports = require("../../build/Release/tree_sitter_dust_binding");
} catch (error1) {
if (error1.code !== 'MODULE_NOT_FOUND') {
throw error1;
}
try {
module.exports = require("../../build/Debug/tree_sitter_dust_binding");
} catch (error2) {
if (error2.code !== 'MODULE_NOT_FOUND') {
throw error2;
}
throw error1
}
}
try {
module.exports.nodeTypeInfo = require("../../src/node-types.json");
} catch (_) {}

View File

@ -1,40 +0,0 @@
fn main() {
let src_dir = std::path::Path::new("src");
let mut c_config = cc::Build::new();
c_config.include(&src_dir);
c_config
.flag_if_supported("-Wno-unused-parameter")
.flag_if_supported("-Wno-unused-but-set-variable")
.flag_if_supported("-Wno-trigraphs");
let parser_path = src_dir.join("parser.c");
c_config.file(&parser_path);
// If your language uses an external scanner written in C,
// then include this block of code:
/*
let scanner_path = src_dir.join("scanner.c");
c_config.file(&scanner_path);
println!("cargo:rerun-if-changed={}", scanner_path.to_str().unwrap());
*/
c_config.compile("parser");
println!("cargo:rerun-if-changed={}", parser_path.to_str().unwrap());
// If your language uses an external scanner written in C++,
// then include this block of code:
/*
let mut cpp_config = cc::Build::new();
cpp_config.cpp(true);
cpp_config.include(&src_dir);
cpp_config
.flag_if_supported("-Wno-unused-parameter")
.flag_if_supported("-Wno-unused-but-set-variable");
let scanner_path = src_dir.join("scanner.cc");
cpp_config.file(&scanner_path);
cpp_config.compile("scanner");
println!("cargo:rerun-if-changed={}", scanner_path.to_str().unwrap());
*/
}

View File

@ -1,52 +0,0 @@
//! This crate provides dust language support for the [tree-sitter][] parsing library.
//!
//! Typically, you will use the [language][language func] function to add this language to a
//! tree-sitter [Parser][], and then use the parser to parse some code:
//!
//! ```
//! let code = "";
//! let mut parser = tree_sitter::Parser::new();
//! parser.set_language(tree_sitter_dust::language()).expect("Error loading dust grammar");
//! let tree = parser.parse(code, None).unwrap();
//! ```
//!
//! [Language]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Language.html
//! [language func]: fn.language.html
//! [Parser]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Parser.html
//! [tree-sitter]: https://tree-sitter.github.io/
use tree_sitter::Language;
extern "C" {
fn tree_sitter_dust() -> Language;
}
/// Get the tree-sitter [Language][] for this grammar.
///
/// [Language]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Language.html
pub fn language() -> Language {
unsafe { tree_sitter_dust() }
}
/// The content of the [`node-types.json`][] file for this grammar.
///
/// [`node-types.json`]: https://tree-sitter.github.io/tree-sitter/using-parsers#static-node-types
pub const NODE_TYPES: &'static str = include_str!("../../src/node-types.json");
// Uncomment these to include any queries that this grammar contains
// pub const HIGHLIGHTS_QUERY: &'static str = include_str!("../../queries/highlights.scm");
// pub const INJECTIONS_QUERY: &'static str = include_str!("../../queries/injections.scm");
// pub const LOCALS_QUERY: &'static str = include_str!("../../queries/locals.scm");
// pub const TAGS_QUERY: &'static str = include_str!("../../queries/tags.scm");
#[cfg(test)]
mod tests {
#[test]
fn test_can_load_grammar() {
let mut parser = tree_sitter::Parser::new();
parser
.set_language(super::language())
.expect("Error loading dust language");
}
}

View File

@ -57,15 +57,15 @@ function <message number> {
(statement (statement
(expression (expression
(tool_call (tool_call
(tool) (output
(expression (expression
(identifier))))) (identifier))))))
(statement (statement
(expression (expression
(tool_call (tool_call
(tool) (output
(expression (expression
(identifier))))))))))) (identifier))))))))))))
================== ==================
Complex Function Call Complex Function Call

View File

@ -54,4 +54,3 @@ x.answer
(statement (statement
(expression (expression
(identifier))))) (identifier)))))

View File

@ -1,8 +1,8 @@
================== ==================
Simple Equality Equality
================== ==================
1 == 1 3 == 1 + 1 + 1
--- ---
@ -15,23 +15,6 @@ Simple Equality
(value (value
(integer))) (integer)))
(logic_operator) (logic_operator)
(expression
(value
(integer))))))))
==================
Complex Equality
==================
1 + 1 + 1 != 3
---
(root
(item
(statement
(expression
(logic
(expression (expression
(math (math
(expression (expression
@ -44,10 +27,72 @@ Complex Equality
(value (value
(integer))))) (integer)))))
(math_operator) (math_operator)
(expression
(value
(integer))))))))))
==================
&&
==================
4 + 2 == 42 && true
---
(root
(item
(statement
(expression
(logic
(expression
(math
(expression
(value
(integer)))
(math_operator)
(expression (expression
(value (value
(integer))))) (integer)))))
(logic_operator) (logic_operator)
(expression (expression
(value (logic
(integer)))))))) (expression
(value
(integer)))
(logic_operator)
(expression
(value
(boolean))))))))))
==================
\||
==================
4 + 2 == 42 || true
---
(root
(item
(statement
(expression
(logic
(expression
(math
(expression
(value
(integer)))
(math_operator)
(expression
(value
(integer)))))
(logic_operator)
(expression
(logic
(expression
(value
(integer)))
(logic_operator)
(expression
(value
(boolean))))))))))

View File

@ -31,14 +31,14 @@ module.exports = grammar({
repeat(prec.left(seq('->', $.expression))) repeat(prec.left(seq('->', $.expression)))
), ),
expression: $ => choice( expression: $ => prec.right(choice(
$.value, $.value,
$.identifier, $.identifier,
$.function_call, $.function_call,
$.tool_call, $.tool_call,
$.math, $.math,
$.logic, $.logic,
), )),
identifier: $ => /[a-z|_|.]+[0-9]?/, identifier: $ => /[a-z|_|.]+[0-9]?/,
@ -106,7 +106,7 @@ module.exports = grammar({
'%', '%',
), ),
logic: $ => prec.left(seq( logic: $ => prec.right(seq(
$.expression, $.expression,
$.logic_operator, $.logic_operator,
$.expression, $.expression,
@ -117,8 +117,6 @@ module.exports = grammar({
'!=', '!=',
'&&', '&&',
'||', '||',
'and',
'or',
), ),
assignment: $ => prec.right(seq( assignment: $ => prec.right(seq(
@ -197,10 +195,12 @@ module.exports = grammar({
)), )),
_tool: $ => choice( _tool: $ => choice(
'input', $.input,
$.output, $.output,
), ),
input: $ => seq('input', $.expression),
output: $ => seq('output', $.expression), output: $ => seq('output', $.expression),
read: $ => seq('read', $.expression),
} }
}); });

View File

@ -110,33 +110,37 @@
] ]
}, },
"expression": { "expression": {
"type": "CHOICE", "type": "PREC_RIGHT",
"members": [ "value": 0,
{ "content": {
"type": "SYMBOL", "type": "CHOICE",
"name": "value" "members": [
}, {
{ "type": "SYMBOL",
"type": "SYMBOL", "name": "value"
"name": "identifier" },
}, {
{ "type": "SYMBOL",
"type": "SYMBOL", "name": "identifier"
"name": "function_call" },
}, {
{ "type": "SYMBOL",
"type": "SYMBOL", "name": "function_call"
"name": "tool_call" },
}, {
{ "type": "SYMBOL",
"type": "SYMBOL", "name": "tool_call"
"name": "math" },
}, {
{ "type": "SYMBOL",
"type": "SYMBOL", "name": "math"
"name": "logic" },
} {
] "type": "SYMBOL",
"name": "logic"
}
]
}
}, },
"identifier": { "identifier": {
"type": "PATTERN", "type": "PATTERN",
@ -451,7 +455,7 @@
] ]
}, },
"logic": { "logic": {
"type": "PREC_LEFT", "type": "PREC_RIGHT",
"value": 0, "value": 0,
"content": { "content": {
"type": "SEQ", "type": "SEQ",
@ -489,14 +493,6 @@
{ {
"type": "STRING", "type": "STRING",
"value": "||" "value": "||"
},
{
"type": "STRING",
"value": "and"
},
{
"type": "STRING",
"value": "or"
} }
] ]
}, },
@ -853,6 +849,19 @@
}, },
"_tool": { "_tool": {
"type": "CHOICE", "type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "input"
},
{
"type": "SYMBOL",
"name": "output"
}
]
},
"input": {
"type": "SEQ",
"members": [ "members": [
{ {
"type": "STRING", "type": "STRING",
@ -860,7 +869,7 @@
}, },
{ {
"type": "SYMBOL", "type": "SYMBOL",
"name": "output" "name": "expression"
} }
] ]
}, },
@ -876,6 +885,19 @@
"name": "expression" "name": "expression"
} }
] ]
},
"read": {
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "read"
},
{
"type": "SYMBOL",
"name": "expression"
}
]
} }
}, },
"extras": [ "extras": [

View File

@ -120,6 +120,21 @@
] ]
} }
}, },
{
"type": "input",
"named": true,
"fields": {},
"children": {
"multiple": false,
"required": true,
"types": [
{
"type": "expression",
"named": true
}
]
}
},
{ {
"type": "insert", "type": "insert",
"named": true, "named": true,
@ -399,8 +414,12 @@
"fields": {}, "fields": {},
"children": { "children": {
"multiple": false, "multiple": false,
"required": false, "required": true,
"types": [ "types": [
{
"type": "input",
"named": true
},
{ {
"type": "output", "type": "output",
"named": true "named": true
@ -550,10 +569,6 @@
"type": "]", "type": "]",
"named": false "named": false
}, },
{
"type": "and",
"named": false
},
{ {
"type": "break", "type": "break",
"named": false "named": false
@ -615,11 +630,11 @@
"named": false "named": false
}, },
{ {
"type": "or", "type": "output",
"named": false "named": false
}, },
{ {
"type": "output", "type": "read",
"named": false "named": false
}, },
{ {

File diff suppressed because it is too large Load Diff