jeff/tree-sitter-dust
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add tests; Improve parsing

Browse Source
This commit is contained in:
Jeff 2023-10-06 16:19:17 -04:00
parent 03c5664851
commit a3dbb19ecc
12 changed files with 4609 additions and 4765 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
binding.gyp
View File

@ -1,19 +0,0 @@
{
"targets": [
{
"target_name": "tree_sitter_dust_binding",
"include_dirs": [
"<!(node -e \"require('nan')\")",
"src"
],
"sources": [
"bindings/node/binding.cc",
"src/parser.c",
# If your language uses an external scanner, add it here.
],
"cflags_c": [
"-std=c99",
]
}
]
}

28
bindings/node/binding.cc
View File

@ -1,28 +0,0 @@
#include "tree_sitter/parser.h"
#include <node.h>
#include "nan.h"
using namespace v8;
extern "C" TSLanguage * tree_sitter_dust();
namespace {
NAN_METHOD(New) {}
void Init(Local<Object> exports, Local<Object> module) {
Local<FunctionTemplate> tpl = Nan::New<FunctionTemplate>(New);
tpl->SetClassName(Nan::New("Language").ToLocalChecked());
tpl->InstanceTemplate()->SetInternalFieldCount(1);
Local<Function> constructor = Nan::GetFunction(tpl).ToLocalChecked();
Local<Object> instance = constructor->NewInstance(Nan::GetCurrentContext()).ToLocalChecked();
Nan::SetInternalFieldPointer(instance, 0, tree_sitter_dust());
Nan::Set(instance, Nan::New("name").ToLocalChecked(), Nan::New("dust").ToLocalChecked());
Nan::Set(module, Nan::New("exports").ToLocalChecked(), instance);
}
NODE_MODULE(tree_sitter_dust_binding, Init)
} // namespace

19
bindings/node/index.js
View File

@ -1,19 +0,0 @@
try {
module.exports = require("../../build/Release/tree_sitter_dust_binding");
} catch (error1) {
if (error1.code !== 'MODULE_NOT_FOUND') {
throw error1;
}
try {
module.exports = require("../../build/Debug/tree_sitter_dust_binding");
} catch (error2) {
if (error2.code !== 'MODULE_NOT_FOUND') {
throw error2;
}
throw error1
}
}
try {
module.exports.nodeTypeInfo = require("../../src/node-types.json");
} catch (_) {}

40
bindings/rust/build.rs
View File

@ -1,40 +0,0 @@
fn main() {
let src_dir = std::path::Path::new("src");
let mut c_config = cc::Build::new();
c_config.include(&src_dir);
c_config
.flag_if_supported("-Wno-unused-parameter")
.flag_if_supported("-Wno-unused-but-set-variable")
.flag_if_supported("-Wno-trigraphs");
let parser_path = src_dir.join("parser.c");
c_config.file(&parser_path);
// If your language uses an external scanner written in C,
// then include this block of code:
/*
let scanner_path = src_dir.join("scanner.c");
c_config.file(&scanner_path);
println!("cargo:rerun-if-changed={}", scanner_path.to_str().unwrap());
*/
c_config.compile("parser");
println!("cargo:rerun-if-changed={}", parser_path.to_str().unwrap());
// If your language uses an external scanner written in C++,
// then include this block of code:
/*
let mut cpp_config = cc::Build::new();
cpp_config.cpp(true);
cpp_config.include(&src_dir);
cpp_config
.flag_if_supported("-Wno-unused-parameter")
.flag_if_supported("-Wno-unused-but-set-variable");
let scanner_path = src_dir.join("scanner.cc");
cpp_config.file(&scanner_path);
cpp_config.compile("scanner");
println!("cargo:rerun-if-changed={}", scanner_path.to_str().unwrap());
*/
}

52
bindings/rust/lib.rs
View File

@ -1,52 +0,0 @@
//! This crate provides dust language support for the [tree-sitter][] parsing library.
//!
//! Typically, you will use the [language][language func] function to add this language to a
//! tree-sitter [Parser][], and then use the parser to parse some code:
//!
//! ```
//! let code = "";
//! let mut parser = tree_sitter::Parser::new();
//! parser.set_language(tree_sitter_dust::language()).expect("Error loading dust grammar");
//! let tree = parser.parse(code, None).unwrap();
//! ```
//!
//! [Language]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Language.html
//! [language func]: fn.language.html
//! [Parser]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Parser.html
//! [tree-sitter]: https://tree-sitter.github.io/
use tree_sitter::Language;
extern "C" {
fn tree_sitter_dust() -> Language;
}
/// Get the tree-sitter [Language][] for this grammar.
///
/// [Language]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Language.html
pub fn language() -> Language {
unsafe { tree_sitter_dust() }
}
/// The content of the [`node-types.json`][] file for this grammar.
///
/// [`node-types.json`]: https://tree-sitter.github.io/tree-sitter/using-parsers#static-node-types
pub const NODE_TYPES: &'static str = include_str!("../../src/node-types.json");
// Uncomment these to include any queries that this grammar contains
// pub const HIGHLIGHTS_QUERY: &'static str = include_str!("../../queries/highlights.scm");
// pub const INJECTIONS_QUERY: &'static str = include_str!("../../queries/injections.scm");
// pub const LOCALS_QUERY: &'static str = include_str!("../../queries/locals.scm");
// pub const TAGS_QUERY: &'static str = include_str!("../../queries/tags.scm");
#[cfg(test)]
mod tests {
#[test]
fn test_can_load_grammar() {
let mut parser = tree_sitter::Parser::new();
parser
.set_language(super::language())
.expect("Error loading dust language");
}
}

12
corpus/functions.txt
View File

@ -57,15 +57,15 @@ function <message number> {
(statement
(expression
(tool_call
(tool)
(expression
(identifier)))))
(output
(expression
(identifier))))))
(statement
(expression
(tool_call
(tool)
(expression
(identifier)))))))))))
(output
(expression
(identifier))))))))))))
==================
Complex Function Call

1
corpus/maps.txt
View File

@ -54,4 +54,3 @@ x.answer
(statement
(expression
(identifier)))))

87
corpus/operators.txt
View File

@ -1,8 +1,8 @@
==================
Simple Equality
Equality
==================
1 == 1
3 == 1 + 1 + 1
---
@ -15,23 +15,6 @@ Simple Equality
(value
(integer)))
(logic_operator)
(expression
(value
(integer))))))))
==================
Complex Equality
==================
1 + 1 + 1 != 3
---
(root
(item
(statement
(expression
(logic
(expression
(math
(expression
@ -44,10 +27,72 @@ Complex Equality
(value
(integer)))))
(math_operator)
(expression
(value
(integer))))))))))
==================
&&
==================
4 + 2 == 42 && true
---
(root
(item
(statement
(expression
(logic
(expression
(math
(expression
(value
(integer)))
(math_operator)
(expression
(value
(integer)))))
(logic_operator)
(expression
(value
(integer))))))))
(logic
(expression
(value
(integer)))
(logic_operator)
(expression
(value
(boolean))))))))))
==================
\||
==================
4 + 2 == 42 || true
---
(root
(item
(statement
(expression
(logic
(expression
(math
(expression
(value
(integer)))
(math_operator)
(expression
(value
(integer)))))
(logic_operator)
(expression
(logic
(expression
(value
(integer)))
(logic_operator)
(expression
(value
(boolean))))))))))

14
grammar.js
View File

@ -31,14 +31,14 @@ module.exports = grammar({
repeat(prec.left(seq('->', $.expression)))
),
expression: $ => choice(
expression: $ => prec.right(choice(
$.value,
$.identifier,
$.function_call,
$.tool_call,
$.math,
$.logic,
),
)),
identifier: $ => /[a-z|_|.]+[0-9]?/,
@ -94,7 +94,7 @@ module.exports = grammar({
math: $ => prec.left(seq(
$.expression,
$.math_operator,
$.math_operator,
$.expression,
)),
@ -106,7 +106,7 @@ module.exports = grammar({
'%',
),
logic: $ => prec.left(seq(
logic: $ => prec.right(seq(
$.expression,
$.logic_operator,
$.expression,
@ -117,8 +117,6 @@ module.exports = grammar({
'!=',
'&&',
'||',
'and',
'or',
),
assignment: $ => prec.right(seq(
@ -197,10 +195,12 @@ module.exports = grammar({
)),
_tool: $ => choice(
'input',
$.input,
$.output,
),
input: $ => seq('input', $.expression),
output: $ => seq('output', $.expression),
read: $ => seq('read', $.expression),
}
});

96
src/grammar.json
View File

@ -110,33 +110,37 @@
]
},
"expression": {
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "value"
},
{
"type": "SYMBOL",
"name": "identifier"
},
{
"type": "SYMBOL",
"name": "function_call"
},
{
"type": "SYMBOL",
"name": "tool_call"
},
{
"type": "SYMBOL",
"name": "math"
},
{
"type": "SYMBOL",
"name": "logic"
}
]
"type": "PREC_RIGHT",
"value": 0,
"content": {
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "value"
},
{
"type": "SYMBOL",
"name": "identifier"
},
{
"type": "SYMBOL",
"name": "function_call"
},
{
"type": "SYMBOL",
"name": "tool_call"
},
{
"type": "SYMBOL",
"name": "math"
},
{
"type": "SYMBOL",
"name": "logic"
}
]
}
},
"identifier": {
"type": "PATTERN",
@ -451,7 +455,7 @@
]
},
"logic": {
"type": "PREC_LEFT",
"type": "PREC_RIGHT",
"value": 0,
"content": {
"type": "SEQ",
@ -489,14 +493,6 @@
{
"type": "STRING",
"value": "||"
},
{
"type": "STRING",
"value": "and"
},
{
"type": "STRING",
"value": "or"
}
]
},
@ -853,6 +849,19 @@
},
"_tool": {
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "input"
},
{
"type": "SYMBOL",
"name": "output"
}
]
},
"input": {
"type": "SEQ",
"members": [
{
"type": "STRING",
@ -860,7 +869,7 @@
},
{
"type": "SYMBOL",
"name": "output"
"name": "expression"
}
]
},
@ -876,6 +885,19 @@
"name": "expression"
}
]
},
"read": {
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "read"
},
{
"type": "SYMBOL",
"name": "expression"
}
]
}
},
"extras": [

29
src/node-types.json
View File

@ -120,6 +120,21 @@
]
}
},
{
"type": "input",
"named": true,
"fields": {},
"children": {
"multiple": false,
"required": true,
"types": [
{
"type": "expression",
"named": true
}
]
}
},
{
"type": "insert",
"named": true,
@ -399,8 +414,12 @@
"fields": {},
"children": {
"multiple": false,
"required": false,
"required": true,
"types": [
{
"type": "input",
"named": true
},
{
"type": "output",
"named": true
@ -550,10 +569,6 @@
"type": "]",
"named": false
},
{
"type": "and",
"named": false
},
{
"type": "break",
"named": false
@ -615,11 +630,11 @@
"named": false
},
{
"type": "or",
"type": "output",
"named": false
},
{
"type": "output",
"type": "read",
"named": false
},
{

8977
src/parser.c
View File

File diff suppressed because it is too large Load Diff