jeff/dust
1
0
Fork 0
Code Issues

Replace tools

Browse Source
This commit is contained in:
Jeff 2023-10-21 13:04:17 -04:00
parent eefb6e5fb6
commit 3b82c6d900
13 changed files with 5180 additions and 6595 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
examples/table.ds
View File

@ -1,24 +1,26 @@
table = create_table <text number bool> [
my_table = table <text number bool> [
["a", 1, true]
["b", 2, true]
["a", 3, true]
]
test_table = create_table <text bool> [
test_table = table <text bool> [
["a", true]
["b", true]
["a", true]
]
select = select <text bool> from table
test_select = select <text bool> from my_table
(assert_equal select, test_table)
(assert_equal test_select, test_table)
test_table = create_table <text number bool> [
test_table = table <text number bool> [
["a", 1, true]
["a", 3, true]
]
select_where = select * from table where text == "a"
test_select_where = select <> from my_table {
text == "a"
}
assert_equal(select_where, test_table);
(assert_equal test_select_where, test_table)

47
src/abstract_tree/function_call.rs
View File

@ -1,39 +1,22 @@
use serde::{Deserialize, Serialize};
use tree_sitter::Node;
use crate::{tool::Tool, AbstractTree, Result, Value, VariableMap};
use crate::{AbstractTree, Error, Result, Value, VariableMap, BUILT_IN_FUNCTIONS};
use super::{expression::Expression, identifier::Identifier};
#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq, PartialOrd, Ord)]
pub struct FunctionCall {
name: FunctionName,
name: Identifier,
arguments: Vec<Expression>,
}
#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq, PartialOrd, Ord)]
pub enum FunctionName {
Identifier(Identifier),
Tool(Tool),
}
impl AbstractTree for FunctionCall {
fn from_syntax_node(source: &str, node: Node) -> Result<Self> {
debug_assert_eq!("function_call", node.kind());
let name_node = node.child(1).unwrap();
let name = match name_node.kind() {
"identifier" => {
FunctionName::Identifier(Identifier::from_syntax_node(source, name_node)?)
}
"tool" => {
let tool_node = name_node.child(0).unwrap();
let tool = Tool::new(tool_node.kind())?;
FunctionName::Tool(tool)
}
_ => panic!(""),
};
let name = Identifier::from_syntax_node(source, name_node)?;
let mut arguments = Vec::new();
@ -51,25 +34,17 @@ impl AbstractTree for FunctionCall {
}
fn run(&self, source: &str, context: &mut VariableMap) -> Result<Value> {
let identifier = match &self.name {
FunctionName::Identifier(identifier) => identifier,
FunctionName::Tool(tool) => {
let mut values = Vec::with_capacity(self.arguments.len());
for expression in &self.arguments {
let value = expression.run(source, context)?;
values.push(value);
}
return tool.run(&values);
}
};
let key = identifier.inner();
let key = self.name.inner();
let definition = if let Some(value) = context.get_value(key)? {
value.as_function().cloned()?
} else {
return Err(crate::Error::FunctionIdentifierNotFound(identifier.clone()));
for function in BUILT_IN_FUNCTIONS {
if key == function.name() {
return function.run(source, context);
}
}
return Err(Error::FunctionIdentifierNotFound(self.name.clone()));
};
let id_expr_pairs = definition.identifiers().iter().zip(self.arguments.iter());

9
src/abstract_tree/mod.rs
View File

@ -8,6 +8,7 @@
pub mod assignment;
pub mod r#async;
pub mod built_in_functions;
pub mod expression;
pub mod filter;
pub mod find;
@ -20,16 +21,16 @@ pub mod logic;
pub mod r#match;
pub mod math;
pub mod remove;
pub mod select;
pub mod statement;
pub mod tool;
pub mod transform;
pub mod value_node;
pub mod r#while;
pub use {
assignment::*, expression::*, filter::*, find::*, function_call::*, identifier::*, if_else::*,
item::*, logic::*, math::*, r#async::*, r#for::*, r#match::*, r#while::*, remove::*,
statement::*, transform::*,
assignment::*, built_in_functions::*, expression::*, filter::*, find::*, function_call::*,
identifier::*, if_else::*, item::*, logic::*, math::*, r#async::*, r#for::*, r#match::*,
r#while::*, remove::*, select::*, statement::*, transform::*,
};
use tree_sitter::Node;

2
src/abstract_tree/remove.rs
View File

@ -1,6 +1,6 @@
use serde::{Deserialize, Serialize};
use crate::{expression, AbstractTree, Expression, Identifier, Item, Value};
use crate::{AbstractTree, Expression, Identifier, Item, Value};
#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq, PartialOrd, Ord)]
pub struct Remove {

71
src/abstract_tree/select.rs Normal file
View File

@ -0,0 +1,71 @@
use serde::{Deserialize, Serialize};
use crate::{identifier, AbstractTree, Expression, Identifier, Item, Table, Value};
#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq, PartialOrd, Ord)]
pub struct Select {
identifiers: Vec<Identifier>,
expression: Expression,
item: Option<Item>,
}
impl AbstractTree for Select {
fn from_syntax_node(source: &str, node: tree_sitter::Node) -> crate::Result<Self> {
let child_count = node.child_count();
let mut identifiers = Vec::new();
for index in 2..child_count - 4 {
let node = node.child(index).unwrap();
if node.kind() == "identifier" {
let identifier = Identifier::from_syntax_node(source, node)?;
identifiers.push(identifier);
}
if node.kind() == ">" {
break;
}
}
let final_node = node.child(child_count - 1).unwrap();
let item = if final_node.kind() == "}" {
let item_node = node.child(child_count - 2).unwrap();
Some(Item::from_syntax_node(source, item_node)?)
} else {
None
};
let expression_node = if item.is_some() {
node.child(child_count - 4).unwrap()
} else {
node.child(child_count - 1).unwrap()
};
let expression = Expression::from_syntax_node(source, expression_node)?;
Ok(Select {
identifiers,
expression,
item,
})
}
fn run(&self, source: &str, context: &mut crate::VariableMap) -> crate::Result<crate::Value> {
let value = self.expression.run(source, context)?;
let table = value.as_table()?;
let column_names = if self.identifiers.len() > 0 {
self.identifiers
.iter()
.cloned()
.map(|identifierier| identifierier.take_inner())
.collect()
} else {
table.column_names().clone()
};
let new_table = table.select(&column_names);
Ok(Value::Table(new_table))
}
}

7
src/abstract_tree/statement.rs
View File

@ -3,7 +3,7 @@ use tree_sitter::Node;
use crate::{
AbstractTree, Assignment, Async, Error, Expression, Filter, Find, For, IfElse, Match, Remove,
Result, Transform, Value, VariableMap, While,
Result, Select, Transform, Value, VariableMap, While,
};
/// Abstract representation of a statement.
@ -23,6 +23,7 @@ pub enum Statement {
Filter(Box<Filter>),
Find(Box<Find>),
Remove(Box<Remove>),
Select(Box<Select>),
}
impl AbstractTree for Statement {
@ -65,6 +66,9 @@ impl AbstractTree for Statement {
"remove" => Ok(Statement::Remove(Box::new(Remove::from_syntax_node(
source, child,
)?))),
"select" => Ok(Statement::Select(Box::new(Select::from_syntax_node(
source, child,
)?))),
_ => Err(Error::UnexpectedSyntaxNode {
expected: "assignment, expression, if...else, while, for, transform, filter, tool or async",
actual: child.kind(),
@ -87,6 +91,7 @@ impl AbstractTree for Statement {
Statement::Filter(filter) => filter.run(source, context),
Statement::Find(find) => find.run(source, context),
Statement::Remove(remove) => remove.run(source, context),
Statement::Select(select) => select.run(source, context),
}
}
}

275
src/abstract_tree/tool.rs
View File

@ -1,275 +0,0 @@
use std::{fs::read_to_string, process::Command};
use rand::{random, thread_rng, Rng};
use serde::{Deserialize, Serialize};
use crate::{evaluate, Error, Result, Table, Value};
#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq, PartialOrd, Ord)]
pub enum Tool {
Assert,
AssertEqual,
Output,
Run,
Read,
Help,
Length,
Raw,
Sh,
Bash,
Fish,
Zsh,
FromCsv,
ToCsv,
FromJson,
ToJson,
Random,
RandomFloat,
RandomInteger,
RandomString,
}
impl Tool {
pub fn new(kind: &str) -> Result<Self> {
let tool = match kind {
"assert" => Tool::Assert,
"assert_equal" => Tool::AssertEqual,
"output" => Tool::Output,
"length" => Tool::Length,
"raw" => Tool::Raw,
"sh" => Tool::Sh,
"bash" => Tool::Bash,
"fish" => Tool::Fish,
"zsh" => Tool::Zsh,
"from_csv" => Tool::FromCsv,
"to_csv" => Tool::ToCsv,
"from_json" => Tool::FromJson,
"to_json" => Tool::ToJson,
"random" => Tool::Random,
"random_integer" => Tool::RandomInteger,
"random_float" => Tool::RandomFloat,
"random_string" => Tool::RandomString,
"read" => Tool::Read,
"help" => Tool::Help,
_ => todo!("Tool name not recognized."),
};
Ok(tool)
}
pub fn run(&self, values: &[Value]) -> Result<Value> {
let value = match self {
Tool::Assert => {
Error::expect_tool_argument_amount("assert", 1, values.len())?;
if values[0].as_boolean()? {
Value::Empty
} else {
return Err(Error::AssertFailed);
}
}
Tool::AssertEqual => {
Error::expect_tool_argument_amount("assert_equal", 2, values.len())?;
if values[0] == values[1] {
Value::Empty
} else {
return Err(Error::AssertEqualFailed {
expected: values[0].clone(),
actual: values[1].clone(),
});
}
}
Tool::Run => {
Error::expect_tool_argument_amount("run", 1, values.len())?;
let file_path = values[0].as_string()?;
let file_contents = read_to_string(file_path)?;
evaluate(&file_contents)?
}
Tool::Output => {
Error::expect_tool_argument_amount("output", 1, values.len())?;
println!("{}", values[0]);
Value::Empty
}
Tool::Length => {
Error::expect_tool_argument_amount("length", 1, values.len())?;
let length = if let Ok(list) = values[0].as_list() {
list.len()
} else if let Ok(map) = values[0].as_map() {
map.len()
} else if let Ok(table) = values[0].as_table() {
table.len()
} else if let Ok(string) = values[0].as_string() {
string.chars().count()
} else {
1
};
Value::Integer(length as i64)
}
Tool::Read => {
Error::expect_tool_argument_amount("read", 1, values.len())?;
let file_contents = read_to_string(values[0].as_string()?)?;
Value::String(file_contents)
}
Tool::Random => todo!(),
Tool::RandomFloat => todo!(),
Tool::RandomString => todo!(),
Tool::RandomInteger => {
if values.len() == 0 {
Value::Integer(random())
} else if values.len() == 2 {
let mut rng = thread_rng();
let range = values[0].as_int()?..=values[1].as_int()?;
let random = rng.gen_range(range);
Value::Integer(random)
} else {
return Err(Error::ExpectedToolArgumentAmount {
tool_name: "random_integer",
expected: 2,
actual: values.len(),
});
}
}
Tool::Help => {
Error::expect_tool_argument_amount("help", 0, values.len())?;
let mut help_table =
Table::new(vec!["name".to_string(), "description".to_string()]);
help_table.insert(vec![
Value::String("help".to_string()),
Value::String("List available tools.".to_string()),
])?;
help_table.insert(vec![
Value::String("assert".to_string()),
Value::String("Panic if an expression is false.".to_string()),
])?;
help_table.insert(vec![
Value::String("assert_equal".to_string()),
Value::String("Panic if two values are not equal.".to_string()),
])?;
help_table.insert(vec![
Value::String("output".to_string()),
Value::String("Emit a value to stdout.".to_string()),
])?;
help_table.insert(vec![
Value::String("read".to_string()),
Value::String("Get a file's content.".to_string()),
])?;
help_table.insert(vec![
Value::String("from_json".to_string()),
Value::String("Convert a JSON string to a value.".to_string()),
])?;
help_table.insert(vec![
Value::String("to_json".to_string()),
Value::String("Convert a value to a JSON string.".to_string()),
])?;
Value::Table(help_table)
}
Tool::Raw => {
let program = values[0].as_string()?;
let mut command = Command::new(program);
for value in &values[1..] {
let arg = value.as_string()?;
command.arg(arg);
}
command.spawn()?.wait()?;
Value::Empty
}
Tool::Sh => {
let mut command = Command::new("sh");
for value in values {
let arg = value.as_string()?;
command.arg(arg);
}
command.spawn()?.wait()?;
Value::Empty
}
Tool::Bash => {
let mut command = Command::new("bash");
for value in values {
let arg = value.as_string()?;
command.arg(arg);
}
command.spawn()?.wait()?;
Value::Empty
}
Tool::Fish => {
let mut command = Command::new("fish");
for value in values {
let arg = value.as_string()?;
command.arg(arg);
}
command.spawn()?.wait()?;
Value::Empty
}
Tool::Zsh => {
let mut command = Command::new("zsh");
for value in values {
let arg = value.as_string()?;
command.arg(arg);
}
command.spawn()?.wait()?;
Value::Empty
}
Tool::FromCsv => todo!(),
Tool::ToCsv => todo!(),
Tool::FromJson => {
Error::expect_tool_argument_amount("from_json", 1, values.len())?;
let json_string = values[0].as_string()?;
serde_json::from_str(json_string)?
}
Tool::ToJson => {
Error::expect_tool_argument_amount("to_json", 1, values.len())?;
let value = &values[0];
let json_string = serde_json::to_string(value)?;
Value::String(json_string)
}
};
Ok(value)
}
}

24
tree-sitter-dust/corpus/assert.txt Normal file
View File

@ -0,0 +1,24 @@
==================
Assert Boolean
==================
assert true
---
==================
Assert Expression
==================
assert 42 % 2 == 0
---
==================
Assert Equal
==================
assert_equal 42 "the answer"
---

12
tree-sitter-dust/corpus/tables.txt
View File

@ -67,7 +67,9 @@ foobar = table <text, number> [
Table Access
==================
select number from foobar where text == 'answer'
select <number> from foobar {
text == 'answer'
}
---
@ -76,7 +78,10 @@ select number from foobar where text == 'answer'
(statement
(select
(identifier)
(identifier)
(expression
(identifier))
(item
(statement
(expression
(logic
(expression
@ -84,8 +89,7 @@ select number from foobar where text == 'answer'
(logic_operator)
(expression
(value
(string)))))))))
(string)))))))))))
==================
Table Insert

45
tree-sitter-dust/grammar.js
View File

@ -163,7 +163,7 @@ module.exports = grammar({
function_call: $ => prec.right(seq(
'(',
choice($.identifier, $.tool),
$.identifier,
repeat(seq($.expression, optional(','))),
')',
)),
@ -226,47 +226,14 @@ module.exports = grammar({
'}',
),
tool: $ => choice(
'assert',
'assert_equal',
'output',
'read',
'write',
'raw',
'sh',
'bash',
'fish',
'zsh',
'random',
'random_boolean',
'random_float',
'random_string',
'random_integer',
'length',
'sort',
'transform',
'filter',
'to_csv',
'from_csv',
'to_json',
'from_json',
'help',
),
select: $ => prec.right(seq(
'select',
$.identifier,
'<',
repeat(seq($.identifier, optional(','))),
'>',
'from',
$.identifier,
optional(
seq('where', $.expression)
),
$.expression,
optional(seq('{', $.item, '}')),
)),
insert: $ => prec.right(seq(

180
tree-sitter-dust/src/grammar.json
View File

@ -662,19 +662,10 @@
"type": "STRING",
"value": "("
},
{
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "identifier"
},
{
"type": "SYMBOL",
"name": "tool"
}
]
},
{
"type": "REPEAT",
"content": {
@ -896,107 +887,6 @@
}
]
},
"tool": {
"type": "CHOICE",
"members": [
{
"type": "STRING",
"value": "assert"
},
{
"type": "STRING",
"value": "assert_equal"
},
{
"type": "STRING",
"value": "output"
},
{
"type": "STRING",
"value": "read"
},
{
"type": "STRING",
"value": "write"
},
{
"type": "STRING",
"value": "raw"
},
{
"type": "STRING",
"value": "sh"
},
{
"type": "STRING",
"value": "bash"
},
{
"type": "STRING",
"value": "fish"
},
{
"type": "STRING",
"value": "zsh"
},
{
"type": "STRING",
"value": "random"
},
{
"type": "STRING",
"value": "random_boolean"
},
{
"type": "STRING",
"value": "random_float"
},
{
"type": "STRING",
"value": "random_string"
},
{
"type": "STRING",
"value": "random_integer"
},
{
"type": "STRING",
"value": "length"
},
{
"type": "STRING",
"value": "sort"
},
{
"type": "STRING",
"value": "transform"
},
{
"type": "STRING",
"value": "filter"
},
{
"type": "STRING",
"value": "to_csv"
},
{
"type": "STRING",
"value": "from_csv"
},
{
"type": "STRING",
"value": "to_json"
},
{
"type": "STRING",
"value": "from_json"
},
{
"type": "STRING",
"value": "help"
}
]
},
"select": {
"type": "PREC_RIGHT",
"value": 0,
@ -1007,17 +897,45 @@
"type": "STRING",
"value": "select"
},
{
"type": "STRING",
"value": "<"
},
{
"type": "REPEAT",
"content": {
"type": "SEQ",
"members": [
{
"type": "SYMBOL",
"name": "identifier"
},
{
"type": "CHOICE",
"members": [
{
"type": "STRING",
"value": ","
},
{
"type": "BLANK"
}
]
}
]
}
},
{
"type": "STRING",
"value": ">"
},
{
"type": "STRING",
"value": "from"
},
{
"type": "SYMBOL",
"name": "identifier"
"name": "expression"
},
{
"type": "CHOICE",
@ -1027,11 +945,15 @@
"members": [
{
"type": "STRING",
"value": "where"
"value": "{"
},
{
"type": "SYMBOL",
"name": "expression"
"name": "item"
},
{
"type": "STRING",
"value": "}"
}
]
},
@ -1115,6 +1037,38 @@
"value": "}"
}
]
},
"assert": {
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "assert"
},
{
"type": "REPEAT",
"content": {
"type": "SYMBOL",
"name": "expression"
}
}
]
},
"assert_equal": {
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "assert"
},
{
"type": "REPEAT",
"content": {
"type": "SYMBOL",
"name": "expression"
}
}
]
}
},
"extras": [

97
tree-sitter-dust/src/node-types.json
View File

@ -220,10 +220,6 @@
{
"type": "identifier",
"named": true
},
{
"type": "tool",
"named": true
}
]
}
@ -443,6 +439,10 @@
{
"type": "identifier",
"named": true
},
{
"type": "item",
"named": true
}
]
}
@ -529,11 +529,6 @@
]
}
},
{
"type": "tool",
"named": true,
"fields": {}
},
{
"type": "transform",
"named": true,
@ -703,18 +698,10 @@
"type": "assert",
"named": false
},
{
"type": "assert_equal",
"named": false
},
{
"type": "async",
"named": false
},
{
"type": "bash",
"named": false
},
{
"type": "else",
"named": false
@ -735,10 +722,6 @@
"type": "find",
"named": false
},
{
"type": "fish",
"named": false
},
{
"type": "float",
"named": true
@ -751,22 +734,10 @@
"type": "from",
"named": false
},
{
"type": "from_csv",
"named": false
},
{
"type": "from_json",
"named": false
},
{
"type": "function",
"named": false
},
{
"type": "help",
"named": false
},
{
"type": "identifier",
"named": true
@ -791,42 +762,6 @@
"type": "into",
"named": false
},
{
"type": "length",
"named": false
},
{
"type": "output",
"named": false
},
{
"type": "random",
"named": false
},
{
"type": "random_boolean",
"named": false
},
{
"type": "random_float",
"named": false
},
{
"type": "random_integer",
"named": false
},
{
"type": "random_string",
"named": false
},
{
"type": "raw",
"named": false
},
{
"type": "read",
"named": false
},
{
"type": "remove",
"named": false
@ -835,14 +770,6 @@
"type": "select",
"named": false
},
{
"type": "sh",
"named": false
},
{
"type": "sort",
"named": false
},
{
"type": "string",
"named": true
@ -851,14 +778,6 @@
"type": "table",
"named": false
},
{
"type": "to_csv",
"named": false
},
{
"type": "to_json",
"named": false
},
{
"type": "transform",
"named": false
@ -875,14 +794,6 @@
"type": "while",
"named": false
},
{
"type": "write",
"named": false
},
{
"type": "zsh",
"named": false
},
{
"type": "{",
"named": false

10970
tree-sitter-dust/src/parser.c
View File

File diff suppressed because it is too large Load Diff