Compare commits

...

7 Commits
main ... 0.3.6

Author SHA1 Message Date
6484b1b307 Merge branch '0.3.6-std' into 0.3.6 2023-11-28 12:07:59 -05:00
2c3c26a0bc Ignore tree sitter's generated files 2023-11-28 12:06:10 -05:00
e5691f8b7e Ignore tree sitter's generated files 2023-11-28 12:04:32 -05:00
32b54c402f Fix example; Write README 2023-11-28 11:38:40 -05:00
69d7b4b1db Fix example 2023-11-28 11:05:54 -05:00
2178c67499 Implement await statements 2023-11-03 21:16:55 -04:00
bc1b88a5fa Begin adding futures 2023-11-03 19:39:34 -04:00
21 changed files with 395 additions and 19713 deletions

1
.gitignore vendored
View File

@ -1,2 +1,3 @@
target/
node_modules/
tree_sitter_dust/src/

View File

@ -28,7 +28,17 @@ if (random_boolean) {
}
```
Dust is an interpreted, strictly typed language with first class functions. It emphasises concurrency by allowing any group of statements to be executed in parallel. Dust includes built-in tooling to import and export data in a variety of formats, including JSON, TOML, YAML and CSV.
Dust enforces strict type checking to make sure your code is correct. Dust does *not* have a null type.
```dust
fib = |i <int>| <int> {
if i <= 1 {
1
} else {
(fib i - 1) + (fib i - 2)
}
}
```
<!--toc:start-->
- [Dust](#dust)
@ -42,9 +52,9 @@ Dust is an interpreted, strictly typed language with first class functions. It e
- [Lists](#lists)
- [Maps](#maps)
- [Loops](#loops)
- [Tables](#tables)
- [Functions](#functions)
- [Concurrency](#concurrency)
- [Acknowledgements](#acknowledgements)
<!--toc:end-->
## Features
@ -140,7 +150,6 @@ Variables have two parts: a key and a value. The key is always a string. The val
- boolean
- list
- map
- table
- function
Here are some examples of variables in dust.
@ -207,56 +216,31 @@ for number in list {
}
```
### Tables
Tables are strict collections, each row must have a value for each column. If a value is "missing" it should be set to an appropriate value for that type. For example, a string can be empty and a number can be set to zero. Dust table declarations consist of a list of column names, which are identifiers enclosed in pointed braces, followed by a list of rows.
An **async for** loop will run the loop operations in parallel using a thread pool.
```dust
animals = table <name species age> [
["rover" "cat" 14]
["spot" "snake" 9]
["bob" "giraffe" 2]
]
```
Querying a table is similar to SQL.
```dust
names = select name from animals
youngins = select species from animals {
age <= 10
async for i in [1 2 3 4 5 6 7 8 9 0] {
(output i)
}
```
The keywords `table` and `insert` make sure that all of the memory used to hold the rows is allocated at once, so it is good practice to group your rows together instead of using a call for each row.
```dust
insert into animals [
["eliza" "ostrich" 4]
["pat" "white rhino" 7]
["jim" "walrus" 9]
]
(assert_equal 6 (length animals))
```
### Functions
Functions are first-class values in dust, so they are assigned to variables like any other value.
```dust
# This simple function has no arguments.
say_hi = || => {
# This simple function has no arguments and no return type.
say_hi = || {
(output "hi")
}
# This function has one argument and will return a value.
add_one = |number| => {
# This function has one argument and will return an integer.
add_one = |number| <int> {
number + 1
}
(say_hi)
(assert_equal (add_one 3), 4)
(assert_equal 4 (add_one 3))
```
You don't need commas when listing arguments and you don't need to add whitespace inside the function body but doing so may make your code easier to read.
@ -274,6 +258,8 @@ async {
}
```
If the final statement in an async block creates a value, the block will return that value just like in a normal block.
```dust
data = async {
(output "Reading a file...")
@ -281,7 +267,7 @@ data = async {
}
```
### Acknowledgements
## Acknowledgements
Dust began as a fork of [evalexpr]. Some of the original code is still in place but the project has dramatically changed and no longer uses any of its parsing or interpreting.

View File

@ -1,6 +1,8 @@
async {
cast = (download "https://api.sampleapis.com/futurama/cast")
characters = (download "https://api.sampleapis.com/futurama/characters")
episodes = (download "https://api.sampleapis.com/futurama/episodes")
}
cast_len = (length (from_json cast))
characters_len = (length (from_json characters))

View File

@ -0,0 +1,53 @@
use rayon::prelude::*;
use serde::{Deserialize, Serialize};
use tree_sitter::Node;
use crate::{AbstractTree, Expression, Map, Result, Value};
#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq, PartialOrd, Ord)]
pub struct Await {
expressions: Vec<Expression>,
}
impl AbstractTree for Await {
fn from_syntax_node(source: &str, node: Node) -> Result<Self> {
debug_assert_eq!("await", node.kind());
let mut expressions = Vec::new();
for index in 2..node.child_count() - 1 {
let child = node.child(index).unwrap();
if child.is_named() {
let expression = Expression::from_syntax_node(source, child)?;
expressions.push(expression);
}
}
Ok(Await { expressions })
}
fn run(&self, source: &str, context: &mut Map) -> Result<Value> {
let expressions = &self.expressions;
expressions
.into_par_iter()
.find_map_first(|expression| {
let mut context = context.clone();
let value = if let Ok(value) = expression.run(source, &mut context) {
value
} else {
return None;
};
let run_result = match value {
Value::Future(block) => block.run(source, &mut context),
_ => return None,
};
Some(run_result)
})
.unwrap_or(Ok(Value::Empty))
}
}

View File

@ -326,11 +326,11 @@ impl AbstractTree for BuiltInFunction {
Value::Map(map) => map.variables()?.len(),
Value::Table(table) => table.len(),
Value::String(string) => string.chars().count(),
Value::Function(_) => todo!(),
Value::Float(_) => todo!(),
Value::Integer(_) => todo!(),
Value::Boolean(_) => todo!(),
Value::Empty => todo!(),
_ => {
return Err(Error::ExpectedCollection {
actual: value.clone(),
});
}
};
Ok(Value::Integer(length as i64))

View File

@ -45,8 +45,9 @@ impl AbstractTree for FunctionCall {
}
fn run(&self, source: &str, context: &mut Map) -> Result<Value> {
let mut function_context = Map::clone_from(context);
let (name, arguments) = match self {
FunctionCall::BuiltIn(function) => return function.run(source, context),
FunctionCall::BuiltIn(function) => return function.run(source, &mut function_context),
FunctionCall::ContextDefined { name, arguments } => (name, arguments),
};

View File

@ -12,8 +12,8 @@ pub struct Select {
impl AbstractTree for Select {
fn from_syntax_node(source: &str, node: Node) -> Result<Self> {
let child_count = node.child_count();
let mut identifiers = Vec::new();
let identifier_list = node.child(1).unwrap();
let identifier_list = node.child(1).unwrap();

View File

@ -107,6 +107,13 @@ impl AbstractTree for ValueNode {
relevant_source: source[child.byte_range()].to_string(),
})
}
_ => return Err(Error::UnexpectedSyntaxNode {
expected:
"string, integer, float, boolean, list, table, map, function, future or empty",
actual: child.kind(),
location: child.start_position(),
relevant_source: source[child.byte_range()].to_string(),
}),
};
Ok(value_node)

View File

@ -30,14 +30,15 @@ pub mod table;
/// value that can be treated as any other.
#[derive(Debug, Clone, Default)]
pub enum Value {
Boolean(bool),
Float(f64),
Function(Function),
Future(Block),
Integer(i64),
List(List),
Map(Map),
Table(Table),
Function(Function),
String(String),
Float(f64),
Integer(i64),
Boolean(bool),
Table(Table),
#[default]
Empty,
}
@ -411,6 +412,8 @@ impl Ord for Value {
(Value::Table(_), _) => Ordering::Greater,
(Value::Function(left), Value::Function(right)) => left.cmp(right),
(Value::Function(_), _) => Ordering::Greater,
(Value::Future(left), Value::Future(right)) => left.cmp(right),
(Value::Future(_), _) => Ordering::Greater,
(Value::Empty, Value::Empty) => Ordering::Equal,
(Value::Empty, _) => Ordering::Less,
}
@ -441,6 +444,7 @@ impl Serialize for Value {
Value::Map(inner) => inner.serialize(serializer),
Value::Table(inner) => inner.serialize(serializer),
Value::Function(inner) => inner.serialize(serializer),
Value::Future(inner) => inner.serialize(serializer),
}
}
}
@ -463,6 +467,7 @@ impl Display for Value {
Value::Map(map) => write!(f, "{map}"),
Value::Table(table) => write!(f, "{table}"),
Value::Function(function) => write!(f, "{function}"),
Value::Future(block) => write!(f, "{block:?}"),
}
}
}

View File

@ -242,6 +242,13 @@ impl From<&Value> for Table {
.insert(vec![Value::Function(function.clone())])
.unwrap();
table
}
Value::Future(block) => {
let mut table = Table::new(vec!["future".to_string()]);
table.insert(vec![Value::Future(block.clone())]).unwrap();
table
}
}

151
src/value/value_type.rs Normal file
View File

@ -0,0 +1,151 @@
use std::{
collections::BTreeMap,
fmt::{self, Debug, Display, Formatter},
};
use serde::{Deserialize, Serialize};
use crate::{value_node::ValueNode, Block, Expression, Function, Identifier, Statement, Value};
/// The type of a `Value`.
#[derive(Clone, Serialize, Deserialize, PartialOrd, Ord)]
pub enum ValueType {
Any,
String,
Float,
Integer,
Boolean,
List(Vec<Expression>),
Empty,
Map(BTreeMap<String, Statement>),
Table {
column_names: Vec<Identifier>,
rows: Box<Expression>,
},
Function(Function),
Future(Block),
}
impl Eq for ValueType {}
impl PartialEq for ValueType {
fn eq(&self, other: &Self) -> bool {
match (self, other) {
(ValueType::Any, _) => true,
(_, ValueType::Any) => true,
(ValueType::String, ValueType::String) => true,
(ValueType::Float, ValueType::Float) => true,
(ValueType::Integer, ValueType::Integer) => true,
(ValueType::Boolean, ValueType::Boolean) => true,
(ValueType::List(left), ValueType::List(right)) => left == right,
(ValueType::Empty, ValueType::Empty) => true,
(ValueType::Map(left), ValueType::Map(right)) => left == right,
(
ValueType::Table {
column_names: left_columns,
rows: left_rows,
},
ValueType::Table {
column_names: right_columns,
rows: right_rows,
},
) => left_columns == right_columns && left_rows == right_rows,
(ValueType::Function(left), ValueType::Function(right)) => left == right,
_ => false,
}
}
}
impl Display for ValueType {
fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
match &self {
ValueType::Any => write!(f, "any"),
ValueType::String => write!(f, "string"),
ValueType::Float => write!(f, "float"),
ValueType::Integer => write!(f, "integer"),
ValueType::Boolean => write!(f, "boolean"),
ValueType::List(list) => {
write!(f, "(")?;
for (index, item) in list.iter().enumerate() {
if index > 0 {
write!(f, ", ")?;
}
write!(f, "{item:?}")?;
}
write!(f, ")")
}
ValueType::Empty => write!(f, "empty"),
ValueType::Map(_map) => write!(f, "map"),
ValueType::Table {
column_names: _,
rows: _,
} => {
write!(f, "table")
}
ValueType::Function(function) => write!(f, "{function}"),
ValueType::Future(_) => write!(f, "future"),
}
}
}
impl Debug for ValueType {
fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
write!(f, "{self}")
}
}
impl From<&Value> for ValueType {
fn from(value: &Value) -> Self {
match value {
Value::String(_) => ValueType::String,
Value::Float(_) => ValueType::Float,
Value::Integer(_) => ValueType::Integer,
Value::Boolean(_) => ValueType::Boolean,
Value::Empty => ValueType::Empty,
Value::List(list) => {
let value_nodes = list
.items()
.iter()
.map(|value| Expression::Value(ValueNode::new(value.value_type(), 0, 0)))
.collect();
ValueType::List(value_nodes)
}
Value::Map(map) => {
let mut value_nodes = BTreeMap::new();
for (key, value) in map.variables().iter() {
let value_type = value.value_type();
let value_node = ValueNode::new(value_type, 0, 0);
let statement = Statement::Expression(Expression::Value(value_node));
value_nodes.insert(key.to_string(), statement);
}
ValueType::Map(value_nodes)
}
Value::Table(table) => ValueType::Table {
column_names: table
.headers()
.iter()
.map(|column_name| Identifier::new(column_name.clone()))
.collect(),
rows: Box::new(Expression::Value(ValueNode::new(
ValueType::List(Vec::with_capacity(0)),
0,
0,
))),
},
Value::Function(function) => ValueType::Function(function.clone()),
Value::Future(block) => ValueType::Future(block.clone()),
}
}
}
impl From<&mut Value> for ValueType {
fn from(value: &mut Value) -> Self {
From::<&Value>::from(value)
}
}

View File

@ -1,20 +1,23 @@
find = |list function| => {
for i in list {
find = |items <list> function <fn>| <any> {
for i in items {
if (function i) {
return i
}
}
}
map = |list function| => {
map = |items <list> function <fn>| <list> {
new_list = []
for i in list {
for i in items {
new_list += (function i)
}
new_list
}
[0 1 2] -> (map |i| => { i - 1})
-> (find |i| => { i == 1 })
foobar <int> = [0 1 2]
-> (map |i <int>| <int> { i - 1 })
-> (find |i <int>| <bool> { i == -1 })
foobar

View File

@ -102,6 +102,13 @@ fn select() {
evaluate(&file_contents).unwrap();
}
#[test]
fn select() {
let file_contents = read_to_string("examples/select.ds").unwrap();
evaluate(&file_contents).unwrap();
}
#[test]
fn table() {
let file_contents = read_to_string("examples/table.ds").unwrap();

View File

@ -103,7 +103,7 @@ Complex Function Call
(foobar
"hi"
42
{
map {
x = 1
y = 2
}

View File

@ -0,0 +1,95 @@
================================================================================
Simple Future
================================================================================
async { (output 'Whaddup') }
--------------------------------------------------------------------------------
(root
(block
(statement
(expression
(value
(future
(block
(statement
(expression
(function_call
(built_in_function
(expression
(value
(string))))))))))))))
================================================================================
Complex Future
================================================================================
async {
if 1 % 2 == 0 {
true
} else {
false
}
'foobar'
}
async { 123 }
'foo'
--------------------------------------------------------------------------------
(root
(block
(statement
(expression
(value
(future
(block
(statement
(if_else
(if
(expression
(logic
(expression
(math
(expression
(value
(integer)))
(math_operator)
(expression
(value
(integer)))))
(logic_operator)
(expression
(value
(integer)))))
(block
(statement
(expression
(value
(boolean))))))
(else
(block
(statement
(expression
(value
(boolean))))))))
(statement
(expression
(value
(string)))))))))
(statement
(expression
(value
(future
(block
(statement
(expression
(value
(integer)))))))))
(statement
(expression
(value
(string))))))

View File

@ -2,7 +2,7 @@
Simple Map
================================================================================
{ answer = 42 }
map { answer = 42 }
--------------------------------------------------------------------------------
@ -21,10 +21,10 @@ Simple Map
Nested Maps
================================================================================
x = {
y = {
x = map {
y = map {
foo = 'bar'
z = {
z = map {
message = 'hiya'
}
}

View File

@ -66,6 +66,7 @@ module.exports = grammar({
$.function,
$.table,
$.map,
$.future,
),
integer: $ => token(prec.left(seq(
@ -91,18 +92,24 @@ module.exports = grammar({
list: $ => seq(
'[',
repeat(prec.left(seq($.expression, optional(',')))),
repeat(prec.right(seq($.expression, optional(',')))),
']',
),
map: $ => seq(
'map',
$.block,
),
future: $ => seq(
'async',
$.block,
),
await: $ => seq(
'await',
'{',
repeat(seq(
$.identifier,
"=",
$.statement,
optional(',')
)),
$._expression_list,
'}',
),
@ -361,4 +368,3 @@ module.exports = grammar({
'reverse',
),
}
});

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@ -1,224 +0,0 @@
#ifndef TREE_SITTER_PARSER_H_
#define TREE_SITTER_PARSER_H_
#ifdef __cplusplus
extern "C" {
#endif
#include <stdbool.h>
#include <stdint.h>
#include <stdlib.h>
#define ts_builtin_sym_error ((TSSymbol)-1)
#define ts_builtin_sym_end 0
#define TREE_SITTER_SERIALIZATION_BUFFER_SIZE 1024
typedef uint16_t TSStateId;
#ifndef TREE_SITTER_API_H_
typedef uint16_t TSSymbol;
typedef uint16_t TSFieldId;
typedef struct TSLanguage TSLanguage;
#endif
typedef struct {
TSFieldId field_id;
uint8_t child_index;
bool inherited;
} TSFieldMapEntry;
typedef struct {
uint16_t index;
uint16_t length;
} TSFieldMapSlice;
typedef struct {
bool visible;
bool named;
bool supertype;
} TSSymbolMetadata;
typedef struct TSLexer TSLexer;
struct TSLexer {
int32_t lookahead;
TSSymbol result_symbol;
void (*advance)(TSLexer *, bool);
void (*mark_end)(TSLexer *);
uint32_t (*get_column)(TSLexer *);
bool (*is_at_included_range_start)(const TSLexer *);
bool (*eof)(const TSLexer *);
};
typedef enum {
TSParseActionTypeShift,
TSParseActionTypeReduce,
TSParseActionTypeAccept,
TSParseActionTypeRecover,
} TSParseActionType;
typedef union {
struct {
uint8_t type;
TSStateId state;
bool extra;
bool repetition;
} shift;
struct {
uint8_t type;
uint8_t child_count;
TSSymbol symbol;
int16_t dynamic_precedence;
uint16_t production_id;
} reduce;
uint8_t type;
} TSParseAction;
typedef struct {
uint16_t lex_state;
uint16_t external_lex_state;
} TSLexMode;
typedef union {
TSParseAction action;
struct {
uint8_t count;
bool reusable;
} entry;
} TSParseActionEntry;
struct TSLanguage {
uint32_t version;
uint32_t symbol_count;
uint32_t alias_count;
uint32_t token_count;
uint32_t external_token_count;
uint32_t state_count;
uint32_t large_state_count;
uint32_t production_id_count;
uint32_t field_count;
uint16_t max_alias_sequence_length;
const uint16_t *parse_table;
const uint16_t *small_parse_table;
const uint32_t *small_parse_table_map;
const TSParseActionEntry *parse_actions;
const char * const *symbol_names;
const char * const *field_names;
const TSFieldMapSlice *field_map_slices;
const TSFieldMapEntry *field_map_entries;
const TSSymbolMetadata *symbol_metadata;
const TSSymbol *public_symbol_map;
const uint16_t *alias_map;
const TSSymbol *alias_sequences;
const TSLexMode *lex_modes;
bool (*lex_fn)(TSLexer *, TSStateId);
bool (*keyword_lex_fn)(TSLexer *, TSStateId);
TSSymbol keyword_capture_token;
struct {
const bool *states;
const TSSymbol *symbol_map;
void *(*create)(void);
void (*destroy)(void *);
bool (*scan)(void *, TSLexer *, const bool *symbol_whitelist);
unsigned (*serialize)(void *, char *);
void (*deserialize)(void *, const char *, unsigned);
} external_scanner;
const TSStateId *primary_state_ids;
};
/*
* Lexer Macros
*/
#define START_LEXER() \
bool result = false; \
bool skip = false; \
bool eof = false; \
int32_t lookahead; \
goto start; \
next_state: \
lexer->advance(lexer, skip); \
start: \
skip = false; \
lookahead = lexer->lookahead;
#define ADVANCE(state_value) \
{ \
state = state_value; \
goto next_state; \
}
#define SKIP(state_value) \
{ \
skip = true; \
state = state_value; \
goto next_state; \
}
#define ACCEPT_TOKEN(symbol_value) \
result = true; \
lexer->result_symbol = symbol_value; \
lexer->mark_end(lexer);
#define END_STATE() return result;
/*
* Parse Table Macros
*/
#define SMALL_STATE(id) id - LARGE_STATE_COUNT
#define STATE(id) id
#define ACTIONS(id) id
#define SHIFT(state_value) \
{{ \
.shift = { \
.type = TSParseActionTypeShift, \
.state = state_value \
} \
}}
#define SHIFT_REPEAT(state_value) \
{{ \
.shift = { \
.type = TSParseActionTypeShift, \
.state = state_value, \
.repetition = true \
} \
}}
#define SHIFT_EXTRA() \
{{ \
.shift = { \
.type = TSParseActionTypeShift, \
.extra = true \
} \
}}
#define REDUCE(symbol_val, child_count_val, ...) \
{{ \
.reduce = { \
.type = TSParseActionTypeReduce, \
.symbol = symbol_val, \
.child_count = child_count_val, \
__VA_ARGS__ \
}, \
}}
#define RECOVER() \
{{ \
.type = TSParseActionTypeRecover \
}}
#define ACCEPT_INPUT() \
{{ \
.type = TSParseActionTypeAccept \
}}
#ifdef __cplusplus
}
#endif
#endif // TREE_SITTER_PARSER_H_