diff --git a/Cargo.lock b/Cargo.lock index 640bfe9..392610e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -468,7 +468,6 @@ dependencies = [ "comfy-table", "csv", "git2", - "json", "rand", "rayon", "reqwest", @@ -909,12 +908,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "json" -version = "0.12.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "078e285eafdfb6c4b434e0d31e8cfcb5115b651496faca5749b88fafd4f23bfd" - [[package]] name = "kv-log-macro" version = "1.0.7" diff --git a/Cargo.toml b/Cargo.toml index 84a469a..312fbc8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,14 +2,19 @@ name = "dust-lang" description = "Data-Oriented Programming Language" version = "0.3.5" -repository = "https://github.com/tree-sitter/tree-sitter-dust" -edition = "2018" +repository = "https://git.jeffa.io/jeff/dust.git" +edition = "2021" license = "MIT" [[bin]] name = "dust" path = "src/main.rs" +[profile.dev] +opt-level = 1 +[profile.dev.package."*"] +opt-level = 3 + [dependencies] ansi_term = "0.12.1" async-std = { version = "1.12.0", features = ["attributes"] } @@ -17,7 +22,6 @@ clap = { version = "4.4.4", features = ["derive"] } comfy-table = "7.0.1" csv = "1.2.2" git2 = "0.18.1" -json = "0.12.4" rand = "0.8.5" rayon = "1.8.0" reqwest = { version = "0.11.20", features = ["blocking", "json"] } diff --git a/README.md b/README.md index 1a7c9db..77a72ed 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Dust -Dust is a programming language and interactive shell. Dust can be used as a replacement for a traditional command line shell, as a scripting language and as a data format. Dust is fast, efficient and easy to learn. +Dust is a general purpose programming language that emphasises concurrency and correctness. A basic dust program: @@ -17,30 +17,53 @@ async { } ``` -Dust is an interpreted, general purpose language with first class functions. It is *data-oriented*, with extensive tools to manage structured and relational data. Dust also includes built-in tooling to import and export data in a variety of formats, including JSON, TOML, YAML and CSV. +You can make *any* block, i.e. `{}`, run its statements in parallel by changing it to `async {}`. + +```dust +if (random_boolean) { + (output "Do something...") +} else async { + (output "Do something else instead...") + (output "And another thing at the same time...") +} +``` + +Dust enforces strict type checking to make sure your code is correct. Dust does *not* have a null type. + +```dust +fib = |i | { + if i <= 1 { + 1 + } else { + (fib i - 1) + (fib i - 2) + } +} +``` - [Dust](#dust) - [Features](#features) - [Usage](#usage) - [Installation](#installation) + - [Benchmarks](#benchmarks) + - [Implementation](#implementation) - [The Dust Programming Language](#the-dust-programming-language) - [Declaring Variables](#declaring-variables) - [Lists](#lists) - [Maps](#maps) - - [Tables](#tables) + - [Loops](#loops) - [Functions](#functions) - [Concurrency](#concurrency) - - [Implementation](#implementation) + - [Acknowledgements](#acknowledgements) ## Features - Simplicity: Dust is designed to be easy to learn. -- Speed: Dust is built on [Tree Sitter] and [Rust] to prioritize performance and correctness. -- Data format: Dust is data-oriented, making it a great language for defining data. -- Format conversion: Effortlessly convert between dust and formats like JSON, CSV and TOML. -- Structured data: Dust can represent data with more than just strings. Lists, maps and tables are easy to make and manage. +- Speed: Dust is built on [Tree Sitter] and [Rust] to prioritize performance and correctness. See [Benchmarks] below. +- Concurrency: Easily and safely write code that runs in parallel. +- Safety: Written in safe, stable Rust. +- Correctness: Type checking makes it easy to write good code that works. ## Usage @@ -58,6 +81,61 @@ You must have the default rust toolchain installed and up-to-date. Install [rust To build from source, clone the repository and build the parser. To do so, enter the `tree-sitter-dust` directory and run `tree-sitter-generate`. In the project root, run `cargo run` to start the shell. To see other command line options, use `cargo run -- --help`. +## Benchmarks + +Dust is at a very early development stage but performs strongly in preliminary benchmarks. The examples given were tested using [Hyperfine] on a single-core cloud instance with 1024 MB RAM. Each test was run 1000 times. The test script is shown below. Each test asks the program to read a JSON file and count the objects. Dust is a command line shell, programming language and data manipulation tool so three appropriate targets were chosen for comparison: nushell, NodeJS and jq. The programs produced identical output with the exception that NodeJS printed in color. + +For the first test, a file with four entries was used. + +| Command | Mean [ms] | Min [ms] | Max [ms] +|:---|---:|---:|---:| +| Dust | 3.1 ± 0.5 | 2.4 | 8.4 | +| jq | 33.7 ± 2.2 | 30.0 | 61.8 | +| NodeJS | 226.4 ± 13.1 | 197.6 | 346.2 | +| Nushell | 51.6 ± 3.7 | 45.4 | 104.3 | + +The second set of data is from the GitHub API, it consists of 100 commits from the jq GitHub repo. + +| Command | Mean [ms] | Min [ms] | Max [ms] | +|:---|---:|---:|---:| +| Dust | 6.8 ± 0.6 | 5.7 | 12.0 | 2.20 ± 0.40 | +| jq | 43.3 ± 3.6 | 37.6 | 81.6 | 13.95 ± 2.49 | +| NodeJS | 224.9 ± 12.3 | 194.8 | 298.5 | +| Nushell | 59.2 ± 5.7 | 49.7 | 125.0 | 19.11 ± 3.55 | + +This data came from CERN, it is a massive file of 100,000 entries. + +| Command | Mean [ms] | Min [ms] | Max [ms] | +|:---|---:|---:|---:| +| Dust | 1080.8 ± 38.7 | 975.3 | 1326.6 | +| jq | 1305.3 ± 64.3 | 1159.7 | 1925.1 | +| NodeJS | 1850.5 ± 72.5 | 1641.9 | 2395.1 | +| Nushell | 1850.5 ± 86.2 | 1625.5 | 2400.7 | + +The tests were run after 5 warmup runs and the cache was cleared before each run. + +```sh +hyperfine \ + --shell none \ + --warmup 5 \ + --prepare "rm -rf /root/.cache" \ + --runs 1000 \ + --parameter-list data_path seaCreatures.json,jq_data.json,dielectron.json \ + --export-markdown test_output.md \ + "dust -c '(length (from_json input))' -p {data_path}" \ + "jq 'length' {data_path}" \ + "node --eval \"require('node:fs').readFile('{data_path}',(err,data)=>{console.log(JSON.parse(data).length)})\"" \ + "nu -c 'open {data_path} | length'" +``` + +## Implementation + +Dust is formally defined as a Tree Sitter grammar in the tree-sitter-dust directory. Tree sitter generates a parser, written in C, from a set of rules defined in JavaScript. Dust itself is a rust binary that calls the C parser using FFI. + +Tests are written in three places: in the Rust library, in Dust as examples and in the Tree Sitter test format. Generally, features are added by implementing and testing the syntax in the tree-sitter-dust repository, then writing library tests to evaluate the new syntax. Implementation tests run the Dust files in the "examples" directory and should be used to demonstrate and verify that features work together. + +Tree Sitter generates a concrete syntax tree, which Dust traverses to create an abstract syntax tree that can run the Dust code. The CST generation is an extra step but it allows easy testing of the parser, defining the language in one file and makes the syntax easy to modify and expand. Because it uses Tree Sitter, developer-friendly features like syntax highlighting and code navigation are already available in any text editor that supports Tree Sitter. + ## The Dust Programming Language Dust is easy to learn. Aside from this guide, the best way to learn Dust is to read the examples and tests to get a better idea of what it can do. @@ -72,7 +150,6 @@ Variables have two parts: a key and a value. The key is always a string. The val - boolean - list - map -- table - function Here are some examples of variables in dust. @@ -91,14 +168,14 @@ Note that strings can be wrapped with any kind of quote: single, double or backt ### Lists -Lists are sequential collections. They can be built by grouping values with square brackets. Commas are optional. Values can be indexed by their position using dot notation with an integer. Dust lists are zero-indexed. +Lists are sequential collections. They can be built by grouping values with square brackets. Commas are optional. Values can be indexed by their position using a colon `:` followed by an integer. Dust lists are zero-indexed. ```dust list = [true 41 "Ok"] -(assert_equal list.0 true) +(assert_equal list:0 true) -the_answer = list.1 + 1 +the_answer = list:1 + 1 (assert_equal the_answer, 42) # You can also use commas when passing values to # a function. @@ -106,7 +183,7 @@ the_answer = list.1 + 1 ### Maps -Maps are flexible collections with arbitrary key-value pairs, similar to JSON objects. A map is created with a pair of curly braces and its entries are variables declared inside those braces. Map contents can be accessed using dot notation. +Maps are flexible collections with arbitrary key-value pairs, similar to JSON objects. A map is created with a pair of curly braces and its entries are variables declared inside those braces. Map contents can be accessed using a colon `:`. ```dust reminder = { @@ -114,7 +191,7 @@ reminder = { tags = ["groceries", "home"] } -(output reminder.message) +(output reminder:message) ``` ### Loops @@ -137,111 +214,43 @@ list = [ 1, 2, 3 ] for number in list { (output number + 1) } - -# The original list is left unchanged. ``` -To create a new list, use a **transform** loop, which modifies the values into a new list without changing the original. +An **async for** loop will run the loop operations in parallel using a thread pool. ```dust -list = [1 2 3] - -new_list = transform number in list { - number - 1 +async for i in [1 2 3 4 5 6 7 8 9 0] { + (output i) } - -(output new_list) -# Output: [ 0 1 2 ] - -(output list) -# Output: [ 1 2 3 ] -``` - -To filter out some of the values in a list, use a **filter** loop. - -```dust -list = filter number in [1 2 3] { - number >= 2 -} - -(output list) -# Output: [ 2 3 ] -``` - -A **find** loop will return a single value, the first item that satisfies the predicate. - -```dust -found = find number in [1 2 1] { - number != 1 -} - -(output found) -# Output: 2 -``` - -### Tables - -Tables are strict collections, each row must have a value for each column. If a value is "missing" it should be set to an appropriate value for that type. For example, a string can be empty and a number can be set to zero. Dust table declarations consist of a list of column names, which are identifiers enclosed in pointed braces, followed by a list of rows. - -```dust -animals = table [ - ["rover" "cat" 14] - ["spot" "snake" 9] - ["bob" "giraffe" 2] -] -``` - -Querying a table is similar to SQL. - -```dust -names = select name from animals -youngins = select species from animals { - age <= 10 -} -``` - -The keywords `table` and `insert` make sure that all of the memory used to hold the rows is allocated at once, so it is good practice to group your rows together instead of using a call for each row. - -```dust -insert into animals [ - ["eliza" "ostrich" 4] - ["pat" "white rhino" 7] - ["jim" "walrus" 9] -] - -(assert_equal 6 (length animals)) ``` ### Functions -Functions are first-class values in dust, so they are assigned to variables like any other value. The function body is wrapped in single parentheses. To create a function, use the "function" keyword. The function's arguments are identifiers inside of a set of pointed braces and the function body is enclosed in curly braces. To call a fuction, invoke its variable name inside a set of parentheses. You don't need commas when listing arguments and you don't need to add whitespace inside the function body but doing so may make your code easier to read. +Functions are first-class values in dust, so they are assigned to variables like any other value. ```dust -say_hi = function <> { +# This simple function has no arguments and no return type. +say_hi = || { (output "hi") } -add_one = function { - (number + 1) +# This function has one argument and will return an integer. +add_one = |number| { + number + 1 } (say_hi) -(assert_equal (add_one 3), 4) +(assert_equal 4 (add_one 3)) ``` -This function simply passes the input to the shell's standard output. - -```dust -print = function { - (output input) -} -``` +You don't need commas when listing arguments and you don't need to add whitespace inside the function body but doing so may make your code easier to read. ### Concurrency -As a language written in Rust, Dust features effortless concurrency anywhere in your code. +Dust features effortless concurrency anywhere in your code. Any block of code can be made to run its contents asynchronously. Dust's concurrency is written in safe Rust and uses a thread pool whose size depends on the number of cores available. ```dust +# An async block will run each statement in its own thread. async { (output (random_integer)) (output (random_float)) @@ -249,7 +258,7 @@ async { } ``` -In an **async** block, each statement is run in parallel. In this case, we want to read from a file and assign the data to a variable. It doesn't matter which statement finishes first, the last statement in the block will be used as the assigned value. If one of the statements in an **async** block produces an error, the other statements will stop running if they have not already finished. +If the final statement in an async block creates a value, the block will return that value just like in a normal block. ```dust data = async { @@ -258,16 +267,12 @@ data = async { } ``` -## Implementation +## Acknowledgements -Dust is formally defined as a Tree Sitter grammar in the tree-sitter-dust directory. Tree sitter generates a parser, written in C, from a set of rules defined in Javascript. Dust itself is a rust binary that calls the C parser using FFI. - -Tests are written in three places: in the Rust library, in Dust as examples and in the Tree Sitter test format. Generally, features are added by implementing and testing the syntax in the tree-sitter-dust repository, then writing library tests to evaluate the new syntax. Implementation tests run the Dust files in the "examples" directory and should be used to demonstrate and verify that features work together. - -Tree Sitter generates a concrete syntax tree, which Dust traverses to create an abstract syntax tree that can run the Dust code. The CST generation is an extra step but it allows easy testing of the parser, defining the language in one file and makes the syntax easy to modify and expand. Because it uses Tree Sitter, developer-friendly features like syntax highlighting and code navigation are already available in any text editor that supports Tree Sitter. +Dust began as a fork of [evalexpr]. Some of the original code is still in place but the project has dramatically changed and no longer uses any of its parsing or interpreting. [Tree Sitter]: https://tree-sitter.github.io/tree-sitter/ [Rust]: https://rust-lang.org -[dnf]: https://dnf.readthedocs.io/en/latest/index.html [evalexpr]: https://github.com/ISibboI/evalexpr [rustup]: https://rustup.rs +[Hyperfine]: https://github.com/sharkdp/hyperfine diff --git a/examples/async.ds b/examples/async.ds index a0ac0d2..5e66524 100644 --- a/examples/async.ds +++ b/examples/async.ds @@ -1,33 +1,19 @@ (output "This will print first.") -(output "This will print second.") -create_random_numbers = |count| => { +create_random_numbers = |count | { numbers = []; while (length numbers) < count { numbers += (random_integer) } + + (output "Made " + count + " numbers.") } -do_a_lot = async { +async { (create_random_numbers 1000) - (output "Made 1000 numbers") -} - -do_some = async { - (create_random_numbers 100) - (output "Made 100 numbers") -} - -do_a_little = async { + (create_random_numbers 100) (create_random_numbers 10) - (output "Made 10 numbers") -} - -await { - do_a_lot - do_some - do_a_little } (output "This will print last.") diff --git a/examples/async_download.ds b/examples/async_download.ds new file mode 100644 index 0000000..88e6d9a --- /dev/null +++ b/examples/async_download.ds @@ -0,0 +1,11 @@ +async { + cast = (download "https://api.sampleapis.com/futurama/cast") + characters = (download "https://api.sampleapis.com/futurama/characters") + episodes = (download "https://api.sampleapis.com/futurama/episodes") +} + +cast_len = (length (from_json cast)) +characters_len = (length (from_json characters)) +episodes_len = (length (from_json episodes)) + +(output [cast_len, characters_len, episodes_len]) diff --git a/examples/clue_solver.ds b/examples/clue_solver.ds index 961db40..301cc02 100644 --- a/examples/clue_solver.ds +++ b/examples/clue_solver.ds @@ -1,44 +1,51 @@ -rooms = ['Library' 'Kitchen'] -suspects = ['White' 'Green'] -weapons = ['Rope' 'Lead_Pipe'] -cards = [rooms suspects weapons] - -take_turn = |current_room opponent_card| => { - (remove_card opponent_card) - (make_guess current_room) -} - -remove_card = |opponent_card| => { - for card_list in cards { - removed = remove card from card_list - card == opponent_card - } - - if (type removed) == 'empty' - output 'Card not found.' +all_cards = { + rooms = ['Library' 'Kitchen' 'Conservatory'] + suspects = ['White' 'Green' 'Scarlett'] + weapons = ['Rope' 'Lead_Pipe' 'Knife'] } -make_guess = |current_room| => { - if ((length suspects) == 1) - && ((length rooms) == 1) - && ((length weapons) == 1) - { +is_ready_to_solve = |cards | { + ((length cards:suspects) == 1) + && ((length cards:rooms) == 1) + && ((length cards:weapons) == 1) +} + +take_turn = |opponent_card , current_room , cards | { + (remove_card opponent_card cards) + (make_guess current_room cards) + cards +} + +remove_card = |opponent_card , cards | { + cards:rooms -= opponent_card + cards:suspects -= opponent_card + cards:weapons -= opponent_card +} + +make_guess = |current_room , cards | { + if (is_ready_to_solve cards) { (output 'It was ' - + suspects:0 - + ' in the ' - + rooms:0 - + ' with the ' - + weapons:0 + + cards:suspects:0 + + ' in the ' + + cards:rooms:0 + + ' with the ' + + cards:weapons:0 + '!') } else { (output 'I accuse ' - + (random suspects) + + (random cards:suspects) + ' in the ' + current_room + ' with the ' - + (random weapons) - + '!') + + (random cards:weapons) + + '.') } } -(make_guess 'Library') +(take_turn 'Rope' 'Kitchen' + (take_turn 'Library' 'Kitchen' + (take_turn 'Conservatory' 'Kitchen' + (take_turn 'White' 'Kitchen' + (take_turn 'Green' 'Kitchen' + (take_turn 'Knife' 'Kitchen' all_cards)))))) + diff --git a/examples/download.ds b/examples/download.ds new file mode 100644 index 0000000..983dbf4 --- /dev/null +++ b/examples/download.ds @@ -0,0 +1,9 @@ +cast = (download "https://api.sampleapis.com/futurama/cast") +characters = (download "https://api.sampleapis.com/futurama/characters") +episodes = (download "https://api.sampleapis.com/futurama/episodes") + +cast_len = (length (from_json cast)) +characters_len = (length (from_json characters)) +episodes_len = (length (from_json episodes)) + +(output [cast_len, characters_len, episodes_len]) diff --git a/examples/fannkuch-redux.ds b/examples/fannkuch-redux.ds deleted file mode 100644 index 92bd81a..0000000 --- a/examples/fannkuch-redux.ds +++ /dev/null @@ -1,21 +0,0 @@ -numbers = (from_json input) -flip_count = 0 -checksum = 0 - -while numbers.0 != 1 { - (reverse numbers 0 numbers.0) - - if flip_count % 2 == 0 { - checksum += flip_count - } else { - checksum -= flip_count - } - - checksum += flip_count * 1 - - flip_count += 1 -} - -(output numbers) -(output flip_count) -(output checksum) diff --git a/examples/fetch.ds b/examples/fetch.ds index b5ebd57..89e656f 100644 --- a/examples/fetch.ds +++ b/examples/fetch.ds @@ -1,8 +1,10 @@ raw_data = (download "https://api.sampleapis.com/futurama/cast") cast_data = (from_json raw_data) -names = transform cast_member in cast_data { - cast_member.name +names = [] + +for cast_member in cast_data { + names += cast_member:name } -(assert_equal "Billy West", names.0) +(assert_equal "Billy West", names:0) diff --git a/examples/fibonacci.ds b/examples/fibonacci.ds index ad5b0b5..62948fd 100644 --- a/examples/fibonacci.ds +++ b/examples/fibonacci.ds @@ -1,4 +1,4 @@ -fib = |i| => { +fib = |i | { if i <= 1 { 1 } else { diff --git a/examples/filter_loop.ds b/examples/filter_loop.ds deleted file mode 100644 index 8b83fb3..0000000 --- a/examples/filter_loop.ds +++ /dev/null @@ -1,3 +0,0 @@ -filter item in (from_json (read 'examples/assets/jq_data.json')) { - (length item.commit.committer.name) <= 10 -} diff --git a/examples/find_loop.ds b/examples/find_loop.ds deleted file mode 100644 index f13733b..0000000 --- a/examples/find_loop.ds +++ /dev/null @@ -1,7 +0,0 @@ -list = [1 2 1 3] - -found = find i in list { - i == 3 -} - -(assert_equal 3 found) diff --git a/examples/fizzbuzz.ds b/examples/fizzbuzz.ds index a007cd4..f2a7c6a 100644 --- a/examples/fizzbuzz.ds +++ b/examples/fizzbuzz.ds @@ -5,13 +5,13 @@ while count <= 15 { divides_by_5 = count % 5 == 0 if divides_by_3 && divides_by_5 { - output 'fizzbuzz' + (output 'fizzbuzz') } else if divides_by_3 { - output 'fizz' + (output 'fizz') } else if divides_by_5 { - output 'buzz' + (output 'buzz') } else { - output count + (output count) } count += 1 diff --git a/examples/jq_data.ds b/examples/jq_data.ds index f9c2586..6c68f83 100644 --- a/examples/jq_data.ds +++ b/examples/jq_data.ds @@ -1,8 +1,12 @@ data = (from_json (read 'examples/assets/jq_data.json')) -transform commit_data in data { - { +new_data = []; + +for commit_data in data { + new_data += { message = commit_data.commit.message name = commit_data.commit.committer.name } } + +new_data diff --git a/examples/list.ds b/examples/list.ds index 65b80a1..c469691 100644 --- a/examples/list.ds +++ b/examples/list.ds @@ -1,7 +1,7 @@ numbers = [1, 2, 3] -x = numbers.{0} -y = numbers.{1} -z = numbers.{2} +x = numbers:0 +y = numbers:1 +z = numbers:2 (assert_equal x + y, z) diff --git a/examples/map.ds b/examples/map.ds index eb9c7a5..f759ce6 100644 --- a/examples/map.ds +++ b/examples/map.ds @@ -5,7 +5,7 @@ dictionary = { (output 'Dust is ' - + dictionary.dust + + dictionary:dust + '! The answer is ' - + dictionary.answer + + dictionary:answer ) diff --git a/examples/remove_loop.ds b/examples/remove_loop.ds deleted file mode 100644 index 9362dc9..0000000 --- a/examples/remove_loop.ds +++ /dev/null @@ -1,8 +0,0 @@ -list = [1 2 1 3] - -removed = remove i from list { - i == 3 -} - -(assert_equal 3 removed) -(assert_equal [1 2 1] list) diff --git a/examples/sea_creatures.ds b/examples/sea_creatures.ds index 6db82df..86a8dac 100644 --- a/examples/sea_creatures.ds +++ b/examples/sea_creatures.ds @@ -8,11 +8,11 @@ data = { } for creature in sea_creatures { - data.creatures += creature.name - data.total_clams += creature.clams + data:creatures += creature:name + data:total_clams += creature:clams - if creature.type == 'dolphin' { - data.dolphin_clams += creature.clams + if creature:type == 'dolphin' { + data:dolphin_clams += creature:clams } } diff --git a/examples/table.ds b/examples/table.ds index 1c2157b..4719b3e 100644 --- a/examples/table.ds +++ b/examples/table.ds @@ -10,11 +10,11 @@ test_table = table |text bool| [ ["a", true] ] -test_select = select |text bool| from my_table +test_select = select |text bool| from my_table; (assert_equal test_select, test_table) -test_table = table |text number bool| [ +test_table = table |number bool| [ [1, true] [3, true] ] diff --git a/examples/transform_loop.ds b/examples/transform_loop.ds deleted file mode 100644 index 00cdfa9..0000000 --- a/examples/transform_loop.ds +++ /dev/null @@ -1,5 +0,0 @@ -data = (from_json (read "examples/assets/jq_data.json")) - -transform item in data { - item:commit:committer:name -} diff --git a/examples/yield.ds b/examples/yield.ds new file mode 100644 index 0000000..8724be3 --- /dev/null +++ b/examples/yield.ds @@ -0,0 +1,15 @@ +1 -> (output) + +add_one = |numbers | { + new_numbers = [] + + for number in numbers { + new_numbers += number + 1 + } + + new_numbers +} + +foo = [1, 2, 3] -> (add_one) + +(assert_equal [2 3 4] foo) diff --git a/src/abstract_tree/assignment.rs b/src/abstract_tree/assignment.rs index b23fcd2..4bd2699 100644 --- a/src/abstract_tree/assignment.rs +++ b/src/abstract_tree/assignment.rs @@ -1,11 +1,12 @@ use serde::{Deserialize, Serialize}; use tree_sitter::Node; -use crate::{AbstractTree, Error, Identifier, Map, Result, Statement, Value}; +use crate::{AbstractTree, Error, Identifier, Map, Result, Statement, Type, Value}; #[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq, PartialOrd, Ord)] pub struct Assignment { identifier: Identifier, + r#type: Option, operator: AssignmentOperator, statement: Statement, } @@ -19,10 +20,23 @@ pub enum AssignmentOperator { impl AbstractTree for Assignment { fn from_syntax_node(source: &str, node: Node) -> Result { - let identifier_node = node.child(0).unwrap(); + Error::expect_syntax_node(source, "assignment", node)?; + + let identifier_node = node.child_by_field_name("identifier").unwrap(); let identifier = Identifier::from_syntax_node(source, identifier_node)?; - let operator_node = node.child(1).unwrap().child(0).unwrap(); + let type_node = node.child_by_field_name("type"); + let r#type = if let Some(type_node) = type_node { + Some(Type::from_syntax_node(source, type_node)?) + } else { + None + }; + + let operator_node = node + .child_by_field_name("assignment_operator") + .unwrap() + .child(0) + .unwrap(); let operator = match operator_node.kind() { "=" => AssignmentOperator::Equal, "+=" => AssignmentOperator::PlusEqual, @@ -37,42 +51,46 @@ impl AbstractTree for Assignment { } }; - let statement_node = node.child(2).unwrap(); + let statement_node = node.child_by_field_name("statement").unwrap(); let statement = Statement::from_syntax_node(source, statement_node)?; Ok(Assignment { identifier, + r#type, operator, statement, }) } fn run(&self, source: &str, context: &mut Map) -> Result { - let key = self.identifier.inner().clone(); + let key = self.identifier.inner(); let value = self.statement.run(source, context)?; - let mut context = context.variables_mut(); let new_value = match self.operator { AssignmentOperator::PlusEqual => { - if let Some(mut previous_value) = context.get(&key).cloned() { + if let Some(mut previous_value) = context.variables()?.get(key).cloned() { previous_value += value; previous_value } else { - Value::Empty + return Err(Error::VariableIdentifierNotFound(key.clone())); } } AssignmentOperator::MinusEqual => { - if let Some(mut previous_value) = context.get(&key).cloned() { + if let Some(mut previous_value) = context.variables()?.get(key).cloned() { previous_value -= value; previous_value } else { - Value::Empty + return Err(Error::VariableIdentifierNotFound(key.clone())); } } AssignmentOperator::Equal => value, }; - context.insert(key, new_value); + if let Some(r#type) = &self.r#type { + r#type.check(&new_value)?; + } + + context.variables_mut()?.insert(key.clone(), new_value); Ok(Value::Empty) } diff --git a/src/abstract_tree/block.rs b/src/abstract_tree/block.rs index 8af949d..d050e05 100644 --- a/src/abstract_tree/block.rs +++ b/src/abstract_tree/block.rs @@ -1,40 +1,84 @@ +use std::sync::RwLock; + +use rayon::prelude::*; use serde::{Deserialize, Serialize}; use tree_sitter::Node; -use crate::{AbstractTree, Result, Statement}; +use crate::{AbstractTree, Error, Map, Result, Statement, Value}; #[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq, PartialOrd, Ord)] pub struct Block { + is_async: bool, statements: Vec, } impl AbstractTree for Block { fn from_syntax_node(source: &str, node: Node) -> Result { - debug_assert_eq!("block", node.kind()); + Error::expect_syntax_node(source, "block", node)?; - let statement_count = node.child_count(); + let first_child = node.child(0).unwrap(); + let is_async = first_child.kind() == "async"; + + let statement_count = if is_async { + node.child_count() - 3 + } else { + node.child_count() - 2 + }; let mut statements = Vec::with_capacity(statement_count); - for index in 0..statement_count { + for index in 1..node.child_count() - 1 { let child_node = node.child(index).unwrap(); - if child_node.kind() == "statement" { + if child_node.is_named() { let statement = Statement::from_syntax_node(source, child_node)?; statements.push(statement); } } - Ok(Block { statements }) + Ok(Block { + is_async, + statements, + }) } - fn run(&self, source: &str, context: &mut crate::Map) -> crate::Result { - for statement in &self.statements[0..self.statements.len() - 1] { - statement.run(source, context)?; + fn run(&self, source: &str, context: &mut Map) -> Result { + if self.is_async { + let statements = &self.statements; + let final_result = RwLock::new(Ok(Value::Empty)); + + statements + .into_par_iter() + .enumerate() + .find_map_first(|(index, statement)| { + if let Statement::Return(expression) = statement { + return Some(expression.run(source, &mut context.clone())); + } + + let result = statement.run(source, &mut context.clone()); + + if result.is_err() { + Some(result) + } else if index == statements.len() - 1 { + let _ = final_result.write().unwrap().as_mut().map(|_| result); + + None + } else { + None + } + }) + .unwrap_or(final_result.into_inner().unwrap()) + } else { + let mut prev_result = None; + + for statement in &self.statements { + if let Statement::Return(expression) = statement { + return expression.run(source, context); + } + + prev_result = Some(statement.run(source, context)); + } + + prev_result.unwrap_or(Ok(Value::Empty)) } - - let final_statement = self.statements.last().unwrap(); - let final_value = final_statement.run(source, context)?; - - Ok(final_value) } } diff --git a/src/abstract_tree/built_in_function.rs b/src/abstract_tree/built_in_function.rs index d4532bf..962e42c 100644 --- a/src/abstract_tree/built_in_function.rs +++ b/src/abstract_tree/built_in_function.rs @@ -11,7 +11,7 @@ use reqwest::blocking::get; use serde::{Deserialize, Serialize}; use tree_sitter::Node; -use crate::{AbstractTree, Error, Expression, List, Map, Result, Table, Value, ValueType}; +use crate::{AbstractTree, Error, Expression, List, Map, Result, Table, Type, Value}; #[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq, PartialOrd, Ord)] pub enum BuiltInFunction { @@ -19,6 +19,7 @@ pub enum BuiltInFunction { Assert(Vec), AssertEqual(Vec), Download(Expression), + Context, Help(Option), Length(Expression), Output(Vec), @@ -30,7 +31,7 @@ pub enum BuiltInFunction { Append(Vec), Metadata(Expression), Move(Vec), - Read(Expression), + Read(Option), Remove(Expression), Write(Vec), @@ -93,6 +94,7 @@ impl AbstractTree for BuiltInFunction { BuiltInFunction::AssertEqual(expressions) } + "context" => BuiltInFunction::Context, "download" => { let expression_node = node.child(1).unwrap(); let expression = Expression::from_syntax_node(source, expression_node)?; @@ -153,8 +155,11 @@ impl AbstractTree for BuiltInFunction { BuiltInFunction::Move(expressions) } "read" => { - let expression_node = node.child(1).unwrap(); - let expression = Expression::from_syntax_node(source, expression_node)?; + let expression = if let Some(node) = node.child(1) { + Some(Expression::from_syntax_node(source, node)?) + } else { + None + }; BuiltInFunction::Read(expression) } @@ -306,6 +311,7 @@ impl AbstractTree for BuiltInFunction { Ok(Value::Empty) } + BuiltInFunction::Context => Ok(Value::Map(context.clone())), BuiltInFunction::Download(expression) => { let value = expression.run(source, context)?; let url = value.as_string()?; @@ -317,7 +323,7 @@ impl AbstractTree for BuiltInFunction { let value = expression.run(source, context)?; let length = match value { Value::List(list) => list.items().len(), - Value::Map(map) => map.len(), + Value::Map(map) => map.variables()?.len(), Value::Table(table) => table.len(), Value::String(string) => string.chars().count(), _ => { @@ -361,9 +367,9 @@ impl AbstractTree for BuiltInFunction { BuiltInFunction::Type(expression) => { let run_expression = expression.run(source, context); let value_type = if let Ok(value) = run_expression { - value.value_type() + value.r#type() } else if let Err(Error::VariableIdentifierNotFound(_)) = run_expression { - ValueType::Empty + Type::Any } else { return run_expression; }; @@ -405,7 +411,7 @@ impl AbstractTree for BuiltInFunction { let metadata_output = Map::new(); { - let mut metadata_variables = metadata_output.variables_mut(); + let mut metadata_variables = metadata_output.variables_mut()?; metadata_variables.insert("type".to_string(), Value::String(file_type)); metadata_variables.insert("size".to_string(), Value::Integer(size)); @@ -428,8 +434,13 @@ impl AbstractTree for BuiltInFunction { Ok(Value::Empty) } BuiltInFunction::Read(expression) => { - let path_value = expression.run(source, context)?; - let path = PathBuf::from(path_value.as_string()?); + let path = if let Some(expression) = expression { + let path_value = expression.run(source, context)?; + + PathBuf::from(path_value.as_string()?) + } else { + PathBuf::from(".") + }; let content = if path.is_dir() { let dir = read_dir(&path)?; let mut contents = Vec::new(); @@ -572,11 +583,8 @@ impl AbstractTree for BuiltInFunction { let value = expressions[0].run(source, context)?; let list = value.as_list()?.items(); - if list.len() < 2 { - return Err(Error::ExpectedMinLengthList { - minimum_len: 2, - actual_len: list.len(), - }); + if list.len() == 1 { + return Ok(list.first().cloned().unwrap()); } let range = 0..list.len(); diff --git a/src/abstract_tree/expression.rs b/src/abstract_tree/expression.rs index fe3ef88..2c5bbe4 100644 --- a/src/abstract_tree/expression.rs +++ b/src/abstract_tree/expression.rs @@ -3,7 +3,7 @@ use tree_sitter::Node; use crate::{ value_node::ValueNode, AbstractTree, BuiltInFunction, Error, Identifier, Index, Map, Result, - Sublist, Value, + Value, Yield, }; use super::{function_call::FunctionCall, logic::Logic, math::Math}; @@ -12,63 +12,58 @@ use super::{function_call::FunctionCall, logic::Logic, math::Math}; pub enum Expression { Value(ValueNode), Identifier(Identifier), - Sublist(Box), Index(Box), Math(Box), Logic(Box), - FunctionCall(FunctionCall), + FunctionCall(Box), Tool(Box), + Yield(Box), } impl AbstractTree for Expression { fn from_syntax_node(source: &str, node: Node) -> Result { - debug_assert_eq!("expression", node.kind()); + Error::expect_syntax_node(source, "expression", node)?; - for index in 0..node.child_count() { - let child = node.child(index).unwrap(); - let expression = match child.kind() { - "value" => Expression::Value(ValueNode::from_syntax_node(source, child)?), - "identifier" => { - Expression::Identifier(Identifier::from_syntax_node(source, child)?) - } - "sublist" => { - Expression::Sublist(Box::new(Sublist::from_syntax_node(source, child)?)) - } - "index" => Expression::Index(Box::new(Index::from_syntax_node(source, child)?)), - "math" => Expression::Math(Box::new(Math::from_syntax_node(source, child)?)), - "logic" => Expression::Logic(Box::new(Logic::from_syntax_node(source, child)?)), - "function_call" => { - Expression::FunctionCall(FunctionCall::from_syntax_node(source, child)?) - } - "tool" => { - Expression::Tool(Box::new(BuiltInFunction::from_syntax_node(source, child)?)) - } - _ => continue, - }; + let child = if node.child(0).unwrap().is_named() { + node.child(0).unwrap() + } else { + node.child(1).unwrap() + }; - return Ok(expression); - } + let expression = match child.kind() { + "value" => Expression::Value(ValueNode::from_syntax_node(source, child)?), + "identifier" => Expression::Identifier(Identifier::from_syntax_node(source, child)?), + "index" => Expression::Index(Box::new(Index::from_syntax_node(source, child)?)), + "math" => Expression::Math(Box::new(Math::from_syntax_node(source, child)?)), + "logic" => Expression::Logic(Box::new(Logic::from_syntax_node(source, child)?)), + "function_call" => { + Expression::FunctionCall(Box::new(FunctionCall::from_syntax_node(source, child)?)) + } + "tool" => Expression::Tool(Box::new(BuiltInFunction::from_syntax_node(source, child)?)), + "yield" => Expression::Yield(Box::new(Yield::from_syntax_node(source, child)?)), + _ => { + return Err(Error::UnexpectedSyntaxNode { + expected: "value, identifier, index, math, logic, function_call or yield", + actual: child.kind(), + location: child.start_position(), + relevant_source: source[child.byte_range()].to_string(), + }) + } + }; - let child = node.child(0).unwrap(); - - Err(Error::UnexpectedSyntaxNode { - expected: "value, identifier, sublist, index, math or function_call", - actual: child.kind(), - location: child.start_position(), - relevant_source: source[child.byte_range()].to_string(), - }) + Ok(expression) } fn run(&self, source: &str, context: &mut Map) -> Result { match self { Expression::Value(value_node) => value_node.run(source, context), Expression::Identifier(identifier) => identifier.run(source, context), - Expression::Sublist(sublist) => sublist.run(source, context), Expression::Math(math) => math.run(source, context), Expression::Logic(logic) => logic.run(source, context), Expression::FunctionCall(function_call) => function_call.run(source, context), Expression::Tool(tool) => tool.run(source, context), Expression::Index(index) => index.run(source, context), + Expression::Yield(r#yield) => r#yield.run(source, context), } } } diff --git a/src/abstract_tree/filter.rs b/src/abstract_tree/filter.rs index 2dda468..d34eb9d 100644 --- a/src/abstract_tree/filter.rs +++ b/src/abstract_tree/filter.rs @@ -19,13 +19,13 @@ impl AbstractTree for Filter { None => None, }; - let item_id_node = node.child(1).unwrap(); + let item_id_node = node.child_by_field_name("item_id").unwrap(); let item_id = Identifier::from_syntax_node(source, item_id_node)?; - let collection_node = node.child(3).unwrap(); + let collection_node = node.child_by_field_name("collection").unwrap(); let collection = Expression::from_syntax_node(source, collection_node)?; - let predicate_node = node.child(5).unwrap(); + let predicate_node = node.child_by_field_name("predicate").unwrap(); let predicate = Block::from_syntax_node(source, predicate_node)?; Ok(Filter { @@ -45,6 +45,7 @@ impl AbstractTree for Filter { Some(expression) => Some(expression.run(source, context)?.as_integer()? as usize), None => None, }; + let loop_context = Map::clone_from(context)?; values.par_iter().try_for_each(|value| { if let Some(max) = count { @@ -53,11 +54,16 @@ impl AbstractTree for Filter { } } - let mut context = Map::new(); + let mut iter_context = loop_context.clone(); - context.variables_mut().insert(key.clone(), value.clone()); + iter_context + .variables_mut()? + .insert(key.clone(), value.clone()); - let should_include = self.predicate.run(source, &mut context)?.as_boolean()?; + let should_include = self + .predicate + .run(source, &mut iter_context)? + .as_boolean()?; if should_include { new_values.items_mut().push(value.clone()); diff --git a/src/abstract_tree/find.rs b/src/abstract_tree/find.rs index 165cd65..8eba0f1 100644 --- a/src/abstract_tree/find.rs +++ b/src/abstract_tree/find.rs @@ -1,7 +1,8 @@ +use rayon::prelude::*; use serde::{Deserialize, Serialize}; use tree_sitter::Node; -use crate::{AbstractTree, Block, Expression, Identifier, Map, Result, Value}; +use crate::{AbstractTree, Block, Error, Expression, Identifier, Map, Result, Value}; #[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq, PartialOrd, Ord)] pub struct Find { @@ -29,21 +30,41 @@ impl AbstractTree for Find { } fn run(&self, source: &str, context: &mut Map) -> Result { - let value = self.expression.run(source, context)?; - let values = value.as_list()?.items(); + let expression_run = self.expression.run(source, context)?; + let list = expression_run.as_list()?.items(); let key = self.identifier.inner(); - let mut context = context.clone(); - for value in values.iter() { - context.variables_mut().insert(key.clone(), value.clone()); + let find_result = list.par_iter().find_map_first(|value| { + let loop_context = Map::clone_from(context).unwrap(); - let should_return = self.item.run(source, &mut context)?.as_boolean()?; + loop_context + .variables_mut() + .unwrap() + .insert(key.clone(), (*value).clone()); - if should_return { - return Ok(value.clone()); + let run_result = self.item.run(source, &mut loop_context.clone()); + + if let Ok(run_result_value) = run_result { + if let Ok(should_return) = run_result_value.as_boolean() { + if should_return { + Some(Ok(value.clone())) + } else { + None + } + } else { + Some(Err(Error::ExpectedBoolean { + actual: value.clone(), + })) + } + } else { + Some(run_result) } - } + }); - Ok(Value::Empty) + if let Some(result) = find_result { + result + } else { + Ok(Value::Empty) + } } } diff --git a/src/abstract_tree/for.rs b/src/abstract_tree/for.rs index 72206c6..7ca36f6 100644 --- a/src/abstract_tree/for.rs +++ b/src/abstract_tree/for.rs @@ -7,19 +7,22 @@ use crate::{AbstractTree, Block, Error, Expression, Identifier, Map, Result, Val #[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq, PartialOrd, Ord)] pub struct For { is_async: bool, - identifier: Identifier, - expression: Expression, - item: Block, + item_id: Identifier, + collection: Expression, + block: Block, } impl AbstractTree for For { fn from_syntax_node(source: &str, node: Node) -> Result { + Error::expect_syntax_node(source, "for", node)?; + let for_node = node.child(0).unwrap(); let is_async = match for_node.kind() { "for" => false, + "async for" => true, _ => { return Err(Error::UnexpectedSyntaxNode { - expected: "for", + expected: "for or async for", actual: for_node.kind(), location: for_node.start_position(), relevant_source: source[for_node.byte_range()].to_string(), @@ -38,32 +41,36 @@ impl AbstractTree for For { Ok(For { is_async, - identifier, - expression, - item, + item_id: identifier, + collection: expression, + block: item, }) } fn run(&self, source: &str, context: &mut Map) -> Result { - let expression_run = self.expression.run(source, context)?; + let expression_run = self.collection.run(source, context)?; let values = expression_run.as_list()?.items(); - let key = self.identifier.inner(); + let key = self.item_id.inner(); if self.is_async { values.par_iter().try_for_each(|value| { - let mut iter_context = Map::new(); + let mut iter_context = Map::clone_from(context)?; iter_context - .variables_mut() + .variables_mut()? .insert(key.clone(), value.clone()); - self.item.run(source, &mut iter_context).map(|_value| ()) + self.block.run(source, &mut iter_context).map(|_value| ()) })?; } else { - for value in values.iter() { - context.variables_mut().insert(key.clone(), value.clone()); + let loop_context = Map::clone_from(context)?; - self.item.run(source, context)?; + for value in values.iter() { + loop_context + .variables_mut()? + .insert(key.clone(), value.clone()); + + self.block.run(source, &mut loop_context.clone())?; } } diff --git a/src/abstract_tree/function_call.rs b/src/abstract_tree/function_call.rs index 02b9954..3c71f13 100644 --- a/src/abstract_tree/function_call.rs +++ b/src/abstract_tree/function_call.rs @@ -3,13 +3,13 @@ use tree_sitter::Node; use crate::{AbstractTree, BuiltInFunction, Error, Map, Result, Value}; -use super::{expression::Expression, identifier::Identifier}; +use super::expression::Expression; #[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq, PartialOrd, Ord)] pub enum FunctionCall { BuiltIn(Box), ContextDefined { - name: Identifier, + name: Expression, arguments: Vec, }, } @@ -18,14 +18,14 @@ impl AbstractTree for FunctionCall { fn from_syntax_node(source: &str, node: Node) -> Result { debug_assert_eq!("function_call", node.kind()); - let function_node = node.child(0).unwrap(); + let function_node = node.child(1).unwrap(); let mut arguments = Vec::new(); - for index in 1..node.child_count() { - let child = node.child(index).unwrap(); + for index in 2..node.child_count() - 1 { + let node = node.child(index).unwrap(); - if child.is_named() { - let expression = Expression::from_syntax_node(source, child)?; + if node.is_named() { + let expression = Expression::from_syntax_node(source, node)?; arguments.push(expression); } @@ -36,12 +36,9 @@ impl AbstractTree for FunctionCall { FunctionCall::BuiltIn(Box::new(function)) } else { - let identifier = Identifier::from_syntax_node(source, function_node)?; + let name = Expression::from_syntax_node(source, function_node)?; - FunctionCall::ContextDefined { - name: identifier, - arguments, - } + FunctionCall::ContextDefined { name, arguments } }; Ok(function_call) @@ -54,23 +51,30 @@ impl AbstractTree for FunctionCall { FunctionCall::ContextDefined { name, arguments } => (name, arguments), }; - let definition = if let Some(value) = context.variables().get(name.inner()) { - value.as_function().cloned()? - } else { - return Err(Error::FunctionIdentifierNotFound(name.clone())); - }; - - if let Some(parameters) = definition.identifiers() { - let parameter_expression_pairs = parameters.iter().zip(arguments.iter()); - - for (identifier, expression) in parameter_expression_pairs { - let key = identifier.clone().take_inner(); - let value = expression.run(source, context)?; - - function_context.variables_mut().insert(key, value); + let function = if let Expression::Identifier(identifier) = name { + if let Some(value) = context.variables()?.get(identifier.inner()) { + value.as_function().cloned() + } else { + return Err(Error::FunctionIdentifierNotFound(identifier.clone())); } + } else { + let name_run = name.run(source, context)?; + + name_run.as_function().cloned() + }?; + + let mut function_context = Map::clone_from(context)?; + let parameter_expression_pairs = function.parameters().iter().zip(arguments.iter()); + + for ((identifier, r#type), expression) in parameter_expression_pairs { + let key = identifier.clone().take_inner(); + let value = expression.run(source, context)?; + + r#type.check(&value)?; + + function_context.variables_mut()?.insert(key, value); } - definition.body().run(source, &mut function_context) + function.run(source, &mut function_context) } } diff --git a/src/abstract_tree/identifier.rs b/src/abstract_tree/identifier.rs index 61bac3a..15689da 100644 --- a/src/abstract_tree/identifier.rs +++ b/src/abstract_tree/identifier.rs @@ -22,7 +22,7 @@ impl Identifier { impl AbstractTree for Identifier { fn from_syntax_node(source: &str, node: Node) -> Result { - debug_assert_eq!("identifier", node.kind()); + Error::expect_syntax_node(source, "identifier", node)?; let identifier = &source[node.byte_range()]; @@ -30,7 +30,7 @@ impl AbstractTree for Identifier { } fn run(&self, _source: &str, context: &mut Map) -> Result { - if let Some(value) = context.variables().get(&self.0) { + if let Some(value) = context.variables()?.get(&self.0) { Ok(value.clone()) } else { Err(Error::VariableIdentifierNotFound(self.inner().clone())) diff --git a/src/abstract_tree/identifier_assignment.rs b/src/abstract_tree/identifier_assignment.rs new file mode 100644 index 0000000..6bb7d0c --- /dev/null +++ b/src/abstract_tree/identifier_assignment.rs @@ -0,0 +1,79 @@ +use serde::{Deserialize, Serialize}; +use tree_sitter::Node; + +use crate::{AbstractTree, Error, Identifier, Map, Result, Statement, Value}; + +#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq, PartialOrd, Ord)] +pub struct IdentifierAssignment { + identifier: Identifier, + operator: AssignmentOperator, + statement: Statement, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq, PartialOrd, Ord)] +pub enum AssignmentOperator { + Equal, + PlusEqual, + MinusEqual, +} + +impl AbstractTree for IdentifierAssignment { + fn from_syntax_node(source: &str, node: Node) -> Result { + let identifier_node = node.child(0).unwrap(); + let identifier = Identifier::from_syntax_node(source, identifier_node)?; + + let operator_node = node.child(1).unwrap().child(0).unwrap(); + let operator = match operator_node.kind() { + "=" => AssignmentOperator::Equal, + "+=" => AssignmentOperator::PlusEqual, + "-=" => AssignmentOperator::MinusEqual, + _ => { + return Err(Error::UnexpectedSyntaxNode { + expected: "=, += or -=", + actual: operator_node.kind(), + location: operator_node.start_position(), + relevant_source: source[operator_node.byte_range()].to_string(), + }) + } + }; + + let statement_node = node.child(2).unwrap(); + let statement = Statement::from_syntax_node(source, statement_node)?; + + Ok(IdentifierAssignment { + identifier, + operator, + statement, + }) + } + + fn run(&self, source: &str, context: &mut Map) -> Result { + let key = self.identifier.inner().clone(); + let value = self.statement.run(source, context)?; + let mut context = context.variables_mut(); + + let new_value = match self.operator { + AssignmentOperator::PlusEqual => { + if let Some(mut previous_value) = context.get(&key).cloned() { + previous_value += value; + previous_value + } else { + Value::Empty + } + } + AssignmentOperator::MinusEqual => { + if let Some(mut previous_value) = context.get(&key).cloned() { + previous_value -= value; + previous_value + } else { + Value::Empty + } + } + AssignmentOperator::Equal => value, + }; + + context.insert(key, new_value); + + Ok(Value::Empty) + } +} diff --git a/src/abstract_tree/index.rs b/src/abstract_tree/index.rs index 17395f6..4de72ad 100644 --- a/src/abstract_tree/index.rs +++ b/src/abstract_tree/index.rs @@ -1,13 +1,13 @@ use serde::{Deserialize, Serialize}; use tree_sitter::Node; -use crate::{AbstractTree, Error, Expression, List, Result, Value}; +use crate::{AbstractTree, Error, Expression, List, Map, Result, Value}; #[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq, PartialOrd, Ord)] pub struct Index { - collection: Expression, - index: Expression, - index_end: Option, + pub collection: Expression, + pub index: Expression, + pub index_end: Option, } impl AbstractTree for Index { @@ -32,10 +32,10 @@ impl AbstractTree for Index { }) } - fn run(&self, source: &str, context: &mut crate::Map) -> crate::Result { - let value = self.collection.run(source, context)?; + fn run(&self, source: &str, context: &mut Map) -> Result { + let collection = self.collection.run(source, context)?; - match value { + match collection { Value::List(list) => { let index = self.index.run(source, context)?.as_integer()? as usize; @@ -50,8 +50,17 @@ impl AbstractTree for Index { Ok(item) } - Value::Map(mut map) => { - let value = self.index.run(source, &mut map)?; + Value::Map(map) => { + let value = if let Expression::Identifier(identifier) = &self.index { + let key = identifier.inner(); + + map.variables()?.get(key).cloned().unwrap_or(Value::Empty) + } else { + let value = self.index.run(source, context)?; + let key = value.as_string()?; + + map.variables()?.get(key).cloned().unwrap_or(Value::Empty) + }; Ok(value) } @@ -61,7 +70,34 @@ impl AbstractTree for Index { Ok(Value::String(item.to_string())) } - _ => Err(Error::ExpectedCollection { actual: value }), + _ => Err(Error::ExpectedCollection { actual: collection }), } } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::evaluate; + + #[test] + fn evaluate_list_index() { + let test = evaluate("x = [1 [2] 3] x:1:0").unwrap(); + + assert_eq!(Value::Integer(2), test); + } + + #[test] + fn evaluate_map_index() { + let test = evaluate("x = {y = {z = 2}} x:y:z").unwrap(); + + assert_eq!(Value::Integer(2), test); + } + + #[test] + fn evaluate_complex_index() { + let test = evaluate("x = [1 2 3]; y = || { 0 } x:((y))").unwrap(); + + assert_eq!(Value::Integer(1), test); + } +} diff --git a/src/abstract_tree/index_assignment.rs b/src/abstract_tree/index_assignment.rs new file mode 100644 index 0000000..af71763 --- /dev/null +++ b/src/abstract_tree/index_assignment.rs @@ -0,0 +1,93 @@ +use serde::{Deserialize, Serialize}; +use tree_sitter::Node; + +use crate::{AbstractTree, Error, Index, Map, Result, Statement, Value}; + +#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq, PartialOrd, Ord)] +pub struct IndexAssignment { + index: Index, + operator: AssignmentOperator, + statement: Statement, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq, PartialOrd, Ord)] +pub enum AssignmentOperator { + Equal, + PlusEqual, + MinusEqual, +} + +impl AbstractTree for IndexAssignment { + fn from_syntax_node(source: &str, node: Node) -> Result { + Error::expect_syntax_node(source, "index_assignment", node)?; + + let index_node = node.child(0).unwrap(); + let index = Index::from_syntax_node(source, index_node)?; + + let operator_node = node.child(1).unwrap().child(0).unwrap(); + let operator = match operator_node.kind() { + "=" => AssignmentOperator::Equal, + "+=" => AssignmentOperator::PlusEqual, + "-=" => AssignmentOperator::MinusEqual, + _ => { + return Err(Error::UnexpectedSyntaxNode { + expected: "=, += or -=", + actual: operator_node.kind(), + location: operator_node.start_position(), + relevant_source: source[operator_node.byte_range()].to_string(), + }) + } + }; + + let statement_node = node.child(2).unwrap(); + let statement = Statement::from_syntax_node(source, statement_node)?; + + Ok(IndexAssignment { + index, + operator, + statement, + }) + } + + fn run(&self, source: &str, context: &mut Map) -> Result { + let index_collection = self.index.collection.run(source, context)?; + let index_context = index_collection.as_map().unwrap_or(&context); + let index_key = if let crate::Expression::Identifier(identifier) = &self.index.index { + identifier.inner() + } else { + return Err(Error::VariableIdentifierNotFound( + self.index.index.run(source, context)?.to_string(), + )); + }; + + let value = self.statement.run(source, &mut context.clone())?; + + let new_value = match self.operator { + AssignmentOperator::PlusEqual => { + if let Some(mut previous_value) = index_context.variables()?.get(index_key).cloned() + { + previous_value += value; + previous_value + } else { + Value::Empty + } + } + AssignmentOperator::MinusEqual => { + if let Some(mut previous_value) = index_context.variables()?.get(index_key).cloned() + { + previous_value -= value; + previous_value + } else { + Value::Empty + } + } + AssignmentOperator::Equal => value, + }; + + index_context + .variables_mut()? + .insert(index_key.clone(), new_value); + + Ok(Value::Empty) + } +} diff --git a/src/abstract_tree/insert.rs b/src/abstract_tree/insert.rs index 3faac90..4a27639 100644 --- a/src/abstract_tree/insert.rs +++ b/src/abstract_tree/insert.rs @@ -36,7 +36,7 @@ impl AbstractTree for Insert { } context - .variables_mut() + .variables_mut()? .insert(table_name, Value::Table(table)); Ok(Value::Empty) diff --git a/src/abstract_tree/mod.rs b/src/abstract_tree/mod.rs index dce5873..909af4d 100644 --- a/src/abstract_tree/mod.rs +++ b/src/abstract_tree/mod.rs @@ -7,7 +7,6 @@ //! examples. pub mod assignment; -pub mod r#await; pub mod block; pub mod built_in_function; pub mod expression; @@ -18,6 +17,7 @@ pub mod function_call; pub mod identifier; pub mod if_else; pub mod index; +pub mod index_assignment; pub mod insert; pub mod logic; pub mod r#match; @@ -25,16 +25,18 @@ pub mod math; pub mod remove; pub mod select; pub mod statement; -pub mod sublist; pub mod transform; +pub mod r#type; +pub mod r#use; pub mod value_node; pub mod r#while; +pub mod r#yield; pub use { assignment::*, block::*, built_in_function::*, expression::*, filter::*, find::*, - function_call::*, identifier::*, if_else::*, index::*, insert::*, logic::*, math::*, - r#await::*, r#await::*, r#for::*, r#match::*, r#while::*, remove::*, select::*, statement::*, - sublist::*, transform::*, value_node::*, + function_call::*, identifier::*, if_else::*, index::*, index_assignment::IndexAssignment, + insert::*, logic::*, math::*, r#for::*, r#match::*, r#type::*, r#use::*, r#while::*, + r#yield::*, remove::*, select::*, statement::*, transform::*, value_node::*, }; use tree_sitter::Node; @@ -55,6 +57,6 @@ pub trait AbstractTree: Sized { /// node's byte range. fn from_syntax_node(source: &str, node: Node) -> Result; - /// Execute dust code by traversing the tree + /// Execute dust code by traversing the tree. fn run(&self, source: &str, context: &mut Map) -> Result; } diff --git a/src/abstract_tree/remove.rs b/src/abstract_tree/remove.rs index 29aace0..e387463 100644 --- a/src/abstract_tree/remove.rs +++ b/src/abstract_tree/remove.rs @@ -1,62 +1,72 @@ +use std::sync::RwLock; + +use rayon::prelude::*; use serde::{Deserialize, Serialize}; use tree_sitter::Node; -use crate::{AbstractTree, Block, Expression, Identifier, Map, Result, Value}; +use crate::{AbstractTree, Block, Error, Expression, Identifier, Map, Result, Value}; #[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq, PartialOrd, Ord)] pub struct Remove { - identifier: Identifier, - expression: Expression, - item: Block, + item_id: Identifier, + collection: Expression, + predicate: Block, } impl AbstractTree for Remove { fn from_syntax_node(source: &str, node: Node) -> Result { let identifier_node = node.child(1).unwrap(); - let identifier = Identifier::from_syntax_node(source, identifier_node)?; + let item_id = Identifier::from_syntax_node(source, identifier_node)?; let expression_node = node.child(3).unwrap(); - let expression = Expression::from_syntax_node(source, expression_node)?; + let collection = Expression::from_syntax_node(source, expression_node)?; - let item_node = node.child(4).unwrap(); - let item = Block::from_syntax_node(source, item_node)?; + let block_node = node.child(4).unwrap(); + let predicate = Block::from_syntax_node(source, block_node)?; Ok(Remove { - identifier, - expression, - item, + item_id, + collection, + predicate, }) } fn run(&self, source: &str, context: &mut Map) -> Result { - let expression_run = self.expression.run(source, context)?; - let values = expression_run.into_inner_list()?; - let key = self.identifier.inner(); - let mut sub_context = context.clone(); - let mut should_remove_index = None; + let value = self.collection.run(source, context)?; + let values = value.as_list()?; + let key = self.item_id.inner(); + let should_remove_index = RwLock::new(None); - for (index, value) in values.items().iter().enumerate() { - sub_context - .variables_mut() - .insert(key.clone(), value.clone()); - - let should_remove = self.item.run(source, &mut sub_context)?.as_boolean()?; - - if should_remove { - should_remove_index = Some(index); - - match &self.expression { - Expression::Identifier(identifier) => { - sub_context - .variables_mut() - .insert(identifier.inner().clone(), Value::List(values.clone())); - } - _ => {} + values + .items() + .par_iter() + .enumerate() + .try_for_each(|(index, value)| { + if should_remove_index.read()?.is_some() { + return Ok(()); } - } - } - if let Some(index) = should_remove_index { + let iter_context = Map::clone_from(context)?; + + iter_context + .variables_mut()? + .insert(key.clone(), value.clone()); + + let should_remove = self + .predicate + .run(source, &mut iter_context.clone())? + .as_boolean()?; + + if should_remove { + let _ = should_remove_index.write()?.insert(index); + } + + Ok::<(), Error>(()) + })?; + + let index = should_remove_index.read()?; + + if let Some(index) = *index { Ok(values.items_mut().remove(index)) } else { Ok(Value::Empty) diff --git a/src/abstract_tree/select.rs b/src/abstract_tree/select.rs index 4f926ef..006c0f9 100644 --- a/src/abstract_tree/select.rs +++ b/src/abstract_tree/select.rs @@ -7,7 +7,7 @@ use crate::{AbstractTree, Block, Expression, Identifier, Map, Result, Table, Val pub struct Select { identifiers: Vec, expression: Expression, - block: Option, + predicate: Option, } impl AbstractTree for Select { @@ -15,6 +15,8 @@ impl AbstractTree for Select { let mut identifiers = Vec::new(); let identifier_list = node.child(1).unwrap(); + let identifier_list = node.child(1).unwrap(); + for index in 1..identifier_list.child_count() - 1 { let node = identifier_list.child(index).unwrap(); @@ -27,8 +29,10 @@ impl AbstractTree for Select { let expression_node = node.child(3).unwrap(); let expression = Expression::from_syntax_node(source, expression_node)?; - let block = if let Some(block_node) = node.child(4) { - Some(Block::from_syntax_node(source, block_node)?) + let final_node = node.child(child_count - 1).unwrap(); + + let predicate = if final_node.kind() == "block" { + Some(Block::from_syntax_node(source, final_node)?) } else { None }; @@ -36,7 +40,7 @@ impl AbstractTree for Select { Ok(Select { identifiers, expression, - block, + predicate, }) } @@ -47,7 +51,7 @@ impl AbstractTree for Select { self.identifiers .iter() .cloned() - .map(|identifierier| identifierier.take_inner()) + .map(|identifier| identifier.take_inner()) .collect() } else { old_table.headers().clone() @@ -56,13 +60,13 @@ impl AbstractTree for Select { for row in old_table.rows() { let mut new_row = Vec::new(); - let mut row_context = Map::new(); + let row_context = Map::new(); for (i, value) in row.iter().enumerate() { let column_name = old_table.headers().get(i).unwrap(); row_context - .variables_mut() + .variables_mut()? .insert(column_name.clone(), value.clone()); let new_table_column_index = @@ -86,8 +90,10 @@ impl AbstractTree for Select { } } - if let Some(where_clause) = &self.block { - let should_include = where_clause.run(source, &mut row_context)?.as_boolean()?; + if let Some(where_clause) = &self.predicate { + let should_include = where_clause + .run(source, &mut row_context.clone())? + .as_boolean()?; if should_include { new_table.insert(new_row)?; diff --git a/src/abstract_tree/statement.rs b/src/abstract_tree/statement.rs index 99ceb22..4ed98c5 100644 --- a/src/abstract_tree/statement.rs +++ b/src/abstract_tree/statement.rs @@ -2,35 +2,34 @@ use serde::{Deserialize, Serialize}; use tree_sitter::Node; use crate::{ - AbstractTree, Assignment, Await, Error, Expression, Filter, Find, For, IfElse, Insert, Map, - Match, Remove, Result, Select, Transform, Value, While, + AbstractTree, Assignment, Block, Error, Expression, Filter, Find, For, IfElse, IndexAssignment, + Insert, Map, Match, Remove, Result, Select, Transform, Use, Value, While, }; /// Abstract representation of a statement. -/// -/// A statement may evaluate to an Empty value when run. If a Statement is an -/// Expression, it will always return a non-empty value when run. #[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq, PartialOrd, Ord)] pub enum Statement { Assignment(Box), - Await(Await), + Return(Expression), Expression(Expression), IfElse(Box), Match(Match), While(Box), - Async(Box), + Block(Box), For(Box), Transform(Box), Filter(Box), Find(Box), Remove(Box), + Use(Use), Select(Box::from(map).unwrap(), Value::Table(table) => table.clone(), Value::Function(function) => { let mut table = Table::new(vec!["function".to_string()]); @@ -287,39 +287,35 @@ impl From<&mut List> for Table { } } -impl From for Table { +impl From for Result
{ fn from(map: Map) -> Self { - let variables = map.variables(); + let variables = map.variables()?; let keys = variables.keys().cloned().collect(); let values = variables.values().cloned().collect(); let mut table = Table::new(keys); - table - .insert(values) - .expect("Failed to create Table from Map. This is a no-op."); + table.insert(values)?; - table + Ok(table) } } -impl From<&Map> for Table { +impl From<&Map> for Result
{ fn from(map: &Map) -> Self { - let variables = map.variables(); + let variables = map.variables()?; let keys = variables.keys().cloned().collect(); let values = variables.values().cloned().collect(); let mut table = Table::new(keys); - table - .insert(values) - .expect("Failed to create Table from Map. This is a no-op."); + table.insert(values)?; - table + Ok(table) } } -impl From<&mut Map> for Table { +impl From<&mut Map> for Result
{ fn from(map: &mut Map) -> Self { - let variables = map.variables(); + let variables = map.variables()?; let keys = variables.keys().cloned().collect(); let values = variables.values().cloned().collect(); let mut table = Table::new(keys); @@ -328,7 +324,7 @@ impl From<&mut Map> for Table { .insert(values) .expect("Failed to create Table from Map. This is a no-op."); - table + Ok(table) } } diff --git a/std/list_functions.ds b/std/list_functions.ds new file mode 100644 index 0000000..db7b766 --- /dev/null +++ b/std/list_functions.ds @@ -0,0 +1,23 @@ +find = |items function | { + for i in items { + if (function i) { + return i + } + } +} + +map = |items function | { + new_list = [] + + for i in items { + new_list += (function i) + } + + new_list +} + +foobar = [0 1 2] + -> (map |i | { i - 1 }) + -> (find |i | { i == -1 }) + +foobar diff --git a/tests/dust_examples.rs b/tests/dust_examples.rs index f1a3327..8803c86 100644 --- a/tests/dust_examples.rs +++ b/tests/dust_examples.rs @@ -2,6 +2,21 @@ use std::fs::read_to_string; use dust_lang::*; +#[test] +fn r#async() { + let file_contents = read_to_string("examples/async.ds").unwrap(); + + evaluate(&file_contents).unwrap(); +} + +#[test] +#[ignore] +fn async_download() { + let file_contents = read_to_string("examples/async_download.ds").unwrap(); + + evaluate(&file_contents).unwrap(); +} + #[test] fn clue_solver() { let file_contents = read_to_string("examples/clue_solver.ds").unwrap(); @@ -24,13 +39,6 @@ fn fibonacci() { evaluate(&file_contents).unwrap(); } -#[test] -fn find_loop() { - let file_contents = read_to_string("examples/find_loop.ds").unwrap(); - - evaluate(&file_contents).unwrap(); -} - #[test] fn fizzbuzz() { let file_contents = read_to_string("examples/fizzbuzz.ds").unwrap(); @@ -53,8 +61,43 @@ fn hello_world() { } #[test] -fn remove_loop() { - let file_contents = read_to_string("examples/remove_loop.ds").unwrap(); +fn jq_data() { + let file_contents = read_to_string("examples/jq_data.ds").unwrap(); + + evaluate(&file_contents).unwrap(); +} + +#[test] +fn list() { + let file_contents = read_to_string("examples/list.ds").unwrap(); + + evaluate(&file_contents).unwrap(); +} + +#[test] +fn map() { + let file_contents = read_to_string("examples/map.ds").unwrap(); + + evaluate(&file_contents).unwrap(); +} + +#[test] +fn random() { + let file_contents = read_to_string("examples/random.ds").unwrap(); + + evaluate(&file_contents).unwrap(); +} + +#[test] +fn sea_creatures() { + let file_contents = read_to_string("examples/sea_creatures.ds").unwrap(); + + evaluate(&file_contents).unwrap(); +} + +#[test] +fn select() { + let file_contents = read_to_string("examples/select.ds").unwrap(); evaluate(&file_contents).unwrap(); } @@ -73,13 +116,6 @@ fn table() { evaluate(&file_contents).unwrap(); } -#[test] -fn transform_loop() { - let file_contents = read_to_string("examples/transform_loop.ds").unwrap(); - - evaluate(&file_contents).unwrap(); -} - #[test] fn variables() { let file_contents = read_to_string("examples/variables.ds").unwrap(); @@ -93,3 +129,10 @@ fn while_loop() { evaluate(&file_contents).unwrap(); } + +#[test] +fn r#yield() { + let file_contents = read_to_string("examples/yield.ds").unwrap(); + + evaluate(&file_contents).unwrap(); +} diff --git a/tree-sitter-dust/corpus/assignment.txt b/tree-sitter-dust/corpus/assignment.txt new file mode 100644 index 0000000..c63a135 --- /dev/null +++ b/tree-sitter-dust/corpus/assignment.txt @@ -0,0 +1,79 @@ +================================================================================ +Simple Assignment +================================================================================ + +x = y + +-------------------------------------------------------------------------------- + +(root + (statement + (assignment + (identifier) + (assignment_operator) + (statement + (expression + (identifier)))))) + +================================================================================ +Simple Assignment with Type +================================================================================ + +x = y + +-------------------------------------------------------------------------------- + +(root + (statement + (assignment + (identifier) + (type) + (assignment_operator) + (statement + (expression + (identifier)))))) + +================================================================================ +Map Item Assignment +================================================================================ + +x:y = 1 + +-------------------------------------------------------------------------------- + +(root + (statement + (index_assignment + (index + (expression + (identifier)) + (expression + (identifier))) + (assignment_operator) + (statement + (expression + (value + (integer))))))) + +================================================================================ +List Item Assignment +================================================================================ + +x:9 = 'foobar' + +-------------------------------------------------------------------------------- + +(root + (statement + (index_assignment + (index + (expression + (identifier)) + (expression + (value + (integer)))) + (assignment_operator) + (statement + (expression + (value + (string))))))) diff --git a/tree-sitter-dust/corpus/async.txt b/tree-sitter-dust/corpus/async.txt new file mode 100644 index 0000000..6f3ecd8 --- /dev/null +++ b/tree-sitter-dust/corpus/async.txt @@ -0,0 +1,71 @@ +================================================================================ +Simple Async Statements +================================================================================ + +async { (output 'Whaddup') } + +-------------------------------------------------------------------------------- + +(root + (statement + (block + (statement + (expression + (function_call + (built_in_function + (expression + (value + (string)))))))))) + +================================================================================ +Complex Async Statements +================================================================================ + +async { + if 1 % 2 == 0 { + true + } else { + false + } + + 'foobar' +} + +-------------------------------------------------------------------------------- + +(root + (statement + (block + (statement + (if_else + (if + (expression + (logic + (expression + (math + (expression + (value + (integer))) + (math_operator) + (expression + (value + (integer))))) + (logic_operator) + (expression + (value + (integer))))) + (block + (statement + (expression + (value + (boolean)))))) + (else + (block + (statement + (expression + (value + (boolean)))))))) + (statement + (expression + (value + (string))))))) diff --git a/tree-sitter-dust/corpus/built_in_function.txt b/tree-sitter-dust/corpus/built_in_function.txt index 7252910..0990736 100644 --- a/tree-sitter-dust/corpus/built_in_function.txt +++ b/tree-sitter-dust/corpus/built_in_function.txt @@ -1,49 +1,36 @@ -================== +================================================================================ Simple Function Call -================== +================================================================================ (output 'hi') ---- +-------------------------------------------------------------------------------- (root - (block - (statement - (expression - (function_call - (built_in_function - (expression - (value - (string))))))))) + (statement + (expression + (function_call + (built_in_function + (expression + (value + (string)))))))) -================== +================================================================================ Nested Function Call -================== +================================================================================ (assert_equal (random_integer) 4) -assert_equal random_integer 4 ---- +-------------------------------------------------------------------------------- (root - (block - (statement - (expression - (function_call - (built_in_function - (expression - (function_call - (built_in_function))) - (expression - (value - (integer))))))) - (statement - (expression - (function_call - (built_in_function - (expression - (function_call - (built_in_function - (expression - (value - (integer)))))))))))) + (statement + (expression + (function_call + (built_in_function + (expression + (function_call + (built_in_function))) + (expression + (value + (integer)))))))) diff --git a/tree-sitter-dust/corpus/comments.txt b/tree-sitter-dust/corpus/comments.txt index ce12a8a..4d98d28 100644 --- a/tree-sitter-dust/corpus/comments.txt +++ b/tree-sitter-dust/corpus/comments.txt @@ -1,53 +1,45 @@ -================== +================================================================================ Full Line Comments -================== +================================================================================ not_a_comment # comment ---- +-------------------------------------------------------------------------------- (root - (block - (statement - (expression - (identifier)))) - (comment)) + (statement + (expression + (identifier)))) -================== +================================================================================ Partial Line Comments -================== +================================================================================ not_a_comment # comment ---- +-------------------------------------------------------------------------------- (root - (block - (statement - (expression - (identifier)))) - (comment)) + (statement + (expression + (identifier)))) -================== +================================================================================ Multiline Comments -================== +================================================================================ # comment # not_a_comment # # comment # "not a comment" ---- +-------------------------------------------------------------------------------- (root - (comment) - (block - (statement - (expression - (identifier))) - (comment) - (comment) - (statement - (expression - (value - (string)))))) + (statement + (expression + (identifier))) + (statement + (expression + (value + (string))))) diff --git a/tree-sitter-dust/corpus/filter.txt b/tree-sitter-dust/corpus/filter.txt deleted file mode 100644 index 0857cd7..0000000 --- a/tree-sitter-dust/corpus/filter.txt +++ /dev/null @@ -1,73 +0,0 @@ -================== -Simple Filter Loop -================== - -filter i in [1, 2, 3] { - i <= 1 -} - ---- - -(root - (block - (statement - (filter - (identifier) - (expression - (value - (list - (expression - (value - (integer))) - (expression - (value - (integer))) - (expression - (value - (integer)))))) - (block - (statement - (expression - (logic - (expression - (identifier)) - (logic_operator) - (expression - (value - (integer))))))))))) - -================== -Nested Filter Loop -================== - -filter i in big_list { - filter j in i { - i != 42 - } -} - ---- - -(root - (block - (statement - (filter - (identifier) - (expression - (identifier)) - (block - (statement - (filter - (identifier) - (expression - (identifier)) - (block - (statement - (expression - (logic - (expression - (identifier)) - (logic_operator) - (expression - (value - (integer)))))))))))))) diff --git a/tree-sitter-dust/corpus/find.txt b/tree-sitter-dust/corpus/find.txt deleted file mode 100644 index 19b785b..0000000 --- a/tree-sitter-dust/corpus/find.txt +++ /dev/null @@ -1,119 +0,0 @@ -================================================================================ -Simple Find Loop -================================================================================ - -find i in [1, 2, 3] { - i <= 3 -} - --------------------------------------------------------------------------------- - -(root - (block - (statement - (find - (identifier) - (expression - (value - (list - (expression - (value - (integer))) - (expression - (value - (integer))) - (expression - (value - (integer)))))) - (block - (statement - (expression - (logic - (expression - (identifier)) - (logic_operator) - (expression - (value - (integer))))))))))) - -================================================================================ -Nested Find Loop -================================================================================ - -find i in ["one", "two", "three"] { - found = find j in i { - i == "e" - } - - if (type found) != 'empty' { - true - } else { - false - } -} - - --------------------------------------------------------------------------------- - -(root - (block - (statement - (find - (identifier) - (expression - (value - (list - (expression - (value - (string))) - (expression - (value - (string))) - (expression - (value - (string)))))) - (block - (statement - (assignment - (identifier) - (assignment_operator) - (statement - (find - (identifier) - (expression - (identifier)) - (block - (statement - (expression - (logic - (expression - (identifier)) - (logic_operator) - (expression - (value - (string))))))))))) - (statement - (if_else - (if - (expression - (logic - (expression - (function_call - (built_in_function - (expression - (identifier))))) - (logic_operator) - (expression - (value - (string))))) - (block - (statement - (expression - (value - (boolean)))))) - (else - (block - (statement - (expression - (value - (boolean))))))))))))) diff --git a/tree-sitter-dust/corpus/for.txt b/tree-sitter-dust/corpus/for.txt index 858ddfb..03c6e66 100644 --- a/tree-sitter-dust/corpus/for.txt +++ b/tree-sitter-dust/corpus/for.txt @@ -2,60 +2,64 @@ Simple For Loop ================================================================================ -for i in [1, 2, 3] output i +for i in [1, 2, 3] { + (output i) +} -------------------------------------------------------------------------------- (root - (block - (statement - (for - (identifier) - (expression - (value - (list - (expression - (value - (integer))) - (expression - (value - (integer))) - (expression - (value - (integer)))))) - (block - (statement + (statement + (for + (identifier) + (expression + (value + (list (expression - (function_call - (built_in_function - (expression - (identifier))))))))))) + (value + (integer))) + (expression + (value + (integer))) + (expression + (value + (integer)))))) + (block + (statement + (expression + (function_call + (built_in_function + (expression + (identifier)))))))))) ================================================================================ Nested For Loop ================================================================================ -for list in list_of_lists for item in list output item +for list in list_of_lists { + for item in list { + (output item) + } +} -------------------------------------------------------------------------------- (root - (block - (statement - (for - (identifier) - (expression - (identifier)) - (block - (statement - (for - (identifier) - (expression - (identifier)) - (block - (statement - (expression - (function_call - (built_in_function - (expression - (identifier)))))))))))))) + (statement + (for + (identifier) + (expression + (identifier)) + (block + (statement + (for + (identifier) + (expression + (identifier)) + (block + (statement + (expression + (function_call + (built_in_function + (expression + (identifier))))))))))))) diff --git a/tree-sitter-dust/corpus/functions.txt b/tree-sitter-dust/corpus/functions.txt index e2b064d..affe8a4 100644 --- a/tree-sitter-dust/corpus/functions.txt +++ b/tree-sitter-dust/corpus/functions.txt @@ -2,45 +2,45 @@ Simple Function ================================================================================ -=> "Hiya" +|| { "Hiya" } -------------------------------------------------------------------------------- (root - (block - (statement - (expression - (value - (function - (block - (statement - (expression - (value - (string))))))))))) + (statement + (expression + (value + (function + (type) + (block + (statement + (expression + (value + (string)))))))))) ================================================================================ Function Assignment ================================================================================ -x = => "Hiya" +x = || { "Hiya" } -------------------------------------------------------------------------------- (root - (block - (statement - (assignment - (identifier) - (assignment_operator) - (statement - (expression - (value - (function - (block - (statement - (expression - (value - (string))))))))))))) + (statement + (assignment + (identifier) + (assignment_operator) + (statement + (expression + (value + (function + (type) + (block + (statement + (expression + (value + (string)))))))))))) ================================================================================ Function Call @@ -51,20 +51,20 @@ Function Call -------------------------------------------------------------------------------- (root - (block - (statement - (expression - (function_call - (identifier) - (expression - (value - (string)))))))) + (statement + (expression + (function_call + (expression + (identifier)) + (expression + (value + (string))))))) ================================================================================ Complex Function ================================================================================ -|message number| => { +|message number | { (output message) (output number) } @@ -72,27 +72,29 @@ Complex Function -------------------------------------------------------------------------------- (root - (block - (statement - (expression - (value - (function - (identifier_list - (identifier) - (identifier)) - (block - (statement - (expression - (function_call - (built_in_function - (expression - (identifier)))))) - (statement - (expression - (function_call - (built_in_function - (expression - (identifier))))))))))))) + (statement + (expression + (value + (function + (parameter + (identifier) + (type)) + (parameter + (identifier) + (type)) + (block + (statement + (expression + (function_call + (built_in_function + (expression + (identifier)))))) + (statement + (expression + (function_call + (built_in_function + (expression + (identifier)))))))))))) ================================================================================ Complex Function Call @@ -110,34 +112,27 @@ Complex Function Call -------------------------------------------------------------------------------- (root - (block - (statement - (expression - (function_call - (identifier) - (expression - (value - (string))) - (expression - (value - (integer))) - (expression - (value - (map - (block - (statement - (assignment - (identifier) - (assignment_operator) - (statement - (expression - (value - (integer)))))) - (statement - (assignment - (identifier) - (assignment_operator) - (statement - (expression - (value - (integer))))))))))))))) + (statement + (expression + (function_call + (expression + (identifier)) + (expression + (value + (string))) + (expression + (value + (integer))) + (expression + (value + (map + (identifier) + (statement + (expression + (value + (integer)))) + (identifier) + (statement + (expression + (value + (integer))))))))))) diff --git a/tree-sitter-dust/corpus/identifiers.txt b/tree-sitter-dust/corpus/identifiers.txt index 7650b4c..1fa0e8c 100644 --- a/tree-sitter-dust/corpus/identifiers.txt +++ b/tree-sitter-dust/corpus/identifiers.txt @@ -1,21 +1,20 @@ -================== +================================================================================ Simple Identifiers -================== +================================================================================ x _y __xyz__ ---- +-------------------------------------------------------------------------------- (root - (block - (statement - (expression - (identifier))) - (statement - (expression - (identifier))) - (statement - (expression - (identifier))))) + (statement + (expression + (identifier))) + (statement + (expression + (identifier))) + (statement + (expression + (identifier)))) diff --git a/tree-sitter-dust/corpus/if_else.txt b/tree-sitter-dust/corpus/if_else.txt index 058ffed..caac6a9 100644 --- a/tree-sitter-dust/corpus/if_else.txt +++ b/tree-sitter-dust/corpus/if_else.txt @@ -1,153 +1,151 @@ -================== +================================================================================ Simple If -================== +================================================================================ if true { "True" } ---- +-------------------------------------------------------------------------------- (root - (block - (statement - (if_else - (if - (expression - (value - (boolean))) - (block - (statement - (expression - (value - (string)))))))))) + (statement + (if_else + (if + (expression + (value + (boolean))) + (block + (statement + (expression + (value + (string))))))))) -================== +================================================================================ Complex If -================== +================================================================================ -if 1 == 1 && 2 == 2 && 3 == 3 "True" +if 1 == 1 && 2 == 2 && 3 == 3 { "True" } ---- +-------------------------------------------------------------------------------- (root - (block - (statement - (if_else - (if - (expression - (logic - (expression - (value - (integer))) - (logic_operator) - (expression - (logic - (expression - (value - (integer))) - (logic_operator) - (expression - (logic - (expression - (value - (integer))) - (logic_operator) - (expression - (logic - (expression - (value - (integer))) - (logic_operator) - (expression - (logic - (expression - (value - (integer))) - (logic_operator) - (expression - (value - (integer))))))))))))) - (block - (statement - (expression - (value - (string)))))))))) + (statement + (if_else + (if + (expression + (logic + (expression + (value + (integer))) + (logic_operator) + (expression + (logic + (expression + (value + (integer))) + (logic_operator) + (expression + (logic + (expression + (value + (integer))) + (logic_operator) + (expression + (logic + (expression + (value + (integer))) + (logic_operator) + (expression + (logic + (expression + (value + (integer))) + (logic_operator) + (expression + (value + (integer))))))))))))) + (block + (statement + (expression + (value + (string))))))))) -================== +================================================================================ Nested If -================== +================================================================================ -if true - if 42 == 12 +if true { + if 42 == 12 { 'hiya' - else + } else { 'bye' + } +} ---- +-------------------------------------------------------------------------------- (root - (block - (statement - (if_else - (if - (expression - (value - (boolean))) - (block - (statement - (if_else - (if - (expression - (logic - (expression - (value - (integer))) - (logic_operator) - (expression - (value - (integer))))) - (block - (statement - (expression - (value - (string)))))) - (else - (block - (statement - (expression - (value - (string)))))))))))))) + (statement + (if_else + (if + (expression + (value + (boolean))) + (block + (statement + (if_else + (if + (expression + (logic + (expression + (value + (integer))) + (logic_operator) + (expression + (value + (integer))))) + (block + (statement + (expression + (value + (string)))))) + (else + (block + (statement + (expression + (value + (string))))))))))))) -================== +================================================================================ If Else -================== +================================================================================ -if false "True" else "False" +if false { "True" } else { "False" } ---- +-------------------------------------------------------------------------------- (root - (block - (statement - (if_else - (if - (expression - (value - (boolean))) - (block - (statement - (expression - (value - (string)))))) - (else - (block - (statement - (expression - (value - (string)))))))))) + (statement + (if_else + (if + (expression + (value + (boolean))) + (block + (statement + (expression + (value + (string)))))) + (else + (block + (statement + (expression + (value + (string))))))))) -================== +================================================================================ If Else If -================== +================================================================================ if 1 == 1 { "math is fun" @@ -155,104 +153,103 @@ if 1 == 1 { "math is broken" } ---- +-------------------------------------------------------------------------------- (root - (block - (statement - (if_else - (if - (expression - (logic - (expression - (value - (integer))) - (logic_operator) - (expression - (value - (integer))))) - (block - (statement - (expression - (value - (string)))))) - (else_if - (expression - (logic - (expression - (value - (integer))) - (logic_operator) - (expression - (value - (integer))))) - (block - (statement - (expression - (value - (string)))))))))) + (statement + (if_else + (if + (expression + (logic + (expression + (value + (integer))) + (logic_operator) + (expression + (value + (integer))))) + (block + (statement + (expression + (value + (string)))))) + (else_if + (expression + (logic + (expression + (value + (integer))) + (logic_operator) + (expression + (value + (integer))))) + (block + (statement + (expression + (value + (string))))))))) -================== +================================================================================ If Else Else If Else -================== +================================================================================ -if false +if false { "no" -else if false +} else if false { "no" -else if 1 + 1 == 9 +} else if 1 + 1 == 9 { "not the answer" -else +} else { "42" +} ---- +-------------------------------------------------------------------------------- (root - (block - (statement - (if_else - (if - (expression - (value - (boolean))) - (block - (statement - (expression - (value - (string)))))) - (else_if - (expression - (value - (boolean))) - (block - (statement - (expression - (value - (string)))))) - (else_if - (expression - (logic - (expression - (math - (expression - (value - (integer))) - (math_operator) - (expression - (value - (integer))))) - (logic_operator) - (expression - (value - (integer))))) - (block - (statement - (expression - (value - (string)))))) - (else - (block - (statement - (expression - (value - (string)))))))))) + (statement + (if_else + (if + (expression + (value + (boolean))) + (block + (statement + (expression + (value + (string)))))) + (else_if + (expression + (value + (boolean))) + (block + (statement + (expression + (value + (string)))))) + (else_if + (expression + (logic + (expression + (math + (expression + (value + (integer))) + (math_operator) + (expression + (value + (integer))))) + (logic_operator) + (expression + (value + (integer))))) + (block + (statement + (expression + (value + (string)))))) + (else + (block + (statement + (expression + (value + (string))))))))) diff --git a/tree-sitter-dust/corpus/index.txt b/tree-sitter-dust/corpus/index.txt index 4fdad7d..f7770d3 100644 --- a/tree-sitter-dust/corpus/index.txt +++ b/tree-sitter-dust/corpus/index.txt @@ -1,6 +1,6 @@ -================== +================================================================================ Simple Indexes -================== +================================================================================ dust_data:1:name @@ -8,87 +8,107 @@ creature:total_clams foobar:1:42 ---- +-------------------------------------------------------------------------------- (root - (block - (statement - (expression - (index - (expression - (index - (expression - (identifier)) - (expression - (value - (integer))))) - (expression - (identifier))))) - (statement - (expression - (index - (expression - (identifier)) - (expression - (identifier))))) - (statement + (statement + (expression + (index (expression (index (expression - (index - (expression - (identifier)) - (expression - (value - (integer))))) + (identifier)) (expression (value - (integer)))))))) + (integer))))) + (expression + (identifier))))) + (statement + (expression + (index + (expression + (identifier)) + (expression + (identifier))))) + (statement + (expression + (index + (expression + (index + (expression + (identifier)) + (expression + (value + (integer))))) + (expression + (value + (integer))))))) -================== +================================================================================ Nested Indexes -================== +================================================================================ [['answers' 'foobar'], 42, 666]:0:1:0..2 ---- +-------------------------------------------------------------------------------- (root - (block - (statement + (statement + (expression + (index (expression (index (expression (index (expression - (index - (expression - (value - (list - (expression - (value - (list - (expression - (value - (string))) - (expression - (value - (string)))))) - (expression - (value - (integer))) - (expression - (value - (integer)))))) - (expression - (value - (integer))))) + (value + (list + (expression + (value + (list + (expression + (value + (string))) + (expression + (value + (string)))))) + (expression + (value + (integer))) + (expression + (value + (integer)))))) (expression (value (integer))))) (expression (value - (integer))) + (integer))))) + (expression + (value + (integer))) + (expression + (value + (integer))))))) + +================================================================================ +Function Call Index +================================================================================ + +x:(y):0 + +-------------------------------------------------------------------------------- + +(root + (statement + (expression + (index + (expression + (index (expression - (value - (integer)))))))) \ No newline at end of file + (identifier)) + (expression + (identifier)))) + (expression + (value + (integer))))))) diff --git a/tree-sitter-dust/corpus/lists.txt b/tree-sitter-dust/corpus/lists.txt index 9e2a422..a8d58f6 100644 --- a/tree-sitter-dust/corpus/lists.txt +++ b/tree-sitter-dust/corpus/lists.txt @@ -1,50 +1,48 @@ -================== +================================================================================ List Declaration -================== +================================================================================ ['answer', 42] ---- +-------------------------------------------------------------------------------- (root - (block - (statement - (expression - (value - (list - (expression - (value - (string))) - (expression - (value - (integer))))))))) + (statement + (expression + (value + (list + (expression + (value + (string))) + (expression + (value + (integer)))))))) -================== +================================================================================ List Nesting -================== +================================================================================ ['answers', [42, [666]]] ---- +-------------------------------------------------------------------------------- (root - (block - (statement - (expression - (value - (list - (expression - (value - (string))) - (expression - (value - (list - (expression - (value - (integer))) - (expression - (value - (list - (expression - (value - (integer))))))))))))))) + (statement + (expression + (value + (list + (expression + (value + (string))) + (expression + (value + (list + (expression + (value + (integer))) + (expression + (value + (list + (expression + (value + (integer)))))))))))))) diff --git a/tree-sitter-dust/corpus/maps.txt b/tree-sitter-dust/corpus/maps.txt index 128ba32..c9aa2b2 100644 --- a/tree-sitter-dust/corpus/maps.txt +++ b/tree-sitter-dust/corpus/maps.txt @@ -7,20 +7,15 @@ map { answer = 42 } -------------------------------------------------------------------------------- (root - (block - (statement - (expression - (value - (map - (block - (statement - (assignment - (identifier) - (assignment_operator) - (statement - (expression - (value - (integer))))))))))))) + (statement + (expression + (value + (map + (identifier) + (statement + (expression + (value + (integer))))))))) ================================================================================ Nested Maps @@ -39,55 +34,36 @@ x = map { -------------------------------------------------------------------------------- (root - (block - (statement - (assignment - (identifier) - (assignment_operator) - (statement - (expression - (value - (map - (block - (statement - (assignment + (statement + (assignment + (identifier) + (assignment_operator) + (statement + (expression + (value + (map + (identifier) + (statement + (expression + (value + (map + (identifier) + (statement + (expression + (value + (string)))) (identifier) - (assignment_operator) (statement (expression (value (map - (block - (statement - (assignment - (identifier) - (assignment_operator) - (statement - (expression - (value - (string)))))) - (statement - (assignment - (identifier) - (assignment_operator) - (statement - (expression - (value - (map - (block - (statement - (assignment - (identifier) - (assignment_operator) - (statement - (expression - (value - (string)))))))))))))))))))) - (statement - (assignment - (identifier) - (assignment_operator) - (statement - (expression - (value - (integer))))))))))))))) + (identifier) + (statement + (expression + (value + (string)))))))))))) + (identifier) + (statement + (expression + (value + (integer))))))))))) diff --git a/tree-sitter-dust/corpus/operators.txt b/tree-sitter-dust/corpus/operators.txt index d79aa44..cad16f9 100644 --- a/tree-sitter-dust/corpus/operators.txt +++ b/tree-sitter-dust/corpus/operators.txt @@ -1,98 +1,95 @@ -================== +================================================================================ \== -================== +================================================================================ 3 == 1 + 1 + 1 ---- +-------------------------------------------------------------------------------- (root - (block - (statement - (expression - (logic - (expression - (value - (integer))) - (logic_operator) - (expression - (math - (expression - (math - (expression - (value - (integer))) - (math_operator) - (expression - (value - (integer))))) - (math_operator) - (expression - (value - (integer)))))))))) + (statement + (expression + (logic + (expression + (value + (integer))) + (logic_operator) + (expression + (math + (expression + (math + (expression + (value + (integer))) + (math_operator) + (expression + (value + (integer))))) + (math_operator) + (expression + (value + (integer))))))))) -================== +================================================================================ && -================== +================================================================================ 4 + 2 == 42 && true ---- +-------------------------------------------------------------------------------- (root - (block - (statement - (expression - (logic - (expression - (math - (expression - (value - (integer))) - (math_operator) - (expression - (value - (integer))))) - (logic_operator) - (expression - (logic - (expression - (value - (integer))) - (logic_operator) - (expression - (value - (boolean)))))))))) + (statement + (expression + (logic + (expression + (math + (expression + (value + (integer))) + (math_operator) + (expression + (value + (integer))))) + (logic_operator) + (expression + (logic + (expression + (value + (integer))) + (logic_operator) + (expression + (value + (boolean))))))))) -================== +================================================================================ \|| -================== +================================================================================ 4 + 2 == 42 || true ---- +-------------------------------------------------------------------------------- (root - (block - (statement - (expression - (logic - (expression - (math - (expression - (value - (integer))) - (math_operator) - (expression - (value - (integer))))) - (logic_operator) - (expression - (logic - (expression - (value - (integer))) - (logic_operator) - (expression - (value - (boolean)))))))))) + (statement + (expression + (logic + (expression + (math + (expression + (value + (integer))) + (math_operator) + (expression + (value + (integer))))) + (logic_operator) + (expression + (logic + (expression + (value + (integer))) + (logic_operator) + (expression + (value + (boolean))))))))) diff --git a/tree-sitter-dust/corpus/reduce.txt b/tree-sitter-dust/corpus/reduce.txt deleted file mode 100644 index 982aa6f..0000000 --- a/tree-sitter-dust/corpus/reduce.txt +++ /dev/null @@ -1,73 +0,0 @@ -================== -Simple Reduce Loop -================== - -reduce i to acc in [1, 2, 3] { - acc += i -} - ---- - -(root - (block - (statement - (reduce - (identifier) - (identifier) - (expression - (value - (list - (expression - (value - (integer))) - (expression - (value - (integer))) - (expression - (value - (integer)))))) - (block - (statement - (assignment - (identifier) - (assignment_operator) - (statement - (expression - (identifier)))))))))) - -================== -Nested Reduce Loop -================== - -reduce i to acc in ["one", "two", "three"] { - acc += i -} - ---- - -(root - (block - (statement - (reduce - (identifier) - (identifier) - (expression - (value - (list - (expression - (value - (string))) - (expression - (value - (string))) - (expression - (value - (string)))))) - (block - (statement - (assignment - (identifier) - (assignment_operator) - (statement - (expression - (identifier)))))))))) diff --git a/tree-sitter-dust/corpus/remove.txt b/tree-sitter-dust/corpus/remove.txt deleted file mode 100644 index a4f4890..0000000 --- a/tree-sitter-dust/corpus/remove.txt +++ /dev/null @@ -1,73 +0,0 @@ -================== -Simple Remove -================== - -remove i from [1, 2, 3] { - i <= 2 -} - ---- - -(root - (block - (statement - (remove - (identifier) - (expression - (value - (list - (expression - (value - (integer))) - (expression - (value - (integer))) - (expression - (value - (integer)))))) - (block - (statement - (expression - (logic - (expression - (identifier)) - (logic_operator) - (expression - (value - (integer))))))))))) - -================== -Nested Remove -================== - -remove i from big_list { - remove j from i { - j != 42 - } -} - ---- - -(root - (block - (statement - (remove - (identifier) - (expression - (identifier)) - (block - (statement - (remove - (identifier) - (expression - (identifier)) - (block - (statement - (expression - (logic - (expression - (identifier)) - (logic_operator) - (expression - (value - (integer)))))))))))))) diff --git a/tree-sitter-dust/corpus/statements.txt b/tree-sitter-dust/corpus/statements.txt index a878f7e..d6ef4d9 100644 --- a/tree-sitter-dust/corpus/statements.txt +++ b/tree-sitter-dust/corpus/statements.txt @@ -1,58 +1,56 @@ -================== +================================================================================ Simple Statements -================== +================================================================================ 1 "one"; x ---- +-------------------------------------------------------------------------------- (root - (block - (statement - (expression - (value - (integer)))) - (statement - (expression - (value - (string)))) - (statement - (expression - (identifier))))) + (statement + (expression + (value + (integer)))) + (statement + (expression + (value + (string)))) + (statement + (expression + (identifier)))) -================== +================================================================================ Simple Assignment -================== +================================================================================ x = 1; y = "one" ---- +-------------------------------------------------------------------------------- (root - (block - (statement - (assignment - (identifier) - (assignment_operator) - (statement - (expression - (value - (integer)))))) - (statement - (assignment - (identifier) - (assignment_operator) - (statement - (expression - (value - (string)))))))) + (statement + (assignment + (identifier) + (assignment_operator) + (statement + (expression + (value + (integer)))))) + (statement + (assignment + (identifier) + (assignment_operator) + (statement + (expression + (value + (string))))))) -================== +================================================================================ Complex Assignment -================== +================================================================================ x = if 1 + 1 == 2 { 'yo' @@ -60,77 +58,75 @@ x = if 1 + 1 == 2 { 'no' } ---- +-------------------------------------------------------------------------------- (root - (block - (statement - (assignment - (identifier) - (assignment_operator) - (statement - (if_else - (if - (expression - (logic - (expression - (math - (expression - (value - (integer))) - (math_operator) - (expression - (value - (integer))))) - (logic_operator) - (expression - (value - (integer))))) - (block - (statement - (expression - (value - (string)))))) - (else - (block - (statement - (expression - (value - (string)))))))))))) + (statement + (assignment + (identifier) + (assignment_operator) + (statement + (if_else + (if + (expression + (logic + (expression + (math + (expression + (value + (integer))) + (math_operator) + (expression + (value + (integer))))) + (logic_operator) + (expression + (value + (integer))))) + (block + (statement + (expression + (value + (string)))))) + (else + (block + (statement + (expression + (value + (string))))))))))) -================== +================================================================================ Expression Precedence -================== +================================================================================ x = 3 == 1 + 2 + 2 ---- +-------------------------------------------------------------------------------- (root - (block - (statement - (assignment - (identifier) - (assignment_operator) - (statement - (expression - (logic - (expression - (value - (integer))) - (logic_operator) - (expression - (math - (expression - (math - (expression - (value - (integer))) - (math_operator) - (expression - (value - (integer))))) - (math_operator) - (expression - (value - (integer)))))))))))) + (statement + (assignment + (identifier) + (assignment_operator) + (statement + (expression + (logic + (expression + (value + (integer))) + (logic_operator) + (expression + (math + (expression + (math + (expression + (value + (integer))) + (math_operator) + (expression + (value + (integer))))) + (math_operator) + (expression + (value + (integer))))))))))) diff --git a/tree-sitter-dust/corpus/tables.txt b/tree-sitter-dust/corpus/tables.txt index 878c4aa..0e7b301 100644 --- a/tree-sitter-dust/corpus/tables.txt +++ b/tree-sitter-dust/corpus/tables.txt @@ -1,6 +1,6 @@ -================== +================================================================================ Table Declaration -================== +================================================================================ table |messages numbers| [ ['hiya' 42] @@ -8,101 +8,98 @@ table |messages numbers| [ ['bar' 99.99] ] ---- +-------------------------------------------------------------------------------- (root - (block - (statement - (expression - (value - (table - (identifier_list - (identifier) - (identifier)) - (expression - (value - (list - (expression - (value - (list - (expression - (value - (string))) - (expression - (value - (integer)))))) - (expression - (value - (list - (expression - (value - (string))) - (expression - (value - (integer)))))) - (expression - (value - (list - (expression - (value - (string))) - (expression - (value - (float))))))))))))))) + (statement + (expression + (value + (table + (identifier_list + (identifier) + (identifier)) + (expression + (value + (list + (expression + (value + (list + (expression + (value + (string))) + (expression + (value + (integer)))))) + (expression + (value + (list + (expression + (value + (string))) + (expression + (value + (integer)))))) + (expression + (value + (list + (expression + (value + (string))) + (expression + (value + (float)))))))))))))) -================== +================================================================================ Table Access -================== +================================================================================ select |number| from foobar { text == 'answer' } ---- +-------------------------------------------------------------------------------- (root - (block - (statement - (select - (identifier_list - (identifier)) - (expression - (identifier)) - (block - (statement - (expression - (logic - (expression - (identifier)) - (logic_operator) - (expression - (value - (string))))))))))) + (statement + (select + (identifier_list + (identifier)) + (expression + (identifier)) + (block + (statement + (expression + (logic + (expression + (identifier)) + (logic_operator) + (expression + (value + (string)))))))))) -================== +================================================================================ Table Insert -================== +================================================================================ insert into foobar [ ['bob was here', 0] ] ---- +-------------------------------------------------------------------------------- (root - (block - (statement - (insert - (identifier) - (expression - (value - (list - (expression - (value - (list - (expression - (value - (string))) - (expression - (value - (integer))))))))))))) + (statement + (insert + (identifier) + (expression + (value + (list + (expression + (value + (list + (expression + (value + (string))) + (expression + (value + (integer)))))))))))) diff --git a/tree-sitter-dust/corpus/transform.txt b/tree-sitter-dust/corpus/transform.txt deleted file mode 100644 index 9b13701..0000000 --- a/tree-sitter-dust/corpus/transform.txt +++ /dev/null @@ -1,88 +0,0 @@ -================================================================================ -Transform Loop -================================================================================ - -transform i in [1, 2, 3] { - (output i) -} - --------------------------------------------------------------------------------- - -(root - (block - (statement - (transform - (identifier) - (expression - (value - (list - (expression - (value - (integer))) - (expression - (value - (integer))) - (expression - (value - (integer)))))) - (block - (statement - (expression - (function_call - (built_in_function - (expression - (identifier))))))))))) - -================================================================================ -Nested Transform Loop -================================================================================ - -transform i in [['one'] ['two'] ['three']] { - transform j in i { - j += 'foobar' - } -} - --------------------------------------------------------------------------------- - -(root - (block - (statement - (transform - (identifier) - (expression - (value - (list - (expression - (value - (list - (expression - (value - (string)))))) - (expression - (value - (list - (expression - (value - (string)))))) - (expression - (value - (list - (expression - (value - (string))))))))) - (block - (statement - (transform - (identifier) - (expression - (identifier)) - (block - (statement - (assignment - (identifier) - (assignment_operator) - (statement - (expression - (value - (string)))))))))))))) diff --git a/tree-sitter-dust/corpus/values.txt b/tree-sitter-dust/corpus/values.txt index 7bc2cc7..239e309 100644 --- a/tree-sitter-dust/corpus/values.txt +++ b/tree-sitter-dust/corpus/values.txt @@ -1,82 +1,79 @@ -================== +================================================================================ Booleans -================== +================================================================================ true false ---- +-------------------------------------------------------------------------------- (root - (block - (statement - (expression - (value - (boolean)))) - (statement - (expression - (value - (boolean)))))) + (statement + (expression + (value + (boolean)))) + (statement + (expression + (value + (boolean))))) -================== +================================================================================ Integers -================== +================================================================================ 1 2 3 456 7 ---- +-------------------------------------------------------------------------------- (root - (block - (statement - (expression - (value - (integer)))) - (statement - (expression - (value - (integer)))) - (statement - (expression - (value - (integer)))) - (statement - (expression - (value - (integer)))) - (statement - (expression - (value - (integer)))))) + (statement + (expression + (value + (integer)))) + (statement + (expression + (value + (integer)))) + (statement + (expression + (value + (integer)))) + (statement + (expression + (value + (integer)))) + (statement + (expression + (value + (integer))))) -================== +================================================================================ Strings -================== +================================================================================ "one" 'two' "three" `four` 'five' ---- +-------------------------------------------------------------------------------- (root - (block - (statement - (expression - (value - (string)))) - (statement - (expression - (value - (string)))) - (statement - (expression - (value - (string)))) - (statement - (expression - (value - (string)))) - (statement - (expression - (value - (string)))))) + (statement + (expression + (value + (string)))) + (statement + (expression + (value + (string)))) + (statement + (expression + (value + (string)))) + (statement + (expression + (value + (string)))) + (statement + (expression + (value + (string))))) diff --git a/tree-sitter-dust/corpus/while.txt b/tree-sitter-dust/corpus/while.txt index aba7a82..4733771 100644 --- a/tree-sitter-dust/corpus/while.txt +++ b/tree-sitter-dust/corpus/while.txt @@ -9,20 +9,19 @@ while true { -------------------------------------------------------------------------------- (root - (block - (statement - (while - (expression - (value - (boolean))) - (block - (statement - (expression - (function_call - (built_in_function - (expression - (value - (string)))))))))))) + (statement + (while + (expression + (value + (boolean))) + (block + (statement + (expression + (function_call + (built_in_function + (expression + (value + (string))))))))))) ================================================================================ Nested While Loop @@ -37,29 +36,28 @@ while (true) { -------------------------------------------------------------------------------- (root - (block - (statement - (while - (expression - (value - (boolean))) - (block - (statement - (while - (expression - (logic - (expression - (identifier)) - (logic_operator) - (expression - (value - (integer))))) - (block - (statement - (assignment - (identifier) - (assignment_operator) - (statement - (expression - (value - (integer)))))))))))))) + (statement + (while + (expression + (value + (boolean))) + (block + (statement + (while + (expression + (logic + (expression + (identifier)) + (logic_operator) + (expression + (value + (integer))))) + (block + (statement + (assignment + (identifier) + (assignment_operator) + (statement + (expression + (value + (integer))))))))))))) diff --git a/tree-sitter-dust/corpus/yield.txt b/tree-sitter-dust/corpus/yield.txt new file mode 100644 index 0000000..1883d2f --- /dev/null +++ b/tree-sitter-dust/corpus/yield.txt @@ -0,0 +1,41 @@ +================================================================================ +Simple Yield +================================================================================ + +1 -> (output) + +-------------------------------------------------------------------------------- + +(root + (statement + (expression + (yield + (expression + (value + (integer))) + (built_in_function))))) + +================================================================================ +Yield Chain +================================================================================ + +x -> (foo) -> (bar) -> (abc) + +-------------------------------------------------------------------------------- + +(root + (statement + (expression + (yield + (expression + (yield + (expression + (yield + (expression + (identifier)) + (expression + (identifier)))) + (expression + (identifier)))) + (expression + (identifier)))))) diff --git a/tree-sitter-dust/grammar.js b/tree-sitter-dust/grammar.js index 719c6a6..54b1a88 100644 --- a/tree-sitter-dust/grammar.js +++ b/tree-sitter-dust/grammar.js @@ -3,57 +3,57 @@ module.exports = grammar({ word: $ => $.identifier, - extras: $ => [ /\s/, $.comment ], - - conflicts: $ => [ - [$.block], - ], + extras: $ => [ /\s/, $._comment ], rules: { - root: $ => $.block, + root: $ => prec(1, repeat1($.statement)), - comment: $ => /[#][^#\n]*[#|\n]/, + _comment: $ => /[#][^#\n]*[#|\n]/, - block: $ => prec.right(choice( + block: $ => seq( + optional('async'), + '{', repeat1($.statement), - seq('{', repeat1($.statement), '}'), - )), + '}', + ), - statement: $ => prec.right(seq( + statement: $ => prec.left(seq( choice( $.assignment, - $.await, + $.block, $.expression, - $.filter, - $.find, $.for, $.if_else, + $.index_assignment, $.insert, $.match, - $.reduce, - $.remove, + $.return, $.select, - $.transform, + $.use, $.while, ), optional(';'), )), - + expression: $ => prec.right(choice( $._expression_kind, seq('(', $._expression_kind, ')'), )), - _expression_kind: $ => prec.right(1, choice( + _expression_kind: $ => prec.right(choice( $.function_call, $.identifier, $.index, $.logic, $.math, $.value, + $.yield, )), - _expression_list: $ => repeat1(prec.right(seq($.expression, optional(',')))), + _expression_list: $ => repeat1(prec.right(seq( + $.expression, + optional(','), + ))), identifier: $ => /[_a-zA-Z]+[_a-zA-Z0-9]?/, @@ -113,7 +113,7 @@ module.exports = grammar({ '}', ), - index: $ => prec.left(seq( + index: $ => prec.left(1, seq( $.expression, ':', $.expression, @@ -155,16 +155,23 @@ module.exports = grammar({ ), assignment: $ => seq( - $.identifier, + field('identifier', $.identifier), + optional(field('type', $.type)), + field('assignment_operator', $.assignment_operator), + field('statement', $.statement), + ), + + index_assignment: $ => seq( + $.index, $.assignment_operator, $.statement, ), - assignment_operator: $ => choice( + assignment_operator: $ => prec.right(choice( "=", "+=", "-=", - ), + )), if_else: $ => prec.right(seq( $.if, @@ -206,50 +213,10 @@ module.exports = grammar({ ), for: $ => seq( - 'for', - $.identifier, - 'in', - $.expression, - $.block, - ), - - transform: $ => seq( - 'transform', - $.identifier, - 'in', - $.expression, - $.block, - ), - - filter: $ => seq( - 'filter', - field('count', optional($.expression)), - field('statement_id', $.identifier), - 'in', - field('collection', $.expression), - field('predicate', $.block), - ), - - find: $ => seq( - 'find', - $.identifier, - 'in', - $.expression, - $.block, - ), - - remove: $ => seq( - 'remove', - $.identifier, - 'from', - $.expression, - $.block, - ), - - reduce: $ => seq( - 'reduce', - $.identifier, - 'to', + choice( + 'for', + 'async for', + ), $.identifier, 'in', $.expression, @@ -270,7 +237,7 @@ module.exports = grammar({ $.identifier, $.expression, )), - + identifier_list: $ => prec.right(choice( seq( '|', @@ -285,19 +252,56 @@ module.exports = grammar({ $.expression, )), + return: $ => seq( + 'return', + $.expression, + ), + + use: $ => seq( + 'use', + $.string, + ), + + type: $ => seq( + '<', + choice( + 'any', + 'bool', + 'fn', + 'int', + 'list', + 'map', + 'str', + 'table', + ), + '>', + ), + function: $ => seq( - field('parameters', optional($.identifier_list)), - '=>', - field('body', $.block), + '|', + repeat($.parameter), + '|', + optional($.type), + $.block, ), - function_call: $ => choice( - $.built_in_function, - $._context_defined_function, - ), - - _context_defined_function: $ => prec.right(seq( + parameter: $ => seq( $.identifier, + $.type, + optional(','), + ), + + function_call: $ => prec.right(1, seq( + '(', + choice( + $.built_in_function, + $._context_defined_function, + ), + ')', + )), + + _context_defined_function: $ => prec.right(1, seq( + $.expression, optional($._expression_list), )), @@ -306,10 +310,22 @@ module.exports = grammar({ optional($._expression_list), )), + yield: $ => prec.left(seq( + $.expression, + '->', + '(', + choice( + $.built_in_function, + $._context_defined_function, + ), + ')', + )), + _built_in_function_name: $ => choice( // General 'assert', 'assert_equal', + 'context', 'download', 'help', 'length', @@ -352,4 +368,3 @@ module.exports = grammar({ 'reverse', ), } -}); \ No newline at end of file