Clean up; Complete async

README.md

@@ -137,8 +137,6 @@ list = [ 1, 2, 3 ]
 for number in list {
     (output number + 1)
 }
-
-# The original list is left unchanged.
 ```
 
 To create a new list, use a **transform** loop, which modifies the values into a new list without changing the original.
@@ -150,6 +148,10 @@ new_list = transform number in list {
     number - 1
 }
 
+list
+    -> filter()
+    -> ()
+
 (output new_list)
 # Output: [ 0 1 2 ]
 

examples/async_download.ds

@@ -0,0 +1,11 @@
+data = async { {
+		cast = (download "https://api.sampleapis.com/futurama/cast")
+		characters = (download "https://api.sampleapis.com/futurama/characters")	
+		episodes = (download "https://api.sampleapis.com/futurama/episodes")
+} }
+
+cast_len = (length (from_json data:cast))
+characters_len = (length (from_json data:characters))
+episodes_len = (length (from_json data:episodes))
+
+(output [cast_len, characters_len, episodes_len ])

examples/clue_solver.ds

@@ -1,44 +1,63 @@
-rooms = ['Library' 'Kitchen']
+all_rooms = ['Library' 'Kitchen']
-suspects = ['White' 'Green']
+all_suspects = ['White' 'Green']
-weapons = ['Rope' 'Lead_Pipe']
+all_weapons = ['Rope' 'Lead_Pipe']
-cards = [rooms suspects weapons]
 
-take_turn = |current_room opponent_card| => {
+is_ready_to_solve = |state| => {
-	(remove_card opponent_card)
+	((length state:suspects) == 1)
-	(make_guess current_room)
+	&& ((length state:rooms) == 1)
-} 
+	&& ((length state:weapons) == 1)
-
-remove_card = |opponent_card| => {
-	for card_list in cards {
-		removed = remove card from card_list 
-			card == opponent_card
-	}
-
-	if (type removed) == 'empty' 
-		output 'Card not found.'
 }
 
-make_guess = |current_room| => {
+take_turn = |opponent_card state| => {
-	if ((length suspects) == 1) 
+	state = (remove_card state opponent_card)
-		&& ((length rooms) == 1)
+	(make_guess state)
-		&& ((length weapons) == 1)
+	state
+} 
+
+remove_card = |state opponent_card| => {
+	rooms = filter card in state:rooms {
+		card != opponent_card
+	}
+	suspects = filter card in state:suspects {
+		card != opponent_card
+	}
+	weapons = filter card in state:weapons {
+		card != opponent_card
+	}
+
 	{
+		current_room = state:current_room
+		rooms = rooms
+		suspects = suspects
+		weapons = weapons
+	}
+}
+
+make_guess = |state| => {
+	if (is_ready_to_solve state) {
 		(output 'It was '
-			+ suspects:0
+			+ state:suspects:0 
-			+ ' in the '
+			+ ' in the ' 
-			+ rooms:0 
+			+ state:rooms:0  
-			+ ' with the '
+			+ ' with the ' 
-			+ weapons:0
+			+ state:weapons:0 
 			+ '!')
 	} else {
 		(output 'I accuse '
-			+ (random suspects)
+			+ (random state:suspects)
 			+ ' in the '
-			+ current_room
+			# + state:current_room
 			+ ' with the '
-			+ (random weapons)
+			+ (random state:weapons)
 			+ '!')
 	}
 }
 
-(make_guess 'Library')
+init_state = {
+	current_room = 'Library'
+	rooms = all_rooms
+	suspects = all_suspects
+	weapons = all_weapons
+}
+
+(take_turn 'Green' (take_turn 'Kitchen' (take_turn 'Rope' init_state)))

examples/fannkuch-redux.ds

@@ -1,21 +0,0 @@
-numbers = (from_json input)
-flip_count = 0
-checksum = 0
-
-while numbers.0 != 1 {
-	(reverse numbers 0 numbers.0)
-	
-	if flip_count % 2 == 0 {
-		checksum += flip_count
-	} else {
-		checksum -= flip_count
-	}
-
-	checksum += flip_count * 1
-	
-	flip_count += 1
-}
-
-(output numbers)
-(output flip_count)
-(output checksum)

examples/select.ds

@@ -0,0 +1,15 @@
+my_table = table |text number bool| [
+	["a", 1, true]
+	["b", 2, true]
+	["a", 3, true]
+]
+
+test_table = table |text bool| [
+	["a", true]
+	["b", true]
+	["a", true]
+]
+
+test_select = select |text bool| from my_table;
+
+(assert_equal test_select, test_table)

examples/table.ds

@@ -10,11 +10,11 @@ test_table = table |text bool| [
 	["a", true]
 ]
 
-test_select = select |text bool| from my_table
+test_select = select |text bool| from my_table;
 
 (assert_equal test_select, test_table)
 
-test_table = table |text number bool| [
+test_table = table |number bool| [
 	[1, true]
 	[3, true]
 ]

src/abstract_tree/async.rs

@@ -29,7 +29,6 @@ impl AbstractTree for Async {
                 let mut context = context.clone();
                 let result = statement.run(source, &mut context);
 
-                result.clone().unwrap();
                 if result.is_err() {
                     Some(result)
                 } else if index == statements.len() - 1 {

src/abstract_tree/block.rs

@@ -5,7 +5,7 @@ use crate::{AbstractTree, Result, Statement};
 
 #[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq, PartialOrd, Ord)]
 pub struct Block {
-    statements: Vec<Statement>,
+    pub statements: Vec<Statement>,
 }
 
 impl Block {

src/abstract_tree/built_in_function.rs

@@ -572,11 +572,8 @@ impl AbstractTree for BuiltInFunction {
                     let value = expressions[0].run(source, context)?;
                     let list = value.as_list()?.items();
 
-                    if list.len() < 2 {
+                    if list.len() == 1 {
-                        return Err(Error::ExpectedMinLengthList {
+                        return Ok(list.first().cloned().unwrap());
-                            minimum_len: 2,
-                            actual_len: list.len(),
-                        });
                     }
 
                     let range = 0..list.len();

src/abstract_tree/filter.rs

@@ -19,13 +19,13 @@ impl AbstractTree for Filter {
             None => None,
         };
 
-        let item_id_node = node.child(1).unwrap();
+        let item_id_node = node.child_by_field_name("item_id").unwrap();
         let item_id = Identifier::from_syntax_node(source, item_id_node)?;
 
-        let collection_node = node.child(3).unwrap();
+        let collection_node = node.child_by_field_name("collection").unwrap();
         let collection = Expression::from_syntax_node(source, collection_node)?;
 
-        let predicate_node = node.child(5).unwrap();
+        let predicate_node = node.child_by_field_name("predicate").unwrap();
         let predicate = Block::from_syntax_node(source, predicate_node)?;
 
         Ok(Filter {
@@ -45,6 +45,7 @@ impl AbstractTree for Filter {
             Some(expression) => Some(expression.run(source, context)?.as_integer()? as usize),
             None => None,
         };
+        let loop_context = Map::clone_from(context);
 
         values.par_iter().try_for_each(|value| {
             if let Some(max) = count {
@@ -53,11 +54,16 @@ impl AbstractTree for Filter {
                 }
             }
 
-            let mut context = Map::new();
+            let mut iter_context = loop_context.clone();
 
-            context.variables_mut().insert(key.clone(), value.clone());
+            iter_context
+                .variables_mut()
+                .insert(key.clone(), value.clone());
 
-            let should_include = self.predicate.run(source, &mut context)?.as_boolean()?;
+            let should_include = self
+                .predicate
+                .run(source, &mut iter_context)?
+                .as_boolean()?;
 
             if should_include {
                 new_values.items_mut().push(value.clone());

src/abstract_tree/for.rs

@@ -7,9 +7,9 @@ use crate::{AbstractTree, Block, Error, Expression, Identifier, Map, Result, Val
 #[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq, PartialOrd, Ord)]
 pub struct For {
     is_async: bool,
-    identifier: Identifier,
+    item_id: Identifier,
-    expression: Expression,
+    collection: Expression,
-    item: Block,
+    block: Block,
 }
 
 impl AbstractTree for For {
@@ -17,9 +17,10 @@ impl AbstractTree for For {
         let for_node = node.child(0).unwrap();
         let is_async = match for_node.kind() {
             "for" => false,
+            "async for" => true,
             _ => {
                 return Err(Error::UnexpectedSyntaxNode {
-                    expected: "for",
+                    expected: "for or async for",
                     actual: for_node.kind(),
                     location: for_node.start_position(),
                     relevant_source: source[for_node.byte_range()].to_string(),
@@ -38,32 +39,35 @@ impl AbstractTree for For {
 
         Ok(For {
             is_async,
-            identifier,
+            item_id: identifier,
-            expression,
+            collection: expression,
-            item,
+            block: item,
         })
     }
 
     fn run(&self, source: &str, context: &mut Map) -> Result<Value> {
-        let expression_run = self.expression.run(source, context)?;
+        let expression_run = self.collection.run(source, context)?;
         let values = expression_run.as_list()?.items();
-        let key = self.identifier.inner();
+        let key = self.item_id.inner();
+        let mut loop_context = Map::clone_from(context);
 
         if self.is_async {
             values.par_iter().try_for_each(|value| {
-                let mut iter_context = Map::new();
+                let mut iter_context = loop_context.clone();
 
                 iter_context
                     .variables_mut()
                     .insert(key.clone(), value.clone());
 
-                self.item.run(source, &mut iter_context).map(|_value| ())
+                self.block.run(source, &mut iter_context).map(|_value| ())
             })?;
         } else {
             for value in values.iter() {
-                context.variables_mut().insert(key.clone(), value.clone());
+                loop_context
+                    .variables_mut()
+                    .insert(key.clone(), value.clone());
 
-                self.item.run(source, context)?;
+                self.block.run(source, &mut loop_context)?;
             }
         }
 

src/abstract_tree/function_call.rs

@@ -22,10 +22,10 @@ impl AbstractTree for FunctionCall {
         let mut arguments = Vec::new();
 
         for index in 1..node.child_count() {
-            let child = node.child(index).unwrap();
+            let node = node.child(index).unwrap();
 
-            if child.is_named() {
+            if node.is_named() {
-                let expression = Expression::from_syntax_node(source, child)?;
+                let expression = Expression::from_syntax_node(source, node)?;
 
                 arguments.push(expression);
             }

src/abstract_tree/remove.rs

@@ -1,63 +1,53 @@
 use serde::{Deserialize, Serialize};
 use tree_sitter::Node;
 
-use crate::{AbstractTree, Block, Expression, Identifier, Map, Result, Value};
+use crate::{AbstractTree, Block, Error, Expression, Identifier, Map, Result, Value};
 
 #[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq, PartialOrd, Ord)]
 pub struct Remove {
-    identifier: Identifier,
+    item_id: Identifier,
-    expression: Expression,
+    collection: Expression,
-    item: Block,
+    predicate: Block,
 }
 
 impl AbstractTree for Remove {
     fn from_syntax_node(source: &str, node: Node) -> Result<Self> {
         let identifier_node = node.child(1).unwrap();
-        let identifier = Identifier::from_syntax_node(source, identifier_node)?;
+        let item_id = Identifier::from_syntax_node(source, identifier_node)?;
 
         let expression_node = node.child(3).unwrap();
-        let expression = Expression::from_syntax_node(source, expression_node)?;
+        let collection = Expression::from_syntax_node(source, expression_node)?;
 
-        let item_node = node.child(4).unwrap();
+        let block_node = node.child(4).unwrap();
-        let item = Block::from_syntax_node(source, item_node)?;
+        let predicate = Block::from_syntax_node(source, block_node)?;
 
         Ok(Remove {
-            identifier,
+            item_id,
-            expression,
+            collection,
-            item,
+            predicate,
         })
     }
 
     fn run(&self, source: &str, context: &mut Map) -> Result<Value> {
-        let expression_run = self.expression.run(source, context)?;
+        let value = self.collection.run(source, context)?;
-        let values = expression_run.into_inner_list()?;
+        let mut values = value.as_list()?.items_mut();
-        let key = self.identifier.inner();
+        let key = self.item_id.inner();
-        let mut sub_context = context.clone();
         let mut should_remove_index = None;
 
-        for (index, value) in values.items().iter().enumerate() {
+        values.iter().enumerate().try_for_each(|(index, value)| {
-            sub_context
+            context.variables_mut().insert(key.clone(), value.clone());
-                .variables_mut()
-                .insert(key.clone(), value.clone());
 
-            let should_remove = self.item.run(source, &mut sub_context)?.as_boolean()?;
+            let should_remove = self.predicate.run(source, context)?.as_boolean()?;
 
             if should_remove {
                 should_remove_index = Some(index);
-
-                match &self.expression {
-                    Expression::Identifier(identifier) => {
-                        sub_context
-                            .variables_mut()
-                            .insert(identifier.inner().clone(), Value::List(values.clone()));
-                    }
-                    _ => {}
-                }
             }
-        }
+
+            Ok::<(), Error>(())
+        })?;
 
         if let Some(index) = should_remove_index {
-            Ok(values.items_mut().remove(index))
+            Ok(values.remove(index))
         } else {
             Ok(Value::Empty)
         }

src/abstract_tree/select.rs

@@ -7,7 +7,7 @@ use crate::{AbstractTree, Block, Expression, Identifier, Map, Result, Table, Val
 pub struct Select {
     identifiers: Vec<Identifier>,
     expression: Expression,
-    item: Option<Block>,
+    predicate: Option<Block>,
 }
 
 impl AbstractTree for Select {
@@ -15,41 +15,32 @@ impl AbstractTree for Select {
         let child_count = node.child_count();
         let mut identifiers = Vec::new();
 
-        for index in 2..child_count - 4 {
+        let identifier_list = node.child(1).unwrap();
-            let node = node.child(index).unwrap();
 
-            if node.kind() == "identifier" {
+        for index in 1..identifier_list.child_count() - 1 {
+            let node = identifier_list.child(index).unwrap();
+
+            if node.is_named() {
                 let identifier = Identifier::from_syntax_node(source, node)?;
                 identifiers.push(identifier);
             }
-
-            if node.kind() == ">" {
-                break;
-            }
         }
 
+        let expression_node = node.child(3).unwrap();
+        let expression = Expression::from_syntax_node(source, expression_node)?;
+
         let final_node = node.child(child_count - 1).unwrap();
 
-        let item = if final_node.kind() == "}" {
+        let predicate = if final_node.kind() == "block" {
-            let item_node = node.child(child_count - 2).unwrap();
+            Some(Block::from_syntax_node(source, final_node)?)
-
-            Some(Block::from_syntax_node(source, item_node)?)
         } else {
             None
         };
 
-        let expression_node = if item.is_some() {
-            node.child(child_count - 4).unwrap()
-        } else {
-            node.child(child_count - 1).unwrap()
-        };
-
-        let expression = Expression::from_syntax_node(source, expression_node)?;
-
         Ok(Select {
             identifiers,
             expression,
-            item,
+            predicate,
         })
     }
 
@@ -60,7 +51,7 @@ impl AbstractTree for Select {
             self.identifiers
                 .iter()
                 .cloned()
-                .map(|identifierier| identifierier.take_inner())
+                .map(|identifier| identifier.take_inner())
                 .collect()
         } else {
             old_table.headers().clone()
@@ -99,7 +90,7 @@ impl AbstractTree for Select {
                 }
             }
 
-            if let Some(where_clause) = &self.item {
+            if let Some(where_clause) = &self.predicate {
                 let should_include = where_clause.run(source, &mut row_context)?.as_boolean()?;
 
                 if should_include {

src/evaluator.rs

@@ -33,9 +33,13 @@ pub fn evaluate(source: &str) -> Result<Value> {
 /// # use dust_lang::*;
 /// let mut context = Map::new();
 ///
-/// context.set_value("one".into(), 1.into());
+/// {
-/// context.set_value("two".into(), 2.into());
+///     let mut variables = context.variables_mut();
-/// context.set_value("three".into(), 3.into());
+///
+///     variables.insert("one".into(), 1.into());
+///     variables.insert("two".into(), 2.into());
+///     variables.insert("three".into(), 3.into());
+/// }
 ///
 /// let dust_code = "four = 4 one + two + three + four";
 ///

tests/dust_examples.rs

@@ -9,6 +9,14 @@ fn clue_solver() {
     evaluate(&file_contents).unwrap();
 }
 
+#[test]
+#[ignore]
+fn download_async() {
+    let file_contents = read_to_string("examples/download_async.ds").unwrap();
+
+    evaluate(&file_contents).unwrap();
+}
+
 #[test]
 #[ignore]
 fn fetch() {
@@ -59,6 +67,13 @@ fn remove_loop() {
     evaluate(&file_contents).unwrap();
 }
 
+#[test]
+fn select() {
+    let file_contents = read_to_string("examples/select.ds").unwrap();
+
+    evaluate(&file_contents).unwrap();
+}
+
 #[test]
 fn table() {
     let file_contents = read_to_string("examples/table.ds").unwrap();

tree-sitter-dust/corpus/comments.txt

@@ -1,52 +1,47 @@
-==================
+================================================================================
 Full Line Comments
-==================
+================================================================================
 
 not_a_comment
 # comment
 
----
+--------------------------------------------------------------------------------
 
 (root
   (block
     (statement
       (expression
-        (identifier))))
+        (identifier)))))
-    (comment))
 
-==================
+================================================================================
 Partial Line Comments
-==================
+================================================================================
 
 not_a_comment # comment
 
----
+--------------------------------------------------------------------------------
 
 (root
   (block
     (statement
       (expression
-        (identifier))))
+        (identifier)))))
-      (comment))
 
-==================
+================================================================================
 Multiline Comments
-==================
+================================================================================
 
 # comment #
 not_a_comment #
 # comment # "not a comment"
 
----
+--------------------------------------------------------------------------------
 
 (root
-  (comment)
   (block
     (statement
       (expression
         (identifier)))
-    (comment)
-    (comment)
     (statement
       (expression
         (value

tree-sitter-dust/grammar.js

@@ -3,7 +3,7 @@ module.exports = grammar({
 
   word: $ => $.identifier,
 
-  extras: $ => [ /\s/, $.comment ],
+  extras: $ => [ /\s/, $._comment ],
 
   conflicts: $ => [
     [$.block],
@@ -13,7 +13,7 @@ module.exports = grammar({
   rules: {
     root: $ => repeat1($.block),
 
-    comment: $ => /[#][^#\n]*[#|\n]/,
+    _comment: $ => /[#][^#\n]*[#|\n]/,
 
     block: $ => choice(
       repeat1($.statement),
@@ -200,7 +200,10 @@ module.exports = grammar({
     ),
 
     for: $ => seq(
-      'for',
+      choice(
+        'for',
+        'async for',
+      ),
       $.identifier,
       'in',
       $.expression,
@@ -218,7 +221,7 @@ module.exports = grammar({
     filter: $ => seq(
       'filter',
       field('count', optional($.expression)),
-      field('statement_id', $.identifier),
+      field('item_id', $.identifier),
       'in',
       field('collection', $.expression),
       field('predicate', $.block),

tree-sitter-dust/src/grammar.json

@@ -9,7 +9,7 @@
         "name": "block"
       }
     },
-    "comment": {
+    "_comment": {
       "type": "PATTERN",
       "value": "[#][^#\\n]*[#|\\n]"
     },
@@ -708,8 +708,17 @@
       "type": "SEQ",
       "members": [
         {
-          "type": "SYMBOL",
+          "type": "CHOICE",
-          "name": "identifier"
+          "members": [
+            {
+              "type": "SYMBOL",
+              "name": "identifier"
+            },
+            {
+              "type": "SYMBOL",
+              "name": "index"
+            }
+          ]
         },
         {
           "type": "SYMBOL",
@@ -875,8 +884,17 @@
       "type": "SEQ",
       "members": [
         {
-          "type": "STRING",
+          "type": "CHOICE",
-          "value": "for"
+          "members": [
+            {
+              "type": "STRING",
+              "value": "for"
+            },
+            {
+              "type": "STRING",
+              "value": "async for"
+            }
+          ]
         },
         {
           "type": "SYMBOL",
@@ -946,7 +964,7 @@
         },
         {
           "type": "FIELD",
-          "name": "statement_id",
+          "name": "item_id",
           "content": {
             "type": "SYMBOL",
             "name": "identifier"
@@ -1428,7 +1446,7 @@
     },
     {
       "type": "SYMBOL",
-      "name": "comment"
+      "name": "_comment"
     }
   ],
   "conflicts": [

tree-sitter-dust/src/node-types.json

@@ -15,6 +15,10 @@
           "type": "identifier",
           "named": true
         },
+        {
+          "type": "index",
+          "named": true
+        },
         {
           "type": "statement",
           "named": true
@@ -170,6 +174,16 @@
           }
         ]
       },
+      "item_id": {
+        "multiple": false,
+        "required": true,
+        "types": [
+          {
+            "type": "identifier",
+            "named": true
+          }
+        ]
+      },
       "predicate": {
         "multiple": false,
         "required": true,
@@ -179,16 +193,6 @@
             "named": true
           }
         ]
-      },
-      "statement_id": {
-        "multiple": false,
-        "required": true,
-        "types": [
-          {
-            "type": "identifier",
-            "named": true
-          }
-        ]
       }
     }
   },
@@ -846,6 +850,10 @@
     "type": "async",
     "named": false
   },
+  {
+    "type": "async for",
+    "named": false
+  },
   {
     "type": "bash",
     "named": false
@@ -854,10 +862,6 @@
     "type": "columns",
     "named": false
   },
-  {
-    "type": "comment",
-    "named": true
-  },
   {
     "type": "download",
     "named": false