Improve soundness of Map type

This commit is contained in:
Jeff 2023-11-05 13:54:29 -05:00
parent a3db9cb9f2
commit 2d85a3ee2b
28 changed files with 79429 additions and 135883 deletions

7
Cargo.lock generated
View File

@ -468,7 +468,6 @@ dependencies = [
"comfy-table",
"csv",
"git2",
"json",
"rand",
"rayon",
"reqwest",
@ -909,12 +908,6 @@ dependencies = [
"wasm-bindgen",
]
[[package]]
name = "json"
version = "0.12.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "078e285eafdfb6c4b434e0d31e8cfcb5115b651496faca5749b88fafd4f23bfd"
[[package]]
name = "kv-log-macro"
version = "1.0.7"

View File

@ -17,7 +17,6 @@ clap = { version = "4.4.4", features = ["derive"] }
comfy-table = "7.0.1"
csv = "1.2.2"
git2 = "0.18.1"
json = "0.12.4"
rand = "0.8.5"
rayon = "1.8.0"
reqwest = { version = "0.11.20", features = ["blocking", "json"] }

View File

@ -1,11 +1,13 @@
data = async { {
raw_data = async {
{
cast = (download "https://api.sampleapis.com/futurama/cast")
characters = (download "https://api.sampleapis.com/futurama/characters")
episodes = (download "https://api.sampleapis.com/futurama/episodes")
} }
}
}
cast_len = (length (from_json data:cast))
characters_len = (length (from_json data:characters))
episodes_len = (length (from_json data:episodes))
(output [cast_len, characters_len, episodes_len ])
(output [cast_len, characters_len, episodes_len])

View File

@ -50,11 +50,11 @@ impl AbstractTree for Assignment {
fn run(&self, source: &str, context: &mut Map) -> Result<Value> {
let key = self.identifier.inner().clone();
let value = self.statement.run(source, context)?;
let mut context = context.variables_mut();
let mut variables = context.variables_mut()?;
let new_value = match self.operator {
AssignmentOperator::PlusEqual => {
if let Some(mut previous_value) = context.get(&key).cloned() {
if let Some(mut previous_value) = variables.get(&key).cloned() {
previous_value += value;
previous_value
} else {
@ -62,7 +62,7 @@ impl AbstractTree for Assignment {
}
}
AssignmentOperator::MinusEqual => {
if let Some(mut previous_value) = context.get(&key).cloned() {
if let Some(mut previous_value) = variables.get(&key).cloned() {
previous_value -= value;
previous_value
} else {
@ -72,7 +72,7 @@ impl AbstractTree for Assignment {
AssignmentOperator::Equal => value,
};
context.insert(key, new_value);
variables.insert(key, new_value);
Ok(Value::Empty)
}

View File

@ -317,7 +317,7 @@ impl AbstractTree for BuiltInFunction {
let value = expression.run(source, context)?;
let length = match value {
Value::List(list) => list.items().len(),
Value::Map(map) => map.len(),
Value::Map(map) => map.variables()?.len(),
Value::Table(table) => table.len(),
Value::String(string) => string.chars().count(),
Value::Function(_) => todo!(),
@ -405,7 +405,7 @@ impl AbstractTree for BuiltInFunction {
let metadata_output = Map::new();
{
let mut metadata_variables = metadata_output.variables_mut();
let mut metadata_variables = metadata_output.variables_mut()?;
metadata_variables.insert("type".to_string(), Value::String(file_type));
metadata_variables.insert("size".to_string(), Value::Integer(size));

View File

@ -45,7 +45,7 @@ impl AbstractTree for Filter {
Some(expression) => Some(expression.run(source, context)?.as_integer()? as usize),
None => None,
};
let loop_context = Map::clone_from(context);
let loop_context = Map::clone_from(context)?;
values.par_iter().try_for_each(|value| {
if let Some(max) = count {
@ -57,7 +57,7 @@ impl AbstractTree for Filter {
let mut iter_context = loop_context.clone();
iter_context
.variables_mut()
.variables_mut()?
.insert(key.clone(), value.clone());
let should_include = self

View File

@ -32,12 +32,12 @@ impl AbstractTree for Find {
let value = self.expression.run(source, context)?;
let values = value.as_list()?.items();
let key = self.identifier.inner();
let mut context = context.clone();
let mut variables = context.variables_mut()?;
for value in values.iter() {
context.variables_mut().insert(key.clone(), value.clone());
variables.insert(key.clone(), value.clone());
let should_return = self.item.run(source, &mut context)?.as_boolean()?;
let should_return = self.item.run(source, &mut context.clone())?.as_boolean()?;
if should_return {
return Ok(value.clone());

View File

@ -49,25 +49,25 @@ impl AbstractTree for For {
let expression_run = self.collection.run(source, context)?;
let values = expression_run.as_list()?.items();
let key = self.item_id.inner();
let mut loop_context = Map::clone_from(context);
let loop_context = Map::clone_from(context)?;
if self.is_async {
values.par_iter().try_for_each(|value| {
let mut iter_context = loop_context.clone();
iter_context
.variables_mut()
.variables_mut()?
.insert(key.clone(), value.clone());
self.block.run(source, &mut iter_context).map(|_value| ())
})?;
} else {
for value in values.iter() {
loop_context
.variables_mut()
.insert(key.clone(), value.clone());
let mut variables = loop_context.variables_mut()?;
self.block.run(source, &mut loop_context)?;
for value in values.iter() {
variables.insert(key.clone(), value.clone());
self.block.run(source, &mut loop_context.clone())?;
}
}

View File

@ -53,21 +53,22 @@ impl AbstractTree for FunctionCall {
FunctionCall::ContextDefined { name, arguments } => (name, arguments),
};
let definition = if let Some(value) = context.variables().get(name.inner()) {
let definition = if let Some(value) = context.variables()?.get(name.inner()) {
value.as_function().cloned()?
} else {
return Err(Error::FunctionIdentifierNotFound(name.clone()));
};
let mut function_context = Map::clone_from(context);
let mut function_context = Map::clone_from(context)?;
if let Some(parameters) = definition.identifiers() {
let parameter_expression_pairs = parameters.iter().zip(arguments.iter());
let mut variables = function_context.variables_mut()?;
for (identifier, expression) in parameter_expression_pairs {
let key = identifier.clone().take_inner();
let value = expression.run(source, context)?;
function_context.variables_mut().insert(key, value);
variables.insert(key, value);
}
}

View File

@ -30,7 +30,7 @@ impl AbstractTree for Identifier {
}
fn run(&self, _source: &str, context: &mut Map) -> Result<Value> {
if let Some(value) = context.variables().get(&self.0) {
if let Some(value) = context.variables()?.get(&self.0) {
Ok(value.clone())
} else {
Err(Error::VariableIdentifierNotFound(self.inner().clone()))

View File

@ -0,0 +1,79 @@
use serde::{Deserialize, Serialize};
use tree_sitter::Node;
use crate::{AbstractTree, Error, Identifier, Map, Result, Statement, Value};
#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq, PartialOrd, Ord)]
pub struct IdentifierAssignment {
identifier: Identifier,
operator: AssignmentOperator,
statement: Statement,
}
#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq, PartialOrd, Ord)]
pub enum AssignmentOperator {
Equal,
PlusEqual,
MinusEqual,
}
impl AbstractTree for IdentifierAssignment {
fn from_syntax_node(source: &str, node: Node) -> Result<Self> {
let identifier_node = node.child(0).unwrap();
let identifier = Identifier::from_syntax_node(source, identifier_node)?;
let operator_node = node.child(1).unwrap().child(0).unwrap();
let operator = match operator_node.kind() {
"=" => AssignmentOperator::Equal,
"+=" => AssignmentOperator::PlusEqual,
"-=" => AssignmentOperator::MinusEqual,
_ => {
return Err(Error::UnexpectedSyntaxNode {
expected: "=, += or -=",
actual: operator_node.kind(),
location: operator_node.start_position(),
relevant_source: source[operator_node.byte_range()].to_string(),
})
}
};
let statement_node = node.child(2).unwrap();
let statement = Statement::from_syntax_node(source, statement_node)?;
Ok(IdentifierAssignment {
identifier,
operator,
statement,
})
}
fn run(&self, source: &str, context: &mut Map) -> Result<Value> {
let key = self.identifier.inner().clone();
let value = self.statement.run(source, context)?;
let mut context = context.variables_mut();
let new_value = match self.operator {
AssignmentOperator::PlusEqual => {
if let Some(mut previous_value) = context.get(&key).cloned() {
previous_value += value;
previous_value
} else {
Value::Empty
}
}
AssignmentOperator::MinusEqual => {
if let Some(mut previous_value) = context.get(&key).cloned() {
previous_value -= value;
previous_value
} else {
Value::Empty
}
}
AssignmentOperator::Equal => value,
};
context.insert(key, new_value);
Ok(Value::Empty)
}
}

View File

@ -0,0 +1,62 @@
use serde::{Deserialize, Serialize};
use tree_sitter::Node;
use crate::{AbstractTree, Error, Index, Map, Result, Statement, Value};
#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq, PartialOrd, Ord)]
pub struct IndexAssignment {
index: Index,
operator: AssignmentOperator,
statement: Statement,
}
#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq, PartialOrd, Ord)]
pub enum AssignmentOperator {
Equal,
PlusEqual,
MinusEqual,
}
impl AbstractTree for IndexAssignment {
fn from_syntax_node(source: &str, node: Node) -> Result<Self> {
let index_node = node.child(0).unwrap();
let index = Index::from_syntax_node(source, index_node)?;
let operator_node = node.child(1).unwrap().child(0).unwrap();
let operator = match operator_node.kind() {
"=" => AssignmentOperator::Equal,
"+=" => AssignmentOperator::PlusEqual,
"-=" => AssignmentOperator::MinusEqual,
_ => {
return Err(Error::UnexpectedSyntaxNode {
expected: "=, += or -=",
actual: operator_node.kind(),
location: operator_node.start_position(),
relevant_source: source[operator_node.byte_range()].to_string(),
})
}
};
let statement_node = node.child(2).unwrap();
let statement = Statement::from_syntax_node(source, statement_node)?;
Ok(IndexAssignment {
index,
operator,
statement,
})
}
fn run(&self, source: &str, context: &mut Map) -> Result<Value> {
let mut left = self.index.run(source, context)?;
let right = self.statement.run(source, context)?;
match self.operator {
AssignmentOperator::PlusEqual => left += right,
AssignmentOperator::MinusEqual => left -= right,
AssignmentOperator::Equal => left = right,
}
Ok(Value::Empty)
}
}

View File

@ -36,7 +36,7 @@ impl AbstractTree for Insert {
}
context
.variables_mut()
.variables_mut()?
.insert(table_name, Value::Table(table));
Ok(Value::Empty)

View File

@ -33,11 +33,15 @@ impl AbstractTree for Remove {
let mut values = value.as_list()?.items_mut();
let key = self.item_id.inner();
let mut should_remove_index = None;
let mut variables = context.variables_mut()?;
values.iter().enumerate().try_for_each(|(index, value)| {
context.variables_mut().insert(key.clone(), value.clone());
variables.insert(key.clone(), value.clone());
let should_remove = self.predicate.run(source, context)?.as_boolean()?;
let should_remove = self
.predicate
.run(source, &mut context.clone())?
.as_boolean()?;
if should_remove {
should_remove_index = Some(index);

View File

@ -60,14 +60,13 @@ impl AbstractTree for Select {
for row in old_table.rows() {
let mut new_row = Vec::new();
let mut row_context = Map::new();
let row_context = Map::new();
let mut row_variables = row_context.variables_mut()?;
for (i, value) in row.iter().enumerate() {
let column_name = old_table.headers().get(i).unwrap();
row_context
.variables_mut()
.insert(column_name.clone(), value.clone());
row_variables.insert(column_name.clone(), value.clone());
let new_table_column_index =
new_table
@ -91,7 +90,9 @@ impl AbstractTree for Select {
}
if let Some(where_clause) = &self.predicate {
let should_include = where_clause.run(source, &mut row_context)?.as_boolean()?;
let should_include = where_clause
.run(source, &mut row_context.clone())?
.as_boolean()?;
if should_include {
new_table.insert(new_row)?;

View File

@ -36,13 +36,15 @@ impl AbstractTree for Transform {
let new_values = values
.par_iter()
.map(|value| {
let mut iter_context = Map::new();
let iter_context = Map::new();
let mut iter_variables = match iter_context.variables_mut() {
Ok(variables) => variables,
Err(_) => return Value::Empty,
};
iter_context
.variables_mut()
.insert(key.clone(), value.clone());
iter_variables.insert(key.clone(), value.clone());
let item_run = self.item.run(source, &mut iter_context);
let item_run = self.item.run(source, &mut iter_context.clone());
match item_run {
Ok(value) => value,

View File

@ -168,10 +168,14 @@ impl AbstractTree for ValueNode {
ValueType::Map(nodes) => {
let map = Map::new();
for (key, node) in nodes {
let value = node.run(source, context)?;
{
let mut variables = map.variables_mut()?;
map.variables_mut().insert(key.clone(), value);
for (key, node) in nodes {
let value = node.run(source, context)?;
variables.insert(key.clone(), value);
}
}
Value::Map(map)

View File

@ -0,0 +1,28 @@
use serde::{Deserialize, Serialize};
use tree_sitter::Node;
use crate::{AbstractTree, Expression, Function, FunctionCall, Result, Value, ValueNode};
#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq, PartialOrd, Ord)]
pub struct Yield {
input: Expression,
call: FunctionCall,
}
impl AbstractTree for Yield {
fn from_syntax_node(source: &str, node: Node) -> Result<Self> {
let input_node = node.child(0).unwrap();
let input = Expression::from_syntax_node(source, input_node)?;
let call_node = node.child(1).unwrap();
let call = FunctionCall::from_syntax_node(source, call_node)?;
Ok(Yield { input, call })
}
fn run(&self, source: &str, context: &mut crate::Map) -> Result<Value> {
let target = self.input.run(source, context)?.as_function()?;
self.call.run(, )
}
}

View File

@ -5,7 +5,7 @@
use crate::{value::Value, Identifier};
use std::{fmt, io, time, string::FromUtf8Error, num::ParseFloatError};
use std::{fmt, io, time, string::FromUtf8Error, num::ParseFloatError, sync::PoisonError};
pub type Result<T> = std::result::Result<T, Error>;
@ -145,6 +145,12 @@ impl Error {
}
}
impl<T> From<PoisonError<T>> for Error {
fn from(value: PoisonError<T>) -> Self {
Error::ToolFailure(value.to_string())
}
}
impl From<FromUtf8Error> for Error {
fn from(value: FromUtf8Error) -> Self {
Error::ToolFailure(value.to_string())
@ -163,12 +169,6 @@ impl From<csv::Error> for Error {
}
}
impl From<json::Error> for Error {
fn from(value: json::Error) -> Self {
Error::ToolFailure(value.to_string())
}
}
impl From<io::Error> for Error {
fn from(value: std::io::Error) -> Self {
Error::ToolFailure(value.to_string())

View File

@ -153,10 +153,12 @@ mod tests {
fn evaluate_map() {
let map = Map::new();
map.variables_mut()
.insert("x".to_string(), Value::Integer(1));
map.variables_mut()
.insert("foo".to_string(), Value::String("bar".to_string()));
{
let mut variables = map.variables_mut().unwrap();
variables.insert("x".to_string(), Value::Integer(1));
variables.insert("foo".to_string(), Value::String("bar".to_string()));
}
assert_eq!(evaluate("{ x = 1, foo = 'bar' }"), Ok(Value::Map(map)));
}

View File

@ -55,6 +55,7 @@ async fn main() {
if let Some(input) = args.input {
context
.variables_mut()
.unwrap()
.insert("input".to_string(), Value::String(input));
}
@ -63,6 +64,7 @@ async fn main() {
context
.variables_mut()
.unwrap()
.insert("input".to_string(), Value::String(file_contents));
}

View File

@ -6,7 +6,7 @@ use std::{
sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard},
};
use crate::{value::Value, List, Table};
use crate::{value::Value, List, Result, Table};
/// A collection dust variables comprised of key-value pairs.
///
@ -25,34 +25,24 @@ impl Map {
}
}
pub fn clone_from(other: &Self) -> Self {
pub fn clone_from(other: &Self) -> Result<Self> {
let mut new_map = BTreeMap::new();
for (key, value) in other.variables().iter() {
for (key, value) in other.variables()?.iter() {
new_map.insert(key.clone(), value.clone());
}
Map {
Ok(Map {
variables: Arc::new(RwLock::new(new_map)),
}
})
}
pub fn variables(&self) -> RwLockReadGuard<BTreeMap<String, Value>> {
self.variables.read().unwrap()
pub fn variables(&self) -> Result<RwLockReadGuard<BTreeMap<String, Value>>> {
Ok(self.variables.read()?)
}
pub fn variables_mut(&self) -> RwLockWriteGuard<BTreeMap<String, Value>> {
self.variables.write().unwrap()
}
/// Returns the number of stored variables.
pub fn len(&self) -> usize {
self.variables.read().unwrap().len()
}
/// Returns true if the length is zero.
pub fn is_empty(&self) -> bool {
self.variables.read().unwrap().is_empty()
pub fn variables_mut(&self) -> Result<RwLockWriteGuard<BTreeMap<String, Value>>> {
Ok(self.variables.write()?)
}
}
@ -104,12 +94,12 @@ impl Display for Map {
}
}
impl From<&Table> for Map {
fn from(value: &Table) -> Self {
impl From<&Table> for Result<Map> {
fn from(value: &Table) -> Result<Map> {
let map = Map::new();
for (row_index, row) in value.rows().iter().enumerate() {
map.variables_mut()
map.variables_mut()?
.insert(
row_index.to_string(),
Value::List(List::with_items(row.clone())),
@ -117,7 +107,7 @@ impl From<&Table> for Map {
.unwrap();
}
map
Ok(map)
}
}

View File

@ -4,7 +4,6 @@ use crate::{
Function, List, Map, Table, ValueType,
};
use json::JsonValue;
use serde::{
de::{MapAccess, SeqAccess, Visitor},
ser::SerializeTuple,
@ -207,7 +206,7 @@ impl Value {
match self {
Value::Table(table) => Ok(table.clone()),
Value::List(list) => Ok(Table::from(list)),
Value::Map(map) => Ok(Table::from(map)),
Value::Map(map) => Result::from(map),
value => Err(Error::ExpectedTable {
actual: value.clone(),
}),
@ -496,82 +495,6 @@ impl From<()> for Value {
}
}
impl TryFrom<JsonValue> for Value {
type Error = Error;
fn try_from(json_value: JsonValue) -> Result<Self> {
use JsonValue::*;
match json_value {
Null => Ok(Value::Empty),
Short(short) => Ok(Value::String(short.to_string())),
String(string) => Ok(Value::String(string)),
Number(number) => Ok(Value::Float(f64::from(number))),
Boolean(boolean) => Ok(Value::Boolean(boolean)),
Object(object) => {
let map = Map::new();
for (key, node_value) in object.iter() {
let value = Value::try_from(node_value)?;
map.variables_mut().insert(key.to_string(), value);
}
Ok(Value::Map(map))
}
Array(array) => {
let mut values = Vec::new();
for json_value in array {
let value = Value::try_from(json_value)?;
values.push(value);
}
Ok(Value::List(List::with_items(values)))
}
}
}
}
impl TryFrom<&JsonValue> for Value {
type Error = Error;
fn try_from(json_value: &JsonValue) -> Result<Self> {
use JsonValue::*;
match json_value {
Null => Ok(Value::Empty),
Short(short) => Ok(Value::String(short.to_string())),
String(string) => Ok(Value::String(string.clone())),
Number(number) => Ok(Value::Float(f64::from(*number))),
Boolean(boolean) => Ok(Value::Boolean(*boolean)),
Object(object) => {
let map = Map::new();
for (key, node_value) in object.iter() {
let value = Value::try_from(node_value)?;
map.variables_mut().insert(key.to_string(), value);
}
Ok(Value::Map(map))
}
Array(array) => {
let mut values = Vec::new();
for json_value in array {
let value = Value::try_from(json_value)?;
values.push(value);
}
Ok(Value::List(List::with_items(values)))
}
}
}
}
impl TryFrom<Value> for String {
type Error = Error;
@ -842,8 +765,12 @@ impl<'de> Visitor<'de> for ValueVisitor {
{
let map = Map::new();
while let Some((key, value)) = access.next_entry()? {
map.variables_mut().insert(key, value);
{
let mut variables = map.variables_mut().unwrap();
while let Some((key, value)) = access.next_entry()? {
variables.insert(key, value);
}
}
Ok(Value::Map(map))

View File

@ -158,7 +158,7 @@ impl Display for Table {
string
}
Value::Map(map) => format!("Map ({} items)", map.len()),
Value::Map(map) => format!("Map ({} items)", map.variables().unwrap().len()),
Value::Table(table) => format!("Table ({} items)", table.len()),
Value::Function(_) => "Function".to_string(),
Value::Empty => "Empty".to_string(),
@ -233,7 +233,7 @@ impl From<&Value> for Table {
}
Value::List(list) => Self::from(list),
Value::Empty => Table::new(Vec::with_capacity(0)),
Value::Map(map) => Self::from(map),
Value::Map(map) => Result::<Table>::from(map).unwrap(),
Value::Table(table) => table.clone(),
Value::Function(function) => {
let mut table = Table::new(vec!["function".to_string()]);
@ -280,39 +280,35 @@ impl From<&mut List> for Table {
}
}
impl From<Map> for Table {
impl From<Map> for Result<Table> {
fn from(map: Map) -> Self {
let variables = map.variables();
let variables = map.variables()?;
let keys = variables.keys().cloned().collect();
let values = variables.values().cloned().collect();
let mut table = Table::new(keys);
table
.insert(values)
.expect("Failed to create Table from Map. This is a no-op.");
table.insert(values)?;
table
Ok(table)
}
}
impl From<&Map> for Table {
impl From<&Map> for Result<Table> {
fn from(map: &Map) -> Self {
let variables = map.variables();
let variables = map.variables()?;
let keys = variables.keys().cloned().collect();
let values = variables.values().cloned().collect();
let mut table = Table::new(keys);
table
.insert(values)
.expect("Failed to create Table from Map. This is a no-op.");
table.insert(values)?;
table
Ok(table)
}
}
impl From<&mut Map> for Table {
impl From<&mut Map> for Result<Table> {
fn from(map: &mut Map) -> Self {
let variables = map.variables();
let variables = map.variables()?;
let keys = variables.keys().cloned().collect();
let values = variables.values().cloned().collect();
let mut table = Table::new(keys);
@ -321,7 +317,7 @@ impl From<&mut Map> for Table {
.insert(values)
.expect("Failed to create Table from Map. This is a no-op.");
table
Ok(table)
}
}

View File

@ -114,7 +114,7 @@ impl From<&Value> for ValueType {
Value::Map(map) => {
let mut value_nodes = BTreeMap::new();
for (key, value) in map.variables().iter() {
for (key, value) in map.variables().unwrap().iter() {
let value_type = value.value_type();
let value_node = ValueNode::new(value_type, 0, 0);
let statement = Statement::Expression(Expression::Value(value_node));

View File

@ -708,17 +708,8 @@
"type": "SEQ",
"members": [
{
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "identifier"
},
{
"type": "SYMBOL",
"name": "index"
}
]
"type": "SYMBOL",
"name": "identifier"
},
{
"type": "SYMBOL",

View File

@ -15,10 +15,6 @@
"type": "identifier",
"named": true
},
{
"type": "index",
"named": true
},
{
"type": "statement",
"named": true

File diff suppressed because it is too large Load Diff