jeff/dust
1
0
Fork 0
Code Issues

Begin passing tests

Browse Source
This commit is contained in:
Jeff 2024-06-22 11:44:09 -04:00
parent 890baa5d51
commit a05d9016f2
10 changed files with 126 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

70
dust-lang/src/lexer.rs
View File

@ -100,7 +100,7 @@ impl Display for Keyword {
#[derive(Copy, Clone, Debug, PartialEq)] #[derive(Copy, Clone, Debug, PartialEq)]
pub enum Symbol { pub enum Symbol {
Plus, Plus,
PlusEquals, PlusEqual,
DoubleAmpersand, DoubleAmpersand,
Colon, Colon,
Comma, Comma,
@ -166,7 +166,7 @@ impl Display for Symbol {
Symbol::Percent => write!(f, "%"), Symbol::Percent => write!(f, "%"),
Symbol::Pipe => write!(f, "|"), Symbol::Pipe => write!(f, "|"),
Symbol::Plus => write!(f, "+"), Symbol::Plus => write!(f, "+"),
Symbol::PlusEquals => write!(f, "+="), Symbol::PlusEqual => write!(f, "+="),
Symbol::Semicolon => write!(f, ";"), Symbol::Semicolon => write!(f, ";"),
Symbol::SkinnyArrow => write!(f, "->"), Symbol::SkinnyArrow => write!(f, "->"),
Symbol::Slash => write!(f, "/"), Symbol::Slash => write!(f, "/"),
@ -251,35 +251,34 @@ pub fn lexer<'src>() -> impl Parser<
delimited_string('`'), delimited_string('`'),
)); ));
let identifier_and_keyword = text::ident().map(|text: &str| match text { let keyword = choice((
"any" => Token::Keyword(Keyword::Any), just("any").to(Token::Keyword(Keyword::Any)),
"async" => Token::Keyword(Keyword::Async), just("async").to(Token::Keyword(Keyword::Async)),
"as" => Token::Keyword(Keyword::As), just("as").to(Token::Keyword(Keyword::As)),
"bool" => Token::Keyword(Keyword::Bool), just("bool").to(Token::Keyword(Keyword::Bool)),
"break" => Token::Keyword(Keyword::Break), just("break").to(Token::Keyword(Keyword::Break)),
"enum" => Token::Keyword(Keyword::Enum), just("enum").to(Token::Keyword(Keyword::Enum)),
"else" => Token::Keyword(Keyword::Else), just("else").to(Token::Keyword(Keyword::Else)),
"float" => Token::Keyword(Keyword::Float), just("float").to(Token::Keyword(Keyword::Float)),
"fn" => Token::Keyword(Keyword::Fn), just("fn").to(Token::Keyword(Keyword::Fn)),
"int" => Token::Keyword(Keyword::Int), just("int").to(Token::Keyword(Keyword::Int)),
"if" => Token::Keyword(Keyword::If), just("if").to(Token::Keyword(Keyword::If)),
"list" => Token::Keyword(Keyword::List), just("list").to(Token::Keyword(Keyword::List)),
"map" => Token::Keyword(Keyword::Map), just("map").to(Token::Keyword(Keyword::Map)),
"none" => Token::Keyword(Keyword::None), just("none").to(Token::Keyword(Keyword::None)),
"range" => Token::Keyword(Keyword::Range), just("range").to(Token::Keyword(Keyword::Range)),
"struct" => Token::Keyword(Keyword::Struct), just("struct").to(Token::Keyword(Keyword::Struct)),
"str" => Token::Keyword(Keyword::Str), just("str").to(Token::Keyword(Keyword::Str)),
"type" => Token::Keyword(Keyword::Type), just("type").to(Token::Keyword(Keyword::Type)),
"loop" => Token::Keyword(Keyword::Loop), just("loop").to(Token::Keyword(Keyword::Loop)),
"while" => Token::Keyword(Keyword::While), just("while").to(Token::Keyword(Keyword::While)),
"JSON_PARSE" => Token::Keyword(Keyword::JsonParse), just("JSON_PARSE").to(Token::Keyword(Keyword::JsonParse)),
"LENGTH" => Token::Keyword(Keyword::Length), just("LENGTH").to(Token::Keyword(Keyword::Length)),
"READ_FILE" => Token::Keyword(Keyword::ReadFile), just("READ_FILE").to(Token::Keyword(Keyword::ReadFile)),
"READ_LINE" => Token::Keyword(Keyword::ReadLine), just("READ_LINE").to(Token::Keyword(Keyword::ReadLine)),
"SLEEP" => Token::Keyword(Keyword::Sleep), just("SLEEP").to(Token::Keyword(Keyword::Sleep)),
"WRITE_LINE" => Token::Keyword(Keyword::WriteLine), just("WRITE_LINE").to(Token::Keyword(Keyword::WriteLine)),
_ => Token::Identifier(text), ));
});
let symbol = choice([ let symbol = choice([
just("!=").to(Token::Symbol(Symbol::NotEqual)), just("!=").to(Token::Symbol(Symbol::NotEqual)),
@ -290,7 +289,7 @@ pub fn lexer<'src>() -> impl Parser<
just("(").to(Token::Symbol(Symbol::ParenOpen)), just("(").to(Token::Symbol(Symbol::ParenOpen)),
just(")").to(Token::Symbol(Symbol::ParenClose)), just(")").to(Token::Symbol(Symbol::ParenClose)),
just("*").to(Token::Symbol(Symbol::Asterisk)), just("*").to(Token::Symbol(Symbol::Asterisk)),
just("+=").to(Token::Symbol(Symbol::PlusEquals)), just("+=").to(Token::Symbol(Symbol::PlusEqual)),
just("+").to(Token::Symbol(Symbol::Plus)), just("+").to(Token::Symbol(Symbol::Plus)),
just(",").to(Token::Symbol(Symbol::Comma)), just(",").to(Token::Symbol(Symbol::Comma)),
just("->").to(Token::Symbol(Symbol::SkinnyArrow)), just("->").to(Token::Symbol(Symbol::SkinnyArrow)),
@ -313,11 +312,13 @@ pub fn lexer<'src>() -> impl Parser<
just("]").to(Token::Symbol(Symbol::SquareClose)), just("]").to(Token::Symbol(Symbol::SquareClose)),
just("__").to(Token::Symbol(Symbol::DoubleUnderscore)), just("__").to(Token::Symbol(Symbol::DoubleUnderscore)),
just("{").to(Token::Symbol(Symbol::CurlyOpen)), just("{").to(Token::Symbol(Symbol::CurlyOpen)),
just("}").to(Token::Symbol(Symbol::CurlyClose)),
just("||").to(Token::Symbol(Symbol::DoublePipe)), just("||").to(Token::Symbol(Symbol::DoublePipe)),
just("|").to(Token::Symbol(Symbol::Pipe)), just("|").to(Token::Symbol(Symbol::Pipe)),
just("}").to(Token::Symbol(Symbol::CurlyClose)),
]); ]);
let identifier = text::ident().map(|text: &str| Token::Identifier(text));
choice(( choice((
line_comment, line_comment,
multi_line_comment, multi_line_comment,
@ -325,8 +326,9 @@ pub fn lexer<'src>() -> impl Parser<
float, float,
integer, integer,
string, string,
identifier_and_keyword, keyword,
symbol, symbol,
identifier,
)) ))
.map_with(|token: Token, state| (token, state.span())) .map_with(|token: Token, state| (token, state.span()))
.padded() .padded()

44
dust-lang/src/lib.rs
View File

@ -14,14 +14,15 @@ use std::{
use abstract_tree::{AbstractTree, Type}; use abstract_tree::{AbstractTree, Type};
use ariadne::{Color, Config, Fmt, Label, Report, ReportKind}; use ariadne::{Color, Config, Fmt, Label, Report, ReportKind};
use chumsky::prelude::*;
use context::Context; use context::Context;
use error::{DustError, RuntimeError, TypeConflict, ValidationError}; use error::{DustError, RuntimeError, TypeConflict, ValidationError};
use lexer::{lex, Token}; use lexer::{lex, Token};
use parser::parse; use parser::{parse, parser};
use rayon::prelude::*; use rayon::prelude::*;
pub use value::Value; pub use value::Value;
pub fn interpret<'src>(source_id: &str, source: &str) -> Result<Option<Value>, InterpreterError> { pub fn interpret(source_id: &str, source: &str) -> Result<Option<Value>, InterpreterError> {
let interpreter = Interpreter::new(Context::new(None)); let interpreter = Interpreter::new(Context::new(None));
interpreter.load_std()?; interpreter.load_std()?;
@ -34,7 +35,7 @@ pub fn interpret_without_std(
) -> Result<Option<Value>, InterpreterError> { ) -> Result<Option<Value>, InterpreterError> {
let interpreter = Interpreter::new(Context::new(None)); let interpreter = Interpreter::new(Context::new(None));
interpreter.run(Arc::from(source_id.to_string()), Arc::from(source)) interpreter.run(Arc::from(source_id), Arc::from(source))
} }
pub struct Interpreter { pub struct Interpreter {
@ -137,18 +138,18 @@ impl Interpreter {
// Always load the core library first because other parts of the standard library may depend // Always load the core library first because other parts of the standard library may depend
// on it. // on it.
self.run(std_core_source.0, std_core_source.1)?; self.run_with_builtins(std_core_source.0, std_core_source.1)?;
let error = if cfg!(test) { let error = if cfg!(test) {
// In debug mode, load the standard library sequentially to get consistent errors. // In debug mode, load the standard library sequentially to get consistent errors.
std_sources std_sources
.into_iter() .into_iter()
.find_map(|(source_id, source)| self.run(source_id, source).err()) .find_map(|(source_id, source)| self.run_with_builtins(source_id, source).err())
} else { } else {
// In release mode, load the standard library asynchronously. // In release mode, load the standard library asynchronously.
std_sources std_sources
.into_par_iter() .into_par_iter()
.find_map_any(|(source_id, source)| self.run(source_id, source).err()) .find_map_any(|(source_id, source)| self.run_with_builtins(source_id, source).err())
}; };
log::info!("Finish loading standard library."); log::info!("Finish loading standard library.");
@ -163,6 +164,37 @@ impl Interpreter {
pub fn sources(&self) -> vec::IntoIter<(Arc<str>, Arc<str>)> { pub fn sources(&self) -> vec::IntoIter<(Arc<str>, Arc<str>)> {
self.sources.read().unwrap().clone().into_iter() self.sources.read().unwrap().clone().into_iter()
} }
fn run_with_builtins(
&self,
source_id: Arc<str>,
source: Arc<str>,
) -> Result<Option<Value>, InterpreterError> {
let mut sources = self.sources.write().unwrap();
sources.clear();
sources.push((source_id.clone(), source.clone()));
let tokens = lex(source.as_ref()).map_err(|errors| InterpreterError {
source_id: source_id.clone(),
errors,
})?;
let abstract_tree = parser(true)
.parse(tokens.spanned((tokens.len()..tokens.len()).into()))
.into_result()
.map_err(|errors| InterpreterError {
source_id: source_id.clone(),
errors: errors
.into_iter()
.map(|error| DustError::from(error))
.collect(),
})?;
let value_option = abstract_tree
.run(&self.context, true)
.map_err(|errors| InterpreterError { source_id, errors })?;
Ok(value_option)
}
} }
#[derive(Debug, PartialEq)] #[derive(Debug, PartialEq)]

46
dust-lang/src/parser/mod.rs
View File

@ -25,7 +25,7 @@ pub type ParserExtra<'src> = extra::Err<Rich<'src, Token<'src>, SimpleSpan>>;
pub fn parse<'src>( pub fn parse<'src>(
tokens: &'src [(Token<'src>, SimpleSpan)], tokens: &'src [(Token<'src>, SimpleSpan)],
) -> Result<AbstractTree, Vec<DustError>> { ) -> Result<AbstractTree, Vec<DustError>> {
let statements = parser(false) parser(false)
.parse(tokens.spanned((tokens.len()..tokens.len()).into())) .parse(tokens.spanned((tokens.len()..tokens.len()).into()))
.into_result() .into_result()
.map_err(|errors| { .map_err(|errors| {
@ -33,14 +33,12 @@ pub fn parse<'src>(
.into_iter() .into_iter()
.map(|error| DustError::from(error)) .map(|error| DustError::from(error))
.collect::<Vec<DustError>>() .collect::<Vec<DustError>>()
})?; })
Ok(AbstractTree::new(statements))
} }
pub fn parser<'src>( pub fn parser<'src>(
allow_built_ins: bool, allow_built_ins: bool,
) -> impl Parser<'src, ParserInput<'src>, Vec<Statement>, ParserExtra<'src>> { ) -> impl Parser<'src, ParserInput<'src>, AbstractTree, ParserExtra<'src>> {
let comment = select_ref! { let comment = select_ref! {
Token::Comment(_) => {} Token::Comment(_) => {}
}; };
@ -330,8 +328,15 @@ pub fn parser<'src>(
) )
}); });
let _built_in_function = |keyword| {
just(Token::Keyword(keyword)).delimited_by(
just(Token::Symbol(Symbol::DoubleUnderscore)),
just(Token::Symbol(Symbol::DoubleUnderscore)),
)
};
let built_in_function_call = choice(( let built_in_function_call = choice((
just(Token::Keyword(Keyword::Length)) _built_in_function(Keyword::Length)
.ignore_then(expression.clone()) .ignore_then(expression.clone())
.map_with(|argument, state| { .map_with(|argument, state| {
Expression::BuiltInFunctionCall( Expression::BuiltInFunctionCall(
@ -339,7 +344,7 @@ pub fn parser<'src>(
.with_position(state.span()), .with_position(state.span()),
) )
}), }),
just(Token::Keyword(Keyword::ReadFile)) _built_in_function(Keyword::ReadFile)
.ignore_then(expression.clone()) .ignore_then(expression.clone())
.map_with(|argument, state| { .map_with(|argument, state| {
Expression::BuiltInFunctionCall( Expression::BuiltInFunctionCall(
@ -347,12 +352,12 @@ pub fn parser<'src>(
.with_position(state.span()), .with_position(state.span()),
) )
}), }),
just(Token::Keyword(Keyword::ReadLine)).map_with(|_, state| { _built_in_function(Keyword::ReadLine).map_with(|_, state| {
Expression::BuiltInFunctionCall( Expression::BuiltInFunctionCall(
Box::new(BuiltInFunctionCall::ReadLine).with_position(state.span()), Box::new(BuiltInFunctionCall::ReadLine).with_position(state.span()),
) )
}), }),
just(Token::Keyword(Keyword::Sleep)) _built_in_function(Keyword::Sleep)
.ignore_then(expression.clone()) .ignore_then(expression.clone())
.map_with(|argument, state| { .map_with(|argument, state| {
Expression::BuiltInFunctionCall( Expression::BuiltInFunctionCall(
@ -360,7 +365,7 @@ pub fn parser<'src>(
.with_position(state.span()), .with_position(state.span()),
) )
}), }),
just(Token::Keyword(Keyword::WriteLine)) _built_in_function(Keyword::WriteLine)
.ignore_then(expression.clone()) .ignore_then(expression.clone())
.map_with(|argument, state| { .map_with(|argument, state| {
Expression::BuiltInFunctionCall( Expression::BuiltInFunctionCall(
@ -368,7 +373,7 @@ pub fn parser<'src>(
.with_position(state.span()), .with_position(state.span()),
) )
}), }),
just(Token::Keyword(Keyword::JsonParse)) _built_in_function(Keyword::JsonParse)
.ignore_then(type_constructor.clone()) .ignore_then(type_constructor.clone())
.then(expression.clone()) .then(expression.clone())
.map_with(|(constructor, argument), state| { .map_with(|(constructor, argument), state| {
@ -378,15 +383,15 @@ pub fn parser<'src>(
) )
}), }),
)) ))
.try_map_with(move |expression, state| { .validate(move |expression, state, emitter| {
if allow_built_ins { if !allow_built_ins {
Ok(expression) emitter.emit(Rich::custom(
} else {
Err(Rich::custom(
state.span(), state.span(),
"Built-in function calls can only be used by the standard library.", "Built-in function calls can only be used by the standard library.",
)) ))
} }
expression
}); });
let turbofish = type_constructor let turbofish = type_constructor
@ -583,9 +588,9 @@ pub fn parser<'src>(
)); ));
choice(( choice((
built_in_function_call,
logic_math_indexes_as_and_function_calls, logic_math_indexes_as_and_function_calls,
enum_instance, enum_instance,
built_in_function_call,
range, range,
function, function,
list, list,
@ -616,7 +621,7 @@ pub fn parser<'src>(
.then(type_specification.clone().or_not()) .then(type_specification.clone().or_not())
.then(choice(( .then(choice((
just(Token::Symbol(Symbol::Equal)).to(AssignmentOperator::Assign), just(Token::Symbol(Symbol::Equal)).to(AssignmentOperator::Assign),
just(Token::Symbol(Symbol::PlusEquals)).to(AssignmentOperator::AddAssign), just(Token::Symbol(Symbol::PlusEqual)).to(AssignmentOperator::AddAssign),
just(Token::Symbol(Symbol::MinusEqual)).to(AssignmentOperator::SubAssign), just(Token::Symbol(Symbol::MinusEqual)).to(AssignmentOperator::SubAssign),
))) )))
.then(statement.clone()) .then(statement.clone())
@ -757,5 +762,8 @@ pub fn parser<'src>(
.then_ignore(just(Token::Symbol(Symbol::Semicolon)).or_not()) .then_ignore(just(Token::Symbol(Symbol::Semicolon)).or_not())
}); });
statement.repeated().collect() statement
.repeated()
.collect()
.map(|statements| AbstractTree::new(statements))
} }

13
dust-lang/src/parser/tests.rs
View File

@ -226,7 +226,7 @@ fn r#as() {
#[test] #[test]
fn built_in_function() { fn built_in_function() {
let tokens = lex("READ_LINE").unwrap(); let tokens = lex("__READ_LINE__").unwrap();
let statements = parser(true) let statements = parser(true)
.parse(tokens.spanned((tokens.len()..tokens.len()).into())) .parse(tokens.spanned((tokens.len()..tokens.len()).into()))
.into_result() .into_result()
@ -241,11 +241,14 @@ fn built_in_function() {
assert_eq!( assert_eq!(
statements[0], statements[0],
Statement::Expression(Expression::BuiltInFunctionCall( Statement::Expression(Expression::BuiltInFunctionCall(
Box::new(BuiltInFunctionCall::ReadLine).with_position((0, 9)) Box::new(BuiltInFunctionCall::ReadLine).with_position((0, 13))
)) ))
); );
}
let tokens = lex("WRITE_LINE 'hiya'").unwrap(); #[test]
fn built_in_function_with_arg() {
let tokens = lex("__WRITE_LINE__ 'hiya'").unwrap();
let statements = parser(true) let statements = parser(true)
.parse(tokens.spanned((tokens.len()..tokens.len()).into())) .parse(tokens.spanned((tokens.len()..tokens.len()).into()))
.into_result() .into_result()
@ -261,9 +264,9 @@ fn built_in_function() {
statements[0], statements[0],
Statement::Expression(Expression::BuiltInFunctionCall( Statement::Expression(Expression::BuiltInFunctionCall(
Box::new(BuiltInFunctionCall::WriteLine(Expression::Value( Box::new(BuiltInFunctionCall::WriteLine(Expression::Value(
ValueNode::String("hiya".to_string()).with_position((11, 17)) ValueNode::String("hiya".to_string()).with_position((15, 21))
))) )))
.with_position((0, 17)) .with_position((0, 21))
)) ))
); );
} }

5
dust-shell/src/cli.rs
View File

@ -86,9 +86,8 @@ pub fn run_shell(context: Context) -> Result<(), io::Error> {
let reports = error.build_reports(); let reports = error.build_reports();
for report in reports { for report in reports {
report let cache = sources(interpreter.sources());
.write_for_stdout(sources(interpreter.sources()), stderr()) report.write_for_stdout(cache, stderr()).unwrap();
.unwrap();
} }
} }
} }

14
dust-shell/src/main.rs
View File

@ -11,6 +11,7 @@ use std::{
fs::read_to_string, fs::read_to_string,
io::{stderr, Write}, io::{stderr, Write},
sync::Arc, sync::Arc,
vec,
}; };
use dust_lang::{context::Context, Interpreter}; use dust_lang::{context::Context, Interpreter};
@ -78,9 +79,9 @@ fn main() {
let (source_id, source): (Arc<str>, Arc<str>) = if let Some(path) = args.path { let (source_id, source): (Arc<str>, Arc<str>) = if let Some(path) = args.path {
let source = read_to_string(&path).unwrap(); let source = read_to_string(&path).unwrap();
(Arc::from(path), Arc::from(source.as_str())) (Arc::from(path.as_str()), Arc::from(source))
} else if let Some(command) = args.command { } else if let Some(command) = args.command {
(Arc::from("command"), Arc::from(command.as_str())) (Arc::from("command"), Arc::from(command))
} else { } else {
match run_shell(context) { match run_shell(context) {
Ok(_) => {} Ok(_) => {}
@ -96,7 +97,12 @@ fn main() {
Err(error) => { Err(error) => {
for report in error.build_reports() { for report in error.build_reports() {
report report
.write_for_stdout(sources(interpreter.sources()), stderr()) .write_for_stdout(
sources::<Arc<str>, Arc<str>, vec::IntoIter<(Arc<str>, Arc<str>)>>(
interpreter.sources(),
),
stderr(),
)
.unwrap(); .unwrap();
} }
} }
@ -120,7 +126,7 @@ fn main() {
return; return;
} }
let run_result = interpreter.run(source_id.clone(), source.clone()); let run_result = interpreter.run(source_id.clone(), source);
match run_result { match run_result {
Ok(value) => { Ok(value) => {

2
std/core.ds
View File

@ -1,3 +1,3 @@
length = fn (input: [any]) -> int { length = fn (input: [any]) -> int {
LENGTH input __LENGTH__ input
} }

2
std/fs.ds
View File

@ -1,5 +1,5 @@
fs = { fs = {
read_file = fn (path: str) -> str { read_file = fn (path: str) -> str {
READ_FILE path __READ_FILE__ path
} }
} }

4
std/io.ds
View File

@ -1,9 +1,9 @@
io = { io = {
read_line = fn () -> str { read_line = fn () -> str {
READ_LINE __READ_LINE__
} }
write_line = fn (output: any) { write_line = fn (output: any) {
WRITE_LINE output __WRITE_LINE__ output
} }
} }

2
std/json.ds
View File

@ -1,5 +1,5 @@
json = { json = {
parse = fn |T| (input: str) -> T { parse = fn |T| (input: str) -> T {
JSON_PARSE T input __JSON_PARSE__ T input
} }
} }