jeff/dust
1
0
Fork 0
Code Issues

Begin passing tests

Browse Source
This commit is contained in:
Jeff 2024-06-22 11:44:09 -04:00
parent 890baa5d51
commit a05d9016f2
10 changed files with 126 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

70
dust-lang/src/lexer.rs
View File

@ -100,7 +100,7 @@ impl Display for Keyword {
#[derive(Copy, Clone, Debug, PartialEq)]
pub enum Symbol {
Plus,
PlusEquals,
PlusEqual,
DoubleAmpersand,
Colon,
Comma,
@ -166,7 +166,7 @@ impl Display for Symbol {
Symbol::Percent => write!(f, "%"),
Symbol::Pipe => write!(f, "|"),
Symbol::Plus => write!(f, "+"),
Symbol::PlusEquals => write!(f, "+="),
Symbol::PlusEqual => write!(f, "+="),
Symbol::Semicolon => write!(f, ";"),
Symbol::SkinnyArrow => write!(f, "->"),
Symbol::Slash => write!(f, "/"),
@ -251,35 +251,34 @@ pub fn lexer<'src>() -> impl Parser<
delimited_string('`'),
));
let identifier_and_keyword = text::ident().map(|text: &str| match text {
"any" => Token::Keyword(Keyword::Any),
"async" => Token::Keyword(Keyword::Async),
"as" => Token::Keyword(Keyword::As),
"bool" => Token::Keyword(Keyword::Bool),
"break" => Token::Keyword(Keyword::Break),
"enum" => Token::Keyword(Keyword::Enum),
"else" => Token::Keyword(Keyword::Else),
"float" => Token::Keyword(Keyword::Float),
"fn" => Token::Keyword(Keyword::Fn),
"int" => Token::Keyword(Keyword::Int),
"if" => Token::Keyword(Keyword::If),
"list" => Token::Keyword(Keyword::List),
"map" => Token::Keyword(Keyword::Map),
"none" => Token::Keyword(Keyword::None),
"range" => Token::Keyword(Keyword::Range),
"struct" => Token::Keyword(Keyword::Struct),
"str" => Token::Keyword(Keyword::Str),
"type" => Token::Keyword(Keyword::Type),
"loop" => Token::Keyword(Keyword::Loop),
"while" => Token::Keyword(Keyword::While),
"JSON_PARSE" => Token::Keyword(Keyword::JsonParse),
"LENGTH" => Token::Keyword(Keyword::Length),
"READ_FILE" => Token::Keyword(Keyword::ReadFile),
"READ_LINE" => Token::Keyword(Keyword::ReadLine),
"SLEEP" => Token::Keyword(Keyword::Sleep),
"WRITE_LINE" => Token::Keyword(Keyword::WriteLine),
_ => Token::Identifier(text),
});
let keyword = choice((
just("any").to(Token::Keyword(Keyword::Any)),
just("async").to(Token::Keyword(Keyword::Async)),
just("as").to(Token::Keyword(Keyword::As)),
just("bool").to(Token::Keyword(Keyword::Bool)),
just("break").to(Token::Keyword(Keyword::Break)),
just("enum").to(Token::Keyword(Keyword::Enum)),
just("else").to(Token::Keyword(Keyword::Else)),
just("float").to(Token::Keyword(Keyword::Float)),
just("fn").to(Token::Keyword(Keyword::Fn)),
just("int").to(Token::Keyword(Keyword::Int)),
just("if").to(Token::Keyword(Keyword::If)),
just("list").to(Token::Keyword(Keyword::List)),
just("map").to(Token::Keyword(Keyword::Map)),
just("none").to(Token::Keyword(Keyword::None)),
just("range").to(Token::Keyword(Keyword::Range)),
just("struct").to(Token::Keyword(Keyword::Struct)),
just("str").to(Token::Keyword(Keyword::Str)),
just("type").to(Token::Keyword(Keyword::Type)),
just("loop").to(Token::Keyword(Keyword::Loop)),
just("while").to(Token::Keyword(Keyword::While)),
just("JSON_PARSE").to(Token::Keyword(Keyword::JsonParse)),
just("LENGTH").to(Token::Keyword(Keyword::Length)),
just("READ_FILE").to(Token::Keyword(Keyword::ReadFile)),
just("READ_LINE").to(Token::Keyword(Keyword::ReadLine)),
just("SLEEP").to(Token::Keyword(Keyword::Sleep)),
just("WRITE_LINE").to(Token::Keyword(Keyword::WriteLine)),
));
let symbol = choice([
just("!=").to(Token::Symbol(Symbol::NotEqual)),
@ -290,7 +289,7 @@ pub fn lexer<'src>() -> impl Parser<
just("(").to(Token::Symbol(Symbol::ParenOpen)),
just(")").to(Token::Symbol(Symbol::ParenClose)),
just("*").to(Token::Symbol(Symbol::Asterisk)),
just("+=").to(Token::Symbol(Symbol::PlusEquals)),
just("+=").to(Token::Symbol(Symbol::PlusEqual)),
just("+").to(Token::Symbol(Symbol::Plus)),
just(",").to(Token::Symbol(Symbol::Comma)),
just("->").to(Token::Symbol(Symbol::SkinnyArrow)),
@ -313,11 +312,13 @@ pub fn lexer<'src>() -> impl Parser<
just("]").to(Token::Symbol(Symbol::SquareClose)),
just("__").to(Token::Symbol(Symbol::DoubleUnderscore)),
just("{").to(Token::Symbol(Symbol::CurlyOpen)),
just("}").to(Token::Symbol(Symbol::CurlyClose)),
just("||").to(Token::Symbol(Symbol::DoublePipe)),
just("|").to(Token::Symbol(Symbol::Pipe)),
just("}").to(Token::Symbol(Symbol::CurlyClose)),
]);
let identifier = text::ident().map(|text: &str| Token::Identifier(text));
choice((
line_comment,
multi_line_comment,
@ -325,8 +326,9 @@ pub fn lexer<'src>() -> impl Parser<
float,
integer,
string,
identifier_and_keyword,
keyword,
symbol,
identifier,
))
.map_with(|token: Token, state| (token, state.span()))
.padded()

44
dust-lang/src/lib.rs
View File

@ -14,14 +14,15 @@ use std::{
use abstract_tree::{AbstractTree, Type};
use ariadne::{Color, Config, Fmt, Label, Report, ReportKind};
use chumsky::prelude::*;
use context::Context;
use error::{DustError, RuntimeError, TypeConflict, ValidationError};
use lexer::{lex, Token};
use parser::parse;
use parser::{parse, parser};
use rayon::prelude::*;
pub use value::Value;
pub fn interpret<'src>(source_id: &str, source: &str) -> Result<Option<Value>, InterpreterError> {
pub fn interpret(source_id: &str, source: &str) -> Result<Option<Value>, InterpreterError> {
let interpreter = Interpreter::new(Context::new(None));
interpreter.load_std()?;
@ -34,7 +35,7 @@ pub fn interpret_without_std(
) -> Result<Option<Value>, InterpreterError> {
let interpreter = Interpreter::new(Context::new(None));
interpreter.run(Arc::from(source_id.to_string()), Arc::from(source))
interpreter.run(Arc::from(source_id), Arc::from(source))
}
pub struct Interpreter {
@ -137,18 +138,18 @@ impl Interpreter {
// Always load the core library first because other parts of the standard library may depend
// on it.
self.run(std_core_source.0, std_core_source.1)?;
self.run_with_builtins(std_core_source.0, std_core_source.1)?;
let error = if cfg!(test) {
// In debug mode, load the standard library sequentially to get consistent errors.
std_sources
.into_iter()
.find_map(|(source_id, source)| self.run(source_id, source).err())
.find_map(|(source_id, source)| self.run_with_builtins(source_id, source).err())
} else {
// In release mode, load the standard library asynchronously.
std_sources
.into_par_iter()
.find_map_any(|(source_id, source)| self.run(source_id, source).err())
.find_map_any(|(source_id, source)| self.run_with_builtins(source_id, source).err())
};
log::info!("Finish loading standard library.");
@ -163,6 +164,37 @@ impl Interpreter {
pub fn sources(&self) -> vec::IntoIter<(Arc<str>, Arc<str>)> {
self.sources.read().unwrap().clone().into_iter()
}
fn run_with_builtins(
&self,
source_id: Arc<str>,
source: Arc<str>,
) -> Result<Option<Value>, InterpreterError> {
let mut sources = self.sources.write().unwrap();
sources.clear();
sources.push((source_id.clone(), source.clone()));
let tokens = lex(source.as_ref()).map_err(|errors| InterpreterError {
source_id: source_id.clone(),
errors,
})?;
let abstract_tree = parser(true)
.parse(tokens.spanned((tokens.len()..tokens.len()).into()))
.into_result()
.map_err(|errors| InterpreterError {
source_id: source_id.clone(),
errors: errors
.into_iter()
.map(|error| DustError::from(error))
.collect(),
})?;
let value_option = abstract_tree
.run(&self.context, true)
.map_err(|errors| InterpreterError { source_id, errors })?;
Ok(value_option)
}
}
#[derive(Debug, PartialEq)]

46
dust-lang/src/parser/mod.rs
View File

@ -25,7 +25,7 @@ pub type ParserExtra<'src> = extra::Err<Rich<'src, Token<'src>, SimpleSpan>>;
pub fn parse<'src>(
tokens: &'src [(Token<'src>, SimpleSpan)],
) -> Result<AbstractTree, Vec<DustError>> {
let statements = parser(false)
parser(false)
.parse(tokens.spanned((tokens.len()..tokens.len()).into()))
.into_result()
.map_err(|errors| {
@ -33,14 +33,12 @@ pub fn parse<'src>(
.into_iter()
.map(|error| DustError::from(error))
.collect::<Vec<DustError>>()
})?;
Ok(AbstractTree::new(statements))
})
}
pub fn parser<'src>(
allow_built_ins: bool,
) -> impl Parser<'src, ParserInput<'src>, Vec<Statement>, ParserExtra<'src>> {
) -> impl Parser<'src, ParserInput<'src>, AbstractTree, ParserExtra<'src>> {
let comment = select_ref! {
Token::Comment(_) => {}
};
@ -330,8 +328,15 @@ pub fn parser<'src>(
)
});
let _built_in_function = |keyword| {
just(Token::Keyword(keyword)).delimited_by(
just(Token::Symbol(Symbol::DoubleUnderscore)),
just(Token::Symbol(Symbol::DoubleUnderscore)),
)
};
let built_in_function_call = choice((
just(Token::Keyword(Keyword::Length))
_built_in_function(Keyword::Length)
.ignore_then(expression.clone())
.map_with(|argument, state| {
Expression::BuiltInFunctionCall(
@ -339,7 +344,7 @@ pub fn parser<'src>(
.with_position(state.span()),
)
}),
just(Token::Keyword(Keyword::ReadFile))
_built_in_function(Keyword::ReadFile)
.ignore_then(expression.clone())
.map_with(|argument, state| {
Expression::BuiltInFunctionCall(
@ -347,12 +352,12 @@ pub fn parser<'src>(
.with_position(state.span()),
)
}),
just(Token::Keyword(Keyword::ReadLine)).map_with(|_, state| {
_built_in_function(Keyword::ReadLine).map_with(|_, state| {
Expression::BuiltInFunctionCall(
Box::new(BuiltInFunctionCall::ReadLine).with_position(state.span()),
)
}),
just(Token::Keyword(Keyword::Sleep))
_built_in_function(Keyword::Sleep)
.ignore_then(expression.clone())
.map_with(|argument, state| {
Expression::BuiltInFunctionCall(
@ -360,7 +365,7 @@ pub fn parser<'src>(
.with_position(state.span()),
)
}),
just(Token::Keyword(Keyword::WriteLine))
_built_in_function(Keyword::WriteLine)
.ignore_then(expression.clone())
.map_with(|argument, state| {
Expression::BuiltInFunctionCall(
@ -368,7 +373,7 @@ pub fn parser<'src>(
.with_position(state.span()),
)
}),
just(Token::Keyword(Keyword::JsonParse))
_built_in_function(Keyword::JsonParse)
.ignore_then(type_constructor.clone())
.then(expression.clone())
.map_with(|(constructor, argument), state| {
@ -378,15 +383,15 @@ pub fn parser<'src>(
)
}),
))
.try_map_with(move |expression, state| {
if allow_built_ins {
Ok(expression)
} else {
Err(Rich::custom(
.validate(move |expression, state, emitter| {
if !allow_built_ins {
emitter.emit(Rich::custom(
state.span(),
"Built-in function calls can only be used by the standard library.",
))
}
expression
});
let turbofish = type_constructor
@ -583,9 +588,9 @@ pub fn parser<'src>(
));
choice((
built_in_function_call,
logic_math_indexes_as_and_function_calls,
enum_instance,
built_in_function_call,
range,
function,
list,
@ -616,7 +621,7 @@ pub fn parser<'src>(
.then(type_specification.clone().or_not())
.then(choice((
just(Token::Symbol(Symbol::Equal)).to(AssignmentOperator::Assign),
just(Token::Symbol(Symbol::PlusEquals)).to(AssignmentOperator::AddAssign),
just(Token::Symbol(Symbol::PlusEqual)).to(AssignmentOperator::AddAssign),
just(Token::Symbol(Symbol::MinusEqual)).to(AssignmentOperator::SubAssign),
)))
.then(statement.clone())
@ -757,5 +762,8 @@ pub fn parser<'src>(
.then_ignore(just(Token::Symbol(Symbol::Semicolon)).or_not())
});
statement.repeated().collect()
statement
.repeated()
.collect()
.map(|statements| AbstractTree::new(statements))
}

13
dust-lang/src/parser/tests.rs
View File

@ -226,7 +226,7 @@ fn r#as() {
#[test]
fn built_in_function() {
let tokens = lex("READ_LINE").unwrap();
let tokens = lex("__READ_LINE__").unwrap();
let statements = parser(true)
.parse(tokens.spanned((tokens.len()..tokens.len()).into()))
.into_result()
@ -241,11 +241,14 @@ fn built_in_function() {
assert_eq!(
statements[0],
Statement::Expression(Expression::BuiltInFunctionCall(
Box::new(BuiltInFunctionCall::ReadLine).with_position((0, 9))
Box::new(BuiltInFunctionCall::ReadLine).with_position((0, 13))
))
);
}
let tokens = lex("WRITE_LINE 'hiya'").unwrap();
#[test]
fn built_in_function_with_arg() {
let tokens = lex("__WRITE_LINE__ 'hiya'").unwrap();
let statements = parser(true)
.parse(tokens.spanned((tokens.len()..tokens.len()).into()))
.into_result()
@ -261,9 +264,9 @@ fn built_in_function() {
statements[0],
Statement::Expression(Expression::BuiltInFunctionCall(
Box::new(BuiltInFunctionCall::WriteLine(Expression::Value(
ValueNode::String("hiya".to_string()).with_position((11, 17))
ValueNode::String("hiya".to_string()).with_position((15, 21))
)))
.with_position((0, 17))
.with_position((0, 21))
))
);
}

5
dust-shell/src/cli.rs
View File

@ -86,9 +86,8 @@ pub fn run_shell(context: Context) -> Result<(), io::Error> {
let reports = error.build_reports();
for report in reports {
report
.write_for_stdout(sources(interpreter.sources()), stderr())
.unwrap();
let cache = sources(interpreter.sources());
report.write_for_stdout(cache, stderr()).unwrap();
}
}
}

14
dust-shell/src/main.rs
View File

@ -11,6 +11,7 @@ use std::{
fs::read_to_string,
io::{stderr, Write},
sync::Arc,
vec,
};
use dust_lang::{context::Context, Interpreter};
@ -78,9 +79,9 @@ fn main() {
let (source_id, source): (Arc<str>, Arc<str>) = if let Some(path) = args.path {
let source = read_to_string(&path).unwrap();
(Arc::from(path), Arc::from(source.as_str()))
(Arc::from(path.as_str()), Arc::from(source))
} else if let Some(command) = args.command {
(Arc::from("command"), Arc::from(command.as_str()))
(Arc::from("command"), Arc::from(command))
} else {
match run_shell(context) {
Ok(_) => {}
@ -96,7 +97,12 @@ fn main() {
Err(error) => {
for report in error.build_reports() {
report
.write_for_stdout(sources(interpreter.sources()), stderr())
.write_for_stdout(
sources::<Arc<str>, Arc<str>, vec::IntoIter<(Arc<str>, Arc<str>)>>(
interpreter.sources(),
),
stderr(),
)
.unwrap();
}
}
@ -120,7 +126,7 @@ fn main() {
return;
}
let run_result = interpreter.run(source_id.clone(), source.clone());
let run_result = interpreter.run(source_id.clone(), source);
match run_result {
Ok(value) => {

2
std/core.ds
View File

@ -1,3 +1,3 @@
length = fn (input: [any]) -> int {
LENGTH input
__LENGTH__ input
}

2
std/fs.ds
View File

@ -1,5 +1,5 @@
fs = {
read_file = fn (path: str) -> str {
READ_FILE path
__READ_FILE__ path
}
}

4
std/io.ds
View File

@ -1,9 +1,9 @@
io = {
read_line = fn () -> str {
READ_LINE
__READ_LINE__
}
write_line = fn (output: any) {
WRITE_LINE output
__WRITE_LINE__ output
}
}

2
std/json.ds
View File

@ -1,5 +1,5 @@
json = {
parse = fn |T| (input: str) -> T {
JSON_PARSE T input
__JSON_PARSE__ T input
}
}