diff --git a/dust-lang/src/lexer.rs b/dust-lang/src/lexer.rs index ed89551..a421da1 100644 --- a/dust-lang/src/lexer.rs +++ b/dust-lang/src/lexer.rs @@ -100,7 +100,7 @@ impl Display for Keyword { #[derive(Copy, Clone, Debug, PartialEq)] pub enum Symbol { Plus, - PlusEquals, + PlusEqual, DoubleAmpersand, Colon, Comma, @@ -166,7 +166,7 @@ impl Display for Symbol { Symbol::Percent => write!(f, "%"), Symbol::Pipe => write!(f, "|"), Symbol::Plus => write!(f, "+"), - Symbol::PlusEquals => write!(f, "+="), + Symbol::PlusEqual => write!(f, "+="), Symbol::Semicolon => write!(f, ";"), Symbol::SkinnyArrow => write!(f, "->"), Symbol::Slash => write!(f, "/"), @@ -251,35 +251,34 @@ pub fn lexer<'src>() -> impl Parser< delimited_string('`'), )); - let identifier_and_keyword = text::ident().map(|text: &str| match text { - "any" => Token::Keyword(Keyword::Any), - "async" => Token::Keyword(Keyword::Async), - "as" => Token::Keyword(Keyword::As), - "bool" => Token::Keyword(Keyword::Bool), - "break" => Token::Keyword(Keyword::Break), - "enum" => Token::Keyword(Keyword::Enum), - "else" => Token::Keyword(Keyword::Else), - "float" => Token::Keyword(Keyword::Float), - "fn" => Token::Keyword(Keyword::Fn), - "int" => Token::Keyword(Keyword::Int), - "if" => Token::Keyword(Keyword::If), - "list" => Token::Keyword(Keyword::List), - "map" => Token::Keyword(Keyword::Map), - "none" => Token::Keyword(Keyword::None), - "range" => Token::Keyword(Keyword::Range), - "struct" => Token::Keyword(Keyword::Struct), - "str" => Token::Keyword(Keyword::Str), - "type" => Token::Keyword(Keyword::Type), - "loop" => Token::Keyword(Keyword::Loop), - "while" => Token::Keyword(Keyword::While), - "JSON_PARSE" => Token::Keyword(Keyword::JsonParse), - "LENGTH" => Token::Keyword(Keyword::Length), - "READ_FILE" => Token::Keyword(Keyword::ReadFile), - "READ_LINE" => Token::Keyword(Keyword::ReadLine), - "SLEEP" => Token::Keyword(Keyword::Sleep), - "WRITE_LINE" => Token::Keyword(Keyword::WriteLine), - _ => Token::Identifier(text), - }); + let keyword = choice(( + just("any").to(Token::Keyword(Keyword::Any)), + just("async").to(Token::Keyword(Keyword::Async)), + just("as").to(Token::Keyword(Keyword::As)), + just("bool").to(Token::Keyword(Keyword::Bool)), + just("break").to(Token::Keyword(Keyword::Break)), + just("enum").to(Token::Keyword(Keyword::Enum)), + just("else").to(Token::Keyword(Keyword::Else)), + just("float").to(Token::Keyword(Keyword::Float)), + just("fn").to(Token::Keyword(Keyword::Fn)), + just("int").to(Token::Keyword(Keyword::Int)), + just("if").to(Token::Keyword(Keyword::If)), + just("list").to(Token::Keyword(Keyword::List)), + just("map").to(Token::Keyword(Keyword::Map)), + just("none").to(Token::Keyword(Keyword::None)), + just("range").to(Token::Keyword(Keyword::Range)), + just("struct").to(Token::Keyword(Keyword::Struct)), + just("str").to(Token::Keyword(Keyword::Str)), + just("type").to(Token::Keyword(Keyword::Type)), + just("loop").to(Token::Keyword(Keyword::Loop)), + just("while").to(Token::Keyword(Keyword::While)), + just("JSON_PARSE").to(Token::Keyword(Keyword::JsonParse)), + just("LENGTH").to(Token::Keyword(Keyword::Length)), + just("READ_FILE").to(Token::Keyword(Keyword::ReadFile)), + just("READ_LINE").to(Token::Keyword(Keyword::ReadLine)), + just("SLEEP").to(Token::Keyword(Keyword::Sleep)), + just("WRITE_LINE").to(Token::Keyword(Keyword::WriteLine)), + )); let symbol = choice([ just("!=").to(Token::Symbol(Symbol::NotEqual)), @@ -290,7 +289,7 @@ pub fn lexer<'src>() -> impl Parser< just("(").to(Token::Symbol(Symbol::ParenOpen)), just(")").to(Token::Symbol(Symbol::ParenClose)), just("*").to(Token::Symbol(Symbol::Asterisk)), - just("+=").to(Token::Symbol(Symbol::PlusEquals)), + just("+=").to(Token::Symbol(Symbol::PlusEqual)), just("+").to(Token::Symbol(Symbol::Plus)), just(",").to(Token::Symbol(Symbol::Comma)), just("->").to(Token::Symbol(Symbol::SkinnyArrow)), @@ -313,11 +312,13 @@ pub fn lexer<'src>() -> impl Parser< just("]").to(Token::Symbol(Symbol::SquareClose)), just("__").to(Token::Symbol(Symbol::DoubleUnderscore)), just("{").to(Token::Symbol(Symbol::CurlyOpen)), + just("}").to(Token::Symbol(Symbol::CurlyClose)), just("||").to(Token::Symbol(Symbol::DoublePipe)), just("|").to(Token::Symbol(Symbol::Pipe)), - just("}").to(Token::Symbol(Symbol::CurlyClose)), ]); + let identifier = text::ident().map(|text: &str| Token::Identifier(text)); + choice(( line_comment, multi_line_comment, @@ -325,8 +326,9 @@ pub fn lexer<'src>() -> impl Parser< float, integer, string, - identifier_and_keyword, + keyword, symbol, + identifier, )) .map_with(|token: Token, state| (token, state.span())) .padded() diff --git a/dust-lang/src/lib.rs b/dust-lang/src/lib.rs index 75e8226..13911d5 100644 --- a/dust-lang/src/lib.rs +++ b/dust-lang/src/lib.rs @@ -14,14 +14,15 @@ use std::{ use abstract_tree::{AbstractTree, Type}; use ariadne::{Color, Config, Fmt, Label, Report, ReportKind}; +use chumsky::prelude::*; use context::Context; use error::{DustError, RuntimeError, TypeConflict, ValidationError}; use lexer::{lex, Token}; -use parser::parse; +use parser::{parse, parser}; use rayon::prelude::*; pub use value::Value; -pub fn interpret<'src>(source_id: &str, source: &str) -> Result, InterpreterError> { +pub fn interpret(source_id: &str, source: &str) -> Result, InterpreterError> { let interpreter = Interpreter::new(Context::new(None)); interpreter.load_std()?; @@ -34,7 +35,7 @@ pub fn interpret_without_std( ) -> Result, InterpreterError> { let interpreter = Interpreter::new(Context::new(None)); - interpreter.run(Arc::from(source_id.to_string()), Arc::from(source)) + interpreter.run(Arc::from(source_id), Arc::from(source)) } pub struct Interpreter { @@ -137,18 +138,18 @@ impl Interpreter { // Always load the core library first because other parts of the standard library may depend // on it. - self.run(std_core_source.0, std_core_source.1)?; + self.run_with_builtins(std_core_source.0, std_core_source.1)?; let error = if cfg!(test) { // In debug mode, load the standard library sequentially to get consistent errors. std_sources .into_iter() - .find_map(|(source_id, source)| self.run(source_id, source).err()) + .find_map(|(source_id, source)| self.run_with_builtins(source_id, source).err()) } else { // In release mode, load the standard library asynchronously. std_sources .into_par_iter() - .find_map_any(|(source_id, source)| self.run(source_id, source).err()) + .find_map_any(|(source_id, source)| self.run_with_builtins(source_id, source).err()) }; log::info!("Finish loading standard library."); @@ -163,6 +164,37 @@ impl Interpreter { pub fn sources(&self) -> vec::IntoIter<(Arc, Arc)> { self.sources.read().unwrap().clone().into_iter() } + + fn run_with_builtins( + &self, + source_id: Arc, + source: Arc, + ) -> Result, InterpreterError> { + let mut sources = self.sources.write().unwrap(); + + sources.clear(); + sources.push((source_id.clone(), source.clone())); + + let tokens = lex(source.as_ref()).map_err(|errors| InterpreterError { + source_id: source_id.clone(), + errors, + })?; + let abstract_tree = parser(true) + .parse(tokens.spanned((tokens.len()..tokens.len()).into())) + .into_result() + .map_err(|errors| InterpreterError { + source_id: source_id.clone(), + errors: errors + .into_iter() + .map(|error| DustError::from(error)) + .collect(), + })?; + let value_option = abstract_tree + .run(&self.context, true) + .map_err(|errors| InterpreterError { source_id, errors })?; + + Ok(value_option) + } } #[derive(Debug, PartialEq)] diff --git a/dust-lang/src/parser/mod.rs b/dust-lang/src/parser/mod.rs index 8da9197..763057e 100644 --- a/dust-lang/src/parser/mod.rs +++ b/dust-lang/src/parser/mod.rs @@ -25,7 +25,7 @@ pub type ParserExtra<'src> = extra::Err, SimpleSpan>>; pub fn parse<'src>( tokens: &'src [(Token<'src>, SimpleSpan)], ) -> Result> { - let statements = parser(false) + parser(false) .parse(tokens.spanned((tokens.len()..tokens.len()).into())) .into_result() .map_err(|errors| { @@ -33,14 +33,12 @@ pub fn parse<'src>( .into_iter() .map(|error| DustError::from(error)) .collect::>() - })?; - - Ok(AbstractTree::new(statements)) + }) } pub fn parser<'src>( allow_built_ins: bool, -) -> impl Parser<'src, ParserInput<'src>, Vec, ParserExtra<'src>> { +) -> impl Parser<'src, ParserInput<'src>, AbstractTree, ParserExtra<'src>> { let comment = select_ref! { Token::Comment(_) => {} }; @@ -330,8 +328,15 @@ pub fn parser<'src>( ) }); + let _built_in_function = |keyword| { + just(Token::Keyword(keyword)).delimited_by( + just(Token::Symbol(Symbol::DoubleUnderscore)), + just(Token::Symbol(Symbol::DoubleUnderscore)), + ) + }; + let built_in_function_call = choice(( - just(Token::Keyword(Keyword::Length)) + _built_in_function(Keyword::Length) .ignore_then(expression.clone()) .map_with(|argument, state| { Expression::BuiltInFunctionCall( @@ -339,7 +344,7 @@ pub fn parser<'src>( .with_position(state.span()), ) }), - just(Token::Keyword(Keyword::ReadFile)) + _built_in_function(Keyword::ReadFile) .ignore_then(expression.clone()) .map_with(|argument, state| { Expression::BuiltInFunctionCall( @@ -347,12 +352,12 @@ pub fn parser<'src>( .with_position(state.span()), ) }), - just(Token::Keyword(Keyword::ReadLine)).map_with(|_, state| { + _built_in_function(Keyword::ReadLine).map_with(|_, state| { Expression::BuiltInFunctionCall( Box::new(BuiltInFunctionCall::ReadLine).with_position(state.span()), ) }), - just(Token::Keyword(Keyword::Sleep)) + _built_in_function(Keyword::Sleep) .ignore_then(expression.clone()) .map_with(|argument, state| { Expression::BuiltInFunctionCall( @@ -360,7 +365,7 @@ pub fn parser<'src>( .with_position(state.span()), ) }), - just(Token::Keyword(Keyword::WriteLine)) + _built_in_function(Keyword::WriteLine) .ignore_then(expression.clone()) .map_with(|argument, state| { Expression::BuiltInFunctionCall( @@ -368,7 +373,7 @@ pub fn parser<'src>( .with_position(state.span()), ) }), - just(Token::Keyword(Keyword::JsonParse)) + _built_in_function(Keyword::JsonParse) .ignore_then(type_constructor.clone()) .then(expression.clone()) .map_with(|(constructor, argument), state| { @@ -378,15 +383,15 @@ pub fn parser<'src>( ) }), )) - .try_map_with(move |expression, state| { - if allow_built_ins { - Ok(expression) - } else { - Err(Rich::custom( + .validate(move |expression, state, emitter| { + if !allow_built_ins { + emitter.emit(Rich::custom( state.span(), "Built-in function calls can only be used by the standard library.", )) } + + expression }); let turbofish = type_constructor @@ -583,9 +588,9 @@ pub fn parser<'src>( )); choice(( + built_in_function_call, logic_math_indexes_as_and_function_calls, enum_instance, - built_in_function_call, range, function, list, @@ -616,7 +621,7 @@ pub fn parser<'src>( .then(type_specification.clone().or_not()) .then(choice(( just(Token::Symbol(Symbol::Equal)).to(AssignmentOperator::Assign), - just(Token::Symbol(Symbol::PlusEquals)).to(AssignmentOperator::AddAssign), + just(Token::Symbol(Symbol::PlusEqual)).to(AssignmentOperator::AddAssign), just(Token::Symbol(Symbol::MinusEqual)).to(AssignmentOperator::SubAssign), ))) .then(statement.clone()) @@ -757,5 +762,8 @@ pub fn parser<'src>( .then_ignore(just(Token::Symbol(Symbol::Semicolon)).or_not()) }); - statement.repeated().collect() + statement + .repeated() + .collect() + .map(|statements| AbstractTree::new(statements)) } diff --git a/dust-lang/src/parser/tests.rs b/dust-lang/src/parser/tests.rs index 298d9f0..43914a3 100644 --- a/dust-lang/src/parser/tests.rs +++ b/dust-lang/src/parser/tests.rs @@ -226,7 +226,7 @@ fn r#as() { #[test] fn built_in_function() { - let tokens = lex("READ_LINE").unwrap(); + let tokens = lex("__READ_LINE__").unwrap(); let statements = parser(true) .parse(tokens.spanned((tokens.len()..tokens.len()).into())) .into_result() @@ -241,11 +241,14 @@ fn built_in_function() { assert_eq!( statements[0], Statement::Expression(Expression::BuiltInFunctionCall( - Box::new(BuiltInFunctionCall::ReadLine).with_position((0, 9)) + Box::new(BuiltInFunctionCall::ReadLine).with_position((0, 13)) )) ); +} - let tokens = lex("WRITE_LINE 'hiya'").unwrap(); +#[test] +fn built_in_function_with_arg() { + let tokens = lex("__WRITE_LINE__ 'hiya'").unwrap(); let statements = parser(true) .parse(tokens.spanned((tokens.len()..tokens.len()).into())) .into_result() @@ -261,9 +264,9 @@ fn built_in_function() { statements[0], Statement::Expression(Expression::BuiltInFunctionCall( Box::new(BuiltInFunctionCall::WriteLine(Expression::Value( - ValueNode::String("hiya".to_string()).with_position((11, 17)) + ValueNode::String("hiya".to_string()).with_position((15, 21)) ))) - .with_position((0, 17)) + .with_position((0, 21)) )) ); } diff --git a/dust-shell/src/cli.rs b/dust-shell/src/cli.rs index 5745340..4a10698 100644 --- a/dust-shell/src/cli.rs +++ b/dust-shell/src/cli.rs @@ -86,9 +86,8 @@ pub fn run_shell(context: Context) -> Result<(), io::Error> { let reports = error.build_reports(); for report in reports { - report - .write_for_stdout(sources(interpreter.sources()), stderr()) - .unwrap(); + let cache = sources(interpreter.sources()); + report.write_for_stdout(cache, stderr()).unwrap(); } } } diff --git a/dust-shell/src/main.rs b/dust-shell/src/main.rs index 9b75da6..bdc9af8 100644 --- a/dust-shell/src/main.rs +++ b/dust-shell/src/main.rs @@ -11,6 +11,7 @@ use std::{ fs::read_to_string, io::{stderr, Write}, sync::Arc, + vec, }; use dust_lang::{context::Context, Interpreter}; @@ -78,9 +79,9 @@ fn main() { let (source_id, source): (Arc, Arc) = if let Some(path) = args.path { let source = read_to_string(&path).unwrap(); - (Arc::from(path), Arc::from(source.as_str())) + (Arc::from(path.as_str()), Arc::from(source)) } else if let Some(command) = args.command { - (Arc::from("command"), Arc::from(command.as_str())) + (Arc::from("command"), Arc::from(command)) } else { match run_shell(context) { Ok(_) => {} @@ -96,7 +97,12 @@ fn main() { Err(error) => { for report in error.build_reports() { report - .write_for_stdout(sources(interpreter.sources()), stderr()) + .write_for_stdout( + sources::, Arc, vec::IntoIter<(Arc, Arc)>>( + interpreter.sources(), + ), + stderr(), + ) .unwrap(); } } @@ -120,7 +126,7 @@ fn main() { return; } - let run_result = interpreter.run(source_id.clone(), source.clone()); + let run_result = interpreter.run(source_id.clone(), source); match run_result { Ok(value) => { diff --git a/std/core.ds b/std/core.ds index a9a0c92..39d555f 100644 --- a/std/core.ds +++ b/std/core.ds @@ -1,3 +1,3 @@ length = fn (input: [any]) -> int { - LENGTH input + __LENGTH__ input } diff --git a/std/fs.ds b/std/fs.ds index 7e92ad3..18e0395 100644 --- a/std/fs.ds +++ b/std/fs.ds @@ -1,5 +1,5 @@ fs = { read_file = fn (path: str) -> str { - READ_FILE path + __READ_FILE__ path } } diff --git a/std/io.ds b/std/io.ds index 4ee1913..004dd37 100644 --- a/std/io.ds +++ b/std/io.ds @@ -1,9 +1,9 @@ io = { read_line = fn () -> str { - READ_LINE + __READ_LINE__ } write_line = fn (output: any) { - WRITE_LINE output + __WRITE_LINE__ output } } diff --git a/std/json.ds b/std/json.ds index 6f90528..69c18b0 100644 --- a/std/json.ds +++ b/std/json.ds @@ -1,5 +1,5 @@ json = { parse = fn |T| (input: str) -> T { - JSON_PARSE T input + __JSON_PARSE__ T input } }