Fix lexing error

This commit is contained in:
Jeff 2024-03-09 20:57:46 -05:00
parent a3591d19af
commit f544bd008e
3 changed files with 81 additions and 30 deletions

View File

@ -63,6 +63,7 @@ impl Display for Operator {
#[derive(Copy, Clone, Debug, PartialEq)] #[derive(Copy, Clone, Debug, PartialEq)]
pub enum Control { pub enum Control {
Arrow,
CurlyOpen, CurlyOpen,
CurlyClose, CurlyClose,
SquareOpen, SquareOpen,
@ -80,6 +81,7 @@ pub enum Control {
impl Display for Control { impl Display for Control {
fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
match self { match self {
Control::Arrow => write!(f, "->"),
Control::CurlyOpen => write!(f, "{{"), Control::CurlyOpen => write!(f, "{{"),
Control::CurlyClose => write!(f, "}}"), Control::CurlyClose => write!(f, "}}"),
Control::SquareOpen => write!(f, "["), Control::SquareOpen => write!(f, "["),
@ -173,10 +175,10 @@ pub fn lexer<'src>() -> impl Parser<
just("&&").padded().to(Operator::And), just("&&").padded().to(Operator::And),
just("==").padded().to(Operator::Equal), just("==").padded().to(Operator::Equal),
just("!=").padded().to(Operator::NotEqual), just("!=").padded().to(Operator::NotEqual),
just(">").padded().to(Operator::Greater),
just(">=").padded().to(Operator::GreaterOrEqual), just(">=").padded().to(Operator::GreaterOrEqual),
just("<").padded().to(Operator::Less),
just("<=").padded().to(Operator::LessOrEqual), just("<=").padded().to(Operator::LessOrEqual),
just(">").padded().to(Operator::Greater),
just("<").padded().to(Operator::Less),
just("!").padded().to(Operator::Not), just("!").padded().to(Operator::Not),
just("!=").padded().to(Operator::NotEqual), just("!=").padded().to(Operator::NotEqual),
just("||").padded().to(Operator::Or), just("||").padded().to(Operator::Or),
@ -194,6 +196,7 @@ pub fn lexer<'src>() -> impl Parser<
.map(Token::Operator); .map(Token::Operator);
let control = choice(( let control = choice((
just("->").padded().to(Control::Arrow),
just("{").padded().to(Control::CurlyOpen), just("{").padded().to(Control::CurlyOpen),
just("}").padded().to(Control::CurlyClose), just("}").padded().to(Control::CurlyClose),
just("[").padded().to(Control::SquareOpen), just("[").padded().to(Control::SquareOpen),
@ -227,7 +230,7 @@ pub fn lexer<'src>() -> impl Parser<
.map(Token::Keyword); .map(Token::Keyword);
choice(( choice((
boolean, float, integer, string, keyword, identifier, operator, control, boolean, float, integer, string, keyword, identifier, control, operator,
)) ))
.map_with(|token, state| (token, state.span())) .map_with(|token, state| (token, state.span()))
.padded() .padded()

View File

@ -66,15 +66,34 @@ pub fn parser<'src>() -> DustParser<'src> {
just(Token::Control(Control::ParenOpen)), just(Token::Control(Control::ParenOpen)),
just(Token::Control(Control::ParenClose)), just(Token::Control(Control::ParenClose)),
) )
.then_ignore(just(Token::Control(Control::Colon))) .then_ignore(just(Token::Control(Control::Arrow)))
.then(r#type.clone()) .then(r#type.clone())
.map(|(parameter_types, return_type)| Type::Function { .map(|(parameter_types, return_type)| Type::Function {
parameter_types, parameter_types,
return_type: Box::new(return_type), return_type: Box::new(return_type),
}); });
let list_of = just(Token::Keyword("list"))
.ignore_then(r#type.clone().delimited_by(
just(Token::Control(Control::ParenOpen)),
just(Token::Control(Control::ParenClose)),
))
.map(|item_type| Type::ListOf(Box::new(item_type)));
let list_exact = r#type
.clone()
.separated_by(just(Token::Control(Control::Comma)))
.collect()
.delimited_by(
just(Token::Control(Control::SquareOpen)),
just(Token::Control(Control::SquareClose)),
)
.map(|types| Type::ListExact(types));
choice(( choice((
function_type, function_type,
list_of,
list_exact,
just(Token::Keyword("any")).to(Type::Any), just(Token::Keyword("any")).to(Type::Any),
just(Token::Keyword("bool")).to(Type::Boolean), just(Token::Keyword("bool")).to(Type::Boolean),
just(Token::Keyword("float")).to(Type::Float), just(Token::Keyword("float")).to(Type::Float),
@ -83,32 +102,13 @@ pub fn parser<'src>() -> DustParser<'src> {
just(Token::Keyword("range")).to(Type::Range), just(Token::Keyword("range")).to(Type::Range),
just(Token::Keyword("str")).to(Type::String), just(Token::Keyword("str")).to(Type::String),
just(Token::Keyword("list")).to(Type::List), just(Token::Keyword("list")).to(Type::List),
))
});
let type_arguments = r#type.clone().delimited_by(
just(Token::Control(Control::ParenOpen)),
just(Token::Control(Control::ParenClose)),
);
just(Token::Control(Control::Colon)).ignore_then(choice((
r#type
.clone()
.separated_by(just(Token::Control(Control::Comma)))
.collect()
.delimited_by(
just(Token::Control(Control::SquareOpen)),
just(Token::Control(Control::SquareClose)),
)
.map(|types| Type::ListExact(types)),
just(Token::Keyword("list"))
.then(type_arguments)
.map(|(_, item_type)| Type::ListOf(Box::new(item_type))),
r#type.clone(),
identifier identifier
.clone() .clone()
.map(|identifier| Type::Custom(identifier)), .map(|identifier| Type::Custom(identifier)),
))) ))
});
just(Token::Control(Control::Colon)).ignore_then(r#type)
}); });
let statement = recursive(|statement| { let statement = recursive(|statement| {
@ -350,12 +350,12 @@ pub fn parser<'src>() -> DustParser<'src> {
.boxed(); .boxed();
choice(( choice((
if_else,
assignment, assignment,
expression_statement, expression_statement,
r#break, r#break,
block_statement, block_statement,
r#loop, r#loop,
if_else,
)) ))
.then_ignore(just(Token::Control(Control::Semicolon)).or_not()) .then_ignore(just(Token::Control(Control::Semicolon)).or_not())
.boxed() .boxed()
@ -374,6 +374,54 @@ mod tests {
use super::*; use super::*;
#[test]
fn types() {
assert_eq!(
parse(&lex("foobar : bool = true").unwrap()).unwrap()[0].0,
Statement::Assignment(Assignment::new(
Identifier::new("foobar"),
Some(Type::Boolean),
AssignmentOperator::Assign,
Statement::Expression(Expression::Value(ValueNode::Boolean(true)))
))
);
assert_eq!(
parse(&lex("foobar : list(bool) = [true]").unwrap()).unwrap()[0].0,
Statement::Assignment(Assignment::new(
Identifier::new("foobar"),
Some(Type::ListOf(Box::new(Type::Boolean))),
AssignmentOperator::Assign,
Statement::Expression(Expression::Value(ValueNode::List(vec![Expression::Value(
ValueNode::Boolean(true)
)])))
))
);
assert_eq!(
parse(&lex("foobar : [bool, str] = [true, '42']").unwrap()).unwrap()[0].0,
Statement::Assignment(Assignment::new(
Identifier::new("foobar"),
Some(Type::ListExact(vec![Type::Boolean, Type::String])),
AssignmentOperator::Assign,
Statement::Expression(Expression::Value(ValueNode::List(vec![
Expression::Value(ValueNode::Boolean(true)),
Expression::Value(ValueNode::String("42".to_string()))
])))
))
);
assert_eq!(
parse(&lex("foobar : () -> any = some_function").unwrap()).unwrap()[0].0,
Statement::Assignment(Assignment::new(
Identifier::new("foobar"),
Some(Type::Function {
parameter_types: vec![],
return_type: Box::new(Type::Any)
}),
AssignmentOperator::Assign,
Statement::Expression(Expression::Identifier(Identifier::new("some_function")))
))
);
}
#[test] #[test]
fn function_call() { fn function_call() {
assert_eq!( assert_eq!(
@ -418,7 +466,7 @@ mod tests {
))]), ))]),
None None
)) ))
) );
} }
#[test] #[test]

View File

@ -38,7 +38,7 @@ fn callback() {
foobar = (cb : () -> str) : str { foobar = (cb : () -> str) : str {
cb() cb()
} }
foobar(() : str 'Hiya') foobar(() : str { 'Hiya' })
", ",
), ),
Ok(Some(Value::string("Hiya".to_string()))) Ok(Some(Value::string("Hiya".to_string())))