From d9889ef2d878fbe83aa1de1e13af9db46d16fa71 Mon Sep 17 00:00:00 2001 From: Jeff Date: Sat, 27 Apr 2024 02:22:26 -0400 Subject: [PATCH] Add comments to lexer --- dust-lang/src/lexer.rs | 91 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 90 insertions(+), 1 deletion(-) diff --git a/dust-lang/src/lexer.rs b/dust-lang/src/lexer.rs index 7d90a09..2de60f0 100644 --- a/dust-lang/src/lexer.rs +++ b/dust-lang/src/lexer.rs @@ -10,6 +10,7 @@ use crate::error::Error; #[derive(Copy, Clone, Debug, PartialEq)] pub enum Token<'src> { Boolean(bool), + Comment(&'src str), Integer(i64), Float(f64), String(&'src str), @@ -23,6 +24,7 @@ impl<'src> Display for Token<'src> { fn fmt(&self, f: &mut Formatter) -> fmt::Result { match self { Token::Boolean(boolean) => write!(f, "{boolean}"), + Token::Comment(comment) => write!(f, "# {comment}"), Token::Integer(integer) => write!(f, "{integer}"), Token::Float(float) => write!(f, "{float}"), Token::String(string) => write!(f, "{string}"), @@ -190,6 +192,24 @@ pub fn lexer<'src>() -> impl Parser< Vec<(Token<'src>, SimpleSpan)>, extra::Err>>, > { + let line_comment = just("//") + .ignore_then( + none_of('\n') + .repeated() + .to_slice() + .map(|text: &str| Token::Comment(text.trim())), + ) + .then_ignore(just('\n').or_not()); + + let multi_line_comment = just("/*") + .ignore_then( + none_of('*') + .repeated() + .to_slice() + .map(|text: &str| Token::Comment(text.trim())), + ) + .then_ignore(just("*/")); + let boolean = choice(( just("true").to(Token::Boolean(true)), just("false").to(Token::Boolean(false)), @@ -308,7 +328,16 @@ pub fn lexer<'src>() -> impl Parser< .map(Token::Keyword); choice(( - boolean, float, integer, string, keyword, identifier, control, operator, + line_comment, + multi_line_comment, + boolean, + float, + integer, + string, + keyword, + identifier, + control, + operator, )) .map_with(|token, state| (token, state.span())) .padded() @@ -320,6 +349,66 @@ pub fn lexer<'src>() -> impl Parser< mod tests { use super::*; + #[test] + fn line_comment() { + assert_eq!( + lex("// 42").unwrap(), + vec![(Token::Comment("42"), (0..5).into())] + ); + + assert_eq!( + lex("1// 42//2").unwrap(), + vec![ + (Token::Integer(1), (0..1).into()), + (Token::Comment("42//2"), (1..9).into()), + ] + ); + assert_eq!( + lex(" + 1 + // 42 + 2 + ") + .unwrap(), + vec![ + (Token::Integer(1), (17..18).into()), + (Token::Comment("42"), (35..41).into()), + (Token::Integer(2), (57..58).into()), + ] + ); + } + + #[test] + fn multi_line_comment() { + assert_eq!( + lex("/* 42 */").unwrap(), + vec![(Token::Comment("42"), (0..8).into())] + ); + + assert_eq!( + lex("1/* 42//2 */").unwrap(), + vec![ + (Token::Integer(1), (0..1).into()), + (Token::Comment("42//2"), (1..12).into()), + ] + ); + assert_eq!( + lex(" + 1 + /* + 42 + */ + 2 + ") + .unwrap(), + vec![ + (Token::Integer(1), (17..18).into()), + (Token::Comment("42"), (35..79).into()), + (Token::Integer(2), (96..97).into()), + ] + ); + } + #[test] fn range() { assert_eq!(