From d9889ef2d878fbe83aa1de1e13af9db46d16fa71 Mon Sep 17 00:00:00 2001
From: Jeff <dev@jeffa.io>
Date: Sat, 27 Apr 2024 02:22:26 -0400
Subject: [PATCH] Add comments to lexer

---
 dust-lang/src/lexer.rs | 91 +++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 90 insertions(+), 1 deletion(-)

diff --git a/dust-lang/src/lexer.rs b/dust-lang/src/lexer.rs
index 7d90a09..2de60f0 100644
--- a/dust-lang/src/lexer.rs
+++ b/dust-lang/src/lexer.rs
@@ -10,6 +10,7 @@ use crate::error::Error;
 #[derive(Copy, Clone, Debug, PartialEq)]
 pub enum Token<'src> {
     Boolean(bool),
+    Comment(&'src str),
     Integer(i64),
     Float(f64),
     String(&'src str),
@@ -23,6 +24,7 @@ impl<'src> Display for Token<'src> {
     fn fmt(&self, f: &mut Formatter) -> fmt::Result {
         match self {
             Token::Boolean(boolean) => write!(f, "{boolean}"),
+            Token::Comment(comment) => write!(f, "# {comment}"),
             Token::Integer(integer) => write!(f, "{integer}"),
             Token::Float(float) => write!(f, "{float}"),
             Token::String(string) => write!(f, "{string}"),
@@ -190,6 +192,24 @@ pub fn lexer<'src>() -> impl Parser<
     Vec<(Token<'src>, SimpleSpan<usize>)>,
     extra::Err<Rich<'src, char, SimpleSpan<usize>>>,
 > {
+    let line_comment = just("//")
+        .ignore_then(
+            none_of('\n')
+                .repeated()
+                .to_slice()
+                .map(|text: &str| Token::Comment(text.trim())),
+        )
+        .then_ignore(just('\n').or_not());
+
+    let multi_line_comment = just("/*")
+        .ignore_then(
+            none_of('*')
+                .repeated()
+                .to_slice()
+                .map(|text: &str| Token::Comment(text.trim())),
+        )
+        .then_ignore(just("*/"));
+
     let boolean = choice((
         just("true").to(Token::Boolean(true)),
         just("false").to(Token::Boolean(false)),
@@ -308,7 +328,16 @@ pub fn lexer<'src>() -> impl Parser<
     .map(Token::Keyword);
 
     choice((
-        boolean, float, integer, string, keyword, identifier, control, operator,
+        line_comment,
+        multi_line_comment,
+        boolean,
+        float,
+        integer,
+        string,
+        keyword,
+        identifier,
+        control,
+        operator,
     ))
     .map_with(|token, state| (token, state.span()))
     .padded()
@@ -320,6 +349,66 @@ pub fn lexer<'src>() -> impl Parser<
 mod tests {
     use super::*;
 
+    #[test]
+    fn line_comment() {
+        assert_eq!(
+            lex("// 42").unwrap(),
+            vec![(Token::Comment("42"), (0..5).into())]
+        );
+
+        assert_eq!(
+            lex("1// 42//2").unwrap(),
+            vec![
+                (Token::Integer(1), (0..1).into()),
+                (Token::Comment("42//2"), (1..9).into()),
+            ]
+        );
+        assert_eq!(
+            lex("
+                1
+                // 42
+                2
+                ")
+            .unwrap(),
+            vec![
+                (Token::Integer(1), (17..18).into()),
+                (Token::Comment("42"), (35..41).into()),
+                (Token::Integer(2), (57..58).into()),
+            ]
+        );
+    }
+
+    #[test]
+    fn multi_line_comment() {
+        assert_eq!(
+            lex("/* 42 */").unwrap(),
+            vec![(Token::Comment("42"), (0..8).into())]
+        );
+
+        assert_eq!(
+            lex("1/* 42//2 */").unwrap(),
+            vec![
+                (Token::Integer(1), (0..1).into()),
+                (Token::Comment("42//2"), (1..12).into()),
+            ]
+        );
+        assert_eq!(
+            lex("
+                1
+                /*
+                    42
+                */
+                2
+                ")
+            .unwrap(),
+            vec![
+                (Token::Integer(1), (17..18).into()),
+                (Token::Comment("42"), (35..79).into()),
+                (Token::Integer(2), (96..97).into()),
+            ]
+        );
+    }
+
     #[test]
     fn range() {
         assert_eq!(