From 8f20e538807de6481014b8cb583c2385ffeafafc Mon Sep 17 00:00:00 2001 From: Jeff Date: Mon, 9 Sep 2024 23:45:06 -0400 Subject: [PATCH] Add support for characters and bytes --- dust-lang/src/lexer.rs | 16 +++++++++++ dust-lang/src/parser.rs | 32 +++++++++++++++++++++- dust-lang/src/token.rs | 9 ++++++ dust-lang/tests/{math.rs => operations.rs} | 0 4 files changed, 56 insertions(+), 1 deletion(-) rename dust-lang/tests/{math.rs => operations.rs} (100%) diff --git a/dust-lang/src/lexer.rs b/dust-lang/src/lexer.rs index 557a6e2..06c09c9 100644 --- a/dust-lang/src/lexer.rs +++ b/dust-lang/src/lexer.rs @@ -424,6 +424,22 @@ impl<'src> Lexer<'src> { } } + if c == 'x' { + self.next_char(); + + while let Some(c) = self.peek_char() { + if c.is_ascii_hexdigit() { + self.next_char(); + } else { + break; + } + } + + let text = &self.source[start_pos..self.position]; + + return Ok((Token::Byte(text), Span(start_pos, self.position))); + } + if c.is_ascii_digit() { self.next_char(); } else { diff --git a/dust-lang/src/parser.rs b/dust-lang/src/parser.rs index df83045..f97005b 100644 --- a/dust-lang/src/parser.rs +++ b/dust-lang/src/parser.rs @@ -115,6 +115,27 @@ impl<'src> Parser<'src> { Ok(()) } + fn parse_byte(&mut self, _allow_assignment: bool) -> Result<(), ParseError> { + if let Token::Byte(text) = self.previous_token { + let byte = u8::from_str_radix(&text[2..], 16)?; + let value = Value::byte(byte); + + self.emit_constant(value)?; + } + + Ok(()) + } + + fn parse_character(&mut self, _allow_assignment: bool) -> Result<(), ParseError> { + if let Token::Character(character) = self.previous_token { + let value = Value::character(character); + + self.emit_constant(value)?; + } + + Ok(()) + } + fn parse_float(&mut self, _allow_assignment: bool) -> Result<(), ParseError> { if let Token::Float(text) = self.previous_token { let float = text.parse::().unwrap(); @@ -418,7 +439,16 @@ impl From<&TokenKind> for ParseRule<'_> { infix: None, precedence: Precedence::None, }, - TokenKind::Character => todo!(), + TokenKind::Byte => ParseRule { + prefix: Some(Parser::parse_byte), + infix: None, + precedence: Precedence::None, + }, + TokenKind::Character => ParseRule { + prefix: Some(Parser::parse_character), + infix: None, + precedence: Precedence::None, + }, TokenKind::Float => ParseRule { prefix: Some(Parser::parse_float), infix: None, diff --git a/dust-lang/src/token.rs b/dust-lang/src/token.rs index 2f2ce04..a0d5fe1 100644 --- a/dust-lang/src/token.rs +++ b/dust-lang/src/token.rs @@ -11,6 +11,7 @@ pub enum Token<'src> { // Hard-coded values Boolean(&'src str), + Byte(&'src str), Character(char), Float(&'src str), Identifier(&'src str), @@ -70,6 +71,7 @@ impl<'src> Token<'src> { match self { Token::Eof => 0, Token::Boolean(text) => text.len(), + Token::Byte(_) => 3, Token::Character(_) => 3, Token::Float(text) => text.len(), Token::Identifier(text) => text.len(), @@ -128,6 +130,7 @@ impl<'src> Token<'src> { Token::Bool => TokenOwned::Bool, Token::Boolean(boolean) => TokenOwned::Boolean(boolean.to_string()), Token::Break => TokenOwned::Break, + Token::Byte(byte) => TokenOwned::Byte(byte.to_string()), Token::Character(character) => TokenOwned::Character(*character), Token::Colon => TokenOwned::Colon, Token::Comma => TokenOwned::Comma, @@ -182,6 +185,7 @@ impl<'src> Token<'src> { Token::Bool => TokenKind::Bool, Token::Boolean(_) => TokenKind::Boolean, Token::Break => TokenKind::Break, + Token::Byte(_) => TokenKind::Byte, Token::Character(_) => TokenKind::Character, Token::Colon => TokenKind::Colon, Token::Comma => TokenKind::Comma, @@ -238,6 +242,7 @@ impl<'src> Display for Token<'src> { Token::Bool => write!(f, "bool"), Token::Boolean(value) => write!(f, "{}", value), Token::Break => write!(f, "break"), + Token::Byte(value) => write!(f, "{}", value), Token::Character(value) => write!(f, "'{}'", value), Token::Colon => write!(f, ":"), Token::Comma => write!(f, ","), @@ -296,6 +301,7 @@ pub enum TokenOwned { // Hard-coded values Boolean(String), + Byte(String), Character(char), Float(String), Integer(String), @@ -357,6 +363,7 @@ impl Display for TokenOwned { TokenOwned::Bool => Token::Bool.fmt(f), TokenOwned::Boolean(boolean) => Token::Boolean(boolean).fmt(f), TokenOwned::Break => Token::Break.fmt(f), + TokenOwned::Byte(byte) => Token::Byte(byte).fmt(f), TokenOwned::Character(character) => Token::Character(*character).fmt(f), TokenOwned::Colon => Token::Colon.fmt(f), TokenOwned::Comma => Token::Comma.fmt(f), @@ -413,6 +420,7 @@ pub enum TokenKind { // Hard-coded values Boolean, + Byte, Character, Float, Integer, @@ -474,6 +482,7 @@ impl Display for TokenKind { TokenKind::Bool => Token::Bool.fmt(f), TokenKind::Boolean => write!(f, "boolean value"), TokenKind::Break => Token::Break.fmt(f), + TokenKind::Byte => write!(f, "byte value"), TokenKind::Character => write!(f, "character value"), TokenKind::Colon => Token::Colon.fmt(f), TokenKind::Comma => Token::Comma.fmt(f), diff --git a/dust-lang/tests/math.rs b/dust-lang/tests/operations.rs similarity index 100% rename from dust-lang/tests/math.rs rename to dust-lang/tests/operations.rs