From 9766777a471b24741f36624b51b0790a0866281d Mon Sep 17 00:00:00 2001 From: Jeff Date: Fri, 9 Aug 2024 00:31:38 -0400 Subject: [PATCH] Expand lexing of complex floats --- dust-lang/src/dust_error.rs | 4 +- dust-lang/src/lex.rs | 89 ++++++++++++++++++++++++++++++++++--- dust-lang/src/token.rs | 13 +++++- dust-lang/src/value.rs | 8 ++++ dust-lang/src/vm.rs | 33 +++++++++----- 5 files changed, 127 insertions(+), 20 deletions(-) diff --git a/dust-lang/src/dust_error.rs b/dust-lang/src/dust_error.rs index 7a942d2..8296754 100644 --- a/dust-lang/src/dust_error.rs +++ b/dust-lang/src/dust_error.rs @@ -21,7 +21,7 @@ impl<'src> DustError<'src> { let title = match &self.vm_error { VmError::AnaylyzerError(_) => "Analyzer error", VmError::ParseError(_) => "Parse error", - VmError::ValueError(_) => "Value error", + VmError::ValueError { .. } => "Value error", VmError::BuiltInFunctionCallError(_) => "Runtime error", _ => "Analysis Failure", }; @@ -36,7 +36,7 @@ impl<'src> DustError<'src> { AnalyzerError::UnexpectedIdentifier { position, .. } => position, }, VmError::ParseError(_) => todo!(), - VmError::ValueError(_) => todo!(), + VmError::ValueError { position, .. } => position, VmError::BuiltInFunctionCallError(_) => todo!(), VmError::ExpectedIdentifier { position } => position, VmError::ExpectedIdentifierOrInteger { position } => position, diff --git a/dust-lang/src/lex.rs b/dust-lang/src/lex.rs index 6faa209..e42476e 100644 --- a/dust-lang/src/lex.rs +++ b/dust-lang/src/lex.rs @@ -106,6 +106,12 @@ impl Lexer { '-' => { if let Some('0'..='9') = self.peek_second_char(source) { self.lex_number(source)? + } else if "-Infinity" == self.peek_chars(source, 9) { + self.position += 9; + ( + Token::Float(f64::NEG_INFINITY), + (self.position - 9, self.position), + ) } else { self.position += 1; (Token::Minus, (self.position - 1, self.position)) @@ -150,7 +156,7 @@ impl Lexer { self.position += 1; (Token::Dot, (self.position - 1, self.position)) } - _ => (Token::Eof, (self.position, self.position)), + _ => return Err(LexError::UnexpectedCharacter(c)), } } else { (Token::Eof, (self.position, self.position)) @@ -188,6 +194,17 @@ impl Lexer { source[self.position..].chars().nth(1) } + /// Peek the next `n` characters without consuming them. + fn peek_chars<'src>(&self, source: &'src str, n: usize) -> &'src str { + let remaining_source = &source[self.position..]; + + if remaining_source.len() < n { + remaining_source + } else { + &remaining_source[..n] + } + } + /// Lex an integer or float token. fn lex_number<'src>(&mut self, source: &'src str) -> Result<(Token<'src>, Span), LexError> { let start_pos = self.position; @@ -206,8 +223,21 @@ impl Lexer { self.next_char(source); - while let Some('0'..='9') = self.peek_char(source) { - self.next_char(source); + loop { + let peek_char = self.peek_char(source); + + if let Some('0'..='9') = peek_char { + self.next_char(source); + } else if let Some('e') = peek_char { + if let Some('0'..='9') = self.peek_second_char(source) { + self.next_char(source); + self.next_char(source); + } else { + break; + } + } else { + break; + } } is_float = true; @@ -242,7 +272,7 @@ impl Lexer { let start_pos = self.position; while let Some(c) = self.peek_char(source) { - if c.is_ascii_alphanumeric() || c == '_' { + if c.is_ascii_alphabetic() || c == '_' { self.next_char(source); } else { break; @@ -253,9 +283,11 @@ impl Lexer { let token = match string { "true" => Token::Boolean(true), "false" => Token::Boolean(false), + "Infinity" => Token::Float(f64::INFINITY), "is_even" => Token::IsEven, "is_odd" => Token::IsOdd, "length" => Token::Length, + "NaN" => Token::Float(f64::NAN), "read_line" => Token::ReadLine, "write_line" => Token::WriteLine, _ => Token::Identifier(string), @@ -298,6 +330,7 @@ impl Default for Lexer { pub enum LexError { FloatError(ParseFloatError), IntegerError(ParseIntError), + UnexpectedCharacter(char), } impl Error for LexError { @@ -305,6 +338,7 @@ impl Error for LexError { match self { Self::FloatError(parse_float_error) => Some(parse_float_error), Self::IntegerError(parse_int_error) => Some(parse_int_error), + Self::UnexpectedCharacter(_) => None, } } } @@ -318,6 +352,9 @@ impl Display for LexError { Self::IntegerError(parse_int_error) => { write!(f, "Failed to parse integer: {}", parse_int_error) } + Self::UnexpectedCharacter(character) => { + write!(f, "Unexpected character: '{}'", character) + } } } } @@ -336,9 +373,51 @@ impl From for LexError { #[cfg(test)] mod tests { - use super::*; + #[test] + fn infinity() { + let input = "Infinity"; + + assert_eq!( + lex(input), + Ok(vec![ + (Token::Float(f64::INFINITY), (0, 8)), + (Token::Eof, (8, 8)), + ]) + ) + } + + #[test] + fn negative_infinity() { + let input = "-Infinity"; + + assert_eq!( + lex(input), + Ok(vec![ + (Token::Float(f64::NEG_INFINITY), (0, 9)), + (Token::Eof, (9, 9)), + ]) + ) + } + + #[test] + fn nan() { + let input = "NaN"; + + assert!(lex(input).is_ok_and(|tokens| tokens[0].0 == Token::Float(f64::NAN))); + } + + #[test] + fn complex_float() { + let input = "42.42e42"; + + assert_eq!( + lex(input), + Ok(vec![(Token::Float(42.42e42), (0, 8)), (Token::Eof, (8, 8)),]) + ) + } + #[test] fn max_integer() { let input = "9223372036854775807"; diff --git a/dust-lang/src/token.rs b/dust-lang/src/token.rs index 565a439..dfb6e83 100644 --- a/dust-lang/src/token.rs +++ b/dust-lang/src/token.rs @@ -4,7 +4,7 @@ use std::fmt::{self, Display, Formatter}; use serde::{Deserialize, Serialize}; /// Source code token. -#[derive(Debug, PartialEq, Serialize, Deserialize)] +#[derive(Debug, Serialize, Deserialize)] pub enum Token<'src> { Eof, @@ -92,6 +92,17 @@ impl<'src> Display for Token<'src> { } } +impl<'src> PartialEq for Token<'src> { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (Token::Float(left), Token::Float(right)) => left.to_bits() == right.to_bits(), + _ => { + matches!(self, other) + } + } + } +} + /// Owned version of `Token`, which owns all the strings. /// /// This is used for errors. diff --git a/dust-lang/src/value.rs b/dust-lang/src/value.rs index cf53d50..0e3f955 100644 --- a/dust-lang/src/value.rs +++ b/dust-lang/src/value.rs @@ -161,6 +161,14 @@ impl Display for Value { match self.inner().as_ref() { ValueInner::Boolean(boolean) => write!(f, "{boolean}"), ValueInner::Float(float) => { + if float == &f64::INFINITY { + return write!(f, "Infinity"); + } + + if float == &f64::NEG_INFINITY { + return write!(f, "-Infinity"); + } + write!(f, "{float}")?; if &float.floor() == float { diff --git a/dust-lang/src/vm.rs b/dust-lang/src/vm.rs index 76e8be9..c3e167f 100644 --- a/dust-lang/src/vm.rs +++ b/dust-lang/src/vm.rs @@ -69,7 +69,12 @@ impl Vm { position: right_span, }); }; - let sum = left.add(&right)?; + let sum = left + .add(&right) + .map_err(|value_error| VmError::ValueError { + error: value_error, + position: (left_span.0, right_span.1), + })?; Ok(Some(sum)) } @@ -199,7 +204,12 @@ impl Vm { position: right_span, }); }; - let product = left.multiply(&right)?; + let product = left + .multiply(&right) + .map_err(|value_error| VmError::ValueError { + error: value_error, + position: (left_span.0, right_span.1), + })?; Ok(Some(product)) } @@ -276,7 +286,12 @@ impl Vm { position: right_span, }); }; - let difference = left.subtract(&right)?; + let difference = + left.subtract(&right) + .map_err(|value_error| VmError::ValueError { + error: value_error, + position: (left_span.0, right_span.1), + })?; Ok(Some(difference)) } @@ -288,7 +303,7 @@ impl Vm { pub enum VmError { AnaylyzerError(AnalyzerError), ParseError(ParseError), - ValueError(ValueError), + ValueError { error: ValueError, position: Span }, // Anaylsis Failures // These should be prevented by running the analyzer before the VM @@ -319,18 +334,12 @@ impl From for VmError { } } -impl From for VmError { - fn from(error: ValueError) -> Self { - Self::ValueError(error) - } -} - impl Error for VmError { fn source(&self) -> Option<&(dyn Error + 'static)> { match self { Self::AnaylyzerError(analyzer_error) => Some(analyzer_error), Self::ParseError(parse_error) => Some(parse_error), - Self::ValueError(value_error) => Some(value_error), + Self::ValueError { error, .. } => Some(error), Self::BuiltInFunctionCallError(built_in_function_error) => { Some(built_in_function_error) } @@ -344,7 +353,7 @@ impl Display for VmError { match self { Self::AnaylyzerError(analyzer_error) => write!(f, "{}", analyzer_error), Self::ParseError(parse_error) => write!(f, "{}", parse_error), - Self::ValueError(value_error) => write!(f, "{}", value_error), + Self::ValueError { error, .. } => write!(f, "{}", error), Self::BuiltInFunctionCallError(built_in_function_error) => { write!(f, "{}", built_in_function_error) }