jeff/dust
1
0
Fork 0
Code Issues

Expand lexing of complex floats

Browse Source
This commit is contained in:
Jeff 2024-08-09 00:31:38 -04:00
parent 60bd8f5352
commit 9766777a47
5 changed files with 127 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
dust-lang/src/dust_error.rs
View File

@ -21,7 +21,7 @@ impl<'src> DustError<'src> {
let title = match &self.vm_error {
VmError::AnaylyzerError(_) => "Analyzer error",
VmError::ParseError(_) => "Parse error",
VmError::ValueError(_) => "Value error",
VmError::ValueError { .. } => "Value error",
VmError::BuiltInFunctionCallError(_) => "Runtime error",
_ => "Analysis Failure",
};
@ -36,7 +36,7 @@ impl<'src> DustError<'src> {
AnalyzerError::UnexpectedIdentifier { position, .. } => position,
},
VmError::ParseError(_) => todo!(),
VmError::ValueError(_) => todo!(),
VmError::ValueError { position, .. } => position,
VmError::BuiltInFunctionCallError(_) => todo!(),
VmError::ExpectedIdentifier { position } => position,
VmError::ExpectedIdentifierOrInteger { position } => position,

87
dust-lang/src/lex.rs
View File

@ -106,6 +106,12 @@ impl Lexer {
'-' => {
if let Some('0'..='9') = self.peek_second_char(source) {
self.lex_number(source)?
} else if "-Infinity" == self.peek_chars(source, 9) {
self.position += 9;
(
Token::Float(f64::NEG_INFINITY),
(self.position - 9, self.position),
)
} else {
self.position += 1;
(Token::Minus, (self.position - 1, self.position))
@ -150,7 +156,7 @@ impl Lexer {
self.position += 1;
(Token::Dot, (self.position - 1, self.position))
}
_ => (Token::Eof, (self.position, self.position)),
_ => return Err(LexError::UnexpectedCharacter(c)),
}
} else {
(Token::Eof, (self.position, self.position))
@ -188,6 +194,17 @@ impl Lexer {
source[self.position..].chars().nth(1)
}
/// Peek the next `n` characters without consuming them.
fn peek_chars<'src>(&self, source: &'src str, n: usize) -> &'src str {
let remaining_source = &source[self.position..];
if remaining_source.len() < n {
remaining_source
} else {
&remaining_source[..n]
}
}
/// Lex an integer or float token.
fn lex_number<'src>(&mut self, source: &'src str) -> Result<(Token<'src>, Span), LexError> {
let start_pos = self.position;
@ -206,8 +223,21 @@ impl Lexer {
self.next_char(source);
while let Some('0'..='9') = self.peek_char(source) {
loop {
let peek_char = self.peek_char(source);
if let Some('0'..='9') = peek_char {
self.next_char(source);
} else if let Some('e') = peek_char {
if let Some('0'..='9') = self.peek_second_char(source) {
self.next_char(source);
self.next_char(source);
} else {
break;
}
} else {
break;
}
}
is_float = true;
@ -242,7 +272,7 @@ impl Lexer {
let start_pos = self.position;
while let Some(c) = self.peek_char(source) {
if c.is_ascii_alphanumeric() || c == '_' {
if c.is_ascii_alphabetic() || c == '_' {
self.next_char(source);
} else {
break;
@ -253,9 +283,11 @@ impl Lexer {
let token = match string {
"true" => Token::Boolean(true),
"false" => Token::Boolean(false),
"Infinity" => Token::Float(f64::INFINITY),
"is_even" => Token::IsEven,
"is_odd" => Token::IsOdd,
"length" => Token::Length,
"NaN" => Token::Float(f64::NAN),
"read_line" => Token::ReadLine,
"write_line" => Token::WriteLine,
_ => Token::Identifier(string),
@ -298,6 +330,7 @@ impl Default for Lexer {
pub enum LexError {
FloatError(ParseFloatError),
IntegerError(ParseIntError),
UnexpectedCharacter(char),
}
impl Error for LexError {
@ -305,6 +338,7 @@ impl Error for LexError {
match self {
Self::FloatError(parse_float_error) => Some(parse_float_error),
Self::IntegerError(parse_int_error) => Some(parse_int_error),
Self::UnexpectedCharacter(_) => None,
}
}
}
@ -318,6 +352,9 @@ impl Display for LexError {
Self::IntegerError(parse_int_error) => {
write!(f, "Failed to parse integer: {}", parse_int_error)
}
Self::UnexpectedCharacter(character) => {
write!(f, "Unexpected character: '{}'", character)
}
}
}
}
@ -336,9 +373,51 @@ impl From<ParseIntError> for LexError {
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn infinity() {
let input = "Infinity";
assert_eq!(
lex(input),
Ok(vec![
(Token::Float(f64::INFINITY), (0, 8)),
(Token::Eof, (8, 8)),
])
)
}
#[test]
fn negative_infinity() {
let input = "-Infinity";
assert_eq!(
lex(input),
Ok(vec![
(Token::Float(f64::NEG_INFINITY), (0, 9)),
(Token::Eof, (9, 9)),
])
)
}
#[test]
fn nan() {
let input = "NaN";
assert!(lex(input).is_ok_and(|tokens| tokens[0].0 == Token::Float(f64::NAN)));
}
#[test]
fn complex_float() {
let input = "42.42e42";
assert_eq!(
lex(input),
Ok(vec![(Token::Float(42.42e42), (0, 8)), (Token::Eof, (8, 8)),])
)
}
#[test]
fn max_integer() {
let input = "9223372036854775807";

13
dust-lang/src/token.rs
View File

@ -4,7 +4,7 @@ use std::fmt::{self, Display, Formatter};
use serde::{Deserialize, Serialize};
/// Source code token.
#[derive(Debug, PartialEq, Serialize, Deserialize)]
#[derive(Debug, Serialize, Deserialize)]
pub enum Token<'src> {
Eof,
@ -92,6 +92,17 @@ impl<'src> Display for Token<'src> {
}
}
impl<'src> PartialEq for Token<'src> {
fn eq(&self, other: &Self) -> bool {
match (self, other) {
(Token::Float(left), Token::Float(right)) => left.to_bits() == right.to_bits(),
_ => {
matches!(self, other)
}
}
}
}
/// Owned version of `Token`, which owns all the strings.
///
/// This is used for errors.

8
dust-lang/src/value.rs
View File

@ -161,6 +161,14 @@ impl Display for Value {
match self.inner().as_ref() {
ValueInner::Boolean(boolean) => write!(f, "{boolean}"),
ValueInner::Float(float) => {
if float == &f64::INFINITY {
return write!(f, "Infinity");
}
if float == &f64::NEG_INFINITY {
return write!(f, "-Infinity");
}
write!(f, "{float}")?;
if &float.floor() == float {

33
dust-lang/src/vm.rs
View File

@ -69,7 +69,12 @@ impl Vm {
position: right_span,
});
};
let sum = left.add(&right)?;
let sum = left
.add(&right)
.map_err(|value_error| VmError::ValueError {
error: value_error,
position: (left_span.0, right_span.1),
})?;
Ok(Some(sum))
}
@ -199,7 +204,12 @@ impl Vm {
position: right_span,
});
};
let product = left.multiply(&right)?;
let product = left
.multiply(&right)
.map_err(|value_error| VmError::ValueError {
error: value_error,
position: (left_span.0, right_span.1),
})?;
Ok(Some(product))
}
@ -276,7 +286,12 @@ impl Vm {
position: right_span,
});
};
let difference = left.subtract(&right)?;
let difference =
left.subtract(&right)
.map_err(|value_error| VmError::ValueError {
error: value_error,
position: (left_span.0, right_span.1),
})?;
Ok(Some(difference))
}
@ -288,7 +303,7 @@ impl Vm {
pub enum VmError {
AnaylyzerError(AnalyzerError),
ParseError(ParseError),
ValueError(ValueError),
ValueError { error: ValueError, position: Span },
// Anaylsis Failures
// These should be prevented by running the analyzer before the VM
@ -319,18 +334,12 @@ impl From<ParseError> for VmError {
}
}
impl From<ValueError> for VmError {
fn from(error: ValueError) -> Self {
Self::ValueError(error)
}
}
impl Error for VmError {
fn source(&self) -> Option<&(dyn Error + 'static)> {
match self {
Self::AnaylyzerError(analyzer_error) => Some(analyzer_error),
Self::ParseError(parse_error) => Some(parse_error),
Self::ValueError(value_error) => Some(value_error),
Self::ValueError { error, .. } => Some(error),
Self::BuiltInFunctionCallError(built_in_function_error) => {
Some(built_in_function_error)
}
@ -344,7 +353,7 @@ impl Display for VmError {
match self {
Self::AnaylyzerError(analyzer_error) => write!(f, "{}", analyzer_error),
Self::ParseError(parse_error) => write!(f, "{}", parse_error),
Self::ValueError(value_error) => write!(f, "{}", value_error),
Self::ValueError { error, .. } => write!(f, "{}", error),
Self::BuiltInFunctionCallError(built_in_function_error) => {
write!(f, "{}", built_in_function_error)
}