337 lines
11 KiB
Rust
337 lines
11 KiB
Rust
//! Token and TokenOwned types.
|
|
use std::fmt::{self, Display, Formatter};
|
|
|
|
use serde::{Deserialize, Serialize};
|
|
|
|
/// Source code token.
|
|
#[derive(Debug, Serialize, Deserialize)]
|
|
pub enum Token<'src> {
|
|
// End of file
|
|
Eof,
|
|
|
|
// Hard-coded values
|
|
Boolean(&'src str),
|
|
Float(&'src str),
|
|
Identifier(&'src str),
|
|
Integer(&'src str),
|
|
String(&'src str),
|
|
|
|
// Keywords
|
|
Else,
|
|
If,
|
|
IsEven,
|
|
IsOdd,
|
|
Length,
|
|
ReadLine,
|
|
ToString,
|
|
While,
|
|
WriteLine,
|
|
|
|
// Symbols
|
|
Bang,
|
|
Comma,
|
|
Dot,
|
|
DoubleAmpersand,
|
|
DoubleEqual,
|
|
DoublePipe,
|
|
Equal,
|
|
Greater,
|
|
GreaterEqual,
|
|
LeftCurlyBrace,
|
|
LeftParenthesis,
|
|
LeftSquareBrace,
|
|
Less,
|
|
LessEqual,
|
|
Minus,
|
|
Percent,
|
|
Plus,
|
|
PlusEqual,
|
|
RightCurlyBrace,
|
|
RightParenthesis,
|
|
RightSquareBrace,
|
|
Semicolon,
|
|
Slash,
|
|
Star,
|
|
}
|
|
|
|
impl<'src> Token<'src> {
|
|
pub fn to_owned(&self) -> TokenOwned {
|
|
match self {
|
|
Token::Bang => TokenOwned::Bang,
|
|
Token::Boolean(boolean) => TokenOwned::Boolean(boolean.to_string()),
|
|
Token::Comma => TokenOwned::Comma,
|
|
Token::Dot => TokenOwned::Dot,
|
|
Token::DoubleAmpersand => TokenOwned::DoubleAmpersand,
|
|
Token::DoubleEqual => TokenOwned::DoubleEqual,
|
|
Token::DoublePipe => TokenOwned::DoublePipe,
|
|
Token::Else => TokenOwned::Else,
|
|
Token::Eof => TokenOwned::Eof,
|
|
Token::Equal => TokenOwned::Equal,
|
|
Token::Float(float) => TokenOwned::Float(float.to_string()),
|
|
Token::Greater => TokenOwned::Greater,
|
|
Token::GreaterEqual => TokenOwned::GreaterOrEqual,
|
|
Token::Identifier(text) => TokenOwned::Identifier(text.to_string()),
|
|
Token::If => TokenOwned::If,
|
|
Token::Integer(integer) => TokenOwned::Integer(integer.to_string()),
|
|
Token::IsEven => TokenOwned::IsEven,
|
|
Token::IsOdd => TokenOwned::IsOdd,
|
|
Token::LeftCurlyBrace => TokenOwned::LeftCurlyBrace,
|
|
Token::LeftParenthesis => TokenOwned::LeftParenthesis,
|
|
Token::LeftSquareBrace => TokenOwned::LeftSquareBrace,
|
|
Token::Length => TokenOwned::Length,
|
|
Token::Less => TokenOwned::Less,
|
|
Token::LessEqual => TokenOwned::LessOrEqual,
|
|
Token::Minus => TokenOwned::Minus,
|
|
Token::Percent => TokenOwned::Percent,
|
|
Token::Plus => TokenOwned::Plus,
|
|
Token::PlusEqual => TokenOwned::PlusEqual,
|
|
Token::ReadLine => TokenOwned::ReadLine,
|
|
Token::RightCurlyBrace => TokenOwned::RightCurlyBrace,
|
|
Token::RightParenthesis => TokenOwned::RightParenthesis,
|
|
Token::RightSquareBrace => TokenOwned::RightSquareBrace,
|
|
Token::Semicolon => TokenOwned::Semicolon,
|
|
Token::Star => TokenOwned::Star,
|
|
Token::Slash => TokenOwned::Slash,
|
|
Token::String(text) => TokenOwned::String(text.to_string()),
|
|
Token::ToString => TokenOwned::ToString,
|
|
Token::While => TokenOwned::While,
|
|
Token::WriteLine => TokenOwned::WriteLine,
|
|
}
|
|
}
|
|
|
|
pub fn as_str(&self) -> &str {
|
|
match self {
|
|
Token::Boolean(boolean_text) => boolean_text,
|
|
Token::Identifier(text) => text,
|
|
Token::Integer(integer_text) => integer_text,
|
|
Token::String(text) => text,
|
|
|
|
Token::Bang => "!",
|
|
Token::Comma => ",",
|
|
Token::Dot => ".",
|
|
Token::DoubleAmpersand => "&&",
|
|
Token::DoubleEqual => "==",
|
|
Token::DoublePipe => "||",
|
|
Token::Else => "else",
|
|
Token::Eof => "EOF",
|
|
Token::Equal => "=",
|
|
Token::Float(_) => "float",
|
|
Token::Greater => ">",
|
|
Token::GreaterEqual => ">=",
|
|
Token::If => "if",
|
|
Token::IsEven => "is_even",
|
|
Token::IsOdd => "is_odd",
|
|
Token::LeftCurlyBrace => "{",
|
|
Token::LeftParenthesis => "(",
|
|
Token::LeftSquareBrace => "[",
|
|
Token::Length => "length",
|
|
Token::Less => "<",
|
|
Token::LessEqual => "<=",
|
|
Token::Minus => "-",
|
|
Token::Percent => "%",
|
|
Token::Plus => "+",
|
|
Token::PlusEqual => "+=",
|
|
Token::ReadLine => "read_line",
|
|
Token::RightCurlyBrace => "}",
|
|
Token::RightParenthesis => ")",
|
|
Token::RightSquareBrace => "]",
|
|
Token::Semicolon => ";",
|
|
Token::Star => "*",
|
|
Token::Slash => "/",
|
|
Token::ToString => "to_string",
|
|
Token::While => "while",
|
|
Token::WriteLine => "write_line",
|
|
}
|
|
}
|
|
|
|
pub fn is_eof(&self) -> bool {
|
|
matches!(self, Token::Eof)
|
|
}
|
|
|
|
pub fn precedence(&self) -> u8 {
|
|
match self {
|
|
Token::Dot => 9,
|
|
Token::Star | Token::Slash | Token::Percent => 8,
|
|
Token::Minus => 7,
|
|
Token::Plus => 6,
|
|
Token::DoubleEqual
|
|
| Token::Less
|
|
| Token::LessEqual
|
|
| Token::Greater
|
|
| Token::GreaterEqual => 5,
|
|
Token::DoubleAmpersand => 4,
|
|
Token::DoublePipe => 3,
|
|
Token::Equal | Token::PlusEqual => 2,
|
|
Token::Semicolon => 1,
|
|
_ => 0,
|
|
}
|
|
}
|
|
|
|
pub fn is_left_associative(&self) -> bool {
|
|
!self.is_right_associative()
|
|
}
|
|
|
|
pub fn is_right_associative(&self) -> bool {
|
|
matches!(self, Token::Equal | Token::PlusEqual)
|
|
}
|
|
|
|
pub fn is_prefix(&self) -> bool {
|
|
matches!(self, Token::Bang | Token::Minus)
|
|
}
|
|
|
|
pub fn is_postfix(&self) -> bool {
|
|
matches!(self, Token::Semicolon)
|
|
}
|
|
}
|
|
|
|
impl<'src> Display for Token<'src> {
|
|
fn fmt(&self, f: &mut Formatter) -> fmt::Result {
|
|
write!(f, "{}", self.as_str())
|
|
}
|
|
}
|
|
|
|
impl<'src> PartialEq for Token<'src> {
|
|
fn eq(&self, other: &Self) -> bool {
|
|
match (self, other) {
|
|
(Token::Bang, Token::Bang) => true,
|
|
(Token::Boolean(left), Token::Boolean(right)) => left == right,
|
|
(Token::Comma, Token::Comma) => true,
|
|
(Token::Dot, Token::Dot) => true,
|
|
(Token::DoubleAmpersand, Token::DoubleAmpersand) => true,
|
|
(Token::DoubleEqual, Token::DoubleEqual) => true,
|
|
(Token::DoublePipe, Token::DoublePipe) => true,
|
|
(Token::Else, Token::Else) => true,
|
|
(Token::Eof, Token::Eof) => true,
|
|
(Token::Equal, Token::Equal) => true,
|
|
(Token::Float(left), Token::Float(right)) => left == right,
|
|
(Token::Greater, Token::Greater) => true,
|
|
(Token::GreaterEqual, Token::GreaterEqual) => true,
|
|
(Token::Identifier(left), Token::Identifier(right)) => left == right,
|
|
(Token::If, Token::If) => true,
|
|
(Token::Integer(left), Token::Integer(right)) => left == right,
|
|
(Token::IsEven, Token::IsEven) => true,
|
|
(Token::IsOdd, Token::IsOdd) => true,
|
|
(Token::LeftCurlyBrace, Token::LeftCurlyBrace) => true,
|
|
(Token::LeftParenthesis, Token::LeftParenthesis) => true,
|
|
(Token::LeftSquareBrace, Token::LeftSquareBrace) => true,
|
|
(Token::Length, Token::Length) => true,
|
|
(Token::Less, Token::Less) => true,
|
|
(Token::LessEqual, Token::LessEqual) => true,
|
|
(Token::Minus, Token::Minus) => true,
|
|
(Token::Percent, Token::Percent) => true,
|
|
(Token::Plus, Token::Plus) => true,
|
|
(Token::PlusEqual, Token::PlusEqual) => true,
|
|
(Token::ReadLine, Token::ReadLine) => true,
|
|
(Token::RightCurlyBrace, Token::RightCurlyBrace) => true,
|
|
(Token::RightParenthesis, Token::RightParenthesis) => true,
|
|
(Token::RightSquareBrace, Token::RightSquareBrace) => true,
|
|
(Token::Semicolon, Token::Semicolon) => true,
|
|
(Token::Star, Token::Star) => true,
|
|
(Token::Slash, Token::Slash) => true,
|
|
(Token::String(left), Token::String(right)) => left == right,
|
|
(Token::While, Token::While) => true,
|
|
(Token::WriteLine, Token::WriteLine) => true,
|
|
_ => false,
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Owned version of `Token`, which owns all the strings.
|
|
///
|
|
/// This is used for errors.
|
|
#[derive(Debug, PartialEq, Clone, Serialize, Deserialize)]
|
|
pub enum TokenOwned {
|
|
Eof,
|
|
|
|
Identifier(String),
|
|
|
|
// Hard-coded values
|
|
Boolean(String),
|
|
Float(String),
|
|
Integer(String),
|
|
String(String),
|
|
|
|
// Keywords
|
|
Else,
|
|
If,
|
|
IsEven,
|
|
IsOdd,
|
|
Length,
|
|
ReadLine,
|
|
ToString,
|
|
While,
|
|
WriteLine,
|
|
|
|
// Symbols
|
|
Bang,
|
|
Comma,
|
|
Dot,
|
|
DoubleAmpersand,
|
|
DoubleEqual,
|
|
DoublePipe,
|
|
Equal,
|
|
Greater,
|
|
GreaterOrEqual,
|
|
LeftCurlyBrace,
|
|
LeftParenthesis,
|
|
LeftSquareBrace,
|
|
Less,
|
|
LessOrEqual,
|
|
Minus,
|
|
Percent,
|
|
Plus,
|
|
PlusEqual,
|
|
RightCurlyBrace,
|
|
RightParenthesis,
|
|
RightSquareBrace,
|
|
Semicolon,
|
|
Star,
|
|
Slash,
|
|
}
|
|
|
|
impl Display for TokenOwned {
|
|
fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
|
|
match self {
|
|
TokenOwned::Bang => Token::Bang.fmt(f),
|
|
TokenOwned::Boolean(boolean) => write!(f, "{boolean}"),
|
|
TokenOwned::Comma => Token::Comma.fmt(f),
|
|
TokenOwned::Dot => Token::Dot.fmt(f),
|
|
TokenOwned::DoubleAmpersand => Token::DoubleAmpersand.fmt(f),
|
|
TokenOwned::DoubleEqual => Token::DoubleEqual.fmt(f),
|
|
TokenOwned::DoublePipe => Token::DoublePipe.fmt(f),
|
|
TokenOwned::Else => Token::Else.fmt(f),
|
|
TokenOwned::Eof => Token::Eof.fmt(f),
|
|
TokenOwned::Equal => Token::Equal.fmt(f),
|
|
TokenOwned::Float(float) => write!(f, "{float}"),
|
|
TokenOwned::Greater => Token::Greater.fmt(f),
|
|
TokenOwned::GreaterOrEqual => Token::GreaterEqual.fmt(f),
|
|
TokenOwned::Identifier(text) => write!(f, "{text}"),
|
|
TokenOwned::If => Token::If.fmt(f),
|
|
TokenOwned::Integer(integer) => write!(f, "{integer}"),
|
|
TokenOwned::IsEven => Token::IsEven.fmt(f),
|
|
TokenOwned::IsOdd => Token::IsOdd.fmt(f),
|
|
TokenOwned::LeftCurlyBrace => Token::LeftCurlyBrace.fmt(f),
|
|
TokenOwned::LeftParenthesis => Token::LeftParenthesis.fmt(f),
|
|
TokenOwned::LeftSquareBrace => Token::LeftSquareBrace.fmt(f),
|
|
TokenOwned::Length => Token::Length.fmt(f),
|
|
TokenOwned::Less => Token::Less.fmt(f),
|
|
TokenOwned::LessOrEqual => Token::LessEqual.fmt(f),
|
|
TokenOwned::Minus => Token::Minus.fmt(f),
|
|
TokenOwned::Percent => Token::Percent.fmt(f),
|
|
TokenOwned::Plus => Token::Plus.fmt(f),
|
|
TokenOwned::PlusEqual => Token::PlusEqual.fmt(f),
|
|
TokenOwned::ReadLine => Token::ReadLine.fmt(f),
|
|
TokenOwned::RightCurlyBrace => Token::RightCurlyBrace.fmt(f),
|
|
TokenOwned::RightParenthesis => Token::RightParenthesis.fmt(f),
|
|
TokenOwned::RightSquareBrace => Token::RightSquareBrace.fmt(f),
|
|
TokenOwned::Semicolon => Token::Semicolon.fmt(f),
|
|
TokenOwned::Star => Token::Star.fmt(f),
|
|
TokenOwned::Slash => Token::Slash.fmt(f),
|
|
TokenOwned::String(string) => write!(f, "{string}"),
|
|
TokenOwned::ToString => Token::ToString.fmt(f),
|
|
TokenOwned::While => Token::While.fmt(f),
|
|
TokenOwned::WriteLine => Token::WriteLine.fmt(f),
|
|
}
|
|
}
|
|
}
|