jeff/dust
1
0
Fork 0
Code Issues

Clean up token.rs

Browse Source
This commit is contained in:
Jeff 2024-12-02 03:50:51 -05:00
parent 8e09d487c3
commit dc002023cd
1 changed files with 7 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
dust-lang/src/token.rs
View File

@ -28,25 +28,22 @@ pub fn write_token_list<W: Write>(tokens: &[(Token, Span)], styled: bool, writer
} }
macro_rules! define_tokens { macro_rules! define_tokens {
($($variant:ident $(($data_type:ty))?),+ $(,)?) => { ($($variant:ident $(($data_type:ty))?),+) => {
/// Source token. /// Source token.
/// ///
/// This is a borrowed type, i.e. some variants contain references to the source text. /// This is a borrowed type, i.e. some variants contain references to the source text.
#[derive(Debug, Clone, Copy, Eq, PartialEq, PartialOrd, Ord, Default, Serialize, Deserialize)] #[derive(Debug, Clone, Copy, Eq, PartialEq, PartialOrd, Ord, Serialize, Deserialize)]
pub enum Token<'src> { pub enum Token<'src> {
#[default]
Eof,
$( $(
$variant $(($data_type))?, $variant $(($data_type))?,
)* )*
} }
#[derive(Debug, PartialEq, Clone)] #[derive(Debug, Clone, Copy, Eq, PartialEq, PartialOrd, Ord, Serialize, Deserialize)]
/// Data-less representation of a source token. /// Data-less representation of a source token.
/// ///
/// If a [Token] borrows from the source text, its TokenKind omits the data. /// If a [Token] borrows from the source text, its TokenKind omits the data.
pub enum TokenKind { pub enum TokenKind {
Eof,
$( $(
$variant, $variant,
)* )*
@ -55,6 +52,8 @@ macro_rules! define_tokens {
} }
define_tokens! { define_tokens! {
Eof,
// Hard-coded values // Hard-coded values
Boolean(&'src str), Boolean(&'src str),
Byte(&'src str), Byte(&'src str),
@ -114,7 +113,7 @@ define_tokens! {
Slash, Slash,
SlashEqual, SlashEqual,
Star, Star,
StarEqual, StarEqual
} }
impl<'src> Token<'src> { impl<'src> Token<'src> {
@ -472,7 +471,7 @@ impl<'src> Display for Token<'src> {
/// Owned representation of a source token. /// Owned representation of a source token.
/// ///
/// If a [Token] borrows from the source text, its TokenOwned omits the data. /// If a [Token] borrows from the source text, its TokenOwned omits the data.
#[derive(Debug, PartialEq, Clone)] #[derive(Debug, Clone, Eq, PartialEq, PartialOrd, Ord, Serialize, Deserialize)]
pub enum TokenOwned { pub enum TokenOwned {
Eof, Eof,