jeff/dust
1
0
Fork 0
Code Issues

Fix lexing bug

Browse Source
This commit is contained in:
Jeff 2024-10-18 19:31:46 -04:00
parent 44659ec34a
commit 95e5b3062d
1 changed files with 7 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
dust-lang/src/lexer.rs
View File

@ -36,13 +36,17 @@ pub fn lex<'tokens, 'src: 'tokens>(
let mut lexer = Lexer::new(source); let mut lexer = Lexer::new(source);
let mut tokens = Vec::new(); let mut tokens = Vec::new();
while !lexer.is_eof() { loop {
let (token, span) = lexer let (token, span) = lexer
.next_token() .next_token()
.map_err(|error| DustError::Lex { error, source })?; .map_err(|error| DustError::Lex { error, source })?;
let length = tokens.len(); let is_eof = matches!(token, Token::Eof);
tokens[length] = (token, span); tokens.push((token, span));
if is_eof {
break;
}
} }
Ok(tokens) Ok(tokens)