jeff/dust
1
0
Fork 0
Code Issues

Fix lexing bug

Browse Source
This commit is contained in:
Jeff 2024-10-18 19:31:46 -04:00
parent 44659ec34a
commit 95e5b3062d
1 changed files with 7 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
dust-lang/src/lexer.rs
View File

@ -36,13 +36,17 @@ pub fn lex<'tokens, 'src: 'tokens>(
let mut lexer = Lexer::new(source);
let mut tokens = Vec::new();
while !lexer.is_eof() {
loop {
let (token, span) = lexer
.next_token()
.map_err(|error| DustError::Lex { error, source })?;
let length = tokens.len();
let is_eof = matches!(token, Token::Eof);
tokens[length] = (token, span);
tokens.push((token, span));
if is_eof {
break;
}
}
Ok(tokens)