Fix token display and update README.md

README.md

@@ -27,7 +27,7 @@ Dust is still in development. This list may change as the language evolves.
 - CLI
   - [X] Run source
   - [X] Compile to chunk and show disassembly
-  - [ ] Tokenize using the lexer and show token list
+  - [X] Tokenize using the lexer and show token list
   - [ ] Format using the formatter and display the output
   - [ ] Compile to and run from intermediate formats
     - [ ] JSON

dust-cli/src/main.rs

@@ -42,7 +42,7 @@ enum CliMode {
         global_arguments: GlobalArguments,
 
         /// Style the disassembly output
-        #[arg(short, long)]
+        #[arg(short, long, default_value = "true")]
         style: bool,
     },
 
@@ -53,7 +53,7 @@ enum CliMode {
         global_arguments: GlobalArguments,
 
         /// Style the disassembly output
-        #[arg(short, long)]
+        #[arg(short, long, default_value = "true")]
         style: bool,
     },
 }

dust-lang/src/token.rs

@@ -9,20 +9,26 @@ use serde::{Deserialize, Serialize};
 
 use crate::Span;
 
-pub fn write_token_list<W: Write>(tokens: &[(Token, Span)], styled: bool, writer: &mut W) {
+pub fn write_token_list<W: Write>(tokens: &[(Token, Span)], style: bool, writer: &mut W) {
     const HEADER: [&str; 2] = ["    TOKEN      POSITION ", "------------- ----------"];
 
-    writeln!(writer, "{}", HEADER[0]).unwrap();
-    writeln!(writer, "{}", HEADER[1]).unwrap();
+    if style {
+        writeln!(writer, "{}", HEADER[0].bold()).unwrap();
+        writeln!(writer, "{}", HEADER[1].bold()).unwrap();
+    } else {
+        writeln!(writer, "{}", HEADER[0]).unwrap();
+        writeln!(writer, "{}", HEADER[1]).unwrap();
+    }
 
     for (token, position) in tokens {
-        let token = if styled {
+        let token = if style {
             format!("{:^13}", token.to_string().bold())
         } else {
             format!("{token:^13}")
         };
+        let position = position.to_string();
 
-        writeln!(writer, "{token} {position:<10}").unwrap();
+        writeln!(writer, "{token:^13} {position:^10}").unwrap();
     }
 }