...

src/compiler/CLI/Program.cs

@@ -80,7 +80,8 @@ foreach (var file in options.Files)
     var content = File.ReadAllText(file);
     var sourceText = new SourceText(file, content);
 
-    var tokenizeResult = Tokenizer.Tokenize(sourceText, out var tokenizerDiagnostics);
+    var tokenizer = new Tokenizer(sourceText);
+    var tokenizeResult = tokenizer.Tokenize(out var tokenizerDiagnostics);
     diagnostics.AddRange(tokenizerDiagnostics);
 
     var parser = new Parser(tokenizeResult);

src/compiler/NubLang/Diagnostics/ConsoleColors.cs

@@ -127,7 +127,8 @@ public static class ConsoleColors
     public static string ColorizeSource(string source)
     {
         var sourceText = new SourceText(string.Empty, source);
-        var tokens = Tokenizer.Tokenize(sourceText, out _);
+        var tokenizer = new Tokenizer(sourceText);
+        var tokens = tokenizer.Tokenize(out _);
         var result = new StringBuilder();
         var lastCharIndex = 0;
 

src/compiler/NubLang/Syntax/Tokenization/Tokenizer.cs

@@ -3,7 +3,7 @@ using NubLang.Diagnostics;
 
 namespace NubLang.Syntax.Tokenization;
 
-public static class Tokenizer
+public sealed class Tokenizer
 {
     private static readonly Dictionary<string, Symbol> Keywords = new()
     {
@@ -62,12 +62,16 @@ public static class Tokenizer
         ['&'] = Symbol.Ampersand,
     };
 
-    private static SourceText _sourceText;
-    private static int _index;
+    private readonly SourceText _sourceText;
+    private int _index;
 
-    public static IEnumerable<Token> Tokenize(SourceText sourceText, out IEnumerable<Diagnostic> diagnostics)
+    public Tokenizer(SourceText sourceText)
     {
         _sourceText = sourceText;
+    }
+
+    public IEnumerable<Token> Tokenize(out IEnumerable<Diagnostic> diagnostics)
+    {
         _index = 0;
 
         List<Token> tokens = [];
@@ -81,7 +85,7 @@ public static class Tokenizer
         return tokens;
     }
 
-    private static Optional<Token> ParseToken()
+    private Optional<Token> ParseToken()
     {
         var startIndex = _index;
 
@@ -226,7 +230,7 @@ public static class Tokenizer
         throw new Exception($"Unknown character {current}");
     }
 
-    private static SourceLocation CreateLocation(int index)
+    private SourceLocation CreateLocation(int index)
     {
         var line = 1;
         var column = 1;
@@ -246,12 +250,12 @@ public static class Tokenizer
         return new SourceLocation(line, column);
     }
 
-    private static SourceSpan CreateSpan(int startIndex)
+    private SourceSpan CreateSpan(int startIndex)
     {
         return new SourceSpan(_sourceText, CreateLocation(startIndex), CreateLocation(_index));
     }
 
-    private static Optional<char> Peek(int offset = 0)
+    private Optional<char> Peek(int offset = 0)
     {
         if (_index + offset < _sourceText.Content.Length)
         {
@@ -261,7 +265,7 @@ public static class Tokenizer
         return Optional<char>.Empty();
     }
 
-    private static Optional<char> Next()
+    private Optional<char> Next()
     {
         if (_index < _sourceText.Content.Length)
         {