diff --git a/src/compiler/CLI/Program.cs b/src/compiler/CLI/Program.cs index d9fba1b..75c354c 100644 --- a/src/compiler/CLI/Program.cs +++ b/src/compiler/CLI/Program.cs @@ -80,7 +80,8 @@ foreach (var file in options.Files) var content = File.ReadAllText(file); var sourceText = new SourceText(file, content); - var tokenizeResult = Tokenizer.Tokenize(sourceText, out var tokenizerDiagnostics); + var tokenizer = new Tokenizer(sourceText); + var tokenizeResult = tokenizer.Tokenize(out var tokenizerDiagnostics); diagnostics.AddRange(tokenizerDiagnostics); var parser = new Parser(tokenizeResult); diff --git a/src/compiler/NubLang/Diagnostics/ConsoleColors.cs b/src/compiler/NubLang/Diagnostics/ConsoleColors.cs index cdfcaf1..3c9f563 100644 --- a/src/compiler/NubLang/Diagnostics/ConsoleColors.cs +++ b/src/compiler/NubLang/Diagnostics/ConsoleColors.cs @@ -15,7 +15,7 @@ public static class ConsoleColors public const string SwapBgAndFg = "\e[7m"; public const string Conceal = "\e[8m"; public const string CrossedOut = "\e[9m"; - + public const string DefaultFont = "\e[10m"; public const string AltFont1 = "\e[11m"; public const string AltFont2 = "\e[12m"; @@ -127,7 +127,8 @@ public static class ConsoleColors public static string ColorizeSource(string source) { var sourceText = new SourceText(string.Empty, source); - var tokens = Tokenizer.Tokenize(sourceText, out _); + var tokenizer = new Tokenizer(sourceText); + var tokens = tokenizer.Tokenize(out _); var result = new StringBuilder(); var lastCharIndex = 0; diff --git a/src/compiler/NubLang/Syntax/Tokenization/Tokenizer.cs b/src/compiler/NubLang/Syntax/Tokenization/Tokenizer.cs index 97d8ecc..49069b6 100644 --- a/src/compiler/NubLang/Syntax/Tokenization/Tokenizer.cs +++ b/src/compiler/NubLang/Syntax/Tokenization/Tokenizer.cs @@ -3,7 +3,7 @@ using NubLang.Diagnostics; namespace NubLang.Syntax.Tokenization; -public static class Tokenizer +public sealed class Tokenizer { private static readonly Dictionary Keywords = new() { @@ -62,12 +62,16 @@ public static class Tokenizer ['&'] = Symbol.Ampersand, }; - private static SourceText _sourceText; - private static int _index; + private readonly SourceText _sourceText; + private int _index; - public static IEnumerable Tokenize(SourceText sourceText, out IEnumerable diagnostics) + public Tokenizer(SourceText sourceText) { _sourceText = sourceText; + } + + public IEnumerable Tokenize(out IEnumerable diagnostics) + { _index = 0; List tokens = []; @@ -81,7 +85,7 @@ public static class Tokenizer return tokens; } - private static Optional ParseToken() + private Optional ParseToken() { var startIndex = _index; @@ -226,7 +230,7 @@ public static class Tokenizer throw new Exception($"Unknown character {current}"); } - private static SourceLocation CreateLocation(int index) + private SourceLocation CreateLocation(int index) { var line = 1; var column = 1; @@ -246,12 +250,12 @@ public static class Tokenizer return new SourceLocation(line, column); } - private static SourceSpan CreateSpan(int startIndex) + private SourceSpan CreateSpan(int startIndex) { return new SourceSpan(_sourceText, CreateLocation(startIndex), CreateLocation(_index)); } - private static Optional Peek(int offset = 0) + private Optional Peek(int offset = 0) { if (_index + offset < _sourceText.Content.Length) { @@ -261,7 +265,7 @@ public static class Tokenizer return Optional.Empty(); } - private static Optional Next() + private Optional Next() { if (_index < _sourceText.Content.Length) {