diangostics

compiler/Compiler/Tokenizer.cs

@@ -3,33 +3,40 @@ using System.Text;
 
 namespace Compiler;
 
-public sealed class Tokenizer(string contents)
+public sealed class Tokenizer(string fileName, string contents)
 {
-    public static List<Token> Tokenize(string contents)
+    public static List<Token> Tokenize(string fileName, string contents, out List<Diagnostic> diagnostics)
     {
-        return new Tokenizer(contents).Tokenize();
+        return new Tokenizer(fileName, contents).Tokenize(out diagnostics);
     }
 
     private int index;
     private int line = 1;
     private int column = 1;
 
-    private List<Token> Tokenize()
+    private List<Token> Tokenize(out List<Diagnostic> diagnostics)
     {
         var tokens = new List<Token>();
-
-        while (true)
+        diagnostics = [];
+        try
         {
-            if (!TryPeek(out var c))
-                break;
-
-            if (char.IsWhiteSpace(c))
+            while (true)
             {
-                Consume();
-                continue;
-            }
+                if (!TryPeek(out var c))
+                    break;
 
-            tokens.Add(ParseToken());
+                if (char.IsWhiteSpace(c))
+                {
+                    Consume();
+                    continue;
+                }
+
+                tokens.Add(ParseToken());
+            }
+        }
+        catch (CompileException e)
+        {
+            diagnostics.Add(e.Diagnostic);
         }
 
         return tokens;
@@ -347,7 +354,7 @@ public sealed class Tokenizer(string contents)
     private char Consume()
     {
         if (index >= contents.Length)
-            throw new Exception("End of tokens");
+            throw new CompileException(Diagnostic.Error("Unexpected end of file").At(fileName, line, column, 0).Build());
 
         var c = contents[index];
 
@@ -469,4 +476,62 @@ public enum Keyword
 public sealed class TokenKeyword(int line, int column, int length, Keyword keyword) : Token(line, column, length)
 {
     public readonly Keyword Keyword = keyword;
+}
+
+public static class TokenExtensions
+{
+    public static string AsString(this Symbol symbol)
+    {
+        return symbol switch
+        {
+            Symbol.OpenCurly => "{",
+            Symbol.CloseCurly => "}",
+            Symbol.OpenParen => "(",
+            Symbol.CloseParen => ")",
+            Symbol.Comma => ",",
+            Symbol.Period => ",",
+            Symbol.Colon => ":",
+            Symbol.Caret => "^",
+            Symbol.Bang => "!",
+            Symbol.Equal => "=",
+            Symbol.EqualEqual => "==",
+            Symbol.BangEqual => "!+",
+            Symbol.LessThan => "<",
+            Symbol.LessThanLessThan => "<<",
+            Symbol.LessThanEqual => "<=",
+            Symbol.GreaterThan => ">",
+            Symbol.GreaterThanGreaterThan => ">>",
+            Symbol.GreaterThanEqual => ">=",
+            Symbol.Plus => "+",
+            Symbol.PlusEqual => "+=",
+            Symbol.Minus => "-",
+            Symbol.MinusEqual => "-=",
+            Symbol.Star => "*",
+            Symbol.StarEqual => "*=",
+            Symbol.ForwardSlash => "/",
+            Symbol.ForwardSlashEqual => "/=",
+            Symbol.Percent => "%",
+            Symbol.PercentEqual => "%=",
+            Symbol.Ampersand => "&",
+            Symbol.AmpersandAmpersand => "&&",
+            Symbol.Pipe => "|",
+            Symbol.PipePipe => "||",
+            _ => throw new ArgumentOutOfRangeException(nameof(symbol), symbol, null)
+        };
+    }
+
+    public static string AsString(this Keyword symbol)
+    {
+        return symbol switch
+        {
+            Keyword.Func => "func",
+            Keyword.Struct => "struct",
+            Keyword.Let => "let",
+            Keyword.If => "if",
+            Keyword.Else => "else",
+            Keyword.While => "while",
+            Keyword.Return => "return",
+            _ => throw new ArgumentOutOfRangeException(nameof(symbol), symbol, null)
+        };
+    }
 }