Syntax highlighting

src/compiler/NubLang/Diagnostics/Diagnostic.cs

@@ -93,6 +93,11 @@ public class Diagnostic
             _ => ConsoleColors.Colorize("unknown", ConsoleColors.Bold + ConsoleColors.White)
         });
 
+        if (FileSpan != null)
+        {
+            sb.Append(ConsoleColors.Colorize($" at {FileSpan.SourceFile.Path}:{FileSpan.Span}", ConsoleColors.Faint));
+        }
+
         sb.Append(": ");
         sb.Append(ConsoleColors.Colorize(Message, ConsoleColors.BrightWhite));
 
@@ -121,6 +126,9 @@ public class Diagnostic
             sb.Append('╮');
             sb.AppendLine();
 
+            var tokenizer = new Tokenizer(FileSpan.SourceFile);
+            var tokens = tokenizer.Tokenize().ToList();
+
             for (var i = contextStartLine; i <= contextEndLine; i++)
             {
                 var line = lines[i - 1];
@@ -128,7 +136,7 @@ public class Diagnostic
                 sb.Append("│ ");
                 sb.Append(i.ToString().PadRight(numberPadding));
                 sb.Append(" │ ");
-                sb.Append(line.PadRight(codePadding));
+                sb.Append(ApplySyntaxHighlighting(line.PadRight(codePadding), i, tokens));
                 sb.Append(" │");
                 sb.AppendLine();
 
@@ -184,6 +192,131 @@ public class Diagnostic
 
         return sb.ToString();
     }
+
+    private static string ApplySyntaxHighlighting(string line, int lineNumber, List<Token> tokens)
+    {
+        var sb = new StringBuilder();
+        var lineTokens = tokens
+            .Where(t => t.FileSpan.Span.Start.Line == lineNumber)
+            .OrderBy(t => t.FileSpan.Span.Start.Column)
+            .ToList();
+
+        if (lineTokens.Count == 0)
+        {
+            return line;
+        }
+
+        var currentColumn = 1;
+
+        foreach (var token in lineTokens)
+        {
+            var tokenStart = token.FileSpan.Span.Start.Column;
+            var tokenEnd = token.FileSpan.Span.End.Column;
+
+            if (tokenStart > currentColumn)
+            {
+                var beforeToken = line.Substring(currentColumn - 1, tokenStart - currentColumn);
+                sb.Append(beforeToken);
+            }
+
+            var tokenLength = tokenEnd - tokenStart;
+            if (tokenStart - 1 + tokenLength <= line.Length)
+            {
+                var tokenText = line.Substring(tokenStart - 1, tokenLength);
+
+                var coloredToken = ColorizeToken(token, tokenText);
+                sb.Append(coloredToken);
+            }
+
+            currentColumn = tokenEnd;
+        }
+
+        if (currentColumn <= line.Length)
+        {
+            var remaining = line[(currentColumn - 1)..];
+            sb.Append(remaining);
+        }
+
+        return sb.ToString();
+    }
+
+    private static string ColorizeToken(Token token, string tokenText)
+    {
+        switch (token)
+        {
+            case IdentifierToken:
+            {
+                return ConsoleColors.Colorize(tokenText, ConsoleColors.BrightWhite);
+            }
+            case LiteralToken literal:
+            {
+                if (literal.Kind == LiteralKind.String)
+                {
+                    return ConsoleColors.Colorize(tokenText, ConsoleColors.Green);
+                }
+
+                return ConsoleColors.Colorize(tokenText, ConsoleColors.Magenta);
+            }
+            case SymbolToken symbolToken:
+            {
+                switch (symbolToken.Symbol)
+                {
+                    case Symbol.Func:
+                    case Symbol.Return:
+                    case Symbol.If:
+                    case Symbol.Else:
+                    case Symbol.While:
+                    case Symbol.Break:
+                    case Symbol.Continue:
+                    case Symbol.Struct:
+                    case Symbol.Let:
+                    case Symbol.Alloc:
+                    case Symbol.Calls:
+                    case Symbol.Interface:
+                    case Symbol.For:
+                    case Symbol.Extern:
+                    {
+                        return ConsoleColors.Colorize(tokenText, ConsoleColors.Bold + ConsoleColors.Blue);
+                    }
+                    case Symbol.Assign:
+                    case Symbol.Bang:
+                    case Symbol.Equal:
+                    case Symbol.NotEqual:
+                    case Symbol.LessThan:
+                    case Symbol.LessThanOrEqual:
+                    case Symbol.GreaterThan:
+                    case Symbol.GreaterThanOrEqual:
+                    case Symbol.Plus:
+                    case Symbol.Minus:
+                    case Symbol.Star:
+                    case Symbol.ForwardSlash:
+                    case Symbol.Caret:
+                    case Symbol.Ampersand:
+                    case Symbol.Arrow:
+                    {
+                        return ConsoleColors.Colorize(tokenText, ConsoleColors.Yellow);
+                    }
+                    case Symbol.Colon:
+                    case Symbol.OpenParen:
+                    case Symbol.CloseParen:
+                    case Symbol.OpenBrace:
+                    case Symbol.CloseBrace:
+                    case Symbol.OpenBracket:
+                    case Symbol.CloseBracket:
+                    case Symbol.Comma:
+                    case Symbol.Period:
+                    case Symbol.Semi:
+                    {
+                        return ConsoleColors.Colorize(tokenText, ConsoleColors.BrightBlack);
+                    }
+                }
+
+                break;
+            }
+        }
+
+        return tokenText;
+    }
 }
 
 public enum DiagnosticSeverity

src/compiler/NubLang/Tokenization/Token.cs

@@ -2,17 +2,17 @@
 
 namespace NubLang.Tokenization;
 
-public abstract class Token(SourceFileSpan? fileSpan)
+public abstract class Token(SourceFileSpan fileSpan)
 {
-    public SourceFileSpan? FileSpan { get; } = fileSpan;
+    public SourceFileSpan FileSpan { get; } = fileSpan;
 }
 
-public class IdentifierToken(SourceFileSpan? fileSpan, string value) : Token(fileSpan)
+public class IdentifierToken(SourceFileSpan fileSpan, string value) : Token(fileSpan)
 {
     public string Value { get; } = value;
 }
 
-public class LiteralToken(SourceFileSpan? fileSpan, LiteralKind kind, string value) : Token(fileSpan)
+public class LiteralToken(SourceFileSpan fileSpan, LiteralKind kind, string value) : Token(fileSpan)
 {
     public LiteralKind Kind { get; } = kind;
     public string Value { get; } = value;
@@ -26,7 +26,7 @@ public enum LiteralKind
     Bool
 }
 
-public class SymbolToken(SourceFileSpan? fileSpan, Symbol symbol) : Token(fileSpan)
+public class SymbolToken(SourceFileSpan fileSpan, Symbol symbol) : Token(fileSpan)
 {
     public Symbol Symbol { get; } = symbol;
 }

src/compiler/NubLang/Tokenization/Tokenizer.cs

@@ -57,20 +57,13 @@ public sealed class Tokenizer
         .Select(kvp => (kvp.Key, kvp.Value))
         .ToArray();
 
-    private readonly string _sourceText;
+    private readonly SourceFile _sourceFile;
-    private readonly SourceFile? _sourceFile;
     private readonly List<Diagnostic> _diagnostics = [];
     private int _index;
 
-    public Tokenizer(string sourceText)
-    {
-        _sourceText = sourceText;
-    }
-
     public Tokenizer(SourceFile sourceFile)
     {
         _sourceFile = sourceFile;
-        _sourceText = sourceFile.GetText();
     }
 
     public IReadOnlyList<Diagnostic> GetDiagnostics() => _diagnostics;
@@ -223,9 +216,9 @@ public sealed class Tokenizer
 
     private Optional<char> Peek(int offset = 0)
     {
-        if (_index + offset < _sourceText.Length)
+        if (_index + offset < _sourceFile.GetText().Length)
         {
-            return _sourceText[_index + offset];
+            return _sourceFile.GetText()[_index + offset];
         }
 
         return Optional<char>.Empty();
@@ -236,26 +229,21 @@ public sealed class Tokenizer
         _index++;
     }
 
-    private SourceFileSpan? GetSourceFileSpan(int tokenStartIndex)
+    private SourceFileSpan GetSourceFileSpan(int tokenStartIndex)
-    {
-        if (_sourceFile != null)
     {
         var start = CalculateSourceLocation(tokenStartIndex);
         var end = CalculateSourceLocation(_index);
         return new SourceFileSpan(_sourceFile, new SourceSpan(start, end));
     }
 
-        return null;
-    }
-
     private SourceLocation CalculateSourceLocation(int index)
     {
         var line = 1;
         var column = 1;
 
-        for (var i = 0; i < index && i < _sourceText.Length; i++)
+        for (var i = 0; i < index && i < _sourceFile.GetText().Length; i++)
         {
-            if (_sourceText[i] == '\n')
+            if (_sourceFile.GetText()[i] == '\n')
             {
                 line++;
                 column = 1;