Diagnostics

example/src/main.nub

@@ -1,5 +1,11 @@
+
+
+
 func main(args: []cstring): i64
 {
-    puts("test")
+    puts("test") %%%
     return 0
 }
+
+
+

src/compiler/NubLang.CLI/Program.cs

@@ -79,11 +79,11 @@ foreach (var file in options.Files)
 
 foreach (var file in options.Files)
 {
-    var tokenizer = new Tokenizer(file.GetText());
+    var tokenizer = new Tokenizer(file);
-    var tokens = tokenizer.Tokenize();
-
     var parser = new Parser();
-    var syntaxTree = parser.Parse(tokens);
+    var syntaxTree = parser.Parse(tokenizer.Tokenize());
+
+    diagnostics.AddRange(tokenizer.GetDiagnostics());
     diagnostics.AddRange(parser.GetDiagnostics());
 
     syntaxTrees.Add(syntaxTree);

src/compiler/NubLang/Code/SourceFile.cs

@@ -26,4 +26,16 @@ public class SourceFile
 
     public static bool operator ==(SourceFile? left, SourceFile? right) => Equals(left, right);
     public static bool operator !=(SourceFile? left, SourceFile? right) => !Equals(left, right);
+}
+
+public class SourceFileSpan
+{
+    public SourceFileSpan(SourceFile sourceFile, SourceSpan span)
+    {
+        SourceFile = sourceFile;
+        Span = span;
+    }
+
+    public SourceFile SourceFile { get; }
+    public SourceSpan Span { get; }
 }

src/compiler/NubLang/Code/SourceLocation.cs

@@ -18,17 +18,6 @@ public readonly struct SourceLocation : IEquatable<SourceLocation>
         return $"{Line}:{Column}";
     }
 
-    public int CompareTo(SourceLocation other)
-    {
-        var lineComparison = Line.CompareTo(other.Line);
-        if (lineComparison == 0)
-        {
-            return Column.CompareTo(other.Column);
-        }
-
-        return lineComparison;
-    }
-
     public override bool Equals(object? obj)
     {
         return obj is SourceLocation other && Equals(other);

src/compiler/NubLang/Diagnostics/Diagnostic.cs

@@ -1,4 +1,6 @@
 using System.Text;
+using NubLang.Code;
+using NubLang.Tokenization;
 
 namespace NubLang.Diagnostics;
 
@@ -8,6 +10,7 @@ public class Diagnostic
     {
         private readonly DiagnosticSeverity _severity;
         private readonly string _message;
+        private SourceFileSpan? _fileSpan;
         private string? _help;
 
         public DiagnosticBuilder(DiagnosticSeverity severity, string message)
@@ -16,13 +19,29 @@ public class Diagnostic
             _message = message;
         }
 
+        public DiagnosticBuilder At(Token token)
+        {
+            At(token.FileSpan);
+            return this;
+        }
+
+        public DiagnosticBuilder At(SourceFileSpan? fileSpan)
+        {
+            if (fileSpan != null)
+            {
+                _fileSpan = fileSpan;
+            }
+
+            return this;
+        }
+
         public DiagnosticBuilder WithHelp(string help)
         {
             _help = help;
             return this;
         }
 
-        public Diagnostic Build() => new(_severity, _message, _help);
+        public Diagnostic Build() => new(_severity, _message, _help, _fileSpan);
     }
 
     public static DiagnosticBuilder Error(string message) => new(DiagnosticSeverity.Error, message);
@@ -32,25 +51,104 @@ public class Diagnostic
     public DiagnosticSeverity Severity { get; }
     public string Message { get; }
     public string? Help { get; }
+    public SourceFileSpan? FileSpan { get; }
 
-    private Diagnostic(DiagnosticSeverity severity, string message, string? help)
+    private Diagnostic(DiagnosticSeverity severity, string message, string? help, SourceFileSpan? fileSpan)
     {
         Severity = severity;
         Message = message;
         Help = help;
+        FileSpan = fileSpan;
     }
 
     public string FormatANSI()
     {
         var sb = new StringBuilder();
 
-        var severityText = GetSeverityText(Severity);
+        sb.Append(Severity switch
-        sb.Append(severityText);
+        {
+            DiagnosticSeverity.Error => ConsoleColors.Colorize("error", ConsoleColors.Bold + ConsoleColors.Red),
+            DiagnosticSeverity.Warning => ConsoleColors.Colorize("warning", ConsoleColors.Bold + ConsoleColors.Yellow),
+            DiagnosticSeverity.Info => ConsoleColors.Colorize("info", ConsoleColors.Bold + ConsoleColors.Blue),
+            _ => ConsoleColors.Colorize("unknown", ConsoleColors.Bold + ConsoleColors.White)
+        });
 
         sb.Append(": ");
         sb.Append(ConsoleColors.Colorize(Message, ConsoleColors.BrightWhite));
 
-        if (!string.IsNullOrEmpty(Help))
+        if (FileSpan != null)
+        {
+            sb.AppendLine();
+            var text = FileSpan.SourceFile.GetText();
+
+            var lines = text.Split('\n');
+
+            var startLine = FileSpan.Span.Start.Line;
+            var endLine = FileSpan.Span.End.Line;
+
+            const int CONTEXT_LINES = 3;
+
+            var contextStartLine = Math.Max(1, startLine - CONTEXT_LINES);
+            var contextEndLine = Math.Min(lines.Length, endLine + CONTEXT_LINES);
+
+            var numberPadding = contextEndLine.ToString().Length;
+            var codePadding = lines.Skip(contextStartLine - 1).Take(contextEndLine - contextStartLine).Max(x => x.Length);
+
+            sb.Append('╭');
+            sb.Append(new string('─', numberPadding + 2));
+            sb.Append('┬');
+            sb.Append(new string('─', codePadding + 2));
+            sb.Append('╮');
+            sb.AppendLine();
+
+            for (var i = contextStartLine; i <= contextEndLine; i++)
+            {
+                var line = lines[i - 1];
+
+                sb.Append("│ ");
+                sb.Append(i.ToString().PadRight(numberPadding));
+                sb.Append(" │ ");
+                sb.Append(line.PadRight(codePadding));
+                sb.Append(" │");
+                sb.AppendLine();
+
+                if (i >= startLine && i <= endLine)
+                {
+                    var markerStartColumn = 1;
+                    var markerEndColumn = line.Length + 1;
+
+                    if (i == startLine)
+                    {
+                        markerStartColumn = FileSpan.Span.Start.Column;
+                    }
+
+                    if (i == endLine)
+                    {
+                        markerEndColumn = FileSpan.Span.End.Column;
+                    }
+
+                    var markerLength = markerEndColumn - markerStartColumn;
+                    var marker = new string('^', markerLength);
+
+                    sb.Append("│ ");
+                    sb.Append(new string(' ', numberPadding));
+                    sb.Append(" │ ");
+                    sb.Append(new string(' ', markerStartColumn - 1));
+                    sb.Append(ConsoleColors.Colorize(marker, ConsoleColors.Red));
+                    sb.Append(new string(' ', codePadding - markerEndColumn + 1));
+                    sb.Append(" │");
+                    sb.AppendLine();
+                }
+            }
+
+            sb.Append('╰');
+            sb.Append(new string('─', numberPadding + 2));
+            sb.Append('┴');
+            sb.Append(new string('─', codePadding + 2));
+            sb.Append('╯');
+        }
+
+        if (Help != null)
         {
             sb.AppendLine();
             sb.Append(ConsoleColors.Colorize($"help: {Help}", ConsoleColors.Cyan));
@@ -58,17 +156,6 @@ public class Diagnostic
 
         return sb.ToString();
     }
-
-    private static string GetSeverityText(DiagnosticSeverity severity)
-    {
-        return severity switch
-        {
-            DiagnosticSeverity.Error => ConsoleColors.Colorize("error", ConsoleColors.Bold + ConsoleColors.Red),
-            DiagnosticSeverity.Warning => ConsoleColors.Colorize("warning", ConsoleColors.Bold + ConsoleColors.Yellow),
-            DiagnosticSeverity.Info => ConsoleColors.Colorize("info", ConsoleColors.Bold + ConsoleColors.Blue),
-            _ => throw new ArgumentOutOfRangeException(nameof(severity), severity, "Unknown diagnostic severity")
-        };
-    }
 }
 
 public enum DiagnosticSeverity

src/compiler/NubLang/Tokenization/Token.cs

@@ -1,13 +1,18 @@
-namespace NubLang.Tokenization;
+using NubLang.Code;
 
-public abstract class Token;
+namespace NubLang.Tokenization;
 
-public class IdentifierToken(string value) : Token
+public abstract class Token(SourceFileSpan? fileSpan)
+{
+    public SourceFileSpan? FileSpan { get; } = fileSpan;
+}
+
+public class IdentifierToken(SourceFileSpan? fileSpan, string value) : Token(fileSpan)
 {
     public string Value { get; } = value;
 }
 
-public class LiteralToken(LiteralKind kind, string value) : Token
+public class LiteralToken(SourceFileSpan? fileSpan, LiteralKind kind, string value) : Token(fileSpan)
 {
     public LiteralKind Kind { get; } = kind;
     public string Value { get; } = value;
@@ -21,7 +26,7 @@ public enum LiteralKind
     Bool
 }
 
-public class SymbolToken(Symbol symbol) : Token
+public class SymbolToken(SourceFileSpan? fileSpan, Symbol symbol) : Token(fileSpan)
 {
     public Symbol Symbol { get; } = symbol;
 }

src/compiler/NubLang/Tokenization/Tokenizer.cs

@@ -1,4 +1,7 @@
-namespace NubLang.Tokenization;
+using NubLang.Code;
+using NubLang.Diagnostics;
+
+namespace NubLang.Tokenization;
 
 public sealed class Tokenizer
 {
@@ -55,6 +58,8 @@ public sealed class Tokenizer
         .ToArray();
 
     private readonly string _sourceText;
+    private readonly SourceFile? _sourceFile;
+    private readonly List<Diagnostic> _diagnostics = [];
     private int _index;
 
     public Tokenizer(string sourceText)
@@ -62,6 +67,14 @@ public sealed class Tokenizer
         _sourceText = sourceText;
     }
 
+    public Tokenizer(SourceFile sourceFile)
+    {
+        _sourceFile = sourceFile;
+        _sourceText = sourceFile.GetText();
+    }
+
+    public IReadOnlyList<Diagnostic> GetDiagnostics() => _diagnostics;
+
     public IEnumerable<Token> Tokenize()
     {
         _index = 0;
@@ -84,6 +97,8 @@ public sealed class Tokenizer
                 continue;
             }
 
+            var tokenStartIndex = _index;
+
             if (char.IsLetter(current) || current == '_')
             {
                 var buffer = string.Empty;
@@ -96,17 +111,17 @@ public sealed class Tokenizer
 
                 if (Keywords.TryGetValue(buffer, out var keywordSymbol))
                 {
-                    yield return new SymbolToken(keywordSymbol);
+                    yield return new SymbolToken(GetSourceFileSpan(tokenStartIndex), keywordSymbol);
                     continue;
                 }
 
                 if (buffer is "true" or "false")
                 {
-                    yield return new LiteralToken(LiteralKind.Bool, buffer);
+                    yield return new LiteralToken(GetSourceFileSpan(tokenStartIndex), LiteralKind.Bool, buffer);
                     continue;
                 }
 
-                yield return new IdentifierToken(buffer);
+                yield return new IdentifierToken(GetSourceFileSpan(tokenStartIndex), buffer);
                 continue;
             }
 
@@ -139,7 +154,7 @@ public sealed class Tokenizer
                     }
                 }
 
-                yield return new LiteralToken(isFloat ? LiteralKind.Float : LiteralKind.Integer, buffer);
+                yield return new LiteralToken(GetSourceFileSpan(tokenStartIndex), isFloat ? LiteralKind.Float : LiteralKind.Integer, buffer);
                 continue;
             }
 
@@ -165,7 +180,7 @@ public sealed class Tokenizer
                     Next();
                 }
 
-                yield return new LiteralToken(LiteralKind.String, buffer);
+                yield return new LiteralToken(GetSourceFileSpan(tokenStartIndex), LiteralKind.String, buffer);
                 continue;
             }
 
@@ -184,7 +199,7 @@ public sealed class Tokenizer
                             Next();
                         }
 
-                        yield return new SymbolToken(symbol);
+                        yield return new SymbolToken(GetSourceFileSpan(tokenStartIndex), symbol);
                         foundMatch = true;
                         break;
                     }
@@ -201,7 +216,8 @@ public sealed class Tokenizer
                 continue;
             }
 
-            throw new Exception($"Unknown character {current}");
+            _diagnostics.Add(Diagnostic.Error($"Unknown token '{current}'").At(GetSourceFileSpan(tokenStartIndex)).Build());
+            Next();
         }
     }
 
@@ -219,4 +235,37 @@ public sealed class Tokenizer
     {
         _index++;
     }
+
+    private SourceFileSpan? GetSourceFileSpan(int tokenStartIndex)
+    {
+        if (_sourceFile != null)
+        {
+            var start = CalculateSourceLocation(tokenStartIndex);
+            var end = CalculateSourceLocation(_index + 1);
+            return new SourceFileSpan(_sourceFile, new SourceSpan(start, end));
+        }
+
+        return null;
+    }
+
+    private SourceLocation CalculateSourceLocation(int index)
+    {
+        var line = 1;
+        var column = 1;
+
+        for (var i = 0; i < index && i < _sourceText.Length; i++)
+        {
+            if (_sourceText[i] == '\n')
+            {
+                line++;
+                column = 1;
+            }
+            else
+            {
+                column++;
+            }
+        }
+
+        return new SourceLocation(line, column);
+    }
 }