From bf89fe02d3ff7c64a18450edc17b806b0eb0bc68 Mon Sep 17 00:00:00 2001 From: nub31 Date: Wed, 23 Jul 2025 00:43:50 +0200 Subject: [PATCH] Diagnostics --- example/src/main.nub | 8 +- src/compiler/NubLang.CLI/Program.cs | 8 +- src/compiler/NubLang/Code/SourceFile.cs | 12 ++ src/compiler/NubLang/Code/SourceLocation.cs | 11 -- .../NubLang/Diagnostics/Diagnostic.cs | 119 +++++++++++++++--- src/compiler/NubLang/Tokenization/Token.cs | 15 ++- .../NubLang/Tokenization/Tokenizer.cs | 65 ++++++++-- 7 files changed, 193 insertions(+), 45 deletions(-) diff --git a/example/src/main.nub b/example/src/main.nub index b97d3d7..07640d6 100644 --- a/example/src/main.nub +++ b/example/src/main.nub @@ -1,5 +1,11 @@ + + + func main(args: []cstring): i64 { - puts("test") + puts("test") %%% return 0 } + + + diff --git a/src/compiler/NubLang.CLI/Program.cs b/src/compiler/NubLang.CLI/Program.cs index d2d6731..52e6ab7 100644 --- a/src/compiler/NubLang.CLI/Program.cs +++ b/src/compiler/NubLang.CLI/Program.cs @@ -79,11 +79,11 @@ foreach (var file in options.Files) foreach (var file in options.Files) { - var tokenizer = new Tokenizer(file.GetText()); - var tokens = tokenizer.Tokenize(); - + var tokenizer = new Tokenizer(file); var parser = new Parser(); - var syntaxTree = parser.Parse(tokens); + var syntaxTree = parser.Parse(tokenizer.Tokenize()); + + diagnostics.AddRange(tokenizer.GetDiagnostics()); diagnostics.AddRange(parser.GetDiagnostics()); syntaxTrees.Add(syntaxTree); diff --git a/src/compiler/NubLang/Code/SourceFile.cs b/src/compiler/NubLang/Code/SourceFile.cs index 0b39ce5..48370c4 100644 --- a/src/compiler/NubLang/Code/SourceFile.cs +++ b/src/compiler/NubLang/Code/SourceFile.cs @@ -26,4 +26,16 @@ public class SourceFile public static bool operator ==(SourceFile? left, SourceFile? right) => Equals(left, right); public static bool operator !=(SourceFile? left, SourceFile? right) => !Equals(left, right); +} + +public class SourceFileSpan +{ + public SourceFileSpan(SourceFile sourceFile, SourceSpan span) + { + SourceFile = sourceFile; + Span = span; + } + + public SourceFile SourceFile { get; } + public SourceSpan Span { get; } } \ No newline at end of file diff --git a/src/compiler/NubLang/Code/SourceLocation.cs b/src/compiler/NubLang/Code/SourceLocation.cs index 6574b4f..38bf900 100644 --- a/src/compiler/NubLang/Code/SourceLocation.cs +++ b/src/compiler/NubLang/Code/SourceLocation.cs @@ -18,17 +18,6 @@ public readonly struct SourceLocation : IEquatable return $"{Line}:{Column}"; } - public int CompareTo(SourceLocation other) - { - var lineComparison = Line.CompareTo(other.Line); - if (lineComparison == 0) - { - return Column.CompareTo(other.Column); - } - - return lineComparison; - } - public override bool Equals(object? obj) { return obj is SourceLocation other && Equals(other); diff --git a/src/compiler/NubLang/Diagnostics/Diagnostic.cs b/src/compiler/NubLang/Diagnostics/Diagnostic.cs index 096101d..1f260bb 100644 --- a/src/compiler/NubLang/Diagnostics/Diagnostic.cs +++ b/src/compiler/NubLang/Diagnostics/Diagnostic.cs @@ -1,4 +1,6 @@ using System.Text; +using NubLang.Code; +using NubLang.Tokenization; namespace NubLang.Diagnostics; @@ -8,6 +10,7 @@ public class Diagnostic { private readonly DiagnosticSeverity _severity; private readonly string _message; + private SourceFileSpan? _fileSpan; private string? _help; public DiagnosticBuilder(DiagnosticSeverity severity, string message) @@ -16,13 +19,29 @@ public class Diagnostic _message = message; } + public DiagnosticBuilder At(Token token) + { + At(token.FileSpan); + return this; + } + + public DiagnosticBuilder At(SourceFileSpan? fileSpan) + { + if (fileSpan != null) + { + _fileSpan = fileSpan; + } + + return this; + } + public DiagnosticBuilder WithHelp(string help) { _help = help; return this; } - public Diagnostic Build() => new(_severity, _message, _help); + public Diagnostic Build() => new(_severity, _message, _help, _fileSpan); } public static DiagnosticBuilder Error(string message) => new(DiagnosticSeverity.Error, message); @@ -32,25 +51,104 @@ public class Diagnostic public DiagnosticSeverity Severity { get; } public string Message { get; } public string? Help { get; } + public SourceFileSpan? FileSpan { get; } - private Diagnostic(DiagnosticSeverity severity, string message, string? help) + private Diagnostic(DiagnosticSeverity severity, string message, string? help, SourceFileSpan? fileSpan) { Severity = severity; Message = message; Help = help; + FileSpan = fileSpan; } public string FormatANSI() { var sb = new StringBuilder(); - var severityText = GetSeverityText(Severity); - sb.Append(severityText); + sb.Append(Severity switch + { + DiagnosticSeverity.Error => ConsoleColors.Colorize("error", ConsoleColors.Bold + ConsoleColors.Red), + DiagnosticSeverity.Warning => ConsoleColors.Colorize("warning", ConsoleColors.Bold + ConsoleColors.Yellow), + DiagnosticSeverity.Info => ConsoleColors.Colorize("info", ConsoleColors.Bold + ConsoleColors.Blue), + _ => ConsoleColors.Colorize("unknown", ConsoleColors.Bold + ConsoleColors.White) + }); sb.Append(": "); sb.Append(ConsoleColors.Colorize(Message, ConsoleColors.BrightWhite)); - if (!string.IsNullOrEmpty(Help)) + if (FileSpan != null) + { + sb.AppendLine(); + var text = FileSpan.SourceFile.GetText(); + + var lines = text.Split('\n'); + + var startLine = FileSpan.Span.Start.Line; + var endLine = FileSpan.Span.End.Line; + + const int CONTEXT_LINES = 3; + + var contextStartLine = Math.Max(1, startLine - CONTEXT_LINES); + var contextEndLine = Math.Min(lines.Length, endLine + CONTEXT_LINES); + + var numberPadding = contextEndLine.ToString().Length; + var codePadding = lines.Skip(contextStartLine - 1).Take(contextEndLine - contextStartLine).Max(x => x.Length); + + sb.Append('╭'); + sb.Append(new string('─', numberPadding + 2)); + sb.Append('┬'); + sb.Append(new string('─', codePadding + 2)); + sb.Append('╮'); + sb.AppendLine(); + + for (var i = contextStartLine; i <= contextEndLine; i++) + { + var line = lines[i - 1]; + + sb.Append("│ "); + sb.Append(i.ToString().PadRight(numberPadding)); + sb.Append(" │ "); + sb.Append(line.PadRight(codePadding)); + sb.Append(" │"); + sb.AppendLine(); + + if (i >= startLine && i <= endLine) + { + var markerStartColumn = 1; + var markerEndColumn = line.Length + 1; + + if (i == startLine) + { + markerStartColumn = FileSpan.Span.Start.Column; + } + + if (i == endLine) + { + markerEndColumn = FileSpan.Span.End.Column; + } + + var markerLength = markerEndColumn - markerStartColumn; + var marker = new string('^', markerLength); + + sb.Append("│ "); + sb.Append(new string(' ', numberPadding)); + sb.Append(" │ "); + sb.Append(new string(' ', markerStartColumn - 1)); + sb.Append(ConsoleColors.Colorize(marker, ConsoleColors.Red)); + sb.Append(new string(' ', codePadding - markerEndColumn + 1)); + sb.Append(" │"); + sb.AppendLine(); + } + } + + sb.Append('╰'); + sb.Append(new string('─', numberPadding + 2)); + sb.Append('┴'); + sb.Append(new string('─', codePadding + 2)); + sb.Append('╯'); + } + + if (Help != null) { sb.AppendLine(); sb.Append(ConsoleColors.Colorize($"help: {Help}", ConsoleColors.Cyan)); @@ -58,17 +156,6 @@ public class Diagnostic return sb.ToString(); } - - private static string GetSeverityText(DiagnosticSeverity severity) - { - return severity switch - { - DiagnosticSeverity.Error => ConsoleColors.Colorize("error", ConsoleColors.Bold + ConsoleColors.Red), - DiagnosticSeverity.Warning => ConsoleColors.Colorize("warning", ConsoleColors.Bold + ConsoleColors.Yellow), - DiagnosticSeverity.Info => ConsoleColors.Colorize("info", ConsoleColors.Bold + ConsoleColors.Blue), - _ => throw new ArgumentOutOfRangeException(nameof(severity), severity, "Unknown diagnostic severity") - }; - } } public enum DiagnosticSeverity diff --git a/src/compiler/NubLang/Tokenization/Token.cs b/src/compiler/NubLang/Tokenization/Token.cs index 8810681..b567864 100644 --- a/src/compiler/NubLang/Tokenization/Token.cs +++ b/src/compiler/NubLang/Tokenization/Token.cs @@ -1,13 +1,18 @@ -namespace NubLang.Tokenization; +using NubLang.Code; -public abstract class Token; +namespace NubLang.Tokenization; -public class IdentifierToken(string value) : Token +public abstract class Token(SourceFileSpan? fileSpan) +{ + public SourceFileSpan? FileSpan { get; } = fileSpan; +} + +public class IdentifierToken(SourceFileSpan? fileSpan, string value) : Token(fileSpan) { public string Value { get; } = value; } -public class LiteralToken(LiteralKind kind, string value) : Token +public class LiteralToken(SourceFileSpan? fileSpan, LiteralKind kind, string value) : Token(fileSpan) { public LiteralKind Kind { get; } = kind; public string Value { get; } = value; @@ -21,7 +26,7 @@ public enum LiteralKind Bool } -public class SymbolToken(Symbol symbol) : Token +public class SymbolToken(SourceFileSpan? fileSpan, Symbol symbol) : Token(fileSpan) { public Symbol Symbol { get; } = symbol; } diff --git a/src/compiler/NubLang/Tokenization/Tokenizer.cs b/src/compiler/NubLang/Tokenization/Tokenizer.cs index 8e3a354..c6e5121 100644 --- a/src/compiler/NubLang/Tokenization/Tokenizer.cs +++ b/src/compiler/NubLang/Tokenization/Tokenizer.cs @@ -1,4 +1,7 @@ -namespace NubLang.Tokenization; +using NubLang.Code; +using NubLang.Diagnostics; + +namespace NubLang.Tokenization; public sealed class Tokenizer { @@ -55,6 +58,8 @@ public sealed class Tokenizer .ToArray(); private readonly string _sourceText; + private readonly SourceFile? _sourceFile; + private readonly List _diagnostics = []; private int _index; public Tokenizer(string sourceText) @@ -62,6 +67,14 @@ public sealed class Tokenizer _sourceText = sourceText; } + public Tokenizer(SourceFile sourceFile) + { + _sourceFile = sourceFile; + _sourceText = sourceFile.GetText(); + } + + public IReadOnlyList GetDiagnostics() => _diagnostics; + public IEnumerable Tokenize() { _index = 0; @@ -84,6 +97,8 @@ public sealed class Tokenizer continue; } + var tokenStartIndex = _index; + if (char.IsLetter(current) || current == '_') { var buffer = string.Empty; @@ -96,17 +111,17 @@ public sealed class Tokenizer if (Keywords.TryGetValue(buffer, out var keywordSymbol)) { - yield return new SymbolToken(keywordSymbol); + yield return new SymbolToken(GetSourceFileSpan(tokenStartIndex), keywordSymbol); continue; } if (buffer is "true" or "false") { - yield return new LiteralToken(LiteralKind.Bool, buffer); + yield return new LiteralToken(GetSourceFileSpan(tokenStartIndex), LiteralKind.Bool, buffer); continue; } - yield return new IdentifierToken(buffer); + yield return new IdentifierToken(GetSourceFileSpan(tokenStartIndex), buffer); continue; } @@ -139,7 +154,7 @@ public sealed class Tokenizer } } - yield return new LiteralToken(isFloat ? LiteralKind.Float : LiteralKind.Integer, buffer); + yield return new LiteralToken(GetSourceFileSpan(tokenStartIndex), isFloat ? LiteralKind.Float : LiteralKind.Integer, buffer); continue; } @@ -165,7 +180,7 @@ public sealed class Tokenizer Next(); } - yield return new LiteralToken(LiteralKind.String, buffer); + yield return new LiteralToken(GetSourceFileSpan(tokenStartIndex), LiteralKind.String, buffer); continue; } @@ -184,7 +199,7 @@ public sealed class Tokenizer Next(); } - yield return new SymbolToken(symbol); + yield return new SymbolToken(GetSourceFileSpan(tokenStartIndex), symbol); foundMatch = true; break; } @@ -201,7 +216,8 @@ public sealed class Tokenizer continue; } - throw new Exception($"Unknown character {current}"); + _diagnostics.Add(Diagnostic.Error($"Unknown token '{current}'").At(GetSourceFileSpan(tokenStartIndex)).Build()); + Next(); } } @@ -219,4 +235,37 @@ public sealed class Tokenizer { _index++; } + + private SourceFileSpan? GetSourceFileSpan(int tokenStartIndex) + { + if (_sourceFile != null) + { + var start = CalculateSourceLocation(tokenStartIndex); + var end = CalculateSourceLocation(_index + 1); + return new SourceFileSpan(_sourceFile, new SourceSpan(start, end)); + } + + return null; + } + + private SourceLocation CalculateSourceLocation(int index) + { + var line = 1; + var column = 1; + + for (var i = 0; i < index && i < _sourceText.Length; i++) + { + if (_sourceText[i] == '\n') + { + line++; + column = 1; + } + else + { + column++; + } + } + + return new SourceLocation(line, column); + } } \ No newline at end of file