From 2d6cf0ba7fdfa1b837c6fddd9f08d8f2536a3de7 Mon Sep 17 00:00:00 2001 From: nub31 Date: Wed, 23 Jul 2025 01:42:52 +0200 Subject: [PATCH] Syntax highlighting --- .../NubLang/Diagnostics/Diagnostic.cs | 135 +++++++++++++++++- src/compiler/NubLang/Tokenization/Token.cs | 10 +- .../NubLang/Tokenization/Tokenizer.cs | 30 ++-- 3 files changed, 148 insertions(+), 27 deletions(-) diff --git a/src/compiler/NubLang/Diagnostics/Diagnostic.cs b/src/compiler/NubLang/Diagnostics/Diagnostic.cs index 7b8534a..4b2bfe4 100644 --- a/src/compiler/NubLang/Diagnostics/Diagnostic.cs +++ b/src/compiler/NubLang/Diagnostics/Diagnostic.cs @@ -93,6 +93,11 @@ public class Diagnostic _ => ConsoleColors.Colorize("unknown", ConsoleColors.Bold + ConsoleColors.White) }); + if (FileSpan != null) + { + sb.Append(ConsoleColors.Colorize($" at {FileSpan.SourceFile.Path}:{FileSpan.Span}", ConsoleColors.Faint)); + } + sb.Append(": "); sb.Append(ConsoleColors.Colorize(Message, ConsoleColors.BrightWhite)); @@ -121,6 +126,9 @@ public class Diagnostic sb.Append('╮'); sb.AppendLine(); + var tokenizer = new Tokenizer(FileSpan.SourceFile); + var tokens = tokenizer.Tokenize().ToList(); + for (var i = contextStartLine; i <= contextEndLine; i++) { var line = lines[i - 1]; @@ -128,7 +136,7 @@ public class Diagnostic sb.Append("│ "); sb.Append(i.ToString().PadRight(numberPadding)); sb.Append(" │ "); - sb.Append(line.PadRight(codePadding)); + sb.Append(ApplySyntaxHighlighting(line.PadRight(codePadding), i, tokens)); sb.Append(" │"); sb.AppendLine(); @@ -184,6 +192,131 @@ public class Diagnostic return sb.ToString(); } + + private static string ApplySyntaxHighlighting(string line, int lineNumber, List tokens) + { + var sb = new StringBuilder(); + var lineTokens = tokens + .Where(t => t.FileSpan.Span.Start.Line == lineNumber) + .OrderBy(t => t.FileSpan.Span.Start.Column) + .ToList(); + + if (lineTokens.Count == 0) + { + return line; + } + + var currentColumn = 1; + + foreach (var token in lineTokens) + { + var tokenStart = token.FileSpan.Span.Start.Column; + var tokenEnd = token.FileSpan.Span.End.Column; + + if (tokenStart > currentColumn) + { + var beforeToken = line.Substring(currentColumn - 1, tokenStart - currentColumn); + sb.Append(beforeToken); + } + + var tokenLength = tokenEnd - tokenStart; + if (tokenStart - 1 + tokenLength <= line.Length) + { + var tokenText = line.Substring(tokenStart - 1, tokenLength); + + var coloredToken = ColorizeToken(token, tokenText); + sb.Append(coloredToken); + } + + currentColumn = tokenEnd; + } + + if (currentColumn <= line.Length) + { + var remaining = line[(currentColumn - 1)..]; + sb.Append(remaining); + } + + return sb.ToString(); + } + + private static string ColorizeToken(Token token, string tokenText) + { + switch (token) + { + case IdentifierToken: + { + return ConsoleColors.Colorize(tokenText, ConsoleColors.BrightWhite); + } + case LiteralToken literal: + { + if (literal.Kind == LiteralKind.String) + { + return ConsoleColors.Colorize(tokenText, ConsoleColors.Green); + } + + return ConsoleColors.Colorize(tokenText, ConsoleColors.Magenta); + } + case SymbolToken symbolToken: + { + switch (symbolToken.Symbol) + { + case Symbol.Func: + case Symbol.Return: + case Symbol.If: + case Symbol.Else: + case Symbol.While: + case Symbol.Break: + case Symbol.Continue: + case Symbol.Struct: + case Symbol.Let: + case Symbol.Alloc: + case Symbol.Calls: + case Symbol.Interface: + case Symbol.For: + case Symbol.Extern: + { + return ConsoleColors.Colorize(tokenText, ConsoleColors.Bold + ConsoleColors.Blue); + } + case Symbol.Assign: + case Symbol.Bang: + case Symbol.Equal: + case Symbol.NotEqual: + case Symbol.LessThan: + case Symbol.LessThanOrEqual: + case Symbol.GreaterThan: + case Symbol.GreaterThanOrEqual: + case Symbol.Plus: + case Symbol.Minus: + case Symbol.Star: + case Symbol.ForwardSlash: + case Symbol.Caret: + case Symbol.Ampersand: + case Symbol.Arrow: + { + return ConsoleColors.Colorize(tokenText, ConsoleColors.Yellow); + } + case Symbol.Colon: + case Symbol.OpenParen: + case Symbol.CloseParen: + case Symbol.OpenBrace: + case Symbol.CloseBrace: + case Symbol.OpenBracket: + case Symbol.CloseBracket: + case Symbol.Comma: + case Symbol.Period: + case Symbol.Semi: + { + return ConsoleColors.Colorize(tokenText, ConsoleColors.BrightBlack); + } + } + + break; + } + } + + return tokenText; + } } public enum DiagnosticSeverity diff --git a/src/compiler/NubLang/Tokenization/Token.cs b/src/compiler/NubLang/Tokenization/Token.cs index b567864..4c473a3 100644 --- a/src/compiler/NubLang/Tokenization/Token.cs +++ b/src/compiler/NubLang/Tokenization/Token.cs @@ -2,17 +2,17 @@ namespace NubLang.Tokenization; -public abstract class Token(SourceFileSpan? fileSpan) +public abstract class Token(SourceFileSpan fileSpan) { - public SourceFileSpan? FileSpan { get; } = fileSpan; + public SourceFileSpan FileSpan { get; } = fileSpan; } -public class IdentifierToken(SourceFileSpan? fileSpan, string value) : Token(fileSpan) +public class IdentifierToken(SourceFileSpan fileSpan, string value) : Token(fileSpan) { public string Value { get; } = value; } -public class LiteralToken(SourceFileSpan? fileSpan, LiteralKind kind, string value) : Token(fileSpan) +public class LiteralToken(SourceFileSpan fileSpan, LiteralKind kind, string value) : Token(fileSpan) { public LiteralKind Kind { get; } = kind; public string Value { get; } = value; @@ -26,7 +26,7 @@ public enum LiteralKind Bool } -public class SymbolToken(SourceFileSpan? fileSpan, Symbol symbol) : Token(fileSpan) +public class SymbolToken(SourceFileSpan fileSpan, Symbol symbol) : Token(fileSpan) { public Symbol Symbol { get; } = symbol; } diff --git a/src/compiler/NubLang/Tokenization/Tokenizer.cs b/src/compiler/NubLang/Tokenization/Tokenizer.cs index dc540a8..f42289f 100644 --- a/src/compiler/NubLang/Tokenization/Tokenizer.cs +++ b/src/compiler/NubLang/Tokenization/Tokenizer.cs @@ -57,20 +57,13 @@ public sealed class Tokenizer .Select(kvp => (kvp.Key, kvp.Value)) .ToArray(); - private readonly string _sourceText; - private readonly SourceFile? _sourceFile; + private readonly SourceFile _sourceFile; private readonly List _diagnostics = []; private int _index; - public Tokenizer(string sourceText) - { - _sourceText = sourceText; - } - public Tokenizer(SourceFile sourceFile) { _sourceFile = sourceFile; - _sourceText = sourceFile.GetText(); } public IReadOnlyList GetDiagnostics() => _diagnostics; @@ -223,9 +216,9 @@ public sealed class Tokenizer private Optional Peek(int offset = 0) { - if (_index + offset < _sourceText.Length) + if (_index + offset < _sourceFile.GetText().Length) { - return _sourceText[_index + offset]; + return _sourceFile.GetText()[_index + offset]; } return Optional.Empty(); @@ -236,16 +229,11 @@ public sealed class Tokenizer _index++; } - private SourceFileSpan? GetSourceFileSpan(int tokenStartIndex) + private SourceFileSpan GetSourceFileSpan(int tokenStartIndex) { - if (_sourceFile != null) - { - var start = CalculateSourceLocation(tokenStartIndex); - var end = CalculateSourceLocation(_index); - return new SourceFileSpan(_sourceFile, new SourceSpan(start, end)); - } - - return null; + var start = CalculateSourceLocation(tokenStartIndex); + var end = CalculateSourceLocation(_index); + return new SourceFileSpan(_sourceFile, new SourceSpan(start, end)); } private SourceLocation CalculateSourceLocation(int index) @@ -253,9 +241,9 @@ public sealed class Tokenizer var line = 1; var column = 1; - for (var i = 0; i < index && i < _sourceText.Length; i++) + for (var i = 0; i < index && i < _sourceFile.GetText().Length; i++) { - if (_sourceText[i] == '\n') + if (_sourceFile.GetText()[i] == '\n') { line++; column = 1;