nub31/nub-lang-archive
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Perf improvements in tokenizer

Browse Source
This commit is contained in:
nub31
2025-09-29 14:11:48 +02:00
parent 3107849915
commit 30081c49d4
10 changed files with 219 additions and 274 deletions
Download Patch File Download Diff File Expand all files Collapse all files

61
compiler/NubLang/Diagnostics/Diagnostic.cs
View File

@@ -11,7 +11,7 @@ public class Diagnostic
{
private readonly DiagnosticSeverity _severity;
private readonly string _message;
private SourceFileSpan? _fileSpan;
private SourceSpan? _span;
private string? _help;
public DiagnosticBuilder(DiagnosticSeverity severity, string message)
@@ -24,12 +24,7 @@ public class Diagnostic
{
if (node != null)
{
var first = node.Tokens.FirstOrDefault();
if (first?.FileSpan != null)
{
var span = SourceSpan.Merge(node.Tokens.Select(x => x.FileSpan.Span));
At(new SourceFileSpan(first.FileSpan.SourceFile, span));
}
_span = SourceSpan.Merge(node.Tokens.Select(x => x.Span));
}
return this;
@@ -39,29 +34,35 @@ public class Diagnostic
{
if (token != null)
{
At(token.FileSpan);
At(token.Span);
}
return this;
}
public DiagnosticBuilder At(SourceFileSpan? fileSpan)
public DiagnosticBuilder At(SourceSpan? span)
{
if (fileSpan != null)
if (span != null)
{
_fileSpan = fileSpan;
_span = span;
}
return this;
}
public DiagnosticBuilder At(string filePath, int line, int column)
{
_span = new SourceSpan(filePath, new SourceLocation(line, column), new SourceLocation(line, column));
return this;
}
public DiagnosticBuilder WithHelp(string help)
{
_help = help;
return this;
}
public Diagnostic Build() => new(_severity, _message, _help, _fileSpan);
public Diagnostic Build() => new(_severity, _message, _help, _span);
}
public static DiagnosticBuilder Error(string message) => new(DiagnosticSeverity.Error, message);
@@ -71,14 +72,14 @@ public class Diagnostic
public DiagnosticSeverity Severity { get; }
public string Message { get; }
public string? Help { get; }
public SourceFileSpan? FileSpan { get; }
public SourceSpan? Span { get; }
private Diagnostic(DiagnosticSeverity severity, string message, string? help, SourceFileSpan? fileSpan)
private Diagnostic(DiagnosticSeverity severity, string message, string? help, SourceSpan? span)
{
Severity = severity;
Message = message;
Help = help;
FileSpan = fileSpan;
Span = span;
}
public string FormatANSI()
@@ -93,23 +94,23 @@ public class Diagnostic
_ => ConsoleColors.Colorize("unknown", ConsoleColors.Bold + ConsoleColors.White)
});
if (FileSpan != null)
if (Span.HasValue)
{
sb.Append(ConsoleColors.Colorize($" at {FileSpan.SourceFile.Path}:{FileSpan.Span}", ConsoleColors.Faint));
sb.Append(ConsoleColors.Colorize($" at {Span.Value}", ConsoleColors.Faint));
}
sb.Append(": ");
sb.Append(ConsoleColors.Colorize(Message, ConsoleColors.BrightWhite));
if (FileSpan != null)
if (Span.HasValue)
{
sb.AppendLine();
var text = FileSpan.SourceFile.GetText();
var text = File.ReadAllText(Span.Value.FilePath);
var lines = text.Split('\n');
var startLine = FileSpan.Span.Start.Line;
var endLine = FileSpan.Span.End.Line;
var startLine = Span.Value.Start.Line;
var endLine = Span.Value.End.Line;
const int CONTEXT_LINES = 3;
@@ -126,8 +127,8 @@ public class Diagnostic
sb.Append('╮');
sb.AppendLine();
var tokenizer = new Tokenizer(FileSpan.SourceFile);
var tokens = tokenizer.Tokenize().ToList();
var tokenizer = new Tokenizer(Span.Value.FilePath, text);
tokenizer.Tokenize();
for (var i = contextStartLine; i <= contextEndLine; i++)
{
@@ -136,7 +137,7 @@ public class Diagnostic
sb.Append("│ ");
sb.Append(i.ToString().PadRight(numberPadding));
sb.Append(" │ ");
sb.Append(ApplySyntaxHighlighting(line.PadRight(codePadding), i, tokens));
sb.Append(ApplySyntaxHighlighting(line.PadRight(codePadding), i, tokenizer.Tokens));
sb.Append(" │");
sb.AppendLine();
@@ -147,12 +148,12 @@ public class Diagnostic
if (i == startLine)
{
markerStartColumn = FileSpan.Span.Start.Column;
markerStartColumn = Span.Value.Start.Column;
}
if (i == endLine)
{
markerEndColumn = FileSpan.Span.End.Column;
markerEndColumn = Span.Value.End.Column;
}
var markerLength = markerEndColumn - markerStartColumn;
@@ -197,8 +198,8 @@ public class Diagnostic
{
var sb = new StringBuilder();
var lineTokens = tokens
.Where(t => t.FileSpan.Span.Start.Line == lineNumber)
.OrderBy(t => t.FileSpan.Span.Start.Column)
.Where(t => t.Span.Start.Line == lineNumber)
.OrderBy(t => t.Span.Start.Column)
.ToList();
if (lineTokens.Count == 0)
@@ -210,8 +211,8 @@ public class Diagnostic
foreach (var token in lineTokens)
{
var tokenStart = token.FileSpan.Span.Start.Column;
var tokenEnd = token.FileSpan.Span.End.Column;
var tokenStart = token.Span.Start.Column;
var tokenEnd = token.Span.End.Column;
if (tokenStart > currentColumn)
{