From 47fef6bc9f41122c9e821f8b3919f913ab19f634 Mon Sep 17 00:00:00 2001 From: nub31 Date: Mon, 3 Nov 2025 17:10:15 +0100 Subject: [PATCH] ... --- compiler/NubLang.LSP/AstExtensions.cs | 34 +++--- compiler/NubLang.LSP/DiagnosticsPublisher.cs | 2 +- compiler/NubLang/Ast/TypeChecker.cs | 78 ++++++------- compiler/NubLang/Diagnostics/Diagnostic.cs | 72 ++++++++---- compiler/NubLang/Diagnostics/SourceSpan.cs | 114 +++++-------------- compiler/NubLang/Syntax/Parser.cs | 32 +++--- compiler/NubLang/Syntax/Tokenizer.cs | 53 ++++----- 7 files changed, 175 insertions(+), 210 deletions(-) diff --git a/compiler/NubLang.LSP/AstExtensions.cs b/compiler/NubLang.LSP/AstExtensions.cs index 15d73b8..567e8df 100644 --- a/compiler/NubLang.LSP/AstExtensions.cs +++ b/compiler/NubLang.LSP/AstExtensions.cs @@ -16,8 +16,8 @@ public static class AstExtensions return new Location { - Uri = node.Tokens.First().Span.FilePath, - Range = new Range(node.Tokens.First().Span.Start.Line - 1, node.Tokens.First().Span.Start.Column - 1, node.Tokens.Last().Span.End.Line - 1, node.Tokens.Last().Span.End.Column - 1) + Uri = node.Tokens.First().Span.SourcePath, + Range = new Range(node.Tokens.First().Span.StartLine - 1, node.Tokens.First().Span.StartColumn - 1, node.Tokens.Last().Span.EndLine - 1, node.Tokens.Last().Span.EndColumn - 1) }; } @@ -25,20 +25,17 @@ public static class AstExtensions { return new Location { - Uri = token.Span.FilePath, - Range = new Range(token.Span.Start.Line - 1, token.Span.Start.Column - 1, token.Span.End.Line - 1, token.Span.End.Column - 1) + Uri = token.Span.SourcePath, + Range = new Range(token.Span.StartLine - 1, token.Span.StartColumn - 1, token.Span.EndLine - 1, token.Span.EndColumn - 1) }; } public static bool ContainsPosition(this Token token, int line, int character) { - var start = token.Span.Start; - var end = token.Span.End; - - var startLine = start.Line - 1; - var startChar = start.Column - 1; - var endLine = end.Line - 1; - var endChar = end.Column - 1; + var startLine = token.Span.StartLine - 1; + var startChar = token.Span.StartColumn - 1; + var endLine = token.Span.EndLine - 1; + var endChar = token.Span.EndColumn - 1; if (line < startLine || line > endLine) return false; @@ -69,13 +66,12 @@ public static class AstExtensions return false; } - var start = node.Tokens.First().Span.Start; - var end = node.Tokens.Last().Span.End; + var span = node.Tokens.First().Span; - var startLine = start.Line - 1; - var startChar = start.Column - 1; - var endLine = end.Line - 1; - var endChar = end.Column - 1; + var startLine = span.StartLine - 1; + var startChar = span.StartColumn - 1; + var endLine = span.EndLine - 1; + var endChar = span.EndColumn - 1; if (line < startLine || line > endLine) return false; @@ -111,8 +107,8 @@ public static class AstExtensions return compilationUnit .SelectMany(x => x.DescendantsAndSelf()) .Where(n => n.ContainsPosition(line, character)) - .OrderBy(n => n.Tokens.First().Span.Start.Line) - .ThenBy(n => n.Tokens.First().Span.Start.Column) + .OrderBy(n => n.Tokens.First().Span.StartLine) + .ThenBy(n => n.Tokens.First().Span.StartColumn) .LastOrDefault(); } } \ No newline at end of file diff --git a/compiler/NubLang.LSP/DiagnosticsPublisher.cs b/compiler/NubLang.LSP/DiagnosticsPublisher.cs index 7364396..5d2fcdd 100644 --- a/compiler/NubLang.LSP/DiagnosticsPublisher.cs +++ b/compiler/NubLang.LSP/DiagnosticsPublisher.cs @@ -37,7 +37,7 @@ public class DiagnosticsPublisher }, Message = $"{nubDiagnostic.Message}\n{(nubDiagnostic.Help == null ? "" : $"help: {nubDiagnostic.Help}")}", Range = nubDiagnostic.Span.HasValue - ? new Range(nubDiagnostic.Span.Value.Start.Line - 1, nubDiagnostic.Span.Value.Start.Column - 1, nubDiagnostic.Span.Value.End.Line - 1, nubDiagnostic.Span.Value.End.Column - 1) + ? new Range(nubDiagnostic.Span.Value.StartLine - 1, nubDiagnostic.Span.Value.StartColumn - 1, nubDiagnostic.Span.Value.EndLine - 1, nubDiagnostic.Span.Value.EndColumn - 1) : new Range(), }; } diff --git a/compiler/NubLang/Ast/TypeChecker.cs b/compiler/NubLang/Ast/TypeChecker.cs index bf747fc..8b96de6 100644 --- a/compiler/NubLang/Ast/TypeChecker.cs +++ b/compiler/NubLang/Ast/TypeChecker.cs @@ -122,7 +122,7 @@ public sealed class TypeChecker { throw new CompileException(Diagnostic .Error($"Type {value.Type} is not assignable to {field.Type} for field {field.NameToken.Value}") - .At(field) + .At(field, _syntaxTree.Tokens) .Build()); } } @@ -144,7 +144,7 @@ public sealed class TypeChecker { throw new CompileException(Diagnostic .Error($"Cannot assign {value.Type} to {target.Type}") - .At(statement.Value) + .At(statement.Value, _syntaxTree.Tokens) .Build()); } @@ -184,7 +184,7 @@ public sealed class TypeChecker return expression switch { FuncCallNode funcCall => new StatementFuncCallNode(statement.Tokens, funcCall), - _ => throw new CompileException(Diagnostic.Error("Expressions statements can only be function calls").At(statement).Build()) + _ => throw new CompileException(Diagnostic.Error("Expressions statements can only be function calls").At(statement, _syntaxTree.Tokens).Build()) }; } @@ -210,7 +210,7 @@ public sealed class TypeChecker { throw new CompileException(Diagnostic .Error($"Cannot assign {assignmentNode.Type} to variable of type {type}") - .At(statement.Assignment) + .At(statement.Assignment, _syntaxTree.Tokens) .Build()); } } @@ -219,7 +219,7 @@ public sealed class TypeChecker { throw new CompileException(Diagnostic .Error($"Cannot infer type of variable {statement.NameToken.Value}") - .At(statement) + .At(statement, _syntaxTree.Tokens) .Build()); } @@ -274,7 +274,7 @@ public sealed class TypeChecker { throw new CompileException(Diagnostic .Error($"Cannot iterate over type {target.Type} which does not have size information") - .At(forSyntax.Target) + .At(forSyntax.Target, _syntaxTree.Tokens) .Build()); } } @@ -337,7 +337,7 @@ public sealed class TypeChecker { throw new CompileException(Diagnostic .Error("Unable to infer target type of cast") - .At(expression) + .At(expression, _syntaxTree.Tokens) .WithHelp("Specify target type where value is used") .Build()); } @@ -348,7 +348,7 @@ public sealed class TypeChecker { Diagnostics.Add(Diagnostic .Warning("Target type of cast is same as the value. Cast is unnecessary") - .At(expression) + .At(expression, _syntaxTree.Tokens) .Build()); return value; @@ -452,7 +452,7 @@ public sealed class TypeChecker { throw new CompileException(Diagnostic .Error("Array indexer must be of type int") - .At(expression.Index) + .At(expression.Index, _syntaxTree.Tokens) .Build()); } @@ -463,7 +463,7 @@ public sealed class TypeChecker NubArrayType arrayType => new ArrayIndexAccessNode(expression.Tokens, arrayType.ElementType, target, index), NubConstArrayType constArrayType => new ConstArrayIndexAccessNode(expression.Tokens, constArrayType.ElementType, target, index), NubSliceType sliceType => new SliceIndexAccessNode(expression.Tokens, sliceType.ElementType, target, index), - _ => throw new CompileException(Diagnostic.Error($"Cannot use array indexer on type {target.Type}").At(expression).Build()) + _ => throw new CompileException(Diagnostic.Error($"Cannot use array indexer on type {target.Type}").At(expression, _syntaxTree.Tokens).Build()) }; } @@ -490,7 +490,7 @@ public sealed class TypeChecker { throw new CompileException(Diagnostic .Error("Unable to infer type of array initializer") - .At(expression) + .At(expression, _syntaxTree.Tokens) .WithHelp("Provide a type for a variable assignment") .Build()); } @@ -503,7 +503,7 @@ public sealed class TypeChecker { throw new CompileException(Diagnostic .Error("Value in array initializer is not the same as the array type") - .At(valueExpression) + .At(valueExpression, _syntaxTree.Tokens) .Build()); } @@ -548,7 +548,7 @@ public sealed class TypeChecker { throw new CompileException(Diagnostic .Error("Equal and not equal operators must must be used with int, float or bool types") - .At(expression.Left) + .At(expression.Left, _syntaxTree.Tokens) .Build()); } @@ -557,7 +557,7 @@ public sealed class TypeChecker { throw new CompileException(Diagnostic .Error($"Expected type {left.Type} from left side of binary expression, but got {right.Type}") - .At(expression.Right) + .At(expression.Right, _syntaxTree.Tokens) .Build()); } @@ -573,7 +573,7 @@ public sealed class TypeChecker { throw new CompileException(Diagnostic .Error("Greater than and less than operators must must be used with int or float types") - .At(expression.Left) + .At(expression.Left, _syntaxTree.Tokens) .Build()); } @@ -582,7 +582,7 @@ public sealed class TypeChecker { throw new CompileException(Diagnostic .Error($"Expected type {left.Type} from left side of binary expression, but got {right.Type}") - .At(expression.Right) + .At(expression.Right, _syntaxTree.Tokens) .Build()); } @@ -596,7 +596,7 @@ public sealed class TypeChecker { throw new CompileException(Diagnostic .Error("Logical and/or must must be used with bool types") - .At(expression.Left) + .At(expression.Left, _syntaxTree.Tokens) .Build()); } @@ -605,7 +605,7 @@ public sealed class TypeChecker { throw new CompileException(Diagnostic .Error($"Expected type {left.Type} from left side of binary expression, but got {right.Type}") - .At(expression.Right) + .At(expression.Right, _syntaxTree.Tokens) .Build()); } @@ -618,7 +618,7 @@ public sealed class TypeChecker { throw new CompileException(Diagnostic .Error("The plus operator must only be used with int and float types") - .At(expression.Left) + .At(expression.Left, _syntaxTree.Tokens) .Build()); } @@ -643,7 +643,7 @@ public sealed class TypeChecker { throw new CompileException(Diagnostic .Error("Math operators must be used with int or float types") - .At(expression.Left) + .At(expression.Left, _syntaxTree.Tokens) .Build()); } @@ -652,7 +652,7 @@ public sealed class TypeChecker { throw new CompileException(Diagnostic .Error($"Expected type {left.Type} from left side of binary expression, but got {right.Type}") - .At(expression.Right) + .At(expression.Right, _syntaxTree.Tokens) .Build()); } @@ -666,7 +666,7 @@ public sealed class TypeChecker { throw new CompileException(Diagnostic .Error("Bitwise operators must be used with int types") - .At(expression.Left) + .At(expression.Left, _syntaxTree.Tokens) .Build()); } @@ -675,7 +675,7 @@ public sealed class TypeChecker { throw new CompileException(Diagnostic .Error("Bitwise operators must be used with int types") - .At(expression.Right) + .At(expression.Right, _syntaxTree.Tokens) .Build()); } @@ -690,7 +690,7 @@ public sealed class TypeChecker { throw new CompileException(Diagnostic .Error("Bitwise operators must be used with int types") - .At(expression.Left) + .At(expression.Left, _syntaxTree.Tokens) .Build()); } @@ -699,7 +699,7 @@ public sealed class TypeChecker { throw new CompileException(Diagnostic .Error($"Expected type {left.Type} from left side of binary expression, but got {right.Type}") - .At(expression.Right) + .At(expression.Right, _syntaxTree.Tokens) .Build()); } @@ -723,7 +723,7 @@ public sealed class TypeChecker { throw new CompileException(Diagnostic .Error("Negation operator must be used with signed integer or float types") - .At(expression) + .At(expression, _syntaxTree.Tokens) .Build()); } @@ -736,7 +736,7 @@ public sealed class TypeChecker { throw new CompileException(Diagnostic .Error("Invert operator must be used with booleans") - .At(expression) + .At(expression, _syntaxTree.Tokens) .Build()); } @@ -756,7 +756,7 @@ public sealed class TypeChecker { throw new CompileException(Diagnostic .Error($"Cannot dereference non-pointer type {target.Type}") - .At(expression) + .At(expression, _syntaxTree.Tokens) .Build()); } @@ -768,14 +768,14 @@ public sealed class TypeChecker var accessor = CheckExpression(expression.Expression); if (accessor.Type is not NubFuncType funcType) { - throw new CompileException(Diagnostic.Error($"Cannot call non-function type {accessor.Type}").At(expression.Expression).Build()); + throw new CompileException(Diagnostic.Error($"Cannot call non-function type {accessor.Type}").At(expression.Expression, _syntaxTree.Tokens).Build()); } if (expression.Parameters.Count != funcType.Parameters.Count) { throw new CompileException(Diagnostic .Error($"Function {funcType} expects {funcType.Parameters.Count} parameters but got {expression.Parameters.Count}") - .At(expression.Parameters.LastOrDefault(expression)) + .At(expression.Parameters.LastOrDefault(expression), _syntaxTree.Tokens) .Build()); } @@ -790,7 +790,7 @@ public sealed class TypeChecker { throw new CompileException(Diagnostic .Error($"Parameter {i + 1} does not match the type {expectedParameterType} for function {funcType}") - .At(parameter) + .At(parameter, _syntaxTree.Tokens) .Build()); } @@ -817,7 +817,7 @@ public sealed class TypeChecker throw new CompileException(Diagnostic .Error($"There is no identifier named {expression.NameToken.Value}") - .At(expression) + .At(expression, _syntaxTree.Tokens) .Build()); } @@ -834,7 +834,7 @@ public sealed class TypeChecker throw new CompileException(Diagnostic .Error($"Module {expression.ModuleToken.Value} does not export a member named {expression.NameToken.Value}") - .At(expression) + .At(expression, _syntaxTree.Tokens) .Build()); } } @@ -909,7 +909,7 @@ public sealed class TypeChecker { throw new CompileException(Diagnostic .Error($"Struct {target.Type} does not have a field with the name {expression.MemberToken.Value}") - .At(expression) + .At(expression, _syntaxTree.Tokens) .Build()); } @@ -919,7 +919,7 @@ public sealed class TypeChecker { throw new CompileException(Diagnostic .Error($"Cannot access struct member {expression.MemberToken.Value} on type {target.Type}") - .At(expression) + .At(expression, _syntaxTree.Tokens) .Build()); } } @@ -950,7 +950,7 @@ public sealed class TypeChecker throw new CompileException(Diagnostic .Error("Cannot get implicit type of struct") .WithHelp("Specify struct type with struct {type_name} syntax") - .At(expression) + .At(expression, _syntaxTree.Tokens) .Build()); } @@ -963,7 +963,7 @@ public sealed class TypeChecker { Diagnostics.AddRange(Diagnostic .Error($"Struct {structType.Name} does not have a field named {initializer.Key}") - .At(initializer.Value) + .At(initializer.Value, _syntaxTree.Tokens) .Build()); continue; @@ -981,7 +981,7 @@ public sealed class TypeChecker { Diagnostics.Add(Diagnostic .Warning($"Fields {string.Join(", ", missingFields)} are not initialized") - .At(expression) + .At(expression, _syntaxTree.Tokens) .Build()); } @@ -1065,7 +1065,7 @@ public sealed class TypeChecker throw new CompileException(Diagnostic .Error($"Type {customType.NameToken.Value} not found in module {module.Name}") - .At(customType) + .At(customType, _syntaxTree.Tokens) .Build()); } diff --git a/compiler/NubLang/Diagnostics/Diagnostic.cs b/compiler/NubLang/Diagnostics/Diagnostic.cs index c8c0bc5..66142ed 100644 --- a/compiler/NubLang/Diagnostics/Diagnostic.cs +++ b/compiler/NubLang/Diagnostics/Diagnostic.cs @@ -11,6 +11,7 @@ public class Diagnostic private readonly string _message; private SourceSpan? _span; private string? _help; + private List? _tokens; public DiagnosticBuilder(DiagnosticSeverity severity, string message) { @@ -18,18 +19,32 @@ public class Diagnostic _message = message; } - public DiagnosticBuilder At(SyntaxNode? node) + public DiagnosticBuilder At(SyntaxNode? node, List? tokens = null) { + if (tokens != null) + { + _tokens = tokens; + } + if (node != null) { - _span = SourceSpan.Merge(node.Tokens.Select(x => x.Span)); + var first = node.Tokens.FirstOrDefault(); + if (first != null) + { + _span = SourceSpan.Merge(node.Tokens.Select(x => x.Span)); + } } return this; } - public DiagnosticBuilder At(Token? token) + public DiagnosticBuilder At(Token? token, List? tokens = null) { + if (tokens != null) + { + _tokens = tokens; + } + if (token != null) { At(token.Span); @@ -48,11 +63,11 @@ public class Diagnostic return this; } - public DiagnosticBuilder At(string filePath, int line, int column) - { - _span = new SourceSpan(filePath, new SourceLocation(line, column), new SourceLocation(line, column)); - return this; - } + // public DiagnosticBuilder At(string filePath, int line, int column) + // { + // _span = new SourceSpan(filePath, new SourceLocation(line, column), new SourceLocation(line, column)); + // return this; + // } public DiagnosticBuilder WithHelp(string help) { @@ -60,20 +75,23 @@ public class Diagnostic return this; } - public Diagnostic Build() => new(_severity, _message, _help, _span); + public Diagnostic Build() => new(_severity, _message, _help, _span, _tokens); } public static DiagnosticBuilder Error(string message) => new(DiagnosticSeverity.Error, message); public static DiagnosticBuilder Warning(string message) => new(DiagnosticSeverity.Warning, message); public static DiagnosticBuilder Info(string message) => new(DiagnosticSeverity.Info, message); + private readonly List? _tokens; + public DiagnosticSeverity Severity { get; } public string Message { get; } public string? Help { get; } public SourceSpan? Span { get; } - private Diagnostic(DiagnosticSeverity severity, string message, string? help, SourceSpan? span) + private Diagnostic(DiagnosticSeverity severity, string message, string? help, SourceSpan? span, List? tokens) { + _tokens = tokens; Severity = severity; Message = message; Help = help; @@ -103,15 +121,12 @@ public class Diagnostic if (Span.HasValue) { sb.AppendLine(); - var text = File.ReadAllText(Span.Value.FilePath); - - var tokenizer = new Tokenizer(); - var tokens = tokenizer.Tokenize(Span.Value.FilePath, text); + var text = Span.Value.Source; var lines = text.Split('\n'); - var startLine = Span.Value.Start.Line; - var endLine = Span.Value.End.Line; + var startLine = Span.Value.StartLine; + var endLine = Span.Value.EndLine; const int CONTEXT_LINES = 3; @@ -144,8 +159,15 @@ public class Diagnostic sb.Append("│ "); sb.Append(i.ToString().PadRight(numberPadding)); sb.Append(" │ "); - sb.Append(ApplySyntaxHighlighting(line.PadRight(codePadding), i, tokens)); - // sb.Append(line.PadRight(codePadding)); + if (_tokens != null) + { + sb.Append(ApplySyntaxHighlighting(line.PadRight(codePadding), i, _tokens)); + } + else + { + sb.Append(line.PadRight(codePadding)); + } + sb.Append(" │"); sb.AppendLine(); @@ -156,12 +178,12 @@ public class Diagnostic if (i == startLine) { - markerStartColumn = Span.Value.Start.Column; + markerStartColumn = Span.Value.StartColumn; } if (i == endLine) { - markerEndColumn = Span.Value.End.Column; + markerEndColumn = Span.Value.EndColumn; } var markerLength = markerEndColumn - markerStartColumn; @@ -206,8 +228,8 @@ public class Diagnostic { var sb = new StringBuilder(); var lineTokens = tokens - .Where(t => t.Span.Start.Line == lineNumber) - .OrderBy(t => t.Span.Start.Column) + .Where(t => t.Span.StartLine == lineNumber) + .OrderBy(t => t.Span.StartColumn) .ToList(); if (lineTokens.Count == 0) @@ -219,8 +241,10 @@ public class Diagnostic foreach (var token in lineTokens) { - var tokenStart = token.Span.Start.Column; - var tokenEnd = token.Span.End.Column; + if (token is WhitespaceToken) continue; + + var tokenStart = token.Span.StartColumn; + var tokenEnd = token.Span.EndColumn; if (tokenStart > currentColumn && currentColumn - 1 < line.Length) { diff --git a/compiler/NubLang/Diagnostics/SourceSpan.cs b/compiler/NubLang/Diagnostics/SourceSpan.cs index 121fe6e..c06b810 100644 --- a/compiler/NubLang/Diagnostics/SourceSpan.cs +++ b/compiler/NubLang/Diagnostics/SourceSpan.cs @@ -1,112 +1,56 @@ namespace NubLang.Diagnostics; -public readonly struct SourceSpan : IEquatable, IComparable +public readonly struct SourceSpan { + private readonly int _startIndex; + private readonly int _endIndex; + public static SourceSpan Merge(params IEnumerable spans) { var spanArray = spans as SourceSpan[] ?? spans.ToArray(); if (spanArray.Length == 0) { - return new SourceSpan(string.Empty, new SourceLocation(0, 0), new SourceLocation(0, 0)); + return new SourceSpan(string.Empty, string.Empty, 0, 0, 0, 0, 0, 0); } - var minStart = spanArray.Min(s => s.Start); - var maxEnd = spanArray.Max(s => s.End); + var first = spanArray.MinBy(x => x._startIndex); + var last = spanArray.MaxBy(x => x._endIndex); - return new SourceSpan(spanArray[0].FilePath, minStart, maxEnd); + return new SourceSpan(first.SourcePath, first.Source, first._startIndex, last._endIndex, first.StartLine, last.EndLine, first.StartColumn, last.EndColumn); } - public SourceSpan(string filePath, SourceLocation start, SourceLocation end) + public SourceSpan(string sourcePath, string source, int startIndex, int endIndex, int startLine, int startColumn, int endLine, int endColumn) { - if (start > end) - { - throw new ArgumentException("Start location cannot be after end location"); - } - - FilePath = filePath; - Start = start; - End = end; + _startIndex = startIndex; + _endIndex = endIndex; + SourcePath = sourcePath; + Source = source; + StartLine = startLine; + StartColumn = startColumn; + EndLine = endLine; + EndColumn = endColumn; } - public string FilePath { get; } - public SourceLocation Start { get; } - public SourceLocation End { get; } + public int StartLine { get; } + public int StartColumn { get; } + public int EndLine { get; } + public int EndColumn { get; } + + public string SourcePath { get; } + public string Source { get; } public override string ToString() { - if (Start == End) + if (StartLine == EndLine && StartColumn == EndColumn) { - return $"{FilePath}:{Start}"; + return $"{SourcePath}:{StartColumn}:{StartColumn}"; } - if (Start.Line == End.Line) + if (StartLine == EndLine) { - return Start.Column == End.Column ? $"{FilePath}:{Start}" : $"{FilePath}:{Start.Line}:{Start.Column}-{End.Column}"; + return $"{SourcePath}:{StartLine}:{StartColumn}-{EndColumn}"; } - return $"{FilePath}:{Start}-{End}"; - } - - public bool Equals(SourceSpan other) => Start == other.Start && End == other.End; - public override bool Equals(object? obj) => obj is SourceSpan other && Equals(other); - public override int GetHashCode() => HashCode.Combine(typeof(SourceSpan), Start, End); - - public static bool operator ==(SourceSpan left, SourceSpan right) => Equals(left, right); - public static bool operator !=(SourceSpan left, SourceSpan right) => !Equals(left, right); - - public static bool operator <(SourceSpan left, SourceSpan right) => left.CompareTo(right) < 0; - public static bool operator <=(SourceSpan left, SourceSpan right) => left.CompareTo(right) <= 0; - public static bool operator >(SourceSpan left, SourceSpan right) => left.CompareTo(right) > 0; - public static bool operator >=(SourceSpan left, SourceSpan right) => left.CompareTo(right) >= 0; - - public int CompareTo(SourceSpan other) - { - var startComparison = Start.CompareTo(other.Start); - return startComparison != 0 ? startComparison : End.CompareTo(other.End); - } -} - -public readonly struct SourceLocation : IEquatable, IComparable -{ - public SourceLocation(int line, int column) - { - Line = line; - Column = column; - } - - public int Line { get; } - public int Column { get; } - - public override string ToString() - { - return $"{Line}:{Column}"; - } - - public override bool Equals(object? obj) - { - return obj is SourceLocation other && Equals(other); - } - - public bool Equals(SourceLocation other) - { - return Line == other.Line && Column == other.Column; - } - - public override int GetHashCode() - { - return HashCode.Combine(typeof(SourceLocation), Line, Column); - } - - public static bool operator ==(SourceLocation left, SourceLocation right) => Equals(left, right); - public static bool operator !=(SourceLocation left, SourceLocation right) => !Equals(left, right); - public static bool operator <(SourceLocation left, SourceLocation right) => left.Line < right.Line || (left.Line == right.Line && left.Column < right.Column); - public static bool operator >(SourceLocation left, SourceLocation right) => left.Line > right.Line || (left.Line == right.Line && left.Column > right.Column); - public static bool operator <=(SourceLocation left, SourceLocation right) => left.Line <= right.Line || (left.Line == right.Line && left.Column <= right.Column); - public static bool operator >=(SourceLocation left, SourceLocation right) => left.Line >= right.Line || (left.Line == right.Line && left.Column >= right.Column); - - public int CompareTo(SourceLocation other) - { - var lineComparison = Line.CompareTo(other.Line); - return lineComparison != 0 ? lineComparison : Column.CompareTo(other.Column); + return $"{SourcePath}:{StartLine}:{StartColumn}-{EndLine}:{EndColumn}"; } } \ No newline at end of file diff --git a/compiler/NubLang/Syntax/Parser.cs b/compiler/NubLang/Syntax/Parser.cs index 1bb36d4..b5c5873 100644 --- a/compiler/NubLang/Syntax/Parser.cs +++ b/compiler/NubLang/Syntax/Parser.cs @@ -62,7 +62,7 @@ public sealed class Parser _ => throw new CompileException(Diagnostic .Error($"Expected 'func', 'struct', 'enum', 'import' or 'module' but found '{keyword.Symbol}'") .WithHelp("Valid top level statements are 'func', 'struct', 'enum', 'import' and 'module'") - .At(keyword) + .At(keyword, _tokens) .Build()) }; @@ -83,7 +83,7 @@ public sealed class Parser } } - return new SyntaxTree(topLevelSyntaxNodes); + return new SyntaxTree(topLevelSyntaxNodes, _tokens); } private ModuleSyntax ParseModule(int startIndex) @@ -190,7 +190,7 @@ public sealed class Parser { throw new CompileException(Diagnostic .Error("Value of enum field must be an integer literal") - .At(CurrentToken) + .At(CurrentToken, _tokens) .Build()); } @@ -461,7 +461,7 @@ public sealed class Parser IdentifierToken identifier => ParseIdentifier(startIndex, identifier), SymbolToken symbolToken => symbolToken.Symbol switch { - Symbol.Ampersand => ParseAddressOf(startIndex), + Symbol.Caret => ParseAddressOf(startIndex), Symbol.OpenParen => ParseParenthesizedExpression(), Symbol.Minus => ParseUnaryNegate(startIndex), Symbol.Bang => ParseUnaryInvert(startIndex), @@ -472,13 +472,13 @@ public sealed class Parser _ => throw new CompileException(Diagnostic .Error($"Unexpected symbol '{symbolToken.Symbol}' in expression") .WithHelp("Expected '(', '-', '!', '[' or '{'") - .At(symbolToken) + .At(symbolToken, _tokens) .Build()) }, _ => throw new CompileException(Diagnostic .Error($"Unexpected token '{token.GetType().Name}' in expression") .WithHelp("Expected literal, identifier, or parenthesized expression") - .At(token) + .At(token, _tokens) .Build()) }; @@ -698,7 +698,7 @@ public sealed class Parser throw new CompileException(Diagnostic .Error("Arbitrary uint size is not supported") .WithHelp("Use u8, u16, u32 or u64") - .At(name) + .At(name, _tokens) .Build()); } @@ -712,7 +712,7 @@ public sealed class Parser throw new CompileException(Diagnostic .Error("Arbitrary int size is not supported") .WithHelp("Use i8, i16, i32 or i64") - .At(name) + .At(name, _tokens) .Build()); } @@ -726,7 +726,7 @@ public sealed class Parser throw new CompileException(Diagnostic .Error("Arbitrary float size is not supported") .WithHelp("Use f32 or f64") - .At(name) + .At(name, _tokens) .Build()); } @@ -810,7 +810,7 @@ public sealed class Parser throw new CompileException(Diagnostic .Error("Invalid type syntax") .WithHelp("Expected type name, '^' for pointer, or '[]' for array") - .At(CurrentToken) + .At(CurrentToken, _tokens) .Build()); } @@ -821,7 +821,7 @@ public sealed class Parser throw new CompileException(Diagnostic .Error("Unexpected end of file") .WithHelp("Expected more tokens to complete the syntax") - .At(_tokens[^1]) + .At(_tokens[^1], _tokens) .Build()); } @@ -838,7 +838,7 @@ public sealed class Parser throw new CompileException(Diagnostic .Error($"Expected symbol, but found {token.GetType().Name}") .WithHelp("This position requires a symbol like '(', ')', '{', '}', etc.") - .At(token) + .At(token, _tokens) .Build()); } @@ -853,7 +853,7 @@ public sealed class Parser throw new CompileException(Diagnostic .Error($"Expected '{expectedSymbol}', but found '{token.Symbol}'") .WithHelp($"Insert '{expectedSymbol}' here") - .At(token) + .At(token, _tokens) .Build()); } } @@ -903,7 +903,7 @@ public sealed class Parser throw new CompileException(Diagnostic .Error($"Expected identifier, but found {token.GetType().Name}") .WithHelp("Provide a valid identifier name here") - .At(token) + .At(token, _tokens) .Build()); } @@ -931,7 +931,7 @@ public sealed class Parser throw new CompileException(Diagnostic .Error($"Expected string literal, but found {token.GetType().Name}") .WithHelp("Provide a valid string literal") - .At(token) + .At(token, _tokens) .Build()); } @@ -953,4 +953,4 @@ public sealed class Parser } } -public record SyntaxTree(List TopLevelSyntaxNodes); \ No newline at end of file +public record SyntaxTree(List TopLevelSyntaxNodes, List Tokens); \ No newline at end of file diff --git a/compiler/NubLang/Syntax/Tokenizer.cs b/compiler/NubLang/Syntax/Tokenizer.cs index b73abd8..8d8e01d 100644 --- a/compiler/NubLang/Syntax/Tokenizer.cs +++ b/compiler/NubLang/Syntax/Tokenizer.cs @@ -43,6 +43,7 @@ public sealed class Tokenizer private Token ParseToken() { + var indexStart = _index; var lineStart = _line; var columnStart = _column; @@ -53,7 +54,7 @@ public sealed class Tokenizer Next(); } - return new WhitespaceToken(CreateSpan(lineStart, columnStart)); + return new WhitespaceToken(CreateSpan(indexStart, lineStart, columnStart)); } if (_content[_index] == '/' && _index + 1 < _content.Length && _content[_index + 1] == '/') @@ -66,23 +67,23 @@ public sealed class Tokenizer Next(); } - return new CommentToken(CreateSpan(lineStart, columnStart), _content.AsSpan(startIndex, _index - startIndex).ToString()); + return new CommentToken(CreateSpan(indexStart, lineStart, columnStart), _content.AsSpan(startIndex, _index - startIndex).ToString()); } if (char.IsDigit(_content[_index])) { - return ParseNumber(lineStart, columnStart); + return ParseNumber(indexStart, lineStart, columnStart); } if (_content[_index] == '"') { - return ParseString(lineStart, columnStart); + return ParseString(indexStart, lineStart, columnStart); } // note(nub31): Look for keywords (longest first in case a keyword fits partially in a larger keyword) for (var i = 8; i >= 1; i--) { - if (TryMatchSymbol(i, lineStart, columnStart, out var token)) + if (TryMatchSymbol(i, indexStart, lineStart, columnStart, out var token)) { return token; } @@ -90,13 +91,13 @@ public sealed class Tokenizer if (char.IsLetter(_content[_index]) || _content[_index] == '_') { - return ParseIdentifier(lineStart, columnStart); + return ParseIdentifier(indexStart, lineStart, columnStart); } throw new CompileException(Diagnostic.Error($"Unknown token '{_content[_index]}'").Build()); } - private Token ParseNumber(int lineStart, int columnStart) + private Token ParseNumber(int indexStart, int lineStart, int columnStart) { var start = _index; var current = _content[_index]; @@ -116,12 +117,12 @@ public sealed class Tokenizer { throw new CompileException(Diagnostic .Error("Invalid hex literal, no digits found") - .At(_fileName, _line, _column) + .At(CreateSpan(_index, _line, _column)) .Build()); } return new IntLiteralToken( - CreateSpan(lineStart, columnStart), + CreateSpan(indexStart, lineStart, columnStart), _content.Substring(start, _index - start), 16); } @@ -141,12 +142,12 @@ public sealed class Tokenizer { throw new CompileException(Diagnostic .Error("Invalid binary literal, no digits found") - .At(_fileName, _line, _column) + .At(CreateSpan(_index, _line, _column)) .Build()); } return new IntLiteralToken( - CreateSpan(lineStart, columnStart), + CreateSpan(indexStart, lineStart, columnStart), _content.Substring(start, _index - start), 2); } @@ -163,7 +164,7 @@ public sealed class Tokenizer { throw new CompileException(Diagnostic .Error("More than one period found in float literal") - .At(_fileName, _line, _column) + .At(CreateSpan(_index, _line, _column)) .Build()); } @@ -183,11 +184,11 @@ public sealed class Tokenizer var buffer = _content.Substring(start, _index - start); return isFloat - ? new FloatLiteralToken(CreateSpan(lineStart, columnStart), buffer) - : new IntLiteralToken(CreateSpan(lineStart, columnStart), buffer, 10); + ? new FloatLiteralToken(CreateSpan(indexStart, lineStart, columnStart), buffer) + : new IntLiteralToken(CreateSpan(indexStart, lineStart, columnStart), buffer, 10); } - private StringLiteralToken ParseString(int lineStart, int columnStart) + private StringLiteralToken ParseString(int indexStart, int lineStart, int columnStart) { Next(); var start = _index; @@ -198,7 +199,7 @@ public sealed class Tokenizer { throw new CompileException(Diagnostic .Error("Unclosed string literal") - .At(_fileName, _line, _column) + .At(CreateSpan(_index, _line, _column)) .Build()); } @@ -208,7 +209,7 @@ public sealed class Tokenizer { throw new CompileException(Diagnostic .Error("Unclosed string literal (newline found)") - .At(_fileName, _line, _column) + .At(CreateSpan(_index, _line, _column)) .Build()); } @@ -216,14 +217,14 @@ public sealed class Tokenizer { var buffer = _content.Substring(start, _index - start); Next(); - return new StringLiteralToken(CreateSpan(lineStart, columnStart), buffer); + return new StringLiteralToken(CreateSpan(indexStart, lineStart, columnStart), buffer); } Next(); } } - private bool TryMatchSymbol(int length, int lineStart, int columnStart, out Token token) + private bool TryMatchSymbol(int length, int indexStart, int lineStart, int columnStart, out Token token) { token = null!; @@ -237,14 +238,14 @@ public sealed class Tokenizer if (span is "true") { Next(4); - token = new BoolLiteralToken(CreateSpan(lineStart, columnStart), true); + token = new BoolLiteralToken(CreateSpan(indexStart, lineStart, columnStart), true); return true; } if (span is "false") { Next(5); - token = new BoolLiteralToken(CreateSpan(lineStart, columnStart), false); + token = new BoolLiteralToken(CreateSpan(indexStart, lineStart, columnStart), false); return true; } @@ -349,14 +350,14 @@ public sealed class Tokenizer } Next(length); - token = new SymbolToken(CreateSpan(lineStart, columnStart), symbol); + token = new SymbolToken(CreateSpan(indexStart, lineStart, columnStart), symbol); return true; } return false; } - private IdentifierToken ParseIdentifier(int lineStart, int columnStart) + private IdentifierToken ParseIdentifier(int indexStart, int lineStart, int columnStart) { var start = _index; @@ -373,12 +374,12 @@ public sealed class Tokenizer } } - return new IdentifierToken(CreateSpan(lineStart, columnStart), _content.Substring(start, _index - start)); + return new IdentifierToken(CreateSpan(indexStart, lineStart, columnStart), _content.Substring(start, _index - start)); } - private SourceSpan CreateSpan(int lineStart, int columnStart) + private SourceSpan CreateSpan(int indexStart, int lineStart, int columnStart) { - return new SourceSpan(_fileName, new SourceLocation(lineStart, columnStart), new SourceLocation(_line, _column)); + return new SourceSpan(_fileName, _content, indexStart, Math.Min(_index, _content.Length), lineStart, columnStart, _line, _column); } private void Next(int count = 1)