...

compiler/NubLang.LSP/AstExtensions.cs

@@ -16,8 +16,8 @@ public static class AstExtensions
 
         return new Location
         {
-            Uri = node.Tokens.First().Span.FilePath,
-            Range = new Range(node.Tokens.First().Span.Start.Line - 1, node.Tokens.First().Span.Start.Column - 1, node.Tokens.Last().Span.End.Line - 1, node.Tokens.Last().Span.End.Column - 1)
+            Uri = node.Tokens.First().Span.SourcePath,
+            Range = new Range(node.Tokens.First().Span.StartLine - 1, node.Tokens.First().Span.StartColumn - 1, node.Tokens.Last().Span.EndLine - 1, node.Tokens.Last().Span.EndColumn - 1)
         };
     }
 
@@ -25,20 +25,17 @@ public static class AstExtensions
     {
         return new Location
         {
-            Uri = token.Span.FilePath,
-            Range = new Range(token.Span.Start.Line - 1, token.Span.Start.Column - 1, token.Span.End.Line - 1, token.Span.End.Column - 1)
+            Uri = token.Span.SourcePath,
+            Range = new Range(token.Span.StartLine - 1, token.Span.StartColumn - 1, token.Span.EndLine - 1, token.Span.EndColumn - 1)
         };
     }
 
     public static bool ContainsPosition(this Token token, int line, int character)
     {
-        var start = token.Span.Start;
-        var end = token.Span.End;
-
-        var startLine = start.Line - 1;
-        var startChar = start.Column - 1;
-        var endLine = end.Line - 1;
-        var endChar = end.Column - 1;
+        var startLine = token.Span.StartLine - 1;
+        var startChar = token.Span.StartColumn - 1;
+        var endLine = token.Span.EndLine - 1;
+        var endChar = token.Span.EndColumn - 1;
 
         if (line < startLine || line > endLine) return false;
 
@@ -69,13 +66,12 @@ public static class AstExtensions
             return false;
         }
 
-        var start = node.Tokens.First().Span.Start;
-        var end = node.Tokens.Last().Span.End;
+        var span = node.Tokens.First().Span;
 
-        var startLine = start.Line - 1;
-        var startChar = start.Column - 1;
-        var endLine = end.Line - 1;
-        var endChar = end.Column - 1;
+        var startLine = span.StartLine - 1;
+        var startChar = span.StartColumn - 1;
+        var endLine = span.EndLine - 1;
+        var endChar = span.EndColumn - 1;
 
         if (line < startLine || line > endLine) return false;
 
@@ -111,8 +107,8 @@ public static class AstExtensions
         return compilationUnit
             .SelectMany(x => x.DescendantsAndSelf())
             .Where(n => n.ContainsPosition(line, character))
-            .OrderBy(n => n.Tokens.First().Span.Start.Line)
-            .ThenBy(n => n.Tokens.First().Span.Start.Column)
+            .OrderBy(n => n.Tokens.First().Span.StartLine)
+            .ThenBy(n => n.Tokens.First().Span.StartColumn)
             .LastOrDefault();
     }
 }

compiler/NubLang.LSP/DiagnosticsPublisher.cs

@@ -37,7 +37,7 @@ public class DiagnosticsPublisher
             },
             Message = $"{nubDiagnostic.Message}\n{(nubDiagnostic.Help == null ? "" : $"help: {nubDiagnostic.Help}")}",
             Range = nubDiagnostic.Span.HasValue
-                ? new Range(nubDiagnostic.Span.Value.Start.Line - 1, nubDiagnostic.Span.Value.Start.Column - 1, nubDiagnostic.Span.Value.End.Line - 1, nubDiagnostic.Span.Value.End.Column - 1)
+                ? new Range(nubDiagnostic.Span.Value.StartLine - 1, nubDiagnostic.Span.Value.StartColumn - 1, nubDiagnostic.Span.Value.EndLine - 1, nubDiagnostic.Span.Value.EndColumn - 1)
                 : new Range(),
         };
     }

compiler/NubLang/Ast/TypeChecker.cs

@@ -122,7 +122,7 @@ public sealed class TypeChecker
                 {
                     throw new CompileException(Diagnostic
                         .Error($"Type {value.Type} is not assignable to {field.Type} for field {field.NameToken.Value}")
-                        .At(field)
+                        .At(field, _syntaxTree.Tokens)
                         .Build());
                 }
             }
@@ -144,7 +144,7 @@ public sealed class TypeChecker
         {
             throw new CompileException(Diagnostic
                 .Error($"Cannot assign {value.Type} to {target.Type}")
-                .At(statement.Value)
+                .At(statement.Value, _syntaxTree.Tokens)
                 .Build());
         }
 
@@ -184,7 +184,7 @@ public sealed class TypeChecker
         return expression switch
         {
             FuncCallNode funcCall => new StatementFuncCallNode(statement.Tokens, funcCall),
-            _ => throw new CompileException(Diagnostic.Error("Expressions statements can only be function calls").At(statement).Build())
+            _ => throw new CompileException(Diagnostic.Error("Expressions statements can only be function calls").At(statement, _syntaxTree.Tokens).Build())
         };
     }
 
@@ -210,7 +210,7 @@ public sealed class TypeChecker
             {
                 throw new CompileException(Diagnostic
                     .Error($"Cannot assign {assignmentNode.Type} to variable of type {type}")
-                    .At(statement.Assignment)
+                    .At(statement.Assignment, _syntaxTree.Tokens)
                     .Build());
             }
         }
@@ -219,7 +219,7 @@ public sealed class TypeChecker
         {
             throw new CompileException(Diagnostic
                 .Error($"Cannot infer type of variable {statement.NameToken.Value}")
-                .At(statement)
+                .At(statement, _syntaxTree.Tokens)
                 .Build());
         }
 
@@ -274,7 +274,7 @@ public sealed class TypeChecker
             {
                 throw new CompileException(Diagnostic
                     .Error($"Cannot iterate over type {target.Type} which does not have size information")
-                    .At(forSyntax.Target)
+                    .At(forSyntax.Target, _syntaxTree.Tokens)
                     .Build());
             }
         }
@@ -337,7 +337,7 @@ public sealed class TypeChecker
         {
             throw new CompileException(Diagnostic
                 .Error("Unable to infer target type of cast")
-                .At(expression)
+                .At(expression, _syntaxTree.Tokens)
                 .WithHelp("Specify target type where value is used")
                 .Build());
         }
@@ -348,7 +348,7 @@ public sealed class TypeChecker
         {
             Diagnostics.Add(Diagnostic
                 .Warning("Target type of cast is same as the value. Cast is unnecessary")
-                .At(expression)
+                .At(expression, _syntaxTree.Tokens)
                 .Build());
 
             return value;
@@ -452,7 +452,7 @@ public sealed class TypeChecker
         {
             throw new CompileException(Diagnostic
                 .Error("Array indexer must be of type int")
-                .At(expression.Index)
+                .At(expression.Index, _syntaxTree.Tokens)
                 .Build());
         }
 
@@ -463,7 +463,7 @@ public sealed class TypeChecker
             NubArrayType arrayType => new ArrayIndexAccessNode(expression.Tokens, arrayType.ElementType, target, index),
             NubConstArrayType constArrayType => new ConstArrayIndexAccessNode(expression.Tokens, constArrayType.ElementType, target, index),
             NubSliceType sliceType => new SliceIndexAccessNode(expression.Tokens, sliceType.ElementType, target, index),
-            _ => throw new CompileException(Diagnostic.Error($"Cannot use array indexer on type {target.Type}").At(expression).Build())
+            _ => throw new CompileException(Diagnostic.Error($"Cannot use array indexer on type {target.Type}").At(expression, _syntaxTree.Tokens).Build())
         };
     }
 
@@ -490,7 +490,7 @@ public sealed class TypeChecker
         {
             throw new CompileException(Diagnostic
                 .Error("Unable to infer type of array initializer")
-                .At(expression)
+                .At(expression, _syntaxTree.Tokens)
                 .WithHelp("Provide a type for a variable assignment")
                 .Build());
         }
@@ -503,7 +503,7 @@ public sealed class TypeChecker
             {
                 throw new CompileException(Diagnostic
                     .Error("Value in array initializer is not the same as the array type")
-                    .At(valueExpression)
+                    .At(valueExpression, _syntaxTree.Tokens)
                     .Build());
             }
 
@@ -548,7 +548,7 @@ public sealed class TypeChecker
                 {
                     throw new CompileException(Diagnostic
                         .Error("Equal and not equal operators must must be used with int, float or bool types")
-                        .At(expression.Left)
+                        .At(expression.Left, _syntaxTree.Tokens)
                         .Build());
                 }
 
@@ -557,7 +557,7 @@ public sealed class TypeChecker
                 {
                     throw new CompileException(Diagnostic
                         .Error($"Expected type {left.Type} from left side of binary expression, but got {right.Type}")
-                        .At(expression.Right)
+                        .At(expression.Right, _syntaxTree.Tokens)
                         .Build());
                 }
 
@@ -573,7 +573,7 @@ public sealed class TypeChecker
                 {
                     throw new CompileException(Diagnostic
                         .Error("Greater than and less than operators must must be used with int or float types")
-                        .At(expression.Left)
+                        .At(expression.Left, _syntaxTree.Tokens)
                         .Build());
                 }
 
@@ -582,7 +582,7 @@ public sealed class TypeChecker
                 {
                     throw new CompileException(Diagnostic
                         .Error($"Expected type {left.Type} from left side of binary expression, but got {right.Type}")
-                        .At(expression.Right)
+                        .At(expression.Right, _syntaxTree.Tokens)
                         .Build());
                 }
 
@@ -596,7 +596,7 @@ public sealed class TypeChecker
                 {
                     throw new CompileException(Diagnostic
                         .Error("Logical and/or must must be used with bool types")
-                        .At(expression.Left)
+                        .At(expression.Left, _syntaxTree.Tokens)
                         .Build());
                 }
 
@@ -605,7 +605,7 @@ public sealed class TypeChecker
                 {
                     throw new CompileException(Diagnostic
                         .Error($"Expected type {left.Type} from left side of binary expression, but got {right.Type}")
-                        .At(expression.Right)
+                        .At(expression.Right, _syntaxTree.Tokens)
                         .Build());
                 }
 
@@ -618,7 +618,7 @@ public sealed class TypeChecker
                 {
                     throw new CompileException(Diagnostic
                         .Error("The plus operator must only be used with int and float types")
-                        .At(expression.Left)
+                        .At(expression.Left, _syntaxTree.Tokens)
                         .Build());
                 }
 
@@ -643,7 +643,7 @@ public sealed class TypeChecker
                 {
                     throw new CompileException(Diagnostic
                         .Error("Math operators must be used with int or float types")
-                        .At(expression.Left)
+                        .At(expression.Left, _syntaxTree.Tokens)
                         .Build());
                 }
 
@@ -652,7 +652,7 @@ public sealed class TypeChecker
                 {
                     throw new CompileException(Diagnostic
                         .Error($"Expected type {left.Type} from left side of binary expression, but got {right.Type}")
-                        .At(expression.Right)
+                        .At(expression.Right, _syntaxTree.Tokens)
                         .Build());
                 }
 
@@ -666,7 +666,7 @@ public sealed class TypeChecker
                 {
                     throw new CompileException(Diagnostic
                         .Error("Bitwise operators must be used with int types")
-                        .At(expression.Left)
+                        .At(expression.Left, _syntaxTree.Tokens)
                         .Build());
                 }
 
@@ -675,7 +675,7 @@ public sealed class TypeChecker
                 {
                     throw new CompileException(Diagnostic
                         .Error("Bitwise operators must be used with int types")
-                        .At(expression.Right)
+                        .At(expression.Right, _syntaxTree.Tokens)
                         .Build());
                 }
 
@@ -690,7 +690,7 @@ public sealed class TypeChecker
                 {
                     throw new CompileException(Diagnostic
                         .Error("Bitwise operators must be used with int types")
-                        .At(expression.Left)
+                        .At(expression.Left, _syntaxTree.Tokens)
                         .Build());
                 }
 
@@ -699,7 +699,7 @@ public sealed class TypeChecker
                 {
                     throw new CompileException(Diagnostic
                         .Error($"Expected type {left.Type} from left side of binary expression, but got {right.Type}")
-                        .At(expression.Right)
+                        .At(expression.Right, _syntaxTree.Tokens)
                         .Build());
                 }
 
@@ -723,7 +723,7 @@ public sealed class TypeChecker
                 {
                     throw new CompileException(Diagnostic
                         .Error("Negation operator must be used with signed integer or float types")
-                        .At(expression)
+                        .At(expression, _syntaxTree.Tokens)
                         .Build());
                 }
 
@@ -736,7 +736,7 @@ public sealed class TypeChecker
                 {
                     throw new CompileException(Diagnostic
                         .Error("Invert operator must be used with booleans")
-                        .At(expression)
+                        .At(expression, _syntaxTree.Tokens)
                         .Build());
                 }
 
@@ -756,7 +756,7 @@ public sealed class TypeChecker
         {
             throw new CompileException(Diagnostic
                 .Error($"Cannot dereference non-pointer type {target.Type}")
-                .At(expression)
+                .At(expression, _syntaxTree.Tokens)
                 .Build());
         }
 
@@ -768,14 +768,14 @@ public sealed class TypeChecker
         var accessor = CheckExpression(expression.Expression);
         if (accessor.Type is not NubFuncType funcType)
         {
-            throw new CompileException(Diagnostic.Error($"Cannot call non-function type {accessor.Type}").At(expression.Expression).Build());
+            throw new CompileException(Diagnostic.Error($"Cannot call non-function type {accessor.Type}").At(expression.Expression, _syntaxTree.Tokens).Build());
         }
 
         if (expression.Parameters.Count != funcType.Parameters.Count)
         {
             throw new CompileException(Diagnostic
                 .Error($"Function {funcType} expects {funcType.Parameters.Count} parameters but got {expression.Parameters.Count}")
-                .At(expression.Parameters.LastOrDefault(expression))
+                .At(expression.Parameters.LastOrDefault(expression), _syntaxTree.Tokens)
                 .Build());
         }
 
@@ -790,7 +790,7 @@ public sealed class TypeChecker
             {
                 throw new CompileException(Diagnostic
                     .Error($"Parameter {i + 1} does not match the type {expectedParameterType} for function {funcType}")
-                    .At(parameter)
+                    .At(parameter, _syntaxTree.Tokens)
                     .Build());
             }
 
@@ -817,7 +817,7 @@ public sealed class TypeChecker
 
         throw new CompileException(Diagnostic
             .Error($"There is no identifier named {expression.NameToken.Value}")
-            .At(expression)
+            .At(expression, _syntaxTree.Tokens)
             .Build());
     }
 
@@ -834,7 +834,7 @@ public sealed class TypeChecker
 
             throw new CompileException(Diagnostic
                 .Error($"Module {expression.ModuleToken.Value} does not export a member named {expression.NameToken.Value}")
-                .At(expression)
+                .At(expression, _syntaxTree.Tokens)
                 .Build());
         }
     }
@@ -909,7 +909,7 @@ public sealed class TypeChecker
                 {
                     throw new CompileException(Diagnostic
                         .Error($"Struct {target.Type} does not have a field with the name {expression.MemberToken.Value}")
-                        .At(expression)
+                        .At(expression, _syntaxTree.Tokens)
                         .Build());
                 }
 
@@ -919,7 +919,7 @@ public sealed class TypeChecker
             {
                 throw new CompileException(Diagnostic
                     .Error($"Cannot access struct member {expression.MemberToken.Value} on type {target.Type}")
-                    .At(expression)
+                    .At(expression, _syntaxTree.Tokens)
                     .Build());
             }
         }
@@ -950,7 +950,7 @@ public sealed class TypeChecker
             throw new CompileException(Diagnostic
                 .Error("Cannot get implicit type of struct")
                 .WithHelp("Specify struct type with struct {type_name} syntax")
-                .At(expression)
+                .At(expression, _syntaxTree.Tokens)
                 .Build());
         }
 
@@ -963,7 +963,7 @@ public sealed class TypeChecker
             {
                 Diagnostics.AddRange(Diagnostic
                     .Error($"Struct {structType.Name} does not have a field named {initializer.Key}")
-                    .At(initializer.Value)
+                    .At(initializer.Value, _syntaxTree.Tokens)
                     .Build());
 
                 continue;
@@ -981,7 +981,7 @@ public sealed class TypeChecker
         {
             Diagnostics.Add(Diagnostic
                 .Warning($"Fields {string.Join(", ", missingFields)} are not initialized")
-                .At(expression)
+                .At(expression, _syntaxTree.Tokens)
                 .Build());
         }
 
@@ -1065,7 +1065,7 @@ public sealed class TypeChecker
 
         throw new CompileException(Diagnostic
             .Error($"Type {customType.NameToken.Value} not found in module {module.Name}")
-            .At(customType)
+            .At(customType, _syntaxTree.Tokens)
             .Build());
     }
 

compiler/NubLang/Diagnostics/Diagnostic.cs

@@ -11,6 +11,7 @@ public class Diagnostic
         private readonly string _message;
         private SourceSpan? _span;
         private string? _help;
+        private List<Token>? _tokens;
 
         public DiagnosticBuilder(DiagnosticSeverity severity, string message)
         {
@@ -18,18 +19,32 @@ public class Diagnostic
             _message = message;
         }
 
-        public DiagnosticBuilder At(SyntaxNode? node)
+        public DiagnosticBuilder At(SyntaxNode? node, List<Token>? tokens = null)
         {
+            if (tokens != null)
+            {
+                _tokens = tokens;
+            }
+
             if (node != null)
+            {
+                var first = node.Tokens.FirstOrDefault();
+                if (first != null)
                 {
                     _span = SourceSpan.Merge(node.Tokens.Select(x => x.Span));
                 }
+            }
 
             return this;
         }
 
-        public DiagnosticBuilder At(Token? token)
+        public DiagnosticBuilder At(Token? token, List<Token>? tokens = null)
         {
+            if (tokens != null)
+            {
+                _tokens = tokens;
+            }
+
             if (token != null)
             {
                 At(token.Span);
@@ -48,11 +63,11 @@ public class Diagnostic
             return this;
         }
 
-        public DiagnosticBuilder At(string filePath, int line, int column)
-        {
-            _span = new SourceSpan(filePath, new SourceLocation(line, column), new SourceLocation(line, column));
-            return this;
-        }
+        // public DiagnosticBuilder At(string filePath, int line, int column)
+        // {
+        //     _span = new SourceSpan(filePath, new SourceLocation(line, column), new SourceLocation(line, column));
+        //     return this;
+        // }
 
         public DiagnosticBuilder WithHelp(string help)
         {
@@ -60,20 +75,23 @@ public class Diagnostic
             return this;
         }
 
-        public Diagnostic Build() => new(_severity, _message, _help, _span);
+        public Diagnostic Build() => new(_severity, _message, _help, _span, _tokens);
     }
 
     public static DiagnosticBuilder Error(string message) => new(DiagnosticSeverity.Error, message);
     public static DiagnosticBuilder Warning(string message) => new(DiagnosticSeverity.Warning, message);
     public static DiagnosticBuilder Info(string message) => new(DiagnosticSeverity.Info, message);
 
+    private readonly List<Token>? _tokens;
+
     public DiagnosticSeverity Severity { get; }
     public string Message { get; }
     public string? Help { get; }
     public SourceSpan? Span { get; }
 
-    private Diagnostic(DiagnosticSeverity severity, string message, string? help, SourceSpan? span)
+    private Diagnostic(DiagnosticSeverity severity, string message, string? help, SourceSpan? span, List<Token>? tokens)
     {
+        _tokens = tokens;
         Severity = severity;
         Message = message;
         Help = help;
@@ -103,15 +121,12 @@ public class Diagnostic
         if (Span.HasValue)
         {
             sb.AppendLine();
-            var text = File.ReadAllText(Span.Value.FilePath);
-
-            var tokenizer = new Tokenizer();
-            var tokens = tokenizer.Tokenize(Span.Value.FilePath, text);
+            var text = Span.Value.Source;
 
             var lines = text.Split('\n');
 
-            var startLine = Span.Value.Start.Line;
-            var endLine = Span.Value.End.Line;
+            var startLine = Span.Value.StartLine;
+            var endLine = Span.Value.EndLine;
 
             const int CONTEXT_LINES = 3;
 
@@ -144,8 +159,15 @@ public class Diagnostic
                 sb.Append("│ ");
                 sb.Append(i.ToString().PadRight(numberPadding));
                 sb.Append(" │ ");
-                sb.Append(ApplySyntaxHighlighting(line.PadRight(codePadding), i, tokens));
-                // sb.Append(line.PadRight(codePadding));
+                if (_tokens != null)
+                {
+                    sb.Append(ApplySyntaxHighlighting(line.PadRight(codePadding), i, _tokens));
+                }
+                else
+                {
+                    sb.Append(line.PadRight(codePadding));
+                }
+
                 sb.Append(" │");
                 sb.AppendLine();
 
@@ -156,12 +178,12 @@ public class Diagnostic
 
                     if (i == startLine)
                     {
-                        markerStartColumn = Span.Value.Start.Column;
+                        markerStartColumn = Span.Value.StartColumn;
                     }
 
                     if (i == endLine)
                     {
-                        markerEndColumn = Span.Value.End.Column;
+                        markerEndColumn = Span.Value.EndColumn;
                     }
 
                     var markerLength = markerEndColumn - markerStartColumn;
@@ -206,8 +228,8 @@ public class Diagnostic
     {
         var sb = new StringBuilder();
         var lineTokens = tokens
-            .Where(t => t.Span.Start.Line == lineNumber)
-            .OrderBy(t => t.Span.Start.Column)
+            .Where(t => t.Span.StartLine == lineNumber)
+            .OrderBy(t => t.Span.StartColumn)
             .ToList();
 
         if (lineTokens.Count == 0)
@@ -219,8 +241,10 @@ public class Diagnostic
 
         foreach (var token in lineTokens)
         {
-            var tokenStart = token.Span.Start.Column;
-            var tokenEnd = token.Span.End.Column;
+            if (token is WhitespaceToken) continue;
+
+            var tokenStart = token.Span.StartColumn;
+            var tokenEnd = token.Span.EndColumn;
 
             if (tokenStart > currentColumn && currentColumn - 1 < line.Length)
             {

compiler/NubLang/Diagnostics/SourceSpan.cs

@@ -1,112 +1,56 @@
 namespace NubLang.Diagnostics;
 
-public readonly struct SourceSpan : IEquatable<SourceSpan>, IComparable<SourceSpan>
+public readonly struct SourceSpan
 {
+    private readonly int _startIndex;
+    private readonly int _endIndex;
+
     public static SourceSpan Merge(params IEnumerable<SourceSpan> spans)
     {
         var spanArray = spans as SourceSpan[] ?? spans.ToArray();
         if (spanArray.Length == 0)
         {
-            return new SourceSpan(string.Empty, new SourceLocation(0, 0), new SourceLocation(0, 0));
+            return new SourceSpan(string.Empty, string.Empty, 0, 0, 0, 0, 0, 0);
         }
 
-        var minStart = spanArray.Min(s => s.Start);
-        var maxEnd = spanArray.Max(s => s.End);
+        var first = spanArray.MinBy(x => x._startIndex);
+        var last = spanArray.MaxBy(x => x._endIndex);
 
-        return new SourceSpan(spanArray[0].FilePath, minStart, maxEnd);
+        return new SourceSpan(first.SourcePath, first.Source, first._startIndex, last._endIndex, first.StartLine, last.EndLine, first.StartColumn, last.EndColumn);
     }
 
-    public SourceSpan(string filePath, SourceLocation start, SourceLocation end)
+    public SourceSpan(string sourcePath, string source, int startIndex, int endIndex, int startLine, int startColumn, int endLine, int endColumn)
     {
-        if (start > end)
-        {
-            throw new ArgumentException("Start location cannot be after end location");
+        _startIndex = startIndex;
+        _endIndex = endIndex;
+        SourcePath = sourcePath;
+        Source = source;
+        StartLine = startLine;
+        StartColumn = startColumn;
+        EndLine = endLine;
+        EndColumn = endColumn;
     }
 
-        FilePath = filePath;
-        Start = start;
-        End = end;
-    }
+    public int StartLine { get; }
+    public int StartColumn { get; }
+    public int EndLine { get; }
+    public int EndColumn { get; }
 
-    public string FilePath { get; }
-    public SourceLocation Start { get; }
-    public SourceLocation End { get; }
+    public string SourcePath { get; }
+    public string Source { get; }
 
     public override string ToString()
     {
-        if (Start == End)
+        if (StartLine == EndLine && StartColumn == EndColumn)
         {
-            return $"{FilePath}:{Start}";
+            return $"{SourcePath}:{StartColumn}:{StartColumn}";
         }
 
-        if (Start.Line == End.Line)
+        if (StartLine == EndLine)
         {
-            return Start.Column == End.Column ? $"{FilePath}:{Start}" : $"{FilePath}:{Start.Line}:{Start.Column}-{End.Column}";
+            return $"{SourcePath}:{StartLine}:{StartColumn}-{EndColumn}";
         }
 
-        return $"{FilePath}:{Start}-{End}";
-    }
-
-    public bool Equals(SourceSpan other) => Start == other.Start && End == other.End;
-    public override bool Equals(object? obj) => obj is SourceSpan other && Equals(other);
-    public override int GetHashCode() => HashCode.Combine(typeof(SourceSpan), Start, End);
-
-    public static bool operator ==(SourceSpan left, SourceSpan right) => Equals(left, right);
-    public static bool operator !=(SourceSpan left, SourceSpan right) => !Equals(left, right);
-
-    public static bool operator <(SourceSpan left, SourceSpan right) => left.CompareTo(right) < 0;
-    public static bool operator <=(SourceSpan left, SourceSpan right) => left.CompareTo(right) <= 0;
-    public static bool operator >(SourceSpan left, SourceSpan right) => left.CompareTo(right) > 0;
-    public static bool operator >=(SourceSpan left, SourceSpan right) => left.CompareTo(right) >= 0;
-
-    public int CompareTo(SourceSpan other)
-    {
-        var startComparison = Start.CompareTo(other.Start);
-        return startComparison != 0 ? startComparison : End.CompareTo(other.End);
-    }
-}
-
-public readonly struct SourceLocation : IEquatable<SourceLocation>, IComparable<SourceLocation>
-{
-    public SourceLocation(int line, int column)
-    {
-        Line = line;
-        Column = column;
-    }
-
-    public int Line { get; }
-    public int Column { get; }
-
-    public override string ToString()
-    {
-        return $"{Line}:{Column}";
-    }
-
-    public override bool Equals(object? obj)
-    {
-        return obj is SourceLocation other && Equals(other);
-    }
-
-    public bool Equals(SourceLocation other)
-    {
-        return Line == other.Line && Column == other.Column;
-    }
-
-    public override int GetHashCode()
-    {
-        return HashCode.Combine(typeof(SourceLocation), Line, Column);
-    }
-
-    public static bool operator ==(SourceLocation left, SourceLocation right) => Equals(left, right);
-    public static bool operator !=(SourceLocation left, SourceLocation right) => !Equals(left, right);
-    public static bool operator <(SourceLocation left, SourceLocation right) => left.Line < right.Line || (left.Line == right.Line && left.Column < right.Column);
-    public static bool operator >(SourceLocation left, SourceLocation right) => left.Line > right.Line || (left.Line == right.Line && left.Column > right.Column);
-    public static bool operator <=(SourceLocation left, SourceLocation right) => left.Line <= right.Line || (left.Line == right.Line && left.Column <= right.Column);
-    public static bool operator >=(SourceLocation left, SourceLocation right) => left.Line >= right.Line || (left.Line == right.Line && left.Column >= right.Column);
-
-    public int CompareTo(SourceLocation other)
-    {
-        var lineComparison = Line.CompareTo(other.Line);
-        return lineComparison != 0 ? lineComparison : Column.CompareTo(other.Column);
+        return $"{SourcePath}:{StartLine}:{StartColumn}-{EndLine}:{EndColumn}";
     }
 }

compiler/NubLang/Syntax/Parser.cs

@@ -62,7 +62,7 @@ public sealed class Parser
                     _ => throw new CompileException(Diagnostic
                         .Error($"Expected 'func', 'struct', 'enum', 'import' or 'module' but found '{keyword.Symbol}'")
                         .WithHelp("Valid top level statements are 'func', 'struct', 'enum', 'import' and 'module'")
-                        .At(keyword)
+                        .At(keyword, _tokens)
                         .Build())
                 };
 
@@ -83,7 +83,7 @@ public sealed class Parser
             }
         }
 
-        return new SyntaxTree(topLevelSyntaxNodes);
+        return new SyntaxTree(topLevelSyntaxNodes, _tokens);
     }
 
     private ModuleSyntax ParseModule(int startIndex)
@@ -190,7 +190,7 @@ public sealed class Parser
                 {
                     throw new CompileException(Diagnostic
                         .Error("Value of enum field must be an integer literal")
-                        .At(CurrentToken)
+                        .At(CurrentToken, _tokens)
                         .Build());
                 }
 
@@ -461,7 +461,7 @@ public sealed class Parser
             IdentifierToken identifier => ParseIdentifier(startIndex, identifier),
             SymbolToken symbolToken => symbolToken.Symbol switch
             {
-                Symbol.Ampersand => ParseAddressOf(startIndex),
+                Symbol.Caret => ParseAddressOf(startIndex),
                 Symbol.OpenParen => ParseParenthesizedExpression(),
                 Symbol.Minus => ParseUnaryNegate(startIndex),
                 Symbol.Bang => ParseUnaryInvert(startIndex),
@@ -472,13 +472,13 @@ public sealed class Parser
                 _ => throw new CompileException(Diagnostic
                     .Error($"Unexpected symbol '{symbolToken.Symbol}' in expression")
                     .WithHelp("Expected '(', '-', '!', '[' or '{'")
-                    .At(symbolToken)
+                    .At(symbolToken, _tokens)
                     .Build())
             },
             _ => throw new CompileException(Diagnostic
                 .Error($"Unexpected token '{token.GetType().Name}' in expression")
                 .WithHelp("Expected literal, identifier, or parenthesized expression")
-                .At(token)
+                .At(token, _tokens)
                 .Build())
         };
 
@@ -698,7 +698,7 @@ public sealed class Parser
                     throw new CompileException(Diagnostic
                         .Error("Arbitrary uint size is not supported")
                         .WithHelp("Use u8, u16, u32 or u64")
-                        .At(name)
+                        .At(name, _tokens)
                         .Build());
                 }
 
@@ -712,7 +712,7 @@ public sealed class Parser
                     throw new CompileException(Diagnostic
                         .Error("Arbitrary int size is not supported")
                         .WithHelp("Use i8, i16, i32 or i64")
-                        .At(name)
+                        .At(name, _tokens)
                         .Build());
                 }
 
@@ -726,7 +726,7 @@ public sealed class Parser
                     throw new CompileException(Diagnostic
                         .Error("Arbitrary float size is not supported")
                         .WithHelp("Use f32 or f64")
-                        .At(name)
+                        .At(name, _tokens)
                         .Build());
                 }
 
@@ -810,7 +810,7 @@ public sealed class Parser
         throw new CompileException(Diagnostic
             .Error("Invalid type syntax")
             .WithHelp("Expected type name, '^' for pointer, or '[]' for array")
-            .At(CurrentToken)
+            .At(CurrentToken, _tokens)
             .Build());
     }
 
@@ -821,7 +821,7 @@ public sealed class Parser
             throw new CompileException(Diagnostic
                 .Error("Unexpected end of file")
                 .WithHelp("Expected more tokens to complete the syntax")
-                .At(_tokens[^1])
+                .At(_tokens[^1], _tokens)
                 .Build());
         }
 
@@ -838,7 +838,7 @@ public sealed class Parser
             throw new CompileException(Diagnostic
                 .Error($"Expected symbol, but found {token.GetType().Name}")
                 .WithHelp("This position requires a symbol like '(', ')', '{', '}', etc.")
-                .At(token)
+                .At(token, _tokens)
                 .Build());
         }
 
@@ -853,7 +853,7 @@ public sealed class Parser
             throw new CompileException(Diagnostic
                 .Error($"Expected '{expectedSymbol}', but found '{token.Symbol}'")
                 .WithHelp($"Insert '{expectedSymbol}' here")
-                .At(token)
+                .At(token, _tokens)
                 .Build());
         }
     }
@@ -903,7 +903,7 @@ public sealed class Parser
             throw new CompileException(Diagnostic
                 .Error($"Expected identifier, but found {token.GetType().Name}")
                 .WithHelp("Provide a valid identifier name here")
-                .At(token)
+                .At(token, _tokens)
                 .Build());
         }
 
@@ -931,7 +931,7 @@ public sealed class Parser
             throw new CompileException(Diagnostic
                 .Error($"Expected string literal, but found {token.GetType().Name}")
                 .WithHelp("Provide a valid string literal")
-                .At(token)
+                .At(token, _tokens)
                 .Build());
         }
 
@@ -953,4 +953,4 @@ public sealed class Parser
     }
 }
 
-public record SyntaxTree(List<TopLevelSyntaxNode> TopLevelSyntaxNodes);
+public record SyntaxTree(List<TopLevelSyntaxNode> TopLevelSyntaxNodes, List<Token> Tokens);

compiler/NubLang/Syntax/Tokenizer.cs

@@ -43,6 +43,7 @@ public sealed class Tokenizer
 
     private Token ParseToken()
     {
+        var indexStart = _index;
         var lineStart = _line;
         var columnStart = _column;
 
@@ -53,7 +54,7 @@ public sealed class Tokenizer
                 Next();
             }
 
-            return new WhitespaceToken(CreateSpan(lineStart, columnStart));
+            return new WhitespaceToken(CreateSpan(indexStart, lineStart, columnStart));
         }
 
         if (_content[_index] == '/' && _index + 1 < _content.Length && _content[_index + 1] == '/')
@@ -66,23 +67,23 @@ public sealed class Tokenizer
                 Next();
             }
 
-            return new CommentToken(CreateSpan(lineStart, columnStart), _content.AsSpan(startIndex, _index - startIndex).ToString());
+            return new CommentToken(CreateSpan(indexStart, lineStart, columnStart), _content.AsSpan(startIndex, _index - startIndex).ToString());
         }
 
         if (char.IsDigit(_content[_index]))
         {
-            return ParseNumber(lineStart, columnStart);
+            return ParseNumber(indexStart, lineStart, columnStart);
         }
 
         if (_content[_index] == '"')
         {
-            return ParseString(lineStart, columnStart);
+            return ParseString(indexStart, lineStart, columnStart);
         }
 
         // note(nub31): Look for keywords (longest first in case a keyword fits partially in a larger keyword)
         for (var i = 8; i >= 1; i--)
         {
-            if (TryMatchSymbol(i, lineStart, columnStart, out var token))
+            if (TryMatchSymbol(i, indexStart, lineStart, columnStart, out var token))
             {
                 return token;
             }
@@ -90,13 +91,13 @@ public sealed class Tokenizer
 
         if (char.IsLetter(_content[_index]) || _content[_index] == '_')
         {
-            return ParseIdentifier(lineStart, columnStart);
+            return ParseIdentifier(indexStart, lineStart, columnStart);
         }
 
         throw new CompileException(Diagnostic.Error($"Unknown token '{_content[_index]}'").Build());
     }
 
-    private Token ParseNumber(int lineStart, int columnStart)
+    private Token ParseNumber(int indexStart, int lineStart, int columnStart)
     {
         var start = _index;
         var current = _content[_index];
@@ -116,12 +117,12 @@ public sealed class Tokenizer
             {
                 throw new CompileException(Diagnostic
                     .Error("Invalid hex literal, no digits found")
-                    .At(_fileName, _line, _column)
+                    .At(CreateSpan(_index, _line, _column))
                     .Build());
             }
 
             return new IntLiteralToken(
-                CreateSpan(lineStart, columnStart),
+                CreateSpan(indexStart, lineStart, columnStart),
                 _content.Substring(start, _index - start),
                 16);
         }
@@ -141,12 +142,12 @@ public sealed class Tokenizer
             {
                 throw new CompileException(Diagnostic
                     .Error("Invalid binary literal, no digits found")
-                    .At(_fileName, _line, _column)
+                    .At(CreateSpan(_index, _line, _column))
                     .Build());
             }
 
             return new IntLiteralToken(
-                CreateSpan(lineStart, columnStart),
+                CreateSpan(indexStart, lineStart, columnStart),
                 _content.Substring(start, _index - start),
                 2);
         }
@@ -163,7 +164,7 @@ public sealed class Tokenizer
                 {
                     throw new CompileException(Diagnostic
                         .Error("More than one period found in float literal")
-                        .At(_fileName, _line, _column)
+                        .At(CreateSpan(_index, _line, _column))
                         .Build());
                 }
 
@@ -183,11 +184,11 @@ public sealed class Tokenizer
         var buffer = _content.Substring(start, _index - start);
 
         return isFloat
-            ? new FloatLiteralToken(CreateSpan(lineStart, columnStart), buffer)
-            : new IntLiteralToken(CreateSpan(lineStart, columnStart), buffer, 10);
+            ? new FloatLiteralToken(CreateSpan(indexStart, lineStart, columnStart), buffer)
+            : new IntLiteralToken(CreateSpan(indexStart, lineStart, columnStart), buffer, 10);
     }
 
-    private StringLiteralToken ParseString(int lineStart, int columnStart)
+    private StringLiteralToken ParseString(int indexStart, int lineStart, int columnStart)
     {
         Next();
         var start = _index;
@@ -198,7 +199,7 @@ public sealed class Tokenizer
             {
                 throw new CompileException(Diagnostic
                     .Error("Unclosed string literal")
-                    .At(_fileName, _line, _column)
+                    .At(CreateSpan(_index, _line, _column))
                     .Build());
             }
 
@@ -208,7 +209,7 @@ public sealed class Tokenizer
             {
                 throw new CompileException(Diagnostic
                     .Error("Unclosed string literal (newline found)")
-                    .At(_fileName, _line, _column)
+                    .At(CreateSpan(_index, _line, _column))
                     .Build());
             }
 
@@ -216,14 +217,14 @@ public sealed class Tokenizer
             {
                 var buffer = _content.Substring(start, _index - start);
                 Next();
-                return new StringLiteralToken(CreateSpan(lineStart, columnStart), buffer);
+                return new StringLiteralToken(CreateSpan(indexStart, lineStart, columnStart), buffer);
             }
 
             Next();
         }
     }
 
-    private bool TryMatchSymbol(int length, int lineStart, int columnStart, out Token token)
+    private bool TryMatchSymbol(int length, int indexStart, int lineStart, int columnStart, out Token token)
     {
         token = null!;
 
@@ -237,14 +238,14 @@ public sealed class Tokenizer
         if (span is "true")
         {
             Next(4);
-            token = new BoolLiteralToken(CreateSpan(lineStart, columnStart), true);
+            token = new BoolLiteralToken(CreateSpan(indexStart, lineStart, columnStart), true);
             return true;
         }
 
         if (span is "false")
         {
             Next(5);
-            token = new BoolLiteralToken(CreateSpan(lineStart, columnStart), false);
+            token = new BoolLiteralToken(CreateSpan(indexStart, lineStart, columnStart), false);
             return true;
         }
 
@@ -349,14 +350,14 @@ public sealed class Tokenizer
             }
 
             Next(length);
-            token = new SymbolToken(CreateSpan(lineStart, columnStart), symbol);
+            token = new SymbolToken(CreateSpan(indexStart, lineStart, columnStart), symbol);
             return true;
         }
 
         return false;
     }
 
-    private IdentifierToken ParseIdentifier(int lineStart, int columnStart)
+    private IdentifierToken ParseIdentifier(int indexStart, int lineStart, int columnStart)
     {
         var start = _index;
 
@@ -373,12 +374,12 @@ public sealed class Tokenizer
             }
         }
 
-        return new IdentifierToken(CreateSpan(lineStart, columnStart), _content.Substring(start, _index - start));
+        return new IdentifierToken(CreateSpan(indexStart, lineStart, columnStart), _content.Substring(start, _index - start));
     }
 
-    private SourceSpan CreateSpan(int lineStart, int columnStart)
+    private SourceSpan CreateSpan(int indexStart, int lineStart, int columnStart)
     {
-        return new SourceSpan(_fileName, new SourceLocation(lineStart, columnStart), new SourceLocation(_line, _column));
+        return new SourceSpan(_fileName, _content, indexStart, Math.Min(_index, _content.Length), lineStart, columnStart, _line, _column);
     }
 
     private void Next(int count = 1)