...

compiler/NubLang/Syntax/Tokenizer.cs

@@ -43,6 +43,7 @@ public sealed class Tokenizer
 
     private Token ParseToken()
     {
+        var indexStart = _index;
         var lineStart = _line;
         var columnStart = _column;
 
@@ -53,7 +54,7 @@ public sealed class Tokenizer
                 Next();
             }
 
-            return new WhitespaceToken(CreateSpan(lineStart, columnStart));
+            return new WhitespaceToken(CreateSpan(indexStart, lineStart, columnStart));
         }
 
         if (_content[_index] == '/' && _index + 1 < _content.Length && _content[_index + 1] == '/')
@@ -66,23 +67,23 @@ public sealed class Tokenizer
                 Next();
             }
 
-            return new CommentToken(CreateSpan(lineStart, columnStart), _content.AsSpan(startIndex, _index - startIndex).ToString());
+            return new CommentToken(CreateSpan(indexStart, lineStart, columnStart), _content.AsSpan(startIndex, _index - startIndex).ToString());
         }
 
         if (char.IsDigit(_content[_index]))
         {
-            return ParseNumber(lineStart, columnStart);
+            return ParseNumber(indexStart, lineStart, columnStart);
         }
 
         if (_content[_index] == '"')
         {
-            return ParseString(lineStart, columnStart);
+            return ParseString(indexStart, lineStart, columnStart);
         }
 
         // note(nub31): Look for keywords (longest first in case a keyword fits partially in a larger keyword)
         for (var i = 8; i >= 1; i--)
         {
-            if (TryMatchSymbol(i, lineStart, columnStart, out var token))
+            if (TryMatchSymbol(i, indexStart, lineStart, columnStart, out var token))
             {
                 return token;
             }
@@ -90,13 +91,13 @@ public sealed class Tokenizer
 
         if (char.IsLetter(_content[_index]) || _content[_index] == '_')
         {
-            return ParseIdentifier(lineStart, columnStart);
+            return ParseIdentifier(indexStart, lineStart, columnStart);
         }
 
         throw new CompileException(Diagnostic.Error($"Unknown token '{_content[_index]}'").Build());
     }
 
-    private Token ParseNumber(int lineStart, int columnStart)
+    private Token ParseNumber(int indexStart, int lineStart, int columnStart)
     {
         var start = _index;
         var current = _content[_index];
@@ -116,12 +117,12 @@ public sealed class Tokenizer
             {
                 throw new CompileException(Diagnostic
                     .Error("Invalid hex literal, no digits found")
-                    .At(_fileName, _line, _column)
+                    .At(CreateSpan(_index, _line, _column))
                     .Build());
             }
 
             return new IntLiteralToken(
-                CreateSpan(lineStart, columnStart),
+                CreateSpan(indexStart, lineStart, columnStart),
                 _content.Substring(start, _index - start),
                 16);
         }
@@ -141,12 +142,12 @@ public sealed class Tokenizer
             {
                 throw new CompileException(Diagnostic
                     .Error("Invalid binary literal, no digits found")
-                    .At(_fileName, _line, _column)
+                    .At(CreateSpan(_index, _line, _column))
                     .Build());
             }
 
             return new IntLiteralToken(
-                CreateSpan(lineStart, columnStart),
+                CreateSpan(indexStart, lineStart, columnStart),
                 _content.Substring(start, _index - start),
                 2);
         }
@@ -163,7 +164,7 @@ public sealed class Tokenizer
                 {
                     throw new CompileException(Diagnostic
                         .Error("More than one period found in float literal")
-                        .At(_fileName, _line, _column)
+                        .At(CreateSpan(_index, _line, _column))
                         .Build());
                 }
 
@@ -183,11 +184,11 @@ public sealed class Tokenizer
         var buffer = _content.Substring(start, _index - start);
 
         return isFloat
-            ? new FloatLiteralToken(CreateSpan(lineStart, columnStart), buffer)
-            : new IntLiteralToken(CreateSpan(lineStart, columnStart), buffer, 10);
+            ? new FloatLiteralToken(CreateSpan(indexStart, lineStart, columnStart), buffer)
+            : new IntLiteralToken(CreateSpan(indexStart, lineStart, columnStart), buffer, 10);
     }
 
-    private StringLiteralToken ParseString(int lineStart, int columnStart)
+    private StringLiteralToken ParseString(int indexStart, int lineStart, int columnStart)
     {
         Next();
         var start = _index;
@@ -198,7 +199,7 @@ public sealed class Tokenizer
             {
                 throw new CompileException(Diagnostic
                     .Error("Unclosed string literal")
-                    .At(_fileName, _line, _column)
+                    .At(CreateSpan(_index, _line, _column))
                     .Build());
             }
 
@@ -208,7 +209,7 @@ public sealed class Tokenizer
             {
                 throw new CompileException(Diagnostic
                     .Error("Unclosed string literal (newline found)")
-                    .At(_fileName, _line, _column)
+                    .At(CreateSpan(_index, _line, _column))
                     .Build());
             }
 
@@ -216,14 +217,14 @@ public sealed class Tokenizer
             {
                 var buffer = _content.Substring(start, _index - start);
                 Next();
-                return new StringLiteralToken(CreateSpan(lineStart, columnStart), buffer);
+                return new StringLiteralToken(CreateSpan(indexStart, lineStart, columnStart), buffer);
             }
 
             Next();
         }
     }
 
-    private bool TryMatchSymbol(int length, int lineStart, int columnStart, out Token token)
+    private bool TryMatchSymbol(int length, int indexStart, int lineStart, int columnStart, out Token token)
     {
         token = null!;
 
@@ -237,14 +238,14 @@ public sealed class Tokenizer
         if (span is "true")
         {
             Next(4);
-            token = new BoolLiteralToken(CreateSpan(lineStart, columnStart), true);
+            token = new BoolLiteralToken(CreateSpan(indexStart, lineStart, columnStart), true);
             return true;
         }
 
         if (span is "false")
         {
             Next(5);
-            token = new BoolLiteralToken(CreateSpan(lineStart, columnStart), false);
+            token = new BoolLiteralToken(CreateSpan(indexStart, lineStart, columnStart), false);
             return true;
         }
 
@@ -349,14 +350,14 @@ public sealed class Tokenizer
             }
 
             Next(length);
-            token = new SymbolToken(CreateSpan(lineStart, columnStart), symbol);
+            token = new SymbolToken(CreateSpan(indexStart, lineStart, columnStart), symbol);
             return true;
         }
 
         return false;
     }
 
-    private IdentifierToken ParseIdentifier(int lineStart, int columnStart)
+    private IdentifierToken ParseIdentifier(int indexStart, int lineStart, int columnStart)
     {
         var start = _index;
 
@@ -373,12 +374,12 @@ public sealed class Tokenizer
             }
         }
 
-        return new IdentifierToken(CreateSpan(lineStart, columnStart), _content.Substring(start, _index - start));
+        return new IdentifierToken(CreateSpan(indexStart, lineStart, columnStart), _content.Substring(start, _index - start));
     }
 
-    private SourceSpan CreateSpan(int lineStart, int columnStart)
+    private SourceSpan CreateSpan(int indexStart, int lineStart, int columnStart)
     {
-        return new SourceSpan(_fileName, new SourceLocation(lineStart, columnStart), new SourceLocation(_line, _column));
+        return new SourceSpan(_fileName, _content, indexStart, Math.Min(_index, _content.Length), lineStart, columnStart, _line, _column);
     }
 
     private void Next(int count = 1)