Store src data in tokens

src/compiler/Nub.Lang/Frontend/Lexing/Lexer.cs

@@ -16,7 +16,7 @@ public class Lexer
         ["struct"] = Symbol.Struct,
     };
 
-    private static readonly Dictionary<string, Modifier> Modifers = new()
+    private static readonly Dictionary<string, Modifier> Modifiers = new()
     {
         ["global"] = Modifier.Global,
         ["extern"] = Modifier.Extern,
@@ -54,14 +54,16 @@ public class Lexer
         ['&'] = Symbol.Ampersand,
     };
 
-    private string _src = string.Empty;
+    private string _src = null!;
+    private string _filePath = null!;
     private int _index;
-
-    public List<Token> Lex(string src)
+    
+    public List<Token> Lex(string src, string filePath)
     {
         _src = src;
+        _filePath = filePath;
         _index = 0;
-
+        
         List<Token> tokens = [];
         while (ParseToken().TryGetValue(out var token))
         {
@@ -82,6 +84,7 @@ public class Lexer
     private Optional<Token> ParseToken()
     {
         ConsumeWhitespace();
+        var startIndex = _index;
 
         string? documentation = null;
         while (Peek() is { Value: '/' } && Peek(1) is { Value: '/' })
@@ -93,44 +96,36 @@ public class Lexer
             {
                 Next();
 
-                if (documentation != null)
-                {
-                    documentation += '\n';
-                }
-
                 while (Peek().TryGetValue(out var character))
                 {
+                    Next();
+                    documentation += character;
                     if (character == '\n')
                     {
-                        Next();
                         break;
                     }
-
-                    documentation += character;
-                    Next();
                 }
             }
             else
             {
                 while (Peek().TryGetValue(out var character))
                 {
+                    Next();
                     if (character == '\n')
                     {
-                        Next();
                         break;
                     }
-
-                    Next();
                 }
             }
         }
 
         if (documentation != null)
         {
-            return new DocumentationToken(documentation);
+            return new DocumentationToken(_filePath, startIndex, _index, documentation);
         }
 
         ConsumeWhitespace();
+        startIndex = _index;
 
         if (!Peek().TryGetValue(out var current))
         {
@@ -149,20 +144,20 @@ public class Lexer
 
             if (Keywords.TryGetValue(buffer, out var keywordSymbol))
             {
-                return new SymbolToken(keywordSymbol);
+                return new SymbolToken(_filePath, startIndex, _index, keywordSymbol);
             }
 
-            if (Modifers.TryGetValue(buffer, out var modifer))
+            if (Modifiers.TryGetValue(buffer, out var modifer))
             {
-                return new ModifierToken(modifer);
+                return new ModifierToken(_filePath, startIndex, _index, modifer);
             }
 
             if (buffer is "true" or "false")
             {
-                return new LiteralToken(NubPrimitiveType.Bool, buffer);
+                return new LiteralToken(_filePath, startIndex, _index, NubPrimitiveType.Bool, buffer);
             }
 
-            return new IdentifierToken(buffer);
+            return new IdentifierToken(_filePath, startIndex, _index, buffer);
         }
 
         if (char.IsDigit(current))
@@ -200,7 +195,7 @@ public class Lexer
                 }
             }
 
-            return new LiteralToken(isFloat ? NubPrimitiveType.F64 : NubPrimitiveType.I64, buffer);
+            return new LiteralToken(_filePath, startIndex, _index, isFloat ? NubPrimitiveType.F64 : NubPrimitiveType.I64, buffer);
         }
 
         // TODO: Revisit this
@@ -220,7 +215,7 @@ public class Lexer
                         Next();
                     }
 
-                    return new SymbolToken(chain.Value);
+                    return new SymbolToken(_filePath, startIndex, _index, chain.Value);
                 }
             }
         }
@@ -228,7 +223,7 @@ public class Lexer
         if (Chars.TryGetValue(current, out var charSymbol))
         {
             Next();
-            return new SymbolToken(charSymbol);
+            return new SymbolToken(_filePath, startIndex, _index, charSymbol);
         }
 
         if (current == '"')
@@ -253,7 +248,7 @@ public class Lexer
                 Next();
             }
 
-            return new LiteralToken(NubPrimitiveType.String, buffer);
+            return new LiteralToken(_filePath, startIndex, _index, NubPrimitiveType.String, buffer);
         }
 
         throw new Exception($"Unknown character {current}");