Store src data in tokens

src/compiler/Nub.Lang/Frontend/Lexing/DocumentationToken.cs

@@ -1,6 +1,6 @@
 namespace Nub.Lang.Frontend.Lexing;
 
-public class DocumentationToken(string documentation) : Token
+public class DocumentationToken(string filePath, int startIndex, int endIndex, string documentation) : Token(filePath, startIndex, endIndex)
 {
     public string Documentation { get; } = documentation;
 }

src/compiler/Nub.Lang/Frontend/Lexing/IdentifierToken.cs

@@ -1,6 +1,6 @@
 namespace Nub.Lang.Frontend.Lexing;
 
-public class IdentifierToken(string value) : Token
+public class IdentifierToken(string filePath, int startIndex, int endIndex, string value) : Token(filePath, startIndex, endIndex)
 {
     public string Value { get; } = value;
 }

src/compiler/Nub.Lang/Frontend/Lexing/Lexer.cs

@@ -16,7 +16,7 @@ public class Lexer
         ["struct"] = Symbol.Struct,
     };
 
-    private static readonly Dictionary<string, Modifier> Modifers = new()
+    private static readonly Dictionary<string, Modifier> Modifiers = new()
     {
         ["global"] = Modifier.Global,
         ["extern"] = Modifier.Extern,
@@ -54,14 +54,16 @@ public class Lexer
         ['&'] = Symbol.Ampersand,
     };
 
-    private string _src = string.Empty;
+    private string _src = null!;
+    private string _filePath = null!;
     private int _index;
-
+    
-    public List<Token> Lex(string src)
+    public List<Token> Lex(string src, string filePath)
     {
         _src = src;
+        _filePath = filePath;
         _index = 0;
-
+        
         List<Token> tokens = [];
         while (ParseToken().TryGetValue(out var token))
         {
@@ -82,6 +84,7 @@ public class Lexer
     private Optional<Token> ParseToken()
     {
         ConsumeWhitespace();
+        var startIndex = _index;
 
         string? documentation = null;
         while (Peek() is { Value: '/' } && Peek(1) is { Value: '/' })
@@ -93,44 +96,36 @@ public class Lexer
             {
                 Next();
 
-                if (documentation != null)
-                {
-                    documentation += '\n';
-                }
-
                 while (Peek().TryGetValue(out var character))
                 {
+                    Next();
+                    documentation += character;
                     if (character == '\n')
                     {
-                        Next();
                         break;
                     }
-
-                    documentation += character;
-                    Next();
                 }
             }
             else
             {
                 while (Peek().TryGetValue(out var character))
                 {
+                    Next();
                     if (character == '\n')
                     {
-                        Next();
                         break;
                     }
-
-                    Next();
                 }
             }
         }
 
         if (documentation != null)
         {
-            return new DocumentationToken(documentation);
+            return new DocumentationToken(_filePath, startIndex, _index, documentation);
         }
 
         ConsumeWhitespace();
+        startIndex = _index;
 
         if (!Peek().TryGetValue(out var current))
         {
@@ -149,20 +144,20 @@ public class Lexer
 
             if (Keywords.TryGetValue(buffer, out var keywordSymbol))
             {
-                return new SymbolToken(keywordSymbol);
+                return new SymbolToken(_filePath, startIndex, _index, keywordSymbol);
             }
 
-            if (Modifers.TryGetValue(buffer, out var modifer))
+            if (Modifiers.TryGetValue(buffer, out var modifer))
             {
-                return new ModifierToken(modifer);
+                return new ModifierToken(_filePath, startIndex, _index, modifer);
             }
 
             if (buffer is "true" or "false")
             {
-                return new LiteralToken(NubPrimitiveType.Bool, buffer);
+                return new LiteralToken(_filePath, startIndex, _index, NubPrimitiveType.Bool, buffer);
             }
 
-            return new IdentifierToken(buffer);
+            return new IdentifierToken(_filePath, startIndex, _index, buffer);
         }
 
         if (char.IsDigit(current))
@@ -200,7 +195,7 @@ public class Lexer
                 }
             }
 
-            return new LiteralToken(isFloat ? NubPrimitiveType.F64 : NubPrimitiveType.I64, buffer);
+            return new LiteralToken(_filePath, startIndex, _index, isFloat ? NubPrimitiveType.F64 : NubPrimitiveType.I64, buffer);
         }
 
         // TODO: Revisit this
@@ -220,7 +215,7 @@ public class Lexer
                         Next();
                     }
 
-                    return new SymbolToken(chain.Value);
+                    return new SymbolToken(_filePath, startIndex, _index, chain.Value);
                 }
             }
         }
@@ -228,7 +223,7 @@ public class Lexer
         if (Chars.TryGetValue(current, out var charSymbol))
         {
             Next();
-            return new SymbolToken(charSymbol);
+            return new SymbolToken(_filePath, startIndex, _index, charSymbol);
         }
 
         if (current == '"')
@@ -253,7 +248,7 @@ public class Lexer
                 Next();
             }
 
-            return new LiteralToken(NubPrimitiveType.String, buffer);
+            return new LiteralToken(_filePath, startIndex, _index, NubPrimitiveType.String, buffer);
         }
 
         throw new Exception($"Unknown character {current}");

src/compiler/Nub.Lang/Frontend/Lexing/LiteralToken.cs

@@ -1,6 +1,6 @@
 namespace Nub.Lang.Frontend.Lexing;
 
-public class LiteralToken(NubType type, string value) : Token
+public class LiteralToken(string filePath, int startIndex, int endIndex, NubType type, string value) : Token(filePath, startIndex, endIndex)
 {
     public NubType Type { get; } = type;
     public string Value { get; } = value;

src/compiler/Nub.Lang/Frontend/Lexing/ModifierToken.cs

@@ -1,8 +1,8 @@
 namespace Nub.Lang.Frontend.Lexing;
 
-public class ModifierToken(Modifier symbol) : Token
+public class ModifierToken(string filePath, int startIndex, int endIndex, Modifier modifier) : Token(filePath, startIndex, endIndex)
 {
-    public Modifier Modifier { get; } = symbol;
+    public Modifier Modifier { get; } = modifier;
 }
 
 public enum Modifier

src/compiler/Nub.Lang/Frontend/Lexing/SymbolToken.cs

@@ -1,6 +1,6 @@
 namespace Nub.Lang.Frontend.Lexing;
 
-public class SymbolToken(Symbol symbol) : Token
+public class SymbolToken(string filePath, int startIndex, int endIndex, Symbol symbol) : Token(filePath, startIndex, endIndex)
 {
     public Symbol Symbol { get; } = symbol;
 }

src/compiler/Nub.Lang/Frontend/Lexing/Token.cs

@@ -1,3 +1,8 @@
 namespace Nub.Lang.Frontend.Lexing;
 
-public abstract class Token;
+public abstract class Token(string filePath, int startIndex, int endIndex)
+{
+    public string FilePath { get; } = filePath;
+    public int StartIndex { get; } = startIndex;
+    public int EndIndex { get; } = endIndex;
+}

src/compiler/Nub.Lang/Frontend/Parsing/Parser.cs

@@ -8,7 +8,7 @@ public class Parser
     private List<Token> _tokens = [];
     private int _index;
 
-    public ModuleNode ParseModule(List<Token> tokens, string path)
+    public ModuleNode ParseModule(List<Token> tokens, string rootFilePath)
     {
         _index = 0;
         _tokens = tokens;
@@ -29,7 +29,7 @@ public class Parser
             }
         }
 
-        return new ModuleNode(path, imports, definitions);
+        return new ModuleNode(rootFilePath, imports, definitions);
     }
 
     private DefinitionNode ParseDefinition()

src/compiler/Nub.Lang/Program.cs

@@ -54,25 +54,25 @@ internal static class Program
         return 0;
     }
 
-    private static List<ModuleNode> RunFrontend(string path)
+    private static List<ModuleNode> RunFrontend(string rootFilePath)
     {
         List<ModuleNode> modules = [];
-        RunFrontend(path, modules);
+        RunFrontend(rootFilePath, modules);
         return modules;
     }
 
-    private static void RunFrontend(string path, List<ModuleNode> modules)
+    private static void RunFrontend(string rootFilePath, List<ModuleNode> modules)
     {
-        var files = Directory.EnumerateFiles(path, "*.nub", SearchOption.TopDirectoryOnly);
+        var filePaths = Directory.EnumerateFiles(rootFilePath, "*.nub", SearchOption.TopDirectoryOnly);
 
         List<Token> tokens = [];
-        foreach (var file in files)
+        foreach (var filePath in filePaths)
         {
-            var src = File.ReadAllText(file);
+            var src = File.ReadAllText(filePath);
-            tokens.AddRange(Lexer.Lex(src));
+            tokens.AddRange(Lexer.Lex(src, filePath));
         }
 
-        var module = Parser.ParseModule(tokens, path);
+        var module = Parser.ParseModule(tokens, rootFilePath);
         modules.Add(module);
 
         foreach (var import in module.Imports)