From 26906ea689295d8f6817545289f9e47974a2406b Mon Sep 17 00:00:00 2001 From: nub31 Date: Sat, 24 May 2025 21:12:44 +0200 Subject: [PATCH] Store src data in tokens --- .../Frontend/Lexing/DocumentationToken.cs | 2 +- .../Frontend/Lexing/IdentifierToken.cs | 2 +- .../Nub.Lang/Frontend/Lexing/Lexer.cs | 49 +++++++++---------- .../Nub.Lang/Frontend/Lexing/LiteralToken.cs | 2 +- .../Nub.Lang/Frontend/Lexing/ModifierToken.cs | 4 +- .../Nub.Lang/Frontend/Lexing/SymbolToken.cs | 2 +- .../Nub.Lang/Frontend/Lexing/Token.cs | 7 ++- .../Nub.Lang/Frontend/Parsing/Parser.cs | 4 +- src/compiler/Nub.Lang/Program.cs | 16 +++--- 9 files changed, 44 insertions(+), 44 deletions(-) diff --git a/src/compiler/Nub.Lang/Frontend/Lexing/DocumentationToken.cs b/src/compiler/Nub.Lang/Frontend/Lexing/DocumentationToken.cs index 7c28db2..422374b 100644 --- a/src/compiler/Nub.Lang/Frontend/Lexing/DocumentationToken.cs +++ b/src/compiler/Nub.Lang/Frontend/Lexing/DocumentationToken.cs @@ -1,6 +1,6 @@ namespace Nub.Lang.Frontend.Lexing; -public class DocumentationToken(string documentation) : Token +public class DocumentationToken(string filePath, int startIndex, int endIndex, string documentation) : Token(filePath, startIndex, endIndex) { public string Documentation { get; } = documentation; } \ No newline at end of file diff --git a/src/compiler/Nub.Lang/Frontend/Lexing/IdentifierToken.cs b/src/compiler/Nub.Lang/Frontend/Lexing/IdentifierToken.cs index 332bb85..92b37c7 100644 --- a/src/compiler/Nub.Lang/Frontend/Lexing/IdentifierToken.cs +++ b/src/compiler/Nub.Lang/Frontend/Lexing/IdentifierToken.cs @@ -1,6 +1,6 @@ namespace Nub.Lang.Frontend.Lexing; -public class IdentifierToken(string value) : Token +public class IdentifierToken(string filePath, int startIndex, int endIndex, string value) : Token(filePath, startIndex, endIndex) { public string Value { get; } = value; } \ No newline at end of file diff --git a/src/compiler/Nub.Lang/Frontend/Lexing/Lexer.cs b/src/compiler/Nub.Lang/Frontend/Lexing/Lexer.cs index 125727d..e868073 100644 --- a/src/compiler/Nub.Lang/Frontend/Lexing/Lexer.cs +++ b/src/compiler/Nub.Lang/Frontend/Lexing/Lexer.cs @@ -16,7 +16,7 @@ public class Lexer ["struct"] = Symbol.Struct, }; - private static readonly Dictionary Modifers = new() + private static readonly Dictionary Modifiers = new() { ["global"] = Modifier.Global, ["extern"] = Modifier.Extern, @@ -54,14 +54,16 @@ public class Lexer ['&'] = Symbol.Ampersand, }; - private string _src = string.Empty; + private string _src = null!; + private string _filePath = null!; private int _index; - - public List Lex(string src) + + public List Lex(string src, string filePath) { _src = src; + _filePath = filePath; _index = 0; - + List tokens = []; while (ParseToken().TryGetValue(out var token)) { @@ -82,6 +84,7 @@ public class Lexer private Optional ParseToken() { ConsumeWhitespace(); + var startIndex = _index; string? documentation = null; while (Peek() is { Value: '/' } && Peek(1) is { Value: '/' }) @@ -93,44 +96,36 @@ public class Lexer { Next(); - if (documentation != null) - { - documentation += '\n'; - } - while (Peek().TryGetValue(out var character)) { + Next(); + documentation += character; if (character == '\n') { - Next(); break; } - - documentation += character; - Next(); } } else { while (Peek().TryGetValue(out var character)) { + Next(); if (character == '\n') { - Next(); break; } - - Next(); } } } if (documentation != null) { - return new DocumentationToken(documentation); + return new DocumentationToken(_filePath, startIndex, _index, documentation); } ConsumeWhitespace(); + startIndex = _index; if (!Peek().TryGetValue(out var current)) { @@ -149,20 +144,20 @@ public class Lexer if (Keywords.TryGetValue(buffer, out var keywordSymbol)) { - return new SymbolToken(keywordSymbol); + return new SymbolToken(_filePath, startIndex, _index, keywordSymbol); } - if (Modifers.TryGetValue(buffer, out var modifer)) + if (Modifiers.TryGetValue(buffer, out var modifer)) { - return new ModifierToken(modifer); + return new ModifierToken(_filePath, startIndex, _index, modifer); } if (buffer is "true" or "false") { - return new LiteralToken(NubPrimitiveType.Bool, buffer); + return new LiteralToken(_filePath, startIndex, _index, NubPrimitiveType.Bool, buffer); } - return new IdentifierToken(buffer); + return new IdentifierToken(_filePath, startIndex, _index, buffer); } if (char.IsDigit(current)) @@ -200,7 +195,7 @@ public class Lexer } } - return new LiteralToken(isFloat ? NubPrimitiveType.F64 : NubPrimitiveType.I64, buffer); + return new LiteralToken(_filePath, startIndex, _index, isFloat ? NubPrimitiveType.F64 : NubPrimitiveType.I64, buffer); } // TODO: Revisit this @@ -220,7 +215,7 @@ public class Lexer Next(); } - return new SymbolToken(chain.Value); + return new SymbolToken(_filePath, startIndex, _index, chain.Value); } } } @@ -228,7 +223,7 @@ public class Lexer if (Chars.TryGetValue(current, out var charSymbol)) { Next(); - return new SymbolToken(charSymbol); + return new SymbolToken(_filePath, startIndex, _index, charSymbol); } if (current == '"') @@ -253,7 +248,7 @@ public class Lexer Next(); } - return new LiteralToken(NubPrimitiveType.String, buffer); + return new LiteralToken(_filePath, startIndex, _index, NubPrimitiveType.String, buffer); } throw new Exception($"Unknown character {current}"); diff --git a/src/compiler/Nub.Lang/Frontend/Lexing/LiteralToken.cs b/src/compiler/Nub.Lang/Frontend/Lexing/LiteralToken.cs index a1b3b20..4797197 100644 --- a/src/compiler/Nub.Lang/Frontend/Lexing/LiteralToken.cs +++ b/src/compiler/Nub.Lang/Frontend/Lexing/LiteralToken.cs @@ -1,6 +1,6 @@ namespace Nub.Lang.Frontend.Lexing; -public class LiteralToken(NubType type, string value) : Token +public class LiteralToken(string filePath, int startIndex, int endIndex, NubType type, string value) : Token(filePath, startIndex, endIndex) { public NubType Type { get; } = type; public string Value { get; } = value; diff --git a/src/compiler/Nub.Lang/Frontend/Lexing/ModifierToken.cs b/src/compiler/Nub.Lang/Frontend/Lexing/ModifierToken.cs index 3446034..0bc95fb 100644 --- a/src/compiler/Nub.Lang/Frontend/Lexing/ModifierToken.cs +++ b/src/compiler/Nub.Lang/Frontend/Lexing/ModifierToken.cs @@ -1,8 +1,8 @@ namespace Nub.Lang.Frontend.Lexing; -public class ModifierToken(Modifier symbol) : Token +public class ModifierToken(string filePath, int startIndex, int endIndex, Modifier modifier) : Token(filePath, startIndex, endIndex) { - public Modifier Modifier { get; } = symbol; + public Modifier Modifier { get; } = modifier; } public enum Modifier diff --git a/src/compiler/Nub.Lang/Frontend/Lexing/SymbolToken.cs b/src/compiler/Nub.Lang/Frontend/Lexing/SymbolToken.cs index 5a2bc47..b1aa398 100644 --- a/src/compiler/Nub.Lang/Frontend/Lexing/SymbolToken.cs +++ b/src/compiler/Nub.Lang/Frontend/Lexing/SymbolToken.cs @@ -1,6 +1,6 @@ namespace Nub.Lang.Frontend.Lexing; -public class SymbolToken(Symbol symbol) : Token +public class SymbolToken(string filePath, int startIndex, int endIndex, Symbol symbol) : Token(filePath, startIndex, endIndex) { public Symbol Symbol { get; } = symbol; } diff --git a/src/compiler/Nub.Lang/Frontend/Lexing/Token.cs b/src/compiler/Nub.Lang/Frontend/Lexing/Token.cs index c63ca90..154083f 100644 --- a/src/compiler/Nub.Lang/Frontend/Lexing/Token.cs +++ b/src/compiler/Nub.Lang/Frontend/Lexing/Token.cs @@ -1,3 +1,8 @@ namespace Nub.Lang.Frontend.Lexing; -public abstract class Token; \ No newline at end of file +public abstract class Token(string filePath, int startIndex, int endIndex) +{ + public string FilePath { get; } = filePath; + public int StartIndex { get; } = startIndex; + public int EndIndex { get; } = endIndex; +} \ No newline at end of file diff --git a/src/compiler/Nub.Lang/Frontend/Parsing/Parser.cs b/src/compiler/Nub.Lang/Frontend/Parsing/Parser.cs index af55174..9b1241b 100644 --- a/src/compiler/Nub.Lang/Frontend/Parsing/Parser.cs +++ b/src/compiler/Nub.Lang/Frontend/Parsing/Parser.cs @@ -8,7 +8,7 @@ public class Parser private List _tokens = []; private int _index; - public ModuleNode ParseModule(List tokens, string path) + public ModuleNode ParseModule(List tokens, string rootFilePath) { _index = 0; _tokens = tokens; @@ -29,7 +29,7 @@ public class Parser } } - return new ModuleNode(path, imports, definitions); + return new ModuleNode(rootFilePath, imports, definitions); } private DefinitionNode ParseDefinition() diff --git a/src/compiler/Nub.Lang/Program.cs b/src/compiler/Nub.Lang/Program.cs index a7335b8..ede202f 100644 --- a/src/compiler/Nub.Lang/Program.cs +++ b/src/compiler/Nub.Lang/Program.cs @@ -54,25 +54,25 @@ internal static class Program return 0; } - private static List RunFrontend(string path) + private static List RunFrontend(string rootFilePath) { List modules = []; - RunFrontend(path, modules); + RunFrontend(rootFilePath, modules); return modules; } - private static void RunFrontend(string path, List modules) + private static void RunFrontend(string rootFilePath, List modules) { - var files = Directory.EnumerateFiles(path, "*.nub", SearchOption.TopDirectoryOnly); + var filePaths = Directory.EnumerateFiles(rootFilePath, "*.nub", SearchOption.TopDirectoryOnly); List tokens = []; - foreach (var file in files) + foreach (var filePath in filePaths) { - var src = File.ReadAllText(file); - tokens.AddRange(Lexer.Lex(src)); + var src = File.ReadAllText(filePath); + tokens.AddRange(Lexer.Lex(src, filePath)); } - var module = Parser.ParseModule(tokens, path); + var module = Parser.ParseModule(tokens, rootFilePath); modules.Add(module); foreach (var import in module.Imports)