nub31/nub-lang-archive-2
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Store src data in tokens

Browse Source
This commit is contained in:
nub31
2025-05-24 21:12:44 +02:00
parent 1ff434e1c3
commit 26906ea689
9 changed files with 44 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/compiler/Nub.Lang/Frontend/Lexing/DocumentationToken.cs
View File

@@ -1,6 +1,6 @@
namespace Nub.Lang.Frontend.Lexing;
public class DocumentationToken(string documentation) : Token
public class DocumentationToken(string filePath, int startIndex, int endIndex, string documentation) : Token(filePath, startIndex, endIndex)
{
public string Documentation { get; } = documentation;
}

2
src/compiler/Nub.Lang/Frontend/Lexing/IdentifierToken.cs
View File

@@ -1,6 +1,6 @@
namespace Nub.Lang.Frontend.Lexing;
public class IdentifierToken(string value) : Token
public class IdentifierToken(string filePath, int startIndex, int endIndex, string value) : Token(filePath, startIndex, endIndex)
{
public string Value { get; } = value;
}

49
src/compiler/Nub.Lang/Frontend/Lexing/Lexer.cs
View File

@@ -16,7 +16,7 @@ public class Lexer
["struct"] = Symbol.Struct,
};
private static readonly Dictionary<string, Modifier> Modifers = new()
private static readonly Dictionary<string, Modifier> Modifiers = new()
{
["global"] = Modifier.Global,
["extern"] = Modifier.Extern,
@@ -54,14 +54,16 @@ public class Lexer
['&'] = Symbol.Ampersand,
};
private string _src = string.Empty;
private string _src = null!;
private string _filePath = null!;
private int _index;
public List<Token> Lex(string src)
public List<Token> Lex(string src, string filePath)
{
_src = src;
_filePath = filePath;
_index = 0;
List<Token> tokens = [];
while (ParseToken().TryGetValue(out var token))
{
@@ -82,6 +84,7 @@ public class Lexer
private Optional<Token> ParseToken()
{
ConsumeWhitespace();
var startIndex = _index;
string? documentation = null;
while (Peek() is { Value: '/' } && Peek(1) is { Value: '/' })
@@ -93,44 +96,36 @@ public class Lexer
{
Next();
if (documentation != null)
{
documentation += '\n';
}
while (Peek().TryGetValue(out var character))
{
Next();
documentation += character;
if (character == '\n')
{
Next();
break;
}
documentation += character;
Next();
}
}
else
{
while (Peek().TryGetValue(out var character))
{
Next();
if (character == '\n')
{
Next();
break;
}
Next();
}
}
}
if (documentation != null)
{
return new DocumentationToken(documentation);
return new DocumentationToken(_filePath, startIndex, _index, documentation);
}
ConsumeWhitespace();
startIndex = _index;
if (!Peek().TryGetValue(out var current))
{
@@ -149,20 +144,20 @@ public class Lexer
if (Keywords.TryGetValue(buffer, out var keywordSymbol))
{
return new SymbolToken(keywordSymbol);
return new SymbolToken(_filePath, startIndex, _index, keywordSymbol);
}
if (Modifers.TryGetValue(buffer, out var modifer))
if (Modifiers.TryGetValue(buffer, out var modifer))
{
return new ModifierToken(modifer);
return new ModifierToken(_filePath, startIndex, _index, modifer);
}
if (buffer is "true" or "false")
{
return new LiteralToken(NubPrimitiveType.Bool, buffer);
return new LiteralToken(_filePath, startIndex, _index, NubPrimitiveType.Bool, buffer);
}
return new IdentifierToken(buffer);
return new IdentifierToken(_filePath, startIndex, _index, buffer);
}
if (char.IsDigit(current))
@@ -200,7 +195,7 @@ public class Lexer
}
}
return new LiteralToken(isFloat ? NubPrimitiveType.F64 : NubPrimitiveType.I64, buffer);
return new LiteralToken(_filePath, startIndex, _index, isFloat ? NubPrimitiveType.F64 : NubPrimitiveType.I64, buffer);
}
// TODO: Revisit this
@@ -220,7 +215,7 @@ public class Lexer
Next();
}
return new SymbolToken(chain.Value);
return new SymbolToken(_filePath, startIndex, _index, chain.Value);
}
}
}
@@ -228,7 +223,7 @@ public class Lexer
if (Chars.TryGetValue(current, out var charSymbol))
{
Next();
return new SymbolToken(charSymbol);
return new SymbolToken(_filePath, startIndex, _index, charSymbol);
}
if (current == '"')
@@ -253,7 +248,7 @@ public class Lexer
Next();
}
return new LiteralToken(NubPrimitiveType.String, buffer);
return new LiteralToken(_filePath, startIndex, _index, NubPrimitiveType.String, buffer);
}
throw new Exception($"Unknown character {current}");

2
src/compiler/Nub.Lang/Frontend/Lexing/LiteralToken.cs
View File

@@ -1,6 +1,6 @@
namespace Nub.Lang.Frontend.Lexing;
public class LiteralToken(NubType type, string value) : Token
public class LiteralToken(string filePath, int startIndex, int endIndex, NubType type, string value) : Token(filePath, startIndex, endIndex)
{
public NubType Type { get; } = type;
public string Value { get; } = value;

4
src/compiler/Nub.Lang/Frontend/Lexing/ModifierToken.cs
View File

@@ -1,8 +1,8 @@
namespace Nub.Lang.Frontend.Lexing;
public class ModifierToken(Modifier symbol) : Token
public class ModifierToken(string filePath, int startIndex, int endIndex, Modifier modifier) : Token(filePath, startIndex, endIndex)
{
public Modifier Modifier { get; } = symbol;
public Modifier Modifier { get; } = modifier;
}
public enum Modifier

2
src/compiler/Nub.Lang/Frontend/Lexing/SymbolToken.cs
View File

@@ -1,6 +1,6 @@
namespace Nub.Lang.Frontend.Lexing;
public class SymbolToken(Symbol symbol) : Token
public class SymbolToken(string filePath, int startIndex, int endIndex, Symbol symbol) : Token(filePath, startIndex, endIndex)
{
public Symbol Symbol { get; } = symbol;
}

7
src/compiler/Nub.Lang/Frontend/Lexing/Token.cs
View File

@@ -1,3 +1,8 @@
namespace Nub.Lang.Frontend.Lexing;
public abstract class Token;
public abstract class Token(string filePath, int startIndex, int endIndex)
{
public string FilePath { get; } = filePath;
public int StartIndex { get; } = startIndex;
public int EndIndex { get; } = endIndex;
}

4
src/compiler/Nub.Lang/Frontend/Parsing/Parser.cs
View File

@@ -8,7 +8,7 @@ public class Parser
private List<Token> _tokens = [];
private int _index;
public ModuleNode ParseModule(List<Token> tokens, string path)
public ModuleNode ParseModule(List<Token> tokens, string rootFilePath)
{
_index = 0;
_tokens = tokens;
@@ -29,7 +29,7 @@ public class Parser
}
}
return new ModuleNode(path, imports, definitions);
return new ModuleNode(rootFilePath, imports, definitions);
}
private DefinitionNode ParseDefinition()

16
src/compiler/Nub.Lang/Program.cs
View File

@@ -54,25 +54,25 @@ internal static class Program
return 0;
}
private static List<ModuleNode> RunFrontend(string path)
private static List<ModuleNode> RunFrontend(string rootFilePath)
{
List<ModuleNode> modules = [];
RunFrontend(path, modules);
RunFrontend(rootFilePath, modules);
return modules;
}
private static void RunFrontend(string path, List<ModuleNode> modules)
private static void RunFrontend(string rootFilePath, List<ModuleNode> modules)
{
var files = Directory.EnumerateFiles(path, "*.nub", SearchOption.TopDirectoryOnly);
var filePaths = Directory.EnumerateFiles(rootFilePath, "*.nub", SearchOption.TopDirectoryOnly);
List<Token> tokens = [];
foreach (var file in files)
foreach (var filePath in filePaths)
{
var src = File.ReadAllText(file);
tokens.AddRange(Lexer.Lex(src));
var src = File.ReadAllText(filePath);
tokens.AddRange(Lexer.Lex(src, filePath));
}
var module = Parser.ParseModule(tokens, path);
var module = Parser.ParseModule(tokens, rootFilePath);
modules.Add(module);
foreach (var import in module.Imports)