nub31/nub-lang-archive-2
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Perf improvements in tokenizer

Browse Source
This commit is contained in:
nub31
2025-09-29 14:11:48 +02:00
parent b5799633bf
commit c74394a849
10 changed files with 219 additions and 274 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
compiler/NubLang/Tokenization/Token.cs
View File

@@ -2,22 +2,6 @@
namespace NubLang.Tokenization;
public abstract class Token(SourceFileSpan fileSpan)
{
public SourceFileSpan FileSpan { get; } = fileSpan;
}
public class IdentifierToken(SourceFileSpan fileSpan, string value) : Token(fileSpan)
{
public string Value { get; } = value;
}
public class LiteralToken(SourceFileSpan fileSpan, LiteralKind kind, string value) : Token(fileSpan)
{
public LiteralKind Kind { get; } = kind;
public string Value { get; } = value;
}
public enum LiteralKind
{
Integer,
@@ -26,11 +10,6 @@ public enum LiteralKind
Bool
}
public class SymbolToken(SourceFileSpan fileSpan, Symbol symbol) : Token(fileSpan)
{
public Symbol Symbol { get; } = symbol;
}
public enum Symbol
{
Func,
@@ -83,4 +62,12 @@ public enum Symbol
Defer,
At,
Enum,
}
}
public abstract record Token(string FileName, SourceSpan Span);
public record IdentifierToken(string FileName, SourceSpan Span, string Value) : Token(FileName, Span);
public record LiteralToken(string FileName, SourceSpan Span, LiteralKind Kind, string Value) : Token(FileName, Span);
public record SymbolToken(string FileName, SourceSpan Span, Symbol Symbol) : Token(FileName, Span);