...

compiler/NubLang/Tokenization/Token.cs

@@ -0,0 +1,82 @@
+using NubLang.Code;
+
+namespace NubLang.Tokenization;
+
+public abstract class Token(SourceFileSpan fileSpan)
+{
+    public SourceFileSpan FileSpan { get; } = fileSpan;
+}
+
+public class IdentifierToken(SourceFileSpan fileSpan, string value) : Token(fileSpan)
+{
+    public string Value { get; } = value;
+}
+
+public class LiteralToken(SourceFileSpan fileSpan, LiteralKind kind, string value) : Token(fileSpan)
+{
+    public LiteralKind Kind { get; } = kind;
+    public string Value { get; } = value;
+}
+
+public enum LiteralKind
+{
+    Integer,
+    Float,
+    String,
+    Bool
+}
+
+public class SymbolToken(SourceFileSpan fileSpan, Symbol symbol) : Token(fileSpan)
+{
+    public Symbol Symbol { get; } = symbol;
+}
+
+public enum Symbol
+{
+    Func,
+    Return,
+    If,
+    Else,
+    While,
+    Break,
+    Continue,
+    Colon,
+    OpenParen,
+    CloseParen,
+    OpenBrace,
+    CloseBrace,
+    OpenBracket,
+    CloseBracket,
+    Comma,
+    Period,
+    Assign,
+    Bang,
+    Equal,
+    NotEqual,
+    LessThan,
+    LessThanOrEqual,
+    GreaterThan,
+    GreaterThanOrEqual,
+    Plus,
+    Minus,
+    Star,
+    ForwardSlash,
+    Struct,
+    Caret,
+    Ampersand,
+    Let,
+    Calls,
+    Interface,
+    For,
+    Extern,
+    Semi,
+    Percent,
+    LeftShift,
+    RightShift,
+    Pipe,
+    And,
+    Or,
+    Module,
+    Import,
+    Export,
+}