nub31/nub-lang-archive
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

...

Browse Source
This commit is contained in:
nub31
2025-09-29 16:57:25 +02:00
parent c0948e856a
commit d560b632c9
6 changed files with 127 additions and 120 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
compiler/NubLang/Tokenization/Token.cs
View File

@@ -2,16 +2,6 @@
namespace NubLang.Tokenization;
public enum LiteralKind
{
Integer,
Hex,
Binary,
Float,
String,
Bool
}
public enum Symbol
{
Func,
@@ -69,6 +59,12 @@ public abstract record Token(string FileName, SourceSpan Span);
public record IdentifierToken(string FileName, SourceSpan Span, string Value) : Token(FileName, Span);
public record LiteralToken(string FileName, SourceSpan Span, LiteralKind Kind, string Value) : Token(FileName, Span);
public record IntLiteralToken(string FileName, SourceSpan Span, string Value, int Base) : Token(FileName, Span);
public record StringLiteralToken(string FileName, SourceSpan Span, string Value) : Token(FileName, Span);
public record BoolLiteralToken(string FileName, SourceSpan Span, bool Value) : Token(FileName, Span);
public record FloatLiteralToken(string FileName, SourceSpan Span, string Value) : Token(FileName, Span);
public record SymbolToken(string FileName, SourceSpan Span, Symbol Symbol) : Token(FileName, Span);

25
compiler/NubLang/Tokenization/Tokenizer.cs
View File

@@ -148,7 +148,7 @@ public sealed class Tokenizer
if (buffer is "true" or "false")
{
return new LiteralToken(_fileName, CreateSpan(lineStart, columnStart), LiteralKind.Bool, buffer);
return new BoolLiteralToken(_fileName, CreateSpan(lineStart, columnStart), Convert.ToBoolean(buffer));
}
return new IdentifierToken(_fileName, CreateSpan(lineStart, columnStart), buffer);
@@ -156,7 +156,6 @@ public sealed class Tokenizer
if (char.IsDigit(current))
{
var isFloat = false;
var buffer = string.Empty;
if (current == '0' && Peek(1) is 'x')
@@ -178,7 +177,7 @@ public sealed class Tokenizer
.Build());
}
return new LiteralToken(_fileName, CreateSpan(lineStart, columnStart), LiteralKind.Hex, buffer);
return new IntLiteralToken(_fileName, CreateSpan(lineStart, columnStart), buffer, 16);
}
if (current == '0' && Peek(1) is 'b')
@@ -200,10 +199,10 @@ public sealed class Tokenizer
.Build());
}
return new LiteralToken(_fileName, CreateSpan(lineStart, columnStart), LiteralKind.Binary, buffer);
return new IntLiteralToken(_fileName, CreateSpan(lineStart, columnStart), buffer, 2);
}
buffer += current;
var isFloat = false;
while (Peek() != null)
{
var next = Peek()!.Value;
@@ -232,12 +231,14 @@ public sealed class Tokenizer
}
}
return new LiteralToken(
_fileName,
CreateSpan(lineStart, columnStart),
isFloat ? LiteralKind.Float : LiteralKind.Integer,
buffer
);
if (isFloat)
{
return new FloatLiteralToken(_fileName, CreateSpan(lineStart, columnStart), buffer);
}
else
{
return new IntLiteralToken(_fileName, CreateSpan(lineStart, columnStart), buffer, 10);
}
}
if (current == '"')
@@ -272,7 +273,7 @@ public sealed class Tokenizer
Next();
}
return new LiteralToken(_fileName, CreateSpan(lineStart, columnStart), LiteralKind.String, buffer);
return new StringLiteralToken(_fileName, CreateSpan(lineStart, columnStart), buffer);
}
foreach (var (pattern, symbol) in OrderedSymbols)