nub31/nub-lang-archive-2
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

...

Browse Source
This commit is contained in:
nub31
2025-09-29 16:23:39 +02:00
parent 933c52ac1e
commit 428d69d242
13 changed files with 568 additions and 474 deletions
Download Patch File Download Diff File Expand all files Collapse all files

52
compiler/NubLang/Tokenization/Tokenizer.cs
View File

@@ -159,6 +159,51 @@ public sealed class Tokenizer
var isFloat = false;
var buffer = string.Empty;
if (current == '0' && Peek(1) is 'x')
{
buffer += "0x";
Next();
Next();
while (Peek() != null && Uri.IsHexDigit(Peek()!.Value))
{
buffer += Peek()!.Value;
Next();
}
if (buffer.Length <= 2)
{
throw new TokenizerException(Diagnostic
.Error("Invalid hex literal, no digits found")
.At(_fileName, _line, _column)
.Build());
}
return new LiteralToken(_fileName, CreateSpan(lineStart, columnStart), LiteralKind.Hex, buffer);
}
if (current == '0' && Peek(1) is 'b')
{
buffer += "0b";
Next();
Next();
while (Peek() != null && (Peek() == '0' || Peek() == '1'))
{
buffer += Peek()!.Value;
Next();
}
if (buffer.Length <= 2)
{
throw new TokenizerException(Diagnostic
.Error("Invalid binary literal, no digits found")
.At(_fileName, _line, _column)
.Build());
}
return new LiteralToken(_fileName, CreateSpan(lineStart, columnStart), LiteralKind.Binary, buffer);
}
buffer += current;
while (Peek() != null)
{
var next = Peek()!.Value;
@@ -187,7 +232,12 @@ public sealed class Tokenizer
}
}
return new LiteralToken(_fileName, CreateSpan(lineStart, columnStart), isFloat ? LiteralKind.Float : LiteralKind.Integer, buffer);
return new LiteralToken(
_fileName,
CreateSpan(lineStart, columnStart),
isFloat ? LiteralKind.Float : LiteralKind.Integer,
buffer
);
}
if (current == '"')