nub31/nub-lang-archive-2
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

...

Browse Source
This commit is contained in:
nub31
2025-09-29 16:57:25 +02:00
parent 428d69d242
commit 0de84375d3
6 changed files with 127 additions and 120 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
compiler/NubLang/Tokenization/Tokenizer.cs
View File

@@ -148,7 +148,7 @@ public sealed class Tokenizer
if (buffer is "true" or "false")
{
return new LiteralToken(_fileName, CreateSpan(lineStart, columnStart), LiteralKind.Bool, buffer);
return new BoolLiteralToken(_fileName, CreateSpan(lineStart, columnStart), Convert.ToBoolean(buffer));
}
return new IdentifierToken(_fileName, CreateSpan(lineStart, columnStart), buffer);
@@ -156,7 +156,6 @@ public sealed class Tokenizer
if (char.IsDigit(current))
{
var isFloat = false;
var buffer = string.Empty;
if (current == '0' && Peek(1) is 'x')
@@ -178,7 +177,7 @@ public sealed class Tokenizer
.Build());
}
return new LiteralToken(_fileName, CreateSpan(lineStart, columnStart), LiteralKind.Hex, buffer);
return new IntLiteralToken(_fileName, CreateSpan(lineStart, columnStart), buffer, 16);
}
if (current == '0' && Peek(1) is 'b')
@@ -200,10 +199,10 @@ public sealed class Tokenizer
.Build());
}
return new LiteralToken(_fileName, CreateSpan(lineStart, columnStart), LiteralKind.Binary, buffer);
return new IntLiteralToken(_fileName, CreateSpan(lineStart, columnStart), buffer, 2);
}
buffer += current;
var isFloat = false;
while (Peek() != null)
{
var next = Peek()!.Value;
@@ -232,12 +231,14 @@ public sealed class Tokenizer
}
}
return new LiteralToken(
_fileName,
CreateSpan(lineStart, columnStart),
isFloat ? LiteralKind.Float : LiteralKind.Integer,
buffer
);
if (isFloat)
{
return new FloatLiteralToken(_fileName, CreateSpan(lineStart, columnStart), buffer);
}
else
{
return new IntLiteralToken(_fileName, CreateSpan(lineStart, columnStart), buffer, 10);
}
}
if (current == '"')
@@ -272,7 +273,7 @@ public sealed class Tokenizer
Next();
}
return new LiteralToken(_fileName, CreateSpan(lineStart, columnStart), LiteralKind.String, buffer);
return new StringLiteralToken(_fileName, CreateSpan(lineStart, columnStart), buffer);
}
foreach (var (pattern, symbol) in OrderedSymbols)