nub31/nub-lang-archive-2
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

...

Browse Source
This commit is contained in:
nub31
2025-09-21 20:44:57 +02:00
parent 642d604ecd
commit 822cdf4bda
13 changed files with 101 additions and 232 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
compiler/NubLang/Tokenization/Tokenizer.cs
View File

@@ -82,17 +82,18 @@ public sealed class Tokenizer
{
_index = 0;
while (Peek().TryGetValue(out var current))
while (Peek() != null)
{
var current = Peek()!.Value;
if (char.IsWhiteSpace(current))
{
Next();
continue;
}
if (current == '/' && Peek(1).TryGetValue(out var nextChar) && nextChar == '/')
if (current == '/' && Peek(1) == '/')
{
while (Peek().TryGetValue(out var ch) && ch != '\n')
while (Peek().HasValue && Peek() != '\n')
{
Next();
}
@@ -106,9 +107,9 @@ public sealed class Tokenizer
{
var buffer = string.Empty;
while (Peek().TryGetValue(out var next) && (char.IsLetterOrDigit(next) || next == '_'))
while (Peek() != null && (char.IsLetterOrDigit(Peek()!.Value) || Peek() == '_'))
{
buffer += next;
buffer += Peek();
Next();
}
@@ -133,8 +134,9 @@ public sealed class Tokenizer
var isFloat = false;
var buffer = string.Empty;
while (Peek().TryGetValue(out var next))
while (Peek() != null)
{
var next = Peek()!.Value;
if (next == '.')
{
if (isFloat)
@@ -168,11 +170,12 @@ public sealed class Tokenizer
while (true)
{
if (!Peek().TryGetValue(out var next))
if (Peek() == null)
{
throw new Exception("Unclosed string literal");
}
var next = Peek()!.Value;
if (next == '"')
{
Next();
@@ -224,14 +227,14 @@ public sealed class Tokenizer
}
}
private Optional<char> Peek(int offset = 0)
private char? Peek(int offset = 0)
{
if (_index + offset < _sourceFile.GetText().Length)
{
return _sourceFile.GetText()[_index + offset];
}
return Optional<char>.Empty();
return null;
}
private void Next()