nub31/nub-lang
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

...

Browse Source
This commit is contained in:
nub31
2026-02-10 23:03:27 +01:00
parent 9d8d8f255a
commit c65fbeba13
8 changed files with 266 additions and 199 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
compiler/Tokenizer.cs
View File

@@ -3,13 +3,21 @@ using System.Text;
namespace Compiler;
public sealed class Tokenizer(string fileName, string contents)
public class Tokenizer
{
public static List<Token>? Tokenize(string fileName, string contents, out List<Diagnostic> diagnostics)
{
return new Tokenizer(fileName, contents).Tokenize(out diagnostics);
}
private Tokenizer(string fileName, string contents)
{
this.fileName = fileName;
this.contents = contents;
}
private readonly string fileName;
private readonly string contents;
private int index;
private int line = 1;
private int column = 1;
@@ -53,7 +61,7 @@ public sealed class Tokenizer(string fileName, string contents)
}
}
if (diagnostics.Any(x => x.Severity == DiagnosticSeverity.Error))
if (diagnostics.Any(x => x.Severity == Diagnostic.DiagnosticSeverity.Error))
return null;
return tokens;
@@ -459,22 +467,22 @@ public abstract class Token(int line, int column, int length)
public int Length { get; } = length;
}
public sealed class TokenIdent(int line, int column, int length, string ident) : Token(line, column, length)
public class TokenIdent(int line, int column, int length, string ident) : Token(line, column, length)
{
public string Ident { get; } = ident;
}
public sealed class TokenIntLiteral(int line, int column, int length, BigInteger value) : Token(line, column, length)
public class TokenIntLiteral(int line, int column, int length, BigInteger value) : Token(line, column, length)
{
public BigInteger Value { get; } = value;
}
public sealed class TokenStringLiteral(int line, int column, int length, string value) : Token(line, column, length)
public class TokenStringLiteral(int line, int column, int length, string value) : Token(line, column, length)
{
public string Value { get; } = value;
}
public sealed class TokenBoolLiteral(int line, int column, int length, bool value) : Token(line, column, length)
public class TokenBoolLiteral(int line, int column, int length, bool value) : Token(line, column, length)
{
public bool Value { get; } = value;
}
@@ -516,7 +524,7 @@ public enum Symbol
PipePipe,
}
public sealed class TokenSymbol(int line, int column, int length, Symbol symbol) : Token(line, column, length)
public class TokenSymbol(int line, int column, int length, Symbol symbol) : Token(line, column, length)
{
public Symbol Symbol { get; } = symbol;
}
@@ -535,7 +543,7 @@ public enum Keyword
Export,
}
public sealed class TokenKeyword(int line, int column, int length, Keyword keyword) : Token(line, column, length)
public class TokenKeyword(int line, int column, int length, Keyword keyword) : Token(line, column, length)
{
public Keyword Keyword { get; } = keyword;
}