nub31/nub-lang
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

...

Browse Source
This commit is contained in:
nub31
2025-10-31 15:18:18 +01:00
parent 7c7624b1bc
commit 40d500fddd
8 changed files with 231 additions and 188 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
compiler/NubLang/Syntax/Tokenizer.cs
View File

@@ -4,25 +4,23 @@ namespace NubLang.Syntax;
public sealed class Tokenizer
{
private readonly string _fileName;
private readonly string _content;
private string _fileName = null!;
private string _content = null!;
private int _index;
private int _line = 1;
private int _column = 1;
public Tokenizer(string fileName, string content)
public List<Diagnostic> Diagnostics { get; set; } = new(16);
public List<Token> Tokens { get; set; } = new(256);
public void Tokenize(string fileName, string content)
{
_fileName = fileName;
_content = content;
}
public List<Diagnostic> Diagnostics { get; } = new(16);
public List<Token> Tokens { get; } = new(256);
Diagnostics = [];
Tokens = [];
public void Tokenize()
{
Diagnostics.Clear();
Tokens.Clear();
_index = 0;
_line = 1;
_column = 1;