nub31/nub-lang-archive-2
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Perf improvements in tokenizer

Browse Source
This commit is contained in:
nub31
2025-09-29 14:11:48 +02:00
parent b5799633bf
commit c74394a849
10 changed files with 219 additions and 274 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
compiler/NubLang/Parsing/Parser.cs
View File

@@ -7,7 +7,6 @@ namespace NubLang.Parsing;
public sealed class Parser
{
private readonly List<Diagnostic> _diagnostics = [];
private readonly HashSet<string> _templateArguments = [];
private List<Token> _tokens = [];
private int _tokenIndex;
@@ -16,14 +15,11 @@ public sealed class Parser
private Token? CurrentToken => _tokenIndex < _tokens.Count ? _tokens[_tokenIndex] : null;
private bool HasToken => CurrentToken != null;
public List<Diagnostic> GetDiagnostics()
{
return _diagnostics;
}
public List<Diagnostic> Diagnostics { get; } = [];
public SyntaxTree Parse(List<Token> tokens)
{
_diagnostics.Clear();
Diagnostics.Clear();
_tokens = tokens;
_tokenIndex = 0;
_moduleName = string.Empty;
@@ -51,7 +47,7 @@ public sealed class Parser
}
catch (ParseException e)
{
_diagnostics.Add(e.Diagnostic);
Diagnostics.Add(e.Diagnostic);
while (HasToken)
{
if (CurrentToken is SymbolToken { Symbol: Symbol.Module or Symbol.Import })
@@ -102,7 +98,7 @@ public sealed class Parser
}
catch (ParseException e)
{
_diagnostics.Add(e.Diagnostic);
Diagnostics.Add(e.Diagnostic);
while (HasToken)
{
if (CurrentToken is SymbolToken { Symbol: Symbol.Extern or Symbol.Func or Symbol.Struct })
@@ -692,7 +688,7 @@ public sealed class Parser
}
catch (ParseException ex)
{
_diagnostics.Add(ex.Diagnostic);
Diagnostics.Add(ex.Diagnostic);
if (HasToken)
{
Next();