nub31/nub-lang-archive-2
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Perf improvements in tokenizer

Browse Source
This commit is contained in:
nub31
2025-09-29 14:11:48 +02:00
parent b5799633bf
commit c74394a849
10 changed files with 219 additions and 274 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
compiler/NubLang.CLI/Options.cs
View File

@@ -1,9 +1,7 @@
using NubLang.Code;
namespace NubLang.CLI;
public class Options
{
public string? OutputPath { get; set; }
public List<SourceFile> Files { get; } = [];
public List<string> Files { get; } = [];
}

21
compiler/NubLang.CLI/Program.cs
View File

@@ -1,6 +1,5 @@
using System.Diagnostics;
using NubLang.CLI;
using NubLang.Code;
using NubLang.Diagnostics;
using NubLang.Generation.QBE;
using NubLang.Modules;
@@ -32,7 +31,7 @@ for (var i = 0; i < args.Length; i++)
}
default:
{
options.Files.Add(new SourceFile(arg));
options.Files.Add(arg);
break;
}
}
@@ -43,7 +42,7 @@ sw.Restart();
foreach (var file in options.Files)
{
if (!File.Exists(file.Path))
if (!File.Exists(file))
{
Console.Error.WriteLine($"File '{file}' does not exist");
return 1;
@@ -58,18 +57,18 @@ var diagnostics = new List<Diagnostic>();
var syntaxTrees = new List<SyntaxTree>();
foreach (var file in options.Files)
{
var tokenizer = new Tokenizer(file);
var tokens = tokenizer.Tokenize().ToList();
diagnostics.AddRange(tokenizer.GetDiagnostics());
var tokenizer = new Tokenizer(file, File.ReadAllText(file));
tokenizer.Tokenize();
diagnostics.AddRange(tokenizer.Diagnostics);
Console.WriteLine($"Tokenize: {Path.GetFileName(file.Path)}: {sw.ElapsedMilliseconds}ms");
Console.WriteLine($" Tokenize: {Path.GetFileName(file)}: {sw.ElapsedMilliseconds}ms");
sw.Restart();
var parser = new Parser();
var syntaxTree = parser.Parse(tokens);
diagnostics.AddRange(parser.GetDiagnostics());
var syntaxTree = parser.Parse(tokenizer.Tokens);
diagnostics.AddRange(parser.Diagnostics);
Console.WriteLine($"Parse: {Path.GetFileName(file.Path)}: {sw.ElapsedMilliseconds}ms");
Console.WriteLine($" Parse: {Path.GetFileName(file)}: {sw.ElapsedMilliseconds}ms");
sw.Restart();
syntaxTrees.Add(syntaxTree);
@@ -91,7 +90,7 @@ foreach (var syntaxTree in syntaxTrees)
var typeChecker = new TypeChecker(syntaxTree, moduleRepository);
typeChecker.Check();
Console.WriteLine($"Type check {syntaxTree.Metadata.ModuleName}: {sw.ElapsedMilliseconds}ms");
Console.WriteLine($" Type check {syntaxTree.Metadata.ModuleName}: {sw.ElapsedMilliseconds}ms");
sw.Restart();
definitions.AddRange(typeChecker.Definitions);