nub31/nub-lang
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Syntax highlighting

Browse Source
This commit is contained in:
nub31
2025-07-23 01:42:52 +02:00
parent f0ab3a5af3
commit 2d6cf0ba7f
3 changed files with 148 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
src/compiler/NubLang/Tokenization/Tokenizer.cs
View File

@@ -57,20 +57,13 @@ public sealed class Tokenizer
.Select(kvp => (kvp.Key, kvp.Value))
.ToArray();
private readonly string _sourceText;
private readonly SourceFile? _sourceFile;
private readonly SourceFile _sourceFile;
private readonly List<Diagnostic> _diagnostics = [];
private int _index;
public Tokenizer(string sourceText)
{
_sourceText = sourceText;
}
public Tokenizer(SourceFile sourceFile)
{
_sourceFile = sourceFile;
_sourceText = sourceFile.GetText();
}
public IReadOnlyList<Diagnostic> GetDiagnostics() => _diagnostics;
@@ -223,9 +216,9 @@ public sealed class Tokenizer
private Optional<char> Peek(int offset = 0)
{
if (_index + offset < _sourceText.Length)
if (_index + offset < _sourceFile.GetText().Length)
{
return _sourceText[_index + offset];
return _sourceFile.GetText()[_index + offset];
}
return Optional<char>.Empty();
@@ -236,16 +229,11 @@ public sealed class Tokenizer
_index++;
}
private SourceFileSpan? GetSourceFileSpan(int tokenStartIndex)
private SourceFileSpan GetSourceFileSpan(int tokenStartIndex)
{
if (_sourceFile != null)
{
var start = CalculateSourceLocation(tokenStartIndex);
var end = CalculateSourceLocation(_index);
return new SourceFileSpan(_sourceFile, new SourceSpan(start, end));
}
return null;
var start = CalculateSourceLocation(tokenStartIndex);
var end = CalculateSourceLocation(_index);
return new SourceFileSpan(_sourceFile, new SourceSpan(start, end));
}
private SourceLocation CalculateSourceLocation(int index)
@@ -253,9 +241,9 @@ public sealed class Tokenizer
var line = 1;
var column = 1;
for (var i = 0; i < index && i < _sourceText.Length; i++)
for (var i = 0; i < index && i < _sourceFile.GetText().Length; i++)
{
if (_sourceText[i] == '\n')
if (_sourceFile.GetText()[i] == '\n')
{
line++;
column = 1;