nub31/nub-lang
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

...

Browse Source
This commit is contained in:
nub31
2025-11-03 17:10:15 +01:00
parent 7d49bf43b7
commit 47fef6bc9f
7 changed files with 175 additions and 210 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
compiler/NubLang/Syntax/Parser.cs
View File

@@ -62,7 +62,7 @@ public sealed class Parser
_ => throw new CompileException(Diagnostic
.Error($"Expected 'func', 'struct', 'enum', 'import' or 'module' but found '{keyword.Symbol}'")
.WithHelp("Valid top level statements are 'func', 'struct', 'enum', 'import' and 'module'")
.At(keyword)
.At(keyword, _tokens)
.Build())
};
@@ -83,7 +83,7 @@ public sealed class Parser
}
}
return new SyntaxTree(topLevelSyntaxNodes);
return new SyntaxTree(topLevelSyntaxNodes, _tokens);
}
private ModuleSyntax ParseModule(int startIndex)
@@ -190,7 +190,7 @@ public sealed class Parser
{
throw new CompileException(Diagnostic
.Error("Value of enum field must be an integer literal")
.At(CurrentToken)
.At(CurrentToken, _tokens)
.Build());
}
@@ -461,7 +461,7 @@ public sealed class Parser
IdentifierToken identifier => ParseIdentifier(startIndex, identifier),
SymbolToken symbolToken => symbolToken.Symbol switch
{
Symbol.Ampersand => ParseAddressOf(startIndex),
Symbol.Caret => ParseAddressOf(startIndex),
Symbol.OpenParen => ParseParenthesizedExpression(),
Symbol.Minus => ParseUnaryNegate(startIndex),
Symbol.Bang => ParseUnaryInvert(startIndex),
@@ -472,13 +472,13 @@ public sealed class Parser
_ => throw new CompileException(Diagnostic
.Error($"Unexpected symbol '{symbolToken.Symbol}' in expression")
.WithHelp("Expected '(', '-', '!', '[' or '{'")
.At(symbolToken)
.At(symbolToken, _tokens)
.Build())
},
_ => throw new CompileException(Diagnostic
.Error($"Unexpected token '{token.GetType().Name}' in expression")
.WithHelp("Expected literal, identifier, or parenthesized expression")
.At(token)
.At(token, _tokens)
.Build())
};
@@ -698,7 +698,7 @@ public sealed class Parser
throw new CompileException(Diagnostic
.Error("Arbitrary uint size is not supported")
.WithHelp("Use u8, u16, u32 or u64")
.At(name)
.At(name, _tokens)
.Build());
}
@@ -712,7 +712,7 @@ public sealed class Parser
throw new CompileException(Diagnostic
.Error("Arbitrary int size is not supported")
.WithHelp("Use i8, i16, i32 or i64")
.At(name)
.At(name, _tokens)
.Build());
}
@@ -726,7 +726,7 @@ public sealed class Parser
throw new CompileException(Diagnostic
.Error("Arbitrary float size is not supported")
.WithHelp("Use f32 or f64")
.At(name)
.At(name, _tokens)
.Build());
}
@@ -810,7 +810,7 @@ public sealed class Parser
throw new CompileException(Diagnostic
.Error("Invalid type syntax")
.WithHelp("Expected type name, '^' for pointer, or '[]' for array")
.At(CurrentToken)
.At(CurrentToken, _tokens)
.Build());
}
@@ -821,7 +821,7 @@ public sealed class Parser
throw new CompileException(Diagnostic
.Error("Unexpected end of file")
.WithHelp("Expected more tokens to complete the syntax")
.At(_tokens[^1])
.At(_tokens[^1], _tokens)
.Build());
}
@@ -838,7 +838,7 @@ public sealed class Parser
throw new CompileException(Diagnostic
.Error($"Expected symbol, but found {token.GetType().Name}")
.WithHelp("This position requires a symbol like '(', ')', '{', '}', etc.")
.At(token)
.At(token, _tokens)
.Build());
}
@@ -853,7 +853,7 @@ public sealed class Parser
throw new CompileException(Diagnostic
.Error($"Expected '{expectedSymbol}', but found '{token.Symbol}'")
.WithHelp($"Insert '{expectedSymbol}' here")
.At(token)
.At(token, _tokens)
.Build());
}
}
@@ -903,7 +903,7 @@ public sealed class Parser
throw new CompileException(Diagnostic
.Error($"Expected identifier, but found {token.GetType().Name}")
.WithHelp("Provide a valid identifier name here")
.At(token)
.At(token, _tokens)
.Build());
}
@@ -931,7 +931,7 @@ public sealed class Parser
throw new CompileException(Diagnostic
.Error($"Expected string literal, but found {token.GetType().Name}")
.WithHelp("Provide a valid string literal")
.At(token)
.At(token, _tokens)
.Build());
}
@@ -953,4 +953,4 @@ public sealed class Parser
}
}
public record SyntaxTree(List<TopLevelSyntaxNode> TopLevelSyntaxNodes);
public record SyntaxTree(List<TopLevelSyntaxNode> TopLevelSyntaxNodes, List<Token> Tokens);

53
compiler/NubLang/Syntax/Tokenizer.cs
View File

@@ -43,6 +43,7 @@ public sealed class Tokenizer
private Token ParseToken()
{
var indexStart = _index;
var lineStart = _line;
var columnStart = _column;
@@ -53,7 +54,7 @@ public sealed class Tokenizer
Next();
}
return new WhitespaceToken(CreateSpan(lineStart, columnStart));
return new WhitespaceToken(CreateSpan(indexStart, lineStart, columnStart));
}
if (_content[_index] == '/' && _index + 1 < _content.Length && _content[_index + 1] == '/')
@@ -66,23 +67,23 @@ public sealed class Tokenizer
Next();
}
return new CommentToken(CreateSpan(lineStart, columnStart), _content.AsSpan(startIndex, _index - startIndex).ToString());
return new CommentToken(CreateSpan(indexStart, lineStart, columnStart), _content.AsSpan(startIndex, _index - startIndex).ToString());
}
if (char.IsDigit(_content[_index]))
{
return ParseNumber(lineStart, columnStart);
return ParseNumber(indexStart, lineStart, columnStart);
}
if (_content[_index] == '"')
{
return ParseString(lineStart, columnStart);
return ParseString(indexStart, lineStart, columnStart);
}
// note(nub31): Look for keywords (longest first in case a keyword fits partially in a larger keyword)
for (var i = 8; i >= 1; i--)
{
if (TryMatchSymbol(i, lineStart, columnStart, out var token))
if (TryMatchSymbol(i, indexStart, lineStart, columnStart, out var token))
{
return token;
}
@@ -90,13 +91,13 @@ public sealed class Tokenizer
if (char.IsLetter(_content[_index]) || _content[_index] == '_')
{
return ParseIdentifier(lineStart, columnStart);
return ParseIdentifier(indexStart, lineStart, columnStart);
}
throw new CompileException(Diagnostic.Error($"Unknown token '{_content[_index]}'").Build());
}
private Token ParseNumber(int lineStart, int columnStart)
private Token ParseNumber(int indexStart, int lineStart, int columnStart)
{
var start = _index;
var current = _content[_index];
@@ -116,12 +117,12 @@ public sealed class Tokenizer
{
throw new CompileException(Diagnostic
.Error("Invalid hex literal, no digits found")
.At(_fileName, _line, _column)
.At(CreateSpan(_index, _line, _column))
.Build());
}
return new IntLiteralToken(
CreateSpan(lineStart, columnStart),
CreateSpan(indexStart, lineStart, columnStart),
_content.Substring(start, _index - start),
16);
}
@@ -141,12 +142,12 @@ public sealed class Tokenizer
{
throw new CompileException(Diagnostic
.Error("Invalid binary literal, no digits found")
.At(_fileName, _line, _column)
.At(CreateSpan(_index, _line, _column))
.Build());
}
return new IntLiteralToken(
CreateSpan(lineStart, columnStart),
CreateSpan(indexStart, lineStart, columnStart),
_content.Substring(start, _index - start),
2);
}
@@ -163,7 +164,7 @@ public sealed class Tokenizer
{
throw new CompileException(Diagnostic
.Error("More than one period found in float literal")
.At(_fileName, _line, _column)
.At(CreateSpan(_index, _line, _column))
.Build());
}
@@ -183,11 +184,11 @@ public sealed class Tokenizer
var buffer = _content.Substring(start, _index - start);
return isFloat
? new FloatLiteralToken(CreateSpan(lineStart, columnStart), buffer)
: new IntLiteralToken(CreateSpan(lineStart, columnStart), buffer, 10);
? new FloatLiteralToken(CreateSpan(indexStart, lineStart, columnStart), buffer)
: new IntLiteralToken(CreateSpan(indexStart, lineStart, columnStart), buffer, 10);
}
private StringLiteralToken ParseString(int lineStart, int columnStart)
private StringLiteralToken ParseString(int indexStart, int lineStart, int columnStart)
{
Next();
var start = _index;
@@ -198,7 +199,7 @@ public sealed class Tokenizer
{
throw new CompileException(Diagnostic
.Error("Unclosed string literal")
.At(_fileName, _line, _column)
.At(CreateSpan(_index, _line, _column))
.Build());
}
@@ -208,7 +209,7 @@ public sealed class Tokenizer
{
throw new CompileException(Diagnostic
.Error("Unclosed string literal (newline found)")
.At(_fileName, _line, _column)
.At(CreateSpan(_index, _line, _column))
.Build());
}
@@ -216,14 +217,14 @@ public sealed class Tokenizer
{
var buffer = _content.Substring(start, _index - start);
Next();
return new StringLiteralToken(CreateSpan(lineStart, columnStart), buffer);
return new StringLiteralToken(CreateSpan(indexStart, lineStart, columnStart), buffer);
}
Next();
}
}
private bool TryMatchSymbol(int length, int lineStart, int columnStart, out Token token)
private bool TryMatchSymbol(int length, int indexStart, int lineStart, int columnStart, out Token token)
{
token = null!;
@@ -237,14 +238,14 @@ public sealed class Tokenizer
if (span is "true")
{
Next(4);
token = new BoolLiteralToken(CreateSpan(lineStart, columnStart), true);
token = new BoolLiteralToken(CreateSpan(indexStart, lineStart, columnStart), true);
return true;
}
if (span is "false")
{
Next(5);
token = new BoolLiteralToken(CreateSpan(lineStart, columnStart), false);
token = new BoolLiteralToken(CreateSpan(indexStart, lineStart, columnStart), false);
return true;
}
@@ -349,14 +350,14 @@ public sealed class Tokenizer
}
Next(length);
token = new SymbolToken(CreateSpan(lineStart, columnStart), symbol);
token = new SymbolToken(CreateSpan(indexStart, lineStart, columnStart), symbol);
return true;
}
return false;
}
private IdentifierToken ParseIdentifier(int lineStart, int columnStart)
private IdentifierToken ParseIdentifier(int indexStart, int lineStart, int columnStart)
{
var start = _index;
@@ -373,12 +374,12 @@ public sealed class Tokenizer
}
}
return new IdentifierToken(CreateSpan(lineStart, columnStart), _content.Substring(start, _index - start));
return new IdentifierToken(CreateSpan(indexStart, lineStart, columnStart), _content.Substring(start, _index - start));
}
private SourceSpan CreateSpan(int lineStart, int columnStart)
private SourceSpan CreateSpan(int indexStart, int lineStart, int columnStart)
{
return new SourceSpan(_fileName, new SourceLocation(lineStart, columnStart), new SourceLocation(_line, _column));
return new SourceSpan(_fileName, _content, indexStart, Math.Min(_index, _content.Length), lineStart, columnStart, _line, _column);
}
private void Next(int count = 1)