nub31/nub-lang-archive-2
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Parser diagnostics

Browse Source
This commit is contained in:
nub31
2025-07-23 01:23:10 +02:00
parent 5e78f63e93
commit f0ab3a5af3
10 changed files with 230 additions and 156 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
example/src/main.nub
View File

@@ -1,11 +1,5 @@
func main(args: []cstring): i64
{
puts("test") %%%
puts("test" =)
return 0
}

8
src/compiler/NubLang.CLI/Program.cs
View File

@@ -80,10 +80,12 @@ foreach (var file in options.Files)
foreach (var file in options.Files)
{
var tokenizer = new Tokenizer(file);
var parser = new Parser();
var syntaxTree = parser.Parse(tokenizer.Tokenize());
var tokens = tokenizer.Tokenize().ToList();
diagnostics.AddRange(tokenizer.GetDiagnostics());
var parser = new Parser();
var syntaxTree = parser.Parse(tokens);
diagnostics.AddRange(parser.GetDiagnostics());
syntaxTrees.Add(syntaxTree);

32
src/compiler/NubLang/Diagnostics/Diagnostic.cs
View File

@@ -1,5 +1,6 @@
using System.Text;
using NubLang.Code;
using NubLang.Parsing.Syntax;
using NubLang.Tokenization;
namespace NubLang.Diagnostics;
@@ -19,9 +20,28 @@ public class Diagnostic
_message = message;
}
public DiagnosticBuilder At(Token token)
public DiagnosticBuilder At(SyntaxNode? node)
{
if (node != null)
{
var first = node.Tokens.FirstOrDefault();
if (first?.FileSpan != null)
{
var span = SourceSpan.Merge(node.Tokens.Select(x => x.FileSpan?.Span ?? SourceSpan.Zero));
At(new SourceFileSpan(first.FileSpan.SourceFile, span));
}
}
return this;
}
public DiagnosticBuilder At(Token? token)
{
if (token != null)
{
At(token.FileSpan);
}
return this;
}
@@ -130,11 +150,19 @@ public class Diagnostic
var markerLength = markerEndColumn - markerStartColumn;
var marker = new string('^', markerLength);
var markerColor = Severity switch
{
DiagnosticSeverity.Info => ConsoleColors.Blue,
DiagnosticSeverity.Warning => ConsoleColors.Yellow,
DiagnosticSeverity.Error => ConsoleColors.Red,
_ => ConsoleColors.White
};
sb.Append("│ ");
sb.Append(new string(' ', numberPadding));
sb.Append(" │ ");
sb.Append(new string(' ', markerStartColumn - 1));
sb.Append(ConsoleColors.Colorize(marker, ConsoleColors.Red));
sb.Append(ConsoleColors.Colorize(marker, markerColor));
sb.Append(new string(' ', codePadding - markerEndColumn + 1));
sb.Append(" │");
sb.AppendLine();

236
src/compiler/NubLang/Parsing/Parser.cs
View File

@@ -7,40 +7,42 @@ namespace NubLang.Parsing;
public sealed class Parser
{
private IEnumerator<Token> _tokenEnumerator = null!;
private readonly List<Diagnostic> _diagnostics = [];
private Token? _currentToken;
private bool _hasCurrentToken;
private IReadOnlyList<Token> _tokens = [];
private int _tokenIndex;
private Token? CurrentToken => _tokenIndex < _tokens.Count ? _tokens[_tokenIndex] : null;
private bool HasToken => CurrentToken != null;
public IReadOnlyList<Diagnostic> GetDiagnostics()
{
return _diagnostics;
}
public SyntaxTree Parse(IEnumerable<Token> tokens)
public SyntaxTree Parse(IReadOnlyList<Token> tokens)
{
_diagnostics.Clear();
_tokenEnumerator = tokens.GetEnumerator();
_hasCurrentToken = _tokenEnumerator.MoveNext();
_currentToken = _hasCurrentToken ? _tokenEnumerator.Current : null;
_tokens = tokens;
_tokenIndex = 0;
var definitions = new List<DefinitionSyntax>();
while (_hasCurrentToken)
while (HasToken)
{
try
{
var startIndex = _tokenIndex;
var keyword = ExpectSymbol();
var definition = keyword.Symbol switch
{
Symbol.Extern => ParseExtern(),
Symbol.Func => ParseFunc(),
Symbol.Struct => ParseStruct(),
Symbol.Interface => ParseInterface(),
Symbol.Extern => ParseExtern(startIndex),
Symbol.Func => ParseFunc(startIndex),
Symbol.Struct => ParseStruct(startIndex),
Symbol.Interface => ParseInterface(startIndex),
_ => throw new ParseException(Diagnostic
.Error($"Expected 'extern', 'func', 'struct' or 'interface' but found '{keyword.Symbol}'")
.WithHelp("Valid definition keywords are 'extern', 'func', 'struct' and 'interface'")
.At(keyword)
.Build())
};
@@ -49,9 +51,9 @@ public sealed class Parser
catch (ParseException ex)
{
_diagnostics.Add(ex.Diagnostic);
while (_hasCurrentToken)
while (HasToken)
{
if (_currentToken is SymbolToken { Symbol: Symbol.Extern or Symbol.Func or Symbol.Struct or Symbol.Interface })
if (CurrentToken is SymbolToken { Symbol: Symbol.Extern or Symbol.Func or Symbol.Struct or Symbol.Interface })
{
break;
}
@@ -61,11 +63,12 @@ public sealed class Parser
}
}
return new SyntaxTree(definitions);
return new SyntaxTree(GetTokens(_tokenIndex), definitions);
}
private FuncSignatureSyntax ParseFuncSignature(FuncParameterSyntax? thisArg = null)
{
var startIndex = _tokenIndex;
List<FuncParameterSyntax> parameters = [];
if (thisArg != null)
@@ -77,43 +80,49 @@ public sealed class Parser
while (!TryExpectSymbol(Symbol.CloseParen))
{
parameters.Add(ParseFuncParameter());
var parameter = ParseFuncParameter();
parameters.Add(parameter);
if (!TryExpectSymbol(Symbol.Comma) && _currentToken is not SymbolToken { Symbol: Symbol.CloseParen })
if (!TryExpectSymbol(Symbol.Comma) && CurrentToken is not SymbolToken { Symbol: Symbol.CloseParen })
{
_diagnostics.Add(Diagnostic
.Warning("Missing comma between function parameters")
.WithHelp("Add a ',' to separate parameters")
.At(CurrentToken)
.Build());
}
}
var returnType = TryExpectSymbol(Symbol.Colon) ? ParseType() : new VoidTypeSyntax();
var returnType = TryExpectSymbol(Symbol.Colon) ? ParseType() : new VoidTypeSyntax([]);
return new FuncSignatureSyntax(parameters, returnType);
return new FuncSignatureSyntax(GetTokens(startIndex), parameters, returnType);
}
private FuncParameterSyntax ParseFuncParameter()
{
var startIndex = _tokenIndex;
var name = ExpectIdentifier();
ExpectSymbol(Symbol.Colon);
var type = ParseType();
return new FuncParameterSyntax(name.Value, type);
return new FuncParameterSyntax(GetTokens(startIndex), name.Value, type);
}
private DefinitionSyntax ParseExtern()
private DefinitionSyntax ParseExtern(int startIndex)
{
var keyword = ExpectSymbol();
return keyword.Symbol switch
{
Symbol.Func => ParseExternFunc(),
_ => throw new ParseException(Diagnostic.Error($"Unexpected symbol {keyword.Symbol} after extern declaration").Build())
Symbol.Func => ParseExternFunc(startIndex),
_ => throw new ParseException(Diagnostic
.Error($"Unexpected symbol {keyword.Symbol} after extern declaration")
.At(keyword)
.Build())
};
}
private ExternFuncSyntax ParseExternFunc()
private ExternFuncSyntax ParseExternFunc(int startIndex)
{
var name = ExpectIdentifier();
@@ -126,19 +135,19 @@ public sealed class Parser
var signature = ParseFuncSignature();
return new ExternFuncSyntax(name.Value, callName, signature);
return new ExternFuncSyntax(GetTokens(startIndex), name.Value, callName, signature);
}
private LocalFuncSyntax ParseFunc()
private LocalFuncSyntax ParseFunc(int startIndex)
{
var name = ExpectIdentifier();
var signature = ParseFuncSignature();
var body = ParseBlock();
return new LocalFuncSyntax(name.Value, signature, body);
return new LocalFuncSyntax(GetTokens(startIndex), name.Value, signature, body);
}
private DefinitionSyntax ParseStruct()
private DefinitionSyntax ParseStruct(int startIndex)
{
var name = ExpectIdentifier();
@@ -151,14 +160,16 @@ public sealed class Parser
while (!TryExpectSymbol(Symbol.CloseBrace))
{
var memberStartIndex = _tokenIndex;
if (TryExpectSymbol(Symbol.Func))
{
var funcName = ExpectIdentifier().Value;
var thisArg = new FuncParameterSyntax("this", new CustomTypeSyntax(name.Value));
var thisArg = new FuncParameterSyntax([], "this", new CustomTypeSyntax([], name.Value));
var funcSignature = ParseFuncSignature(thisArg);
var funcBody = ParseBlock();
funcs.Add(new StructFuncSyntax(funcName, funcSignature, funcBody));
funcs.Add(new StructFuncSyntax(GetTokens(memberStartIndex), funcName, funcSignature, funcBody));
}
else
{
@@ -173,14 +184,14 @@ public sealed class Parser
fieldValue = ParseExpression();
}
fields.Add(new StructFieldSyntax(fieldIndex++, fieldName, fieldType, fieldValue));
fields.Add(new StructFieldSyntax(GetTokens(memberStartIndex), fieldIndex++, fieldName, fieldType, fieldValue));
}
}
return new StructSyntax(name.Value, fields, funcs);
return new StructSyntax(GetTokens(startIndex), name.Value, fields, funcs);
}
private InterfaceSyntax ParseInterface()
private InterfaceSyntax ParseInterface(int startIndex)
{
var name = ExpectIdentifier();
@@ -190,20 +201,22 @@ public sealed class Parser
while (!TryExpectSymbol(Symbol.CloseBrace))
{
var funcStartIndex = _tokenIndex;
ExpectSymbol(Symbol.Func);
var funcName = ExpectIdentifier().Value;
var signature = ParseFuncSignature();
functions.Add(new InterfaceFuncSyntax(funcName, signature));
functions.Add(new InterfaceFuncSyntax(GetTokens(funcStartIndex), funcName, signature));
}
return new InterfaceSyntax(name.Value, functions);
return new InterfaceSyntax(GetTokens(startIndex), name.Value, functions);
}
private StatementSyntax ParseStatement()
{
if (_currentToken is SymbolToken symbol)
if (CurrentToken is SymbolToken symbol)
{
switch (symbol.Symbol)
{
@@ -227,19 +240,21 @@ public sealed class Parser
private StatementSyntax ParseStatementExpression()
{
var startIndex = _tokenIndex;
var expr = ParseExpression();
if (TryExpectSymbol(Symbol.Assign))
{
var value = ParseExpression();
return new AssignmentSyntax(expr, value);
return new AssignmentSyntax(GetTokens(startIndex), expr, value);
}
return new StatementExpressionSyntax(expr);
return new StatementExpressionSyntax(GetTokens(startIndex), expr);
}
private VariableDeclarationSyntax ParseVariableDeclaration()
{
var startIndex = _tokenIndex;
ExpectSymbol(Symbol.Let);
var name = ExpectIdentifier().Value;
@@ -255,23 +270,26 @@ public sealed class Parser
assignment = ParseExpression();
}
return new VariableDeclarationSyntax(name, explicitType, assignment);
return new VariableDeclarationSyntax(GetTokens(startIndex), name, explicitType, assignment);
}
private StatementSyntax ParseBreak()
{
var startIndex = _tokenIndex;
ExpectSymbol(Symbol.Break);
return new BreakSyntax();
return new BreakSyntax(GetTokens(startIndex));
}
private StatementSyntax ParseContinue()
{
var startIndex = _tokenIndex;
ExpectSymbol(Symbol.Continue);
return new ContinueSyntax();
return new ContinueSyntax(GetTokens(startIndex));
}
private ReturnSyntax ParseReturn()
{
var startIndex = _tokenIndex;
ExpectSymbol(Symbol.Return);
var value = Optional<ExpressionSyntax>.Empty();
@@ -281,11 +299,12 @@ public sealed class Parser
value = ParseExpression();
}
return new ReturnSyntax(value);
return new ReturnSyntax(GetTokens(startIndex), value);
}
private IfSyntax ParseIf()
{
var startIndex = _tokenIndex;
ExpectSymbol(Symbol.If);
var condition = ParseExpression();
var body = ParseBlock();
@@ -298,26 +317,28 @@ public sealed class Parser
: (Variant<IfSyntax, BlockSyntax>)ParseBlock();
}
return new IfSyntax(condition, body, elseStatement);
return new IfSyntax(GetTokens(startIndex), condition, body, elseStatement);
}
private WhileSyntax ParseWhile()
{
var startIndex = _tokenIndex;
ExpectSymbol(Symbol.While);
var condition = ParseExpression();
var body = ParseBlock();
return new WhileSyntax(condition, body);
return new WhileSyntax(GetTokens(startIndex), condition, body);
}
private ExpressionSyntax ParseExpression(int precedence = 0)
{
var startIndex = _tokenIndex;
var left = ParsePrimaryExpression();
while (_currentToken is SymbolToken symbolToken && TryGetBinaryOperator(symbolToken.Symbol, out var op) && GetBinaryOperatorPrecedence(op.Value) >= precedence)
while (CurrentToken is SymbolToken symbolToken && TryGetBinaryOperator(symbolToken.Symbol, out var op) && GetBinaryOperatorPrecedence(op.Value) >= precedence)
{
Next();
var right = ParseExpression(GetBinaryOperatorPrecedence(op.Value) + 1);
left = new BinaryExpressionSyntax(left, op.Value, right);
left = new BinaryExpressionSyntax(GetTokens(startIndex), left, op.Value, right);
}
return left;
@@ -383,27 +404,30 @@ public sealed class Parser
private ExpressionSyntax ParsePrimaryExpression()
{
var startIndex = _tokenIndex;
var token = ExpectToken();
var expr = token switch
{
LiteralToken literal => new LiteralSyntax(literal.Value, literal.Kind),
IdentifierToken identifier => new IdentifierSyntax(identifier.Value),
LiteralToken literal => new LiteralSyntax(GetTokens(startIndex), literal.Value, literal.Kind),
IdentifierToken identifier => new IdentifierSyntax(GetTokens(startIndex), identifier.Value),
SymbolToken symbolToken => symbolToken.Symbol switch
{
Symbol.Func => ParseArrowFunction(),
Symbol.OpenParen => ParseParenthesizedExpression(),
Symbol.Minus => new UnaryExpressionSyntax(UnaryOperatorSyntax.Negate, ParsePrimaryExpression()),
Symbol.Bang => new UnaryExpressionSyntax(UnaryOperatorSyntax.Invert, ParsePrimaryExpression()),
Symbol.OpenBracket => ParseArrayInitializer(),
Symbol.Alloc => ParseStructInitializer(),
Symbol.Minus => new UnaryExpressionSyntax(GetTokens(startIndex), UnaryOperatorSyntax.Negate, ParsePrimaryExpression()),
Symbol.Bang => new UnaryExpressionSyntax(GetTokens(startIndex), UnaryOperatorSyntax.Invert, ParsePrimaryExpression()),
Symbol.OpenBracket => ParseArrayInitializer(startIndex),
Symbol.Alloc => ParseStructInitializer(startIndex),
_ => throw new ParseException(Diagnostic
.Error($"Unexpected symbol '{symbolToken.Symbol}' in expression")
.WithHelp("Expected literal, identifier, or '(' to start expression")
.At(symbolToken)
.Build())
},
_ => throw new ParseException(Diagnostic
.Error($"Unexpected token '{token.GetType().Name}' in expression")
.WithHelp("Expected literal, identifier, or parenthesized expression")
.At(token)
.Build())
};
@@ -412,29 +436,32 @@ public sealed class Parser
private ExpressionSyntax ParseArrowFunction()
{
var startIndex = _tokenIndex;
List<ArrowFuncParameterSyntax> parameters = [];
ExpectSymbol(Symbol.OpenParen);
while (!TryExpectSymbol(Symbol.CloseParen))
{
var parameterIndex = _tokenIndex;
var name = ExpectIdentifier();
parameters.Add(new ArrowFuncParameterSyntax(name.Value));
parameters.Add(new ArrowFuncParameterSyntax(GetTokens(parameterIndex), name.Value));
}
ExpectSymbol(Symbol.Arrow);
BlockSyntax body;
if (_currentToken is SymbolToken { Symbol: Symbol.OpenBrace })
if (CurrentToken is SymbolToken { Symbol: Symbol.OpenBrace })
{
var bodyStartIndex = _tokenIndex;
var returnValue = ParseExpression();
var arrowExpression = new ReturnSyntax(returnValue);
body = new BlockSyntax([arrowExpression]);
var arrowExpression = new ReturnSyntax(GetTokens(bodyStartIndex), returnValue);
body = new BlockSyntax(GetTokens(bodyStartIndex), [arrowExpression]);
}
else
{
body = ParseBlock();
}
return new ArrowFuncSyntax(parameters, body);
return new ArrowFuncSyntax(GetTokens(startIndex), parameters, body);
}
private ExpressionSyntax ParseParenthesizedExpression()
@@ -444,15 +471,15 @@ public sealed class Parser
return expression;
}
private ExpressionSyntax ParseArrayInitializer()
private ExpressionSyntax ParseArrayInitializer(int startIndex)
{
var capacity = ParseExpression();
ExpectSymbol(Symbol.CloseBracket);
var type = ParseType();
return new ArrayInitializerSyntax(capacity, type);
return new ArrayInitializerSyntax(GetTokens(startIndex), capacity, type);
}
private ExpressionSyntax ParseStructInitializer()
private ExpressionSyntax ParseStructInitializer(int startIndex)
{
var type = ParseType();
Dictionary<string, ExpressionSyntax> initializers = [];
@@ -465,29 +492,30 @@ public sealed class Parser
initializers.Add(name, value);
}
return new StructInitializerSyntax(type, initializers);
return new StructInitializerSyntax(GetTokens(startIndex), type, initializers);
}
private ExpressionSyntax ParsePostfixOperators(ExpressionSyntax expr)
{
while (_hasCurrentToken)
var startIndex = _tokenIndex;
while (HasToken)
{
if (TryExpectSymbol(Symbol.Ampersand))
{
expr = new AddressOfSyntax(expr);
expr = new AddressOfSyntax(GetTokens(startIndex), expr);
continue;
}
if (TryExpectSymbol(Symbol.Caret))
{
expr = new DereferenceSyntax(expr);
expr = new DereferenceSyntax(GetTokens(startIndex), expr);
continue;
}
if (TryExpectSymbol(Symbol.Period))
{
var structMember = ExpectIdentifier().Value;
expr = new MemberAccessSyntax(expr, structMember);
expr = new MemberAccessSyntax(GetTokens(startIndex), expr, structMember);
continue;
}
@@ -495,7 +523,7 @@ public sealed class Parser
{
var index = ParseExpression();
ExpectSymbol(Symbol.CloseBracket);
expr = new ArrayIndexAccessSyntax(expr, index);
expr = new ArrayIndexAccessSyntax(GetTokens(startIndex), expr, index);
continue;
}
@@ -504,17 +532,19 @@ public sealed class Parser
var parameters = new List<ExpressionSyntax>();
while (!TryExpectSymbol(Symbol.CloseParen))
{
parameters.Add(ParseExpression());
if (!TryExpectSymbol(Symbol.Comma) && _currentToken is not SymbolToken { Symbol: Symbol.CloseParen })
var parameter = ParseExpression();
parameters.Add(parameter);
if (!TryExpectSymbol(Symbol.Comma) && CurrentToken is not SymbolToken { Symbol: Symbol.CloseParen })
{
_diagnostics.Add(Diagnostic
.Warning("Missing comma between function arguments")
.WithHelp("Add a ',' to separate arguments")
.At(CurrentToken)
.Build());
}
}
expr = new FuncCallSyntax(expr, parameters);
expr = new FuncCallSyntax(GetTokens(startIndex), expr, parameters);
continue;
}
@@ -526,6 +556,7 @@ public sealed class Parser
private BlockSyntax ParseBlock()
{
var startIndex = _tokenIndex;
ExpectSymbol(Symbol.OpenBrace);
List<StatementSyntax> statements = [];
while (!TryExpectSymbol(Symbol.CloseBrace))
@@ -541,37 +572,38 @@ public sealed class Parser
}
}
return new BlockSyntax(statements);
return new BlockSyntax(GetTokens(startIndex), statements);
}
private TypeSyntax ParseType()
{
var startIndex = _tokenIndex;
if (TryExpectIdentifier(out var name))
{
return name.Value switch
{
"void" => new VoidTypeSyntax(),
"string" => new StringTypeSyntax(),
"cstring" => new CStringTypeSyntax(),
"i64" => new PrimitiveTypeSyntax(PrimitiveTypeSyntaxKind.I64),
"i32" => new PrimitiveTypeSyntax(PrimitiveTypeSyntaxKind.I32),
"i16" => new PrimitiveTypeSyntax(PrimitiveTypeSyntaxKind.I16),
"i8" => new PrimitiveTypeSyntax(PrimitiveTypeSyntaxKind.I8),
"u64" => new PrimitiveTypeSyntax(PrimitiveTypeSyntaxKind.U64),
"u32" => new PrimitiveTypeSyntax(PrimitiveTypeSyntaxKind.U32),
"u16" => new PrimitiveTypeSyntax(PrimitiveTypeSyntaxKind.U16),
"u8" => new PrimitiveTypeSyntax(PrimitiveTypeSyntaxKind.U8),
"f64" => new PrimitiveTypeSyntax(PrimitiveTypeSyntaxKind.F64),
"f32" => new PrimitiveTypeSyntax(PrimitiveTypeSyntaxKind.F32),
"bool" => new PrimitiveTypeSyntax(PrimitiveTypeSyntaxKind.Bool),
_ => new CustomTypeSyntax(name.Value)
"void" => new VoidTypeSyntax(GetTokens(startIndex)),
"string" => new StringTypeSyntax(GetTokens(startIndex)),
"cstring" => new CStringTypeSyntax(GetTokens(startIndex)),
"i64" => new PrimitiveTypeSyntax(GetTokens(startIndex), PrimitiveTypeSyntaxKind.I64),
"i32" => new PrimitiveTypeSyntax(GetTokens(startIndex), PrimitiveTypeSyntaxKind.I32),
"i16" => new PrimitiveTypeSyntax(GetTokens(startIndex), PrimitiveTypeSyntaxKind.I16),
"i8" => new PrimitiveTypeSyntax(GetTokens(startIndex), PrimitiveTypeSyntaxKind.I8),
"u64" => new PrimitiveTypeSyntax(GetTokens(startIndex), PrimitiveTypeSyntaxKind.U64),
"u32" => new PrimitiveTypeSyntax(GetTokens(startIndex), PrimitiveTypeSyntaxKind.U32),
"u16" => new PrimitiveTypeSyntax(GetTokens(startIndex), PrimitiveTypeSyntaxKind.U16),
"u8" => new PrimitiveTypeSyntax(GetTokens(startIndex), PrimitiveTypeSyntaxKind.U8),
"f64" => new PrimitiveTypeSyntax(GetTokens(startIndex), PrimitiveTypeSyntaxKind.F64),
"f32" => new PrimitiveTypeSyntax(GetTokens(startIndex), PrimitiveTypeSyntaxKind.F32),
"bool" => new PrimitiveTypeSyntax(GetTokens(startIndex), PrimitiveTypeSyntaxKind.Bool),
_ => new CustomTypeSyntax(GetTokens(startIndex), name.Value)
};
}
if (TryExpectSymbol(Symbol.Caret))
{
var baseType = ParseType();
return new PointerTypeSyntax(baseType);
return new PointerTypeSyntax(GetTokens(startIndex), baseType);
}
if (TryExpectSymbol(Symbol.Func))
@@ -582,44 +614,47 @@ public sealed class Parser
{
var parameter = ParseType();
parameters.Add(parameter);
if (!TryExpectSymbol(Symbol.Comma) && _currentToken is not SymbolToken { Symbol: Symbol.CloseParen })
if (!TryExpectSymbol(Symbol.Comma) && CurrentToken is not SymbolToken { Symbol: Symbol.CloseParen })
{
_diagnostics.Add(Diagnostic
.Warning("Missing comma between func type arguments")
.WithHelp("Add a ',' to separate arguments")
.At(CurrentToken)
.Build());
}
}
var returnType = TryExpectSymbol(Symbol.Colon) ? ParseType() : new VoidTypeSyntax();
var returnType = TryExpectSymbol(Symbol.Colon) ? ParseType() : new VoidTypeSyntax([]);
return new FuncTypeSyntax(parameters, returnType);
return new FuncTypeSyntax(GetTokens(startIndex), parameters, returnType);
}
if (TryExpectSymbol(Symbol.OpenBracket))
{
ExpectSymbol(Symbol.CloseBracket);
var baseType = ParseType();
return new ArrayTypeSyntax(baseType);
return new ArrayTypeSyntax(GetTokens(startIndex), baseType);
}
throw new ParseException(Diagnostic
.Error("Invalid type syntax")
.WithHelp("Expected type name, '^' for pointer, or '[]' for array")
.At(CurrentToken)
.Build());
}
private Token ExpectToken()
{
if (!_hasCurrentToken)
if (!HasToken)
{
throw new ParseException(Diagnostic
.Error("Unexpected end of file")
.WithHelp("Expected more tokens to complete the syntax")
.At(_tokens[^1])
.Build());
}
var token = _currentToken!;
var token = CurrentToken!;
Next();
return token;
}
@@ -632,6 +667,7 @@ public sealed class Parser
throw new ParseException(Diagnostic
.Error($"Expected symbol, but found {token.GetType().Name}")
.WithHelp("This position requires a symbol like '(', ')', '{', '}', etc.")
.At(token)
.Build());
}
@@ -646,13 +682,14 @@ public sealed class Parser
throw new ParseException(Diagnostic
.Error($"Expected '{expectedSymbol}', but found '{token.Symbol}'")
.WithHelp($"Insert '{expectedSymbol}' here")
.At(token)
.Build());
}
}
private bool TryExpectSymbol(Symbol symbol)
{
if (_currentToken is SymbolToken symbolToken && symbolToken.Symbol == symbol)
if (CurrentToken is SymbolToken symbolToken && symbolToken.Symbol == symbol)
{
Next();
return true;
@@ -663,7 +700,7 @@ public sealed class Parser
private bool TryExpectIdentifier([NotNullWhen(true)] out IdentifierToken? identifier)
{
if (_currentToken is IdentifierToken identifierToken)
if (CurrentToken is IdentifierToken identifierToken)
{
identifier = identifierToken;
Next();
@@ -682,6 +719,7 @@ public sealed class Parser
throw new ParseException(Diagnostic
.Error($"Expected identifier, but found {token.GetType().Name}")
.WithHelp("Provide a valid identifier name here")
.At(token)
.Build());
}
@@ -690,8 +728,12 @@ public sealed class Parser
private void Next()
{
_hasCurrentToken = _tokenEnumerator.MoveNext();
_currentToken = _hasCurrentToken ? _tokenEnumerator.Current : null;
_tokenIndex++;
}
private IEnumerable<Token> GetTokens(int tokenStartIndex)
{
return _tokens.Skip(tokenStartIndex).Take(_tokenIndex - tokenStartIndex);
}
}

22
src/compiler/NubLang/Parsing/Syntax/DefinitionSyntax.cs
View File

@@ -1,8 +1,10 @@
using NubLang.Tokenization;
namespace NubLang.Parsing.Syntax;
public abstract record DefinitionSyntax : SyntaxNode;
public abstract record DefinitionSyntax(IEnumerable<Token> Tokens) : SyntaxNode(Tokens);
public record FuncParameterSyntax(string Name, TypeSyntax Type) : SyntaxNode
public record FuncParameterSyntax(IEnumerable<Token> Tokens, string Name, TypeSyntax Type) : SyntaxNode(Tokens)
{
public override IEnumerable<SyntaxNode> GetChildren()
{
@@ -10,7 +12,7 @@ public record FuncParameterSyntax(string Name, TypeSyntax Type) : SyntaxNode
}
}
public record FuncSignatureSyntax(IReadOnlyList<FuncParameterSyntax> Parameters, TypeSyntax ReturnType) : SyntaxNode
public record FuncSignatureSyntax(IEnumerable<Token> Tokens, IReadOnlyList<FuncParameterSyntax> Parameters, TypeSyntax ReturnType) : SyntaxNode(Tokens)
{
public override IEnumerable<SyntaxNode> GetChildren()
{
@@ -23,7 +25,7 @@ public record FuncSignatureSyntax(IReadOnlyList<FuncParameterSyntax> Parameters,
}
}
public record LocalFuncSyntax(string Name, FuncSignatureSyntax Signature, BlockSyntax Body) : DefinitionSyntax
public record LocalFuncSyntax(IEnumerable<Token> Tokens, string Name, FuncSignatureSyntax Signature, BlockSyntax Body) : DefinitionSyntax(Tokens)
{
public override IEnumerable<SyntaxNode> GetChildren()
{
@@ -32,7 +34,7 @@ public record LocalFuncSyntax(string Name, FuncSignatureSyntax Signature, BlockS
}
}
public record ExternFuncSyntax(string Name, string CallName, FuncSignatureSyntax Signature) : DefinitionSyntax
public record ExternFuncSyntax(IEnumerable<Token> Tokens, string Name, string CallName, FuncSignatureSyntax Signature) : DefinitionSyntax(Tokens)
{
public override IEnumerable<SyntaxNode> GetChildren()
{
@@ -40,7 +42,7 @@ public record ExternFuncSyntax(string Name, string CallName, FuncSignatureSyntax
}
}
public record StructFieldSyntax(int Index, string Name, TypeSyntax Type, Optional<ExpressionSyntax> Value) : SyntaxNode
public record StructFieldSyntax(IEnumerable<Token> Tokens, int Index, string Name, TypeSyntax Type, Optional<ExpressionSyntax> Value) : SyntaxNode(Tokens)
{
public override IEnumerable<SyntaxNode> GetChildren()
{
@@ -52,7 +54,7 @@ public record StructFieldSyntax(int Index, string Name, TypeSyntax Type, Optiona
}
}
public record StructFuncSyntax(string Name, FuncSignatureSyntax Signature, BlockSyntax Body) : SyntaxNode
public record StructFuncSyntax(IEnumerable<Token> Tokens, string Name, FuncSignatureSyntax Signature, BlockSyntax Body) : SyntaxNode(Tokens)
{
public override IEnumerable<SyntaxNode> GetChildren()
{
@@ -61,7 +63,7 @@ public record StructFuncSyntax(string Name, FuncSignatureSyntax Signature, Block
}
}
public record StructSyntax(string Name, IReadOnlyList<StructFieldSyntax> Fields, IReadOnlyList<StructFuncSyntax> Functions) : DefinitionSyntax
public record StructSyntax(IEnumerable<Token> Tokens, string Name, IReadOnlyList<StructFieldSyntax> Fields, IReadOnlyList<StructFuncSyntax> Functions) : DefinitionSyntax(Tokens)
{
public override IEnumerable<SyntaxNode> GetChildren()
{
@@ -77,7 +79,7 @@ public record StructSyntax(string Name, IReadOnlyList<StructFieldSyntax> Fields,
}
}
public record InterfaceFuncSyntax(string Name, FuncSignatureSyntax Signature) : SyntaxNode
public record InterfaceFuncSyntax(IEnumerable<Token> Tokens, string Name, FuncSignatureSyntax Signature) : SyntaxNode(Tokens)
{
public override IEnumerable<SyntaxNode> GetChildren()
{
@@ -85,7 +87,7 @@ public record InterfaceFuncSyntax(string Name, FuncSignatureSyntax Signature) :
}
}
public record InterfaceSyntax(string Name, IReadOnlyList<InterfaceFuncSyntax> Functions) : DefinitionSyntax
public record InterfaceSyntax(IEnumerable<Token> Tokens, string Name, IReadOnlyList<InterfaceFuncSyntax> Functions) : DefinitionSyntax(Tokens)
{
public override IEnumerable<SyntaxNode> GetChildren()
{

28
src/compiler/NubLang/Parsing/Syntax/ExpressionSyntax.cs
View File

@@ -22,9 +22,9 @@ public enum BinaryOperatorSyntax
Divide
}
public abstract record ExpressionSyntax : SyntaxNode;
public abstract record ExpressionSyntax(IEnumerable<Token> Tokens) : SyntaxNode(Tokens);
public record BinaryExpressionSyntax(ExpressionSyntax Left, BinaryOperatorSyntax OperatorSyntax, ExpressionSyntax Right) : ExpressionSyntax
public record BinaryExpressionSyntax(IEnumerable<Token> Tokens, ExpressionSyntax Left, BinaryOperatorSyntax OperatorSyntax, ExpressionSyntax Right) : ExpressionSyntax(Tokens)
{
public override IEnumerable<SyntaxNode> GetChildren()
{
@@ -33,7 +33,7 @@ public record BinaryExpressionSyntax(ExpressionSyntax Left, BinaryOperatorSyntax
}
}
public record UnaryExpressionSyntax(UnaryOperatorSyntax OperatorSyntax, ExpressionSyntax Operand) : ExpressionSyntax
public record UnaryExpressionSyntax(IEnumerable<Token> Tokens, UnaryOperatorSyntax OperatorSyntax, ExpressionSyntax Operand) : ExpressionSyntax(Tokens)
{
public override IEnumerable<SyntaxNode> GetChildren()
{
@@ -41,7 +41,7 @@ public record UnaryExpressionSyntax(UnaryOperatorSyntax OperatorSyntax, Expressi
}
}
public record FuncCallSyntax(ExpressionSyntax Expression, IReadOnlyList<ExpressionSyntax> Parameters) : ExpressionSyntax
public record FuncCallSyntax(IEnumerable<Token> Tokens, ExpressionSyntax Expression, IReadOnlyList<ExpressionSyntax> Parameters) : ExpressionSyntax(Tokens)
{
public override IEnumerable<SyntaxNode> GetChildren()
{
@@ -53,12 +53,12 @@ public record FuncCallSyntax(ExpressionSyntax Expression, IReadOnlyList<Expressi
}
}
public record IdentifierSyntax(string Name) : ExpressionSyntax
public record IdentifierSyntax(IEnumerable<Token> Tokens, string Name) : ExpressionSyntax(Tokens)
{
public override IEnumerable<SyntaxNode> GetChildren() => [];
}
public record ArrayInitializerSyntax(ExpressionSyntax Capacity, TypeSyntax ElementType) : ExpressionSyntax
public record ArrayInitializerSyntax(IEnumerable<Token> Tokens, ExpressionSyntax Capacity, TypeSyntax ElementType) : ExpressionSyntax(Tokens)
{
public override IEnumerable<SyntaxNode> GetChildren()
{
@@ -67,7 +67,7 @@ public record ArrayInitializerSyntax(ExpressionSyntax Capacity, TypeSyntax Eleme
}
}
public record ArrayIndexAccessSyntax(ExpressionSyntax Target, ExpressionSyntax Index) : ExpressionSyntax
public record ArrayIndexAccessSyntax(IEnumerable<Token> Tokens, ExpressionSyntax Target, ExpressionSyntax Index) : ExpressionSyntax(Tokens)
{
public override IEnumerable<SyntaxNode> GetChildren()
{
@@ -76,12 +76,12 @@ public record ArrayIndexAccessSyntax(ExpressionSyntax Target, ExpressionSyntax I
}
}
public record ArrowFuncParameterSyntax(string Name) : ExpressionSyntax
public record ArrowFuncParameterSyntax(IEnumerable<Token> Tokens, string Name) : ExpressionSyntax(Tokens)
{
public override IEnumerable<SyntaxNode> GetChildren() => [];
}
public record ArrowFuncSyntax(IReadOnlyList<ArrowFuncParameterSyntax> Parameters, BlockSyntax Body) : ExpressionSyntax
public record ArrowFuncSyntax(IEnumerable<Token> Tokens, IReadOnlyList<ArrowFuncParameterSyntax> Parameters, BlockSyntax Body) : ExpressionSyntax(Tokens)
{
public override IEnumerable<SyntaxNode> GetChildren()
{
@@ -94,7 +94,7 @@ public record ArrowFuncSyntax(IReadOnlyList<ArrowFuncParameterSyntax> Parameters
}
}
public record AddressOfSyntax(ExpressionSyntax Expression) : ExpressionSyntax
public record AddressOfSyntax(IEnumerable<Token> Tokens, ExpressionSyntax Expression) : ExpressionSyntax(Tokens)
{
public override IEnumerable<SyntaxNode> GetChildren()
{
@@ -102,12 +102,12 @@ public record AddressOfSyntax(ExpressionSyntax Expression) : ExpressionSyntax
}
}
public record LiteralSyntax(string Value, LiteralKind Kind) : ExpressionSyntax
public record LiteralSyntax(IEnumerable<Token> Tokens, string Value, LiteralKind Kind) : ExpressionSyntax(Tokens)
{
public override IEnumerable<SyntaxNode> GetChildren() => [];
}
public record MemberAccessSyntax(ExpressionSyntax Target, string Member) : ExpressionSyntax
public record MemberAccessSyntax(IEnumerable<Token> Tokens, ExpressionSyntax Target, string Member) : ExpressionSyntax(Tokens)
{
public override IEnumerable<SyntaxNode> GetChildren()
{
@@ -115,7 +115,7 @@ public record MemberAccessSyntax(ExpressionSyntax Target, string Member) : Expre
}
}
public record StructInitializerSyntax(TypeSyntax StructType, Dictionary<string, ExpressionSyntax> Initializers) : ExpressionSyntax
public record StructInitializerSyntax(IEnumerable<Token> Tokens, TypeSyntax StructType, Dictionary<string, ExpressionSyntax> Initializers) : ExpressionSyntax(Tokens)
{
public override IEnumerable<SyntaxNode> GetChildren()
{
@@ -127,7 +127,7 @@ public record StructInitializerSyntax(TypeSyntax StructType, Dictionary<string,
}
}
public record DereferenceSyntax(ExpressionSyntax Expression) : ExpressionSyntax
public record DereferenceSyntax(IEnumerable<Token> Tokens, ExpressionSyntax Expression) : ExpressionSyntax(Tokens)
{
public override IEnumerable<SyntaxNode> GetChildren()
{

20
src/compiler/NubLang/Parsing/Syntax/StatementSyntax.cs
View File

@@ -1,8 +1,10 @@
using NubLang.Tokenization;
namespace NubLang.Parsing.Syntax;
public abstract record StatementSyntax : SyntaxNode;
public abstract record StatementSyntax(IEnumerable<Token> Tokens) : SyntaxNode(Tokens);
public record StatementExpressionSyntax(ExpressionSyntax Expression) : StatementSyntax
public record StatementExpressionSyntax(IEnumerable<Token> Tokens, ExpressionSyntax Expression) : StatementSyntax(Tokens)
{
public override IEnumerable<SyntaxNode> GetChildren()
{
@@ -10,7 +12,7 @@ public record StatementExpressionSyntax(ExpressionSyntax Expression) : Statement
}
}
public record ReturnSyntax(Optional<ExpressionSyntax> Value) : StatementSyntax
public record ReturnSyntax(IEnumerable<Token> Tokens, Optional<ExpressionSyntax> Value) : StatementSyntax(Tokens)
{
public override IEnumerable<SyntaxNode> GetChildren()
{
@@ -21,7 +23,7 @@ public record ReturnSyntax(Optional<ExpressionSyntax> Value) : StatementSyntax
}
}
public record AssignmentSyntax(ExpressionSyntax Target, ExpressionSyntax Value) : StatementSyntax
public record AssignmentSyntax(IEnumerable<Token> Tokens, ExpressionSyntax Target, ExpressionSyntax Value) : StatementSyntax(Tokens)
{
public override IEnumerable<SyntaxNode> GetChildren()
{
@@ -30,7 +32,7 @@ public record AssignmentSyntax(ExpressionSyntax Target, ExpressionSyntax Value)
}
}
public record IfSyntax(ExpressionSyntax Condition, BlockSyntax Body, Optional<Variant<IfSyntax, BlockSyntax>> Else) : StatementSyntax
public record IfSyntax(IEnumerable<Token> Tokens, ExpressionSyntax Condition, BlockSyntax Body, Optional<Variant<IfSyntax, BlockSyntax>> Else) : StatementSyntax(Tokens)
{
public override IEnumerable<SyntaxNode> GetChildren()
{
@@ -50,7 +52,7 @@ public record IfSyntax(ExpressionSyntax Condition, BlockSyntax Body, Optional<Va
}
}
public record VariableDeclarationSyntax(string Name, Optional<TypeSyntax> ExplicitType, Optional<ExpressionSyntax> Assignment) : StatementSyntax
public record VariableDeclarationSyntax(IEnumerable<Token> Tokens, string Name, Optional<TypeSyntax> ExplicitType, Optional<ExpressionSyntax> Assignment) : StatementSyntax(Tokens)
{
public override IEnumerable<SyntaxNode> GetChildren()
{
@@ -66,17 +68,17 @@ public record VariableDeclarationSyntax(string Name, Optional<TypeSyntax> Explic
}
}
public record ContinueSyntax : StatementSyntax
public record ContinueSyntax(IEnumerable<Token> Tokens) : StatementSyntax(Tokens)
{
public override IEnumerable<SyntaxNode> GetChildren() => [];
}
public record BreakSyntax : StatementSyntax
public record BreakSyntax(IEnumerable<Token> Tokens) : StatementSyntax(Tokens)
{
public override IEnumerable<SyntaxNode> GetChildren() => [];
}
public record WhileSyntax(ExpressionSyntax Condition, BlockSyntax Body) : StatementSyntax
public record WhileSyntax(IEnumerable<Token> Tokens, ExpressionSyntax Condition, BlockSyntax Body) : StatementSyntax(Tokens)
{
public override IEnumerable<SyntaxNode> GetChildren() => [];
}

8
src/compiler/NubLang/Parsing/Syntax/SyntaxNode.cs
View File

@@ -1,6 +1,8 @@
using NubLang.Tokenization;
namespace NubLang.Parsing.Syntax;
public abstract record SyntaxNode
public abstract record SyntaxNode(IEnumerable<Token> Tokens)
{
public abstract IEnumerable<SyntaxNode> GetChildren();
@@ -18,7 +20,7 @@ public abstract record SyntaxNode
}
}
public record SyntaxTree(IReadOnlyList<DefinitionSyntax> Definitions) : SyntaxNode
public record SyntaxTree(IEnumerable<Token> Tokens, IReadOnlyList<DefinitionSyntax> Definitions) : SyntaxNode(Tokens)
{
public override IEnumerable<SyntaxNode> GetChildren()
{
@@ -29,7 +31,7 @@ public record SyntaxTree(IReadOnlyList<DefinitionSyntax> Definitions) : SyntaxNo
}
}
public record BlockSyntax(IReadOnlyList<StatementSyntax> Statements) : SyntaxNode
public record BlockSyntax(IEnumerable<Token> Tokens, IReadOnlyList<StatementSyntax> Statements) : SyntaxNode(Tokens)
{
public override IEnumerable<SyntaxNode> GetChildren()
{

20
src/compiler/NubLang/Parsing/Syntax/TypeSyntax.cs
View File

@@ -1,3 +1,5 @@
using NubLang.Tokenization;
namespace NubLang.Parsing.Syntax;
public enum PrimitiveTypeSyntaxKind
@@ -15,7 +17,7 @@ public enum PrimitiveTypeSyntaxKind
Bool
}
public abstract record TypeSyntax : SyntaxNode
public abstract record TypeSyntax(IEnumerable<Token> Tokens) : SyntaxNode(Tokens)
{
public string MangledName()
{
@@ -34,7 +36,7 @@ public abstract record TypeSyntax : SyntaxNode
}
}
public record FuncTypeSyntax(IReadOnlyList<TypeSyntax> Parameters, TypeSyntax ReturnType) : TypeSyntax
public record FuncTypeSyntax(IEnumerable<Token> Tokens, IReadOnlyList<TypeSyntax> Parameters, TypeSyntax ReturnType) : TypeSyntax(Tokens)
{
public override IEnumerable<SyntaxNode> GetChildren()
{
@@ -47,7 +49,7 @@ public record FuncTypeSyntax(IReadOnlyList<TypeSyntax> Parameters, TypeSyntax Re
}
}
public record PointerTypeSyntax(TypeSyntax BaseType) : TypeSyntax
public record PointerTypeSyntax(IEnumerable<Token> Tokens, TypeSyntax BaseType) : TypeSyntax(Tokens)
{
public override IEnumerable<SyntaxNode> GetChildren()
{
@@ -55,32 +57,32 @@ public record PointerTypeSyntax(TypeSyntax BaseType) : TypeSyntax
}
}
public record VoidTypeSyntax : TypeSyntax
public record VoidTypeSyntax(IEnumerable<Token> Tokens) : TypeSyntax(Tokens)
{
public override IEnumerable<SyntaxNode> GetChildren() => [];
}
public record PrimitiveTypeSyntax(PrimitiveTypeSyntaxKind SyntaxKind) : TypeSyntax
public record PrimitiveTypeSyntax(IEnumerable<Token> Tokens, PrimitiveTypeSyntaxKind SyntaxKind) : TypeSyntax(Tokens)
{
public override IEnumerable<SyntaxNode> GetChildren() => [];
}
public record CStringTypeSyntax : TypeSyntax
public record CStringTypeSyntax(IEnumerable<Token> Tokens) : TypeSyntax(Tokens)
{
public override IEnumerable<SyntaxNode> GetChildren() => [];
}
public record StringTypeSyntax : TypeSyntax
public record StringTypeSyntax(IEnumerable<Token> Tokens) : TypeSyntax(Tokens)
{
public override IEnumerable<SyntaxNode> GetChildren() => [];
}
public record CustomTypeSyntax(string Name) : TypeSyntax
public record CustomTypeSyntax(IEnumerable<Token> Tokens, string Name) : TypeSyntax(Tokens)
{
public override IEnumerable<SyntaxNode> GetChildren() => [];
}
public record ArrayTypeSyntax(TypeSyntax BaseType) : TypeSyntax
public record ArrayTypeSyntax(IEnumerable<Token> Tokens, TypeSyntax BaseType) : TypeSyntax(Tokens)
{
public override IEnumerable<SyntaxNode> GetChildren()
{

2
src/compiler/NubLang/Tokenization/Tokenizer.cs
View File

@@ -241,7 +241,7 @@ public sealed class Tokenizer
if (_sourceFile != null)
{
var start = CalculateSourceLocation(tokenStartIndex);
var end = CalculateSourceLocation(_index + 1);
var end = CalculateSourceLocation(_index);
return new SourceFileSpan(_sourceFile, new SourceSpan(start, end));
}