From 9cbc2210618098ba989d4af031b29573f7b8480e Mon Sep 17 00:00:00 2001 From: nub31 Date: Wed, 23 Jul 2025 01:23:10 +0200 Subject: [PATCH] Parser diagnostics --- example/src/main.nub | 8 +- src/compiler/NubLang.CLI/Program.cs | 8 +- .../NubLang/Diagnostics/Diagnostic.cs | 34 ++- src/compiler/NubLang/Parsing/Parser.cs | 236 +++++++++++------- .../Parsing/Syntax/DefinitionSyntax.cs | 22 +- .../Parsing/Syntax/ExpressionSyntax.cs | 28 +-- .../NubLang/Parsing/Syntax/StatementSyntax.cs | 20 +- .../NubLang/Parsing/Syntax/SyntaxNode.cs | 8 +- .../NubLang/Parsing/Syntax/TypeSyntax.cs | 20 +- .../NubLang/Tokenization/Tokenizer.cs | 2 +- 10 files changed, 230 insertions(+), 156 deletions(-) diff --git a/example/src/main.nub b/example/src/main.nub index 07640d6..ff60048 100644 --- a/example/src/main.nub +++ b/example/src/main.nub @@ -1,11 +1,5 @@ - - - func main(args: []cstring): i64 { - puts("test") %%% + puts("test" =) return 0 } - - - diff --git a/src/compiler/NubLang.CLI/Program.cs b/src/compiler/NubLang.CLI/Program.cs index 52e6ab7..7fc401b 100644 --- a/src/compiler/NubLang.CLI/Program.cs +++ b/src/compiler/NubLang.CLI/Program.cs @@ -80,10 +80,12 @@ foreach (var file in options.Files) foreach (var file in options.Files) { var tokenizer = new Tokenizer(file); - var parser = new Parser(); - var syntaxTree = parser.Parse(tokenizer.Tokenize()); - + var tokens = tokenizer.Tokenize().ToList(); diagnostics.AddRange(tokenizer.GetDiagnostics()); + + var parser = new Parser(); + var syntaxTree = parser.Parse(tokens); + diagnostics.AddRange(parser.GetDiagnostics()); syntaxTrees.Add(syntaxTree); diff --git a/src/compiler/NubLang/Diagnostics/Diagnostic.cs b/src/compiler/NubLang/Diagnostics/Diagnostic.cs index 1f260bb..7b8534a 100644 --- a/src/compiler/NubLang/Diagnostics/Diagnostic.cs +++ b/src/compiler/NubLang/Diagnostics/Diagnostic.cs @@ -1,5 +1,6 @@ using System.Text; using NubLang.Code; +using NubLang.Parsing.Syntax; using NubLang.Tokenization; namespace NubLang.Diagnostics; @@ -19,9 +20,28 @@ public class Diagnostic _message = message; } - public DiagnosticBuilder At(Token token) + public DiagnosticBuilder At(SyntaxNode? node) { - At(token.FileSpan); + if (node != null) + { + var first = node.Tokens.FirstOrDefault(); + if (first?.FileSpan != null) + { + var span = SourceSpan.Merge(node.Tokens.Select(x => x.FileSpan?.Span ?? SourceSpan.Zero)); + At(new SourceFileSpan(first.FileSpan.SourceFile, span)); + } + } + + return this; + } + + public DiagnosticBuilder At(Token? token) + { + if (token != null) + { + At(token.FileSpan); + } + return this; } @@ -130,11 +150,19 @@ public class Diagnostic var markerLength = markerEndColumn - markerStartColumn; var marker = new string('^', markerLength); + var markerColor = Severity switch + { + DiagnosticSeverity.Info => ConsoleColors.Blue, + DiagnosticSeverity.Warning => ConsoleColors.Yellow, + DiagnosticSeverity.Error => ConsoleColors.Red, + _ => ConsoleColors.White + }; + sb.Append("│ "); sb.Append(new string(' ', numberPadding)); sb.Append(" │ "); sb.Append(new string(' ', markerStartColumn - 1)); - sb.Append(ConsoleColors.Colorize(marker, ConsoleColors.Red)); + sb.Append(ConsoleColors.Colorize(marker, markerColor)); sb.Append(new string(' ', codePadding - markerEndColumn + 1)); sb.Append(" │"); sb.AppendLine(); diff --git a/src/compiler/NubLang/Parsing/Parser.cs b/src/compiler/NubLang/Parsing/Parser.cs index 1b5c758..9b52cfb 100644 --- a/src/compiler/NubLang/Parsing/Parser.cs +++ b/src/compiler/NubLang/Parsing/Parser.cs @@ -7,40 +7,42 @@ namespace NubLang.Parsing; public sealed class Parser { - private IEnumerator _tokenEnumerator = null!; private readonly List _diagnostics = []; - private Token? _currentToken; - private bool _hasCurrentToken; + private IReadOnlyList _tokens = []; + private int _tokenIndex; + + private Token? CurrentToken => _tokenIndex < _tokens.Count ? _tokens[_tokenIndex] : null; + private bool HasToken => CurrentToken != null; public IReadOnlyList GetDiagnostics() { return _diagnostics; } - public SyntaxTree Parse(IEnumerable tokens) + public SyntaxTree Parse(IReadOnlyList tokens) { _diagnostics.Clear(); - _tokenEnumerator = tokens.GetEnumerator(); - _hasCurrentToken = _tokenEnumerator.MoveNext(); - _currentToken = _hasCurrentToken ? _tokenEnumerator.Current : null; + _tokens = tokens; + _tokenIndex = 0; var definitions = new List(); - while (_hasCurrentToken) + while (HasToken) { try { + var startIndex = _tokenIndex; var keyword = ExpectSymbol(); - var definition = keyword.Symbol switch { - Symbol.Extern => ParseExtern(), - Symbol.Func => ParseFunc(), - Symbol.Struct => ParseStruct(), - Symbol.Interface => ParseInterface(), + Symbol.Extern => ParseExtern(startIndex), + Symbol.Func => ParseFunc(startIndex), + Symbol.Struct => ParseStruct(startIndex), + Symbol.Interface => ParseInterface(startIndex), _ => throw new ParseException(Diagnostic .Error($"Expected 'extern', 'func', 'struct' or 'interface' but found '{keyword.Symbol}'") .WithHelp("Valid definition keywords are 'extern', 'func', 'struct' and 'interface'") + .At(keyword) .Build()) }; @@ -49,9 +51,9 @@ public sealed class Parser catch (ParseException ex) { _diagnostics.Add(ex.Diagnostic); - while (_hasCurrentToken) + while (HasToken) { - if (_currentToken is SymbolToken { Symbol: Symbol.Extern or Symbol.Func or Symbol.Struct or Symbol.Interface }) + if (CurrentToken is SymbolToken { Symbol: Symbol.Extern or Symbol.Func or Symbol.Struct or Symbol.Interface }) { break; } @@ -61,11 +63,12 @@ public sealed class Parser } } - return new SyntaxTree(definitions); + return new SyntaxTree(GetTokens(_tokenIndex), definitions); } private FuncSignatureSyntax ParseFuncSignature(FuncParameterSyntax? thisArg = null) { + var startIndex = _tokenIndex; List parameters = []; if (thisArg != null) @@ -77,43 +80,49 @@ public sealed class Parser while (!TryExpectSymbol(Symbol.CloseParen)) { - parameters.Add(ParseFuncParameter()); + var parameter = ParseFuncParameter(); + parameters.Add(parameter); - if (!TryExpectSymbol(Symbol.Comma) && _currentToken is not SymbolToken { Symbol: Symbol.CloseParen }) + if (!TryExpectSymbol(Symbol.Comma) && CurrentToken is not SymbolToken { Symbol: Symbol.CloseParen }) { _diagnostics.Add(Diagnostic .Warning("Missing comma between function parameters") .WithHelp("Add a ',' to separate parameters") + .At(CurrentToken) .Build()); } } - var returnType = TryExpectSymbol(Symbol.Colon) ? ParseType() : new VoidTypeSyntax(); + var returnType = TryExpectSymbol(Symbol.Colon) ? ParseType() : new VoidTypeSyntax([]); - return new FuncSignatureSyntax(parameters, returnType); + return new FuncSignatureSyntax(GetTokens(startIndex), parameters, returnType); } private FuncParameterSyntax ParseFuncParameter() { + var startIndex = _tokenIndex; var name = ExpectIdentifier(); ExpectSymbol(Symbol.Colon); var type = ParseType(); - return new FuncParameterSyntax(name.Value, type); + return new FuncParameterSyntax(GetTokens(startIndex), name.Value, type); } - private DefinitionSyntax ParseExtern() + private DefinitionSyntax ParseExtern(int startIndex) { var keyword = ExpectSymbol(); return keyword.Symbol switch { - Symbol.Func => ParseExternFunc(), - _ => throw new ParseException(Diagnostic.Error($"Unexpected symbol {keyword.Symbol} after extern declaration").Build()) + Symbol.Func => ParseExternFunc(startIndex), + _ => throw new ParseException(Diagnostic + .Error($"Unexpected symbol {keyword.Symbol} after extern declaration") + .At(keyword) + .Build()) }; } - private ExternFuncSyntax ParseExternFunc() + private ExternFuncSyntax ParseExternFunc(int startIndex) { var name = ExpectIdentifier(); @@ -126,19 +135,19 @@ public sealed class Parser var signature = ParseFuncSignature(); - return new ExternFuncSyntax(name.Value, callName, signature); + return new ExternFuncSyntax(GetTokens(startIndex), name.Value, callName, signature); } - private LocalFuncSyntax ParseFunc() + private LocalFuncSyntax ParseFunc(int startIndex) { var name = ExpectIdentifier(); var signature = ParseFuncSignature(); var body = ParseBlock(); - return new LocalFuncSyntax(name.Value, signature, body); + return new LocalFuncSyntax(GetTokens(startIndex), name.Value, signature, body); } - private DefinitionSyntax ParseStruct() + private DefinitionSyntax ParseStruct(int startIndex) { var name = ExpectIdentifier(); @@ -151,14 +160,16 @@ public sealed class Parser while (!TryExpectSymbol(Symbol.CloseBrace)) { + var memberStartIndex = _tokenIndex; + if (TryExpectSymbol(Symbol.Func)) { var funcName = ExpectIdentifier().Value; - var thisArg = new FuncParameterSyntax("this", new CustomTypeSyntax(name.Value)); + var thisArg = new FuncParameterSyntax([], "this", new CustomTypeSyntax([], name.Value)); var funcSignature = ParseFuncSignature(thisArg); var funcBody = ParseBlock(); - funcs.Add(new StructFuncSyntax(funcName, funcSignature, funcBody)); + funcs.Add(new StructFuncSyntax(GetTokens(memberStartIndex), funcName, funcSignature, funcBody)); } else { @@ -173,14 +184,14 @@ public sealed class Parser fieldValue = ParseExpression(); } - fields.Add(new StructFieldSyntax(fieldIndex++, fieldName, fieldType, fieldValue)); + fields.Add(new StructFieldSyntax(GetTokens(memberStartIndex), fieldIndex++, fieldName, fieldType, fieldValue)); } } - return new StructSyntax(name.Value, fields, funcs); + return new StructSyntax(GetTokens(startIndex), name.Value, fields, funcs); } - private InterfaceSyntax ParseInterface() + private InterfaceSyntax ParseInterface(int startIndex) { var name = ExpectIdentifier(); @@ -190,20 +201,22 @@ public sealed class Parser while (!TryExpectSymbol(Symbol.CloseBrace)) { + var funcStartIndex = _tokenIndex; + ExpectSymbol(Symbol.Func); var funcName = ExpectIdentifier().Value; var signature = ParseFuncSignature(); - functions.Add(new InterfaceFuncSyntax(funcName, signature)); + functions.Add(new InterfaceFuncSyntax(GetTokens(funcStartIndex), funcName, signature)); } - return new InterfaceSyntax(name.Value, functions); + return new InterfaceSyntax(GetTokens(startIndex), name.Value, functions); } private StatementSyntax ParseStatement() { - if (_currentToken is SymbolToken symbol) + if (CurrentToken is SymbolToken symbol) { switch (symbol.Symbol) { @@ -227,19 +240,21 @@ public sealed class Parser private StatementSyntax ParseStatementExpression() { + var startIndex = _tokenIndex; var expr = ParseExpression(); if (TryExpectSymbol(Symbol.Assign)) { var value = ParseExpression(); - return new AssignmentSyntax(expr, value); + return new AssignmentSyntax(GetTokens(startIndex), expr, value); } - return new StatementExpressionSyntax(expr); + return new StatementExpressionSyntax(GetTokens(startIndex), expr); } private VariableDeclarationSyntax ParseVariableDeclaration() { + var startIndex = _tokenIndex; ExpectSymbol(Symbol.Let); var name = ExpectIdentifier().Value; @@ -255,23 +270,26 @@ public sealed class Parser assignment = ParseExpression(); } - return new VariableDeclarationSyntax(name, explicitType, assignment); + return new VariableDeclarationSyntax(GetTokens(startIndex), name, explicitType, assignment); } private StatementSyntax ParseBreak() { + var startIndex = _tokenIndex; ExpectSymbol(Symbol.Break); - return new BreakSyntax(); + return new BreakSyntax(GetTokens(startIndex)); } private StatementSyntax ParseContinue() { + var startIndex = _tokenIndex; ExpectSymbol(Symbol.Continue); - return new ContinueSyntax(); + return new ContinueSyntax(GetTokens(startIndex)); } private ReturnSyntax ParseReturn() { + var startIndex = _tokenIndex; ExpectSymbol(Symbol.Return); var value = Optional.Empty(); @@ -281,11 +299,12 @@ public sealed class Parser value = ParseExpression(); } - return new ReturnSyntax(value); + return new ReturnSyntax(GetTokens(startIndex), value); } private IfSyntax ParseIf() { + var startIndex = _tokenIndex; ExpectSymbol(Symbol.If); var condition = ParseExpression(); var body = ParseBlock(); @@ -298,26 +317,28 @@ public sealed class Parser : (Variant)ParseBlock(); } - return new IfSyntax(condition, body, elseStatement); + return new IfSyntax(GetTokens(startIndex), condition, body, elseStatement); } private WhileSyntax ParseWhile() { + var startIndex = _tokenIndex; ExpectSymbol(Symbol.While); var condition = ParseExpression(); var body = ParseBlock(); - return new WhileSyntax(condition, body); + return new WhileSyntax(GetTokens(startIndex), condition, body); } private ExpressionSyntax ParseExpression(int precedence = 0) { + var startIndex = _tokenIndex; var left = ParsePrimaryExpression(); - while (_currentToken is SymbolToken symbolToken && TryGetBinaryOperator(symbolToken.Symbol, out var op) && GetBinaryOperatorPrecedence(op.Value) >= precedence) + while (CurrentToken is SymbolToken symbolToken && TryGetBinaryOperator(symbolToken.Symbol, out var op) && GetBinaryOperatorPrecedence(op.Value) >= precedence) { Next(); var right = ParseExpression(GetBinaryOperatorPrecedence(op.Value) + 1); - left = new BinaryExpressionSyntax(left, op.Value, right); + left = new BinaryExpressionSyntax(GetTokens(startIndex), left, op.Value, right); } return left; @@ -383,27 +404,30 @@ public sealed class Parser private ExpressionSyntax ParsePrimaryExpression() { + var startIndex = _tokenIndex; var token = ExpectToken(); var expr = token switch { - LiteralToken literal => new LiteralSyntax(literal.Value, literal.Kind), - IdentifierToken identifier => new IdentifierSyntax(identifier.Value), + LiteralToken literal => new LiteralSyntax(GetTokens(startIndex), literal.Value, literal.Kind), + IdentifierToken identifier => new IdentifierSyntax(GetTokens(startIndex), identifier.Value), SymbolToken symbolToken => symbolToken.Symbol switch { Symbol.Func => ParseArrowFunction(), Symbol.OpenParen => ParseParenthesizedExpression(), - Symbol.Minus => new UnaryExpressionSyntax(UnaryOperatorSyntax.Negate, ParsePrimaryExpression()), - Symbol.Bang => new UnaryExpressionSyntax(UnaryOperatorSyntax.Invert, ParsePrimaryExpression()), - Symbol.OpenBracket => ParseArrayInitializer(), - Symbol.Alloc => ParseStructInitializer(), + Symbol.Minus => new UnaryExpressionSyntax(GetTokens(startIndex), UnaryOperatorSyntax.Negate, ParsePrimaryExpression()), + Symbol.Bang => new UnaryExpressionSyntax(GetTokens(startIndex), UnaryOperatorSyntax.Invert, ParsePrimaryExpression()), + Symbol.OpenBracket => ParseArrayInitializer(startIndex), + Symbol.Alloc => ParseStructInitializer(startIndex), _ => throw new ParseException(Diagnostic .Error($"Unexpected symbol '{symbolToken.Symbol}' in expression") .WithHelp("Expected literal, identifier, or '(' to start expression") + .At(symbolToken) .Build()) }, _ => throw new ParseException(Diagnostic .Error($"Unexpected token '{token.GetType().Name}' in expression") .WithHelp("Expected literal, identifier, or parenthesized expression") + .At(token) .Build()) }; @@ -412,29 +436,32 @@ public sealed class Parser private ExpressionSyntax ParseArrowFunction() { + var startIndex = _tokenIndex; List parameters = []; ExpectSymbol(Symbol.OpenParen); while (!TryExpectSymbol(Symbol.CloseParen)) { + var parameterIndex = _tokenIndex; var name = ExpectIdentifier(); - parameters.Add(new ArrowFuncParameterSyntax(name.Value)); + parameters.Add(new ArrowFuncParameterSyntax(GetTokens(parameterIndex), name.Value)); } ExpectSymbol(Symbol.Arrow); BlockSyntax body; - if (_currentToken is SymbolToken { Symbol: Symbol.OpenBrace }) + if (CurrentToken is SymbolToken { Symbol: Symbol.OpenBrace }) { + var bodyStartIndex = _tokenIndex; var returnValue = ParseExpression(); - var arrowExpression = new ReturnSyntax(returnValue); - body = new BlockSyntax([arrowExpression]); + var arrowExpression = new ReturnSyntax(GetTokens(bodyStartIndex), returnValue); + body = new BlockSyntax(GetTokens(bodyStartIndex), [arrowExpression]); } else { body = ParseBlock(); } - return new ArrowFuncSyntax(parameters, body); + return new ArrowFuncSyntax(GetTokens(startIndex), parameters, body); } private ExpressionSyntax ParseParenthesizedExpression() @@ -444,15 +471,15 @@ public sealed class Parser return expression; } - private ExpressionSyntax ParseArrayInitializer() + private ExpressionSyntax ParseArrayInitializer(int startIndex) { var capacity = ParseExpression(); ExpectSymbol(Symbol.CloseBracket); var type = ParseType(); - return new ArrayInitializerSyntax(capacity, type); + return new ArrayInitializerSyntax(GetTokens(startIndex), capacity, type); } - private ExpressionSyntax ParseStructInitializer() + private ExpressionSyntax ParseStructInitializer(int startIndex) { var type = ParseType(); Dictionary initializers = []; @@ -465,29 +492,30 @@ public sealed class Parser initializers.Add(name, value); } - return new StructInitializerSyntax(type, initializers); + return new StructInitializerSyntax(GetTokens(startIndex), type, initializers); } private ExpressionSyntax ParsePostfixOperators(ExpressionSyntax expr) { - while (_hasCurrentToken) + var startIndex = _tokenIndex; + while (HasToken) { if (TryExpectSymbol(Symbol.Ampersand)) { - expr = new AddressOfSyntax(expr); + expr = new AddressOfSyntax(GetTokens(startIndex), expr); continue; } if (TryExpectSymbol(Symbol.Caret)) { - expr = new DereferenceSyntax(expr); + expr = new DereferenceSyntax(GetTokens(startIndex), expr); continue; } if (TryExpectSymbol(Symbol.Period)) { var structMember = ExpectIdentifier().Value; - expr = new MemberAccessSyntax(expr, structMember); + expr = new MemberAccessSyntax(GetTokens(startIndex), expr, structMember); continue; } @@ -495,7 +523,7 @@ public sealed class Parser { var index = ParseExpression(); ExpectSymbol(Symbol.CloseBracket); - expr = new ArrayIndexAccessSyntax(expr, index); + expr = new ArrayIndexAccessSyntax(GetTokens(startIndex), expr, index); continue; } @@ -504,17 +532,19 @@ public sealed class Parser var parameters = new List(); while (!TryExpectSymbol(Symbol.CloseParen)) { - parameters.Add(ParseExpression()); - if (!TryExpectSymbol(Symbol.Comma) && _currentToken is not SymbolToken { Symbol: Symbol.CloseParen }) + var parameter = ParseExpression(); + parameters.Add(parameter); + if (!TryExpectSymbol(Symbol.Comma) && CurrentToken is not SymbolToken { Symbol: Symbol.CloseParen }) { _diagnostics.Add(Diagnostic .Warning("Missing comma between function arguments") .WithHelp("Add a ',' to separate arguments") + .At(CurrentToken) .Build()); } } - expr = new FuncCallSyntax(expr, parameters); + expr = new FuncCallSyntax(GetTokens(startIndex), expr, parameters); continue; } @@ -526,6 +556,7 @@ public sealed class Parser private BlockSyntax ParseBlock() { + var startIndex = _tokenIndex; ExpectSymbol(Symbol.OpenBrace); List statements = []; while (!TryExpectSymbol(Symbol.CloseBrace)) @@ -541,37 +572,38 @@ public sealed class Parser } } - return new BlockSyntax(statements); + return new BlockSyntax(GetTokens(startIndex), statements); } private TypeSyntax ParseType() { + var startIndex = _tokenIndex; if (TryExpectIdentifier(out var name)) { return name.Value switch { - "void" => new VoidTypeSyntax(), - "string" => new StringTypeSyntax(), - "cstring" => new CStringTypeSyntax(), - "i64" => new PrimitiveTypeSyntax(PrimitiveTypeSyntaxKind.I64), - "i32" => new PrimitiveTypeSyntax(PrimitiveTypeSyntaxKind.I32), - "i16" => new PrimitiveTypeSyntax(PrimitiveTypeSyntaxKind.I16), - "i8" => new PrimitiveTypeSyntax(PrimitiveTypeSyntaxKind.I8), - "u64" => new PrimitiveTypeSyntax(PrimitiveTypeSyntaxKind.U64), - "u32" => new PrimitiveTypeSyntax(PrimitiveTypeSyntaxKind.U32), - "u16" => new PrimitiveTypeSyntax(PrimitiveTypeSyntaxKind.U16), - "u8" => new PrimitiveTypeSyntax(PrimitiveTypeSyntaxKind.U8), - "f64" => new PrimitiveTypeSyntax(PrimitiveTypeSyntaxKind.F64), - "f32" => new PrimitiveTypeSyntax(PrimitiveTypeSyntaxKind.F32), - "bool" => new PrimitiveTypeSyntax(PrimitiveTypeSyntaxKind.Bool), - _ => new CustomTypeSyntax(name.Value) + "void" => new VoidTypeSyntax(GetTokens(startIndex)), + "string" => new StringTypeSyntax(GetTokens(startIndex)), + "cstring" => new CStringTypeSyntax(GetTokens(startIndex)), + "i64" => new PrimitiveTypeSyntax(GetTokens(startIndex), PrimitiveTypeSyntaxKind.I64), + "i32" => new PrimitiveTypeSyntax(GetTokens(startIndex), PrimitiveTypeSyntaxKind.I32), + "i16" => new PrimitiveTypeSyntax(GetTokens(startIndex), PrimitiveTypeSyntaxKind.I16), + "i8" => new PrimitiveTypeSyntax(GetTokens(startIndex), PrimitiveTypeSyntaxKind.I8), + "u64" => new PrimitiveTypeSyntax(GetTokens(startIndex), PrimitiveTypeSyntaxKind.U64), + "u32" => new PrimitiveTypeSyntax(GetTokens(startIndex), PrimitiveTypeSyntaxKind.U32), + "u16" => new PrimitiveTypeSyntax(GetTokens(startIndex), PrimitiveTypeSyntaxKind.U16), + "u8" => new PrimitiveTypeSyntax(GetTokens(startIndex), PrimitiveTypeSyntaxKind.U8), + "f64" => new PrimitiveTypeSyntax(GetTokens(startIndex), PrimitiveTypeSyntaxKind.F64), + "f32" => new PrimitiveTypeSyntax(GetTokens(startIndex), PrimitiveTypeSyntaxKind.F32), + "bool" => new PrimitiveTypeSyntax(GetTokens(startIndex), PrimitiveTypeSyntaxKind.Bool), + _ => new CustomTypeSyntax(GetTokens(startIndex), name.Value) }; } if (TryExpectSymbol(Symbol.Caret)) { var baseType = ParseType(); - return new PointerTypeSyntax(baseType); + return new PointerTypeSyntax(GetTokens(startIndex), baseType); } if (TryExpectSymbol(Symbol.Func)) @@ -582,44 +614,47 @@ public sealed class Parser { var parameter = ParseType(); parameters.Add(parameter); - if (!TryExpectSymbol(Symbol.Comma) && _currentToken is not SymbolToken { Symbol: Symbol.CloseParen }) + if (!TryExpectSymbol(Symbol.Comma) && CurrentToken is not SymbolToken { Symbol: Symbol.CloseParen }) { _diagnostics.Add(Diagnostic .Warning("Missing comma between func type arguments") .WithHelp("Add a ',' to separate arguments") + .At(CurrentToken) .Build()); } } - var returnType = TryExpectSymbol(Symbol.Colon) ? ParseType() : new VoidTypeSyntax(); + var returnType = TryExpectSymbol(Symbol.Colon) ? ParseType() : new VoidTypeSyntax([]); - return new FuncTypeSyntax(parameters, returnType); + return new FuncTypeSyntax(GetTokens(startIndex), parameters, returnType); } if (TryExpectSymbol(Symbol.OpenBracket)) { ExpectSymbol(Symbol.CloseBracket); var baseType = ParseType(); - return new ArrayTypeSyntax(baseType); + return new ArrayTypeSyntax(GetTokens(startIndex), baseType); } throw new ParseException(Diagnostic .Error("Invalid type syntax") .WithHelp("Expected type name, '^' for pointer, or '[]' for array") + .At(CurrentToken) .Build()); } private Token ExpectToken() { - if (!_hasCurrentToken) + if (!HasToken) { throw new ParseException(Diagnostic .Error("Unexpected end of file") .WithHelp("Expected more tokens to complete the syntax") + .At(_tokens[^1]) .Build()); } - var token = _currentToken!; + var token = CurrentToken!; Next(); return token; } @@ -632,6 +667,7 @@ public sealed class Parser throw new ParseException(Diagnostic .Error($"Expected symbol, but found {token.GetType().Name}") .WithHelp("This position requires a symbol like '(', ')', '{', '}', etc.") + .At(token) .Build()); } @@ -646,13 +682,14 @@ public sealed class Parser throw new ParseException(Diagnostic .Error($"Expected '{expectedSymbol}', but found '{token.Symbol}'") .WithHelp($"Insert '{expectedSymbol}' here") + .At(token) .Build()); } } private bool TryExpectSymbol(Symbol symbol) { - if (_currentToken is SymbolToken symbolToken && symbolToken.Symbol == symbol) + if (CurrentToken is SymbolToken symbolToken && symbolToken.Symbol == symbol) { Next(); return true; @@ -663,7 +700,7 @@ public sealed class Parser private bool TryExpectIdentifier([NotNullWhen(true)] out IdentifierToken? identifier) { - if (_currentToken is IdentifierToken identifierToken) + if (CurrentToken is IdentifierToken identifierToken) { identifier = identifierToken; Next(); @@ -682,6 +719,7 @@ public sealed class Parser throw new ParseException(Diagnostic .Error($"Expected identifier, but found {token.GetType().Name}") .WithHelp("Provide a valid identifier name here") + .At(token) .Build()); } @@ -690,8 +728,12 @@ public sealed class Parser private void Next() { - _hasCurrentToken = _tokenEnumerator.MoveNext(); - _currentToken = _hasCurrentToken ? _tokenEnumerator.Current : null; + _tokenIndex++; + } + + private IEnumerable GetTokens(int tokenStartIndex) + { + return _tokens.Skip(tokenStartIndex).Take(_tokenIndex - tokenStartIndex); } } diff --git a/src/compiler/NubLang/Parsing/Syntax/DefinitionSyntax.cs b/src/compiler/NubLang/Parsing/Syntax/DefinitionSyntax.cs index 91be4f3..2cfeb55 100644 --- a/src/compiler/NubLang/Parsing/Syntax/DefinitionSyntax.cs +++ b/src/compiler/NubLang/Parsing/Syntax/DefinitionSyntax.cs @@ -1,8 +1,10 @@ +using NubLang.Tokenization; + namespace NubLang.Parsing.Syntax; -public abstract record DefinitionSyntax : SyntaxNode; +public abstract record DefinitionSyntax(IEnumerable Tokens) : SyntaxNode(Tokens); -public record FuncParameterSyntax(string Name, TypeSyntax Type) : SyntaxNode +public record FuncParameterSyntax(IEnumerable Tokens, string Name, TypeSyntax Type) : SyntaxNode(Tokens) { public override IEnumerable GetChildren() { @@ -10,7 +12,7 @@ public record FuncParameterSyntax(string Name, TypeSyntax Type) : SyntaxNode } } -public record FuncSignatureSyntax(IReadOnlyList Parameters, TypeSyntax ReturnType) : SyntaxNode +public record FuncSignatureSyntax(IEnumerable Tokens, IReadOnlyList Parameters, TypeSyntax ReturnType) : SyntaxNode(Tokens) { public override IEnumerable GetChildren() { @@ -23,7 +25,7 @@ public record FuncSignatureSyntax(IReadOnlyList Parameters, } } -public record LocalFuncSyntax(string Name, FuncSignatureSyntax Signature, BlockSyntax Body) : DefinitionSyntax +public record LocalFuncSyntax(IEnumerable Tokens, string Name, FuncSignatureSyntax Signature, BlockSyntax Body) : DefinitionSyntax(Tokens) { public override IEnumerable GetChildren() { @@ -32,7 +34,7 @@ public record LocalFuncSyntax(string Name, FuncSignatureSyntax Signature, BlockS } } -public record ExternFuncSyntax(string Name, string CallName, FuncSignatureSyntax Signature) : DefinitionSyntax +public record ExternFuncSyntax(IEnumerable Tokens, string Name, string CallName, FuncSignatureSyntax Signature) : DefinitionSyntax(Tokens) { public override IEnumerable GetChildren() { @@ -40,7 +42,7 @@ public record ExternFuncSyntax(string Name, string CallName, FuncSignatureSyntax } } -public record StructFieldSyntax(int Index, string Name, TypeSyntax Type, Optional Value) : SyntaxNode +public record StructFieldSyntax(IEnumerable Tokens, int Index, string Name, TypeSyntax Type, Optional Value) : SyntaxNode(Tokens) { public override IEnumerable GetChildren() { @@ -52,7 +54,7 @@ public record StructFieldSyntax(int Index, string Name, TypeSyntax Type, Optiona } } -public record StructFuncSyntax(string Name, FuncSignatureSyntax Signature, BlockSyntax Body) : SyntaxNode +public record StructFuncSyntax(IEnumerable Tokens, string Name, FuncSignatureSyntax Signature, BlockSyntax Body) : SyntaxNode(Tokens) { public override IEnumerable GetChildren() { @@ -61,7 +63,7 @@ public record StructFuncSyntax(string Name, FuncSignatureSyntax Signature, Block } } -public record StructSyntax(string Name, IReadOnlyList Fields, IReadOnlyList Functions) : DefinitionSyntax +public record StructSyntax(IEnumerable Tokens, string Name, IReadOnlyList Fields, IReadOnlyList Functions) : DefinitionSyntax(Tokens) { public override IEnumerable GetChildren() { @@ -77,7 +79,7 @@ public record StructSyntax(string Name, IReadOnlyList Fields, } } -public record InterfaceFuncSyntax(string Name, FuncSignatureSyntax Signature) : SyntaxNode +public record InterfaceFuncSyntax(IEnumerable Tokens, string Name, FuncSignatureSyntax Signature) : SyntaxNode(Tokens) { public override IEnumerable GetChildren() { @@ -85,7 +87,7 @@ public record InterfaceFuncSyntax(string Name, FuncSignatureSyntax Signature) : } } -public record InterfaceSyntax(string Name, IReadOnlyList Functions) : DefinitionSyntax +public record InterfaceSyntax(IEnumerable Tokens, string Name, IReadOnlyList Functions) : DefinitionSyntax(Tokens) { public override IEnumerable GetChildren() { diff --git a/src/compiler/NubLang/Parsing/Syntax/ExpressionSyntax.cs b/src/compiler/NubLang/Parsing/Syntax/ExpressionSyntax.cs index e715314..be41736 100644 --- a/src/compiler/NubLang/Parsing/Syntax/ExpressionSyntax.cs +++ b/src/compiler/NubLang/Parsing/Syntax/ExpressionSyntax.cs @@ -22,9 +22,9 @@ public enum BinaryOperatorSyntax Divide } -public abstract record ExpressionSyntax : SyntaxNode; +public abstract record ExpressionSyntax(IEnumerable Tokens) : SyntaxNode(Tokens); -public record BinaryExpressionSyntax(ExpressionSyntax Left, BinaryOperatorSyntax OperatorSyntax, ExpressionSyntax Right) : ExpressionSyntax +public record BinaryExpressionSyntax(IEnumerable Tokens, ExpressionSyntax Left, BinaryOperatorSyntax OperatorSyntax, ExpressionSyntax Right) : ExpressionSyntax(Tokens) { public override IEnumerable GetChildren() { @@ -33,7 +33,7 @@ public record BinaryExpressionSyntax(ExpressionSyntax Left, BinaryOperatorSyntax } } -public record UnaryExpressionSyntax(UnaryOperatorSyntax OperatorSyntax, ExpressionSyntax Operand) : ExpressionSyntax +public record UnaryExpressionSyntax(IEnumerable Tokens, UnaryOperatorSyntax OperatorSyntax, ExpressionSyntax Operand) : ExpressionSyntax(Tokens) { public override IEnumerable GetChildren() { @@ -41,7 +41,7 @@ public record UnaryExpressionSyntax(UnaryOperatorSyntax OperatorSyntax, Expressi } } -public record FuncCallSyntax(ExpressionSyntax Expression, IReadOnlyList Parameters) : ExpressionSyntax +public record FuncCallSyntax(IEnumerable Tokens, ExpressionSyntax Expression, IReadOnlyList Parameters) : ExpressionSyntax(Tokens) { public override IEnumerable GetChildren() { @@ -53,12 +53,12 @@ public record FuncCallSyntax(ExpressionSyntax Expression, IReadOnlyList Tokens, string Name) : ExpressionSyntax(Tokens) { public override IEnumerable GetChildren() => []; } -public record ArrayInitializerSyntax(ExpressionSyntax Capacity, TypeSyntax ElementType) : ExpressionSyntax +public record ArrayInitializerSyntax(IEnumerable Tokens, ExpressionSyntax Capacity, TypeSyntax ElementType) : ExpressionSyntax(Tokens) { public override IEnumerable GetChildren() { @@ -67,7 +67,7 @@ public record ArrayInitializerSyntax(ExpressionSyntax Capacity, TypeSyntax Eleme } } -public record ArrayIndexAccessSyntax(ExpressionSyntax Target, ExpressionSyntax Index) : ExpressionSyntax +public record ArrayIndexAccessSyntax(IEnumerable Tokens, ExpressionSyntax Target, ExpressionSyntax Index) : ExpressionSyntax(Tokens) { public override IEnumerable GetChildren() { @@ -76,12 +76,12 @@ public record ArrayIndexAccessSyntax(ExpressionSyntax Target, ExpressionSyntax I } } -public record ArrowFuncParameterSyntax(string Name) : ExpressionSyntax +public record ArrowFuncParameterSyntax(IEnumerable Tokens, string Name) : ExpressionSyntax(Tokens) { public override IEnumerable GetChildren() => []; } -public record ArrowFuncSyntax(IReadOnlyList Parameters, BlockSyntax Body) : ExpressionSyntax +public record ArrowFuncSyntax(IEnumerable Tokens, IReadOnlyList Parameters, BlockSyntax Body) : ExpressionSyntax(Tokens) { public override IEnumerable GetChildren() { @@ -94,7 +94,7 @@ public record ArrowFuncSyntax(IReadOnlyList Parameters } } -public record AddressOfSyntax(ExpressionSyntax Expression) : ExpressionSyntax +public record AddressOfSyntax(IEnumerable Tokens, ExpressionSyntax Expression) : ExpressionSyntax(Tokens) { public override IEnumerable GetChildren() { @@ -102,12 +102,12 @@ public record AddressOfSyntax(ExpressionSyntax Expression) : ExpressionSyntax } } -public record LiteralSyntax(string Value, LiteralKind Kind) : ExpressionSyntax +public record LiteralSyntax(IEnumerable Tokens, string Value, LiteralKind Kind) : ExpressionSyntax(Tokens) { public override IEnumerable GetChildren() => []; } -public record MemberAccessSyntax(ExpressionSyntax Target, string Member) : ExpressionSyntax +public record MemberAccessSyntax(IEnumerable Tokens, ExpressionSyntax Target, string Member) : ExpressionSyntax(Tokens) { public override IEnumerable GetChildren() { @@ -115,7 +115,7 @@ public record MemberAccessSyntax(ExpressionSyntax Target, string Member) : Expre } } -public record StructInitializerSyntax(TypeSyntax StructType, Dictionary Initializers) : ExpressionSyntax +public record StructInitializerSyntax(IEnumerable Tokens, TypeSyntax StructType, Dictionary Initializers) : ExpressionSyntax(Tokens) { public override IEnumerable GetChildren() { @@ -127,7 +127,7 @@ public record StructInitializerSyntax(TypeSyntax StructType, Dictionary Tokens, ExpressionSyntax Expression) : ExpressionSyntax(Tokens) { public override IEnumerable GetChildren() { diff --git a/src/compiler/NubLang/Parsing/Syntax/StatementSyntax.cs b/src/compiler/NubLang/Parsing/Syntax/StatementSyntax.cs index 5ad40e7..4601ab5 100644 --- a/src/compiler/NubLang/Parsing/Syntax/StatementSyntax.cs +++ b/src/compiler/NubLang/Parsing/Syntax/StatementSyntax.cs @@ -1,8 +1,10 @@ +using NubLang.Tokenization; + namespace NubLang.Parsing.Syntax; -public abstract record StatementSyntax : SyntaxNode; +public abstract record StatementSyntax(IEnumerable Tokens) : SyntaxNode(Tokens); -public record StatementExpressionSyntax(ExpressionSyntax Expression) : StatementSyntax +public record StatementExpressionSyntax(IEnumerable Tokens, ExpressionSyntax Expression) : StatementSyntax(Tokens) { public override IEnumerable GetChildren() { @@ -10,7 +12,7 @@ public record StatementExpressionSyntax(ExpressionSyntax Expression) : Statement } } -public record ReturnSyntax(Optional Value) : StatementSyntax +public record ReturnSyntax(IEnumerable Tokens, Optional Value) : StatementSyntax(Tokens) { public override IEnumerable GetChildren() { @@ -21,7 +23,7 @@ public record ReturnSyntax(Optional Value) : StatementSyntax } } -public record AssignmentSyntax(ExpressionSyntax Target, ExpressionSyntax Value) : StatementSyntax +public record AssignmentSyntax(IEnumerable Tokens, ExpressionSyntax Target, ExpressionSyntax Value) : StatementSyntax(Tokens) { public override IEnumerable GetChildren() { @@ -30,7 +32,7 @@ public record AssignmentSyntax(ExpressionSyntax Target, ExpressionSyntax Value) } } -public record IfSyntax(ExpressionSyntax Condition, BlockSyntax Body, Optional> Else) : StatementSyntax +public record IfSyntax(IEnumerable Tokens, ExpressionSyntax Condition, BlockSyntax Body, Optional> Else) : StatementSyntax(Tokens) { public override IEnumerable GetChildren() { @@ -50,7 +52,7 @@ public record IfSyntax(ExpressionSyntax Condition, BlockSyntax Body, Optional ExplicitType, Optional Assignment) : StatementSyntax +public record VariableDeclarationSyntax(IEnumerable Tokens, string Name, Optional ExplicitType, Optional Assignment) : StatementSyntax(Tokens) { public override IEnumerable GetChildren() { @@ -66,17 +68,17 @@ public record VariableDeclarationSyntax(string Name, Optional Explic } } -public record ContinueSyntax : StatementSyntax +public record ContinueSyntax(IEnumerable Tokens) : StatementSyntax(Tokens) { public override IEnumerable GetChildren() => []; } -public record BreakSyntax : StatementSyntax +public record BreakSyntax(IEnumerable Tokens) : StatementSyntax(Tokens) { public override IEnumerable GetChildren() => []; } -public record WhileSyntax(ExpressionSyntax Condition, BlockSyntax Body) : StatementSyntax +public record WhileSyntax(IEnumerable Tokens, ExpressionSyntax Condition, BlockSyntax Body) : StatementSyntax(Tokens) { public override IEnumerable GetChildren() => []; } \ No newline at end of file diff --git a/src/compiler/NubLang/Parsing/Syntax/SyntaxNode.cs b/src/compiler/NubLang/Parsing/Syntax/SyntaxNode.cs index 76ab7f1..37f9ac0 100644 --- a/src/compiler/NubLang/Parsing/Syntax/SyntaxNode.cs +++ b/src/compiler/NubLang/Parsing/Syntax/SyntaxNode.cs @@ -1,6 +1,8 @@ +using NubLang.Tokenization; + namespace NubLang.Parsing.Syntax; -public abstract record SyntaxNode +public abstract record SyntaxNode(IEnumerable Tokens) { public abstract IEnumerable GetChildren(); @@ -18,7 +20,7 @@ public abstract record SyntaxNode } } -public record SyntaxTree(IReadOnlyList Definitions) : SyntaxNode +public record SyntaxTree(IEnumerable Tokens, IReadOnlyList Definitions) : SyntaxNode(Tokens) { public override IEnumerable GetChildren() { @@ -29,7 +31,7 @@ public record SyntaxTree(IReadOnlyList Definitions) : SyntaxNo } } -public record BlockSyntax(IReadOnlyList Statements) : SyntaxNode +public record BlockSyntax(IEnumerable Tokens, IReadOnlyList Statements) : SyntaxNode(Tokens) { public override IEnumerable GetChildren() { diff --git a/src/compiler/NubLang/Parsing/Syntax/TypeSyntax.cs b/src/compiler/NubLang/Parsing/Syntax/TypeSyntax.cs index 43a316b..50fd274 100644 --- a/src/compiler/NubLang/Parsing/Syntax/TypeSyntax.cs +++ b/src/compiler/NubLang/Parsing/Syntax/TypeSyntax.cs @@ -1,3 +1,5 @@ +using NubLang.Tokenization; + namespace NubLang.Parsing.Syntax; public enum PrimitiveTypeSyntaxKind @@ -15,7 +17,7 @@ public enum PrimitiveTypeSyntaxKind Bool } -public abstract record TypeSyntax : SyntaxNode +public abstract record TypeSyntax(IEnumerable Tokens) : SyntaxNode(Tokens) { public string MangledName() { @@ -34,7 +36,7 @@ public abstract record TypeSyntax : SyntaxNode } } -public record FuncTypeSyntax(IReadOnlyList Parameters, TypeSyntax ReturnType) : TypeSyntax +public record FuncTypeSyntax(IEnumerable Tokens, IReadOnlyList Parameters, TypeSyntax ReturnType) : TypeSyntax(Tokens) { public override IEnumerable GetChildren() { @@ -47,7 +49,7 @@ public record FuncTypeSyntax(IReadOnlyList Parameters, TypeSyntax Re } } -public record PointerTypeSyntax(TypeSyntax BaseType) : TypeSyntax +public record PointerTypeSyntax(IEnumerable Tokens, TypeSyntax BaseType) : TypeSyntax(Tokens) { public override IEnumerable GetChildren() { @@ -55,32 +57,32 @@ public record PointerTypeSyntax(TypeSyntax BaseType) : TypeSyntax } } -public record VoidTypeSyntax : TypeSyntax +public record VoidTypeSyntax(IEnumerable Tokens) : TypeSyntax(Tokens) { public override IEnumerable GetChildren() => []; } -public record PrimitiveTypeSyntax(PrimitiveTypeSyntaxKind SyntaxKind) : TypeSyntax +public record PrimitiveTypeSyntax(IEnumerable Tokens, PrimitiveTypeSyntaxKind SyntaxKind) : TypeSyntax(Tokens) { public override IEnumerable GetChildren() => []; } -public record CStringTypeSyntax : TypeSyntax +public record CStringTypeSyntax(IEnumerable Tokens) : TypeSyntax(Tokens) { public override IEnumerable GetChildren() => []; } -public record StringTypeSyntax : TypeSyntax +public record StringTypeSyntax(IEnumerable Tokens) : TypeSyntax(Tokens) { public override IEnumerable GetChildren() => []; } -public record CustomTypeSyntax(string Name) : TypeSyntax +public record CustomTypeSyntax(IEnumerable Tokens, string Name) : TypeSyntax(Tokens) { public override IEnumerable GetChildren() => []; } -public record ArrayTypeSyntax(TypeSyntax BaseType) : TypeSyntax +public record ArrayTypeSyntax(IEnumerable Tokens, TypeSyntax BaseType) : TypeSyntax(Tokens) { public override IEnumerable GetChildren() { diff --git a/src/compiler/NubLang/Tokenization/Tokenizer.cs b/src/compiler/NubLang/Tokenization/Tokenizer.cs index c6e5121..dc540a8 100644 --- a/src/compiler/NubLang/Tokenization/Tokenizer.cs +++ b/src/compiler/NubLang/Tokenization/Tokenizer.cs @@ -241,7 +241,7 @@ public sealed class Tokenizer if (_sourceFile != null) { var start = CalculateSourceLocation(tokenStartIndex); - var end = CalculateSourceLocation(_index + 1); + var end = CalculateSourceLocation(_index); return new SourceFileSpan(_sourceFile, new SourceSpan(start, end)); }