using System.Diagnostics.CodeAnalysis; namespace Compiler; public class Parser { public static Ast? Parse(string fileName, List tokens, out List diagnostics) { return new Parser(fileName, tokens).Parse(out diagnostics); } private Parser(string fileName, List tokens) { this.fileName = fileName; this.tokens = tokens; } private readonly string fileName; private readonly List tokens; private int index; private Ast? Parse(out List diagnostics) { var definitions = new List(); diagnostics = []; TokenIdent? moduleName = null; try { ExpectKeyword(Keyword.Module); moduleName = ExpectIdent(); while (Peek() != null) { definitions.Add(ParseDefinition()); } } catch (CompileException e) { diagnostics.Add(e.Diagnostic); } if (moduleName == null || diagnostics.Any(x => x.Severity == Diagnostic.DiagnosticSeverity.Error)) return null; return new Ast(fileName, moduleName, definitions); } private NodeDefinition ParseDefinition() { var startIndex = index; Dictionary modifiers = []; while (true) { if (Peek() is TokenKeyword keyword) { switch (keyword.Keyword) { case Keyword.Export: Next(); modifiers[Keyword.Export] = keyword; break; case Keyword.Packed: Next(); modifiers[Keyword.Packed] = keyword; break; default: goto modifier_done; } } } modifier_done: if (TryExpectKeyword(Keyword.Func)) { var exported = modifiers.Remove(Keyword.Export); foreach (var modifier in modifiers) // todo(nub31): Add to diagnostics instead of throwing throw new CompileException(Diagnostic.Error("Invalid modifier for function").At(fileName, modifier.Value).Build()); var name = ExpectIdent(); var parameters = new List(); ExpectSymbol(Symbol.OpenParen); while (!TryExpectSymbol(Symbol.CloseParen)) { var paramStartIndex = index; var parameterName = ExpectIdent(); ExpectSymbol(Symbol.Colon); var parameterType = ParseType(); parameters.Add(new NodeDefinitionFunc.Param(TokensFrom(paramStartIndex), parameterName, parameterType)); } ExpectSymbol(Symbol.Colon); var returnType = ParseType(); var body = ParseStatement(); return new NodeDefinitionFunc(TokensFrom(startIndex), exported, name, parameters, body, returnType); } if (TryExpectKeyword(Keyword.Struct)) { var exported = modifiers.Remove(Keyword.Export); var packed = modifiers.Remove(Keyword.Packed); foreach (var modifier in modifiers) // todo(nub31): Add to diagnostics instead of throwing throw new CompileException(Diagnostic.Error("Invalid modifier for struct").At(fileName, modifier.Value).Build()); var name = ExpectIdent(); var fields = new List(); ExpectSymbol(Symbol.OpenCurly); while (!TryExpectSymbol(Symbol.CloseCurly)) { var fieldStartIndex = index; var fieldName = ExpectIdent(); ExpectSymbol(Symbol.Colon); var fieldType = ParseType(); fields.Add(new NodeDefinitionStruct.Field(TokensFrom(fieldStartIndex), fieldName, fieldType)); } return new NodeDefinitionStruct(TokensFrom(startIndex), exported, packed, name, fields); } if (TryExpectKeyword(Keyword.Enum)) { var exported = modifiers.Remove(Keyword.Export); foreach (var modifier in modifiers) // todo(nub31): Add to diagnostics instead of throwing throw new CompileException(Diagnostic.Error("Invalid modifier for struct").At(fileName, modifier.Value).Build()); var name = ExpectIdent(); var variants = new List(); ExpectSymbol(Symbol.OpenCurly); while (!TryExpectSymbol(Symbol.CloseCurly)) { var variantsStartIndex = index; var variantName = ExpectIdent(); var variantFields = new List(); if (TryExpectSymbol(Symbol.OpenCurly)) { while (!TryExpectSymbol(Symbol.CloseCurly)) { var fieldStartIndex = index; var fieldName = ExpectIdent(); ExpectSymbol(Symbol.Colon); var fieldType = ParseType(); variantFields.Add(new NodeDefinitionEnum.Variant.Field(TokensFrom(fieldStartIndex), fieldName, fieldType)); } } variants.Add(new NodeDefinitionEnum.Variant(TokensFrom(variantsStartIndex), variantName, variantFields)); } return new NodeDefinitionEnum(TokensFrom(startIndex), exported, name, variants); } if (TryExpectKeyword(Keyword.Let)) { var exported = modifiers.Remove(Keyword.Export); foreach (var modifier in modifiers) // todo(nub31): Add to diagnostics instead of throwing throw new CompileException(Diagnostic.Error("Invalid modifier for global variable").At(fileName, modifier.Value).Build()); var name = ExpectIdent(); ExpectSymbol(Symbol.Colon); var type = ParseType(); return new NodeDefinitionGlobalVariable(TokensFrom(startIndex), exported, name, type); } throw new CompileException(Diagnostic.Error("Not a valid definition").At(fileName, Peek()).Build()); } private NodeStatement ParseStatement() { var startIndex = index; if (TryExpectSymbol(Symbol.OpenCurly)) { var statements = new List(); while (!TryExpectSymbol(Symbol.CloseCurly)) statements.Add(ParseStatement()); return new NodeStatementBlock(TokensFrom(startIndex), statements); } if (TryExpectKeyword(Keyword.Return)) { var value = ParseExpression(); return new NodeStatementReturn(TokensFrom(startIndex), value); } if (TryExpectKeyword(Keyword.Let)) { var name = ExpectIdent(); ExpectSymbol(Symbol.Colon); var type = ParseType(); ExpectSymbol(Symbol.Equal); var value = ParseExpression(); return new NodeStatementVariableDeclaration(TokensFrom(startIndex), name, type, value); } if (TryExpectKeyword(Keyword.If)) { var condition = ParseExpression(); var thenBlock = ParseStatement(); NodeStatement? elseBlock = null; if (TryExpectKeyword(Keyword.Else)) elseBlock = ParseStatement(); return new NodeStatementIf(TokensFrom(startIndex), condition, thenBlock, elseBlock); } if (TryExpectKeyword(Keyword.While)) { var condition = ParseExpression(); var thenBlock = ParseStatement(); return new NodeStatementWhile(TokensFrom(startIndex), condition, thenBlock); } var target = ParseExpression(); if (TryExpectSymbol(Symbol.Equal)) { var value = ParseExpression(); return new NodeStatementAssignment(TokensFrom(startIndex), target, value); } return new NodeStatementExpression(TokensFrom(startIndex), target); } private NodeExpression ParseExpression(int minPrecedence = -1) { var startIndex = index; var left = ParseExpressionLeaf(); while (TryPeekBinaryOperator(out var op) && GetPrecedence(op) >= minPrecedence) { Next(); var right = ParseExpression(GetPrecedence(op) + 1); left = new NodeExpressionBinary(TokensFrom(startIndex), left, op, right); } return left; } private static int GetPrecedence(NodeExpressionBinary.Op operation) { return operation switch { NodeExpressionBinary.Op.Multiply => 10, NodeExpressionBinary.Op.Divide => 10, NodeExpressionBinary.Op.Modulo => 10, NodeExpressionBinary.Op.Add => 9, NodeExpressionBinary.Op.Subtract => 9, NodeExpressionBinary.Op.LeftShift => 8, NodeExpressionBinary.Op.RightShift => 8, NodeExpressionBinary.Op.GreaterThan => 7, NodeExpressionBinary.Op.GreaterThanOrEqual => 7, NodeExpressionBinary.Op.LessThan => 7, NodeExpressionBinary.Op.LessThanOrEqual => 7, NodeExpressionBinary.Op.Equal => 7, NodeExpressionBinary.Op.NotEqual => 7, // NodeExpressionBinary.Op.BitwiseAnd => 6, // NodeExpressionBinary.Op.BitwiseXor => 5, // NodeExpressionBinary.Op.BitwiseOr => 4, NodeExpressionBinary.Op.LogicalAnd => 3, NodeExpressionBinary.Op.LogicalOr => 2, _ => throw new ArgumentOutOfRangeException(nameof(operation), operation, null) }; } private NodeExpression ParseExpressionLeaf() { var startIndex = index; NodeExpression expr; if (TryExpectSymbol(Symbol.OpenParen)) { var value = ParseExpression(); ExpectSymbol(Symbol.CloseParen); expr = value; } else if (TryExpectSymbol(Symbol.Minus)) { var target = ParseExpression(); expr = new NodeExpressionUnary(TokensFrom(startIndex), target, NodeExpressionUnary.Op.Negate); } else if (TryExpectSymbol(Symbol.Bang)) { var target = ParseExpression(); expr = new NodeExpressionUnary(TokensFrom(startIndex), target, NodeExpressionUnary.Op.Invert); } else if (TryExpectIntLiteral(out var intLiteral)) { expr = new NodeExpressionIntLiteral(TokensFrom(startIndex), intLiteral); } else if (TryExpectStringLiteral(out var stringLiteral)) { expr = new NodeExpressionStringLiteral(TokensFrom(startIndex), stringLiteral); } else if (TryExpectBoolLiteral(out var boolLiteral)) { expr = new NodeExpressionBoolLiteral(TokensFrom(startIndex), boolLiteral); } else if (TryExpectIdent(out var ident)) { if (TryExpectSymbol(Symbol.ColonColon)) { var name = ExpectIdent(); expr = new NodeExpressionModuleIdent(TokensFrom(startIndex), ident, name); } else { expr = new NodeExpressionLocalIdent(TokensFrom(startIndex), ident); } } else if (TryExpectKeyword(Keyword.Struct)) { var module = ExpectIdent(); ExpectSymbol(Symbol.ColonColon); var name = ExpectIdent(); var initializers = new List(); ExpectSymbol(Symbol.OpenCurly); while (!TryExpectSymbol(Symbol.CloseCurly)) { var initializerStartIndex = startIndex; var fieldName = ExpectIdent(); ExpectSymbol(Symbol.Equal); var fieldValue = ParseExpression(); initializers.Add(new NodeExpressionStructLiteral.Initializer(TokensFrom(initializerStartIndex), fieldName, fieldValue)); } expr = new NodeExpressionStructLiteral(TokensFrom(startIndex), module, name, initializers); } else { throw new CompileException(Diagnostic.Error("Expected start of expression").At(fileName, Peek()).Build()); } while (true) { if (TryExpectSymbol(Symbol.Period)) { var name = ExpectIdent(); expr = new NodeExpressionMemberAccess(TokensFrom(startIndex), expr, name); } else if (TryExpectSymbol(Symbol.OpenParen)) { var parameters = new List(); while (!TryExpectSymbol(Symbol.CloseParen)) parameters.Add(ParseExpression()); expr = new NodeExpressionFuncCall(TokensFrom(startIndex), expr, parameters); } else { break; } } return expr; } private NodeType ParseType() { var startIndex = index; if (TryExpectSymbol(Symbol.Caret)) { var to = ParseType(); return new NodeTypePointer(TokensFrom(startIndex), to); } if (TryExpectKeyword(Keyword.Func)) { var parameters = new List(); ExpectSymbol(Symbol.OpenParen); while (!TryExpectSymbol(Symbol.CloseParen)) { parameters.Add(ParseType()); } ExpectSymbol(Symbol.Colon); var returnType = ParseType(); return new NodeTypeFunc(TokensFrom(startIndex), parameters, returnType); } if (TryExpectIdent(out var ident)) { switch (ident.Ident) { case "void": return new NodeTypeVoid(TokensFrom(startIndex)); case "string": return new NodeTypeString(TokensFrom(startIndex)); case "bool": return new NodeTypeBool(TokensFrom(startIndex)); case "i8": return new NodeTypeSInt(TokensFrom(startIndex), 8); case "i16": return new NodeTypeSInt(TokensFrom(startIndex), 16); case "i32": return new NodeTypeSInt(TokensFrom(startIndex), 32); case "i64": return new NodeTypeSInt(TokensFrom(startIndex), 64); case "u8": return new NodeTypeUInt(TokensFrom(startIndex), 8); case "u16": return new NodeTypeUInt(TokensFrom(startIndex), 16); case "u32": return new NodeTypeUInt(TokensFrom(startIndex), 32); case "u64": return new NodeTypeUInt(TokensFrom(startIndex), 64); default: ExpectSymbol(Symbol.ColonColon); var name = ExpectIdent(); return new NodeTypeCustom(TokensFrom(startIndex), ident, name); } } throw new CompileException(Diagnostic.Error("Expected type").At(fileName, Peek()).Build()); } private List TokensFrom(int startIndex) { return tokens.GetRange(startIndex, index - startIndex); } private void ExpectKeyword(Keyword keyword) { if (Peek() is TokenKeyword token && token.Keyword == keyword) { Next(); return; } throw new CompileException(Diagnostic.Error($"Expected '{keyword.AsString()}'").At(fileName, Peek()).Build()); } private bool TryExpectKeyword(Keyword keyword) { if (Peek() is TokenKeyword token && token.Keyword == keyword) { Next(); return true; } return false; } private void ExpectSymbol(Symbol symbol) { if (Peek() is TokenSymbol token && token.Symbol == symbol) { Next(); return; } throw new CompileException(Diagnostic.Error($"Expected '{symbol.AsString()}'").At(fileName, Peek()).Build()); } private bool TryExpectSymbol(Symbol symbol) { if (Peek() is TokenSymbol token && token.Symbol == symbol) { Next(); return true; } return false; } private TokenIdent ExpectIdent() { if (Peek() is TokenIdent token) { Next(); return token; } throw new CompileException(Diagnostic.Error("Expected identifier").At(fileName, Peek()).Build()); } private bool TryExpectIdent([NotNullWhen(true)] out TokenIdent? ident) { if (Peek() is TokenIdent token) { Next(); ident = token; return true; } ident = null; return false; } private bool TryExpectIntLiteral([NotNullWhen(true)] out TokenIntLiteral? intLiteral) { if (Peek() is TokenIntLiteral token) { Next(); intLiteral = token; return true; } intLiteral = null; return false; } private bool TryExpectStringLiteral([NotNullWhen(true)] out TokenStringLiteral? stringLiteral) { if (Peek() is TokenStringLiteral token) { Next(); stringLiteral = token; return true; } stringLiteral = null; return false; } private bool TryExpectBoolLiteral([NotNullWhen(true)] out TokenBoolLiteral? boolLiteral) { if (Peek() is TokenBoolLiteral token) { Next(); boolLiteral = token; return true; } boolLiteral = null; return false; } private void Next() { if (index >= tokens.Count) throw new CompileException(Diagnostic.Error("Unexpected end of tokens").At(fileName, Peek()).Build()); index += 1; } private Token? Peek(int offset = 0) { if (index + offset >= tokens.Count) return null; return tokens[index + offset]; } private bool TryPeekBinaryOperator(out NodeExpressionBinary.Op op) { if (Peek() is not TokenSymbol token) { op = default; return false; } switch (token.Symbol) { case Symbol.Plus: op = NodeExpressionBinary.Op.Add; return true; case Symbol.Minus: op = NodeExpressionBinary.Op.Subtract; return true; case Symbol.Star: op = NodeExpressionBinary.Op.Multiply; return true; case Symbol.ForwardSlash: op = NodeExpressionBinary.Op.Divide; return true; case Symbol.Percent: op = NodeExpressionBinary.Op.Modulo; return true; case Symbol.BangEqual: op = NodeExpressionBinary.Op.NotEqual; return true; case Symbol.EqualEqual: op = NodeExpressionBinary.Op.Equal; return true; case Symbol.LessThan: op = NodeExpressionBinary.Op.LessThan; return true; case Symbol.LessThanEqual: op = NodeExpressionBinary.Op.LessThanOrEqual; return true; case Symbol.GreaterThan: op = NodeExpressionBinary.Op.GreaterThan; return true; case Symbol.GreaterThanEqual: op = NodeExpressionBinary.Op.GreaterThanOrEqual; return true; case Symbol.LessThanLessThan: op = NodeExpressionBinary.Op.LeftShift; return true; case Symbol.GreaterThanGreaterThan: op = NodeExpressionBinary.Op.RightShift; return true; case Symbol.AmpersandAmpersand: op = NodeExpressionBinary.Op.LogicalAnd; return true; case Symbol.PipePipe: op = NodeExpressionBinary.Op.LogicalOr; return true; default: op = default; return false; } } } public class Ast(string fileName, TokenIdent moduleName, List definitions) { public string FileName { get; } = fileName; public TokenIdent ModuleName { get; } = moduleName; public List Definitions { get; } = definitions; } public abstract class Node(List tokens) { public List Tokens { get; } = tokens; } public abstract class NodeDefinition(List tokens) : Node(tokens); public class NodeDefinitionFunc(List tokens, bool exported, TokenIdent name, List parameters, NodeStatement body, NodeType returnType) : NodeDefinition(tokens) { public bool Exported { get; } = exported; public TokenIdent Name { get; } = name; public List Parameters { get; } = parameters; public NodeStatement Body { get; } = body; public NodeType ReturnType { get; } = returnType; public class Param(List tokens, TokenIdent name, NodeType type) : Node(tokens) { public TokenIdent Name { get; } = name; public NodeType Type { get; } = type; } } public class NodeDefinitionStruct(List tokens, bool exported, bool packed, TokenIdent name, List fields) : NodeDefinition(tokens) { public bool Exported { get; } = exported; public bool Packed { get; } = packed; public TokenIdent Name { get; } = name; public List Fields { get; } = fields; public class Field(List tokens, TokenIdent name, NodeType type) : Node(tokens) { public TokenIdent Name { get; } = name; public NodeType Type { get; } = type; } } public class NodeDefinitionEnum(List tokens, bool exported, TokenIdent name, List variants) : NodeDefinition(tokens) { public bool Exported { get; } = exported; public TokenIdent Name { get; } = name; public List Variants { get; } = variants; public class Variant(List tokens, TokenIdent name, List fields) : Node(tokens) { public TokenIdent Name { get; } = name; public List Fields { get; } = fields; public class Field(List tokens, TokenIdent name, NodeType type) : Node(tokens) { public TokenIdent Name { get; } = name; public NodeType Type { get; } = type; } } } public class NodeDefinitionGlobalVariable(List tokens, bool exported, TokenIdent name, NodeType type) : NodeDefinition(tokens) { public bool Exported { get; } = exported; public TokenIdent Name { get; } = name; public NodeType Type { get; } = type; } public abstract class NodeStatement(List tokens) : Node(tokens); public class NodeStatementBlock(List tokens, List statements) : NodeStatement(tokens) { public List Statements { get; } = statements; } public class NodeStatementExpression(List tokens, NodeExpression expression) : NodeStatement(tokens) { public NodeExpression Expression { get; } = expression; } public class NodeStatementReturn(List tokens, NodeExpression value) : NodeStatement(tokens) { public NodeExpression Value { get; } = value; } public class NodeStatementVariableDeclaration(List tokens, TokenIdent name, NodeType type, NodeExpression value) : NodeStatement(tokens) { public TokenIdent Name { get; } = name; public NodeType Type { get; } = type; public NodeExpression Value { get; } = value; } public class NodeStatementAssignment(List tokens, NodeExpression target, NodeExpression value) : NodeStatement(tokens) { public NodeExpression Target { get; } = target; public NodeExpression Value { get; } = value; } public class NodeStatementIf(List tokens, NodeExpression condition, NodeStatement thenBlock, NodeStatement? elseBlock) : NodeStatement(tokens) { public NodeExpression Condition { get; } = condition; public NodeStatement ThenBlock { get; } = thenBlock; public NodeStatement? ElseBlock { get; } = elseBlock; } public class NodeStatementWhile(List tokens, NodeExpression condition, NodeStatement block) : NodeStatement(tokens) { public NodeExpression Condition { get; } = condition; public NodeStatement Block { get; } = block; } public abstract class NodeExpression(List tokens) : Node(tokens); public class NodeExpressionIntLiteral(List tokens, TokenIntLiteral value) : NodeExpression(tokens) { public TokenIntLiteral Value { get; } = value; } public class NodeExpressionStringLiteral(List tokens, TokenStringLiteral value) : NodeExpression(tokens) { public TokenStringLiteral Value { get; } = value; } public class NodeExpressionBoolLiteral(List tokens, TokenBoolLiteral value) : NodeExpression(tokens) { public TokenBoolLiteral Value { get; } = value; } public class NodeExpressionStructLiteral(List tokens, TokenIdent module, TokenIdent name, List initializers) : NodeExpression(tokens) { public TokenIdent Module { get; } = module; public TokenIdent Name { get; } = name; public List Initializers { get; } = initializers; public class Initializer(List tokens, TokenIdent name, NodeExpression value) : Node(tokens) { public TokenIdent Name { get; } = name; public NodeExpression Value { get; } = value; } } public class NodeExpressionMemberAccess(List tokens, NodeExpression target, TokenIdent name) : NodeExpression(tokens) { public NodeExpression Target { get; } = target; public TokenIdent Name { get; } = name; } public class NodeExpressionFuncCall(List tokens, NodeExpression target, List parameters) : NodeExpression(tokens) { public NodeExpression Target { get; } = target; public List Parameters { get; } = parameters; } public class NodeExpressionLocalIdent(List tokens, TokenIdent value) : NodeExpression(tokens) { public TokenIdent Value { get; } = value; } public class NodeExpressionModuleIdent(List tokens, TokenIdent module, TokenIdent value) : NodeExpression(tokens) { public TokenIdent Module { get; } = module; public TokenIdent Value { get; } = value; } public class NodeExpressionBinary(List tokens, NodeExpression left, NodeExpressionBinary.Op operation, NodeExpression right) : NodeExpression(tokens) { public NodeExpression Left { get; } = left; public Op Operation { get; } = operation; public NodeExpression Right { get; } = right; public enum Op { Add, Subtract, Multiply, Divide, Modulo, Equal, NotEqual, LessThan, LessThanOrEqual, GreaterThan, GreaterThanOrEqual, LeftShift, RightShift, // BitwiseAnd, // BitwiseXor, // BitwiseOr, LogicalAnd, LogicalOr, } } public class NodeExpressionUnary(List tokens, NodeExpression target, NodeExpressionUnary.Op op) : NodeExpression(tokens) { public NodeExpression Target { get; } = target; public Op Operation { get; } = op; public enum Op { Negate, Invert, } } public abstract class NodeType(List tokens) : Node(tokens); public class NodeTypeVoid(List tokens) : NodeType(tokens); public class NodeTypeUInt(List tokens, int width) : NodeType(tokens) { public int Width { get; } = width; } public class NodeTypeSInt(List tokens, int width) : NodeType(tokens) { public int Width { get; } = width; } public class NodeTypeBool(List tokens) : NodeType(tokens); public class NodeTypeString(List tokens) : NodeType(tokens); public class NodeTypeCustom(List tokens, TokenIdent module, TokenIdent name) : NodeType(tokens) { public TokenIdent Module { get; } = module; public TokenIdent Name { get; } = name; } public class NodeTypePointer(List tokens, NodeType to) : NodeType(tokens) { public NodeType To { get; } = to; } public class NodeTypeFunc(List tokens, List parameters, NodeType returnType) : NodeType(tokens) { public List Parameters { get; } = parameters; public NodeType ReturnType { get; } = returnType; }