This commit is contained in:
nub31
2025-06-22 18:18:42 +02:00
parent 88d85c6849
commit 33b101681d
39 changed files with 154 additions and 163 deletions

View File

@@ -130,7 +130,7 @@ public static class ConsoleColors
public static string ColorizeSource(string source)
{
var sourceText = new SourceText(string.Empty, source);
var tokens = Tokenizer.Tokenize(sourceText).Value;
var tokens = Tokenizer.Tokenize(sourceText, out _);
var result = new StringBuilder();
var lastCharIndex = 0;

View File

@@ -63,7 +63,7 @@ public class Diagnostic
Help = help;
}
public string Format()
public string FormatANSI()
{
var sb = new StringBuilder();

View File

@@ -1,20 +0,0 @@
namespace Syntax.Diagnostics;
public class DiagnosticsResult(List<Diagnostic> diagnostics)
{
public bool HasErrors => diagnostics.Any(d => d.Severity == DiagnosticSeverity.Error);
public void PrintAllDiagnostics()
{
foreach (var diagnostic in diagnostics)
{
Console.Error.WriteLine(diagnostic.Format());
Console.Error.WriteLine();
}
}
}
public class DiagnosticsResult<TResult>(List<Diagnostic> diagnostics, TResult value) : DiagnosticsResult(diagnostics)
{
public TResult Value { get; } = value;
}

View File

@@ -3,7 +3,7 @@ using Syntax.Tokenization;
namespace Syntax.Parsing.Definitions;
public abstract class DefinitionNode(IReadOnlyList<Token> tokens, Optional<string> documentation, string @namespace) : Node(tokens)
public abstract class DefinitionNode(IEnumerable<Token> tokens, Optional<string> documentation, string @namespace) : Node(tokens)
{
public Optional<string> Documentation { get; } = documentation;
public string Namespace { get; set; } = @namespace;

View File

@@ -22,7 +22,7 @@ public interface IFuncSignature
public string ToString() => $"{Name}({string.Join(", ", Parameters.Select(p => p.ToString()))}){": " + ReturnType}";
}
public class LocalFuncDefinitionNode(IReadOnlyList<Token> tokens, Optional<string> documentation, string @namespace, string name, List<FuncParameter> parameters, BlockNode body, NubType returnType, bool exported) : DefinitionNode(tokens, documentation, @namespace), IFuncSignature
public class LocalFuncDefinitionNode(IEnumerable<Token> tokens, Optional<string> documentation, string @namespace, string name, List<FuncParameter> parameters, BlockNode body, NubType returnType, bool exported) : DefinitionNode(tokens, documentation, @namespace), IFuncSignature
{
public string Name { get; } = name;
public List<FuncParameter> Parameters { get; } = parameters;
@@ -33,7 +33,7 @@ public class LocalFuncDefinitionNode(IReadOnlyList<Token> tokens, Optional<strin
public override string ToString() => $"{Name}({string.Join(", ", Parameters.Select(p => p.ToString()))}){": " + ReturnType}";
}
public class ExternFuncDefinitionNode(IReadOnlyList<Token> tokens, Optional<string> documentation, string @namespace, string name, string callName, List<FuncParameter> parameters, NubType returnType) : DefinitionNode(tokens, documentation, @namespace), IFuncSignature
public class ExternFuncDefinitionNode(IEnumerable<Token> tokens, Optional<string> documentation, string @namespace, string name, string callName, List<FuncParameter> parameters, NubType returnType) : DefinitionNode(tokens, documentation, @namespace), IFuncSignature
{
public string Name { get; } = name;
public string CallName { get; } = callName;

View File

@@ -12,7 +12,7 @@ public class StructField(string name, NubType type, Optional<ExpressionNode> val
public Optional<ExpressionNode> Value { get; } = value;
}
public class StructDefinitionNode(IReadOnlyList<Token> tokens, Optional<string> documentation, string @namespace, string name, List<StructField> fields) : DefinitionNode(tokens, documentation, @namespace)
public class StructDefinitionNode(IEnumerable<Token> tokens, Optional<string> documentation, string @namespace, string name, List<StructField> fields) : DefinitionNode(tokens, documentation, @namespace)
{
public string Name { get; } = name;
public List<StructField> Fields { get; } = fields;

View File

@@ -2,7 +2,7 @@ using Syntax.Tokenization;
namespace Syntax.Parsing.Expressions;
public class AddressOfNode(IReadOnlyList<Token> tokens, LValueNode expression) : ExpressionNode(tokens)
public class AddressOfNode(IEnumerable<Token> tokens, LValueNode expression) : ExpressionNode(tokens)
{
public LValueNode Expression { get; } = expression;
}

View File

@@ -5,7 +5,7 @@ using Syntax.Typing;
namespace Syntax.Parsing.Expressions;
public class AnonymousFuncNode(IReadOnlyList<Token> tokens, List<FuncParameter> parameters, BlockNode body, NubType returnType) : ExpressionNode(tokens)
public class AnonymousFuncNode(IEnumerable<Token> tokens, List<FuncParameter> parameters, BlockNode body, NubType returnType) : ExpressionNode(tokens)
{
public List<FuncParameter> Parameters { get; } = parameters;
public BlockNode Body { get; } = body;

View File

@@ -2,7 +2,7 @@ using Syntax.Tokenization;
namespace Syntax.Parsing.Expressions;
public class ArrayIndexAccessNode(IReadOnlyList<Token> tokens, ExpressionNode array, ExpressionNode index) : LValueNode(tokens)
public class ArrayIndexAccessNode(IEnumerable<Token> tokens, ExpressionNode array, ExpressionNode index) : LValueNode(tokens)
{
public ExpressionNode Array { get; } = array;
public ExpressionNode Index { get; } = index;

View File

@@ -3,7 +3,7 @@ using Syntax.Typing;
namespace Syntax.Parsing.Expressions;
public class ArrayInitializerNode(IReadOnlyList<Token> tokens, ExpressionNode capacity, NubType elementType) : ExpressionNode(tokens)
public class ArrayInitializerNode(IEnumerable<Token> tokens, ExpressionNode capacity, NubType elementType) : ExpressionNode(tokens)
{
public ExpressionNode Capacity { get; } = capacity;
public NubType ElementType { get; } = elementType;

View File

@@ -2,7 +2,7 @@
namespace Syntax.Parsing.Expressions;
public class BinaryExpressionNode(IReadOnlyList<Token> tokens, ExpressionNode left, BinaryExpressionOperator @operator, ExpressionNode right) : ExpressionNode(tokens)
public class BinaryExpressionNode(IEnumerable<Token> tokens, ExpressionNode left, BinaryExpressionOperator @operator, ExpressionNode right) : ExpressionNode(tokens)
{
public ExpressionNode Left { get; } = left;
public BinaryExpressionOperator Operator { get; } = @operator;

View File

@@ -2,7 +2,7 @@ using Syntax.Tokenization;
namespace Syntax.Parsing.Expressions;
public class DereferenceNode(IReadOnlyList<Token> tokens, ExpressionNode expression) : LValueNode(tokens)
public class DereferenceNode(IEnumerable<Token> tokens, ExpressionNode expression) : LValueNode(tokens)
{
public ExpressionNode Expression { get; } = expression;
}

View File

@@ -3,7 +3,7 @@ using Syntax.Typing;
namespace Syntax.Parsing.Expressions;
public abstract class ExpressionNode(IReadOnlyList<Token> tokens) : Node(tokens)
public abstract class ExpressionNode(IEnumerable<Token> tokens) : Node(tokens)
{
private NubType? _type;
public NubType Type
@@ -13,4 +13,4 @@ public abstract class ExpressionNode(IReadOnlyList<Token> tokens) : Node(tokens)
}
}
public abstract class LValueNode(IReadOnlyList<Token> tokens) : ExpressionNode(tokens);
public abstract class LValueNode(IEnumerable<Token> tokens) : ExpressionNode(tokens);

View File

@@ -3,7 +3,7 @@ using Syntax.Typing;
namespace Syntax.Parsing.Expressions;
public class FixedArrayInitializerNode(IReadOnlyList<Token> tokens, NubType elementType, int capacity) : ExpressionNode(tokens)
public class FixedArrayInitializerNode(IEnumerable<Token> tokens, NubType elementType, int capacity) : ExpressionNode(tokens)
{
public NubType ElementType { get; } = elementType;
public int Capacity { get; } = capacity;

View File

@@ -2,7 +2,7 @@
namespace Syntax.Parsing.Expressions;
public class FuncCallNode(IReadOnlyList<Token> tokens, ExpressionNode expression, List<ExpressionNode> parameters) : ExpressionNode(tokens)
public class FuncCallNode(IEnumerable<Token> tokens, ExpressionNode expression, List<ExpressionNode> parameters) : ExpressionNode(tokens)
{
public ExpressionNode Expression = expression;
public List<ExpressionNode> Parameters { get; } = parameters;

View File

@@ -3,7 +3,7 @@ using Syntax.Tokenization;
namespace Syntax.Parsing.Expressions;
public class IdentifierNode(IReadOnlyList<Token> tokens, Optional<string> @namespace, string name) : LValueNode(tokens)
public class IdentifierNode(IEnumerable<Token> tokens, Optional<string> @namespace, string name) : LValueNode(tokens)
{
public Optional<string> Namespace { get; } = @namespace;
public string Name { get; } = name;

View File

@@ -2,7 +2,7 @@
namespace Syntax.Parsing.Expressions;
public class LiteralNode(IReadOnlyList<Token> tokens, string literal, LiteralKind kind) : ExpressionNode(tokens)
public class LiteralNode(IEnumerable<Token> tokens, string literal, LiteralKind kind) : ExpressionNode(tokens)
{
public string Literal { get; } = literal;
public LiteralKind Kind { get; } = kind;

View File

@@ -2,7 +2,7 @@
namespace Syntax.Parsing.Expressions;
public class MemberAccessNode(IReadOnlyList<Token> tokens, ExpressionNode expression, string member) : LValueNode(tokens)
public class MemberAccessNode(IEnumerable<Token> tokens, ExpressionNode expression, string member) : LValueNode(tokens)
{
public ExpressionNode Expression { get; } = expression;
public string Member { get; } = member;

View File

@@ -3,7 +3,7 @@ using Syntax.Typing;
namespace Syntax.Parsing.Expressions;
public class StructInitializerNode(IReadOnlyList<Token> tokens, NubStructType structType, Dictionary<string, ExpressionNode> initializers) : ExpressionNode(tokens)
public class StructInitializerNode(IEnumerable<Token> tokens, NubStructType structType, Dictionary<string, ExpressionNode> initializers) : ExpressionNode(tokens)
{
public NubStructType StructType { get; } = structType;
public Dictionary<string, ExpressionNode> Initializers { get; } = initializers;

View File

@@ -2,7 +2,7 @@ using Syntax.Tokenization;
namespace Syntax.Parsing.Expressions;
public class UnaryExpressionNode(IReadOnlyList<Token> tokens, UnaryExpressionOperator @operator, ExpressionNode operand) : ExpressionNode(tokens)
public class UnaryExpressionNode(IEnumerable<Token> tokens, UnaryExpressionOperator @operator, ExpressionNode operand) : ExpressionNode(tokens)
{
public UnaryExpressionOperator Operator { get; } = @operator;
public ExpressionNode Operand { get; } = operand;

View File

@@ -2,7 +2,7 @@
namespace Syntax.Parsing;
public abstract class Node(IReadOnlyList<Token> tokens)
public abstract class Node(IEnumerable<Token> tokens)
{
public IReadOnlyList<Token> Tokens { get; set; } = tokens;
public IEnumerable<Token> Tokens { get; set; } = tokens;
}

View File

@@ -14,10 +14,10 @@ public static class Parser
{
private static string _namespace = null!;
private static List<Diagnostic> _diagnostics = [];
private static List<Token> _tokens = [];
private static IEnumerable<Token> _tokens = [];
private static int _index;
public static DiagnosticsResult<CompilationUnit?> ParseFile(List<Token> tokens)
public static CompilationUnit? ParseFile(IEnumerable<Token> tokens, out IEnumerable<Diagnostic> diagnostics)
{
_tokens = tokens;
_namespace = null!;
@@ -33,7 +33,8 @@ public static class Parser
catch (ParseException ex)
{
_diagnostics.Add(ex.Diagnostic);
return new DiagnosticsResult<CompilationUnit?>(_diagnostics, null);
diagnostics = _diagnostics;
return null;
}
try
@@ -46,7 +47,8 @@ public static class Parser
definitions.Add(definition);
}
return new DiagnosticsResult<CompilationUnit?>(_diagnostics, new CompilationUnit(_namespace, definitions));
diagnostics = _diagnostics;
return new CompilationUnit(_namespace, definitions);
}
catch (ParseException ex)
{
@@ -54,7 +56,8 @@ public static class Parser
RecoverToNextDefinition();
}
return new DiagnosticsResult<CompilationUnit?>(_diagnostics, null);
diagnostics = _diagnostics;
return null;
}
private static DefinitionNode ParseDefinition()
@@ -63,7 +66,7 @@ public static class Parser
List<ModifierToken> modifiers = [];
List<string> documentationParts = [];
while (_index < _tokens.Count && _tokens[_index] is DocumentationToken commentToken)
while (_index < _tokens.Count() && _tokens.ElementAt(_index) is DocumentationToken commentToken)
{
documentationParts.Add(commentToken.Documentation);
_index++;
@@ -898,14 +901,14 @@ public static class Parser
private static Optional<Token> Peek(int offset = 0)
{
var peekIndex = _index + offset;
while (peekIndex < _tokens.Count && _tokens[peekIndex] is DocumentationToken)
while (peekIndex < _tokens.Count() && _tokens.ElementAt(peekIndex) is DocumentationToken)
{
peekIndex++;
}
if (peekIndex < _tokens.Count)
if (peekIndex < _tokens.Count())
{
return _tokens[peekIndex];
return _tokens.ElementAt(peekIndex);
}
return Optional<Token>.Empty();
@@ -913,7 +916,7 @@ public static class Parser
private static void Next()
{
while (_index < _tokens.Count && _tokens[_index] is DocumentationToken)
while (_index < _tokens.Count() && _tokens.ElementAt(_index) is DocumentationToken)
{
_index++;
}
@@ -921,9 +924,9 @@ public static class Parser
_index++;
}
private static IReadOnlyList<Token> GetTokensForNode(int startIndex)
private static IEnumerable<Token> GetTokensForNode(int startIndex)
{
return _tokens[startIndex..Math.Min(_index, _tokens.Count - 1)];
return _tokens.Skip(startIndex).Take(Math.Min(_index, _tokens.Count() - 1) - startIndex);
}
}

View File

@@ -3,7 +3,7 @@ using Syntax.Tokenization;
namespace Syntax.Parsing.Statements;
public class ArrayIndexAssignmentNode(IReadOnlyList<Token> tokens, ArrayIndexAccessNode arrayIndexAccess, ExpressionNode value) : StatementNode(tokens)
public class ArrayIndexAssignmentNode(IEnumerable<Token> tokens, ArrayIndexAccessNode arrayIndexAccess, ExpressionNode value) : StatementNode(tokens)
{
public ArrayIndexAccessNode ArrayIndexAccess { get; } = arrayIndexAccess;
public ExpressionNode Value { get; } = value;

View File

@@ -2,7 +2,7 @@
namespace Syntax.Parsing.Statements;
public class BlockNode(IReadOnlyList<Token> tokens, List<StatementNode> statements) : Node(tokens)
public class BlockNode(IEnumerable<Token> tokens, List<StatementNode> statements) : Node(tokens)
{
public List<StatementNode> Statements { get; } = statements;
}

View File

@@ -2,4 +2,4 @@ using Syntax.Tokenization;
namespace Syntax.Parsing.Statements;
public class BreakNode(IReadOnlyList<Token> tokens) : StatementNode(tokens);
public class BreakNode(IEnumerable<Token> tokens) : StatementNode(tokens);

View File

@@ -2,4 +2,4 @@ using Syntax.Tokenization;
namespace Syntax.Parsing.Statements;
public class ContinueNode(IReadOnlyList<Token> tokens) : StatementNode(tokens);
public class ContinueNode(IEnumerable<Token> tokens) : StatementNode(tokens);

View File

@@ -3,7 +3,7 @@ using Syntax.Tokenization;
namespace Syntax.Parsing.Statements;
public class DereferenceAssignmentNode(IReadOnlyList<Token> tokens, DereferenceNode dereference, ExpressionNode value) : StatementNode(tokens)
public class DereferenceAssignmentNode(IEnumerable<Token> tokens, DereferenceNode dereference, ExpressionNode value) : StatementNode(tokens)
{
public DereferenceNode Dereference { get; } = dereference;
public ExpressionNode Value { get; } = value;

View File

@@ -4,7 +4,7 @@ using Syntax.Tokenization;
namespace Syntax.Parsing.Statements;
public class IfNode(IReadOnlyList<Token> tokens, ExpressionNode condition, BlockNode body, Optional<Variant<IfNode, BlockNode>> @else) : StatementNode(tokens)
public class IfNode(IEnumerable<Token> tokens, ExpressionNode condition, BlockNode body, Optional<Variant<IfNode, BlockNode>> @else) : StatementNode(tokens)
{
public ExpressionNode Condition { get; } = condition;
public BlockNode Body { get; } = body;

View File

@@ -3,7 +3,7 @@ using Syntax.Tokenization;
namespace Syntax.Parsing.Statements;
public class MemberAssignmentNode(IReadOnlyList<Token> tokens, MemberAccessNode expression, ExpressionNode value) : StatementNode(tokens)
public class MemberAssignmentNode(IEnumerable<Token> tokens, MemberAccessNode expression, ExpressionNode value) : StatementNode(tokens)
{
public MemberAccessNode MemberAccess { get; } = expression;
public ExpressionNode Value { get; } = value;

View File

@@ -4,7 +4,7 @@ using Syntax.Tokenization;
namespace Syntax.Parsing.Statements;
public class ReturnNode(IReadOnlyList<Token> tokens, Optional<ExpressionNode> value) : StatementNode(tokens)
public class ReturnNode(IEnumerable<Token> tokens, Optional<ExpressionNode> value) : StatementNode(tokens)
{
public Optional<ExpressionNode> Value { get; } = value;
}

View File

@@ -3,7 +3,7 @@ using Syntax.Tokenization;
namespace Syntax.Parsing.Statements;
public class StatementExpressionNode(IReadOnlyList<Token> tokens, ExpressionNode expression) : StatementNode(tokens)
public class StatementExpressionNode(IEnumerable<Token> tokens, ExpressionNode expression) : StatementNode(tokens)
{
public ExpressionNode Expression { get; } = expression;
}

View File

@@ -2,4 +2,4 @@
namespace Syntax.Parsing.Statements;
public abstract class StatementNode(IReadOnlyList<Token> tokens) : Node(tokens);
public abstract class StatementNode(IEnumerable<Token> tokens) : Node(tokens);

View File

@@ -3,7 +3,7 @@ using Syntax.Tokenization;
namespace Syntax.Parsing.Statements;
public class VariableAssignmentNode(IReadOnlyList<Token> tokens, IdentifierNode identifier, ExpressionNode value) : StatementNode(tokens)
public class VariableAssignmentNode(IEnumerable<Token> tokens, IdentifierNode identifier, ExpressionNode value) : StatementNode(tokens)
{
public IdentifierNode Identifier { get; } = identifier;
public ExpressionNode Value { get; } = value;

View File

@@ -3,7 +3,7 @@ using Syntax.Typing;
namespace Syntax.Parsing.Statements;
public class VariableDeclarationNode(IReadOnlyList<Token> tokens, string name, NubType type) : StatementNode(tokens)
public class VariableDeclarationNode(IEnumerable<Token> tokens, string name, NubType type) : StatementNode(tokens)
{
public string Name { get; } = name;
public NubType Type { get; } = type;

View File

@@ -3,7 +3,7 @@ using Syntax.Tokenization;
namespace Syntax.Parsing.Statements;
public class WhileNode(IReadOnlyList<Token> tokens, ExpressionNode condition, BlockNode body) : StatementNode(tokens)
public class WhileNode(IEnumerable<Token> tokens, ExpressionNode condition, BlockNode body) : StatementNode(tokens)
{
public ExpressionNode Condition { get; } = condition;
public BlockNode Body { get; } = body;

View File

@@ -62,19 +62,21 @@ public static class Tokenizer
private static SourceText _sourceText;
private static int _index;
public static DiagnosticsResult<List<Token>> Tokenize(SourceText sourceText)
public static IEnumerable<Token> Tokenize(SourceText sourceText, out IEnumerable<Diagnostic> diagnostics)
{
_sourceText = sourceText;
_index = 0;
List<Token> tokens = [];
while (ParseToken().TryGetValue(out var token))
{
tokens.Add(token);
}
return new DiagnosticsResult<List<Token>>([], tokens);
// TODO: Implement diagnostics
diagnostics = [];
return tokens;
}
private static void ConsumeWhitespace()
@@ -99,7 +101,7 @@ public static class Tokenizer
{
Next();
Next();
if (Peek().TryGetValue(out var thirdChar) && thirdChar == '/')
{
Next();
@@ -109,6 +111,7 @@ public static class Tokenizer
buffer += character;
Next();
}
Next();
return new DocumentationToken(CreateSpan(startIndex), buffer);
}
@@ -117,6 +120,7 @@ public static class Tokenizer
{
Next();
}
Next();
return ParseToken();
}

View File

@@ -18,7 +18,7 @@ public static class TypeChecker
private static NubType? _currentFunctionReturnType;
private static Queue<AnonymousFuncNode> _anonymousFunctions = [];
public static DiagnosticsResult Check(CompilationUnit compilationUnit, DefinitionTable definitionTable)
public static void Check(CompilationUnit compilationUnit, DefinitionTable definitionTable, out IEnumerable<Diagnostic> diagnostics)
{
_compilationUnit = compilationUnit;
_definitionTable = definitionTable;
@@ -43,8 +43,8 @@ public static class TypeChecker
CheckFuncDef(func.Parameters, func.Body, func.ReturnType);
}
return new DiagnosticsResult(_diagnostics);
diagnostics = _diagnostics;
}
private static void CheckStructDef(StructDefinitionNode structDef)