nub31/nub-lang
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

WIP: dev #1

Draft
nub31 wants to merge 103 commits from dev into master
pull from: dev
merge into: nub31:master
Conversation 0 Commits 103 Files Changed 75 +4 -4187
31 changed files with 4 additions and 4187 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 5105a689df - Show all commits

4
compiler/.gitignore vendored
View File

@@ -1,4 +1,4 @@
# Common IntelliJ Platform excludes
# Common IntelliJ Platform excludes
# User specific
**/.idea/**/workspace.xml
@@ -32,3 +32,5 @@ _UpgradeReport_Files/
Thumbs.db
Desktop.ini
.DS_Store
out.c

13
compiler/.idea/.idea.Compiler/.idea/.gitignore generated vendored
View File

@@ -1,13 +0,0 @@
# Default ignored files
/shelf/
/workspace.xml
# Rider ignored files
/modules.xml
/projectSettingsUpdater.xml
/contentModel.xml
/.idea.Compiler.iml
# Editor-based HTTP Client requests
/httpRequests/
# Datasource local storage ignored files
/dataSources/
/dataSources.local.xml

1
compiler/.idea/.idea.Compiler/.idea/.name generated
View File

@@ -1 +0,0 @@
Compiler

4
compiler/.idea/.idea.Compiler/.idea/encodings.xml generated
View File

@@ -1,4 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="Encoding" addBOMForNewFiles="with BOM under Windows, with no BOM otherwise" />
</project>

10
compiler/.idea/.idea.Compiler/.idea/indexLayout.xml generated
View File

@@ -1,10 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="UserContentModel">
<attachedFolders>
<Path>Runtime</Path>
</attachedFolders>
<explicitIncludes />
<explicitExcludes />
</component>
</project>

6
compiler/.idea/.idea.Compiler/.idea/vcs.xml generated
View File

@@ -1,6 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="VcsDirectoryMappings">
<mapping directory="$PROJECT_DIR$/.." vcs="Git" />
</component>
</project>

0
compiler/Compiler/Compiler.csproj → compiler/Compiler.csproj
View File

1
compiler/Compiler/.gitignore vendored
View File

@@ -1 +0,0 @@
out.c

0
compiler/Compiler/Diagnostic.cs → compiler/Diagnostic.cs
View File

0
compiler/Compiler/Generator.cs → compiler/Generator.cs
View File

16
compiler/NubLang.CLI/NubLang.CLI.csproj
View File

@@ -1,16 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<AssemblyName>nubc</AssemblyName>
<OutputType>Exe</OutputType>
<TargetFramework>net9.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<PublishAot>true</PublishAot>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\NubLang\NubLang.csproj" />
</ItemGroup>
</Project>

43
compiler/NubLang.CLI/Program.cs
View File

@@ -1,43 +0,0 @@
using NubLang.Ast;
using NubLang.Diagnostics;
using NubLang.Syntax;
var diagnostics = new List<Diagnostic>();
var syntaxTrees = new List<SyntaxTree>();
foreach (var file in args)
{
var tokenizer = new Tokenizer(file, File.ReadAllText(file));
tokenizer.Tokenize();
diagnostics.AddRange(tokenizer.Diagnostics);
var parser = new Parser();
var syntaxTree = parser.Parse(tokenizer.Tokens);
diagnostics.AddRange(parser.Diagnostics);
syntaxTrees.Add(syntaxTree);
}
var modules = Module.Collect(syntaxTrees);
var compilationUnits = new List<CompilationUnit>();
for (var i = 0; i < args.Length; i++)
{
var typeChecker = new TypeChecker(syntaxTrees[i], modules);
var compilationUnit = typeChecker.Check();
compilationUnits.Add(compilationUnit);
diagnostics.AddRange(typeChecker.Diagnostics);
}
foreach (var diagnostic in diagnostics)
{
Console.Error.WriteLine(diagnostic.FormatANSI());
}
if (diagnostics.Any(diagnostic => diagnostic.Severity == DiagnosticSeverity.Error))
{
return 1;
}
return 0;

15
compiler/NubLang/Ast/CompilationUnit.cs
View File

@@ -1,15 +0,0 @@
namespace NubLang.Ast;
public sealed class CompilationUnit
{
public CompilationUnit(List<FuncNode> functions, List<NubStructType> importedStructTypes, List<FuncPrototypeNode> importedFunctions)
{
Functions = functions;
ImportedStructTypes = importedStructTypes;
ImportedFunctions = importedFunctions;
}
public List<FuncNode> Functions { get; }
public List<NubStructType> ImportedStructTypes { get; }
public List<FuncPrototypeNode> ImportedFunctions { get; }
}

590
compiler/NubLang/Ast/Node.cs
View File

@@ -1,590 +0,0 @@
using NubLang.Syntax;
namespace NubLang.Ast;
public abstract class Node(List<Token> tokens)
{
public List<Token> Tokens { get; } = tokens;
public abstract IEnumerable<Node> Children();
public IEnumerable<Node> Descendants()
{
foreach (var child in Children())
{
foreach (var descendant in child.DescendantsAndSelf())
{
yield return descendant;
}
}
}
public IEnumerable<Node> DescendantsAndSelf()
{
yield return this;
foreach (var descendant in Descendants())
{
yield return descendant;
}
}
}
#region Definitions
public abstract class DefinitionNode(List<Token> tokens, string module, string name) : Node(tokens)
{
public string Module { get; } = module;
public string Name { get; } = name;
}
public class FuncParameterNode(List<Token> tokens, string name, NubType type) : Node(tokens)
{
public string Name { get; } = name;
public NubType Type { get; } = type;
public override IEnumerable<Node> Children()
{
return [];
}
}
public class FuncPrototypeNode(List<Token> tokens, string module, string name, string? externSymbol, List<FuncParameterNode> parameters, NubType returnType) : Node(tokens)
{
public string Module { get; } = module;
public string Name { get; } = name;
public string? ExternSymbol { get; } = externSymbol;
public List<FuncParameterNode> Parameters { get; } = parameters;
public NubType ReturnType { get; } = returnType;
public override IEnumerable<Node> Children()
{
return Parameters;
}
}
public class FuncNode(List<Token> tokens, FuncPrototypeNode prototype, BlockNode? body) : DefinitionNode(tokens, prototype.Module, prototype.Name)
{
public FuncPrototypeNode Prototype { get; } = prototype;
public BlockNode? Body { get; } = body;
public override IEnumerable<Node> Children()
{
yield return Prototype;
if (Body != null)
{
yield return Body;
}
}
}
#endregion
#region Statements
public abstract class StatementNode(List<Token> tokens) : Node(tokens);
public abstract class TerminalStatementNode(List<Token> tokens) : StatementNode(tokens);
public class BlockNode(List<Token> tokens, List<StatementNode> statements) : StatementNode(tokens)
{
public List<StatementNode> Statements { get; } = statements;
public override IEnumerable<Node> Children()
{
return Statements;
}
}
public class StatementFuncCallNode(List<Token> tokens, FuncCallNode funcCall) : StatementNode(tokens)
{
public FuncCallNode FuncCall { get; } = funcCall;
public override IEnumerable<Node> Children()
{
yield return FuncCall;
}
}
public class ReturnNode(List<Token> tokens, ExpressionNode? value) : TerminalStatementNode(tokens)
{
public ExpressionNode? Value { get; } = value;
public override IEnumerable<Node> Children()
{
if (Value != null) yield return Value;
}
}
public class AssignmentNode(List<Token> tokens, LValueExpressionNode target, ExpressionNode value) : StatementNode(tokens)
{
public LValueExpressionNode Target { get; } = target;
public ExpressionNode Value { get; } = value;
public override IEnumerable<Node> Children()
{
yield return Target;
yield return Value;
}
}
public class IfNode(List<Token> tokens, ExpressionNode condition, BlockNode body, Variant<IfNode, BlockNode>? @else) : StatementNode(tokens)
{
public ExpressionNode Condition { get; } = condition;
public BlockNode Body { get; } = body;
public Variant<IfNode, BlockNode>? Else { get; } = @else;
public override IEnumerable<Node> Children()
{
yield return Condition;
yield return Body;
if (Else.HasValue)
{
yield return Else.Value.Match<Node>(x => x, x => x);
}
}
}
public class VariableDeclarationNode(List<Token> tokens, string name, ExpressionNode? assignment, NubType type) : StatementNode(tokens)
{
public string Name { get; } = name;
public ExpressionNode? Assignment { get; } = assignment;
public NubType Type { get; } = type;
public override IEnumerable<Node> Children()
{
if (Assignment != null) yield return Assignment;
}
}
public class ContinueNode(List<Token> tokens) : TerminalStatementNode(tokens)
{
public override IEnumerable<Node> Children()
{
return [];
}
}
public class BreakNode(List<Token> tokens) : TerminalStatementNode(tokens)
{
public override IEnumerable<Node> Children()
{
return [];
}
}
public class WhileNode(List<Token> tokens, ExpressionNode condition, BlockNode body) : StatementNode(tokens)
{
public ExpressionNode Condition { get; } = condition;
public BlockNode Body { get; } = body;
public override IEnumerable<Node> Children()
{
yield return Condition;
yield return Body;
}
}
public class ForSliceNode(List<Token> tokens, string elementName, string? indexName, ExpressionNode target, BlockNode body) : StatementNode(tokens)
{
public string ElementName { get; } = elementName;
public string? IndexName { get; } = indexName;
public ExpressionNode Target { get; } = target;
public BlockNode Body { get; } = body;
public override IEnumerable<Node> Children()
{
yield return Target;
yield return Body;
}
}
public class ForConstArrayNode(List<Token> tokens, string elementName, string? indexName, ExpressionNode target, BlockNode body) : StatementNode(tokens)
{
public string ElementName { get; } = elementName;
public string? IndexName { get; } = indexName;
public ExpressionNode Target { get; } = target;
public BlockNode Body { get; } = body;
public override IEnumerable<Node> Children()
{
yield return Target;
yield return Body;
}
}
public class DeferNode(List<Token> tokens, StatementNode statement) : StatementNode(tokens)
{
public StatementNode Statement { get; } = statement;
public override IEnumerable<Node> Children()
{
yield return Statement;
}
}
#endregion
#region Expressions
public enum UnaryOperator
{
Negate,
Invert
}
public enum BinaryOperator
{
Equal,
NotEqual,
GreaterThan,
GreaterThanOrEqual,
LessThan,
LessThanOrEqual,
LogicalAnd,
LogicalOr,
Plus,
Minus,
Multiply,
Divide,
Modulo,
LeftShift,
RightShift,
BitwiseAnd,
BitwiseXor,
BitwiseOr
}
public abstract class ExpressionNode(List<Token> tokens, NubType type) : Node(tokens)
{
public NubType Type { get; } = type;
}
public abstract class LValueExpressionNode(List<Token> tokens, NubType type) : ExpressionNode(tokens, type);
public abstract class RValueExpressionNode(List<Token> tokens, NubType type) : ExpressionNode(tokens, type);
public abstract class IntermediateExpression(List<Token> tokens) : ExpressionNode(tokens, new NubVoidType());
public class StringLiteralNode(List<Token> tokens, string value) : RValueExpressionNode(tokens, new NubStringType())
{
public string Value { get; } = value;
public override IEnumerable<Node> Children()
{
return [];
}
}
public class CStringLiteralNode(List<Token> tokens, string value) : RValueExpressionNode(tokens, new NubPointerType(new NubIntType(true, 8)))
{
public string Value { get; } = value;
public override IEnumerable<Node> Children()
{
return [];
}
}
public class I8LiteralNode(List<Token> tokens, sbyte value) : RValueExpressionNode(tokens, new NubIntType(true, 8))
{
public sbyte Value { get; } = value;
public override IEnumerable<Node> Children()
{
return [];
}
}
public class I16LiteralNode(List<Token> tokens, short value) : RValueExpressionNode(tokens, new NubIntType(true, 16))
{
public short Value { get; } = value;
public override IEnumerable<Node> Children()
{
return [];
}
}
public class I32LiteralNode(List<Token> tokens, int value) : RValueExpressionNode(tokens, new NubIntType(true, 32))
{
public int Value { get; } = value;
public override IEnumerable<Node> Children()
{
return [];
}
}
public class I64LiteralNode(List<Token> tokens, long value) : RValueExpressionNode(tokens, new NubIntType(true, 64))
{
public long Value { get; } = value;
public override IEnumerable<Node> Children()
{
return [];
}
}
public class U8LiteralNode(List<Token> tokens, byte value) : RValueExpressionNode(tokens, new NubIntType(false, 8))
{
public byte Value { get; } = value;
public override IEnumerable<Node> Children()
{
return [];
}
}
public class U16LiteralNode(List<Token> tokens, ushort value) : RValueExpressionNode(tokens, new NubIntType(false, 16))
{
public ushort Value { get; } = value;
public override IEnumerable<Node> Children()
{
return [];
}
}
public class U32LiteralNode(List<Token> tokens, uint value) : RValueExpressionNode(tokens, new NubIntType(false, 32))
{
public uint Value { get; } = value;
public override IEnumerable<Node> Children()
{
return [];
}
}
public class U64LiteralNode(List<Token> tokens, ulong value) : RValueExpressionNode(tokens, new NubIntType(false, 64))
{
public ulong Value { get; } = value;
public override IEnumerable<Node> Children()
{
return [];
}
}
public class Float32LiteralNode(List<Token> tokens, float value) : RValueExpressionNode(tokens, new NubFloatType(32))
{
public float Value { get; } = value;
public override IEnumerable<Node> Children()
{
return [];
}
}
public class Float64LiteralNode(List<Token> tokens, double value) : RValueExpressionNode(tokens, new NubFloatType(64))
{
public double Value { get; } = value;
public override IEnumerable<Node> Children()
{
return [];
}
}
public class BoolLiteralNode(List<Token> tokens, NubType type, bool value) : RValueExpressionNode(tokens, type)
{
public bool Value { get; } = value;
public override IEnumerable<Node> Children()
{
return [];
}
}
public class BinaryExpressionNode(List<Token> tokens, NubType type, ExpressionNode left, BinaryOperator @operator, ExpressionNode right) : RValueExpressionNode(tokens, type)
{
public ExpressionNode Left { get; } = left;
public BinaryOperator Operator { get; } = @operator;
public ExpressionNode Right { get; } = right;
public override IEnumerable<Node> Children()
{
yield return Left;
yield return Right;
}
}
public class UnaryExpressionNode(List<Token> tokens, NubType type, UnaryOperator @operator, ExpressionNode operand) : RValueExpressionNode(tokens, type)
{
public UnaryOperator Operator { get; } = @operator;
public ExpressionNode Operand { get; } = operand;
public override IEnumerable<Node> Children()
{
yield return Operand;
}
}
public class FuncCallNode(List<Token> tokens, NubType type, ExpressionNode expression, List<ExpressionNode> parameters) : RValueExpressionNode(tokens, type)
{
public ExpressionNode Expression { get; } = expression;
public List<ExpressionNode> Parameters { get; } = parameters;
public override IEnumerable<Node> Children()
{
yield return Expression;
foreach (var expressionNode in Parameters)
{
yield return expressionNode;
}
}
}
public class VariableIdentifierNode(List<Token> tokens, NubType type, string name) : LValueExpressionNode(tokens, type)
{
public string Name { get; } = name;
public override IEnumerable<Node> Children()
{
return [];
}
}
public class FuncIdentifierNode(List<Token> tokens, NubType type, string module, string name, string? externSymbol) : RValueExpressionNode(tokens, type)
{
public string Module { get; } = module;
public string Name { get; } = name;
public string? ExternSymbol { get; } = externSymbol;
public override IEnumerable<Node> Children()
{
return [];
}
}
public class ArrayInitializerNode(List<Token> tokens, NubType type, List<ExpressionNode> values) : RValueExpressionNode(tokens, type)
{
public List<ExpressionNode> Values { get; } = values;
public override IEnumerable<Node> Children()
{
return Values;
}
}
public class ConstArrayInitializerNode(List<Token> tokens, NubType type, List<ExpressionNode> values) : RValueExpressionNode(tokens, type)
{
public List<ExpressionNode> Values { get; } = values;
public override IEnumerable<Node> Children()
{
return Values;
}
}
public class ArrayIndexAccessNode(List<Token> tokens, NubType type, ExpressionNode target, ExpressionNode index) : LValueExpressionNode(tokens, type)
{
public ExpressionNode Target { get; } = target;
public ExpressionNode Index { get; } = index;
public override IEnumerable<Node> Children()
{
yield return Target;
yield return Index;
}
}
public class ConstArrayIndexAccessNode(List<Token> tokens, NubType type, ExpressionNode target, ExpressionNode index) : LValueExpressionNode(tokens, type)
{
public ExpressionNode Target { get; } = target;
public ExpressionNode Index { get; } = index;
public override IEnumerable<Node> Children()
{
yield return Target;
yield return Index;
}
}
public class SliceIndexAccessNode(List<Token> tokens, NubType type, ExpressionNode target, ExpressionNode index) : LValueExpressionNode(tokens, type)
{
public ExpressionNode Target { get; } = target;
public ExpressionNode Index { get; } = index;
public override IEnumerable<Node> Children()
{
yield return Target;
yield return Index;
}
}
public class AddressOfNode(List<Token> tokens, NubType type, LValueExpressionNode lValue) : RValueExpressionNode(tokens, type)
{
public LValueExpressionNode LValue { get; } = lValue;
public override IEnumerable<Node> Children()
{
yield return LValue;
}
}
public class StructFieldAccessNode(List<Token> tokens, NubType type, ExpressionNode target, string field) : LValueExpressionNode(tokens, type)
{
public ExpressionNode Target { get; } = target;
public string Field { get; } = field;
public override IEnumerable<Node> Children()
{
yield return Target;
}
}
public class StructInitializerNode(List<Token> tokens, NubType type, Dictionary<string, ExpressionNode> initializers) : RValueExpressionNode(tokens, type)
{
public Dictionary<string, ExpressionNode> Initializers { get; } = initializers;
public override IEnumerable<Node> Children()
{
foreach (var initializer in Initializers)
{
yield return initializer.Value;
}
}
}
public class DereferenceNode(List<Token> tokens, NubType type, ExpressionNode target) : LValueExpressionNode(tokens, type)
{
public ExpressionNode Target { get; } = target;
public override IEnumerable<Node> Children()
{
yield return Target;
}
}
public class SizeNode(List<Token> tokens, NubType TargetType) : RValueExpressionNode(tokens, new NubIntType(false, 64))
{
public NubType TargetType { get; } = TargetType;
public override IEnumerable<Node> Children()
{
return [];
}
}
public class CastNode(List<Token> tokens, NubType type, ExpressionNode value) : RValueExpressionNode(tokens, type)
{
public ExpressionNode Value { get; } = value;
public override IEnumerable<Node> Children()
{
yield return Value;
}
}
public class EnumReferenceIntermediateNode(List<Token> tokens, string module, string name) : IntermediateExpression(tokens)
{
public string Module { get; } = module;
public string Name { get; } = name;
public override IEnumerable<Node> Children()
{
return [];
}
}
#endregion

165
compiler/NubLang/Ast/NubType.cs
View File

@@ -1,165 +0,0 @@
using System.Security.Cryptography;
using System.Text;
namespace NubLang.Ast;
public abstract class NubType : IEquatable<NubType>
{
public override bool Equals(object? obj) => obj is NubType other && Equals(other);
public abstract bool Equals(NubType? other);
public abstract override int GetHashCode();
public abstract override string ToString();
public static bool operator ==(NubType? left, NubType? right) => Equals(left, right);
public static bool operator !=(NubType? left, NubType? right) => !Equals(left, right);
}
public class NubVoidType : NubType
{
public override string ToString() => "void";
public override bool Equals(NubType? other) => other is NubVoidType;
public override int GetHashCode() => HashCode.Combine(typeof(NubVoidType));
}
public sealed class NubIntType(bool signed, int width) : NubType
{
public bool Signed { get; } = signed;
public int Width { get; } = width;
public override string ToString() => $"{(Signed ? "i" : "u")}{Width}";
public override bool Equals(NubType? other) => other is NubIntType @int && @int.Width == Width && @int.Signed == Signed;
public override int GetHashCode() => HashCode.Combine(typeof(NubIntType), Signed, Width);
}
public sealed class NubFloatType(int width) : NubType
{
public int Width { get; } = width;
public override string ToString() => $"f{Width}";
public override bool Equals(NubType? other) => other is NubFloatType @float && @float.Width == Width;
public override int GetHashCode() => HashCode.Combine(typeof(NubFloatType), Width);
}
public class NubBoolType : NubType
{
public override string ToString() => "bool";
public override bool Equals(NubType? other) => other is NubBoolType;
public override int GetHashCode() => HashCode.Combine(typeof(NubBoolType));
}
public sealed class NubPointerType(NubType baseType) : NubType
{
public NubType BaseType { get; } = baseType;
public override string ToString() => "^" + BaseType;
public override bool Equals(NubType? other) => other is NubPointerType pointer && BaseType.Equals(pointer.BaseType);
public override int GetHashCode() => HashCode.Combine(typeof(NubPointerType), BaseType);
}
public class NubFuncType(List<NubType> parameters, NubType returnType) : NubType
{
public List<NubType> Parameters { get; } = parameters;
public NubType ReturnType { get; } = returnType;
public override string ToString() => $"func({string.Join(", ", Parameters)}): {ReturnType}";
public override bool Equals(NubType? other) => other is NubFuncType func && ReturnType.Equals(func.ReturnType) && Parameters.SequenceEqual(func.Parameters);
public override int GetHashCode()
{
var hash = new HashCode();
hash.Add(typeof(NubFuncType));
hash.Add(ReturnType);
foreach (var param in Parameters)
{
hash.Add(param);
}
return hash.ToHashCode();
}
}
public class NubStructType(string module, string name, List<NubStructFieldType> fields) : NubType
{
public string Module { get; } = module;
public string Name { get; } = name;
public List<NubStructFieldType> Fields { get; set; } = fields;
public override string ToString() => $"{Module}::{Name}";
public override bool Equals(NubType? other) => other is NubStructType structType && Name == structType.Name && Module == structType.Module;
public override int GetHashCode() => HashCode.Combine(typeof(NubStructType), Module, Name);
}
public class NubStructFieldType(string name, NubType type, bool hasDefaultValue)
{
public string Name { get; } = name;
public NubType Type { get; } = type;
public bool HasDefaultValue { get; } = hasDefaultValue;
}
public class NubSliceType(NubType elementType) : NubType
{
public NubType ElementType { get; } = elementType;
public override string ToString() => "[]" + ElementType;
public override bool Equals(NubType? other) => other is NubSliceType slice && ElementType.Equals(slice.ElementType);
public override int GetHashCode() => HashCode.Combine(typeof(NubSliceType), ElementType);
}
public class NubConstArrayType(NubType elementType, long size) : NubType
{
public NubType ElementType { get; } = elementType;
public long Size { get; } = size;
public override string ToString() => $"[{Size}]{ElementType}";
public override bool Equals(NubType? other) => other is NubConstArrayType array && ElementType.Equals(array.ElementType) && Size == array.Size;
public override int GetHashCode() => HashCode.Combine(typeof(NubConstArrayType), ElementType, Size);
}
public class NubArrayType(NubType elementType) : NubType
{
public NubType ElementType { get; } = elementType;
public override string ToString() => $"[?]{ElementType}";
public override bool Equals(NubType? other) => other is NubArrayType array && ElementType.Equals(array.ElementType);
public override int GetHashCode() => HashCode.Combine(typeof(NubArrayType), ElementType);
}
public class NubStringType : NubType
{
public override string ToString() => "string";
public override bool Equals(NubType? other) => other is NubStringType;
public override int GetHashCode() => HashCode.Combine(typeof(NubStringType));
}
public static class NameMangler
{
public static string Mangle(params IEnumerable<NubType> types)
{
var readable = string.Join(":", types.Select(EncodeType));
return ComputeShortHash(readable);
}
private static string EncodeType(NubType node) => node switch
{
NubVoidType => "V",
NubBoolType => "B",
NubIntType i => (i.Signed ? "I" : "U") + i.Width,
NubFloatType f => "F" + f.Width,
NubStringType => "S",
NubArrayType a => $"A({EncodeType(a.ElementType)})",
NubConstArrayType ca => $"CA({EncodeType(ca.ElementType)})",
NubSliceType a => $"SL{EncodeType(a.ElementType)}()",
NubPointerType p => $"P({EncodeType(p.BaseType)})",
NubFuncType fn => $"FN({string.Join(":", fn.Parameters.Select(EncodeType))}:{EncodeType(fn.ReturnType)})",
NubStructType st => $"ST({st.Module}:{st.Name})",
_ => throw new NotSupportedException($"Cannot encode type: {node}")
};
private static string ComputeShortHash(string input)
{
var bytes = Encoding.UTF8.GetBytes(input);
var hash = SHA256.HashData(bytes);
return Convert.ToHexString(hash[..8]).ToLower();
}
}

1167
compiler/NubLang/Ast/TypeChecker.cs
View File

File diff suppressed because it is too large Load Diff

398
compiler/NubLang/Diagnostics/Diagnostic.cs
View File

@@ -1,398 +0,0 @@
using System.Text;
using NubLang.Syntax;
namespace NubLang.Diagnostics;
public class Diagnostic
{
public class DiagnosticBuilder
{
private readonly DiagnosticSeverity _severity;
private readonly string _message;
private SourceSpan? _span;
private string? _help;
public DiagnosticBuilder(DiagnosticSeverity severity, string message)
{
_severity = severity;
_message = message;
}
public DiagnosticBuilder At(SyntaxNode? node)
{
if (node != null)
{
_span = SourceSpan.Merge(node.Tokens.Select(x => x.Span));
}
return this;
}
public DiagnosticBuilder At(Token? token)
{
if (token != null)
{
At(token.Span);
}
return this;
}
public DiagnosticBuilder At(SourceSpan? span)
{
if (span != null)
{
_span = span;
}
return this;
}
public DiagnosticBuilder At(string filePath, int line, int column)
{
_span = new SourceSpan(filePath, new SourceLocation(line, column), new SourceLocation(line, column));
return this;
}
public DiagnosticBuilder WithHelp(string help)
{
_help = help;
return this;
}
public Diagnostic Build() => new(_severity, _message, _help, _span);
}
public static DiagnosticBuilder Error(string message) => new(DiagnosticSeverity.Error, message);
public static DiagnosticBuilder Warning(string message) => new(DiagnosticSeverity.Warning, message);
public static DiagnosticBuilder Info(string message) => new(DiagnosticSeverity.Info, message);
public DiagnosticSeverity Severity { get; }
public string Message { get; }
public string? Help { get; }
public SourceSpan? Span { get; }
private Diagnostic(DiagnosticSeverity severity, string message, string? help, SourceSpan? span)
{
Severity = severity;
Message = message;
Help = help;
Span = span;
}
public string FormatANSI()
{
var sb = new StringBuilder();
sb.Append(Severity switch
{
DiagnosticSeverity.Error => ConsoleColors.Colorize("error", ConsoleColors.Bold + ConsoleColors.Red),
DiagnosticSeverity.Warning => ConsoleColors.Colorize("warning", ConsoleColors.Bold + ConsoleColors.Yellow),
DiagnosticSeverity.Info => ConsoleColors.Colorize("info", ConsoleColors.Bold + ConsoleColors.Blue),
_ => ConsoleColors.Colorize("unknown", ConsoleColors.Bold + ConsoleColors.White)
});
if (Span.HasValue)
{
sb.Append(ConsoleColors.Colorize($" at {Span.Value}", ConsoleColors.Faint));
}
sb.Append(": ");
sb.Append(ConsoleColors.Colorize(Message, ConsoleColors.BrightWhite));
if (Span.HasValue)
{
sb.AppendLine();
var text = File.ReadAllText(Span.Value.FilePath);
var tokenizer = new Tokenizer(Span.Value.FilePath, text);
tokenizer.Tokenize();
var lines = text.Split('\n');
var startLine = Span.Value.Start.Line;
var endLine = Span.Value.End.Line;
const int CONTEXT_LINES = 3;
var contextStartLine = Math.Max(1, startLine - CONTEXT_LINES);
var contextEndLine = Math.Min(lines.Length, endLine + CONTEXT_LINES);
var numberPadding = contextEndLine.ToString().Length;
var codePadding = 0;
for (var i = contextStartLine - 1; i < contextEndLine && i < lines.Length; i++)
{
var lineLength = lines[i].Length;
if (lineLength > codePadding)
{
codePadding = lineLength;
}
}
sb.Append('╭');
sb.Append(new string('─', numberPadding + 2));
sb.Append('┬');
sb.Append(new string('─', codePadding + 2));
sb.Append('╮');
sb.AppendLine();
for (var i = contextStartLine; i <= contextEndLine; i++)
{
var line = lines[i - 1];
sb.Append("│ ");
sb.Append(i.ToString().PadRight(numberPadding));
sb.Append(" │ ");
sb.Append(ApplySyntaxHighlighting(line.PadRight(codePadding), i, tokenizer.Tokens));
// sb.Append(line.PadRight(codePadding));
sb.Append(" │");
sb.AppendLine();
if (i >= startLine && i <= endLine)
{
var markerStartColumn = 1;
var markerEndColumn = line.Length;
if (i == startLine)
{
markerStartColumn = Span.Value.Start.Column;
}
if (i == endLine)
{
markerEndColumn = Span.Value.End.Column;
}
var markerLength = markerEndColumn - markerStartColumn;
var marker = new string('^', markerLength);
var markerColor = Severity switch
{
DiagnosticSeverity.Info => ConsoleColors.Blue,
DiagnosticSeverity.Warning => ConsoleColors.Yellow,
DiagnosticSeverity.Error => ConsoleColors.Red,
_ => ConsoleColors.White
};
sb.Append("│ ");
sb.Append(new string(' ', numberPadding));
sb.Append(" │ ");
sb.Append(new string(' ', markerStartColumn - 1));
sb.Append(ConsoleColors.Colorize(marker, markerColor));
sb.Append(new string(' ', codePadding - (markerStartColumn - 1) - markerLength));
sb.Append(" │");
sb.AppendLine();
}
}
sb.Append('╰');
sb.Append(new string('─', numberPadding + 2));
sb.Append('┴');
sb.Append(new string('─', codePadding + 2));
sb.Append('╯');
}
if (Help != null)
{
sb.AppendLine();
sb.Append(ConsoleColors.Colorize($"help: {Help}", ConsoleColors.Cyan));
}
return sb.ToString();
}
private static string ApplySyntaxHighlighting(string line, int lineNumber, List<Token> tokens)
{
var sb = new StringBuilder();
var lineTokens = tokens
.Where(t => t.Span.Start.Line == lineNumber)
.OrderBy(t => t.Span.Start.Column)
.ToList();
if (lineTokens.Count == 0)
{
return line;
}
var currentColumn = 1;
foreach (var token in lineTokens)
{
var tokenStart = token.Span.Start.Column;
var tokenEnd = token.Span.End.Column;
if (tokenStart > currentColumn && currentColumn - 1 < line.Length)
{
var beforeLength = Math.Min(tokenStart - currentColumn, line.Length - (currentColumn - 1));
if (beforeLength > 0)
{
var beforeToken = line.Substring(currentColumn - 1, beforeLength);
sb.Append(beforeToken);
}
}
var tokenLength = tokenEnd - tokenStart;
if (tokenStart >= 1 && tokenStart - 1 < line.Length && tokenLength > 0)
{
var availableLength = line.Length - (tokenStart - 1);
var actualLength = Math.Min(tokenLength, availableLength);
if (actualLength > 0)
{
var tokenText = line.Substring(tokenStart - 1, actualLength);
var coloredToken = ColorizeToken(token, tokenText);
sb.Append(coloredToken);
}
}
currentColumn = tokenEnd;
}
if (currentColumn - 1 < line.Length)
{
var remaining = line[(currentColumn - 1)..];
sb.Append(remaining);
}
return sb.ToString();
}
private static string ColorizeToken(Token token, string tokenText)
{
switch (token)
{
case IdentifierToken:
{
return ConsoleColors.Colorize(tokenText, ConsoleColors.BrightWhite);
}
case StringLiteralToken:
{
return ConsoleColors.Colorize(tokenText, ConsoleColors.Green);
}
case IntLiteralToken:
case FloatLiteralToken:
case BoolLiteralToken:
{
return ConsoleColors.Colorize(tokenText, ConsoleColors.Magenta);
}
case SymbolToken symbolToken:
{
switch (symbolToken.Symbol)
{
case Symbol.Func:
case Symbol.Return:
case Symbol.If:
case Symbol.Else:
case Symbol.While:
case Symbol.Break:
case Symbol.Continue:
case Symbol.Struct:
case Symbol.Let:
case Symbol.Extern:
case Symbol.For:
case Symbol.In:
{
return ConsoleColors.Colorize(tokenText, ConsoleColors.Bold + ConsoleColors.Blue);
}
case Symbol.Assign:
case Symbol.Bang:
case Symbol.Equal:
case Symbol.NotEqual:
case Symbol.LessThan:
case Symbol.LessThanOrEqual:
case Symbol.GreaterThan:
case Symbol.GreaterThanOrEqual:
case Symbol.Plus:
case Symbol.Minus:
case Symbol.Star:
case Symbol.ForwardSlash:
case Symbol.Caret:
case Symbol.Ampersand:
{
return ConsoleColors.Colorize(tokenText, ConsoleColors.Yellow);
}
case Symbol.Colon:
case Symbol.OpenParen:
case Symbol.CloseParen:
case Symbol.OpenBrace:
case Symbol.CloseBrace:
case Symbol.OpenBracket:
case Symbol.CloseBracket:
case Symbol.Comma:
case Symbol.Period:
case Symbol.Semi:
{
return ConsoleColors.Colorize(tokenText, ConsoleColors.BrightBlack);
}
}
break;
}
}
return tokenText;
}
}
public enum DiagnosticSeverity
{
Info,
Warning,
Error
}
public static class ConsoleColors
{
public const string Reset = "\e[0m";
public const string Bold = "\e[1m";
public const string Faint = "\e[2m";
public const string Italic = "\e[3m";
public const string Underline = "\e[4m";
public const string SlowBlink = "\e[5m";
public const string RapidBlink = "\e[6m";
public const string SwapBgAndFg = "\e[7m";
public const string Conceal = "\e[8m";
public const string CrossedOut = "\e[9m";
public const string DefaultFont = "\e[10m";
public const string AltFont1 = "\e[11m";
public const string AltFont2 = "\e[12m";
public const string AltFont3 = "\e[13m";
public const string AltFont4 = "\e[14m";
public const string AltFont5 = "\e[15m";
public const string AltFont6 = "\e[16m";
public const string AltFont7 = "\e[17m";
public const string AltFont8 = "\e[18m";
public const string AltFont9 = "\e[19m";
public const string Black = "\e[30m";
public const string Red = "\e[31m";
public const string Green = "\e[32m";
public const string Yellow = "\e[33m";
public const string Blue = "\e[34m";
public const string Magenta = "\e[35m";
public const string Cyan = "\e[36m";
public const string White = "\e[37m";
public const string BrightBlack = "\e[90m";
public const string BrightRed = "\e[91m";
public const string BrightGreen = "\e[92m";
public const string BrightYellow = "\e[93m";
public const string BrightBlue = "\e[94m";
public const string BrightMagenta = "\e[95m";
public const string BrightCyan = "\e[96m";
public const string BrightWhite = "\e[97m";
private static bool IsColorSupported()
{
var term = Environment.GetEnvironmentVariable("TERM");
var colorTerm = Environment.GetEnvironmentVariable("COLORTERM");
return !string.IsNullOrEmpty(term) || !string.IsNullOrEmpty(colorTerm) || !Console.IsOutputRedirected;
}
public static string Colorize(string text, string color)
{
return IsColorSupported() ? $"{color}{text}{Reset}" : text;
}
}

112
compiler/NubLang/Diagnostics/SourceSpan.cs
View File

@@ -1,112 +0,0 @@
namespace NubLang.Diagnostics;
public readonly struct SourceSpan : IEquatable<SourceSpan>, IComparable<SourceSpan>
{
public static SourceSpan Merge(params IEnumerable<SourceSpan> spans)
{
var spanArray = spans as SourceSpan[] ?? spans.ToArray();
if (spanArray.Length == 0)
{
return new SourceSpan(string.Empty, new SourceLocation(0, 0), new SourceLocation(0, 0));
}
var minStart = spanArray.Min(s => s.Start);
var maxEnd = spanArray.Max(s => s.End);
return new SourceSpan(spanArray[0].FilePath, minStart, maxEnd);
}
public SourceSpan(string filePath, SourceLocation start, SourceLocation end)
{
if (start > end)
{
throw new ArgumentException("Start location cannot be after end location");
}
FilePath = filePath;
Start = start;
End = end;
}
public string FilePath { get; }
public SourceLocation Start { get; }
public SourceLocation End { get; }
public override string ToString()
{
if (Start == End)
{
return $"{FilePath}:{Start}";
}
if (Start.Line == End.Line)
{
return Start.Column == End.Column ? $"{FilePath}:{Start}" : $"{FilePath}:{Start.Line}:{Start.Column}-{End.Column}";
}
return $"{FilePath}:{Start}-{End}";
}
public bool Equals(SourceSpan other) => Start == other.Start && End == other.End;
public override bool Equals(object? obj) => obj is SourceSpan other && Equals(other);
public override int GetHashCode() => HashCode.Combine(typeof(SourceSpan), Start, End);
public static bool operator ==(SourceSpan left, SourceSpan right) => Equals(left, right);
public static bool operator !=(SourceSpan left, SourceSpan right) => !Equals(left, right);
public static bool operator <(SourceSpan left, SourceSpan right) => left.CompareTo(right) < 0;
public static bool operator <=(SourceSpan left, SourceSpan right) => left.CompareTo(right) <= 0;
public static bool operator >(SourceSpan left, SourceSpan right) => left.CompareTo(right) > 0;
public static bool operator >=(SourceSpan left, SourceSpan right) => left.CompareTo(right) >= 0;
public int CompareTo(SourceSpan other)
{
var startComparison = Start.CompareTo(other.Start);
return startComparison != 0 ? startComparison : End.CompareTo(other.End);
}
}
public readonly struct SourceLocation : IEquatable<SourceLocation>, IComparable<SourceLocation>
{
public SourceLocation(int line, int column)
{
Line = line;
Column = column;
}
public int Line { get; }
public int Column { get; }
public override string ToString()
{
return $"{Line}:{Column}";
}
public override bool Equals(object? obj)
{
return obj is SourceLocation other && Equals(other);
}
public bool Equals(SourceLocation other)
{
return Line == other.Line && Column == other.Column;
}
public override int GetHashCode()
{
return HashCode.Combine(typeof(SourceLocation), Line, Column);
}
public static bool operator ==(SourceLocation left, SourceLocation right) => Equals(left, right);
public static bool operator !=(SourceLocation left, SourceLocation right) => !Equals(left, right);
public static bool operator <(SourceLocation left, SourceLocation right) => left.Line < right.Line || (left.Line == right.Line && left.Column < right.Column);
public static bool operator >(SourceLocation left, SourceLocation right) => left.Line > right.Line || (left.Line == right.Line && left.Column > right.Column);
public static bool operator <=(SourceLocation left, SourceLocation right) => left.Line <= right.Line || (left.Line == right.Line && left.Column <= right.Column);
public static bool operator >=(SourceLocation left, SourceLocation right) => left.Line >= right.Line || (left.Line == right.Line && left.Column >= right.Column);
public int CompareTo(SourceLocation other)
{
var lineComparison = Line.CompareTo(other.Line);
return lineComparison != 0 ? lineComparison : Column.CompareTo(other.Column);
}
}

10
compiler/NubLang/NubLang.csproj
View File

@@ -1,10 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net9.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<IsAotCompatible>true</IsAotCompatible>
</PropertyGroup>
</Project>

6
compiler/NubLang/Sugar/DeSugar.cs
View File

@@ -1,6 +0,0 @@
namespace NubLang.Sugar;
public class DeSugar
{
}

47
compiler/NubLang/Syntax/Module.cs
View File

@@ -1,47 +0,0 @@
namespace NubLang.Syntax;
public sealed class Module
{
public static Dictionary<string, Module> Collect(List<SyntaxTree> syntaxTrees)
{
var modules = new Dictionary<string, Module>();
foreach (var syntaxTree in syntaxTrees)
{
if (!modules.TryGetValue(syntaxTree.ModuleName, out var module))
{
module = new Module();
modules.Add(syntaxTree.ModuleName, module);
}
module._definitions.AddRange(syntaxTree.Definitions);
}
return modules;
}
private readonly List<DefinitionSyntax> _definitions = [];
public List<StructSyntax> Structs(bool includePrivate)
{
return _definitions
.OfType<StructSyntax>()
.Where(x => x.Exported || includePrivate)
.ToList();
}
public List<FuncSyntax> Functions(bool includePrivate)
{
return _definitions
.OfType<FuncSyntax>()
.Where(x => x.Exported || includePrivate)
.ToList();
}
public List<EnumSyntax> Enums(bool includePrivate)
{
return _definitions
.OfType<EnumSyntax>()
.Where(x => x.Exported || includePrivate)
.ToList();
}
}

951
compiler/NubLang/Syntax/Parser.cs
View File

@@ -1,951 +0,0 @@
using System.Diagnostics.CodeAnalysis;
using NubLang.Diagnostics;
namespace NubLang.Syntax;
public sealed class Parser
{
private List<Token> _tokens = [];
private int _tokenIndex;
private Token? CurrentToken => _tokenIndex < _tokens.Count ? _tokens[_tokenIndex] : null;
private bool HasToken => CurrentToken != null;
public List<Diagnostic> Diagnostics { get; } = [];
public SyntaxTree Parse(List<Token> tokens)
{
Diagnostics.Clear();
_tokens = tokens;
_tokenIndex = 0;
string? moduleName = null;
var imports = new List<string>();
var definitions = new List<DefinitionSyntax>();
while (HasToken)
{
try
{
var startIndex = _tokenIndex;
if (TryExpectSymbol(Symbol.Import))
{
var name = ExpectStringLiteral();
if (imports.Contains(name.Value))
{
Diagnostics.Add(Diagnostic
.Warning($"Module {name.Value} is imported twice")
.At(name)
.WithHelp($"Remove duplicate import \"{name.Value}\"")
.Build());
}
else
{
imports.Add(name.Value);
}
continue;
}
if (TryExpectSymbol(Symbol.Module))
{
if (moduleName != null)
{
throw new ParseException(Diagnostic
.Error("Module is declared more than once")
.At(CurrentToken)
.WithHelp("Remove duplicate module declaration")
.Build());
}
moduleName = ExpectStringLiteral().Value;
continue;
}
var exported = TryExpectSymbol(Symbol.Export);
if (TryExpectSymbol(Symbol.Extern))
{
var externSymbol = ExpectStringLiteral();
ExpectSymbol(Symbol.Func);
definitions.Add(ParseFunc(startIndex, exported, externSymbol.Value));
continue;
}
var keyword = ExpectSymbol();
DefinitionSyntax definition = keyword.Symbol switch
{
Symbol.Func => ParseFunc(startIndex, exported, null),
Symbol.Struct => ParseStruct(startIndex, exported),
Symbol.Enum => ParseEnum(startIndex, exported),
_ => throw new ParseException(Diagnostic
.Error($"Expected 'func', 'struct', 'enum', 'import' or 'module' but found '{keyword.Symbol}'")
.WithHelp("Valid top level statements are 'func', 'struct', 'enum', 'import' and 'module'")
.At(keyword)
.Build())
};
definitions.Add(definition);
}
catch (ParseException e)
{
Diagnostics.Add(e.Diagnostic);
while (HasToken)
{
if (CurrentToken is SymbolToken { Symbol: Symbol.Extern or Symbol.Func or Symbol.Struct })
{
break;
}
Next();
}
}
}
return new SyntaxTree(definitions, moduleName ?? "default", imports);
}
private FuncParameterSyntax ParseFuncParameter()
{
var startIndex = _tokenIndex;
var name = ExpectIdentifier();
ExpectSymbol(Symbol.Colon);
var type = ParseType();
return new FuncParameterSyntax(GetTokens(startIndex), name.Value, type);
}
private FuncSyntax ParseFunc(int startIndex, bool exported, string? externSymbol)
{
var name = ExpectIdentifier();
List<FuncParameterSyntax> parameters = [];
ExpectSymbol(Symbol.OpenParen);
while (!TryExpectSymbol(Symbol.CloseParen))
{
parameters.Add(ParseFuncParameter());
if (!TryExpectSymbol(Symbol.Comma))
{
ExpectSymbol(Symbol.CloseParen);
break;
}
}
var returnType = TryExpectSymbol(Symbol.Colon) ? ParseType() : new VoidTypeSyntax([]);
var prototype = new FuncPrototypeSyntax(GetTokens(startIndex), name.Value, exported, externSymbol, parameters, returnType);
BlockSyntax? body = null;
var bodyStartIndex = _tokenIndex;
if (TryExpectSymbol(Symbol.OpenBrace))
{
body = ParseBlock(bodyStartIndex);
}
return new FuncSyntax(GetTokens(startIndex), prototype, body);
}
private StructSyntax ParseStruct(int startIndex, bool exported)
{
var name = ExpectIdentifier();
ExpectSymbol(Symbol.OpenBrace);
List<StructFieldSyntax> fields = [];
while (!TryExpectSymbol(Symbol.CloseBrace))
{
var memberStartIndex = _tokenIndex;
var fieldName = ExpectIdentifier().Value;
ExpectSymbol(Symbol.Colon);
var fieldType = ParseType();
ExpressionSyntax? fieldValue = null;
if (TryExpectSymbol(Symbol.Assign))
{
fieldValue = ParseExpression();
}
fields.Add(new StructFieldSyntax(GetTokens(memberStartIndex), fieldName, fieldType, fieldValue));
}
return new StructSyntax(GetTokens(startIndex), name.Value, exported, fields);
}
private EnumSyntax ParseEnum(int startIndex, bool exported)
{
var name = ExpectIdentifier();
TypeSyntax? type = null;
if (TryExpectSymbol(Symbol.Colon))
{
type = ParseType();
}
List<EnumFieldSyntax> fields = [];
ExpectSymbol(Symbol.OpenBrace);
long value = -1;
while (!TryExpectSymbol(Symbol.CloseBrace))
{
var memberStartIndex = _tokenIndex;
var fieldName = ExpectIdentifier().Value;
long fieldValue;
if (TryExpectSymbol(Symbol.Assign))
{
if (!TryExpectIntLiteral(out var intLiteralToken))
{
throw new ParseException(Diagnostic
.Error("Value of enum field must be an integer literal")
.At(CurrentToken)
.Build());
}
fieldValue = Convert.ToInt64(intLiteralToken.Value, intLiteralToken.Base);
value = fieldValue;
}
else
{
fieldValue = value + 1;
value = fieldValue;
}
fields.Add(new EnumFieldSyntax(GetTokens(memberStartIndex), fieldName, fieldValue));
}
return new EnumSyntax(GetTokens(startIndex), name.Value, exported, type, fields);
}
private StatementSyntax ParseStatement()
{
var startIndex = _tokenIndex;
if (TryExpectSymbol(out var symbol))
{
switch (symbol)
{
case Symbol.OpenBrace:
return ParseBlock(startIndex);
case Symbol.Return:
return ParseReturn(startIndex);
case Symbol.If:
return ParseIf(startIndex);
case Symbol.While:
return ParseWhile(startIndex);
case Symbol.For:
return ParseFor(startIndex);
case Symbol.Let:
return ParseVariableDeclaration(startIndex);
case Symbol.Defer:
return ParseDefer(startIndex);
case Symbol.Break:
return new BreakSyntax(GetTokens(startIndex));
case Symbol.Continue:
return new ContinueSyntax(GetTokens(startIndex));
}
}
var expr = ParseExpression();
if (TryExpectSymbol(Symbol.Assign))
{
var value = ParseExpression();
return new AssignmentSyntax(GetTokens(startIndex), expr, value);
}
return new StatementExpressionSyntax(GetTokens(startIndex), expr);
}
private VariableDeclarationSyntax ParseVariableDeclaration(int startIndex)
{
var name = ExpectIdentifier().Value;
TypeSyntax? explicitType = null;
if (TryExpectSymbol(Symbol.Colon))
{
explicitType = ParseType();
}
ExpressionSyntax? assignment = null;
if (TryExpectSymbol(Symbol.Assign))
{
assignment = ParseExpression();
}
return new VariableDeclarationSyntax(GetTokens(startIndex), name, explicitType, assignment);
}
private DeferSyntax ParseDefer(int startIndex)
{
var statement = ParseStatement();
return new DeferSyntax(GetTokens(startIndex), statement);
}
private ReturnSyntax ParseReturn(int startIndex)
{
ExpressionSyntax? value = null;
if (!TryExpectSymbol(Symbol.Semi))
{
value = ParseExpression();
}
return new ReturnSyntax(GetTokens(startIndex), value);
}
private IfSyntax ParseIf(int startIndex)
{
var condition = ParseExpression();
var body = ParseBlock();
Variant<IfSyntax, BlockSyntax>? elseStatement = null;
var elseStartIndex = _tokenIndex;
if (TryExpectSymbol(Symbol.Else))
{
if (TryExpectSymbol(Symbol.If))
{
elseStatement = (Variant<IfSyntax, BlockSyntax>)ParseIf(elseStartIndex);
}
else
{
elseStatement = (Variant<IfSyntax, BlockSyntax>)ParseBlock();
}
}
return new IfSyntax(GetTokens(startIndex), condition, body, elseStatement);
}
private WhileSyntax ParseWhile(int startIndex)
{
var condition = ParseExpression();
var body = ParseBlock();
return new WhileSyntax(GetTokens(startIndex), condition, body);
}
private ForSyntax ParseFor(int startIndex)
{
var itemName = ExpectIdentifier().Value;
string? indexName = null;
if (TryExpectSymbol(Symbol.Comma))
{
indexName = ExpectIdentifier().Value;
}
ExpectSymbol(Symbol.In);
var target = ParseExpression();
var body = ParseBlock();
return new ForSyntax(GetTokens(startIndex), itemName, indexName, target, body);
}
private ExpressionSyntax ParseExpression(int precedence = 0)
{
var startIndex = _tokenIndex;
var left = ParsePrimaryExpression();
while (CurrentToken is SymbolToken symbolToken && TryGetBinaryOperator(symbolToken.Symbol, out var op) && GetBinaryOperatorPrecedence(op.Value) >= precedence)
{
Next();
var right = ParseExpression(GetBinaryOperatorPrecedence(op.Value) + 1);
left = new BinaryExpressionSyntax(GetTokens(startIndex), left, op.Value, right);
}
return left;
}
private static int GetBinaryOperatorPrecedence(BinaryOperatorSyntax operatorSyntax)
{
return operatorSyntax switch
{
BinaryOperatorSyntax.Multiply => 10,
BinaryOperatorSyntax.Divide => 10,
BinaryOperatorSyntax.Modulo => 10,
BinaryOperatorSyntax.Plus => 9,
BinaryOperatorSyntax.Minus => 9,
BinaryOperatorSyntax.LeftShift => 8,
BinaryOperatorSyntax.RightShift => 8,
BinaryOperatorSyntax.GreaterThan => 7,
BinaryOperatorSyntax.GreaterThanOrEqual => 7,
BinaryOperatorSyntax.LessThan => 7,
BinaryOperatorSyntax.LessThanOrEqual => 7,
BinaryOperatorSyntax.Equal => 7,
BinaryOperatorSyntax.NotEqual => 7,
BinaryOperatorSyntax.BitwiseAnd => 6,
BinaryOperatorSyntax.BitwiseXor => 5,
BinaryOperatorSyntax.BitwiseOr => 4,
BinaryOperatorSyntax.LogicalAnd => 3,
BinaryOperatorSyntax.LogicalOr => 2,
_ => throw new ArgumentOutOfRangeException(nameof(operatorSyntax), operatorSyntax, null)
};
}
private bool TryGetBinaryOperator(Symbol symbol, [NotNullWhen(true)] out BinaryOperatorSyntax? binaryExpressionOperator)
{
switch (symbol)
{
case Symbol.Equal:
binaryExpressionOperator = BinaryOperatorSyntax.Equal;
return true;
case Symbol.NotEqual:
binaryExpressionOperator = BinaryOperatorSyntax.NotEqual;
return true;
case Symbol.LessThan:
binaryExpressionOperator = BinaryOperatorSyntax.LessThan;
return true;
case Symbol.LessThanOrEqual:
binaryExpressionOperator = BinaryOperatorSyntax.LessThanOrEqual;
return true;
case Symbol.GreaterThan:
binaryExpressionOperator = BinaryOperatorSyntax.GreaterThan;
return true;
case Symbol.GreaterThanOrEqual:
binaryExpressionOperator = BinaryOperatorSyntax.GreaterThanOrEqual;
return true;
case Symbol.And:
binaryExpressionOperator = BinaryOperatorSyntax.LogicalAnd;
return true;
case Symbol.Or:
binaryExpressionOperator = BinaryOperatorSyntax.LogicalOr;
return true;
case Symbol.Plus:
binaryExpressionOperator = BinaryOperatorSyntax.Plus;
return true;
case Symbol.Minus:
binaryExpressionOperator = BinaryOperatorSyntax.Minus;
return true;
case Symbol.Star:
binaryExpressionOperator = BinaryOperatorSyntax.Multiply;
return true;
case Symbol.ForwardSlash:
binaryExpressionOperator = BinaryOperatorSyntax.Divide;
return true;
case Symbol.Percent:
binaryExpressionOperator = BinaryOperatorSyntax.Modulo;
return true;
case Symbol.LeftShift:
binaryExpressionOperator = BinaryOperatorSyntax.LeftShift;
return true;
case Symbol.RightShift:
binaryExpressionOperator = BinaryOperatorSyntax.RightShift;
return true;
case Symbol.Ampersand:
binaryExpressionOperator = BinaryOperatorSyntax.BitwiseAnd;
return true;
case Symbol.Pipe:
binaryExpressionOperator = BinaryOperatorSyntax.BitwiseOr;
return true;
case Symbol.Caret:
binaryExpressionOperator = BinaryOperatorSyntax.BitwiseXor;
return true;
default:
binaryExpressionOperator = null;
return false;
}
}
private ExpressionSyntax ParsePrimaryExpression()
{
var startIndex = _tokenIndex;
var token = ExpectToken();
var expr = token switch
{
BoolLiteralToken boolLiteral => new BoolLiteralSyntax(GetTokens(startIndex), boolLiteral.Value),
StringLiteralToken stringLiteral => new StringLiteralSyntax(GetTokens(startIndex), stringLiteral.Value),
FloatLiteralToken floatLiteral => new FloatLiteralSyntax(GetTokens(startIndex), floatLiteral.Value),
IntLiteralToken intLiteral => new IntLiteralSyntax(GetTokens(startIndex), intLiteral.Value, intLiteral.Base),
IdentifierToken identifier => ParseIdentifier(startIndex, identifier),
SymbolToken symbolToken => symbolToken.Symbol switch
{
Symbol.OpenParen => ParseParenthesizedExpression(),
Symbol.Minus => new UnaryExpressionSyntax(GetTokens(startIndex), UnaryOperatorSyntax.Negate, ParsePrimaryExpression()),
Symbol.Bang => new UnaryExpressionSyntax(GetTokens(startIndex), UnaryOperatorSyntax.Invert, ParsePrimaryExpression()),
Symbol.OpenBracket => ParseArrayInitializer(startIndex),
Symbol.OpenBrace => new StructInitializerSyntax(GetTokens(startIndex), null, ParseStructInitializerBody()),
Symbol.Struct => ParseStructInitializer(startIndex),
Symbol.At => ParseBuiltinFunction(startIndex),
_ => throw new ParseException(Diagnostic
.Error($"Unexpected symbol '{symbolToken.Symbol}' in expression")
.WithHelp("Expected '(', '-', '!', '[' or '{'")
.At(symbolToken)
.Build())
},
_ => throw new ParseException(Diagnostic
.Error($"Unexpected token '{token.GetType().Name}' in expression")
.WithHelp("Expected literal, identifier, or parenthesized expression")
.At(token)
.Build())
};
return ParsePostfixOperators(expr);
}
private ExpressionSyntax ParseBuiltinFunction(int startIndex)
{
var name = ExpectIdentifier();
ExpectSymbol(Symbol.OpenParen);
switch (name.Value)
{
case "size":
{
var type = ParseType();
ExpectSymbol(Symbol.CloseParen);
return new SizeSyntax(GetTokens(startIndex), type);
}
case "cast":
{
var expression = ParseExpression();
ExpectSymbol(Symbol.CloseParen);
return new CastSyntax(GetTokens(startIndex), expression);
}
default:
{
throw new ParseException(Diagnostic.Error($"Unknown builtin {name.Value}").At(name).Build());
}
}
}
private ExpressionSyntax ParseIdentifier(int startIndex, IdentifierToken identifier)
{
if (TryExpectSymbol(Symbol.DoubleColon))
{
var name = ExpectIdentifier();
return new ModuleIdentifierSyntax(GetTokens(startIndex), identifier.Value, name.Value);
}
return new LocalIdentifierSyntax(GetTokens(startIndex), identifier.Value);
}
private ExpressionSyntax ParseParenthesizedExpression()
{
var expression = ParseExpression();
ExpectSymbol(Symbol.CloseParen);
return expression;
}
private ExpressionSyntax ParsePostfixOperators(ExpressionSyntax expr)
{
var startIndex = _tokenIndex;
while (HasToken)
{
if (TryExpectSymbol(Symbol.Ampersand))
{
expr = new AddressOfSyntax(GetTokens(startIndex), expr);
continue;
}
if (TryExpectSymbol(Symbol.Caret))
{
expr = new DereferenceSyntax(GetTokens(startIndex), expr);
continue;
}
if (TryExpectSymbol(Symbol.Period))
{
var member = ExpectIdentifier().Value;
expr = new MemberAccessSyntax(GetTokens(startIndex), expr, member);
continue;
}
if (TryExpectSymbol(Symbol.OpenBracket))
{
var index = ParseExpression();
ExpectSymbol(Symbol.CloseBracket);
expr = new ArrayIndexAccessSyntax(GetTokens(startIndex), expr, index);
continue;
}
if (TryExpectSymbol(Symbol.OpenParen))
{
var parameters = new List<ExpressionSyntax>();
while (!TryExpectSymbol(Symbol.CloseParen))
{
parameters.Add(ParseExpression());
if (!TryExpectSymbol(Symbol.Comma))
{
ExpectSymbol(Symbol.CloseParen);
break;
}
}
expr = new FuncCallSyntax(GetTokens(startIndex), expr, parameters);
continue;
}
break;
}
return expr;
}
private ExpressionSyntax ParseArrayInitializer(int startIndex)
{
var values = new List<ExpressionSyntax>();
while (!TryExpectSymbol(Symbol.CloseBracket))
{
values.Add(ParseExpression());
if (!TryExpectSymbol(Symbol.Comma))
{
ExpectSymbol(Symbol.CloseBracket);
break;
}
}
return new ArrayInitializerSyntax(GetTokens(startIndex), values);
}
private StructInitializerSyntax ParseStructInitializer(int startIndex)
{
TypeSyntax? type = null;
if (!TryExpectSymbol(Symbol.OpenBrace))
{
type = ParseType();
ExpectSymbol(Symbol.OpenBrace);
}
var initializers = ParseStructInitializerBody();
return new StructInitializerSyntax(GetTokens(startIndex), type, initializers);
}
private Dictionary<string, ExpressionSyntax> ParseStructInitializerBody()
{
Dictionary<string, ExpressionSyntax> initializers = [];
while (!TryExpectSymbol(Symbol.CloseBrace))
{
var name = ExpectIdentifier().Value;
ExpectSymbol(Symbol.Assign);
var value = ParseExpression();
initializers.Add(name, value);
}
return initializers;
}
private BlockSyntax ParseBlock()
{
var startIndex = _tokenIndex;
ExpectSymbol(Symbol.OpenBrace);
return ParseBlock(startIndex);
}
private BlockSyntax ParseBlock(int startIndex)
{
List<StatementSyntax> statements = [];
while (!TryExpectSymbol(Symbol.CloseBrace))
{
try
{
statements.Add(ParseStatement());
}
catch (ParseException ex)
{
Diagnostics.Add(ex.Diagnostic);
if (HasToken)
{
Next();
}
else
{
break;
}
}
}
return new BlockSyntax(GetTokens(startIndex), statements);
}
private TypeSyntax ParseType()
{
var startIndex = _tokenIndex;
if (TryExpectIdentifier(out var name))
{
if (name.Value[0] == 'u' && int.TryParse(name.Value[1..], out var size))
{
if (size is not 8 and not 16 and not 32 and not 64)
{
throw new ParseException(Diagnostic
.Error("Arbitrary uint size is not supported")
.WithHelp("Use u8, u16, u32 or u64")
.At(name)
.Build());
}
return new IntTypeSyntax(GetTokens(startIndex), false, size);
}
if (name.Value[0] == 'i' && int.TryParse(name.Value[1..], out size))
{
if (size is not 8 and not 16 and not 32 and not 64)
{
throw new ParseException(Diagnostic
.Error("Arbitrary int size is not supported")
.WithHelp("Use i8, i16, i32 or i64")
.At(name)
.Build());
}
return new IntTypeSyntax(GetTokens(startIndex), true, size);
}
if (name.Value[0] == 'f' && int.TryParse(name.Value[1..], out size))
{
if (size is not 32 and not 64)
{
throw new ParseException(Diagnostic
.Error("Arbitrary float size is not supported")
.WithHelp("Use f32 or f64")
.At(name)
.Build());
}
return new FloatTypeSyntax(GetTokens(startIndex), size);
}
switch (name.Value)
{
case "void":
return new VoidTypeSyntax(GetTokens(startIndex));
case "string":
return new StringTypeSyntax(GetTokens(startIndex));
case "bool":
return new BoolTypeSyntax(GetTokens(startIndex));
default:
{
string? module = null;
if (TryExpectSymbol(Symbol.DoubleColon))
{
var customTypeName = ExpectIdentifier();
module = name.Value;
name = customTypeName;
}
return new CustomTypeSyntax(GetTokens(startIndex), module, name.Value);
}
}
}
if (TryExpectSymbol(Symbol.Caret))
{
var baseType = ParseType();
return new PointerTypeSyntax(GetTokens(startIndex), baseType);
}
if (TryExpectSymbol(Symbol.Func))
{
ExpectSymbol(Symbol.OpenParen);
List<TypeSyntax> parameters = [];
while (!TryExpectSymbol(Symbol.CloseParen))
{
parameters.Add(ParseType());
if (!TryExpectSymbol(Symbol.Comma))
{
ExpectSymbol(Symbol.CloseParen);
break;
}
}
var returnType = TryExpectSymbol(Symbol.Colon)
? ParseType()
: new VoidTypeSyntax([]);
return new FuncTypeSyntax(GetTokens(startIndex), parameters, returnType);
}
if (TryExpectSymbol(Symbol.OpenBracket))
{
if (TryExpectIntLiteral(out var intLiteral))
{
ExpectSymbol(Symbol.CloseBracket);
var baseType = ParseType();
return new ConstArrayTypeSyntax(GetTokens(startIndex), baseType, Convert.ToInt64(intLiteral.Value, intLiteral.Base));
}
else if (TryExpectSymbol(Symbol.QuestionMark))
{
ExpectSymbol(Symbol.CloseBracket);
var baseType = ParseType();
return new ArrayTypeSyntax(GetTokens(startIndex), baseType);
}
else
{
ExpectSymbol(Symbol.CloseBracket);
var baseType = ParseType();
return new SliceTypeSyntax(GetTokens(startIndex), baseType);
}
}
throw new ParseException(Diagnostic
.Error("Invalid type syntax")
.WithHelp("Expected type name, '^' for pointer, or '[]' for array")
.At(CurrentToken)
.Build());
}
private Token ExpectToken()
{
if (!HasToken)
{
throw new ParseException(Diagnostic
.Error("Unexpected end of file")
.WithHelp("Expected more tokens to complete the syntax")
.At(_tokens[^1])
.Build());
}
var token = CurrentToken!;
Next();
return token;
}
private SymbolToken ExpectSymbol()
{
var token = ExpectToken();
if (token is not SymbolToken symbol)
{
throw new ParseException(Diagnostic
.Error($"Expected symbol, but found {token.GetType().Name}")
.WithHelp("This position requires a symbol like '(', ')', '{', '}', etc.")
.At(token)
.Build());
}
return symbol;
}
private void ExpectSymbol(Symbol expectedSymbol)
{
var token = ExpectSymbol();
if (token.Symbol != expectedSymbol)
{
throw new ParseException(Diagnostic
.Error($"Expected '{expectedSymbol}', but found '{token.Symbol}'")
.WithHelp($"Insert '{expectedSymbol}' here")
.At(token)
.Build());
}
}
private bool TryExpectSymbol(out Symbol symbol)
{
if (CurrentToken is SymbolToken symbolToken)
{
Next();
symbol = symbolToken.Symbol;
return true;
}
symbol = default;
return false;
}
private bool TryExpectSymbol(Symbol symbol)
{
if (CurrentToken is SymbolToken symbolToken && symbolToken.Symbol == symbol)
{
Next();
return true;
}
return false;
}
private bool TryExpectIdentifier([NotNullWhen(true)] out IdentifierToken? identifier)
{
if (CurrentToken is IdentifierToken identifierToken)
{
identifier = identifierToken;
Next();
return true;
}
identifier = null;
return false;
}
private IdentifierToken ExpectIdentifier()
{
var token = ExpectToken();
if (token is not IdentifierToken identifier)
{
throw new ParseException(Diagnostic
.Error($"Expected identifier, but found {token.GetType().Name}")
.WithHelp("Provide a valid identifier name here")
.At(token)
.Build());
}
return identifier;
}
private bool TryExpectIntLiteral([NotNullWhen(true)] out IntLiteralToken? stringLiteral)
{
if (CurrentToken is IntLiteralToken token)
{
stringLiteral = token;
Next();
return true;
}
stringLiteral = null;
return false;
}
private StringLiteralToken ExpectStringLiteral()
{
var token = ExpectToken();
if (token is not StringLiteralToken identifier)
{
throw new ParseException(Diagnostic
.Error($"Expected string literal, but found {token.GetType().Name}")
.WithHelp("Provide a valid string literal")
.At(token)
.Build());
}
return identifier;
}
private void Next()
{
_tokenIndex++;
}
private List<Token> GetTokens(int tokenStartIndex)
{
return _tokens.Skip(tokenStartIndex).Take(_tokenIndex - tokenStartIndex).ToList();
}
}
public record SyntaxTree(List<DefinitionSyntax> Definitions, string ModuleName, List<string> Imports);
public class ParseException : Exception
{
public Diagnostic Diagnostic { get; }
public ParseException(Diagnostic diagnostic) : base(diagnostic.Message)
{
Diagnostic = diagnostic;
}
}

147
compiler/NubLang/Syntax/Syntax.cs
View File

@@ -1,147 +0,0 @@
namespace NubLang.Syntax;
public abstract record SyntaxNode(List<Token> Tokens);
#region Definitions
public abstract record DefinitionSyntax(List<Token> Tokens, string Name, bool Exported) : SyntaxNode(Tokens);
public record FuncParameterSyntax(List<Token> Tokens, string Name, TypeSyntax Type) : SyntaxNode(Tokens);
public record FuncPrototypeSyntax(List<Token> Tokens, string Name, bool Exported, string? ExternSymbol, List<FuncParameterSyntax> Parameters, TypeSyntax ReturnType) : SyntaxNode(Tokens);
public record FuncSyntax(List<Token> Tokens, FuncPrototypeSyntax Prototype, BlockSyntax? Body) : DefinitionSyntax(Tokens, Prototype.Name, Prototype.Exported);
public record StructFieldSyntax(List<Token> Tokens, string Name, TypeSyntax Type, ExpressionSyntax? Value) : SyntaxNode(Tokens);
public record StructSyntax(List<Token> Tokens, string Name, bool Exported, List<StructFieldSyntax> Fields) : DefinitionSyntax(Tokens, Name, Exported);
public record EnumFieldSyntax(List<Token> Tokens, string Name, long Value) : SyntaxNode(Tokens);
public record EnumSyntax(List<Token> Tokens, string Name, bool Exported, TypeSyntax? Type, List<EnumFieldSyntax> Fields) : DefinitionSyntax(Tokens, Name, Exported);
public enum UnaryOperatorSyntax
{
Negate,
Invert
}
public enum BinaryOperatorSyntax
{
Equal,
NotEqual,
GreaterThan,
GreaterThanOrEqual,
LessThan,
LessThanOrEqual,
LogicalAnd,
LogicalOr,
Plus,
Minus,
Multiply,
Divide,
Modulo,
LeftShift,
RightShift,
BitwiseAnd,
BitwiseXor,
BitwiseOr,
}
#endregion
#region Statements
public abstract record StatementSyntax(List<Token> Tokens) : SyntaxNode(Tokens);
public record BlockSyntax(List<Token> Tokens, List<StatementSyntax> Statements) : StatementSyntax(Tokens);
public record StatementExpressionSyntax(List<Token> Tokens, ExpressionSyntax Expression) : StatementSyntax(Tokens);
public record ReturnSyntax(List<Token> Tokens, ExpressionSyntax? Value) : StatementSyntax(Tokens);
public record AssignmentSyntax(List<Token> Tokens, ExpressionSyntax Target, ExpressionSyntax Value) : StatementSyntax(Tokens);
public record IfSyntax(List<Token> Tokens, ExpressionSyntax Condition, BlockSyntax Body, Variant<IfSyntax, BlockSyntax>? Else) : StatementSyntax(Tokens);
public record VariableDeclarationSyntax(List<Token> Tokens, string Name, TypeSyntax? ExplicitType, ExpressionSyntax? Assignment) : StatementSyntax(Tokens);
public record ContinueSyntax(List<Token> Tokens) : StatementSyntax(Tokens);
public record BreakSyntax(List<Token> Tokens) : StatementSyntax(Tokens);
public record DeferSyntax(List<Token> Tokens, StatementSyntax Statement) : StatementSyntax(Tokens);
public record WhileSyntax(List<Token> Tokens, ExpressionSyntax Condition, BlockSyntax Body) : StatementSyntax(Tokens);
public record ForSyntax(List<Token> Tokens, string ElementName, string? IndexName, ExpressionSyntax Target, BlockSyntax Body) : StatementSyntax(Tokens);
#endregion
#region Expressions
public abstract record ExpressionSyntax(List<Token> Tokens) : SyntaxNode(Tokens);
public record BinaryExpressionSyntax(List<Token> Tokens, ExpressionSyntax Left, BinaryOperatorSyntax Operator, ExpressionSyntax Right) : ExpressionSyntax(Tokens);
public record UnaryExpressionSyntax(List<Token> Tokens, UnaryOperatorSyntax Operator, ExpressionSyntax Operand) : ExpressionSyntax(Tokens);
public record FuncCallSyntax(List<Token> Tokens, ExpressionSyntax Expression, List<ExpressionSyntax> Parameters) : ExpressionSyntax(Tokens);
public record LocalIdentifierSyntax(List<Token> Tokens, string Name) : ExpressionSyntax(Tokens);
public record ModuleIdentifierSyntax(List<Token> Tokens, string Module, string Name) : ExpressionSyntax(Tokens);
public record ArrayInitializerSyntax(List<Token> Tokens, List<ExpressionSyntax> Values) : ExpressionSyntax(Tokens);
public record ArrayIndexAccessSyntax(List<Token> Tokens, ExpressionSyntax Target, ExpressionSyntax Index) : ExpressionSyntax(Tokens);
public record AddressOfSyntax(List<Token> Tokens, ExpressionSyntax Target) : ExpressionSyntax(Tokens);
public record IntLiteralSyntax(List<Token> Tokens, string Value, int Base) : ExpressionSyntax(Tokens);
public record StringLiteralSyntax(List<Token> Tokens, string Value) : ExpressionSyntax(Tokens);
public record BoolLiteralSyntax(List<Token> Tokens, bool Value) : ExpressionSyntax(Tokens);
public record FloatLiteralSyntax(List<Token> Tokens, string Value) : ExpressionSyntax(Tokens);
public record MemberAccessSyntax(List<Token> Tokens, ExpressionSyntax Target, string Member) : ExpressionSyntax(Tokens);
public record StructInitializerSyntax(List<Token> Tokens, TypeSyntax? StructType, Dictionary<string, ExpressionSyntax> Initializers) : ExpressionSyntax(Tokens);
public record DereferenceSyntax(List<Token> Tokens, ExpressionSyntax Target) : ExpressionSyntax(Tokens);
public record SizeSyntax(List<Token> Tokens, TypeSyntax Type) : ExpressionSyntax(Tokens);
public record CastSyntax(List<Token> Tokens, ExpressionSyntax Value) : ExpressionSyntax(Tokens);
#endregion
#region Types
public abstract record TypeSyntax(List<Token> Tokens) : SyntaxNode(Tokens);
public record FuncTypeSyntax(List<Token> Tokens, List<TypeSyntax> Parameters, TypeSyntax ReturnType) : TypeSyntax(Tokens);
public record PointerTypeSyntax(List<Token> Tokens, TypeSyntax BaseType) : TypeSyntax(Tokens);
public record VoidTypeSyntax(List<Token> Tokens) : TypeSyntax(Tokens);
public record IntTypeSyntax(List<Token> Tokens, bool Signed, int Width) : TypeSyntax(Tokens);
public record FloatTypeSyntax(List<Token> Tokens, int Width) : TypeSyntax(Tokens);
public record BoolTypeSyntax(List<Token> Tokens) : TypeSyntax(Tokens);
public record StringTypeSyntax(List<Token> Tokens) : TypeSyntax(Tokens);
public record SliceTypeSyntax(List<Token> Tokens, TypeSyntax BaseType) : TypeSyntax(Tokens);
public record ArrayTypeSyntax(List<Token> Tokens, TypeSyntax BaseType) : TypeSyntax(Tokens);
public record ConstArrayTypeSyntax(List<Token> Tokens, TypeSyntax BaseType, long Size) : TypeSyntax(Tokens);
public record CustomTypeSyntax(List<Token> Tokens, string? Module, string Name) : TypeSyntax(Tokens);
#endregion

77
compiler/NubLang/Syntax/Token.cs
View File

@@ -1,77 +0,0 @@
using NubLang.Diagnostics;
namespace NubLang.Syntax;
public enum Symbol
{
// Control
If,
Else,
While,
For,
In,
Break,
Continue,
Return,
Let,
Defer,
// Declaration
Func,
Struct,
Enum,
Import,
Module,
// Modifier
Extern,
Export,
Colon,
DoubleColon,
OpenParen,
CloseParen,
OpenBrace,
CloseBrace,
OpenBracket,
CloseBracket,
Comma,
Period,
Assign,
Bang,
Equal,
NotEqual,
LessThan,
LessThanOrEqual,
GreaterThan,
GreaterThanOrEqual,
Plus,
Minus,
Star,
ForwardSlash,
Caret,
Ampersand,
Semi,
Percent,
LeftShift,
RightShift,
Pipe,
And,
Or,
At,
QuestionMark,
}
public abstract record Token(SourceSpan Span);
public record IdentifierToken(SourceSpan Span, string Value) : Token(Span);
public record IntLiteralToken(SourceSpan Span, string Value, int Base) : Token(Span);
public record StringLiteralToken(SourceSpan Span, string Value) : Token(Span);
public record BoolLiteralToken(SourceSpan Span, bool Value) : Token(Span);
public record FloatLiteralToken(SourceSpan Span, string Value) : Token(Span);
public record SymbolToken(SourceSpan Span, Symbol Symbol) : Token(Span);

331
compiler/NubLang/Syntax/Tokenizer.cs
View File

@@ -1,331 +0,0 @@
using NubLang.Diagnostics;
namespace NubLang.Syntax;
public sealed class Tokenizer
{
private static readonly Dictionary<string, Symbol> Keywords = new()
{
["func"] = Symbol.Func,
["if"] = Symbol.If,
["else"] = Symbol.Else,
["while"] = Symbol.While,
["for"] = Symbol.For,
["in"] = Symbol.In,
["break"] = Symbol.Break,
["continue"] = Symbol.Continue,
["return"] = Symbol.Return,
["struct"] = Symbol.Struct,
["let"] = Symbol.Let,
["extern"] = Symbol.Extern,
["module"] = Symbol.Module,
["export"] = Symbol.Export,
["import"] = Symbol.Import,
["defer"] = Symbol.Defer,
["enum"] = Symbol.Enum,
};
private static readonly Dictionary<char[], Symbol> Symbols = new()
{
[['=', '=']] = Symbol.Equal,
[['!', '=']] = Symbol.NotEqual,
[['<', '=']] = Symbol.LessThanOrEqual,
[['>', '=']] = Symbol.GreaterThanOrEqual,
[['<', '<']] = Symbol.LeftShift,
[['>', '>']] = Symbol.RightShift,
[['&', '&']] = Symbol.And,
[['|', '|']] = Symbol.Or,
[[':', ':']] = Symbol.DoubleColon,
[[':']] = Symbol.Colon,
[['(']] = Symbol.OpenParen,
[[')']] = Symbol.CloseParen,
[['{']] = Symbol.OpenBrace,
[['}']] = Symbol.CloseBrace,
[['[']] = Symbol.OpenBracket,
[[']']] = Symbol.CloseBracket,
[[',']] = Symbol.Comma,
[['.']] = Symbol.Period,
[['=']] = Symbol.Assign,
[['<']] = Symbol.LessThan,
[['>']] = Symbol.GreaterThan,
[['+']] = Symbol.Plus,
[['-']] = Symbol.Minus,
[['*']] = Symbol.Star,
[['/']] = Symbol.ForwardSlash,
[['!']] = Symbol.Bang,
[['^']] = Symbol.Caret,
[['&']] = Symbol.Ampersand,
[[';']] = Symbol.Semi,
[['%']] = Symbol.Percent,
[['|']] = Symbol.Pipe,
[['@']] = Symbol.At,
[['?']] = Symbol.QuestionMark,
};
private static readonly (char[] Pattern, Symbol Symbol)[] OrderedSymbols = Symbols
.OrderByDescending(kvp => kvp.Key.Length)
.Select(kvp => (kvp.Key, kvp.Value))
.ToArray();
private readonly string _fileName;
private readonly string _content;
private int _index = 0;
private int _line = 1;
private int _column = 1;
public Tokenizer(string fileName, string content)
{
_fileName = fileName;
_content = content;
}
public List<Diagnostic> Diagnostics { get; } = [];
public List<Token> Tokens { get; } = [];
public void Tokenize()
{
Diagnostics.Clear();
Tokens.Clear();
_index = 0;
_line = 1;
_column = 1;
while (Peek().HasValue)
{
try
{
var current = Peek()!.Value;
if (char.IsWhiteSpace(current))
{
if (current is '\n')
{
_line += 1;
// note(nub31): Next increments the column, so 0 is correct here
_column = 0;
}
Next();
continue;
}
if (current == '/' && Peek(1) == '/')
{
// note(nub31): Keep newline so next iteration increments the line counter
while (Peek() is not '\n')
{
Next();
}
continue;
}
Tokens.Add(ParseToken(current, _line, _column));
}
catch (TokenizerException e)
{
Diagnostics.Add(e.Diagnostic);
Next();
}
}
}
private Token ParseToken(char current, int lineStart, int columnStart)
{
if (char.IsLetter(current) || current == '_')
{
var buffer = string.Empty;
while (Peek() != null && (char.IsLetterOrDigit(Peek()!.Value) || Peek() == '_'))
{
buffer += Peek();
Next();
}
if (Keywords.TryGetValue(buffer, out var keywordSymbol))
{
return new SymbolToken(CreateSpan(lineStart, columnStart), keywordSymbol);
}
if (buffer is "true" or "false")
{
return new BoolLiteralToken(CreateSpan(lineStart, columnStart), Convert.ToBoolean(buffer));
}
return new IdentifierToken(CreateSpan(lineStart, columnStart), buffer);
}
if (char.IsDigit(current))
{
var buffer = string.Empty;
if (current == '0' && Peek(1) is 'x')
{
buffer += "0x";
Next();
Next();
while (Peek() != null && Uri.IsHexDigit(Peek()!.Value))
{
buffer += Peek()!.Value;
Next();
}
if (buffer.Length <= 2)
{
throw new TokenizerException(Diagnostic
.Error("Invalid hex literal, no digits found")
.At(_fileName, _line, _column)
.Build());
}
return new IntLiteralToken(CreateSpan(lineStart, columnStart), buffer, 16);
}
if (current == '0' && Peek(1) is 'b')
{
buffer += "0b";
Next();
Next();
while (Peek() != null && (Peek() == '0' || Peek() == '1'))
{
buffer += Peek()!.Value;
Next();
}
if (buffer.Length <= 2)
{
throw new TokenizerException(Diagnostic
.Error("Invalid binary literal, no digits found")
.At(_fileName, _line, _column)
.Build());
}
return new IntLiteralToken(CreateSpan(lineStart, columnStart), buffer, 2);
}
var isFloat = false;
while (Peek() != null)
{
var next = Peek()!.Value;
if (next == '.')
{
if (isFloat)
{
throw new TokenizerException(Diagnostic
.Error("More than one period found in float literal")
.At(_fileName, _line, _column)
.Build());
}
isFloat = true;
buffer += next;
Next();
}
else if (char.IsDigit(next))
{
buffer += next;
Next();
}
else
{
break;
}
}
if (isFloat)
{
return new FloatLiteralToken(CreateSpan(lineStart, columnStart), buffer);
}
else
{
return new IntLiteralToken(CreateSpan(lineStart, columnStart), buffer, 10);
}
}
if (current == '"')
{
Next();
var buffer = string.Empty;
while (true)
{
var next = Peek();
if (!next.HasValue)
{
throw new TokenizerException(Diagnostic
.Error("Unclosed string literal")
.At(_fileName, _line, _column)
.Build());
}
if (next is '\n')
{
_line += 1;
break;
}
if (next is '"')
{
Next();
break;
}
buffer += next;
Next();
}
return new StringLiteralToken(CreateSpan(lineStart, columnStart), buffer);
}
foreach (var (pattern, symbol) in OrderedSymbols)
{
for (var i = 0; i < pattern.Length; i++)
{
var c = Peek(i);
if (!c.HasValue || c.Value != pattern[i]) break;
if (i == pattern.Length - 1)
{
for (var j = 0; j <= i; j++)
{
Next();
}
return new SymbolToken(CreateSpan(lineStart, columnStart), symbol);
}
}
}
throw new TokenizerException(Diagnostic.Error($"Unknown token '{current}'").Build());
}
private SourceSpan CreateSpan(int lineStart, int columnStart)
{
return new SourceSpan(_fileName, new SourceLocation(lineStart, columnStart), new SourceLocation(_line, _column));
}
private char? Peek(int offset = 0)
{
if (_index + offset < _content.Length)
{
return _content[_index + offset];
}
return null;
}
private void Next()
{
_index += 1;
_column += 1;
}
}
public class TokenizerException : Exception
{
public Diagnostic Diagnostic { get; }
public TokenizerException(Diagnostic diagnostic) : base(diagnostic.Message)
{
Diagnostic = diagnostic;
}
}

75
compiler/NubLang/Variant.cs
View File

@@ -1,75 +0,0 @@
using System.Diagnostics.CodeAnalysis;
namespace NubLang;
public readonly struct Variant<T1, T2> where T1 : notnull where T2 : notnull
{
public Variant()
{
throw new InvalidOperationException("Variant must be initialized with a value");
}
public Variant(T1 value)
{
_value = value;
}
public Variant(T2 value)
{
_value = value;
}
private readonly object _value;
public void Match(Action<T1> on1, Action<T2> on2)
{
switch (_value)
{
case T1 v1:
on1(v1);
break;
case T2 v2:
on2(v2);
break;
default:
throw new InvalidCastException();
}
}
public T Match<T>(Func<T1, T> on1, Func<T2, T> on2)
{
return _value switch
{
T1 v1 => on1(v1),
T2 v2 => on2(v2),
_ => throw new InvalidCastException()
};
}
public bool IsCase1([NotNullWhen(true)] out T1? value)
{
if (_value is T1 converted)
{
value = converted;
return true;
}
value = default;
return false;
}
public bool IsCase2([NotNullWhen(true)] out T2? value)
{
if (_value is T2 converted)
{
value = converted;
return true;
}
value = default;
return false;
}
public static implicit operator Variant<T1, T2>(T1 value) => new(value);
public static implicit operator Variant<T1, T2>(T2 value) => new(value);
}

0
compiler/Compiler/Parser.cs → compiler/Parser.cs
View File

0
compiler/Compiler/Program.cs → compiler/Program.cs
View File

0
compiler/Compiler/Tokenizer.cs → compiler/Tokenizer.cs
View File

0
compiler/Compiler/TypeChecker.cs → compiler/TypeChecker.cs
View File

0
compiler/Compiler/test.nub → compiler/test.nub
View File